Update layout

app.py

@@ -1,14 +1,8 @@
 from apps import mlm, vqa
-import os
 import streamlit as st
 from session import _get_state
 from multiapp import MultiApp
-
-
-def read_markdown(path, parent="./sections/"):
-    with open(os.path.join(parent, path)) as f:
-        return f.read()
-
+from apps.utils import read_markdown
 
 def main():
     state = _get_state()
@@ -24,12 +18,6 @@ def main():
         "[Gunjan Chhablani](https://huggingface.co/gchhablani), [Bhavitvya Malik](https://huggingface.co/bhavitvyamalik)"
     )
 
-    image_col, intro_col = st.beta_columns([3, 8])
-    image_col.image("./misc/mvqa-logo-3-white.png", use_column_width="always")
-    intro_col.write(read_markdown("intro.md"))
-    with st.beta_expander("Usage"):
-        st.write(read_markdown("usage.md"))
-
     with st.beta_expander("Article"):
         st.write(read_markdown("abstract.md"))
         st.write(read_markdown("caveats.md"))

apps/mlm.py

@@ -12,7 +12,7 @@ import pandas as pd
 import os
 import matplotlib.pyplot as plt
 from mtranslate import translate
-
+from .utils import read_markdown
 
 from .model.flax_clip_vision_bert.modeling_clip_vision_bert import (
     FlaxCLIPVisionBertForMaskedLM,
@@ -25,6 +25,10 @@ def softmax(logits):
 def app(state):
     mlm_state = state
 
+    with st.beta_expander("Usage"):
+        st.write(read_markdown("mlm_usage.md"))
+    st.write(read_markdown("mlm_intro.md"))
+
     # @st.cache(persist=False) # TODO: Make this work with mlm_state. Currently not supported.
     def predict(transformed_image, caption_inputs):
         outputs = mlm_state.mlm_model(pixel_values=transformed_image, **caption_inputs)

apps/utils.py

@@ -1,5 +1,5 @@
 import json
-
+import os
 import numpy as np
 import plotly.express as px
 import torch
@@ -81,3 +81,8 @@ def plotly_express_horizontal_bar_plot(values, labels):
         orientation="h",
     )
     return fig
+
+
+def read_markdown(path, parent="./sections/"):
+    with open(os.path.join(parent, path)) as f:
+        return f.read()

apps/vqa.py

@@ -14,7 +14,7 @@ import matplotlib.pyplot as plt
 import json
 
 from mtranslate import translate
-
+from .utils import read_markdown
 
 from .model.flax_clip_vision_bert.modeling_clip_vision_bert import (
     FlaxCLIPVisionBertForSequenceClassification,
@@ -28,6 +28,10 @@ def softmax(logits):
 def app(state):
     vqa_state = state
 
+    with st.beta_expander("Usage"):
+        st.write(read_markdown("vqa_usage.md"))
+    st.write(read_markdown("vqa_intro.md"))
+
     # @st.cache(persist=False)
     def predict(transformed_image, question_inputs):
         return np.array(

multiapp.py

@@ -10,6 +10,7 @@ class MultiApp:
         self.apps.append({"title": title, "function": func})
 
     def run(self):
+        logo = st.image("./misc/mvqa-logo-3-white.png")
         st.sidebar.header("Tasks")
         app = st.sidebar.radio(
             "", self.apps, format_func=lambda app: app["title"]

sections/mlm_intro.md

@@ -0,0 +1,5 @@
+This demo uses a [CLIP-Vision-Bert model checkpoint](https://huggingface.co/flax-community/clip-vision-bert-cc12m-70k) pre-trained using text-only Masked LM on approximately 10 million image-text pairs taken from the [Conceptual 12M dataset](https://github.com/google-research-datasets/conceptual-12m) translated using [MBart](https://huggingface.co/transformers/model_doc/mbart.html). The translations are performed in the following four languages: English, French, German and Spanish, giving 2.5M examples in each language.
+
+The model can be used for mask-filling as shown in this demo. The caption can be present or written in any of the following: English, French, German and Spanish.
+
+For more details, click on `Usage` or `Article` 🤗 above.

sections/mlm_usage.md

@@ -0,0 +1,7 @@
+- This demo loads the `FlaxCLIPVisionBertForMaskedLM` present in the `model` directory of this repository. The checkpoint is loaded from [`flax-community/clip-vision-bert-cc12m-70k`](https://huggingface.co/flax-community/clip-vision-bert-cc12m-70k) which is pre-trained checkpoint with 70k steps. 100 random validation set examples are present in the `cc12m_data/vqa_val.tsv` with respective images in the `cc12m_data/images_data` directory.
+
+- We provide `English Translation` of the caption for users who are not well-acquainted with the other languages. This is done using `mtranslate` to keep things flexible enough and needs internet connection as it uses the Google Translate API.
+
+- The model predicts the scores for tokens from the `bert-base-multilingual-uncased` checkpoint.
+
+- The top-5 predictions are displayed below and their respective confidence scores are shown in form of a bar plot.

sections/{intro.md → vqa_intro.md}

@@ -1,5 +1,5 @@
-This demo uses a [ViTBert model checkpoint](https://huggingface.co/flax-community/multilingual-vqa-pt-60k-ft/tree/main/ckpt-5999) fine-tuned on a [MarianMT](https://huggingface.co/transformers/model_doc/marian.html)-translated version of the [VQA v2 dataset](https://visualqa.org/challenge.html). The fine-tuning is performed after pre-training using text-only Masked LM on approximately 10 million image-text pairs taken from the [Conceptual 12M dataset](https://github.com/google-research-datasets/conceptual-12m) translated using [MBart](https://huggingface.co/transformers/model_doc/mbart.html). The translations are performed in the following four languages: English, French, German and Spanish.
+This demo uses a [CLIP-Vision-Bert model checkpoint](https://huggingface.co/flax-community/clip-vision-bert-vqa-ft-6k) fine-tuned on a [MarianMT](https://huggingface.co/transformers/model_doc/marian.html)-translated version of the [VQA v2 dataset](https://visualqa.org/challenge.html). The fine-tuning is performed after pre-training using text-only Masked LM on approximately 10 million image-text pairs taken from the [Conceptual 12M dataset](https://github.com/google-research-datasets/conceptual-12m) translated using [MBart](https://huggingface.co/transformers/model_doc/mbart.html). The translations are performed in the following four languages: English, French, German and Spanish.
 
 The model predicts one out of 3129 classes in English which can be found [here](https://huggingface.co/spaces/flax-community/Multilingual-VQA/blob/main/answer_reverse_mapping.json), and then the translated versions are provided based on the language chosen as `Answer Language`. The question can be present or written in any of the following: English, French, German and Spanish.
 
-For more details, click on `Usage` or `Article` 🤗 below.
+For more details, click on `Usage` or `Article` 🤗 above.

sections/{usage.md → vqa_usage.md}

@@ -1,4 +1,4 @@
-- This demo loads the `FlaxCLIPVisionBertForSequenceClassificationModel` present in the `model` directory of this repository. The checkpoint is loaded from `ckpt/ckpt-60k-5999` which is pre-trained checkpoint with 60k steps and 5999 fine-tuning steps. 100 random validation set examples are present in the `dummy_vqa_multilingual.tsv` with respective images in the `images/val2014` directory.
+- This demo loads the `FlaxCLIPVisionBertForSequenceClassification` present in the `model` directory of this repository. The checkpoint is loaded from [`flax-community/clip-vision-bert-vqa-ft-6k`](https://huggingface.co/flax-community/clip-vision-bert-vqa-ft-6k) which is pre-trained checkpoint with 60k steps and 6k fine-tuning steps. 100 random validation set examples are present in the `dummy_vqa_multilingual.tsv` with respective images in the `images/val2014` directory.
 
 - We provide `English Translation` of the question for users who are not well-acquainted with the other languages. This is done using `mtranslate` to keep things flexible enough and needs internet connection as it uses the Google Translate API.