first version

.gitattributes

@@ -29,3 +29,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,86 @@
+import json
+import gradio as gr
+import os
+import random
+# load data
+
+folder_path="./data/fid-light-xl/"
+
+tasks = [
+"fever",
+"hotpotqa",
+"nq",
+"trivia_qa",
+"structured_zeroshot",
+"trex",
+]
+
+labels = [
+"FEVER",
+"HotpotQA",
+"NQ",
+"TriviaQA",
+"zsRE",
+"T-REx",
+]
+
+modes = ["Perfect (Both text & provenance are correct)", "Double Failure (Both text & provenance are wrong)", 
+"Correct Text, Wrong Provenance (R-Precision < 1)", "Wrong Text, Correct Provenance (R-Precision == 1)"]
+modes_map = {modes[0]:"perfect", modes[1]:"double_failure", modes[2]:"wrong_passage", modes[3]:"wrong_text"}
+
+data = {}
+total_num_per_task = {}
+for task in tasks:
+    data[task] = {}
+    total_num_per_task[task] = 0
+    for mode in modes:
+        data[task][mode] = []
+        with open(os.path.join(folder_path,"examples_"+task+"_"+modes_map[mode]+".json")) as f:
+            data[task][mode] = json.load(f)
+            total_num_per_task[task] += len(data[task][mode])
+
+def render_examples(selected_mode):
+    all_rendered=[]
+    for i,task in enumerate(tasks):
+            examples = random.sample(data[task][selected_mode], 10)
+            rendered_examples = "## Statistics\nNumber of examples in this category: **"+str(len(data[task][selected_mode]))+"** ("+ \
+            str(round(len(data[task][selected_mode])/total_num_per_task[task]*100,2)) +"%)\n\n----\n"
+            for example in examples:
+                
+                def render_prov(provenance):
+                    t=""
+                    for i, prov in enumerate(provenance):
+                        t+=f"**{i+1})** {prov['text']}\n\n"
+                    if len(provenance) == 0:
+                        t+="<< No provenance returned >>\n\n"
+                    return t
+
+                rendered_examples += "### Query \n"+example["query"]+ "\n"+ \
+                "### Target Text"+ "\n" + "\n\n".join(example["target_text"])+ "\n"
+
+                if "target_provenance" in example:
+                    rendered_examples+="### Target Provenance"+ "\n"+ render_prov(example["target_provenance"])+ "\n"
+
+                rendered_examples += "### Output Text"+ "\n"+ str(example["output_text"])+ "\n"+ \
+                "### Output Provenance"+ "\n"+ render_prov(example["output_provenance"])+ "\n"+ \
+                "\n----\n"
+
+            all_rendered.append(rendered_examples)
+    return all_rendered
+
+with gr.Blocks() as interface:
+    gr.Markdown(
+        "# FiD-Light Output Explorer \n"+
+        "This is a random data output explorer for the FiD-Light model on six KILT tasks (static dev set results).\n\n"+
+        "*Every time you click on a result split we load up a new set of 10 random examples from this split for all the tasks.*")
+    selected = gr.Radio(modes, value=modes[0], label="Result Split",interactive=True)
+    text_fields = []
+    init_data = render_examples(selected.value)
+    for i,task in enumerate(tasks):
+        with gr.Tab(labels[i]):
+            text_fields.append(gr.Markdown(init_data[i]))
+
+    selected.change(fn=render_examples, inputs=selected, outputs=text_fields)
+
+
+interface.launch()

data/fid-light-xl/examples_fever_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f82d268b96fe0d033d919dd15fc088e6aa263f9556004defe267effb9a03b3f
+size 606446

data/fid-light-xl/examples_fever_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95f67986b46f86d9f7ac3e24dd3ee03bf16908f201fdfaeb0535d2ea47d824d5
+size 9316478

data/fid-light-xl/examples_fever_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b93921d430e1ff6003c7fb8f3be00745626eb833ffb0d5973cf3bb80435ef240
+size 6237847

data/fid-light-xl/examples_fever_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:213d05496a016036d729b43e3f81091a88e13d648edd1d4e1419de150572003a
+size 515883

data/fid-light-xl/examples_hotpotqa_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edea6033b3069f30f132439ff82fe3f6221c21e2ae499bb86dc7cefbc16d8e11
+size 6109804

data/fid-light-xl/examples_hotpotqa_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:753ff346246fdcdffd1740741488f340800596a99277c1b5bace50b9d034297f
+size 2850921

data/fid-light-xl/examples_hotpotqa_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb82cdee0d0c9857d5b49f433d91b3cc9b33c6e594f66f8e9b061df76b1fe48
+size 2468631

data/fid-light-xl/examples_hotpotqa_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ffe0f3b81602c700094cda3e2f8ece2e841521463f959f78d957539dd710504
+size 1704857

data/fid-light-xl/examples_nq_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69230904c5e1d1691873a0721c9c40432a8c5498509006d6d29912e3a3ac5697
+size 1775024

data/fid-light-xl/examples_nq_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:932c40101b6a1a63a3f70f24988ae8088b1b42fd8f1d517b31e78fafd808673b
+size 1405505

data/fid-light-xl/examples_nq_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3e11e481f40eb60333dd7f79c01df58bc8cddc112ad00fc5f9ee2757411f8a5
+size 1410061

data/fid-light-xl/examples_nq_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e16392f4b4978cc527f33f8acffb984f1c2449ff802b8b641450e5bd38239c1
+size 412578

data/fid-light-xl/examples_structured_zeroshot_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6356fa2832e9dbd6b911b366c3465ebe842a6a0849b4e791914f4cae6fe425fc
+size 609081

data/fid-light-xl/examples_structured_zeroshot_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7febc9502f8a37c58b1e75e92376653cef6decc8d7724a24b9f55fd60d9a3a08
+size 2675759

data/fid-light-xl/examples_structured_zeroshot_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa2bfc71932d2d865431ea588b0f198a57f2558275496536b6b31f9632a73e7
+size 202664

data/fid-light-xl/examples_structured_zeroshot_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b13fabffe200e31a2747be38f939414a92164970105c040b86145fd1befce88b
+size 467929

data/fid-light-xl/examples_trex_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b2252234ab014faca7256e7e7bfdaf35558b303a35703fe7c2ad813fa8f8e6
+size 820891

data/fid-light-xl/examples_trex_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:306ffb86c9651c0b68a7563d45a6d1012b1143becc5ca3c66849ebdc774db74f
+size 3093308

data/fid-light-xl/examples_trex_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54af5eb0793b699332d0b86355a53463c476dce65634da9230867b15c1fde49d
+size 1914568

data/fid-light-xl/examples_trex_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98aa636cfc17eb315acf2fc59ff5dbe4d028e84e5b8a62b8d263005ad5cfdf4
+size 217426

data/fid-light-xl/examples_trivia_qa_double_failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48bf33ae75f3b580471b24b1f25f5632c06cbd209a671d42e4fc9d826131a3ef
+size 4247896

data/fid-light-xl/examples_trivia_qa_perfect.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1a73cb61596a47823693ba46555a9d8752d65db9164cf05c8e05b5a2b38c18
+size 3134464

data/fid-light-xl/examples_trivia_qa_wrong_passage.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb5e1ffae74755072698eb990253b19b455c5a60c60b32fb3a7a5340171fdac9
+size 17325722

data/fid-light-xl/examples_trivia_qa_wrong_text.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:663afee7920818126328cd6e347c7a35a0d3afefec0ea48a5976cd97ed46bed2
+size 346876