model update

README.md

@@ -46,236 +46,242 @@ model-index:
     - name: MoverScore
       type: moverscore
       value: 0.6499011626820898
+    - name: QAAlignedF1Score (BERTScore)
+      type: qa_aligned_f1_score_bertscore
+      value: 0.9553719665829591
+    - name: QAAlignedF1Score (MoverScore)
+      type: qa_aligned_f1_score_moverscore
+      value: 0.7082452551815105
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
-      name: lmqg/qg_squadshifts
-      type: reddit
-      args: reddit
+      name: lmqg/qg_subjqa
+      type: tripadvisor
+      args: tripadvisor
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.059525104157825456
+      value: 8.380171318718442e-07
     - name: ROUGE-L
       type: rouge-l
-      value: 0.22365090580055863
+      value: 0.1402922852924756
     - name: METEOR
       type: meteor
-      value: 0.21499800504546457
+      value: 0.1372146070365174
     - name: BERTScore
       type: bertscore
-      value: 0.9095144685254328
+      value: 0.8891002409937424
     - name: MoverScore
       type: moverscore
-      value: 0.6059332247878408
+      value: 0.5604572211470809
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_squadshifts
-      type: new_wiki
-      args: new_wiki
+      type: amazon
+      args: amazon
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.11118273173452982
+      value: 0.06530369842068952
     - name: ROUGE-L
       type: rouge-l
-      value: 0.2967546690273089
+      value: 0.25030985091008146
     - name: METEOR
       type: meteor
-      value: 0.27315087810722966
+      value: 0.2229994442645732
     - name: BERTScore
       type: bertscore
-      value: 0.9322739617807421
+      value: 0.9092814804525936
     - name: MoverScore
       type: moverscore
-      value: 0.6623000084761579
+      value: 0.6086538514008419
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_subjqa
-      type: tripadvisor
-      args: tripadvisor
+      type: books
+      args: books
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 8.380171318718442e-07
+      value: 0.006278914808207679
     - name: ROUGE-L
       type: rouge-l
-      value: 0.1402922852924756
+      value: 0.12368226019088967
     - name: METEOR
       type: meteor
-      value: 0.1372146070365174
+      value: 0.11576293675813865
     - name: BERTScore
       type: bertscore
-      value: 0.8891002409937424
+      value: 0.8807110440044503
     - name: MoverScore
       type: moverscore
-      value: 0.5604572211470809
+      value: 0.5555905941686486
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
-      name: lmqg/qg_squadshifts
-      type: nyt
-      args: nyt
+      name: lmqg/qg_subjqa
+      type: restaurants
+      args: restaurants
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.08117757543966063
+      value: 1.1301750984972448e-06
     - name: ROUGE-L
       type: rouge-l
-      value: 0.25292097720734297
+      value: 0.13083168975354642
     - name: METEOR
       type: meteor
-      value: 0.25254205113198686
+      value: 0.12419733006916912
     - name: BERTScore
       type: bertscore
-      value: 0.9249009759439454
+      value: 0.8797711839570719
     - name: MoverScore
       type: moverscore
-      value: 0.6406329128556304
+      value: 0.5542757411268555
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_subjqa
-      type: restaurants
-      args: restaurants
+      type: movies
+      args: movies
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 1.1301750984972448e-06
+      value: 1.0121579426501661e-06
     - name: ROUGE-L
       type: rouge-l
-      value: 0.13083168975354642
+      value: 0.12508697028506718
     - name: METEOR
       type: meteor
-      value: 0.12419733006916912
+      value: 0.11862284941640638
     - name: BERTScore
       type: bertscore
-      value: 0.8797711839570719
+      value: 0.8748829724726739
     - name: MoverScore
       type: moverscore
-      value: 0.5542757411268555
+      value: 0.5528899173535703
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_subjqa
-      type: electronics
-      args: electronics
+      type: grocery
+      args: grocery
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.00866799444965211
+      value: 0.00528043272450429
     - name: ROUGE-L
       type: rouge-l
-      value: 0.1601628874804186
+      value: 0.12343711316491492
     - name: METEOR
       type: meteor
-      value: 0.15348605312210778
+      value: 0.15133496445452477
     - name: BERTScore
       type: bertscore
-      value: 0.8783386920680519
+      value: 0.8778951253890991
     - name: MoverScore
       type: moverscore
-      value: 0.5634845371093992
+      value: 0.5701949938103265
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
-      name: lmqg/qg_subjqa
-      type: books
-      args: books
+      name: lmqg/qg_squadshifts
+      type: nyt
+      args: nyt
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.006278914808207679
+      value: 0.08117757543966063
     - name: ROUGE-L
       type: rouge-l
-      value: 0.12368226019088967
+      value: 0.25292097720734297
     - name: METEOR
       type: meteor
-      value: 0.11576293675813865
+      value: 0.25254205113198686
     - name: BERTScore
       type: bertscore
-      value: 0.8807110440044503
+      value: 0.9249009759439454
     - name: MoverScore
       type: moverscore
-      value: 0.5555905941686486
+      value: 0.6406329128556304
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_subjqa
-      type: movies
-      args: movies
+      type: electronics
+      args: electronics
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 1.0121579426501661e-06
+      value: 0.00866799444965211
     - name: ROUGE-L
       type: rouge-l
-      value: 0.12508697028506718
+      value: 0.1601628874804186
     - name: METEOR
       type: meteor
-      value: 0.11862284941640638
+      value: 0.15348605312210778
     - name: BERTScore
       type: bertscore
-      value: 0.8748829724726739
+      value: 0.8783386920680519
     - name: MoverScore
       type: moverscore
-      value: 0.5528899173535703
+      value: 0.5634845371093992
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
-      name: lmqg/qg_subjqa
-      type: grocery
-      args: grocery
+      name: lmqg/qg_squadshifts
+      type: new_wiki
+      args: new_wiki
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.00528043272450429
+      value: 0.11118273173452982
     - name: ROUGE-L
       type: rouge-l
-      value: 0.12343711316491492
+      value: 0.2967546690273089
     - name: METEOR
       type: meteor
-      value: 0.15133496445452477
+      value: 0.27315087810722966
     - name: BERTScore
       type: bertscore
-      value: 0.8778951253890991
+      value: 0.9322739617807421
     - name: MoverScore
       type: moverscore
-      value: 0.5701949938103265
+      value: 0.6623000084761579
   - task:
       name: Text2text Generation
       type: text2text-generation
     dataset:
       name: lmqg/qg_squadshifts
-      type: amazon
-      args: amazon
+      type: reddit
+      args: reddit
     metrics:
     - name: BLEU4
       type: bleu4
-      value: 0.06530369842068952
+      value: 0.059525104157825456
     - name: ROUGE-L
       type: rouge-l
-      value: 0.25030985091008146
+      value: 0.22365090580055863
     - name: METEOR
       type: meteor
-      value: 0.2229994442645732
+      value: 0.21499800504546457
     - name: BERTScore
       type: bertscore
-      value: 0.9092814804525936
+      value: 0.9095144685254328
     - name: MoverScore
       type: moverscore
-      value: 0.6086538514008419
+      value: 0.6059332247878408
 ---
 
 # Model Card of `lmqg/bart-large-squad`
@@ -342,21 +348,28 @@ question = pipe('<hl> Beyonce <hl> further expanded her acting career, starring
 | [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) | default | 0.262 | 0.538 | 0.271 | 0.91 | 0.65 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval/metric.first.sentence.paragraph_answer.question.lmqg_qg_squad.default.json) | 
 
 
+### Metrics (QAG)
+
+| Dataset | Type | QA Aligned F1 Score (BERTScore) | QA Aligned F1 Score (MoverScore) | Link |
+|:--------|:-----|--------------------------------:|---------------------------------:|-----:|
+| [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) | default | 0.955 | 0.708 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval/metric.first.answer.paragraph.questions_answers.lmqg_qg_squad.default.json) | 
+    
+
 
 ### Out-of-domain Metrics
         
 | Dataset | Type | BLEU4 | ROUGE-L | METEOR | BERTScore | MoverScore | Link |
 |:--------|:-----|------:|--------:|-------:|----------:|-----------:|-----:|
-| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | reddit | 0.06 | 0.224 | 0.215 | 0.91 | 0.606 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json) |
-| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | new_wiki | 0.111 | 0.297 | 0.273 | 0.932 | 0.662 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json) |
 | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | tripadvisor | 0.0 | 0.14 | 0.137 | 0.889 | 0.56 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.json) |
-| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | nyt | 0.081 | 0.253 | 0.253 | 0.925 | 0.641 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json) |
-| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | restaurants | 0.0 | 0.131 | 0.124 | 0.88 | 0.554 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json) |
-| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | electronics | 0.009 | 0.16 | 0.153 | 0.878 | 0.563 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | amazon | 0.065 | 0.25 | 0.223 | 0.909 | 0.609 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json) |
 | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | books | 0.006 | 0.124 | 0.116 | 0.881 | 0.556 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.books.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | restaurants | 0.0 | 0.131 | 0.124 | 0.88 | 0.554 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json) |
 | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | movies | 0.0 | 0.125 | 0.119 | 0.875 | 0.553 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.movies.json) |
 | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | grocery | 0.005 | 0.123 | 0.151 | 0.878 | 0.57 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.grocery.json) |
-| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | amazon | 0.065 | 0.25 | 0.223 | 0.909 | 0.609 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | nyt | 0.081 | 0.253 | 0.253 | 0.925 | 0.641 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | electronics | 0.009 | 0.16 | 0.153 | 0.878 | 0.563 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | new_wiki | 0.111 | 0.297 | 0.273 | 0.932 | 0.662 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | reddit | 0.06 | 0.224 | 0.215 | 0.91 | 0.606 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json) |
 
 
 ## Training hyperparameters