pyserini/2cr/mrtydi.yaml

conditions:
  # mDPR, tied encoders, pFT w/ MS MARCO, FT all
  - name: mdpr-tied-pft-msmarco-ft-all.ar
    eval_key: mrtydi-v1.1-arabic
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9505
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.6913
            R@100: 0.9165
      - split: test
        scores:
          - MRR@100: 0.6949
            R@100: 0.9004
  - name: mdpr-tied-pft-msmarco-ft-all.bn
    eval_key: mrtydi-v1.1-bengali
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9620
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.5897
            R@100: 0.8977
      - split: test
        scores:
          - MRR@100: 0.6228
            R@100: 0.9550
  - name: mdpr-tied-pft-msmarco-ft-all.en
    eval_key: mrtydi-v1.1-english
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.8278
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.5357
            R@100: 0.8884
      - split: test
        scores:
          - MRR@100: 0.4916
            R@100: 0.8414
  - name: mdpr-tied-pft-msmarco-ft-all.fi
    eval_key: mrtydi-v1.1-finnish
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9577
            R@100: 0.9997
      - split: dev
        scores:
          - MRR@100: 0.6626
            R@100: 0.9171
      - split: test
        scores:
          - MRR@100: 0.5595
            R@100: 0.8563
  - name: mdpr-tied-pft-msmarco-ft-all.id
    eval_key: mrtydi-v1.1-indonesian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9469
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.6294
            R@100: 0.9150
      - split: test
        scores:
          - MRR@100: 0.5783
            R@100: 0.8609
  - name: mdpr-tied-pft-msmarco-ft-all.ja
    eval_key: mrtydi-v1.1-japanese
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.8802
            R@100: 0.9997
      - split: dev
        scores:
          - MRR@100: 0.5505
            R@100: 0.8696
      - split: test
        scores:
          - MRR@100: 0.5007
            R@100: 0.8130
  - name: mdpr-tied-pft-msmarco-ft-all.ko
    eval_key: mrtydi-v1.1-korean
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9195
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.5645
            R@100: 0.8663
      - split: test
        scores:
          - MRR@100: 0.4861
            R@100: 0.7854
  - name: mdpr-tied-pft-msmarco-ft-all.ru
    eval_key: mrtydi-v1.1-russian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.8473
            R@100: 0.9994
      - split: dev
        scores:
          - MRR@100: 0.5104
            R@100: 0.8720
      - split: test
        scores:
          - MRR@100: 0.5161
            R@100: 0.8432
  - name: mdpr-tied-pft-msmarco-ft-all.sw
    eval_key: mrtydi-v1.1-swahili
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9515
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.6404
            R@100: 0.9018
      - split: test
        scores:
          - MRR@100: 0.6438
            R@100: 0.8756
  - name: mdpr-tied-pft-msmarco-ft-all.te
    eval_key: mrtydi-v1.1-telugu
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9679
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.7962
            R@100: 0.9593
      - split: test
        scores:
          - MRR@100: 0.8908
            R@100: 0.9659
  - name: mdpr-tied-pft-msmarco-ft-all.th
    eval_key: mrtydi-v1.1-thai
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.9504
            R@100: 1.0000
      - split: dev
        scores:
          - MRR@100: 0.6670
            R@100: 0.9114
      - split: test
        scores:
          - MRR@100: 0.6175
            R@100: 0.8826

  # mDPR, tied encoders, pFT w/ MS MARCO
  - name: mdpr-tied-pft-msmarco.ar
    eval_key: mrtydi-v1.1-arabic
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3957
            R@100: 0.7818
      - split: dev
        scores:
          - MRR@100: 0.3978
            R@100: 0.7778
      - split: test
        scores:
          - MRR@100: 0.4414
            R@100: 0.7971
  - name: mdpr-tied-pft-msmarco.bn
    eval_key: mrtydi-v1.1-bengali
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2920
            R@100: 0.7323
      - split: dev
        scores:
          - MRR@100: 0.2993
            R@100: 0.7318
      - split: test
        scores:
          - MRR@100: 0.3969
            R@100: 0.7838
  - name: mdpr-tied-pft-msmarco.en
    eval_key: mrtydi-v1.1-english
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3374
            R@100: 0.8111
      - split: dev
        scores:
          - MRR@100: 0.3451
            R@100: 0.7995
      - split: test
        scores:
          - MRR@100: 0.3270
            R@100: 0.7536
  - name: mdpr-tied-pft-msmarco.fi
    eval_key: mrtydi-v1.1-finnish
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3668
            R@100: 0.7337
      - split: dev
        scores:
          - MRR@100: 0.3636
            R@100: 0.7371
      - split: test
        scores:
          - MRR@100: 0.2750
            R@100: 0.6471
  - name: mdpr-tied-pft-msmarco.id
    eval_key: mrtydi-v1.1-indonesian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2794
            R@100: 0.7044
      - split: dev
        scores:
          - MRR@100: 0.2853
            R@100: 0.7198
      - split: test
        scores:
          - MRR@100: 0.3520
            R@100: 0.7356
  - name: mdpr-tied-pft-msmarco.ja
    eval_key: mrtydi-v1.1-japanese
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3089
            R@100: 0.7603
      - split: dev
        scores:
          - MRR@100: 0.3108
            R@100: 0.7597
      - split: test
        scores:
          - MRR@100: 0.3107
            R@100: 0.7317
  - name: mdpr-tied-pft-msmarco.ko
    eval_key: mrtydi-v1.1-korean
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3003
            R@100: 0.6907
      - split: dev
        scores:
          - MRR@100: 0.3017
            R@100: 0.7046
      - split: test
        scores:
          - MRR@100: 0.2820
            R@100: 0.6172
  - name: mdpr-tied-pft-msmarco.ru
    eval_key: mrtydi-v1.1-russian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2856
            R@100: 0.7305
      - split: dev
        scores:
          - MRR@100: 0.2943
            R@100: 0.7404
      - split: test
        scores:
          - MRR@100: 0.3561
            R@100: 0.7432
  - name: mdpr-tied-pft-msmarco.sw
    eval_key: mrtydi-v1.1-swahili
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2491
            R@100: 0.5195
      - split: dev
        scores:
          - MRR@100: 0.2447
            R@100: 0.5266
      - split: test
        scores:
          - MRR@100: 0.3418
            R@100: 0.6343
  - name: mdpr-tied-pft-msmarco.te
    eval_key: mrtydi-v1.1-telugu
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3059
            R@100: 0.7510
      - split: dev
        scores:
          - MRR@100: 0.2995
            R@100: 0.7355
      - split: test
        scores:
          - MRR@100: 0.3102
            R@100: 0.7817
  - name: mdpr-tied-pft-msmarco.th
    eval_key: mrtydi-v1.1-thai
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2334
            R@100: 0.5851
      - split: dev
        scores:
          - MRR@100: 0.2407
            R@100: 0.5795
      - split: test
        scores:
          - MRR@100: 0.2693
            R@100: 0.5945

  # mDPR, tied encoders, pFT w/ NQ
  - name: mdpr-tied-pft-nq.ar
    eval_key: mrtydi-v1.1-arabic
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2087
            R@100: 0.5854
      - split: dev
        scores:
          - MRR@100: 0.2132
            R@100: 0.5868
      - split: test
        scores:
          - MRR@100: 0.2214
            R@100: 0.6001
  - name: mdpr-tied-pft-nq.bn
    eval_key: mrtydi-v1.1-bengali
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2371
            R@100: 0.6281
      - split: dev
        scores:
          - MRR@100: 0.2414
            R@100: 0.6409
      - split: test
        scores:
          - MRR@100: 0.2535
            R@100: 0.7072
  - name: mdpr-tied-pft-nq.en
    eval_key: mrtydi-v1.1-english
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2441
            R@100: 0.7217
      - split: dev
        scores:
          - MRR@100: 0.2359
            R@100: 0.7187
      - split: test
        scores:
          - MRR@100: 0.2433
            R@100: 0.6893
  - name: mdpr-tied-pft-nq.fi
    eval_key: mrtydi-v1.1-finnish
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2996
            R@100: 0.6787
      - split: dev
        scores:
          - MRR@100: 0.3252
            R@100: 0.7037
      - split: test
        scores:
          - MRR@100: 0.2444
            R@100: 0.6401
  - name: mdpr-tied-pft-nq.id
    eval_key: mrtydi-v1.1-indonesian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2706
            R@100: 0.7322
      - split: dev
        scores:
          - MRR@100: 0.2719
            R@100: 0.7394
      - split: test
        scores:
          - MRR@100: 0.2815
            R@100: 0.6914
  - name: mdpr-tied-pft-nq.ja
    eval_key: mrtydi-v1.1-japanese
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2165
            R@100: 0.6043
      - split: dev
        scores:
          - MRR@100: 0.2299
            R@100: 0.6239
      - split: test
        scores:
          - MRR@100: 0.2058
            R@100: 0.5734
  - name: mdpr-tied-pft-nq.ko
    eval_key: mrtydi-v1.1-korean
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2527
            R@100: 0.6556
      - split: dev
        scores:
          - MRR@100: 0.2680
            R@100: 0.6271
      - split: test
        scores:
          - MRR@100: 0.2234
            R@100: 0.5499
  - name: mdpr-tied-pft-nq.ru
    eval_key: mrtydi-v1.1-russian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2160
            R@100: 0.6262
      - split: dev
        scores:
          - MRR@100: 0.2263
            R@100: 0.6444
      - split: test
        scores:
          - MRR@100: 0.2501
            R@100: 0.6181
  - name: mdpr-tied-pft-nq.sw
    eval_key: mrtydi-v1.1-swahili
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2383
            R@100: 0.5707
      - split: dev
        scores:
          - MRR@100: 0.2543
            R@100: 0.6138
      - split: test
        scores:
          - MRR@100: 0.2621
            R@100: 0.5965
  - name: mdpr-tied-pft-nq.te
    eval_key: mrtydi-v1.1-telugu
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1483
            R@100: 0.4162
      - split: dev
        scores:
          - MRR@100: 0.1494
            R@100: 0.3967
      - split: test
        scores:
          - MRR@100: 0.0970
            R@100: 0.2454
  - name: mdpr-tied-pft-nq.th
    eval_key: mrtydi-v1.1-thai
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1426
            R@100: 0.4717
      - split: dev
        scores:
          - MRR@100: 0.1618
            R@100: 0.4637
      - split: test
        scores:
          - MRR@100: 0.1575
            R@100: 0.4550

  # mDPR, split encoders, pFT w/ NQ
  - name: mdpr-split-pft-nq.ar
    eval_key: mrtydi-v1.1-arabic
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2510
            R@100: 0.6384
      - split: dev
        scores:
          - MRR@100: 0.2449
            R@100: 0.6334
      - split: test
        scores:
          - MRR@100: 0.2907
            R@100: 0.6502
  - name: mdpr-split-pft-nq.bn
    eval_key: mrtydi-v1.1-bengali
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2293
            R@100: 0.6454
      - split: dev
        scores:
          - MRR@100: 0.2367
            R@100: 0.6511
      - split: test
        scores:
          - MRR@100: 0.2911
            R@100: 0.7793
  - name: mdpr-split-pft-nq.en
    eval_key: mrtydi-v1.1-english
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2862
            R@100: 0.7372
      - split: dev
        scores:
          - MRR@100: 0.2821
            R@100: 0.7437
      - split: test
        scores:
          - MRR@100: 0.2907
            R@100: 0.6779
  - name: mdpr-split-pft-nq.fi
    eval_key: mrtydi-v1.1-finnish
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2473
            R@100: 0.6289
      - split: dev
        scores:
          - MRR@100: 0.2466
            R@100: 0.6283
      - split: test
        scores:
          - MRR@100: 0.2050
            R@100: 0.5680
  - name: mdpr-split-pft-nq.id
    eval_key: mrtydi-v1.1-indonesian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2351
            R@100: 0.6952
      - split: dev
        scores:
          - MRR@100: 0.2475
            R@100: 0.7181
      - split: test
        scores:
          - MRR@100: 0.2705
            R@100: 0.6848
  - name: mdpr-split-pft-nq.ja
    eval_key: mrtydi-v1.1-japanese
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1967
            R@100: 0.5983
      - split: dev
        scores:
          - MRR@100: 0.2055
            R@100: 0.6142
      - split: test
        scores:
          - MRR@100: 0.2119
            R@100: 0.5840
  - name: mdpr-split-pft-nq.ko
    eval_key: mrtydi-v1.1-korean
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2383
            R@100: 0.6180
      - split: dev
        scores:
          - MRR@100: 0.2343
            R@100: 0.6238
      - split: test
        scores:
          - MRR@100: 0.2345
            R@100: 0.5325
  - name: mdpr-split-pft-nq.ru
    eval_key: mrtydi-v1.1-russian
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2317
            R@100: 0.6534
      - split: dev
        scores:
          - MRR@100: 0.2490
            R@100: 0.6553
      - split: test
        scores:
          - MRR@100: 0.2820
            R@100: 0.6474
  - name: mdpr-split-pft-nq.sw
    eval_key: mrtydi-v1.1-swahili
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1457
            R@100: 0.4481
      - split: dev
        scores:
          - MRR@100: 0.1547
            R@100: 0.4724
      - split: test
        scores:
          - MRR@100: 0.1883
            R@100: 0.5281
  - name: mdpr-split-pft-nq.te
    eval_key: mrtydi-v1.1-telugu
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1489
            R@100: 0.4905
      - split: dev
        scores:
          - MRR@100: 0.1503
            R@100: 0.4934
      - split: test
        scores:
          - MRR@100: 0.1099
            R@100: 0.3661
  - name: mdpr-split-pft-nq.th
    eval_key: mrtydi-v1.1-thai
    command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-nq --output $output --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1603
            R@100: 0.4983
      - split: dev
        scores:
          - MRR@100: 0.1584
            R@100: 0.5083
      - split: test
        scores:
          - MRR@100: 0.1709
            R@100: 0.5146

  # BM25
  - name: bm25.ar
    eval_key: mrtydi-v1.1-arabic
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ar --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3356
            R@100: 0.7944
      - split: dev
        scores:
          - MRR@100: 0.3462
            R@100: 0.7872
      - split: test
        scores:
          - MRR@100: 0.3682
            R@100: 0.7928
  - name: bm25.bn
    eval_key: mrtydi-v1.1-bengali
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language bn --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3566
          - R@100: 0.8336
      - split: dev
        scores:
          - MRR@100: 0.3385
          - R@100: 0.8432
      - split: test
        scores:
          - MRR@100: 0.4182
          - R@100: 0.8694
  - name: bm25.en
    eval_key: mrtydi-v1.1-english
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language en --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.1592
          - R@100: 0.5785
      - split: dev
        scores:
          - MRR@100: 0.1685
          - R@100: 0.6196
      - split: test
        scores:
          - MRR@100: 0.1404
          - R@100: 0.5365
  - name: bm25.fi
    eval_key: mrtydi-v1.1-finnish
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language fi --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.4101
          - R@100: 0.8198
      - split: dev
        scores:
          - MRR@100: 0.4136
          - R@100: 0.8285
      - split: test
        scores:
          - MRR@100: 0.2836
          - R@100: 0.7196
  - name: bm25.id
    eval_key: mrtydi-v1.1-indonesian
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language id --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2972
          - R@100: 0.7948
      - split: dev
        scores:
          - MRR@100: 0.2937
          - R@100: 0.7827
      - split: test
        scores:
          - MRR@100: 0.3762
          - R@100: 0.8426
  - name: bm25.ja
    eval_key: mrtydi-v1.1-japanese
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ja --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2262
          - R@100: 0.7290
      - split: dev
        scores:
          - MRR@100: 0.2250
          - R@100: 0.7252
      - split: test
        scores:
          - MRR@100: 0.2125
          - R@100: 0.6431
  - name: bm25.ko
    eval_key: mrtydi-v1.1-korean
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ko --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2596
          - R@100: 0.6178
      - split: dev
        scores:
          - MRR@100: 0.2888
          - R@100: 0.6733
      - split: test
        scores:
          - MRR@100: 0.2848
          - R@100: 0.6188
  - name: bm25.ru
    eval_key: mrtydi-v1.1-russian
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ru --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2229
          - R@100: 0.5779
      - split: dev
        scores:
          - MRR@100: 0.2202
          - R@100: 0.5760
      - split: test
        scores:
          - MRR@100: 0.3163
          - R@100: 0.6541
  - name: bm25.sw
    eval_key: mrtydi-v1.1-swahili
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language sw --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.2610
          - R@100: 0.5903
      - split: dev
        scores:
          - MRR@100: 0.2693
          - R@100: 0.5789
      - split: test
        scores:
          - MRR@100: 0.3893
          - R@100: 0.7642
  - name: bm25.te
    eval_key: mrtydi-v1.1-telugu
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language te --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.4204
          - R@100: 0.8229
      - split: dev
        scores:
          - MRR@100: 0.4269
          - R@100: 0.8362
      - split: test
        scores:
          - MRR@100: 0.5283
          - R@100: 0.8971
  - name: bm25.th
    eval_key: mrtydi-v1.1-thai
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language th --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai --output $output --bm25 --hits 100
    splits:
      - split: train
        scores:
          - MRR@100: 0.3543
          - R@100: 0.8349
      - split: dev
        scores:
          - MRR@100: 0.3586
          - R@100: 0.8536
      - split: test
        scores:
          - MRR@100: 0.4012
          - R@100: 0.8529