pyserini/2cr/msmarco-v2-doc.yaml

conditions:
  - name: bm25-doc-default
    display: BM25 doc (k1=0.9, b=0.4)
    display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (1a)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-slim --topics $topics --output $output --bm25
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.1572
            R@1K: 0.8054
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1659
            R@1K: 0.8029
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2126
            nDCG@10: 0.5116
            MRR@100: 0.8367
            R@100: 0.3195
            R@1K: 0.6739
  - name: bm25-doc-segmented-default
    display: BM25 doc segmented (k1=0.9, b=0.4)
    display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (1b)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-slim --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.1896
            R@1K: 0.8542
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1930
            R@1K: 0.8549
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2436
            nDCG@10: 0.5776
            MRR@100: 0.8937
            R@100: 0.3478
            R@1K: 0.6930
  - name: bm25-rm3-doc-default
    display: BM25+RM3 doc (k1=0.9, b=0.4)
    display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (1c)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-full --topics $topics --output $output --bm25 --rm3
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.0974
            R@1K: 0.7699
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1033
            R@1K: 0.7736
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2452
            nDCG@10: 0.5304
            MRR@100: 0.7914
            R@100: 0.3376
            R@1K: 0.7341
  - name: bm25-rm3-doc-segmented-default
    display: BM25+RM3 doc segmented (k1=0.9, b=0.4)
    display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (1d)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-full --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.1660
            R@1K: 0.8608
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1702
            R@1K: 0.8639
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2936
            nDCG@10: 0.6189
            MRR@100: 0.9076
            R@100: 0.3890
            R@1K: 0.7678
  - name: bm25-d2q-t5-doc-default
    display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4)
    display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (2a)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5 --topics $topics --output $output --bm25
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2011
            R@1K: 0.8614
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2012
            R@1K: 0.8568
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2387
            nDCG@10: 0.5792
            MRR@100: 0.8866
            R@100: 0.3443
            R@1K: 0.7066
  - name: bm25-d2q-t5-doc-segmented-default
    display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
    display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (2b)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2226
            R@1K: 0.8982
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2234
            R@1K: 0.8952
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2683
            nDCG@10: 0.6289
            MRR@100: 0.9454
            R@100: 0.3656
            R@1K: 0.7202
  - name: bm25-rm3-d2q-t5-doc-default
    display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4)
    display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (2c)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.1141
            R@1K: 0.8191
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1170
            R@1K: 0.8247
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2611
            nDCG@10: 0.5375
            MRR@100: 0.8255
            R@100: 0.3580
            R@1K: 0.7574
  - name: bm25-rm3-d2q-t5-doc-segmented-default
    display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
    display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
    display-row: (2d)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.1975
            R@1K: 0.9002
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.1978
            R@1K: 0.8972
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.3191
            nDCG@10: 0.6559
            MRR@100: 0.8989
            R@100: 0.4131
            R@1K: 0.7948
  - name: unicoil-noexp
    display: "uniCOIL (noexp): pre-encoded"
    display-html: "uniCOIL (noexp): pre-encoded queries"
    display-row: (3a)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev-unicoil-noexp
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2231
            R@1K: 0.8987
      - topic_key: msmarco-v2-doc-dev2-unicoil-noexp
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2314
            R@1K: 0.8995
      - topic_key: dl21-unicoil-noexp
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2587
            nDCG@10: 0.6495
            MRR@100: 0.9282
            R@100: 0.3563
            R@1K: 0.6787
  - name: unicoil-noexp-otf
    display: "uniCOIL (noexp): otf"
    display-html: "uniCOIL (noexp): on-the-fly query inference"
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2232
            R@1K: 0.8987
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2314
            R@1K: 0.8993
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2589
            nDCG@10: 0.6501
            MRR@100: 0.9282
            R@100: 0.3574
            R@1K: 0.6782
  - name: unicoil
    display: "uniCOIL (w/ doc2query-T5): pre-encoded"
    display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries"
    display-row: (3b)
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev-unicoil
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2419
            R@1K: 0.9122
      - topic_key: msmarco-v2-doc-dev2-unicoil
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2445
            R@1K: 0.9172
      - topic_key: dl21-unicoil
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2718
            nDCG@10: 0.6783
            MRR@100: 0.9684
            R@100: 0.3700
            R@1K: 0.7069
  - name: unicoil-otf
    display: "uniCOIL (w/ doc2query-T5): otf"
    display-html: "uniCOIL (w/ doc2query-T5): on-the-fly query inference"
    command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
    topics:
      - topic_key: msmarco-v2-doc-dev
        eval_key: msmarco-v2-doc-dev
        scores:
          - MRR@100: 0.2419
            R@1K: 0.9120
      - topic_key: msmarco-v2-doc-dev2
        eval_key: msmarco-v2-doc-dev2
        scores:
          - MRR@100: 0.2447
            R@1K: 0.9174
      - topic_key: dl21
        eval_key: dl21-doc
        scores:
          - MAP@100: 0.2720
            nDCG@10: 0.6782
            MRR@100: 0.9684
            R@100: 0.3702
            R@1K: 0.7071