Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18065,0.28673,0.33189,0.38885,0.42307,0.44384,0.45175,0.45545,0.45915
+map,0.18065,0.25984,0.28459,0.30805,0.31766,0.32112,0.32184,0.32203,0.32212
+precision,0.18065,0.12162,0.09516,0.06522,0.0393,0.01778,0.00937,0.00482,0.001
+recall,0.18065,0.36486,0.47582,0.6522,0.78592,0.88905,0.93741,0.96373,0.99502
+mrr,0.18563,0.2615,0.28628,0.31024,0.31984,0.32333,0.32401,0.32421,0.3243
+recall_cap,0.18563,0.36273,0.47297,0.6522,0.78592,0.89047,0.9367,0.96373,0.99502
+hole,0.66785,0.68445,0.68677,0.68876,0.7287,0.80622,0.84315,0.86613,0.87967

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18065,
+            "NDCG@3": 0.28673,
+            "NDCG@5": 0.33189,
+            "NDCG@10": 0.38885,
+            "NDCG@20": 0.42307,
+            "NDCG@50": 0.44384,
+            "NDCG@100": 0.45175,
+            "NDCG@200": 0.45545,
+            "NDCG@1000": 0.45915
+        },
+        "map": {
+            "MAP@1": 0.18065,
+            "MAP@3": 0.25984,
+            "MAP@5": 0.28459,
+            "MAP@10": 0.30805,
+            "MAP@20": 0.31766,
+            "MAP@50": 0.32112,
+            "MAP@100": 0.32184,
+            "MAP@200": 0.32203,
+            "MAP@1000": 0.32212
+        },
+        "precision": {
+            "P@1": 0.18065,
+            "P@3": 0.12162,
+            "P@5": 0.09516,
+            "P@10": 0.06522,
+            "P@20": 0.0393,
+            "P@50": 0.01778,
+            "P@100": 0.00937,
+            "P@200": 0.00482,
+            "P@1000": 0.001
+        },
+        "recall": {
+            "Recall@1": 0.18065,
+            "Recall@3": 0.36486,
+            "Recall@5": 0.47582,
+            "Recall@10": 0.6522,
+            "Recall@20": 0.78592,
+            "Recall@50": 0.88905,
+            "Recall@100": 0.93741,
+            "Recall@200": 0.96373,
+            "Recall@1000": 0.99502
+        },
+        "mrr": {
+            "MRR@1": 0.18563,
+            "MRR@3": 0.2615,
+            "MRR@5": 0.28628,
+            "MRR@10": 0.31024,
+            "MRR@20": 0.31984,
+            "MRR@50": 0.32333,
+            "MRR@100": 0.32401,
+            "MRR@200": 0.32421,
+            "MRR@1000": 0.3243
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18563,
+            "R_cap@3": 0.36273,
+            "R_cap@5": 0.47297,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.78592,
+            "R_cap@50": 0.89047,
+            "R_cap@100": 0.9367,
+            "R_cap@200": 0.96373,
+            "R_cap@1000": 0.99502
+        },
+        "hole": {
+            "Hole@1": 0.66785,
+            "Hole@3": 0.68445,
+            "Hole@5": 0.68677,
+            "Hole@10": 0.68876,
+            "Hole@20": 0.7287,
+            "Hole@50": 0.80622,
+            "Hole@100": 0.84315,
+            "Hole@200": 0.86613,
+            "Hole@1000": 0.87967
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.20719,
+    "eval_beir-msmarco_recall@10": 0.35679,
+    "eval_beir-msmarco_recall@20": 0.46104,
+    "eval_beir-msmarco_recall@100": 0.67851,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.29868,
+    "eval_beir-dbpedia-entity_recall@10": 0.19285,
+    "eval_beir-dbpedia-entity_recall@20": 0.25779,
+    "eval_beir-dbpedia-entity_recall@100": 0.44014,
+    "eval_beir-fever_ndcg@10": 0.57055,
+    "eval_beir-fever_recall@10": 0.75716,
+    "eval_beir-fever_recall@20": 0.81739,
+    "eval_beir-fever_recall@100": 0.90338,
+    "eval_beir-climate-fever_ndcg@10": 0.15352,
+    "eval_beir-climate-fever_recall@10": 0.19198,
+    "eval_beir-climate-fever_recall@20": 0.24603,
+    "eval_beir-climate-fever_recall@100": 0.41676,
+    "eval_beir-hotpotqa_ndcg@10": 0.50295,
+    "eval_beir-hotpotqa_recall@10": 0.53862,
+    "eval_beir-hotpotqa_recall@20": 0.58913,
+    "eval_beir-hotpotqa_recall@100": 0.70223,
+    "eval_beir-avg_ndcg@10": 0.346578,
+    "eval_beir-avg_recall@10": 0.40747999999999995,
+    "eval_beir-avg_recall@20": 0.47427600000000003,
+    "eval_beir-avg_recall@100": 0.628204
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13616,0.11999,0.13185,0.15352,0.17229,0.19776,0.21563,0.23358,0.26448
+map,0.06621,0.08801,0.09729,0.10607,0.11106,0.11572,0.11776,0.11902,0.12006
+precision,0.13616,0.08426,0.06645,0.04671,0.03137,0.01812,0.01142,0.00711,0.00202
+recall,0.06621,0.11046,0.14208,0.19198,0.24603,0.33811,0.41676,0.50799,0.69866
+mrr,0.13616,0.18198,0.19507,0.20745,0.21469,0.2189,0.22021,0.22085,0.22118
+recall_cap,0.13616,0.11737,0.14208,0.19198,0.24603,0.33811,0.41676,0.50799,0.69866
+hole,0.67036,0.73181,0.75622,0.78339,0.81287,0.84128,0.86154,0.88163,0.92799

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13616,
+            "NDCG@3": 0.11999,
+            "NDCG@5": 0.13185,
+            "NDCG@10": 0.15352,
+            "NDCG@20": 0.17229,
+            "NDCG@50": 0.19776,
+            "NDCG@100": 0.21563,
+            "NDCG@200": 0.23358,
+            "NDCG@1000": 0.26448
+        },
+        "map": {
+            "MAP@1": 0.06621,
+            "MAP@3": 0.08801,
+            "MAP@5": 0.09729,
+            "MAP@10": 0.10607,
+            "MAP@20": 0.11106,
+            "MAP@50": 0.11572,
+            "MAP@100": 0.11776,
+            "MAP@200": 0.11902,
+            "MAP@1000": 0.12006
+        },
+        "precision": {
+            "P@1": 0.13616,
+            "P@3": 0.08426,
+            "P@5": 0.06645,
+            "P@10": 0.04671,
+            "P@20": 0.03137,
+            "P@50": 0.01812,
+            "P@100": 0.01142,
+            "P@200": 0.00711,
+            "P@1000": 0.00202
+        },
+        "recall": {
+            "Recall@1": 0.06621,
+            "Recall@3": 0.11046,
+            "Recall@5": 0.14208,
+            "Recall@10": 0.19198,
+            "Recall@20": 0.24603,
+            "Recall@50": 0.33811,
+            "Recall@100": 0.41676,
+            "Recall@200": 0.50799,
+            "Recall@1000": 0.69866
+        },
+        "mrr": {
+            "MRR@1": 0.13616,
+            "MRR@3": 0.18198,
+            "MRR@5": 0.19507,
+            "MRR@10": 0.20745,
+            "MRR@20": 0.21469,
+            "MRR@50": 0.2189,
+            "MRR@100": 0.22021,
+            "MRR@200": 0.22085,
+            "MRR@1000": 0.22118
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13616,
+            "R_cap@3": 0.11737,
+            "R_cap@5": 0.14208,
+            "R_cap@10": 0.19198,
+            "R_cap@20": 0.24603,
+            "R_cap@50": 0.33811,
+            "R_cap@100": 0.41676,
+            "R_cap@200": 0.50799,
+            "R_cap@1000": 0.69866
+        },
+        "hole": {
+            "Hole@1": 0.67036,
+            "Hole@3": 0.73181,
+            "Hole@5": 0.75622,
+            "Hole@10": 0.78339,
+            "Hole@20": 0.81287,
+            "Hole@50": 0.84128,
+            "Hole@100": 0.86154,
+            "Hole@200": 0.88163,
+            "Hole@1000": 0.92799
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19588083333333337,0.23468916666666664,0.252925,0.27400583333333334,0.29131500000000005,0.31248666666666663,0.32535083333333337,0.33661499999999994,0.35514249999999997
+map,0.16519666666666669,0.20994416666666668,0.22194833333333333,0.2316575,0.23698166666666665,0.24089250000000004,0.24225583333333325,0.243,0.24357083333333338
+precision,0.19588083333333337,0.10911749999999999,0.07942166666666667,0.049599166666666666,0.030192499999999997,0.015421666666666665,0.009026666666666667,0.005214166666666666,0.00134
+recall,0.16519666666666669,0.2599008333333333,0.30588583333333336,0.3683874999999999,0.43202916666666685,0.5295,0.6010533333333333,0.6726200000000001,0.8146341666666667
+mrr,0.19588083333333337,0.24322500000000002,0.2548241666666667,0.26416333333333336,0.26886833333333326,0.27205,0.2730416666666667,0.27352166666666666,0.2738508333333334
+recall_cap,0.19588083333333337,0.2656725,0.3085091666666667,0.3695791666666666,0.4325691666666667,0.5297108333333332,0.6011341666666666,0.6726291666666667,0.8146341666666667
+hole,0.7236174999999999,0.8075649999999999,0.8362941666666668,0.8651483333333333,0.8852116666666666,0.9016124999999998,0.9098649999999999,0.9159975,0.9253083333333333

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19588083333333337,
+            "NDCG@3": 0.23468916666666664,
+            "NDCG@5": 0.252925,
+            "NDCG@10": 0.27400583333333334,
+            "NDCG@20": 0.29131500000000005,
+            "NDCG@50": 0.31248666666666663,
+            "NDCG@100": 0.32535083333333337,
+            "NDCG@200": 0.33661499999999994,
+            "NDCG@1000": 0.35514249999999997
+        },
+        "map": {
+            "MAP@1": 0.16519666666666669,
+            "MAP@3": 0.20994416666666668,
+            "MAP@5": 0.22194833333333333,
+            "MAP@10": 0.2316575,
+            "MAP@20": 0.23698166666666665,
+            "MAP@50": 0.24089250000000004,
+            "MAP@100": 0.24225583333333325,
+            "MAP@200": 0.243,
+            "MAP@1000": 0.24357083333333338
+        },
+        "precision": {
+            "P@1": 0.19588083333333337,
+            "P@3": 0.10911749999999999,
+            "P@5": 0.07942166666666667,
+            "P@10": 0.049599166666666666,
+            "P@20": 0.030192499999999997,
+            "P@50": 0.015421666666666665,
+            "P@100": 0.009026666666666667,
+            "P@200": 0.005214166666666666,
+            "P@1000": 0.00134
+        },
+        "recall": {
+            "Recall@1": 0.16519666666666669,
+            "Recall@3": 0.2599008333333333,
+            "Recall@5": 0.30588583333333336,
+            "Recall@10": 0.3683874999999999,
+            "Recall@20": 0.43202916666666685,
+            "Recall@50": 0.5295,
+            "Recall@100": 0.6010533333333333,
+            "Recall@200": 0.6726200000000001,
+            "Recall@1000": 0.8146341666666667
+        },
+        "mrr": {
+            "MRR@1": 0.19588083333333337,
+            "MRR@3": 0.24322500000000002,
+            "MRR@5": 0.2548241666666667,
+            "MRR@10": 0.26416333333333336,
+            "MRR@20": 0.26886833333333326,
+            "MRR@50": 0.27205,
+            "MRR@100": 0.2730416666666667,
+            "MRR@200": 0.27352166666666666,
+            "MRR@1000": 0.2738508333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19588083333333337,
+            "R_cap@3": 0.2656725,
+            "R_cap@5": 0.3085091666666667,
+            "R_cap@10": 0.3695791666666666,
+            "R_cap@20": 0.4325691666666667,
+            "R_cap@50": 0.5297108333333332,
+            "R_cap@100": 0.6011341666666666,
+            "R_cap@200": 0.6726291666666667,
+            "R_cap@1000": 0.8146341666666667
+        },
+        "hole": {
+            "Hole@1": 0.7236174999999999,
+            "Hole@3": 0.8075649999999999,
+            "Hole@5": 0.8362941666666668,
+            "Hole@10": 0.8651483333333333,
+            "Hole@20": 0.8852116666666666,
+            "Hole@50": 0.9016124999999998,
+            "Hole@100": 0.9098649999999999,
+            "Hole@200": 0.9159975,
+            "Hole@1000": 0.9253083333333333
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.355,0.32217,0.30425,0.29868,0.2991,0.32222,0.35248,0.3783,0.43149
+map,0.06591,0.09994,0.11384,0.13585,0.15867,0.18447,0.19685,0.2034,0.21057
+precision,0.4475,0.36417,0.304,0.245,0.18937,0.1229,0.08362,0.05335,0.01845
+recall,0.06591,0.11526,0.14065,0.19285,0.25779,0.35794,0.44014,0.51938,0.70279
+mrr,0.4475,0.54125,0.55838,0.56899,0.57305,0.57471,0.57512,0.5753,0.57534
+recall_cap,0.4475,0.38208,0.33929,0.31914,0.31582,0.36846,0.44481,0.52228,0.70327
+hole,0.345,0.41667,0.478,0.5425,0.62325,0.72785,0.80015,0.85739,0.93601

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.355,
+            "NDCG@3": 0.32217,
+            "NDCG@5": 0.30425,
+            "NDCG@10": 0.29868,
+            "NDCG@20": 0.2991,
+            "NDCG@50": 0.32222,
+            "NDCG@100": 0.35248,
+            "NDCG@200": 0.3783,
+            "NDCG@1000": 0.43149
+        },
+        "map": {
+            "MAP@1": 0.06591,
+            "MAP@3": 0.09994,
+            "MAP@5": 0.11384,
+            "MAP@10": 0.13585,
+            "MAP@20": 0.15867,
+            "MAP@50": 0.18447,
+            "MAP@100": 0.19685,
+            "MAP@200": 0.2034,
+            "MAP@1000": 0.21057
+        },
+        "precision": {
+            "P@1": 0.4475,
+            "P@3": 0.36417,
+            "P@5": 0.304,
+            "P@10": 0.245,
+            "P@20": 0.18937,
+            "P@50": 0.1229,
+            "P@100": 0.08362,
+            "P@200": 0.05335,
+            "P@1000": 0.01845
+        },
+        "recall": {
+            "Recall@1": 0.06591,
+            "Recall@3": 0.11526,
+            "Recall@5": 0.14065,
+            "Recall@10": 0.19285,
+            "Recall@20": 0.25779,
+            "Recall@50": 0.35794,
+            "Recall@100": 0.44014,
+            "Recall@200": 0.51938,
+            "Recall@1000": 0.70279
+        },
+        "mrr": {
+            "MRR@1": 0.4475,
+            "MRR@3": 0.54125,
+            "MRR@5": 0.55838,
+            "MRR@10": 0.56899,
+            "MRR@20": 0.57305,
+            "MRR@50": 0.57471,
+            "MRR@100": 0.57512,
+            "MRR@200": 0.5753,
+            "MRR@1000": 0.57534
+        },
+        "recall_cap": {
+            "R_cap@1": 0.4475,
+            "R_cap@3": 0.38208,
+            "R_cap@5": 0.33929,
+            "R_cap@10": 0.31914,
+            "R_cap@20": 0.31582,
+            "R_cap@50": 0.36846,
+            "R_cap@100": 0.44481,
+            "R_cap@200": 0.52228,
+            "R_cap@1000": 0.70327
+        },
+        "hole": {
+            "Hole@1": 0.345,
+            "Hole@3": 0.41667,
+            "Hole@5": 0.478,
+            "Hole@10": 0.5425,
+            "Hole@20": 0.62325,
+            "Hole@50": 0.72785,
+            "Hole@100": 0.80015,
+            "Hole@200": 0.85739,
+            "Hole@1000": 0.93601
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39949,0.50745,0.54328,0.57055,0.58649,0.59856,0.60344,0.60657,0.61037
+map,0.37155,0.4683,0.48868,0.50029,0.50494,0.50705,0.50753,0.50772,0.50783
+precision,0.39949,0.21237,0.14656,0.08252,0.04479,0.0193,0.01002,0.00516,0.00108
+recall,0.37155,0.58843,0.67523,0.75716,0.81739,0.87511,0.90338,0.92382,0.95326
+mrr,0.39949,0.50003,0.52062,0.53237,0.5367,0.53857,0.53895,0.53908,0.53914
+recall_cap,0.39949,0.58953,0.6755,0.75717,0.81739,0.87511,0.90338,0.92382,0.95326
+hole,0.57651,0.76093,0.83021,0.8995,0.94221,0.97272,0.98443,0.99095,0.99684

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39949,
+            "NDCG@3": 0.50745,
+            "NDCG@5": 0.54328,
+            "NDCG@10": 0.57055,
+            "NDCG@20": 0.58649,
+            "NDCG@50": 0.59856,
+            "NDCG@100": 0.60344,
+            "NDCG@200": 0.60657,
+            "NDCG@1000": 0.61037
+        },
+        "map": {
+            "MAP@1": 0.37155,
+            "MAP@3": 0.4683,
+            "MAP@5": 0.48868,
+            "MAP@10": 0.50029,
+            "MAP@20": 0.50494,
+            "MAP@50": 0.50705,
+            "MAP@100": 0.50753,
+            "MAP@200": 0.50772,
+            "MAP@1000": 0.50783
+        },
+        "precision": {
+            "P@1": 0.39949,
+            "P@3": 0.21237,
+            "P@5": 0.14656,
+            "P@10": 0.08252,
+            "P@20": 0.04479,
+            "P@50": 0.0193,
+            "P@100": 0.01002,
+            "P@200": 0.00516,
+            "P@1000": 0.00108
+        },
+        "recall": {
+            "Recall@1": 0.37155,
+            "Recall@3": 0.58843,
+            "Recall@5": 0.67523,
+            "Recall@10": 0.75716,
+            "Recall@20": 0.81739,
+            "Recall@50": 0.87511,
+            "Recall@100": 0.90338,
+            "Recall@200": 0.92382,
+            "Recall@1000": 0.95326
+        },
+        "mrr": {
+            "MRR@1": 0.39949,
+            "MRR@3": 0.50003,
+            "MRR@5": 0.52062,
+            "MRR@10": 0.53237,
+            "MRR@20": 0.5367,
+            "MRR@50": 0.53857,
+            "MRR@100": 0.53895,
+            "MRR@200": 0.53908,
+            "MRR@1000": 0.53914
+        },
+        "recall_cap": {
+            "R_cap@1": 0.39949,
+            "R_cap@3": 0.58953,
+            "R_cap@5": 0.6755,
+            "R_cap@10": 0.75717,
+            "R_cap@20": 0.81739,
+            "R_cap@50": 0.87511,
+            "R_cap@100": 0.90338,
+            "R_cap@200": 0.92382,
+            "R_cap@1000": 0.95326
+        },
+        "hole": {
+            "Hole@1": 0.57651,
+            "Hole@3": 0.76093,
+            "Hole@5": 0.83021,
+            "Hole@10": 0.8995,
+            "Hole@20": 0.94221,
+            "Hole@50": 0.97272,
+            "Hole@100": 0.98443,
+            "Hole@200": 0.99095,
+            "Hole@1000": 0.99684
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22222,0.21064,0.21727,0.23933,0.26541,0.29324,0.31053,0.3257,0.3544
+map,0.11722,0.15973,0.17028,0.18279,0.19128,0.19672,0.19887,0.20002,0.20103
+precision,0.22222,0.13837,0.10185,0.06698,0.04367,0.02281,0.01364,0.00793,0.00215
+recall,0.11722,0.19567,0.22813,0.29241,0.37557,0.48607,0.56617,0.64939,0.83391
+mrr,0.22222,0.27469,0.28634,0.29721,0.30394,0.30787,0.30886,0.30933,0.30963
+recall_cap,0.22222,0.21656,0.23423,0.29288,0.37557,0.48607,0.56617,0.64939,0.83391
+hole,0.71914,0.80916,0.84321,0.8733,0.89583,0.9162,0.92281,0.92897,0.93637

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22222,
+            "NDCG@3": 0.21064,
+            "NDCG@5": 0.21727,
+            "NDCG@10": 0.23933,
+            "NDCG@20": 0.26541,
+            "NDCG@50": 0.29324,
+            "NDCG@100": 0.31053,
+            "NDCG@200": 0.3257,
+            "NDCG@1000": 0.3544
+        },
+        "map": {
+            "MAP@1": 0.11722,
+            "MAP@3": 0.15973,
+            "MAP@5": 0.17028,
+            "MAP@10": 0.18279,
+            "MAP@20": 0.19128,
+            "MAP@50": 0.19672,
+            "MAP@100": 0.19887,
+            "MAP@200": 0.20002,
+            "MAP@1000": 0.20103
+        },
+        "precision": {
+            "P@1": 0.22222,
+            "P@3": 0.13837,
+            "P@5": 0.10185,
+            "P@10": 0.06698,
+            "P@20": 0.04367,
+            "P@50": 0.02281,
+            "P@100": 0.01364,
+            "P@200": 0.00793,
+            "P@1000": 0.00215
+        },
+        "recall": {
+            "Recall@1": 0.11722,
+            "Recall@3": 0.19567,
+            "Recall@5": 0.22813,
+            "Recall@10": 0.29241,
+            "Recall@20": 0.37557,
+            "Recall@50": 0.48607,
+            "Recall@100": 0.56617,
+            "Recall@200": 0.64939,
+            "Recall@1000": 0.83391
+        },
+        "mrr": {
+            "MRR@1": 0.22222,
+            "MRR@3": 0.27469,
+            "MRR@5": 0.28634,
+            "MRR@10": 0.29721,
+            "MRR@20": 0.30394,
+            "MRR@50": 0.30787,
+            "MRR@100": 0.30886,
+            "MRR@200": 0.30933,
+            "MRR@1000": 0.30963
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22222,
+            "R_cap@3": 0.21656,
+            "R_cap@5": 0.23423,
+            "R_cap@10": 0.29288,
+            "R_cap@20": 0.37557,
+            "R_cap@50": 0.48607,
+            "R_cap@100": 0.56617,
+            "R_cap@200": 0.64939,
+            "R_cap@1000": 0.83391
+        },
+        "hole": {
+            "Hole@1": 0.71914,
+            "Hole@3": 0.80916,
+            "Hole@5": 0.84321,
+            "Hole@10": 0.8733,
+            "Hole@20": 0.89583,
+            "Hole@50": 0.9162,
+            "Hole@100": 0.92281,
+            "Hole@200": 0.92897,
+            "Hole@1000": 0.93637
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.57542,0.45244,0.47841,0.50295,0.51863,0.53481,0.54412,0.55116,0.56292
+map,0.28771,0.3763,0.39453,0.40744,0.413,0.41656,0.41772,0.41824,0.41863
+precision,0.57542,0.28377,0.19079,0.10772,0.05891,0.02622,0.01404,0.00743,0.00165
+recall,0.28771,0.42566,0.47698,0.53862,0.58913,0.65544,0.70223,0.74321,0.82694
+mrr,0.57542,0.64461,0.65528,0.66264,0.66564,0.66703,0.66738,0.66752,0.6676
+recall_cap,0.57542,0.42566,0.47698,0.53862,0.58913,0.65544,0.70223,0.74321,0.82694
+hole,0.41756,0.70488,0.79681,0.88022,0.92963,0.96304,0.97586,0.98298,0.99027

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.57542,
+            "NDCG@3": 0.45244,
+            "NDCG@5": 0.47841,
+            "NDCG@10": 0.50295,
+            "NDCG@20": 0.51863,
+            "NDCG@50": 0.53481,
+            "NDCG@100": 0.54412,
+            "NDCG@200": 0.55116,
+            "NDCG@1000": 0.56292
+        },
+        "map": {
+            "MAP@1": 0.28771,
+            "MAP@3": 0.3763,
+            "MAP@5": 0.39453,
+            "MAP@10": 0.40744,
+            "MAP@20": 0.413,
+            "MAP@50": 0.41656,
+            "MAP@100": 0.41772,
+            "MAP@200": 0.41824,
+            "MAP@1000": 0.41863
+        },
+        "precision": {
+            "P@1": 0.57542,
+            "P@3": 0.28377,
+            "P@5": 0.19079,
+            "P@10": 0.10772,
+            "P@20": 0.05891,
+            "P@50": 0.02622,
+            "P@100": 0.01404,
+            "P@200": 0.00743,
+            "P@1000": 0.00165
+        },
+        "recall": {
+            "Recall@1": 0.28771,
+            "Recall@3": 0.42566,
+            "Recall@5": 0.47698,
+            "Recall@10": 0.53862,
+            "Recall@20": 0.58913,
+            "Recall@50": 0.65544,
+            "Recall@100": 0.70223,
+            "Recall@200": 0.74321,
+            "Recall@1000": 0.82694
+        },
+        "mrr": {
+            "MRR@1": 0.57542,
+            "MRR@3": 0.64461,
+            "MRR@5": 0.65528,
+            "MRR@10": 0.66264,
+            "MRR@20": 0.66564,
+            "MRR@50": 0.66703,
+            "MRR@100": 0.66738,
+            "MRR@200": 0.66752,
+            "MRR@1000": 0.6676
+        },
+        "recall_cap": {
+            "R_cap@1": 0.57542,
+            "R_cap@3": 0.42566,
+            "R_cap@5": 0.47698,
+            "R_cap@10": 0.53862,
+            "R_cap@20": 0.58913,
+            "R_cap@50": 0.65544,
+            "R_cap@100": 0.70223,
+            "R_cap@200": 0.74321,
+            "R_cap@1000": 0.82694
+        },
+        "hole": {
+            "Hole@1": 0.41756,
+            "Hole@3": 0.70488,
+            "Hole@5": 0.79681,
+            "Hole@10": 0.88022,
+            "Hole@20": 0.92963,
+            "Hole@50": 0.96304,
+            "Hole@100": 0.97586,
+            "Hole@200": 0.98298,
+            "Hole@1000": 0.99027
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.08854,0.14623,0.17456,0.20719,0.23389,0.2607,0.27464,0.28578,0.30227
+map,0.08606,0.13031,0.14613,0.15969,0.16712,0.17151,0.17277,0.17336,0.17376
+precision,0.08854,0.06514,0.05332,0.03718,0.02405,0.01242,0.00711,0.00398,0.00095
+recall,0.08606,0.18927,0.25725,0.35679,0.46104,0.59392,0.67851,0.75634,0.89596
+mrr,0.08854,0.1335,0.14937,0.16295,0.1703,0.17462,0.17585,0.17641,0.17678
+recall_cap,0.08854,0.18928,0.25725,0.35679,0.46104,0.59392,0.67851,0.75634,0.89596
+hole,0.91032,0.93395,0.94596,0.96196,0.97505,0.98663,0.99192,0.99506,0.99811

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.08854,
+            "NDCG@3": 0.14623,
+            "NDCG@5": 0.17456,
+            "NDCG@10": 0.20719,
+            "NDCG@20": 0.23389,
+            "NDCG@50": 0.2607,
+            "NDCG@100": 0.27464,
+            "NDCG@200": 0.28578,
+            "NDCG@1000": 0.30227
+        },
+        "map": {
+            "MAP@1": 0.08606,
+            "MAP@3": 0.13031,
+            "MAP@5": 0.14613,
+            "MAP@10": 0.15969,
+            "MAP@20": 0.16712,
+            "MAP@50": 0.17151,
+            "MAP@100": 0.17277,
+            "MAP@200": 0.17336,
+            "MAP@1000": 0.17376
+        },
+        "precision": {
+            "P@1": 0.08854,
+            "P@3": 0.06514,
+            "P@5": 0.05332,
+            "P@10": 0.03718,
+            "P@20": 0.02405,
+            "P@50": 0.01242,
+            "P@100": 0.00711,
+            "P@200": 0.00398,
+            "P@1000": 0.00095
+        },
+        "recall": {
+            "Recall@1": 0.08606,
+            "Recall@3": 0.18927,
+            "Recall@5": 0.25725,
+            "Recall@10": 0.35679,
+            "Recall@20": 0.46104,
+            "Recall@50": 0.59392,
+            "Recall@100": 0.67851,
+            "Recall@200": 0.75634,
+            "Recall@1000": 0.89596
+        },
+        "mrr": {
+            "MRR@1": 0.08854,
+            "MRR@3": 0.1335,
+            "MRR@5": 0.14937,
+            "MRR@10": 0.16295,
+            "MRR@20": 0.1703,
+            "MRR@50": 0.17462,
+            "MRR@100": 0.17585,
+            "MRR@200": 0.17641,
+            "MRR@1000": 0.17678
+        },
+        "recall_cap": {
+            "R_cap@1": 0.08854,
+            "R_cap@3": 0.18928,
+            "R_cap@5": 0.25725,
+            "R_cap@10": 0.35679,
+            "R_cap@20": 0.46104,
+            "R_cap@50": 0.59392,
+            "R_cap@100": 0.67851,
+            "R_cap@200": 0.75634,
+            "R_cap@1000": 0.89596
+        },
+        "hole": {
+            "Hole@1": 0.91032,
+            "Hole@3": 0.93395,
+            "Hole@5": 0.94596,
+            "Hole@10": 0.96196,
+            "Hole@20": 0.97505,
+            "Hole@50": 0.98663,
+            "Hole@100": 0.99192,
+            "Hole@200": 0.99506,
+            "Hole@1000": 0.99811
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.40248,0.36778,0.34609,0.31333,0.29072,0.27832,0.28703,0.30409,0.3767
+map,0.04185,0.07922,0.09288,0.10904,0.12083,0.13169,0.13815,0.14314,0.15187
+precision,0.42105,0.34778,0.30341,0.23189,0.17152,0.11065,0.07551,0.05034,0.02038
+recall,0.04185,0.09406,0.11671,0.15154,0.18654,0.23733,0.29552,0.36976,0.6217
+mrr,0.42724,0.49536,0.50867,0.51683,0.52172,0.52321,0.52365,0.52399,0.52416
+recall_cap,0.42724,0.36687,0.33731,0.29357,0.26743,0.26871,0.31016,0.37472,0.6217
+hole,0.08359,0.09391,0.0935,0.09783,0.11099,0.1218,0.12579,0.1272,0.1333

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.40248,
+            "NDCG@3": 0.36778,
+            "NDCG@5": 0.34609,
+            "NDCG@10": 0.31333,
+            "NDCG@20": 0.29072,
+            "NDCG@50": 0.27832,
+            "NDCG@100": 0.28703,
+            "NDCG@200": 0.30409,
+            "NDCG@1000": 0.3767
+        },
+        "map": {
+            "MAP@1": 0.04185,
+            "MAP@3": 0.07922,
+            "MAP@5": 0.09288,
+            "MAP@10": 0.10904,
+            "MAP@20": 0.12083,
+            "MAP@50": 0.13169,
+            "MAP@100": 0.13815,
+            "MAP@200": 0.14314,
+            "MAP@1000": 0.15187
+        },
+        "precision": {
+            "P@1": 0.42105,
+            "P@3": 0.34778,
+            "P@5": 0.30341,
+            "P@10": 0.23189,
+            "P@20": 0.17152,
+            "P@50": 0.11065,
+            "P@100": 0.07551,
+            "P@200": 0.05034,
+            "P@1000": 0.02038
+        },
+        "recall": {
+            "Recall@1": 0.04185,
+            "Recall@3": 0.09406,
+            "Recall@5": 0.11671,
+            "Recall@10": 0.15154,
+            "Recall@20": 0.18654,
+            "Recall@50": 0.23733,
+            "Recall@100": 0.29552,
+            "Recall@200": 0.36976,
+            "Recall@1000": 0.6217
+        },
+        "mrr": {
+            "MRR@1": 0.42724,
+            "MRR@3": 0.49536,
+            "MRR@5": 0.50867,
+            "MRR@10": 0.51683,
+            "MRR@20": 0.52172,
+            "MRR@50": 0.52321,
+            "MRR@100": 0.52365,
+            "MRR@200": 0.52399,
+            "MRR@1000": 0.52416
+        },
+        "recall_cap": {
+            "R_cap@1": 0.42724,
+            "R_cap@3": 0.36687,
+            "R_cap@5": 0.33731,
+            "R_cap@10": 0.29357,
+            "R_cap@20": 0.26743,
+            "R_cap@50": 0.26871,
+            "R_cap@100": 0.31016,
+            "R_cap@200": 0.37472,
+            "R_cap@1000": 0.6217
+        },
+        "hole": {
+            "Hole@1": 0.08359,
+            "Hole@3": 0.09391,
+            "Hole@5": 0.0935,
+            "Hole@10": 0.09783,
+            "Hole@20": 0.11099,
+            "Hole@50": 0.1218,
+            "Hole@100": 0.12579,
+            "Hole@200": 0.1272,
+            "Hole@1000": 0.1333
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.12891,0.19221,0.2285,0.27242,0.30319,0.33231,0.34668,0.35581,0.36667
+map,0.11334,0.16952,0.19029,0.20905,0.21795,0.22312,0.22452,0.22505,0.22534
+precision,0.12891,0.09222,0.07567,0.05348,0.03384,0.01699,0.00953,0.00516,0.00114
+recall,0.11334,0.23899,0.32339,0.45355,0.56969,0.70824,0.79193,0.85337,0.94127
+mrr,0.1292,0.18883,0.20899,0.22687,0.23489,0.23913,0.24028,0.24068,0.2409
+recall_cap,0.1292,0.23904,0.32339,0.45355,0.56969,0.70824,0.79193,0.85337,0.94127
+hole,0.86761,0.90604,0.92254,0.94412,0.96338,0.98031,0.98787,0.99231,0.99646

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.12891,
+            "NDCG@3": 0.19221,
+            "NDCG@5": 0.2285,
+            "NDCG@10": 0.27242,
+            "NDCG@20": 0.30319,
+            "NDCG@50": 0.33231,
+            "NDCG@100": 0.34668,
+            "NDCG@200": 0.35581,
+            "NDCG@1000": 0.36667
+        },
+        "map": {
+            "MAP@1": 0.11334,
+            "MAP@3": 0.16952,
+            "MAP@5": 0.19029,
+            "MAP@10": 0.20905,
+            "MAP@20": 0.21795,
+            "MAP@50": 0.22312,
+            "MAP@100": 0.22452,
+            "MAP@200": 0.22505,
+            "MAP@1000": 0.22534
+        },
+        "precision": {
+            "P@1": 0.12891,
+            "P@3": 0.09222,
+            "P@5": 0.07567,
+            "P@10": 0.05348,
+            "P@20": 0.03384,
+            "P@50": 0.01699,
+            "P@100": 0.00953,
+            "P@200": 0.00516,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.11334,
+            "Recall@3": 0.23899,
+            "Recall@5": 0.32339,
+            "Recall@10": 0.45355,
+            "Recall@20": 0.56969,
+            "Recall@50": 0.70824,
+            "Recall@100": 0.79193,
+            "Recall@200": 0.85337,
+            "Recall@1000": 0.94127
+        },
+        "mrr": {
+            "MRR@1": 0.1292,
+            "MRR@3": 0.18883,
+            "MRR@5": 0.20899,
+            "MRR@10": 0.22687,
+            "MRR@20": 0.23489,
+            "MRR@50": 0.23913,
+            "MRR@100": 0.24028,
+            "MRR@200": 0.24068,
+            "MRR@1000": 0.2409
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1292,
+            "R_cap@3": 0.23904,
+            "R_cap@5": 0.32339,
+            "R_cap@10": 0.45355,
+            "R_cap@20": 0.56969,
+            "R_cap@50": 0.70824,
+            "R_cap@100": 0.79193,
+            "R_cap@200": 0.85337,
+            "R_cap@1000": 0.94127
+        },
+        "hole": {
+            "Hole@1": 0.86761,
+            "Hole@3": 0.90604,
+            "Hole@5": 0.92254,
+            "Hole@10": 0.94412,
+            "Hole@20": 0.96338,
+            "Hole@50": 0.98031,
+            "Hole@100": 0.98787,
+            "Hole@200": 0.99231,
+            "Hole@1000": 0.99646
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6722,0.73396,0.75688,0.77732,0.79002,0.79851,0.80169,0.80334,0.80505
+map,0.58723,0.69289,0.71314,0.72619,0.73182,0.7344,0.73498,0.73517,0.73527
+precision,0.6722,0.322,0.21572,0.12086,0.0658,0.02842,0.01474,0.00754,0.00156
+recall,0.58723,0.7687,0.82882,0.88759,0.9305,0.96358,0.97812,0.98674,0.99719
+mrr,0.6718,0.74277,0.75414,0.76039,0.76273,0.76345,0.76358,0.76362,0.76364
+recall_cap,0.6718,0.78125,0.83357,0.88894,0.93076,0.96361,0.97812,0.98674,0.99719
+hole,0.3077,0.64423,0.74672,0.83875,0.89242,0.93003,0.94472,0.9534,0.96232

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6722,
+            "NDCG@3": 0.73396,
+            "NDCG@5": 0.75688,
+            "NDCG@10": 0.77732,
+            "NDCG@20": 0.79002,
+            "NDCG@50": 0.79851,
+            "NDCG@100": 0.80169,
+            "NDCG@200": 0.80334,
+            "NDCG@1000": 0.80505
+        },
+        "map": {
+            "MAP@1": 0.58723,
+            "MAP@3": 0.69289,
+            "MAP@5": 0.71314,
+            "MAP@10": 0.72619,
+            "MAP@20": 0.73182,
+            "MAP@50": 0.7344,
+            "MAP@100": 0.73498,
+            "MAP@200": 0.73517,
+            "MAP@1000": 0.73527
+        },
+        "precision": {
+            "P@1": 0.6722,
+            "P@3": 0.322,
+            "P@5": 0.21572,
+            "P@10": 0.12086,
+            "P@20": 0.0658,
+            "P@50": 0.02842,
+            "P@100": 0.01474,
+            "P@200": 0.00754,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.58723,
+            "Recall@3": 0.7687,
+            "Recall@5": 0.82882,
+            "Recall@10": 0.88759,
+            "Recall@20": 0.9305,
+            "Recall@50": 0.96358,
+            "Recall@100": 0.97812,
+            "Recall@200": 0.98674,
+            "Recall@1000": 0.99719
+        },
+        "mrr": {
+            "MRR@1": 0.6718,
+            "MRR@3": 0.74277,
+            "MRR@5": 0.75414,
+            "MRR@10": 0.76039,
+            "MRR@20": 0.76273,
+            "MRR@50": 0.76345,
+            "MRR@100": 0.76358,
+            "MRR@200": 0.76362,
+            "MRR@1000": 0.76364
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6718,
+            "R_cap@3": 0.78125,
+            "R_cap@5": 0.83357,
+            "R_cap@10": 0.88894,
+            "R_cap@20": 0.93076,
+            "R_cap@50": 0.96361,
+            "R_cap@100": 0.97812,
+            "R_cap@200": 0.98674,
+            "R_cap@1000": 0.99719
+        },
+        "hole": {
+            "Hole@1": 0.3077,
+            "Hole@3": 0.64423,
+            "Hole@5": 0.74672,
+            "Hole@10": 0.83875,
+            "Hole@20": 0.89242,
+            "Hole@50": 0.93003,
+            "Hole@100": 0.94472,
+            "Hole@200": 0.9534,
+            "Hole@1000": 0.96232
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.179,0.14353,0.12346,0.1498,0.17405,0.20361,0.22099,0.23697,0.27327
+map,0.03643,0.0628,0.07378,0.08572,0.09402,0.10029,0.10254,0.10383,0.10521
+precision,0.179,0.13367,0.1068,0.0773,0.0527,0.02984,0.01805,0.0107,0.00306
+recall,0.03643,0.08143,0.10843,0.15677,0.21367,0.30232,0.36587,0.43345,0.622
+mrr,0.179,0.24117,0.25542,0.26871,0.27548,0.27969,0.28074,0.28118,0.28151
+recall_cap,0.179,0.13367,0.10843,0.15677,0.21367,0.30232,0.36587,0.43345,0.622
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.179,
+            "NDCG@3": 0.14353,
+            "NDCG@5": 0.12346,
+            "NDCG@10": 0.1498,
+            "NDCG@20": 0.17405,
+            "NDCG@50": 0.20361,
+            "NDCG@100": 0.22099,
+            "NDCG@200": 0.23697,
+            "NDCG@1000": 0.27327
+        },
+        "map": {
+            "MAP@1": 0.03643,
+            "MAP@3": 0.0628,
+            "MAP@5": 0.07378,
+            "MAP@10": 0.08572,
+            "MAP@20": 0.09402,
+            "MAP@50": 0.10029,
+            "MAP@100": 0.10254,
+            "MAP@200": 0.10383,
+            "MAP@1000": 0.10521
+        },
+        "precision": {
+            "P@1": 0.179,
+            "P@3": 0.13367,
+            "P@5": 0.1068,
+            "P@10": 0.0773,
+            "P@20": 0.0527,
+            "P@50": 0.02984,
+            "P@100": 0.01805,
+            "P@200": 0.0107,
+            "P@1000": 0.00306
+        },
+        "recall": {
+            "Recall@1": 0.03643,
+            "Recall@3": 0.08143,
+            "Recall@5": 0.10843,
+            "Recall@10": 0.15677,
+            "Recall@20": 0.21367,
+            "Recall@50": 0.30232,
+            "Recall@100": 0.36587,
+            "Recall@200": 0.43345,
+            "Recall@1000": 0.622
+        },
+        "mrr": {
+            "MRR@1": 0.179,
+            "MRR@3": 0.24117,
+            "MRR@5": 0.25542,
+            "MRR@10": 0.26871,
+            "MRR@20": 0.27548,
+            "MRR@50": 0.27969,
+            "MRR@100": 0.28074,
+            "MRR@200": 0.28118,
+            "MRR@1000": 0.28151
+        },
+        "recall_cap": {
+            "R_cap@1": 0.179,
+            "R_cap@3": 0.13367,
+            "R_cap@5": 0.10843,
+            "R_cap@10": 0.15677,
+            "R_cap@20": 0.21367,
+            "R_cap@50": 0.30232,
+            "R_cap@100": 0.36587,
+            "R_cap@200": 0.43345,
+            "R_cap@1000": 0.622
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.52333,0.6237,0.64232,0.66223,0.67037,0.6844,0.68597,0.69097,0.69688
+map,0.49872,0.59041,0.60497,0.61398,0.61644,0.61911,0.61923,0.6195,0.61965
+precision,0.52333,0.25,0.16333,0.089,0.04633,0.0202,0.0102,0.0053,0.00111
+recall,0.49872,0.68961,0.7365,0.79411,0.82411,0.89156,0.90156,0.936,0.986
+mrr,0.52333,0.61,0.61933,0.62722,0.6294,0.63119,0.63131,0.63156,0.63169
+recall_cap,0.52333,0.69333,0.7365,0.79411,0.82411,0.89156,0.90156,0.936,0.986
+hole,0.43667,0.71222,0.79333,0.85433,0.89467,0.91753,0.9284,0.93535,0.94049

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.52333,
+            "NDCG@3": 0.6237,
+            "NDCG@5": 0.64232,
+            "NDCG@10": 0.66223,
+            "NDCG@20": 0.67037,
+            "NDCG@50": 0.6844,
+            "NDCG@100": 0.68597,
+            "NDCG@200": 0.69097,
+            "NDCG@1000": 0.69688
+        },
+        "map": {
+            "MAP@1": 0.49872,
+            "MAP@3": 0.59041,
+            "MAP@5": 0.60497,
+            "MAP@10": 0.61398,
+            "MAP@20": 0.61644,
+            "MAP@50": 0.61911,
+            "MAP@100": 0.61923,
+            "MAP@200": 0.6195,
+            "MAP@1000": 0.61965
+        },
+        "precision": {
+            "P@1": 0.52333,
+            "P@3": 0.25,
+            "P@5": 0.16333,
+            "P@10": 0.089,
+            "P@20": 0.04633,
+            "P@50": 0.0202,
+            "P@100": 0.0102,
+            "P@200": 0.0053,
+            "P@1000": 0.00111
+        },
+        "recall": {
+            "Recall@1": 0.49872,
+            "Recall@3": 0.68961,
+            "Recall@5": 0.7365,
+            "Recall@10": 0.79411,
+            "Recall@20": 0.82411,
+            "Recall@50": 0.89156,
+            "Recall@100": 0.90156,
+            "Recall@200": 0.936,
+            "Recall@1000": 0.986
+        },
+        "mrr": {
+            "MRR@1": 0.52333,
+            "MRR@3": 0.61,
+            "MRR@5": 0.61933,
+            "MRR@10": 0.62722,
+            "MRR@20": 0.6294,
+            "MRR@50": 0.63119,
+            "MRR@100": 0.63131,
+            "MRR@200": 0.63156,
+            "MRR@1000": 0.63169
+        },
+        "recall_cap": {
+            "R_cap@1": 0.52333,
+            "R_cap@3": 0.69333,
+            "R_cap@5": 0.7365,
+            "R_cap@10": 0.79411,
+            "R_cap@20": 0.82411,
+            "R_cap@50": 0.89156,
+            "R_cap@100": 0.90156,
+            "R_cap@200": 0.936,
+            "R_cap@1000": 0.986
+        },
+        "hole": {
+            "Hole@1": 0.43667,
+            "Hole@3": 0.71222,
+            "Hole@5": 0.79333,
+            "Hole@10": 0.85433,
+            "Hole@20": 0.89467,
+            "Hole@50": 0.91753,
+            "Hole@100": 0.9284,
+            "Hole@200": 0.93535,
+            "Hole@1000": 0.94049
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7730453513050343,
+    "eval_senteval-sickr_spearman": 0.7176593438481393,
+    "eval_senteval-avg_sts": 0.7453523475765869
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7675016313639117,
+                0.0
+            ],
+            "spearman": [
+                0.7404488937353225,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7658521041744508,
+                1.4020329449305176e-289
+            ],
+            "spearman": [
+                0.7730453513050343,
+                2.066561283121993e-298
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7623056519318484,
+                1.8448021245198884e-262
+            ],
+            "spearman": [
+                0.7572290027800557,
+                5.3120116722114526e-257
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7647570029076096,
+                "mean": 0.7652197958234037,
+                "wmean": 0.7663843913985655
+            },
+            "spearman": {
+                "all": 0.7534738635333297,
+                "mean": 0.7569077492734708,
+                "wmean": 0.7487978108339844
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7707517065205873,
+                0.0
+            ],
+            "spearman": [
+                0.6949522587450165,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.778349180191507,
+                9.711324138536703e-103
+            ],
+            "spearman": [
+                0.7176593438481393,
+                2.743332620783351e-80
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7642987657770804,
+                0.0
+            ],
+            "spearman": [
+                0.6847455515560564,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7678373187987577,
+                "mean": 0.7711332174963915,
+                "wmean": 0.7679316297393041
+            },
+            "spearman": {
+                "all": 0.6909477744903625,
+                "mean": 0.6991190513830707,
+                "wmean": 0.6910301368785469
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7730453513050343,
+    "eval_senteval-sickr_spearman": 0.7176593438481393,
+    "eval_senteval-avg_sts": 0.7453523475765869
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.73,0.65212,0.62264,0.58943,0.56316,0.4955,0.44313,0.3814,0.40665
+map,0.00216,0.00538,0.00815,0.01415,0.02525,0.04725,0.07527,0.1094,0.18357
+precision,0.78,0.69333,0.656,0.622,0.606,0.518,0.4602,0.3797,0.18474
+recall,0.00216,0.00563,0.00877,0.01621,0.03086,0.06224,0.10692,0.16993,0.38745
+mrr,0.76,0.80667,0.81567,0.82656,0.82656,0.82656,0.82656,0.82656,0.82656
+recall_cap,0.76,0.69333,0.656,0.622,0.606,0.518,0.4602,0.38311,0.3888
+hole,0.12,0.09333,0.124,0.142,0.152,0.1788,0.2032,0.2504,0.38076

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.73,
+            "NDCG@3": 0.65212,
+            "NDCG@5": 0.62264,
+            "NDCG@10": 0.58943,
+            "NDCG@20": 0.56316,
+            "NDCG@50": 0.4955,
+            "NDCG@100": 0.44313,
+            "NDCG@200": 0.3814,
+            "NDCG@1000": 0.40665
+        },
+        "map": {
+            "MAP@1": 0.00216,
+            "MAP@3": 0.00538,
+            "MAP@5": 0.00815,
+            "MAP@10": 0.01415,
+            "MAP@20": 0.02525,
+            "MAP@50": 0.04725,
+            "MAP@100": 0.07527,
+            "MAP@200": 0.1094,
+            "MAP@1000": 0.18357
+        },
+        "precision": {
+            "P@1": 0.78,
+            "P@3": 0.69333,
+            "P@5": 0.656,
+            "P@10": 0.622,
+            "P@20": 0.606,
+            "P@50": 0.518,
+            "P@100": 0.4602,
+            "P@200": 0.3797,
+            "P@1000": 0.18474
+        },
+        "recall": {
+            "Recall@1": 0.00216,
+            "Recall@3": 0.00563,
+            "Recall@5": 0.00877,
+            "Recall@10": 0.01621,
+            "Recall@20": 0.03086,
+            "Recall@50": 0.06224,
+            "Recall@100": 0.10692,
+            "Recall@200": 0.16993,
+            "Recall@1000": 0.38745
+        },
+        "mrr": {
+            "MRR@1": 0.76,
+            "MRR@3": 0.80667,
+            "MRR@5": 0.81567,
+            "MRR@10": 0.82656,
+            "MRR@20": 0.82656,
+            "MRR@50": 0.82656,
+            "MRR@100": 0.82656,
+            "MRR@200": 0.82656,
+            "MRR@1000": 0.82656
+        },
+        "recall_cap": {
+            "R_cap@1": 0.76,
+            "R_cap@3": 0.69333,
+            "R_cap@5": 0.656,
+            "R_cap@10": 0.622,
+            "R_cap@20": 0.606,
+            "R_cap@50": 0.518,
+            "R_cap@100": 0.4602,
+            "R_cap@200": 0.38311,
+            "R_cap@1000": 0.3888
+        },
+        "hole": {
+            "Hole@1": 0.12,
+            "Hole@3": 0.09333,
+            "Hole@5": 0.124,
+            "Hole@10": 0.142,
+            "Hole@20": 0.152,
+            "Hole@50": 0.1788,
+            "Hole@100": 0.2032,
+            "Hole@200": 0.2504,
+            "Hole@1000": 0.38076
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18367,0.19359,0.17838,0.18989,0.20531,0.26302,0.31702,0.35305,0.42979
+map,0.01235,0.03167,0.04256,0.0714,0.0903,0.10947,0.12275,0.12951,0.13723
+precision,0.18367,0.20408,0.17959,0.17959,0.1398,0.09714,0.07122,0.04541,0.01465
+recall,0.01235,0.04811,0.07298,0.1387,0.21105,0.33022,0.45315,0.54881,0.79733
+mrr,0.18367,0.32993,0.34422,0.37151,0.37658,0.38123,0.38123,0.38123,0.38127
+recall_cap,0.18367,0.20408,0.17959,0.19906,0.21666,0.33022,0.45315,0.54881,0.79733
+hole,0.73469,0.7619,0.78776,0.78367,0.82959,0.8698,0.90265,0.92939,0.96884

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18367,
+            "NDCG@3": 0.19359,
+            "NDCG@5": 0.17838,
+            "NDCG@10": 0.18989,
+            "NDCG@20": 0.20531,
+            "NDCG@50": 0.26302,
+            "NDCG@100": 0.31702,
+            "NDCG@200": 0.35305,
+            "NDCG@1000": 0.42979
+        },
+        "map": {
+            "MAP@1": 0.01235,
+            "MAP@3": 0.03167,
+            "MAP@5": 0.04256,
+            "MAP@10": 0.0714,
+            "MAP@20": 0.0903,
+            "MAP@50": 0.10947,
+            "MAP@100": 0.12275,
+            "MAP@200": 0.12951,
+            "MAP@1000": 0.13723
+        },
+        "precision": {
+            "P@1": 0.18367,
+            "P@3": 0.20408,
+            "P@5": 0.17959,
+            "P@10": 0.17959,
+            "P@20": 0.1398,
+            "P@50": 0.09714,
+            "P@100": 0.07122,
+            "P@200": 0.04541,
+            "P@1000": 0.01465
+        },
+        "recall": {
+            "Recall@1": 0.01235,
+            "Recall@3": 0.04811,
+            "Recall@5": 0.07298,
+            "Recall@10": 0.1387,
+            "Recall@20": 0.21105,
+            "Recall@50": 0.33022,
+            "Recall@100": 0.45315,
+            "Recall@200": 0.54881,
+            "Recall@1000": 0.79733
+        },
+        "mrr": {
+            "MRR@1": 0.18367,
+            "MRR@3": 0.32993,
+            "MRR@5": 0.34422,
+            "MRR@10": 0.37151,
+            "MRR@20": 0.37658,
+            "MRR@50": 0.38123,
+            "MRR@100": 0.38123,
+            "MRR@200": 0.38123,
+            "MRR@1000": 0.38127
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18367,
+            "R_cap@3": 0.20408,
+            "R_cap@5": 0.17959,
+            "R_cap@10": 0.19906,
+            "R_cap@20": 0.21666,
+            "R_cap@50": 0.33022,
+            "R_cap@100": 0.45315,
+            "R_cap@200": 0.54881,
+            "R_cap@1000": 0.79733
+        },
+        "hole": {
+            "Hole@1": 0.73469,
+            "Hole@3": 0.7619,
+            "Hole@5": 0.78776,
+            "Hole@10": 0.78367,
+            "Hole@20": 0.82959,
+            "Hole@50": 0.8698,
+            "Hole@100": 0.90265,
+            "Hole@200": 0.92939,
+            "Hole@1000": 0.96884
+        }
+    }
+}

cc.moco.topic50.bs1024.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=false
+export NUM_WORKER=4
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.T03b_topic50.moco-2e14.contriever256-special50.bert-base-uncased.avg.dot.qd128.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file /export/home/data/search/upr/cc/T03B_PileCC_topic.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special50% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=13h4vrrq
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18065,0.28673,0.33189,0.38885,0.42307,0.44384,0.45175,0.45545,0.45915
+map,0.18065,0.25984,0.28459,0.30805,0.31766,0.32112,0.32184,0.32203,0.32212
+precision,0.18065,0.12162,0.09516,0.06522,0.0393,0.01778,0.00937,0.00482,0.001
+recall,0.18065,0.36486,0.47582,0.6522,0.78592,0.88905,0.93741,0.96373,0.99502
+mrr,0.18563,0.2615,0.28628,0.31024,0.31984,0.32333,0.32401,0.32421,0.3243
+recall_cap,0.18563,0.36273,0.47297,0.6522,0.78592,0.89047,0.9367,0.96373,0.99502
+hole,0.66785,0.68445,0.68677,0.68876,0.7287,0.80622,0.84315,0.86613,0.87967

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18065,
+            "NDCG@3": 0.28673,
+            "NDCG@5": 0.33189,
+            "NDCG@10": 0.38885,
+            "NDCG@20": 0.42307,
+            "NDCG@50": 0.44384,
+            "NDCG@100": 0.45175,
+            "NDCG@200": 0.45545,
+            "NDCG@1000": 0.45915
+        },
+        "map": {
+            "MAP@1": 0.18065,
+            "MAP@3": 0.25984,
+            "MAP@5": 0.28459,
+            "MAP@10": 0.30805,
+            "MAP@20": 0.31766,
+            "MAP@50": 0.32112,
+            "MAP@100": 0.32184,
+            "MAP@200": 0.32203,
+            "MAP@1000": 0.32212
+        },
+        "precision": {
+            "P@1": 0.18065,
+            "P@3": 0.12162,
+            "P@5": 0.09516,
+            "P@10": 0.06522,
+            "P@20": 0.0393,
+            "P@50": 0.01778,
+            "P@100": 0.00937,
+            "P@200": 0.00482,
+            "P@1000": 0.001
+        },
+        "recall": {
+            "Recall@1": 0.18065,
+            "Recall@3": 0.36486,
+            "Recall@5": 0.47582,
+            "Recall@10": 0.6522,
+            "Recall@20": 0.78592,
+            "Recall@50": 0.88905,
+            "Recall@100": 0.93741,
+            "Recall@200": 0.96373,
+            "Recall@1000": 0.99502
+        },
+        "mrr": {
+            "MRR@1": 0.18563,
+            "MRR@3": 0.2615,
+            "MRR@5": 0.28628,
+            "MRR@10": 0.31024,
+            "MRR@20": 0.31984,
+            "MRR@50": 0.32333,
+            "MRR@100": 0.32401,
+            "MRR@200": 0.32421,
+            "MRR@1000": 0.3243
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18563,
+            "R_cap@3": 0.36273,
+            "R_cap@5": 0.47297,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.78592,
+            "R_cap@50": 0.89047,
+            "R_cap@100": 0.9367,
+            "R_cap@200": 0.96373,
+            "R_cap@1000": 0.99502
+        },
+        "hole": {
+            "Hole@1": 0.66785,
+            "Hole@3": 0.68445,
+            "Hole@5": 0.68677,
+            "Hole@10": 0.68876,
+            "Hole@20": 0.7287,
+            "Hole@50": 0.80622,
+            "Hole@100": 0.84315,
+            "Hole@200": 0.86613,
+            "Hole@1000": 0.87967
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.31333,
+    "eval_beir-nfcorpus_recall@10": 0.15154,
+    "eval_beir-nfcorpus_recall@20": 0.18654,
+    "eval_beir-nfcorpus_recall@100": 0.29552,
+    "eval_beir-fiqa_ndcg@10": 0.23933,
+    "eval_beir-fiqa_recall@10": 0.29241,
+    "eval_beir-fiqa_recall@20": 0.37557,
+    "eval_beir-fiqa_recall@100": 0.56617,
+    "eval_beir-arguana_ndcg@10": 0.38885,
+    "eval_beir-arguana_recall@10": 0.6522,
+    "eval_beir-arguana_recall@20": 0.78592,
+    "eval_beir-arguana_recall@100": 0.93741,
+    "eval_beir-scidocs_ndcg@10": 0.1498,
+    "eval_beir-scidocs_recall@10": 0.15677,
+    "eval_beir-scidocs_recall@20": 0.21367,
+    "eval_beir-scidocs_recall@100": 0.36587,
+    "eval_beir-scifact_ndcg@10": 0.66223,
+    "eval_beir-scifact_recall@10": 0.79411,
+    "eval_beir-scifact_recall@20": 0.82411,
+    "eval_beir-scifact_recall@100": 0.90156,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18989,
+    "eval_beir-webis-touche2020_recall@10": 0.1387,
+    "eval_beir-webis-touche2020_recall@20": 0.21105,
+    "eval_beir-webis-touche2020_recall@100": 0.45315,
+    "eval_beir-cqadupstack_ndcg@10": 0.27400583333333334,
+    "eval_beir-cqadupstack_recall@10": 0.3683874999999999,
+    "eval_beir-cqadupstack_recall@20": 0.43202916666666685,
+    "eval_beir-cqadupstack_recall@100": 0.6010533333333333,
+    "eval_beir-trec-covid_ndcg@10": 0.58943,
+    "eval_beir-trec-covid_recall@10": 0.622,
+    "eval_beir-trec-covid_recall@20": 0.606,
+    "eval_beir-trec-covid_recall@100": 0.4602,
+    "eval_beir-quora_ndcg@10": 0.77732,
+    "eval_beir-quora_recall@10": 0.88759,
+    "eval_beir-quora_recall@20": 0.9305,
+    "eval_beir-quora_recall@100": 0.97812,
+    "eval_beir-nq_ndcg@10": 0.27242,
+    "eval_beir-nq_recall@10": 0.45355,
+    "eval_beir-nq_recall@20": 0.56969,
+    "eval_beir-nq_recall@100": 0.79193,
+    "eval_beir-avg_ndcg@10": 0.3856605833333333,
+    "eval_beir-avg_recall@10": 0.45172575000000004,
+    "eval_beir-avg_recall@20": 0.5135079166666667,
+    "eval_beir-avg_recall@100": 0.6350983333333333
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19588083333333337,0.23468916666666664,0.252925,0.27400583333333334,0.29131500000000005,0.31248666666666663,0.32535083333333337,0.33661499999999994,0.35514249999999997
+map,0.16519666666666669,0.20994416666666668,0.22194833333333333,0.2316575,0.23698166666666665,0.24089250000000004,0.24225583333333325,0.243,0.24357083333333338
+precision,0.19588083333333337,0.10911749999999999,0.07942166666666667,0.049599166666666666,0.030192499999999997,0.015421666666666665,0.009026666666666667,0.005214166666666666,0.00134
+recall,0.16519666666666669,0.2599008333333333,0.30588583333333336,0.3683874999999999,0.43202916666666685,0.5295,0.6010533333333333,0.6726200000000001,0.8146341666666667
+mrr,0.19588083333333337,0.24322500000000002,0.2548241666666667,0.26416333333333336,0.26886833333333326,0.27205,0.2730416666666667,0.27352166666666666,0.2738508333333334
+recall_cap,0.19588083333333337,0.2656725,0.3085091666666667,0.3695791666666666,0.4325691666666667,0.5297108333333332,0.6011341666666666,0.6726291666666667,0.8146341666666667
+hole,0.7236174999999999,0.8075649999999999,0.8362941666666668,0.8651483333333333,0.8852116666666666,0.9016124999999998,0.9098649999999999,0.9159975,0.9253083333333333

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19588083333333337,
+            "NDCG@3": 0.23468916666666664,
+            "NDCG@5": 0.252925,
+            "NDCG@10": 0.27400583333333334,
+            "NDCG@20": 0.29131500000000005,
+            "NDCG@50": 0.31248666666666663,
+            "NDCG@100": 0.32535083333333337,
+            "NDCG@200": 0.33661499999999994,
+            "NDCG@1000": 0.35514249999999997
+        },
+        "map": {
+            "MAP@1": 0.16519666666666669,
+            "MAP@3": 0.20994416666666668,
+            "MAP@5": 0.22194833333333333,
+            "MAP@10": 0.2316575,
+            "MAP@20": 0.23698166666666665,
+            "MAP@50": 0.24089250000000004,
+            "MAP@100": 0.24225583333333325,
+            "MAP@200": 0.243,
+            "MAP@1000": 0.24357083333333338
+        },
+        "precision": {
+            "P@1": 0.19588083333333337,
+            "P@3": 0.10911749999999999,
+            "P@5": 0.07942166666666667,
+            "P@10": 0.049599166666666666,
+            "P@20": 0.030192499999999997,
+            "P@50": 0.015421666666666665,
+            "P@100": 0.009026666666666667,
+            "P@200": 0.005214166666666666,
+            "P@1000": 0.00134
+        },
+        "recall": {
+            "Recall@1": 0.16519666666666669,
+            "Recall@3": 0.2599008333333333,
+            "Recall@5": 0.30588583333333336,
+            "Recall@10": 0.3683874999999999,
+            "Recall@20": 0.43202916666666685,
+            "Recall@50": 0.5295,
+            "Recall@100": 0.6010533333333333,
+            "Recall@200": 0.6726200000000001,
+            "Recall@1000": 0.8146341666666667
+        },
+        "mrr": {
+            "MRR@1": 0.19588083333333337,
+            "MRR@3": 0.24322500000000002,
+            "MRR@5": 0.2548241666666667,
+            "MRR@10": 0.26416333333333336,
+            "MRR@20": 0.26886833333333326,
+            "MRR@50": 0.27205,
+            "MRR@100": 0.2730416666666667,
+            "MRR@200": 0.27352166666666666,
+            "MRR@1000": 0.2738508333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19588083333333337,
+            "R_cap@3": 0.2656725,
+            "R_cap@5": 0.3085091666666667,
+            "R_cap@10": 0.3695791666666666,
+            "R_cap@20": 0.4325691666666667,
+            "R_cap@50": 0.5297108333333332,
+            "R_cap@100": 0.6011341666666666,
+            "R_cap@200": 0.6726291666666667,
+            "R_cap@1000": 0.8146341666666667
+        },
+        "hole": {
+            "Hole@1": 0.7236174999999999,
+            "Hole@3": 0.8075649999999999,
+            "Hole@5": 0.8362941666666668,
+            "Hole@10": 0.8651483333333333,
+            "Hole@20": 0.8852116666666666,
+            "Hole@50": 0.9016124999999998,
+            "Hole@100": 0.9098649999999999,
+            "Hole@200": 0.9159975,
+            "Hole@1000": 0.9253083333333333
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22222,0.21064,0.21727,0.23933,0.26541,0.29324,0.31053,0.3257,0.3544
+map,0.11722,0.15973,0.17028,0.18279,0.19128,0.19672,0.19887,0.20002,0.20103
+precision,0.22222,0.13837,0.10185,0.06698,0.04367,0.02281,0.01364,0.00793,0.00215
+recall,0.11722,0.19567,0.22813,0.29241,0.37557,0.48607,0.56617,0.64939,0.83391
+mrr,0.22222,0.27469,0.28634,0.29721,0.30394,0.30787,0.30886,0.30933,0.30963
+recall_cap,0.22222,0.21656,0.23423,0.29288,0.37557,0.48607,0.56617,0.64939,0.83391
+hole,0.71914,0.80916,0.84321,0.8733,0.89583,0.9162,0.92281,0.92897,0.93637

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22222,
+            "NDCG@3": 0.21064,
+            "NDCG@5": 0.21727,
+            "NDCG@10": 0.23933,
+            "NDCG@20": 0.26541,
+            "NDCG@50": 0.29324,
+            "NDCG@100": 0.31053,
+            "NDCG@200": 0.3257,
+            "NDCG@1000": 0.3544
+        },
+        "map": {
+            "MAP@1": 0.11722,
+            "MAP@3": 0.15973,
+            "MAP@5": 0.17028,
+            "MAP@10": 0.18279,
+            "MAP@20": 0.19128,
+            "MAP@50": 0.19672,
+            "MAP@100": 0.19887,
+            "MAP@200": 0.20002,
+            "MAP@1000": 0.20103
+        },
+        "precision": {
+            "P@1": 0.22222,
+            "P@3": 0.13837,
+            "P@5": 0.10185,
+            "P@10": 0.06698,
+            "P@20": 0.04367,
+            "P@50": 0.02281,
+            "P@100": 0.01364,
+            "P@200": 0.00793,
+            "P@1000": 0.00215
+        },
+        "recall": {
+            "Recall@1": 0.11722,
+            "Recall@3": 0.19567,
+            "Recall@5": 0.22813,
+            "Recall@10": 0.29241,
+            "Recall@20": 0.37557,
+            "Recall@50": 0.48607,
+            "Recall@100": 0.56617,
+            "Recall@200": 0.64939,
+            "Recall@1000": 0.83391
+        },
+        "mrr": {
+            "MRR@1": 0.22222,
+            "MRR@3": 0.27469,
+            "MRR@5": 0.28634,
+            "MRR@10": 0.29721,
+            "MRR@20": 0.30394,
+            "MRR@50": 0.30787,
+            "MRR@100": 0.30886,
+            "MRR@200": 0.30933,
+            "MRR@1000": 0.30963
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22222,
+            "R_cap@3": 0.21656,
+            "R_cap@5": 0.23423,
+            "R_cap@10": 0.29288,
+            "R_cap@20": 0.37557,
+            "R_cap@50": 0.48607,
+            "R_cap@100": 0.56617,
+            "R_cap@200": 0.64939,
+            "R_cap@1000": 0.83391
+        },
+        "hole": {
+            "Hole@1": 0.71914,
+            "Hole@3": 0.80916,
+            "Hole@5": 0.84321,
+            "Hole@10": 0.8733,
+            "Hole@20": 0.89583,
+            "Hole@50": 0.9162,
+            "Hole@100": 0.92281,
+            "Hole@200": 0.92897,
+            "Hole@1000": 0.93637
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.40248,0.36778,0.34609,0.31333,0.29072,0.27832,0.28703,0.30409,0.3767
+map,0.04185,0.07922,0.09288,0.10904,0.12083,0.13169,0.13815,0.14314,0.15187
+precision,0.42105,0.34778,0.30341,0.23189,0.17152,0.11065,0.07551,0.05034,0.02038
+recall,0.04185,0.09406,0.11671,0.15154,0.18654,0.23733,0.29552,0.36976,0.6217
+mrr,0.42724,0.49536,0.50867,0.51683,0.52172,0.52321,0.52365,0.52399,0.52416
+recall_cap,0.42724,0.36687,0.33731,0.29357,0.26743,0.26871,0.31016,0.37472,0.6217
+hole,0.08359,0.09391,0.0935,0.09783,0.11099,0.1218,0.12579,0.1272,0.1333

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.40248,
+            "NDCG@3": 0.36778,
+            "NDCG@5": 0.34609,
+            "NDCG@10": 0.31333,
+            "NDCG@20": 0.29072,
+            "NDCG@50": 0.27832,
+            "NDCG@100": 0.28703,
+            "NDCG@200": 0.30409,
+            "NDCG@1000": 0.3767
+        },
+        "map": {
+            "MAP@1": 0.04185,
+            "MAP@3": 0.07922,
+            "MAP@5": 0.09288,
+            "MAP@10": 0.10904,
+            "MAP@20": 0.12083,
+            "MAP@50": 0.13169,
+            "MAP@100": 0.13815,
+            "MAP@200": 0.14314,
+            "MAP@1000": 0.15187
+        },
+        "precision": {
+            "P@1": 0.42105,
+            "P@3": 0.34778,
+            "P@5": 0.30341,
+            "P@10": 0.23189,
+            "P@20": 0.17152,
+            "P@50": 0.11065,
+            "P@100": 0.07551,
+            "P@200": 0.05034,
+            "P@1000": 0.02038
+        },
+        "recall": {
+            "Recall@1": 0.04185,
+            "Recall@3": 0.09406,
+            "Recall@5": 0.11671,
+            "Recall@10": 0.15154,
+            "Recall@20": 0.18654,
+            "Recall@50": 0.23733,
+            "Recall@100": 0.29552,
+            "Recall@200": 0.36976,
+            "Recall@1000": 0.6217
+        },
+        "mrr": {
+            "MRR@1": 0.42724,
+            "MRR@3": 0.49536,
+            "MRR@5": 0.50867,
+            "MRR@10": 0.51683,
+            "MRR@20": 0.52172,
+            "MRR@50": 0.52321,
+            "MRR@100": 0.52365,
+            "MRR@200": 0.52399,
+            "MRR@1000": 0.52416
+        },
+        "recall_cap": {
+            "R_cap@1": 0.42724,
+            "R_cap@3": 0.36687,
+            "R_cap@5": 0.33731,
+            "R_cap@10": 0.29357,
+            "R_cap@20": 0.26743,
+            "R_cap@50": 0.26871,
+            "R_cap@100": 0.31016,
+            "R_cap@200": 0.37472,
+            "R_cap@1000": 0.6217
+        },
+        "hole": {
+            "Hole@1": 0.08359,
+            "Hole@3": 0.09391,
+            "Hole@5": 0.0935,
+            "Hole@10": 0.09783,
+            "Hole@20": 0.11099,
+            "Hole@50": 0.1218,
+            "Hole@100": 0.12579,
+            "Hole@200": 0.1272,
+            "Hole@1000": 0.1333
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.12891,0.19221,0.2285,0.27242,0.30319,0.33231,0.34668,0.35581,0.36667
+map,0.11334,0.16952,0.19029,0.20905,0.21795,0.22312,0.22452,0.22505,0.22534
+precision,0.12891,0.09222,0.07567,0.05348,0.03384,0.01699,0.00953,0.00516,0.00114
+recall,0.11334,0.23899,0.32339,0.45355,0.56969,0.70824,0.79193,0.85337,0.94127
+mrr,0.1292,0.18883,0.20899,0.22687,0.23489,0.23913,0.24028,0.24068,0.2409
+recall_cap,0.1292,0.23904,0.32339,0.45355,0.56969,0.70824,0.79193,0.85337,0.94127
+hole,0.86761,0.90604,0.92254,0.94412,0.96338,0.98031,0.98787,0.99231,0.99646

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.12891,
+            "NDCG@3": 0.19221,
+            "NDCG@5": 0.2285,
+            "NDCG@10": 0.27242,
+            "NDCG@20": 0.30319,
+            "NDCG@50": 0.33231,
+            "NDCG@100": 0.34668,
+            "NDCG@200": 0.35581,
+            "NDCG@1000": 0.36667
+        },
+        "map": {
+            "MAP@1": 0.11334,
+            "MAP@3": 0.16952,
+            "MAP@5": 0.19029,
+            "MAP@10": 0.20905,
+            "MAP@20": 0.21795,
+            "MAP@50": 0.22312,
+            "MAP@100": 0.22452,
+            "MAP@200": 0.22505,
+            "MAP@1000": 0.22534
+        },
+        "precision": {
+            "P@1": 0.12891,
+            "P@3": 0.09222,
+            "P@5": 0.07567,
+            "P@10": 0.05348,
+            "P@20": 0.03384,
+            "P@50": 0.01699,
+            "P@100": 0.00953,
+            "P@200": 0.00516,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.11334,
+            "Recall@3": 0.23899,
+            "Recall@5": 0.32339,
+            "Recall@10": 0.45355,
+            "Recall@20": 0.56969,
+            "Recall@50": 0.70824,
+            "Recall@100": 0.79193,
+            "Recall@200": 0.85337,
+            "Recall@1000": 0.94127
+        },
+        "mrr": {
+            "MRR@1": 0.1292,
+            "MRR@3": 0.18883,
+            "MRR@5": 0.20899,
+            "MRR@10": 0.22687,
+            "MRR@20": 0.23489,
+            "MRR@50": 0.23913,
+            "MRR@100": 0.24028,
+            "MRR@200": 0.24068,
+            "MRR@1000": 0.2409
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1292,
+            "R_cap@3": 0.23904,
+            "R_cap@5": 0.32339,
+            "R_cap@10": 0.45355,
+            "R_cap@20": 0.56969,
+            "R_cap@50": 0.70824,
+            "R_cap@100": 0.79193,
+            "R_cap@200": 0.85337,
+            "R_cap@1000": 0.94127
+        },
+        "hole": {
+            "Hole@1": 0.86761,
+            "Hole@3": 0.90604,
+            "Hole@5": 0.92254,
+            "Hole@10": 0.94412,
+            "Hole@20": 0.96338,
+            "Hole@50": 0.98031,
+            "Hole@100": 0.98787,
+            "Hole@200": 0.99231,
+            "Hole@1000": 0.99646
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6722,0.73396,0.75688,0.77732,0.79002,0.79851,0.80169,0.80334,0.80505
+map,0.58723,0.69289,0.71314,0.72619,0.73182,0.7344,0.73498,0.73517,0.73527
+precision,0.6722,0.322,0.21572,0.12086,0.0658,0.02842,0.01474,0.00754,0.00156
+recall,0.58723,0.7687,0.82882,0.88759,0.9305,0.96358,0.97812,0.98674,0.99719
+mrr,0.6718,0.74277,0.75414,0.76039,0.76273,0.76345,0.76358,0.76362,0.76364
+recall_cap,0.6718,0.78125,0.83357,0.88894,0.93076,0.96361,0.97812,0.98674,0.99719
+hole,0.3077,0.64423,0.74672,0.83875,0.89242,0.93003,0.94472,0.9534,0.96232

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6722,
+            "NDCG@3": 0.73396,
+            "NDCG@5": 0.75688,
+            "NDCG@10": 0.77732,
+            "NDCG@20": 0.79002,
+            "NDCG@50": 0.79851,
+            "NDCG@100": 0.80169,
+            "NDCG@200": 0.80334,
+            "NDCG@1000": 0.80505
+        },
+        "map": {
+            "MAP@1": 0.58723,
+            "MAP@3": 0.69289,
+            "MAP@5": 0.71314,
+            "MAP@10": 0.72619,
+            "MAP@20": 0.73182,
+            "MAP@50": 0.7344,
+            "MAP@100": 0.73498,
+            "MAP@200": 0.73517,
+            "MAP@1000": 0.73527
+        },
+        "precision": {
+            "P@1": 0.6722,
+            "P@3": 0.322,
+            "P@5": 0.21572,
+            "P@10": 0.12086,
+            "P@20": 0.0658,
+            "P@50": 0.02842,
+            "P@100": 0.01474,
+            "P@200": 0.00754,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.58723,
+            "Recall@3": 0.7687,
+            "Recall@5": 0.82882,
+            "Recall@10": 0.88759,
+            "Recall@20": 0.9305,
+            "Recall@50": 0.96358,
+            "Recall@100": 0.97812,
+            "Recall@200": 0.98674,
+            "Recall@1000": 0.99719
+        },
+        "mrr": {
+            "MRR@1": 0.6718,
+            "MRR@3": 0.74277,
+            "MRR@5": 0.75414,
+            "MRR@10": 0.76039,
+            "MRR@20": 0.76273,
+            "MRR@50": 0.76345,
+            "MRR@100": 0.76358,
+            "MRR@200": 0.76362,
+            "MRR@1000": 0.76364
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6718,
+            "R_cap@3": 0.78125,
+            "R_cap@5": 0.83357,
+            "R_cap@10": 0.88894,
+            "R_cap@20": 0.93076,
+            "R_cap@50": 0.96361,
+            "R_cap@100": 0.97812,
+            "R_cap@200": 0.98674,
+            "R_cap@1000": 0.99719
+        },
+        "hole": {
+            "Hole@1": 0.3077,
+            "Hole@3": 0.64423,
+            "Hole@5": 0.74672,
+            "Hole@10": 0.83875,
+            "Hole@20": 0.89242,
+            "Hole@50": 0.93003,
+            "Hole@100": 0.94472,
+            "Hole@200": 0.9534,
+            "Hole@1000": 0.96232
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.179,0.14353,0.12346,0.1498,0.17405,0.20361,0.22099,0.23697,0.27327
+map,0.03643,0.0628,0.07378,0.08572,0.09402,0.10029,0.10254,0.10383,0.10521
+precision,0.179,0.13367,0.1068,0.0773,0.0527,0.02984,0.01805,0.0107,0.00306
+recall,0.03643,0.08143,0.10843,0.15677,0.21367,0.30232,0.36587,0.43345,0.622
+mrr,0.179,0.24117,0.25542,0.26871,0.27548,0.27969,0.28074,0.28118,0.28151
+recall_cap,0.179,0.13367,0.10843,0.15677,0.21367,0.30232,0.36587,0.43345,0.622
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0