Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1458,0.25465,0.30138,0.36665,0.40931,0.43032,0.43752,0.43972,0.44131
+map,0.1458,0.227,0.25268,0.27931,0.29133,0.29487,0.29552,0.29563,0.29567
+precision,0.1458,0.11166,0.0899,0.06529,0.04097,0.01846,0.00967,0.00491,0.001
+recall,0.1458,0.33499,0.4495,0.65292,0.81935,0.92319,0.96728,0.98293,0.99644
+mrr,0.15434,0.22973,0.25537,0.28239,0.29444,0.298,0.29866,0.29878,0.29882
+recall_cap,0.15434,0.33357,0.44666,0.6522,0.81863,0.92248,0.96657,0.98293,0.99644
+hole,0.68919,0.67236,0.66401,0.64339,0.6697,0.7468,0.78873,0.8117,0.82983

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1458,
+            "NDCG@3": 0.25465,
+            "NDCG@5": 0.30138,
+            "NDCG@10": 0.36665,
+            "NDCG@20": 0.40931,
+            "NDCG@50": 0.43032,
+            "NDCG@100": 0.43752,
+            "NDCG@200": 0.43972,
+            "NDCG@1000": 0.44131
+        },
+        "map": {
+            "MAP@1": 0.1458,
+            "MAP@3": 0.227,
+            "MAP@5": 0.25268,
+            "MAP@10": 0.27931,
+            "MAP@20": 0.29133,
+            "MAP@50": 0.29487,
+            "MAP@100": 0.29552,
+            "MAP@200": 0.29563,
+            "MAP@1000": 0.29567
+        },
+        "precision": {
+            "P@1": 0.1458,
+            "P@3": 0.11166,
+            "P@5": 0.0899,
+            "P@10": 0.06529,
+            "P@20": 0.04097,
+            "P@50": 0.01846,
+            "P@100": 0.00967,
+            "P@200": 0.00491,
+            "P@1000": 0.001
+        },
+        "recall": {
+            "Recall@1": 0.1458,
+            "Recall@3": 0.33499,
+            "Recall@5": 0.4495,
+            "Recall@10": 0.65292,
+            "Recall@20": 0.81935,
+            "Recall@50": 0.92319,
+            "Recall@100": 0.96728,
+            "Recall@200": 0.98293,
+            "Recall@1000": 0.99644
+        },
+        "mrr": {
+            "MRR@1": 0.15434,
+            "MRR@3": 0.22973,
+            "MRR@5": 0.25537,
+            "MRR@10": 0.28239,
+            "MRR@20": 0.29444,
+            "MRR@50": 0.298,
+            "MRR@100": 0.29866,
+            "MRR@200": 0.29878,
+            "MRR@1000": 0.29882
+        },
+        "recall_cap": {
+            "R_cap@1": 0.15434,
+            "R_cap@3": 0.33357,
+            "R_cap@5": 0.44666,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.81863,
+            "R_cap@50": 0.92248,
+            "R_cap@100": 0.96657,
+            "R_cap@200": 0.98293,
+            "R_cap@1000": 0.99644
+        },
+        "hole": {
+            "Hole@1": 0.68919,
+            "Hole@3": 0.67236,
+            "Hole@5": 0.66401,
+            "Hole@10": 0.64339,
+            "Hole@20": 0.6697,
+            "Hole@50": 0.7468,
+            "Hole@100": 0.78873,
+            "Hole@200": 0.8117,
+            "Hole@1000": 0.82983
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,14 @@
+{
+    "eval_beir-climate-fever_ndcg@10": 0.12105,
+    "eval_beir-climate-fever_recall@10": 0.1548,
+    "eval_beir-climate-fever_recall@20": 0.20165,
+    "eval_beir-climate-fever_recall@100": 0.37817,
+    "eval_beir-hotpotqa_ndcg@10": 0.49429,
+    "eval_beir-hotpotqa_recall@10": 0.51587,
+    "eval_beir-hotpotqa_recall@20": 0.5661,
+    "eval_beir-hotpotqa_recall@100": 0.66826,
+    "eval_beir-avg_ndcg@10": 0.30767,
+    "eval_beir-avg_recall@10": 0.33533500000000005,
+    "eval_beir-avg_recall@20": 0.383875,
+    "eval_beir-avg_recall@100": 0.523215
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.10163,0.09468,0.10288,0.12105,0.13715,0.16023,0.18066,0.20061,0.23369
+map,0.04886,0.06966,0.07541,0.08233,0.08658,0.09046,0.09268,0.09405,0.09519
+precision,0.10163,0.06688,0.05186,0.03707,0.02515,0.01494,0.01013,0.00659,0.00197
+recall,0.04886,0.091,0.1126,0.1548,0.20165,0.28685,0.37817,0.4811,0.68076
+mrr,0.10163,0.13985,0.15076,0.16243,0.16859,0.1731,0.17473,0.17546,0.17585
+recall_cap,0.10163,0.09598,0.1126,0.1548,0.20165,0.28685,0.37817,0.4811,0.68076
+hole,0.71987,0.75744,0.77433,0.79401,0.81498,0.83872,0.85514,0.87124,0.9135

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.10163,
+            "NDCG@3": 0.09468,
+            "NDCG@5": 0.10288,
+            "NDCG@10": 0.12105,
+            "NDCG@20": 0.13715,
+            "NDCG@50": 0.16023,
+            "NDCG@100": 0.18066,
+            "NDCG@200": 0.20061,
+            "NDCG@1000": 0.23369
+        },
+        "map": {
+            "MAP@1": 0.04886,
+            "MAP@3": 0.06966,
+            "MAP@5": 0.07541,
+            "MAP@10": 0.08233,
+            "MAP@20": 0.08658,
+            "MAP@50": 0.09046,
+            "MAP@100": 0.09268,
+            "MAP@200": 0.09405,
+            "MAP@1000": 0.09519
+        },
+        "precision": {
+            "P@1": 0.10163,
+            "P@3": 0.06688,
+            "P@5": 0.05186,
+            "P@10": 0.03707,
+            "P@20": 0.02515,
+            "P@50": 0.01494,
+            "P@100": 0.01013,
+            "P@200": 0.00659,
+            "P@1000": 0.00197
+        },
+        "recall": {
+            "Recall@1": 0.04886,
+            "Recall@3": 0.091,
+            "Recall@5": 0.1126,
+            "Recall@10": 0.1548,
+            "Recall@20": 0.20165,
+            "Recall@50": 0.28685,
+            "Recall@100": 0.37817,
+            "Recall@200": 0.4811,
+            "Recall@1000": 0.68076
+        },
+        "mrr": {
+            "MRR@1": 0.10163,
+            "MRR@3": 0.13985,
+            "MRR@5": 0.15076,
+            "MRR@10": 0.16243,
+            "MRR@20": 0.16859,
+            "MRR@50": 0.1731,
+            "MRR@100": 0.17473,
+            "MRR@200": 0.17546,
+            "MRR@1000": 0.17585
+        },
+        "recall_cap": {
+            "R_cap@1": 0.10163,
+            "R_cap@3": 0.09598,
+            "R_cap@5": 0.1126,
+            "R_cap@10": 0.1548,
+            "R_cap@20": 0.20165,
+            "R_cap@50": 0.28685,
+            "R_cap@100": 0.37817,
+            "R_cap@200": 0.4811,
+            "R_cap@1000": 0.68076
+        },
+        "hole": {
+            "Hole@1": 0.71987,
+            "Hole@3": 0.75744,
+            "Hole@5": 0.77433,
+            "Hole@10": 0.79401,
+            "Hole@20": 0.81498,
+            "Hole@50": 0.83872,
+            "Hole@100": 0.85514,
+            "Hole@200": 0.87124,
+            "Hole@1000": 0.9135
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22085083333333333,0.2611408333333333,0.280635,0.3031158333333333,0.3222125,0.34235499999999996,0.35566999999999993,0.36617166666666673,0.38425166666666666
+map,0.18674833333333338,0.234905,0.24751333333333336,0.25813250000000004,0.26415083333333333,0.26800750000000007,0.26944166666666663,0.2701408333333333,0.27071833333333334
+precision,0.22085083333333333,0.12022333333333333,0.08697666666666669,0.05404916666666667,0.032967500000000004,0.016465000000000004,0.009618333333333335,0.005469166666666667,0.0014016666666666667
+recall,0.18674833333333338,0.28826999999999997,0.3373541666666666,0.40381749999999994,0.4736041666666666,0.5654874999999999,0.6389233333333334,0.7054,0.8424908333333333
+mrr,0.22085083333333333,0.27007333333333333,0.2825991666666667,0.2922741666666666,0.2973,0.30019083333333335,0.3012291666666667,0.3016791666666666,0.30198583333333334
+recall_cap,0.22085083333333333,0.2941683333333333,0.3401016666666666,0.4049400000000001,0.47411749999999997,0.5656558333333334,0.6390225,0.7053058333333334,0.8424908333333333
+hole,0.706795,0.8030316666666667,0.8332233333333333,0.8648841666666668,0.8864941666666667,0.9045466666666666,0.9131933333333335,0.9195391666666667,0.9277475000000001

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22085083333333333,
+            "NDCG@3": 0.2611408333333333,
+            "NDCG@5": 0.280635,
+            "NDCG@10": 0.3031158333333333,
+            "NDCG@20": 0.3222125,
+            "NDCG@50": 0.34235499999999996,
+            "NDCG@100": 0.35566999999999993,
+            "NDCG@200": 0.36617166666666673,
+            "NDCG@1000": 0.38425166666666666
+        },
+        "map": {
+            "MAP@1": 0.18674833333333338,
+            "MAP@3": 0.234905,
+            "MAP@5": 0.24751333333333336,
+            "MAP@10": 0.25813250000000004,
+            "MAP@20": 0.26415083333333333,
+            "MAP@50": 0.26800750000000007,
+            "MAP@100": 0.26944166666666663,
+            "MAP@200": 0.2701408333333333,
+            "MAP@1000": 0.27071833333333334
+        },
+        "precision": {
+            "P@1": 0.22085083333333333,
+            "P@3": 0.12022333333333333,
+            "P@5": 0.08697666666666669,
+            "P@10": 0.05404916666666667,
+            "P@20": 0.032967500000000004,
+            "P@50": 0.016465000000000004,
+            "P@100": 0.009618333333333335,
+            "P@200": 0.005469166666666667,
+            "P@1000": 0.0014016666666666667
+        },
+        "recall": {
+            "Recall@1": 0.18674833333333338,
+            "Recall@3": 0.28826999999999997,
+            "Recall@5": 0.3373541666666666,
+            "Recall@10": 0.40381749999999994,
+            "Recall@20": 0.4736041666666666,
+            "Recall@50": 0.5654874999999999,
+            "Recall@100": 0.6389233333333334,
+            "Recall@200": 0.7054,
+            "Recall@1000": 0.8424908333333333
+        },
+        "mrr": {
+            "MRR@1": 0.22085083333333333,
+            "MRR@3": 0.27007333333333333,
+            "MRR@5": 0.2825991666666667,
+            "MRR@10": 0.2922741666666666,
+            "MRR@20": 0.2973,
+            "MRR@50": 0.30019083333333335,
+            "MRR@100": 0.3012291666666667,
+            "MRR@200": 0.3016791666666666,
+            "MRR@1000": 0.30198583333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22085083333333333,
+            "R_cap@3": 0.2941683333333333,
+            "R_cap@5": 0.3401016666666666,
+            "R_cap@10": 0.4049400000000001,
+            "R_cap@20": 0.47411749999999997,
+            "R_cap@50": 0.5656558333333334,
+            "R_cap@100": 0.6390225,
+            "R_cap@200": 0.7053058333333334,
+            "R_cap@1000": 0.8424908333333333
+        },
+        "hole": {
+            "Hole@1": 0.706795,
+            "Hole@3": 0.8030316666666667,
+            "Hole@5": 0.8332233333333333,
+            "Hole@10": 0.8648841666666668,
+            "Hole@20": 0.8864941666666667,
+            "Hole@50": 0.9045466666666666,
+            "Hole@100": 0.9131933333333335,
+            "Hole@200": 0.9195391666666667,
+            "Hole@1000": 0.9277475000000001
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.38375,0.34452,0.32075,0.3116,0.31519,0.34148,0.37337,0.40051,0.45298
+map,0.06984,0.11272,0.1275,0.15026,0.17365,0.20164,0.21506,0.2224,0.22998
+precision,0.48,0.3825,0.313,0.246,0.19025,0.12565,0.08555,0.05503,0.01866
+recall,0.06984,0.12841,0.15703,0.20675,0.27058,0.37447,0.4652,0.54588,0.72027
+mrr,0.48,0.57875,0.59213,0.60054,0.60404,0.60587,0.60632,0.60643,0.60647
+recall_cap,0.48,0.40167,0.35417,0.32848,0.32834,0.38431,0.4686,0.54797,0.7206
+hole,0.325,0.41333,0.48,0.5545,0.63012,0.73025,0.79923,0.8556,0.93489

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.38375,
+            "NDCG@3": 0.34452,
+            "NDCG@5": 0.32075,
+            "NDCG@10": 0.3116,
+            "NDCG@20": 0.31519,
+            "NDCG@50": 0.34148,
+            "NDCG@100": 0.37337,
+            "NDCG@200": 0.40051,
+            "NDCG@1000": 0.45298
+        },
+        "map": {
+            "MAP@1": 0.06984,
+            "MAP@3": 0.11272,
+            "MAP@5": 0.1275,
+            "MAP@10": 0.15026,
+            "MAP@20": 0.17365,
+            "MAP@50": 0.20164,
+            "MAP@100": 0.21506,
+            "MAP@200": 0.2224,
+            "MAP@1000": 0.22998
+        },
+        "precision": {
+            "P@1": 0.48,
+            "P@3": 0.3825,
+            "P@5": 0.313,
+            "P@10": 0.246,
+            "P@20": 0.19025,
+            "P@50": 0.12565,
+            "P@100": 0.08555,
+            "P@200": 0.05503,
+            "P@1000": 0.01866
+        },
+        "recall": {
+            "Recall@1": 0.06984,
+            "Recall@3": 0.12841,
+            "Recall@5": 0.15703,
+            "Recall@10": 0.20675,
+            "Recall@20": 0.27058,
+            "Recall@50": 0.37447,
+            "Recall@100": 0.4652,
+            "Recall@200": 0.54588,
+            "Recall@1000": 0.72027
+        },
+        "mrr": {
+            "MRR@1": 0.48,
+            "MRR@3": 0.57875,
+            "MRR@5": 0.59213,
+            "MRR@10": 0.60054,
+            "MRR@20": 0.60404,
+            "MRR@50": 0.60587,
+            "MRR@100": 0.60632,
+            "MRR@200": 0.60643,
+            "MRR@1000": 0.60647
+        },
+        "recall_cap": {
+            "R_cap@1": 0.48,
+            "R_cap@3": 0.40167,
+            "R_cap@5": 0.35417,
+            "R_cap@10": 0.32848,
+            "R_cap@20": 0.32834,
+            "R_cap@50": 0.38431,
+            "R_cap@100": 0.4686,
+            "R_cap@200": 0.54797,
+            "R_cap@1000": 0.7206
+        },
+        "hole": {
+            "Hole@1": 0.325,
+            "Hole@3": 0.41333,
+            "Hole@5": 0.48,
+            "Hole@10": 0.5545,
+            "Hole@20": 0.63012,
+            "Hole@50": 0.73025,
+            "Hole@100": 0.79923,
+            "Hole@200": 0.8556,
+            "Hole@1000": 0.93489
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.4685,0.58291,0.61555,0.63923,0.65243,0.66162,0.66512,0.66738,0.66975
+map,0.43439,0.54012,0.55886,0.56904,0.57297,0.57463,0.57501,0.57516,0.57524
+precision,0.4685,0.24182,0.1628,0.08954,0.04771,0.02015,0.01036,0.00529,0.00109
+recall,0.43439,0.66842,0.74702,0.81819,0.86817,0.91176,0.93145,0.94589,0.9631
+mrr,0.4685,0.57831,0.59725,0.60729,0.61065,0.61202,0.61225,0.61232,0.61235
+recall_cap,0.4685,0.66964,0.74736,0.8182,0.86817,0.91176,0.93145,0.94589,0.9631
+hole,0.50285,0.72937,0.81251,0.891,0.93745,0.97033,0.98308,0.9901,0.99652

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.4685,
+            "NDCG@3": 0.58291,
+            "NDCG@5": 0.61555,
+            "NDCG@10": 0.63923,
+            "NDCG@20": 0.65243,
+            "NDCG@50": 0.66162,
+            "NDCG@100": 0.66512,
+            "NDCG@200": 0.66738,
+            "NDCG@1000": 0.66975
+        },
+        "map": {
+            "MAP@1": 0.43439,
+            "MAP@3": 0.54012,
+            "MAP@5": 0.55886,
+            "MAP@10": 0.56904,
+            "MAP@20": 0.57297,
+            "MAP@50": 0.57463,
+            "MAP@100": 0.57501,
+            "MAP@200": 0.57516,
+            "MAP@1000": 0.57524
+        },
+        "precision": {
+            "P@1": 0.4685,
+            "P@3": 0.24182,
+            "P@5": 0.1628,
+            "P@10": 0.08954,
+            "P@20": 0.04771,
+            "P@50": 0.02015,
+            "P@100": 0.01036,
+            "P@200": 0.00529,
+            "P@1000": 0.00109
+        },
+        "recall": {
+            "Recall@1": 0.43439,
+            "Recall@3": 0.66842,
+            "Recall@5": 0.74702,
+            "Recall@10": 0.81819,
+            "Recall@20": 0.86817,
+            "Recall@50": 0.91176,
+            "Recall@100": 0.93145,
+            "Recall@200": 0.94589,
+            "Recall@1000": 0.9631
+        },
+        "mrr": {
+            "MRR@1": 0.4685,
+            "MRR@3": 0.57831,
+            "MRR@5": 0.59725,
+            "MRR@10": 0.60729,
+            "MRR@20": 0.61065,
+            "MRR@50": 0.61202,
+            "MRR@100": 0.61225,
+            "MRR@200": 0.61232,
+            "MRR@1000": 0.61235
+        },
+        "recall_cap": {
+            "R_cap@1": 0.4685,
+            "R_cap@3": 0.66964,
+            "R_cap@5": 0.74736,
+            "R_cap@10": 0.8182,
+            "R_cap@20": 0.86817,
+            "R_cap@50": 0.91176,
+            "R_cap@100": 0.93145,
+            "R_cap@200": 0.94589,
+            "R_cap@1000": 0.9631
+        },
+        "hole": {
+            "Hole@1": 0.50285,
+            "Hole@3": 0.72937,
+            "Hole@5": 0.81251,
+            "Hole@10": 0.891,
+            "Hole@20": 0.93745,
+            "Hole@50": 0.97033,
+            "Hole@100": 0.98308,
+            "Hole@200": 0.9901,
+            "Hole@1000": 0.99652
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.24846,0.23935,0.25446,0.28062,0.3056,0.32979,0.34963,0.36675,0.39175
+map,0.12932,0.18415,0.2016,0.21602,0.22481,0.22978,0.23215,0.23351,0.23443
+precision,0.24846,0.16049,0.12191,0.08071,0.05031,0.02531,0.01525,0.00889,0.00225
+recall,0.12932,0.22337,0.27665,0.35412,0.43214,0.52487,0.61765,0.71066,0.8749
+mrr,0.24846,0.3053,0.31957,0.33238,0.3379,0.34081,0.34213,0.34257,0.34278
+recall_cap,0.24846,0.24691,0.28418,0.35454,0.43214,0.52487,0.61765,0.71066,0.8749
+hole,0.69599,0.78447,0.82068,0.86034,0.8892,0.91321,0.92128,0.92703,0.93571

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.24846,
+            "NDCG@3": 0.23935,
+            "NDCG@5": 0.25446,
+            "NDCG@10": 0.28062,
+            "NDCG@20": 0.3056,
+            "NDCG@50": 0.32979,
+            "NDCG@100": 0.34963,
+            "NDCG@200": 0.36675,
+            "NDCG@1000": 0.39175
+        },
+        "map": {
+            "MAP@1": 0.12932,
+            "MAP@3": 0.18415,
+            "MAP@5": 0.2016,
+            "MAP@10": 0.21602,
+            "MAP@20": 0.22481,
+            "MAP@50": 0.22978,
+            "MAP@100": 0.23215,
+            "MAP@200": 0.23351,
+            "MAP@1000": 0.23443
+        },
+        "precision": {
+            "P@1": 0.24846,
+            "P@3": 0.16049,
+            "P@5": 0.12191,
+            "P@10": 0.08071,
+            "P@20": 0.05031,
+            "P@50": 0.02531,
+            "P@100": 0.01525,
+            "P@200": 0.00889,
+            "P@1000": 0.00225
+        },
+        "recall": {
+            "Recall@1": 0.12932,
+            "Recall@3": 0.22337,
+            "Recall@5": 0.27665,
+            "Recall@10": 0.35412,
+            "Recall@20": 0.43214,
+            "Recall@50": 0.52487,
+            "Recall@100": 0.61765,
+            "Recall@200": 0.71066,
+            "Recall@1000": 0.8749
+        },
+        "mrr": {
+            "MRR@1": 0.24846,
+            "MRR@3": 0.3053,
+            "MRR@5": 0.31957,
+            "MRR@10": 0.33238,
+            "MRR@20": 0.3379,
+            "MRR@50": 0.34081,
+            "MRR@100": 0.34213,
+            "MRR@200": 0.34257,
+            "MRR@1000": 0.34278
+        },
+        "recall_cap": {
+            "R_cap@1": 0.24846,
+            "R_cap@3": 0.24691,
+            "R_cap@5": 0.28418,
+            "R_cap@10": 0.35454,
+            "R_cap@20": 0.43214,
+            "R_cap@50": 0.52487,
+            "R_cap@100": 0.61765,
+            "R_cap@200": 0.71066,
+            "R_cap@1000": 0.8749
+        },
+        "hole": {
+            "Hole@1": 0.69599,
+            "Hole@3": 0.78447,
+            "Hole@5": 0.82068,
+            "Hole@10": 0.86034,
+            "Hole@20": 0.8892,
+            "Hole@50": 0.91321,
+            "Hole@100": 0.92128,
+            "Hole@200": 0.92703,
+            "Hole@1000": 0.93571
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.59217,0.44989,0.47309,0.49429,0.50996,0.5251,0.53311,0.53972,0.55201
+map,0.29608,0.3726,0.38817,0.3994,0.40513,0.40845,0.40944,0.40992,0.41032
+precision,0.59217,0.27788,0.18512,0.10317,0.05661,0.02512,0.01337,0.00707,0.00159
+recall,0.29608,0.41681,0.4628,0.51587,0.5661,0.62809,0.66826,0.70689,0.79467
+mrr,0.59217,0.65318,0.66396,0.67034,0.67307,0.67445,0.67478,0.67492,0.675
+recall_cap,0.59217,0.41681,0.4628,0.51587,0.5661,0.62809,0.66826,0.70689,0.79467
+hole,0.40162,0.71195,0.80432,0.88632,0.93342,0.9654,0.97758,0.98425,0.99081

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.59217,
+            "NDCG@3": 0.44989,
+            "NDCG@5": 0.47309,
+            "NDCG@10": 0.49429,
+            "NDCG@20": 0.50996,
+            "NDCG@50": 0.5251,
+            "NDCG@100": 0.53311,
+            "NDCG@200": 0.53972,
+            "NDCG@1000": 0.55201
+        },
+        "map": {
+            "MAP@1": 0.29608,
+            "MAP@3": 0.3726,
+            "MAP@5": 0.38817,
+            "MAP@10": 0.3994,
+            "MAP@20": 0.40513,
+            "MAP@50": 0.40845,
+            "MAP@100": 0.40944,
+            "MAP@200": 0.40992,
+            "MAP@1000": 0.41032
+        },
+        "precision": {
+            "P@1": 0.59217,
+            "P@3": 0.27788,
+            "P@5": 0.18512,
+            "P@10": 0.10317,
+            "P@20": 0.05661,
+            "P@50": 0.02512,
+            "P@100": 0.01337,
+            "P@200": 0.00707,
+            "P@1000": 0.00159
+        },
+        "recall": {
+            "Recall@1": 0.29608,
+            "Recall@3": 0.41681,
+            "Recall@5": 0.4628,
+            "Recall@10": 0.51587,
+            "Recall@20": 0.5661,
+            "Recall@50": 0.62809,
+            "Recall@100": 0.66826,
+            "Recall@200": 0.70689,
+            "Recall@1000": 0.79467
+        },
+        "mrr": {
+            "MRR@1": 0.59217,
+            "MRR@3": 0.65318,
+            "MRR@5": 0.66396,
+            "MRR@10": 0.67034,
+            "MRR@20": 0.67307,
+            "MRR@50": 0.67445,
+            "MRR@100": 0.67478,
+            "MRR@200": 0.67492,
+            "MRR@1000": 0.675
+        },
+        "recall_cap": {
+            "R_cap@1": 0.59217,
+            "R_cap@3": 0.41681,
+            "R_cap@5": 0.4628,
+            "R_cap@10": 0.51587,
+            "R_cap@20": 0.5661,
+            "R_cap@50": 0.62809,
+            "R_cap@100": 0.66826,
+            "R_cap@200": 0.70689,
+            "R_cap@1000": 0.79467
+        },
+        "hole": {
+            "Hole@1": 0.40162,
+            "Hole@3": 0.71195,
+            "Hole@5": 0.80432,
+            "Hole@10": 0.88632,
+            "Hole@20": 0.93342,
+            "Hole@50": 0.9654,
+            "Hole@100": 0.97758,
+            "Hole@200": 0.98425,
+            "Hole@1000": 0.99081
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.07865,0.14084,0.17087,0.20651,0.2327,0.25783,0.27105,0.28319,0.29862
+map,0.07665,0.12377,0.14045,0.15525,0.1625,0.16668,0.16788,0.16852,0.16888
+precision,0.07865,0.06437,0.05381,0.03838,0.0246,0.01248,0.00711,0.00401,0.00095
+recall,0.07665,0.18771,0.26018,0.36899,0.47128,0.59502,0.67481,0.76033,0.89163
+mrr,0.07865,0.12655,0.14329,0.15812,0.16529,0.16938,0.17053,0.17114,0.17148
+recall_cap,0.07865,0.18758,0.26018,0.36899,0.47128,0.59502,0.67481,0.76033,0.89163
+hole,0.92106,0.93524,0.94562,0.96085,0.97458,0.98665,0.99204,0.99511,0.99822

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.07865,
+            "NDCG@3": 0.14084,
+            "NDCG@5": 0.17087,
+            "NDCG@10": 0.20651,
+            "NDCG@20": 0.2327,
+            "NDCG@50": 0.25783,
+            "NDCG@100": 0.27105,
+            "NDCG@200": 0.28319,
+            "NDCG@1000": 0.29862
+        },
+        "map": {
+            "MAP@1": 0.07665,
+            "MAP@3": 0.12377,
+            "MAP@5": 0.14045,
+            "MAP@10": 0.15525,
+            "MAP@20": 0.1625,
+            "MAP@50": 0.16668,
+            "MAP@100": 0.16788,
+            "MAP@200": 0.16852,
+            "MAP@1000": 0.16888
+        },
+        "precision": {
+            "P@1": 0.07865,
+            "P@3": 0.06437,
+            "P@5": 0.05381,
+            "P@10": 0.03838,
+            "P@20": 0.0246,
+            "P@50": 0.01248,
+            "P@100": 0.00711,
+            "P@200": 0.00401,
+            "P@1000": 0.00095
+        },
+        "recall": {
+            "Recall@1": 0.07665,
+            "Recall@3": 0.18771,
+            "Recall@5": 0.26018,
+            "Recall@10": 0.36899,
+            "Recall@20": 0.47128,
+            "Recall@50": 0.59502,
+            "Recall@100": 0.67481,
+            "Recall@200": 0.76033,
+            "Recall@1000": 0.89163
+        },
+        "mrr": {
+            "MRR@1": 0.07865,
+            "MRR@3": 0.12655,
+            "MRR@5": 0.14329,
+            "MRR@10": 0.15812,
+            "MRR@20": 0.16529,
+            "MRR@50": 0.16938,
+            "MRR@100": 0.17053,
+            "MRR@200": 0.17114,
+            "MRR@1000": 0.17148
+        },
+        "recall_cap": {
+            "R_cap@1": 0.07865,
+            "R_cap@3": 0.18758,
+            "R_cap@5": 0.26018,
+            "R_cap@10": 0.36899,
+            "R_cap@20": 0.47128,
+            "R_cap@50": 0.59502,
+            "R_cap@100": 0.67481,
+            "R_cap@200": 0.76033,
+            "R_cap@1000": 0.89163
+        },
+        "hole": {
+            "Hole@1": 0.92106,
+            "Hole@3": 0.93524,
+            "Hole@5": 0.94562,
+            "Hole@10": 0.96085,
+            "Hole@20": 0.97458,
+            "Hole@50": 0.98665,
+            "Hole@100": 0.99204,
+            "Hole@200": 0.99511,
+            "Hole@1000": 0.99822
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.4226,0.39679,0.36804,0.34134,0.32087,0.30701,0.31615,0.33327,0.40435
+map,0.05571,0.09379,0.1076,0.1265,0.14033,0.15186,0.15879,0.16411,0.17343
+precision,0.43653,0.37771,0.31765,0.25418,0.18839,0.11783,0.08053,0.05351,0.02111
+recall,0.05571,0.10869,0.13097,0.16925,0.20788,0.26467,0.31962,0.39443,0.64151
+mrr,0.43653,0.52167,0.5308,0.53731,0.54116,0.54308,0.54363,0.54396,0.54412
+recall_cap,0.43653,0.39783,0.35619,0.32178,0.29557,0.29771,0.33502,0.39953,0.64151
+hole,0.0805,0.07843,0.08669,0.09938,0.10836,0.11944,0.12644,0.13033,0.13478

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.4226,
+            "NDCG@3": 0.39679,
+            "NDCG@5": 0.36804,
+            "NDCG@10": 0.34134,
+            "NDCG@20": 0.32087,
+            "NDCG@50": 0.30701,
+            "NDCG@100": 0.31615,
+            "NDCG@200": 0.33327,
+            "NDCG@1000": 0.40435
+        },
+        "map": {
+            "MAP@1": 0.05571,
+            "MAP@3": 0.09379,
+            "MAP@5": 0.1076,
+            "MAP@10": 0.1265,
+            "MAP@20": 0.14033,
+            "MAP@50": 0.15186,
+            "MAP@100": 0.15879,
+            "MAP@200": 0.16411,
+            "MAP@1000": 0.17343
+        },
+        "precision": {
+            "P@1": 0.43653,
+            "P@3": 0.37771,
+            "P@5": 0.31765,
+            "P@10": 0.25418,
+            "P@20": 0.18839,
+            "P@50": 0.11783,
+            "P@100": 0.08053,
+            "P@200": 0.05351,
+            "P@1000": 0.02111
+        },
+        "recall": {
+            "Recall@1": 0.05571,
+            "Recall@3": 0.10869,
+            "Recall@5": 0.13097,
+            "Recall@10": 0.16925,
+            "Recall@20": 0.20788,
+            "Recall@50": 0.26467,
+            "Recall@100": 0.31962,
+            "Recall@200": 0.39443,
+            "Recall@1000": 0.64151
+        },
+        "mrr": {
+            "MRR@1": 0.43653,
+            "MRR@3": 0.52167,
+            "MRR@5": 0.5308,
+            "MRR@10": 0.53731,
+            "MRR@20": 0.54116,
+            "MRR@50": 0.54308,
+            "MRR@100": 0.54363,
+            "MRR@200": 0.54396,
+            "MRR@1000": 0.54412
+        },
+        "recall_cap": {
+            "R_cap@1": 0.43653,
+            "R_cap@3": 0.39783,
+            "R_cap@5": 0.35619,
+            "R_cap@10": 0.32178,
+            "R_cap@20": 0.29557,
+            "R_cap@50": 0.29771,
+            "R_cap@100": 0.33502,
+            "R_cap@200": 0.39953,
+            "R_cap@1000": 0.64151
+        },
+        "hole": {
+            "Hole@1": 0.0805,
+            "Hole@3": 0.07843,
+            "Hole@5": 0.08669,
+            "Hole@10": 0.09938,
+            "Hole@20": 0.10836,
+            "Hole@50": 0.11944,
+            "Hole@100": 0.12644,
+            "Hole@200": 0.13033,
+            "Hole@1000": 0.13478
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1263,0.1948,0.23184,0.27574,0.31019,0.33737,0.34981,0.35952,0.36958
+map,0.11097,0.17079,0.19176,0.21075,0.22104,0.22576,0.22697,0.22753,0.2278
+precision,0.1263,0.09511,0.07746,0.05458,0.0355,0.01739,0.00961,0.00521,0.00115
+recall,0.11097,0.24488,0.33196,0.46125,0.59007,0.72019,0.7927,0.85808,0.93996
+mrr,0.12659,0.19028,0.21087,0.22842,0.23687,0.24093,0.24189,0.24234,0.24254
+recall_cap,0.12659,0.24493,0.33196,0.46125,0.59007,0.72019,0.7927,0.85808,0.93996
+hole,0.87283,0.90324,0.92097,0.94371,0.96246,0.98022,0.98807,0.9925,0.99676

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1263,
+            "NDCG@3": 0.1948,
+            "NDCG@5": 0.23184,
+            "NDCG@10": 0.27574,
+            "NDCG@20": 0.31019,
+            "NDCG@50": 0.33737,
+            "NDCG@100": 0.34981,
+            "NDCG@200": 0.35952,
+            "NDCG@1000": 0.36958
+        },
+        "map": {
+            "MAP@1": 0.11097,
+            "MAP@3": 0.17079,
+            "MAP@5": 0.19176,
+            "MAP@10": 0.21075,
+            "MAP@20": 0.22104,
+            "MAP@50": 0.22576,
+            "MAP@100": 0.22697,
+            "MAP@200": 0.22753,
+            "MAP@1000": 0.2278
+        },
+        "precision": {
+            "P@1": 0.1263,
+            "P@3": 0.09511,
+            "P@5": 0.07746,
+            "P@10": 0.05458,
+            "P@20": 0.0355,
+            "P@50": 0.01739,
+            "P@100": 0.00961,
+            "P@200": 0.00521,
+            "P@1000": 0.00115
+        },
+        "recall": {
+            "Recall@1": 0.11097,
+            "Recall@3": 0.24488,
+            "Recall@5": 0.33196,
+            "Recall@10": 0.46125,
+            "Recall@20": 0.59007,
+            "Recall@50": 0.72019,
+            "Recall@100": 0.7927,
+            "Recall@200": 0.85808,
+            "Recall@1000": 0.93996
+        },
+        "mrr": {
+            "MRR@1": 0.12659,
+            "MRR@3": 0.19028,
+            "MRR@5": 0.21087,
+            "MRR@10": 0.22842,
+            "MRR@20": 0.23687,
+            "MRR@50": 0.24093,
+            "MRR@100": 0.24189,
+            "MRR@200": 0.24234,
+            "MRR@1000": 0.24254
+        },
+        "recall_cap": {
+            "R_cap@1": 0.12659,
+            "R_cap@3": 0.24493,
+            "R_cap@5": 0.33196,
+            "R_cap@10": 0.46125,
+            "R_cap@20": 0.59007,
+            "R_cap@50": 0.72019,
+            "R_cap@100": 0.7927,
+            "R_cap@200": 0.85808,
+            "R_cap@1000": 0.93996
+        },
+        "hole": {
+            "Hole@1": 0.87283,
+            "Hole@3": 0.90324,
+            "Hole@5": 0.92097,
+            "Hole@10": 0.94371,
+            "Hole@20": 0.96246,
+            "Hole@50": 0.98022,
+            "Hole@100": 0.98807,
+            "Hole@200": 0.9925,
+            "Hole@1000": 0.99676
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6897,0.74408,0.76601,0.78549,0.79663,0.80497,0.80825,0.81002,0.81195
+map,0.60075,0.70413,0.7239,0.73673,0.74194,0.74438,0.74492,0.74512,0.74524
+precision,0.6897,0.326,0.21826,0.12171,0.06578,0.02827,0.01462,0.00748,0.00155
+recall,0.60075,0.77438,0.83167,0.88855,0.92615,0.95942,0.9752,0.98443,0.99636
+mrr,0.6894,0.75502,0.7656,0.77143,0.77332,0.77411,0.77426,0.7743,0.77432
+recall_cap,0.6894,0.78705,0.83681,0.88983,0.92641,0.95944,0.9752,0.98443,0.99636
+hole,0.2918,0.64107,0.74518,0.83811,0.89266,0.93065,0.94517,0.95374,0.96222

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6897,
+            "NDCG@3": 0.74408,
+            "NDCG@5": 0.76601,
+            "NDCG@10": 0.78549,
+            "NDCG@20": 0.79663,
+            "NDCG@50": 0.80497,
+            "NDCG@100": 0.80825,
+            "NDCG@200": 0.81002,
+            "NDCG@1000": 0.81195
+        },
+        "map": {
+            "MAP@1": 0.60075,
+            "MAP@3": 0.70413,
+            "MAP@5": 0.7239,
+            "MAP@10": 0.73673,
+            "MAP@20": 0.74194,
+            "MAP@50": 0.74438,
+            "MAP@100": 0.74492,
+            "MAP@200": 0.74512,
+            "MAP@1000": 0.74524
+        },
+        "precision": {
+            "P@1": 0.6897,
+            "P@3": 0.326,
+            "P@5": 0.21826,
+            "P@10": 0.12171,
+            "P@20": 0.06578,
+            "P@50": 0.02827,
+            "P@100": 0.01462,
+            "P@200": 0.00748,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.60075,
+            "Recall@3": 0.77438,
+            "Recall@5": 0.83167,
+            "Recall@10": 0.88855,
+            "Recall@20": 0.92615,
+            "Recall@50": 0.95942,
+            "Recall@100": 0.9752,
+            "Recall@200": 0.98443,
+            "Recall@1000": 0.99636
+        },
+        "mrr": {
+            "MRR@1": 0.6894,
+            "MRR@3": 0.75502,
+            "MRR@5": 0.7656,
+            "MRR@10": 0.77143,
+            "MRR@20": 0.77332,
+            "MRR@50": 0.77411,
+            "MRR@100": 0.77426,
+            "MRR@200": 0.7743,
+            "MRR@1000": 0.77432
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6894,
+            "R_cap@3": 0.78705,
+            "R_cap@5": 0.83681,
+            "R_cap@10": 0.88983,
+            "R_cap@20": 0.92641,
+            "R_cap@50": 0.95944,
+            "R_cap@100": 0.9752,
+            "R_cap@200": 0.98443,
+            "R_cap@1000": 0.99636
+        },
+        "hole": {
+            "Hole@1": 0.2918,
+            "Hole@3": 0.64107,
+            "Hole@5": 0.74518,
+            "Hole@10": 0.83811,
+            "Hole@20": 0.89266,
+            "Hole@50": 0.93065,
+            "Hole@100": 0.94517,
+            "Hole@200": 0.95374,
+            "Hole@1000": 0.96222
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.192,0.14881,0.13068,0.15962,0.18417,0.21593,0.23574,0.25272,0.28886
+map,0.03908,0.06637,0.07823,0.09173,0.10044,0.10749,0.11018,0.11157,0.11296
+precision,0.192,0.13667,0.1134,0.083,0.0556,0.0316,0.01937,0.01145,0.00322
+recall,0.03908,0.08328,0.11523,0.16823,0.22532,0.32033,0.39283,0.4648,0.65352
+mrr,0.192,0.24867,0.26757,0.2826,0.28888,0.29292,0.29393,0.2944,0.29467
+recall_cap,0.192,0.13667,0.11523,0.16823,0.22532,0.32033,0.39283,0.4648,0.65352
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.192,
+            "NDCG@3": 0.14881,
+            "NDCG@5": 0.13068,
+            "NDCG@10": 0.15962,
+            "NDCG@20": 0.18417,
+            "NDCG@50": 0.21593,
+            "NDCG@100": 0.23574,
+            "NDCG@200": 0.25272,
+            "NDCG@1000": 0.28886
+        },
+        "map": {
+            "MAP@1": 0.03908,
+            "MAP@3": 0.06637,
+            "MAP@5": 0.07823,
+            "MAP@10": 0.09173,
+            "MAP@20": 0.10044,
+            "MAP@50": 0.10749,
+            "MAP@100": 0.11018,
+            "MAP@200": 0.11157,
+            "MAP@1000": 0.11296
+        },
+        "precision": {
+            "P@1": 0.192,
+            "P@3": 0.13667,
+            "P@5": 0.1134,
+            "P@10": 0.083,
+            "P@20": 0.0556,
+            "P@50": 0.0316,
+            "P@100": 0.01937,
+            "P@200": 0.01145,
+            "P@1000": 0.00322
+        },
+        "recall": {
+            "Recall@1": 0.03908,
+            "Recall@3": 0.08328,
+            "Recall@5": 0.11523,
+            "Recall@10": 0.16823,
+            "Recall@20": 0.22532,
+            "Recall@50": 0.32033,
+            "Recall@100": 0.39283,
+            "Recall@200": 0.4648,
+            "Recall@1000": 0.65352
+        },
+        "mrr": {
+            "MRR@1": 0.192,
+            "MRR@3": 0.24867,
+            "MRR@5": 0.26757,
+            "MRR@10": 0.2826,
+            "MRR@20": 0.28888,
+            "MRR@50": 0.29292,
+            "MRR@100": 0.29393,
+            "MRR@200": 0.2944,
+            "MRR@1000": 0.29467
+        },
+        "recall_cap": {
+            "R_cap@1": 0.192,
+            "R_cap@3": 0.13667,
+            "R_cap@5": 0.11523,
+            "R_cap@10": 0.16823,
+            "R_cap@20": 0.22532,
+            "R_cap@50": 0.32033,
+            "R_cap@100": 0.39283,
+            "R_cap@200": 0.4648,
+            "R_cap@1000": 0.65352
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.49667,0.56063,0.5832,0.618,0.63668,0.64841,0.65303,0.65582,0.66222
+map,0.48222,0.53771,0.55207,0.56804,0.57421,0.5764,0.57689,0.57703,0.57722
+precision,0.49667,0.21556,0.14467,0.08467,0.04683,0.02007,0.01037,0.00528,0.00111
+recall,0.48222,0.602,0.65778,0.76,0.82856,0.88467,0.911,0.931,0.98333
+mrr,0.49667,0.555,0.56767,0.58091,0.58502,0.58662,0.58706,0.5872,0.58737
+recall_cap,0.49667,0.605,0.65778,0.76,0.82856,0.88467,0.911,0.931,0.98333
+hole,0.48,0.74444,0.808,0.86233,0.902,0.9238,0.9323,0.9359,0.94129

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.49667,
+            "NDCG@3": 0.56063,
+            "NDCG@5": 0.5832,
+            "NDCG@10": 0.618,
+            "NDCG@20": 0.63668,
+            "NDCG@50": 0.64841,
+            "NDCG@100": 0.65303,
+            "NDCG@200": 0.65582,
+            "NDCG@1000": 0.66222
+        },
+        "map": {
+            "MAP@1": 0.48222,
+            "MAP@3": 0.53771,
+            "MAP@5": 0.55207,
+            "MAP@10": 0.56804,
+            "MAP@20": 0.57421,
+            "MAP@50": 0.5764,
+            "MAP@100": 0.57689,
+            "MAP@200": 0.57703,
+            "MAP@1000": 0.57722
+        },
+        "precision": {
+            "P@1": 0.49667,
+            "P@3": 0.21556,
+            "P@5": 0.14467,
+            "P@10": 0.08467,
+            "P@20": 0.04683,
+            "P@50": 0.02007,
+            "P@100": 0.01037,
+            "P@200": 0.00528,
+            "P@1000": 0.00111
+        },
+        "recall": {
+            "Recall@1": 0.48222,
+            "Recall@3": 0.602,
+            "Recall@5": 0.65778,
+            "Recall@10": 0.76,
+            "Recall@20": 0.82856,
+            "Recall@50": 0.88467,
+            "Recall@100": 0.911,
+            "Recall@200": 0.931,
+            "Recall@1000": 0.98333
+        },
+        "mrr": {
+            "MRR@1": 0.49667,
+            "MRR@3": 0.555,
+            "MRR@5": 0.56767,
+            "MRR@10": 0.58091,
+            "MRR@20": 0.58502,
+            "MRR@50": 0.58662,
+            "MRR@100": 0.58706,
+            "MRR@200": 0.5872,
+            "MRR@1000": 0.58737
+        },
+        "recall_cap": {
+            "R_cap@1": 0.49667,
+            "R_cap@3": 0.605,
+            "R_cap@5": 0.65778,
+            "R_cap@10": 0.76,
+            "R_cap@20": 0.82856,
+            "R_cap@50": 0.88467,
+            "R_cap@100": 0.911,
+            "R_cap@200": 0.931,
+            "R_cap@1000": 0.98333
+        },
+        "hole": {
+            "Hole@1": 0.48,
+            "Hole@3": 0.74444,
+            "Hole@5": 0.808,
+            "Hole@10": 0.86233,
+            "Hole@20": 0.902,
+            "Hole@50": 0.9238,
+            "Hole@100": 0.9323,
+            "Hole@200": 0.9359,
+            "Hole@1000": 0.94129
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7430861317995934,
+    "eval_senteval-sickr_spearman": 0.7323591663810709,
+    "eval_senteval-avg_sts": 0.7377226490903321
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.700217766857111,
+                0.0
+            ],
+            "spearman": [
+                0.694596243514894,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7188725565702827,
+                7.642460243707613e-239
+            ],
+            "spearman": [
+                0.7430861317995934,
+                1.3229042265692801e-263
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.6697149917661123,
+                3.5510311886663796e-180
+            ],
+            "spearman": [
+                0.7028355722100523,
+                6.396511507534624e-206
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7003515412165919,
+                "mean": 0.6962684383978353,
+                "wmean": 0.698585738312752
+            },
+            "spearman": {
+                "all": 0.708944371429387,
+                "mean": 0.7135059825081799,
+                "wmean": 0.7043432146203266
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7900787687078513,
+                0.0
+            ],
+            "spearman": [
+                0.7162798220982042,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7941456305052318,
+                8.704175570070149e-110
+            ],
+            "spearman": [
+                0.7323591663810709,
+                3.70700656086251e-85
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7827901659487767,
+                0.0
+            ],
+            "spearman": [
+                0.7066436772359563,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7866207722959027,
+                "mean": 0.7890048550539532,
+                "wmean": 0.7866661047715896
+            },
+            "spearman": {
+                "all": 0.7122486034563766,
+                "mean": 0.7184275552384105,
+                "wmean": 0.7123070595722788
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7430861317995934,
+    "eval_senteval-sickr_spearman": 0.7323591663810709,
+    "eval_senteval-avg_sts": 0.7377226490903321
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.61,0.63123,0.60405,0.58244,0.55117,0.49432,0.43674,0.37626,0.40298
+map,0.00181,0.00533,0.00779,0.01433,0.02492,0.04887,0.07645,0.11131,0.18572
+precision,0.66,0.7,0.652,0.628,0.588,0.5188,0.449,0.3701,0.18128
+recall,0.00181,0.00583,0.00862,0.01669,0.03021,0.06328,0.10597,0.16785,0.38057
+mrr,0.66,0.78333,0.78333,0.78333,0.78487,0.78567,0.78589,0.78589,0.78589
+recall_cap,0.66,0.7,0.652,0.628,0.588,0.5188,0.4492,0.37335,0.38221
+hole,0.26,0.18667,0.228,0.226,0.226,0.2568,0.3028,0.3433,0.45968

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.61,
+            "NDCG@3": 0.63123,
+            "NDCG@5": 0.60405,
+            "NDCG@10": 0.58244,
+            "NDCG@20": 0.55117,
+            "NDCG@50": 0.49432,
+            "NDCG@100": 0.43674,
+            "NDCG@200": 0.37626,
+            "NDCG@1000": 0.40298
+        },
+        "map": {
+            "MAP@1": 0.00181,
+            "MAP@3": 0.00533,
+            "MAP@5": 0.00779,
+            "MAP@10": 0.01433,
+            "MAP@20": 0.02492,
+            "MAP@50": 0.04887,
+            "MAP@100": 0.07645,
+            "MAP@200": 0.11131,
+            "MAP@1000": 0.18572
+        },
+        "precision": {
+            "P@1": 0.66,
+            "P@3": 0.7,
+            "P@5": 0.652,
+            "P@10": 0.628,
+            "P@20": 0.588,
+            "P@50": 0.5188,
+            "P@100": 0.449,
+            "P@200": 0.3701,
+            "P@1000": 0.18128
+        },
+        "recall": {
+            "Recall@1": 0.00181,
+            "Recall@3": 0.00583,
+            "Recall@5": 0.00862,
+            "Recall@10": 0.01669,
+            "Recall@20": 0.03021,
+            "Recall@50": 0.06328,
+            "Recall@100": 0.10597,
+            "Recall@200": 0.16785,
+            "Recall@1000": 0.38057
+        },
+        "mrr": {
+            "MRR@1": 0.66,
+            "MRR@3": 0.78333,
+            "MRR@5": 0.78333,
+            "MRR@10": 0.78333,
+            "MRR@20": 0.78487,
+            "MRR@50": 0.78567,
+            "MRR@100": 0.78589,
+            "MRR@200": 0.78589,
+            "MRR@1000": 0.78589
+        },
+        "recall_cap": {
+            "R_cap@1": 0.66,
+            "R_cap@3": 0.7,
+            "R_cap@5": 0.652,
+            "R_cap@10": 0.628,
+            "R_cap@20": 0.588,
+            "R_cap@50": 0.5188,
+            "R_cap@100": 0.4492,
+            "R_cap@200": 0.37335,
+            "R_cap@1000": 0.38221
+        },
+        "hole": {
+            "Hole@1": 0.26,
+            "Hole@3": 0.18667,
+            "Hole@5": 0.228,
+            "Hole@10": 0.226,
+            "Hole@20": 0.226,
+            "Hole@50": 0.2568,
+            "Hole@100": 0.3028,
+            "Hole@200": 0.3433,
+            "Hole@1000": 0.45968
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.20408,0.18027,0.17318,0.18565,0.21149,0.27352,0.31876,0.35829,0.43774
+map,0.02085,0.04242,0.05288,0.07506,0.09542,0.11655,0.12782,0.13519,0.14333
+precision,0.20408,0.18367,0.17143,0.16735,0.1398,0.09755,0.06837,0.04449,0.01467
+recall,0.02085,0.05119,0.07492,0.13326,0.20962,0.33334,0.43843,0.54287,0.80178
+mrr,0.20408,0.29592,0.31939,0.34798,0.35971,0.36191,0.36222,0.36222,0.36228
+recall_cap,0.20408,0.18367,0.17551,0.18546,0.2149,0.33334,0.43843,0.54287,0.80178
+hole,0.71429,0.71429,0.75102,0.76939,0.80918,0.86163,0.89816,0.92837,0.96918

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.20408,
+            "NDCG@3": 0.18027,
+            "NDCG@5": 0.17318,
+            "NDCG@10": 0.18565,
+            "NDCG@20": 0.21149,
+            "NDCG@50": 0.27352,
+            "NDCG@100": 0.31876,
+            "NDCG@200": 0.35829,
+            "NDCG@1000": 0.43774
+        },
+        "map": {
+            "MAP@1": 0.02085,
+            "MAP@3": 0.04242,
+            "MAP@5": 0.05288,
+            "MAP@10": 0.07506,
+            "MAP@20": 0.09542,
+            "MAP@50": 0.11655,
+            "MAP@100": 0.12782,
+            "MAP@200": 0.13519,
+            "MAP@1000": 0.14333
+        },
+        "precision": {
+            "P@1": 0.20408,
+            "P@3": 0.18367,
+            "P@5": 0.17143,
+            "P@10": 0.16735,
+            "P@20": 0.1398,
+            "P@50": 0.09755,
+            "P@100": 0.06837,
+            "P@200": 0.04449,
+            "P@1000": 0.01467
+        },
+        "recall": {
+            "Recall@1": 0.02085,
+            "Recall@3": 0.05119,
+            "Recall@5": 0.07492,
+            "Recall@10": 0.13326,
+            "Recall@20": 0.20962,
+            "Recall@50": 0.33334,
+            "Recall@100": 0.43843,
+            "Recall@200": 0.54287,
+            "Recall@1000": 0.80178
+        },
+        "mrr": {
+            "MRR@1": 0.20408,
+            "MRR@3": 0.29592,
+            "MRR@5": 0.31939,
+            "MRR@10": 0.34798,
+            "MRR@20": 0.35971,
+            "MRR@50": 0.36191,
+            "MRR@100": 0.36222,
+            "MRR@200": 0.36222,
+            "MRR@1000": 0.36228
+        },
+        "recall_cap": {
+            "R_cap@1": 0.20408,
+            "R_cap@3": 0.18367,
+            "R_cap@5": 0.17551,
+            "R_cap@10": 0.18546,
+            "R_cap@20": 0.2149,
+            "R_cap@50": 0.33334,
+            "R_cap@100": 0.43843,
+            "R_cap@200": 0.54287,
+            "R_cap@1000": 0.80178
+        },
+        "hole": {
+            "Hole@1": 0.71429,
+            "Hole@3": 0.71429,
+            "Hole@5": 0.75102,
+            "Hole@10": 0.76939,
+            "Hole@20": 0.80918,
+            "Hole@50": 0.86163,
+            "Hole@100": 0.89816,
+            "Hole@200": 0.92837,
+            "Hole@1000": 0.96918
+        }
+    }
+}

cc.inbatch.topic.bs1024.gpu8.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=false
+export NUM_WORKER=4
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.T03b_topic.inbatch.contriever256-special50.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --train_file /export/home/data/search/upr/cc/T03B_PileCC_topic.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+# export WANDB_RUN_ID=zl4sljkg
+# nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1458,0.25465,0.30138,0.36665,0.40931,0.43032,0.43752,0.43972,0.44131
+map,0.1458,0.227,0.25268,0.27931,0.29133,0.29487,0.29552,0.29563,0.29567
+precision,0.1458,0.11166,0.0899,0.06529,0.04097,0.01846,0.00967,0.00491,0.001
+recall,0.1458,0.33499,0.4495,0.65292,0.81935,0.92319,0.96728,0.98293,0.99644
+mrr,0.15434,0.22973,0.25537,0.28239,0.29444,0.298,0.29866,0.29878,0.29882
+recall_cap,0.15434,0.33357,0.44666,0.6522,0.81863,0.92248,0.96657,0.98293,0.99644
+hole,0.68919,0.67236,0.66401,0.64339,0.6697,0.7468,0.78873,0.8117,0.82983

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1458,
+            "NDCG@3": 0.25465,
+            "NDCG@5": 0.30138,
+            "NDCG@10": 0.36665,
+            "NDCG@20": 0.40931,
+            "NDCG@50": 0.43032,
+            "NDCG@100": 0.43752,
+            "NDCG@200": 0.43972,
+            "NDCG@1000": 0.44131
+        },
+        "map": {
+            "MAP@1": 0.1458,
+            "MAP@3": 0.227,
+            "MAP@5": 0.25268,
+            "MAP@10": 0.27931,
+            "MAP@20": 0.29133,
+            "MAP@50": 0.29487,
+            "MAP@100": 0.29552,
+            "MAP@200": 0.29563,
+            "MAP@1000": 0.29567
+        },
+        "precision": {
+            "P@1": 0.1458,
+            "P@3": 0.11166,
+            "P@5": 0.0899,
+            "P@10": 0.06529,
+            "P@20": 0.04097,
+            "P@50": 0.01846,
+            "P@100": 0.00967,
+            "P@200": 0.00491,
+            "P@1000": 0.001
+        },
+        "recall": {
+            "Recall@1": 0.1458,
+            "Recall@3": 0.33499,
+            "Recall@5": 0.4495,
+            "Recall@10": 0.65292,
+            "Recall@20": 0.81935,
+            "Recall@50": 0.92319,
+            "Recall@100": 0.96728,
+            "Recall@200": 0.98293,
+            "Recall@1000": 0.99644
+        },
+        "mrr": {
+            "MRR@1": 0.15434,
+            "MRR@3": 0.22973,
+            "MRR@5": 0.25537,
+            "MRR@10": 0.28239,
+            "MRR@20": 0.29444,
+            "MRR@50": 0.298,
+            "MRR@100": 0.29866,
+            "MRR@200": 0.29878,
+            "MRR@1000": 0.29882
+        },
+        "recall_cap": {
+            "R_cap@1": 0.15434,
+            "R_cap@3": 0.33357,
+            "R_cap@5": 0.44666,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.81863,
+            "R_cap@50": 0.92248,
+            "R_cap@100": 0.96657,
+            "R_cap@200": 0.98293,
+            "R_cap@1000": 0.99644
+        },
+        "hole": {
+            "Hole@1": 0.68919,
+            "Hole@3": 0.67236,
+            "Hole@5": 0.66401,
+            "Hole@10": 0.64339,
+            "Hole@20": 0.6697,
+            "Hole@50": 0.7468,
+            "Hole@100": 0.78873,
+            "Hole@200": 0.8117,
+            "Hole@1000": 0.82983
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.34134,
+    "eval_beir-nfcorpus_recall@10": 0.16925,
+    "eval_beir-nfcorpus_recall@20": 0.20788,
+    "eval_beir-nfcorpus_recall@100": 0.31962,
+    "eval_beir-fiqa_ndcg@10": 0.28062,
+    "eval_beir-fiqa_recall@10": 0.35412,
+    "eval_beir-fiqa_recall@20": 0.43214,
+    "eval_beir-fiqa_recall@100": 0.61765,
+    "eval_beir-arguana_ndcg@10": 0.36665,
+    "eval_beir-arguana_recall@10": 0.65292,
+    "eval_beir-arguana_recall@20": 0.81935,
+    "eval_beir-arguana_recall@100": 0.96728,
+    "eval_beir-scidocs_ndcg@10": 0.15962,
+    "eval_beir-scidocs_recall@10": 0.16823,
+    "eval_beir-scidocs_recall@20": 0.22532,
+    "eval_beir-scidocs_recall@100": 0.39283,
+    "eval_beir-scifact_ndcg@10": 0.618,
+    "eval_beir-scifact_recall@10": 0.76,
+    "eval_beir-scifact_recall@20": 0.82856,
+    "eval_beir-scifact_recall@100": 0.911,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18565,
+    "eval_beir-webis-touche2020_recall@10": 0.13326,
+    "eval_beir-webis-touche2020_recall@20": 0.20962,
+    "eval_beir-webis-touche2020_recall@100": 0.43843,
+    "eval_beir-cqadupstack_ndcg@10": 0.3031158333333333,
+    "eval_beir-cqadupstack_recall@10": 0.40381749999999994,
+    "eval_beir-cqadupstack_recall@20": 0.4736041666666666,
+    "eval_beir-cqadupstack_recall@100": 0.6389233333333334,
+    "eval_beir-trec-covid_ndcg@10": 0.58244,
+    "eval_beir-trec-covid_recall@10": 0.628,
+    "eval_beir-trec-covid_recall@20": 0.588,
+    "eval_beir-trec-covid_recall@100": 0.4492,
+    "eval_beir-quora_ndcg@10": 0.78549,
+    "eval_beir-quora_recall@10": 0.88855,
+    "eval_beir-quora_recall@20": 0.92615,
+    "eval_beir-quora_recall@100": 0.9752,
+    "eval_beir-nq_ndcg@10": 0.27574,
+    "eval_beir-nq_recall@10": 0.46125,
+    "eval_beir-nq_recall@20": 0.59007,
+    "eval_beir-nq_recall@100": 0.7927,
+    "eval_beir-avg_ndcg@10": 0.38986658333333335,
+    "eval_beir-avg_recall@10": 0.46193975,
+    "eval_beir-avg_recall@20": 0.5300694166666666,
+    "eval_beir-avg_recall@100": 0.6502833333333334
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22085083333333333,0.2611408333333333,0.280635,0.3031158333333333,0.3222125,0.34235499999999996,0.35566999999999993,0.36617166666666673,0.38425166666666666
+map,0.18674833333333338,0.234905,0.24751333333333336,0.25813250000000004,0.26415083333333333,0.26800750000000007,0.26944166666666663,0.2701408333333333,0.27071833333333334
+precision,0.22085083333333333,0.12022333333333333,0.08697666666666669,0.05404916666666667,0.032967500000000004,0.016465000000000004,0.009618333333333335,0.005469166666666667,0.0014016666666666667
+recall,0.18674833333333338,0.28826999999999997,0.3373541666666666,0.40381749999999994,0.4736041666666666,0.5654874999999999,0.6389233333333334,0.7054,0.8424908333333333
+mrr,0.22085083333333333,0.27007333333333333,0.2825991666666667,0.2922741666666666,0.2973,0.30019083333333335,0.3012291666666667,0.3016791666666666,0.30198583333333334
+recall_cap,0.22085083333333333,0.2941683333333333,0.3401016666666666,0.4049400000000001,0.47411749999999997,0.5656558333333334,0.6390225,0.7053058333333334,0.8424908333333333
+hole,0.706795,0.8030316666666667,0.8332233333333333,0.8648841666666668,0.8864941666666667,0.9045466666666666,0.9131933333333335,0.9195391666666667,0.9277475000000001

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22085083333333333,
+            "NDCG@3": 0.2611408333333333,
+            "NDCG@5": 0.280635,
+            "NDCG@10": 0.3031158333333333,
+            "NDCG@20": 0.3222125,
+            "NDCG@50": 0.34235499999999996,
+            "NDCG@100": 0.35566999999999993,
+            "NDCG@200": 0.36617166666666673,
+            "NDCG@1000": 0.38425166666666666
+        },
+        "map": {
+            "MAP@1": 0.18674833333333338,
+            "MAP@3": 0.234905,
+            "MAP@5": 0.24751333333333336,
+            "MAP@10": 0.25813250000000004,
+            "MAP@20": 0.26415083333333333,
+            "MAP@50": 0.26800750000000007,
+            "MAP@100": 0.26944166666666663,
+            "MAP@200": 0.2701408333333333,
+            "MAP@1000": 0.27071833333333334
+        },
+        "precision": {
+            "P@1": 0.22085083333333333,
+            "P@3": 0.12022333333333333,
+            "P@5": 0.08697666666666669,
+            "P@10": 0.05404916666666667,
+            "P@20": 0.032967500000000004,
+            "P@50": 0.016465000000000004,
+            "P@100": 0.009618333333333335,
+            "P@200": 0.005469166666666667,
+            "P@1000": 0.0014016666666666667
+        },
+        "recall": {
+            "Recall@1": 0.18674833333333338,
+            "Recall@3": 0.28826999999999997,
+            "Recall@5": 0.3373541666666666,
+            "Recall@10": 0.40381749999999994,
+            "Recall@20": 0.4736041666666666,
+            "Recall@50": 0.5654874999999999,
+            "Recall@100": 0.6389233333333334,
+            "Recall@200": 0.7054,
+            "Recall@1000": 0.8424908333333333
+        },
+        "mrr": {
+            "MRR@1": 0.22085083333333333,
+            "MRR@3": 0.27007333333333333,
+            "MRR@5": 0.2825991666666667,
+            "MRR@10": 0.2922741666666666,
+            "MRR@20": 0.2973,
+            "MRR@50": 0.30019083333333335,
+            "MRR@100": 0.3012291666666667,
+            "MRR@200": 0.3016791666666666,
+            "MRR@1000": 0.30198583333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22085083333333333,
+            "R_cap@3": 0.2941683333333333,
+            "R_cap@5": 0.3401016666666666,
+            "R_cap@10": 0.4049400000000001,
+            "R_cap@20": 0.47411749999999997,
+            "R_cap@50": 0.5656558333333334,
+            "R_cap@100": 0.6390225,
+            "R_cap@200": 0.7053058333333334,
+            "R_cap@1000": 0.8424908333333333
+        },
+        "hole": {
+            "Hole@1": 0.706795,
+            "Hole@3": 0.8030316666666667,
+            "Hole@5": 0.8332233333333333,
+            "Hole@10": 0.8648841666666668,
+            "Hole@20": 0.8864941666666667,
+            "Hole@50": 0.9045466666666666,
+            "Hole@100": 0.9131933333333335,
+            "Hole@200": 0.9195391666666667,
+            "Hole@1000": 0.9277475000000001
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.24846,0.23935,0.25446,0.28062,0.3056,0.32979,0.34963,0.36675,0.39175
+map,0.12932,0.18415,0.2016,0.21602,0.22481,0.22978,0.23215,0.23351,0.23443
+precision,0.24846,0.16049,0.12191,0.08071,0.05031,0.02531,0.01525,0.00889,0.00225
+recall,0.12932,0.22337,0.27665,0.35412,0.43214,0.52487,0.61765,0.71066,0.8749
+mrr,0.24846,0.3053,0.31957,0.33238,0.3379,0.34081,0.34213,0.34257,0.34278
+recall_cap,0.24846,0.24691,0.28418,0.35454,0.43214,0.52487,0.61765,0.71066,0.8749
+hole,0.69599,0.78447,0.82068,0.86034,0.8892,0.91321,0.92128,0.92703,0.93571

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.24846,
+            "NDCG@3": 0.23935,
+            "NDCG@5": 0.25446,
+            "NDCG@10": 0.28062,
+            "NDCG@20": 0.3056,
+            "NDCG@50": 0.32979,
+            "NDCG@100": 0.34963,
+            "NDCG@200": 0.36675,
+            "NDCG@1000": 0.39175
+        },
+        "map": {
+            "MAP@1": 0.12932,
+            "MAP@3": 0.18415,
+            "MAP@5": 0.2016,
+            "MAP@10": 0.21602,
+            "MAP@20": 0.22481,
+            "MAP@50": 0.22978,
+            "MAP@100": 0.23215,
+            "MAP@200": 0.23351,
+            "MAP@1000": 0.23443
+        },
+        "precision": {
+            "P@1": 0.24846,
+            "P@3": 0.16049,
+            "P@5": 0.12191,
+            "P@10": 0.08071,
+            "P@20": 0.05031,
+            "P@50": 0.02531,
+            "P@100": 0.01525,
+            "P@200": 0.00889,
+            "P@1000": 0.00225
+        },
+        "recall": {
+            "Recall@1": 0.12932,
+            "Recall@3": 0.22337,
+            "Recall@5": 0.27665,
+            "Recall@10": 0.35412,
+            "Recall@20": 0.43214,
+            "Recall@50": 0.52487,
+            "Recall@100": 0.61765,
+            "Recall@200": 0.71066,
+            "Recall@1000": 0.8749
+        },
+        "mrr": {
+            "MRR@1": 0.24846,
+            "MRR@3": 0.3053,
+            "MRR@5": 0.31957,
+            "MRR@10": 0.33238,
+            "MRR@20": 0.3379,
+            "MRR@50": 0.34081,
+            "MRR@100": 0.34213,
+            "MRR@200": 0.34257,
+            "MRR@1000": 0.34278
+        },
+        "recall_cap": {
+            "R_cap@1": 0.24846,
+            "R_cap@3": 0.24691,
+            "R_cap@5": 0.28418,
+            "R_cap@10": 0.35454,
+            "R_cap@20": 0.43214,
+            "R_cap@50": 0.52487,
+            "R_cap@100": 0.61765,
+            "R_cap@200": 0.71066,
+            "R_cap@1000": 0.8749
+        },
+        "hole": {
+            "Hole@1": 0.69599,
+            "Hole@3": 0.78447,
+            "Hole@5": 0.82068,
+            "Hole@10": 0.86034,
+            "Hole@20": 0.8892,
+            "Hole@50": 0.91321,
+            "Hole@100": 0.92128,
+            "Hole@200": 0.92703,
+            "Hole@1000": 0.93571
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.4226,0.39679,0.36804,0.34134,0.32087,0.30701,0.31615,0.33327,0.40435
+map,0.05571,0.09379,0.1076,0.1265,0.14033,0.15186,0.15879,0.16411,0.17343
+precision,0.43653,0.37771,0.31765,0.25418,0.18839,0.11783,0.08053,0.05351,0.02111
+recall,0.05571,0.10869,0.13097,0.16925,0.20788,0.26467,0.31962,0.39443,0.64151
+mrr,0.43653,0.52167,0.5308,0.53731,0.54116,0.54308,0.54363,0.54396,0.54412
+recall_cap,0.43653,0.39783,0.35619,0.32178,0.29557,0.29771,0.33502,0.39953,0.64151
+hole,0.0805,0.07843,0.08669,0.09938,0.10836,0.11944,0.12644,0.13033,0.13478

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.4226,
+            "NDCG@3": 0.39679,
+            "NDCG@5": 0.36804,
+            "NDCG@10": 0.34134,
+            "NDCG@20": 0.32087,
+            "NDCG@50": 0.30701,
+            "NDCG@100": 0.31615,
+            "NDCG@200": 0.33327,
+            "NDCG@1000": 0.40435
+        },
+        "map": {
+            "MAP@1": 0.05571,
+            "MAP@3": 0.09379,
+            "MAP@5": 0.1076,
+            "MAP@10": 0.1265,
+            "MAP@20": 0.14033,
+            "MAP@50": 0.15186,
+            "MAP@100": 0.15879,
+            "MAP@200": 0.16411,
+            "MAP@1000": 0.17343
+        },
+        "precision": {
+            "P@1": 0.43653,
+            "P@3": 0.37771,
+            "P@5": 0.31765,
+            "P@10": 0.25418,
+            "P@20": 0.18839,
+            "P@50": 0.11783,
+            "P@100": 0.08053,
+            "P@200": 0.05351,
+            "P@1000": 0.02111
+        },
+        "recall": {
+            "Recall@1": 0.05571,
+            "Recall@3": 0.10869,
+            "Recall@5": 0.13097,
+            "Recall@10": 0.16925,
+            "Recall@20": 0.20788,
+            "Recall@50": 0.26467,
+            "Recall@100": 0.31962,
+            "Recall@200": 0.39443,
+            "Recall@1000": 0.64151
+        },
+        "mrr": {
+            "MRR@1": 0.43653,
+            "MRR@3": 0.52167,
+            "MRR@5": 0.5308,
+            "MRR@10": 0.53731,
+            "MRR@20": 0.54116,
+            "MRR@50": 0.54308,
+            "MRR@100": 0.54363,
+            "MRR@200": 0.54396,
+            "MRR@1000": 0.54412
+        },
+        "recall_cap": {
+            "R_cap@1": 0.43653,
+            "R_cap@3": 0.39783,
+            "R_cap@5": 0.35619,
+            "R_cap@10": 0.32178,
+            "R_cap@20": 0.29557,
+            "R_cap@50": 0.29771,
+            "R_cap@100": 0.33502,
+            "R_cap@200": 0.39953,
+            "R_cap@1000": 0.64151
+        },
+        "hole": {
+            "Hole@1": 0.0805,
+            "Hole@3": 0.07843,
+            "Hole@5": 0.08669,
+            "Hole@10": 0.09938,
+            "Hole@20": 0.10836,
+            "Hole@50": 0.11944,
+            "Hole@100": 0.12644,
+            "Hole@200": 0.13033,
+            "Hole@1000": 0.13478
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1263,0.1948,0.23184,0.27574,0.31019,0.33737,0.34981,0.35952,0.36958
+map,0.11097,0.17079,0.19176,0.21075,0.22104,0.22576,0.22697,0.22753,0.2278
+precision,0.1263,0.09511,0.07746,0.05458,0.0355,0.01739,0.00961,0.00521,0.00115
+recall,0.11097,0.24488,0.33196,0.46125,0.59007,0.72019,0.7927,0.85808,0.93996
+mrr,0.12659,0.19028,0.21087,0.22842,0.23687,0.24093,0.24189,0.24234,0.24254
+recall_cap,0.12659,0.24493,0.33196,0.46125,0.59007,0.72019,0.7927,0.85808,0.93996
+hole,0.87283,0.90324,0.92097,0.94371,0.96246,0.98022,0.98807,0.9925,0.99676

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1263,
+            "NDCG@3": 0.1948,
+            "NDCG@5": 0.23184,
+            "NDCG@10": 0.27574,
+            "NDCG@20": 0.31019,
+            "NDCG@50": 0.33737,
+            "NDCG@100": 0.34981,
+            "NDCG@200": 0.35952,
+            "NDCG@1000": 0.36958
+        },
+        "map": {
+            "MAP@1": 0.11097,
+            "MAP@3": 0.17079,
+            "MAP@5": 0.19176,
+            "MAP@10": 0.21075,
+            "MAP@20": 0.22104,
+            "MAP@50": 0.22576,
+            "MAP@100": 0.22697,
+            "MAP@200": 0.22753,
+            "MAP@1000": 0.2278
+        },
+        "precision": {
+            "P@1": 0.1263,
+            "P@3": 0.09511,
+            "P@5": 0.07746,
+            "P@10": 0.05458,
+            "P@20": 0.0355,
+            "P@50": 0.01739,
+            "P@100": 0.00961,
+            "P@200": 0.00521,
+            "P@1000": 0.00115
+        },
+        "recall": {
+            "Recall@1": 0.11097,
+            "Recall@3": 0.24488,
+            "Recall@5": 0.33196,
+            "Recall@10": 0.46125,
+            "Recall@20": 0.59007,
+            "Recall@50": 0.72019,
+            "Recall@100": 0.7927,
+            "Recall@200": 0.85808,
+            "Recall@1000": 0.93996
+        },
+        "mrr": {
+            "MRR@1": 0.12659,
+            "MRR@3": 0.19028,
+            "MRR@5": 0.21087,
+            "MRR@10": 0.22842,
+            "MRR@20": 0.23687,
+            "MRR@50": 0.24093,
+            "MRR@100": 0.24189,
+            "MRR@200": 0.24234,
+            "MRR@1000": 0.24254
+        },
+        "recall_cap": {
+            "R_cap@1": 0.12659,
+            "R_cap@3": 0.24493,
+            "R_cap@5": 0.33196,
+            "R_cap@10": 0.46125,
+            "R_cap@20": 0.59007,
+            "R_cap@50": 0.72019,
+            "R_cap@100": 0.7927,
+            "R_cap@200": 0.85808,
+            "R_cap@1000": 0.93996
+        },
+        "hole": {
+            "Hole@1": 0.87283,
+            "Hole@3": 0.90324,
+            "Hole@5": 0.92097,
+            "Hole@10": 0.94371,
+            "Hole@20": 0.96246,
+            "Hole@50": 0.98022,
+            "Hole@100": 0.98807,
+            "Hole@200": 0.9925,
+            "Hole@1000": 0.99676
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6897,0.74408,0.76601,0.78549,0.79663,0.80497,0.80825,0.81002,0.81195
+map,0.60075,0.70413,0.7239,0.73673,0.74194,0.74438,0.74492,0.74512,0.74524
+precision,0.6897,0.326,0.21826,0.12171,0.06578,0.02827,0.01462,0.00748,0.00155
+recall,0.60075,0.77438,0.83167,0.88855,0.92615,0.95942,0.9752,0.98443,0.99636
+mrr,0.6894,0.75502,0.7656,0.77143,0.77332,0.77411,0.77426,0.7743,0.77432
+recall_cap,0.6894,0.78705,0.83681,0.88983,0.92641,0.95944,0.9752,0.98443,0.99636
+hole,0.2918,0.64107,0.74518,0.83811,0.89266,0.93065,0.94517,0.95374,0.96222

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6897,
+            "NDCG@3": 0.74408,
+            "NDCG@5": 0.76601,
+            "NDCG@10": 0.78549,
+            "NDCG@20": 0.79663,
+            "NDCG@50": 0.80497,
+            "NDCG@100": 0.80825,
+            "NDCG@200": 0.81002,
+            "NDCG@1000": 0.81195
+        },
+        "map": {
+            "MAP@1": 0.60075,
+            "MAP@3": 0.70413,
+            "MAP@5": 0.7239,
+            "MAP@10": 0.73673,
+            "MAP@20": 0.74194,
+            "MAP@50": 0.74438,
+            "MAP@100": 0.74492,
+            "MAP@200": 0.74512,
+            "MAP@1000": 0.74524
+        },
+        "precision": {
+            "P@1": 0.6897,
+            "P@3": 0.326,
+            "P@5": 0.21826,
+            "P@10": 0.12171,
+            "P@20": 0.06578,
+            "P@50": 0.02827,
+            "P@100": 0.01462,
+            "P@200": 0.00748,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.60075,
+            "Recall@3": 0.77438,
+            "Recall@5": 0.83167,
+            "Recall@10": 0.88855,
+            "Recall@20": 0.92615,
+            "Recall@50": 0.95942,
+            "Recall@100": 0.9752,
+            "Recall@200": 0.98443,
+            "Recall@1000": 0.99636
+        },
+        "mrr": {
+            "MRR@1": 0.6894,
+            "MRR@3": 0.75502,
+            "MRR@5": 0.7656,
+            "MRR@10": 0.77143,
+            "MRR@20": 0.77332,
+            "MRR@50": 0.77411,
+            "MRR@100": 0.77426,
+            "MRR@200": 0.7743,
+            "MRR@1000": 0.77432
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6894,
+            "R_cap@3": 0.78705,
+            "R_cap@5": 0.83681,
+            "R_cap@10": 0.88983,
+            "R_cap@20": 0.92641,
+            "R_cap@50": 0.95944,
+            "R_cap@100": 0.9752,
+            "R_cap@200": 0.98443,
+            "R_cap@1000": 0.99636
+        },
+        "hole": {
+            "Hole@1": 0.2918,
+            "Hole@3": 0.64107,
+            "Hole@5": 0.74518,
+            "Hole@10": 0.83811,
+            "Hole@20": 0.89266,
+            "Hole@50": 0.93065,
+            "Hole@100": 0.94517,
+            "Hole@200": 0.95374,
+            "Hole@1000": 0.96222
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.192,0.14881,0.13068,0.15962,0.18417,0.21593,0.23574,0.25272,0.28886
+map,0.03908,0.06637,0.07823,0.09173,0.10044,0.10749,0.11018,0.11157,0.11296
+precision,0.192,0.13667,0.1134,0.083,0.0556,0.0316,0.01937,0.01145,0.00322
+recall,0.03908,0.08328,0.11523,0.16823,0.22532,0.32033,0.39283,0.4648,0.65352
+mrr,0.192,0.24867,0.26757,0.2826,0.28888,0.29292,0.29393,0.2944,0.29467
+recall_cap,0.192,0.13667,0.11523,0.16823,0.22532,0.32033,0.39283,0.4648,0.65352
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0