--- pipeline_tag: sentence-similarity tags: - sentence-transformers - feature-extraction - sentence-similarity - mteb language: en license: apache-2.0 datasets: - s2orc - flax-sentence-embeddings/stackexchange_xml - MS Marco - gooaq - yahoo_answers_topics - code_search_net - search_qa - eli5 - snli - multi_nli - wikihow - natural_questions - trivia_qa - embedding-data/sentence-compression - embedding-data/flickr30k-captions - embedding-data/altlex - embedding-data/simple-wiki - embedding-data/QQP - embedding-data/SPECTER - embedding-data/PAQ_pairs - embedding-data/WikiAnswers model-index: - name: all-mpnet-base-v2 results: - task: type: Classification dataset: type: mteb/amazon_counterfactual name: MTEB AmazonCounterfactualClassification (en) config: en split: test revision: 2d8a100785abf0ae21420d2a55b0c56e3e1ea996 metrics: - type: accuracy value: 65.26865671641791 - type: ap value: 28.47453420428918 - type: f1 value: 59.3470101009448 - task: type: Classification dataset: type: mteb/amazon_polarity name: MTEB AmazonPolarityClassification config: default split: test revision: 80714f8dcf8cefc218ef4f8c5a966dd83f75a0e1 metrics: - type: accuracy value: 67.13145 - type: ap value: 61.842060778903786 - type: f1 value: 66.79987305640383 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (en) config: en split: test revision: c379a6705fec24a2493fa68e011692605f44e119 metrics: - type: accuracy value: 31.920000000000005 - type: f1 value: 31.2465193896153 - task: type: Retrieval dataset: type: arguana name: MTEB ArguAna config: default split: test revision: 5b3e3697907184a9b77a3c99ee9ea1a9cbb1e4e3 metrics: - type: map_at_1 value: 23.186 - type: map_at_10 value: 37.692 - type: map_at_100 value: 38.986 - type: map_at_1000 value: 38.991 - type: map_at_3 value: 32.622 - type: map_at_5 value: 35.004999999999995 - type: ndcg_at_1 value: 23.186 - type: ndcg_at_10 value: 46.521 - type: ndcg_at_100 value: 51.954 - type: ndcg_at_1000 value: 52.087 - type: ndcg_at_3 value: 35.849 - type: ndcg_at_5 value: 40.12 - type: precision_at_1 value: 23.186 - type: precision_at_10 value: 7.510999999999999 - type: precision_at_100 value: 0.9860000000000001 - type: precision_at_1000 value: 0.1 - type: precision_at_3 value: 15.078 - type: precision_at_5 value: 11.110000000000001 - type: recall_at_1 value: 23.186 - type: recall_at_10 value: 75.107 - type: recall_at_100 value: 98.649 - type: recall_at_1000 value: 99.644 - type: recall_at_3 value: 45.235 - type: recall_at_5 value: 55.547999999999995 - task: type: Clustering dataset: type: mteb/arxiv-clustering-p2p name: MTEB ArxivClusteringP2P config: default split: test revision: 0bbdb47bcbe3a90093699aefeed338a0f28a7ee8 metrics: - type: v_measure value: 48.37886340922374 - task: type: Clustering dataset: type: mteb/arxiv-clustering-s2s name: MTEB ArxivClusteringS2S config: default split: test revision: b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3 metrics: - type: v_measure value: 39.72488615315985 - task: type: Reranking dataset: type: mteb/askubuntudupquestions-reranking name: MTEB AskUbuntuDupQuestions config: default split: test revision: 4d853f94cd57d85ec13805aeeac3ae3e5eb4c49c metrics: - type: map value: 65.85199009344481 - type: mrr value: 78.47700391329201 - task: type: STS dataset: type: mteb/biosses-sts name: MTEB BIOSSES config: default split: test revision: 9ee918f184421b6bd48b78f6c714d86546106103 metrics: - type: cos_sim_pearson value: 84.47737119217858 - type: cos_sim_spearman value: 80.43195317854409 - type: euclidean_pearson value: 82.20496332547978 - type: euclidean_spearman value: 80.43195317854409 - type: manhattan_pearson value: 81.4836610720397 - type: manhattan_spearman value: 79.65904400101908 - task: type: Classification dataset: type: mteb/banking77 name: MTEB Banking77Classification config: default split: test revision: 44fa15921b4c889113cc5df03dd4901b49161ab7 metrics: - type: accuracy value: 81.8603896103896 - type: f1 value: 81.28027245637479 - task: type: Clustering dataset: type: mteb/biorxiv-clustering-p2p name: MTEB BiorxivClusteringP2P config: default split: test revision: 11d0121201d1f1f280e8cc8f3d98fb9c4d9f9c55 metrics: - type: v_measure value: 39.616605133625185 - task: type: Clustering dataset: type: mteb/biorxiv-clustering-s2s name: MTEB BiorxivClusteringS2S config: default split: test revision: c0fab014e1bcb8d3a5e31b2088972a1e01547dc1 metrics: - type: v_measure value: 35.02442407186902 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackAndroidRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 36.036 - type: map_at_10 value: 49.302 - type: map_at_100 value: 50.956 - type: map_at_1000 value: 51.080000000000005 - type: map_at_3 value: 45.237 - type: map_at_5 value: 47.353 - type: ndcg_at_1 value: 45.207 - type: ndcg_at_10 value: 56.485 - type: ndcg_at_100 value: 61.413 - type: ndcg_at_1000 value: 62.870000000000005 - type: ndcg_at_3 value: 51.346000000000004 - type: ndcg_at_5 value: 53.486 - type: precision_at_1 value: 45.207 - type: precision_at_10 value: 11.144 - type: precision_at_100 value: 1.735 - type: precision_at_1000 value: 0.22100000000000003 - type: precision_at_3 value: 24.94 - type: precision_at_5 value: 17.997 - type: recall_at_1 value: 36.036 - type: recall_at_10 value: 69.191 - type: recall_at_100 value: 89.423 - type: recall_at_1000 value: 98.425 - type: recall_at_3 value: 53.849999999999994 - type: recall_at_5 value: 60.107 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackEnglishRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 32.92 - type: map_at_10 value: 45.739999999999995 - type: map_at_100 value: 47.309 - type: map_at_1000 value: 47.443000000000005 - type: map_at_3 value: 42.154 - type: map_at_5 value: 44.207 - type: ndcg_at_1 value: 42.229 - type: ndcg_at_10 value: 52.288999999999994 - type: ndcg_at_100 value: 57.04900000000001 - type: ndcg_at_1000 value: 58.788 - type: ndcg_at_3 value: 47.531 - type: ndcg_at_5 value: 49.861 - type: precision_at_1 value: 42.229 - type: precision_at_10 value: 10.299 - type: precision_at_100 value: 1.68 - type: precision_at_1000 value: 0.213 - type: precision_at_3 value: 23.673 - type: precision_at_5 value: 17.006 - type: recall_at_1 value: 32.92 - type: recall_at_10 value: 63.865 - type: recall_at_100 value: 84.06700000000001 - type: recall_at_1000 value: 94.536 - type: recall_at_3 value: 49.643 - type: recall_at_5 value: 56.119 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackGamingRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 40.695 - type: map_at_10 value: 53.787 - type: map_at_100 value: 54.778000000000006 - type: map_at_1000 value: 54.827000000000005 - type: map_at_3 value: 50.151999999999994 - type: map_at_5 value: 52.207 - type: ndcg_at_1 value: 46.52 - type: ndcg_at_10 value: 60.026 - type: ndcg_at_100 value: 63.81099999999999 - type: ndcg_at_1000 value: 64.741 - type: ndcg_at_3 value: 53.83 - type: ndcg_at_5 value: 56.928999999999995 - type: precision_at_1 value: 46.52 - type: precision_at_10 value: 9.754999999999999 - type: precision_at_100 value: 1.2670000000000001 - type: precision_at_1000 value: 0.13799999999999998 - type: precision_at_3 value: 24.096 - type: precision_at_5 value: 16.689999999999998 - type: recall_at_1 value: 40.695 - type: recall_at_10 value: 75.181 - type: recall_at_100 value: 91.479 - type: recall_at_1000 value: 98.06899999999999 - type: recall_at_3 value: 58.707 - type: recall_at_5 value: 66.295 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackGisRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 29.024 - type: map_at_10 value: 38.438 - type: map_at_100 value: 39.576 - type: map_at_1000 value: 39.645 - type: map_at_3 value: 34.827999999999996 - type: map_at_5 value: 36.947 - type: ndcg_at_1 value: 31.299 - type: ndcg_at_10 value: 44.268 - type: ndcg_at_100 value: 49.507 - type: ndcg_at_1000 value: 51.205999999999996 - type: ndcg_at_3 value: 37.248999999999995 - type: ndcg_at_5 value: 40.861999999999995 - type: precision_at_1 value: 31.299 - type: precision_at_10 value: 6.949 - type: precision_at_100 value: 1.012 - type: precision_at_1000 value: 0.11900000000000001 - type: precision_at_3 value: 15.518 - type: precision_at_5 value: 11.366999999999999 - type: recall_at_1 value: 29.024 - type: recall_at_10 value: 60.404 - type: recall_at_100 value: 83.729 - type: recall_at_1000 value: 96.439 - type: recall_at_3 value: 41.65 - type: recall_at_5 value: 50.263999999999996 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackMathematicaRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 17.774 - type: map_at_10 value: 28.099 - type: map_at_100 value: 29.603 - type: map_at_1000 value: 29.709999999999997 - type: map_at_3 value: 25.036 - type: map_at_5 value: 26.657999999999998 - type: ndcg_at_1 value: 22.139 - type: ndcg_at_10 value: 34.205999999999996 - type: ndcg_at_100 value: 40.844 - type: ndcg_at_1000 value: 43.144 - type: ndcg_at_3 value: 28.732999999999997 - type: ndcg_at_5 value: 31.252000000000002 - type: precision_at_1 value: 22.139 - type: precision_at_10 value: 6.567 - type: precision_at_100 value: 1.147 - type: precision_at_1000 value: 0.146 - type: precision_at_3 value: 14.386 - type: precision_at_5 value: 10.423 - type: recall_at_1 value: 17.774 - type: recall_at_10 value: 48.32 - type: recall_at_100 value: 76.373 - type: recall_at_1000 value: 92.559 - type: recall_at_3 value: 33.478 - type: recall_at_5 value: 39.872 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackPhysicsRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 31.885 - type: map_at_10 value: 44.289 - type: map_at_100 value: 45.757999999999996 - type: map_at_1000 value: 45.86 - type: map_at_3 value: 40.459 - type: map_at_5 value: 42.662 - type: ndcg_at_1 value: 39.75 - type: ndcg_at_10 value: 50.975 - type: ndcg_at_100 value: 56.528999999999996 - type: ndcg_at_1000 value: 58.06099999999999 - type: ndcg_at_3 value: 45.327 - type: ndcg_at_5 value: 48.041 - type: precision_at_1 value: 39.75 - type: precision_at_10 value: 9.557 - type: precision_at_100 value: 1.469 - type: precision_at_1000 value: 0.17700000000000002 - type: precision_at_3 value: 22.073 - type: precision_at_5 value: 15.765 - type: recall_at_1 value: 31.885 - type: recall_at_10 value: 64.649 - type: recall_at_100 value: 87.702 - type: recall_at_1000 value: 97.327 - type: recall_at_3 value: 48.61 - type: recall_at_5 value: 55.882 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackProgrammersRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 26.454 - type: map_at_10 value: 37.756 - type: map_at_100 value: 39.225 - type: map_at_1000 value: 39.332 - type: map_at_3 value: 34.115 - type: map_at_5 value: 35.942 - type: ndcg_at_1 value: 32.42 - type: ndcg_at_10 value: 44.165 - type: ndcg_at_100 value: 50.202000000000005 - type: ndcg_at_1000 value: 52.188 - type: ndcg_at_3 value: 38.381 - type: ndcg_at_5 value: 40.849000000000004 - type: precision_at_1 value: 32.42 - type: precision_at_10 value: 8.482000000000001 - type: precision_at_100 value: 1.332 - type: precision_at_1000 value: 0.169 - type: precision_at_3 value: 18.683 - type: precision_at_5 value: 13.539000000000001 - type: recall_at_1 value: 26.454 - type: recall_at_10 value: 57.937000000000005 - type: recall_at_100 value: 83.76 - type: recall_at_1000 value: 96.82600000000001 - type: recall_at_3 value: 41.842 - type: recall_at_5 value: 48.285 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 27.743666666666666 - type: map_at_10 value: 38.75416666666667 - type: map_at_100 value: 40.133250000000004 - type: map_at_1000 value: 40.24616666666667 - type: map_at_3 value: 35.267250000000004 - type: map_at_5 value: 37.132749999999994 - type: ndcg_at_1 value: 33.14358333333333 - type: ndcg_at_10 value: 44.95916666666667 - type: ndcg_at_100 value: 50.46375 - type: ndcg_at_1000 value: 52.35508333333334 - type: ndcg_at_3 value: 39.17883333333334 - type: ndcg_at_5 value: 41.79724999999999 - type: precision_at_1 value: 33.14358333333333 - type: precision_at_10 value: 8.201083333333333 - type: precision_at_100 value: 1.3085 - type: precision_at_1000 value: 0.1665833333333333 - type: precision_at_3 value: 18.405583333333333 - type: precision_at_5 value: 13.233166666666666 - type: recall_at_1 value: 27.743666666666666 - type: recall_at_10 value: 58.91866666666667 - type: recall_at_100 value: 82.76216666666666 - type: recall_at_1000 value: 95.56883333333333 - type: recall_at_3 value: 42.86925 - type: recall_at_5 value: 49.553333333333335 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackStatsRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 25.244 - type: map_at_10 value: 33.464 - type: map_at_100 value: 34.633 - type: map_at_1000 value: 34.721999999999994 - type: map_at_3 value: 30.784 - type: map_at_5 value: 32.183 - type: ndcg_at_1 value: 28.681 - type: ndcg_at_10 value: 38.149 - type: ndcg_at_100 value: 43.856 - type: ndcg_at_1000 value: 46.026 - type: ndcg_at_3 value: 33.318 - type: ndcg_at_5 value: 35.454 - type: precision_at_1 value: 28.681 - type: precision_at_10 value: 6.304 - type: precision_at_100 value: 0.992 - type: precision_at_1000 value: 0.125 - type: precision_at_3 value: 14.673 - type: precision_at_5 value: 10.245 - type: recall_at_1 value: 25.244 - type: recall_at_10 value: 49.711 - type: recall_at_100 value: 75.928 - type: recall_at_1000 value: 91.79899999999999 - type: recall_at_3 value: 36.325 - type: recall_at_5 value: 41.752 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackTexRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 18.857 - type: map_at_10 value: 27.794 - type: map_at_100 value: 29.186 - type: map_at_1000 value: 29.323 - type: map_at_3 value: 24.779 - type: map_at_5 value: 26.459 - type: ndcg_at_1 value: 23.227999999999998 - type: ndcg_at_10 value: 33.353 - type: ndcg_at_100 value: 39.598 - type: ndcg_at_1000 value: 42.268 - type: ndcg_at_3 value: 28.054000000000002 - type: ndcg_at_5 value: 30.566 - type: precision_at_1 value: 23.227999999999998 - type: precision_at_10 value: 6.397 - type: precision_at_100 value: 1.129 - type: precision_at_1000 value: 0.155 - type: precision_at_3 value: 13.616 - type: precision_at_5 value: 10.116999999999999 - type: recall_at_1 value: 18.857 - type: recall_at_10 value: 45.797 - type: recall_at_100 value: 73.615 - type: recall_at_1000 value: 91.959 - type: recall_at_3 value: 31.129 - type: recall_at_5 value: 37.565 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackUnixRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 27.486 - type: map_at_10 value: 39.164 - type: map_at_100 value: 40.543 - type: map_at_1000 value: 40.636 - type: map_at_3 value: 35.52 - type: map_at_5 value: 37.355 - type: ndcg_at_1 value: 32.275999999999996 - type: ndcg_at_10 value: 45.414 - type: ndcg_at_100 value: 51.254 - type: ndcg_at_1000 value: 53.044000000000004 - type: ndcg_at_3 value: 39.324999999999996 - type: ndcg_at_5 value: 41.835 - type: precision_at_1 value: 32.275999999999996 - type: precision_at_10 value: 8.144 - type: precision_at_100 value: 1.237 - type: precision_at_1000 value: 0.15 - type: precision_at_3 value: 18.501 - type: precision_at_5 value: 13.134 - type: recall_at_1 value: 27.486 - type: recall_at_10 value: 60.449 - type: recall_at_100 value: 85.176 - type: recall_at_1000 value: 97.087 - type: recall_at_3 value: 43.59 - type: recall_at_5 value: 50.08899999999999 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackWebmastersRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 26.207 - type: map_at_10 value: 37.255 - type: map_at_100 value: 39.043 - type: map_at_1000 value: 39.273 - type: map_at_3 value: 33.487 - type: map_at_5 value: 35.441 - type: ndcg_at_1 value: 31.423000000000002 - type: ndcg_at_10 value: 44.235 - type: ndcg_at_100 value: 50.49 - type: ndcg_at_1000 value: 52.283 - type: ndcg_at_3 value: 37.602000000000004 - type: ndcg_at_5 value: 40.518 - type: precision_at_1 value: 31.423000000000002 - type: precision_at_10 value: 8.715 - type: precision_at_100 value: 1.7590000000000001 - type: precision_at_1000 value: 0.257 - type: precision_at_3 value: 17.523 - type: precision_at_5 value: 13.161999999999999 - type: recall_at_1 value: 26.207 - type: recall_at_10 value: 59.17099999999999 - type: recall_at_100 value: 86.166 - type: recall_at_1000 value: 96.54700000000001 - type: recall_at_3 value: 41.18 - type: recall_at_5 value: 48.083999999999996 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackWordpressRetrieval config: default split: test revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db metrics: - type: map_at_1 value: 20.342 - type: map_at_10 value: 29.962 - type: map_at_100 value: 30.989 - type: map_at_1000 value: 31.102999999999998 - type: map_at_3 value: 26.656000000000002 - type: map_at_5 value: 28.179 - type: ndcg_at_1 value: 22.551 - type: ndcg_at_10 value: 35.945 - type: ndcg_at_100 value: 41.012 - type: ndcg_at_1000 value: 43.641999999999996 - type: ndcg_at_3 value: 29.45 - type: ndcg_at_5 value: 31.913999999999998 - type: precision_at_1 value: 22.551 - type: precision_at_10 value: 6.1 - type: precision_at_100 value: 0.943 - type: precision_at_1000 value: 0.129 - type: precision_at_3 value: 13.184999999999999 - type: precision_at_5 value: 9.353 - type: recall_at_1 value: 20.342 - type: recall_at_10 value: 52.349000000000004 - type: recall_at_100 value: 75.728 - type: recall_at_1000 value: 95.253 - type: recall_at_3 value: 34.427 - type: recall_at_5 value: 40.326 - task: type: Retrieval dataset: type: climate-fever name: MTEB ClimateFEVER config: default split: test revision: 392b78eb68c07badcd7c2cd8f39af108375dfcce metrics: - type: map_at_1 value: 7.71 - type: map_at_10 value: 14.81 - type: map_at_100 value: 16.536 - type: map_at_1000 value: 16.744999999999997 - type: map_at_3 value: 12.109 - type: map_at_5 value: 13.613 - type: ndcg_at_1 value: 18.046 - type: ndcg_at_10 value: 21.971 - type: ndcg_at_100 value: 29.468 - type: ndcg_at_1000 value: 33.428999999999995 - type: ndcg_at_3 value: 17.227999999999998 - type: ndcg_at_5 value: 19.189999999999998 - type: precision_at_1 value: 18.046 - type: precision_at_10 value: 7.192 - type: precision_at_100 value: 1.51 - type: precision_at_1000 value: 0.22499999999999998 - type: precision_at_3 value: 13.312 - type: precision_at_5 value: 10.775 - type: recall_at_1 value: 7.71 - type: recall_at_10 value: 27.908 - type: recall_at_100 value: 54.452 - type: recall_at_1000 value: 76.764 - type: recall_at_3 value: 16.64 - type: recall_at_5 value: 21.631 - task: type: Retrieval dataset: type: dbpedia-entity name: MTEB DBPedia config: default split: test revision: f097057d03ed98220bc7309ddb10b71a54d667d6 metrics: - type: map_at_1 value: 6.8180000000000005 - type: map_at_10 value: 14.591000000000001 - type: map_at_100 value: 19.855999999999998 - type: map_at_1000 value: 21.178 - type: map_at_3 value: 10.345 - type: map_at_5 value: 12.367 - type: ndcg_at_1 value: 39.25 - type: ndcg_at_10 value: 32.088 - type: ndcg_at_100 value: 36.019 - type: ndcg_at_1000 value: 43.649 - type: ndcg_at_3 value: 35.132999999999996 - type: ndcg_at_5 value: 33.777 - type: precision_at_1 value: 49.5 - type: precision_at_10 value: 25.624999999999996 - type: precision_at_100 value: 8.043 - type: precision_at_1000 value: 1.7409999999999999 - type: precision_at_3 value: 38.417 - type: precision_at_5 value: 33.2 - type: recall_at_1 value: 6.8180000000000005 - type: recall_at_10 value: 20.399 - type: recall_at_100 value: 42.8 - type: recall_at_1000 value: 68.081 - type: recall_at_3 value: 11.928999999999998 - type: recall_at_5 value: 15.348999999999998 - task: type: Classification dataset: type: mteb/emotion name: MTEB EmotionClassification config: default split: test revision: 829147f8f75a25f005913200eb5ed41fae320aa1 metrics: - type: accuracy value: 39.725 - type: f1 value: 35.19385687310605 - task: type: Retrieval dataset: type: fever name: MTEB FEVER config: default split: test revision: 1429cf27e393599b8b359b9b72c666f96b2525f9 metrics: - type: map_at_1 value: 31.901000000000003 - type: map_at_10 value: 44.156 - type: map_at_100 value: 44.901 - type: map_at_1000 value: 44.939 - type: map_at_3 value: 41.008 - type: map_at_5 value: 42.969 - type: ndcg_at_1 value: 34.263 - type: ndcg_at_10 value: 50.863 - type: ndcg_at_100 value: 54.336 - type: ndcg_at_1000 value: 55.297 - type: ndcg_at_3 value: 44.644 - type: ndcg_at_5 value: 48.075 - type: precision_at_1 value: 34.263 - type: precision_at_10 value: 7.542999999999999 - type: precision_at_100 value: 0.9400000000000001 - type: precision_at_1000 value: 0.104 - type: precision_at_3 value: 18.912000000000003 - type: precision_at_5 value: 13.177 - type: recall_at_1 value: 31.901000000000003 - type: recall_at_10 value: 68.872 - type: recall_at_100 value: 84.468 - type: recall_at_1000 value: 91.694 - type: recall_at_3 value: 52.272 - type: recall_at_5 value: 60.504999999999995 - task: type: Retrieval dataset: type: fiqa name: MTEB FiQA2018 config: default split: test revision: 41b686a7f28c59bcaaa5791efd47c67c8ebe28be metrics: - type: map_at_1 value: 24.4 - type: map_at_10 value: 41.117 - type: map_at_100 value: 43.167 - type: map_at_1000 value: 43.323 - type: map_at_3 value: 35.744 - type: map_at_5 value: 38.708 - type: ndcg_at_1 value: 49.074 - type: ndcg_at_10 value: 49.963 - type: ndcg_at_100 value: 56.564 - type: ndcg_at_1000 value: 58.931999999999995 - type: ndcg_at_3 value: 45.489000000000004 - type: ndcg_at_5 value: 47.133 - type: precision_at_1 value: 49.074 - type: precision_at_10 value: 13.889000000000001 - type: precision_at_100 value: 2.091 - type: precision_at_1000 value: 0.251 - type: precision_at_3 value: 30.658 - type: precision_at_5 value: 22.593 - type: recall_at_1 value: 24.4 - type: recall_at_10 value: 58.111999999999995 - type: recall_at_100 value: 81.96900000000001 - type: recall_at_1000 value: 96.187 - type: recall_at_3 value: 41.661 - type: recall_at_5 value: 49.24 - task: type: Retrieval dataset: type: hotpotqa name: MTEB HotpotQA config: default split: test revision: 766870b35a1b9ca65e67a0d1913899973551fc6c metrics: - type: map_at_1 value: 22.262 - type: map_at_10 value: 31.266 - type: map_at_100 value: 32.202 - type: map_at_1000 value: 32.300000000000004 - type: map_at_3 value: 28.874 - type: map_at_5 value: 30.246000000000002 - type: ndcg_at_1 value: 44.524 - type: ndcg_at_10 value: 39.294000000000004 - type: ndcg_at_100 value: 43.296 - type: ndcg_at_1000 value: 45.561 - type: ndcg_at_3 value: 35.013 - type: ndcg_at_5 value: 37.177 - type: precision_at_1 value: 44.524 - type: precision_at_10 value: 8.52 - type: precision_at_100 value: 1.169 - type: precision_at_1000 value: 0.147 - type: precision_at_3 value: 22.003 - type: precision_at_5 value: 14.914 - type: recall_at_1 value: 22.262 - type: recall_at_10 value: 42.6 - type: recall_at_100 value: 58.46 - type: recall_at_1000 value: 73.565 - type: recall_at_3 value: 33.005 - type: recall_at_5 value: 37.286 - task: type: Classification dataset: type: mteb/imdb name: MTEB ImdbClassification config: default split: test revision: 8d743909f834c38949e8323a8a6ce8721ea6c7f4 metrics: - type: accuracy value: 70.7156 - type: ap value: 64.89470531959896 - type: f1 value: 70.53051887683772 - task: type: Retrieval dataset: type: msmarco name: MTEB MSMARCO config: default split: dev revision: e6838a846e2408f22cf5cc337ebc83e0bcf77849 metrics: - type: map_at_1 value: 21.174 - type: map_at_10 value: 33.0 - type: map_at_100 value: 34.178 - type: map_at_1000 value: 34.227000000000004 - type: map_at_3 value: 29.275000000000002 - type: map_at_5 value: 31.341 - type: ndcg_at_1 value: 21.776999999999997 - type: ndcg_at_10 value: 39.745999999999995 - type: ndcg_at_100 value: 45.488 - type: ndcg_at_1000 value: 46.733999999999995 - type: ndcg_at_3 value: 32.086 - type: ndcg_at_5 value: 35.792 - type: precision_at_1 value: 21.776999999999997 - type: precision_at_10 value: 6.324000000000001 - type: precision_at_100 value: 0.922 - type: precision_at_1000 value: 0.10300000000000001 - type: precision_at_3 value: 13.696 - type: precision_at_5 value: 10.100000000000001 - type: recall_at_1 value: 21.174 - type: recall_at_10 value: 60.488 - type: recall_at_100 value: 87.234 - type: recall_at_1000 value: 96.806 - type: recall_at_3 value: 39.582 - type: recall_at_5 value: 48.474000000000004 - task: type: Classification dataset: type: mteb/mtop_domain name: MTEB MTOPDomainClassification (en) config: en split: test revision: a7e2a951126a26fc8c6a69f835f33a346ba259e3 metrics: - type: accuracy value: 92.07934336525308 - type: f1 value: 91.93440027035814 - task: type: Classification dataset: type: mteb/mtop_intent name: MTEB MTOPIntentClassification (en) config: en split: test revision: 6299947a7777084cc2d4b64235bf7190381ce755 metrics: - type: accuracy value: 70.20975832193344 - type: f1 value: 48.571776628850074 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (en) config: en split: test revision: 072a486a144adf7f4479a4a0dddb2152e161e1ea metrics: - type: accuracy value: 69.56624075319435 - type: f1 value: 67.64419185784621 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (en) config: en split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 76.01210490921318 - type: f1 value: 75.1934366365826 - task: type: Clustering dataset: type: mteb/medrxiv-clustering-p2p name: MTEB MedrxivClusteringP2P config: default split: test revision: dcefc037ef84348e49b0d29109e891c01067226b metrics: - type: v_measure value: 35.58002813186373 - task: type: Clustering dataset: type: mteb/medrxiv-clustering-s2s name: MTEB MedrxivClusteringS2S config: default split: test revision: 3cd0e71dfbe09d4de0f9e5ecba43e7ce280959dc metrics: - type: v_measure value: 32.872725562410444 - task: type: Reranking dataset: type: mteb/mind_small name: MTEB MindSmallReranking config: default split: test revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 metrics: - type: map value: 30.965343604861328 - type: mrr value: 31.933710165863594 - task: type: Retrieval dataset: type: nfcorpus name: MTEB NFCorpus config: default split: test revision: 7eb63cc0c1eb59324d709ebed25fcab851fa7610 metrics: - type: map_at_1 value: 4.938 - type: map_at_10 value: 12.034 - type: map_at_100 value: 15.675 - type: map_at_1000 value: 17.18 - type: map_at_3 value: 8.471 - type: map_at_5 value: 10.128 - type: ndcg_at_1 value: 40.402 - type: ndcg_at_10 value: 33.289 - type: ndcg_at_100 value: 31.496000000000002 - type: ndcg_at_1000 value: 40.453 - type: ndcg_at_3 value: 37.841 - type: ndcg_at_5 value: 36.215 - type: precision_at_1 value: 41.796 - type: precision_at_10 value: 25.294 - type: precision_at_100 value: 8.381 - type: precision_at_1000 value: 2.1260000000000003 - type: precision_at_3 value: 36.429 - type: precision_at_5 value: 32.446000000000005 - type: recall_at_1 value: 4.938 - type: recall_at_10 value: 16.637 - type: recall_at_100 value: 33.853 - type: recall_at_1000 value: 66.07 - type: recall_at_3 value: 9.818 - type: recall_at_5 value: 12.544 - task: type: Retrieval dataset: type: nq name: MTEB NQ config: default split: test revision: 6062aefc120bfe8ece5897809fb2e53bfe0d128c metrics: - type: map_at_1 value: 27.124 - type: map_at_10 value: 42.418 - type: map_at_100 value: 43.633 - type: map_at_1000 value: 43.66 - type: map_at_3 value: 37.766 - type: map_at_5 value: 40.482 - type: ndcg_at_1 value: 30.794 - type: ndcg_at_10 value: 50.449999999999996 - type: ndcg_at_100 value: 55.437999999999995 - type: ndcg_at_1000 value: 56.084 - type: ndcg_at_3 value: 41.678 - type: ndcg_at_5 value: 46.257 - type: precision_at_1 value: 30.794 - type: precision_at_10 value: 8.656 - type: precision_at_100 value: 1.141 - type: precision_at_1000 value: 0.12 - type: precision_at_3 value: 19.37 - type: precision_at_5 value: 14.218 - type: recall_at_1 value: 27.124 - type: recall_at_10 value: 72.545 - type: recall_at_100 value: 93.938 - type: recall_at_1000 value: 98.788 - type: recall_at_3 value: 49.802 - type: recall_at_5 value: 60.426 - task: type: Retrieval dataset: type: quora name: MTEB QuoraRetrieval config: default split: test revision: 6205996560df11e3a3da9ab4f926788fc30a7db4 metrics: - type: map_at_1 value: 69.33500000000001 - type: map_at_10 value: 83.554 - type: map_at_100 value: 84.237 - type: map_at_1000 value: 84.251 - type: map_at_3 value: 80.456 - type: map_at_5 value: 82.395 - type: ndcg_at_1 value: 80.06 - type: ndcg_at_10 value: 87.46199999999999 - type: ndcg_at_100 value: 88.774 - type: ndcg_at_1000 value: 88.864 - type: ndcg_at_3 value: 84.437 - type: ndcg_at_5 value: 86.129 - type: precision_at_1 value: 80.06 - type: precision_at_10 value: 13.418 - type: precision_at_100 value: 1.536 - type: precision_at_1000 value: 0.157 - type: precision_at_3 value: 37.103 - type: precision_at_5 value: 24.522 - type: recall_at_1 value: 69.33500000000001 - type: recall_at_10 value: 95.03200000000001 - type: recall_at_100 value: 99.559 - type: recall_at_1000 value: 99.98700000000001 - type: recall_at_3 value: 86.404 - type: recall_at_5 value: 91.12400000000001 - task: type: Clustering dataset: type: mteb/reddit-clustering name: MTEB RedditClustering config: default split: test revision: b2805658ae38990172679479369a78b86de8c390 metrics: - type: v_measure value: 54.824256698437324 - task: type: Clustering dataset: type: mteb/reddit-clustering-p2p name: MTEB RedditClusteringP2P config: default split: test revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 metrics: - type: v_measure value: 56.768972678049366 - task: type: Retrieval dataset: type: scidocs name: MTEB SCIDOCS config: default split: test revision: 5c59ef3e437a0a9651c8fe6fde943e7dce59fba5 metrics: - type: map_at_1 value: 5.192 - type: map_at_10 value: 14.426 - type: map_at_100 value: 17.18 - type: map_at_1000 value: 17.580000000000002 - type: map_at_3 value: 9.94 - type: map_at_5 value: 12.077 - type: ndcg_at_1 value: 25.5 - type: ndcg_at_10 value: 23.765 - type: ndcg_at_100 value: 33.664 - type: ndcg_at_1000 value: 39.481 - type: ndcg_at_3 value: 21.813 - type: ndcg_at_5 value: 19.285 - type: precision_at_1 value: 25.5 - type: precision_at_10 value: 12.690000000000001 - type: precision_at_100 value: 2.71 - type: precision_at_1000 value: 0.409 - type: precision_at_3 value: 20.732999999999997 - type: precision_at_5 value: 17.24 - type: recall_at_1 value: 5.192 - type: recall_at_10 value: 25.712000000000003 - type: recall_at_100 value: 54.99699999999999 - type: recall_at_1000 value: 82.97200000000001 - type: recall_at_3 value: 12.631999999999998 - type: recall_at_5 value: 17.497 - task: type: STS dataset: type: mteb/sickr-sts name: MTEB SICK-R config: default split: test revision: 20a6d6f312dd54037fe07a32d58e5e168867909d metrics: - type: cos_sim_pearson value: 84.00280838354293 - type: cos_sim_spearman value: 80.5854192844009 - type: euclidean_pearson value: 80.55974827073891 - type: euclidean_spearman value: 80.58541460172292 - type: manhattan_pearson value: 80.27294578437488 - type: manhattan_spearman value: 80.33176193921884 - task: type: STS dataset: type: mteb/sts12-sts name: MTEB STS12 config: default split: test revision: fdf84275bb8ce4b49c971d02e84dd1abc677a50f metrics: - type: cos_sim_pearson value: 83.2801353818369 - type: cos_sim_spearman value: 72.63427853822449 - type: euclidean_pearson value: 79.01343235899544 - type: euclidean_spearman value: 72.63178302036903 - type: manhattan_pearson value: 78.65899981586094 - type: manhattan_spearman value: 72.26646573268035 - task: type: STS dataset: type: mteb/sts13-sts name: MTEB STS13 config: default split: test revision: 1591bfcbe8c69d4bf7fe2a16e2451017832cafb9 metrics: - type: cos_sim_pearson value: 83.20700572036095 - type: cos_sim_spearman value: 83.48499016384896 - type: euclidean_pearson value: 82.82555353364394 - type: euclidean_spearman value: 83.48499008964005 - type: manhattan_pearson value: 82.46034885462956 - type: manhattan_spearman value: 83.09829447251937 - task: type: STS dataset: type: mteb/sts14-sts name: MTEB STS14 config: default split: test revision: e2125984e7df8b7871f6ae9949cf6b6795e7c54b metrics: - type: cos_sim_pearson value: 82.27113025749529 - type: cos_sim_spearman value: 78.0001371342168 - type: euclidean_pearson value: 80.62651938409732 - type: euclidean_spearman value: 78.0001341029446 - type: manhattan_pearson value: 80.25786381999085 - type: manhattan_spearman value: 77.68750207429126 - task: type: STS dataset: type: mteb/sts15-sts name: MTEB STS15 config: default split: test revision: 1cd7298cac12a96a373b6a2f18738bb3e739a9b6 metrics: - type: cos_sim_pearson value: 84.98824030948605 - type: cos_sim_spearman value: 85.66275391649481 - type: euclidean_pearson value: 84.88733530073506 - type: euclidean_spearman value: 85.66275062257034 - type: manhattan_pearson value: 84.70100813924223 - type: manhattan_spearman value: 85.50318526944764 - task: type: STS dataset: type: mteb/sts16-sts name: MTEB STS16 config: default split: test revision: 360a0b2dff98700d09e634a01e1cc1624d3e42cd metrics: - type: cos_sim_pearson value: 78.82478639193744 - type: cos_sim_spearman value: 80.03011315645662 - type: euclidean_pearson value: 79.84794502236802 - type: euclidean_spearman value: 80.03011258077692 - type: manhattan_pearson value: 79.47012152325492 - type: manhattan_spearman value: 79.60652985087651 - task: type: STS dataset: type: mteb/sts17-crosslingual-sts name: MTEB STS17 (en-en) config: en-en split: test revision: 9fc37e8c632af1c87a3d23e685d49552a02582a0 metrics: - type: cos_sim_pearson value: 90.90804154377126 - type: cos_sim_spearman value: 90.59523263123734 - type: euclidean_pearson value: 89.8466957775513 - type: euclidean_spearman value: 90.59523263123734 - type: manhattan_pearson value: 89.82268413033941 - type: manhattan_spearman value: 90.68706496728889 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (en) config: en split: test revision: 2de6ce8c1921b71a755b262c6b57fef195dd7906 metrics: - type: cos_sim_pearson value: 66.78771571400975 - type: cos_sim_spearman value: 67.94534221542501 - type: euclidean_pearson value: 68.62534447097993 - type: euclidean_spearman value: 67.94534221542501 - type: manhattan_pearson value: 68.35916011329631 - type: manhattan_spearman value: 67.58212723406085 - task: type: STS dataset: type: mteb/stsbenchmark-sts name: MTEB STSBenchmark config: default split: test revision: 8913289635987208e6e7c72789e4be2fe94b6abd metrics: - type: cos_sim_pearson value: 84.03996099800993 - type: cos_sim_spearman value: 83.421898505618 - type: euclidean_pearson value: 83.78671249317563 - type: euclidean_spearman value: 83.4219042133061 - type: manhattan_pearson value: 83.44085827249334 - type: manhattan_spearman value: 83.02901331535297 - task: type: Reranking dataset: type: mteb/scidocs-reranking name: MTEB SciDocsRR config: default split: test revision: 56a6d0140cf6356659e2a7c1413286a774468d44 metrics: - type: map value: 88.65396986895777 - type: mrr value: 96.60209525405604 - task: type: Retrieval dataset: type: scifact name: MTEB SciFact config: default split: test revision: a75ae049398addde9b70f6b268875f5cbce99089 metrics: - type: map_at_1 value: 51.456 - type: map_at_10 value: 60.827 - type: map_at_100 value: 61.595 - type: map_at_1000 value: 61.629999999999995 - type: map_at_3 value: 57.518 - type: map_at_5 value: 59.435 - type: ndcg_at_1 value: 53.333 - type: ndcg_at_10 value: 65.57 - type: ndcg_at_100 value: 68.911 - type: ndcg_at_1000 value: 69.65299999999999 - type: ndcg_at_3 value: 60.009 - type: ndcg_at_5 value: 62.803 - type: precision_at_1 value: 53.333 - type: precision_at_10 value: 8.933 - type: precision_at_100 value: 1.0699999999999998 - type: precision_at_1000 value: 0.11299999999999999 - type: precision_at_3 value: 23.333000000000002 - type: precision_at_5 value: 15.8 - type: recall_at_1 value: 51.456 - type: recall_at_10 value: 79.011 - type: recall_at_100 value: 94.167 - type: recall_at_1000 value: 99.667 - type: recall_at_3 value: 64.506 - type: recall_at_5 value: 71.211 - task: type: PairClassification dataset: type: mteb/sprintduplicatequestions-pairclassification name: MTEB SprintDuplicateQuestions config: default split: test revision: 5a8256d0dff9c4bd3be3ba3e67e4e70173f802ea metrics: - type: cos_sim_accuracy value: 99.65940594059406 - type: cos_sim_ap value: 90.1455141683116 - type: cos_sim_f1 value: 82.26044226044226 - type: cos_sim_precision value: 80.8695652173913 - type: cos_sim_recall value: 83.7 - type: dot_accuracy value: 99.65940594059406 - type: dot_ap value: 90.1455141683116 - type: dot_f1 value: 82.26044226044226 - type: dot_precision value: 80.8695652173913 - type: dot_recall value: 83.7 - type: euclidean_accuracy value: 99.65940594059406 - type: euclidean_ap value: 90.14551416831162 - type: euclidean_f1 value: 82.26044226044226 - type: euclidean_precision value: 80.8695652173913 - type: euclidean_recall value: 83.7 - type: manhattan_accuracy value: 99.64950495049504 - type: manhattan_ap value: 89.5492617367771 - type: manhattan_f1 value: 81.58280410356619 - type: manhattan_precision value: 79.75167144221585 - type: manhattan_recall value: 83.5 - type: max_accuracy value: 99.65940594059406 - type: max_ap value: 90.14551416831162 - type: max_f1 value: 82.26044226044226 - task: type: Clustering dataset: type: mteb/stackexchange-clustering name: MTEB StackExchangeClustering config: default split: test revision: 70a89468f6dccacc6aa2b12a6eac54e74328f235 metrics: - type: v_measure value: 53.80048409076929 - task: type: Clustering dataset: type: mteb/stackexchange-clustering-p2p name: MTEB StackExchangeClusteringP2P config: default split: test revision: d88009ab563dd0b16cfaf4436abaf97fa3550cf0 metrics: - type: v_measure value: 34.280269334397545 - task: type: Reranking dataset: type: mteb/stackoverflowdupquestions-reranking name: MTEB StackOverflowDupQuestions config: default split: test revision: ef807ea29a75ec4f91b50fd4191cb4ee4589a9f9 metrics: - type: map value: 51.97907654945493 - type: mrr value: 52.82873376623376 - task: type: Summarization dataset: type: mteb/summeval name: MTEB SummEval config: default split: test revision: 8753c2788d36c01fc6f05d03fe3f7268d63f9122 metrics: - type: cos_sim_pearson value: 28.364293841556304 - type: cos_sim_spearman value: 27.485869639926136 - type: dot_pearson value: 28.364295910221145 - type: dot_spearman value: 27.485869639926136 - task: type: Retrieval dataset: type: trec-covid name: MTEB TRECCOVID config: default split: test revision: 2c8041b2c07a79b6f7ba8fe6acc72e5d9f92d217 metrics: - type: map_at_1 value: 0.19499999999999998 - type: map_at_10 value: 1.218 - type: map_at_100 value: 7.061000000000001 - type: map_at_1000 value: 19.735 - type: map_at_3 value: 0.46499999999999997 - type: map_at_5 value: 0.672 - type: ndcg_at_1 value: 60.0 - type: ndcg_at_10 value: 51.32600000000001 - type: ndcg_at_100 value: 41.74 - type: ndcg_at_1000 value: 43.221 - type: ndcg_at_3 value: 54.989 - type: ndcg_at_5 value: 52.905 - type: precision_at_1 value: 66.0 - type: precision_at_10 value: 55.60000000000001 - type: precision_at_100 value: 43.34 - type: precision_at_1000 value: 19.994 - type: precision_at_3 value: 59.333000000000006 - type: precision_at_5 value: 57.199999999999996 - type: recall_at_1 value: 0.19499999999999998 - type: recall_at_10 value: 1.473 - type: recall_at_100 value: 10.596 - type: recall_at_1000 value: 42.466 - type: recall_at_3 value: 0.49899999999999994 - type: recall_at_5 value: 0.76 - task: type: Retrieval dataset: type: webis-touche2020 name: MTEB Touche2020 config: default split: test revision: 527b7d77e16e343303e68cb6af11d6e18b9f7b3b metrics: - type: map_at_1 value: 1.997 - type: map_at_10 value: 7.5569999999999995 - type: map_at_100 value: 12.238 - type: map_at_1000 value: 13.773 - type: map_at_3 value: 4.334 - type: map_at_5 value: 5.5 - type: ndcg_at_1 value: 22.448999999999998 - type: ndcg_at_10 value: 19.933999999999997 - type: ndcg_at_100 value: 30.525999999999996 - type: ndcg_at_1000 value: 43.147999999999996 - type: ndcg_at_3 value: 22.283 - type: ndcg_at_5 value: 21.224 - type: precision_at_1 value: 24.490000000000002 - type: precision_at_10 value: 17.551 - type: precision_at_100 value: 6.4079999999999995 - type: precision_at_1000 value: 1.463 - type: precision_at_3 value: 23.128999999999998 - type: precision_at_5 value: 20.816000000000003 - type: recall_at_1 value: 1.997 - type: recall_at_10 value: 13.001999999999999 - type: recall_at_100 value: 40.98 - type: recall_at_1000 value: 79.40899999999999 - type: recall_at_3 value: 5.380999999999999 - type: recall_at_5 value: 7.721 - task: type: Classification dataset: type: mteb/toxic_conversations_50k name: MTEB ToxicConversationsClassification config: default split: test revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de metrics: - type: accuracy value: 60.861200000000004 - type: ap value: 11.39641747026629 - type: f1 value: 47.80230380517065 - task: type: Classification dataset: type: mteb/tweet_sentiment_extraction name: MTEB TweetSentimentExtractionClassification config: default split: test revision: 62146448f05be9e52a36b8ee9936447ea787eede metrics: - type: accuracy value: 55.464063384267114 - type: f1 value: 55.759039643764666 - task: type: Clustering dataset: type: mteb/twentynewsgroups-clustering name: MTEB TwentyNewsgroupsClustering config: default split: test revision: 091a54f9a36281ce7d6590ec8c75dd485e7e01d4 metrics: - type: v_measure value: 49.74455348083809 - task: type: PairClassification dataset: type: mteb/twittersemeval2015-pairclassification name: MTEB TwitterSemEval2015 config: default split: test revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 metrics: - type: cos_sim_accuracy value: 86.07617571675507 - type: cos_sim_ap value: 73.85398650568216 - type: cos_sim_f1 value: 68.50702798531087 - type: cos_sim_precision value: 65.86316045775506 - type: cos_sim_recall value: 71.37203166226914 - type: dot_accuracy value: 86.07617571675507 - type: dot_ap value: 73.85398346238429 - type: dot_f1 value: 68.50702798531087 - type: dot_precision value: 65.86316045775506 - type: dot_recall value: 71.37203166226914 - type: euclidean_accuracy value: 86.07617571675507 - type: euclidean_ap value: 73.85398625060357 - type: euclidean_f1 value: 68.50702798531087 - type: euclidean_precision value: 65.86316045775506 - type: euclidean_recall value: 71.37203166226914 - type: manhattan_accuracy value: 85.98676759849795 - type: manhattan_ap value: 73.86874126878737 - type: manhattan_f1 value: 68.55096559662361 - type: manhattan_precision value: 66.51774633904195 - type: manhattan_recall value: 70.71240105540898 - type: max_accuracy value: 86.07617571675507 - type: max_ap value: 73.86874126878737 - type: max_f1 value: 68.55096559662361 - task: type: PairClassification dataset: type: mteb/twitterurlcorpus-pairclassification name: MTEB TwitterURLCorpus config: default split: test revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf metrics: - type: cos_sim_accuracy value: 88.51631932316529 - type: cos_sim_ap value: 85.10831084479727 - type: cos_sim_f1 value: 77.14563397129186 - type: cos_sim_precision value: 74.9709386806161 - type: cos_sim_recall value: 79.45026178010471 - type: dot_accuracy value: 88.51631932316529 - type: dot_ap value: 85.10831188797107 - type: dot_f1 value: 77.14563397129186 - type: dot_precision value: 74.9709386806161 - type: dot_recall value: 79.45026178010471 - type: euclidean_accuracy value: 88.51631932316529 - type: euclidean_ap value: 85.10829618408616 - type: euclidean_f1 value: 77.14563397129186 - type: euclidean_precision value: 74.9709386806161 - type: euclidean_recall value: 79.45026178010471 - type: manhattan_accuracy value: 88.50467652423643 - type: manhattan_ap value: 85.08329502055064 - type: manhattan_f1 value: 77.11157455683002 - type: manhattan_precision value: 74.67541834968263 - type: manhattan_recall value: 79.71204188481676 - type: max_accuracy value: 88.51631932316529 - type: max_ap value: 85.10831188797107 - type: max_f1 value: 77.14563397129186 --- # all-mpnet-base-v2 This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search. ## Usage (Sentence-Transformers) Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed: ``` pip install -U sentence-transformers ``` Then you can use the model like this: ```python from sentence_transformers import SentenceTransformer sentences = ["This is an example sentence", "Each sentence is converted"] model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2') embeddings = model.encode(sentences) print(embeddings) ``` ## Usage (HuggingFace Transformers) Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings. ```python from transformers import AutoTokenizer, AutoModel import torch import torch.nn.functional as F #Mean Pooling - Take attention mask into account for correct averaging def mean_pooling(model_output, attention_mask): token_embeddings = model_output[0] #First element of model_output contains all token embeddings input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) # Sentences we want sentence embeddings for sentences = ['This is an example sentence', 'Each sentence is converted'] # Load model from HuggingFace Hub tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-mpnet-base-v2') model = AutoModel.from_pretrained('sentence-transformers/all-mpnet-base-v2') # Tokenize sentences encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt') # Compute token embeddings with torch.no_grad(): model_output = model(**encoded_input) # Perform pooling sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) # Normalize embeddings sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) print("Sentence embeddings:") print(sentence_embeddings) ``` ## Evaluation Results For an automated evaluation of this model, see *MTEB*: https://huggingface.co/spaces/mteb/leaderboard or the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name=sentence-transformers/all-MiniLM-L12-v2) ------ ## Background The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised contrastive learning objective. We used the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model and fine-tuned in on a 1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset. We developped this model during the [Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), organized by Hugging Face. We developped this model as part of the project: [Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks. ## Intended uses Our model is intented to be used as a sentence and short paragraph encoder. Given an input text, it ouptuts a vector which captures the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks. By default, input text longer than 384 word pieces is truncated. ## Training procedure ### Pre-training We use the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model. Please refer to the model card for more detailed information about the pre-training procedure. ### Fine-tuning We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch. We then apply the cross entropy loss by comparing with true pairs. #### Hyper parameters We trained ou model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core). We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`. #### Training data We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences. We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file. | Dataset | Paper | Number of training tuples | |--------------------------------------------------------|:----------------------------------------:|:--------------------------:| | [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 | | [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 | | [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 | | [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 | | [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 | | [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs | - | 25,316,456 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs | - | 21,396,559 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs | - | 21,396,559 | | [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 | | [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 | | [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 | | [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 | | [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395| | [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 | | [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 | | [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 | | [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 | | [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 | | [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 | | AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 | | [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 | | [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 | | [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 | | [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 | | [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 | | [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 | | [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 | | [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 | | [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 | | **Total** | | **1,170,060,424** |