|
--- |
|
pipeline_tag: sentence-similarity |
|
tags: |
|
- sentence-transformers |
|
- feature-extraction |
|
- sentence-similarity |
|
- mteb |
|
language: en |
|
license: apache-2.0 |
|
datasets: |
|
- s2orc |
|
- flax-sentence-embeddings/stackexchange_xml |
|
- MS Marco |
|
- gooaq |
|
- yahoo_answers_topics |
|
- code_search_net |
|
- search_qa |
|
- eli5 |
|
- snli |
|
- multi_nli |
|
- wikihow |
|
- natural_questions |
|
- trivia_qa |
|
- embedding-data/sentence-compression |
|
- embedding-data/flickr30k-captions |
|
- embedding-data/altlex |
|
- embedding-data/simple-wiki |
|
- embedding-data/QQP |
|
- embedding-data/SPECTER |
|
- embedding-data/PAQ_pairs |
|
- embedding-data/WikiAnswers |
|
model-index: |
|
- name: all-mpnet-base-v2 |
|
results: |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en) |
|
config: en |
|
split: test |
|
revision: 2d8a100785abf0ae21420d2a55b0c56e3e1ea996 |
|
metrics: |
|
- type: accuracy |
|
value: 65.26865671641791 |
|
- type: ap |
|
value: 28.47453420428918 |
|
- type: f1 |
|
value: 59.3470101009448 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_polarity |
|
name: MTEB AmazonPolarityClassification |
|
config: default |
|
split: test |
|
revision: 80714f8dcf8cefc218ef4f8c5a966dd83f75a0e1 |
|
metrics: |
|
- type: accuracy |
|
value: 67.13145 |
|
- type: ap |
|
value: 61.842060778903786 |
|
- type: f1 |
|
value: 66.79987305640383 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (en) |
|
config: en |
|
split: test |
|
revision: c379a6705fec24a2493fa68e011692605f44e119 |
|
metrics: |
|
- type: accuracy |
|
value: 31.920000000000005 |
|
- type: f1 |
|
value: 31.2465193896153 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: 5b3e3697907184a9b77a3c99ee9ea1a9cbb1e4e3 |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.186 |
|
- type: map_at_10 |
|
value: 37.692 |
|
- type: map_at_100 |
|
value: 38.986 |
|
- type: map_at_1000 |
|
value: 38.991 |
|
- type: map_at_3 |
|
value: 32.622 |
|
- type: map_at_5 |
|
value: 35.004999999999995 |
|
- type: ndcg_at_1 |
|
value: 23.186 |
|
- type: ndcg_at_10 |
|
value: 46.521 |
|
- type: ndcg_at_100 |
|
value: 51.954 |
|
- type: ndcg_at_1000 |
|
value: 52.087 |
|
- type: ndcg_at_3 |
|
value: 35.849 |
|
- type: ndcg_at_5 |
|
value: 40.12 |
|
- type: precision_at_1 |
|
value: 23.186 |
|
- type: precision_at_10 |
|
value: 7.510999999999999 |
|
- type: precision_at_100 |
|
value: 0.9860000000000001 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 15.078 |
|
- type: precision_at_5 |
|
value: 11.110000000000001 |
|
- type: recall_at_1 |
|
value: 23.186 |
|
- type: recall_at_10 |
|
value: 75.107 |
|
- type: recall_at_100 |
|
value: 98.649 |
|
- type: recall_at_1000 |
|
value: 99.644 |
|
- type: recall_at_3 |
|
value: 45.235 |
|
- type: recall_at_5 |
|
value: 55.547999999999995 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-p2p |
|
name: MTEB ArxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 0bbdb47bcbe3a90093699aefeed338a0f28a7ee8 |
|
metrics: |
|
- type: v_measure |
|
value: 48.37886340922374 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-s2s |
|
name: MTEB ArxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3 |
|
metrics: |
|
- type: v_measure |
|
value: 39.72488615315985 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 4d853f94cd57d85ec13805aeeac3ae3e5eb4c49c |
|
metrics: |
|
- type: map |
|
value: 65.85199009344481 |
|
- type: mrr |
|
value: 78.47700391329201 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: 9ee918f184421b6bd48b78f6c714d86546106103 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.47737119217858 |
|
- type: cos_sim_spearman |
|
value: 80.43195317854409 |
|
- type: euclidean_pearson |
|
value: 82.20496332547978 |
|
- type: euclidean_spearman |
|
value: 80.43195317854409 |
|
- type: manhattan_pearson |
|
value: 81.4836610720397 |
|
- type: manhattan_spearman |
|
value: 79.65904400101908 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/banking77 |
|
name: MTEB Banking77Classification |
|
config: default |
|
split: test |
|
revision: 44fa15921b4c889113cc5df03dd4901b49161ab7 |
|
metrics: |
|
- type: accuracy |
|
value: 81.8603896103896 |
|
- type: f1 |
|
value: 81.28027245637479 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-p2p |
|
name: MTEB BiorxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 11d0121201d1f1f280e8cc8f3d98fb9c4d9f9c55 |
|
metrics: |
|
- type: v_measure |
|
value: 39.616605133625185 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-s2s |
|
name: MTEB BiorxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: c0fab014e1bcb8d3a5e31b2088972a1e01547dc1 |
|
metrics: |
|
- type: v_measure |
|
value: 35.02442407186902 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackAndroidRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 36.036 |
|
- type: map_at_10 |
|
value: 49.302 |
|
- type: map_at_100 |
|
value: 50.956 |
|
- type: map_at_1000 |
|
value: 51.080000000000005 |
|
- type: map_at_3 |
|
value: 45.237 |
|
- type: map_at_5 |
|
value: 47.353 |
|
- type: ndcg_at_1 |
|
value: 45.207 |
|
- type: ndcg_at_10 |
|
value: 56.485 |
|
- type: ndcg_at_100 |
|
value: 61.413 |
|
- type: ndcg_at_1000 |
|
value: 62.870000000000005 |
|
- type: ndcg_at_3 |
|
value: 51.346000000000004 |
|
- type: ndcg_at_5 |
|
value: 53.486 |
|
- type: precision_at_1 |
|
value: 45.207 |
|
- type: precision_at_10 |
|
value: 11.144 |
|
- type: precision_at_100 |
|
value: 1.735 |
|
- type: precision_at_1000 |
|
value: 0.22100000000000003 |
|
- type: precision_at_3 |
|
value: 24.94 |
|
- type: precision_at_5 |
|
value: 17.997 |
|
- type: recall_at_1 |
|
value: 36.036 |
|
- type: recall_at_10 |
|
value: 69.191 |
|
- type: recall_at_100 |
|
value: 89.423 |
|
- type: recall_at_1000 |
|
value: 98.425 |
|
- type: recall_at_3 |
|
value: 53.849999999999994 |
|
- type: recall_at_5 |
|
value: 60.107 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackEnglishRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 32.92 |
|
- type: map_at_10 |
|
value: 45.739999999999995 |
|
- type: map_at_100 |
|
value: 47.309 |
|
- type: map_at_1000 |
|
value: 47.443000000000005 |
|
- type: map_at_3 |
|
value: 42.154 |
|
- type: map_at_5 |
|
value: 44.207 |
|
- type: ndcg_at_1 |
|
value: 42.229 |
|
- type: ndcg_at_10 |
|
value: 52.288999999999994 |
|
- type: ndcg_at_100 |
|
value: 57.04900000000001 |
|
- type: ndcg_at_1000 |
|
value: 58.788 |
|
- type: ndcg_at_3 |
|
value: 47.531 |
|
- type: ndcg_at_5 |
|
value: 49.861 |
|
- type: precision_at_1 |
|
value: 42.229 |
|
- type: precision_at_10 |
|
value: 10.299 |
|
- type: precision_at_100 |
|
value: 1.68 |
|
- type: precision_at_1000 |
|
value: 0.213 |
|
- type: precision_at_3 |
|
value: 23.673 |
|
- type: precision_at_5 |
|
value: 17.006 |
|
- type: recall_at_1 |
|
value: 32.92 |
|
- type: recall_at_10 |
|
value: 63.865 |
|
- type: recall_at_100 |
|
value: 84.06700000000001 |
|
- type: recall_at_1000 |
|
value: 94.536 |
|
- type: recall_at_3 |
|
value: 49.643 |
|
- type: recall_at_5 |
|
value: 56.119 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGamingRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 40.695 |
|
- type: map_at_10 |
|
value: 53.787 |
|
- type: map_at_100 |
|
value: 54.778000000000006 |
|
- type: map_at_1000 |
|
value: 54.827000000000005 |
|
- type: map_at_3 |
|
value: 50.151999999999994 |
|
- type: map_at_5 |
|
value: 52.207 |
|
- type: ndcg_at_1 |
|
value: 46.52 |
|
- type: ndcg_at_10 |
|
value: 60.026 |
|
- type: ndcg_at_100 |
|
value: 63.81099999999999 |
|
- type: ndcg_at_1000 |
|
value: 64.741 |
|
- type: ndcg_at_3 |
|
value: 53.83 |
|
- type: ndcg_at_5 |
|
value: 56.928999999999995 |
|
- type: precision_at_1 |
|
value: 46.52 |
|
- type: precision_at_10 |
|
value: 9.754999999999999 |
|
- type: precision_at_100 |
|
value: 1.2670000000000001 |
|
- type: precision_at_1000 |
|
value: 0.13799999999999998 |
|
- type: precision_at_3 |
|
value: 24.096 |
|
- type: precision_at_5 |
|
value: 16.689999999999998 |
|
- type: recall_at_1 |
|
value: 40.695 |
|
- type: recall_at_10 |
|
value: 75.181 |
|
- type: recall_at_100 |
|
value: 91.479 |
|
- type: recall_at_1000 |
|
value: 98.06899999999999 |
|
- type: recall_at_3 |
|
value: 58.707 |
|
- type: recall_at_5 |
|
value: 66.295 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGisRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 29.024 |
|
- type: map_at_10 |
|
value: 38.438 |
|
- type: map_at_100 |
|
value: 39.576 |
|
- type: map_at_1000 |
|
value: 39.645 |
|
- type: map_at_3 |
|
value: 34.827999999999996 |
|
- type: map_at_5 |
|
value: 36.947 |
|
- type: ndcg_at_1 |
|
value: 31.299 |
|
- type: ndcg_at_10 |
|
value: 44.268 |
|
- type: ndcg_at_100 |
|
value: 49.507 |
|
- type: ndcg_at_1000 |
|
value: 51.205999999999996 |
|
- type: ndcg_at_3 |
|
value: 37.248999999999995 |
|
- type: ndcg_at_5 |
|
value: 40.861999999999995 |
|
- type: precision_at_1 |
|
value: 31.299 |
|
- type: precision_at_10 |
|
value: 6.949 |
|
- type: precision_at_100 |
|
value: 1.012 |
|
- type: precision_at_1000 |
|
value: 0.11900000000000001 |
|
- type: precision_at_3 |
|
value: 15.518 |
|
- type: precision_at_5 |
|
value: 11.366999999999999 |
|
- type: recall_at_1 |
|
value: 29.024 |
|
- type: recall_at_10 |
|
value: 60.404 |
|
- type: recall_at_100 |
|
value: 83.729 |
|
- type: recall_at_1000 |
|
value: 96.439 |
|
- type: recall_at_3 |
|
value: 41.65 |
|
- type: recall_at_5 |
|
value: 50.263999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackMathematicaRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.774 |
|
- type: map_at_10 |
|
value: 28.099 |
|
- type: map_at_100 |
|
value: 29.603 |
|
- type: map_at_1000 |
|
value: 29.709999999999997 |
|
- type: map_at_3 |
|
value: 25.036 |
|
- type: map_at_5 |
|
value: 26.657999999999998 |
|
- type: ndcg_at_1 |
|
value: 22.139 |
|
- type: ndcg_at_10 |
|
value: 34.205999999999996 |
|
- type: ndcg_at_100 |
|
value: 40.844 |
|
- type: ndcg_at_1000 |
|
value: 43.144 |
|
- type: ndcg_at_3 |
|
value: 28.732999999999997 |
|
- type: ndcg_at_5 |
|
value: 31.252000000000002 |
|
- type: precision_at_1 |
|
value: 22.139 |
|
- type: precision_at_10 |
|
value: 6.567 |
|
- type: precision_at_100 |
|
value: 1.147 |
|
- type: precision_at_1000 |
|
value: 0.146 |
|
- type: precision_at_3 |
|
value: 14.386 |
|
- type: precision_at_5 |
|
value: 10.423 |
|
- type: recall_at_1 |
|
value: 17.774 |
|
- type: recall_at_10 |
|
value: 48.32 |
|
- type: recall_at_100 |
|
value: 76.373 |
|
- type: recall_at_1000 |
|
value: 92.559 |
|
- type: recall_at_3 |
|
value: 33.478 |
|
- type: recall_at_5 |
|
value: 39.872 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackPhysicsRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 31.885 |
|
- type: map_at_10 |
|
value: 44.289 |
|
- type: map_at_100 |
|
value: 45.757999999999996 |
|
- type: map_at_1000 |
|
value: 45.86 |
|
- type: map_at_3 |
|
value: 40.459 |
|
- type: map_at_5 |
|
value: 42.662 |
|
- type: ndcg_at_1 |
|
value: 39.75 |
|
- type: ndcg_at_10 |
|
value: 50.975 |
|
- type: ndcg_at_100 |
|
value: 56.528999999999996 |
|
- type: ndcg_at_1000 |
|
value: 58.06099999999999 |
|
- type: ndcg_at_3 |
|
value: 45.327 |
|
- type: ndcg_at_5 |
|
value: 48.041 |
|
- type: precision_at_1 |
|
value: 39.75 |
|
- type: precision_at_10 |
|
value: 9.557 |
|
- type: precision_at_100 |
|
value: 1.469 |
|
- type: precision_at_1000 |
|
value: 0.17700000000000002 |
|
- type: precision_at_3 |
|
value: 22.073 |
|
- type: precision_at_5 |
|
value: 15.765 |
|
- type: recall_at_1 |
|
value: 31.885 |
|
- type: recall_at_10 |
|
value: 64.649 |
|
- type: recall_at_100 |
|
value: 87.702 |
|
- type: recall_at_1000 |
|
value: 97.327 |
|
- type: recall_at_3 |
|
value: 48.61 |
|
- type: recall_at_5 |
|
value: 55.882 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackProgrammersRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.454 |
|
- type: map_at_10 |
|
value: 37.756 |
|
- type: map_at_100 |
|
value: 39.225 |
|
- type: map_at_1000 |
|
value: 39.332 |
|
- type: map_at_3 |
|
value: 34.115 |
|
- type: map_at_5 |
|
value: 35.942 |
|
- type: ndcg_at_1 |
|
value: 32.42 |
|
- type: ndcg_at_10 |
|
value: 44.165 |
|
- type: ndcg_at_100 |
|
value: 50.202000000000005 |
|
- type: ndcg_at_1000 |
|
value: 52.188 |
|
- type: ndcg_at_3 |
|
value: 38.381 |
|
- type: ndcg_at_5 |
|
value: 40.849000000000004 |
|
- type: precision_at_1 |
|
value: 32.42 |
|
- type: precision_at_10 |
|
value: 8.482000000000001 |
|
- type: precision_at_100 |
|
value: 1.332 |
|
- type: precision_at_1000 |
|
value: 0.169 |
|
- type: precision_at_3 |
|
value: 18.683 |
|
- type: precision_at_5 |
|
value: 13.539000000000001 |
|
- type: recall_at_1 |
|
value: 26.454 |
|
- type: recall_at_10 |
|
value: 57.937000000000005 |
|
- type: recall_at_100 |
|
value: 83.76 |
|
- type: recall_at_1000 |
|
value: 96.82600000000001 |
|
- type: recall_at_3 |
|
value: 41.842 |
|
- type: recall_at_5 |
|
value: 48.285 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.743666666666666 |
|
- type: map_at_10 |
|
value: 38.75416666666667 |
|
- type: map_at_100 |
|
value: 40.133250000000004 |
|
- type: map_at_1000 |
|
value: 40.24616666666667 |
|
- type: map_at_3 |
|
value: 35.267250000000004 |
|
- type: map_at_5 |
|
value: 37.132749999999994 |
|
- type: ndcg_at_1 |
|
value: 33.14358333333333 |
|
- type: ndcg_at_10 |
|
value: 44.95916666666667 |
|
- type: ndcg_at_100 |
|
value: 50.46375 |
|
- type: ndcg_at_1000 |
|
value: 52.35508333333334 |
|
- type: ndcg_at_3 |
|
value: 39.17883333333334 |
|
- type: ndcg_at_5 |
|
value: 41.79724999999999 |
|
- type: precision_at_1 |
|
value: 33.14358333333333 |
|
- type: precision_at_10 |
|
value: 8.201083333333333 |
|
- type: precision_at_100 |
|
value: 1.3085 |
|
- type: precision_at_1000 |
|
value: 0.1665833333333333 |
|
- type: precision_at_3 |
|
value: 18.405583333333333 |
|
- type: precision_at_5 |
|
value: 13.233166666666666 |
|
- type: recall_at_1 |
|
value: 27.743666666666666 |
|
- type: recall_at_10 |
|
value: 58.91866666666667 |
|
- type: recall_at_100 |
|
value: 82.76216666666666 |
|
- type: recall_at_1000 |
|
value: 95.56883333333333 |
|
- type: recall_at_3 |
|
value: 42.86925 |
|
- type: recall_at_5 |
|
value: 49.553333333333335 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackStatsRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.244 |
|
- type: map_at_10 |
|
value: 33.464 |
|
- type: map_at_100 |
|
value: 34.633 |
|
- type: map_at_1000 |
|
value: 34.721999999999994 |
|
- type: map_at_3 |
|
value: 30.784 |
|
- type: map_at_5 |
|
value: 32.183 |
|
- type: ndcg_at_1 |
|
value: 28.681 |
|
- type: ndcg_at_10 |
|
value: 38.149 |
|
- type: ndcg_at_100 |
|
value: 43.856 |
|
- type: ndcg_at_1000 |
|
value: 46.026 |
|
- type: ndcg_at_3 |
|
value: 33.318 |
|
- type: ndcg_at_5 |
|
value: 35.454 |
|
- type: precision_at_1 |
|
value: 28.681 |
|
- type: precision_at_10 |
|
value: 6.304 |
|
- type: precision_at_100 |
|
value: 0.992 |
|
- type: precision_at_1000 |
|
value: 0.125 |
|
- type: precision_at_3 |
|
value: 14.673 |
|
- type: precision_at_5 |
|
value: 10.245 |
|
- type: recall_at_1 |
|
value: 25.244 |
|
- type: recall_at_10 |
|
value: 49.711 |
|
- type: recall_at_100 |
|
value: 75.928 |
|
- type: recall_at_1000 |
|
value: 91.79899999999999 |
|
- type: recall_at_3 |
|
value: 36.325 |
|
- type: recall_at_5 |
|
value: 41.752 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackTexRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 18.857 |
|
- type: map_at_10 |
|
value: 27.794 |
|
- type: map_at_100 |
|
value: 29.186 |
|
- type: map_at_1000 |
|
value: 29.323 |
|
- type: map_at_3 |
|
value: 24.779 |
|
- type: map_at_5 |
|
value: 26.459 |
|
- type: ndcg_at_1 |
|
value: 23.227999999999998 |
|
- type: ndcg_at_10 |
|
value: 33.353 |
|
- type: ndcg_at_100 |
|
value: 39.598 |
|
- type: ndcg_at_1000 |
|
value: 42.268 |
|
- type: ndcg_at_3 |
|
value: 28.054000000000002 |
|
- type: ndcg_at_5 |
|
value: 30.566 |
|
- type: precision_at_1 |
|
value: 23.227999999999998 |
|
- type: precision_at_10 |
|
value: 6.397 |
|
- type: precision_at_100 |
|
value: 1.129 |
|
- type: precision_at_1000 |
|
value: 0.155 |
|
- type: precision_at_3 |
|
value: 13.616 |
|
- type: precision_at_5 |
|
value: 10.116999999999999 |
|
- type: recall_at_1 |
|
value: 18.857 |
|
- type: recall_at_10 |
|
value: 45.797 |
|
- type: recall_at_100 |
|
value: 73.615 |
|
- type: recall_at_1000 |
|
value: 91.959 |
|
- type: recall_at_3 |
|
value: 31.129 |
|
- type: recall_at_5 |
|
value: 37.565 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackUnixRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.486 |
|
- type: map_at_10 |
|
value: 39.164 |
|
- type: map_at_100 |
|
value: 40.543 |
|
- type: map_at_1000 |
|
value: 40.636 |
|
- type: map_at_3 |
|
value: 35.52 |
|
- type: map_at_5 |
|
value: 37.355 |
|
- type: ndcg_at_1 |
|
value: 32.275999999999996 |
|
- type: ndcg_at_10 |
|
value: 45.414 |
|
- type: ndcg_at_100 |
|
value: 51.254 |
|
- type: ndcg_at_1000 |
|
value: 53.044000000000004 |
|
- type: ndcg_at_3 |
|
value: 39.324999999999996 |
|
- type: ndcg_at_5 |
|
value: 41.835 |
|
- type: precision_at_1 |
|
value: 32.275999999999996 |
|
- type: precision_at_10 |
|
value: 8.144 |
|
- type: precision_at_100 |
|
value: 1.237 |
|
- type: precision_at_1000 |
|
value: 0.15 |
|
- type: precision_at_3 |
|
value: 18.501 |
|
- type: precision_at_5 |
|
value: 13.134 |
|
- type: recall_at_1 |
|
value: 27.486 |
|
- type: recall_at_10 |
|
value: 60.449 |
|
- type: recall_at_100 |
|
value: 85.176 |
|
- type: recall_at_1000 |
|
value: 97.087 |
|
- type: recall_at_3 |
|
value: 43.59 |
|
- type: recall_at_5 |
|
value: 50.08899999999999 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWebmastersRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.207 |
|
- type: map_at_10 |
|
value: 37.255 |
|
- type: map_at_100 |
|
value: 39.043 |
|
- type: map_at_1000 |
|
value: 39.273 |
|
- type: map_at_3 |
|
value: 33.487 |
|
- type: map_at_5 |
|
value: 35.441 |
|
- type: ndcg_at_1 |
|
value: 31.423000000000002 |
|
- type: ndcg_at_10 |
|
value: 44.235 |
|
- type: ndcg_at_100 |
|
value: 50.49 |
|
- type: ndcg_at_1000 |
|
value: 52.283 |
|
- type: ndcg_at_3 |
|
value: 37.602000000000004 |
|
- type: ndcg_at_5 |
|
value: 40.518 |
|
- type: precision_at_1 |
|
value: 31.423000000000002 |
|
- type: precision_at_10 |
|
value: 8.715 |
|
- type: precision_at_100 |
|
value: 1.7590000000000001 |
|
- type: precision_at_1000 |
|
value: 0.257 |
|
- type: precision_at_3 |
|
value: 17.523 |
|
- type: precision_at_5 |
|
value: 13.161999999999999 |
|
- type: recall_at_1 |
|
value: 26.207 |
|
- type: recall_at_10 |
|
value: 59.17099999999999 |
|
- type: recall_at_100 |
|
value: 86.166 |
|
- type: recall_at_1000 |
|
value: 96.54700000000001 |
|
- type: recall_at_3 |
|
value: 41.18 |
|
- type: recall_at_5 |
|
value: 48.083999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWordpressRetrieval |
|
config: default |
|
split: test |
|
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db |
|
metrics: |
|
- type: map_at_1 |
|
value: 20.342 |
|
- type: map_at_10 |
|
value: 29.962 |
|
- type: map_at_100 |
|
value: 30.989 |
|
- type: map_at_1000 |
|
value: 31.102999999999998 |
|
- type: map_at_3 |
|
value: 26.656000000000002 |
|
- type: map_at_5 |
|
value: 28.179 |
|
- type: ndcg_at_1 |
|
value: 22.551 |
|
- type: ndcg_at_10 |
|
value: 35.945 |
|
- type: ndcg_at_100 |
|
value: 41.012 |
|
- type: ndcg_at_1000 |
|
value: 43.641999999999996 |
|
- type: ndcg_at_3 |
|
value: 29.45 |
|
- type: ndcg_at_5 |
|
value: 31.913999999999998 |
|
- type: precision_at_1 |
|
value: 22.551 |
|
- type: precision_at_10 |
|
value: 6.1 |
|
- type: precision_at_100 |
|
value: 0.943 |
|
- type: precision_at_1000 |
|
value: 0.129 |
|
- type: precision_at_3 |
|
value: 13.184999999999999 |
|
- type: precision_at_5 |
|
value: 9.353 |
|
- type: recall_at_1 |
|
value: 20.342 |
|
- type: recall_at_10 |
|
value: 52.349000000000004 |
|
- type: recall_at_100 |
|
value: 75.728 |
|
- type: recall_at_1000 |
|
value: 95.253 |
|
- type: recall_at_3 |
|
value: 34.427 |
|
- type: recall_at_5 |
|
value: 40.326 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: climate-fever |
|
name: MTEB ClimateFEVER |
|
config: default |
|
split: test |
|
revision: 392b78eb68c07badcd7c2cd8f39af108375dfcce |
|
metrics: |
|
- type: map_at_1 |
|
value: 7.71 |
|
- type: map_at_10 |
|
value: 14.81 |
|
- type: map_at_100 |
|
value: 16.536 |
|
- type: map_at_1000 |
|
value: 16.744999999999997 |
|
- type: map_at_3 |
|
value: 12.109 |
|
- type: map_at_5 |
|
value: 13.613 |
|
- type: ndcg_at_1 |
|
value: 18.046 |
|
- type: ndcg_at_10 |
|
value: 21.971 |
|
- type: ndcg_at_100 |
|
value: 29.468 |
|
- type: ndcg_at_1000 |
|
value: 33.428999999999995 |
|
- type: ndcg_at_3 |
|
value: 17.227999999999998 |
|
- type: ndcg_at_5 |
|
value: 19.189999999999998 |
|
- type: precision_at_1 |
|
value: 18.046 |
|
- type: precision_at_10 |
|
value: 7.192 |
|
- type: precision_at_100 |
|
value: 1.51 |
|
- type: precision_at_1000 |
|
value: 0.22499999999999998 |
|
- type: precision_at_3 |
|
value: 13.312 |
|
- type: precision_at_5 |
|
value: 10.775 |
|
- type: recall_at_1 |
|
value: 7.71 |
|
- type: recall_at_10 |
|
value: 27.908 |
|
- type: recall_at_100 |
|
value: 54.452 |
|
- type: recall_at_1000 |
|
value: 76.764 |
|
- type: recall_at_3 |
|
value: 16.64 |
|
- type: recall_at_5 |
|
value: 21.631 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: dbpedia-entity |
|
name: MTEB DBPedia |
|
config: default |
|
split: test |
|
revision: f097057d03ed98220bc7309ddb10b71a54d667d6 |
|
metrics: |
|
- type: map_at_1 |
|
value: 6.8180000000000005 |
|
- type: map_at_10 |
|
value: 14.591000000000001 |
|
- type: map_at_100 |
|
value: 19.855999999999998 |
|
- type: map_at_1000 |
|
value: 21.178 |
|
- type: map_at_3 |
|
value: 10.345 |
|
- type: map_at_5 |
|
value: 12.367 |
|
- type: ndcg_at_1 |
|
value: 39.25 |
|
- type: ndcg_at_10 |
|
value: 32.088 |
|
- type: ndcg_at_100 |
|
value: 36.019 |
|
- type: ndcg_at_1000 |
|
value: 43.649 |
|
- type: ndcg_at_3 |
|
value: 35.132999999999996 |
|
- type: ndcg_at_5 |
|
value: 33.777 |
|
- type: precision_at_1 |
|
value: 49.5 |
|
- type: precision_at_10 |
|
value: 25.624999999999996 |
|
- type: precision_at_100 |
|
value: 8.043 |
|
- type: precision_at_1000 |
|
value: 1.7409999999999999 |
|
- type: precision_at_3 |
|
value: 38.417 |
|
- type: precision_at_5 |
|
value: 33.2 |
|
- type: recall_at_1 |
|
value: 6.8180000000000005 |
|
- type: recall_at_10 |
|
value: 20.399 |
|
- type: recall_at_100 |
|
value: 42.8 |
|
- type: recall_at_1000 |
|
value: 68.081 |
|
- type: recall_at_3 |
|
value: 11.928999999999998 |
|
- type: recall_at_5 |
|
value: 15.348999999999998 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/emotion |
|
name: MTEB EmotionClassification |
|
config: default |
|
split: test |
|
revision: 829147f8f75a25f005913200eb5ed41fae320aa1 |
|
metrics: |
|
- type: accuracy |
|
value: 39.725 |
|
- type: f1 |
|
value: 35.19385687310605 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fever |
|
name: MTEB FEVER |
|
config: default |
|
split: test |
|
revision: 1429cf27e393599b8b359b9b72c666f96b2525f9 |
|
metrics: |
|
- type: map_at_1 |
|
value: 31.901000000000003 |
|
- type: map_at_10 |
|
value: 44.156 |
|
- type: map_at_100 |
|
value: 44.901 |
|
- type: map_at_1000 |
|
value: 44.939 |
|
- type: map_at_3 |
|
value: 41.008 |
|
- type: map_at_5 |
|
value: 42.969 |
|
- type: ndcg_at_1 |
|
value: 34.263 |
|
- type: ndcg_at_10 |
|
value: 50.863 |
|
- type: ndcg_at_100 |
|
value: 54.336 |
|
- type: ndcg_at_1000 |
|
value: 55.297 |
|
- type: ndcg_at_3 |
|
value: 44.644 |
|
- type: ndcg_at_5 |
|
value: 48.075 |
|
- type: precision_at_1 |
|
value: 34.263 |
|
- type: precision_at_10 |
|
value: 7.542999999999999 |
|
- type: precision_at_100 |
|
value: 0.9400000000000001 |
|
- type: precision_at_1000 |
|
value: 0.104 |
|
- type: precision_at_3 |
|
value: 18.912000000000003 |
|
- type: precision_at_5 |
|
value: 13.177 |
|
- type: recall_at_1 |
|
value: 31.901000000000003 |
|
- type: recall_at_10 |
|
value: 68.872 |
|
- type: recall_at_100 |
|
value: 84.468 |
|
- type: recall_at_1000 |
|
value: 91.694 |
|
- type: recall_at_3 |
|
value: 52.272 |
|
- type: recall_at_5 |
|
value: 60.504999999999995 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fiqa |
|
name: MTEB FiQA2018 |
|
config: default |
|
split: test |
|
revision: 41b686a7f28c59bcaaa5791efd47c67c8ebe28be |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.4 |
|
- type: map_at_10 |
|
value: 41.117 |
|
- type: map_at_100 |
|
value: 43.167 |
|
- type: map_at_1000 |
|
value: 43.323 |
|
- type: map_at_3 |
|
value: 35.744 |
|
- type: map_at_5 |
|
value: 38.708 |
|
- type: ndcg_at_1 |
|
value: 49.074 |
|
- type: ndcg_at_10 |
|
value: 49.963 |
|
- type: ndcg_at_100 |
|
value: 56.564 |
|
- type: ndcg_at_1000 |
|
value: 58.931999999999995 |
|
- type: ndcg_at_3 |
|
value: 45.489000000000004 |
|
- type: ndcg_at_5 |
|
value: 47.133 |
|
- type: precision_at_1 |
|
value: 49.074 |
|
- type: precision_at_10 |
|
value: 13.889000000000001 |
|
- type: precision_at_100 |
|
value: 2.091 |
|
- type: precision_at_1000 |
|
value: 0.251 |
|
- type: precision_at_3 |
|
value: 30.658 |
|
- type: precision_at_5 |
|
value: 22.593 |
|
- type: recall_at_1 |
|
value: 24.4 |
|
- type: recall_at_10 |
|
value: 58.111999999999995 |
|
- type: recall_at_100 |
|
value: 81.96900000000001 |
|
- type: recall_at_1000 |
|
value: 96.187 |
|
- type: recall_at_3 |
|
value: 41.661 |
|
- type: recall_at_5 |
|
value: 49.24 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: hotpotqa |
|
name: MTEB HotpotQA |
|
config: default |
|
split: test |
|
revision: 766870b35a1b9ca65e67a0d1913899973551fc6c |
|
metrics: |
|
- type: map_at_1 |
|
value: 22.262 |
|
- type: map_at_10 |
|
value: 31.266 |
|
- type: map_at_100 |
|
value: 32.202 |
|
- type: map_at_1000 |
|
value: 32.300000000000004 |
|
- type: map_at_3 |
|
value: 28.874 |
|
- type: map_at_5 |
|
value: 30.246000000000002 |
|
- type: ndcg_at_1 |
|
value: 44.524 |
|
- type: ndcg_at_10 |
|
value: 39.294000000000004 |
|
- type: ndcg_at_100 |
|
value: 43.296 |
|
- type: ndcg_at_1000 |
|
value: 45.561 |
|
- type: ndcg_at_3 |
|
value: 35.013 |
|
- type: ndcg_at_5 |
|
value: 37.177 |
|
- type: precision_at_1 |
|
value: 44.524 |
|
- type: precision_at_10 |
|
value: 8.52 |
|
- type: precision_at_100 |
|
value: 1.169 |
|
- type: precision_at_1000 |
|
value: 0.147 |
|
- type: precision_at_3 |
|
value: 22.003 |
|
- type: precision_at_5 |
|
value: 14.914 |
|
- type: recall_at_1 |
|
value: 22.262 |
|
- type: recall_at_10 |
|
value: 42.6 |
|
- type: recall_at_100 |
|
value: 58.46 |
|
- type: recall_at_1000 |
|
value: 73.565 |
|
- type: recall_at_3 |
|
value: 33.005 |
|
- type: recall_at_5 |
|
value: 37.286 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/imdb |
|
name: MTEB ImdbClassification |
|
config: default |
|
split: test |
|
revision: 8d743909f834c38949e8323a8a6ce8721ea6c7f4 |
|
metrics: |
|
- type: accuracy |
|
value: 70.7156 |
|
- type: ap |
|
value: 64.89470531959896 |
|
- type: f1 |
|
value: 70.53051887683772 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: msmarco |
|
name: MTEB MSMARCO |
|
config: default |
|
split: dev |
|
revision: e6838a846e2408f22cf5cc337ebc83e0bcf77849 |
|
metrics: |
|
- type: map_at_1 |
|
value: 21.174 |
|
- type: map_at_10 |
|
value: 33.0 |
|
- type: map_at_100 |
|
value: 34.178 |
|
- type: map_at_1000 |
|
value: 34.227000000000004 |
|
- type: map_at_3 |
|
value: 29.275000000000002 |
|
- type: map_at_5 |
|
value: 31.341 |
|
- type: ndcg_at_1 |
|
value: 21.776999999999997 |
|
- type: ndcg_at_10 |
|
value: 39.745999999999995 |
|
- type: ndcg_at_100 |
|
value: 45.488 |
|
- type: ndcg_at_1000 |
|
value: 46.733999999999995 |
|
- type: ndcg_at_3 |
|
value: 32.086 |
|
- type: ndcg_at_5 |
|
value: 35.792 |
|
- type: precision_at_1 |
|
value: 21.776999999999997 |
|
- type: precision_at_10 |
|
value: 6.324000000000001 |
|
- type: precision_at_100 |
|
value: 0.922 |
|
- type: precision_at_1000 |
|
value: 0.10300000000000001 |
|
- type: precision_at_3 |
|
value: 13.696 |
|
- type: precision_at_5 |
|
value: 10.100000000000001 |
|
- type: recall_at_1 |
|
value: 21.174 |
|
- type: recall_at_10 |
|
value: 60.488 |
|
- type: recall_at_100 |
|
value: 87.234 |
|
- type: recall_at_1000 |
|
value: 96.806 |
|
- type: recall_at_3 |
|
value: 39.582 |
|
- type: recall_at_5 |
|
value: 48.474000000000004 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (en) |
|
config: en |
|
split: test |
|
revision: a7e2a951126a26fc8c6a69f835f33a346ba259e3 |
|
metrics: |
|
- type: accuracy |
|
value: 92.07934336525308 |
|
- type: f1 |
|
value: 91.93440027035814 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 6299947a7777084cc2d4b64235bf7190381ce755 |
|
metrics: |
|
- type: accuracy |
|
value: 70.20975832193344 |
|
- type: f1 |
|
value: 48.571776628850074 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 072a486a144adf7f4479a4a0dddb2152e161e1ea |
|
metrics: |
|
- type: accuracy |
|
value: 69.56624075319435 |
|
- type: f1 |
|
value: 67.64419185784621 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (en) |
|
config: en |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 76.01210490921318 |
|
- type: f1 |
|
value: 75.1934366365826 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-p2p |
|
name: MTEB MedrxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: dcefc037ef84348e49b0d29109e891c01067226b |
|
metrics: |
|
- type: v_measure |
|
value: 35.58002813186373 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-s2s |
|
name: MTEB MedrxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 3cd0e71dfbe09d4de0f9e5ecba43e7ce280959dc |
|
metrics: |
|
- type: v_measure |
|
value: 32.872725562410444 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 30.965343604861328 |
|
- type: mrr |
|
value: 31.933710165863594 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nfcorpus |
|
name: MTEB NFCorpus |
|
config: default |
|
split: test |
|
revision: 7eb63cc0c1eb59324d709ebed25fcab851fa7610 |
|
metrics: |
|
- type: map_at_1 |
|
value: 4.938 |
|
- type: map_at_10 |
|
value: 12.034 |
|
- type: map_at_100 |
|
value: 15.675 |
|
- type: map_at_1000 |
|
value: 17.18 |
|
- type: map_at_3 |
|
value: 8.471 |
|
- type: map_at_5 |
|
value: 10.128 |
|
- type: ndcg_at_1 |
|
value: 40.402 |
|
- type: ndcg_at_10 |
|
value: 33.289 |
|
- type: ndcg_at_100 |
|
value: 31.496000000000002 |
|
- type: ndcg_at_1000 |
|
value: 40.453 |
|
- type: ndcg_at_3 |
|
value: 37.841 |
|
- type: ndcg_at_5 |
|
value: 36.215 |
|
- type: precision_at_1 |
|
value: 41.796 |
|
- type: precision_at_10 |
|
value: 25.294 |
|
- type: precision_at_100 |
|
value: 8.381 |
|
- type: precision_at_1000 |
|
value: 2.1260000000000003 |
|
- type: precision_at_3 |
|
value: 36.429 |
|
- type: precision_at_5 |
|
value: 32.446000000000005 |
|
- type: recall_at_1 |
|
value: 4.938 |
|
- type: recall_at_10 |
|
value: 16.637 |
|
- type: recall_at_100 |
|
value: 33.853 |
|
- type: recall_at_1000 |
|
value: 66.07 |
|
- type: recall_at_3 |
|
value: 9.818 |
|
- type: recall_at_5 |
|
value: 12.544 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nq |
|
name: MTEB NQ |
|
config: default |
|
split: test |
|
revision: 6062aefc120bfe8ece5897809fb2e53bfe0d128c |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.124 |
|
- type: map_at_10 |
|
value: 42.418 |
|
- type: map_at_100 |
|
value: 43.633 |
|
- type: map_at_1000 |
|
value: 43.66 |
|
- type: map_at_3 |
|
value: 37.766 |
|
- type: map_at_5 |
|
value: 40.482 |
|
- type: ndcg_at_1 |
|
value: 30.794 |
|
- type: ndcg_at_10 |
|
value: 50.449999999999996 |
|
- type: ndcg_at_100 |
|
value: 55.437999999999995 |
|
- type: ndcg_at_1000 |
|
value: 56.084 |
|
- type: ndcg_at_3 |
|
value: 41.678 |
|
- type: ndcg_at_5 |
|
value: 46.257 |
|
- type: precision_at_1 |
|
value: 30.794 |
|
- type: precision_at_10 |
|
value: 8.656 |
|
- type: precision_at_100 |
|
value: 1.141 |
|
- type: precision_at_1000 |
|
value: 0.12 |
|
- type: precision_at_3 |
|
value: 19.37 |
|
- type: precision_at_5 |
|
value: 14.218 |
|
- type: recall_at_1 |
|
value: 27.124 |
|
- type: recall_at_10 |
|
value: 72.545 |
|
- type: recall_at_100 |
|
value: 93.938 |
|
- type: recall_at_1000 |
|
value: 98.788 |
|
- type: recall_at_3 |
|
value: 49.802 |
|
- type: recall_at_5 |
|
value: 60.426 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: quora |
|
name: MTEB QuoraRetrieval |
|
config: default |
|
split: test |
|
revision: 6205996560df11e3a3da9ab4f926788fc30a7db4 |
|
metrics: |
|
- type: map_at_1 |
|
value: 69.33500000000001 |
|
- type: map_at_10 |
|
value: 83.554 |
|
- type: map_at_100 |
|
value: 84.237 |
|
- type: map_at_1000 |
|
value: 84.251 |
|
- type: map_at_3 |
|
value: 80.456 |
|
- type: map_at_5 |
|
value: 82.395 |
|
- type: ndcg_at_1 |
|
value: 80.06 |
|
- type: ndcg_at_10 |
|
value: 87.46199999999999 |
|
- type: ndcg_at_100 |
|
value: 88.774 |
|
- type: ndcg_at_1000 |
|
value: 88.864 |
|
- type: ndcg_at_3 |
|
value: 84.437 |
|
- type: ndcg_at_5 |
|
value: 86.129 |
|
- type: precision_at_1 |
|
value: 80.06 |
|
- type: precision_at_10 |
|
value: 13.418 |
|
- type: precision_at_100 |
|
value: 1.536 |
|
- type: precision_at_1000 |
|
value: 0.157 |
|
- type: precision_at_3 |
|
value: 37.103 |
|
- type: precision_at_5 |
|
value: 24.522 |
|
- type: recall_at_1 |
|
value: 69.33500000000001 |
|
- type: recall_at_10 |
|
value: 95.03200000000001 |
|
- type: recall_at_100 |
|
value: 99.559 |
|
- type: recall_at_1000 |
|
value: 99.98700000000001 |
|
- type: recall_at_3 |
|
value: 86.404 |
|
- type: recall_at_5 |
|
value: 91.12400000000001 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering |
|
name: MTEB RedditClustering |
|
config: default |
|
split: test |
|
revision: b2805658ae38990172679479369a78b86de8c390 |
|
metrics: |
|
- type: v_measure |
|
value: 54.824256698437324 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering-p2p |
|
name: MTEB RedditClusteringP2P |
|
config: default |
|
split: test |
|
revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 |
|
metrics: |
|
- type: v_measure |
|
value: 56.768972678049366 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scidocs |
|
name: MTEB SCIDOCS |
|
config: default |
|
split: test |
|
revision: 5c59ef3e437a0a9651c8fe6fde943e7dce59fba5 |
|
metrics: |
|
- type: map_at_1 |
|
value: 5.192 |
|
- type: map_at_10 |
|
value: 14.426 |
|
- type: map_at_100 |
|
value: 17.18 |
|
- type: map_at_1000 |
|
value: 17.580000000000002 |
|
- type: map_at_3 |
|
value: 9.94 |
|
- type: map_at_5 |
|
value: 12.077 |
|
- type: ndcg_at_1 |
|
value: 25.5 |
|
- type: ndcg_at_10 |
|
value: 23.765 |
|
- type: ndcg_at_100 |
|
value: 33.664 |
|
- type: ndcg_at_1000 |
|
value: 39.481 |
|
- type: ndcg_at_3 |
|
value: 21.813 |
|
- type: ndcg_at_5 |
|
value: 19.285 |
|
- type: precision_at_1 |
|
value: 25.5 |
|
- type: precision_at_10 |
|
value: 12.690000000000001 |
|
- type: precision_at_100 |
|
value: 2.71 |
|
- type: precision_at_1000 |
|
value: 0.409 |
|
- type: precision_at_3 |
|
value: 20.732999999999997 |
|
- type: precision_at_5 |
|
value: 17.24 |
|
- type: recall_at_1 |
|
value: 5.192 |
|
- type: recall_at_10 |
|
value: 25.712000000000003 |
|
- type: recall_at_100 |
|
value: 54.99699999999999 |
|
- type: recall_at_1000 |
|
value: 82.97200000000001 |
|
- type: recall_at_3 |
|
value: 12.631999999999998 |
|
- type: recall_at_5 |
|
value: 17.497 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: 20a6d6f312dd54037fe07a32d58e5e168867909d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.00280838354293 |
|
- type: cos_sim_spearman |
|
value: 80.5854192844009 |
|
- type: euclidean_pearson |
|
value: 80.55974827073891 |
|
- type: euclidean_spearman |
|
value: 80.58541460172292 |
|
- type: manhattan_pearson |
|
value: 80.27294578437488 |
|
- type: manhattan_spearman |
|
value: 80.33176193921884 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: fdf84275bb8ce4b49c971d02e84dd1abc677a50f |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 83.2801353818369 |
|
- type: cos_sim_spearman |
|
value: 72.63427853822449 |
|
- type: euclidean_pearson |
|
value: 79.01343235899544 |
|
- type: euclidean_spearman |
|
value: 72.63178302036903 |
|
- type: manhattan_pearson |
|
value: 78.65899981586094 |
|
- type: manhattan_spearman |
|
value: 72.26646573268035 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 1591bfcbe8c69d4bf7fe2a16e2451017832cafb9 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 83.20700572036095 |
|
- type: cos_sim_spearman |
|
value: 83.48499016384896 |
|
- type: euclidean_pearson |
|
value: 82.82555353364394 |
|
- type: euclidean_spearman |
|
value: 83.48499008964005 |
|
- type: manhattan_pearson |
|
value: 82.46034885462956 |
|
- type: manhattan_spearman |
|
value: 83.09829447251937 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: e2125984e7df8b7871f6ae9949cf6b6795e7c54b |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 82.27113025749529 |
|
- type: cos_sim_spearman |
|
value: 78.0001371342168 |
|
- type: euclidean_pearson |
|
value: 80.62651938409732 |
|
- type: euclidean_spearman |
|
value: 78.0001341029446 |
|
- type: manhattan_pearson |
|
value: 80.25786381999085 |
|
- type: manhattan_spearman |
|
value: 77.68750207429126 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: 1cd7298cac12a96a373b6a2f18738bb3e739a9b6 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.98824030948605 |
|
- type: cos_sim_spearman |
|
value: 85.66275391649481 |
|
- type: euclidean_pearson |
|
value: 84.88733530073506 |
|
- type: euclidean_spearman |
|
value: 85.66275062257034 |
|
- type: manhattan_pearson |
|
value: 84.70100813924223 |
|
- type: manhattan_spearman |
|
value: 85.50318526944764 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 360a0b2dff98700d09e634a01e1cc1624d3e42cd |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 78.82478639193744 |
|
- type: cos_sim_spearman |
|
value: 80.03011315645662 |
|
- type: euclidean_pearson |
|
value: 79.84794502236802 |
|
- type: euclidean_spearman |
|
value: 80.03011258077692 |
|
- type: manhattan_pearson |
|
value: 79.47012152325492 |
|
- type: manhattan_spearman |
|
value: 79.60652985087651 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: 9fc37e8c632af1c87a3d23e685d49552a02582a0 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 90.90804154377126 |
|
- type: cos_sim_spearman |
|
value: 90.59523263123734 |
|
- type: euclidean_pearson |
|
value: 89.8466957775513 |
|
- type: euclidean_spearman |
|
value: 90.59523263123734 |
|
- type: manhattan_pearson |
|
value: 89.82268413033941 |
|
- type: manhattan_spearman |
|
value: 90.68706496728889 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: 2de6ce8c1921b71a755b262c6b57fef195dd7906 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 66.78771571400975 |
|
- type: cos_sim_spearman |
|
value: 67.94534221542501 |
|
- type: euclidean_pearson |
|
value: 68.62534447097993 |
|
- type: euclidean_spearman |
|
value: 67.94534221542501 |
|
- type: manhattan_pearson |
|
value: 68.35916011329631 |
|
- type: manhattan_spearman |
|
value: 67.58212723406085 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: 8913289635987208e6e7c72789e4be2fe94b6abd |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.03996099800993 |
|
- type: cos_sim_spearman |
|
value: 83.421898505618 |
|
- type: euclidean_pearson |
|
value: 83.78671249317563 |
|
- type: euclidean_spearman |
|
value: 83.4219042133061 |
|
- type: manhattan_pearson |
|
value: 83.44085827249334 |
|
- type: manhattan_spearman |
|
value: 83.02901331535297 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: 56a6d0140cf6356659e2a7c1413286a774468d44 |
|
metrics: |
|
- type: map |
|
value: 88.65396986895777 |
|
- type: mrr |
|
value: 96.60209525405604 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scifact |
|
name: MTEB SciFact |
|
config: default |
|
split: test |
|
revision: a75ae049398addde9b70f6b268875f5cbce99089 |
|
metrics: |
|
- type: map_at_1 |
|
value: 51.456 |
|
- type: map_at_10 |
|
value: 60.827 |
|
- type: map_at_100 |
|
value: 61.595 |
|
- type: map_at_1000 |
|
value: 61.629999999999995 |
|
- type: map_at_3 |
|
value: 57.518 |
|
- type: map_at_5 |
|
value: 59.435 |
|
- type: ndcg_at_1 |
|
value: 53.333 |
|
- type: ndcg_at_10 |
|
value: 65.57 |
|
- type: ndcg_at_100 |
|
value: 68.911 |
|
- type: ndcg_at_1000 |
|
value: 69.65299999999999 |
|
- type: ndcg_at_3 |
|
value: 60.009 |
|
- type: ndcg_at_5 |
|
value: 62.803 |
|
- type: precision_at_1 |
|
value: 53.333 |
|
- type: precision_at_10 |
|
value: 8.933 |
|
- type: precision_at_100 |
|
value: 1.0699999999999998 |
|
- type: precision_at_1000 |
|
value: 0.11299999999999999 |
|
- type: precision_at_3 |
|
value: 23.333000000000002 |
|
- type: precision_at_5 |
|
value: 15.8 |
|
- type: recall_at_1 |
|
value: 51.456 |
|
- type: recall_at_10 |
|
value: 79.011 |
|
- type: recall_at_100 |
|
value: 94.167 |
|
- type: recall_at_1000 |
|
value: 99.667 |
|
- type: recall_at_3 |
|
value: 64.506 |
|
- type: recall_at_5 |
|
value: 71.211 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/sprintduplicatequestions-pairclassification |
|
name: MTEB SprintDuplicateQuestions |
|
config: default |
|
split: test |
|
revision: 5a8256d0dff9c4bd3be3ba3e67e4e70173f802ea |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 99.65940594059406 |
|
- type: cos_sim_ap |
|
value: 90.1455141683116 |
|
- type: cos_sim_f1 |
|
value: 82.26044226044226 |
|
- type: cos_sim_precision |
|
value: 80.8695652173913 |
|
- type: cos_sim_recall |
|
value: 83.7 |
|
- type: dot_accuracy |
|
value: 99.65940594059406 |
|
- type: dot_ap |
|
value: 90.1455141683116 |
|
- type: dot_f1 |
|
value: 82.26044226044226 |
|
- type: dot_precision |
|
value: 80.8695652173913 |
|
- type: dot_recall |
|
value: 83.7 |
|
- type: euclidean_accuracy |
|
value: 99.65940594059406 |
|
- type: euclidean_ap |
|
value: 90.14551416831162 |
|
- type: euclidean_f1 |
|
value: 82.26044226044226 |
|
- type: euclidean_precision |
|
value: 80.8695652173913 |
|
- type: euclidean_recall |
|
value: 83.7 |
|
- type: manhattan_accuracy |
|
value: 99.64950495049504 |
|
- type: manhattan_ap |
|
value: 89.5492617367771 |
|
- type: manhattan_f1 |
|
value: 81.58280410356619 |
|
- type: manhattan_precision |
|
value: 79.75167144221585 |
|
- type: manhattan_recall |
|
value: 83.5 |
|
- type: max_accuracy |
|
value: 99.65940594059406 |
|
- type: max_ap |
|
value: 90.14551416831162 |
|
- type: max_f1 |
|
value: 82.26044226044226 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering |
|
name: MTEB StackExchangeClustering |
|
config: default |
|
split: test |
|
revision: 70a89468f6dccacc6aa2b12a6eac54e74328f235 |
|
metrics: |
|
- type: v_measure |
|
value: 53.80048409076929 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering-p2p |
|
name: MTEB StackExchangeClusteringP2P |
|
config: default |
|
split: test |
|
revision: d88009ab563dd0b16cfaf4436abaf97fa3550cf0 |
|
metrics: |
|
- type: v_measure |
|
value: 34.280269334397545 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: ef807ea29a75ec4f91b50fd4191cb4ee4589a9f9 |
|
metrics: |
|
- type: map |
|
value: 51.97907654945493 |
|
- type: mrr |
|
value: 52.82873376623376 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: 8753c2788d36c01fc6f05d03fe3f7268d63f9122 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 28.364293841556304 |
|
- type: cos_sim_spearman |
|
value: 27.485869639926136 |
|
- type: dot_pearson |
|
value: 28.364295910221145 |
|
- type: dot_spearman |
|
value: 27.485869639926136 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: trec-covid |
|
name: MTEB TRECCOVID |
|
config: default |
|
split: test |
|
revision: 2c8041b2c07a79b6f7ba8fe6acc72e5d9f92d217 |
|
metrics: |
|
- type: map_at_1 |
|
value: 0.19499999999999998 |
|
- type: map_at_10 |
|
value: 1.218 |
|
- type: map_at_100 |
|
value: 7.061000000000001 |
|
- type: map_at_1000 |
|
value: 19.735 |
|
- type: map_at_3 |
|
value: 0.46499999999999997 |
|
- type: map_at_5 |
|
value: 0.672 |
|
- type: ndcg_at_1 |
|
value: 60.0 |
|
- type: ndcg_at_10 |
|
value: 51.32600000000001 |
|
- type: ndcg_at_100 |
|
value: 41.74 |
|
- type: ndcg_at_1000 |
|
value: 43.221 |
|
- type: ndcg_at_3 |
|
value: 54.989 |
|
- type: ndcg_at_5 |
|
value: 52.905 |
|
- type: precision_at_1 |
|
value: 66.0 |
|
- type: precision_at_10 |
|
value: 55.60000000000001 |
|
- type: precision_at_100 |
|
value: 43.34 |
|
- type: precision_at_1000 |
|
value: 19.994 |
|
- type: precision_at_3 |
|
value: 59.333000000000006 |
|
- type: precision_at_5 |
|
value: 57.199999999999996 |
|
- type: recall_at_1 |
|
value: 0.19499999999999998 |
|
- type: recall_at_10 |
|
value: 1.473 |
|
- type: recall_at_100 |
|
value: 10.596 |
|
- type: recall_at_1000 |
|
value: 42.466 |
|
- type: recall_at_3 |
|
value: 0.49899999999999994 |
|
- type: recall_at_5 |
|
value: 0.76 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: webis-touche2020 |
|
name: MTEB Touche2020 |
|
config: default |
|
split: test |
|
revision: 527b7d77e16e343303e68cb6af11d6e18b9f7b3b |
|
metrics: |
|
- type: map_at_1 |
|
value: 1.997 |
|
- type: map_at_10 |
|
value: 7.5569999999999995 |
|
- type: map_at_100 |
|
value: 12.238 |
|
- type: map_at_1000 |
|
value: 13.773 |
|
- type: map_at_3 |
|
value: 4.334 |
|
- type: map_at_5 |
|
value: 5.5 |
|
- type: ndcg_at_1 |
|
value: 22.448999999999998 |
|
- type: ndcg_at_10 |
|
value: 19.933999999999997 |
|
- type: ndcg_at_100 |
|
value: 30.525999999999996 |
|
- type: ndcg_at_1000 |
|
value: 43.147999999999996 |
|
- type: ndcg_at_3 |
|
value: 22.283 |
|
- type: ndcg_at_5 |
|
value: 21.224 |
|
- type: precision_at_1 |
|
value: 24.490000000000002 |
|
- type: precision_at_10 |
|
value: 17.551 |
|
- type: precision_at_100 |
|
value: 6.4079999999999995 |
|
- type: precision_at_1000 |
|
value: 1.463 |
|
- type: precision_at_3 |
|
value: 23.128999999999998 |
|
- type: precision_at_5 |
|
value: 20.816000000000003 |
|
- type: recall_at_1 |
|
value: 1.997 |
|
- type: recall_at_10 |
|
value: 13.001999999999999 |
|
- type: recall_at_100 |
|
value: 40.98 |
|
- type: recall_at_1000 |
|
value: 79.40899999999999 |
|
- type: recall_at_3 |
|
value: 5.380999999999999 |
|
- type: recall_at_5 |
|
value: 7.721 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/toxic_conversations_50k |
|
name: MTEB ToxicConversationsClassification |
|
config: default |
|
split: test |
|
revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de |
|
metrics: |
|
- type: accuracy |
|
value: 60.861200000000004 |
|
- type: ap |
|
value: 11.39641747026629 |
|
- type: f1 |
|
value: 47.80230380517065 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/tweet_sentiment_extraction |
|
name: MTEB TweetSentimentExtractionClassification |
|
config: default |
|
split: test |
|
revision: 62146448f05be9e52a36b8ee9936447ea787eede |
|
metrics: |
|
- type: accuracy |
|
value: 55.464063384267114 |
|
- type: f1 |
|
value: 55.759039643764666 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/twentynewsgroups-clustering |
|
name: MTEB TwentyNewsgroupsClustering |
|
config: default |
|
split: test |
|
revision: 091a54f9a36281ce7d6590ec8c75dd485e7e01d4 |
|
metrics: |
|
- type: v_measure |
|
value: 49.74455348083809 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twittersemeval2015-pairclassification |
|
name: MTEB TwitterSemEval2015 |
|
config: default |
|
split: test |
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 86.07617571675507 |
|
- type: cos_sim_ap |
|
value: 73.85398650568216 |
|
- type: cos_sim_f1 |
|
value: 68.50702798531087 |
|
- type: cos_sim_precision |
|
value: 65.86316045775506 |
|
- type: cos_sim_recall |
|
value: 71.37203166226914 |
|
- type: dot_accuracy |
|
value: 86.07617571675507 |
|
- type: dot_ap |
|
value: 73.85398346238429 |
|
- type: dot_f1 |
|
value: 68.50702798531087 |
|
- type: dot_precision |
|
value: 65.86316045775506 |
|
- type: dot_recall |
|
value: 71.37203166226914 |
|
- type: euclidean_accuracy |
|
value: 86.07617571675507 |
|
- type: euclidean_ap |
|
value: 73.85398625060357 |
|
- type: euclidean_f1 |
|
value: 68.50702798531087 |
|
- type: euclidean_precision |
|
value: 65.86316045775506 |
|
- type: euclidean_recall |
|
value: 71.37203166226914 |
|
- type: manhattan_accuracy |
|
value: 85.98676759849795 |
|
- type: manhattan_ap |
|
value: 73.86874126878737 |
|
- type: manhattan_f1 |
|
value: 68.55096559662361 |
|
- type: manhattan_precision |
|
value: 66.51774633904195 |
|
- type: manhattan_recall |
|
value: 70.71240105540898 |
|
- type: max_accuracy |
|
value: 86.07617571675507 |
|
- type: max_ap |
|
value: 73.86874126878737 |
|
- type: max_f1 |
|
value: 68.55096559662361 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twitterurlcorpus-pairclassification |
|
name: MTEB TwitterURLCorpus |
|
config: default |
|
split: test |
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 88.51631932316529 |
|
- type: cos_sim_ap |
|
value: 85.10831084479727 |
|
- type: cos_sim_f1 |
|
value: 77.14563397129186 |
|
- type: cos_sim_precision |
|
value: 74.9709386806161 |
|
- type: cos_sim_recall |
|
value: 79.45026178010471 |
|
- type: dot_accuracy |
|
value: 88.51631932316529 |
|
- type: dot_ap |
|
value: 85.10831188797107 |
|
- type: dot_f1 |
|
value: 77.14563397129186 |
|
- type: dot_precision |
|
value: 74.9709386806161 |
|
- type: dot_recall |
|
value: 79.45026178010471 |
|
- type: euclidean_accuracy |
|
value: 88.51631932316529 |
|
- type: euclidean_ap |
|
value: 85.10829618408616 |
|
- type: euclidean_f1 |
|
value: 77.14563397129186 |
|
- type: euclidean_precision |
|
value: 74.9709386806161 |
|
- type: euclidean_recall |
|
value: 79.45026178010471 |
|
- type: manhattan_accuracy |
|
value: 88.50467652423643 |
|
- type: manhattan_ap |
|
value: 85.08329502055064 |
|
- type: manhattan_f1 |
|
value: 77.11157455683002 |
|
- type: manhattan_precision |
|
value: 74.67541834968263 |
|
- type: manhattan_recall |
|
value: 79.71204188481676 |
|
- type: max_accuracy |
|
value: 88.51631932316529 |
|
- type: max_ap |
|
value: 85.10831188797107 |
|
- type: max_f1 |
|
value: 77.14563397129186 |
|
--- |
|
|
|
|
|
# all-mpnet-base-v2 |
|
This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search. |
|
|
|
## Usage (Sentence-Transformers) |
|
Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed: |
|
|
|
``` |
|
pip install -U sentence-transformers |
|
``` |
|
|
|
Then you can use the model like this: |
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
sentences = ["This is an example sentence", "Each sentence is converted"] |
|
|
|
model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2') |
|
embeddings = model.encode(sentences) |
|
print(embeddings) |
|
``` |
|
|
|
## Usage (HuggingFace Transformers) |
|
Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings. |
|
|
|
```python |
|
from transformers import AutoTokenizer, AutoModel |
|
import torch |
|
import torch.nn.functional as F |
|
|
|
#Mean Pooling - Take attention mask into account for correct averaging |
|
def mean_pooling(model_output, attention_mask): |
|
token_embeddings = model_output[0] #First element of model_output contains all token embeddings |
|
input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() |
|
return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) |
|
|
|
|
|
# Sentences we want sentence embeddings for |
|
sentences = ['This is an example sentence', 'Each sentence is converted'] |
|
|
|
# Load model from HuggingFace Hub |
|
tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-mpnet-base-v2') |
|
model = AutoModel.from_pretrained('sentence-transformers/all-mpnet-base-v2') |
|
|
|
# Tokenize sentences |
|
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt') |
|
|
|
# Compute token embeddings |
|
with torch.no_grad(): |
|
model_output = model(**encoded_input) |
|
|
|
# Perform pooling |
|
sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) |
|
|
|
# Normalize embeddings |
|
sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) |
|
|
|
print("Sentence embeddings:") |
|
print(sentence_embeddings) |
|
``` |
|
|
|
## Evaluation Results |
|
|
|
For an automated evaluation of this model, see *MTEB*: https://huggingface.co/spaces/mteb/leaderboard or the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name=sentence-transformers/all-MiniLM-L12-v2) |
|
|
|
------ |
|
|
|
## Background |
|
|
|
The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised |
|
contrastive learning objective. We used the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model and fine-tuned in on a |
|
1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset. |
|
|
|
We developped this model during the |
|
[Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), |
|
organized by Hugging Face. We developped this model as part of the project: |
|
[Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks. |
|
|
|
## Intended uses |
|
|
|
Our model is intented to be used as a sentence and short paragraph encoder. Given an input text, it ouptuts a vector which captures |
|
the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks. |
|
|
|
By default, input text longer than 384 word pieces is truncated. |
|
|
|
|
|
## Training procedure |
|
|
|
### Pre-training |
|
|
|
We use the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model. Please refer to the model card for more detailed information about the pre-training procedure. |
|
|
|
### Fine-tuning |
|
|
|
We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch. |
|
We then apply the cross entropy loss by comparing with true pairs. |
|
|
|
#### Hyper parameters |
|
|
|
We trained ou model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core). |
|
We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with |
|
a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`. |
|
|
|
#### Training data |
|
|
|
We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences. |
|
We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file. |
|
|
|
|
|
| Dataset | Paper | Number of training tuples | |
|
|--------------------------------------------------------|:----------------------------------------:|:--------------------------:| |
|
| [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 | |
|
| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 | |
|
| [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 | |
|
| [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 | |
|
| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 | |
|
| [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs | - | 25,316,456 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs | - | 21,396,559 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs | - | 21,396,559 | |
|
| [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 | |
|
| [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 | |
|
| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 | |
|
| [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 | |
|
| [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395| |
|
| [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 | |
|
| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 | |
|
| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 | |
|
| [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 | |
|
| [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 | |
|
| [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 | |
|
| AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 | |
|
| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 | |
|
| [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 | |
|
| [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 | |
|
| [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 | |
|
| [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 | |
|
| [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 | |
|
| [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 | |
|
| [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 | |
|
| [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 | |
|
| **Total** | | **1,170,060,424** | |
