diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,5 +1,6886 @@ ---- -license: other -license_name: amazon-service-terms -license_link: https://aws.amazon.com/service-terms/ ---- +--- +license: other +license_name: amazon-service-terms +license_link: https://aws.amazon.com/service-terms/ +language: +- en +- fr +- de +- es +- ja +- zh +- hi +- ar +- it +- pt +- sv +- ko +- he +- cs +- tr +- tl +- ru +- nl +- pl +- ta +- mr +- ml +- te +- kn +- vi +- id +- fa +- hu +- el +- ro +- da +- th +- fi +- sk +- uk +- 'no' +- bg +- ca +- sr +- hr +- lt +- sl +- et +- la +- bn +- lv +- ms +- bs +- sq +- az +- gl +- is +- ka +- mk +- eu +- hy +- ne +- ur +- kk +- mn +- be +- uz +- km +- nn +- gu +- my +- cy +- eo +- si +- tt +- sw +- af +- ga +- pa +- ku +- ky +- tg +- or +- lo +- fo +- mt +- so +- lb +- am +- oc +- jv +- ha +- ps +- sa +- fy +- mg +- as +- ba +- br +- tk +- co +- dv +- rw +- ht +- yi +- sd +- zu +- gd +- bo +- ug +- mi +- rm +- xh +- su +- yo +tags: +- mteb +model-index: +- name: Titan-text-embeddings-v2 + results: + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en) + config: en + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 79.31343283582089 + - type: ap + value: 43.9465851246623 + - type: f1 + value: 73.6131343594374 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (de) + config: de + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 70.94218415417559 + - type: ap + value: 82.30115528468109 + - type: f1 + value: 69.37963699148699 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en-ext) + config: en-ext + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 82.29385307346327 + - type: ap + value: 29.956638709449372 + - type: f1 + value: 68.88158061498754 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (ja) + config: ja + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 80.06423982869379 + - type: ap + value: 25.2439835379337 + - type: f1 + value: 65.53837311569734 + - task: + type: Classification + dataset: + type: mteb/amazon_polarity + name: MTEB AmazonPolarityClassification + config: default + split: test + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + metrics: + - type: accuracy + value: 76.66435 + - type: ap + value: 70.76988138513991 + - type: f1 + value: 76.54117595647566 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (en) + config: en + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 35.276 + - type: f1 + value: 34.90637768461089 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (de) + config: de + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 38.826 + - type: f1 + value: 37.71339372044998 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (es) + config: es + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 39.385999999999996 + - type: f1 + value: 38.24347249789392 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (fr) + config: fr + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 39.472 + - type: f1 + value: 38.37157729490788 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (ja) + config: ja + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 35.897999999999996 + - type: f1 + value: 35.187204289589346 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (zh) + config: zh + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 36.068 + - type: f1 + value: 35.042441064207175 + - task: + type: Retrieval + dataset: + type: arguana + name: MTEB ArguAna + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 27.027 + - type: map_at_10 + value: 42.617 + - type: map_at_100 + value: 43.686 + - type: map_at_1000 + value: 43.695 + - type: map_at_3 + value: 37.684 + - type: map_at_5 + value: 40.532000000000004 + - type: mrr_at_1 + value: 27.667 + - type: mrr_at_10 + value: 42.88 + - type: mrr_at_100 + value: 43.929 + - type: mrr_at_1000 + value: 43.938 + - type: mrr_at_3 + value: 37.933 + - type: mrr_at_5 + value: 40.774 + - type: ndcg_at_1 + value: 27.027 + - type: ndcg_at_10 + value: 51.312000000000005 + - type: ndcg_at_100 + value: 55.696 + - type: ndcg_at_1000 + value: 55.896 + - type: ndcg_at_3 + value: 41.124 + - type: ndcg_at_5 + value: 46.283 + - type: precision_at_1 + value: 27.027 + - type: precision_at_10 + value: 7.9159999999999995 + - type: precision_at_100 + value: 0.979 + - type: precision_at_1000 + value: 0.099 + - type: precision_at_3 + value: 17.022000000000002 + - type: precision_at_5 + value: 12.731 + - type: recall_at_1 + value: 27.027 + - type: recall_at_10 + value: 79.161 + - type: recall_at_100 + value: 97.937 + - type: recall_at_1000 + value: 99.431 + - type: recall_at_3 + value: 51.06699999999999 + - type: recall_at_5 + value: 63.656 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-p2p + name: MTEB ArxivClusteringP2P + config: default + split: test + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + metrics: + - type: v_measure + value: 41.775131599226874 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-s2s + name: MTEB ArxivClusteringS2S + config: default + split: test + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + metrics: + - type: v_measure + value: 34.134214263072494 + - task: + type: Reranking + dataset: + type: mteb/askubuntudupquestions-reranking + name: MTEB AskUbuntuDupQuestions + config: default + split: test + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + metrics: + - type: map + value: 63.2885651257187 + - type: mrr + value: 76.37712702809655 + - task: + type: STS + dataset: + type: mteb/biosses-sts + name: MTEB BIOSSES + config: default + split: test + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + metrics: + - type: cos_sim_pearson + value: 89.53738990667027 + - type: cos_sim_spearman + value: 87.13210584606783 + - type: euclidean_pearson + value: 87.33265405736388 + - type: euclidean_spearman + value: 87.18632394893399 + - type: manhattan_pearson + value: 87.33673166528312 + - type: manhattan_spearman + value: 86.9736685010257 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (de-en) + config: de-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 98.32985386221294 + - type: f1 + value: 98.18371607515658 + - type: precision + value: 98.1106471816284 + - type: recall + value: 98.32985386221294 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (fr-en) + config: fr-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 98.20603125687872 + - type: f1 + value: 98.04461075647515 + - type: precision + value: 97.96390050627338 + - type: recall + value: 98.20603125687872 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (ru-en) + config: ru-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 94.8874263941808 + - type: f1 + value: 94.57568410114305 + - type: precision + value: 94.42096755570951 + - type: recall + value: 94.8874263941808 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (zh-en) + config: zh-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 96.78778304370721 + - type: f1 + value: 96.75267684746358 + - type: precision + value: 96.73512374934175 + - type: recall + value: 96.78778304370721 + - task: + type: Classification + dataset: + type: mteb/banking77 + name: MTEB Banking77Classification + config: default + split: test + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + metrics: + - type: accuracy + value: 84.3051948051948 + - type: f1 + value: 83.97876601554812 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-p2p + name: MTEB BiorxivClusteringP2P + config: default + split: test + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + metrics: + - type: v_measure + value: 35.005716163806575 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-s2s + name: MTEB BiorxivClusteringS2S + config: default + split: test + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + metrics: + - type: v_measure + value: 30.999141295578852 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackAndroidRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 36.153 + - type: map_at_10 + value: 48.742000000000004 + - type: map_at_100 + value: 50.253 + - type: map_at_1000 + value: 50.373999999999995 + - type: map_at_3 + value: 45.089 + - type: map_at_5 + value: 47.08 + - type: mrr_at_1 + value: 44.635000000000005 + - type: mrr_at_10 + value: 54.715 + - type: mrr_at_100 + value: 55.300000000000004 + - type: mrr_at_1000 + value: 55.337 + - type: mrr_at_3 + value: 52.527 + - type: mrr_at_5 + value: 53.76499999999999 + - type: ndcg_at_1 + value: 44.635000000000005 + - type: ndcg_at_10 + value: 55.31 + - type: ndcg_at_100 + value: 60.084 + - type: ndcg_at_1000 + value: 61.645 + - type: ndcg_at_3 + value: 50.876999999999995 + - type: ndcg_at_5 + value: 52.764 + - type: precision_at_1 + value: 44.635000000000005 + - type: precision_at_10 + value: 10.687000000000001 + - type: precision_at_100 + value: 1.66 + - type: precision_at_1000 + value: 0.212 + - type: precision_at_3 + value: 24.94 + - type: precision_at_5 + value: 17.596999999999998 + - type: recall_at_1 + value: 36.153 + - type: recall_at_10 + value: 67.308 + - type: recall_at_100 + value: 87.199 + - type: recall_at_1000 + value: 96.904 + - type: recall_at_3 + value: 53.466 + - type: recall_at_5 + value: 59.512 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackEnglishRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 32.0 + - type: map_at_10 + value: 43.646 + - type: map_at_100 + value: 44.933 + - type: map_at_1000 + value: 45.049 + - type: map_at_3 + value: 40.333999999999996 + - type: map_at_5 + value: 42.108000000000004 + - type: mrr_at_1 + value: 40.382 + - type: mrr_at_10 + value: 49.738 + - type: mrr_at_100 + value: 50.331 + - type: mrr_at_1000 + value: 50.364 + - type: mrr_at_3 + value: 47.442 + - type: mrr_at_5 + value: 48.719 + - type: ndcg_at_1 + value: 40.382 + - type: ndcg_at_10 + value: 49.808 + - type: ndcg_at_100 + value: 54.053 + - type: ndcg_at_1000 + value: 55.753 + - type: ndcg_at_3 + value: 45.355000000000004 + - type: ndcg_at_5 + value: 47.215 + - type: precision_at_1 + value: 40.382 + - type: precision_at_10 + value: 9.58 + - type: precision_at_100 + value: 1.488 + - type: precision_at_1000 + value: 0.192 + - type: precision_at_3 + value: 22.272 + - type: precision_at_5 + value: 15.604999999999999 + - type: recall_at_1 + value: 32.0 + - type: recall_at_10 + value: 60.839 + - type: recall_at_100 + value: 78.869 + - type: recall_at_1000 + value: 89.384 + - type: recall_at_3 + value: 47.226 + - type: recall_at_5 + value: 52.864 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGamingRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 44.084 + - type: map_at_10 + value: 56.591 + - type: map_at_100 + value: 57.533 + - type: map_at_1000 + value: 57.583 + - type: map_at_3 + value: 53.356 + - type: map_at_5 + value: 55.236 + - type: mrr_at_1 + value: 50.532999999999994 + - type: mrr_at_10 + value: 59.974000000000004 + - type: mrr_at_100 + value: 60.557 + - type: mrr_at_1000 + value: 60.584 + - type: mrr_at_3 + value: 57.774 + - type: mrr_at_5 + value: 59.063 + - type: ndcg_at_1 + value: 50.532999999999994 + - type: ndcg_at_10 + value: 62.265 + - type: ndcg_at_100 + value: 65.78 + - type: ndcg_at_1000 + value: 66.76299999999999 + - type: ndcg_at_3 + value: 57.154 + - type: ndcg_at_5 + value: 59.708000000000006 + - type: precision_at_1 + value: 50.532999999999994 + - type: precision_at_10 + value: 9.85 + - type: precision_at_100 + value: 1.247 + - type: precision_at_1000 + value: 0.13699999999999998 + - type: precision_at_3 + value: 25.434 + - type: precision_at_5 + value: 17.279 + - type: recall_at_1 + value: 44.084 + - type: recall_at_10 + value: 75.576 + - type: recall_at_100 + value: 90.524 + - type: recall_at_1000 + value: 97.38799999999999 + - type: recall_at_3 + value: 61.792 + - type: recall_at_5 + value: 68.112 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGisRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 29.203000000000003 + - type: map_at_10 + value: 38.078 + - type: map_at_100 + value: 39.144 + - type: map_at_1000 + value: 39.222 + - type: map_at_3 + value: 35.278999999999996 + - type: map_at_5 + value: 36.812 + - type: mrr_at_1 + value: 31.299 + - type: mrr_at_10 + value: 39.879 + - type: mrr_at_100 + value: 40.832 + - type: mrr_at_1000 + value: 40.891 + - type: mrr_at_3 + value: 37.513999999999996 + - type: mrr_at_5 + value: 38.802 + - type: ndcg_at_1 + value: 31.299 + - type: ndcg_at_10 + value: 43.047999999999995 + - type: ndcg_at_100 + value: 48.101 + - type: ndcg_at_1000 + value: 49.958999999999996 + - type: ndcg_at_3 + value: 37.778 + - type: ndcg_at_5 + value: 40.257 + - type: precision_at_1 + value: 31.299 + - type: precision_at_10 + value: 6.508 + - type: precision_at_100 + value: 0.9530000000000001 + - type: precision_at_1000 + value: 0.11399999999999999 + - type: precision_at_3 + value: 15.744 + - type: precision_at_5 + value: 10.893 + - type: recall_at_1 + value: 29.203000000000003 + - type: recall_at_10 + value: 56.552 + - type: recall_at_100 + value: 79.21000000000001 + - type: recall_at_1000 + value: 92.884 + - type: recall_at_3 + value: 42.441 + - type: recall_at_5 + value: 48.399 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackMathematicaRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 19.029 + - type: map_at_10 + value: 28.410000000000004 + - type: map_at_100 + value: 29.773 + - type: map_at_1000 + value: 29.887000000000004 + - type: map_at_3 + value: 25.374000000000002 + - type: map_at_5 + value: 27.162 + - type: mrr_at_1 + value: 23.632 + - type: mrr_at_10 + value: 33.0 + - type: mrr_at_100 + value: 34.043 + - type: mrr_at_1000 + value: 34.105999999999995 + - type: mrr_at_3 + value: 30.245 + - type: mrr_at_5 + value: 31.830000000000002 + - type: ndcg_at_1 + value: 23.632 + - type: ndcg_at_10 + value: 34.192 + - type: ndcg_at_100 + value: 40.29 + - type: ndcg_at_1000 + value: 42.753 + - type: ndcg_at_3 + value: 28.811999999999998 + - type: ndcg_at_5 + value: 31.46 + - type: precision_at_1 + value: 23.632 + - type: precision_at_10 + value: 6.455 + - type: precision_at_100 + value: 1.095 + - type: precision_at_1000 + value: 0.14200000000000002 + - type: precision_at_3 + value: 14.096 + - type: precision_at_5 + value: 10.448 + - type: recall_at_1 + value: 19.029 + - type: recall_at_10 + value: 47.278999999999996 + - type: recall_at_100 + value: 72.977 + - type: recall_at_1000 + value: 90.17699999999999 + - type: recall_at_3 + value: 32.519 + - type: recall_at_5 + value: 39.156 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackPhysicsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 30.983 + - type: map_at_10 + value: 42.595 + - type: map_at_100 + value: 43.906 + - type: map_at_1000 + value: 44.001000000000005 + - type: map_at_3 + value: 39.245000000000005 + - type: map_at_5 + value: 41.14 + - type: mrr_at_1 + value: 38.114 + - type: mrr_at_10 + value: 48.181000000000004 + - type: mrr_at_100 + value: 48.935 + - type: mrr_at_1000 + value: 48.972 + - type: mrr_at_3 + value: 45.877 + - type: mrr_at_5 + value: 47.249 + - type: ndcg_at_1 + value: 38.114 + - type: ndcg_at_10 + value: 48.793 + - type: ndcg_at_100 + value: 54.001999999999995 + - type: ndcg_at_1000 + value: 55.749 + - type: ndcg_at_3 + value: 43.875 + - type: ndcg_at_5 + value: 46.23 + - type: precision_at_1 + value: 38.114 + - type: precision_at_10 + value: 8.98 + - type: precision_at_100 + value: 1.3390000000000002 + - type: precision_at_1000 + value: 0.166 + - type: precision_at_3 + value: 21.303 + - type: precision_at_5 + value: 15.072 + - type: recall_at_1 + value: 30.983 + - type: recall_at_10 + value: 61.47 + - type: recall_at_100 + value: 83.14399999999999 + - type: recall_at_1000 + value: 94.589 + - type: recall_at_3 + value: 47.019 + - type: recall_at_5 + value: 53.445 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackProgrammersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 29.707 + - type: map_at_10 + value: 40.900999999999996 + - type: map_at_100 + value: 42.369 + - type: map_at_1000 + value: 42.455 + - type: map_at_3 + value: 37.416 + - type: map_at_5 + value: 39.483000000000004 + - type: mrr_at_1 + value: 36.301 + - type: mrr_at_10 + value: 46.046 + - type: mrr_at_100 + value: 46.922999999999995 + - type: mrr_at_1000 + value: 46.964 + - type: mrr_at_3 + value: 43.436 + - type: mrr_at_5 + value: 45.04 + - type: ndcg_at_1 + value: 36.301 + - type: ndcg_at_10 + value: 46.955999999999996 + - type: ndcg_at_100 + value: 52.712 + - type: ndcg_at_1000 + value: 54.447 + - type: ndcg_at_3 + value: 41.643 + - type: ndcg_at_5 + value: 44.305 + - type: precision_at_1 + value: 36.301 + - type: precision_at_10 + value: 8.607 + - type: precision_at_100 + value: 1.34 + - type: precision_at_1000 + value: 0.164 + - type: precision_at_3 + value: 19.901 + - type: precision_at_5 + value: 14.429 + - type: recall_at_1 + value: 29.707 + - type: recall_at_10 + value: 59.559 + - type: recall_at_100 + value: 83.60499999999999 + - type: recall_at_1000 + value: 95.291 + - type: recall_at_3 + value: 44.774 + - type: recall_at_5 + value: 51.67 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 29.455416666666668 + - type: map_at_10 + value: 39.61333333333334 + - type: map_at_100 + value: 40.85875 + - type: map_at_1000 + value: 40.96791666666667 + - type: map_at_3 + value: 36.48874999999999 + - type: map_at_5 + value: 38.24341666666667 + - type: mrr_at_1 + value: 34.80258333333334 + - type: mrr_at_10 + value: 43.783 + - type: mrr_at_100 + value: 44.591833333333334 + - type: mrr_at_1000 + value: 44.64208333333333 + - type: mrr_at_3 + value: 41.38974999999999 + - type: mrr_at_5 + value: 42.74566666666667 + - type: ndcg_at_1 + value: 34.80258333333334 + - type: ndcg_at_10 + value: 45.2705 + - type: ndcg_at_100 + value: 50.31224999999999 + - type: ndcg_at_1000 + value: 52.27916666666667 + - type: ndcg_at_3 + value: 40.2745 + - type: ndcg_at_5 + value: 42.61575 + - type: precision_at_1 + value: 34.80258333333334 + - type: precision_at_10 + value: 7.97075 + - type: precision_at_100 + value: 1.2400000000000002 + - type: precision_at_1000 + value: 0.1595 + - type: precision_at_3 + value: 18.627583333333337 + - type: precision_at_5 + value: 13.207000000000003 + - type: recall_at_1 + value: 29.455416666666668 + - type: recall_at_10 + value: 57.66091666666665 + - type: recall_at_100 + value: 79.51966666666665 + - type: recall_at_1000 + value: 93.01883333333333 + - type: recall_at_3 + value: 43.580416666666665 + - type: recall_at_5 + value: 49.7025 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackStatsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 27.569 + - type: map_at_10 + value: 34.73 + - type: map_at_100 + value: 35.708 + - type: map_at_1000 + value: 35.808 + - type: map_at_3 + value: 32.62 + - type: map_at_5 + value: 33.556999999999995 + - type: mrr_at_1 + value: 31.135 + - type: mrr_at_10 + value: 37.833 + - type: mrr_at_100 + value: 38.68 + - type: mrr_at_1000 + value: 38.749 + - type: mrr_at_3 + value: 35.915 + - type: mrr_at_5 + value: 36.751 + - type: ndcg_at_1 + value: 31.135 + - type: ndcg_at_10 + value: 39.047 + - type: ndcg_at_100 + value: 43.822 + - type: ndcg_at_1000 + value: 46.249 + - type: ndcg_at_3 + value: 35.115 + - type: ndcg_at_5 + value: 36.49 + - type: precision_at_1 + value: 31.135 + - type: precision_at_10 + value: 6.058 + - type: precision_at_100 + value: 0.923 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_3 + value: 15.031 + - type: precision_at_5 + value: 10.030999999999999 + - type: recall_at_1 + value: 27.569 + - type: recall_at_10 + value: 49.332 + - type: recall_at_100 + value: 70.967 + - type: recall_at_1000 + value: 88.876 + - type: recall_at_3 + value: 37.858999999999995 + - type: recall_at_5 + value: 41.589 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackTexRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 19.677 + - type: map_at_10 + value: 28.097 + - type: map_at_100 + value: 29.24 + - type: map_at_1000 + value: 29.365000000000002 + - type: map_at_3 + value: 25.566 + - type: map_at_5 + value: 26.852999999999998 + - type: mrr_at_1 + value: 23.882 + - type: mrr_at_10 + value: 31.851000000000003 + - type: mrr_at_100 + value: 32.757 + - type: mrr_at_1000 + value: 32.83 + - type: mrr_at_3 + value: 29.485 + - type: mrr_at_5 + value: 30.744 + - type: ndcg_at_1 + value: 23.882 + - type: ndcg_at_10 + value: 33.154 + - type: ndcg_at_100 + value: 38.491 + - type: ndcg_at_1000 + value: 41.274 + - type: ndcg_at_3 + value: 28.648 + - type: ndcg_at_5 + value: 30.519000000000002 + - type: precision_at_1 + value: 23.882 + - type: precision_at_10 + value: 6.117999999999999 + - type: precision_at_100 + value: 1.0330000000000001 + - type: precision_at_1000 + value: 0.145 + - type: precision_at_3 + value: 13.73 + - type: precision_at_5 + value: 9.794 + - type: recall_at_1 + value: 19.677 + - type: recall_at_10 + value: 44.444 + - type: recall_at_100 + value: 68.477 + - type: recall_at_1000 + value: 88.23 + - type: recall_at_3 + value: 31.708 + - type: recall_at_5 + value: 36.599 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackUnixRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 30.489 + - type: map_at_10 + value: 40.883 + - type: map_at_100 + value: 42.058 + - type: map_at_1000 + value: 42.152 + - type: map_at_3 + value: 37.525999999999996 + - type: map_at_5 + value: 39.753 + - type: mrr_at_1 + value: 35.541 + - type: mrr_at_10 + value: 44.842999999999996 + - type: mrr_at_100 + value: 45.673 + - type: mrr_at_1000 + value: 45.723 + - type: mrr_at_3 + value: 42.397 + - type: mrr_at_5 + value: 43.937 + - type: ndcg_at_1 + value: 35.541 + - type: ndcg_at_10 + value: 46.504 + - type: ndcg_at_100 + value: 51.637 + - type: ndcg_at_1000 + value: 53.535 + - type: ndcg_at_3 + value: 41.127 + - type: ndcg_at_5 + value: 44.17 + - type: precision_at_1 + value: 35.541 + - type: precision_at_10 + value: 7.864 + - type: precision_at_100 + value: 1.165 + - type: precision_at_1000 + value: 0.14300000000000002 + - type: precision_at_3 + value: 18.688 + - type: precision_at_5 + value: 13.507 + - type: recall_at_1 + value: 30.489 + - type: recall_at_10 + value: 59.378 + - type: recall_at_100 + value: 81.38300000000001 + - type: recall_at_1000 + value: 94.294 + - type: recall_at_3 + value: 44.946000000000005 + - type: recall_at_5 + value: 52.644999999999996 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWebmastersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 29.981 + - type: map_at_10 + value: 39.688 + - type: map_at_100 + value: 41.400999999999996 + - type: map_at_1000 + value: 41.634 + - type: map_at_3 + value: 36.047000000000004 + - type: map_at_5 + value: 38.064 + - type: mrr_at_1 + value: 35.375 + - type: mrr_at_10 + value: 44.169000000000004 + - type: mrr_at_100 + value: 45.07 + - type: mrr_at_1000 + value: 45.113 + - type: mrr_at_3 + value: 41.502 + - type: mrr_at_5 + value: 43.034 + - type: ndcg_at_1 + value: 35.375 + - type: ndcg_at_10 + value: 45.959 + - type: ndcg_at_100 + value: 51.688 + - type: ndcg_at_1000 + value: 53.714 + - type: ndcg_at_3 + value: 40.457 + - type: ndcg_at_5 + value: 43.08 + - type: precision_at_1 + value: 35.375 + - type: precision_at_10 + value: 8.953 + - type: precision_at_100 + value: 1.709 + - type: precision_at_1000 + value: 0.253 + - type: precision_at_3 + value: 18.775 + - type: precision_at_5 + value: 14.032 + - type: recall_at_1 + value: 29.981 + - type: recall_at_10 + value: 57.896 + - type: recall_at_100 + value: 83.438 + - type: recall_at_1000 + value: 95.608 + - type: recall_at_3 + value: 42.327 + - type: recall_at_5 + value: 49.069 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWordpressRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 24.59 + - type: map_at_10 + value: 32.999 + - type: map_at_100 + value: 33.987 + - type: map_at_1000 + value: 34.085 + - type: map_at_3 + value: 30.013 + - type: map_at_5 + value: 31.673000000000002 + - type: mrr_at_1 + value: 26.802 + - type: mrr_at_10 + value: 35.167 + - type: mrr_at_100 + value: 36.001 + - type: mrr_at_1000 + value: 36.071999999999996 + - type: mrr_at_3 + value: 32.562999999999995 + - type: mrr_at_5 + value: 34.014 + - type: ndcg_at_1 + value: 26.802 + - type: ndcg_at_10 + value: 38.21 + - type: ndcg_at_100 + value: 43.086999999999996 + - type: ndcg_at_1000 + value: 45.509 + - type: ndcg_at_3 + value: 32.452999999999996 + - type: ndcg_at_5 + value: 35.191 + - type: precision_at_1 + value: 26.802 + - type: precision_at_10 + value: 5.989 + - type: precision_at_100 + value: 0.928 + - type: precision_at_1000 + value: 0.125 + - type: precision_at_3 + value: 13.617 + - type: precision_at_5 + value: 9.797 + - type: recall_at_1 + value: 24.59 + - type: recall_at_10 + value: 52.298 + - type: recall_at_100 + value: 74.443 + - type: recall_at_1000 + value: 92.601 + - type: recall_at_3 + value: 36.888 + - type: recall_at_5 + value: 43.37 + - task: + type: Retrieval + dataset: + type: climate-fever + name: MTEB ClimateFEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 9.798 + - type: map_at_10 + value: 15.983 + - type: map_at_100 + value: 17.18 + - type: map_at_1000 + value: 17.329 + - type: map_at_3 + value: 13.594000000000001 + - type: map_at_5 + value: 14.984 + - type: mrr_at_1 + value: 21.564 + - type: mrr_at_10 + value: 31.415 + - type: mrr_at_100 + value: 32.317 + - type: mrr_at_1000 + value: 32.376 + - type: mrr_at_3 + value: 28.360000000000003 + - type: mrr_at_5 + value: 30.194 + - type: ndcg_at_1 + value: 21.564 + - type: ndcg_at_10 + value: 22.762 + - type: ndcg_at_100 + value: 28.199 + - type: ndcg_at_1000 + value: 31.284 + - type: ndcg_at_3 + value: 18.746 + - type: ndcg_at_5 + value: 20.434 + - type: precision_at_1 + value: 21.564 + - type: precision_at_10 + value: 6.755999999999999 + - type: precision_at_100 + value: 1.258 + - type: precision_at_1000 + value: 0.182 + - type: precision_at_3 + value: 13.507 + - type: precision_at_5 + value: 10.541 + - type: recall_at_1 + value: 9.798 + - type: recall_at_10 + value: 27.407999999999998 + - type: recall_at_100 + value: 46.659 + - type: recall_at_1000 + value: 64.132 + - type: recall_at_3 + value: 17.541999999999998 + - type: recall_at_5 + value: 22.137999999999998 + - task: + type: Retrieval + dataset: + type: dbpedia-entity + name: MTEB DBPedia + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 8.276 + - type: map_at_10 + value: 18.003 + - type: map_at_100 + value: 23.759 + - type: map_at_1000 + value: 25.105 + - type: map_at_3 + value: 13.812 + - type: map_at_5 + value: 15.659999999999998 + - type: mrr_at_1 + value: 63.0 + - type: mrr_at_10 + value: 71.812 + - type: mrr_at_100 + value: 72.205 + - type: mrr_at_1000 + value: 72.21300000000001 + - type: mrr_at_3 + value: 70.375 + - type: mrr_at_5 + value: 71.188 + - type: ndcg_at_1 + value: 50.5 + - type: ndcg_at_10 + value: 36.954 + - type: ndcg_at_100 + value: 40.083999999999996 + - type: ndcg_at_1000 + value: 47.661 + - type: ndcg_at_3 + value: 42.666 + - type: ndcg_at_5 + value: 39.581 + - type: precision_at_1 + value: 63.0 + - type: precision_at_10 + value: 28.249999999999996 + - type: precision_at_100 + value: 8.113 + - type: precision_at_1000 + value: 1.7149999999999999 + - type: precision_at_3 + value: 47.083000000000006 + - type: precision_at_5 + value: 38.65 + - type: recall_at_1 + value: 8.276 + - type: recall_at_10 + value: 23.177 + - type: recall_at_100 + value: 45.321 + - type: recall_at_1000 + value: 68.742 + - type: recall_at_3 + value: 15.473 + - type: recall_at_5 + value: 18.276 + - task: + type: Classification + dataset: + type: mteb/emotion + name: MTEB EmotionClassification + config: default + split: test + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + metrics: + - type: accuracy + value: 55.605000000000004 + - type: f1 + value: 49.86208997523934 + - task: + type: Retrieval + dataset: + type: fever + name: MTEB FEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 80.079 + - type: map_at_10 + value: 85.143 + - type: map_at_100 + value: 85.287 + - type: map_at_1000 + value: 85.297 + - type: map_at_3 + value: 84.533 + - type: map_at_5 + value: 84.953 + - type: mrr_at_1 + value: 86.424 + - type: mrr_at_10 + value: 91.145 + - type: mrr_at_100 + value: 91.212 + - type: mrr_at_1000 + value: 91.213 + - type: mrr_at_3 + value: 90.682 + - type: mrr_at_5 + value: 91.013 + - type: ndcg_at_1 + value: 86.424 + - type: ndcg_at_10 + value: 88.175 + - type: ndcg_at_100 + value: 88.77199999999999 + - type: ndcg_at_1000 + value: 88.967 + - type: ndcg_at_3 + value: 87.265 + - type: ndcg_at_5 + value: 87.813 + - type: precision_at_1 + value: 86.424 + - type: precision_at_10 + value: 10.012 + - type: precision_at_100 + value: 1.042 + - type: precision_at_1000 + value: 0.107 + - type: precision_at_3 + value: 32.228 + - type: precision_at_5 + value: 19.724 + - type: recall_at_1 + value: 80.079 + - type: recall_at_10 + value: 91.96600000000001 + - type: recall_at_100 + value: 94.541 + - type: recall_at_1000 + value: 95.824 + - type: recall_at_3 + value: 89.213 + - type: recall_at_5 + value: 90.791 + - task: + type: Retrieval + dataset: + type: fiqa + name: MTEB FiQA2018 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.006999999999998 + - type: map_at_10 + value: 36.923 + - type: map_at_100 + value: 38.932 + - type: map_at_1000 + value: 39.096 + - type: map_at_3 + value: 32.322 + - type: map_at_5 + value: 35.119 + - type: mrr_at_1 + value: 45.37 + - type: mrr_at_10 + value: 53.418 + - type: mrr_at_100 + value: 54.174 + - type: mrr_at_1000 + value: 54.20700000000001 + - type: mrr_at_3 + value: 51.132 + - type: mrr_at_5 + value: 52.451 + - type: ndcg_at_1 + value: 45.37 + - type: ndcg_at_10 + value: 44.799 + - type: ndcg_at_100 + value: 51.605000000000004 + - type: ndcg_at_1000 + value: 54.30500000000001 + - type: ndcg_at_3 + value: 41.33 + - type: ndcg_at_5 + value: 42.608000000000004 + - type: precision_at_1 + value: 45.37 + - type: precision_at_10 + value: 12.33 + - type: precision_at_100 + value: 1.9349999999999998 + - type: precision_at_1000 + value: 0.241 + - type: precision_at_3 + value: 27.828999999999997 + - type: precision_at_5 + value: 20.432 + - type: recall_at_1 + value: 23.006999999999998 + - type: recall_at_10 + value: 51.06699999999999 + - type: recall_at_100 + value: 75.917 + - type: recall_at_1000 + value: 92.331 + - type: recall_at_3 + value: 36.544 + - type: recall_at_5 + value: 43.449 + - task: + type: Retrieval + dataset: + type: hotpotqa + name: MTEB HotpotQA + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 38.196999999999996 + - type: map_at_10 + value: 55.554 + - type: map_at_100 + value: 56.309 + - type: map_at_1000 + value: 56.37799999999999 + - type: map_at_3 + value: 53.123 + - type: map_at_5 + value: 54.626 + - type: mrr_at_1 + value: 76.39399999999999 + - type: mrr_at_10 + value: 81.75 + - type: mrr_at_100 + value: 81.973 + - type: mrr_at_1000 + value: 81.982 + - type: mrr_at_3 + value: 80.79499999999999 + - type: mrr_at_5 + value: 81.393 + - type: ndcg_at_1 + value: 76.39399999999999 + - type: ndcg_at_10 + value: 64.14800000000001 + - type: ndcg_at_100 + value: 66.90899999999999 + - type: ndcg_at_1000 + value: 68.277 + - type: ndcg_at_3 + value: 60.529999999999994 + - type: ndcg_at_5 + value: 62.513 + - type: precision_at_1 + value: 76.39399999999999 + - type: precision_at_10 + value: 12.967999999999998 + - type: precision_at_100 + value: 1.5150000000000001 + - type: precision_at_1000 + value: 0.16999999999999998 + - type: precision_at_3 + value: 37.884 + - type: precision_at_5 + value: 24.294 + - type: recall_at_1 + value: 38.196999999999996 + - type: recall_at_10 + value: 64.84100000000001 + - type: recall_at_100 + value: 75.726 + - type: recall_at_1000 + value: 84.794 + - type: recall_at_3 + value: 56.826 + - type: recall_at_5 + value: 60.736000000000004 + - task: + type: Classification + dataset: + type: mteb/imdb + name: MTEB ImdbClassification + config: default + split: test + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + metrics: + - type: accuracy + value: 82.3912 + - type: ap + value: 76.3949298163793 + - type: f1 + value: 82.30848699417406 + - task: + type: Retrieval + dataset: + type: msmarco + name: MTEB MSMARCO + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 19.454 + - type: map_at_10 + value: 31.22 + - type: map_at_100 + value: 32.475 + - type: map_at_1000 + value: 32.532 + - type: map_at_3 + value: 27.419 + - type: map_at_5 + value: 29.608 + - type: mrr_at_1 + value: 20.072000000000003 + - type: mrr_at_10 + value: 31.813999999999997 + - type: mrr_at_100 + value: 33.01 + - type: mrr_at_1000 + value: 33.062000000000005 + - type: mrr_at_3 + value: 28.055999999999997 + - type: mrr_at_5 + value: 30.218 + - type: ndcg_at_1 + value: 20.072000000000003 + - type: ndcg_at_10 + value: 38.0 + - type: ndcg_at_100 + value: 44.038 + - type: ndcg_at_1000 + value: 45.43 + - type: ndcg_at_3 + value: 30.219 + - type: ndcg_at_5 + value: 34.127 + - type: precision_at_1 + value: 20.072000000000003 + - type: precision_at_10 + value: 6.159 + - type: precision_at_100 + value: 0.9169999999999999 + - type: precision_at_1000 + value: 0.104 + - type: precision_at_3 + value: 13.071 + - type: precision_at_5 + value: 9.814 + - type: recall_at_1 + value: 19.454 + - type: recall_at_10 + value: 58.931 + - type: recall_at_100 + value: 86.886 + - type: recall_at_1000 + value: 97.425 + - type: recall_at_3 + value: 37.697 + - type: recall_at_5 + value: 47.101 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (en) + config: en + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 90.46283629730961 + - type: f1 + value: 90.22448402668293 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (de) + config: de + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 86.91462383770076 + - type: f1 + value: 85.77767304705436 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (es) + config: es + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 87.73849232821881 + - type: f1 + value: 87.33680109229385 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (fr) + config: fr + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 86.22298778578141 + - type: f1 + value: 85.88868176519013 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (hi) + config: hi + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 82.91860882036572 + - type: f1 + value: 81.38044567838352 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (th) + config: th + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 69.90235081374323 + - type: f1 + value: 68.12897827044782 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (en) + config: en + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 66.0031919744642 + - type: f1 + value: 48.13490278120492 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (de) + config: de + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 63.260073260073256 + - type: f1 + value: 42.627167415555505 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (es) + config: es + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 65.06004002668445 + - type: f1 + value: 44.90527231209402 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (fr) + config: fr + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 59.42687128092702 + - type: f1 + value: 41.79584710899656 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (hi) + config: hi + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 59.078522768017216 + - type: f1 + value: 40.398016878580734 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (th) + config: th + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 43.750452079565996 + - type: f1 + value: 28.985320742729865 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (af) + config: af + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 47.59919300605245 + - type: f1 + value: 44.27505749600044 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (am) + config: am + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 31.56691324815064 + - type: f1 + value: 30.34952276390722 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ar) + config: ar + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 52.62945527908541 + - type: f1 + value: 49.689536347222386 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (az) + config: az + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 50.0941492938803 + - type: f1 + value: 48.47831879848094 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (bn) + config: bn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 46.540013449899135 + - type: f1 + value: 44.25663324630171 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (cy) + config: cy + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 44.25689307330195 + - type: f1 + value: 42.06066077477426 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (da) + config: da + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 55.05716207128446 + - type: f1 + value: 52.41516089202158 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (de) + config: de + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 61.86953597848015 + - type: f1 + value: 58.45989820228606 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (el) + config: el + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 47.02084734364493 + - type: f1 + value: 45.21525882986924 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (en) + config: en + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 69.24008069939475 + - type: f1 + value: 68.27971089998472 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (es) + config: es + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 62.53530598520511 + - type: f1 + value: 61.83588971206536 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fa) + config: fa + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 55.19166106254204 + - type: f1 + value: 52.335787325774 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fi) + config: fi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 48.43308675184936 + - type: f1 + value: 45.841102061239184 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fr) + config: fr + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 64.26698049764627 + - type: f1 + value: 62.25607481996241 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (he) + config: he + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 57.619367854741085 + - type: f1 + value: 54.93671211092237 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hi) + config: hi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 57.53530598520511 + - type: f1 + value: 55.36413211751344 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hu) + config: hu + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 45.66913248150638 + - type: f1 + value: 42.52092657926257 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hy) + config: hy + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 39.19973100201749 + - type: f1 + value: 37.194613407773566 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (id) + config: id + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 54.99663752521856 + - type: f1 + value: 53.875181150315356 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (is) + config: is + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 43.143913920645595 + - type: f1 + value: 41.756257561394456 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (it) + config: it + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 60.99529253530599 + - type: f1 + value: 59.103812128183705 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ja) + config: ja + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 64.29051782111634 + - type: f1 + value: 62.5268914542489 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (jv) + config: jv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 43.69199731002017 + - type: f1 + value: 41.71651113018154 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ka) + config: ka + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 38.34566240753194 + - type: f1 + value: 36.935911015227894 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (km) + config: km + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 34.21654337592467 + - type: f1 + value: 32.067289455027755 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (kn) + config: kn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 51.785474108944186 + - type: f1 + value: 49.29285691779668 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ko) + config: ko + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 59.58977807666444 + - type: f1 + value: 57.81630371862734 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (lv) + config: lv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 46.53665097511768 + - type: f1 + value: 44.8386852929464 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ml) + config: ml + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 54.468728984532625 + - type: f1 + value: 52.13613631138983 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (mn) + config: mn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 40.67921990585071 + - type: f1 + value: 39.87218130311539 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ms) + config: ms + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 51.2441156691325 + - type: f1 + value: 48.93351041227674 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (my) + config: my + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 31.76193678547411 + - type: f1 + value: 29.917012787908785 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nb) + config: nb + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 54.40820443846671 + - type: f1 + value: 51.232049156874396 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nl) + config: nl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 60.8170813718897 + - type: f1 + value: 57.74887572270486 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pl) + config: pl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 60.067249495628786 + - type: f1 + value: 57.60151669462318 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pt) + config: pt + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 62.73705447209146 + - type: f1 + value: 61.14377989075874 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ro) + config: ro + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 49.68392737054472 + - type: f1 + value: 48.07062918679129 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ru) + config: ru + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 60.85406859448555 + - type: f1 + value: 58.48852652838252 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sl) + config: sl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 48.58776059179556 + - type: f1 + value: 46.92163099241966 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sq) + config: sq + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 47.16879623402824 + - type: f1 + value: 45.8155066134247 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sv) + config: sv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 62.41425689307329 + - type: f1 + value: 60.097954878192574 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sw) + config: sw + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 45.97175521183591 + - type: f1 + value: 44.29275283000346 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ta) + config: ta + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 53.597848016139885 + - type: f1 + value: 51.54318966923094 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (te) + config: te + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 53.44653665097512 + - type: f1 + value: 51.60095623356469 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (th) + config: th + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 46.173503698722264 + - type: f1 + value: 46.311285276929105 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tl) + config: tl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 49.47881640887693 + - type: f1 + value: 46.63989802589145 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tr) + config: tr + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 58.02958977807666 + - type: f1 + value: 55.34728796730868 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ur) + config: ur + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 39.26361802286483 + - type: f1 + value: 37.61201358829197 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (vi) + config: vi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 52.15534633490249 + - type: f1 + value: 50.438951980623145 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-CN) + config: zh-CN + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 63.39946200403498 + - type: f1 + value: 62.152249150179664 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-TW) + config: zh-TW + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 58.207800941492934 + - type: f1 + value: 58.318584465398104 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (af) + config: af + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 58.069939475453936 + - type: f1 + value: 55.04073616892449 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (am) + config: am + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 38.214525891055814 + - type: f1 + value: 36.42184260742777 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ar) + config: ar + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 57.47141896435777 + - type: f1 + value: 57.22453431938479 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (az) + config: az + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 54.37121721587089 + - type: f1 + value: 53.004976087120134 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (bn) + config: bn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 52.71687962340283 + - type: f1 + value: 51.140151342341646 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (cy) + config: cy + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 49.502353732347004 + - type: f1 + value: 45.74604753969847 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (da) + config: da + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 64.25689307330195 + - type: f1 + value: 62.25355539317913 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (de) + config: de + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 71.27774041694688 + - type: f1 + value: 70.26880477280841 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (el) + config: el + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 52.420981842636195 + - type: f1 + value: 50.824547366213565 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (en) + config: en + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 74.11230665770006 + - type: f1 + value: 73.00723710263364 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (es) + config: es + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 67.04102219233356 + - type: f1 + value: 66.7904194512351 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fa) + config: fa + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 60.1714862138534 + - type: f1 + value: 58.781208933846095 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fi) + config: fi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 54.04841963685272 + - type: f1 + value: 51.185007148328545 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fr) + config: fr + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 69.76462676529927 + - type: f1 + value: 68.85227238388136 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (he) + config: he + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 62.84801613987895 + - type: f1 + value: 61.18395865529196 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hi) + config: hi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 62.17888365837256 + - type: f1 + value: 60.40570575783401 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hu) + config: hu + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 53.52051109616678 + - type: f1 + value: 51.210696278552014 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hy) + config: hy + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 45.94821788836584 + - type: f1 + value: 43.65062337089374 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (id) + config: id + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 60.33288500336248 + - type: f1 + value: 59.50436947982156 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (is) + config: is + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 50.09751176866174 + - type: f1 + value: 47.293838685239 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (it) + config: it + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 66.49293880295897 + - type: f1 + value: 65.96586462307134 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ja) + config: ja + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 68.35911230665769 + - type: f1 + value: 67.77840431764355 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (jv) + config: jv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 50.585070611970416 + - type: f1 + value: 47.957277125670295 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ka) + config: ka + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 42.76059179556153 + - type: f1 + value: 40.446327361325565 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (km) + config: km + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 40.648957632817755 + - type: f1 + value: 37.231284508608276 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (kn) + config: kn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 57.24613315400134 + - type: f1 + value: 55.14523425690653 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ko) + config: ko + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 63.839946200403496 + - type: f1 + value: 62.6239063060589 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (lv) + config: lv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 53.14391392064559 + - type: f1 + value: 50.08744471966442 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ml) + config: ml + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 58.8399462004035 + - type: f1 + value: 57.586991117740794 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (mn) + config: mn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 44.81842636180229 + - type: f1 + value: 42.82813975084655 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ms) + config: ms + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 58.90047074646939 + - type: f1 + value: 56.640503134745714 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (my) + config: my + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 38.52051109616678 + - type: f1 + value: 36.504553927569454 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nb) + config: nb + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 64.63685272360458 + - type: f1 + value: 62.88129994502907 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nl) + config: nl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 67.54203093476798 + - type: f1 + value: 66.02745142287087 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pl) + config: pl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 64.00470746469402 + - type: f1 + value: 62.91845058355313 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pt) + config: pt + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 65.69939475453934 + - type: f1 + value: 65.37413822081011 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ro) + config: ro + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 57.19905850706121 + - type: f1 + value: 55.08271383695852 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ru) + config: ru + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 65.42367182246134 + - type: f1 + value: 64.61962307022019 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sl) + config: sl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 55.147948890383326 + - type: f1 + value: 53.2933851469903 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sq) + config: sq + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 55.679219905850715 + - type: f1 + value: 52.80159603468007 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sv) + config: sv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 69.42165433759246 + - type: f1 + value: 67.99984081248608 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sw) + config: sw + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 52.30329522528581 + - type: f1 + value: 50.10810382364662 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ta) + config: ta + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 56.186953597848024 + - type: f1 + value: 55.51656586643505 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (te) + config: te + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 58.019502353732356 + - type: f1 + value: 56.260726586358736 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (th) + config: th + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 52.55548083389374 + - type: f1 + value: 51.139712264362714 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tl) + config: tl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 57.43443174176194 + - type: f1 + value: 55.76244076715635 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tr) + config: tr + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 61.55346334902488 + - type: f1 + value: 61.25819823057803 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ur) + config: ur + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 47.114996637525216 + - type: f1 + value: 45.20428169546973 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (vi) + config: vi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 56.83254875588434 + - type: f1 + value: 56.00919757601416 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-CN) + config: zh-CN + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 69.57969065232012 + - type: f1 + value: 69.17378512156806 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-TW) + config: zh-TW + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 64.02488231338263 + - type: f1 + value: 64.09790488949963 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-p2p + name: MTEB MedrxivClusteringP2P + config: default + split: test + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + metrics: + - type: v_measure + value: 29.71446786877363 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-s2s + name: MTEB MedrxivClusteringS2S + config: default + split: test + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + metrics: + - type: v_measure + value: 28.003624498407547 + - task: + type: Reranking + dataset: + type: mteb/mind_small + name: MTEB MindSmallReranking + config: default + split: test + revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 + metrics: + - type: map + value: 31.29671894458151 + - type: mrr + value: 32.44455140124599 + - task: + type: Retrieval + dataset: + type: nfcorpus + name: MTEB NFCorpus + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 6.127 + - type: map_at_10 + value: 13.047 + - type: map_at_100 + value: 15.754000000000001 + - type: map_at_1000 + value: 16.930999999999997 + - type: map_at_3 + value: 9.876999999999999 + - type: map_at_5 + value: 11.265 + - type: mrr_at_1 + value: 45.511 + - type: mrr_at_10 + value: 54.75600000000001 + - type: mrr_at_100 + value: 55.33 + - type: mrr_at_1000 + value: 55.374 + - type: mrr_at_3 + value: 53.147999999999996 + - type: mrr_at_5 + value: 53.952999999999996 + - type: ndcg_at_1 + value: 43.653 + - type: ndcg_at_10 + value: 33.936 + - type: ndcg_at_100 + value: 29.952 + - type: ndcg_at_1000 + value: 38.356 + - type: ndcg_at_3 + value: 40.018 + - type: ndcg_at_5 + value: 37.102000000000004 + - type: precision_at_1 + value: 45.511 + - type: precision_at_10 + value: 24.768 + - type: precision_at_100 + value: 7.13 + - type: precision_at_1000 + value: 1.928 + - type: precision_at_3 + value: 37.461 + - type: precision_at_5 + value: 31.703 + - type: recall_at_1 + value: 6.127 + - type: recall_at_10 + value: 16.512999999999998 + - type: recall_at_100 + value: 29.057 + - type: recall_at_1000 + value: 59.25899999999999 + - type: recall_at_3 + value: 10.940999999999999 + - type: recall_at_5 + value: 12.925 + - task: + type: Retrieval + dataset: + type: nq + name: MTEB NQ + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 32.228 + - type: map_at_10 + value: 47.56 + - type: map_at_100 + value: 48.539 + - type: map_at_1000 + value: 48.567 + - type: map_at_3 + value: 43.214999999999996 + - type: map_at_5 + value: 45.799 + - type: mrr_at_1 + value: 36.53 + - type: mrr_at_10 + value: 50.004000000000005 + - type: mrr_at_100 + value: 50.737 + - type: mrr_at_1000 + value: 50.758 + - type: mrr_at_3 + value: 46.543 + - type: mrr_at_5 + value: 48.672 + - type: ndcg_at_1 + value: 36.501 + - type: ndcg_at_10 + value: 55.103 + - type: ndcg_at_100 + value: 59.156 + - type: ndcg_at_1000 + value: 59.821999999999996 + - type: ndcg_at_3 + value: 47.089 + - type: ndcg_at_5 + value: 51.35999999999999 + - type: precision_at_1 + value: 36.501 + - type: precision_at_10 + value: 9.046999999999999 + - type: precision_at_100 + value: 1.13 + - type: precision_at_1000 + value: 0.11900000000000001 + - type: precision_at_3 + value: 21.398 + - type: precision_at_5 + value: 15.307 + - type: recall_at_1 + value: 32.228 + - type: recall_at_10 + value: 75.608 + - type: recall_at_100 + value: 93.062 + - type: recall_at_1000 + value: 98.059 + - type: recall_at_3 + value: 55.021 + - type: recall_at_5 + value: 64.873 + - task: + type: Retrieval + dataset: + type: quora + name: MTEB QuoraRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 70.623 + - type: map_at_10 + value: 84.705 + - type: map_at_100 + value: 85.333 + - type: map_at_1000 + value: 85.348 + - type: map_at_3 + value: 81.736 + - type: map_at_5 + value: 83.616 + - type: mrr_at_1 + value: 81.28 + - type: mrr_at_10 + value: 87.518 + - type: mrr_at_100 + value: 87.619 + - type: mrr_at_1000 + value: 87.62 + - type: mrr_at_3 + value: 86.545 + - type: mrr_at_5 + value: 87.238 + - type: ndcg_at_1 + value: 81.28999999999999 + - type: ndcg_at_10 + value: 88.412 + - type: ndcg_at_100 + value: 89.603 + - type: ndcg_at_1000 + value: 89.696 + - type: ndcg_at_3 + value: 85.563 + - type: ndcg_at_5 + value: 87.17 + - type: precision_at_1 + value: 81.28999999999999 + - type: precision_at_10 + value: 13.439 + - type: precision_at_100 + value: 1.5310000000000001 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_3 + value: 37.437 + - type: precision_at_5 + value: 24.662 + - type: recall_at_1 + value: 70.623 + - type: recall_at_10 + value: 95.531 + - type: recall_at_100 + value: 99.58 + - type: recall_at_1000 + value: 99.978 + - type: recall_at_3 + value: 87.368 + - type: recall_at_5 + value: 91.898 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering + name: MTEB RedditClustering + config: default + split: test + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + metrics: + - type: v_measure + value: 49.53241309124786 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering-p2p + name: MTEB RedditClusteringP2P + config: default + split: test + revision: 282350215ef01743dc01b456c7f5241fa8937f16 + metrics: + - type: v_measure + value: 59.712004482915994 + - task: + type: Retrieval + dataset: + type: scidocs + name: MTEB SCIDOCS + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 5.313 + - type: map_at_10 + value: 13.447000000000001 + - type: map_at_100 + value: 15.491 + - type: map_at_1000 + value: 15.784999999999998 + - type: map_at_3 + value: 9.58 + - type: map_at_5 + value: 11.562 + - type: mrr_at_1 + value: 26.200000000000003 + - type: mrr_at_10 + value: 37.212 + - type: mrr_at_100 + value: 38.190000000000005 + - type: mrr_at_1000 + value: 38.242 + - type: mrr_at_3 + value: 34.067 + - type: mrr_at_5 + value: 35.862 + - type: ndcg_at_1 + value: 26.200000000000003 + - type: ndcg_at_10 + value: 21.979000000000003 + - type: ndcg_at_100 + value: 29.726999999999997 + - type: ndcg_at_1000 + value: 34.766000000000005 + - type: ndcg_at_3 + value: 21.16 + - type: ndcg_at_5 + value: 18.478 + - type: precision_at_1 + value: 26.200000000000003 + - type: precision_at_10 + value: 11.25 + - type: precision_at_100 + value: 2.241 + - type: precision_at_1000 + value: 0.345 + - type: precision_at_3 + value: 19.633 + - type: precision_at_5 + value: 16.14 + - type: recall_at_1 + value: 5.313 + - type: recall_at_10 + value: 22.808 + - type: recall_at_100 + value: 45.540000000000006 + - type: recall_at_1000 + value: 70.043 + - type: recall_at_3 + value: 11.932 + - type: recall_at_5 + value: 16.347 + - task: + type: STS + dataset: + type: mteb/sickr-sts + name: MTEB SICK-R + config: default + split: test + revision: a6ea5a8cab320b040a23452cc28066d9beae2cee + metrics: + - type: cos_sim_pearson + value: 75.95540796619258 + - type: cos_sim_spearman + value: 76.49462277620303 + - type: euclidean_pearson + value: 71.67643435507317 + - type: euclidean_spearman + value: 76.4915921108082 + - type: manhattan_pearson + value: 71.71412560074847 + - type: manhattan_spearman + value: 76.46738312094736 + - task: + type: STS + dataset: + type: mteb/sts12-sts + name: MTEB STS12 + config: default + split: test + revision: a0d554a64d88156834ff5ae9920b964011b16384 + metrics: + - type: cos_sim_pearson + value: 81.48773267615617 + - type: cos_sim_spearman + value: 74.99867664033701 + - type: euclidean_pearson + value: 76.0885798115032 + - type: euclidean_spearman + value: 74.99438208715942 + - type: manhattan_pearson + value: 76.09382557464033 + - type: manhattan_spearman + value: 74.96139353538533 + - task: + type: STS + dataset: + type: mteb/sts13-sts + name: MTEB STS13 + config: default + split: test + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + metrics: + - type: cos_sim_pearson + value: 88.19022560804167 + - type: cos_sim_spearman + value: 87.9128142106699 + - type: euclidean_pearson + value: 85.51390183763914 + - type: euclidean_spearman + value: 87.89995488057309 + - type: manhattan_pearson + value: 85.44945034816052 + - type: manhattan_spearman + value: 87.791458898378 + - task: + type: STS + dataset: + type: mteb/sts14-sts + name: MTEB STS14 + config: default + split: test + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + metrics: + - type: cos_sim_pearson + value: 85.17877898640924 + - type: cos_sim_spearman + value: 82.25544088807465 + - type: euclidean_pearson + value: 82.36395988835416 + - type: euclidean_spearman + value: 82.26359924974219 + - type: manhattan_pearson + value: 82.39219808999891 + - type: manhattan_spearman + value: 82.27757404868157 + - task: + type: STS + dataset: + type: mteb/sts15-sts + name: MTEB STS15 + config: default + split: test + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + metrics: + - type: cos_sim_pearson + value: 87.66865350602554 + - type: cos_sim_spearman + value: 87.87150169810872 + - type: euclidean_pearson + value: 85.41520650056647 + - type: euclidean_spearman + value: 87.86636613654022 + - type: manhattan_pearson + value: 85.38710485867502 + - type: manhattan_spearman + value: 87.83513424575301 + - task: + type: STS + dataset: + type: mteb/sts16-sts + name: MTEB STS16 + config: default + split: test + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + metrics: + - type: cos_sim_pearson + value: 80.75527643407175 + - type: cos_sim_spearman + value: 80.9239008594745 + - type: euclidean_pearson + value: 79.37682746800515 + - type: euclidean_spearman + value: 80.91978947194092 + - type: manhattan_pearson + value: 79.38884189990698 + - type: manhattan_spearman + value: 80.91771608341014 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ko-ko) + config: ko-ko + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 80.24344311909609 + - type: cos_sim_spearman + value: 80.78933956176022 + - type: euclidean_pearson + value: 76.95229806538676 + - type: euclidean_spearman + value: 80.79706724032172 + - type: manhattan_pearson + value: 76.90212135774246 + - type: manhattan_spearman + value: 80.68727415384441 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ar-ar) + config: ar-ar + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 77.33891809228084 + - type: cos_sim_spearman + value: 79.37912430317627 + - type: euclidean_pearson + value: 72.56919843951036 + - type: euclidean_spearman + value: 79.3091436905072 + - type: manhattan_pearson + value: 72.4282811588754 + - type: manhattan_spearman + value: 78.90144894538078 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-ar) + config: en-ar + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 59.68908656739356 + - type: cos_sim_spearman + value: 58.76110210983758 + - type: euclidean_pearson + value: 59.14749159577439 + - type: euclidean_spearman + value: 59.015997032145016 + - type: manhattan_pearson + value: 57.907675340322676 + - type: manhattan_spearman + value: 57.07751173022352 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-de) + config: en-de + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 75.53325164873934 + - type: cos_sim_spearman + value: 76.13104388846271 + - type: euclidean_pearson + value: 74.61931031522006 + - type: euclidean_spearman + value: 75.96875166459931 + - type: manhattan_pearson + value: 74.82154350849251 + - type: manhattan_spearman + value: 76.64455924104236 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-en) + config: en-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 85.4228376590724 + - type: cos_sim_spearman + value: 87.22764976624408 + - type: euclidean_pearson + value: 81.94975688107507 + - type: euclidean_spearman + value: 87.19193932664932 + - type: manhattan_pearson + value: 82.0043964628936 + - type: manhattan_spearman + value: 87.09130430957818 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-tr) + config: en-tr + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 57.5627552601949 + - type: cos_sim_spearman + value: 55.5263144563657 + - type: euclidean_pearson + value: 57.00569241610482 + - type: euclidean_spearman + value: 55.35291811479459 + - type: manhattan_pearson + value: 56.99656284623506 + - type: manhattan_spearman + value: 55.593673744709946 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (es-en) + config: es-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 69.93801311909735 + - type: cos_sim_spearman + value: 72.2581115470475 + - type: euclidean_pearson + value: 68.24881290268563 + - type: euclidean_spearman + value: 72.60813652864522 + - type: manhattan_pearson + value: 67.86369874088834 + - type: manhattan_spearman + value: 71.92346382988023 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (es-es) + config: es-es + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 86.20555264114785 + - type: cos_sim_spearman + value: 85.0588060013836 + - type: euclidean_pearson + value: 81.78229090166155 + - type: euclidean_spearman + value: 85.09687374900614 + - type: manhattan_pearson + value: 81.77449099980244 + - type: manhattan_spearman + value: 84.70331476222177 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (fr-en) + config: fr-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 73.786793911605 + - type: cos_sim_spearman + value: 75.63094397551554 + - type: euclidean_pearson + value: 71.64292842519251 + - type: euclidean_spearman + value: 75.60215267384011 + - type: manhattan_pearson + value: 72.2124078037642 + - type: manhattan_spearman + value: 76.34546028465175 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (it-en) + config: it-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 69.62139987106455 + - type: cos_sim_spearman + value: 71.35872226722493 + - type: euclidean_pearson + value: 68.50103697766141 + - type: euclidean_spearman + value: 71.24590187948473 + - type: manhattan_pearson + value: 68.89236562525663 + - type: manhattan_spearman + value: 71.77994400789173 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (nl-en) + config: nl-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 71.62728174871292 + - type: cos_sim_spearman + value: 71.98655715409397 + - type: euclidean_pearson + value: 70.27026741609356 + - type: euclidean_spearman + value: 72.14004669693777 + - type: manhattan_pearson + value: 70.46335140108751 + - type: manhattan_spearman + value: 72.6638254374311 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (en) + config: en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 71.10248717637424 + - type: cos_sim_spearman + value: 68.5905931564714 + - type: euclidean_pearson + value: 71.23290000423759 + - type: euclidean_spearman + value: 68.6419513130457 + - type: manhattan_pearson + value: 71.6886015250234 + - type: manhattan_spearman + value: 69.47543660368697 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de) + config: de + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 59.010555056244776 + - type: cos_sim_spearman + value: 60.121771179899255 + - type: euclidean_pearson + value: 53.04527785573465 + - type: euclidean_spearman + value: 60.121771179899255 + - type: manhattan_pearson + value: 52.931480071124234 + - type: manhattan_spearman + value: 60.03868409331775 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es) + config: es + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 70.6833028374664 + - type: cos_sim_spearman + value: 68.57396263856863 + - type: euclidean_pearson + value: 68.30905084522986 + - type: euclidean_spearman + value: 68.57396263856863 + - type: manhattan_pearson + value: 70.91400657516918 + - type: manhattan_spearman + value: 72.72240857808112 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl) + config: pl + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 36.948290734279645 + - type: cos_sim_spearman + value: 42.07722031011005 + - type: euclidean_pearson + value: 22.539446972018467 + - type: euclidean_spearman + value: 42.07722031011005 + - type: manhattan_pearson + value: 24.119402246951786 + - type: manhattan_spearman + value: 45.80525501822569 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (tr) + config: tr + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 66.97840719036533 + - type: cos_sim_spearman + value: 66.62430648804775 + - type: euclidean_pearson + value: 66.89526587772023 + - type: euclidean_spearman + value: 66.62430648804775 + - type: manhattan_pearson + value: 68.6929895225091 + - type: manhattan_spearman + value: 68.91772708432867 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ar) + config: ar + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 56.65098289103698 + - type: cos_sim_spearman + value: 57.436674670689214 + - type: euclidean_pearson + value: 51.79149892785239 + - type: euclidean_spearman + value: 57.436674670689214 + - type: manhattan_pearson + value: 52.64807953938707 + - type: manhattan_spearman + value: 58.94583987372767 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ru) + config: ru + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 60.669531297510225 + - type: cos_sim_spearman + value: 61.71342510003327 + - type: euclidean_pearson + value: 55.821871433553504 + - type: euclidean_spearman + value: 61.71342510003327 + - type: manhattan_pearson + value: 57.77073441351117 + - type: manhattan_spearman + value: 65.20759033207 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh) + config: zh + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 64.34728960310699 + - type: cos_sim_spearman + value: 64.03565302589584 + - type: euclidean_pearson + value: 61.958942333930544 + - type: euclidean_spearman + value: 64.03565302589584 + - type: manhattan_pearson + value: 64.65072672727923 + - type: manhattan_spearman + value: 67.82569969943107 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr) + config: fr + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 82.47120815594353 + - type: cos_sim_spearman + value: 81.46916544955101 + - type: euclidean_pearson + value: 79.21753533489019 + - type: euclidean_spearman + value: 81.46916544955101 + - type: manhattan_pearson + value: 78.26605518839271 + - type: manhattan_spearman + value: 81.29749169339514 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-en) + config: de-en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 63.31467231933632 + - type: cos_sim_spearman + value: 53.36160506603274 + - type: euclidean_pearson + value: 64.98434169416196 + - type: euclidean_spearman + value: 53.36160506603274 + - type: manhattan_pearson + value: 69.6837006629638 + - type: manhattan_spearman + value: 60.85384324700893 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-en) + config: es-en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 79.99425127770438 + - type: cos_sim_spearman + value: 77.41308957007035 + - type: euclidean_pearson + value: 79.69441265626801 + - type: euclidean_spearman + value: 77.41308957007035 + - type: manhattan_pearson + value: 80.3726291667624 + - type: manhattan_spearman + value: 79.0414050644631 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (it) + config: it + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 79.13469287716659 + - type: cos_sim_spearman + value: 79.27976881582065 + - type: euclidean_pearson + value: 77.65964425780172 + - type: euclidean_spearman + value: 79.27976881582065 + - type: manhattan_pearson + value: 77.64158710257945 + - type: manhattan_spearman + value: 79.22242281895944 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl-en) + config: pl-en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 76.303314995599 + - type: cos_sim_spearman + value: 77.4991345414335 + - type: euclidean_pearson + value: 74.88826621426401 + - type: euclidean_spearman + value: 77.4991345414335 + - type: manhattan_pearson + value: 77.70223488989319 + - type: manhattan_spearman + value: 79.69746987627822 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh-en) + config: zh-en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 70.87814957197239 + - type: cos_sim_spearman + value: 69.86785751801642 + - type: euclidean_pearson + value: 68.68630146548654 + - type: euclidean_spearman + value: 69.8615799070054 + - type: manhattan_pearson + value: 61.83743315022061 + - type: manhattan_spearman + value: 64.35346450347738 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-it) + config: es-it + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 74.1484689923211 + - type: cos_sim_spearman + value: 74.69046355179742 + - type: euclidean_pearson + value: 73.03951899271793 + - type: euclidean_spearman + value: 74.69820632954205 + - type: manhattan_pearson + value: 73.36810146930709 + - type: manhattan_spearman + value: 75.33154135287258 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-fr) + config: de-fr + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 51.43125921362742 + - type: cos_sim_spearman + value: 58.25341239774093 + - type: euclidean_pearson + value: 48.00689582162098 + - type: euclidean_spearman + value: 58.533194841668426 + - type: manhattan_pearson + value: 46.11721778230745 + - type: manhattan_spearman + value: 55.026889052448134 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-pl) + config: de-pl + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 40.066205533538046 + - type: cos_sim_spearman + value: 48.46991890841381 + - type: euclidean_pearson + value: 42.29606506858651 + - type: euclidean_spearman + value: 48.34674249441531 + - type: manhattan_pearson + value: 41.70680990555484 + - type: manhattan_spearman + value: 47.54609580342499 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr-pl) + config: fr-pl + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 82.26527545520592 + - type: cos_sim_spearman + value: 73.24670207647144 + - type: euclidean_pearson + value: 81.78699781584893 + - type: euclidean_spearman + value: 73.24670207647144 + - type: manhattan_pearson + value: 83.14172292187807 + - type: manhattan_spearman + value: 73.24670207647144 + - task: + type: STS + dataset: + type: mteb/stsbenchmark-sts + name: MTEB STSBenchmark + config: default + split: test + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + metrics: + - type: cos_sim_pearson + value: 81.51438108053523 + - type: cos_sim_spearman + value: 81.9481311864648 + - type: euclidean_pearson + value: 78.6683040592179 + - type: euclidean_spearman + value: 81.9535649926177 + - type: manhattan_pearson + value: 78.65396325536754 + - type: manhattan_spearman + value: 81.96918240343872 + - task: + type: Reranking + dataset: + type: mteb/scidocs-reranking + name: MTEB SciDocsRR + config: default + split: test + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + metrics: + - type: map + value: 80.6689275068653 + - type: mrr + value: 95.021337594867 + - task: + type: Retrieval + dataset: + type: scifact + name: MTEB SciFact + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 55.193999999999996 + - type: map_at_10 + value: 65.814 + - type: map_at_100 + value: 66.428 + - type: map_at_1000 + value: 66.447 + - type: map_at_3 + value: 63.304 + - type: map_at_5 + value: 64.64 + - type: mrr_at_1 + value: 57.99999999999999 + - type: mrr_at_10 + value: 66.957 + - type: mrr_at_100 + value: 67.405 + - type: mrr_at_1000 + value: 67.422 + - type: mrr_at_3 + value: 65.0 + - type: mrr_at_5 + value: 66.183 + - type: ndcg_at_1 + value: 57.99999999999999 + - type: ndcg_at_10 + value: 70.523 + - type: ndcg_at_100 + value: 72.987 + - type: ndcg_at_1000 + value: 73.605 + - type: ndcg_at_3 + value: 66.268 + - type: ndcg_at_5 + value: 68.27600000000001 + - type: precision_at_1 + value: 57.99999999999999 + - type: precision_at_10 + value: 9.467 + - type: precision_at_100 + value: 1.073 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_3 + value: 26.444000000000003 + - type: precision_at_5 + value: 17.2 + - type: recall_at_1 + value: 55.193999999999996 + - type: recall_at_10 + value: 83.52199999999999 + - type: recall_at_100 + value: 94.5 + - type: recall_at_1000 + value: 99.667 + - type: recall_at_3 + value: 71.989 + - type: recall_at_5 + value: 77.31700000000001 + - task: + type: PairClassification + dataset: + type: mteb/sprintduplicatequestions-pairclassification + name: MTEB SprintDuplicateQuestions + config: default + split: test + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + metrics: + - type: cos_sim_accuracy + value: 99.73465346534654 + - type: cos_sim_ap + value: 92.91719494015508 + - type: cos_sim_f1 + value: 86.46200301962756 + - type: cos_sim_precision + value: 87.03140830800406 + - type: cos_sim_recall + value: 85.9 + - type: dot_accuracy + value: 99.73663366336633 + - type: dot_ap + value: 92.90802848215259 + - type: dot_f1 + value: 86.46200301962756 + - type: dot_precision + value: 87.03140830800406 + - type: dot_recall + value: 85.9 + - type: euclidean_accuracy + value: 99.73465346534654 + - type: euclidean_ap + value: 92.91627363446204 + - type: euclidean_f1 + value: 86.43469490670702 + - type: euclidean_precision + value: 87.18209562563581 + - type: euclidean_recall + value: 85.7 + - type: manhattan_accuracy + value: 99.73663366336633 + - type: manhattan_ap + value: 92.90219877406929 + - type: manhattan_f1 + value: 86.31471040492056 + - type: manhattan_precision + value: 88.53838065194533 + - type: manhattan_recall + value: 84.2 + - type: max_accuracy + value: 99.73663366336633 + - type: max_ap + value: 92.91719494015508 + - type: max_f1 + value: 86.46200301962756 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering + name: MTEB StackExchangeClustering + config: default + split: test + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + metrics: + - type: v_measure + value: 60.73098998430779 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering-p2p + name: MTEB StackExchangeClusteringP2P + config: default + split: test + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + metrics: + - type: v_measure + value: 34.64256206757585 + - task: + type: Reranking + dataset: + type: mteb/stackoverflowdupquestions-reranking + name: MTEB StackOverflowDupQuestions + config: default + split: test + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + metrics: + - type: map + value: 54.749150614295694 + - type: mrr + value: 55.78880984211867 + - task: + type: Summarization + dataset: + type: mteb/summeval + name: MTEB SummEval + config: default + split: test + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + metrics: + - type: cos_sim_pearson + value: 28.863577054305907 + - type: cos_sim_spearman + value: 27.538596944829774 + - type: dot_pearson + value: 28.93043755116643 + - type: dot_spearman + value: 27.733110516733987 + - task: + type: Retrieval + dataset: + type: trec-covid + name: MTEB TRECCOVID + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 0.22899999999999998 + - type: map_at_10 + value: 2.078 + - type: map_at_100 + value: 12.024 + - type: map_at_1000 + value: 29.036 + - type: map_at_3 + value: 0.681 + - type: map_at_5 + value: 1.083 + - type: mrr_at_1 + value: 86.0 + - type: mrr_at_10 + value: 92.667 + - type: mrr_at_100 + value: 92.667 + - type: mrr_at_1000 + value: 92.667 + - type: mrr_at_3 + value: 92.667 + - type: mrr_at_5 + value: 92.667 + - type: ndcg_at_1 + value: 82.0 + - type: ndcg_at_10 + value: 80.746 + - type: ndcg_at_100 + value: 61.090999999999994 + - type: ndcg_at_1000 + value: 55.034000000000006 + - type: ndcg_at_3 + value: 82.419 + - type: ndcg_at_5 + value: 81.018 + - type: precision_at_1 + value: 86.0 + - type: precision_at_10 + value: 86.2 + - type: precision_at_100 + value: 62.68 + - type: precision_at_1000 + value: 24.032 + - type: precision_at_3 + value: 88.667 + - type: precision_at_5 + value: 86.0 + - type: recall_at_1 + value: 0.22899999999999998 + - type: recall_at_10 + value: 2.263 + - type: recall_at_100 + value: 15.238999999999999 + - type: recall_at_1000 + value: 51.937 + - type: recall_at_3 + value: 0.719 + - type: recall_at_5 + value: 1.15 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (sqi-eng) + config: sqi-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 19.400000000000002 + - type: f1 + value: 15.386076064970075 + - type: precision + value: 14.253878834615676 + - type: recall + value: 19.400000000000002 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fry-eng) + config: fry-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 42.19653179190752 + - type: f1 + value: 37.726396917148364 + - type: precision + value: 36.14643545279384 + - type: recall + value: 42.19653179190752 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kur-eng) + config: kur-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 18.536585365853657 + - type: f1 + value: 13.512010347376199 + - type: precision + value: 12.034068912117693 + - type: recall + value: 18.536585365853657 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tur-eng) + config: tur-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 81.69999999999999 + - type: f1 + value: 77.37888888888888 + - type: precision + value: 75.49583333333332 + - type: recall + value: 81.69999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (deu-eng) + config: deu-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 97.39999999999999 + - type: f1 + value: 96.56666666666666 + - type: precision + value: 96.16666666666667 + - type: recall + value: 97.39999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nld-eng) + config: nld-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 90.0 + - type: f1 + value: 87.22333333333333 + - type: precision + value: 85.89166666666667 + - type: recall + value: 90.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ron-eng) + config: ron-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 64.7 + - type: f1 + value: 59.10904761904763 + - type: precision + value: 56.91968253968254 + - type: recall + value: 64.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ang-eng) + config: ang-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 38.80597014925373 + - type: f1 + value: 30.890784174366264 + - type: precision + value: 28.327114427860696 + - type: recall + value: 38.80597014925373 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ido-eng) + config: ido-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 53.900000000000006 + - type: f1 + value: 48.294138583638585 + - type: precision + value: 46.333495670995674 + - type: recall + value: 53.900000000000006 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (jav-eng) + config: jav-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.707317073170733 + - type: f1 + value: 8.999999999999998 + - type: precision + value: 8.175377468060395 + - type: recall + value: 11.707317073170733 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (isl-eng) + config: isl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 15.9 + - type: f1 + value: 12.451226269430602 + - type: precision + value: 11.404807799760325 + - type: recall + value: 15.9 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (slv-eng) + config: slv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 41.919805589307416 + - type: f1 + value: 35.880619060297064 + - type: precision + value: 33.77682308241239 + - type: recall + value: 41.919805589307416 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cym-eng) + config: cym-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 10.956521739130434 + - type: f1 + value: 9.098715976676996 + - type: precision + value: 8.659935858401333 + - type: recall + value: 10.956521739130434 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kaz-eng) + config: kaz-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.652173913043478 + - type: f1 + value: 9.154324883225136 + - type: precision + value: 8.505898125360801 + - type: recall + value: 11.652173913043478 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (est-eng) + config: est-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.700000000000001 + - type: f1 + value: 7.431679431679432 + - type: precision + value: 6.799925118740907 + - type: recall + value: 9.700000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (heb-eng) + config: heb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 77.5 + - type: f1 + value: 72.39999999999999 + - type: precision + value: 70.13444444444444 + - type: recall + value: 77.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gla-eng) + config: gla-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 5.548854041013269 + - type: f1 + value: 4.233155465362944 + - type: precision + value: 3.948150869646547 + - type: recall + value: 5.548854041013269 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mar-eng) + config: mar-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 73.5 + - type: f1 + value: 67.35333333333332 + - type: precision + value: 64.63666666666666 + - type: recall + value: 73.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lat-eng) + config: lat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 27.700000000000003 + - type: f1 + value: 21.152765495941964 + - type: precision + value: 19.27832403707404 + - type: recall + value: 27.700000000000003 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bel-eng) + config: bel-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 48.1 + - type: f1 + value: 41.21001443001443 + - type: precision + value: 38.628495670995676 + - type: recall + value: 48.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pms-eng) + config: pms-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 40.0 + - type: f1 + value: 34.32060003488575 + - type: precision + value: 32.32134353741497 + - type: recall + value: 40.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gle-eng) + config: gle-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.800000000000001 + - type: f1 + value: 4.3954389450190465 + - type: precision + value: 3.893838027469606 + - type: recall + value: 6.800000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pes-eng) + config: pes-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 51.800000000000004 + - type: f1 + value: 45.04222943722944 + - type: precision + value: 42.541984126984126 + - type: recall + value: 51.800000000000004 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nob-eng) + config: nob-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 83.1 + - type: f1 + value: 79.20675324675324 + - type: precision + value: 77.44944444444444 + - type: recall + value: 83.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bul-eng) + config: bul-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 66.8 + - type: f1 + value: 60.25746031746031 + - type: precision + value: 57.55250000000001 + - type: recall + value: 66.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cbk-eng) + config: cbk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 63.6 + - type: f1 + value: 56.73421356421356 + - type: precision + value: 54.02218253968254 + - type: recall + value: 63.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hun-eng) + config: hun-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 17.599999999999998 + - type: f1 + value: 13.17699134199134 + - type: precision + value: 11.77444805194805 + - type: recall + value: 17.599999999999998 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (uig-eng) + config: uig-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 2.0 + - type: f1 + value: 1.3126923076923078 + - type: precision + value: 1.104952380952381 + - type: recall + value: 2.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (rus-eng) + config: rus-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 88.3 + - type: f1 + value: 84.96333333333334 + - type: precision + value: 83.38333333333333 + - type: recall + value: 88.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (spa-eng) + config: spa-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 94.69999999999999 + - type: f1 + value: 93.12333333333333 + - type: precision + value: 92.375 + - type: recall + value: 94.69999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hye-eng) + config: hye-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 0.6738544474393532 + - type: f1 + value: 0.3690849566291394 + - type: precision + value: 0.3305452159899599 + - type: recall + value: 0.6738544474393532 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tel-eng) + config: tel-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 71.7948717948718 + - type: f1 + value: 65.37037037037037 + - type: precision + value: 62.46438746438747 + - type: recall + value: 71.7948717948718 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (afr-eng) + config: afr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 56.699999999999996 + - type: f1 + value: 50.58054945054945 + - type: precision + value: 48.313047619047616 + - type: recall + value: 56.699999999999996 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mon-eng) + config: mon-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.863636363636363 + - type: f1 + value: 10.948429096156369 + - type: precision + value: 10.227287994137523 + - type: recall + value: 13.863636363636363 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (arz-eng) + config: arz-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 62.473794549266245 + - type: f1 + value: 56.04172906059699 + - type: precision + value: 53.26694619147448 + - type: recall + value: 62.473794549266245 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hrv-eng) + config: hrv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 40.0 + - type: f1 + value: 34.62948179271708 + - type: precision + value: 32.699030910609864 + - type: recall + value: 40.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nov-eng) + config: nov-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 60.311284046692606 + - type: f1 + value: 54.06182447038479 + - type: precision + value: 51.757921067259595 + - type: recall + value: 60.311284046692606 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gsw-eng) + config: gsw-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 43.58974358974359 + - type: f1 + value: 37.042359350051655 + - type: precision + value: 34.75783475783476 + - type: recall + value: 43.58974358974359 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nds-eng) + config: nds-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 56.49999999999999 + - type: f1 + value: 49.471269841269844 + - type: precision + value: 46.742182539682545 + - type: recall + value: 56.49999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ukr-eng) + config: ukr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 71.5 + - type: f1 + value: 65.32880952380951 + - type: precision + value: 62.71261904761904 + - type: recall + value: 71.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (uzb-eng) + config: uzb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.448598130841122 + - type: f1 + value: 7.861361294691689 + - type: precision + value: 6.961045509526818 + - type: recall + value: 11.448598130841122 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lit-eng) + config: lit-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.5 + - type: f1 + value: 10.448586132968154 + - type: precision + value: 9.624691955878397 + - type: recall + value: 13.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ina-eng) + config: ina-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 82.19999999999999 + - type: f1 + value: 78.25366946778712 + - type: precision + value: 76.54291666666667 + - type: recall + value: 82.19999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lfn-eng) + config: lfn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 53.5 + - type: f1 + value: 47.48505411255411 + - type: precision + value: 45.29801587301587 + - type: recall + value: 53.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (zsm-eng) + config: zsm-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 61.1 + - type: f1 + value: 54.60758056758057 + - type: precision + value: 52.16455433455434 + - type: recall + value: 61.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ita-eng) + config: ita-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 85.1 + - type: f1 + value: 81.98506715506716 + - type: precision + value: 80.64754901960784 + - type: recall + value: 85.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cmn-eng) + config: cmn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 89.2 + - type: f1 + value: 86.13333333333333 + - type: precision + value: 84.65 + - type: recall + value: 89.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lvs-eng) + config: lvs-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.600000000000001 + - type: f1 + value: 10.721816580317723 + - type: precision + value: 9.97922024538847 + - type: recall + value: 13.600000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (glg-eng) + config: glg-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 79.0 + - type: f1 + value: 74.2652380952381 + - type: precision + value: 72.18690476190476 + - type: recall + value: 79.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ceb-eng) + config: ceb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 12.833333333333332 + - type: f1 + value: 10.45993265993266 + - type: precision + value: 9.849548907882243 + - type: recall + value: 12.833333333333332 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bre-eng) + config: bre-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 8.3 + - type: f1 + value: 5.457311371692176 + - type: precision + value: 4.8466941508148595 + - type: recall + value: 8.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ben-eng) + config: ben-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 26.3 + - type: f1 + value: 20.851341154819416 + - type: precision + value: 19.1173617945522 + - type: recall + value: 26.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swg-eng) + config: swg-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 41.964285714285715 + - type: f1 + value: 36.38605442176871 + - type: precision + value: 34.523809523809526 + - type: recall + value: 41.964285714285715 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (arq-eng) + config: arq-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 26.454445664105382 + - type: f1 + value: 20.67692765826684 + - type: precision + value: 18.684070229075715 + - type: recall + value: 26.454445664105382 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kab-eng) + config: kab-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 2.8000000000000003 + - type: f1 + value: 1.9487240537240536 + - type: precision + value: 1.7766582325720255 + - type: recall + value: 2.8000000000000003 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fra-eng) + config: fra-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 91.5 + - type: f1 + value: 89.39 + - type: precision + value: 88.425 + - type: recall + value: 91.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (por-eng) + config: por-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 91.5 + - type: f1 + value: 89.38333333333333 + - type: precision + value: 88.36666666666667 + - type: recall + value: 91.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tat-eng) + config: tat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.2 + - type: f1 + value: 6.672282438325198 + - type: precision + value: 6.046073589145276 + - type: recall + value: 9.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (oci-eng) + config: oci-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 45.2 + - type: f1 + value: 39.12095238095238 + - type: precision + value: 36.820952380952384 + - type: recall + value: 45.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pol-eng) + config: pol-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 86.8 + - type: f1 + value: 83.35000000000001 + - type: precision + value: 81.825 + - type: recall + value: 86.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (war-eng) + config: war-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.5 + - type: f1 + value: 10.66862856136998 + - type: precision + value: 9.845928551928552 + - type: recall + value: 13.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (aze-eng) + config: aze-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 33.4 + - type: f1 + value: 27.78153389993659 + - type: precision + value: 25.778055555555557 + - type: recall + value: 33.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (vie-eng) + config: vie-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 57.699999999999996 + - type: f1 + value: 50.440714285714286 + - type: precision + value: 47.64396825396825 + - type: recall + value: 57.699999999999996 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nno-eng) + config: nno-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 62.2 + - type: f1 + value: 56.0098625351257 + - type: precision + value: 53.691914098972916 + - type: recall + value: 62.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cha-eng) + config: cha-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 27.00729927007299 + - type: f1 + value: 22.798053527980535 + - type: precision + value: 21.107055961070557 + - type: recall + value: 27.00729927007299 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mhr-eng) + config: mhr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.2 + - type: f1 + value: 4.295544090473964 + - type: precision + value: 3.913153952193392 + - type: recall + value: 6.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dan-eng) + config: dan-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 77.10000000000001 + - type: f1 + value: 72.49333333333334 + - type: precision + value: 70.53368637110017 + - type: recall + value: 77.10000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ell-eng) + config: ell-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 15.2 + - type: f1 + value: 10.429591693330824 + - type: precision + value: 9.145801926831338 + - type: recall + value: 15.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (amh-eng) + config: amh-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 1.7857142857142856 + - type: f1 + value: 0.3635204081632653 + - type: precision + value: 0.205026455026455 + - type: recall + value: 1.7857142857142856 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pam-eng) + config: pam-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.4 + - type: f1 + value: 4.8412763053939525 + - type: precision + value: 4.444087810337809 + - type: recall + value: 6.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hsb-eng) + config: hsb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 43.47826086956522 + - type: f1 + value: 37.13266949291794 + - type: precision + value: 34.655332590115194 + - type: recall + value: 43.47826086956522 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (srp-eng) + config: srp-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 42.0 + - type: f1 + value: 35.412229437229435 + - type: precision + value: 32.907539682539685 + - type: recall + value: 42.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (epo-eng) + config: epo-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 36.0 + - type: f1 + value: 30.53874458874459 + - type: precision + value: 28.711192408382807 + - type: recall + value: 36.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kzj-eng) + config: kzj-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.9 + - type: f1 + value: 5.80190114561213 + - type: precision + value: 5.298527531836355 + - type: recall + value: 7.9 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (awa-eng) + config: awa-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 49.35064935064935 + - type: f1 + value: 41.57805638325119 + - type: precision + value: 38.87445887445887 + - type: recall + value: 49.35064935064935 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fao-eng) + config: fao-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 25.572519083969464 + - type: f1 + value: 21.338006776938073 + - type: precision + value: 20.194474736459465 + - type: recall + value: 25.572519083969464 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mal-eng) + config: mal-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 79.62154294032024 + - type: f1 + value: 74.47355652595827 + - type: precision + value: 72.2076661814653 + - type: recall + value: 79.62154294032024 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ile-eng) + config: ile-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 68.0 + - type: f1 + value: 61.80859649122807 + - type: precision + value: 59.30381381381381 + - type: recall + value: 68.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bos-eng) + config: bos-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 42.93785310734463 + - type: f1 + value: 36.72617201306135 + - type: precision + value: 34.72641059505466 + - type: recall + value: 42.93785310734463 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cor-eng) + config: cor-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 5.5 + - type: f1 + value: 3.8651658986175113 + - type: precision + value: 3.4432814407814405 + - type: recall + value: 5.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cat-eng) + config: cat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 69.19999999999999 + - type: f1 + value: 63.41880952380953 + - type: precision + value: 61.07913419913419 + - type: recall + value: 69.19999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (eus-eng) + config: eus-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 15.4 + - type: f1 + value: 11.672122577122575 + - type: precision + value: 10.59919974661354 + - type: recall + value: 15.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (yue-eng) + config: yue-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 58.5 + - type: f1 + value: 51.31880452880453 + - type: precision + value: 48.60550125313283 + - type: recall + value: 58.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swe-eng) + config: swe-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 89.3 + - type: f1 + value: 86.32666666666667 + - type: precision + value: 84.98333333333333 + - type: recall + value: 89.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dtp-eng) + config: dtp-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 5.7 + - type: f1 + value: 3.8739805216757546 + - type: precision + value: 3.4734608954367014 + - type: recall + value: 5.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kat-eng) + config: kat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 0.8042895442359249 + - type: f1 + value: 0.7596067917783735 + - type: precision + value: 0.7372654155495978 + - type: recall + value: 0.8042895442359249 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (jpn-eng) + config: jpn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 89.7 + - type: f1 + value: 86.92333333333333 + - type: precision + value: 85.64166666666667 + - type: recall + value: 89.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (csb-eng) + config: csb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 26.08695652173913 + - type: f1 + value: 20.517863778733343 + - type: precision + value: 18.901098901098898 + - type: recall + value: 26.08695652173913 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (xho-eng) + config: xho-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 12.676056338028168 + - type: f1 + value: 9.526324614352783 + - type: precision + value: 9.006292657908235 + - type: recall + value: 12.676056338028168 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (orv-eng) + config: orv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 24.910179640718564 + - type: f1 + value: 19.645099411566473 + - type: precision + value: 17.676076418591386 + - type: recall + value: 24.910179640718564 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ind-eng) + config: ind-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 61.4 + - type: f1 + value: 54.64269841269841 + - type: precision + value: 51.981071428571425 + - type: recall + value: 61.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tuk-eng) + config: tuk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.330049261083744 + - type: f1 + value: 9.610016420361248 + - type: precision + value: 9.123781574258464 + - type: recall + value: 11.330049261083744 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (max-eng) + config: max-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 27.816901408450708 + - type: f1 + value: 22.51925345174495 + - type: precision + value: 21.10468365750056 + - type: recall + value: 27.816901408450708 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swh-eng) + config: swh-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.282051282051283 + - type: f1 + value: 7.777167097237831 + - type: precision + value: 7.050109879436802 + - type: recall + value: 11.282051282051283 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hin-eng) + config: hin-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 86.0 + - type: f1 + value: 82.05857142857143 + - type: precision + value: 80.25 + - type: recall + value: 86.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dsb-eng) + config: dsb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 34.44676409185804 + - type: f1 + value: 28.296517215097587 + - type: precision + value: 26.16624956236465 + - type: recall + value: 34.44676409185804 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ber-eng) + config: ber-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.199999999999999 + - type: f1 + value: 5.500051631938041 + - type: precision + value: 5.164411510424442 + - type: recall + value: 7.199999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tam-eng) + config: tam-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 71.9869706840391 + - type: f1 + value: 65.79339227547696 + - type: precision + value: 63.16503800217155 + - type: recall + value: 71.9869706840391 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (slk-eng) + config: slk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 70.89999999999999 + - type: f1 + value: 65.4152380952381 + - type: precision + value: 63.106666666666655 + - type: recall + value: 70.89999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tgl-eng) + config: tgl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 21.0 + - type: f1 + value: 17.86438197644649 + - type: precision + value: 16.84469948469949 + - type: recall + value: 21.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ast-eng) + config: ast-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 62.20472440944882 + - type: f1 + value: 55.81364829396325 + - type: precision + value: 53.262092238470196 + - type: recall + value: 62.20472440944882 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mkd-eng) + config: mkd-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 41.8 + - type: f1 + value: 34.724603174603175 + - type: precision + value: 32.040277777777774 + - type: recall + value: 41.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (khm-eng) + config: khm-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 0.41551246537396125 + - type: f1 + value: 0.3462603878116343 + - type: precision + value: 0.32317636195752536 + - type: recall + value: 0.41551246537396125 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ces-eng) + config: ces-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 85.6 + - type: f1 + value: 81.81333333333333 + - type: precision + value: 80.08333333333334 + - type: recall + value: 85.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tzl-eng) + config: tzl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 31.73076923076923 + - type: f1 + value: 26.097374847374844 + - type: precision + value: 24.31891025641026 + - type: recall + value: 31.73076923076923 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (urd-eng) + config: urd-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.6 + - type: f1 + value: 6.598392371412457 + - type: precision + value: 5.855494356434758 + - type: recall + value: 9.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ara-eng) + config: ara-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 83.5 + - type: f1 + value: 79.65190476190476 + - type: precision + value: 77.875 + - type: recall + value: 83.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kor-eng) + config: kor-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 80.5 + - type: f1 + value: 75.75999999999999 + - type: precision + value: 73.60333333333332 + - type: recall + value: 80.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (yid-eng) + config: yid-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 2.1226415094339623 + - type: f1 + value: 1.4622641509433962 + - type: precision + value: 1.2637578616352203 + - type: recall + value: 2.1226415094339623 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fin-eng) + config: fin-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 23.0 + - type: f1 + value: 18.111780719280716 + - type: precision + value: 16.497738095238095 + - type: recall + value: 23.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tha-eng) + config: tha-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 4.562043795620438 + - type: f1 + value: 3.1632119907667358 + - type: precision + value: 2.8806772100567724 + - type: recall + value: 4.562043795620438 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (wuu-eng) + config: wuu-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 75.9 + - type: f1 + value: 70.57690476190476 + - type: precision + value: 68.19761904761904 + - type: recall + value: 75.9 + - task: + type: Retrieval + dataset: + type: webis-touche2020 + name: MTEB Touche2020 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 2.804 + - type: map_at_10 + value: 11.267000000000001 + - type: map_at_100 + value: 17.034 + - type: map_at_1000 + value: 18.733 + - type: map_at_3 + value: 6.071 + - type: map_at_5 + value: 8.187 + - type: mrr_at_1 + value: 34.694 + - type: mrr_at_10 + value: 50.504000000000005 + - type: mrr_at_100 + value: 51.162 + - type: mrr_at_1000 + value: 51.162 + - type: mrr_at_3 + value: 45.918 + - type: mrr_at_5 + value: 49.082 + - type: ndcg_at_1 + value: 33.672999999999995 + - type: ndcg_at_10 + value: 27.478 + - type: ndcg_at_100 + value: 37.961 + - type: ndcg_at_1000 + value: 50.117 + - type: ndcg_at_3 + value: 30.156 + - type: ndcg_at_5 + value: 29.293999999999997 + - type: precision_at_1 + value: 34.694 + - type: precision_at_10 + value: 24.082 + - type: precision_at_100 + value: 7.632999999999999 + - type: precision_at_1000 + value: 1.569 + - type: precision_at_3 + value: 30.612000000000002 + - type: precision_at_5 + value: 29.387999999999998 + - type: recall_at_1 + value: 2.804 + - type: recall_at_10 + value: 17.785 + - type: recall_at_100 + value: 47.452 + - type: recall_at_1000 + value: 84.687 + - type: recall_at_3 + value: 6.9190000000000005 + - type: recall_at_5 + value: 10.807 + - task: + type: Classification + dataset: + type: mteb/toxic_conversations_50k + name: MTEB ToxicConversationsClassification + config: default + split: test + revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c + metrics: + - type: accuracy + value: 74.5162 + - type: ap + value: 15.022137849208509 + - type: f1 + value: 56.77914300422838 + - task: + type: Classification + dataset: + type: mteb/tweet_sentiment_extraction + name: MTEB TweetSentimentExtractionClassification + config: default + split: test + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + metrics: + - type: accuracy + value: 59.589700056593095 + - type: f1 + value: 59.93893560752363 + - task: + type: Clustering + dataset: + type: mteb/twentynewsgroups-clustering + name: MTEB TwentyNewsgroupsClustering + config: default + split: test + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + metrics: + - type: v_measure + value: 40.11538634360855 + - task: + type: PairClassification + dataset: + type: mteb/twittersemeval2015-pairclassification + name: MTEB TwitterSemEval2015 + config: default + split: test + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + metrics: + - type: cos_sim_accuracy + value: 83.97806520832091 + - type: cos_sim_ap + value: 67.80381341664686 + - type: cos_sim_f1 + value: 63.01665268958908 + - type: cos_sim_precision + value: 57.713407943822695 + - type: cos_sim_recall + value: 69.39313984168865 + - type: dot_accuracy + value: 83.9899862907552 + - type: dot_ap + value: 67.80914960711299 + - type: dot_f1 + value: 63.0287144048612 + - type: dot_precision + value: 57.46252444058223 + - type: dot_recall + value: 69.78891820580475 + - type: euclidean_accuracy + value: 83.9601835846695 + - type: euclidean_ap + value: 67.79862461635126 + - type: euclidean_f1 + value: 63.02426882389545 + - type: euclidean_precision + value: 59.64664310954063 + - type: euclidean_recall + value: 66.80738786279683 + - type: manhattan_accuracy + value: 83.94230196101806 + - type: manhattan_ap + value: 67.78560087328111 + - type: manhattan_f1 + value: 63.10622881851117 + - type: manhattan_precision + value: 56.63939584644431 + - type: manhattan_recall + value: 71.2401055408971 + - type: max_accuracy + value: 83.9899862907552 + - type: max_ap + value: 67.80914960711299 + - type: max_f1 + value: 63.10622881851117 + - task: + type: PairClassification + dataset: + type: mteb/twitterurlcorpus-pairclassification + name: MTEB TwitterURLCorpus + config: default + split: test + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + metrics: + - type: cos_sim_accuracy + value: 89.04994760740482 + - type: cos_sim_ap + value: 85.71231674852108 + - type: cos_sim_f1 + value: 78.92350867093619 + - type: cos_sim_precision + value: 74.07807645549101 + - type: cos_sim_recall + value: 84.44718201416693 + - type: dot_accuracy + value: 89.05188807389295 + - type: dot_ap + value: 85.71776365526502 + - type: dot_f1 + value: 78.92055922835156 + - type: dot_precision + value: 74.34152317430069 + - type: dot_recall + value: 84.10070834616569 + - type: euclidean_accuracy + value: 89.05188807389295 + - type: euclidean_ap + value: 85.7114644968015 + - type: euclidean_f1 + value: 78.9458525345622 + - type: euclidean_precision + value: 74.14119556397078 + - type: euclidean_recall + value: 84.41638435478903 + - type: manhattan_accuracy + value: 89.06547133930997 + - type: manhattan_ap + value: 85.70658730333459 + - type: manhattan_f1 + value: 78.91009741543552 + - type: manhattan_precision + value: 74.00714719169308 + - type: manhattan_recall + value: 84.5087773329227 + - type: max_accuracy + value: 89.06547133930997 + - type: max_ap + value: 85.71776365526502 + - type: max_f1 + value: 78.9458525345622 +--- + +## Bedrock Titan Text Embeddings v2 +This repository contains the MTEB scores and usage examples of Bedrock Titan Text Embeddings v2. You can use the embedding model either via the Bedrock streaming API or via Bedrock's batch jobs. For RAG use cases we recommend the former to embed queries during search (latency optimized) and the latter to index corpus (throughput optimized). + +## Using Bedrock's streaming API + +```python +import json +import boto3 +class TitanEmbeddings(object): + accept = "application/json" + content_type = "application/json" + + def __init__(self, model_id="amazon.titan-embed-text-v2"): + self.bedrock = boto3.client(service_name='bedrock-runtime') + self.model_id = model_id + def __call__(self, text, dimensions, normalize=True): + """ + Returns Titan Embeddings + Args: + text (str): text to embed + dimensions (int): Number of output dimensions. + normalize (bool): Whether to return the normalized embedding or not. + Return: + List[float]: Embedding + + """ + body = json.dumps({ + "inputText": text, + "dimensions": dimensions, + "normalize": normalize + }) + response = self.bedrock.invoke_model( + body=body, modelId=self.model_id, accept=self.accept, contentType=self.content_type + ) + response_body = json.loads(response.get('body').read()) + return response_body['embedding'] + +if __name__ == '__main__': + """ + Entrypoint for Amazon Titan Embeddings V2 - Text example. + """ + dimensions = 1024 + normalize = True + + titan_embeddings_v2 = TitanEmbeddings(model_id="amazon.titan-embed-text-v2") + + input_text = "What are the different services that you offer?" + embedding = titan_embeddings_v2(input_text, dimensions, normalize) + + print(f"{input_text=}") + print(f"{embedding[:10]=}") + +``` + + +## Using Bedrock's batch jobs + +```python +import requests +from aws_requests_auth.boto_utils import BotoAWSRequestsAuth + +region = "us-east-1" +base_uri = f"preprod.{region}.controlplane.bedrock.aws.dev" +batch_job_uri = f"https://{base_uri}/model-invocation-job/" + +role_arn = "arn:aws:iam::750485122747:role/BedrockServiceRole" +payload = { + "inputDataConfig": { + "s3InputDataConfig": { + "s3Uri": "s3://gnovack-bedrock-data/batch-input/", + "s3InputFormat": "JSONL" + } + }, + "jobName": "embeddings-v2-test-job-9", + "modelId": "amazon.titan-embed-text-v2", + "outputDataConfig": { + "s3OutputDataConfig": { + "s3Uri": "s3://gnovack-bedrock-data/batch-output/" + } + }, + "roleArn": role_arn +} + +request_auth = BotoAWSRequestsAuth( + aws_host=base_uri, + aws_region=region, + aws_service="bedrock" +) + + +response= requests.request("POST", batch_job_uri, json=payload, auth=request_auth) +print(response.json()) +``` \ No newline at end of file