|
--- |
|
license: bigscience-bloom-rail-1.0 |
|
language: |
|
- ak |
|
- ar |
|
- as |
|
- bm |
|
- bn |
|
- ca |
|
- code |
|
- en |
|
- es |
|
- eu |
|
- fon |
|
- fr |
|
- gu |
|
- hi |
|
- id |
|
- ig |
|
- ki |
|
- kn |
|
- lg |
|
- ln |
|
- ml |
|
- mr |
|
- ne |
|
- nso |
|
- ny |
|
- or |
|
- pa |
|
- pt |
|
- rn |
|
- rw |
|
- sn |
|
- st |
|
- sw |
|
- ta |
|
- te |
|
- tn |
|
- ts |
|
- tum |
|
- tw |
|
- ur |
|
- vi |
|
- wo |
|
- xh |
|
- yo |
|
- zh |
|
- zhs |
|
- zht |
|
- zu |
|
tags: |
|
- mteb |
|
model-index: |
|
- name: udever-bloom-560m |
|
results: |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 25.170024237678657 |
|
- type: cos_sim_spearman |
|
value: 25.32025098111752 |
|
- type: euclidean_pearson |
|
value: 25.34284673812859 |
|
- type: euclidean_spearman |
|
value: 25.52812937004611 |
|
- type: manhattan_pearson |
|
value: 25.734179522960822 |
|
- type: manhattan_spearman |
|
value: 25.92247507041032 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 32.3359541791282 |
|
- type: cos_sim_spearman |
|
value: 33.45815274836323 |
|
- type: euclidean_pearson |
|
value: 35.14748229440635 |
|
- type: euclidean_spearman |
|
value: 33.377829932851334 |
|
- type: manhattan_pearson |
|
value: 35.359130773295625 |
|
- type: manhattan_spearman |
|
value: 33.524469762932426 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en) |
|
config: en |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 72.35820895522389 |
|
- type: ap |
|
value: 35.45566303125099 |
|
- type: f1 |
|
value: 66.49474786522534 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (de) |
|
config: de |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 66.423982869379 |
|
- type: ap |
|
value: 78.32781372746805 |
|
- type: f1 |
|
value: 64.24959400774807 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en-ext) |
|
config: en-ext |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 73.65817091454274 |
|
- type: ap |
|
value: 21.73416645163647 |
|
- type: f1 |
|
value: 60.52120070712094 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (ja) |
|
config: ja |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 56.86295503211991 |
|
- type: ap |
|
value: 12.906256075113513 |
|
- type: f1 |
|
value: 46.68625513679152 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_polarity |
|
name: MTEB AmazonPolarityClassification |
|
config: default |
|
split: test |
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
metrics: |
|
- type: accuracy |
|
value: 83.8095 |
|
- type: ap |
|
value: 78.5195717101614 |
|
- type: f1 |
|
value: 83.74169093676316 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (en) |
|
config: en |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 38.97 |
|
- type: f1 |
|
value: 38.57853211177342 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (de) |
|
config: de |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 26.846000000000004 |
|
- type: f1 |
|
value: 26.473886891677306 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (es) |
|
config: es |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 38.974 |
|
- type: f1 |
|
value: 38.31719230291287 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (fr) |
|
config: fr |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 38.38799999999999 |
|
- type: f1 |
|
value: 37.53319978613875 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (ja) |
|
config: ja |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 28.311999999999998 |
|
- type: f1 |
|
value: 27.988313617729755 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (zh) |
|
config: zh |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 35.704 |
|
- type: f1 |
|
value: 34.863182924437254 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 21.053 |
|
- type: map_at_10 |
|
value: 35.811 |
|
- type: map_at_100 |
|
value: 37.035000000000004 |
|
- type: map_at_1000 |
|
value: 37.055 |
|
- type: map_at_3 |
|
value: 30.666 |
|
- type: map_at_5 |
|
value: 33.525 |
|
- type: mrr_at_1 |
|
value: 21.266 |
|
- type: mrr_at_10 |
|
value: 35.906 |
|
- type: mrr_at_100 |
|
value: 37.122 |
|
- type: mrr_at_1000 |
|
value: 37.141999999999996 |
|
- type: mrr_at_3 |
|
value: 30.714000000000002 |
|
- type: mrr_at_5 |
|
value: 33.576 |
|
- type: ndcg_at_1 |
|
value: 21.053 |
|
- type: ndcg_at_10 |
|
value: 44.545 |
|
- type: ndcg_at_100 |
|
value: 49.844 |
|
- type: ndcg_at_1000 |
|
value: 50.298 |
|
- type: ndcg_at_3 |
|
value: 33.889 |
|
- type: ndcg_at_5 |
|
value: 39.059 |
|
- type: precision_at_1 |
|
value: 21.053 |
|
- type: precision_at_10 |
|
value: 7.269 |
|
- type: precision_at_100 |
|
value: 0.96 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 14.414 |
|
- type: precision_at_5 |
|
value: 11.166 |
|
- type: recall_at_1 |
|
value: 21.053 |
|
- type: recall_at_10 |
|
value: 72.688 |
|
- type: recall_at_100 |
|
value: 96.017 |
|
- type: recall_at_1000 |
|
value: 99.431 |
|
- type: recall_at_3 |
|
value: 43.242999999999995 |
|
- type: recall_at_5 |
|
value: 55.832 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-p2p |
|
name: MTEB ArxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
metrics: |
|
- type: v_measure |
|
value: 40.26646269393896 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-s2s |
|
name: MTEB ArxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
metrics: |
|
- type: v_measure |
|
value: 32.00218289816601 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
metrics: |
|
- type: map |
|
value: 57.381567373603424 |
|
- type: mrr |
|
value: 70.09431473420392 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.14803223261677 |
|
- type: cos_sim_spearman |
|
value: 84.43626128689064 |
|
- type: euclidean_pearson |
|
value: 85.03130036472703 |
|
- type: euclidean_spearman |
|
value: 84.05974668365359 |
|
- type: manhattan_pearson |
|
value: 85.59339889467545 |
|
- type: manhattan_spearman |
|
value: 83.86938090025696 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 44.19468290937555 |
|
- type: cos_sim_spearman |
|
value: 43.93025426799595 |
|
- type: euclidean_pearson |
|
value: 45.273900549350735 |
|
- type: euclidean_spearman |
|
value: 45.07419415738924 |
|
- type: manhattan_pearson |
|
value: 45.469211385235376 |
|
- type: manhattan_spearman |
|
value: 45.27440191151001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (de-en) |
|
config: de-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 11.440501043841337 |
|
- type: f1 |
|
value: 11.295895880968951 |
|
- type: precision |
|
value: 11.237446950317073 |
|
- type: recall |
|
value: 11.440501043841337 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (fr-en) |
|
config: fr-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 96.53312788906008 |
|
- type: f1 |
|
value: 96.18093770636143 |
|
- type: precision |
|
value: 96.00667693888035 |
|
- type: recall |
|
value: 96.53312788906008 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (ru-en) |
|
config: ru-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 1.6972635954277795 |
|
- type: f1 |
|
value: 1.5885146938143124 |
|
- type: precision |
|
value: 1.5581125970067466 |
|
- type: recall |
|
value: 1.6972635954277795 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (zh-en) |
|
config: zh-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 96.31384939441811 |
|
- type: f1 |
|
value: 96.15587151132175 |
|
- type: precision |
|
value: 96.07688256977357 |
|
- type: recall |
|
value: 96.31384939441811 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/banking77 |
|
name: MTEB Banking77Classification |
|
config: default |
|
split: test |
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
metrics: |
|
- type: accuracy |
|
value: 80.97402597402598 |
|
- type: f1 |
|
value: 80.88177660652944 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-p2p |
|
name: MTEB BiorxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
metrics: |
|
- type: v_measure |
|
value: 33.266950159712465 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-s2s |
|
name: MTEB BiorxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
metrics: |
|
- type: v_measure |
|
value: 28.65092446021672 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 35.21075820650184 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 35.121931960714484 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 63.41256934884578 |
|
- type: mrr |
|
value: 68.6492857142857 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 63.663067375541104 |
|
- type: mrr |
|
value: 68.92075396825396 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackAndroidRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.997 |
|
- type: map_at_10 |
|
value: 35.477 |
|
- type: map_at_100 |
|
value: 36.722 |
|
- type: map_at_1000 |
|
value: 36.849 |
|
- type: map_at_3 |
|
value: 32.083 |
|
- type: map_at_5 |
|
value: 33.884 |
|
- type: mrr_at_1 |
|
value: 32.046 |
|
- type: mrr_at_10 |
|
value: 41.455999999999996 |
|
- type: mrr_at_100 |
|
value: 42.214 |
|
- type: mrr_at_1000 |
|
value: 42.268 |
|
- type: mrr_at_3 |
|
value: 38.722 |
|
- type: mrr_at_5 |
|
value: 40.266999999999996 |
|
- type: ndcg_at_1 |
|
value: 32.046 |
|
- type: ndcg_at_10 |
|
value: 41.705999999999996 |
|
- type: ndcg_at_100 |
|
value: 46.695 |
|
- type: ndcg_at_1000 |
|
value: 49.128 |
|
- type: ndcg_at_3 |
|
value: 36.6 |
|
- type: ndcg_at_5 |
|
value: 38.725 |
|
- type: precision_at_1 |
|
value: 32.046 |
|
- type: precision_at_10 |
|
value: 8.197000000000001 |
|
- type: precision_at_100 |
|
value: 1.323 |
|
- type: precision_at_1000 |
|
value: 0.183 |
|
- type: precision_at_3 |
|
value: 18.073 |
|
- type: precision_at_5 |
|
value: 13.047 |
|
- type: recall_at_1 |
|
value: 24.997 |
|
- type: recall_at_10 |
|
value: 54.013 |
|
- type: recall_at_100 |
|
value: 75.29400000000001 |
|
- type: recall_at_1000 |
|
value: 91.611 |
|
- type: recall_at_3 |
|
value: 38.627 |
|
- type: recall_at_5 |
|
value: 45.019999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackEnglishRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.194 |
|
- type: map_at_10 |
|
value: 30.076000000000004 |
|
- type: map_at_100 |
|
value: 31.0 |
|
- type: map_at_1000 |
|
value: 31.125999999999998 |
|
- type: map_at_3 |
|
value: 28.137 |
|
- type: map_at_5 |
|
value: 29.206 |
|
- type: mrr_at_1 |
|
value: 28.535 |
|
- type: mrr_at_10 |
|
value: 34.833999999999996 |
|
- type: mrr_at_100 |
|
value: 35.504999999999995 |
|
- type: mrr_at_1000 |
|
value: 35.57 |
|
- type: mrr_at_3 |
|
value: 33.089 |
|
- type: mrr_at_5 |
|
value: 34.115 |
|
- type: ndcg_at_1 |
|
value: 28.535 |
|
- type: ndcg_at_10 |
|
value: 34.285 |
|
- type: ndcg_at_100 |
|
value: 38.286 |
|
- type: ndcg_at_1000 |
|
value: 41.007 |
|
- type: ndcg_at_3 |
|
value: 31.395 |
|
- type: ndcg_at_5 |
|
value: 32.687 |
|
- type: precision_at_1 |
|
value: 28.535 |
|
- type: precision_at_10 |
|
value: 6.166 |
|
- type: precision_at_100 |
|
value: 1.042 |
|
- type: precision_at_1000 |
|
value: 0.155 |
|
- type: precision_at_3 |
|
value: 14.862 |
|
- type: precision_at_5 |
|
value: 10.331 |
|
- type: recall_at_1 |
|
value: 23.194 |
|
- type: recall_at_10 |
|
value: 41.648 |
|
- type: recall_at_100 |
|
value: 58.999 |
|
- type: recall_at_1000 |
|
value: 77.46300000000001 |
|
- type: recall_at_3 |
|
value: 32.931 |
|
- type: recall_at_5 |
|
value: 36.736999999999995 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGamingRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 31.899 |
|
- type: map_at_10 |
|
value: 42.657000000000004 |
|
- type: map_at_100 |
|
value: 43.717 |
|
- type: map_at_1000 |
|
value: 43.79 |
|
- type: map_at_3 |
|
value: 39.635 |
|
- type: map_at_5 |
|
value: 41.538000000000004 |
|
- type: mrr_at_1 |
|
value: 36.864999999999995 |
|
- type: mrr_at_10 |
|
value: 46.137 |
|
- type: mrr_at_100 |
|
value: 46.946 |
|
- type: mrr_at_1000 |
|
value: 46.986 |
|
- type: mrr_at_3 |
|
value: 43.469 |
|
- type: mrr_at_5 |
|
value: 45.262 |
|
- type: ndcg_at_1 |
|
value: 36.864999999999995 |
|
- type: ndcg_at_10 |
|
value: 48.164 |
|
- type: ndcg_at_100 |
|
value: 52.769999999999996 |
|
- type: ndcg_at_1000 |
|
value: 54.393 |
|
- type: ndcg_at_3 |
|
value: 42.887 |
|
- type: ndcg_at_5 |
|
value: 45.871 |
|
- type: precision_at_1 |
|
value: 36.864999999999995 |
|
- type: precision_at_10 |
|
value: 7.843 |
|
- type: precision_at_100 |
|
value: 1.102 |
|
- type: precision_at_1000 |
|
value: 0.13 |
|
- type: precision_at_3 |
|
value: 19.352 |
|
- type: precision_at_5 |
|
value: 13.618 |
|
- type: recall_at_1 |
|
value: 31.899 |
|
- type: recall_at_10 |
|
value: 61.131 |
|
- type: recall_at_100 |
|
value: 81.504 |
|
- type: recall_at_1000 |
|
value: 93.146 |
|
- type: recall_at_3 |
|
value: 46.971000000000004 |
|
- type: recall_at_5 |
|
value: 54.42399999999999 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGisRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.621000000000002 |
|
- type: map_at_10 |
|
value: 23.621 |
|
- type: map_at_100 |
|
value: 24.636 |
|
- type: map_at_1000 |
|
value: 24.739 |
|
- type: map_at_3 |
|
value: 21.623 |
|
- type: map_at_5 |
|
value: 22.511 |
|
- type: mrr_at_1 |
|
value: 19.096 |
|
- type: mrr_at_10 |
|
value: 25.288 |
|
- type: mrr_at_100 |
|
value: 26.238 |
|
- type: mrr_at_1000 |
|
value: 26.314 |
|
- type: mrr_at_3 |
|
value: 23.202 |
|
- type: mrr_at_5 |
|
value: 24.213 |
|
- type: ndcg_at_1 |
|
value: 19.096 |
|
- type: ndcg_at_10 |
|
value: 27.529999999999998 |
|
- type: ndcg_at_100 |
|
value: 32.763 |
|
- type: ndcg_at_1000 |
|
value: 35.538 |
|
- type: ndcg_at_3 |
|
value: 23.362 |
|
- type: ndcg_at_5 |
|
value: 24.961 |
|
- type: precision_at_1 |
|
value: 19.096 |
|
- type: precision_at_10 |
|
value: 4.417999999999999 |
|
- type: precision_at_100 |
|
value: 0.739 |
|
- type: precision_at_1000 |
|
value: 0.10300000000000001 |
|
- type: precision_at_3 |
|
value: 9.981 |
|
- type: precision_at_5 |
|
value: 6.959999999999999 |
|
- type: recall_at_1 |
|
value: 17.621000000000002 |
|
- type: recall_at_10 |
|
value: 38.079 |
|
- type: recall_at_100 |
|
value: 62.499 |
|
- type: recall_at_1000 |
|
value: 83.783 |
|
- type: recall_at_3 |
|
value: 26.687 |
|
- type: recall_at_5 |
|
value: 30.459000000000003 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackMathematicaRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 11.019 |
|
- type: map_at_10 |
|
value: 15.869 |
|
- type: map_at_100 |
|
value: 17.078 |
|
- type: map_at_1000 |
|
value: 17.205000000000002 |
|
- type: map_at_3 |
|
value: 13.794 |
|
- type: map_at_5 |
|
value: 14.814 |
|
- type: mrr_at_1 |
|
value: 13.930000000000001 |
|
- type: mrr_at_10 |
|
value: 19.172 |
|
- type: mrr_at_100 |
|
value: 20.325 |
|
- type: mrr_at_1000 |
|
value: 20.415 |
|
- type: mrr_at_3 |
|
value: 17.122999999999998 |
|
- type: mrr_at_5 |
|
value: 18.124000000000002 |
|
- type: ndcg_at_1 |
|
value: 13.930000000000001 |
|
- type: ndcg_at_10 |
|
value: 19.646 |
|
- type: ndcg_at_100 |
|
value: 25.684 |
|
- type: ndcg_at_1000 |
|
value: 29.14 |
|
- type: ndcg_at_3 |
|
value: 15.614 |
|
- type: ndcg_at_5 |
|
value: 17.247 |
|
- type: precision_at_1 |
|
value: 13.930000000000001 |
|
- type: precision_at_10 |
|
value: 3.868 |
|
- type: precision_at_100 |
|
value: 0.8 |
|
- type: precision_at_1000 |
|
value: 0.125 |
|
- type: precision_at_3 |
|
value: 7.420999999999999 |
|
- type: precision_at_5 |
|
value: 5.672 |
|
- type: recall_at_1 |
|
value: 11.019 |
|
- type: recall_at_10 |
|
value: 28.116000000000003 |
|
- type: recall_at_100 |
|
value: 54.794 |
|
- type: recall_at_1000 |
|
value: 79.838 |
|
- type: recall_at_3 |
|
value: 17.124 |
|
- type: recall_at_5 |
|
value: 21.086 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackPhysicsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.791 |
|
- type: map_at_10 |
|
value: 33.442 |
|
- type: map_at_100 |
|
value: 34.719 |
|
- type: map_at_1000 |
|
value: 34.849000000000004 |
|
- type: map_at_3 |
|
value: 30.885 |
|
- type: map_at_5 |
|
value: 32.245000000000005 |
|
- type: mrr_at_1 |
|
value: 30.606 |
|
- type: mrr_at_10 |
|
value: 38.922000000000004 |
|
- type: mrr_at_100 |
|
value: 39.822 |
|
- type: mrr_at_1000 |
|
value: 39.881 |
|
- type: mrr_at_3 |
|
value: 36.622 |
|
- type: mrr_at_5 |
|
value: 37.907000000000004 |
|
- type: ndcg_at_1 |
|
value: 30.606 |
|
- type: ndcg_at_10 |
|
value: 38.867000000000004 |
|
- type: ndcg_at_100 |
|
value: 44.364 |
|
- type: ndcg_at_1000 |
|
value: 47.073 |
|
- type: ndcg_at_3 |
|
value: 34.63 |
|
- type: ndcg_at_5 |
|
value: 36.479 |
|
- type: precision_at_1 |
|
value: 30.606 |
|
- type: precision_at_10 |
|
value: 7.0360000000000005 |
|
- type: precision_at_100 |
|
value: 1.174 |
|
- type: precision_at_1000 |
|
value: 0.16 |
|
- type: precision_at_3 |
|
value: 16.522000000000002 |
|
- type: precision_at_5 |
|
value: 11.588 |
|
- type: recall_at_1 |
|
value: 24.791 |
|
- type: recall_at_10 |
|
value: 49.736000000000004 |
|
- type: recall_at_100 |
|
value: 72.67099999999999 |
|
- type: recall_at_1000 |
|
value: 91.29599999999999 |
|
- type: recall_at_3 |
|
value: 37.345 |
|
- type: recall_at_5 |
|
value: 42.400999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackProgrammersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 20.669999999999998 |
|
- type: map_at_10 |
|
value: 28.605000000000004 |
|
- type: map_at_100 |
|
value: 29.769000000000002 |
|
- type: map_at_1000 |
|
value: 29.881999999999998 |
|
- type: map_at_3 |
|
value: 25.886 |
|
- type: map_at_5 |
|
value: 27.317999999999998 |
|
- type: mrr_at_1 |
|
value: 25.457 |
|
- type: mrr_at_10 |
|
value: 33.423 |
|
- type: mrr_at_100 |
|
value: 34.269 |
|
- type: mrr_at_1000 |
|
value: 34.336 |
|
- type: mrr_at_3 |
|
value: 30.974 |
|
- type: mrr_at_5 |
|
value: 32.23 |
|
- type: ndcg_at_1 |
|
value: 25.457 |
|
- type: ndcg_at_10 |
|
value: 33.785 |
|
- type: ndcg_at_100 |
|
value: 39.145 |
|
- type: ndcg_at_1000 |
|
value: 41.772 |
|
- type: ndcg_at_3 |
|
value: 29.014 |
|
- type: ndcg_at_5 |
|
value: 31.019999999999996 |
|
- type: precision_at_1 |
|
value: 25.457 |
|
- type: precision_at_10 |
|
value: 6.2330000000000005 |
|
- type: precision_at_100 |
|
value: 1.045 |
|
- type: precision_at_1000 |
|
value: 0.145 |
|
- type: precision_at_3 |
|
value: 13.813 |
|
- type: precision_at_5 |
|
value: 9.863 |
|
- type: recall_at_1 |
|
value: 20.669999999999998 |
|
- type: recall_at_10 |
|
value: 44.651 |
|
- type: recall_at_100 |
|
value: 68.037 |
|
- type: recall_at_1000 |
|
value: 86.282 |
|
- type: recall_at_3 |
|
value: 31.381999999999998 |
|
- type: recall_at_5 |
|
value: 36.778 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 19.796583333333338 |
|
- type: map_at_10 |
|
value: 26.900166666666664 |
|
- type: map_at_100 |
|
value: 27.956583333333334 |
|
- type: map_at_1000 |
|
value: 28.08083333333333 |
|
- type: map_at_3 |
|
value: 24.598416666666665 |
|
- type: map_at_5 |
|
value: 25.81791666666667 |
|
- type: mrr_at_1 |
|
value: 23.68591666666667 |
|
- type: mrr_at_10 |
|
value: 30.65558333333333 |
|
- type: mrr_at_100 |
|
value: 31.503583333333335 |
|
- type: mrr_at_1000 |
|
value: 31.576083333333333 |
|
- type: mrr_at_3 |
|
value: 28.50525 |
|
- type: mrr_at_5 |
|
value: 29.690666666666665 |
|
- type: ndcg_at_1 |
|
value: 23.68591666666667 |
|
- type: ndcg_at_10 |
|
value: 31.425000000000004 |
|
- type: ndcg_at_100 |
|
value: 36.34316666666666 |
|
- type: ndcg_at_1000 |
|
value: 39.164249999999996 |
|
- type: ndcg_at_3 |
|
value: 27.330083333333338 |
|
- type: ndcg_at_5 |
|
value: 29.14408333333333 |
|
- type: precision_at_1 |
|
value: 23.68591666666667 |
|
- type: precision_at_10 |
|
value: 5.5862500000000015 |
|
- type: precision_at_100 |
|
value: 0.9571666666666666 |
|
- type: precision_at_1000 |
|
value: 0.13866666666666666 |
|
- type: precision_at_3 |
|
value: 12.663499999999999 |
|
- type: precision_at_5 |
|
value: 9.035333333333332 |
|
- type: recall_at_1 |
|
value: 19.796583333333338 |
|
- type: recall_at_10 |
|
value: 41.289416666666675 |
|
- type: recall_at_100 |
|
value: 63.251250000000006 |
|
- type: recall_at_1000 |
|
value: 83.4515 |
|
- type: recall_at_3 |
|
value: 29.727916666666665 |
|
- type: recall_at_5 |
|
value: 34.45824999999999 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackStatsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 16.121 |
|
- type: map_at_10 |
|
value: 22.104 |
|
- type: map_at_100 |
|
value: 23.003 |
|
- type: map_at_1000 |
|
value: 23.108 |
|
- type: map_at_3 |
|
value: 20.233 |
|
- type: map_at_5 |
|
value: 21.186 |
|
- type: mrr_at_1 |
|
value: 18.865000000000002 |
|
- type: mrr_at_10 |
|
value: 24.951 |
|
- type: mrr_at_100 |
|
value: 25.779000000000003 |
|
- type: mrr_at_1000 |
|
value: 25.863999999999997 |
|
- type: mrr_at_3 |
|
value: 23.083000000000002 |
|
- type: mrr_at_5 |
|
value: 24.049 |
|
- type: ndcg_at_1 |
|
value: 18.865000000000002 |
|
- type: ndcg_at_10 |
|
value: 26.031 |
|
- type: ndcg_at_100 |
|
value: 30.589 |
|
- type: ndcg_at_1000 |
|
value: 33.565 |
|
- type: ndcg_at_3 |
|
value: 22.369 |
|
- type: ndcg_at_5 |
|
value: 23.932000000000002 |
|
- type: precision_at_1 |
|
value: 18.865000000000002 |
|
- type: precision_at_10 |
|
value: 4.324999999999999 |
|
- type: precision_at_100 |
|
value: 0.722 |
|
- type: precision_at_1000 |
|
value: 0.104 |
|
- type: precision_at_3 |
|
value: 10.072000000000001 |
|
- type: precision_at_5 |
|
value: 7.086 |
|
- type: recall_at_1 |
|
value: 16.121 |
|
- type: recall_at_10 |
|
value: 35.577 |
|
- type: recall_at_100 |
|
value: 56.298 |
|
- type: recall_at_1000 |
|
value: 79.089 |
|
- type: recall_at_3 |
|
value: 25.239 |
|
- type: recall_at_5 |
|
value: 29.242 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackTexRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.968 |
|
- type: map_at_10 |
|
value: 15.639 |
|
- type: map_at_100 |
|
value: 16.459 |
|
- type: map_at_1000 |
|
value: 16.584 |
|
- type: map_at_3 |
|
value: 14.127 |
|
- type: map_at_5 |
|
value: 14.911 |
|
- type: mrr_at_1 |
|
value: 13.73 |
|
- type: mrr_at_10 |
|
value: 18.822 |
|
- type: mrr_at_100 |
|
value: 19.592000000000002 |
|
- type: mrr_at_1000 |
|
value: 19.683999999999997 |
|
- type: mrr_at_3 |
|
value: 17.223 |
|
- type: mrr_at_5 |
|
value: 18.082 |
|
- type: ndcg_at_1 |
|
value: 13.73 |
|
- type: ndcg_at_10 |
|
value: 18.881999999999998 |
|
- type: ndcg_at_100 |
|
value: 23.182 |
|
- type: ndcg_at_1000 |
|
value: 26.479000000000003 |
|
- type: ndcg_at_3 |
|
value: 16.067999999999998 |
|
- type: ndcg_at_5 |
|
value: 17.265 |
|
- type: precision_at_1 |
|
value: 13.73 |
|
- type: precision_at_10 |
|
value: 3.544 |
|
- type: precision_at_100 |
|
value: 0.679 |
|
- type: precision_at_1000 |
|
value: 0.11199999999999999 |
|
- type: precision_at_3 |
|
value: 7.674 |
|
- type: precision_at_5 |
|
value: 5.561 |
|
- type: recall_at_1 |
|
value: 10.968 |
|
- type: recall_at_10 |
|
value: 25.596000000000004 |
|
- type: recall_at_100 |
|
value: 45.411 |
|
- type: recall_at_1000 |
|
value: 69.555 |
|
- type: recall_at_3 |
|
value: 17.582 |
|
- type: recall_at_5 |
|
value: 20.785 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackUnixRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 20.886 |
|
- type: map_at_10 |
|
value: 27.029999999999998 |
|
- type: map_at_100 |
|
value: 27.968 |
|
- type: map_at_1000 |
|
value: 28.108 |
|
- type: map_at_3 |
|
value: 25.001 |
|
- type: map_at_5 |
|
value: 26.185000000000002 |
|
- type: mrr_at_1 |
|
value: 24.067 |
|
- type: mrr_at_10 |
|
value: 30.756 |
|
- type: mrr_at_100 |
|
value: 31.593 |
|
- type: mrr_at_1000 |
|
value: 31.685999999999996 |
|
- type: mrr_at_3 |
|
value: 28.793999999999997 |
|
- type: mrr_at_5 |
|
value: 29.997 |
|
- type: ndcg_at_1 |
|
value: 24.067 |
|
- type: ndcg_at_10 |
|
value: 31.095 |
|
- type: ndcg_at_100 |
|
value: 35.893 |
|
- type: ndcg_at_1000 |
|
value: 39.158 |
|
- type: ndcg_at_3 |
|
value: 27.321 |
|
- type: ndcg_at_5 |
|
value: 29.247 |
|
- type: precision_at_1 |
|
value: 24.067 |
|
- type: precision_at_10 |
|
value: 5.103 |
|
- type: precision_at_100 |
|
value: 0.8460000000000001 |
|
- type: precision_at_1000 |
|
value: 0.125 |
|
- type: precision_at_3 |
|
value: 12.065 |
|
- type: precision_at_5 |
|
value: 8.601 |
|
- type: recall_at_1 |
|
value: 20.886 |
|
- type: recall_at_10 |
|
value: 39.797 |
|
- type: recall_at_100 |
|
value: 61.399 |
|
- type: recall_at_1000 |
|
value: 84.555 |
|
- type: recall_at_3 |
|
value: 29.721999999999998 |
|
- type: recall_at_5 |
|
value: 34.455999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWebmastersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 21.394 |
|
- type: map_at_10 |
|
value: 28.303 |
|
- type: map_at_100 |
|
value: 29.726000000000003 |
|
- type: map_at_1000 |
|
value: 29.955 |
|
- type: map_at_3 |
|
value: 25.705 |
|
- type: map_at_5 |
|
value: 26.989 |
|
- type: mrr_at_1 |
|
value: 25.691999999999997 |
|
- type: mrr_at_10 |
|
value: 32.495000000000005 |
|
- type: mrr_at_100 |
|
value: 33.461999999999996 |
|
- type: mrr_at_1000 |
|
value: 33.534000000000006 |
|
- type: mrr_at_3 |
|
value: 30.137999999999998 |
|
- type: mrr_at_5 |
|
value: 31.383 |
|
- type: ndcg_at_1 |
|
value: 25.691999999999997 |
|
- type: ndcg_at_10 |
|
value: 33.300000000000004 |
|
- type: ndcg_at_100 |
|
value: 39.062000000000005 |
|
- type: ndcg_at_1000 |
|
value: 42.176 |
|
- type: ndcg_at_3 |
|
value: 28.859 |
|
- type: ndcg_at_5 |
|
value: 30.805 |
|
- type: precision_at_1 |
|
value: 25.691999999999997 |
|
- type: precision_at_10 |
|
value: 6.383 |
|
- type: precision_at_100 |
|
value: 1.387 |
|
- type: precision_at_1000 |
|
value: 0.22899999999999998 |
|
- type: precision_at_3 |
|
value: 13.439 |
|
- type: precision_at_5 |
|
value: 9.959999999999999 |
|
- type: recall_at_1 |
|
value: 21.394 |
|
- type: recall_at_10 |
|
value: 42.853 |
|
- type: recall_at_100 |
|
value: 69.284 |
|
- type: recall_at_1000 |
|
value: 89.646 |
|
- type: recall_at_3 |
|
value: 29.786 |
|
- type: recall_at_5 |
|
value: 34.797 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWordpressRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 13.999 |
|
- type: map_at_10 |
|
value: 19.979 |
|
- type: map_at_100 |
|
value: 20.682000000000002 |
|
- type: map_at_1000 |
|
value: 20.775 |
|
- type: map_at_3 |
|
value: 18.072 |
|
- type: map_at_5 |
|
value: 19.028 |
|
- type: mrr_at_1 |
|
value: 15.342 |
|
- type: mrr_at_10 |
|
value: 21.611 |
|
- type: mrr_at_100 |
|
value: 22.298000000000002 |
|
- type: mrr_at_1000 |
|
value: 22.375 |
|
- type: mrr_at_3 |
|
value: 19.624 |
|
- type: mrr_at_5 |
|
value: 20.659 |
|
- type: ndcg_at_1 |
|
value: 15.342 |
|
- type: ndcg_at_10 |
|
value: 23.809 |
|
- type: ndcg_at_100 |
|
value: 27.685 |
|
- type: ndcg_at_1000 |
|
value: 30.542 |
|
- type: ndcg_at_3 |
|
value: 19.842000000000002 |
|
- type: ndcg_at_5 |
|
value: 21.490000000000002 |
|
- type: precision_at_1 |
|
value: 15.342 |
|
- type: precision_at_10 |
|
value: 3.9190000000000005 |
|
- type: precision_at_100 |
|
value: 0.627 |
|
- type: precision_at_1000 |
|
value: 0.093 |
|
- type: precision_at_3 |
|
value: 8.688 |
|
- type: precision_at_5 |
|
value: 6.1370000000000005 |
|
- type: recall_at_1 |
|
value: 13.999 |
|
- type: recall_at_10 |
|
value: 34.276 |
|
- type: recall_at_100 |
|
value: 52.825 |
|
- type: recall_at_1000 |
|
value: 75.154 |
|
- type: recall_at_3 |
|
value: 23.339 |
|
- type: recall_at_5 |
|
value: 27.314 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: climate-fever |
|
name: MTEB ClimateFEVER |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 8.27 |
|
- type: map_at_10 |
|
value: 14.161999999999999 |
|
- type: map_at_100 |
|
value: 15.775 |
|
- type: map_at_1000 |
|
value: 15.947 |
|
- type: map_at_3 |
|
value: 11.701 |
|
- type: map_at_5 |
|
value: 12.952 |
|
- type: mrr_at_1 |
|
value: 18.632 |
|
- type: mrr_at_10 |
|
value: 28.871000000000002 |
|
- type: mrr_at_100 |
|
value: 29.985 |
|
- type: mrr_at_1000 |
|
value: 30.037999999999997 |
|
- type: mrr_at_3 |
|
value: 25.451 |
|
- type: mrr_at_5 |
|
value: 27.366 |
|
- type: ndcg_at_1 |
|
value: 18.632 |
|
- type: ndcg_at_10 |
|
value: 21.017 |
|
- type: ndcg_at_100 |
|
value: 28.022999999999996 |
|
- type: ndcg_at_1000 |
|
value: 31.518 |
|
- type: ndcg_at_3 |
|
value: 16.611 |
|
- type: ndcg_at_5 |
|
value: 18.149 |
|
- type: precision_at_1 |
|
value: 18.632 |
|
- type: precision_at_10 |
|
value: 6.736000000000001 |
|
- type: precision_at_100 |
|
value: 1.414 |
|
- type: precision_at_1000 |
|
value: 0.20600000000000002 |
|
- type: precision_at_3 |
|
value: 12.313 |
|
- type: precision_at_5 |
|
value: 9.759 |
|
- type: recall_at_1 |
|
value: 8.27 |
|
- type: recall_at_10 |
|
value: 26.218999999999998 |
|
- type: recall_at_100 |
|
value: 50.77 |
|
- type: recall_at_1000 |
|
value: 70.8 |
|
- type: recall_at_3 |
|
value: 15.526000000000002 |
|
- type: recall_at_5 |
|
value: 19.724 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.598 |
|
- type: map_at_10 |
|
value: 15.869 |
|
- type: map_at_100 |
|
value: 17.081 |
|
- type: map_at_1000 |
|
value: 17.267 |
|
- type: map_at_3 |
|
value: 13.877 |
|
- type: map_at_5 |
|
value: 14.884 |
|
- type: mrr_at_1 |
|
value: 17.279 |
|
- type: mrr_at_10 |
|
value: 22.554 |
|
- type: mrr_at_100 |
|
value: 23.521 |
|
- type: mrr_at_1000 |
|
value: 23.619 |
|
- type: mrr_at_3 |
|
value: 20.647 |
|
- type: mrr_at_5 |
|
value: 21.625 |
|
- type: ndcg_at_1 |
|
value: 17.279 |
|
- type: ndcg_at_10 |
|
value: 20.029 |
|
- type: ndcg_at_100 |
|
value: 25.968000000000004 |
|
- type: ndcg_at_1000 |
|
value: 30.158 |
|
- type: ndcg_at_3 |
|
value: 16.947000000000003 |
|
- type: ndcg_at_5 |
|
value: 18.069 |
|
- type: precision_at_1 |
|
value: 17.279 |
|
- type: precision_at_10 |
|
value: 4.704 |
|
- type: precision_at_100 |
|
value: 0.9690000000000001 |
|
- type: precision_at_1000 |
|
value: 0.152 |
|
- type: precision_at_3 |
|
value: 9.777 |
|
- type: precision_at_5 |
|
value: 7.207 |
|
- type: recall_at_1 |
|
value: 10.598 |
|
- type: recall_at_10 |
|
value: 26.034000000000002 |
|
- type: recall_at_100 |
|
value: 51.385999999999996 |
|
- type: recall_at_1000 |
|
value: 80.49 |
|
- type: recall_at_3 |
|
value: 16.834 |
|
- type: recall_at_5 |
|
value: 20.317 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 70.40288634996993 |
|
- type: cos_sim_ap |
|
value: 78.43387766087626 |
|
- type: cos_sim_f1 |
|
value: 73.09982840415867 |
|
- type: cos_sim_precision |
|
value: 64.31616341030195 |
|
- type: cos_sim_recall |
|
value: 84.66214636427402 |
|
- type: dot_accuracy |
|
value: 65.52014431749849 |
|
- type: dot_ap |
|
value: 70.89507344960353 |
|
- type: dot_f1 |
|
value: 70.7030509759333 |
|
- type: dot_precision |
|
value: 59.43922255854708 |
|
- type: dot_recall |
|
value: 87.2340425531915 |
|
- type: euclidean_accuracy |
|
value: 69.84966927239927 |
|
- type: euclidean_ap |
|
value: 78.08825177727368 |
|
- type: euclidean_f1 |
|
value: 72.68394399761692 |
|
- type: euclidean_precision |
|
value: 63.16879530548844 |
|
- type: euclidean_recall |
|
value: 85.57400046761748 |
|
- type: manhattan_accuracy |
|
value: 69.9579073962718 |
|
- type: manhattan_ap |
|
value: 78.38355697667261 |
|
- type: manhattan_f1 |
|
value: 73.06507508663844 |
|
- type: manhattan_precision |
|
value: 62.10112911143839 |
|
- type: manhattan_recall |
|
value: 88.73041851765257 |
|
- type: max_accuracy |
|
value: 70.40288634996993 |
|
- type: max_ap |
|
value: 78.43387766087626 |
|
- type: max_f1 |
|
value: 73.09982840415867 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.973 |
|
- type: map_at_10 |
|
value: 30.074 |
|
- type: map_at_100 |
|
value: 31.05 |
|
- type: map_at_1000 |
|
value: 31.147000000000002 |
|
- type: map_at_3 |
|
value: 27.977 |
|
- type: map_at_5 |
|
value: 29.247 |
|
- type: mrr_at_1 |
|
value: 24.025 |
|
- type: mrr_at_10 |
|
value: 30.093999999999998 |
|
- type: mrr_at_100 |
|
value: 31.068 |
|
- type: mrr_at_1000 |
|
value: 31.165 |
|
- type: mrr_at_3 |
|
value: 27.994000000000003 |
|
- type: mrr_at_5 |
|
value: 29.243000000000002 |
|
- type: ndcg_at_1 |
|
value: 24.025 |
|
- type: ndcg_at_10 |
|
value: 33.566 |
|
- type: ndcg_at_100 |
|
value: 38.818999999999996 |
|
- type: ndcg_at_1000 |
|
value: 41.477000000000004 |
|
- type: ndcg_at_3 |
|
value: 29.293000000000003 |
|
- type: ndcg_at_5 |
|
value: 31.564999999999998 |
|
- type: precision_at_1 |
|
value: 24.025 |
|
- type: precision_at_10 |
|
value: 4.489 |
|
- type: precision_at_100 |
|
value: 0.709 |
|
- type: precision_at_1000 |
|
value: 0.092 |
|
- type: precision_at_3 |
|
value: 11.064 |
|
- type: precision_at_5 |
|
value: 7.734000000000001 |
|
- type: recall_at_1 |
|
value: 23.973 |
|
- type: recall_at_10 |
|
value: 44.731 |
|
- type: recall_at_100 |
|
value: 70.52199999999999 |
|
- type: recall_at_1000 |
|
value: 91.491 |
|
- type: recall_at_3 |
|
value: 33.087 |
|
- type: recall_at_5 |
|
value: 38.567 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: dbpedia-entity |
|
name: MTEB DBPedia |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 6.950000000000001 |
|
- type: map_at_10 |
|
value: 13.236999999999998 |
|
- type: map_at_100 |
|
value: 16.137 |
|
- type: map_at_1000 |
|
value: 16.785 |
|
- type: map_at_3 |
|
value: 10.378 |
|
- type: map_at_5 |
|
value: 11.62 |
|
- type: mrr_at_1 |
|
value: 54.0 |
|
- type: mrr_at_10 |
|
value: 61.861 |
|
- type: mrr_at_100 |
|
value: 62.436 |
|
- type: mrr_at_1000 |
|
value: 62.456 |
|
- type: mrr_at_3 |
|
value: 60.458 |
|
- type: mrr_at_5 |
|
value: 61.208 |
|
- type: ndcg_at_1 |
|
value: 43.75 |
|
- type: ndcg_at_10 |
|
value: 28.224 |
|
- type: ndcg_at_100 |
|
value: 29.244999999999997 |
|
- type: ndcg_at_1000 |
|
value: 34.410000000000004 |
|
- type: ndcg_at_3 |
|
value: 33.955 |
|
- type: ndcg_at_5 |
|
value: 30.597 |
|
- type: precision_at_1 |
|
value: 54.0 |
|
- type: precision_at_10 |
|
value: 20.825 |
|
- type: precision_at_100 |
|
value: 5.462 |
|
- type: precision_at_1000 |
|
value: 1.1320000000000001 |
|
- type: precision_at_3 |
|
value: 37.0 |
|
- type: precision_at_5 |
|
value: 28.849999999999998 |
|
- type: recall_at_1 |
|
value: 6.950000000000001 |
|
- type: recall_at_10 |
|
value: 17.159 |
|
- type: recall_at_100 |
|
value: 31.657999999999998 |
|
- type: recall_at_1000 |
|
value: 49.155 |
|
- type: recall_at_3 |
|
value: 11.393 |
|
- type: recall_at_5 |
|
value: 13.568 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 16.333000000000002 |
|
- type: map_at_10 |
|
value: 44.080999999999996 |
|
- type: map_at_100 |
|
value: 47.958 |
|
- type: map_at_1000 |
|
value: 48.183 |
|
- type: map_at_3 |
|
value: 31.468 |
|
- type: map_at_5 |
|
value: 38.213 |
|
- type: mrr_at_1 |
|
value: 63.0 |
|
- type: mrr_at_10 |
|
value: 72.006 |
|
- type: mrr_at_100 |
|
value: 72.299 |
|
- type: mrr_at_1000 |
|
value: 72.313 |
|
- type: mrr_at_3 |
|
value: 70.375 |
|
- type: mrr_at_5 |
|
value: 71.33 |
|
- type: ndcg_at_1 |
|
value: 63.0 |
|
- type: ndcg_at_10 |
|
value: 56.044000000000004 |
|
- type: ndcg_at_100 |
|
value: 63.629999999999995 |
|
- type: ndcg_at_1000 |
|
value: 66.156 |
|
- type: ndcg_at_3 |
|
value: 55.85 |
|
- type: ndcg_at_5 |
|
value: 53.559 |
|
- type: precision_at_1 |
|
value: 63.0 |
|
- type: precision_at_10 |
|
value: 27.279999999999998 |
|
- type: precision_at_100 |
|
value: 4.005 |
|
- type: precision_at_1000 |
|
value: 0.462 |
|
- type: precision_at_3 |
|
value: 49.633 |
|
- type: precision_at_5 |
|
value: 40.6 |
|
- type: recall_at_1 |
|
value: 16.333000000000002 |
|
- type: recall_at_10 |
|
value: 57.152 |
|
- type: recall_at_100 |
|
value: 80.231 |
|
- type: recall_at_1000 |
|
value: 92.95400000000001 |
|
- type: recall_at_3 |
|
value: 34.793 |
|
- type: recall_at_5 |
|
value: 44.989000000000004 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 33.7 |
|
- type: map_at_10 |
|
value: 42.327999999999996 |
|
- type: map_at_100 |
|
value: 43.230000000000004 |
|
- type: map_at_1000 |
|
value: 43.274 |
|
- type: map_at_3 |
|
value: 39.883 |
|
- type: map_at_5 |
|
value: 41.178 |
|
- type: mrr_at_1 |
|
value: 33.7 |
|
- type: mrr_at_10 |
|
value: 42.327999999999996 |
|
- type: mrr_at_100 |
|
value: 43.230000000000004 |
|
- type: mrr_at_1000 |
|
value: 43.274 |
|
- type: mrr_at_3 |
|
value: 39.883 |
|
- type: mrr_at_5 |
|
value: 41.178 |
|
- type: ndcg_at_1 |
|
value: 33.7 |
|
- type: ndcg_at_10 |
|
value: 46.996 |
|
- type: ndcg_at_100 |
|
value: 51.629000000000005 |
|
- type: ndcg_at_1000 |
|
value: 52.823 |
|
- type: ndcg_at_3 |
|
value: 41.891 |
|
- type: ndcg_at_5 |
|
value: 44.232 |
|
- type: precision_at_1 |
|
value: 33.7 |
|
- type: precision_at_10 |
|
value: 6.1899999999999995 |
|
- type: precision_at_100 |
|
value: 0.8410000000000001 |
|
- type: precision_at_1000 |
|
value: 0.094 |
|
- type: precision_at_3 |
|
value: 15.9 |
|
- type: precision_at_5 |
|
value: 10.68 |
|
- type: recall_at_1 |
|
value: 33.7 |
|
- type: recall_at_10 |
|
value: 61.9 |
|
- type: recall_at_100 |
|
value: 84.1 |
|
- type: recall_at_1000 |
|
value: 93.60000000000001 |
|
- type: recall_at_3 |
|
value: 47.699999999999996 |
|
- type: recall_at_5 |
|
value: 53.400000000000006 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/emotion |
|
name: MTEB EmotionClassification |
|
config: default |
|
split: test |
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
metrics: |
|
- type: accuracy |
|
value: 44.76500000000001 |
|
- type: f1 |
|
value: 40.46330006682868 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fever |
|
name: MTEB FEVER |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 45.078 |
|
- type: map_at_10 |
|
value: 55.443 |
|
- type: map_at_100 |
|
value: 56.03900000000001 |
|
- type: map_at_1000 |
|
value: 56.067 |
|
- type: map_at_3 |
|
value: 53.174 |
|
- type: map_at_5 |
|
value: 54.510999999999996 |
|
- type: mrr_at_1 |
|
value: 48.575 |
|
- type: mrr_at_10 |
|
value: 59.194 |
|
- type: mrr_at_100 |
|
value: 59.760999999999996 |
|
- type: mrr_at_1000 |
|
value: 59.784000000000006 |
|
- type: mrr_at_3 |
|
value: 56.896 |
|
- type: mrr_at_5 |
|
value: 58.282000000000004 |
|
- type: ndcg_at_1 |
|
value: 48.575 |
|
- type: ndcg_at_10 |
|
value: 61.096 |
|
- type: ndcg_at_100 |
|
value: 63.94800000000001 |
|
- type: ndcg_at_1000 |
|
value: 64.68199999999999 |
|
- type: ndcg_at_3 |
|
value: 56.58 |
|
- type: ndcg_at_5 |
|
value: 58.928000000000004 |
|
- type: precision_at_1 |
|
value: 48.575 |
|
- type: precision_at_10 |
|
value: 8.18 |
|
- type: precision_at_100 |
|
value: 0.968 |
|
- type: precision_at_1000 |
|
value: 0.104 |
|
- type: precision_at_3 |
|
value: 22.662 |
|
- type: precision_at_5 |
|
value: 14.881 |
|
- type: recall_at_1 |
|
value: 45.078 |
|
- type: recall_at_10 |
|
value: 75.057 |
|
- type: recall_at_100 |
|
value: 88.05199999999999 |
|
- type: recall_at_1000 |
|
value: 93.58999999999999 |
|
- type: recall_at_3 |
|
value: 62.77700000000001 |
|
- type: recall_at_5 |
|
value: 68.50699999999999 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fiqa |
|
name: MTEB FiQA2018 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 11.097999999999999 |
|
- type: map_at_10 |
|
value: 18.288 |
|
- type: map_at_100 |
|
value: 19.903000000000002 |
|
- type: map_at_1000 |
|
value: 20.108 |
|
- type: map_at_3 |
|
value: 15.576 |
|
- type: map_at_5 |
|
value: 16.997999999999998 |
|
- type: mrr_at_1 |
|
value: 23.302 |
|
- type: mrr_at_10 |
|
value: 30.978 |
|
- type: mrr_at_100 |
|
value: 32.072 |
|
- type: mrr_at_1000 |
|
value: 32.15 |
|
- type: mrr_at_3 |
|
value: 28.549000000000003 |
|
- type: mrr_at_5 |
|
value: 29.931 |
|
- type: ndcg_at_1 |
|
value: 23.302 |
|
- type: ndcg_at_10 |
|
value: 24.488 |
|
- type: ndcg_at_100 |
|
value: 31.052999999999997 |
|
- type: ndcg_at_1000 |
|
value: 35.124 |
|
- type: ndcg_at_3 |
|
value: 21.215999999999998 |
|
- type: ndcg_at_5 |
|
value: 22.314999999999998 |
|
- type: precision_at_1 |
|
value: 23.302 |
|
- type: precision_at_10 |
|
value: 7.13 |
|
- type: precision_at_100 |
|
value: 1.3559999999999999 |
|
- type: precision_at_1000 |
|
value: 0.20600000000000002 |
|
- type: precision_at_3 |
|
value: 14.198 |
|
- type: precision_at_5 |
|
value: 10.895000000000001 |
|
- type: recall_at_1 |
|
value: 11.097999999999999 |
|
- type: recall_at_10 |
|
value: 30.352 |
|
- type: recall_at_100 |
|
value: 54.937999999999995 |
|
- type: recall_at_1000 |
|
value: 79.586 |
|
- type: recall_at_3 |
|
value: 19.486 |
|
- type: recall_at_5 |
|
value: 23.860999999999997 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: hotpotqa |
|
name: MTEB HotpotQA |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.325 |
|
- type: map_at_10 |
|
value: 37.305 |
|
- type: map_at_100 |
|
value: 38.0 |
|
- type: map_at_1000 |
|
value: 38.065 |
|
- type: map_at_3 |
|
value: 35.219 |
|
- type: map_at_5 |
|
value: 36.466 |
|
- type: mrr_at_1 |
|
value: 56.650999999999996 |
|
- type: mrr_at_10 |
|
value: 63.574 |
|
- type: mrr_at_100 |
|
value: 63.966 |
|
- type: mrr_at_1000 |
|
value: 63.992000000000004 |
|
- type: mrr_at_3 |
|
value: 62.107 |
|
- type: mrr_at_5 |
|
value: 62.976 |
|
- type: ndcg_at_1 |
|
value: 56.650999999999996 |
|
- type: ndcg_at_10 |
|
value: 46.046 |
|
- type: ndcg_at_100 |
|
value: 48.916 |
|
- type: ndcg_at_1000 |
|
value: 50.410999999999994 |
|
- type: ndcg_at_3 |
|
value: 42.516999999999996 |
|
- type: ndcg_at_5 |
|
value: 44.374 |
|
- type: precision_at_1 |
|
value: 56.650999999999996 |
|
- type: precision_at_10 |
|
value: 9.392 |
|
- type: precision_at_100 |
|
value: 1.166 |
|
- type: precision_at_1000 |
|
value: 0.13699999999999998 |
|
- type: precision_at_3 |
|
value: 26.068 |
|
- type: precision_at_5 |
|
value: 17.11 |
|
- type: recall_at_1 |
|
value: 28.325 |
|
- type: recall_at_10 |
|
value: 46.961999999999996 |
|
- type: recall_at_100 |
|
value: 58.318999999999996 |
|
- type: recall_at_1000 |
|
value: 68.298 |
|
- type: recall_at_3 |
|
value: 39.102 |
|
- type: recall_at_5 |
|
value: 42.775 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 40.461716044632546 |
|
- type: f1 |
|
value: 33.890745966734315 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/imdb |
|
name: MTEB ImdbClassification |
|
config: default |
|
split: test |
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
metrics: |
|
- type: accuracy |
|
value: 72.21000000000001 |
|
- type: ap |
|
value: 66.59963731769069 |
|
- type: f1 |
|
value: 71.97616824840041 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 78.25515947467167 |
|
- type: ap |
|
value: 38.265118237185064 |
|
- type: f1 |
|
value: 70.73962826410575 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 63.98362797180168 |
|
- type: cos_sim_spearman |
|
value: 71.97575564053473 |
|
- type: euclidean_pearson |
|
value: 70.56052438394708 |
|
- type: euclidean_spearman |
|
value: 72.48267176371337 |
|
- type: manhattan_pearson |
|
value: 70.7156268448442 |
|
- type: manhattan_spearman |
|
value: 72.61065396802094 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 55.775 |
|
- type: map_at_10 |
|
value: 65.074 |
|
- type: map_at_100 |
|
value: 65.596 |
|
- type: map_at_1000 |
|
value: 65.618 |
|
- type: map_at_3 |
|
value: 62.92 |
|
- type: map_at_5 |
|
value: 64.277 |
|
- type: mrr_at_1 |
|
value: 57.708000000000006 |
|
- type: mrr_at_10 |
|
value: 65.824 |
|
- type: mrr_at_100 |
|
value: 66.286 |
|
- type: mrr_at_1000 |
|
value: 66.306 |
|
- type: mrr_at_3 |
|
value: 63.871 |
|
- type: mrr_at_5 |
|
value: 65.093 |
|
- type: ndcg_at_1 |
|
value: 57.708000000000006 |
|
- type: ndcg_at_10 |
|
value: 69.309 |
|
- type: ndcg_at_100 |
|
value: 71.723 |
|
- type: ndcg_at_1000 |
|
value: 72.313 |
|
- type: ndcg_at_3 |
|
value: 65.134 |
|
- type: ndcg_at_5 |
|
value: 67.476 |
|
- type: precision_at_1 |
|
value: 57.708000000000006 |
|
- type: precision_at_10 |
|
value: 8.668 |
|
- type: precision_at_100 |
|
value: 0.989 |
|
- type: precision_at_1000 |
|
value: 0.104 |
|
- type: precision_at_3 |
|
value: 24.837999999999997 |
|
- type: precision_at_5 |
|
value: 16.128999999999998 |
|
- type: recall_at_1 |
|
value: 55.775 |
|
- type: recall_at_10 |
|
value: 81.702 |
|
- type: recall_at_100 |
|
value: 92.785 |
|
- type: recall_at_1000 |
|
value: 97.425 |
|
- type: recall_at_3 |
|
value: 70.587 |
|
- type: recall_at_5 |
|
value: 76.199 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: msmarco |
|
name: MTEB MSMARCO |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.771 |
|
- type: map_at_10 |
|
value: 28.16 |
|
- type: map_at_100 |
|
value: 29.363 |
|
- type: map_at_1000 |
|
value: 29.431 |
|
- type: map_at_3 |
|
value: 24.767 |
|
- type: map_at_5 |
|
value: 26.706999999999997 |
|
- type: mrr_at_1 |
|
value: 18.252 |
|
- type: mrr_at_10 |
|
value: 28.666000000000004 |
|
- type: mrr_at_100 |
|
value: 29.837000000000003 |
|
- type: mrr_at_1000 |
|
value: 29.898999999999997 |
|
- type: mrr_at_3 |
|
value: 25.308000000000003 |
|
- type: mrr_at_5 |
|
value: 27.226 |
|
- type: ndcg_at_1 |
|
value: 18.252 |
|
- type: ndcg_at_10 |
|
value: 34.176 |
|
- type: ndcg_at_100 |
|
value: 40.138 |
|
- type: ndcg_at_1000 |
|
value: 41.923 |
|
- type: ndcg_at_3 |
|
value: 27.214 |
|
- type: ndcg_at_5 |
|
value: 30.695 |
|
- type: precision_at_1 |
|
value: 18.252 |
|
- type: precision_at_10 |
|
value: 5.503 |
|
- type: precision_at_100 |
|
value: 0.8500000000000001 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 11.667 |
|
- type: precision_at_5 |
|
value: 8.754000000000001 |
|
- type: recall_at_1 |
|
value: 17.771 |
|
- type: recall_at_10 |
|
value: 52.781 |
|
- type: recall_at_100 |
|
value: 80.638 |
|
- type: recall_at_1000 |
|
value: 94.46 |
|
- type: recall_at_3 |
|
value: 33.767 |
|
- type: recall_at_5 |
|
value: 42.172 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (en) |
|
config: en |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 89.93388052895577 |
|
- type: f1 |
|
value: 89.55553145791954 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (de) |
|
config: de |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 68.42490842490842 |
|
- type: f1 |
|
value: 67.01398674117826 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (es) |
|
config: es |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 88.2121414276184 |
|
- type: f1 |
|
value: 87.61981627763988 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (fr) |
|
config: fr |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 85.49013466958974 |
|
- type: f1 |
|
value: 85.09758510104221 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (hi) |
|
config: hi |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 84.22732162065257 |
|
- type: f1 |
|
value: 83.24580378090367 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (th) |
|
config: th |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 53.171790235081374 |
|
- type: f1 |
|
value: 51.93028909966765 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 66.5640674874601 |
|
- type: f1 |
|
value: 49.856876973153966 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (de) |
|
config: de |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 49.171597633136095 |
|
- type: f1 |
|
value: 32.166022205347545 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (es) |
|
config: es |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 65.71714476317545 |
|
- type: f1 |
|
value: 45.748971341625136 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (fr) |
|
config: fr |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 62.65267773253993 |
|
- type: f1 |
|
value: 45.904472624086026 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (hi) |
|
config: hi |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 61.8752240946576 |
|
- type: f1 |
|
value: 40.7359613185448 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (th) |
|
config: th |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 41.67088607594936 |
|
- type: f1 |
|
value: 28.12210726419673 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (af) |
|
config: af |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 43.29186281102892 |
|
- type: f1 |
|
value: 41.83461350696014 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (am) |
|
config: am |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 23.214525891055814 |
|
- type: f1 |
|
value: 22.364131190189962 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ar) |
|
config: ar |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 53.38264963012777 |
|
- type: f1 |
|
value: 50.74546702709091 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (az) |
|
config: az |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 39.55951580363147 |
|
- type: f1 |
|
value: 39.07769075741216 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (bn) |
|
config: bn |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 56.73839946200403 |
|
- type: f1 |
|
value: 54.36728741542025 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (cy) |
|
config: cy |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 39.99663752521857 |
|
- type: f1 |
|
value: 38.709817953652596 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (da) |
|
config: da |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 46.933422999327504 |
|
- type: f1 |
|
value: 45.32022679895763 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (de) |
|
config: de |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 45.820443846671154 |
|
- type: f1 |
|
value: 42.853155158197886 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (el) |
|
config: el |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 37.874915938130464 |
|
- type: f1 |
|
value: 35.9849010888881 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 66.08944182918628 |
|
- type: f1 |
|
value: 64.5039080809391 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (es) |
|
config: es |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 61.17350369872226 |
|
- type: f1 |
|
value: 60.0792530132073 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (fa) |
|
config: fa |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 45.652320107599195 |
|
- type: f1 |
|
value: 44.28182554287625 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (fi) |
|
config: fi |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 40.282447881640884 |
|
- type: f1 |
|
value: 38.79927524886836 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (fr) |
|
config: fr |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 62.60591795561533 |
|
- type: f1 |
|
value: 61.01451309609411 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (he) |
|
config: he |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 32.225958305312716 |
|
- type: f1 |
|
value: 30.903299940417906 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (hi) |
|
config: hi |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 59.46200403496974 |
|
- type: f1 |
|
value: 57.34556231956785 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (hu) |
|
config: hu |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 40.907868190988566 |
|
- type: f1 |
|
value: 39.74702259997524 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (hy) |
|
config: hy |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 29.939475453934094 |
|
- type: f1 |
|
value: 28.462353413371353 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (id) |
|
config: id |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 59.14256893073302 |
|
- type: f1 |
|
value: 57.24600767871435 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (is) |
|
config: is |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 39.620040349697376 |
|
- type: f1 |
|
value: 38.414866180464735 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (it) |
|
config: it |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 51.772024209818426 |
|
- type: f1 |
|
value: 51.05050942366993 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ja) |
|
config: ja |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 53.749159381304636 |
|
- type: f1 |
|
value: 52.04563008527909 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (jv) |
|
config: jv |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 46.29455279085406 |
|
- type: f1 |
|
value: 43.84047527739209 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ka) |
|
config: ka |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 25.107599193006045 |
|
- type: f1 |
|
value: 24.58731463875415 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (km) |
|
config: km |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 27.21923335574984 |
|
- type: f1 |
|
value: 25.964338481976796 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (kn) |
|
config: kn |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 47.96906523201077 |
|
- type: f1 |
|
value: 45.32239408435578 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ko) |
|
config: ko |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 40.53799596503026 |
|
- type: f1 |
|
value: 39.15655510771227 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (lv) |
|
config: lv |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 43.140551445864155 |
|
- type: f1 |
|
value: 42.12232733095163 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ml) |
|
config: ml |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 53.69199731002017 |
|
- type: f1 |
|
value: 50.67085509122796 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (mn) |
|
config: mn |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 33.37256220578346 |
|
- type: f1 |
|
value: 33.39335560955231 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ms) |
|
config: ms |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 51.94014794889038 |
|
- type: f1 |
|
value: 50.6207021226521 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (my) |
|
config: my |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 25.322797579018157 |
|
- type: f1 |
|
value: 23.94164121951907 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (nb) |
|
config: nb |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 44.11903160726294 |
|
- type: f1 |
|
value: 43.016752983579536 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (nl) |
|
config: nl |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 44.03496973772697 |
|
- type: f1 |
|
value: 42.322828283176754 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (pl) |
|
config: pl |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 41.63080026899798 |
|
- type: f1 |
|
value: 39.58824644978166 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (pt) |
|
config: pt |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 61.7350369872226 |
|
- type: f1 |
|
value: 59.956752206079386 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ro) |
|
config: ro |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 45.72629455279086 |
|
- type: f1 |
|
value: 44.731249269647826 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ru) |
|
config: ru |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 47.61264290517822 |
|
- type: f1 |
|
value: 45.5280995218491 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (sl) |
|
config: sl |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 42.82784129119032 |
|
- type: f1 |
|
value: 41.37165985220223 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (sq) |
|
config: sq |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 43.61466039004707 |
|
- type: f1 |
|
value: 43.164498227815535 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (sv) |
|
config: sv |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 44.64021519838602 |
|
- type: f1 |
|
value: 43.04775030948548 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (sw) |
|
config: sw |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 45.54808338937458 |
|
- type: f1 |
|
value: 44.011677633779975 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ta) |
|
config: ta |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 51.2441156691325 |
|
- type: f1 |
|
value: 48.73592932403811 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (te) |
|
config: te |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 47.43443174176195 |
|
- type: f1 |
|
value: 45.08686598891457 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (th) |
|
config: th |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 36.87962340282448 |
|
- type: f1 |
|
value: 36.50540864756967 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (tl) |
|
config: tl |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 45.9280430396772 |
|
- type: f1 |
|
value: 44.57216865343283 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (tr) |
|
config: tr |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 38.591123066577 |
|
- type: f1 |
|
value: 37.886312373767446 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (ur) |
|
config: ur |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 51.85272360457296 |
|
- type: f1 |
|
value: 49.43461566216979 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (vi) |
|
config: vi |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 58.72225958305313 |
|
- type: f1 |
|
value: 56.95500715299434 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 63.74915938130464 |
|
- type: f1 |
|
value: 62.35543158488615 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-TW) |
|
config: zh-TW |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 59.95292535305985 |
|
- type: f1 |
|
value: 59.73499569346673 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (af) |
|
config: af |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 47.42098184263618 |
|
- type: f1 |
|
value: 45.22541854557743 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (am) |
|
config: am |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 24.707464694014796 |
|
- type: f1 |
|
value: 24.033506081882468 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ar) |
|
config: ar |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 62.09145931405515 |
|
- type: f1 |
|
value: 62.22048940230962 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (az) |
|
config: az |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 39.25016812373907 |
|
- type: f1 |
|
value: 38.35431952425269 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (bn) |
|
config: bn |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 63.37256220578345 |
|
- type: f1 |
|
value: 63.12728180326932 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (cy) |
|
config: cy |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 39.172831203765966 |
|
- type: f1 |
|
value: 37.078841372640234 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (da) |
|
config: da |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 49.11230665770006 |
|
- type: f1 |
|
value: 46.489580286547245 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (de) |
|
config: de |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 50.7128446536651 |
|
- type: f1 |
|
value: 48.27782602378952 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (el) |
|
config: el |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 39.46536650975118 |
|
- type: f1 |
|
value: 37.4365280056047 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (en) |
|
config: en |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 73.26160053799597 |
|
- type: f1 |
|
value: 73.4478249967817 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (es) |
|
config: es |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 68.31203765971756 |
|
- type: f1 |
|
value: 68.70554437788068 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (fa) |
|
config: fa |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 45.652320107599195 |
|
- type: f1 |
|
value: 44.55357745265521 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (fi) |
|
config: fi |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 38.94754539340955 |
|
- type: f1 |
|
value: 36.48927336173062 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (fr) |
|
config: fr |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 68.69872225958305 |
|
- type: f1 |
|
value: 68.81347966311543 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (he) |
|
config: he |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 32.131809011432416 |
|
- type: f1 |
|
value: 30.212230946937474 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (hi) |
|
config: hi |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 65.57498318762609 |
|
- type: f1 |
|
value: 65.16084751135229 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (hu) |
|
config: hu |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 42.965702757229316 |
|
- type: f1 |
|
value: 40.575896627739105 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (hy) |
|
config: hy |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 32.125084061869536 |
|
- type: f1 |
|
value: 30.708056882129476 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (id) |
|
config: id |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 65.10759919300607 |
|
- type: f1 |
|
value: 64.5007800119315 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (is) |
|
config: is |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 40.83725622057834 |
|
- type: f1 |
|
value: 37.855774705520886 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (it) |
|
config: it |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 54.55279085406859 |
|
- type: f1 |
|
value: 52.73318944173822 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ja) |
|
config: ja |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 57.14525891055817 |
|
- type: f1 |
|
value: 55.96714177558203 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (jv) |
|
config: jv |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 49.30060524546065 |
|
- type: f1 |
|
value: 47.82999154670342 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ka) |
|
config: ka |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 25.85743106926698 |
|
- type: f1 |
|
value: 24.974946990729716 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (km) |
|
config: km |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 31.180228648285137 |
|
- type: f1 |
|
value: 28.22387838219335 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (kn) |
|
config: kn |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 53.00941492938802 |
|
- type: f1 |
|
value: 52.39610045092559 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ko) |
|
config: ko |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 40.24546065904505 |
|
- type: f1 |
|
value: 38.99779773215032 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (lv) |
|
config: lv |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 41.88298587760592 |
|
- type: f1 |
|
value: 39.53867071594289 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ml) |
|
config: ml |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 59.078681909885674 |
|
- type: f1 |
|
value: 58.47368723772022 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (mn) |
|
config: mn |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 33.33893745796907 |
|
- type: f1 |
|
value: 32.113466354321226 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ms) |
|
config: ms |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 57.454606590450574 |
|
- type: f1 |
|
value: 56.13075383338251 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (my) |
|
config: my |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 27.19569603227976 |
|
- type: f1 |
|
value: 26.300773160344015 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (nb) |
|
config: nb |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 46.78547410894418 |
|
- type: f1 |
|
value: 44.233771335183015 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (nl) |
|
config: nl |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 48.4196368527236 |
|
- type: f1 |
|
value: 45.55838648206857 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (pl) |
|
config: pl |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 41.63080026899798 |
|
- type: f1 |
|
value: 40.77775839499525 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (pt) |
|
config: pt |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 66.408876933423 |
|
- type: f1 |
|
value: 66.7358693871042 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ro) |
|
config: ro |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 50.077336919973106 |
|
- type: f1 |
|
value: 48.572749739090014 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ru) |
|
config: ru |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 49.942837928715534 |
|
- type: f1 |
|
value: 49.34771836662566 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (sl) |
|
config: sl |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 43.43308675184936 |
|
- type: f1 |
|
value: 41.818008297000986 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (sq) |
|
config: sq |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 44.082044384667114 |
|
- type: f1 |
|
value: 43.25002746432129 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (sv) |
|
config: sv |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 46.45258910558171 |
|
- type: f1 |
|
value: 44.00958237591922 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (sw) |
|
config: sw |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 49.53261600537996 |
|
- type: f1 |
|
value: 48.01969699634672 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ta) |
|
config: ta |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 56.792199058507066 |
|
- type: f1 |
|
value: 56.54421925671813 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (te) |
|
config: te |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 54.0114324142569 |
|
- type: f1 |
|
value: 52.29830350891558 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (th) |
|
config: th |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 38.584398117014125 |
|
- type: f1 |
|
value: 36.551426239639575 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (tl) |
|
config: tl |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 48.07330195023538 |
|
- type: f1 |
|
value: 46.463553675519975 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (tr) |
|
config: tr |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 40.645595158036315 |
|
- type: f1 |
|
value: 40.21280676607986 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (ur) |
|
config: ur |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 57.74714189643577 |
|
- type: f1 |
|
value: 56.8673027258351 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (vi) |
|
config: vi |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 65.83389374579693 |
|
- type: f1 |
|
value: 66.11273939782248 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 72.38735709482181 |
|
- type: f1 |
|
value: 72.89481650271512 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-TW) |
|
config: zh-TW |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 69.63685272360458 |
|
- type: f1 |
|
value: 70.72285841806938 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 30.8 |
|
- type: map_at_10 |
|
value: 34.782000000000004 |
|
- type: map_at_100 |
|
value: 35.333999999999996 |
|
- type: map_at_1000 |
|
value: 35.405 |
|
- type: map_at_3 |
|
value: 34.0 |
|
- type: map_at_5 |
|
value: 34.345 |
|
- type: mrr_at_1 |
|
value: 30.8 |
|
- type: mrr_at_10 |
|
value: 34.782000000000004 |
|
- type: mrr_at_100 |
|
value: 35.333999999999996 |
|
- type: mrr_at_1000 |
|
value: 35.405 |
|
- type: mrr_at_3 |
|
value: 34.0 |
|
- type: mrr_at_5 |
|
value: 34.345 |
|
- type: ndcg_at_1 |
|
value: 30.8 |
|
- type: ndcg_at_10 |
|
value: 36.675000000000004 |
|
- type: ndcg_at_100 |
|
value: 39.633 |
|
- type: ndcg_at_1000 |
|
value: 41.904 |
|
- type: ndcg_at_3 |
|
value: 35.028 |
|
- type: ndcg_at_5 |
|
value: 35.648 |
|
- type: precision_at_1 |
|
value: 30.8 |
|
- type: precision_at_10 |
|
value: 4.26 |
|
- type: precision_at_100 |
|
value: 0.571 |
|
- type: precision_at_1000 |
|
value: 0.076 |
|
- type: precision_at_3 |
|
value: 12.667 |
|
- type: precision_at_5 |
|
value: 7.9 |
|
- type: recall_at_1 |
|
value: 30.8 |
|
- type: recall_at_10 |
|
value: 42.6 |
|
- type: recall_at_100 |
|
value: 57.099999999999994 |
|
- type: recall_at_1000 |
|
value: 75.8 |
|
- type: recall_at_3 |
|
value: 38.0 |
|
- type: recall_at_5 |
|
value: 39.5 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-p2p |
|
name: MTEB MedrxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
metrics: |
|
- type: v_measure |
|
value: 27.84536559870361 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-s2s |
|
name: MTEB MedrxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
metrics: |
|
- type: v_measure |
|
value: 27.714921841841605 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 30.52145905910035 |
|
- type: mrr |
|
value: 31.551577344311845 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 23.6853605350459 |
|
- type: mrr |
|
value: 22.341269841269842 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 63.16666666666666 |
|
- type: f1 |
|
value: 63.09453591106835 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nfcorpus |
|
name: MTEB NFCorpus |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 3.7060000000000004 |
|
- type: map_at_10 |
|
value: 9.032 |
|
- type: map_at_100 |
|
value: 11.395 |
|
- type: map_at_1000 |
|
value: 12.713 |
|
- type: map_at_3 |
|
value: 6.502 |
|
- type: map_at_5 |
|
value: 7.8100000000000005 |
|
- type: mrr_at_1 |
|
value: 37.461 |
|
- type: mrr_at_10 |
|
value: 45.839999999999996 |
|
- type: mrr_at_100 |
|
value: 46.513 |
|
- type: mrr_at_1000 |
|
value: 46.571 |
|
- type: mrr_at_3 |
|
value: 43.55 |
|
- type: mrr_at_5 |
|
value: 44.773 |
|
- type: ndcg_at_1 |
|
value: 35.913000000000004 |
|
- type: ndcg_at_10 |
|
value: 27.340999999999998 |
|
- type: ndcg_at_100 |
|
value: 25.197000000000003 |
|
- type: ndcg_at_1000 |
|
value: 34.632000000000005 |
|
- type: ndcg_at_3 |
|
value: 31.952 |
|
- type: ndcg_at_5 |
|
value: 30.244 |
|
- type: precision_at_1 |
|
value: 37.461 |
|
- type: precision_at_10 |
|
value: 20.495 |
|
- type: precision_at_100 |
|
value: 6.551 |
|
- type: precision_at_1000 |
|
value: 1.966 |
|
- type: precision_at_3 |
|
value: 30.753000000000004 |
|
- type: precision_at_5 |
|
value: 26.935 |
|
- type: recall_at_1 |
|
value: 3.7060000000000004 |
|
- type: recall_at_10 |
|
value: 12.958 |
|
- type: recall_at_100 |
|
value: 26.582 |
|
- type: recall_at_1000 |
|
value: 59.724 |
|
- type: recall_at_3 |
|
value: 7.503 |
|
- type: recall_at_5 |
|
value: 9.808 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nq |
|
name: MTEB NQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 22.201999999999998 |
|
- type: map_at_10 |
|
value: 33.76 |
|
- type: map_at_100 |
|
value: 34.867 |
|
- type: map_at_1000 |
|
value: 34.92 |
|
- type: map_at_3 |
|
value: 30.233999999999998 |
|
- type: map_at_5 |
|
value: 32.291 |
|
- type: mrr_at_1 |
|
value: 25.232 |
|
- type: mrr_at_10 |
|
value: 36.239 |
|
- type: mrr_at_100 |
|
value: 37.119 |
|
- type: mrr_at_1000 |
|
value: 37.162 |
|
- type: mrr_at_3 |
|
value: 33.213 |
|
- type: mrr_at_5 |
|
value: 35.02 |
|
- type: ndcg_at_1 |
|
value: 25.232 |
|
- type: ndcg_at_10 |
|
value: 40.046 |
|
- type: ndcg_at_100 |
|
value: 45.025 |
|
- type: ndcg_at_1000 |
|
value: 46.459 |
|
- type: ndcg_at_3 |
|
value: 33.343 |
|
- type: ndcg_at_5 |
|
value: 36.801 |
|
- type: precision_at_1 |
|
value: 25.232 |
|
- type: precision_at_10 |
|
value: 6.796 |
|
- type: precision_at_100 |
|
value: 0.959 |
|
- type: precision_at_1000 |
|
value: 0.109 |
|
- type: precision_at_3 |
|
value: 15.276 |
|
- type: precision_at_5 |
|
value: 11.17 |
|
- type: recall_at_1 |
|
value: 22.201999999999998 |
|
- type: recall_at_10 |
|
value: 56.733 |
|
- type: recall_at_100 |
|
value: 79.041 |
|
- type: recall_at_1000 |
|
value: 90.08500000000001 |
|
- type: recall_at_3 |
|
value: 39.412000000000006 |
|
- type: recall_at_5 |
|
value: 47.352 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 62.53383865728208 |
|
- type: cos_sim_ap |
|
value: 66.3197921045625 |
|
- type: cos_sim_f1 |
|
value: 69.3385214007782 |
|
- type: cos_sim_precision |
|
value: 54.89833641404805 |
|
- type: cos_sim_recall |
|
value: 94.08658922914466 |
|
- type: dot_accuracy |
|
value: 59.7184623714131 |
|
- type: dot_ap |
|
value: 61.53586693000539 |
|
- type: dot_f1 |
|
value: 68.26923076923077 |
|
- type: dot_precision |
|
value: 52.53272623790552 |
|
- type: dot_recall |
|
value: 97.46568109820485 |
|
- type: euclidean_accuracy |
|
value: 62.912831618841366 |
|
- type: euclidean_ap |
|
value: 67.15479155849464 |
|
- type: euclidean_f1 |
|
value: 70.64071370640713 |
|
- type: euclidean_precision |
|
value: 57.34035549703752 |
|
- type: euclidean_recall |
|
value: 91.97465681098205 |
|
- type: manhattan_accuracy |
|
value: 63.50839198700595 |
|
- type: manhattan_ap |
|
value: 67.55807251483273 |
|
- type: manhattan_f1 |
|
value: 70.58356490670901 |
|
- type: manhattan_precision |
|
value: 56.55216284987278 |
|
- type: manhattan_recall |
|
value: 93.8753959873284 |
|
- type: max_accuracy |
|
value: 63.50839198700595 |
|
- type: max_ap |
|
value: 67.55807251483273 |
|
- type: max_f1 |
|
value: 70.64071370640713 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 87.11 |
|
- type: ap |
|
value: 84.20351278644551 |
|
- type: f1 |
|
value: 87.10043002123766 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 13.050279647770473 |
|
- type: cos_sim_spearman |
|
value: 14.227909232579874 |
|
- type: euclidean_pearson |
|
value: 16.372629300358096 |
|
- type: euclidean_spearman |
|
value: 14.68140021547196 |
|
- type: manhattan_pearson |
|
value: 16.266960163157336 |
|
- type: manhattan_spearman |
|
value: 14.627750758965616 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 30.56036276943463 |
|
- type: cos_sim_spearman |
|
value: 32.918859292204 |
|
- type: euclidean_pearson |
|
value: 31.679745438037195 |
|
- type: euclidean_spearman |
|
value: 33.68461814972644 |
|
- type: manhattan_pearson |
|
value: 31.994557954084563 |
|
- type: manhattan_spearman |
|
value: 33.97758185204816 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: quora |
|
name: MTEB QuoraRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 68.327 |
|
- type: map_at_10 |
|
value: 81.938 |
|
- type: map_at_100 |
|
value: 82.581 |
|
- type: map_at_1000 |
|
value: 82.60300000000001 |
|
- type: map_at_3 |
|
value: 78.89399999999999 |
|
- type: map_at_5 |
|
value: 80.816 |
|
- type: mrr_at_1 |
|
value: 78.75 |
|
- type: mrr_at_10 |
|
value: 85.302 |
|
- type: mrr_at_100 |
|
value: 85.432 |
|
- type: mrr_at_1000 |
|
value: 85.434 |
|
- type: mrr_at_3 |
|
value: 84.128 |
|
- type: mrr_at_5 |
|
value: 84.91199999999999 |
|
- type: ndcg_at_1 |
|
value: 78.74 |
|
- type: ndcg_at_10 |
|
value: 86.042 |
|
- type: ndcg_at_100 |
|
value: 87.468 |
|
- type: ndcg_at_1000 |
|
value: 87.641 |
|
- type: ndcg_at_3 |
|
value: 82.799 |
|
- type: ndcg_at_5 |
|
value: 84.603 |
|
- type: precision_at_1 |
|
value: 78.74 |
|
- type: precision_at_10 |
|
value: 13.071 |
|
- type: precision_at_100 |
|
value: 1.508 |
|
- type: precision_at_1000 |
|
value: 0.156 |
|
- type: precision_at_3 |
|
value: 36.08 |
|
- type: precision_at_5 |
|
value: 23.87 |
|
- type: recall_at_1 |
|
value: 68.327 |
|
- type: recall_at_10 |
|
value: 93.962 |
|
- type: recall_at_100 |
|
value: 99.054 |
|
- type: recall_at_1000 |
|
value: 99.9 |
|
- type: recall_at_3 |
|
value: 84.788 |
|
- type: recall_at_5 |
|
value: 89.73 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering |
|
name: MTEB RedditClustering |
|
config: default |
|
split: test |
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
metrics: |
|
- type: v_measure |
|
value: 41.337989152483956 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering-p2p |
|
name: MTEB RedditClusteringP2P |
|
config: default |
|
split: test |
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
|
metrics: |
|
- type: v_measure |
|
value: 51.2046136625677 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scidocs |
|
name: MTEB SCIDOCS |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 3.763 |
|
- type: map_at_10 |
|
value: 8.785 |
|
- type: map_at_100 |
|
value: 10.266 |
|
- type: map_at_1000 |
|
value: 10.506 |
|
- type: map_at_3 |
|
value: 6.551 |
|
- type: map_at_5 |
|
value: 7.670000000000001 |
|
- type: mrr_at_1 |
|
value: 18.5 |
|
- type: mrr_at_10 |
|
value: 27.771 |
|
- type: mrr_at_100 |
|
value: 28.842000000000002 |
|
- type: mrr_at_1000 |
|
value: 28.913 |
|
- type: mrr_at_3 |
|
value: 24.767 |
|
- type: mrr_at_5 |
|
value: 26.457000000000004 |
|
- type: ndcg_at_1 |
|
value: 18.5 |
|
- type: ndcg_at_10 |
|
value: 15.312000000000001 |
|
- type: ndcg_at_100 |
|
value: 21.599 |
|
- type: ndcg_at_1000 |
|
value: 26.473999999999997 |
|
- type: ndcg_at_3 |
|
value: 14.821000000000002 |
|
- type: ndcg_at_5 |
|
value: 12.836 |
|
- type: precision_at_1 |
|
value: 18.5 |
|
- type: precision_at_10 |
|
value: 7.779999999999999 |
|
- type: precision_at_100 |
|
value: 1.69 |
|
- type: precision_at_1000 |
|
value: 0.28700000000000003 |
|
- type: precision_at_3 |
|
value: 13.667000000000002 |
|
- type: precision_at_5 |
|
value: 11.08 |
|
- type: recall_at_1 |
|
value: 3.763 |
|
- type: recall_at_10 |
|
value: 15.798000000000002 |
|
- type: recall_at_100 |
|
value: 34.313 |
|
- type: recall_at_1000 |
|
value: 58.318000000000005 |
|
- type: recall_at_3 |
|
value: 8.312999999999999 |
|
- type: recall_at_5 |
|
value: 11.238 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.33402689861924 |
|
- type: cos_sim_spearman |
|
value: 78.52738315932625 |
|
- type: euclidean_pearson |
|
value: 80.800678573052 |
|
- type: euclidean_spearman |
|
value: 77.86666946799137 |
|
- type: manhattan_pearson |
|
value: 81.03106755866989 |
|
- type: manhattan_spearman |
|
value: 78.0676393879487 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.86998503723257 |
|
- type: cos_sim_spearman |
|
value: 74.07437934108376 |
|
- type: euclidean_pearson |
|
value: 80.91626452869946 |
|
- type: euclidean_spearman |
|
value: 76.88419802521403 |
|
- type: manhattan_pearson |
|
value: 81.50196980117957 |
|
- type: manhattan_spearman |
|
value: 77.2456891009073 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.19616084290932 |
|
- type: cos_sim_spearman |
|
value: 81.80834431353927 |
|
- type: euclidean_pearson |
|
value: 81.25429737195789 |
|
- type: euclidean_spearman |
|
value: 82.00934127307355 |
|
- type: manhattan_pearson |
|
value: 81.67403556759655 |
|
- type: manhattan_spearman |
|
value: 82.42359818976753 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.50884725941148 |
|
- type: cos_sim_spearman |
|
value: 77.0493522248929 |
|
- type: euclidean_pearson |
|
value: 79.15856111178543 |
|
- type: euclidean_spearman |
|
value: 77.24292975474096 |
|
- type: manhattan_pearson |
|
value: 79.22641788874807 |
|
- type: manhattan_spearman |
|
value: 77.37101663798234 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 83.75652767224308 |
|
- type: cos_sim_spearman |
|
value: 84.61113973428688 |
|
- type: euclidean_pearson |
|
value: 83.73646379542737 |
|
- type: euclidean_spearman |
|
value: 84.47126779405652 |
|
- type: manhattan_pearson |
|
value: 83.89617307570857 |
|
- type: manhattan_spearman |
|
value: 84.6073703393468 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.16302763567215 |
|
- type: cos_sim_spearman |
|
value: 83.08923353997561 |
|
- type: euclidean_pearson |
|
value: 80.08338016232464 |
|
- type: euclidean_spearman |
|
value: 80.40181608724076 |
|
- type: manhattan_pearson |
|
value: 80.02358856208708 |
|
- type: manhattan_spearman |
|
value: 80.30032329982274 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (ko-ko) |
|
config: ko-ko |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 56.45965932801117 |
|
- type: cos_sim_spearman |
|
value: 57.28270045199294 |
|
- type: euclidean_pearson |
|
value: 57.3615782157595 |
|
- type: euclidean_spearman |
|
value: 56.94348399074146 |
|
- type: manhattan_pearson |
|
value: 57.9426531718209 |
|
- type: manhattan_spearman |
|
value: 57.61844831263504 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (ar-ar) |
|
config: ar-ar |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 80.2973366536596 |
|
- type: cos_sim_spearman |
|
value: 80.60259304741632 |
|
- type: euclidean_pearson |
|
value: 78.30266089843892 |
|
- type: euclidean_spearman |
|
value: 78.06065126709282 |
|
- type: manhattan_pearson |
|
value: 78.61370380599344 |
|
- type: manhattan_spearman |
|
value: 78.45738598619143 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-ar) |
|
config: en-ar |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 72.35020162217042 |
|
- type: cos_sim_spearman |
|
value: 72.59857902847162 |
|
- type: euclidean_pearson |
|
value: 65.03547299350457 |
|
- type: euclidean_spearman |
|
value: 64.16617373109685 |
|
- type: manhattan_pearson |
|
value: 65.68996569454929 |
|
- type: manhattan_spearman |
|
value: 64.88542254595046 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-de) |
|
config: en-de |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 39.766484883595425 |
|
- type: cos_sim_spearman |
|
value: 40.3429946300341 |
|
- type: euclidean_pearson |
|
value: 39.47427150040957 |
|
- type: euclidean_spearman |
|
value: 39.072525589079696 |
|
- type: manhattan_pearson |
|
value: 40.56345338078474 |
|
- type: manhattan_spearman |
|
value: 40.444629078138036 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 88.83798941013089 |
|
- type: cos_sim_spearman |
|
value: 89.15159294402415 |
|
- type: euclidean_pearson |
|
value: 87.9810618414505 |
|
- type: euclidean_spearman |
|
value: 87.90818542026535 |
|
- type: manhattan_pearson |
|
value: 88.06116863048229 |
|
- type: manhattan_spearman |
|
value: 88.00182442010694 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-tr) |
|
config: en-tr |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 7.416028059666332 |
|
- type: cos_sim_spearman |
|
value: 6.792945857606915 |
|
- type: euclidean_pearson |
|
value: 11.485332917116061 |
|
- type: euclidean_spearman |
|
value: 9.793932873423419 |
|
- type: manhattan_pearson |
|
value: 9.148469412558393 |
|
- type: manhattan_spearman |
|
value: 7.803450524017845 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (es-en) |
|
config: es-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 80.16381852152489 |
|
- type: cos_sim_spearman |
|
value: 81.80324089694928 |
|
- type: euclidean_pearson |
|
value: 76.41433274302783 |
|
- type: euclidean_spearman |
|
value: 77.15238726996526 |
|
- type: manhattan_pearson |
|
value: 77.08610108551368 |
|
- type: manhattan_spearman |
|
value: 77.99971298324311 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (es-es) |
|
config: es-es |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 85.11032272383456 |
|
- type: cos_sim_spearman |
|
value: 85.64528002839239 |
|
- type: euclidean_pearson |
|
value: 85.54301672487198 |
|
- type: euclidean_spearman |
|
value: 84.21727806530393 |
|
- type: manhattan_pearson |
|
value: 85.57145576255618 |
|
- type: manhattan_spearman |
|
value: 84.07127479487694 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (fr-en) |
|
config: fr-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 79.73703272230806 |
|
- type: cos_sim_spearman |
|
value: 79.9424510113259 |
|
- type: euclidean_pearson |
|
value: 77.64485173960838 |
|
- type: euclidean_spearman |
|
value: 77.54693014468836 |
|
- type: manhattan_pearson |
|
value: 77.96911553781774 |
|
- type: manhattan_spearman |
|
value: 77.87266778206842 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (it-en) |
|
config: it-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 37.260672179617515 |
|
- type: cos_sim_spearman |
|
value: 34.80434004457536 |
|
- type: euclidean_pearson |
|
value: 38.55806751295782 |
|
- type: euclidean_spearman |
|
value: 36.129700913023115 |
|
- type: manhattan_pearson |
|
value: 40.74316244582763 |
|
- type: manhattan_spearman |
|
value: 38.60667540883322 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (nl-en) |
|
config: nl-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 38.038311386574456 |
|
- type: cos_sim_spearman |
|
value: 33.576193063894195 |
|
- type: euclidean_pearson |
|
value: 33.712663568034316 |
|
- type: euclidean_spearman |
|
value: 32.560617375956916 |
|
- type: manhattan_pearson |
|
value: 35.60457167895616 |
|
- type: manhattan_spearman |
|
value: 34.63036216555931 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 61.01583638162472 |
|
- type: cos_sim_spearman |
|
value: 62.92281428893316 |
|
- type: euclidean_pearson |
|
value: 62.939630289711815 |
|
- type: euclidean_spearman |
|
value: 64.15209661725994 |
|
- type: manhattan_pearson |
|
value: 64.24261705090608 |
|
- type: manhattan_spearman |
|
value: 64.78283158164017 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (de) |
|
config: de |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 21.529440799555704 |
|
- type: cos_sim_spearman |
|
value: 26.62727800620091 |
|
- type: euclidean_pearson |
|
value: 16.837244578590123 |
|
- type: euclidean_spearman |
|
value: 25.012107525591425 |
|
- type: manhattan_pearson |
|
value: 18.445531476179454 |
|
- type: manhattan_spearman |
|
value: 27.070240480795153 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (es) |
|
config: es |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 49.655500043363624 |
|
- type: cos_sim_spearman |
|
value: 56.31248457847469 |
|
- type: euclidean_pearson |
|
value: 48.787154598246616 |
|
- type: euclidean_spearman |
|
value: 52.90454409579225 |
|
- type: manhattan_pearson |
|
value: 55.392327232639836 |
|
- type: manhattan_spearman |
|
value: 57.3726886727899 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (pl) |
|
config: pl |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 2.9137753115190304 |
|
- type: cos_sim_spearman |
|
value: 15.062114976486532 |
|
- type: euclidean_pearson |
|
value: -2.034404984782681 |
|
- type: euclidean_spearman |
|
value: 14.683481835467338 |
|
- type: manhattan_pearson |
|
value: -0.22204468354050833 |
|
- type: manhattan_spearman |
|
value: 15.526420635759743 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (tr) |
|
config: tr |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 4.3616620418459915 |
|
- type: cos_sim_spearman |
|
value: 22.11078316878173 |
|
- type: euclidean_pearson |
|
value: 15.111514877123403 |
|
- type: euclidean_spearman |
|
value: 21.232869644925973 |
|
- type: manhattan_pearson |
|
value: 19.71276925909529 |
|
- type: manhattan_spearman |
|
value: 25.704469862313466 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (ar) |
|
config: ar |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 44.25888840250496 |
|
- type: cos_sim_spearman |
|
value: 54.82352971568842 |
|
- type: euclidean_pearson |
|
value: 48.00261414068268 |
|
- type: euclidean_spearman |
|
value: 53.3721608428832 |
|
- type: manhattan_pearson |
|
value: 50.6442021864215 |
|
- type: manhattan_spearman |
|
value: 55.352339945631954 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (ru) |
|
config: ru |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 0.08233514100531068 |
|
- type: cos_sim_spearman |
|
value: 28.771721168834276 |
|
- type: euclidean_pearson |
|
value: 10.783524938899138 |
|
- type: euclidean_spearman |
|
value: 24.67831010432439 |
|
- type: manhattan_pearson |
|
value: 16.98415610436092 |
|
- type: manhattan_spearman |
|
value: 25.81670115913176 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh) |
|
config: zh |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 36.86678706245425 |
|
- type: cos_sim_spearman |
|
value: 40.9736918674032 |
|
- type: euclidean_pearson |
|
value: 26.42365971768556 |
|
- type: euclidean_spearman |
|
value: 30.479818788692054 |
|
- type: manhattan_pearson |
|
value: 41.08694658968258 |
|
- type: manhattan_spearman |
|
value: 45.080877435751084 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (fr) |
|
config: fr |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 75.98114217777062 |
|
- type: cos_sim_spearman |
|
value: 78.7295845730892 |
|
- type: euclidean_pearson |
|
value: 76.99433076522276 |
|
- type: euclidean_spearman |
|
value: 79.71421663258973 |
|
- type: manhattan_pearson |
|
value: 78.65656344143478 |
|
- type: manhattan_spearman |
|
value: 80.60968909615123 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (de-en) |
|
config: de-en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 47.33261398683554 |
|
- type: cos_sim_spearman |
|
value: 49.547954534754666 |
|
- type: euclidean_pearson |
|
value: 48.23362592012922 |
|
- type: euclidean_spearman |
|
value: 49.17277986369927 |
|
- type: manhattan_pearson |
|
value: 49.06792311033889 |
|
- type: manhattan_spearman |
|
value: 51.27529282708198 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (es-en) |
|
config: es-en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 66.10070360470756 |
|
- type: cos_sim_spearman |
|
value: 71.03150249855938 |
|
- type: euclidean_pearson |
|
value: 67.05372897033872 |
|
- type: euclidean_spearman |
|
value: 69.73291838049877 |
|
- type: manhattan_pearson |
|
value: 70.34740916239467 |
|
- type: manhattan_spearman |
|
value: 72.40053406658815 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (it) |
|
config: it |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 56.581317404418904 |
|
- type: cos_sim_spearman |
|
value: 62.61318021096797 |
|
- type: euclidean_pearson |
|
value: 57.4403074342031 |
|
- type: euclidean_spearman |
|
value: 60.04897783631694 |
|
- type: manhattan_pearson |
|
value: 58.441729285803014 |
|
- type: manhattan_spearman |
|
value: 60.70510326005463 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (pl-en) |
|
config: pl-en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 47.064414464023905 |
|
- type: cos_sim_spearman |
|
value: 43.716659075869465 |
|
- type: euclidean_pearson |
|
value: 43.81699490724336 |
|
- type: euclidean_spearman |
|
value: 43.784380306563726 |
|
- type: manhattan_pearson |
|
value: 53.664583329563264 |
|
- type: manhattan_spearman |
|
value: 45.399271192350135 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh-en) |
|
config: zh-en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 63.585903017365055 |
|
- type: cos_sim_spearman |
|
value: 63.90147651068459 |
|
- type: euclidean_pearson |
|
value: 50.21918146173064 |
|
- type: euclidean_spearman |
|
value: 53.02530618040754 |
|
- type: manhattan_pearson |
|
value: 62.7472089813117 |
|
- type: manhattan_spearman |
|
value: 63.90440606248973 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (es-it) |
|
config: es-it |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 59.06715980430013 |
|
- type: cos_sim_spearman |
|
value: 61.2993294424547 |
|
- type: euclidean_pearson |
|
value: 53.67335552456426 |
|
- type: euclidean_spearman |
|
value: 55.32940583953816 |
|
- type: manhattan_pearson |
|
value: 58.08097600675386 |
|
- type: manhattan_spearman |
|
value: 57.1966250850173 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (de-fr) |
|
config: de-fr |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 18.94271219818519 |
|
- type: cos_sim_spearman |
|
value: 22.355519793818935 |
|
- type: euclidean_pearson |
|
value: 14.336479135636187 |
|
- type: euclidean_spearman |
|
value: 18.862751864788684 |
|
- type: manhattan_pearson |
|
value: 14.481730447681057 |
|
- type: manhattan_spearman |
|
value: 17.572142526671563 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (de-pl) |
|
config: de-pl |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 20.644357537446464 |
|
- type: cos_sim_spearman |
|
value: 35.32083671407284 |
|
- type: euclidean_pearson |
|
value: 28.24720906134992 |
|
- type: euclidean_spearman |
|
value: 46.437508077438395 |
|
- type: manhattan_pearson |
|
value: 42.09834718968137 |
|
- type: manhattan_spearman |
|
value: 53.02744622635869 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (fr-pl) |
|
config: fr-pl |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 71.84986730523782 |
|
- type: cos_sim_spearman |
|
value: 73.24670207647144 |
|
- type: euclidean_pearson |
|
value: 62.450055500805604 |
|
- type: euclidean_spearman |
|
value: 61.97797868009122 |
|
- type: manhattan_pearson |
|
value: 56.32083882980946 |
|
- type: manhattan_spearman |
|
value: 39.440531887330785 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 78.11479317838469 |
|
- type: cos_sim_spearman |
|
value: 77.7709743500025 |
|
- type: euclidean_pearson |
|
value: 78.83834281752932 |
|
- type: euclidean_spearman |
|
value: 78.21978829646487 |
|
- type: manhattan_pearson |
|
value: 79.36075578990533 |
|
- type: manhattan_spearman |
|
value: 78.72958965446072 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 82.92539499228975 |
|
- type: cos_sim_spearman |
|
value: 83.63025944536395 |
|
- type: euclidean_pearson |
|
value: 81.54744230098872 |
|
- type: euclidean_spearman |
|
value: 81.08707735758752 |
|
- type: manhattan_pearson |
|
value: 81.50252353111375 |
|
- type: manhattan_spearman |
|
value: 81.00641210322735 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
metrics: |
|
- type: map |
|
value: 75.12690809334019 |
|
- type: mrr |
|
value: 92.28846951886169 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scifact |
|
name: MTEB SciFact |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 47.15 |
|
- type: map_at_10 |
|
value: 56.748 |
|
- type: map_at_100 |
|
value: 57.528999999999996 |
|
- type: map_at_1000 |
|
value: 57.56400000000001 |
|
- type: map_at_3 |
|
value: 53.691 |
|
- type: map_at_5 |
|
value: 55.656000000000006 |
|
- type: mrr_at_1 |
|
value: 49.667 |
|
- type: mrr_at_10 |
|
value: 58.24700000000001 |
|
- type: mrr_at_100 |
|
value: 58.855000000000004 |
|
- type: mrr_at_1000 |
|
value: 58.888 |
|
- type: mrr_at_3 |
|
value: 55.72200000000001 |
|
- type: mrr_at_5 |
|
value: 57.272 |
|
- type: ndcg_at_1 |
|
value: 49.667 |
|
- type: ndcg_at_10 |
|
value: 61.739 |
|
- type: ndcg_at_100 |
|
value: 65.17399999999999 |
|
- type: ndcg_at_1000 |
|
value: 66.122 |
|
- type: ndcg_at_3 |
|
value: 56.266000000000005 |
|
- type: ndcg_at_5 |
|
value: 59.357000000000006 |
|
- type: precision_at_1 |
|
value: 49.667 |
|
- type: precision_at_10 |
|
value: 8.5 |
|
- type: precision_at_100 |
|
value: 1.04 |
|
- type: precision_at_1000 |
|
value: 0.11199999999999999 |
|
- type: precision_at_3 |
|
value: 22.111 |
|
- type: precision_at_5 |
|
value: 15.133 |
|
- type: recall_at_1 |
|
value: 47.15 |
|
- type: recall_at_10 |
|
value: 75.52799999999999 |
|
- type: recall_at_100 |
|
value: 91.167 |
|
- type: recall_at_1000 |
|
value: 98.667 |
|
- type: recall_at_3 |
|
value: 60.978 |
|
- type: recall_at_5 |
|
value: 68.839 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/sprintduplicatequestions-pairclassification |
|
name: MTEB SprintDuplicateQuestions |
|
config: default |
|
split: test |
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 99.71188118811881 |
|
- type: cos_sim_ap |
|
value: 92.0858173884619 |
|
- type: cos_sim_f1 |
|
value: 85.48864758144126 |
|
- type: cos_sim_precision |
|
value: 84.40545808966861 |
|
- type: cos_sim_recall |
|
value: 86.6 |
|
- type: dot_accuracy |
|
value: 99.57722772277228 |
|
- type: dot_ap |
|
value: 83.92226742515372 |
|
- type: dot_f1 |
|
value: 78.85091629519565 |
|
- type: dot_precision |
|
value: 78.11579980372915 |
|
- type: dot_recall |
|
value: 79.60000000000001 |
|
- type: euclidean_accuracy |
|
value: 99.6970297029703 |
|
- type: euclidean_ap |
|
value: 91.69378964699095 |
|
- type: euclidean_f1 |
|
value: 85.08771929824562 |
|
- type: euclidean_precision |
|
value: 82.98479087452472 |
|
- type: euclidean_recall |
|
value: 87.3 |
|
- type: manhattan_accuracy |
|
value: 99.7019801980198 |
|
- type: manhattan_ap |
|
value: 92.00969741996086 |
|
- type: manhattan_f1 |
|
value: 84.95752123938031 |
|
- type: manhattan_precision |
|
value: 84.91508491508492 |
|
- type: manhattan_recall |
|
value: 85.0 |
|
- type: max_accuracy |
|
value: 99.71188118811881 |
|
- type: max_ap |
|
value: 92.0858173884619 |
|
- type: max_f1 |
|
value: 85.48864758144126 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering |
|
name: MTEB StackExchangeClustering |
|
config: default |
|
split: test |
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
metrics: |
|
- type: v_measure |
|
value: 54.50675991473899 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering-p2p |
|
name: MTEB StackExchangeClusteringP2P |
|
config: default |
|
split: test |
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
metrics: |
|
- type: v_measure |
|
value: 31.12415042272221 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
metrics: |
|
- type: map |
|
value: 47.37961638353922 |
|
- type: mrr |
|
value: 48.04425558102029 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 31.358583236464177 |
|
- type: cos_sim_spearman |
|
value: 32.06044850511017 |
|
- type: dot_pearson |
|
value: 30.36343303587471 |
|
- type: dot_spearman |
|
value: 30.303932242144704 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 63.73951666189072 |
|
- type: mrr |
|
value: 73.54706021429108 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 16.892 |
|
- type: map_at_10 |
|
value: 40.215 |
|
- type: map_at_100 |
|
value: 43.9 |
|
- type: map_at_1000 |
|
value: 44.185 |
|
- type: map_at_3 |
|
value: 30.008000000000003 |
|
- type: map_at_5 |
|
value: 35.465 |
|
- type: mrr_at_1 |
|
value: 63.931000000000004 |
|
- type: mrr_at_10 |
|
value: 70.35 |
|
- type: mrr_at_100 |
|
value: 70.762 |
|
- type: mrr_at_1000 |
|
value: 70.784 |
|
- type: mrr_at_3 |
|
value: 68.863 |
|
- type: mrr_at_5 |
|
value: 69.758 |
|
- type: ndcg_at_1 |
|
value: 63.931000000000004 |
|
- type: ndcg_at_10 |
|
value: 51.573 |
|
- type: ndcg_at_100 |
|
value: 59.067 |
|
- type: ndcg_at_1000 |
|
value: 62.388 |
|
- type: ndcg_at_3 |
|
value: 55.422000000000004 |
|
- type: ndcg_at_5 |
|
value: 52.322 |
|
- type: precision_at_1 |
|
value: 63.931000000000004 |
|
- type: precision_at_10 |
|
value: 25.373 |
|
- type: precision_at_100 |
|
value: 3.894 |
|
- type: precision_at_1000 |
|
value: 0.47400000000000003 |
|
- type: precision_at_3 |
|
value: 48.083 |
|
- type: precision_at_5 |
|
value: 38.513 |
|
- type: recall_at_1 |
|
value: 16.892 |
|
- type: recall_at_10 |
|
value: 49.945 |
|
- type: recall_at_100 |
|
value: 73.41499999999999 |
|
- type: recall_at_1000 |
|
value: 89.776 |
|
- type: recall_at_3 |
|
value: 32.544000000000004 |
|
- type: recall_at_5 |
|
value: 40.501 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 44.153999999999996 |
|
- type: f1 |
|
value: 42.69123774230511 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: trec-covid |
|
name: MTEB TRECCOVID |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 0.22300000000000003 |
|
- type: map_at_10 |
|
value: 1.7999999999999998 |
|
- type: map_at_100 |
|
value: 9.098 |
|
- type: map_at_1000 |
|
value: 20.59 |
|
- type: map_at_3 |
|
value: 0.6459999999999999 |
|
- type: map_at_5 |
|
value: 1.006 |
|
- type: mrr_at_1 |
|
value: 84.0 |
|
- type: mrr_at_10 |
|
value: 91.5 |
|
- type: mrr_at_100 |
|
value: 91.5 |
|
- type: mrr_at_1000 |
|
value: 91.5 |
|
- type: mrr_at_3 |
|
value: 91.0 |
|
- type: mrr_at_5 |
|
value: 91.5 |
|
- type: ndcg_at_1 |
|
value: 80.0 |
|
- type: ndcg_at_10 |
|
value: 72.992 |
|
- type: ndcg_at_100 |
|
value: 51.778999999999996 |
|
- type: ndcg_at_1000 |
|
value: 44.473 |
|
- type: ndcg_at_3 |
|
value: 77.531 |
|
- type: ndcg_at_5 |
|
value: 74.685 |
|
- type: precision_at_1 |
|
value: 84.0 |
|
- type: precision_at_10 |
|
value: 78.60000000000001 |
|
- type: precision_at_100 |
|
value: 52.800000000000004 |
|
- type: precision_at_1000 |
|
value: 19.736 |
|
- type: precision_at_3 |
|
value: 83.333 |
|
- type: precision_at_5 |
|
value: 80.0 |
|
- type: recall_at_1 |
|
value: 0.22300000000000003 |
|
- type: recall_at_10 |
|
value: 2.016 |
|
- type: recall_at_100 |
|
value: 12.21 |
|
- type: recall_at_1000 |
|
value: 41.427 |
|
- type: recall_at_3 |
|
value: 0.6839999999999999 |
|
- type: recall_at_5 |
|
value: 1.083 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (sqi-eng) |
|
config: sqi-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 11.0 |
|
- type: f1 |
|
value: 8.487309997179562 |
|
- type: precision |
|
value: 7.935185890268856 |
|
- type: recall |
|
value: 11.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fry-eng) |
|
config: fry-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 23.699421965317917 |
|
- type: f1 |
|
value: 18.09982567208001 |
|
- type: precision |
|
value: 16.582017825552963 |
|
- type: recall |
|
value: 23.699421965317917 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kur-eng) |
|
config: kur-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.780487804878048 |
|
- type: f1 |
|
value: 6.484836753129436 |
|
- type: precision |
|
value: 5.916220801747723 |
|
- type: recall |
|
value: 8.780487804878048 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tur-eng) |
|
config: tur-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.0 |
|
- type: f1 |
|
value: 3.493223480735001 |
|
- type: precision |
|
value: 3.1492116349139385 |
|
- type: recall |
|
value: 5.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (deu-eng) |
|
config: deu-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 33.6 |
|
- type: f1 |
|
value: 29.339340352229065 |
|
- type: precision |
|
value: 27.997920626374693 |
|
- type: recall |
|
value: 33.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nld-eng) |
|
config: nld-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 20.200000000000003 |
|
- type: f1 |
|
value: 16.330981736231458 |
|
- type: precision |
|
value: 15.250949969794044 |
|
- type: recall |
|
value: 20.200000000000003 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ron-eng) |
|
config: ron-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 19.6 |
|
- type: f1 |
|
value: 14.951120083366323 |
|
- type: precision |
|
value: 13.617335362707001 |
|
- type: recall |
|
value: 19.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ang-eng) |
|
config: ang-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 20.149253731343283 |
|
- type: f1 |
|
value: 13.312899786780385 |
|
- type: precision |
|
value: 11.979388770433545 |
|
- type: recall |
|
value: 20.149253731343283 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ido-eng) |
|
config: ido-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 31.4 |
|
- type: f1 |
|
value: 26.21323201417634 |
|
- type: precision |
|
value: 24.607830064672168 |
|
- type: recall |
|
value: 31.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (jav-eng) |
|
config: jav-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 18.048780487804876 |
|
- type: f1 |
|
value: 14.347798542920492 |
|
- type: precision |
|
value: 13.301672920575362 |
|
- type: recall |
|
value: 18.048780487804876 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (isl-eng) |
|
config: isl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.2 |
|
- type: f1 |
|
value: 3.2713297295122503 |
|
- type: precision |
|
value: 2.978548911585725 |
|
- type: recall |
|
value: 5.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (slv-eng) |
|
config: slv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.411907654921021 |
|
- type: f1 |
|
value: 5.412915976323278 |
|
- type: precision |
|
value: 4.975402373122839 |
|
- type: recall |
|
value: 7.411907654921021 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cym-eng) |
|
config: cym-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.521739130434783 |
|
- type: f1 |
|
value: 5.871393789897329 |
|
- type: precision |
|
value: 5.350472658912557 |
|
- type: recall |
|
value: 8.521739130434783 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kaz-eng) |
|
config: kaz-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.565217391304348 |
|
- type: f1 |
|
value: 0.7422394530145001 |
|
- type: precision |
|
value: 0.7201734373569025 |
|
- type: recall |
|
value: 1.565217391304348 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (est-eng) |
|
config: est-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.3 |
|
- type: f1 |
|
value: 3.0838354401589694 |
|
- type: precision |
|
value: 2.709942839090994 |
|
- type: recall |
|
value: 5.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (heb-eng) |
|
config: heb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.8 |
|
- type: f1 |
|
value: 0.24583802742178057 |
|
- type: precision |
|
value: 0.18710578268453032 |
|
- type: recall |
|
value: 0.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gla-eng) |
|
config: gla-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.945717732207479 |
|
- type: f1 |
|
value: 2.7266734043909437 |
|
- type: precision |
|
value: 2.3247505400014186 |
|
- type: recall |
|
value: 4.945717732207479 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mar-eng) |
|
config: mar-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 54.2 |
|
- type: f1 |
|
value: 47.22780366692132 |
|
- type: precision |
|
value: 44.740178571428565 |
|
- type: recall |
|
value: 54.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lat-eng) |
|
config: lat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 25.8 |
|
- type: f1 |
|
value: 19.547406382656526 |
|
- type: precision |
|
value: 17.80766233766234 |
|
- type: recall |
|
value: 25.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bel-eng) |
|
config: bel-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.283031457969928 |
|
- type: precision |
|
value: 3.0361515007649467 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pms-eng) |
|
config: pms-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 22.476190476190478 |
|
- type: f1 |
|
value: 17.494204011570957 |
|
- type: precision |
|
value: 16.16236240785113 |
|
- type: recall |
|
value: 22.476190476190478 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gle-eng) |
|
config: gle-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.3 |
|
- type: f1 |
|
value: 3.461898170471662 |
|
- type: precision |
|
value: 2.975546957350575 |
|
- type: recall |
|
value: 6.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pes-eng) |
|
config: pes-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.6 |
|
- type: f1 |
|
value: 5.874235156578609 |
|
- type: precision |
|
value: 5.201352547725499 |
|
- type: recall |
|
value: 8.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nob-eng) |
|
config: nob-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 15.2 |
|
- type: f1 |
|
value: 11.908986787697534 |
|
- type: precision |
|
value: 11.090628985937808 |
|
- type: recall |
|
value: 15.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bul-eng) |
|
config: bul-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.9 |
|
- type: f1 |
|
value: 4.58348360335125 |
|
- type: precision |
|
value: 4.183620994869927 |
|
- type: recall |
|
value: 6.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cbk-eng) |
|
config: cbk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 62.1 |
|
- type: f1 |
|
value: 55.70845598845599 |
|
- type: precision |
|
value: 53.22281746031747 |
|
- type: recall |
|
value: 62.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hun-eng) |
|
config: hun-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.8 |
|
- type: f1 |
|
value: 3.246932234432234 |
|
- type: precision |
|
value: 2.9738765839703265 |
|
- type: recall |
|
value: 4.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (uig-eng) |
|
config: uig-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.8999999999999999 |
|
- type: f1 |
|
value: 0.5331481481481481 |
|
- type: precision |
|
value: 0.4918990604783396 |
|
- type: recall |
|
value: 0.8999999999999999 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (rus-eng) |
|
config: rus-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 31.7 |
|
- type: f1 |
|
value: 25.22406237037816 |
|
- type: precision |
|
value: 23.27273155929038 |
|
- type: recall |
|
value: 31.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (spa-eng) |
|
config: spa-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 96.5 |
|
- type: f1 |
|
value: 95.48333333333333 |
|
- type: precision |
|
value: 95.0 |
|
- type: recall |
|
value: 96.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hye-eng) |
|
config: hye-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.40431266846361186 |
|
- type: f1 |
|
value: 0.22521185350542844 |
|
- type: precision |
|
value: 0.20245384171411912 |
|
- type: recall |
|
value: 0.40431266846361186 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tel-eng) |
|
config: tel-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 43.162393162393165 |
|
- type: f1 |
|
value: 35.83662064431295 |
|
- type: precision |
|
value: 33.66590199923534 |
|
- type: recall |
|
value: 43.162393162393165 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (afr-eng) |
|
config: afr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 12.2 |
|
- type: f1 |
|
value: 9.007009351120605 |
|
- type: precision |
|
value: 8.26509907921979 |
|
- type: recall |
|
value: 12.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mon-eng) |
|
config: mon-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 2.0454545454545454 |
|
- type: f1 |
|
value: 0.846869670733307 |
|
- type: precision |
|
value: 0.719285857023819 |
|
- type: recall |
|
value: 2.0454545454545454 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (arz-eng) |
|
config: arz-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 56.18448637316562 |
|
- type: f1 |
|
value: 49.41850369523325 |
|
- type: precision |
|
value: 46.84486373165618 |
|
- type: recall |
|
value: 56.18448637316562 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hrv-eng) |
|
config: hrv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.4 |
|
- type: f1 |
|
value: 6.274306734742452 |
|
- type: precision |
|
value: 5.854786915151029 |
|
- type: recall |
|
value: 8.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nov-eng) |
|
config: nov-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 45.13618677042802 |
|
- type: f1 |
|
value: 38.784818726452976 |
|
- type: precision |
|
value: 36.65848310789945 |
|
- type: recall |
|
value: 45.13618677042802 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gsw-eng) |
|
config: gsw-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 23.076923076923077 |
|
- type: f1 |
|
value: 17.501757501757503 |
|
- type: precision |
|
value: 16.06289721674337 |
|
- type: recall |
|
value: 23.076923076923077 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nds-eng) |
|
config: nds-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 15.8 |
|
- type: f1 |
|
value: 11.834682187321722 |
|
- type: precision |
|
value: 10.871016304088595 |
|
- type: recall |
|
value: 15.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ukr-eng) |
|
config: ukr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.3 |
|
- type: f1 |
|
value: 4.929314970921539 |
|
- type: precision |
|
value: 4.427714750128542 |
|
- type: recall |
|
value: 7.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (uzb-eng) |
|
config: uzb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.14018691588785 |
|
- type: f1 |
|
value: 2.543797914741945 |
|
- type: precision |
|
value: 2.1476927403586066 |
|
- type: recall |
|
value: 5.14018691588785 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lit-eng) |
|
config: lit-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.0 |
|
- type: f1 |
|
value: 3.173243817101591 |
|
- type: precision |
|
value: 2.8643206769285485 |
|
- type: recall |
|
value: 5.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ina-eng) |
|
config: ina-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 69.5 |
|
- type: f1 |
|
value: 63.89614902641219 |
|
- type: precision |
|
value: 61.628650793650785 |
|
- type: recall |
|
value: 69.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lfn-eng) |
|
config: lfn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 41.8 |
|
- type: f1 |
|
value: 37.523909714712914 |
|
- type: precision |
|
value: 36.054581750900766 |
|
- type: recall |
|
value: 41.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (zsm-eng) |
|
config: zsm-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 79.2 |
|
- type: f1 |
|
value: 74.88805555555554 |
|
- type: precision |
|
value: 73.05083333333333 |
|
- type: recall |
|
value: 79.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ita-eng) |
|
config: ita-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 43.5 |
|
- type: f1 |
|
value: 37.28660019590605 |
|
- type: precision |
|
value: 35.18067447433519 |
|
- type: recall |
|
value: 43.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cmn-eng) |
|
config: cmn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 94.5 |
|
- type: f1 |
|
value: 92.95 |
|
- type: precision |
|
value: 92.2 |
|
- type: recall |
|
value: 94.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lvs-eng) |
|
config: lvs-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.2 |
|
- type: f1 |
|
value: 3.5297755651484026 |
|
- type: precision |
|
value: 3.190013722690584 |
|
- type: recall |
|
value: 5.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (glg-eng) |
|
config: glg-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 74.7 |
|
- type: f1 |
|
value: 69.2602380952381 |
|
- type: precision |
|
value: 67.03261904761905 |
|
- type: recall |
|
value: 74.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ceb-eng) |
|
config: ceb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.0 |
|
- type: f1 |
|
value: 5.639611303143687 |
|
- type: precision |
|
value: 5.209856824277429 |
|
- type: recall |
|
value: 8.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bre-eng) |
|
config: bre-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.1 |
|
- type: f1 |
|
value: 3.847611167634209 |
|
- type: precision |
|
value: 3.3324923687423693 |
|
- type: recall |
|
value: 6.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ben-eng) |
|
config: ben-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 75.5 |
|
- type: f1 |
|
value: 70.14214285714286 |
|
- type: precision |
|
value: 67.88761904761904 |
|
- type: recall |
|
value: 75.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swg-eng) |
|
config: swg-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 20.535714285714285 |
|
- type: f1 |
|
value: 16.437074829931973 |
|
- type: precision |
|
value: 15.459837781266353 |
|
- type: recall |
|
value: 20.535714285714285 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (arq-eng) |
|
config: arq-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 21.405049396267835 |
|
- type: f1 |
|
value: 16.162968480476714 |
|
- type: precision |
|
value: 14.506603642481391 |
|
- type: recall |
|
value: 21.405049396267835 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kab-eng) |
|
config: kab-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.4000000000000001 |
|
- type: f1 |
|
value: 0.8861559696342305 |
|
- type: precision |
|
value: 0.7898232323232323 |
|
- type: recall |
|
value: 1.4000000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fra-eng) |
|
config: fra-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 93.5 |
|
- type: f1 |
|
value: 91.65333333333334 |
|
- type: precision |
|
value: 90.80833333333332 |
|
- type: recall |
|
value: 93.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (por-eng) |
|
config: por-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 93.8 |
|
- type: f1 |
|
value: 92.08333333333333 |
|
- type: precision |
|
value: 91.23333333333333 |
|
- type: recall |
|
value: 93.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tat-eng) |
|
config: tat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.3 |
|
- type: f1 |
|
value: 0.9654912597950575 |
|
- type: precision |
|
value: 0.911237853823405 |
|
- type: recall |
|
value: 1.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (oci-eng) |
|
config: oci-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 35.5 |
|
- type: f1 |
|
value: 29.385868020868024 |
|
- type: precision |
|
value: 27.38218614718615 |
|
- type: recall |
|
value: 35.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pol-eng) |
|
config: pol-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.3 |
|
- type: f1 |
|
value: 5.625495291471218 |
|
- type: precision |
|
value: 5.006352187769519 |
|
- type: recall |
|
value: 8.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (war-eng) |
|
config: war-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 9.3 |
|
- type: f1 |
|
value: 7.188871139201601 |
|
- type: precision |
|
value: 6.68110313042221 |
|
- type: recall |
|
value: 9.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (aze-eng) |
|
config: aze-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.4368196711816386 |
|
- type: precision |
|
value: 3.1516575755476186 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (vie-eng) |
|
config: vie-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 94.5 |
|
- type: f1 |
|
value: 92.85666666666667 |
|
- type: precision |
|
value: 92.07499999999999 |
|
- type: recall |
|
value: 94.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nno-eng) |
|
config: nno-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.9 |
|
- type: f1 |
|
value: 8.052880589619718 |
|
- type: precision |
|
value: 7.2833020438680816 |
|
- type: recall |
|
value: 10.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cha-eng) |
|
config: cha-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 21.897810218978105 |
|
- type: f1 |
|
value: 16.459096459096457 |
|
- type: precision |
|
value: 14.99391727493917 |
|
- type: recall |
|
value: 21.897810218978105 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mhr-eng) |
|
config: mhr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.8 |
|
- type: f1 |
|
value: 0.43900258600589265 |
|
- type: precision |
|
value: 0.42151473277789064 |
|
- type: recall |
|
value: 0.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dan-eng) |
|
config: dan-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 14.899999999999999 |
|
- type: f1 |
|
value: 11.403181682754628 |
|
- type: precision |
|
value: 10.506373051667312 |
|
- type: recall |
|
value: 14.899999999999999 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ell-eng) |
|
config: ell-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.9 |
|
- type: f1 |
|
value: 0.8872641689515834 |
|
- type: precision |
|
value: 0.7857231069685399 |
|
- type: recall |
|
value: 1.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (amh-eng) |
|
config: amh-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.1904761904761905 |
|
- type: f1 |
|
value: 0.20847048496818082 |
|
- type: precision |
|
value: 0.11904761904761904 |
|
- type: recall |
|
value: 1.1904761904761905 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pam-eng) |
|
config: pam-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.3 |
|
- type: f1 |
|
value: 3.784571880595977 |
|
- type: precision |
|
value: 3.4556477020719782 |
|
- type: recall |
|
value: 5.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hsb-eng) |
|
config: hsb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 9.316770186335404 |
|
- type: f1 |
|
value: 6.80343720685027 |
|
- type: precision |
|
value: 6.316650292717499 |
|
- type: recall |
|
value: 9.316770186335404 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (srp-eng) |
|
config: srp-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.8999999999999995 |
|
- type: f1 |
|
value: 4.5486926228313695 |
|
- type: precision |
|
value: 4.311121913612427 |
|
- type: recall |
|
value: 5.8999999999999995 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (epo-eng) |
|
config: epo-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 18.099999999999998 |
|
- type: f1 |
|
value: 13.4170874831821 |
|
- type: precision |
|
value: 12.178193046524806 |
|
- type: recall |
|
value: 18.099999999999998 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kzj-eng) |
|
config: kzj-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.3999999999999995 |
|
- type: f1 |
|
value: 3.3905735425765524 |
|
- type: precision |
|
value: 3.2588935800436625 |
|
- type: recall |
|
value: 4.3999999999999995 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (awa-eng) |
|
config: awa-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 37.66233766233766 |
|
- type: f1 |
|
value: 30.539579468150897 |
|
- type: precision |
|
value: 28.60288100547841 |
|
- type: recall |
|
value: 37.66233766233766 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fao-eng) |
|
config: fao-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 12.213740458015266 |
|
- type: f1 |
|
value: 8.297822182308039 |
|
- type: precision |
|
value: 7.463649581970193 |
|
- type: recall |
|
value: 12.213740458015266 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mal-eng) |
|
config: mal-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 78.31149927219796 |
|
- type: f1 |
|
value: 73.35759340126152 |
|
- type: precision |
|
value: 71.26394953905871 |
|
- type: recall |
|
value: 78.31149927219796 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ile-eng) |
|
config: ile-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 51.800000000000004 |
|
- type: f1 |
|
value: 44.24010323010323 |
|
- type: precision |
|
value: 41.450707972582975 |
|
- type: recall |
|
value: 51.800000000000004 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bos-eng) |
|
config: bos-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 13.27683615819209 |
|
- type: f1 |
|
value: 9.167320569156727 |
|
- type: precision |
|
value: 8.200402665583079 |
|
- type: recall |
|
value: 13.27683615819209 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cor-eng) |
|
config: cor-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.8 |
|
- type: f1 |
|
value: 3.1268763352790283 |
|
- type: precision |
|
value: 2.84393718699601 |
|
- type: recall |
|
value: 4.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cat-eng) |
|
config: cat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 85.1 |
|
- type: f1 |
|
value: 81.55 |
|
- type: precision |
|
value: 79.98166666666665 |
|
- type: recall |
|
value: 85.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (eus-eng) |
|
config: eus-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 48.3 |
|
- type: f1 |
|
value: 42.347894491129786 |
|
- type: precision |
|
value: 40.36040404040404 |
|
- type: recall |
|
value: 48.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (yue-eng) |
|
config: yue-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 78.8 |
|
- type: f1 |
|
value: 74.35484848484847 |
|
- type: precision |
|
value: 72.43277777777777 |
|
- type: recall |
|
value: 78.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swe-eng) |
|
config: swe-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 13.900000000000002 |
|
- type: f1 |
|
value: 10.718252991153888 |
|
- type: precision |
|
value: 9.835761434404196 |
|
- type: recall |
|
value: 13.900000000000002 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dtp-eng) |
|
config: dtp-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.371714825002496 |
|
- type: precision |
|
value: 3.085928254003479 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kat-eng) |
|
config: kat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.5361930294906166 |
|
- type: f1 |
|
value: 0.40389703692021933 |
|
- type: precision |
|
value: 0.40302666854804575 |
|
- type: recall |
|
value: 0.5361930294906166 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (jpn-eng) |
|
config: jpn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 55.300000000000004 |
|
- type: f1 |
|
value: 48.83353113553113 |
|
- type: precision |
|
value: 46.48630659536542 |
|
- type: recall |
|
value: 55.300000000000004 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (csb-eng) |
|
config: csb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.300395256916996 |
|
- type: f1 |
|
value: 5.261552988548536 |
|
- type: precision |
|
value: 4.724388115499655 |
|
- type: recall |
|
value: 8.300395256916996 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (xho-eng) |
|
config: xho-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.450704225352112 |
|
- type: f1 |
|
value: 4.829974470478787 |
|
- type: precision |
|
value: 4.337585798478816 |
|
- type: recall |
|
value: 8.450704225352112 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (orv-eng) |
|
config: orv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.0778443113772456 |
|
- type: f1 |
|
value: 0.5373251562068135 |
|
- type: precision |
|
value: 0.5107640721914694 |
|
- type: recall |
|
value: 1.0778443113772456 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ind-eng) |
|
config: ind-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 88.5 |
|
- type: f1 |
|
value: 85.46333333333334 |
|
- type: precision |
|
value: 84.1 |
|
- type: recall |
|
value: 88.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tuk-eng) |
|
config: tuk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.41871921182266 |
|
- type: f1 |
|
value: 2.8063639248802965 |
|
- type: precision |
|
value: 2.2699550039451513 |
|
- type: recall |
|
value: 5.41871921182266 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (max-eng) |
|
config: max-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 40.49295774647887 |
|
- type: f1 |
|
value: 33.455454951933824 |
|
- type: precision |
|
value: 31.4339393461183 |
|
- type: recall |
|
value: 40.49295774647887 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swh-eng) |
|
config: swh-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 18.974358974358974 |
|
- type: f1 |
|
value: 14.517578026097205 |
|
- type: precision |
|
value: 13.3510327465177 |
|
- type: recall |
|
value: 18.974358974358974 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hin-eng) |
|
config: hin-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 88.5 |
|
- type: f1 |
|
value: 85.34666666666666 |
|
- type: precision |
|
value: 83.89999999999999 |
|
- type: recall |
|
value: 88.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dsb-eng) |
|
config: dsb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.1419624217119 |
|
- type: f1 |
|
value: 5.830783012763732 |
|
- type: precision |
|
value: 5.4408714223116545 |
|
- type: recall |
|
value: 8.1419624217119 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ber-eng) |
|
config: ber-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.800000000000001 |
|
- type: f1 |
|
value: 3.9245687335866406 |
|
- type: precision |
|
value: 3.5535667824951584 |
|
- type: recall |
|
value: 5.800000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tam-eng) |
|
config: tam-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 68.40390879478826 |
|
- type: f1 |
|
value: 62.25738069386277 |
|
- type: precision |
|
value: 60.10935318752908 |
|
- type: recall |
|
value: 68.40390879478826 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (slk-eng) |
|
config: slk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.1 |
|
- type: f1 |
|
value: 5.4876787833762135 |
|
- type: precision |
|
value: 5.126663482701374 |
|
- type: recall |
|
value: 7.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tgl-eng) |
|
config: tgl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.9 |
|
- type: f1 |
|
value: 6.519531004112515 |
|
- type: precision |
|
value: 5.987707404636394 |
|
- type: recall |
|
value: 8.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ast-eng) |
|
config: ast-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 66.92913385826772 |
|
- type: f1 |
|
value: 59.96062992125984 |
|
- type: precision |
|
value: 57.13348331458567 |
|
- type: recall |
|
value: 66.92913385826772 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mkd-eng) |
|
config: mkd-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.3 |
|
- type: f1 |
|
value: 2.765805343607201 |
|
- type: precision |
|
value: 2.5247851243177144 |
|
- type: recall |
|
value: 4.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (khm-eng) |
|
config: khm-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.41551246537396125 |
|
- type: f1 |
|
value: 0.1497838495760933 |
|
- type: precision |
|
value: 0.14429034844729552 |
|
- type: recall |
|
value: 0.41551246537396125 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ces-eng) |
|
config: ces-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 5.800000000000001 |
|
- type: f1 |
|
value: 3.761224995516873 |
|
- type: precision |
|
value: 3.2689210175496086 |
|
- type: recall |
|
value: 5.800000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tzl-eng) |
|
config: tzl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 16.346153846153847 |
|
- type: f1 |
|
value: 14.524291497975709 |
|
- type: precision |
|
value: 13.995726495726496 |
|
- type: recall |
|
value: 16.346153846153847 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (urd-eng) |
|
config: urd-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 67.80000000000001 |
|
- type: f1 |
|
value: 61.615800865800864 |
|
- type: precision |
|
value: 59.12333333333334 |
|
- type: recall |
|
value: 67.80000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ara-eng) |
|
config: ara-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 83.8 |
|
- type: f1 |
|
value: 80.08857142857143 |
|
- type: precision |
|
value: 78.46666666666667 |
|
- type: recall |
|
value: 83.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kor-eng) |
|
config: kor-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.2 |
|
- type: f1 |
|
value: 2.6507751588440254 |
|
- type: precision |
|
value: 2.335273168189835 |
|
- type: recall |
|
value: 4.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (yid-eng) |
|
config: yid-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.4716981132075472 |
|
- type: f1 |
|
value: 0.19293763102725367 |
|
- type: precision |
|
value: 0.1622040325564188 |
|
- type: recall |
|
value: 0.4716981132075472 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fin-eng) |
|
config: fin-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.5001791555125235 |
|
- type: precision |
|
value: 3.277940522301425 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tha-eng) |
|
config: tha-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.9124087591240875 |
|
- type: f1 |
|
value: 0.5083420229405631 |
|
- type: precision |
|
value: 0.4674562188049969 |
|
- type: recall |
|
value: 0.9124087591240875 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (wuu-eng) |
|
config: wuu-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 79.4 |
|
- type: f1 |
|
value: 74.62333333333333 |
|
- type: precision |
|
value: 72.52333333333334 |
|
- type: recall |
|
value: 79.4 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 51.02719281751054 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 48.31885339280247 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: webis-touche2020 |
|
name: MTEB Touche2020 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 2.426 |
|
- type: map_at_10 |
|
value: 9.029 |
|
- type: map_at_100 |
|
value: 14.299999999999999 |
|
- type: map_at_1000 |
|
value: 15.798000000000002 |
|
- type: map_at_3 |
|
value: 4.626 |
|
- type: map_at_5 |
|
value: 6.221 |
|
- type: mrr_at_1 |
|
value: 32.653 |
|
- type: mrr_at_10 |
|
value: 46.608 |
|
- type: mrr_at_100 |
|
value: 47.195 |
|
- type: mrr_at_1000 |
|
value: 47.208 |
|
- type: mrr_at_3 |
|
value: 41.837 |
|
- type: mrr_at_5 |
|
value: 43.673 |
|
- type: ndcg_at_1 |
|
value: 29.592000000000002 |
|
- type: ndcg_at_10 |
|
value: 23.354 |
|
- type: ndcg_at_100 |
|
value: 33.875 |
|
- type: ndcg_at_1000 |
|
value: 45.369 |
|
- type: ndcg_at_3 |
|
value: 25.734 |
|
- type: ndcg_at_5 |
|
value: 23.873 |
|
- type: precision_at_1 |
|
value: 32.653 |
|
- type: precision_at_10 |
|
value: 21.224 |
|
- type: precision_at_100 |
|
value: 7.122000000000001 |
|
- type: precision_at_1000 |
|
value: 1.459 |
|
- type: precision_at_3 |
|
value: 26.531 |
|
- type: precision_at_5 |
|
value: 24.082 |
|
- type: recall_at_1 |
|
value: 2.426 |
|
- type: recall_at_10 |
|
value: 15.622 |
|
- type: recall_at_100 |
|
value: 44.318999999999996 |
|
- type: recall_at_1000 |
|
value: 78.632 |
|
- type: recall_at_3 |
|
value: 5.798 |
|
- type: recall_at_5 |
|
value: 8.927 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/toxic_conversations_50k |
|
name: MTEB ToxicConversationsClassification |
|
config: default |
|
split: test |
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
|
metrics: |
|
- type: accuracy |
|
value: 67.9606 |
|
- type: ap |
|
value: 12.665547829558923 |
|
- type: f1 |
|
value: 52.10043478110198 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/tweet_sentiment_extraction |
|
name: MTEB TweetSentimentExtractionClassification |
|
config: default |
|
split: test |
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
metrics: |
|
- type: accuracy |
|
value: 59.601018675721576 |
|
- type: f1 |
|
value: 59.91486569196274 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/twentynewsgroups-clustering |
|
name: MTEB TwentyNewsgroupsClustering |
|
config: default |
|
split: test |
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
metrics: |
|
- type: v_measure |
|
value: 37.881729581540135 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twittersemeval2015-pairclassification |
|
name: MTEB TwitterSemEval2015 |
|
config: default |
|
split: test |
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 83.68003814746379 |
|
- type: cos_sim_ap |
|
value: 65.95659315362258 |
|
- type: cos_sim_f1 |
|
value: 61.94669484560291 |
|
- type: cos_sim_precision |
|
value: 55.80706579225725 |
|
- type: cos_sim_recall |
|
value: 69.6042216358839 |
|
- type: dot_accuracy |
|
value: 81.97532335936103 |
|
- type: dot_ap |
|
value: 58.99091918849294 |
|
- type: dot_f1 |
|
value: 57.098765432098766 |
|
- type: dot_precision |
|
value: 51.8990073370738 |
|
- type: dot_recall |
|
value: 63.45646437994723 |
|
- type: euclidean_accuracy |
|
value: 83.18531322644095 |
|
- type: euclidean_ap |
|
value: 64.5631762106556 |
|
- type: euclidean_f1 |
|
value: 61.150808574652125 |
|
- type: euclidean_precision |
|
value: 58.25173155003582 |
|
- type: euclidean_recall |
|
value: 64.35356200527704 |
|
- type: manhattan_accuracy |
|
value: 83.14358943792097 |
|
- type: manhattan_ap |
|
value: 64.73090464118813 |
|
- type: manhattan_f1 |
|
value: 61.228384019081695 |
|
- type: manhattan_precision |
|
value: 55.86507072905332 |
|
- type: manhattan_recall |
|
value: 67.73087071240106 |
|
- type: max_accuracy |
|
value: 83.68003814746379 |
|
- type: max_ap |
|
value: 65.95659315362258 |
|
- type: max_f1 |
|
value: 61.94669484560291 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twitterurlcorpus-pairclassification |
|
name: MTEB TwitterURLCorpus |
|
config: default |
|
split: test |
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 88.7161873714441 |
|
- type: cos_sim_ap |
|
value: 85.10870963707444 |
|
- type: cos_sim_f1 |
|
value: 77.88396923766146 |
|
- type: cos_sim_precision |
|
value: 75.59791274097695 |
|
- type: cos_sim_recall |
|
value: 80.31259624268556 |
|
- type: dot_accuracy |
|
value: 87.74595412737222 |
|
- type: dot_ap |
|
value: 81.22910623983562 |
|
- type: dot_f1 |
|
value: 76.08511889448344 |
|
- type: dot_precision |
|
value: 72.78672385908163 |
|
- type: dot_recall |
|
value: 79.69664305512781 |
|
- type: euclidean_accuracy |
|
value: 88.13404742500097 |
|
- type: euclidean_ap |
|
value: 84.03032098854915 |
|
- type: euclidean_f1 |
|
value: 76.3909440662918 |
|
- type: euclidean_precision |
|
value: 73.51894047279977 |
|
- type: euclidean_recall |
|
value: 79.49645826917154 |
|
- type: manhattan_accuracy |
|
value: 88.13598789148911 |
|
- type: manhattan_ap |
|
value: 84.13258714083858 |
|
- type: manhattan_f1 |
|
value: 76.44922164566346 |
|
- type: manhattan_precision |
|
value: 73.70640365923384 |
|
- type: manhattan_recall |
|
value: 79.40406529103788 |
|
- type: max_accuracy |
|
value: 88.7161873714441 |
|
- type: max_ap |
|
value: 85.10870963707444 |
|
- type: max_f1 |
|
value: 77.88396923766146 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 41.8 |
|
- type: map_at_10 |
|
value: 50.57000000000001 |
|
- type: map_at_100 |
|
value: 51.271 |
|
- type: map_at_1000 |
|
value: 51.31099999999999 |
|
- type: map_at_3 |
|
value: 48.283 |
|
- type: map_at_5 |
|
value: 49.633 |
|
- type: mrr_at_1 |
|
value: 41.8 |
|
- type: mrr_at_10 |
|
value: 50.57000000000001 |
|
- type: mrr_at_100 |
|
value: 51.271 |
|
- type: mrr_at_1000 |
|
value: 51.31099999999999 |
|
- type: mrr_at_3 |
|
value: 48.283 |
|
- type: mrr_at_5 |
|
value: 49.633 |
|
- type: ndcg_at_1 |
|
value: 41.8 |
|
- type: ndcg_at_10 |
|
value: 55.071999999999996 |
|
- type: ndcg_at_100 |
|
value: 58.604 |
|
- type: ndcg_at_1000 |
|
value: 59.679 |
|
- type: ndcg_at_3 |
|
value: 50.394000000000005 |
|
- type: ndcg_at_5 |
|
value: 52.825 |
|
- type: precision_at_1 |
|
value: 41.8 |
|
- type: precision_at_10 |
|
value: 6.93 |
|
- type: precision_at_100 |
|
value: 0.861 |
|
- type: precision_at_1000 |
|
value: 0.095 |
|
- type: precision_at_3 |
|
value: 18.833 |
|
- type: precision_at_5 |
|
value: 12.479999999999999 |
|
- type: recall_at_1 |
|
value: 41.8 |
|
- type: recall_at_10 |
|
value: 69.3 |
|
- type: recall_at_100 |
|
value: 86.1 |
|
- type: recall_at_1000 |
|
value: 94.6 |
|
- type: recall_at_3 |
|
value: 56.49999999999999 |
|
- type: recall_at_5 |
|
value: 62.4 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 80.65 |
|
- type: ap |
|
value: 59.927241826012924 |
|
- type: f1 |
|
value: 78.72456184299979 |
|
--- |
|
|
|
# Model Card for udever-bloom |
|
|
|
<!-- Provide a quick summary of what the model is/does. --> |
|
|
|
`udever-bloom-560m` is finetuned from [bigscience/bloom-560m](https://huggingface.co/bigscience/bloom-560m) via [BitFit](https://aclanthology.org/2022.acl-short.1/) on MS MARCO Passage Ranking, SNLI and MultiNLI data. |
|
It is a universal embedding model across tasks, natural and programming languages. |
|
(From the technical view, `udever` is merely with some minor improvements to `sgpt-bloom`) |
|
|
|
<img width="338" height="259" src="https://user-images.githubusercontent.com/26690193/277643721-cdb7f227-cae5-40e1-b6e1-a201bde00339.png" /> |
|
|
|
|
|
## Model Details |
|
|
|
### Model Description |
|
|
|
- **Developed by:** Alibaba Group |
|
- **Model type:** Transformer-based Language Model (decoder-only) |
|
- **Language(s) (NLP):** Multiple; see [bloom training data](https://huggingface.co/bigscience/bloom-560m#training-data) |
|
- **Finetuned from model :** [bigscience/bloom-560m](https://huggingface.co/bigscience/bloom-560m) |
|
|
|
### Model Sources |
|
|
|
<!-- Provide the basic links for the model. --> |
|
|
|
- **Repository:** [github.com/izhx/uni-rep](https://github.com/izhx/uni-rep) |
|
- **Paper :** [Language Models are Universal Embedders](https://arxiv.org/pdf/2310.08232.pdf) |
|
- **Training Date :** 2023-06 |
|
|
|
|
|
### Checkpoints |
|
- [udever-bloom-560m](https://huggingface.co/izhx/udever-bloom-560m) |
|
- [udever-bloom-1b1](https://huggingface.co/izhx/udever-bloom-1b1) |
|
- [udever-bloom-3b](https://huggingface.co/izhx/udever-bloom-3b) |
|
- [udever-bloom-7b1](https://huggingface.co/izhx/udever-bloom-7b1) |
|
|
|
On ModelScope / 魔搭社区: [udever-bloom-560m](https://modelscope.cn/models/damo/udever-bloom-560m), [udever-bloom-1b1](https://modelscope.cn/models/damo/udever-bloom-1b1), [udever-bloom-3b](https://modelscope.cn/models/damo/udever-bloom-3b), [udever-bloom-7b1](https://modelscope.cn/models/damo/udever-bloom-7b1) |
|
|
|
|
|
## How to Get Started with the Model |
|
|
|
Use the code below to get started with the model. |
|
|
|
```python |
|
import torch |
|
from transformers import AutoTokenizer, BloomModel |
|
|
|
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-560m') |
|
model = BloomModel.from_pretrained('izhx/udever-bloom-560m') |
|
|
|
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]' |
|
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod]) |
|
|
|
if tokenizer.padding_side != 'left': |
|
print('!!!', tokenizer.padding_side) |
|
tokenizer.padding_side = 'left' |
|
|
|
|
|
def encode(texts: list, is_query: bool = True, max_length=300): |
|
bos = boq if is_query else bod |
|
eos_id = eoq_id if is_query else eod_id |
|
texts = [bos + t for t in texts] |
|
encoding = tokenizer( |
|
texts, truncation=True, max_length=max_length - 1, padding=True |
|
) |
|
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']): |
|
ids.append(eos_id) |
|
mask.append(1) |
|
inputs = tokenizer.pad(encoding, return_tensors='pt') |
|
with torch.inference_mode(): |
|
outputs = model(**inputs) |
|
embeds = outputs.last_hidden_state[:, -1] |
|
return embeds |
|
|
|
encode(['I am Bert', 'You are Elmo']) |
|
|
|
``` |
|
|
|
## Training Details |
|
|
|
### Training Data |
|
|
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. --> |
|
|
|
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86) |
|
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz) |
|
|
|
|
|
### Training Procedure |
|
|
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. --> |
|
|
|
#### Preprocessing |
|
|
|
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). |
|
Negatives for SNLI and MultiNLI are randomly sampled. |
|
|
|
|
|
#### Training Hyperparameters |
|
|
|
- **Training regime:** tf32, BitFit |
|
- **Batch size:** 1024 |
|
- **Epochs:** 3 |
|
- **Optimizer:** AdamW |
|
- **Learning rate:** 1e-4 |
|
- **Scheduler:** constant with warmup. |
|
- **Warmup:** 0.25 epoch |
|
|
|
|
|
## Evaluation |
|
|
|
### Table 1: Massive Text Embedding Benchmark [MTEB](https://huggingface.co/spaces/mteb/leaderboard) |
|
|
|
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. | |
|
|-----------------------------|--------------|--------------|--------------|--------------|--------------|--------------|--------------|--------| |
|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 | |
|
|| |
|
| bge-large-en-v1.5 | **64.23** | **75.97** | 46.08| **87.12** | **60.03** | **54.29** | 83.11| 31.61 | |
|
| bge-base-en-v1.5 | 63.55| 75.53| 45.77| 86.55| 58.86| 53.25| 82.4| 31.07 | |
|
| gte-large | 63.13| 73.33| **46.84** | 85| 59.13| 52.22| **83.35** | 31.66 | |
|
| gte-base | 62.39| 73.01| 46.2| 84.57| 58.61| 51.14| 82.3| 31.17 | |
|
| e5-large-v2 | 62.25| 75.24| 44.49| 86.03| 56.61| 50.56| 82.05| 30.19 | |
|
| instructor-xl | 61.79| 73.12| 44.74| 86.62| 57.29| 49.26| 83.06| 32.32 | |
|
| instructor-large | 61.59| 73.86| 45.29| 85.89| 57.54| 47.57| 83.15| 31.84 | |
|
| e5-base-v2 | 61.5 | 73.84| 43.8| 85.73| 55.91| 50.29| 81.05| 30.28 | |
|
| e5-large | 61.42| 73.14| 43.33| 85.94| 56.53| 49.99| 82.06| 30.97 | |
|
| text-embedding-ada-002 (OpenAI API) | 60.99| 70.93| 45.9 | 84.89| 56.32| 49.25| 80.97| 30.8 | |
|
| e5-base | 60.44| 72.63| 42.11| 85.09| 55.7 | 48.75| 80.96| 31.01 | |
|
| SGPT-5.8B-msmarco | 58.93| 68.13| 40.34| 82 | 56.56| 50.25| 78.1 | 31.46 | |
|
| sgpt-bloom-7b1-msmarco | 57.59| 66.19| 38.93| 81.9 | 55.65| 48.22| 77.74| **33.6** | |
|
|| |
|
| Udever-bloom-560m | 55.80| 68.04| 36.89| 81.05| 52.60| 41.19| 79.93| 32.06 | |
|
| Udever-bloom-1b1 | 58.28| 70.18| 39.11| 83.11| 54.28| 45.27| 81.52| 31.10 | |
|
| Udever-bloom-3b | 59.86| 71.91| 40.74| 84.06| 54.90| 47.67| 82.37| 30.62 | |
|
| Udever-bloom-7b1 | 60.63 | 72.13| 40.81| 85.40| 55.91| 49.34| 83.01| 30.97 | |
|
|
|
|
|
### Table 2: [CodeSearchNet](https://github.com/github/CodeSearchNet) |
|
|
|
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. | |
|
|-|-|-|-|-|-|-|-| |
|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 | |
|
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 | |
|
| cpt-code S | **97.7** | **86.3** | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 | |
|
| cpt-code M | 97.5 | 85.5 | **99.9** | **94.4** | **86.5** | **97.2** | **93.5** | |
|
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 | |
|
|| |
|
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 | |
|
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 | |
|
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 | |
|
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 | |
|
|
|
|
|
### Table 3: Chinese multi-domain retrieval [Multi-cpr](https://dl.acm.org/doi/10.1145/3477495.3531736) |
|
|
|
| | | |E-commerce | | Entertainment video | | Medical | | |
|
|--|--|--|--|--|--|--|--|--| |
|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k | |
|
|| |
|
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 | |
|
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 | |
|
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 | |
|
| DPR-2 | In-Domain | BERT-CT | 0.289 | **0.926** | 0.263 | **0.935** | 0.339 | **0.769** | |
|
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 | |
|
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 | |
|
|| |
|
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 | |
|
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 | |
|
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 | |
|
| Udever-bloom-7b1 | General | BLOOM | **0.296** | 0.889 | **0.267** | 0.907 | **0.343** | 0.705 | |
|
|
|
#### More results refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 3. |
|
|
|
|
|
|
|
## Technical Specifications |
|
|
|
### Model Architecture and Objective |
|
|
|
- Model: [bigscience/bloom-560m](https://huggingface.co/bigscience/bloom-560m). |
|
- Objective: Constrastive loss with hard negatives (refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 2.2). |
|
|
|
|
|
### Compute Infrastructure |
|
|
|
- Nvidia A100 SXM4 80GB. |
|
- torch 2.0.0, transformers 4.29.2. |
|
|
|
|
|
## Citation |
|
|
|
**BibTeX:** |
|
|
|
```BibTeX |
|
@article{zhang2023language, |
|
title={Language Models are Universal Embedders}, |
|
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min}, |
|
journal={arXiv preprint arXiv:2310.08232}, |
|
year={2023} |
|
} |
|
``` |
|
|
