metadata
license: bigscience-bloom-rail-1.0
language:
- ak
- ar
- as
- bm
- bn
- ca
- code
- en
- es
- eu
- fon
- fr
- gu
- hi
- id
- ig
- ki
- kn
- lg
- ln
- ml
- mr
- ne
- nso
- ny
- or
- pa
- pt
- rn
- rw
- sn
- st
- sw
- ta
- te
- tn
- ts
- tum
- tw
- ur
- vi
- wo
- xh
- yo
- zh
- zhs
- zht
- zu
tags:
- mteb
model-index:
- name: udever-bloom-7b1
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 31.3788313486292
- type: cos_sim_spearman
value: 31.87117445808444
- type: euclidean_pearson
value: 30.66886666881808
- type: euclidean_spearman
value: 31.28368681542041
- type: manhattan_pearson
value: 30.679984531432936
- type: manhattan_spearman
value: 31.22208726593753
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 38.403248424956764
- type: cos_sim_spearman
value: 38.798254852046504
- type: euclidean_pearson
value: 41.154981142995084
- type: euclidean_spearman
value: 38.73503172297125
- type: manhattan_pearson
value: 41.20226384035751
- type: manhattan_spearman
value: 38.77085234568287
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 73.11940298507463
- type: ap
value: 35.692863077186466
- type: f1
value: 67.02733552778966
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
value: 88.885175
- type: ap
value: 84.75400736514149
- type: f1
value: 88.85806225869703
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 43.202
- type: f1
value: 42.63847450850621
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.676
- type: map_at_10
value: 42.539
- type: map_at_100
value: 43.383
- type: map_at_1000
value: 43.39
- type: map_at_3
value: 36.996
- type: map_at_5
value: 40.175
- type: mrr_at_1
value: 26.387
- type: mrr_at_10
value: 42.792
- type: mrr_at_100
value: 43.637
- type: mrr_at_1000
value: 43.644
- type: mrr_at_3
value: 37.21
- type: mrr_at_5
value: 40.407
- type: ndcg_at_1
value: 25.676
- type: ndcg_at_10
value: 52.207
- type: ndcg_at_100
value: 55.757999999999996
- type: ndcg_at_1000
value: 55.913999999999994
- type: ndcg_at_3
value: 40.853
- type: ndcg_at_5
value: 46.588
- type: precision_at_1
value: 25.676
- type: precision_at_10
value: 8.314
- type: precision_at_100
value: 0.985
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 17.354
- type: precision_at_5
value: 13.200999999999999
- type: recall_at_1
value: 25.676
- type: recall_at_10
value: 83.14399999999999
- type: recall_at_100
value: 98.506
- type: recall_at_1000
value: 99.644
- type: recall_at_3
value: 52.063
- type: recall_at_5
value: 66.003
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
value: 45.66024127046263
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 38.418361433667336
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 61.60189642383972
- type: mrr
value: 75.26678538451391
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
value: 87.85884182572595
- type: cos_sim_spearman
value: 85.5242378844044
- type: euclidean_pearson
value: 85.37705073557146
- type: euclidean_spearman
value: 84.65132642825964
- type: manhattan_pearson
value: 85.42179213807349
- type: manhattan_spearman
value: 84.6959057572829
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 47.81802155652125
- type: cos_sim_spearman
value: 47.66691834501235
- type: euclidean_pearson
value: 47.781824357030935
- type: euclidean_spearman
value: 48.03322284408188
- type: manhattan_pearson
value: 47.871159981038346
- type: manhattan_spearman
value: 48.18240784527666
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (de-en)
config: de-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 88.29853862212944
- type: f1
value: 87.70994966904566
- type: precision
value: 87.43152897902377
- type: recall
value: 88.29853862212944
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (fr-en)
config: fr-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 98.6022452124147
- type: f1
value: 98.40597255851495
- type: precision
value: 98.30875339349916
- type: recall
value: 98.6022452124147
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (ru-en)
config: ru-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 79.64669206789054
- type: f1
value: 78.74831345770036
- type: precision
value: 78.33899087865143
- type: recall
value: 79.64669206789054
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (zh-en)
config: zh-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 98.78883622959452
- type: f1
value: 98.7712831314727
- type: precision
value: 98.76250658241179
- type: recall
value: 98.78883622959452
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 85.36363636363637
- type: f1
value: 85.33381612267455
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 35.54276849354455
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 32.18953191097238
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.00041315364012
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.35255790689628
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 70.54141681949504
- type: mrr
value: 74.81400793650795
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 71.3534829537025
- type: mrr
value: 75.85095238095238
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 32.5
- type: map_at_10
value: 43.37
- type: map_at_100
value: 44.926
- type: map_at_1000
value: 45.047
- type: map_at_3
value: 40.083999999999996
- type: map_at_5
value: 41.71
- type: mrr_at_1
value: 40.343
- type: mrr_at_10
value: 49.706
- type: mrr_at_100
value: 50.470000000000006
- type: mrr_at_1000
value: 50.515
- type: mrr_at_3
value: 47.306
- type: mrr_at_5
value: 48.379
- type: ndcg_at_1
value: 40.343
- type: ndcg_at_10
value: 49.461
- type: ndcg_at_100
value: 55.084999999999994
- type: ndcg_at_1000
value: 56.994
- type: ndcg_at_3
value: 44.896
- type: ndcg_at_5
value: 46.437
- type: precision_at_1
value: 40.343
- type: precision_at_10
value: 9.27
- type: precision_at_100
value: 1.5190000000000001
- type: precision_at_1000
value: 0.197
- type: precision_at_3
value: 21.412
- type: precision_at_5
value: 15.021
- type: recall_at_1
value: 32.5
- type: recall_at_10
value: 60.857000000000006
- type: recall_at_100
value: 83.761
- type: recall_at_1000
value: 96.003
- type: recall_at_3
value: 46.675
- type: recall_at_5
value: 51.50900000000001
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 26.931
- type: map_at_10
value: 35.769
- type: map_at_100
value: 36.8
- type: map_at_1000
value: 36.925999999999995
- type: map_at_3
value: 33.068999999999996
- type: map_at_5
value: 34.615
- type: mrr_at_1
value: 34.013
- type: mrr_at_10
value: 41.293
- type: mrr_at_100
value: 41.945
- type: mrr_at_1000
value: 42.002
- type: mrr_at_3
value: 39.204
- type: mrr_at_5
value: 40.436
- type: ndcg_at_1
value: 34.013
- type: ndcg_at_10
value: 40.935
- type: ndcg_at_100
value: 44.879999999999995
- type: ndcg_at_1000
value: 47.342
- type: ndcg_at_3
value: 37.071
- type: ndcg_at_5
value: 38.903
- type: precision_at_1
value: 34.013
- type: precision_at_10
value: 7.617999999999999
- type: precision_at_100
value: 1.185
- type: precision_at_1000
value: 0.169
- type: precision_at_3
value: 17.855999999999998
- type: precision_at_5
value: 12.65
- type: recall_at_1
value: 26.931
- type: recall_at_10
value: 50.256
- type: recall_at_100
value: 67.026
- type: recall_at_1000
value: 83.138
- type: recall_at_3
value: 38.477
- type: recall_at_5
value: 43.784
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 38.474000000000004
- type: map_at_10
value: 50.486
- type: map_at_100
value: 51.620999999999995
- type: map_at_1000
value: 51.675000000000004
- type: map_at_3
value: 47.64
- type: map_at_5
value: 49.187999999999995
- type: mrr_at_1
value: 43.824000000000005
- type: mrr_at_10
value: 53.910000000000004
- type: mrr_at_100
value: 54.601
- type: mrr_at_1000
value: 54.632000000000005
- type: mrr_at_3
value: 51.578
- type: mrr_at_5
value: 52.922999999999995
- type: ndcg_at_1
value: 43.824000000000005
- type: ndcg_at_10
value: 56.208000000000006
- type: ndcg_at_100
value: 60.624
- type: ndcg_at_1000
value: 61.78
- type: ndcg_at_3
value: 51.27
- type: ndcg_at_5
value: 53.578
- type: precision_at_1
value: 43.824000000000005
- type: precision_at_10
value: 8.978
- type: precision_at_100
value: 1.216
- type: precision_at_1000
value: 0.136
- type: precision_at_3
value: 22.884
- type: precision_at_5
value: 15.498000000000001
- type: recall_at_1
value: 38.474000000000004
- type: recall_at_10
value: 69.636
- type: recall_at_100
value: 88.563
- type: recall_at_1000
value: 96.86200000000001
- type: recall_at_3
value: 56.347
- type: recall_at_5
value: 61.980000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 23.13
- type: map_at_10
value: 31.892
- type: map_at_100
value: 32.938
- type: map_at_1000
value: 33.025999999999996
- type: map_at_3
value: 29.072
- type: map_at_5
value: 30.775000000000002
- type: mrr_at_1
value: 25.197999999999997
- type: mrr_at_10
value: 34.224
- type: mrr_at_100
value: 35.149
- type: mrr_at_1000
value: 35.215999999999994
- type: mrr_at_3
value: 31.563000000000002
- type: mrr_at_5
value: 33.196
- type: ndcg_at_1
value: 25.197999999999997
- type: ndcg_at_10
value: 37.117
- type: ndcg_at_100
value: 42.244
- type: ndcg_at_1000
value: 44.432
- type: ndcg_at_3
value: 31.604
- type: ndcg_at_5
value: 34.543
- type: precision_at_1
value: 25.197999999999997
- type: precision_at_10
value: 5.876
- type: precision_at_100
value: 0.886
- type: precision_at_1000
value: 0.11100000000000002
- type: precision_at_3
value: 13.672
- type: precision_at_5
value: 9.831
- type: recall_at_1
value: 23.13
- type: recall_at_10
value: 50.980000000000004
- type: recall_at_100
value: 74.565
- type: recall_at_1000
value: 90.938
- type: recall_at_3
value: 36.038
- type: recall_at_5
value: 43.326
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.317
- type: map_at_10
value: 24.517
- type: map_at_100
value: 25.771
- type: map_at_1000
value: 25.915
- type: map_at_3
value: 22.332
- type: map_at_5
value: 23.526
- type: mrr_at_1
value: 21.766
- type: mrr_at_10
value: 29.096
- type: mrr_at_100
value: 30.165
- type: mrr_at_1000
value: 30.253000000000004
- type: mrr_at_3
value: 27.114
- type: mrr_at_5
value: 28.284
- type: ndcg_at_1
value: 21.766
- type: ndcg_at_10
value: 29.060999999999996
- type: ndcg_at_100
value: 35.107
- type: ndcg_at_1000
value: 38.339
- type: ndcg_at_3
value: 25.121
- type: ndcg_at_5
value: 26.953
- type: precision_at_1
value: 21.766
- type: precision_at_10
value: 5.274
- type: precision_at_100
value: 0.958
- type: precision_at_1000
value: 0.13699999999999998
- type: precision_at_3
value: 11.816
- type: precision_at_5
value: 8.433
- type: recall_at_1
value: 17.317
- type: recall_at_10
value: 38.379999999999995
- type: recall_at_100
value: 64.792
- type: recall_at_1000
value: 87.564
- type: recall_at_3
value: 27.737000000000002
- type: recall_at_5
value: 32.340999999999994
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 28.876
- type: map_at_10
value: 40.02
- type: map_at_100
value: 41.367
- type: map_at_1000
value: 41.482
- type: map_at_3
value: 36.651
- type: map_at_5
value: 38.411
- type: mrr_at_1
value: 35.804
- type: mrr_at_10
value: 45.946999999999996
- type: mrr_at_100
value: 46.696
- type: mrr_at_1000
value: 46.741
- type: mrr_at_3
value: 43.118
- type: mrr_at_5
value: 44.74
- type: ndcg_at_1
value: 35.804
- type: ndcg_at_10
value: 46.491
- type: ndcg_at_100
value: 51.803
- type: ndcg_at_1000
value: 53.845
- type: ndcg_at_3
value: 40.97
- type: ndcg_at_5
value: 43.431
- type: precision_at_1
value: 35.804
- type: precision_at_10
value: 8.595
- type: precision_at_100
value: 1.312
- type: precision_at_1000
value: 0.167
- type: precision_at_3
value: 19.634
- type: precision_at_5
value: 13.879
- type: recall_at_1
value: 28.876
- type: recall_at_10
value: 59.952000000000005
- type: recall_at_100
value: 81.978
- type: recall_at_1000
value: 95.03399999999999
- type: recall_at_3
value: 44.284
- type: recall_at_5
value: 50.885999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.238
- type: map_at_10
value: 34.276
- type: map_at_100
value: 35.65
- type: map_at_1000
value: 35.769
- type: map_at_3
value: 31.227
- type: map_at_5
value: 33.046
- type: mrr_at_1
value: 30.137000000000004
- type: mrr_at_10
value: 39.473
- type: mrr_at_100
value: 40.400999999999996
- type: mrr_at_1000
value: 40.455000000000005
- type: mrr_at_3
value: 36.891
- type: mrr_at_5
value: 38.391999999999996
- type: ndcg_at_1
value: 30.137000000000004
- type: ndcg_at_10
value: 40.08
- type: ndcg_at_100
value: 46.01
- type: ndcg_at_1000
value: 48.36
- type: ndcg_at_3
value: 35.163
- type: ndcg_at_5
value: 37.583
- type: precision_at_1
value: 30.137000000000004
- type: precision_at_10
value: 7.466
- type: precision_at_100
value: 1.228
- type: precision_at_1000
value: 0.16199999999999998
- type: precision_at_3
value: 17.122999999999998
- type: precision_at_5
value: 12.283
- type: recall_at_1
value: 24.238
- type: recall_at_10
value: 52.078
- type: recall_at_100
value: 77.643
- type: recall_at_1000
value: 93.49199999999999
- type: recall_at_3
value: 38.161
- type: recall_at_5
value: 44.781
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.915250000000004
- type: map_at_10
value: 33.98191666666666
- type: map_at_100
value: 35.19166666666667
- type: map_at_1000
value: 35.30983333333333
- type: map_at_3
value: 31.27391666666666
- type: map_at_5
value: 32.74366666666666
- type: mrr_at_1
value: 29.800749999999994
- type: mrr_at_10
value: 38.235749999999996
- type: mrr_at_100
value: 39.10616666666667
- type: mrr_at_1000
value: 39.166583333333335
- type: mrr_at_3
value: 35.91033333333334
- type: mrr_at_5
value: 37.17766666666667
- type: ndcg_at_1
value: 29.800749999999994
- type: ndcg_at_10
value: 39.287833333333325
- type: ndcg_at_100
value: 44.533833333333334
- type: ndcg_at_1000
value: 46.89608333333333
- type: ndcg_at_3
value: 34.676
- type: ndcg_at_5
value: 36.75208333333333
- type: precision_at_1
value: 29.800749999999994
- type: precision_at_10
value: 6.9134166666666665
- type: precision_at_100
value: 1.1206666666666665
- type: precision_at_1000
value: 0.15116666666666667
- type: precision_at_3
value: 16.069083333333335
- type: precision_at_5
value: 11.337916666666668
- type: recall_at_1
value: 24.915250000000004
- type: recall_at_10
value: 50.86333333333334
- type: recall_at_100
value: 73.85574999999999
- type: recall_at_1000
value: 90.24041666666666
- type: recall_at_3
value: 37.80116666666666
- type: recall_at_5
value: 43.263
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.853
- type: map_at_10
value: 30.349999999999998
- type: map_at_100
value: 31.341
- type: map_at_1000
value: 31.44
- type: map_at_3
value: 28.294999999999998
- type: map_at_5
value: 29.412
- type: mrr_at_1
value: 25.919999999999998
- type: mrr_at_10
value: 33.194
- type: mrr_at_100
value: 34.071
- type: mrr_at_1000
value: 34.136
- type: mrr_at_3
value: 31.391000000000002
- type: mrr_at_5
value: 32.311
- type: ndcg_at_1
value: 25.919999999999998
- type: ndcg_at_10
value: 34.691
- type: ndcg_at_100
value: 39.83
- type: ndcg_at_1000
value: 42.193000000000005
- type: ndcg_at_3
value: 30.91
- type: ndcg_at_5
value: 32.634
- type: precision_at_1
value: 25.919999999999998
- type: precision_at_10
value: 5.521
- type: precision_at_100
value: 0.882
- type: precision_at_1000
value: 0.117
- type: precision_at_3
value: 13.547999999999998
- type: precision_at_5
value: 9.293999999999999
- type: recall_at_1
value: 22.853
- type: recall_at_10
value: 45.145
- type: recall_at_100
value: 69.158
- type: recall_at_1000
value: 86.354
- type: recall_at_3
value: 34.466
- type: recall_at_5
value: 39.044000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.151
- type: map_at_10
value: 23.674
- type: map_at_100
value: 24.738
- type: map_at_1000
value: 24.864
- type: map_at_3
value: 21.514
- type: map_at_5
value: 22.695
- type: mrr_at_1
value: 20.991
- type: mrr_at_10
value: 27.612
- type: mrr_at_100
value: 28.526
- type: mrr_at_1000
value: 28.603
- type: mrr_at_3
value: 25.618999999999996
- type: mrr_at_5
value: 26.674
- type: ndcg_at_1
value: 20.991
- type: ndcg_at_10
value: 27.983000000000004
- type: ndcg_at_100
value: 33.190999999999995
- type: ndcg_at_1000
value: 36.172
- type: ndcg_at_3
value: 24.195
- type: ndcg_at_5
value: 25.863999999999997
- type: precision_at_1
value: 20.991
- type: precision_at_10
value: 5.093
- type: precision_at_100
value: 0.8959999999999999
- type: precision_at_1000
value: 0.132
- type: precision_at_3
value: 11.402
- type: precision_at_5
value: 8.197000000000001
- type: recall_at_1
value: 17.151
- type: recall_at_10
value: 37.025000000000006
- type: recall_at_100
value: 60.787
- type: recall_at_1000
value: 82.202
- type: recall_at_3
value: 26.19
- type: recall_at_5
value: 30.657
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.463
- type: map_at_10
value: 34.372
- type: map_at_100
value: 35.475
- type: map_at_1000
value: 35.582
- type: map_at_3
value: 31.791000000000004
- type: map_at_5
value: 33.292
- type: mrr_at_1
value: 30.784
- type: mrr_at_10
value: 38.948
- type: mrr_at_100
value: 39.792
- type: mrr_at_1000
value: 39.857
- type: mrr_at_3
value: 36.614000000000004
- type: mrr_at_5
value: 37.976
- type: ndcg_at_1
value: 30.784
- type: ndcg_at_10
value: 39.631
- type: ndcg_at_100
value: 44.747
- type: ndcg_at_1000
value: 47.172
- type: ndcg_at_3
value: 34.976
- type: ndcg_at_5
value: 37.241
- type: precision_at_1
value: 30.784
- type: precision_at_10
value: 6.622999999999999
- type: precision_at_100
value: 1.04
- type: precision_at_1000
value: 0.135
- type: precision_at_3
value: 16.014
- type: precision_at_5
value: 11.286999999999999
- type: recall_at_1
value: 25.463
- type: recall_at_10
value: 51.23799999999999
- type: recall_at_100
value: 73.4
- type: recall_at_1000
value: 90.634
- type: recall_at_3
value: 38.421
- type: recall_at_5
value: 44.202999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 23.714
- type: map_at_10
value: 32.712
- type: map_at_100
value: 34.337
- type: map_at_1000
value: 34.556
- type: map_at_3
value: 29.747
- type: map_at_5
value: 31.208000000000002
- type: mrr_at_1
value: 29.051
- type: mrr_at_10
value: 37.589
- type: mrr_at_100
value: 38.638
- type: mrr_at_1000
value: 38.692
- type: mrr_at_3
value: 35.079
- type: mrr_at_5
value: 36.265
- type: ndcg_at_1
value: 29.051
- type: ndcg_at_10
value: 38.681
- type: ndcg_at_100
value: 44.775999999999996
- type: ndcg_at_1000
value: 47.354
- type: ndcg_at_3
value: 33.888
- type: ndcg_at_5
value: 35.854
- type: precision_at_1
value: 29.051
- type: precision_at_10
value: 7.489999999999999
- type: precision_at_100
value: 1.518
- type: precision_at_1000
value: 0.241
- type: precision_at_3
value: 16.008
- type: precision_at_5
value: 11.66
- type: recall_at_1
value: 23.714
- type: recall_at_10
value: 50.324000000000005
- type: recall_at_100
value: 77.16
- type: recall_at_1000
value: 93.186
- type: recall_at_3
value: 36.356
- type: recall_at_5
value: 41.457
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 18.336
- type: map_at_10
value: 26.345000000000002
- type: map_at_100
value: 27.336
- type: map_at_1000
value: 27.436
- type: map_at_3
value: 23.865
- type: map_at_5
value: 25.046000000000003
- type: mrr_at_1
value: 19.778000000000002
- type: mrr_at_10
value: 27.837
- type: mrr_at_100
value: 28.82
- type: mrr_at_1000
value: 28.897000000000002
- type: mrr_at_3
value: 25.446999999999996
- type: mrr_at_5
value: 26.556
- type: ndcg_at_1
value: 19.778000000000002
- type: ndcg_at_10
value: 31.115
- type: ndcg_at_100
value: 36.109
- type: ndcg_at_1000
value: 38.769999999999996
- type: ndcg_at_3
value: 26.048
- type: ndcg_at_5
value: 28.004
- type: precision_at_1
value: 19.778000000000002
- type: precision_at_10
value: 5.157
- type: precision_at_100
value: 0.808
- type: precision_at_1000
value: 0.11
- type: precision_at_3
value: 11.459999999999999
- type: precision_at_5
value: 8.022
- type: recall_at_1
value: 18.336
- type: recall_at_10
value: 44.489000000000004
- type: recall_at_100
value: 67.43599999999999
- type: recall_at_1000
value: 87.478
- type: recall_at_3
value: 30.462
- type: recall_at_5
value: 35.188
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 10.747
- type: map_at_10
value: 18.625
- type: map_at_100
value: 20.465
- type: map_at_1000
value: 20.639
- type: map_at_3
value: 15.57
- type: map_at_5
value: 17.089
- type: mrr_at_1
value: 24.169
- type: mrr_at_10
value: 35.96
- type: mrr_at_100
value: 36.888
- type: mrr_at_1000
value: 36.931999999999995
- type: mrr_at_3
value: 32.443
- type: mrr_at_5
value: 34.433
- type: ndcg_at_1
value: 24.169
- type: ndcg_at_10
value: 26.791999999999998
- type: ndcg_at_100
value: 34.054
- type: ndcg_at_1000
value: 37.285000000000004
- type: ndcg_at_3
value: 21.636
- type: ndcg_at_5
value: 23.394000000000002
- type: precision_at_1
value: 24.169
- type: precision_at_10
value: 8.476
- type: precision_at_100
value: 1.6209999999999998
- type: precision_at_1000
value: 0.22200000000000003
- type: precision_at_3
value: 16.156000000000002
- type: precision_at_5
value: 12.520999999999999
- type: recall_at_1
value: 10.747
- type: recall_at_10
value: 32.969
- type: recall_at_100
value: 57.99999999999999
- type: recall_at_1000
value: 76.12299999999999
- type: recall_at_3
value: 20.315
- type: recall_at_5
value: 25.239
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 14.751
- type: map_at_10
value: 22.03
- type: map_at_100
value: 23.471
- type: map_at_1000
value: 23.644000000000002
- type: map_at_3
value: 19.559
- type: map_at_5
value: 20.863
- type: mrr_at_1
value: 23.581
- type: mrr_at_10
value: 29.863
- type: mrr_at_100
value: 30.839
- type: mrr_at_1000
value: 30.925000000000004
- type: mrr_at_3
value: 27.894000000000002
- type: mrr_at_5
value: 28.965999999999998
- type: ndcg_at_1
value: 23.581
- type: ndcg_at_10
value: 26.996
- type: ndcg_at_100
value: 33.537
- type: ndcg_at_1000
value: 37.307
- type: ndcg_at_3
value: 23.559
- type: ndcg_at_5
value: 24.839
- type: precision_at_1
value: 23.581
- type: precision_at_10
value: 6.209
- type: precision_at_100
value: 1.165
- type: precision_at_1000
value: 0.165
- type: precision_at_3
value: 13.62
- type: precision_at_5
value: 9.882
- type: recall_at_1
value: 14.751
- type: recall_at_10
value: 34.075
- type: recall_at_100
value: 61.877
- type: recall_at_1000
value: 88.212
- type: recall_at_3
value: 23.519000000000002
- type: recall_at_5
value: 27.685
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 76.36800962116656
- type: cos_sim_ap
value: 85.14376065556142
- type: cos_sim_f1
value: 77.81474723623485
- type: cos_sim_precision
value: 71.92460317460318
- type: cos_sim_recall
value: 84.75566986205284
- type: dot_accuracy
value: 71.94227300060132
- type: dot_ap
value: 79.03676891584456
- type: dot_f1
value: 74.95833333333334
- type: dot_precision
value: 67.59346233327072
- type: dot_recall
value: 84.12438625204582
- type: euclidean_accuracy
value: 76.043295249549
- type: euclidean_ap
value: 85.28765360616536
- type: euclidean_f1
value: 78.01733248784612
- type: euclidean_precision
value: 71.1861137897782
- type: euclidean_recall
value: 86.29880757540333
- type: manhattan_accuracy
value: 76.17558628983764
- type: manhattan_ap
value: 85.52739323094916
- type: manhattan_f1
value: 78.30788804071246
- type: manhattan_precision
value: 71.63918525703201
- type: manhattan_recall
value: 86.34556932429273
- type: max_accuracy
value: 76.36800962116656
- type: max_ap
value: 85.52739323094916
- type: max_f1
value: 78.30788804071246
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 56.164
- type: map_at_10
value: 64.575
- type: map_at_100
value: 65.098
- type: map_at_1000
value: 65.118
- type: map_at_3
value: 62.329
- type: map_at_5
value: 63.535
- type: mrr_at_1
value: 56.269999999999996
- type: mrr_at_10
value: 64.63600000000001
- type: mrr_at_100
value: 65.14
- type: mrr_at_1000
value: 65.16
- type: mrr_at_3
value: 62.522
- type: mrr_at_5
value: 63.57000000000001
- type: ndcg_at_1
value: 56.269999999999996
- type: ndcg_at_10
value: 68.855
- type: ndcg_at_100
value: 71.47099999999999
- type: ndcg_at_1000
value: 72.02499999999999
- type: ndcg_at_3
value: 64.324
- type: ndcg_at_5
value: 66.417
- type: precision_at_1
value: 56.269999999999996
- type: precision_at_10
value: 8.303
- type: precision_at_100
value: 0.9570000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 23.427999999999997
- type: precision_at_5
value: 15.09
- type: recall_at_1
value: 56.164
- type: recall_at_10
value: 82.271
- type: recall_at_100
value: 94.626
- type: recall_at_1000
value: 99.05199999999999
- type: recall_at_3
value: 69.94200000000001
- type: recall_at_5
value: 74.947
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 8.686
- type: map_at_10
value: 17.766000000000002
- type: map_at_100
value: 23.507
- type: map_at_1000
value: 24.757
- type: map_at_3
value: 13.238
- type: map_at_5
value: 15.161
- type: mrr_at_1
value: 65.25
- type: mrr_at_10
value: 72.88
- type: mrr_at_100
value: 73.246
- type: mrr_at_1000
value: 73.261
- type: mrr_at_3
value: 71.542
- type: mrr_at_5
value: 72.392
- type: ndcg_at_1
value: 53.75
- type: ndcg_at_10
value: 37.623
- type: ndcg_at_100
value: 40.302
- type: ndcg_at_1000
value: 47.471999999999994
- type: ndcg_at_3
value: 43.324
- type: ndcg_at_5
value: 39.887
- type: precision_at_1
value: 65.25
- type: precision_at_10
value: 28.749999999999996
- type: precision_at_100
value: 8.34
- type: precision_at_1000
value: 1.703
- type: precision_at_3
value: 46.583000000000006
- type: precision_at_5
value: 38
- type: recall_at_1
value: 8.686
- type: recall_at_10
value: 22.966
- type: recall_at_100
value: 44.3
- type: recall_at_1000
value: 67.77499999999999
- type: recall_at_3
value: 14.527999999999999
- type: recall_at_5
value: 17.617
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 22.439
- type: map_at_10
value: 68.484
- type: map_at_100
value: 71.67999999999999
- type: map_at_1000
value: 71.761
- type: map_at_3
value: 46.373999999999995
- type: map_at_5
value: 58.697
- type: mrr_at_1
value: 80.65
- type: mrr_at_10
value: 86.53
- type: mrr_at_100
value: 86.624
- type: mrr_at_1000
value: 86.631
- type: mrr_at_3
value: 85.95
- type: mrr_at_5
value: 86.297
- type: ndcg_at_1
value: 80.65
- type: ndcg_at_10
value: 78.075
- type: ndcg_at_100
value: 82.014
- type: ndcg_at_1000
value: 82.903
- type: ndcg_at_3
value: 75.785
- type: ndcg_at_5
value: 74.789
- type: precision_at_1
value: 80.65
- type: precision_at_10
value: 38.425
- type: precision_at_100
value: 4.62
- type: precision_at_1000
value: 0.483
- type: precision_at_3
value: 68.25
- type: precision_at_5
value: 57.92
- type: recall_at_1
value: 22.439
- type: recall_at_10
value: 80.396
- type: recall_at_100
value: 92.793
- type: recall_at_1000
value: 97.541
- type: recall_at_3
value: 49.611
- type: recall_at_5
value: 65.065
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 43.9
- type: map_at_10
value: 53.394
- type: map_at_100
value: 54.078
- type: map_at_1000
value: 54.105000000000004
- type: map_at_3
value: 50.583
- type: map_at_5
value: 52.443
- type: mrr_at_1
value: 43.9
- type: mrr_at_10
value: 53.394
- type: mrr_at_100
value: 54.078
- type: mrr_at_1000
value: 54.105000000000004
- type: mrr_at_3
value: 50.583
- type: mrr_at_5
value: 52.443
- type: ndcg_at_1
value: 43.9
- type: ndcg_at_10
value: 58.341
- type: ndcg_at_100
value: 61.753
- type: ndcg_at_1000
value: 62.525
- type: ndcg_at_3
value: 52.699
- type: ndcg_at_5
value: 56.042
- type: precision_at_1
value: 43.9
- type: precision_at_10
value: 7.3999999999999995
- type: precision_at_100
value: 0.901
- type: precision_at_1000
value: 0.096
- type: precision_at_3
value: 19.6
- type: precision_at_5
value: 13.38
- type: recall_at_1
value: 43.9
- type: recall_at_10
value: 74
- type: recall_at_100
value: 90.10000000000001
- type: recall_at_1000
value: 96.3
- type: recall_at_3
value: 58.8
- type: recall_at_5
value: 66.9
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
config: default
split: test
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
value: 48.765
- type: f1
value: 44.2791193129597
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 56.89999999999999
- type: map_at_10
value: 68.352
- type: map_at_100
value: 68.768
- type: map_at_1000
value: 68.782
- type: map_at_3
value: 66.27300000000001
- type: map_at_5
value: 67.67699999999999
- type: mrr_at_1
value: 61.476
- type: mrr_at_10
value: 72.662
- type: mrr_at_100
value: 72.993
- type: mrr_at_1000
value: 72.99799999999999
- type: mrr_at_3
value: 70.75200000000001
- type: mrr_at_5
value: 72.056
- type: ndcg_at_1
value: 61.476
- type: ndcg_at_10
value: 73.98400000000001
- type: ndcg_at_100
value: 75.744
- type: ndcg_at_1000
value: 76.036
- type: ndcg_at_3
value: 70.162
- type: ndcg_at_5
value: 72.482
- type: precision_at_1
value: 61.476
- type: precision_at_10
value: 9.565
- type: precision_at_100
value: 1.054
- type: precision_at_1000
value: 0.109
- type: precision_at_3
value: 27.943
- type: precision_at_5
value: 18.056
- type: recall_at_1
value: 56.89999999999999
- type: recall_at_10
value: 87.122
- type: recall_at_100
value: 94.742
- type: recall_at_1000
value: 96.70100000000001
- type: recall_at_3
value: 76.911
- type: recall_at_5
value: 82.607
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.610999999999997
- type: map_at_10
value: 29.12
- type: map_at_100
value: 30.958000000000002
- type: map_at_1000
value: 31.151
- type: map_at_3
value: 25.369000000000003
- type: map_at_5
value: 27.445000000000004
- type: mrr_at_1
value: 35.185
- type: mrr_at_10
value: 44.533
- type: mrr_at_100
value: 45.385
- type: mrr_at_1000
value: 45.432
- type: mrr_at_3
value: 42.258
- type: mrr_at_5
value: 43.608999999999995
- type: ndcg_at_1
value: 35.185
- type: ndcg_at_10
value: 36.696
- type: ndcg_at_100
value: 43.491
- type: ndcg_at_1000
value: 46.800000000000004
- type: ndcg_at_3
value: 33.273
- type: ndcg_at_5
value: 34.336
- type: precision_at_1
value: 35.185
- type: precision_at_10
value: 10.309
- type: precision_at_100
value: 1.719
- type: precision_at_1000
value: 0.231
- type: precision_at_3
value: 22.479
- type: precision_at_5
value: 16.481
- type: recall_at_1
value: 17.610999999999997
- type: recall_at_10
value: 43.29
- type: recall_at_100
value: 68.638
- type: recall_at_1000
value: 88.444
- type: recall_at_3
value: 30.303
- type: recall_at_5
value: 35.856
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 34.18
- type: map_at_10
value: 47.753
- type: map_at_100
value: 48.522
- type: map_at_1000
value: 48.596000000000004
- type: map_at_3
value: 45.222
- type: map_at_5
value: 46.793
- type: mrr_at_1
value: 68.35900000000001
- type: mrr_at_10
value: 74.503
- type: mrr_at_100
value: 74.811
- type: mrr_at_1000
value: 74.82799999999999
- type: mrr_at_3
value: 73.347
- type: mrr_at_5
value: 74.06700000000001
- type: ndcg_at_1
value: 68.35900000000001
- type: ndcg_at_10
value: 56.665
- type: ndcg_at_100
value: 59.629
- type: ndcg_at_1000
value: 61.222
- type: ndcg_at_3
value: 52.81400000000001
- type: ndcg_at_5
value: 54.94
- type: precision_at_1
value: 68.35900000000001
- type: precision_at_10
value: 11.535
- type: precision_at_100
value: 1.388
- type: precision_at_1000
value: 0.16
- type: precision_at_3
value: 32.784
- type: precision_at_5
value: 21.348
- type: recall_at_1
value: 34.18
- type: recall_at_10
value: 57.677
- type: recall_at_100
value: 69.379
- type: recall_at_1000
value: 80.061
- type: recall_at_3
value: 49.175999999999995
- type: recall_at_5
value: 53.369
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 46.23316660253944
- type: f1
value: 39.09397722262806
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
config: default
split: test
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
value: 78.46119999999999
- type: ap
value: 72.53477126781094
- type: f1
value: 78.28701752379332
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.16510318949344
- type: ap
value: 50.10324581565756
- type: f1
value: 78.34748161287605
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 68.71925879533819
- type: cos_sim_spearman
value: 75.33926640820977
- type: euclidean_pearson
value: 74.59557932790653
- type: euclidean_spearman
value: 75.76006440878783
- type: manhattan_pearson
value: 74.7461963483351
- type: manhattan_spearman
value: 75.87111519308131
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 66.249
- type: map_at_10
value: 75.236
- type: map_at_100
value: 75.581
- type: map_at_1000
value: 75.593
- type: map_at_3
value: 73.463
- type: map_at_5
value: 74.602
- type: mrr_at_1
value: 68.42399999999999
- type: mrr_at_10
value: 75.81099999999999
- type: mrr_at_100
value: 76.115
- type: mrr_at_1000
value: 76.126
- type: mrr_at_3
value: 74.26899999999999
- type: mrr_at_5
value: 75.24300000000001
- type: ndcg_at_1
value: 68.42399999999999
- type: ndcg_at_10
value: 78.81700000000001
- type: ndcg_at_100
value: 80.379
- type: ndcg_at_1000
value: 80.667
- type: ndcg_at_3
value: 75.476
- type: ndcg_at_5
value: 77.38199999999999
- type: precision_at_1
value: 68.42399999999999
- type: precision_at_10
value: 9.491
- type: precision_at_100
value: 1.027
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 28.352
- type: precision_at_5
value: 18.043
- type: recall_at_1
value: 66.249
- type: recall_at_10
value: 89.238
- type: recall_at_100
value: 96.319
- type: recall_at_1000
value: 98.524
- type: recall_at_3
value: 80.438
- type: recall_at_5
value: 84.95
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.083000000000002
- type: map_at_10
value: 35.251
- type: map_at_100
value: 36.461
- type: map_at_1000
value: 36.507
- type: map_at_3
value: 31.474999999999998
- type: map_at_5
value: 33.658
- type: mrr_at_1
value: 23.724999999999998
- type: mrr_at_10
value: 35.88
- type: mrr_at_100
value: 37.021
- type: mrr_at_1000
value: 37.062
- type: mrr_at_3
value: 32.159
- type: mrr_at_5
value: 34.325
- type: ndcg_at_1
value: 23.724999999999998
- type: ndcg_at_10
value: 42.018
- type: ndcg_at_100
value: 47.764
- type: ndcg_at_1000
value: 48.916
- type: ndcg_at_3
value: 34.369
- type: ndcg_at_5
value: 38.266
- type: precision_at_1
value: 23.724999999999998
- type: precision_at_10
value: 6.553000000000001
- type: precision_at_100
value: 0.942
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 14.532
- type: precision_at_5
value: 10.696
- type: recall_at_1
value: 23.083000000000002
- type: recall_at_10
value: 62.739
- type: recall_at_100
value: 89.212
- type: recall_at_1000
value: 97.991
- type: recall_at_3
value: 42.064
- type: recall_at_5
value: 51.417
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
config: en
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 93.43365253077975
- type: f1
value: 93.07455671032345
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
config: en
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 71.72822617419061
- type: f1
value: 55.6093871673643
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (en)
config: en
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 72.03765971755212
- type: f1
value: 70.88235592002572
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (en)
config: en
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 76.86281102891728
- type: f1
value: 77.15496923811003
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 41.8
- type: map_at_10
value: 46.993
- type: map_at_100
value: 47.534
- type: map_at_1000
value: 47.587
- type: map_at_3
value: 45.717
- type: map_at_5
value: 46.357
- type: mrr_at_1
value: 42
- type: mrr_at_10
value: 47.093
- type: mrr_at_100
value: 47.634
- type: mrr_at_1000
value: 47.687000000000005
- type: mrr_at_3
value: 45.817
- type: mrr_at_5
value: 46.457
- type: ndcg_at_1
value: 41.8
- type: ndcg_at_10
value: 49.631
- type: ndcg_at_100
value: 52.53
- type: ndcg_at_1000
value: 54.238
- type: ndcg_at_3
value: 46.949000000000005
- type: ndcg_at_5
value: 48.102000000000004
- type: precision_at_1
value: 41.8
- type: precision_at_10
value: 5.800000000000001
- type: precision_at_100
value: 0.722
- type: precision_at_1000
value: 0.086
- type: precision_at_3
value: 16.833000000000002
- type: precision_at_5
value: 10.66
- type: recall_at_1
value: 41.8
- type: recall_at_10
value: 57.99999999999999
- type: recall_at_100
value: 72.2
- type: recall_at_1000
value: 86.3
- type: recall_at_3
value: 50.5
- type: recall_at_5
value: 53.300000000000004
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
config: default
split: test
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
value: 30.949060810392886
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
config: default
split: test
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
value: 28.87339864059011
- task:
type: Reranking
dataset:
type: mteb/mind_small
name: MTEB MindSmallReranking
config: default
split: test
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
metrics:
- type: map
value: 31.217934626189926
- type: mrr
value: 32.27509143911496
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 26.691638884089574
- type: mrr
value: 25.15674603174603
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 68.35666666666667
- type: f1
value: 68.30294399725629
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 5.759
- type: map_at_10
value: 13.425999999999998
- type: map_at_100
value: 16.988
- type: map_at_1000
value: 18.512
- type: map_at_3
value: 9.737
- type: map_at_5
value: 11.558
- type: mrr_at_1
value: 48.297000000000004
- type: mrr_at_10
value: 56.788000000000004
- type: mrr_at_100
value: 57.306000000000004
- type: mrr_at_1000
value: 57.349000000000004
- type: mrr_at_3
value: 54.386
- type: mrr_at_5
value: 56.135000000000005
- type: ndcg_at_1
value: 46.285
- type: ndcg_at_10
value: 36.016
- type: ndcg_at_100
value: 32.984
- type: ndcg_at_1000
value: 42.093
- type: ndcg_at_3
value: 41.743
- type: ndcg_at_5
value: 39.734
- type: precision_at_1
value: 48.297000000000004
- type: precision_at_10
value: 26.779999999999998
- type: precision_at_100
value: 8.505
- type: precision_at_1000
value: 2.1420000000000003
- type: precision_at_3
value: 39.422000000000004
- type: precision_at_5
value: 34.675
- type: recall_at_1
value: 5.759
- type: recall_at_10
value: 17.251
- type: recall_at_100
value: 33.323
- type: recall_at_1000
value: 66.759
- type: recall_at_3
value: 10.703
- type: recall_at_5
value: 13.808000000000002
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 31.696999999999996
- type: map_at_10
value: 46.099000000000004
- type: map_at_100
value: 47.143
- type: map_at_1000
value: 47.178
- type: map_at_3
value: 41.948
- type: map_at_5
value: 44.504
- type: mrr_at_1
value: 35.717999999999996
- type: mrr_at_10
value: 48.653
- type: mrr_at_100
value: 49.456
- type: mrr_at_1000
value: 49.479
- type: mrr_at_3
value: 45.283
- type: mrr_at_5
value: 47.422
- type: ndcg_at_1
value: 35.689
- type: ndcg_at_10
value: 53.312000000000005
- type: ndcg_at_100
value: 57.69
- type: ndcg_at_1000
value: 58.489000000000004
- type: ndcg_at_3
value: 45.678999999999995
- type: ndcg_at_5
value: 49.897000000000006
- type: precision_at_1
value: 35.689
- type: precision_at_10
value: 8.685
- type: precision_at_100
value: 1.111
- type: precision_at_1000
value: 0.11900000000000001
- type: precision_at_3
value: 20.558
- type: precision_at_5
value: 14.802999999999999
- type: recall_at_1
value: 31.696999999999996
- type: recall_at_10
value: 72.615
- type: recall_at_100
value: 91.563
- type: recall_at_1000
value: 97.52300000000001
- type: recall_at_3
value: 53.203
- type: recall_at_5
value: 62.836000000000006
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 67.94802382241473
- type: cos_sim_ap
value: 72.1545049768353
- type: cos_sim_f1
value: 71.24658780709737
- type: cos_sim_precision
value: 62.589928057553955
- type: cos_sim_recall
value: 82.68215417106653
- type: dot_accuracy
value: 63.56253383865729
- type: dot_ap
value: 66.5298825401086
- type: dot_f1
value: 69.31953840031835
- type: dot_precision
value: 55.61941251596424
- type: dot_recall
value: 91.97465681098205
- type: euclidean_accuracy
value: 69.46399566865186
- type: euclidean_ap
value: 73.63177936887436
- type: euclidean_f1
value: 72.91028446389497
- type: euclidean_precision
value: 62.25710014947683
- type: euclidean_recall
value: 87.96198521647307
- type: manhattan_accuracy
value: 69.89713048186248
- type: manhattan_ap
value: 74.11555425121965
- type: manhattan_f1
value: 72.8923476005188
- type: manhattan_precision
value: 61.71303074670571
- type: manhattan_recall
value: 89.01795142555439
- type: max_accuracy
value: 69.89713048186248
- type: max_ap
value: 74.11555425121965
- type: max_f1
value: 72.91028446389497
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 90.93
- type: ap
value: 88.66185083484555
- type: f1
value: 90.91685771516175
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 14.385178129184318
- type: cos_sim_spearman
value: 17.246549728263478
- type: euclidean_pearson
value: 18.921969136664913
- type: euclidean_spearman
value: 17.245713577354014
- type: manhattan_pearson
value: 18.98503959815216
- type: manhattan_spearman
value: 17.37740013639568
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 32.04198138050403
- type: cos_sim_spearman
value: 34.4844617563846
- type: euclidean_pearson
value: 34.2634608256121
- type: euclidean_spearman
value: 36.322207068208066
- type: manhattan_pearson
value: 34.414939622012284
- type: manhattan_spearman
value: 36.49437789416394
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 70.858
- type: map_at_10
value: 84.516
- type: map_at_100
value: 85.138
- type: map_at_1000
value: 85.153
- type: map_at_3
value: 81.487
- type: map_at_5
value: 83.41199999999999
- type: mrr_at_1
value: 81.55
- type: mrr_at_10
value: 87.51400000000001
- type: mrr_at_100
value: 87.607
- type: mrr_at_1000
value: 87.60900000000001
- type: mrr_at_3
value: 86.49
- type: mrr_at_5
value: 87.21
- type: ndcg_at_1
value: 81.57
- type: ndcg_at_10
value: 88.276
- type: ndcg_at_100
value: 89.462
- type: ndcg_at_1000
value: 89.571
- type: ndcg_at_3
value: 85.294
- type: ndcg_at_5
value: 86.979
- type: precision_at_1
value: 81.57
- type: precision_at_10
value: 13.389999999999999
- type: precision_at_100
value: 1.532
- type: precision_at_1000
value: 0.157
- type: precision_at_3
value: 37.2
- type: precision_at_5
value: 24.544
- type: recall_at_1
value: 70.858
- type: recall_at_10
value: 95.428
- type: recall_at_100
value: 99.46000000000001
- type: recall_at_1000
value: 99.98
- type: recall_at_3
value: 86.896
- type: recall_at_5
value: 91.617
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
config: default
split: test
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
value: 47.90089115942085
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
config: default
split: test
revision: 282350215ef01743dc01b456c7f5241fa8937f16
metrics:
- type: v_measure
value: 55.948584594903515
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.513
- type: map_at_10
value: 11.189
- type: map_at_100
value: 13.034
- type: map_at_1000
value: 13.312
- type: map_at_3
value: 8.124
- type: map_at_5
value: 9.719999999999999
- type: mrr_at_1
value: 22.1
- type: mrr_at_10
value: 32.879999999999995
- type: mrr_at_100
value: 33.916000000000004
- type: mrr_at_1000
value: 33.982
- type: mrr_at_3
value: 29.633
- type: mrr_at_5
value: 31.663000000000004
- type: ndcg_at_1
value: 22.1
- type: ndcg_at_10
value: 18.944
- type: ndcg_at_100
value: 26.240000000000002
- type: ndcg_at_1000
value: 31.282
- type: ndcg_at_3
value: 18.17
- type: ndcg_at_5
value: 15.976
- type: precision_at_1
value: 22.1
- type: precision_at_10
value: 9.700000000000001
- type: precision_at_100
value: 2.025
- type: precision_at_1000
value: 0.32299999999999995
- type: precision_at_3
value: 16.933
- type: precision_at_5
value: 14.02
- type: recall_at_1
value: 4.513
- type: recall_at_10
value: 19.723
- type: recall_at_100
value: 41.117
- type: recall_at_1000
value: 65.718
- type: recall_at_3
value: 10.333
- type: recall_at_5
value: 14.252
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
config: default
split: test
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
metrics:
- type: cos_sim_pearson
value: 85.93526522406187
- type: cos_sim_spearman
value: 81.4067321748142
- type: euclidean_pearson
value: 82.23783344725466
- type: euclidean_spearman
value: 80.88990344685583
- type: manhattan_pearson
value: 82.3367264631989
- type: manhattan_spearman
value: 80.9278067738814
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
config: default
split: test
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
value: 85.23458296088118
- type: cos_sim_spearman
value: 77.47310329678291
- type: euclidean_pearson
value: 83.73584591194671
- type: euclidean_spearman
value: 80.15616176452284
- type: manhattan_pearson
value: 84.03063128849925
- type: manhattan_spearman
value: 80.36472448270416
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
config: default
split: test
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
value: 86.11807249122802
- type: cos_sim_spearman
value: 86.37854318479079
- type: euclidean_pearson
value: 86.65850909046301
- type: euclidean_spearman
value: 87.85344963531178
- type: manhattan_pearson
value: 86.77920459868837
- type: manhattan_spearman
value: 87.97331161741792
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
config: default
split: test
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
value: 84.4649953305265
- type: cos_sim_spearman
value: 81.17166984686445
- type: euclidean_pearson
value: 82.36880883967271
- type: euclidean_spearman
value: 81.28206358558401
- type: manhattan_pearson
value: 82.56994704487155
- type: manhattan_spearman
value: 81.52094918949243
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
config: default
split: test
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
value: 87.5328930220188
- type: cos_sim_spearman
value: 88.23398394823562
- type: euclidean_pearson
value: 88.0817998861656
- type: euclidean_spearman
value: 88.68995789914679
- type: manhattan_pearson
value: 88.11885742601258
- type: manhattan_spearman
value: 88.7318106493293
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
config: default
split: test
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
value: 84.81883368511858
- type: cos_sim_spearman
value: 86.28679308000675
- type: euclidean_pearson
value: 84.33705182713047
- type: euclidean_spearman
value: 84.83018555455023
- type: manhattan_pearson
value: 84.3271850394614
- type: manhattan_spearman
value: 84.77974015415639
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
config: en-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 90.71845282522295
- type: cos_sim_spearman
value: 90.6215253553308
- type: euclidean_pearson
value: 89.486847313806
- type: euclidean_spearman
value: 89.11692037511729
- type: manhattan_pearson
value: 89.53911733450684
- type: manhattan_spearman
value: 89.2507288145461
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
config: en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 65.81961557635002
- type: cos_sim_spearman
value: 65.01437718770094
- type: euclidean_pearson
value: 66.53720271639384
- type: euclidean_spearman
value: 65.66538718470727
- type: manhattan_pearson
value: 66.85160833477023
- type: manhattan_spearman
value: 65.86253623736344
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 81.74904608584143
- type: cos_sim_spearman
value: 82.02672847550606
- type: euclidean_pearson
value: 81.47843718306068
- type: euclidean_spearman
value: 81.7259314292303
- type: manhattan_pearson
value: 81.70320276859634
- type: manhattan_spearman
value: 81.94903024173293
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
config: default
split: test
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
value: 87.37129233774877
- type: cos_sim_spearman
value: 88.02311088852667
- type: euclidean_pearson
value: 85.864664021262
- type: euclidean_spearman
value: 86.24775921494894
- type: manhattan_pearson
value: 85.85401868812795
- type: manhattan_spearman
value: 86.22999105137849
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
config: default
split: test
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
value: 80.2684105571225
- type: mrr
value: 94.3528194753685
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 55.161
- type: map_at_10
value: 64.794
- type: map_at_100
value: 65.66499999999999
- type: map_at_1000
value: 65.684
- type: map_at_3
value: 62.326
- type: map_at_5
value: 63.863
- type: mrr_at_1
value: 58.333
- type: mrr_at_10
value: 66.396
- type: mrr_at_100
value: 67.07300000000001
- type: mrr_at_1000
value: 67.092
- type: mrr_at_3
value: 64.61099999999999
- type: mrr_at_5
value: 65.744
- type: ndcg_at_1
value: 58.333
- type: ndcg_at_10
value: 69.294
- type: ndcg_at_100
value: 72.612
- type: ndcg_at_1000
value: 73.083
- type: ndcg_at_3
value: 65.226
- type: ndcg_at_5
value: 67.44
- type: precision_at_1
value: 58.333
- type: precision_at_10
value: 9.2
- type: precision_at_100
value: 1.083
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 25.667
- type: precision_at_5
value: 16.866999999999997
- type: recall_at_1
value: 55.161
- type: recall_at_10
value: 81.289
- type: recall_at_100
value: 95.333
- type: recall_at_1000
value: 99
- type: recall_at_3
value: 70.45
- type: recall_at_5
value: 76.128
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
config: default
split: test
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
value: 99.81980198019802
- type: cos_sim_ap
value: 95.61939598272275
- type: cos_sim_f1
value: 91.00684261974584
- type: cos_sim_precision
value: 89.0057361376673
- type: cos_sim_recall
value: 93.10000000000001
- type: dot_accuracy
value: 99.78910891089109
- type: dot_ap
value: 94.52852299178002
- type: dot_f1
value: 89.2586989409985
- type: dot_precision
value: 90.03051881993896
- type: dot_recall
value: 88.5
- type: euclidean_accuracy
value: 99.81782178217821
- type: euclidean_ap
value: 95.41313424258671
- type: euclidean_f1
value: 90.91806515301086
- type: euclidean_precision
value: 89.76608187134502
- type: euclidean_recall
value: 92.10000000000001
- type: manhattan_accuracy
value: 99.81584158415842
- type: manhattan_ap
value: 95.52722650384223
- type: manhattan_f1
value: 90.86444007858546
- type: manhattan_precision
value: 89.28571428571429
- type: manhattan_recall
value: 92.5
- type: max_accuracy
value: 99.81980198019802
- type: max_ap
value: 95.61939598272275
- type: max_f1
value: 91.00684261974584
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
config: default
split: test
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
value: 60.2736951820551
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
config: default
split: test
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
value: 32.34316824844043
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
config: default
split: test
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
value: 50.55034024386463
- type: mrr
value: 51.468598803157626
- task:
type: Summarization
dataset:
type: mteb/summeval
name: MTEB SummEval
config: default
split: test
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
metrics:
- type: cos_sim_pearson
value: 31.20772719310616
- type: cos_sim_spearman
value: 30.966269993937523
- type: dot_pearson
value: 30.866563682880965
- type: dot_spearman
value: 29.906699130890875
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 67.87990805824984
- type: mrr
value: 78.16078682657897
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.009
- type: map_at_10
value: 71.319
- type: map_at_100
value: 74.895
- type: map_at_1000
value: 74.995
- type: map_at_3
value: 50.778
- type: map_at_5
value: 62.00599999999999
- type: mrr_at_1
value: 87.41
- type: mrr_at_10
value: 90.18599999999999
- type: mrr_at_100
value: 90.29700000000001
- type: mrr_at_1000
value: 90.302
- type: mrr_at_3
value: 89.701
- type: mrr_at_5
value: 89.992
- type: ndcg_at_1
value: 87.41
- type: ndcg_at_10
value: 79.822
- type: ndcg_at_100
value: 83.877
- type: ndcg_at_1000
value: 84.882
- type: ndcg_at_3
value: 82.391
- type: ndcg_at_5
value: 80.339
- type: precision_at_1
value: 87.41
- type: precision_at_10
value: 39.546
- type: precision_at_100
value: 4.824
- type: precision_at_1000
value: 0.507
- type: precision_at_3
value: 72.129
- type: precision_at_5
value: 59.915
- type: recall_at_1
value: 26.009
- type: recall_at_10
value: 78.144
- type: recall_at_100
value: 91.375
- type: recall_at_1000
value: 96.42399999999999
- type: recall_at_3
value: 52.529
- type: recall_at_5
value: 65.46
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 47.803000000000004
- type: f1
value: 46.298520969605775
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.252
- type: map_at_10
value: 2.181
- type: map_at_100
value: 12.82
- type: map_at_1000
value: 30.307000000000002
- type: map_at_3
value: 0.716
- type: map_at_5
value: 1.133
- type: mrr_at_1
value: 96
- type: mrr_at_10
value: 98
- type: mrr_at_100
value: 98
- type: mrr_at_1000
value: 98
- type: mrr_at_3
value: 98
- type: mrr_at_5
value: 98
- type: ndcg_at_1
value: 92
- type: ndcg_at_10
value: 83.818
- type: ndcg_at_100
value: 63.327999999999996
- type: ndcg_at_1000
value: 55.883
- type: ndcg_at_3
value: 87.16199999999999
- type: ndcg_at_5
value: 85.03
- type: precision_at_1
value: 96
- type: precision_at_10
value: 88
- type: precision_at_100
value: 64.94
- type: precision_at_1000
value: 24.688
- type: precision_at_3
value: 91.333
- type: precision_at_5
value: 88.8
- type: recall_at_1
value: 0.252
- type: recall_at_10
value: 2.326
- type: recall_at_100
value: 15.665000000000001
- type: recall_at_1000
value: 52.559999999999995
- type: recall_at_3
value: 0.735
- type: recall_at_5
value: 1.175
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (sqi-eng)
config: sqi-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19
- type: f1
value: 15.331629955575188
- type: precision
value: 14.38509724403208
- type: recall
value: 19
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fry-eng)
config: fry-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 39.884393063583815
- type: f1
value: 32.369942196531795
- type: precision
value: 30.036929993577395
- type: recall
value: 39.884393063583815
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kur-eng)
config: kur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.365853658536585
- type: f1
value: 12.49755078527547
- type: precision
value: 11.840415442997939
- type: recall
value: 15.365853658536585
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tur-eng)
config: tur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.1
- type: f1
value: 8.955359175928436
- type: precision
value: 8.324461412770235
- type: recall
value: 11.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (deu-eng)
config: deu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.7
- type: f1
value: 85.06214285714286
- type: precision
value: 83.98761904761905
- type: recall
value: 87.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nld-eng)
config: nld-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 56.00000000000001
- type: f1
value: 49.8456850459482
- type: precision
value: 47.80084415584415
- type: recall
value: 56.00000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ron-eng)
config: ron-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 38.1
- type: f1
value: 33.85465329991646
- type: precision
value: 32.37519841269841
- type: recall
value: 38.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ang-eng)
config: ang-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 42.53731343283582
- type: f1
value: 34.67903986560703
- type: precision
value: 32.17128642501776
- type: recall
value: 42.53731343283582
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ido-eng)
config: ido-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 53.900000000000006
- type: f1
value: 47.83909812409812
- type: precision
value: 45.67887667887668
- type: recall
value: 53.900000000000006
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jav-eng)
config: jav-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 26.34146341463415
- type: f1
value: 22.264125162260022
- type: precision
value: 21.384015912351636
- type: recall
value: 26.34146341463415
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (isl-eng)
config: isl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.2
- type: f1
value: 8.001233870597419
- type: precision
value: 7.383838204560821
- type: recall
value: 10.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slv-eng)
config: slv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 17.253948967193196
- type: f1
value: 13.055189087650387
- type: precision
value: 12.105642744272275
- type: recall
value: 17.253948967193196
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cym-eng)
config: cym-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.26086956521739
- type: f1
value: 8.31837824011737
- type: precision
value: 7.879315672736052
- type: recall
value: 10.26086956521739
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kaz-eng)
config: kaz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.826086956521738
- type: f1
value: 9.663030581871162
- type: precision
value: 9.152605557273077
- type: recall
value: 11.826086956521738
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (est-eng)
config: est-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.800000000000001
- type: f1
value: 5.608697757594542
- type: precision
value: 5.333727335466467
- type: recall
value: 6.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (heb-eng)
config: heb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.3
- type: f1
value: 7.4866384899217335
- type: precision
value: 6.580321536442861
- type: recall
value: 11.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gla-eng)
config: gla-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.101326899879373
- type: f1
value: 3.0988364784130122
- type: precision
value: 2.925923150618102
- type: recall
value: 4.101326899879373
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mar-eng)
config: mar-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 76.3
- type: f1
value: 71.55912698412699
- type: precision
value: 69.55511904761904
- type: recall
value: 76.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lat-eng)
config: lat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 53.6
- type: f1
value: 46.74811085685228
- type: precision
value: 44.41049616018656
- type: recall
value: 53.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bel-eng)
config: bel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.400000000000002
- type: f1
value: 18.485309948823105
- type: precision
value: 17.12104734130107
- type: recall
value: 23.400000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pms-eng)
config: pms-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 41.523809523809526
- type: f1
value: 36.577269291555005
- type: precision
value: 35.00219198790627
- type: recall
value: 41.523809523809526
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gle-eng)
config: gle-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.909842412258181
- type: precision
value: 3.7099694121032796
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pes-eng)
config: pes-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 26.900000000000002
- type: f1
value: 21.587309161426806
- type: precision
value: 19.877234126984124
- type: recall
value: 26.900000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nob-eng)
config: nob-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 37.3
- type: f1
value: 31.940531675926408
- type: precision
value: 30.414405457464277
- type: recall
value: 37.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bul-eng)
config: bul-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 34.4
- type: f1
value: 28.460500740394355
- type: precision
value: 26.630818170746558
- type: recall
value: 34.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cbk-eng)
config: cbk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 67.5
- type: f1
value: 61.492367158984806
- type: precision
value: 59.23266755904913
- type: recall
value: 67.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hun-eng)
config: hun-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.9
- type: f1
value: 6.652063929922994
- type: precision
value: 6.392931096681097
- type: recall
value: 7.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uig-eng)
config: uig-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.6
- type: f1
value: 2.0216271963330783
- type: precision
value: 1.9467343791901313
- type: recall
value: 2.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (rus-eng)
config: rus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 76.6
- type: f1
value: 71.23357142857142
- type: precision
value: 69.03261904761905
- type: recall
value: 76.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (spa-eng)
config: spa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 98.6
- type: f1
value: 98.13333333333333
- type: precision
value: 97.89999999999999
- type: recall
value: 98.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hye-eng)
config: hye-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.8867924528301887
- type: f1
value: 0.9184016421339141
- type: precision
value: 0.8343646123610833
- type: recall
value: 1.8867924528301887
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tel-eng)
config: tel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 84.1880341880342
- type: f1
value: 80.56369556369557
- type: precision
value: 79.02421652421653
- type: recall
value: 84.1880341880342
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (afr-eng)
config: afr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 27.200000000000003
- type: f1
value: 22.55873107448107
- type: precision
value: 21.13610950874723
- type: recall
value: 27.200000000000003
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mon-eng)
config: mon-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.090909090909092
- type: f1
value: 7.37323521273764
- type: precision
value: 7.01229523252768
- type: recall
value: 9.090909090909092
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arz-eng)
config: arz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 79.24528301886792
- type: f1
value: 74.80483178596387
- type: precision
value: 72.8336827393431
- type: recall
value: 79.24528301886792
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hrv-eng)
config: hrv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.3
- type: f1
value: 17.754399705471684
- type: precision
value: 16.81516621898026
- type: recall
value: 21.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nov-eng)
config: nov-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 59.92217898832685
- type: f1
value: 54.92807451951421
- type: precision
value: 53.071150639244024
- type: recall
value: 59.92217898832685
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gsw-eng)
config: gsw-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 30.76923076923077
- type: f1
value: 23.70099036765703
- type: precision
value: 21.666666666666664
- type: recall
value: 30.76923076923077
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nds-eng)
config: nds-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.6
- type: f1
value: 29.87713276919159
- type: precision
value: 28.07062211509473
- type: recall
value: 35.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ukr-eng)
config: ukr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 38.1
- type: f1
value: 31.123585858585855
- type: precision
value: 28.995893769823304
- type: recall
value: 38.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uzb-eng)
config: uzb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.74766355140187
- type: f1
value: 8.280338473537247
- type: precision
value: 7.806134675293554
- type: recall
value: 10.74766355140187
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lit-eng)
config: lit-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.6
- type: f1
value: 5.872095040470223
- type: precision
value: 5.557777361527362
- type: recall
value: 7.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ina-eng)
config: ina-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 86.8
- type: f1
value: 83.72833333333332
- type: precision
value: 82.4259523809524
- type: recall
value: 86.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lfn-eng)
config: lfn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 52
- type: f1
value: 46.48058132211534
- type: precision
value: 44.52753032676945
- type: recall
value: 52
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (zsm-eng)
config: zsm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 90.4
- type: f1
value: 88.10999999999999
- type: precision
value: 87.10333333333334
- type: recall
value: 90.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ita-eng)
config: ita-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 79.5
- type: f1
value: 74.95746031746032
- type: precision
value: 73.03249999999998
- type: recall
value: 79.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cmn-eng)
config: cmn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.7
- type: f1
value: 95.7
- type: precision
value: 95.21666666666667
- type: recall
value: 96.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lvs-eng)
config: lvs-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.7
- type: f1
value: 8.576412755390276
- type: precision
value: 8.046714349557488
- type: recall
value: 10.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (glg-eng)
config: glg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 86
- type: f1
value: 82.54523809523809
- type: precision
value: 81.06166666666665
- type: recall
value: 86
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ceb-eng)
config: ceb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11
- type: f1
value: 9.080509354193564
- type: precision
value: 8.57587968815845
- type: recall
value: 11
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bre-eng)
config: bre-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.6
- type: f1
value: 7.409451659451658
- type: precision
value: 6.8121069441897415
- type: recall
value: 9.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ben-eng)
config: ben-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.1
- type: f1
value: 83.88999999999999
- type: precision
value: 82.395
- type: recall
value: 87.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swg-eng)
config: swg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 34.82142857142857
- type: f1
value: 29.175170068027214
- type: precision
value: 27.40499084249084
- type: recall
value: 34.82142857142857
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arq-eng)
config: arq-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 42.9198682766191
- type: f1
value: 37.21120707205811
- type: precision
value: 35.23526784229309
- type: recall
value: 42.9198682766191
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kab-eng)
config: kab-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.3
- type: f1
value: 1.5401826425879608
- type: precision
value: 1.424235527544351
- type: recall
value: 2.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fra-eng)
config: fra-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 95.6
- type: f1
value: 94.32333333333334
- type: precision
value: 93.72500000000001
- type: recall
value: 95.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (por-eng)
config: por-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 95.5
- type: f1
value: 94.43333333333334
- type: precision
value: 93.89999999999999
- type: recall
value: 95.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tat-eng)
config: tat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.7
- type: f1
value: 4.9622522983552395
- type: precision
value: 4.528962761017515
- type: recall
value: 6.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (oci-eng)
config: oci-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 50.8
- type: f1
value: 45.736438587556236
- type: precision
value: 44.010822829131655
- type: recall
value: 50.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pol-eng)
config: pol-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.9
- type: f1
value: 20.267261904761906
- type: precision
value: 19.16142408316321
- type: recall
value: 23.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (war-eng)
config: war-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.4
- type: f1
value: 11.232209832252995
- type: precision
value: 10.714445160103056
- type: recall
value: 13.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (aze-eng)
config: aze-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.299999999999999
- type: f1
value: 8.161916387744503
- type: precision
value: 7.678631905405786
- type: recall
value: 10.299999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (vie-eng)
config: vie-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.7
- type: f1
value: 95.83333333333334
- type: precision
value: 95.41666666666667
- type: recall
value: 96.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nno-eng)
config: nno-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 24.9
- type: f1
value: 20.794749162495066
- type: precision
value: 19.575997295469914
- type: recall
value: 24.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cha-eng)
config: cha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.11678832116788
- type: f1
value: 26.960375391032326
- type: precision
value: 25.498078211502524
- type: recall
value: 32.11678832116788
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mhr-eng)
config: mhr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.251889552842259
- type: precision
value: 2.9281137342615295
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dan-eng)
config: dan-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 38.9
- type: f1
value: 33.59595154442981
- type: precision
value: 31.906759791342587
- type: recall
value: 38.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ell-eng)
config: ell-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.900000000000002
- type: f1
value: 13.082818919542666
- type: precision
value: 12.125554724968518
- type: recall
value: 16.900000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (amh-eng)
config: amh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.5952380952380952
- type: f1
value: 0.09920634920634923
- type: precision
value: 0.05411255411255411
- type: recall
value: 0.5952380952380952
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pam-eng)
config: pam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.1
- type: f1
value: 7.033911671727207
- type: precision
value: 6.759952905986985
- type: recall
value: 8.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hsb-eng)
config: hsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.149068322981368
- type: f1
value: 13.314287609382625
- type: precision
value: 12.588291889534126
- type: recall
value: 16.149068322981368
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (srp-eng)
config: srp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 22.3
- type: f1
value: 18.754672526177103
- type: precision
value: 17.77463320976479
- type: recall
value: 22.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (epo-eng)
config: epo-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 39.5
- type: f1
value: 33.91659439373835
- type: precision
value: 32.244738455988454
- type: recall
value: 39.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kzj-eng)
config: kzj-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.5
- type: f1
value: 6.300929449087343
- type: precision
value: 6.05555758176835
- type: recall
value: 7.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (awa-eng)
config: awa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 57.14285714285714
- type: f1
value: 53.011353725639445
- type: precision
value: 51.78829107400536
- type: recall
value: 57.14285714285714
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fao-eng)
config: fao-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.030534351145036
- type: f1
value: 14.424487352192786
- type: precision
value: 13.98739301411057
- type: recall
value: 16.030534351145036
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mal-eng)
config: mal-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.21542940320232
- type: f1
value: 95.0509461426492
- type: precision
value: 94.46870451237264
- type: recall
value: 96.21542940320232
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ile-eng)
config: ile-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 69.1
- type: f1
value: 63.649573934837086
- type: precision
value: 61.44357142857143
- type: recall
value: 69.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bos-eng)
config: bos-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.728813559322035
- type: f1
value: 19.281200536513545
- type: precision
value: 18.11042731593579
- type: recall
value: 23.728813559322035
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cor-eng)
config: cor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.8602777777777777
- type: precision
value: 3.553962393468025
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cat-eng)
config: cat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 92.10000000000001
- type: f1
value: 90.24190476190476
- type: precision
value: 89.41666666666667
- type: recall
value: 92.10000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (eus-eng)
config: eus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 78.8
- type: f1
value: 74.53390756302521
- type: precision
value: 72.79386904761904
- type: recall
value: 78.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yue-eng)
config: yue-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 91.60000000000001
- type: f1
value: 89.39
- type: precision
value: 88.375
- type: recall
value: 91.60000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swe-eng)
config: swe-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.4
- type: f1
value: 27.824399979105863
- type: precision
value: 26.434715247715246
- type: recall
value: 32.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dtp-eng)
config: dtp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.800000000000001
- type: f1
value: 5.258204523374802
- type: precision
value: 4.940595825661615
- type: recall
value: 6.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kat-eng)
config: kat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.0107238605898123
- type: f1
value: 1.4770600435024532
- type: precision
value: 1.4215975441361408
- type: recall
value: 2.0107238605898123
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jpn-eng)
config: jpn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.2
- type: f1
value: 83.88333333333333
- type: precision
value: 82.44166666666668
- type: recall
value: 87.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (csb-eng)
config: csb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.229249011857709
- type: f1
value: 11.043453048700425
- type: precision
value: 10.285902503293807
- type: recall
value: 14.229249011857709
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (xho-eng)
config: xho-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.859154929577464
- type: f1
value: 7.960154086914651
- type: precision
value: 7.679678785726838
- type: recall
value: 9.859154929577464
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (orv-eng)
config: orv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.574850299401197
- type: f1
value: 8.435162337247867
- type: precision
value: 7.5408084342568324
- type: recall
value: 12.574850299401197
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ind-eng)
config: ind-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 93.5
- type: f1
value: 91.90666666666665
- type: precision
value: 91.14166666666668
- type: recall
value: 93.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tuk-eng)
config: tuk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.866995073891626
- type: f1
value: 6.8479221927497775
- type: precision
value: 6.431102386508143
- type: recall
value: 8.866995073891626
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (max-eng)
config: max-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 46.12676056338028
- type: f1
value: 41.447273383893105
- type: precision
value: 39.80374351371386
- type: recall
value: 46.12676056338028
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swh-eng)
config: swh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.38461538461539
- type: f1
value: 27.80912253371418
- type: precision
value: 25.588007434161277
- type: recall
value: 35.38461538461539
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hin-eng)
config: hin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.1
- type: f1
value: 94.88333333333333
- type: precision
value: 94.3
- type: recall
value: 96.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dsb-eng)
config: dsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.16283924843424
- type: f1
value: 15.00273275898725
- type: precision
value: 14.135773519036146
- type: recall
value: 18.16283924843424
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ber-eng)
config: ber-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.4
- type: f1
value: 5.169780886652615
- type: precision
value: 4.901094815916798
- type: recall
value: 6.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tam-eng)
config: tam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 85.66775244299674
- type: f1
value: 81.86753528773072
- type: precision
value: 80.13029315960912
- type: recall
value: 85.66775244299674
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slk-eng)
config: slk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.7
- type: f1
value: 12.296409553542203
- type: precision
value: 11.643939628482972
- type: recall
value: 14.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tgl-eng)
config: tgl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.000000000000002
- type: f1
value: 11.188658083109301
- type: precision
value: 10.439068547503426
- type: recall
value: 14.000000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ast-eng)
config: ast-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 74.01574803149606
- type: f1
value: 68.3727034120735
- type: precision
value: 66.06299212598424
- type: recall
value: 74.01574803149606
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mkd-eng)
config: mkd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 20.200000000000003
- type: f1
value: 15.584321026350167
- type: precision
value: 14.220359087863855
- type: recall
value: 20.200000000000003
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (khm-eng)
config: khm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.2465373961218837
- type: f1
value: 0.7009849184364421
- type: precision
value: 0.6369121979354991
- type: recall
value: 1.2465373961218837
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ces-eng)
config: ces-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.5
- type: f1
value: 12.992671904350203
- type: precision
value: 12.323623108157992
- type: recall
value: 15.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tzl-eng)
config: tzl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 37.5
- type: f1
value: 32.70299145299145
- type: precision
value: 31.066176470588236
- type: recall
value: 37.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (urd-eng)
config: urd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 86.2
- type: f1
value: 82.87166666666667
- type: precision
value: 81.44261904761906
- type: recall
value: 86.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ara-eng)
config: ara-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 92.5
- type: f1
value: 90.61666666666667
- type: precision
value: 89.71666666666668
- type: recall
value: 92.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kor-eng)
config: kor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 51.7
- type: f1
value: 44.78806599832916
- type: precision
value: 42.26749389499389
- type: recall
value: 51.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yid-eng)
config: yid-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.9433962264150944
- type: f1
value: 0.48704516529471925
- type: precision
value: 0.41179094097726165
- type: recall
value: 0.9433962264150944
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fin-eng)
config: fin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.800000000000001
- type: f1
value: 5.480668860234897
- type: precision
value: 5.195067371791852
- type: recall
value: 6.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tha-eng)
config: tha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.036496350364963
- type: f1
value: 6.784271238735886
- type: precision
value: 6.159462364744479
- type: recall
value: 10.036496350364963
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (wuu-eng)
config: wuu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 90.3
- type: f1
value: 87.91499999999999
- type: precision
value: 86.82595238095237
- type: recall
value: 90.3
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 49.19154423543331
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 47.76345036893387
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 2.396
- type: map_at_10
value: 9.994
- type: map_at_100
value: 16.067999999999998
- type: map_at_1000
value: 17.59
- type: map_at_3
value: 4.733
- type: map_at_5
value: 6.7589999999999995
- type: mrr_at_1
value: 28.571
- type: mrr_at_10
value: 47.678
- type: mrr_at_100
value: 48.311
- type: mrr_at_1000
value: 48.317
- type: mrr_at_3
value: 43.878
- type: mrr_at_5
value: 46.224
- type: ndcg_at_1
value: 25.509999999999998
- type: ndcg_at_10
value: 25.189
- type: ndcg_at_100
value: 36.179
- type: ndcg_at_1000
value: 47.562
- type: ndcg_at_3
value: 26.858999999999998
- type: ndcg_at_5
value: 26.825
- type: precision_at_1
value: 28.571
- type: precision_at_10
value: 23.469
- type: precision_at_100
value: 7.550999999999999
- type: precision_at_1000
value: 1.51
- type: precision_at_3
value: 29.252
- type: precision_at_5
value: 28.571
- type: recall_at_1
value: 2.396
- type: recall_at_10
value: 16.551
- type: recall_at_100
value: 46.438
- type: recall_at_1000
value: 81.04
- type: recall_at_3
value: 6.145
- type: recall_at_5
value: 9.728
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
config: default
split: test
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
metrics:
- type: accuracy
value: 71.5842
- type: ap
value: 14.770823761227014
- type: f1
value: 55.22772349179383
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
config: default
split: test
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
value: 62.13921901528015
- type: f1
value: 62.450042974251694
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
config: default
split: test
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
value: 40.81463922932671
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
config: default
split: test
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
value: 85.86755677415509
- type: cos_sim_ap
value: 73.8131664470889
- type: cos_sim_f1
value: 68.03196803196803
- type: cos_sim_precision
value: 64.58036984352773
- type: cos_sim_recall
value: 71.87335092348285
- type: dot_accuracy
value: 84.58604041246946
- type: dot_ap
value: 69.43165607336826
- type: dot_f1
value: 65.84285381207741
- type: dot_precision
value: 58.980785296574766
- type: dot_recall
value: 74.51187335092348
- type: euclidean_accuracy
value: 85.60529296060082
- type: euclidean_ap
value: 72.48939155702391
- type: euclidean_f1
value: 66.84775898259045
- type: euclidean_precision
value: 62.822000464144814
- type: euclidean_recall
value: 71.42480211081794
- type: manhattan_accuracy
value: 85.5456875484294
- type: manhattan_ap
value: 72.37178636434892
- type: manhattan_f1
value: 66.6751398068124
- type: manhattan_precision
value: 64.32074546346249
- type: manhattan_recall
value: 69.2084432717678
- type: max_accuracy
value: 85.86755677415509
- type: max_ap
value: 73.8131664470889
- type: max_f1
value: 68.03196803196803
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
config: default
split: test
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
value: 89.39341017580627
- type: cos_sim_ap
value: 86.7769866448429
- type: cos_sim_f1
value: 79.26586570354536
- type: cos_sim_precision
value: 76.02149017390076
- type: cos_sim_recall
value: 82.79950723744996
- type: dot_accuracy
value: 89.15861373074087
- type: dot_ap
value: 85.15235322715995
- type: dot_f1
value: 78.97118887294403
- type: dot_precision
value: 75.6290083867785
- type: dot_recall
value: 82.62242069602709
- type: euclidean_accuracy
value: 89.0266620095471
- type: euclidean_ap
value: 86.18904940615533
- type: euclidean_f1
value: 78.37750135208222
- type: euclidean_precision
value: 73.70312605953754
- type: euclidean_recall
value: 83.68493994456422
- type: manhattan_accuracy
value: 88.98397174680794
- type: manhattan_ap
value: 86.18302538523727
- type: manhattan_f1
value: 78.42197035745423
- type: manhattan_precision
value: 74.23658872077029
- type: manhattan_recall
value: 83.10748383122882
- type: max_accuracy
value: 89.39341017580627
- type: max_ap
value: 86.7769866448429
- type: max_f1
value: 79.26586570354536
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 46.9
- type: map_at_10
value: 57.399
- type: map_at_100
value: 57.976000000000006
- type: map_at_1000
value: 58.00300000000001
- type: map_at_3
value: 54.967
- type: map_at_5
value: 56.562
- type: mrr_at_1
value: 46.800000000000004
- type: mrr_at_10
value: 57.349000000000004
- type: mrr_at_100
value: 57.926
- type: mrr_at_1000
value: 57.952999999999996
- type: mrr_at_3
value: 54.917
- type: mrr_at_5
value: 56.51199999999999
- type: ndcg_at_1
value: 46.9
- type: ndcg_at_10
value: 62.437
- type: ndcg_at_100
value: 65.273
- type: ndcg_at_1000
value: 65.999
- type: ndcg_at_3
value: 57.524
- type: ndcg_at_5
value: 60.402
- type: precision_at_1
value: 46.9
- type: precision_at_10
value: 7.82
- type: precision_at_100
value: 0.915
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 21.633
- type: precision_at_5
value: 14.38
- type: recall_at_1
value: 46.9
- type: recall_at_10
value: 78.2
- type: recall_at_100
value: 91.5
- type: recall_at_1000
value: 97.2
- type: recall_at_3
value: 64.9
- type: recall_at_5
value: 71.89999999999999
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.68
- type: ap
value: 66.4749730574293
- type: f1
value: 82.93606561551698
Model Card for udever-bloom
udever-bloom-7b1 is finetuned from bigscience/bloom-7b1 via BitFit on MS MARCO Passage Ranking, SNLI and MultiNLI data.
It is a universal embedding model across tasks, natural and programming languages.
(From the technical view, udever is merely with some minor improvements to sgpt-bloom)
Model Details
Model Description
- Developed by: Alibaba Group
- Model type: Transformer-based Language Model (decoder-only)
- Language(s) (NLP): Multiple; see bloom training data
- Finetuned from model : bigscience/bloom-7b1
Model Sources
- Repository: github.com/izhx/uni-rep
- Paper : Language Models are Universal Embedders
- Training Date : 2023-06
Checkpoints
On ModelScope / 魔搭社区: udever-bloom-560m, udever-bloom-1b1, udever-bloom-3b, udever-bloom-7b1
How to Get Started with the Model
Use the code below to get started with the model.
import torch
from transformers import AutoTokenizer, BloomModel
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-7b1')
model = BloomModel.from_pretrained('izhx/udever-bloom-7b1')
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]'
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod])
if tokenizer.padding_side != 'left':
print('!!!', tokenizer.padding_side)
tokenizer.padding_side = 'left'
def encode(texts: list, is_query: bool = True, max_length=300):
bos = boq if is_query else bod
eos_id = eoq_id if is_query else eod_id
texts = [bos + t for t in texts]
encoding = tokenizer(
texts, truncation=True, max_length=max_length - 1, padding=True
)
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']):
ids.append(eos_id)
mask.append(1)
inputs = tokenizer.pad(encoding, return_tensors='pt')
with torch.inference_mode():
outputs = model(**inputs)
embeds = outputs.last_hidden_state[:, -1]
return embeds
encode(['I am Bert', 'You are Elmo'])
Training Details
Training Data
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86)
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz)
Training Procedure
Preprocessing
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). Negatives for SNLI and MultiNLI are randomly sampled.
Training Hyperparameters
- Training regime: tf32, BitFit
- Batch size: 1024
- Epochs: 3
- Optimizer: AdamW
- Learning rate: 1e-4
- Scheduler: constant with warmup.
- Warmup: 0.25 epoch
Evaluation
Table 1: Massive Text Embedding Benchmark MTEB
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. |
|---|---|---|---|---|---|---|---|---|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 |
| bge-large-en-v1.5 | 64.23 | 75.97 | 46.08 | 87.12 | 60.03 | 54.29 | 83.11 | 31.61 |
| bge-base-en-v1.5 | 63.55 | 75.53 | 45.77 | 86.55 | 58.86 | 53.25 | 82.4 | 31.07 |
| gte-large | 63.13 | 73.33 | 46.84 | 85 | 59.13 | 52.22 | 83.35 | 31.66 |
| gte-base | 62.39 | 73.01 | 46.2 | 84.57 | 58.61 | 51.14 | 82.3 | 31.17 |
| e5-large-v2 | 62.25 | 75.24 | 44.49 | 86.03 | 56.61 | 50.56 | 82.05 | 30.19 |
| instructor-xl | 61.79 | 73.12 | 44.74 | 86.62 | 57.29 | 49.26 | 83.06 | 32.32 |
| instructor-large | 61.59 | 73.86 | 45.29 | 85.89 | 57.54 | 47.57 | 83.15 | 31.84 |
| e5-base-v2 | 61.5 | 73.84 | 43.8 | 85.73 | 55.91 | 50.29 | 81.05 | 30.28 |
| e5-large | 61.42 | 73.14 | 43.33 | 85.94 | 56.53 | 49.99 | 82.06 | 30.97 |
| text-embedding-ada-002 (OpenAI API) | 60.99 | 70.93 | 45.9 | 84.89 | 56.32 | 49.25 | 80.97 | 30.8 |
| e5-base | 60.44 | 72.63 | 42.11 | 85.09 | 55.7 | 48.75 | 80.96 | 31.01 |
| SGPT-5.8B-msmarco | 58.93 | 68.13 | 40.34 | 82 | 56.56 | 50.25 | 78.1 | 31.46 |
| sgpt-bloom-7b1-msmarco | 57.59 | 66.19 | 38.93 | 81.9 | 55.65 | 48.22 | 77.74 | 33.6 |
| Udever-bloom-560m | 55.80 | 68.04 | 36.89 | 81.05 | 52.60 | 41.19 | 79.93 | 32.06 |
| Udever-bloom-1b1 | 58.28 | 70.18 | 39.11 | 83.11 | 54.28 | 45.27 | 81.52 | 31.10 |
| Udever-bloom-3b | 59.86 | 71.91 | 40.74 | 84.06 | 54.90 | 47.67 | 82.37 | 30.62 |
| Udever-bloom-7b1 | 60.63 | 72.13 | 40.81 | 85.40 | 55.91 | 49.34 | 83.01 | 30.97 |
Table 2: CodeSearchNet
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. |
|---|---|---|---|---|---|---|---|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 |
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 |
| cpt-code S | 97.7 | 86.3 | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 |
| cpt-code M | 97.5 | 85.5 | 99.9 | 94.4 | 86.5 | 97.2 | 93.5 |
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 |
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 |
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 |
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 |
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 |
Table 3: Chinese multi-domain retrieval Multi-cpr
| E-commerce | Entertainment video | Medical | ||||||
|---|---|---|---|---|---|---|---|---|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k |
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 |
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 |
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 |
| DPR-2 | In-Domain | BERT-CT | 0.289 | 0.926 | 0.263 | 0.935 | 0.339 | 0.769 |
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 |
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 |
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 |
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 |
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 |
| Udever-bloom-7b1 | General | BLOOM | 0.296 | 0.889 | 0.267 | 0.907 | 0.343 | 0.705 |
More results refer to paper section 3.
Technical Specifications
Model Architecture and Objective
- Model: bigscience/bloom-7b1.
- Objective: Constrastive loss with hard negatives (refer to paper section 2.2).
Compute Infrastructure
- Nvidia A100 SXM4 80GB.
- torch 2.0.0, transformers 4.29.2.
Citation
BibTeX:
@article{zhang2023language,
title={Language Models are Universal Embedders},
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min},
journal={arXiv preprint arXiv:2310.08232},
year={2023}
}