metadata
tags:
- mteb
model-index:
- name: tao-8k
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 46.6327281304144
- type: cos_sim_spearman
value: 48.842454434123376
- type: euclidean_pearson
value: 46.94481399008005
- type: euclidean_spearman
value: 48.842454434123376
- type: manhattan_pearson
value: 46.89375935801324
- type: manhattan_spearman
value: 48.78990181105918
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 51.29442837260785
- type: cos_sim_spearman
value: 52.652094634834
- type: euclidean_pearson
value: 54.86278112546793
- type: euclidean_spearman
value: 52.65209238258423
- type: manhattan_pearson
value: 54.8164800665497
- type: manhattan_spearman
value: 52.626711935726014
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 41.51200000000001
- type: f1
value: 39.47955832883091
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 63.27653562193512
- type: cos_sim_spearman
value: 65.37293598647585
- type: euclidean_pearson
value: 63.91367659963474
- type: euclidean_spearman
value: 65.37294637878077
- type: manhattan_pearson
value: 63.89671277983551
- type: manhattan_spearman
value: 65.35510625635355
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 39.92148459596857
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.7800929733979
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 84.56370955233704
- type: mrr
value: 87.14396825396825
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 85.4719112626303
- type: mrr
value: 88.25107142857142
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 24.314
- type: map_at_10
value: 36.157000000000004
- type: map_at_100
value: 38.004
- type: map_at_1000
value: 38.129999999999995
- type: map_at_3
value: 32.141999999999996
- type: map_at_5
value: 34.414
- type: mrr_at_1
value: 37.384
- type: mrr_at_10
value: 45.261
- type: mrr_at_100
value: 46.271
- type: mrr_at_1000
value: 46.32
- type: mrr_at_3
value: 42.760999999999996
- type: mrr_at_5
value: 44.219
- type: ndcg_at_1
value: 37.384
- type: ndcg_at_10
value: 42.599
- type: ndcg_at_100
value: 50.068999999999996
- type: ndcg_at_1000
value: 52.221
- type: ndcg_at_3
value: 37.551
- type: ndcg_at_5
value: 39.711
- type: precision_at_1
value: 37.384
- type: precision_at_10
value: 9.532
- type: precision_at_100
value: 1.554
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 21.205
- type: precision_at_5
value: 15.539
- type: recall_at_1
value: 24.314
- type: recall_at_10
value: 52.463
- type: recall_at_100
value: 83.86099999999999
- type: recall_at_1000
value: 98.17399999999999
- type: recall_at_3
value: 37.341
- type: recall_at_5
value: 43.952999999999996
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 78.80938063740228
- type: cos_sim_ap
value: 87.42519095434638
- type: cos_sim_f1
value: 80.08597528210638
- type: cos_sim_precision
value: 74.10501193317423
- type: cos_sim_recall
value: 87.11713818096797
- type: dot_accuracy
value: 78.80938063740228
- type: dot_ap
value: 87.44023261310717
- type: dot_f1
value: 80.08597528210638
- type: dot_precision
value: 74.10501193317423
- type: dot_recall
value: 87.11713818096797
- type: euclidean_accuracy
value: 78.80938063740228
- type: euclidean_ap
value: 87.42517285949802
- type: euclidean_f1
value: 80.08597528210638
- type: euclidean_precision
value: 74.10501193317423
- type: euclidean_recall
value: 87.11713818096797
- type: manhattan_accuracy
value: 78.90559230306675
- type: manhattan_ap
value: 87.38730802838026
- type: manhattan_f1
value: 80.1043138107139
- type: manhattan_precision
value: 74.82744620381648
- type: manhattan_recall
value: 86.1819032031798
- type: max_accuracy
value: 78.90559230306675
- type: max_ap
value: 87.44023261310717
- type: max_f1
value: 80.1043138107139
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 69.863
- type: map_at_10
value: 77.865
- type: map_at_100
value: 78.21900000000001
- type: map_at_1000
value: 78.22200000000001
- type: map_at_3
value: 76.335
- type: map_at_5
value: 77.179
- type: mrr_at_1
value: 70.074
- type: mrr_at_10
value: 77.89
- type: mrr_at_100
value: 78.235
- type: mrr_at_1000
value: 78.238
- type: mrr_at_3
value: 76.466
- type: mrr_at_5
value: 77.241
- type: ndcg_at_1
value: 70.074
- type: ndcg_at_10
value: 81.375
- type: ndcg_at_100
value: 82.918
- type: ndcg_at_1000
value: 83.019
- type: ndcg_at_3
value: 78.32000000000001
- type: ndcg_at_5
value: 79.824
- type: precision_at_1
value: 70.074
- type: precision_at_10
value: 9.325999999999999
- type: precision_at_100
value: 1.001
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 28.17
- type: precision_at_5
value: 17.682000000000002
- type: recall_at_1
value: 69.863
- type: recall_at_10
value: 92.202
- type: recall_at_100
value: 99.05199999999999
- type: recall_at_1000
value: 99.895
- type: recall_at_3
value: 83.93
- type: recall_at_5
value: 87.566
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 25.730999999999998
- type: map_at_10
value: 80.765
- type: map_at_100
value: 83.486
- type: map_at_1000
value: 83.521
- type: map_at_3
value: 55.745999999999995
- type: map_at_5
value: 70.473
- type: mrr_at_1
value: 89.55
- type: mrr_at_10
value: 93.028
- type: mrr_at_100
value: 93.093
- type: mrr_at_1000
value: 93.096
- type: mrr_at_3
value: 92.80000000000001
- type: mrr_at_5
value: 92.92200000000001
- type: ndcg_at_1
value: 89.55
- type: ndcg_at_10
value: 87.898
- type: ndcg_at_100
value: 90.366
- type: ndcg_at_1000
value: 90.715
- type: ndcg_at_3
value: 86.497
- type: ndcg_at_5
value: 85.533
- type: precision_at_1
value: 89.55
- type: precision_at_10
value: 42.305
- type: precision_at_100
value: 4.82
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_3
value: 77.833
- type: precision_at_5
value: 65.81
- type: recall_at_1
value: 25.730999999999998
- type: recall_at_10
value: 89.409
- type: recall_at_100
value: 97.62100000000001
- type: recall_at_1000
value: 99.565
- type: recall_at_3
value: 58.298
- type: recall_at_5
value: 75.315
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 49.6
- type: map_at_10
value: 59.34
- type: map_at_100
value: 59.894999999999996
- type: map_at_1000
value: 59.913000000000004
- type: map_at_3
value: 56.667
- type: map_at_5
value: 58.196999999999996
- type: mrr_at_1
value: 49.6
- type: mrr_at_10
value: 59.34
- type: mrr_at_100
value: 59.894999999999996
- type: mrr_at_1000
value: 59.913000000000004
- type: mrr_at_3
value: 56.667
- type: mrr_at_5
value: 58.196999999999996
- type: ndcg_at_1
value: 49.6
- type: ndcg_at_10
value: 64.461
- type: ndcg_at_100
value: 67.08800000000001
- type: ndcg_at_1000
value: 67.578
- type: ndcg_at_3
value: 58.962
- type: ndcg_at_5
value: 61.741
- type: precision_at_1
value: 49.6
- type: precision_at_10
value: 8.07
- type: precision_at_100
value: 0.928
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 21.867
- type: precision_at_5
value: 14.48
- type: recall_at_1
value: 49.6
- type: recall_at_10
value: 80.7
- type: recall_at_100
value: 92.80000000000001
- type: recall_at_1000
value: 96.7
- type: recall_at_3
value: 65.60000000000001
- type: recall_at_5
value: 72.39999999999999
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 47.44132358599462
- type: f1
value: 34.814352930577854
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.43527204502813
- type: ap
value: 55.197728692877554
- type: f1
value: 81.22331922899193
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 72.21054197899034
- type: cos_sim_spearman
value: 77.10172371889475
- type: euclidean_pearson
value: 76.15914782847307
- type: euclidean_spearman
value: 77.10173036795658
- type: manhattan_pearson
value: 76.16257390318928
- type: manhattan_spearman
value: 77.10538180843567
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 26.968179320629726
- type: mrr
value: 25.664285714285718
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 66.674
- type: map_at_10
value: 75.624
- type: map_at_100
value: 75.96199999999999
- type: map_at_1000
value: 75.973
- type: map_at_3
value: 73.9
- type: map_at_5
value: 75.007
- type: mrr_at_1
value: 68.89699999999999
- type: mrr_at_10
value: 76.212
- type: mrr_at_100
value: 76.506
- type: mrr_at_1000
value: 76.517
- type: mrr_at_3
value: 74.72999999999999
- type: mrr_at_5
value: 75.65899999999999
- type: ndcg_at_1
value: 68.89699999999999
- type: ndcg_at_10
value: 79.19
- type: ndcg_at_100
value: 80.681
- type: ndcg_at_1000
value: 80.97999999999999
- type: ndcg_at_3
value: 75.954
- type: ndcg_at_5
value: 77.792
- type: precision_at_1
value: 68.89699999999999
- type: precision_at_10
value: 9.519
- type: precision_at_100
value: 1.026
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 28.548000000000002
- type: precision_at_5
value: 18.117
- type: recall_at_1
value: 66.674
- type: recall_at_10
value: 89.55499999999999
- type: recall_at_100
value: 96.26
- type: recall_at_1000
value: 98.598
- type: recall_at_3
value: 81.029
- type: recall_at_5
value: 85.37700000000001
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 68.13718897108271
- type: f1
value: 66.00508413016382
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 72.542030934768
- type: f1
value: 71.87970959109703
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 51.2
- type: map_at_10
value: 57.211999999999996
- type: map_at_100
value: 57.74
- type: map_at_1000
value: 57.791000000000004
- type: map_at_3
value: 55.900000000000006
- type: map_at_5
value: 56.665
- type: mrr_at_1
value: 51.300000000000004
- type: mrr_at_10
value: 57.252
- type: mrr_at_100
value: 57.789
- type: mrr_at_1000
value: 57.84
- type: mrr_at_3
value: 55.95
- type: mrr_at_5
value: 56.715
- type: ndcg_at_1
value: 51.2
- type: ndcg_at_10
value: 59.998
- type: ndcg_at_100
value: 62.971999999999994
- type: ndcg_at_1000
value: 64.453
- type: ndcg_at_3
value: 57.321
- type: ndcg_at_5
value: 58.711
- type: precision_at_1
value: 51.2
- type: precision_at_10
value: 6.87
- type: precision_at_100
value: 0.835
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 20.467
- type: precision_at_5
value: 12.959999999999999
- type: recall_at_1
value: 51.2
- type: recall_at_10
value: 68.7
- type: recall_at_100
value: 83.5
- type: recall_at_1000
value: 95.39999999999999
- type: recall_at_3
value: 61.4
- type: recall_at_5
value: 64.8
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 73.33000000000001
- type: f1
value: 72.76740880461465
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 75.09474824038982
- type: cos_sim_ap
value: 79.49093167837522
- type: cos_sim_f1
value: 77.762619372442
- type: cos_sim_precision
value: 68.29073482428115
- type: cos_sim_recall
value: 90.28511087645195
- type: dot_accuracy
value: 75.09474824038982
- type: dot_ap
value: 79.49093167837522
- type: dot_f1
value: 77.762619372442
- type: dot_precision
value: 68.29073482428115
- type: dot_recall
value: 90.28511087645195
- type: euclidean_accuracy
value: 75.09474824038982
- type: euclidean_ap
value: 79.49093167837522
- type: euclidean_f1
value: 77.762619372442
- type: euclidean_precision
value: 68.29073482428115
- type: euclidean_recall
value: 90.28511087645195
- type: manhattan_accuracy
value: 74.93232268543584
- type: manhattan_ap
value: 79.50256779527038
- type: manhattan_f1
value: 77.3749426342359
- type: manhattan_precision
value: 68.42532467532467
- type: manhattan_recall
value: 89.01795142555439
- type: max_accuracy
value: 75.09474824038982
- type: max_ap
value: 79.50256779527038
- type: max_f1
value: 77.762619372442
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 91.71
- type: ap
value: 89.30664330630684
- type: f1
value: 91.69380669543091
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 27.87844586552044
- type: cos_sim_spearman
value: 33.55828345961726
- type: euclidean_pearson
value: 34.008422591348754
- type: euclidean_spearman
value: 33.55828173553759
- type: manhattan_pearson
value: 33.97354762221951
- type: manhattan_spearman
value: 33.55061748217219
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 37.16475906990342
- type: cos_sim_spearman
value: 39.02023124990304
- type: euclidean_pearson
value: 37.12905621621282
- type: euclidean_spearman
value: 39.02017798495793
- type: manhattan_pearson
value: 37.16400100601629
- type: manhattan_spearman
value: 39.027383935772335
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.7431509369159
- type: cos_sim_spearman
value: 69.10355047922879
- type: euclidean_pearson
value: 67.48723360063258
- type: euclidean_spearman
value: 69.10355047922879
- type: manhattan_pearson
value: 67.55981324291854
- type: manhattan_spearman
value: 69.1816947077302
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 78.27412453529412
- type: cos_sim_spearman
value: 78.74292565872022
- type: euclidean_pearson
value: 77.95359390335884
- type: euclidean_spearman
value: 78.7428438579602
- type: manhattan_pearson
value: 77.99252788851469
- type: manhattan_spearman
value: 78.80401873296358
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.42334440897298
- type: mrr
value: 76.24570128209263
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 27.323999999999998
- type: map_at_10
value: 76.752
- type: map_at_100
value: 80.39
- type: map_at_1000
value: 80.457
- type: map_at_3
value: 53.93
- type: map_at_5
value: 66.263
- type: mrr_at_1
value: 89.90899999999999
- type: mrr_at_10
value: 92.35
- type: mrr_at_100
value: 92.43599999999999
- type: mrr_at_1000
value: 92.44
- type: mrr_at_3
value: 91.92
- type: mrr_at_5
value: 92.192
- type: ndcg_at_1
value: 89.90899999999999
- type: ndcg_at_10
value: 84.352
- type: ndcg_at_100
value: 87.978
- type: ndcg_at_1000
value: 88.631
- type: ndcg_at_3
value: 85.845
- type: ndcg_at_5
value: 84.35000000000001
- type: precision_at_1
value: 89.90899999999999
- type: precision_at_10
value: 41.985
- type: precision_at_100
value: 5.007000000000001
- type: precision_at_1000
value: 0.516
- type: precision_at_3
value: 75.146
- type: precision_at_5
value: 62.92100000000001
- type: recall_at_1
value: 27.323999999999998
- type: recall_at_10
value: 83.221
- type: recall_at_100
value: 95.088
- type: recall_at_1000
value: 98.436
- type: recall_at_3
value: 55.58
- type: recall_at_5
value: 69.594
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 50.453
- type: f1
value: 48.736715267813835
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 59.153574405500706
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 52.79421409479782
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 56.699999999999996
- type: map_at_10
value: 66.834
- type: map_at_100
value: 67.313
- type: map_at_1000
value: 67.325
- type: map_at_3
value: 65.017
- type: map_at_5
value: 65.927
- type: mrr_at_1
value: 56.699999999999996
- type: mrr_at_10
value: 66.834
- type: mrr_at_100
value: 67.313
- type: mrr_at_1000
value: 67.325
- type: mrr_at_3
value: 65.017
- type: mrr_at_5
value: 65.927
- type: ndcg_at_1
value: 56.699999999999996
- type: ndcg_at_10
value: 71.576
- type: ndcg_at_100
value: 73.79400000000001
- type: ndcg_at_1000
value: 74.08200000000001
- type: ndcg_at_3
value: 67.73400000000001
- type: ndcg_at_5
value: 69.378
- type: precision_at_1
value: 56.699999999999996
- type: precision_at_10
value: 8.64
- type: precision_at_100
value: 0.9650000000000001
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 25.2
- type: precision_at_5
value: 15.920000000000002
- type: recall_at_1
value: 56.699999999999996
- type: recall_at_10
value: 86.4
- type: recall_at_100
value: 96.5
- type: recall_at_1000
value: 98.7
- type: recall_at_3
value: 75.6
- type: recall_at_5
value: 79.60000000000001
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.83
- type: ap
value: 70.2908139255317
- type: f1
value: 85.19267443803346
a try for emebdding model