metadata
tags:
- mteb
model-index:
- name: mist-zh
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 44.734816122831546
- type: cos_sim_spearman
value: 46.97006123331873
- type: euclidean_pearson
value: 45.38062036005061
- type: euclidean_spearman
value: 46.97006123331873
- type: manhattan_pearson
value: 45.25100462997557
- type: manhattan_spearman
value: 46.85418008817015
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 49.23835317471939
- type: cos_sim_spearman
value: 51.29611473119322
- type: euclidean_pearson
value: 53.41533188991713
- type: euclidean_spearman
value: 51.29611360495954
- type: manhattan_pearson
value: 53.42662771302782
- type: manhattan_spearman
value: 51.29682402789285
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 38.855999999999995
- type: f1
value: 36.96137480741953
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 61.79575529204537
- type: cos_sim_spearman
value: 64.96308773217001
- type: euclidean_pearson
value: 63.38747223113914
- type: euclidean_spearman
value: 64.96309119412786
- type: manhattan_pearson
value: 63.36833986897711
- type: manhattan_spearman
value: 64.95000035386369
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 40.26570556670306
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 37.68621168788469
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 84.40938491415716
- type: mrr
value: 86.86722222222222
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 85.2507433210034
- type: mrr
value: 87.58742063492063
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 24.043999999999997
- type: map_at_10
value: 35.311
- type: map_at_100
value: 37.125
- type: map_at_1000
value: 37.26
- type: map_at_3
value: 31.342
- type: map_at_5
value: 33.613
- type: mrr_at_1
value: 36.909
- type: mrr_at_10
value: 44.373000000000005
- type: mrr_at_100
value: 45.367000000000004
- type: mrr_at_1000
value: 45.422000000000004
- type: mrr_at_3
value: 41.927
- type: mrr_at_5
value: 43.292
- type: ndcg_at_1
value: 36.909
- type: ndcg_at_10
value: 41.666
- type: ndcg_at_100
value: 48.915
- type: ndcg_at_1000
value: 51.348000000000006
- type: ndcg_at_3
value: 36.592
- type: ndcg_at_5
value: 38.787
- type: precision_at_1
value: 36.909
- type: precision_at_10
value: 9.327
- type: precision_at_100
value: 1.5230000000000001
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 20.671999999999997
- type: precision_at_5
value: 15.179
- type: recall_at_1
value: 24.043999999999997
- type: recall_at_10
value: 51.370000000000005
- type: recall_at_100
value: 81.569
- type: recall_at_1000
value: 98.053
- type: recall_at_3
value: 36.120000000000005
- type: recall_at_5
value: 42.829
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 75.92303066746842
- type: cos_sim_ap
value: 84.39741959629595
- type: cos_sim_f1
value: 77.28710064333224
- type: cos_sim_precision
value: 72.41520228851655
- type: cos_sim_recall
value: 82.8618190320318
- type: dot_accuracy
value: 75.92303066746842
- type: dot_ap
value: 84.39592659189601
- type: dot_f1
value: 77.28710064333224
- type: dot_precision
value: 72.41520228851655
- type: dot_recall
value: 82.8618190320318
- type: euclidean_accuracy
value: 75.92303066746842
- type: euclidean_ap
value: 84.39741904478117
- type: euclidean_f1
value: 77.28710064333224
- type: euclidean_precision
value: 72.41520228851655
- type: euclidean_recall
value: 82.8618190320318
- type: manhattan_accuracy
value: 75.83884546001202
- type: manhattan_ap
value: 84.39482592167423
- type: manhattan_f1
value: 77.2419718612394
- type: manhattan_precision
value: 71.43424711958681
- type: manhattan_recall
value: 84.07762450315643
- type: max_accuracy
value: 75.92303066746842
- type: max_ap
value: 84.39741959629595
- type: max_f1
value: 77.28710064333224
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 67.65
- type: map_at_10
value: 75.672
- type: map_at_100
value: 76.005
- type: map_at_1000
value: 76.007
- type: map_at_3
value: 73.867
- type: map_at_5
value: 74.949
- type: mrr_at_1
value: 67.756
- type: mrr_at_10
value: 75.64
- type: mrr_at_100
value: 75.973
- type: mrr_at_1000
value: 75.97500000000001
- type: mrr_at_3
value: 73.867
- type: mrr_at_5
value: 74.984
- type: ndcg_at_1
value: 67.861
- type: ndcg_at_10
value: 79.393
- type: ndcg_at_100
value: 81.04400000000001
- type: ndcg_at_1000
value: 81.15299999999999
- type: ndcg_at_3
value: 75.767
- type: ndcg_at_5
value: 77.714
- type: precision_at_1
value: 67.861
- type: precision_at_10
value: 9.199
- type: precision_at_100
value: 0.9979999999999999
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 27.222
- type: precision_at_5
value: 17.302
- type: recall_at_1
value: 67.65
- type: recall_at_10
value: 90.938
- type: recall_at_100
value: 98.736
- type: recall_at_1000
value: 99.684
- type: recall_at_3
value: 81.138
- type: recall_at_5
value: 85.827
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 25.407000000000004
- type: map_at_10
value: 79.001
- type: map_at_100
value: 81.98299999999999
- type: map_at_1000
value: 82.021
- type: map_at_3
value: 54.25600000000001
- type: map_at_5
value: 68.918
- type: mrr_at_1
value: 89.14999999999999
- type: mrr_at_10
value: 92.548
- type: mrr_at_100
value: 92.61399999999999
- type: mrr_at_1000
value: 92.616
- type: mrr_at_3
value: 92.175
- type: mrr_at_5
value: 92.432
- type: ndcg_at_1
value: 89.14999999999999
- type: ndcg_at_10
value: 86.588
- type: ndcg_at_100
value: 89.48700000000001
- type: ndcg_at_1000
value: 89.84100000000001
- type: ndcg_at_3
value: 85.00999999999999
- type: ndcg_at_5
value: 84.301
- type: precision_at_1
value: 89.14999999999999
- type: precision_at_10
value: 41.71
- type: precision_at_100
value: 4.807
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_3
value: 76.417
- type: precision_at_5
value: 64.95
- type: recall_at_1
value: 25.407000000000004
- type: recall_at_10
value: 88.221
- type: recall_at_100
value: 97.527
- type: recall_at_1000
value: 99.396
- type: recall_at_3
value: 56.751
- type: recall_at_5
value: 74.191
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 47.599999999999994
- type: map_at_10
value: 57.15
- type: map_at_100
value: 57.789
- type: map_at_1000
value: 57.80800000000001
- type: map_at_3
value: 54.467
- type: map_at_5
value: 56.016999999999996
- type: mrr_at_1
value: 47.599999999999994
- type: mrr_at_10
value: 57.15
- type: mrr_at_100
value: 57.789
- type: mrr_at_1000
value: 57.80800000000001
- type: mrr_at_3
value: 54.467
- type: mrr_at_5
value: 56.016999999999996
- type: ndcg_at_1
value: 47.599999999999994
- type: ndcg_at_10
value: 62.304
- type: ndcg_at_100
value: 65.32900000000001
- type: ndcg_at_1000
value: 65.837
- type: ndcg_at_3
value: 56.757000000000005
- type: ndcg_at_5
value: 59.575
- type: precision_at_1
value: 47.599999999999994
- type: precision_at_10
value: 7.870000000000001
- type: precision_at_100
value: 0.9259999999999999
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 21.133
- type: precision_at_5
value: 14.06
- type: recall_at_1
value: 47.599999999999994
- type: recall_at_10
value: 78.7
- type: recall_at_100
value: 92.60000000000001
- type: recall_at_1000
value: 96.6
- type: recall_at_3
value: 63.4
- type: recall_at_5
value: 70.3
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 48.28010773374375
- type: f1
value: 35.536995302144916
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.8405253283302
- type: ap
value: 52.35323515091401
- type: f1
value: 79.50069160202494
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 69.68404288794713
- type: cos_sim_spearman
value: 77.06824442481803
- type: euclidean_pearson
value: 75.47746745802166
- type: euclidean_spearman
value: 77.06825328995878
- type: manhattan_pearson
value: 75.46220581621667
- type: manhattan_spearman
value: 77.05100926919137
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 65.36800000000001
- type: map_at_10
value: 74.29400000000001
- type: map_at_100
value: 74.653
- type: map_at_1000
value: 74.664
- type: map_at_3
value: 72.416
- type: map_at_5
value: 73.658
- type: mrr_at_1
value: 67.50699999999999
- type: mrr_at_10
value: 74.85300000000001
- type: mrr_at_100
value: 75.17399999999999
- type: mrr_at_1000
value: 75.184
- type: mrr_at_3
value: 73.235
- type: mrr_at_5
value: 74.298
- type: ndcg_at_1
value: 67.50699999999999
- type: ndcg_at_10
value: 77.948
- type: ndcg_at_100
value: 79.55499999999999
- type: ndcg_at_1000
value: 79.864
- type: ndcg_at_3
value: 74.434
- type: ndcg_at_5
value: 76.504
- type: precision_at_1
value: 67.50699999999999
- type: precision_at_10
value: 9.423
- type: precision_at_100
value: 1.022
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 27.975
- type: precision_at_5
value: 17.891000000000002
- type: recall_at_1
value: 65.36800000000001
- type: recall_at_10
value: 88.633
- type: recall_at_100
value: 95.889
- type: recall_at_1000
value: 98.346
- type: recall_at_3
value: 79.404
- type: recall_at_5
value: 84.292
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 67.45124411566913
- type: f1
value: 64.77175074397455
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 73.08002689979824
- type: f1
value: 72.65358173635958
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.699999999999996
- type: map_at_10
value: 54.871
- type: map_at_100
value: 55.381
- type: map_at_1000
value: 55.43599999999999
- type: map_at_3
value: 53.367
- type: map_at_5
value: 54.257
- type: mrr_at_1
value: 48.699999999999996
- type: mrr_at_10
value: 54.871
- type: mrr_at_100
value: 55.381
- type: mrr_at_1000
value: 55.43599999999999
- type: mrr_at_3
value: 53.367
- type: mrr_at_5
value: 54.257
- type: ndcg_at_1
value: 48.699999999999996
- type: ndcg_at_10
value: 57.94200000000001
- type: ndcg_at_100
value: 60.75000000000001
- type: ndcg_at_1000
value: 62.400999999999996
- type: ndcg_at_3
value: 54.867
- type: ndcg_at_5
value: 56.493
- type: precision_at_1
value: 48.699999999999996
- type: precision_at_10
value: 6.76
- type: precision_at_100
value: 0.815
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 19.733
- type: precision_at_5
value: 12.64
- type: recall_at_1
value: 48.699999999999996
- type: recall_at_10
value: 67.60000000000001
- type: recall_at_100
value: 81.5
- type: recall_at_1000
value: 94.89999999999999
- type: recall_at_3
value: 59.199999999999996
- type: recall_at_5
value: 63.2
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 71.27000000000001
- type: f1
value: 70.46516219039894
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 69.89713048186248
- type: cos_sim_ap
value: 74.75296949416844
- type: cos_sim_f1
value: 73.0820399113082
- type: cos_sim_precision
value: 62.99694189602446
- type: cos_sim_recall
value: 87.0116156282999
- type: dot_accuracy
value: 69.89713048186248
- type: dot_ap
value: 74.75289228002875
- type: dot_f1
value: 73.0820399113082
- type: dot_precision
value: 62.99694189602446
- type: dot_recall
value: 87.0116156282999
- type: euclidean_accuracy
value: 69.89713048186248
- type: euclidean_ap
value: 74.75289228002875
- type: euclidean_f1
value: 73.0820399113082
- type: euclidean_precision
value: 62.99694189602446
- type: euclidean_recall
value: 87.0116156282999
- type: manhattan_accuracy
value: 69.9512723335138
- type: manhattan_ap
value: 74.63572749955489
- type: manhattan_f1
value: 72.80663465735486
- type: manhattan_precision
value: 62.05357142857143
- type: manhattan_recall
value: 88.0675818373812
- type: max_accuracy
value: 69.9512723335138
- type: max_ap
value: 74.75296949416844
- type: max_f1
value: 73.0820399113082
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 91.38
- type: ap
value: 89.14371766660247
- type: f1
value: 91.3668296299526
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 23.621683997579606
- type: cos_sim_spearman
value: 29.46714129804792
- type: euclidean_pearson
value: 29.841725912733487
- type: euclidean_spearman
value: 29.466951993706992
- type: manhattan_pearson
value: 29.853598937043625
- type: manhattan_spearman
value: 29.42340511723847
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 34.86196986379606
- type: cos_sim_spearman
value: 37.316873994339986
- type: euclidean_pearson
value: 35.52672274329054
- type: euclidean_spearman
value: 37.316799507511014
- type: manhattan_pearson
value: 35.55879437844226
- type: manhattan_spearman
value: 37.369433247035474
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 68.7924534800626
- type: cos_sim_spearman
value: 69.45014686127368
- type: euclidean_pearson
value: 69.12500964503516
- type: euclidean_spearman
value: 69.45014686127368
- type: manhattan_pearson
value: 70.53825064823806
- type: manhattan_spearman
value: 70.67595198226869
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 79.02281275805849
- type: cos_sim_spearman
value: 79.69275718339352
- type: euclidean_pearson
value: 79.39660648560955
- type: euclidean_spearman
value: 79.69291851788452
- type: manhattan_pearson
value: 79.3382690172365
- type: manhattan_spearman
value: 79.63605584076028
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.1994271234341
- type: mrr
value: 75.76681067371655
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.594
- type: map_at_10
value: 75.27199999999999
- type: map_at_100
value: 78.96
- type: map_at_1000
value: 79.032
- type: map_at_3
value: 52.76
- type: map_at_5
value: 64.967
- type: mrr_at_1
value: 88.721
- type: mrr_at_10
value: 91.38
- type: mrr_at_100
value: 91.484
- type: mrr_at_1000
value: 91.489
- type: mrr_at_3
value: 90.901
- type: mrr_at_5
value: 91.21000000000001
- type: ndcg_at_1
value: 88.721
- type: ndcg_at_10
value: 83.099
- type: ndcg_at_100
value: 86.938
- type: ndcg_at_1000
value: 87.644
- type: ndcg_at_3
value: 84.573
- type: ndcg_at_5
value: 83.131
- type: precision_at_1
value: 88.721
- type: precision_at_10
value: 41.506
- type: precision_at_100
value: 4.99
- type: precision_at_1000
value: 0.515
- type: precision_at_3
value: 74.214
- type: precision_at_5
value: 62.244
- type: recall_at_1
value: 26.594
- type: recall_at_10
value: 82.121
- type: recall_at_100
value: 94.643
- type: recall_at_1000
value: 98.261
- type: recall_at_3
value: 54.539
- type: recall_at_5
value: 68.573
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 51.845
- type: f1
value: 49.97529772676145
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 62.34936773593232
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 58.65057354232379
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 52.2
- type: map_at_10
value: 62.669
- type: map_at_100
value: 63.239000000000004
- type: map_at_1000
value: 63.253
- type: map_at_3
value: 60.267
- type: map_at_5
value: 61.772000000000006
- type: mrr_at_1
value: 52.2
- type: mrr_at_10
value: 62.669
- type: mrr_at_100
value: 63.239000000000004
- type: mrr_at_1000
value: 63.253
- type: mrr_at_3
value: 60.267
- type: mrr_at_5
value: 61.772000000000006
- type: ndcg_at_1
value: 52.2
- type: ndcg_at_10
value: 67.583
- type: ndcg_at_100
value: 70.30499999999999
- type: ndcg_at_1000
value: 70.652
- type: ndcg_at_3
value: 62.775999999999996
- type: ndcg_at_5
value: 65.47
- type: precision_at_1
value: 52.2
- type: precision_at_10
value: 8.290000000000001
- type: precision_at_100
value: 0.955
- type: precision_at_1000
value: 0.098
- type: precision_at_3
value: 23.333000000000002
- type: precision_at_5
value: 15.299999999999999
- type: recall_at_1
value: 52.2
- type: recall_at_10
value: 82.89999999999999
- type: recall_at_100
value: 95.5
- type: recall_at_1000
value: 98.2
- type: recall_at_3
value: 70
- type: recall_at_5
value: 76.5
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.64999999999999
- type: ap
value: 69.90209999390807
- type: f1
value: 84.9231810656075