--- tags: - mteb model-index: - name: mist-zh results: - task: type: STS dataset: type: C-MTEB/AFQMC name: MTEB AFQMC config: default split: validation revision: None metrics: - type: cos_sim_pearson value: 44.734816122831546 - type: cos_sim_spearman value: 46.97006123331873 - type: euclidean_pearson value: 45.38062036005061 - type: euclidean_spearman value: 46.97006123331873 - type: manhattan_pearson value: 45.25100462997557 - type: manhattan_spearman value: 46.85418008817015 - task: type: STS dataset: type: C-MTEB/ATEC name: MTEB ATEC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 49.23835317471939 - type: cos_sim_spearman value: 51.29611473119322 - type: euclidean_pearson value: 53.41533188991713 - type: euclidean_spearman value: 51.29611360495954 - type: manhattan_pearson value: 53.42662771302782 - type: manhattan_spearman value: 51.29682402789285 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (zh) config: zh split: test revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy value: 38.855999999999995 - type: f1 value: 36.96137480741953 - task: type: STS dataset: type: C-MTEB/BQ name: MTEB BQ config: default split: test revision: None metrics: - type: cos_sim_pearson value: 61.79575529204537 - type: cos_sim_spearman value: 64.96308773217001 - type: euclidean_pearson value: 63.38747223113914 - type: euclidean_spearman value: 64.96309119412786 - type: manhattan_pearson value: 63.36833986897711 - type: manhattan_spearman value: 64.95000035386369 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringP2P name: MTEB CLSClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 40.26570556670306 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringS2S name: MTEB CLSClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 37.68621168788469 - task: type: Reranking dataset: type: C-MTEB/CMedQAv1-reranking name: MTEB CMedQAv1 config: default split: test revision: None metrics: - type: map value: 84.40938491415716 - type: mrr value: 86.86722222222222 - task: type: Reranking dataset: type: C-MTEB/CMedQAv2-reranking name: MTEB CMedQAv2 config: default split: test revision: None metrics: - type: map value: 85.2507433210034 - type: mrr value: 87.58742063492063 - task: type: Retrieval dataset: type: C-MTEB/CmedqaRetrieval name: MTEB CmedqaRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 24.043999999999997 - type: map_at_10 value: 35.311 - type: map_at_100 value: 37.125 - type: map_at_1000 value: 37.26 - type: map_at_3 value: 31.342 - type: map_at_5 value: 33.613 - type: mrr_at_1 value: 36.909 - type: mrr_at_10 value: 44.373000000000005 - type: mrr_at_100 value: 45.367000000000004 - type: mrr_at_1000 value: 45.422000000000004 - type: mrr_at_3 value: 41.927 - type: mrr_at_5 value: 43.292 - type: ndcg_at_1 value: 36.909 - type: ndcg_at_10 value: 41.666 - type: ndcg_at_100 value: 48.915 - type: ndcg_at_1000 value: 51.348000000000006 - type: ndcg_at_3 value: 36.592 - type: ndcg_at_5 value: 38.787 - type: precision_at_1 value: 36.909 - type: precision_at_10 value: 9.327 - type: precision_at_100 value: 1.5230000000000001 - type: precision_at_1000 value: 0.183 - type: precision_at_3 value: 20.671999999999997 - type: precision_at_5 value: 15.179 - type: recall_at_1 value: 24.043999999999997 - type: recall_at_10 value: 51.370000000000005 - type: recall_at_100 value: 81.569 - type: recall_at_1000 value: 98.053 - type: recall_at_3 value: 36.120000000000005 - type: recall_at_5 value: 42.829 - task: type: PairClassification dataset: type: C-MTEB/CMNLI name: MTEB Cmnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 75.92303066746842 - type: cos_sim_ap value: 84.39741959629595 - type: cos_sim_f1 value: 77.28710064333224 - type: cos_sim_precision value: 72.41520228851655 - type: cos_sim_recall value: 82.8618190320318 - type: dot_accuracy value: 75.92303066746842 - type: dot_ap value: 84.39592659189601 - type: dot_f1 value: 77.28710064333224 - type: dot_precision value: 72.41520228851655 - type: dot_recall value: 82.8618190320318 - type: euclidean_accuracy value: 75.92303066746842 - type: euclidean_ap value: 84.39741904478117 - type: euclidean_f1 value: 77.28710064333224 - type: euclidean_precision value: 72.41520228851655 - type: euclidean_recall value: 82.8618190320318 - type: manhattan_accuracy value: 75.83884546001202 - type: manhattan_ap value: 84.39482592167423 - type: manhattan_f1 value: 77.2419718612394 - type: manhattan_precision value: 71.43424711958681 - type: manhattan_recall value: 84.07762450315643 - type: max_accuracy value: 75.92303066746842 - type: max_ap value: 84.39741959629595 - type: max_f1 value: 77.28710064333224 - task: type: Retrieval dataset: type: C-MTEB/CovidRetrieval name: MTEB CovidRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 67.65 - type: map_at_10 value: 75.672 - type: map_at_100 value: 76.005 - type: map_at_1000 value: 76.007 - type: map_at_3 value: 73.867 - type: map_at_5 value: 74.949 - type: mrr_at_1 value: 67.756 - type: mrr_at_10 value: 75.64 - type: mrr_at_100 value: 75.973 - type: mrr_at_1000 value: 75.97500000000001 - type: mrr_at_3 value: 73.867 - type: mrr_at_5 value: 74.984 - type: ndcg_at_1 value: 67.861 - type: ndcg_at_10 value: 79.393 - type: ndcg_at_100 value: 81.04400000000001 - type: ndcg_at_1000 value: 81.15299999999999 - type: ndcg_at_3 value: 75.767 - type: ndcg_at_5 value: 77.714 - type: precision_at_1 value: 67.861 - type: precision_at_10 value: 9.199 - type: precision_at_100 value: 0.9979999999999999 - type: precision_at_1000 value: 0.101 - type: precision_at_3 value: 27.222 - type: precision_at_5 value: 17.302 - type: recall_at_1 value: 67.65 - type: recall_at_10 value: 90.938 - type: recall_at_100 value: 98.736 - type: recall_at_1000 value: 99.684 - type: recall_at_3 value: 81.138 - type: recall_at_5 value: 85.827 - task: type: Retrieval dataset: type: C-MTEB/DuRetrieval name: MTEB DuRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 25.407000000000004 - type: map_at_10 value: 79.001 - type: map_at_100 value: 81.98299999999999 - type: map_at_1000 value: 82.021 - type: map_at_3 value: 54.25600000000001 - type: map_at_5 value: 68.918 - type: mrr_at_1 value: 89.14999999999999 - type: mrr_at_10 value: 92.548 - type: mrr_at_100 value: 92.61399999999999 - type: mrr_at_1000 value: 92.616 - type: mrr_at_3 value: 92.175 - type: mrr_at_5 value: 92.432 - type: ndcg_at_1 value: 89.14999999999999 - type: ndcg_at_10 value: 86.588 - type: ndcg_at_100 value: 89.48700000000001 - type: ndcg_at_1000 value: 89.84100000000001 - type: ndcg_at_3 value: 85.00999999999999 - type: ndcg_at_5 value: 84.301 - type: precision_at_1 value: 89.14999999999999 - type: precision_at_10 value: 41.71 - type: precision_at_100 value: 4.807 - type: precision_at_1000 value: 0.48900000000000005 - type: precision_at_3 value: 76.417 - type: precision_at_5 value: 64.95 - type: recall_at_1 value: 25.407000000000004 - type: recall_at_10 value: 88.221 - type: recall_at_100 value: 97.527 - type: recall_at_1000 value: 99.396 - type: recall_at_3 value: 56.751 - type: recall_at_5 value: 74.191 - task: type: Retrieval dataset: type: C-MTEB/EcomRetrieval name: MTEB EcomRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 47.599999999999994 - type: map_at_10 value: 57.15 - type: map_at_100 value: 57.789 - type: map_at_1000 value: 57.80800000000001 - type: map_at_3 value: 54.467 - type: map_at_5 value: 56.016999999999996 - type: mrr_at_1 value: 47.599999999999994 - type: mrr_at_10 value: 57.15 - type: mrr_at_100 value: 57.789 - type: mrr_at_1000 value: 57.80800000000001 - type: mrr_at_3 value: 54.467 - type: mrr_at_5 value: 56.016999999999996 - type: ndcg_at_1 value: 47.599999999999994 - type: ndcg_at_10 value: 62.304 - type: ndcg_at_100 value: 65.32900000000001 - type: ndcg_at_1000 value: 65.837 - type: ndcg_at_3 value: 56.757000000000005 - type: ndcg_at_5 value: 59.575 - type: precision_at_1 value: 47.599999999999994 - type: precision_at_10 value: 7.870000000000001 - type: precision_at_100 value: 0.9259999999999999 - type: precision_at_1000 value: 0.097 - type: precision_at_3 value: 21.133 - type: precision_at_5 value: 14.06 - type: recall_at_1 value: 47.599999999999994 - type: recall_at_10 value: 78.7 - type: recall_at_100 value: 92.60000000000001 - type: recall_at_1000 value: 96.6 - type: recall_at_3 value: 63.4 - type: recall_at_5 value: 70.3 - task: type: Classification dataset: type: C-MTEB/IFlyTek-classification name: MTEB IFlyTek config: default split: validation revision: None metrics: - type: accuracy value: 48.28010773374375 - type: f1 value: 35.536995302144916 - task: type: Classification dataset: type: C-MTEB/JDReview-classification name: MTEB JDReview config: default split: test revision: None metrics: - type: accuracy value: 84.8405253283302 - type: ap value: 52.35323515091401 - type: f1 value: 79.50069160202494 - task: type: STS dataset: type: C-MTEB/LCQMC name: MTEB LCQMC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 69.68404288794713 - type: cos_sim_spearman value: 77.06824442481803 - type: euclidean_pearson value: 75.47746745802166 - type: euclidean_spearman value: 77.06825328995878 - type: manhattan_pearson value: 75.46220581621667 - type: manhattan_spearman value: 77.05100926919137 - task: type: Retrieval dataset: type: C-MTEB/MMarcoRetrieval name: MTEB MMarcoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 65.36800000000001 - type: map_at_10 value: 74.29400000000001 - type: map_at_100 value: 74.653 - type: map_at_1000 value: 74.664 - type: map_at_3 value: 72.416 - type: map_at_5 value: 73.658 - type: mrr_at_1 value: 67.50699999999999 - type: mrr_at_10 value: 74.85300000000001 - type: mrr_at_100 value: 75.17399999999999 - type: mrr_at_1000 value: 75.184 - type: mrr_at_3 value: 73.235 - type: mrr_at_5 value: 74.298 - type: ndcg_at_1 value: 67.50699999999999 - type: ndcg_at_10 value: 77.948 - type: ndcg_at_100 value: 79.55499999999999 - type: ndcg_at_1000 value: 79.864 - type: ndcg_at_3 value: 74.434 - type: ndcg_at_5 value: 76.504 - type: precision_at_1 value: 67.50699999999999 - type: precision_at_10 value: 9.423 - type: precision_at_100 value: 1.022 - type: precision_at_1000 value: 0.105 - type: precision_at_3 value: 27.975 - type: precision_at_5 value: 17.891000000000002 - type: recall_at_1 value: 65.36800000000001 - type: recall_at_10 value: 88.633 - type: recall_at_100 value: 95.889 - type: recall_at_1000 value: 98.346 - type: recall_at_3 value: 79.404 - type: recall_at_5 value: 84.292 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (zh-CN) config: zh-CN split: test revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 metrics: - type: accuracy value: 67.45124411566913 - type: f1 value: 64.77175074397455 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (zh-CN) config: zh-CN split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 73.08002689979824 - type: f1 value: 72.65358173635958 - task: type: Retrieval dataset: type: C-MTEB/MedicalRetrieval name: MTEB MedicalRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 48.699999999999996 - type: map_at_10 value: 54.871 - type: map_at_100 value: 55.381 - type: map_at_1000 value: 55.43599999999999 - type: map_at_3 value: 53.367 - type: map_at_5 value: 54.257 - type: mrr_at_1 value: 48.699999999999996 - type: mrr_at_10 value: 54.871 - type: mrr_at_100 value: 55.381 - type: mrr_at_1000 value: 55.43599999999999 - type: mrr_at_3 value: 53.367 - type: mrr_at_5 value: 54.257 - type: ndcg_at_1 value: 48.699999999999996 - type: ndcg_at_10 value: 57.94200000000001 - type: ndcg_at_100 value: 60.75000000000001 - type: ndcg_at_1000 value: 62.400999999999996 - type: ndcg_at_3 value: 54.867 - type: ndcg_at_5 value: 56.493 - type: precision_at_1 value: 48.699999999999996 - type: precision_at_10 value: 6.76 - type: precision_at_100 value: 0.815 - type: precision_at_1000 value: 0.095 - type: precision_at_3 value: 19.733 - type: precision_at_5 value: 12.64 - type: recall_at_1 value: 48.699999999999996 - type: recall_at_10 value: 67.60000000000001 - type: recall_at_100 value: 81.5 - type: recall_at_1000 value: 94.89999999999999 - type: recall_at_3 value: 59.199999999999996 - type: recall_at_5 value: 63.2 - task: type: Classification dataset: type: C-MTEB/MultilingualSentiment-classification name: MTEB MultilingualSentiment config: default split: validation revision: None metrics: - type: accuracy value: 71.27000000000001 - type: f1 value: 70.46516219039894 - task: type: PairClassification dataset: type: C-MTEB/OCNLI name: MTEB Ocnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 69.89713048186248 - type: cos_sim_ap value: 74.75296949416844 - type: cos_sim_f1 value: 73.0820399113082 - type: cos_sim_precision value: 62.99694189602446 - type: cos_sim_recall value: 87.0116156282999 - type: dot_accuracy value: 69.89713048186248 - type: dot_ap value: 74.75289228002875 - type: dot_f1 value: 73.0820399113082 - type: dot_precision value: 62.99694189602446 - type: dot_recall value: 87.0116156282999 - type: euclidean_accuracy value: 69.89713048186248 - type: euclidean_ap value: 74.75289228002875 - type: euclidean_f1 value: 73.0820399113082 - type: euclidean_precision value: 62.99694189602446 - type: euclidean_recall value: 87.0116156282999 - type: manhattan_accuracy value: 69.9512723335138 - type: manhattan_ap value: 74.63572749955489 - type: manhattan_f1 value: 72.80663465735486 - type: manhattan_precision value: 62.05357142857143 - type: manhattan_recall value: 88.0675818373812 - type: max_accuracy value: 69.9512723335138 - type: max_ap value: 74.75296949416844 - type: max_f1 value: 73.0820399113082 - task: type: Classification dataset: type: C-MTEB/OnlineShopping-classification name: MTEB OnlineShopping config: default split: test revision: None metrics: - type: accuracy value: 91.38 - type: ap value: 89.14371766660247 - type: f1 value: 91.3668296299526 - task: type: STS dataset: type: C-MTEB/PAWSX name: MTEB PAWSX config: default split: test revision: None metrics: - type: cos_sim_pearson value: 23.621683997579606 - type: cos_sim_spearman value: 29.46714129804792 - type: euclidean_pearson value: 29.841725912733487 - type: euclidean_spearman value: 29.466951993706992 - type: manhattan_pearson value: 29.853598937043625 - type: manhattan_spearman value: 29.42340511723847 - task: type: STS dataset: type: C-MTEB/QBQTC name: MTEB QBQTC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 34.86196986379606 - type: cos_sim_spearman value: 37.316873994339986 - type: euclidean_pearson value: 35.52672274329054 - type: euclidean_spearman value: 37.316799507511014 - type: manhattan_pearson value: 35.55879437844226 - type: manhattan_spearman value: 37.369433247035474 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (zh) config: zh split: test revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 metrics: - type: cos_sim_pearson value: 68.7924534800626 - type: cos_sim_spearman value: 69.45014686127368 - type: euclidean_pearson value: 69.12500964503516 - type: euclidean_spearman value: 69.45014686127368 - type: manhattan_pearson value: 70.53825064823806 - type: manhattan_spearman value: 70.67595198226869 - task: type: STS dataset: type: C-MTEB/STSB name: MTEB STSB config: default split: test revision: None metrics: - type: cos_sim_pearson value: 79.02281275805849 - type: cos_sim_spearman value: 79.69275718339352 - type: euclidean_pearson value: 79.39660648560955 - type: euclidean_spearman value: 79.69291851788452 - type: manhattan_pearson value: 79.3382690172365 - type: manhattan_spearman value: 79.63605584076028 - task: type: Reranking dataset: type: C-MTEB/T2Reranking name: MTEB T2Reranking config: default split: dev revision: None metrics: - type: map value: 66.1994271234341 - type: mrr value: 75.76681067371655 - task: type: Retrieval dataset: type: C-MTEB/T2Retrieval name: MTEB T2Retrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 26.594 - type: map_at_10 value: 75.27199999999999 - type: map_at_100 value: 78.96 - type: map_at_1000 value: 79.032 - type: map_at_3 value: 52.76 - type: map_at_5 value: 64.967 - type: mrr_at_1 value: 88.721 - type: mrr_at_10 value: 91.38 - type: mrr_at_100 value: 91.484 - type: mrr_at_1000 value: 91.489 - type: mrr_at_3 value: 90.901 - type: mrr_at_5 value: 91.21000000000001 - type: ndcg_at_1 value: 88.721 - type: ndcg_at_10 value: 83.099 - type: ndcg_at_100 value: 86.938 - type: ndcg_at_1000 value: 87.644 - type: ndcg_at_3 value: 84.573 - type: ndcg_at_5 value: 83.131 - type: precision_at_1 value: 88.721 - type: precision_at_10 value: 41.506 - type: precision_at_100 value: 4.99 - type: precision_at_1000 value: 0.515 - type: precision_at_3 value: 74.214 - type: precision_at_5 value: 62.244 - type: recall_at_1 value: 26.594 - type: recall_at_10 value: 82.121 - type: recall_at_100 value: 94.643 - type: recall_at_1000 value: 98.261 - type: recall_at_3 value: 54.539 - type: recall_at_5 value: 68.573 - task: type: Classification dataset: type: C-MTEB/TNews-classification name: MTEB TNews config: default split: validation revision: None metrics: - type: accuracy value: 51.845 - type: f1 value: 49.97529772676145 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringP2P name: MTEB ThuNewsClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 62.34936773593232 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringS2S name: MTEB ThuNewsClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 58.65057354232379 - task: type: Retrieval dataset: type: C-MTEB/VideoRetrieval name: MTEB VideoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 52.2 - type: map_at_10 value: 62.669 - type: map_at_100 value: 63.239000000000004 - type: map_at_1000 value: 63.253 - type: map_at_3 value: 60.267 - type: map_at_5 value: 61.772000000000006 - type: mrr_at_1 value: 52.2 - type: mrr_at_10 value: 62.669 - type: mrr_at_100 value: 63.239000000000004 - type: mrr_at_1000 value: 63.253 - type: mrr_at_3 value: 60.267 - type: mrr_at_5 value: 61.772000000000006 - type: ndcg_at_1 value: 52.2 - type: ndcg_at_10 value: 67.583 - type: ndcg_at_100 value: 70.30499999999999 - type: ndcg_at_1000 value: 70.652 - type: ndcg_at_3 value: 62.775999999999996 - type: ndcg_at_5 value: 65.47 - type: precision_at_1 value: 52.2 - type: precision_at_10 value: 8.290000000000001 - type: precision_at_100 value: 0.955 - type: precision_at_1000 value: 0.098 - type: precision_at_3 value: 23.333000000000002 - type: precision_at_5 value: 15.299999999999999 - type: recall_at_1 value: 52.2 - type: recall_at_10 value: 82.89999999999999 - type: recall_at_100 value: 95.5 - type: recall_at_1000 value: 98.2 - type: recall_at_3 value: 70.0 - type: recall_at_5 value: 76.5 - task: type: Classification dataset: type: C-MTEB/waimai-classification name: MTEB Waimai config: default split: test revision: None metrics: - type: accuracy value: 86.64999999999999 - type: ap value: 69.90209999390807 - type: f1 value: 84.9231810656075 - task: type: Reranking dataset: type: C-MTEB/Mmarco-reranking name: MTEB MMarcoReranking config: default split: dev revision: None metrics: - type: map value: 28.09825725416088 - type: mrr value: 26.95912698412698 ---