metadata
pipeline_tag: sentence-similarity
tags:
- sentence-transformers
- feature-extraction
- sentence-similarity
model-index:
- name: SGPT-125M-weightedmean-nli-bitfit
results:
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
metrics:
- type: v_measure
value: 0.28301902023313874
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
metrics:
- type: cos_sim_pearson
value: 0.76401935081936
- type: cos_sim_spearman
value: 0.7723446219694267
- type: euclidean_pearson
value: 0.7461017160439877
- type: euclidean_spearman
value: 0.7585871531365609
- type: manhattan_pearson
value: 0.7483034779539725
- type: manhattan_spearman
value: 0.759594899358843
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
metrics:
- type: v_measure
value: 0.3474248247787077
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
metrics:
- type: accuracy
value: 0.35098
- type: f1
value: 0.34732656514357263
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (de)
metrics:
- type: accuracy
value: 0.24516
- type: f1
value: 0.2421748200448397
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (es)
metrics:
- type: accuracy
value: 0.29097999999999996
- type: f1
value: 0.28620040162757093
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (fr)
metrics:
- type: accuracy
value: 0.27396
- type: f1
value: 0.27146888644986283
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (ja)
metrics:
- type: accuracy
value: 0.21724000000000002
- type: f1
value: 0.2137230564276654
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
metrics:
- type: accuracy
value: 0.23975999999999997
- type: f1
value: 0.23741137981755484
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (de-en)
metrics:
- type: accuracy
value: 0.010960334029227558
- type: f1
value: 0.01092553931802366
- type: precision
value: 0.010908141962421711
- type: recall
value: 0.010960334029227558
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (fr-en)
metrics:
- type: accuracy
value: 0.00022011886418666079
- type: f1
value: 0.00022011886418666079
- type: precision
value: 0.00022011886418666079
- type: recall
value: 0.00022011886418666079
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (ru-en)
metrics:
- type: accuracy
value: 0
- type: f1
value: 0
- type: precision
value: 0
- type: recall
value: 0
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (zh-en)
metrics:
- type: accuracy
value: 0
- type: f1
value: 0
- type: precision
value: 0
- type: recall
value: 0
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
metrics:
- type: accuracy
value: 0.8151846785225718
- type: f1
value: 0.81648869152345
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (de)
metrics:
- type: accuracy
value: 0.6037475345167653
- type: f1
value: 0.5845264937551703
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (es)
metrics:
- type: accuracy
value: 0.6736824549699799
- type: f1
value: 0.6535927434998515
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (fr)
metrics:
- type: accuracy
value: 0.6312871907297212
- type: f1
value: 0.6137620329272278
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (hi)
metrics:
- type: accuracy
value: 0.47045536034420943
- type: f1
value: 0.46203899126445613
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (th)
metrics:
- type: accuracy
value: 0.5228209764918625
- type: f1
value: 0.5075489206473579
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
metrics:
- type: map_at_1
value: 0.0808
- type: map_at_10
value: 0.11691
- type: map_at_100
value: 0.12312
- type: map_at_1000
value: 0.12439
- type: map_at_3
value: 0.10344
- type: map_at_5
value: 0.10996
- type: ndcg_at_1
value: 0.10697
- type: ndcg_at_10
value: 0.1448
- type: ndcg_at_100
value: 0.18161
- type: ndcg_at_1000
value: 0.21886
- type: ndcg_at_3
value: 0.11872
- type: ndcg_at_5
value: 0.12834
- type: precision_at_1
value: 0.10697
- type: precision_at_10
value: 0.02811
- type: precision_at_100
value: 0.00551
- type: precision_at_1000
value: 0.00102
- type: precision_at_3
value: 0.05804
- type: precision_at_5
value: 0.04154
- type: recall_at_1
value: 0.0808
- type: recall_at_10
value: 0.20235
- type: recall_at_100
value: 0.37526
- type: recall_at_1000
value: 0.65106
- type: recall_at_3
value: 0.12804
- type: recall_at_5
value: 0.15499
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
metrics:
- type: accuracy
value: 0.6588059701492537
- type: ap
value: 0.28685493163579784
- type: f1
value: 0.5979951005816335
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (de)
metrics:
- type: accuracy
value: 0.5907922912205568
- type: ap
value: 0.7391887421019034
- type: f1
value: 0.566316368658711
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en-ext)
metrics:
- type: accuracy
value: 0.6491754122938531
- type: ap
value: 0.16360681214864226
- type: f1
value: 0.5312659206152377
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (ja)
metrics:
- type: accuracy
value: 0.56423982869379
- type: ap
value: 0.12143003571907898
- type: f1
value: 0.45763637779874716
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
metrics:
- type: map_at_1
value: 0.06496
- type: map_at_10
value: 0.09243
- type: map_at_100
value: 0.09841
- type: map_at_1000
value: 0.09946
- type: map_at_3
value: 0.08395
- type: map_at_5
value: 0.08872
- type: ndcg_at_1
value: 0.08224
- type: ndcg_at_10
value: 0.1124
- type: ndcg_at_100
value: 0.14525
- type: ndcg_at_1000
value: 0.17686
- type: ndcg_at_3
value: 0.09617
- type: ndcg_at_5
value: 0.1037
- type: precision_at_1
value: 0.08224
- type: precision_at_10
value: 0.02082
- type: precision_at_100
value: 0.00443
- type: precision_at_1000
value: 0.00085
- type: precision_at_3
value: 0.04623
- type: precision_at_5
value: 0.03331
- type: recall_at_1
value: 0.06496
- type: recall_at_10
value: 0.1531
- type: recall_at_100
value: 0.3068
- type: recall_at_1000
value: 0.54335
- type: recall_at_3
value: 0.10691
- type: recall_at_5
value: 0.12688
- task:
type: Reranking
dataset:
type: mteb/mind_small
name: MTEB MindSmallReranking
metrics:
- type: map
value: 0.2926934104146833
- type: mrr
value: 0.3013214087687572
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
metrics:
- type: map_at_1
value: 0.01227
- type: map_at_10
value: 0.03081
- type: map_at_100
value: 0.04104
- type: map_at_1000
value: 0.04989
- type: map_at_3
value: 0.02221
- type: map_at_5
value: 0.02535
- type: ndcg_at_1
value: 0.15015
- type: ndcg_at_10
value: 0.11805
- type: ndcg_at_100
value: 0.12452
- type: ndcg_at_1000
value: 0.22284
- type: ndcg_at_3
value: 0.13257
- type: ndcg_at_5
value: 0.12199
- type: precision_at_1
value: 0.16409
- type: precision_at_10
value: 0.09102
- type: precision_at_100
value: 0.03678
- type: precision_at_1000
value: 0.01609
- type: precision_at_3
value: 0.12797
- type: precision_at_5
value: 0.10464
- type: recall_at_1
value: 0.01227
- type: recall_at_10
value: 0.05838
- type: recall_at_100
value: 0.15716
- type: recall_at_1000
value: 0.48837
- type: recall_at_3
value: 0.02828
- type: recall_at_5
value: 0.03697
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
metrics:
- type: map_at_1
value: 0.0288
- type: map_at_10
value: 0.04914
- type: map_at_100
value: 0.05459
- type: map_at_1000
value: 0.05538
- type: map_at_3
value: 0.04087
- type: map_at_5
value: 0.04518
- type: ndcg_at_1
value: 0.02937
- type: ndcg_at_10
value: 0.06273
- type: ndcg_at_100
value: 0.09426
- type: ndcg_at_1000
value: 0.12033
- type: ndcg_at_3
value: 0.04513
- type: ndcg_at_5
value: 0.05292
- type: precision_at_1
value: 0.02937
- type: precision_at_10
value: 0.01089
- type: precision_at_100
value: 0.00277
- type: precision_at_1000
value: 0.00051
- type: precision_at_3
value: 0.01929
- type: precision_at_5
value: 0.01547
- type: recall_at_1
value: 0.0288
- type: recall_at_10
value: 0.10578
- type: recall_at_100
value: 0.26267
- type: recall_at_1000
value: 0.4759
- type: recall_at_3
value: 0.05673
- type: recall_at_5
value: 0.07545
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
metrics:
- type: map_at_1
value: 0.13843
- type: map_at_10
value: 0.17496
- type: map_at_100
value: 0.18304
- type: map_at_1000
value: 0.18426
- type: map_at_3
value: 0.16225
- type: map_at_5
value: 0.1683
- type: ndcg_at_1
value: 0.16698
- type: ndcg_at_10
value: 0.20301
- type: ndcg_at_100
value: 0.24523
- type: ndcg_at_1000
value: 0.27784
- type: ndcg_at_3
value: 0.17822
- type: ndcg_at_5
value: 0.18794
- type: precision_at_1
value: 0.16698
- type: precision_at_10
value: 0.03358
- type: precision_at_100
value: 0.00618
- type: precision_at_1000
value: 0.00101
- type: precision_at_3
value: 0.07898
- type: precision_at_5
value: 0.05429
- type: recall_at_1
value: 0.13843
- type: recall_at_10
value: 0.25888
- type: recall_at_100
value: 0.45028
- type: recall_at_1000
value: 0.68991
- type: recall_at_3
value: 0.18851
- type: recall_at_5
value: 0.21462
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
metrics:
- type: cos_sim_pearson
value: 0.8020938796088339
- type: cos_sim_spearman
value: 0.6916914010333395
- type: euclidean_pearson
value: 0.7933415250097545
- type: euclidean_spearman
value: 0.7146707320292746
- type: manhattan_pearson
value: 0.7973669837981976
- type: manhattan_spearman
value: 0.7187919511134903
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
metrics:
- type: v_measure
value: 0.4459127540530939
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
metrics:
- type: map
value: 0.6835710819755543
- type: mrr
value: 0.8805442832403617
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
metrics:
- type: map_at_1
value: 0.13442
- type: map_at_10
value: 0.24275
- type: map_at_100
value: 0.25588
- type: map_at_1000
value: 0.25659
- type: map_at_3
value: 0.20092
- type: map_at_5
value: 0.2244
- type: ndcg_at_1
value: 0.13442
- type: ndcg_at_10
value: 0.3104
- type: ndcg_at_100
value: 0.37529
- type: ndcg_at_1000
value: 0.39348
- type: ndcg_at_3
value: 0.22342
- type: ndcg_at_5
value: 0.26596
- type: precision_at_1
value: 0.13442
- type: precision_at_10
value: 0.05299
- type: precision_at_100
value: 0.00836
- type: precision_at_1000
value: 0.00098
- type: precision_at_3
value: 0.09625
- type: precision_at_5
value: 0.07852
- type: recall_at_1
value: 0.13442
- type: recall_at_10
value: 0.52987
- type: recall_at_100
value: 0.83642
- type: recall_at_1000
value: 0.97795
- type: recall_at_3
value: 0.28876
- type: recall_at_5
value: 0.3926
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
metrics:
- type: map
value: 0.5263439984994702
- type: mrr
value: 0.6575704612408213
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
metrics:
- type: accuracy
value: 0.5482173174872665
- type: f1
value: 0.5514729314789282
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
metrics:
- type: v_measure
value: 0.2467870651472156
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
metrics:
- type: map_at_1
value: 0.09676
- type: map_at_10
value: 0.13351
- type: map_at_100
value: 0.13919
- type: map_at_1000
value: 0.1401
- type: map_at_3
value: 0.12223
- type: map_at_5
value: 0.12812
- type: ndcg_at_1
value: 0.19352
- type: ndcg_at_10
value: 0.17727
- type: ndcg_at_100
value: 0.20837
- type: ndcg_at_1000
value: 0.23412
- type: ndcg_at_3
value: 0.15317
- type: ndcg_at_5
value: 0.16436
- type: precision_at_1
value: 0.19352
- type: precision_at_10
value: 0.03993
- type: precision_at_100
value: 0.00651
- type: precision_at_1000
value: 0.001
- type: precision_at_3
value: 0.09669
- type: precision_at_5
value: 0.0669
- type: recall_at_1
value: 0.09676
- type: recall_at_10
value: 0.19966
- type: recall_at_100
value: 0.32573
- type: recall_at_1000
value: 0.49905
- type: recall_at_3
value: 0.14504
- type: recall_at_5
value: 0.16725
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
metrics:
- type: map_at_1
value: 0.00645
- type: map_at_10
value: 0.04116
- type: map_at_100
value: 0.07527
- type: map_at_1000
value: 0.08678
- type: map_at_3
value: 0.01602
- type: map_at_5
value: 0.026
- type: ndcg_at_1
value: 0.10204
- type: ndcg_at_10
value: 0.1227
- type: ndcg_at_100
value: 0.22461
- type: ndcg_at_1000
value: 0.33543
- type: ndcg_at_3
value: 0.09982
- type: ndcg_at_5
value: 0.11498
- type: precision_at_1
value: 0.10204
- type: precision_at_10
value: 0.12245
- type: precision_at_100
value: 0.05286
- type: precision_at_1000
value: 0.01263
- type: precision_at_3
value: 0.10884
- type: precision_at_5
value: 0.13061
- type: recall_at_1
value: 0.00645
- type: recall_at_10
value: 0.08996
- type: recall_at_100
value: 0.33666
- type: recall_at_1000
value: 0.67704
- type: recall_at_3
value: 0.02504
- type: recall_at_5
value: 0.0495
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
metrics:
- type: map_at_1
value: 0.18222
- type: map_at_10
value: 0.24506
- type: map_at_100
value: 0.25611
- type: map_at_1000
value: 0.25758
- type: map_at_3
value: 0.22265
- type: map_at_5
value: 0.23698
- type: ndcg_at_1
value: 0.23033
- type: ndcg_at_10
value: 0.28719
- type: ndcg_at_100
value: 0.33748
- type: ndcg_at_1000
value: 0.37056
- type: ndcg_at_3
value: 0.2524
- type: ndcg_at_5
value: 0.2712
- type: precision_at_1
value: 0.23033
- type: precision_at_10
value: 0.05408
- type: precision_at_100
value: 0.01004
- type: precision_at_1000
value: 0.00158
- type: precision_at_3
value: 0.11874
- type: precision_at_5
value: 0.08927
- type: recall_at_1
value: 0.18222
- type: recall_at_10
value: 0.36355
- type: recall_at_100
value: 0.58724
- type: recall_at_1000
value: 0.81335
- type: recall_at_3
value: 0.26334
- type: recall_at_5
value: 0.314
- task:
type: Summarization
dataset:
type: mteb/summeval
name: MTEB SummEval
metrics:
- type: cos_sim_pearson
value: 0.3056303767714449
- type: cos_sim_spearman
value: 0.30256847004390486
- type: dot_pearson
value: 0.29453520030995006
- type: dot_spearman
value: 0.2956173255092678
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
metrics:
- type: accuracy
value: 0.62896
- type: ap
value: 0.5847769349850157
- type: f1
value: 0.6267885149592086
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
metrics:
- type: cos_sim_pearson
value: 0.7905293131911804
- type: cos_sim_spearman
value: 0.7973794782598049
- type: euclidean_pearson
value: 0.7817016171851057
- type: euclidean_spearman
value: 0.7876038607583106
- type: manhattan_pearson
value: 0.784994607532332
- type: manhattan_spearman
value: 0.7913026720132872
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
metrics:
- type: v_measure
value: 0.24932123582259286
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
metrics:
- type: map_at_1
value: 0.03714
- type: map_at_10
value: 0.06926
- type: map_at_100
value: 0.07879
- type: map_at_1000
value: 0.08032
- type: map_at_3
value: 0.05504
- type: map_at_5
value: 0.06357
- type: ndcg_at_1
value: 0.0886
- type: ndcg_at_10
value: 0.11007
- type: ndcg_at_100
value: 0.16154
- type: ndcg_at_1000
value: 0.19668
- type: ndcg_at_3
value: 0.08103
- type: ndcg_at_5
value: 0.09456
- type: precision_at_1
value: 0.0886
- type: precision_at_10
value: 0.0372
- type: precision_at_100
value: 0.00917
- type: precision_at_1000
value: 0.00156
- type: precision_at_3
value: 0.06254
- type: precision_at_5
value: 0.05381
- type: recall_at_1
value: 0.03714
- type: recall_at_10
value: 0.14382
- type: recall_at_100
value: 0.33166
- type: recall_at_1000
value: 0.53444
- type: recall_at_3
value: 0.07523
- type: recall_at_5
value: 0.1091
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
metrics:
- type: cos_sim_pearson
value: 0.7535551963935667
- type: cos_sim_spearman
value: 0.7098892671568665
- type: euclidean_pearson
value: 0.7324467338564629
- type: euclidean_spearman
value: 0.7197533151639425
- type: manhattan_pearson
value: 0.7327765593599381
- type: manhattan_spearman
value: 0.722221421456084
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
metrics:
- type: map_at_1
value: 0.12058
- type: map_at_10
value: 0.16051
- type: map_at_100
value: 0.16772
- type: map_at_1000
value: 0.16871
- type: map_at_3
value: 0.1478
- type: map_at_5
value: 0.155
- type: ndcg_at_1
value: 0.1535
- type: ndcg_at_10
value: 0.18804
- type: ndcg_at_100
value: 0.22346
- type: ndcg_at_1000
value: 0.25007
- type: ndcg_at_3
value: 0.16768
- type: ndcg_at_5
value: 0.17692
- type: precision_at_1
value: 0.1535
- type: precision_at_10
value: 0.0351
- type: precision_at_100
value: 0.00664
- type: precision_at_1000
value: 0.00111
- type: precision_at_3
value: 0.07983
- type: precision_at_5
value: 0.05656
- type: recall_at_1
value: 0.12058
- type: recall_at_10
value: 0.23644
- type: recall_at_100
value: 0.3976
- type: recall_at_1000
value: 0.5856
- type: recall_at_3
value: 0.17542
- type: recall_at_5
value: 0.20232
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
metrics:
- type: map_at_1
value: 0.21183
- type: map_at_10
value: 0.289
- type: map_at_100
value: 0.29858
- type: map_at_1000
value: 0.29954
- type: map_at_3
value: 0.2658
- type: map_at_5
value: 0.27912
- type: ndcg_at_1
value: 0.24765
- type: ndcg_at_10
value: 0.3334
- type: ndcg_at_100
value: 0.37997
- type: ndcg_at_1000
value: 0.40416
- type: ndcg_at_3
value: 0.29045
- type: ndcg_at_5
value: 0.31121
- type: precision_at_1
value: 0.24765
- type: precision_at_10
value: 0.05599
- type: precision_at_100
value: 0.0087
- type: precision_at_1000
value: 0.00115
- type: precision_at_3
value: 0.13271
- type: precision_at_5
value: 0.09367
- type: recall_at_1
value: 0.21183
- type: recall_at_10
value: 0.43875
- type: recall_at_100
value: 0.65005
- type: recall_at_1000
value: 0.83017
- type: recall_at_3
value: 0.32232
- type: recall_at_5
value: 0.37308
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
metrics:
- type: map_at_1
value: 0.03637
- type: map_at_10
value: 0.06084
- type: map_at_100
value: 0.06919
- type: map_at_1000
value: 0.07108
- type: map_at_3
value: 0.05071
- type: map_at_5
value: 0.05565
- type: ndcg_at_1
value: 0.07407
- type: ndcg_at_10
value: 0.0894
- type: ndcg_at_100
value: 0.13595
- type: ndcg_at_1000
value: 0.1829
- type: ndcg_at_3
value: 0.07393
- type: ndcg_at_5
value: 0.07854
- type: precision_at_1
value: 0.07407
- type: precision_at_10
value: 0.02778
- type: precision_at_100
value: 0.0075
- type: precision_at_1000
value: 0.00154
- type: precision_at_3
value: 0.05144
- type: precision_at_5
value: 0.03981
- type: recall_at_1
value: 0.03637
- type: recall_at_10
value: 0.11821
- type: recall_at_100
value: 0.3018
- type: recall_at_1000
value: 0.60207
- type: recall_at_3
value: 0.06839
- type: recall_at_5
value: 0.08649
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (af)
metrics:
- type: accuracy
value: 0.3779421654337593
- type: f1
value: 0.3681580701507746
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (am)
metrics:
- type: accuracy
value: 0.23722259583053126
- type: f1
value: 0.23235269695764274
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ar)
metrics:
- type: accuracy
value: 0.2964021519838601
- type: f1
value: 0.28273175327650135
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (az)
metrics:
- type: accuracy
value: 0.39475453934095495
- type: f1
value: 0.39259973614151206
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (bn)
metrics:
- type: accuracy
value: 0.26550100874243443
- type: f1
value: 0.25607924873522975
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (cy)
metrics:
- type: accuracy
value: 0.38782784129119036
- type: f1
value: 0.3764180582626517
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (da)
metrics:
- type: accuracy
value: 0.43557498318762605
- type: f1
value: 0.4135305173800667
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (de)
metrics:
- type: accuracy
value: 0.4039340954942838
- type: f1
value: 0.38333932195289344
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (el)
metrics:
- type: accuracy
value: 0.3728648285137861
- type: f1
value: 0.36640059066802844
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (en)
metrics:
- type: accuracy
value: 0.5808002689979825
- type: f1
value: 0.5649243881660991
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (es)
metrics:
- type: accuracy
value: 0.411768661735037
- type: f1
value: 0.4066779962225799
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fa)
metrics:
- type: accuracy
value: 0.36422326832548757
- type: f1
value: 0.34644173804288503
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fi)
metrics:
- type: accuracy
value: 0.3875588433086752
- type: f1
value: 0.3726725894668694
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fr)
metrics:
- type: accuracy
value: 0.43671822461331533
- type: f1
value: 0.423518466245666
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (he)
metrics:
- type: accuracy
value: 0.3198049764626766
- type: f1
value: 0.3055792887280901
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hi)
metrics:
- type: accuracy
value: 0.2803967720242098
- type: f1
value: 0.28428418145508305
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hu)
metrics:
- type: accuracy
value: 0.3813718897108272
- type: f1
value: 0.3705740698819687
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hy)
metrics:
- type: accuracy
value: 0.2605245460659045
- type: f1
value: 0.2525483953344816
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (id)
metrics:
- type: accuracy
value: 0.41156691324815065
- type: f1
value: 0.40837150332476047
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (is)
metrics:
- type: accuracy
value: 0.38628110289172835
- type: f1
value: 0.37676919012460314
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (it)
metrics:
- type: accuracy
value: 0.440383322125084
- type: f1
value: 0.43772590108774556
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ja)
metrics:
- type: accuracy
value: 0.46207128446536655
- type: f1
value: 0.44666328759408236
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (jv)
metrics:
- type: accuracy
value: 0.3760591795561533
- type: f1
value: 0.36581071742378013
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ka)
metrics:
- type: accuracy
value: 0.24472091459314052
- type: f1
value: 0.24238209697895607
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (km)
metrics:
- type: accuracy
value: 0.2623739071956961
- type: f1
value: 0.2537878315084505
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (kn)
metrics:
- type: accuracy
value: 0.17831203765971754
- type: f1
value: 0.17275078420466344
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ko)
metrics:
- type: accuracy
value: 0.37266308002689974
- type: f1
value: 0.3692473791708214
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (lv)
metrics:
- type: accuracy
value: 0.4093140551445864
- type: f1
value: 0.4082522788964197
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ml)
metrics:
- type: accuracy
value: 0.1788500336247478
- type: f1
value: 0.17621569082971816
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (mn)
metrics:
- type: accuracy
value: 0.3297579018157364
- type: f1
value: 0.33402014633349664
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ms)
metrics:
- type: accuracy
value: 0.40911230665770015
- type: f1
value: 0.4009538559124075
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (my)
metrics:
- type: accuracy
value: 0.17834566240753194
- type: f1
value: 0.17006381849454313
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nb)
metrics:
- type: accuracy
value: 0.3947881640887693
- type: f1
value: 0.37819934317839304
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nl)
metrics:
- type: accuracy
value: 0.4176193678547412
- type: f1
value: 0.40281991759509694
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pl)
metrics:
- type: accuracy
value: 0.4261936785474109
- type: f1
value: 0.40836739146499046
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pt)
metrics:
- type: accuracy
value: 0.44542703429724273
- type: f1
value: 0.43452431642784484
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ro)
metrics:
- type: accuracy
value: 0.3996973772696705
- type: f1
value: 0.3874209466530094
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ru)
metrics:
- type: accuracy
value: 0.37461331540013454
- type: f1
value: 0.3691132021821187
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sl)
metrics:
- type: accuracy
value: 0.3828850033624748
- type: f1
value: 0.3737259394049676
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sq)
metrics:
- type: accuracy
value: 0.4095494283792872
- type: f1
value: 0.3976770790286908
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sv)
metrics:
- type: accuracy
value: 0.4185272360457296
- type: f1
value: 0.4042848260365438
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sw)
metrics:
- type: accuracy
value: 0.3832885003362475
- type: f1
value: 0.3690334596675622
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ta)
metrics:
- type: accuracy
value: 0.19031607262945527
- type: f1
value: 0.18665103063257613
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (te)
metrics:
- type: accuracy
value: 0.1938466711499664
- type: f1
value: 0.19186399376652535
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (th)
metrics:
- type: accuracy
value: 0.34088769334229996
- type: f1
value: 0.3420383086009429
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tl)
metrics:
- type: accuracy
value: 0.40285810356422325
- type: f1
value: 0.39361500249640413
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tr)
metrics:
- type: accuracy
value: 0.38860121049092133
- type: f1
value: 0.3781916859627235
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ur)
metrics:
- type: accuracy
value: 0.27834566240753195
- type: f1
value: 0.26898389386106486
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (vi)
metrics:
- type: accuracy
value: 0.38705447209145927
- type: f1
value: 0.3828002644202441
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
metrics:
- type: accuracy
value: 0.45780094149293876
- type: f1
value: 0.4421526778674136
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-TW)
metrics:
- type: accuracy
value: 0.4232010759919301
- type: f1
value: 0.4225772977490916
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
metrics:
- type: accuracy
value: 0.74938225
- type: ap
value: 0.6958187110320567
- type: f1
value: 0.7472744058439321
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
metrics:
- type: map_at_1
value: 0.01764
- type: map_at_10
value: 0.0386
- type: map_at_100
value: 0.05457
- type: map_at_1000
value: 0.05938
- type: map_at_3
value: 0.02667
- type: map_at_5
value: 0.0322
- type: ndcg_at_1
value: 0.14
- type: ndcg_at_10
value: 0.10868
- type: ndcg_at_100
value: 0.12866
- type: ndcg_at_1000
value: 0.1743
- type: ndcg_at_3
value: 0.11943
- type: ndcg_at_5
value: 0.1166
- type: precision_at_1
value: 0.1925
- type: precision_at_10
value: 0.10275
- type: precision_at_100
value: 0.03527
- type: precision_at_1000
value: 0.00912
- type: precision_at_3
value: 0.14917
- type: precision_at_5
value: 0.135
- type: recall_at_1
value: 0.01764
- type: recall_at_10
value: 0.06609
- type: recall_at_100
value: 0.17616
- type: recall_at_1000
value: 0.33085
- type: recall_at_3
value: 0.03115
- type: recall_at_5
value: 0.04605
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
metrics:
- type: map_at_1
value: 0.11497
- type: map_at_10
value: 0.15744
- type: map_at_100
value: 0.163
- type: map_at_1000
value: 0.16365
- type: map_at_3
value: 0.1444
- type: map_at_5
value: 0.1518
- type: ndcg_at_1
value: 0.12346
- type: ndcg_at_10
value: 0.18399
- type: ndcg_at_100
value: 0.21399
- type: ndcg_at_1000
value: 0.23442
- type: ndcg_at_3
value: 0.15695
- type: ndcg_at_5
value: 0.17027
- type: precision_at_1
value: 0.12346
- type: precision_at_10
value: 0.02798
- type: precision_at_100
value: 0.00445
- type: precision_at_1000
value: 0.00063
- type: precision_at_3
value: 0.06586
- type: precision_at_5
value: 0.04665
- type: recall_at_1
value: 0.11497
- type: recall_at_10
value: 0.25636
- type: recall_at_100
value: 0.39894
- type: recall_at_1000
value: 0.56181
- type: recall_at_3
value: 0.18273
- type: recall_at_5
value: 0.21474
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
metrics:
- type: map_at_1
value: 0.12598
- type: map_at_10
value: 0.17304
- type: map_at_100
value: 0.18209
- type: map_at_1000
value: 0.18328
- type: map_at_3
value: 0.15784
- type: map_at_5
value: 0.1667
- type: ndcg_at_1
value: 0.15868
- type: ndcg_at_10
value: 0.20623
- type: ndcg_at_100
value: 0.25093
- type: ndcg_at_1000
value: 0.28498
- type: ndcg_at_3
value: 0.17912
- type: ndcg_at_5
value: 0.19198
- type: precision_at_1
value: 0.15868
- type: precision_at_10
value: 0.03767
- type: precision_at_100
value: 0.00716
- type: precision_at_1000
value: 0.00118
- type: precision_at_3
value: 0.08638
- type: precision_at_5
value: 0.0621
- type: recall_at_1
value: 0.12598
- type: recall_at_10
value: 0.27144
- type: recall_at_100
value: 0.46817
- type: recall_at_1000
value: 0.71861
- type: recall_at_3
value: 0.19231
- type: recall_at_5
value: 0.22716
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
metrics:
- type: cos_sim_pearson
value: 0.5917638344661753
- type: cos_sim_spearman
value: 0.5963676007113087
- type: euclidean_pearson
value: 0.5668753290255448
- type: euclidean_spearman
value: 0.5761328025857448
- type: manhattan_pearson
value: 0.5692312052723706
- type: manhattan_spearman
value: 0.5776774918418505
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de)
metrics:
- type: cos_sim_pearson
value: 0.10322254716987457
- type: cos_sim_spearman
value: 0.110033092996862
- type: euclidean_pearson
value: 0.06006926471684402
- type: euclidean_spearman
value: 0.10972140246688376
- type: manhattan_pearson
value: 0.05933298751861177
- type: manhattan_spearman
value: 0.11030111585680233
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es)
metrics:
- type: cos_sim_pearson
value: 0.4338031880545056
- type: cos_sim_spearman
value: 0.4305358201410913
- type: euclidean_pearson
value: 0.42723271963625525
- type: euclidean_spearman
value: 0.4255163899944477
- type: manhattan_pearson
value: 0.44015574997805873
- type: manhattan_spearman
value: 0.43124732216158546
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl)
metrics:
- type: cos_sim_pearson
value: 0.042912905043631364
- type: cos_sim_spearman
value: 0.1491272748789348
- type: euclidean_pearson
value: 0.032855132112394485
- type: euclidean_spearman
value: 0.16575204463951024
- type: manhattan_pearson
value: 0.03239877672346581
- type: manhattan_spearman
value: 0.16841985772913856
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (tr)
metrics:
- type: cos_sim_pearson
value: 0.041027394985558165
- type: cos_sim_spearman
value: 0.03818238576547375
- type: euclidean_pearson
value: 0.023181033496453556
- type: euclidean_spearman
value: 0.051826811802703564
- type: manhattan_pearson
value: 0.04800617926525645
- type: manhattan_spearman
value: 0.06738401400306251
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ar)
metrics:
- type: cos_sim_pearson
value: 0.0238765395226737
- type: cos_sim_spearman
value: 0.051738993911623274
- type: euclidean_pearson
value: 0.030710263954769824
- type: euclidean_spearman
value: 0.050492229090398195
- type: manhattan_pearson
value: 0.0378263141098617
- type: manhattan_spearman
value: 0.05042238232170212
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ru)
metrics:
- type: cos_sim_pearson
value: 0.07673549067267635
- type: cos_sim_spearman
value: 0.03363121525687889
- type: euclidean_pearson
value: 0.0464331702652217
- type: euclidean_spearman
value: 0.036129205171334326
- type: manhattan_pearson
value: 0.040112317360761963
- type: manhattan_spearman
value: 0.03233959766173701
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
metrics:
- type: cos_sim_pearson
value: 0.0006167614416104335
- type: cos_sim_spearman
value: 0.06521685391703255
- type: euclidean_pearson
value: 0.048845725790690325
- type: euclidean_spearman
value: 0.0559058032900239
- type: manhattan_pearson
value: 0.06139838096573896
- type: manhattan_spearman
value: 0.050060884837066215
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr)
metrics:
- type: cos_sim_pearson
value: 0.5319490347682836
- type: cos_sim_spearman
value: 0.5456055727079527
- type: euclidean_pearson
value: 0.5255574442039842
- type: euclidean_spearman
value: 0.5294640154371587
- type: manhattan_pearson
value: 0.532759930404542
- type: manhattan_spearman
value: 0.5317456150351015
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-en)
metrics:
- type: cos_sim_pearson
value: 0.5115115853012214
- type: cos_sim_spearman
value: 0.5392692508173665
- type: euclidean_pearson
value: 0.4455629287737235
- type: euclidean_spearman
value: 0.46222372143731383
- type: manhattan_pearson
value: 0.42831322151459006
- type: manhattan_spearman
value: 0.4570991764985799
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-en)
metrics:
- type: cos_sim_pearson
value: 0.30361948851267917
- type: cos_sim_spearman
value: 0.32739632941633834
- type: euclidean_pearson
value: 0.2983135800843496
- type: euclidean_spearman
value: 0.3111440600132692
- type: manhattan_pearson
value: 0.31264502938148286
- type: manhattan_spearman
value: 0.33311204075347495
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (it)
metrics:
- type: cos_sim_pearson
value: 0.3523883630335275
- type: cos_sim_spearman
value: 0.33677970820867037
- type: euclidean_pearson
value: 0.34878640693874546
- type: euclidean_spearman
value: 0.33525189235133496
- type: manhattan_pearson
value: 0.3422761246389947
- type: manhattan_spearman
value: 0.32713218497609176
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl-en)
metrics:
- type: cos_sim_pearson
value: 0.19809302548119545
- type: cos_sim_spearman
value: 0.205403702021155
- type: euclidean_pearson
value: 0.23006803962133016
- type: euclidean_spearman
value: 0.2296270653079511
- type: manhattan_pearson
value: 0.2540168317585851
- type: manhattan_spearman
value: 0.25421508137540866
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh-en)
metrics:
- type: cos_sim_pearson
value: 0.20393500955410487
- type: cos_sim_spearman
value: 0.267057136930116
- type: euclidean_pearson
value: 0.18168376767724584
- type: euclidean_spearman
value: 0.19260826601517245
- type: manhattan_pearson
value: 0.18302619990671526
- type: manhattan_spearman
value: 0.194691037846159
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-it)
metrics:
- type: cos_sim_pearson
value: 0.36589199830751484
- type: cos_sim_spearman
value: 0.3598972209997404
- type: euclidean_pearson
value: 0.4104511254757421
- type: euclidean_spearman
value: 0.39322301680629834
- type: manhattan_pearson
value: 0.4136802503205308
- type: manhattan_spearman
value: 0.4076270030293609
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-fr)
metrics:
- type: cos_sim_pearson
value: 0.26350936227950084
- type: cos_sim_spearman
value: 0.25108218032460344
- type: euclidean_pearson
value: 0.2861681094744849
- type: euclidean_spearman
value: 0.2735099020394359
- type: manhattan_pearson
value: 0.30527977072984513
- type: manhattan_spearman
value: 0.2640333999064081
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-pl)
metrics:
- type: cos_sim_pearson
value: 0.20056269198600324
- type: cos_sim_spearman
value: 0.20939990379746756
- type: euclidean_pearson
value: 0.18942765438962197
- type: euclidean_spearman
value: 0.21709842967237447
- type: manhattan_pearson
value: 0.23643909798655122
- type: manhattan_spearman
value: 0.2358828328071473
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr-pl)
metrics:
- type: cos_sim_pearson
value: 0.19563740271419394
- type: cos_sim_spearman
value: 0.05634361698190111
- type: euclidean_pearson
value: 0.16833522619239474
- type: euclidean_spearman
value: 0.16903085094570333
- type: manhattan_pearson
value: 0.058053927126608146
- type: manhattan_spearman
value: 0.16903085094570333
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (af)
metrics:
- type: accuracy
value: 0.40245460659045057
- type: f1
value: 0.3879924050989544
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (am)
metrics:
- type: accuracy
value: 0.2568930733019502
- type: f1
value: 0.2548816627916271
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ar)
metrics:
- type: accuracy
value: 0.3239744451916611
- type: f1
value: 0.31863029579075774
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (az)
metrics:
- type: accuracy
value: 0.4053127101546738
- type: f1
value: 0.39707079033948933
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (bn)
metrics:
- type: accuracy
value: 0.2723268325487559
- type: f1
value: 0.2644365328185879
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (cy)
metrics:
- type: accuracy
value: 0.3869872225958305
- type: f1
value: 0.3655930387892567
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (da)
metrics:
- type: accuracy
value: 0.4475453934095494
- type: f1
value: 0.4287356484024154
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (de)
metrics:
- type: accuracy
value: 0.41355077336919976
- type: f1
value: 0.3982365179458047
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (el)
metrics:
- type: accuracy
value: 0.3843981170141224
- type: f1
value: 0.3702538368296387
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (en)
metrics:
- type: accuracy
value: 0.6633826496301277
- type: f1
value: 0.6589634765029931
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (es)
metrics:
- type: accuracy
value: 0.4417955615332885
- type: f1
value: 0.4310228811620319
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fa)
metrics:
- type: accuracy
value: 0.3482851378614661
- type: f1
value: 0.33959524415028025
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fi)
metrics:
- type: accuracy
value: 0.40561533288500334
- type: f1
value: 0.38049390117336274
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fr)
metrics:
- type: accuracy
value: 0.45917955615332884
- type: f1
value: 0.4465741971572902
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (he)
metrics:
- type: accuracy
value: 0.3208473436449227
- type: f1
value: 0.2953932929808133
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hi)
metrics:
- type: accuracy
value: 0.28369199731002015
- type: f1
value: 0.2752902837981212
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hu)
metrics:
- type: accuracy
value: 0.3949226630800269
- type: f1
value: 0.37327234047050406
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hy)
metrics:
- type: accuracy
value: 0.2590450571620713
- type: f1
value: 0.24547396574853445
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (id)
metrics:
- type: accuracy
value: 0.4095830531271016
- type: f1
value: 0.40177843177422223
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (is)
metrics:
- type: accuracy
value: 0.38564223268325487
- type: f1
value: 0.3735307758495248
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (it)
metrics:
- type: accuracy
value: 0.4658708809683928
- type: f1
value: 0.44103900526804984
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ja)
metrics:
- type: accuracy
value: 0.4624747814391393
- type: f1
value: 0.454107101796664
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (jv)
metrics:
- type: accuracy
value: 0.396570275722932
- type: f1
value: 0.3882737576832412
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ka)
metrics:
- type: accuracy
value: 0.2527908540685945
- type: f1
value: 0.23662661686788491
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (km)
metrics:
- type: accuracy
value: 0.2897108271687962
- type: f1
value: 0.27195758324189245
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (kn)
metrics:
- type: accuracy
value: 0.1927370544720915
- type: f1
value: 0.18694271924323635
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ko)
metrics:
- type: accuracy
value: 0.3572965702757229
- type: f1
value: 0.3438287006177308
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (lv)
metrics:
- type: accuracy
value: 0.3957296570275723
- type: f1
value: 0.38074945140886923
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ml)
metrics:
- type: accuracy
value: 0.19895763281775386
- type: f1
value: 0.20009313648468288
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (mn)
metrics:
- type: accuracy
value: 0.32431069266980495
- type: f1
value: 0.31395958664782575
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ms)
metrics:
- type: accuracy
value: 0.42323470073974445
- type: f1
value: 0.4081374026314701
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (my)
metrics:
- type: accuracy
value: 0.20864156018829857
- type: f1
value: 0.20409870408935435
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nb)
metrics:
- type: accuracy
value: 0.4047074646940148
- type: f1
value: 0.3919044149415904
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nl)
metrics:
- type: accuracy
value: 0.43591123066577
- type: f1
value: 0.4143420363064241
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pl)
metrics:
- type: accuracy
value: 0.41876260928043046
- type: f1
value: 0.4119211767666761
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pt)
metrics:
- type: accuracy
value: 0.46308002689979827
- type: f1
value: 0.4525536730126799
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ro)
metrics:
- type: accuracy
value: 0.4252521856086079
- type: f1
value: 0.4102418109296485
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ru)
metrics:
- type: accuracy
value: 0.3594821788836584
- type: f1
value: 0.3508598314806566
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sl)
metrics:
- type: accuracy
value: 0.3869199731002017
- type: f1
value: 0.3768119408674127
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sq)
metrics:
- type: accuracy
value: 0.4047410894418292
- type: f1
value: 0.39480530387013596
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sv)
metrics:
- type: accuracy
value: 0.41523201075991933
- type: f1
value: 0.40200979960243827
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sw)
metrics:
- type: accuracy
value: 0.39549428379287155
- type: f1
value: 0.3818556124333806
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ta)
metrics:
- type: accuracy
value: 0.228782784129119
- type: f1
value: 0.22239467186721457
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (te)
metrics:
- type: accuracy
value: 0.2051445864156019
- type: f1
value: 0.1999904788553022
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (th)
metrics:
- type: accuracy
value: 0.34926025554808343
- type: f1
value: 0.33240167172157226
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tl)
metrics:
- type: accuracy
value: 0.4074983187626093
- type: f1
value: 0.3930274328728882
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tr)
metrics:
- type: accuracy
value: 0.3906859448554136
- type: f1
value: 0.39215420396629713
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ur)
metrics:
- type: accuracy
value: 0.29747814391392063
- type: f1
value: 0.2826183689222045
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (vi)
metrics:
- type: accuracy
value: 0.3802286482851379
- type: f1
value: 0.37874243860869694
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
metrics:
- type: accuracy
value: 0.48550773369199723
- type: f1
value: 0.46739962588264905
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-TW)
metrics:
- type: accuracy
value: 0.45178211163416276
- type: f1
value: 0.4484809741811729
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
metrics:
- type: map_at_1
value: 0.61697
- type: map_at_10
value: 0.74204
- type: map_at_100
value: 0.75023
- type: map_at_1000
value: 0.75059
- type: map_at_3
value: 0.71265
- type: map_at_5
value: 0.73001
- type: ndcg_at_1
value: 0.7095
- type: ndcg_at_10
value: 0.7896
- type: ndcg_at_100
value: 0.8126
- type: ndcg_at_1000
value: 0.81679
- type: ndcg_at_3
value: 0.75246
- type: ndcg_at_5
value: 0.77092
- type: precision_at_1
value: 0.7095
- type: precision_at_10
value: 0.11998
- type: precision_at_100
value: 0.01451
- type: precision_at_1000
value: 0.00154
- type: precision_at_3
value: 0.3263
- type: precision_at_5
value: 0.21574
- type: recall_at_1
value: 0.61697
- type: recall_at_10
value: 0.88233
- type: recall_at_100
value: 0.96961
- type: recall_at_1000
value: 0.99401
- type: recall_at_3
value: 0.77689
- type: recall_at_5
value: 0.82745
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
metrics:
- type: cos_sim_pearson
value: 0.8096286245858941
- type: cos_sim_spearman
value: 0.7457093488947429
- type: euclidean_pearson
value: 0.7550377970259401
- type: euclidean_spearman
value: 0.7174980046229991
- type: manhattan_pearson
value: 0.7532568360913819
- type: manhattan_spearman
value: 0.7180676733410375
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
metrics:
- type: cos_sim_accuracy
value: 0.8663018589668956
- type: cos_sim_accuracy_threshold
value: 0.6738145351409912
- type: cos_sim_ap
value: 0.805106377126291
- type: cos_sim_f1
value: 0.7270810586950793
- type: cos_sim_f1_threshold
value: 0.6406128406524658
- type: cos_sim_precision
value: 0.7114123627790466
- type: cos_sim_recall
value: 0.743455497382199
- type: dot_accuracy
value: 0.8241743315092949
- type: dot_accuracy_threshold
value: 967.1823120117188
- type: dot_ap
value: 0.692393381283664
- type: dot_f1
value: 0.6561346624814597
- type: dot_f1_threshold
value: 831.1060791015625
- type: dot_precision
value: 0.5943260638630257
- type: dot_recall
value: 0.7322913458577148
- type: euclidean_accuracy
value: 0.8649435324251951
- type: euclidean_accuracy_threshold
value: 30.077878952026367
- type: euclidean_ap
value: 0.8028100477250927
- type: euclidean_f1
value: 0.7258242344489099
- type: euclidean_f1_threshold
value: 32.570228576660156
- type: euclidean_precision
value: 0.6744662568576906
- type: euclidean_recall
value: 0.7856482907299045
- type: manhattan_accuracy
value: 0.8659525749990298
- type: manhattan_accuracy_threshold
value: 625.0921020507812
- type: manhattan_ap
value: 0.8037850832566262
- type: manhattan_f1
value: 0.7259435321233073
- type: manhattan_f1_threshold
value: 679.8679809570312
- type: manhattan_precision
value: 0.6819350473612991
- type: manhattan_recall
value: 0.7760240221743148
- type: max_accuracy
value: 0.8663018589668956
- type: max_ap
value: 0.805106377126291
- type: max_f1
value: 0.7270810586950793
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
metrics:
- type: v_measure
value: 0.23080939123955474
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ko-ko)
metrics:
- type: cos_sim_pearson
value: 0.430464619152799
- type: cos_sim_spearman
value: 0.4565606588928089
- type: euclidean_pearson
value: 0.45694377883554993
- type: euclidean_spearman
value: 0.4508552742346606
- type: manhattan_pearson
value: 0.45871666989036813
- type: manhattan_spearman
value: 0.45155963016434164
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ar-ar)
metrics:
- type: cos_sim_pearson
value: 0.5327469278912148
- type: cos_sim_spearman
value: 0.541611320762379
- type: euclidean_pearson
value: 0.5597026429327157
- type: euclidean_spearman
value: 0.5471320909074608
- type: manhattan_pearson
value: 0.5612511774278802
- type: manhattan_spearman
value: 0.5522875659158676
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-ar)
metrics:
- type: cos_sim_pearson
value: 0.015482997790039945
- type: cos_sim_spearman
value: 0.01720838634736358
- type: euclidean_pearson
value: 0.06727915670345885
- type: euclidean_spearman
value: 0.06112826908474543
- type: manhattan_pearson
value: 0.0494386093060865
- type: manhattan_spearman
value: 0.05018174110623732
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-de)
metrics:
- type: cos_sim_pearson
value: 0.275420218362265
- type: cos_sim_spearman
value: 0.2548383843103101
- type: euclidean_pearson
value: 0.06268684143856358
- type: euclidean_spearman
value: 0.058779614210916785
- type: manhattan_pearson
value: 0.026672377392278606
- type: manhattan_spearman
value: 0.025683839956554773
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
metrics:
- type: cos_sim_pearson
value: 0.8532029757646663
- type: cos_sim_spearman
value: 0.8732720847297224
- type: euclidean_pearson
value: 0.8112594485791255
- type: euclidean_spearman
value: 0.811531079489332
- type: manhattan_pearson
value: 0.8132899414704019
- type: manhattan_spearman
value: 0.813897040261192
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-tr)
metrics:
- type: cos_sim_pearson
value: 0.0437162299241808
- type: cos_sim_spearman
value: 0.020879072561774542
- type: euclidean_pearson
value: 0.030725243785454597
- type: euclidean_spearman
value: 0.05372133927948353
- type: manhattan_pearson
value: 0.04867795293367359
- type: manhattan_spearman
value: 0.07939706984001878
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-en)
metrics:
- type: cos_sim_pearson
value: 0.20306030448858603
- type: cos_sim_spearman
value: 0.2193220782551375
- type: euclidean_pearson
value: 0.03878631934602361
- type: euclidean_spearman
value: 0.05171796902725965
- type: manhattan_pearson
value: 0.0713020644036815
- type: manhattan_spearman
value: 0.07707315591498748
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-es)
metrics:
- type: cos_sim_pearson
value: 0.6681873207478459
- type: cos_sim_spearman
value: 0.6780273445636502
- type: euclidean_pearson
value: 0.7060654682977268
- type: euclidean_spearman
value: 0.694566208379486
- type: manhattan_pearson
value: 0.7095484618966419
- type: manhattan_spearman
value: 0.6978323323058773
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (fr-en)
metrics:
- type: cos_sim_pearson
value: 0.21366487281202604
- type: cos_sim_spearman
value: 0.18906275286984808
- type: euclidean_pearson
value: 0.023390998579461995
- type: euclidean_spearman
value: 0.04151213674012541
- type: manhattan_pearson
value: 0.02234831868844863
- type: manhattan_spearman
value: 0.045552913285014415
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (it-en)
metrics:
- type: cos_sim_pearson
value: 0.20731531772510847
- type: cos_sim_spearman
value: 0.163855949033176
- type: euclidean_pearson
value: 0.08734648741714238
- type: euclidean_spearman
value: 0.1075672244732182
- type: manhattan_pearson
value: 0.07536654126608877
- type: manhattan_spearman
value: 0.08330065460047295
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (nl-en)
metrics:
- type: cos_sim_pearson
value: 0.2661843502408425
- type: cos_sim_spearman
value: 0.23488974089577816
- type: euclidean_pearson
value: 0.031310350304707864
- type: euclidean_spearman
value: 0.031242598481634666
- type: manhattan_pearson
value: 0.011096752982707007
- type: manhattan_spearman
value: 0.014591693078765849
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
metrics:
- type: map_at_1
value: 0.00113
- type: map_at_10
value: 0.00733
- type: map_at_100
value: 0.03313
- type: map_at_1000
value: 0.07355
- type: map_at_3
value: 0.00282
- type: map_at_5
value: 0.00414
- type: ndcg_at_1
value: 0.42
- type: ndcg_at_10
value: 0.3931
- type: ndcg_at_100
value: 0.26904
- type: ndcg_at_1000
value: 0.23778
- type: ndcg_at_3
value: 0.42776
- type: ndcg_at_5
value: 0.41554
- type: precision_at_1
value: 0.48
- type: precision_at_10
value: 0.43
- type: precision_at_100
value: 0.2708
- type: precision_at_1000
value: 0.11014
- type: precision_at_3
value: 0.48
- type: precision_at_5
value: 0.456
- type: recall_at_1
value: 0.00113
- type: recall_at_10
value: 0.00976
- type: recall_at_100
value: 0.05888
- type: recall_at_1000
value: 0.22635
- type: recall_at_3
value: 0.00329
- type: recall_at_5
value: 0.00518
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
metrics:
- type: map_at_1
value: 0.21556
- type: map_at_10
value: 0.27982
- type: map_at_100
value: 0.28937
- type: map_at_1000
value: 0.29058
- type: map_at_3
value: 0.25644
- type: map_at_5
value: 0.26996
- type: ndcg_at_1
value: 0.23333
- type: ndcg_at_10
value: 0.31787
- type: ndcg_at_100
value: 0.36648
- type: ndcg_at_1000
value: 0.39936
- type: ndcg_at_3
value: 0.27299
- type: ndcg_at_5
value: 0.29659
- type: precision_at_1
value: 0.23333
- type: precision_at_10
value: 0.04867
- type: precision_at_100
value: 0.00743
- type: precision_at_1000
value: 0.00102
- type: precision_at_3
value: 0.11333
- type: precision_at_5
value: 0.08133
- type: recall_at_1
value: 0.21556
- type: recall_at_10
value: 0.42333
- type: recall_at_100
value: 0.65706
- type: recall_at_1000
value: 0.91489
- type: recall_at_3
value: 0.30361
- type: recall_at_5
value: 0.36222
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
metrics:
- type: map_at_1
value: 0.0172
- type: map_at_10
value: 0.03824
- type: map_at_100
value: 0.04727
- type: map_at_1000
value: 0.04932
- type: map_at_3
value: 0.02867
- type: map_at_5
value: 0.03323
- type: ndcg_at_1
value: 0.085
- type: ndcg_at_10
value: 0.07133
- type: ndcg_at_100
value: 0.11911
- type: ndcg_at_1000
value: 0.16962
- type: ndcg_at_3
value: 0.06763
- type: ndcg_at_5
value: 0.05832
- type: precision_at_1
value: 0.085
- type: precision_at_10
value: 0.0368
- type: precision_at_100
value: 0.01067
- type: precision_at_1000
value: 0.0023
- type: precision_at_3
value: 0.06233
- type: precision_at_5
value: 0.0502
- type: recall_at_1
value: 0.0172
- type: recall_at_10
value: 0.07487
- type: recall_at_100
value: 0.21683
- type: recall_at_1000
value: 0.46688
- type: recall_at_3
value: 0.03798
- type: recall_at_5
value: 0.05113
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
metrics:
- type: map_at_1
value: 0.03515
- type: map_at_10
value: 0.05884
- type: map_at_100
value: 0.0651
- type: map_at_1000
value: 0.06599
- type: map_at_3
value: 0.04892
- type: map_at_5
value: 0.05391
- type: ndcg_at_1
value: 0.04056
- type: ndcg_at_10
value: 0.07626
- type: ndcg_at_100
value: 0.1108
- type: ndcg_at_1000
value: 0.13793
- type: ndcg_at_3
value: 0.05537
- type: ndcg_at_5
value: 0.0645
- type: precision_at_1
value: 0.04056
- type: precision_at_10
value: 0.01457
- type: precision_at_100
value: 0.00347
- type: precision_at_1000
value: 0.00061
- type: precision_at_3
value: 0.02607
- type: precision_at_5
value: 0.02086
- type: recall_at_1
value: 0.03515
- type: recall_at_10
value: 0.12312
- type: recall_at_100
value: 0.28713
- type: recall_at_1000
value: 0.50027
- type: recall_at_3
value: 0.06701
- type: recall_at_5
value: 0.08816
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
metrics:
- type: cos_sim_pearson
value: 0.7604750373932828
- type: cos_sim_spearman
value: 0.7793230986462234
- type: euclidean_pearson
value: 0.758320302521164
- type: euclidean_spearman
value: 0.7683154481579385
- type: manhattan_pearson
value: 0.7598713517720608
- type: manhattan_spearman
value: 0.7695479705521506
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
metrics:
- type: accuracy
value: 0.42225
- type: f1
value: 0.3756351654211211
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
metrics:
- type: map_at_1
value: 0.13757
- type: map_at_10
value: 0.1927
- type: map_at_100
value: 0.20461
- type: map_at_1000
value: 0.20641
- type: map_at_3
value: 0.17865
- type: map_at_5
value: 0.18618
- type: ndcg_at_1
value: 0.16996
- type: ndcg_at_10
value: 0.22774
- type: ndcg_at_100
value: 0.27675
- type: ndcg_at_1000
value: 0.31145
- type: ndcg_at_3
value: 0.20691
- type: ndcg_at_5
value: 0.21741
- type: precision_at_1
value: 0.16996
- type: precision_at_10
value: 0.04545
- type: precision_at_100
value: 0.01036
- type: precision_at_1000
value: 0.00185
- type: precision_at_3
value: 0.10145
- type: precision_at_5
value: 0.07391
- type: recall_at_1
value: 0.13757
- type: recall_at_10
value: 0.28234
- type: recall_at_100
value: 0.51055
- type: recall_at_1000
value: 0.75353
- type: recall_at_3
value: 0.21794
- type: recall_at_5
value: 0.24614
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
metrics:
- type: v_measure
value: 0.41007999100992665
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
metrics:
- type: map_at_1
value: 0.11351
- type: map_at_10
value: 0.14953
- type: map_at_100
value: 0.15623
- type: map_at_1000
value: 0.15716
- type: map_at_3
value: 0.13603
- type: map_at_5
value: 0.14343
- type: ndcg_at_1
value: 0.12429
- type: ndcg_at_10
value: 0.17319
- type: ndcg_at_100
value: 0.2099
- type: ndcg_at_1000
value: 0.23899
- type: ndcg_at_3
value: 0.14605
- type: ndcg_at_5
value: 0.1589
- type: precision_at_1
value: 0.12429
- type: precision_at_10
value: 0.02701
- type: precision_at_100
value: 0.00487
- type: precision_at_1000
value: 0.00078
- type: precision_at_3
value: 0.06026
- type: precision_at_5
value: 0.04384
- type: recall_at_1
value: 0.11351
- type: recall_at_10
value: 0.23536
- type: recall_at_100
value: 0.40942
- type: recall_at_1000
value: 0.6405
- type: recall_at_3
value: 0.16195
- type: recall_at_5
value: 0.19264
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
metrics:
- type: cos_sim_pearson
value: 0.8000905671833967
- type: cos_sim_spearman
value: 0.7954269211027273
- type: euclidean_pearson
value: 0.7951954544247442
- type: euclidean_spearman
value: 0.7893670303434288
- type: manhattan_pearson
value: 0.7947610653340678
- type: manhattan_spearman
value: 0.7907344156719612
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
metrics:
- type: accuracy
value: 0.7467857142857142
- type: f1
value: 0.7461743413995573
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
metrics:
- type: map_at_1
value: 0.12307
- type: map_at_10
value: 0.1544
- type: map_at_100
value: 0.16033
- type: map_at_1000
value: 0.1614
- type: map_at_3
value: 0.14393
- type: map_at_5
value: 0.14856
- type: ndcg_at_1
value: 0.14571
- type: ndcg_at_10
value: 0.17685
- type: ndcg_at_100
value: 0.20882
- type: ndcg_at_1000
value: 0.23888
- type: ndcg_at_3
value: 0.15739
- type: ndcg_at_5
value: 0.16391
- type: precision_at_1
value: 0.14571
- type: precision_at_10
value: 0.02883
- type: precision_at_100
value: 0.00491
- type: precision_at_1000
value: 0.0008
- type: precision_at_3
value: 0.07004
- type: precision_at_5
value: 0.04693
- type: recall_at_1
value: 0.12307
- type: recall_at_10
value: 0.22566
- type: recall_at_100
value: 0.37469
- type: recall_at_1000
value: 0.6055
- type: recall_at_3
value: 0.16742
- type: recall_at_5
value: 0.18634
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
metrics:
- type: cos_sim_pearson
value: 0.7278000135012542
- type: cos_sim_spearman
value: 0.7092812216947605
- type: euclidean_pearson
value: 0.771169214949292
- type: euclidean_spearman
value: 0.7710175681583312
- type: manhattan_pearson
value: 0.7684527031837596
- type: manhattan_spearman
value: 0.7707043080084379
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
metrics:
- type: v_measure
value: 0.2893427045246491
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
metrics:
- type: v_measure
value: 0.28230204578753637
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
metrics:
- type: accuracy
value: 0.627862
- type: ap
value: 0.10958454618347832
- type: f1
value: 0.48372434170467626
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
metrics:
- type: v_measure
value: 0.2824295128553035
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
metrics:
- type: cos_sim_accuracy
value: 0.815640460153782
- type: cos_sim_accuracy_threshold
value: 0.7118978500366211
- type: cos_sim_ap
value: 0.5709409536692154
- type: cos_sim_f1
value: 0.5529607083563918
- type: cos_sim_f1_threshold
value: 0.5981647968292236
- type: cos_sim_precision
value: 0.47626310772163966
- type: cos_sim_recall
value: 0.6591029023746702
- type: dot_accuracy
value: 0.788162365142755
- type: dot_accuracy_threshold
value: 1049.799072265625
- type: dot_ap
value: 0.4742989400382077
- type: dot_f1
value: 0.5125944584382871
- type: dot_f1_threshold
value: 723.3736572265625
- type: dot_precision
value: 0.4255838271174625
- type: dot_recall
value: 0.6443271767810026
- type: euclidean_accuracy
value: 0.8029445073612684
- type: euclidean_accuracy_threshold
value: 26.134265899658203
- type: euclidean_ap
value: 0.5342012231336148
- type: euclidean_f1
value: 0.5186778356350464
- type: euclidean_f1_threshold
value: 31.25627326965332
- type: euclidean_precision
value: 0.454203013481364
- type: euclidean_recall
value: 0.604485488126649
- type: manhattan_accuracy
value: 0.802884901949097
- type: manhattan_accuracy_threshold
value: 560.0760498046875
- type: manhattan_ap
value: 0.5343205271323233
- type: manhattan_f1
value: 0.520141655599823
- type: manhattan_f1_threshold
value: 658.3975830078125
- type: manhattan_precision
value: 0.44796035074342355
- type: manhattan_recall
value: 0.6200527704485488
- type: max_accuracy
value: 0.815640460153782
- type: max_ap
value: 0.5709409536692154
- type: max_f1
value: 0.5529607083563918
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
metrics:
- type: accuracy
value: 0.582421340629275
- type: f1
value: 0.40116960466226426
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (de)
metrics:
- type: accuracy
value: 0.4506903353057199
- type: f1
value: 0.30468468273374966
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (es)
metrics:
- type: accuracy
value: 0.4880920613742495
- type: f1
value: 0.3265985375400447
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (fr)
metrics:
- type: accuracy
value: 0.4433761352959599
- type: f1
value: 0.2930204743560644
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (hi)
metrics:
- type: accuracy
value: 0.34198637504481894
- type: f1
value: 0.2206370603224841
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (th)
metrics:
- type: accuracy
value: 0.4311030741410488
- type: f1
value: 0.2692408933648504
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
metrics:
- type: v_measure
value: 0.3375741018380938
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
metrics:
- type: map_at_1
value: 0.13909
- type: map_at_10
value: 0.19256
- type: map_at_100
value: 0.20286
- type: map_at_1000
value: 0.20429
- type: map_at_3
value: 0.17399
- type: map_at_5
value: 0.18399
- type: ndcg_at_1
value: 0.17421
- type: ndcg_at_10
value: 0.23106
- type: ndcg_at_100
value: 0.28129
- type: ndcg_at_1000
value: 0.31481
- type: ndcg_at_3
value: 0.19789
- type: ndcg_at_5
value: 0.21237
- type: precision_at_1
value: 0.17421
- type: precision_at_10
value: 0.04331
- type: precision_at_100
value: 0.00839
- type: precision_at_1000
value: 0.00131
- type: precision_at_3
value: 0.094
- type: precision_at_5
value: 0.06776
- type: recall_at_1
value: 0.13909
- type: recall_at_10
value: 0.31087
- type: recall_at_100
value: 0.52946
- type: recall_at_1000
value: 0.76546
- type: recall_at_3
value: 0.21351
- type: recall_at_5
value: 0.25265
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
metrics:
- type: map
value: 0.3996520488022785
- type: mrr
value: 0.40189248047703935
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
metrics:
- type: map_at_1
value: 0.12738416666666666
- type: map_at_10
value: 0.17235916666666667
- type: map_at_100
value: 0.1806333333333333
- type: map_at_1000
value: 0.18184333333333333
- type: map_at_3
value: 0.1574775
- type: map_at_5
value: 0.1657825
- type: ndcg_at_1
value: 0.15487416666666665
- type: ndcg_at_10
value: 0.20290166666666667
- type: ndcg_at_100
value: 0.24412916666666662
- type: ndcg_at_1000
value: 0.27586333333333335
- type: ndcg_at_3
value: 0.17622083333333333
- type: ndcg_at_5
value: 0.18859916666666668
- type: precision_at_1
value: 0.15487416666666665
- type: precision_at_10
value: 0.036226666666666664
- type: precision_at_100
value: 0.006820833333333333
- type: precision_at_1000
value: 0.0011216666666666666
- type: precision_at_3
value: 0.08163749999999999
- type: precision_at_5
value: 0.058654166666666674
- type: recall_at_1
value: 0.12738416666666666
- type: recall_at_10
value: 0.26599416666666664
- type: recall_at_100
value: 0.4541258333333334
- type: recall_at_1000
value: 0.687565
- type: recall_at_3
value: 0.19008166666666668
- type: recall_at_5
value: 0.2224991666666667
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
metrics:
- type: cos_sim_accuracy
value: 0.9949306930693069
- type: cos_sim_accuracy_threshold
value: 0.7870972752571106
- type: cos_sim_ap
value: 0.7773085502917281
- type: cos_sim_f1
value: 0.7178978681209718
- type: cos_sim_f1_threshold
value: 0.7572916746139526
- type: cos_sim_precision
value: 0.711897738446411
- type: cos_sim_recall
value: 0.724
- type: dot_accuracy
value: 0.9908118811881188
- type: dot_accuracy_threshold
value: 1571.5850830078125
- type: dot_ap
value: 0.30267748833368235
- type: dot_f1
value: 0.34335201222618444
- type: dot_f1_threshold
value: 1329.530029296875
- type: dot_precision
value: 0.34994807892004154
- type: dot_recall
value: 0.337
- type: euclidean_accuracy
value: 0.9951683168316832
- type: euclidean_accuracy_threshold
value: 25.715721130371094
- type: euclidean_ap
value: 0.7864498778235628
- type: euclidean_f1
value: 0.7309149972929074
- type: euclidean_f1_threshold
value: 26.336116790771484
- type: euclidean_precision
value: 0.7969303423848878
- type: euclidean_recall
value: 0.675
- type: manhattan_accuracy
value: 0.9953168316831683
- type: manhattan_accuracy_threshold
value: 534.224609375
- type: manhattan_ap
value: 0.7945274878693959
- type: manhattan_f1
value: 0.7419863373620599
- type: manhattan_f1_threshold
value: 562.244140625
- type: manhattan_precision
value: 0.7818383167220376
- type: manhattan_recall
value: 0.706
- type: max_accuracy
value: 0.9953168316831683
- type: max_ap
value: 0.7945274878693959
- type: max_f1
value: 0.7419863373620599
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
metrics:
- type: map_at_1
value: 0.09057
- type: map_at_10
value: 0.12721
- type: map_at_100
value: 0.1345
- type: map_at_1000
value: 0.13564
- type: map_at_3
value: 0.1134
- type: map_at_5
value: 0.12245
- type: ndcg_at_1
value: 0.09797
- type: ndcg_at_10
value: 0.15091
- type: ndcg_at_100
value: 0.18886
- type: ndcg_at_1000
value: 0.2229
- type: ndcg_at_3
value: 0.12365
- type: ndcg_at_5
value: 0.13931
- type: precision_at_1
value: 0.09797
- type: precision_at_10
value: 0.02477
- type: precision_at_100
value: 0.00466
- type: precision_at_1000
value: 0.00082
- type: precision_at_3
value: 0.05299
- type: precision_at_5
value: 0.04067
- type: recall_at_1
value: 0.09057
- type: recall_at_10
value: 0.21319
- type: recall_at_100
value: 0.38999
- type: recall_at_1000
value: 0.65374
- type: recall_at_3
value: 0.14331
- type: recall_at_5
value: 0.17917
SGPT-125M-weightedmean-nli-bitfit
Usage
For usage instructions, refer to our codebase: https://github.com/Muennighoff/sgpt
Evaluation Results
For eval results, refer to the eval folder or our paper: https://arxiv.org/abs/2202.08904
Training
The model was trained with the parameters:
DataLoader:
sentence_transformers.datasets.NoDuplicatesDataLoader.NoDuplicatesDataLoader of length 8807 with parameters:
{'batch_size': 64}
Loss:
sentence_transformers.losses.MultipleNegativesRankingLoss.MultipleNegativesRankingLoss with parameters:
{'scale': 20.0, 'similarity_fct': 'cos_sim'}
Parameters of the fit()-Method:
{
"epochs": 1,
"evaluation_steps": 880,
"evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
"max_grad_norm": 1,
"optimizer_class": "<class 'transformers.optimization.AdamW'>",
"optimizer_params": {
"lr": 0.0002
},
"scheduler": "WarmupLinear",
"steps_per_epoch": null,
"warmup_steps": 881,
"weight_decay": 0.01
}
Full Model Architecture
SentenceTransformer(
(0): Transformer({'max_seq_length': 75, 'do_lower_case': False}) with Transformer model: GPTNeoModel
(1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': True, 'pooling_mode_lasttoken': False})
)
Citing & Authors
@article{muennighoff2022sgpt,
title={SGPT: GPT Sentence Embeddings for Semantic Search},
author={Muennighoff, Niklas},
journal={arXiv preprint arXiv:2202.08904},
year={2022}
}