|
---
|
|
tags:
|
|
- mteb
|
|
- transformers
|
|
model-index:
|
|
- name: speed-embedding-7b-instruct
|
|
results:
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/amazon_counterfactual
|
|
name: MTEB AmazonCounterfactualClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
|
|
metrics:
|
|
- type: accuracy
|
|
value: 76.67164179104478
|
|
- type: ap
|
|
value: 39.07181577576136
|
|
- type: f1
|
|
value: 70.25085237742982
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/amazon_polarity
|
|
name: MTEB AmazonPolarityClassification
|
|
config: default
|
|
split: test
|
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
|
|
metrics:
|
|
- type: accuracy
|
|
value: 96.1775
|
|
- type: ap
|
|
value: 94.84308844303422
|
|
- type: f1
|
|
value: 96.17546959843244
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/amazon_reviews_multi
|
|
name: MTEB AmazonReviewsClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
|
|
metrics:
|
|
- type: accuracy
|
|
value: 56.278000000000006
|
|
- type: f1
|
|
value: 55.45101875980304
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: arguana
|
|
name: MTEB ArguAna
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 33.642
|
|
- type: ndcg_at_3
|
|
value: 49.399
|
|
- type: ndcg_at_5
|
|
value: 54.108999999999995
|
|
- type: ndcg_at_10
|
|
value: 59.294999999999995
|
|
- type: ndcg_at_100
|
|
value: 62.015
|
|
- type: map_at_1
|
|
value: 33.642
|
|
- type: map_at_3
|
|
value: 45.507
|
|
- type: map_at_5
|
|
value: 48.1
|
|
- type: map_at_10
|
|
value: 50.248000000000005
|
|
- type: map_at_100
|
|
value: 50.954
|
|
- type: recall_at_1
|
|
value: 33.642
|
|
- type: recall_at_3
|
|
value: 60.669
|
|
- type: recall_at_5
|
|
value: 72.191
|
|
- type: recall_at_10
|
|
value: 88.193
|
|
- type: recall_at_100
|
|
value: 99.431
|
|
- type: precision_at_1
|
|
value: 33.642
|
|
- type: precision_at_3
|
|
value: 20.223
|
|
- type: precision_at_5
|
|
value: 14.438
|
|
- type: precision_at_10
|
|
value: 8.819
|
|
- type: precision_at_100
|
|
value: 0.9939999999999999
|
|
- type: mrr_at_1
|
|
value: 33.997
|
|
- type: mrr_at_3
|
|
value: 45.614
|
|
- type: mrr_at_5
|
|
value: 48.263
|
|
- type: mrr_at_10
|
|
value: 50.388999999999996
|
|
- type: mrr_at_100
|
|
value: 51.102000000000004
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/arxiv-clustering-p2p
|
|
name: MTEB ArxivClusteringP2P
|
|
config: default
|
|
split: test
|
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
|
|
metrics:
|
|
- type: v_measure
|
|
value: 51.1249344529392
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/arxiv-clustering-s2s
|
|
name: MTEB ArxivClusteringS2S
|
|
config: default
|
|
split: test
|
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
|
|
metrics:
|
|
- type: v_measure
|
|
value: 47.01575217563573
|
|
- task:
|
|
type: Reranking
|
|
dataset:
|
|
type: mteb/askubuntudupquestions-reranking
|
|
name: MTEB AskUbuntuDupQuestions
|
|
config: default
|
|
split: test
|
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
|
|
metrics:
|
|
- type: map
|
|
value: 67.2259454062751
|
|
- type: mrr
|
|
value: 79.37508244294948
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/biosses-sts
|
|
name: MTEB BIOSSES
|
|
config: default
|
|
split: test
|
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 89.5312396547344
|
|
- type: cos_sim_spearman
|
|
value: 87.1447567367366
|
|
- type: euclidean_pearson
|
|
value: 88.67110804544821
|
|
- type: euclidean_spearman
|
|
value: 87.1447567367366
|
|
- type: manhattan_pearson
|
|
value: 89.06983994154335
|
|
- type: manhattan_spearman
|
|
value: 87.59115245033443
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/banking77
|
|
name: MTEB Banking77Classification
|
|
config: default
|
|
split: test
|
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
|
|
metrics:
|
|
- type: accuracy
|
|
value: 88.63636363636364
|
|
- type: f1
|
|
value: 88.58740097633193
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/biorxiv-clustering-p2p
|
|
name: MTEB BiorxivClusteringP2P
|
|
config: default
|
|
split: test
|
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
|
|
metrics:
|
|
- type: v_measure
|
|
value: 41.99753263006505
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/biorxiv-clustering-s2s
|
|
name: MTEB BiorxivClusteringS2S
|
|
config: default
|
|
split: test
|
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
|
|
metrics:
|
|
- type: v_measure
|
|
value: 39.623067884052666
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: BeIR/cqadupstack
|
|
name: MTEB CQADupstackRetrieval
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 30.904666666666664
|
|
- type: ndcg_at_3
|
|
value: 36.32808333333333
|
|
- type: ndcg_at_5
|
|
value: 38.767250000000004
|
|
- type: ndcg_at_10
|
|
value: 41.62008333333333
|
|
- type: ndcg_at_100
|
|
value: 47.118083333333324
|
|
- type: map_at_1
|
|
value: 25.7645
|
|
- type: map_at_3
|
|
value: 32.6235
|
|
- type: map_at_5
|
|
value: 34.347
|
|
- type: map_at_10
|
|
value: 35.79658333333333
|
|
- type: map_at_100
|
|
value: 37.10391666666666
|
|
- type: recall_at_1
|
|
value: 25.7645
|
|
- type: recall_at_3
|
|
value: 39.622666666666674
|
|
- type: recall_at_5
|
|
value: 45.938750000000006
|
|
- type: recall_at_10
|
|
value: 54.43816666666667
|
|
- type: recall_at_100
|
|
value: 78.66183333333333
|
|
- type: precision_at_1
|
|
value: 30.904666666666664
|
|
- type: precision_at_3
|
|
value: 17.099083333333333
|
|
- type: precision_at_5
|
|
value: 12.278416666666669
|
|
- type: precision_at_10
|
|
value: 7.573083333333335
|
|
- type: precision_at_100
|
|
value: 1.22275
|
|
- type: mrr_at_1
|
|
value: 30.904666666666664
|
|
- type: mrr_at_3
|
|
value: 37.458333333333336
|
|
- type: mrr_at_5
|
|
value: 38.97333333333333
|
|
- type: mrr_at_10
|
|
value: 40.10316666666666
|
|
- type: mrr_at_100
|
|
value: 41.004250000000006
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: climate-fever
|
|
name: MTEB ClimateFEVER
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 38.046
|
|
- type: ndcg_at_3
|
|
value: 31.842
|
|
- type: ndcg_at_5
|
|
value: 33.698
|
|
- type: ndcg_at_10
|
|
value: 37.765
|
|
- type: ndcg_at_100
|
|
value: 44.998
|
|
- type: map_at_1
|
|
value: 16.682
|
|
- type: map_at_3
|
|
value: 23.624000000000002
|
|
- type: map_at_5
|
|
value: 25.812
|
|
- type: map_at_10
|
|
value: 28.017999999999997
|
|
- type: map_at_100
|
|
value: 30.064999999999998
|
|
- type: recall_at_1
|
|
value: 16.682
|
|
- type: recall_at_3
|
|
value: 28.338
|
|
- type: recall_at_5
|
|
value: 34.486
|
|
- type: recall_at_10
|
|
value: 43.474000000000004
|
|
- type: recall_at_100
|
|
value: 67.984
|
|
- type: precision_at_1
|
|
value: 38.046
|
|
- type: precision_at_3
|
|
value: 23.779
|
|
- type: precision_at_5
|
|
value: 17.849999999999998
|
|
- type: precision_at_10
|
|
value: 11.642
|
|
- type: precision_at_100
|
|
value: 1.9429999999999998
|
|
- type: mrr_at_1
|
|
value: 38.046
|
|
- type: mrr_at_3
|
|
value: 46.764
|
|
- type: mrr_at_5
|
|
value: 48.722
|
|
- type: mrr_at_10
|
|
value: 49.976
|
|
- type: mrr_at_100
|
|
value: 50.693999999999996
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: dbpedia-entity
|
|
name: MTEB DBPedia
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 63.24999999999999
|
|
- type: ndcg_at_3
|
|
value: 54.005
|
|
- type: ndcg_at_5
|
|
value: 51.504000000000005
|
|
- type: ndcg_at_10
|
|
value: 49.738
|
|
- type: ndcg_at_100
|
|
value: 54.754000000000005
|
|
- type: map_at_1
|
|
value: 10.639
|
|
- type: map_at_3
|
|
value: 16.726
|
|
- type: map_at_5
|
|
value: 20.101
|
|
- type: map_at_10
|
|
value: 24.569
|
|
- type: map_at_100
|
|
value: 35.221999999999994
|
|
- type: recall_at_1
|
|
value: 10.639
|
|
- type: recall_at_3
|
|
value: 17.861
|
|
- type: recall_at_5
|
|
value: 22.642
|
|
- type: recall_at_10
|
|
value: 30.105999999999998
|
|
- type: recall_at_100
|
|
value: 60.92999999999999
|
|
- type: precision_at_1
|
|
value: 75.0
|
|
- type: precision_at_3
|
|
value: 58.083
|
|
- type: precision_at_5
|
|
value: 50.0
|
|
- type: precision_at_10
|
|
value: 40.35
|
|
- type: precision_at_100
|
|
value: 12.659999999999998
|
|
- type: mrr_at_1
|
|
value: 75.0
|
|
- type: mrr_at_3
|
|
value: 80.042
|
|
- type: mrr_at_5
|
|
value: 80.779
|
|
- type: mrr_at_10
|
|
value: 81.355
|
|
- type: mrr_at_100
|
|
value: 81.58
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/emotion
|
|
name: MTEB EmotionClassification
|
|
config: default
|
|
split: test
|
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
|
|
metrics:
|
|
- type: accuracy
|
|
value: 51.025
|
|
- type: f1
|
|
value: 47.08253474922065
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: fever
|
|
name: MTEB FEVER
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 82.163
|
|
- type: ndcg_at_3
|
|
value: 86.835
|
|
- type: ndcg_at_5
|
|
value: 87.802
|
|
- type: ndcg_at_10
|
|
value: 88.529
|
|
- type: ndcg_at_100
|
|
value: 89.17
|
|
- type: map_at_1
|
|
value: 76.335
|
|
- type: map_at_3
|
|
value: 83.91499999999999
|
|
- type: map_at_5
|
|
value: 84.64500000000001
|
|
- type: map_at_10
|
|
value: 85.058
|
|
- type: map_at_100
|
|
value: 85.257
|
|
- type: recall_at_1
|
|
value: 76.335
|
|
- type: recall_at_3
|
|
value: 90.608
|
|
- type: recall_at_5
|
|
value: 93.098
|
|
- type: recall_at_10
|
|
value: 95.173
|
|
- type: recall_at_100
|
|
value: 97.59299999999999
|
|
- type: precision_at_1
|
|
value: 82.163
|
|
- type: precision_at_3
|
|
value: 33.257999999999996
|
|
- type: precision_at_5
|
|
value: 20.654
|
|
- type: precision_at_10
|
|
value: 10.674999999999999
|
|
- type: precision_at_100
|
|
value: 1.122
|
|
- type: mrr_at_1
|
|
value: 82.163
|
|
- type: mrr_at_3
|
|
value: 88.346
|
|
- type: mrr_at_5
|
|
value: 88.791
|
|
- type: mrr_at_10
|
|
value: 88.97699999999999
|
|
- type: mrr_at_100
|
|
value: 89.031
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: fiqa
|
|
name: MTEB FiQA2018
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 55.093
|
|
- type: ndcg_at_3
|
|
value: 52.481
|
|
- type: ndcg_at_5
|
|
value: 53.545
|
|
- type: ndcg_at_10
|
|
value: 56.053
|
|
- type: ndcg_at_100
|
|
value: 62.53999999999999
|
|
- type: map_at_1
|
|
value: 29.189999999999998
|
|
- type: map_at_3
|
|
value: 42.603
|
|
- type: map_at_5
|
|
value: 45.855000000000004
|
|
- type: map_at_10
|
|
value: 48.241
|
|
- type: map_at_100
|
|
value: 50.300999999999995
|
|
- type: recall_at_1
|
|
value: 29.189999999999998
|
|
- type: recall_at_3
|
|
value: 47.471999999999994
|
|
- type: recall_at_5
|
|
value: 54.384
|
|
- type: recall_at_10
|
|
value: 62.731
|
|
- type: recall_at_100
|
|
value: 86.02300000000001
|
|
- type: precision_at_1
|
|
value: 55.093
|
|
- type: precision_at_3
|
|
value: 34.979
|
|
- type: precision_at_5
|
|
value: 25.278
|
|
- type: precision_at_10
|
|
value: 15.231
|
|
- type: precision_at_100
|
|
value: 2.2190000000000003
|
|
- type: mrr_at_1
|
|
value: 55.093
|
|
- type: mrr_at_3
|
|
value: 61.317
|
|
- type: mrr_at_5
|
|
value: 62.358999999999995
|
|
- type: mrr_at_10
|
|
value: 63.165000000000006
|
|
- type: mrr_at_100
|
|
value: 63.81
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: hotpotqa
|
|
name: MTEB HotpotQA
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 78.866
|
|
- type: ndcg_at_3
|
|
value: 70.128
|
|
- type: ndcg_at_5
|
|
value: 73.017
|
|
- type: ndcg_at_10
|
|
value: 75.166
|
|
- type: ndcg_at_100
|
|
value: 77.97500000000001
|
|
- type: map_at_1
|
|
value: 39.433
|
|
- type: map_at_3
|
|
value: 64.165
|
|
- type: map_at_5
|
|
value: 66.503
|
|
- type: map_at_10
|
|
value: 67.822
|
|
- type: map_at_100
|
|
value: 68.675
|
|
- type: recall_at_1
|
|
value: 39.433
|
|
- type: recall_at_3
|
|
value: 69.03399999999999
|
|
- type: recall_at_5
|
|
value: 74.74
|
|
- type: recall_at_10
|
|
value: 80.108
|
|
- type: recall_at_100
|
|
value: 90.81700000000001
|
|
- type: precision_at_1
|
|
value: 78.866
|
|
- type: precision_at_3
|
|
value: 46.022999999999996
|
|
- type: precision_at_5
|
|
value: 29.896
|
|
- type: precision_at_10
|
|
value: 16.022
|
|
- type: precision_at_100
|
|
value: 1.8159999999999998
|
|
- type: mrr_at_1
|
|
value: 78.866
|
|
- type: mrr_at_3
|
|
value: 83.91
|
|
- type: mrr_at_5
|
|
value: 84.473
|
|
- type: mrr_at_10
|
|
value: 84.769
|
|
- type: mrr_at_100
|
|
value: 84.953
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/imdb
|
|
name: MTEB ImdbClassification
|
|
config: default
|
|
split: test
|
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
|
|
metrics:
|
|
- type: accuracy
|
|
value: 94.87799999999999
|
|
- type: ap
|
|
value: 92.5831019543702
|
|
- type: f1
|
|
value: 94.87675087619891
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: msmarco
|
|
name: MTEB MSMARCO
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 23.195
|
|
- type: ndcg_at_3
|
|
value: 34.419
|
|
- type: ndcg_at_5
|
|
value: 38.665
|
|
- type: ndcg_at_10
|
|
value: 42.549
|
|
- type: ndcg_at_100
|
|
value: 48.256
|
|
- type: map_at_1
|
|
value: 22.508
|
|
- type: map_at_3
|
|
value: 31.346
|
|
- type: map_at_5
|
|
value: 33.73
|
|
- type: map_at_10
|
|
value: 35.365
|
|
- type: map_at_100
|
|
value: 36.568
|
|
- type: recall_at_1
|
|
value: 22.508
|
|
- type: recall_at_3
|
|
value: 42.63
|
|
- type: recall_at_5
|
|
value: 52.827999999999996
|
|
- type: recall_at_10
|
|
value: 64.645
|
|
- type: recall_at_100
|
|
value: 90.852
|
|
- type: precision_at_1
|
|
value: 23.195
|
|
- type: precision_at_3
|
|
value: 14.752
|
|
- type: precision_at_5
|
|
value: 11.0
|
|
- type: precision_at_10
|
|
value: 6.755
|
|
- type: precision_at_100
|
|
value: 0.96
|
|
- type: mrr_at_1
|
|
value: 23.195
|
|
- type: mrr_at_3
|
|
value: 32.042
|
|
- type: mrr_at_5
|
|
value: 34.388000000000005
|
|
- type: mrr_at_10
|
|
value: 35.974000000000004
|
|
- type: mrr_at_100
|
|
value: 37.114000000000004
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/mtop_domain
|
|
name: MTEB MTOPDomainClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
|
|
metrics:
|
|
- type: accuracy
|
|
value: 95.84587323301413
|
|
- type: f1
|
|
value: 95.69948889844318
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/mtop_intent
|
|
name: MTEB MTOPIntentClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
|
|
metrics:
|
|
- type: accuracy
|
|
value: 87.08162334701322
|
|
- type: f1
|
|
value: 72.237783326283
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/amazon_massive_intent
|
|
name: MTEB MassiveIntentClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
|
|
metrics:
|
|
- type: accuracy
|
|
value: 80.19502353732346
|
|
- type: f1
|
|
value: 77.732184986995
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/amazon_massive_scenario
|
|
name: MTEB MassiveScenarioClassification (en)
|
|
config: en
|
|
split: test
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634
|
|
metrics:
|
|
- type: accuracy
|
|
value: 82.26630800268998
|
|
- type: f1
|
|
value: 82.12747916248556
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/medrxiv-clustering-p2p
|
|
name: MTEB MedrxivClusteringP2P
|
|
config: default
|
|
split: test
|
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
|
|
metrics:
|
|
- type: v_measure
|
|
value: 36.95240450167033
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/medrxiv-clustering-s2s
|
|
name: MTEB MedrxivClusteringS2S
|
|
config: default
|
|
split: test
|
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
|
|
metrics:
|
|
- type: v_measure
|
|
value: 36.27758530931266
|
|
- task:
|
|
type: Reranking
|
|
dataset:
|
|
type: mteb/mind_small
|
|
name: MTEB MindSmallReranking
|
|
config: default
|
|
split: test
|
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
|
|
metrics:
|
|
- type: map
|
|
value: 33.35707665482982
|
|
- type: mrr
|
|
value: 34.60987842278547
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: nfcorpus
|
|
name: MTEB NFCorpus
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 47.522999999999996
|
|
- type: ndcg_at_3
|
|
value: 44.489000000000004
|
|
- type: ndcg_at_5
|
|
value: 41.92
|
|
- type: ndcg_at_10
|
|
value: 38.738
|
|
- type: ndcg_at_100
|
|
value: 35.46
|
|
- type: map_at_1
|
|
value: 5.357
|
|
- type: map_at_3
|
|
value: 10.537
|
|
- type: map_at_5
|
|
value: 12.062000000000001
|
|
- type: map_at_10
|
|
value: 14.264
|
|
- type: map_at_100
|
|
value: 18.442
|
|
- type: recall_at_1
|
|
value: 5.357
|
|
- type: recall_at_3
|
|
value: 12.499
|
|
- type: recall_at_5
|
|
value: 14.809
|
|
- type: recall_at_10
|
|
value: 18.765
|
|
- type: recall_at_100
|
|
value: 36.779
|
|
- type: precision_at_1
|
|
value: 49.226
|
|
- type: precision_at_3
|
|
value: 41.899
|
|
- type: precision_at_5
|
|
value: 36.718
|
|
- type: precision_at_10
|
|
value: 29.287999999999997
|
|
- type: precision_at_100
|
|
value: 9.22
|
|
- type: mrr_at_1
|
|
value: 49.845
|
|
- type: mrr_at_3
|
|
value: 57.121
|
|
- type: mrr_at_5
|
|
value: 58.172999999999995
|
|
- type: mrr_at_10
|
|
value: 58.906000000000006
|
|
- type: mrr_at_100
|
|
value: 59.486000000000004
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: nq
|
|
name: MTEB NQ
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 42.815999999999995
|
|
- type: ndcg_at_3
|
|
value: 53.766999999999996
|
|
- type: ndcg_at_5
|
|
value: 57.957
|
|
- type: ndcg_at_10
|
|
value: 61.661
|
|
- type: ndcg_at_100
|
|
value: 65.218
|
|
- type: map_at_1
|
|
value: 38.364
|
|
- type: map_at_3
|
|
value: 49.782
|
|
- type: map_at_5
|
|
value: 52.319
|
|
- type: map_at_10
|
|
value: 54.07300000000001
|
|
- type: map_at_100
|
|
value: 54.983000000000004
|
|
- type: recall_at_1
|
|
value: 38.364
|
|
- type: recall_at_3
|
|
value: 61.744
|
|
- type: recall_at_5
|
|
value: 71.32300000000001
|
|
- type: recall_at_10
|
|
value: 82.015
|
|
- type: recall_at_100
|
|
value: 96.978
|
|
- type: precision_at_1
|
|
value: 42.815999999999995
|
|
- type: precision_at_3
|
|
value: 23.976
|
|
- type: precision_at_5
|
|
value: 16.866
|
|
- type: precision_at_10
|
|
value: 9.806
|
|
- type: precision_at_100
|
|
value: 1.1769999999999998
|
|
- type: mrr_at_1
|
|
value: 42.845
|
|
- type: mrr_at_3
|
|
value: 53.307
|
|
- type: mrr_at_5
|
|
value: 55.434000000000005
|
|
- type: mrr_at_10
|
|
value: 56.702
|
|
- type: mrr_at_100
|
|
value: 57.342000000000006
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: quora
|
|
name: MTEB QuoraRetrieval
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 82.46
|
|
- type: ndcg_at_3
|
|
value: 86.774
|
|
- type: ndcg_at_5
|
|
value: 88.256
|
|
- type: ndcg_at_10
|
|
value: 89.35
|
|
- type: ndcg_at_100
|
|
value: 90.46499999999999
|
|
- type: map_at_1
|
|
value: 71.562
|
|
- type: map_at_3
|
|
value: 82.948
|
|
- type: map_at_5
|
|
value: 84.786
|
|
- type: map_at_10
|
|
value: 85.82300000000001
|
|
- type: map_at_100
|
|
value: 86.453
|
|
- type: recall_at_1
|
|
value: 71.562
|
|
- type: recall_at_3
|
|
value: 88.51
|
|
- type: recall_at_5
|
|
value: 92.795
|
|
- type: recall_at_10
|
|
value: 95.998
|
|
- type: recall_at_100
|
|
value: 99.701
|
|
- type: precision_at_1
|
|
value: 82.46
|
|
- type: precision_at_3
|
|
value: 38.1
|
|
- type: precision_at_5
|
|
value: 24.990000000000002
|
|
- type: precision_at_10
|
|
value: 13.553999999999998
|
|
- type: precision_at_100
|
|
value: 1.539
|
|
- type: mrr_at_1
|
|
value: 82.43
|
|
- type: mrr_at_3
|
|
value: 87.653
|
|
- type: mrr_at_5
|
|
value: 88.26899999999999
|
|
- type: mrr_at_10
|
|
value: 88.505
|
|
- type: mrr_at_100
|
|
value: 88.601
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/reddit-clustering
|
|
name: MTEB RedditClustering
|
|
config: default
|
|
split: test
|
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
|
|
metrics:
|
|
- type: v_measure
|
|
value: 57.928338007609256
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/reddit-clustering-p2p
|
|
name: MTEB RedditClusteringP2P
|
|
config: default
|
|
split: test
|
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16
|
|
metrics:
|
|
- type: v_measure
|
|
value: 65.28915417473826
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: scidocs
|
|
name: MTEB SCIDOCS
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 17.2
|
|
- type: ndcg_at_3
|
|
value: 15.856
|
|
- type: ndcg_at_5
|
|
value: 13.983
|
|
- type: ndcg_at_10
|
|
value: 16.628999999999998
|
|
- type: ndcg_at_100
|
|
value: 23.845
|
|
- type: map_at_1
|
|
value: 3.4750000000000005
|
|
- type: map_at_3
|
|
value: 6.905
|
|
- type: map_at_5
|
|
value: 8.254
|
|
- type: map_at_10
|
|
value: 9.474
|
|
- type: map_at_100
|
|
value: 11.242
|
|
- type: recall_at_1
|
|
value: 3.4750000000000005
|
|
- type: recall_at_3
|
|
value: 9.298
|
|
- type: recall_at_5
|
|
value: 12.817
|
|
- type: recall_at_10
|
|
value: 17.675
|
|
- type: recall_at_100
|
|
value: 38.678000000000004
|
|
- type: precision_at_1
|
|
value: 17.2
|
|
- type: precision_at_3
|
|
value: 15.299999999999999
|
|
- type: precision_at_5
|
|
value: 12.64
|
|
- type: precision_at_10
|
|
value: 8.72
|
|
- type: precision_at_100
|
|
value: 1.907
|
|
- type: mrr_at_1
|
|
value: 17.2
|
|
- type: mrr_at_3
|
|
value: 25.55
|
|
- type: mrr_at_5
|
|
value: 27.485
|
|
- type: mrr_at_10
|
|
value: 28.809
|
|
- type: mrr_at_100
|
|
value: 29.964000000000002
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sickr-sts
|
|
name: MTEB SICK-R
|
|
config: default
|
|
split: test
|
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 86.10434430387332
|
|
- type: cos_sim_spearman
|
|
value: 82.46041161692649
|
|
- type: euclidean_pearson
|
|
value: 83.4010092798136
|
|
- type: euclidean_spearman
|
|
value: 82.46040715308601
|
|
- type: manhattan_pearson
|
|
value: 83.6702316837156
|
|
- type: manhattan_spearman
|
|
value: 82.72271392303014
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts12-sts
|
|
name: MTEB STS12
|
|
config: default
|
|
split: test
|
|
revision: a0d554a64d88156834ff5ae9920b964011b16384
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 87.3179771524676
|
|
- type: cos_sim_spearman
|
|
value: 80.15194914870666
|
|
- type: euclidean_pearson
|
|
value: 84.54005271342946
|
|
- type: euclidean_spearman
|
|
value: 80.15194914870666
|
|
- type: manhattan_pearson
|
|
value: 85.24410357734307
|
|
- type: manhattan_spearman
|
|
value: 80.78274673604562
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts13-sts
|
|
name: MTEB STS13
|
|
config: default
|
|
split: test
|
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 89.2691354894402
|
|
- type: cos_sim_spearman
|
|
value: 89.94300436293618
|
|
- type: euclidean_pearson
|
|
value: 89.5600067781475
|
|
- type: euclidean_spearman
|
|
value: 89.942989691344
|
|
- type: manhattan_pearson
|
|
value: 89.80327997794308
|
|
- type: manhattan_spearman
|
|
value: 90.3964860275568
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts14-sts
|
|
name: MTEB STS14
|
|
config: default
|
|
split: test
|
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 87.68003396295498
|
|
- type: cos_sim_spearman
|
|
value: 86.23848649310362
|
|
- type: euclidean_pearson
|
|
value: 87.0702308813695
|
|
- type: euclidean_spearman
|
|
value: 86.23848649310362
|
|
- type: manhattan_pearson
|
|
value: 87.24495415360472
|
|
- type: manhattan_spearman
|
|
value: 86.58198464997109
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts15-sts
|
|
name: MTEB STS15
|
|
config: default
|
|
split: test
|
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 90.25643329096215
|
|
- type: cos_sim_spearman
|
|
value: 91.19520084590636
|
|
- type: euclidean_pearson
|
|
value: 90.68579446788728
|
|
- type: euclidean_spearman
|
|
value: 91.19519611831312
|
|
- type: manhattan_pearson
|
|
value: 90.83476867273104
|
|
- type: manhattan_spearman
|
|
value: 91.4569817842705
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts16-sts
|
|
name: MTEB STS16
|
|
config: default
|
|
split: test
|
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 86.41175694023282
|
|
- type: cos_sim_spearman
|
|
value: 88.18744495989392
|
|
- type: euclidean_pearson
|
|
value: 87.60085709987156
|
|
- type: euclidean_spearman
|
|
value: 88.18773792681107
|
|
- type: manhattan_pearson
|
|
value: 87.83199472909764
|
|
- type: manhattan_spearman
|
|
value: 88.45824161471776
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts17-crosslingual-sts
|
|
name: MTEB STS17 (en-en)
|
|
config: en-en
|
|
split: test
|
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 91.78311335565503
|
|
- type: cos_sim_spearman
|
|
value: 91.93416269793802
|
|
- type: euclidean_pearson
|
|
value: 91.84163160890154
|
|
- type: euclidean_spearman
|
|
value: 91.93416269793802
|
|
- type: manhattan_pearson
|
|
value: 91.77053255749301
|
|
- type: manhattan_spearman
|
|
value: 91.67392623286098
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/sts22-crosslingual-sts
|
|
name: MTEB STS22 (en)
|
|
config: en
|
|
split: test
|
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 68.2137857919086
|
|
- type: cos_sim_spearman
|
|
value: 68.31928639693375
|
|
- type: euclidean_pearson
|
|
value: 69.96072053688385
|
|
- type: euclidean_spearman
|
|
value: 68.31928639693375
|
|
- type: manhattan_pearson
|
|
value: 70.47736299273389
|
|
- type: manhattan_spearman
|
|
value: 68.72439259356818
|
|
- task:
|
|
type: STS
|
|
dataset:
|
|
type: mteb/stsbenchmark-sts
|
|
name: MTEB STSBenchmark
|
|
config: default
|
|
split: test
|
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 88.16092476703817
|
|
- type: cos_sim_spearman
|
|
value: 89.20507562822989
|
|
- type: euclidean_pearson
|
|
value: 88.91358225424611
|
|
- type: euclidean_spearman
|
|
value: 89.20505548241839
|
|
- type: manhattan_pearson
|
|
value: 88.98787306839809
|
|
- type: manhattan_spearman
|
|
value: 89.37338458483269
|
|
- task:
|
|
type: Reranking
|
|
dataset:
|
|
type: mteb/scidocs-reranking
|
|
name: MTEB SciDocsRR
|
|
config: default
|
|
split: test
|
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
|
|
metrics:
|
|
- type: map
|
|
value: 87.29108971888714
|
|
- type: mrr
|
|
value: 96.62042024787124
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: scifact
|
|
name: MTEB SciFact
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 63.333
|
|
- type: ndcg_at_3
|
|
value: 72.768
|
|
- type: ndcg_at_5
|
|
value: 75.124
|
|
- type: ndcg_at_10
|
|
value: 77.178
|
|
- type: ndcg_at_100
|
|
value: 78.769
|
|
- type: map_at_1
|
|
value: 60.9
|
|
- type: map_at_3
|
|
value: 69.69999999999999
|
|
- type: map_at_5
|
|
value: 71.345
|
|
- type: map_at_10
|
|
value: 72.36200000000001
|
|
- type: map_at_100
|
|
value: 72.783
|
|
- type: recall_at_1
|
|
value: 60.9
|
|
- type: recall_at_3
|
|
value: 79.172
|
|
- type: recall_at_5
|
|
value: 84.917
|
|
- type: recall_at_10
|
|
value: 90.756
|
|
- type: recall_at_100
|
|
value: 97.667
|
|
- type: precision_at_1
|
|
value: 63.333
|
|
- type: precision_at_3
|
|
value: 28.555999999999997
|
|
- type: precision_at_5
|
|
value: 18.8
|
|
- type: precision_at_10
|
|
value: 10.233
|
|
- type: precision_at_100
|
|
value: 1.107
|
|
- type: mrr_at_1
|
|
value: 63.333
|
|
- type: mrr_at_3
|
|
value: 71.27799999999999
|
|
- type: mrr_at_5
|
|
value: 72.478
|
|
- type: mrr_at_10
|
|
value: 73.163
|
|
- type: mrr_at_100
|
|
value: 73.457
|
|
- task:
|
|
type: PairClassification
|
|
dataset:
|
|
type: mteb/sprintduplicatequestions-pairclassification
|
|
name: MTEB SprintDuplicateQuestions
|
|
config: default
|
|
split: test
|
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
|
|
metrics:
|
|
- type: cos_sim_accuracy
|
|
value: 99.8009900990099
|
|
- type: cos_sim_ap
|
|
value: 95.46920445134404
|
|
- type: cos_sim_f1
|
|
value: 89.70814132104455
|
|
- type: cos_sim_precision
|
|
value: 91.9202518363064
|
|
- type: cos_sim_recall
|
|
value: 87.6
|
|
- type: dot_accuracy
|
|
value: 99.8009900990099
|
|
- type: dot_ap
|
|
value: 95.46920445134404
|
|
- type: dot_f1
|
|
value: 89.70814132104455
|
|
- type: dot_precision
|
|
value: 91.9202518363064
|
|
- type: dot_recall
|
|
value: 87.6
|
|
- type: euclidean_accuracy
|
|
value: 99.8009900990099
|
|
- type: euclidean_ap
|
|
value: 95.46924273007079
|
|
- type: euclidean_f1
|
|
value: 89.70814132104455
|
|
- type: euclidean_precision
|
|
value: 91.9202518363064
|
|
- type: euclidean_recall
|
|
value: 87.6
|
|
- type: manhattan_accuracy
|
|
value: 99.81188118811882
|
|
- type: manhattan_ap
|
|
value: 95.77631677784113
|
|
- type: manhattan_f1
|
|
value: 90.26639344262296
|
|
- type: manhattan_precision
|
|
value: 92.5420168067227
|
|
- type: manhattan_recall
|
|
value: 88.1
|
|
- type: max_accuracy
|
|
value: 99.81188118811882
|
|
- type: max_ap
|
|
value: 95.77631677784113
|
|
- type: max_f1
|
|
value: 90.26639344262296
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/stackexchange-clustering
|
|
name: MTEB StackExchangeClustering
|
|
config: default
|
|
split: test
|
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
|
|
metrics:
|
|
- type: v_measure
|
|
value: 71.59238280333025
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/stackexchange-clustering-p2p
|
|
name: MTEB StackExchangeClusteringP2P
|
|
config: default
|
|
split: test
|
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
|
|
metrics:
|
|
- type: v_measure
|
|
value: 39.012562075214035
|
|
- task:
|
|
type: Reranking
|
|
dataset:
|
|
type: mteb/stackoverflowdupquestions-reranking
|
|
name: MTEB StackOverflowDupQuestions
|
|
config: default
|
|
split: test
|
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
|
|
metrics:
|
|
- type: map
|
|
value: 55.16521497700657
|
|
- type: mrr
|
|
value: 56.1779427680163
|
|
- task:
|
|
type: Summarization
|
|
dataset:
|
|
type: mteb/summeval
|
|
name: MTEB SummEval
|
|
config: default
|
|
split: test
|
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
|
|
metrics:
|
|
- type: cos_sim_pearson
|
|
value: 31.04402552863106
|
|
- type: cos_sim_spearman
|
|
value: 31.05558230938988
|
|
- type: dot_pearson
|
|
value: 31.04400838015153
|
|
- type: dot_spearman
|
|
value: 31.05558230938988
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: trec-covid
|
|
name: MTEB TRECCOVID
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 91.0
|
|
- type: ndcg_at_3
|
|
value: 92.34599999999999
|
|
- type: ndcg_at_5
|
|
value: 90.89399999999999
|
|
- type: ndcg_at_10
|
|
value: 87.433
|
|
- type: ndcg_at_100
|
|
value: 67.06400000000001
|
|
- type: map_at_1
|
|
value: 0.241
|
|
- type: map_at_3
|
|
value: 0.735
|
|
- type: map_at_5
|
|
value: 1.216
|
|
- type: map_at_10
|
|
value: 2.317
|
|
- type: map_at_100
|
|
value: 14.151
|
|
- type: recall_at_1
|
|
value: 0.241
|
|
- type: recall_at_3
|
|
value: 0.76
|
|
- type: recall_at_5
|
|
value: 1.254
|
|
- type: recall_at_10
|
|
value: 2.421
|
|
- type: recall_at_100
|
|
value: 16.715
|
|
- type: precision_at_1
|
|
value: 94.0
|
|
- type: precision_at_3
|
|
value: 96.0
|
|
- type: precision_at_5
|
|
value: 94.8
|
|
- type: precision_at_10
|
|
value: 91.4
|
|
- type: precision_at_100
|
|
value: 68.24
|
|
- type: mrr_at_1
|
|
value: 94.0
|
|
- type: mrr_at_3
|
|
value: 96.667
|
|
- type: mrr_at_5
|
|
value: 96.667
|
|
- type: mrr_at_10
|
|
value: 96.667
|
|
- type: mrr_at_100
|
|
value: 96.667
|
|
- task:
|
|
type: Retrieval
|
|
dataset:
|
|
type: webis-touche2020
|
|
name: MTEB Touche2020
|
|
config: default
|
|
split: test
|
|
revision: None
|
|
metrics:
|
|
- type: ndcg_at_1
|
|
value: 26.531
|
|
- type: ndcg_at_3
|
|
value: 27.728
|
|
- type: ndcg_at_5
|
|
value: 25.668000000000003
|
|
- type: ndcg_at_10
|
|
value: 25.785999999999998
|
|
- type: ndcg_at_100
|
|
value: 35.623
|
|
- type: map_at_1
|
|
value: 2.076
|
|
- type: map_at_3
|
|
value: 5.29
|
|
- type: map_at_5
|
|
value: 7.292999999999999
|
|
- type: map_at_10
|
|
value: 9.81
|
|
- type: map_at_100
|
|
value: 15.461
|
|
- type: recall_at_1
|
|
value: 2.076
|
|
- type: recall_at_3
|
|
value: 6.7250000000000005
|
|
- type: recall_at_5
|
|
value: 9.808
|
|
- type: recall_at_10
|
|
value: 16.467000000000002
|
|
- type: recall_at_100
|
|
value: 45.109
|
|
- type: precision_at_1
|
|
value: 28.571
|
|
- type: precision_at_3
|
|
value: 29.252
|
|
- type: precision_at_5
|
|
value: 25.714
|
|
- type: precision_at_10
|
|
value: 23.265
|
|
- type: precision_at_100
|
|
value: 7.184
|
|
- type: mrr_at_1
|
|
value: 28.571
|
|
- type: mrr_at_3
|
|
value: 42.857
|
|
- type: mrr_at_5
|
|
value: 44.184
|
|
- type: mrr_at_10
|
|
value: 47.564
|
|
- type: mrr_at_100
|
|
value: 48.142
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/toxic_conversations_50k
|
|
name: MTEB ToxicConversationsClassification
|
|
config: default
|
|
split: test
|
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
|
|
metrics:
|
|
- type: accuracy
|
|
value: 68.43159999999999
|
|
- type: ap
|
|
value: 14.08119146524032
|
|
- type: f1
|
|
value: 53.26032318755336
|
|
- task:
|
|
type: Classification
|
|
dataset:
|
|
type: mteb/tweet_sentiment_extraction
|
|
name: MTEB TweetSentimentExtractionClassification
|
|
config: default
|
|
split: test
|
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
|
|
metrics:
|
|
- type: accuracy
|
|
value: 63.82852292020373
|
|
- type: f1
|
|
value: 64.14509521870399
|
|
- task:
|
|
type: Clustering
|
|
dataset:
|
|
type: mteb/twentynewsgroups-clustering
|
|
name: MTEB TwentyNewsgroupsClustering
|
|
config: default
|
|
split: test
|
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
|
|
metrics:
|
|
- type: v_measure
|
|
value: 55.252554461698566
|
|
- task:
|
|
type: PairClassification
|
|
dataset:
|
|
type: mteb/twittersemeval2015-pairclassification
|
|
name: MTEB TwitterSemEval2015
|
|
config: default
|
|
split: test
|
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
|
|
metrics:
|
|
- type: cos_sim_accuracy
|
|
value: 88.54383978065208
|
|
- type: cos_sim_ap
|
|
value: 81.67495128150328
|
|
- type: cos_sim_f1
|
|
value: 74.58161532864419
|
|
- type: cos_sim_precision
|
|
value: 69.00807899461401
|
|
- type: cos_sim_recall
|
|
value: 81.13456464379946
|
|
- type: dot_accuracy
|
|
value: 88.54383978065208
|
|
- type: dot_ap
|
|
value: 81.6748330747088
|
|
- type: dot_f1
|
|
value: 74.58161532864419
|
|
- type: dot_precision
|
|
value: 69.00807899461401
|
|
- type: dot_recall
|
|
value: 81.13456464379946
|
|
- type: euclidean_accuracy
|
|
value: 88.54383978065208
|
|
- type: euclidean_ap
|
|
value: 81.67496006818212
|
|
- type: euclidean_f1
|
|
value: 74.58161532864419
|
|
- type: euclidean_precision
|
|
value: 69.00807899461401
|
|
- type: euclidean_recall
|
|
value: 81.13456464379946
|
|
- type: manhattan_accuracy
|
|
value: 88.40674733265782
|
|
- type: manhattan_ap
|
|
value: 81.56036996969941
|
|
- type: manhattan_f1
|
|
value: 74.33063129452223
|
|
- type: manhattan_precision
|
|
value: 69.53125
|
|
- type: manhattan_recall
|
|
value: 79.84168865435356
|
|
- type: max_accuracy
|
|
value: 88.54383978065208
|
|
- type: max_ap
|
|
value: 81.67496006818212
|
|
- type: max_f1
|
|
value: 74.58161532864419
|
|
- task:
|
|
type: PairClassification
|
|
dataset:
|
|
type: mteb/twitterurlcorpus-pairclassification
|
|
name: MTEB TwitterURLCorpus
|
|
config: default
|
|
split: test
|
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
|
|
metrics:
|
|
- type: cos_sim_accuracy
|
|
value: 89.75627740908915
|
|
- type: cos_sim_ap
|
|
value: 87.41911504007292
|
|
- type: cos_sim_f1
|
|
value: 79.91742008969888
|
|
- type: cos_sim_precision
|
|
value: 74.31484178472131
|
|
- type: cos_sim_recall
|
|
value: 86.43363104404065
|
|
- type: dot_accuracy
|
|
value: 89.75627740908915
|
|
- type: dot_ap
|
|
value: 87.41910845717851
|
|
- type: dot_f1
|
|
value: 79.91742008969888
|
|
- type: dot_precision
|
|
value: 74.31484178472131
|
|
- type: dot_recall
|
|
value: 86.43363104404065
|
|
- type: euclidean_accuracy
|
|
value: 89.75627740908915
|
|
- type: euclidean_ap
|
|
value: 87.41912150448005
|
|
- type: euclidean_f1
|
|
value: 79.91742008969888
|
|
- type: euclidean_precision
|
|
value: 74.31484178472131
|
|
- type: euclidean_recall
|
|
value: 86.43363104404065
|
|
- type: manhattan_accuracy
|
|
value: 89.76597974152986
|
|
- type: manhattan_ap
|
|
value: 87.49835162128704
|
|
- type: manhattan_f1
|
|
value: 80.05401656994779
|
|
- type: manhattan_precision
|
|
value: 76.10158906390951
|
|
- type: manhattan_recall
|
|
value: 84.43948259932245
|
|
- type: max_accuracy
|
|
value: 89.76597974152986
|
|
- type: max_ap
|
|
value: 87.49835162128704
|
|
- type: max_f1
|
|
value: 80.05401656994779
|
|
language:
|
|
- en
|
|
license: mit
|
|
---
|
|
|
|
## SPEED-embedding-7b-instruct
|
|
|
|
[Little Giants: Synthesizing High-Quality Embedding Data at Scale](https://arxiv.org/pdf/2410.18634.pdf). Haonan Chen, Liang Wang, Nan Yang, Yutao Zhu, Ziliang Zhao, Furu Wei, Zhicheng Dou, arXiv 2024
|
|
|
|
This model has 32 layers and the embedding size is 4096.
|
|
|
|
## Usage
|
|
|
|
Below is an example to encode queries and passages from the MS-MARCO passage ranking dataset.
|
|
|
|
### Transformers
|
|
|
|
```python
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from torch import Tensor
|
|
from transformers import AutoTokenizer, AutoModel
|
|
|
|
|
|
def last_token_pool(last_hidden_states: Tensor,
|
|
attention_mask: Tensor) -> Tensor:
|
|
left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])
|
|
if left_padding:
|
|
return last_hidden_states[:, -1]
|
|
else:
|
|
sequence_lengths = attention_mask.sum(dim=1) - 1
|
|
batch_size = last_hidden_states.shape[0]
|
|
return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths]
|
|
|
|
|
|
def get_detailed_instruct(task_description: str, query: str) -> str:
|
|
return f'Instruct: {task_description}\nQuery: {query}'
|
|
|
|
|
|
# Each query must come with a one-sentence instruction that describes the task
|
|
task = 'Given a web search query, retrieve relevant passages that answer the query'
|
|
queries = [
|
|
get_detailed_instruct(task, 'how much protein should a female eat'),
|
|
get_detailed_instruct(task, 'summit define')
|
|
]
|
|
# No need to add instruction for retrieval documents
|
|
documents = [
|
|
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
|
|
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."
|
|
]
|
|
input_texts = queries + documents
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained('Haon-Chen/speed-embedding-7b-instruct')
|
|
model = AutoModel.from_pretrained('Haon-Chen/speed-embedding-7b-instruct')
|
|
|
|
max_length = 4096
|
|
# Tokenize the input texts
|
|
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
|
|
|
|
outputs = model(**batch_dict)
|
|
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
|
|
|
|
# normalize embeddings
|
|
embeddings = F.normalize(embeddings, p=2, dim=1)
|
|
scores = (embeddings[:2] @ embeddings[2:].T) * 100
|
|
print(scores.tolist())
|
|
```
|
|
|
|
## MTEB Benchmark Evaluation
|
|
|
|
Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results
|
|
on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB benchmark](https://arxiv.org/abs/2210.07316).
|
|
|
|
## FAQ
|
|
|
|
**1. Do I need to add instructions to the query?**
|
|
|
|
Yes, this is how the model is trained, otherwise you will see a performance degradation.
|
|
The task definition should be a one-sentence instruction that describes the task.
|
|
This is a way to customize text embeddings for different scenarios through natural language instructions.
|
|
|
|
Please check out [unilm/e5/utils.py](https://github.com/microsoft/unilm/blob/9c0f1ff7ca53431fe47d2637dfe253643d94185b/e5/utils.py#L106) for instructions we used for evaluation.
|
|
|
|
On the other hand, there is no need to add instructions to the document side.
|
|
|
|
**2. Why are my reproduced results slightly different from reported in the model card?**
|
|
|
|
Different versions of `transformers` and `pytorch` could cause negligible but non-zero performance differences.
|
|
|
|
**3. Where are the LoRA-only weights?**
|
|
|
|
You can find the LoRA-only weights at [https://huggingface.co/Haon-Chen/speed-embedding-7b-instruct/tree/main/lora](https://huggingface.co/Haon-Chen/speed-embedding-7b-instruct/tree/main/lora).
|
|
|
|
## Citation
|
|
|
|
If you find our paper or models helpful, please consider cite as follows:
|
|
|
|
```bibtex
|
|
@article{chen2024little,
|
|
title={Little Giants: Synthesizing High-Quality Embedding Data at Scale},
|
|
author={Chen, Haonan and Wang, Liang and Yang, Nan and Zhu, Yutao and Zhao, Ziliang and Wei, Furu and Dou, Zhicheng},
|
|
journal={arXiv preprint arXiv:2410.18634},
|
|
year={2024}
|
|
}
|
|
```
|
|
|
|
## Limitations
|
|
|
|
Using this model for inputs longer than 4096 tokens is not recommended.
|
|
|
