metadata
license: apache-2.0
model-index:
- name: vstackai-law-1
results:
- dataset:
config: default
name: MTEB AILACasedocs (default)
revision: 4106e6bcc72e0698d714ea8b101355e3e238431a
split: test
type: mteb/AILA_casedocs
metrics:
- type: main_score
value: 45.881
- type: map_at_1
value: 15.626000000000001
- type: map_at_10
value: 35.655
- type: map_at_100
value: 39.715
- type: map_at_1000
value: 39.889
- type: map_at_20
value: 37.102000000000004
- type: map_at_3
value: 27.089999999999996
- type: map_at_5
value: 31.025999999999996
- type: mrr_at_1
value: 46
- type: mrr_at_10
value: 54.604761904761915
- type: mrr_at_100
value: 55.25030966503531
- type: mrr_at_1000
value: 55.28445318355383
- type: mrr_at_20
value: 54.921136173767756
- type: mrr_at_3
value: 51.66666666666666
- type: mrr_at_5
value: 52.166666666666664
- type: nauc_map_at_1000_diff1
value: 15.57495845891449
- type: nauc_map_at_1000_max
value: 23.143422477463474
- type: nauc_map_at_1000_std
value: -8.945561626866498
- type: nauc_map_at_100_diff1
value: 15.506630125824048
- type: nauc_map_at_100_max
value: 23.282567495936846
- type: nauc_map_at_100_std
value: -8.911449134456035
- type: nauc_map_at_10_diff1
value: 15.618163394870141
- type: nauc_map_at_10_max
value: 22.07816596666465
- type: nauc_map_at_10_std
value: -10.298462021115187
- type: nauc_map_at_1_diff1
value: 13.37923142053057
- type: nauc_map_at_1_max
value: 13.483533976115272
- type: nauc_map_at_1_std
value: -6.849119244118052
- type: nauc_map_at_20_diff1
value: 15.678610092128679
- type: nauc_map_at_20_max
value: 23.01239046943845
- type: nauc_map_at_20_std
value: -10.384062772234483
- type: nauc_map_at_3_diff1
value: 21.805983515218617
- type: nauc_map_at_3_max
value: 15.777843249645866
- type: nauc_map_at_3_std
value: -15.963631005395404
- type: nauc_map_at_5_diff1
value: 20.668846473212096
- type: nauc_map_at_5_max
value: 21.465743982369055
- type: nauc_map_at_5_std
value: -13.06765002103128
- type: nauc_mrr_at_1000_diff1
value: 18.950498468308886
- type: nauc_mrr_at_1000_max
value: 26.651641047762148
- type: nauc_mrr_at_1000_std
value: -4.101624493270528
- type: nauc_mrr_at_100_diff1
value: 18.92883047034938
- type: nauc_mrr_at_100_max
value: 26.722955290521792
- type: nauc_mrr_at_100_std
value: -4.027052366530224
- type: nauc_mrr_at_10_diff1
value: 18.60683936270885
- type: nauc_mrr_at_10_max
value: 26.16665005770204
- type: nauc_mrr_at_10_std
value: -4.698063142209917
- type: nauc_mrr_at_1_diff1
value: 17.54487398491498
- type: nauc_mrr_at_1_max
value: 26.099182675882364
- type: nauc_mrr_at_1_std
value: -7.442642768915407
- type: nauc_mrr_at_20_diff1
value: 19.0367989604306
- type: nauc_mrr_at_20_max
value: 26.630737835733974
- type: nauc_mrr_at_20_std
value: -4.11566988702261
- type: nauc_mrr_at_3_diff1
value: 21.267024293259933
- type: nauc_mrr_at_3_max
value: 26.69418345236406
- type: nauc_mrr_at_3_std
value: -3.612150752894062
- type: nauc_mrr_at_5_diff1
value: 19.90001865151827
- type: nauc_mrr_at_5_max
value: 27.66941220085612
- type: nauc_mrr_at_5_std
value: -2.7974974728343893
- type: nauc_ndcg_at_1000_diff1
value: 15.901300620232577
- type: nauc_ndcg_at_1000_max
value: 25.612869768853642
- type: nauc_ndcg_at_1000_std
value: -5.309417560619307
- type: nauc_ndcg_at_100_diff1
value: 14.95954368378741
- type: nauc_ndcg_at_100_max
value: 27.23572900207926
- type: nauc_ndcg_at_100_std
value: -3.5899694438310528
- type: nauc_ndcg_at_10_diff1
value: 14.060694457887331
- type: nauc_ndcg_at_10_max
value: 23.459115385918828
- type: nauc_ndcg_at_10_std
value: -7.547036607345263
- type: nauc_ndcg_at_1_diff1
value: 17.54487398491498
- type: nauc_ndcg_at_1_max
value: 26.099182675882364
- type: nauc_ndcg_at_1_std
value: -7.442642768915407
- type: nauc_ndcg_at_20_diff1
value: 14.406205372080914
- type: nauc_ndcg_at_20_max
value: 26.249115297438518
- type: nauc_ndcg_at_20_std
value: -6.498773153759981
- type: nauc_ndcg_at_3_diff1
value: 15.967994683338851
- type: nauc_ndcg_at_3_max
value: 23.71128286998367
- type: nauc_ndcg_at_3_std
value: -7.166430376561868
- type: nauc_ndcg_at_5_diff1
value: 16.371530698236622
- type: nauc_ndcg_at_5_max
value: 25.725132293180273
- type: nauc_ndcg_at_5_std
value: -3.8545166535972943
- type: nauc_precision_at_1000_diff1
value: -10.41717795536071
- type: nauc_precision_at_1000_max
value: 4.7029594790402625
- type: nauc_precision_at_1000_std
value: 14.173763073581878
- type: nauc_precision_at_100_diff1
value: -11.074302280828615
- type: nauc_precision_at_100_max
value: 8.244938186343832
- type: nauc_precision_at_100_std
value: 15.285975835662343
- type: nauc_precision_at_10_diff1
value: -10.837400355480538
- type: nauc_precision_at_10_max
value: 22.832192981934664
- type: nauc_precision_at_10_std
value: 10.05034983325703
- type: nauc_precision_at_1_diff1
value: 17.54487398491498
- type: nauc_precision_at_1_max
value: 26.099182675882364
- type: nauc_precision_at_1_std
value: -7.442642768915407
- type: nauc_precision_at_20_diff1
value: -9.411528844386767
- type: nauc_precision_at_20_max
value: 22.63946047993228
- type: nauc_precision_at_20_std
value: 10.267476763547872
- type: nauc_precision_at_3_diff1
value: 12.089219490235562
- type: nauc_precision_at_3_max
value: 21.867564149285098
- type: nauc_precision_at_3_std
value: -7.900490900126483
- type: nauc_precision_at_5_diff1
value: 6.088160593562308
- type: nauc_precision_at_5_max
value: 25.24343016238549
- type: nauc_precision_at_5_std
value: -0.2798984072447858
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: -5.301190251888983
- type: nauc_recall_at_100_max
value: 61.552650883513884
- type: nauc_recall_at_100_std
value: 41.63529646496176
- type: nauc_recall_at_10_diff1
value: 3.4992492433968714
- type: nauc_recall_at_10_max
value: 14.516376506894568
- type: nauc_recall_at_10_std
value: -8.155441184017405
- type: nauc_recall_at_1_diff1
value: 13.37923142053057
- type: nauc_recall_at_1_max
value: 13.483533976115272
- type: nauc_recall_at_1_std
value: -6.849119244118052
- type: nauc_recall_at_20_diff1
value: 3.2751273323204697
- type: nauc_recall_at_20_max
value: 22.900211937193326
- type: nauc_recall_at_20_std
value: -1.1031026054527242
- type: nauc_recall_at_3_diff1
value: 22.819746834360018
- type: nauc_recall_at_3_max
value: 13.682536574839425
- type: nauc_recall_at_3_std
value: -14.733108963214727
- type: nauc_recall_at_5_diff1
value: 22.45051937631727
- type: nauc_recall_at_5_max
value: 24.19540105383745
- type: nauc_recall_at_5_std
value: -7.468653169210884
- type: ndcg_at_1
value: 46
- type: ndcg_at_10
value: 45.881
- type: ndcg_at_100
value: 57.25900000000001
- type: ndcg_at_1000
value: 58.528000000000006
- type: ndcg_at_20
value: 48.413000000000004
- type: ndcg_at_3
value: 42.709
- type: ndcg_at_5
value: 41.253
- type: precision_at_1
value: 46
- type: precision_at_10
value: 19.400000000000002
- type: precision_at_100
value: 3.66
- type: precision_at_1000
value: 0.38999999999999996
- type: precision_at_20
value: 11.3
- type: precision_at_3
value: 34.666999999999994
- type: precision_at_5
value: 26.8
- type: recall_at_1
value: 15.626000000000001
- type: recall_at_10
value: 53.722
- type: recall_at_100
value: 93.312
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 62.011
- type: recall_at_3
value: 30.44
- type: recall_at_5
value: 35.836
task:
type: Retrieval
- dataset:
config: default
name: MTEB AILAStatutes (default)
revision: ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e
split: test
type: mteb/AILA_statutes
metrics:
- type: main_score
value: 47.122
- type: map_at_1
value: 11.4
- type: map_at_10
value: 30.796
- type: map_at_100
value: 37.822
- type: map_at_1000
value: 37.822
- type: map_at_20
value: 34.211000000000006
- type: map_at_3
value: 20.089000000000002
- type: map_at_5
value: 25.304
- type: mrr_at_1
value: 50
- type: mrr_at_10
value: 67.45714285714284
- type: mrr_at_100
value: 67.45714285714284
- type: mrr_at_1000
value: 67.45714285714284
- type: mrr_at_20
value: 67.45714285714284
- type: mrr_at_3
value: 63.66666666666666
- type: mrr_at_5
value: 66.06666666666666
- type: nauc_map_at_1000_diff1
value: 13.231953724763489
- type: nauc_map_at_1000_max
value: 36.25413964500182
- type: nauc_map_at_1000_std
value: 16.12109134339587
- type: nauc_map_at_100_diff1
value: 13.231953724763489
- type: nauc_map_at_100_max
value: 36.25413964500182
- type: nauc_map_at_100_std
value: 16.12109134339587
- type: nauc_map_at_10_diff1
value: 17.94211109348458
- type: nauc_map_at_10_max
value: 39.28669806176402
- type: nauc_map_at_10_std
value: 13.594341769002869
- type: nauc_map_at_1_diff1
value: 30.100915555842434
- type: nauc_map_at_1_max
value: 33.83683525374127
- type: nauc_map_at_1_std
value: 20.814102357206625
- type: nauc_map_at_20_diff1
value: 13.652796877938734
- type: nauc_map_at_20_max
value: 35.83818028836039
- type: nauc_map_at_20_std
value: 14.37656196108266
- type: nauc_map_at_3_diff1
value: 34.486067078257975
- type: nauc_map_at_3_max
value: 31.543224895382878
- type: nauc_map_at_3_std
value: -5.527795564447535
- type: nauc_map_at_5_diff1
value: 26.851187948582773
- type: nauc_map_at_5_max
value: 38.89484793235884
- type: nauc_map_at_5_std
value: 3.4001045162742694
- type: nauc_mrr_at_1000_diff1
value: 26.39698483562357
- type: nauc_mrr_at_1000_max
value: 30.835723241593847
- type: nauc_mrr_at_1000_std
value: 20.683462333907705
- type: nauc_mrr_at_100_diff1
value: 26.39698483562357
- type: nauc_mrr_at_100_max
value: 30.835723241593847
- type: nauc_mrr_at_100_std
value: 20.683462333907705
- type: nauc_mrr_at_10_diff1
value: 26.39698483562357
- type: nauc_mrr_at_10_max
value: 30.835723241593847
- type: nauc_mrr_at_10_std
value: 20.683462333907705
- type: nauc_mrr_at_1_diff1
value: 27.95469686875413
- type: nauc_mrr_at_1_max
value: 31.818787475016645
- type: nauc_mrr_at_1_std
value: 19.56295802798131
- type: nauc_mrr_at_20_diff1
value: 26.39698483562357
- type: nauc_mrr_at_20_max
value: 30.835723241593847
- type: nauc_mrr_at_20_std
value: 20.683462333907705
- type: nauc_mrr_at_3_diff1
value: 27.65080554514802
- type: nauc_mrr_at_3_max
value: 32.553977955855814
- type: nauc_mrr_at_3_std
value: 18.78114112837834
- type: nauc_mrr_at_5_diff1
value: 26.384655469886393
- type: nauc_mrr_at_5_max
value: 31.664536690997192
- type: nauc_mrr_at_5_std
value: 23.119445085922425
- type: nauc_ndcg_at_1000_diff1
value: 13.624306343440482
- type: nauc_ndcg_at_1000_max
value: 33.949287066404395
- type: nauc_ndcg_at_1000_std
value: 18.873031773538894
- type: nauc_ndcg_at_100_diff1
value: 13.624306343440482
- type: nauc_ndcg_at_100_max
value: 33.949287066404395
- type: nauc_ndcg_at_100_std
value: 18.873031773538894
- type: nauc_ndcg_at_10_diff1
value: 17.95882248371854
- type: nauc_ndcg_at_10_max
value: 38.42611161111767
- type: nauc_ndcg_at_10_std
value: 18.382289466996596
- type: nauc_ndcg_at_1_diff1
value: 27.95469686875413
- type: nauc_ndcg_at_1_max
value: 31.818787475016645
- type: nauc_ndcg_at_1_std
value: 19.56295802798131
- type: nauc_ndcg_at_20_diff1
value: 10.108403510160493
- type: nauc_ndcg_at_20_max
value: 33.30645155135527
- type: nauc_ndcg_at_20_std
value: 19.106968240050197
- type: nauc_ndcg_at_3_diff1
value: 26.02063027004639
- type: nauc_ndcg_at_3_max
value: 29.85933476459198
- type: nauc_ndcg_at_3_std
value: 0.04135555148143812
- type: nauc_ndcg_at_5_diff1
value: 24.51049558894743
- type: nauc_ndcg_at_5_max
value: 39.030804792387535
- type: nauc_ndcg_at_5_std
value: 10.626686520599742
- type: nauc_precision_at_1000_diff1
value: -52.64814035601246
- type: nauc_precision_at_1000_max
value: -13.856314374050852
- type: nauc_precision_at_1000_std
value: 17.593896928340197
- type: nauc_precision_at_100_diff1
value: -52.648140356011965
- type: nauc_precision_at_100_max
value: -13.856314374050674
- type: nauc_precision_at_100_std
value: 17.593896928340225
- type: nauc_precision_at_10_diff1
value: -11.6303596329846
- type: nauc_precision_at_10_max
value: 26.23949752159085
- type: nauc_precision_at_10_std
value: 25.41101749534196
- type: nauc_precision_at_1_diff1
value: 27.95469686875413
- type: nauc_precision_at_1_max
value: 31.818787475016645
- type: nauc_precision_at_1_std
value: 19.56295802798131
- type: nauc_precision_at_20_diff1
value: -26.535056035995154
- type: nauc_precision_at_20_max
value: 11.922354543765811
- type: nauc_precision_at_20_std
value: 23.095498544921597
- type: nauc_precision_at_3_diff1
value: 21.2869202175386
- type: nauc_precision_at_3_max
value: 31.7866363919357
- type: nauc_precision_at_3_std
value: -3.858197066525092
- type: nauc_precision_at_5_diff1
value: 1.8680405985613282
- type: nauc_precision_at_5_max
value: 35.366076833199564
- type: nauc_precision_at_5_std
value: 16.706786604164005
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: .nan
- type: nauc_recall_at_100_max
value: .nan
- type: nauc_recall_at_100_std
value: .nan
- type: nauc_recall_at_10_diff1
value: 8.112331861587329
- type: nauc_recall_at_10_max
value: 34.89423965499535
- type: nauc_recall_at_10_std
value: 21.6246021152069
- type: nauc_recall_at_1_diff1
value: 30.100915555842434
- type: nauc_recall_at_1_max
value: 33.83683525374127
- type: nauc_recall_at_1_std
value: 20.814102357206625
- type: nauc_recall_at_20_diff1
value: -11.669499704119648
- type: nauc_recall_at_20_max
value: 22.30570195122146
- type: nauc_recall_at_20_std
value: 23.238122093323348
- type: nauc_recall_at_3_diff1
value: 33.29080234285882
- type: nauc_recall_at_3_max
value: 30.134100912744604
- type: nauc_recall_at_3_std
value: -12.43448322513982
- type: nauc_recall_at_5_diff1
value: 18.77021416083913
- type: nauc_recall_at_5_max
value: 40.54971590909089
- type: nauc_recall_at_5_std
value: 9.419471153846175
- type: ndcg_at_1
value: 50
- type: ndcg_at_10
value: 47.122
- type: ndcg_at_100
value: 63.287000000000006
- type: ndcg_at_1000
value: 63.287000000000006
- type: ndcg_at_20
value: 53.172
- type: ndcg_at_3
value: 39.36
- type: ndcg_at_5
value: 38.586
- type: precision_at_1
value: 50
- type: precision_at_10
value: 23
- type: precision_at_100
value: 4.34
- type: precision_at_1000
value: 0.434
- type: precision_at_20
value: 14.799999999999999
- type: precision_at_3
value: 35.333
- type: precision_at_5
value: 31.2
- type: recall_at_1
value: 11.4
- type: recall_at_10
value: 52.532999999999994
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 67
- type: recall_at_3
value: 25.467000000000002
- type: recall_at_5
value: 36.467
task:
type: Retrieval
- dataset:
config: default
name: MTEB GerDaLIRSmall (default)
revision: 48327de6ee192e9610f3069789719788957c7abd
split: test
type: mteb/GerDaLIRSmall
metrics:
- type: main_score
value: 50.041999999999994
- type: map_at_1
value: 32.192
- type: map_at_10
value: 44.156
- type: map_at_100
value: 45.08
- type: map_at_1000
value: 45.126
- type: map_at_20
value: 44.743
- type: map_at_3
value: 40.886
- type: map_at_5
value: 42.834
- type: mrr_at_1
value: 35.67108059506294
- type: mrr_at_10
value: 46.08966424562292
- type: mrr_at_100
value: 46.85163949774394
- type: mrr_at_1000
value: 46.88987506054722
- type: mrr_at_20
value: 46.56428992878071
- type: mrr_at_3
value: 43.33415072748076
- type: mrr_at_5
value: 44.98406081412419
- type: nauc_map_at_1000_diff1
value: 47.08157584565528
- type: nauc_map_at_1000_max
value: 43.206128601312486
- type: nauc_map_at_1000_std
value: 4.745982286256461
- type: nauc_map_at_100_diff1
value: 47.07078139806274
- type: nauc_map_at_100_max
value: 43.23099197560626
- type: nauc_map_at_100_std
value: 4.781780686055857
- type: nauc_map_at_10_diff1
value: 47.06888215588044
- type: nauc_map_at_10_max
value: 42.993085843512404
- type: nauc_map_at_10_std
value: 4.137168566460313
- type: nauc_map_at_1_diff1
value: 50.99589527197208
- type: nauc_map_at_1_max
value: 35.91682256310639
- type: nauc_map_at_1_std
value: -1.167131810256268
- type: nauc_map_at_20_diff1
value: 47.04454709015284
- type: nauc_map_at_20_max
value: 43.25308050978819
- type: nauc_map_at_20_std
value: 4.655176251983949
- type: nauc_map_at_3_diff1
value: 47.477586403020254
- type: nauc_map_at_3_max
value: 41.070269193209505
- type: nauc_map_at_3_std
value: 1.8419013437125402
- type: nauc_map_at_5_diff1
value: 47.02373499346413
- type: nauc_map_at_5_max
value: 42.16258581830696
- type: nauc_map_at_5_std
value: 3.0623218224529687
- type: nauc_mrr_at_1000_diff1
value: 47.80488822997101
- type: nauc_mrr_at_1000_max
value: 44.30020606841032
- type: nauc_mrr_at_1000_std
value: 5.636122646285231
- type: nauc_mrr_at_100_diff1
value: 47.79392263287222
- type: nauc_mrr_at_100_max
value: 44.32088767447976
- type: nauc_mrr_at_100_std
value: 5.668194634702436
- type: nauc_mrr_at_10_diff1
value: 47.76641281589811
- type: nauc_mrr_at_10_max
value: 44.279337066083805
- type: nauc_mrr_at_10_std
value: 5.324066003258172
- type: nauc_mrr_at_1_diff1
value: 51.827466333842494
- type: nauc_mrr_at_1_max
value: 39.9726801468231
- type: nauc_mrr_at_1_std
value: 0.7288717627709872
- type: nauc_mrr_at_20_diff1
value: 47.764880161368986
- type: nauc_mrr_at_20_max
value: 44.37823842067007
- type: nauc_mrr_at_20_std
value: 5.622281146435594
- type: nauc_mrr_at_3_diff1
value: 48.09440765276413
- type: nauc_mrr_at_3_max
value: 43.08121834325808
- type: nauc_mrr_at_3_std
value: 3.463004783984404
- type: nauc_mrr_at_5_diff1
value: 47.722106520188504
- type: nauc_mrr_at_5_max
value: 43.75524931306371
- type: nauc_mrr_at_5_std
value: 4.544429473258565
- type: nauc_ndcg_at_1000_diff1
value: 46.173190405196785
- type: nauc_ndcg_at_1000_max
value: 45.74626009789115
- type: nauc_ndcg_at_1000_std
value: 8.766536922367903
- type: nauc_ndcg_at_100_diff1
value: 45.83562302519218
- type: nauc_ndcg_at_100_max
value: 46.439699221515376
- type: nauc_ndcg_at_100_std
value: 9.948770560612058
- type: nauc_ndcg_at_10_diff1
value: 45.75120503110639
- type: nauc_ndcg_at_10_max
value: 45.97030387145503
- type: nauc_ndcg_at_10_std
value: 7.585101690722181
- type: nauc_ndcg_at_1_diff1
value: 51.80241384384428
- type: nauc_ndcg_at_1_max
value: 39.954443617615624
- type: nauc_ndcg_at_1_std
value: 0.6967907136634796
- type: nauc_ndcg_at_20_diff1
value: 45.653758645238895
- type: nauc_ndcg_at_20_max
value: 46.71180118809166
- type: nauc_ndcg_at_20_std
value: 9.250603495033369
- type: nauc_ndcg_at_3_diff1
value: 46.451364520561924
- type: nauc_ndcg_at_3_max
value: 42.65149578608251
- type: nauc_ndcg_at_3_std
value: 3.18214445444501
- type: nauc_ndcg_at_5_diff1
value: 45.70310380498345
- type: nauc_ndcg_at_5_max
value: 44.210803603105866
- type: nauc_ndcg_at_5_std
value: 5.164266026630885
- type: nauc_precision_at_1000_diff1
value: 1.5935204591909151
- type: nauc_precision_at_1000_max
value: 24.49290419273546
- type: nauc_precision_at_1000_std
value: 23.419221041007578
- type: nauc_precision_at_100_diff1
value: 11.340541238515147
- type: nauc_precision_at_100_max
value: 38.03551764407295
- type: nauc_precision_at_100_std
value: 31.0523811487812
- type: nauc_precision_at_10_diff1
value: 25.73178507519107
- type: nauc_precision_at_10_max
value: 46.868875006839545
- type: nauc_precision_at_10_std
value: 20.062754134173872
- type: nauc_precision_at_1_diff1
value: 51.80241384384428
- type: nauc_precision_at_1_max
value: 39.954443617615624
- type: nauc_precision_at_1_std
value: 0.6967907136634796
- type: nauc_precision_at_20_diff1
value: 20.43139551018563
- type: nauc_precision_at_20_max
value: 46.00136497143788
- type: nauc_precision_at_20_std
value: 26.43348589267883
- type: nauc_precision_at_3_diff1
value: 36.477540595476235
- type: nauc_precision_at_3_max
value: 45.27812276802484
- type: nauc_precision_at_3_std
value: 7.988192977691942
- type: nauc_precision_at_5_diff1
value: 30.711545494807886
- type: nauc_precision_at_5_max
value: 45.84532014309742
- type: nauc_precision_at_5_std
value: 12.810198584032126
- type: nauc_recall_at_1000_diff1
value: 34.222675530936584
- type: nauc_recall_at_1000_max
value: 64.84714200625044
- type: nauc_recall_at_1000_std
value: 58.96182247732879
- type: nauc_recall_at_100_diff1
value: 35.36855555958951
- type: nauc_recall_at_100_max
value: 62.91068693686705
- type: nauc_recall_at_100_std
value: 46.54158522159893
- type: nauc_recall_at_10_diff1
value: 39.32976276229237
- type: nauc_recall_at_10_max
value: 53.86613555456697
- type: nauc_recall_at_10_std
value: 18.601960247067957
- type: nauc_recall_at_1_diff1
value: 50.99589527197208
- type: nauc_recall_at_1_max
value: 35.91682256310639
- type: nauc_recall_at_1_std
value: -1.167131810256268
- type: nauc_recall_at_20_diff1
value: 37.57662491377409
- type: nauc_recall_at_20_max
value: 58.543627483010816
- type: nauc_recall_at_20_std
value: 28.511725371993336
- type: nauc_recall_at_3_diff1
value: 42.4797724119925
- type: nauc_recall_at_3_max
value: 43.96696921591431
- type: nauc_recall_at_3_std
value: 4.855189723191562
- type: nauc_recall_at_5_diff1
value: 40.20239865372907
- type: nauc_recall_at_5_max
value: 47.80150483467517
- type: nauc_recall_at_5_std
value: 9.903701080938268
- type: ndcg_at_1
value: 35.679
- type: ndcg_at_10
value: 50.041999999999994
- type: ndcg_at_100
value: 54.193000000000005
- type: ndcg_at_1000
value: 55.444
- type: ndcg_at_20
value: 51.961
- type: ndcg_at_3
value: 43.942
- type: ndcg_at_5
value: 47.121
- type: precision_at_1
value: 35.679
- type: precision_at_10
value: 7.733
- type: precision_at_100
value: 0.997
- type: precision_at_1000
value: 0.11100000000000002
- type: precision_at_20
value: 4.304
- type: precision_at_3
value: 19.165
- type: precision_at_5
value: 13.366
- type: recall_at_1
value: 32.192
- type: recall_at_10
value: 66.226
- type: recall_at_100
value: 84.985
- type: recall_at_1000
value: 94.76299999999999
- type: recall_at_20
value: 73.456
- type: recall_at_3
value: 50.071
- type: recall_at_5
value: 57.60999999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB LeCaRDv2 (default)
revision: b78e18688c3d012a33dc3676597c1d1b2243ce1c
split: test
type: mteb/LeCaRDv2
metrics:
- type: main_score
value: 75.955
- type: map_at_1
value: 4.099
- type: map_at_10
value: 27.639999999999997
- type: map_at_100
value: 63.919000000000004
- type: map_at_1000
value: 64.742
- type: map_at_20
value: 43.64
- type: map_at_3
value: 11.017000000000001
- type: map_at_5
value: 16.487
- type: mrr_at_1
value: 86.79245283018868
- type: mrr_at_10
value: 91.09613656783468
- type: mrr_at_100
value: 91.24099956647127
- type: mrr_at_1000
value: 91.24099956647127
- type: mrr_at_20
value: 91.24099956647127
- type: mrr_at_3
value: 90.67085953878407
- type: mrr_at_5
value: 90.7966457023061
- type: nauc_map_at_1000_diff1
value: -0.1165116760309431
- type: nauc_map_at_1000_max
value: 36.09282039931682
- type: nauc_map_at_1000_std
value: 16.717641031560422
- type: nauc_map_at_100_diff1
value: -0.1977916865120706
- type: nauc_map_at_100_max
value: 35.95558832098128
- type: nauc_map_at_100_std
value: 17.15965620332017
- type: nauc_map_at_10_diff1
value: 11.88383387650193
- type: nauc_map_at_10_max
value: 19.323749519368256
- type: nauc_map_at_10_std
value: -13.368970916127784
- type: nauc_map_at_1_diff1
value: 12.468762885123516
- type: nauc_map_at_1_max
value: 5.597229875368963
- type: nauc_map_at_1_std
value: -25.835685303037202
- type: nauc_map_at_20_diff1
value: 4.625997058170944
- type: nauc_map_at_20_max
value: 23.42312594320493
- type: nauc_map_at_20_std
value: -1.2842557193275415
- type: nauc_map_at_3_diff1
value: 16.457500025698952
- type: nauc_map_at_3_max
value: 12.260630248199256
- type: nauc_map_at_3_std
value: -23.60500260290979
- type: nauc_map_at_5_diff1
value: 15.924351493208256
- type: nauc_map_at_5_max
value: 17.553142481774472
- type: nauc_map_at_5_std
value: -17.604184344296943
- type: nauc_mrr_at_1000_diff1
value: 23.24803044899622
- type: nauc_mrr_at_1000_max
value: 67.6449236362031
- type: nauc_mrr_at_1000_std
value: 18.966799318760344
- type: nauc_mrr_at_100_diff1
value: 23.24803044899622
- type: nauc_mrr_at_100_max
value: 67.6449236362031
- type: nauc_mrr_at_100_std
value: 18.966799318760344
- type: nauc_mrr_at_10_diff1
value: 22.7515823299183
- type: nauc_mrr_at_10_max
value: 67.04946200016059
- type: nauc_mrr_at_10_std
value: 19.187654338444936
- type: nauc_mrr_at_1_diff1
value: 29.056072093682022
- type: nauc_mrr_at_1_max
value: 68.55895633768269
- type: nauc_mrr_at_1_std
value: 15.092113581290862
- type: nauc_mrr_at_20_diff1
value: 23.24803044899622
- type: nauc_mrr_at_20_max
value: 67.6449236362031
- type: nauc_mrr_at_20_std
value: 18.966799318760344
- type: nauc_mrr_at_3_diff1
value: 23.287597081925508
- type: nauc_mrr_at_3_max
value: 68.05169166666252
- type: nauc_mrr_at_3_std
value: 19.785092034648617
- type: nauc_mrr_at_5_diff1
value: 23.11480929003818
- type: nauc_mrr_at_5_max
value: 67.99383983419222
- type: nauc_mrr_at_5_std
value: 19.052480486459373
- type: nauc_ndcg_at_1000_diff1
value: 1.5782017453966002
- type: nauc_ndcg_at_1000_max
value: 41.5302298842471
- type: nauc_ndcg_at_1000_std
value: 20.50865790647901
- type: nauc_ndcg_at_100_diff1
value: -2.1894348746609804
- type: nauc_ndcg_at_100_max
value: 37.14267089909159
- type: nauc_ndcg_at_100_std
value: 25.276579300026942
- type: nauc_ndcg_at_10_diff1
value: 4.916325074632776
- type: nauc_ndcg_at_10_max
value: 41.10158619113785
- type: nauc_ndcg_at_10_std
value: 12.855454525343141
- type: nauc_ndcg_at_1_diff1
value: 29.056072093682022
- type: nauc_ndcg_at_1_max
value: 68.55895633768269
- type: nauc_ndcg_at_1_std
value: 15.092113581290862
- type: nauc_ndcg_at_20_diff1
value: -0.7532516396275979
- type: nauc_ndcg_at_20_max
value: 32.78457335101816
- type: nauc_ndcg_at_20_std
value: 13.463927254088695
- type: nauc_ndcg_at_3_diff1
value: 12.68114834388015
- type: nauc_ndcg_at_3_max
value: 54.41470867032895
- type: nauc_ndcg_at_3_std
value: 18.565606565635647
- type: nauc_ndcg_at_5_diff1
value: 10.193606786989358
- type: nauc_ndcg_at_5_max
value: 51.401706187544825
- type: nauc_ndcg_at_5_std
value: 19.805481146848177
- type: nauc_precision_at_1000_diff1
value: -11.29314155191534
- type: nauc_precision_at_1000_max
value: 36.90077388824628
- type: nauc_precision_at_1000_std
value: 42.40123140196757
- type: nauc_precision_at_100_diff1
value: -12.974354162087675
- type: nauc_precision_at_100_max
value: 34.81416396301287
- type: nauc_precision_at_100_std
value: 45.724028299929955
- type: nauc_precision_at_10_diff1
value: 0.25354300446169603
- type: nauc_precision_at_10_max
value: 34.7650903227545
- type: nauc_precision_at_10_std
value: 14.426822867124692
- type: nauc_precision_at_1_diff1
value: 29.056072093682022
- type: nauc_precision_at_1_max
value: 68.55895633768269
- type: nauc_precision_at_1_std
value: 15.092113581290862
- type: nauc_precision_at_20_diff1
value: -4.398113098730461
- type: nauc_precision_at_20_max
value: 29.644722534077783
- type: nauc_precision_at_20_std
value: 20.498896408366953
- type: nauc_precision_at_3_diff1
value: 8.511537696975939
- type: nauc_precision_at_3_max
value: 51.02291732645718
- type: nauc_precision_at_3_std
value: 19.018537614881883
- type: nauc_precision_at_5_diff1
value: 5.968041335751909
- type: nauc_precision_at_5_max
value: 47.08751216174023
- type: nauc_precision_at_5_std
value: 21.422128104399597
- type: nauc_recall_at_1000_diff1
value: -114.68537043092684
- type: nauc_recall_at_1000_max
value: -17.258899842714303
- type: nauc_recall_at_1000_std
value: 35.64737342608127
- type: nauc_recall_at_100_diff1
value: -21.918374369212724
- type: nauc_recall_at_100_max
value: 9.323860457981256
- type: nauc_recall_at_100_std
value: 41.88023148573863
- type: nauc_recall_at_10_diff1
value: 6.872030552051965
- type: nauc_recall_at_10_max
value: 3.87403864340295
- type: nauc_recall_at_10_std
value: -21.003946521947647
- type: nauc_recall_at_1_diff1
value: 12.468762885123516
- type: nauc_recall_at_1_max
value: 5.597229875368963
- type: nauc_recall_at_1_std
value: -25.835685303037202
- type: nauc_recall_at_20_diff1
value: -0.957514361288361
- type: nauc_recall_at_20_max
value: 6.322474574744254
- type: nauc_recall_at_20_std
value: -7.517845037342536
- type: nauc_recall_at_3_diff1
value: 11.6795437402863
- type: nauc_recall_at_3_max
value: 4.802113241170994
- type: nauc_recall_at_3_std
value: -25.562583690610108
- type: nauc_recall_at_5_diff1
value: 11.34475005606284
- type: nauc_recall_at_5_max
value: 8.318487558973697
- type: nauc_recall_at_5_std
value: -21.462212735159852
- type: ndcg_at_1
value: 86.792
- type: ndcg_at_10
value: 75.955
- type: ndcg_at_100
value: 83.532
- type: ndcg_at_1000
value: 86.085
- type: ndcg_at_20
value: 70.61699999999999
- type: ndcg_at_3
value: 83.672
- type: ndcg_at_5
value: 80.988
- type: precision_at_1
value: 86.792
- type: precision_at_10
value: 71.572
- type: precision_at_100
value: 22.962
- type: precision_at_1000
value: 2.448
- type: precision_at_20
value: 62.075
- type: precision_at_3
value: 83.019
- type: precision_at_5
value: 79.119
- type: recall_at_1
value: 4.099
- type: recall_at_10
value: 30.963
- type: recall_at_100
value: 93.66799999999999
- type: recall_at_1000
value: 99.91
- type: recall_at_20
value: 51.763999999999996
- type: recall_at_3
value: 11.644
- type: recall_at_5
value: 17.646
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalBenchConsumerContractsQA (default)
revision: b23590301ec94e8087e2850b21d43d4956b1cca9
split: test
type: mteb/legalbench_consumer_contracts_qa
metrics:
- type: main_score
value: 81.722
- type: map_at_1
value: 66.919
- type: map_at_10
value: 77.11099999999999
- type: map_at_100
value: 77.365
- type: map_at_1000
value: 77.365
- type: map_at_20
value: 77.32
- type: map_at_3
value: 75.042
- type: map_at_5
value: 76.519
- type: mrr_at_1
value: 67.17171717171718
- type: mrr_at_10
value: 77.23735369568702
- type: mrr_at_100
value: 77.49038149701025
- type: mrr_at_1000
value: 77.49038149701025
- type: mrr_at_20
value: 77.44484086340144
- type: mrr_at_3
value: 75.16835016835017
- type: mrr_at_5
value: 76.64562289562288
- type: nauc_map_at_1000_diff1
value: 52.97349214688111
- type: nauc_map_at_1000_max
value: 36.6960251962721
- type: nauc_map_at_1000_std
value: 5.3061037068017995
- type: nauc_map_at_100_diff1
value: 52.97349214688111
- type: nauc_map_at_100_max
value: 36.6960251962721
- type: nauc_map_at_100_std
value: 5.3061037068017995
- type: nauc_map_at_10_diff1
value: 52.75678159019962
- type: nauc_map_at_10_max
value: 36.540933731741696
- type: nauc_map_at_10_std
value: 5.301224105047955
- type: nauc_map_at_1_diff1
value: 57.67214955091278
- type: nauc_map_at_1_max
value: 35.78220074855998
- type: nauc_map_at_1_std
value: 3.4210250671003326
- type: nauc_map_at_20_diff1
value: 52.932588393157765
- type: nauc_map_at_20_max
value: 36.63003364305272
- type: nauc_map_at_20_std
value: 5.379295465437984
- type: nauc_map_at_3_diff1
value: 52.480331430472546
- type: nauc_map_at_3_max
value: 38.091587355109084
- type: nauc_map_at_3_std
value: 5.195570015494146
- type: nauc_map_at_5_diff1
value: 51.82108883079516
- type: nauc_map_at_5_max
value: 36.87971971720586
- type: nauc_map_at_5_std
value: 6.1809111678690725
- type: nauc_mrr_at_1000_diff1
value: 52.50999035933979
- type: nauc_mrr_at_1000_max
value: 36.34762852524654
- type: nauc_mrr_at_1000_std
value: 4.968842635399586
- type: nauc_mrr_at_100_diff1
value: 52.50999035933979
- type: nauc_mrr_at_100_max
value: 36.34762852524654
- type: nauc_mrr_at_100_std
value: 4.968842635399586
- type: nauc_mrr_at_10_diff1
value: 52.30508629262912
- type: nauc_mrr_at_10_max
value: 36.209224098805336
- type: nauc_mrr_at_10_std
value: 4.966404389655018
- type: nauc_mrr_at_1_diff1
value: 57.04712337185792
- type: nauc_mrr_at_1_max
value: 35.30097575331179
- type: nauc_mrr_at_1_std
value: 2.9296181705495568
- type: nauc_mrr_at_20_diff1
value: 52.4700212348044
- type: nauc_mrr_at_20_max
value: 36.282215725584074
- type: nauc_mrr_at_20_std
value: 5.043076008887179
- type: nauc_mrr_at_3_diff1
value: 52.06774147045164
- type: nauc_mrr_at_3_max
value: 37.79424665327426
- type: nauc_mrr_at_3_std
value: 4.890057068652614
- type: nauc_mrr_at_5_diff1
value: 51.37929937217122
- type: nauc_mrr_at_5_max
value: 36.55880654826355
- type: nauc_mrr_at_5_std
value: 5.858835123935254
- type: nauc_ndcg_at_1000_diff1
value: 52.35035912300684
- type: nauc_ndcg_at_1000_max
value: 36.68201866972285
- type: nauc_ndcg_at_1000_std
value: 5.7202049594831035
- type: nauc_ndcg_at_100_diff1
value: 52.35035912300684
- type: nauc_ndcg_at_100_max
value: 36.68201866972285
- type: nauc_ndcg_at_100_std
value: 5.7202049594831035
- type: nauc_ndcg_at_10_diff1
value: 51.28316055116551
- type: nauc_ndcg_at_10_max
value: 35.87712251784529
- type: nauc_ndcg_at_10_std
value: 6.021170340566541
- type: nauc_ndcg_at_1_diff1
value: 57.67214955091278
- type: nauc_ndcg_at_1_max
value: 35.78220074855998
- type: nauc_ndcg_at_1_std
value: 3.4210250671003326
- type: nauc_ndcg_at_20_diff1
value: 52.026541138524394
- type: nauc_ndcg_at_20_max
value: 36.23499300677105
- type: nauc_ndcg_at_20_std
value: 6.3316005762687695
- type: nauc_ndcg_at_3_diff1
value: 50.317993712307285
- type: nauc_ndcg_at_3_max
value: 39.27048927075592
- type: nauc_ndcg_at_3_std
value: 6.266648429252955
- type: nauc_ndcg_at_5_diff1
value: 48.69195472599095
- type: nauc_ndcg_at_5_max
value: 36.920018592423126
- type: nauc_ndcg_at_5_std
value: 8.451954024556523
- type: nauc_precision_at_1000_diff1
value: .nan
- type: nauc_precision_at_1000_max
value: .nan
- type: nauc_precision_at_1000_std
value: .nan
- type: nauc_precision_at_100_diff1
value: .nan
- type: nauc_precision_at_100_max
value: .nan
- type: nauc_precision_at_100_std
value: .nan
- type: nauc_precision_at_10_diff1
value: 34.43489322486972
- type: nauc_precision_at_10_max
value: 23.186302881363822
- type: nauc_precision_at_10_std
value: 13.793599801222541
- type: nauc_precision_at_1_diff1
value: 57.67214955091278
- type: nauc_precision_at_1_max
value: 35.78220074855998
- type: nauc_precision_at_1_std
value: 3.4210250671003326
- type: nauc_precision_at_20_diff1
value: 32.55853898108205
- type: nauc_precision_at_20_max
value: 11.404195591060642
- type: nauc_precision_at_20_std
value: 43.5675136741475
- type: nauc_precision_at_3_diff1
value: 40.852371923000376
- type: nauc_precision_at_3_max
value: 44.53164299608135
- type: nauc_precision_at_3_std
value: 11.109027031191964
- type: nauc_precision_at_5_diff1
value: 26.165078486515554
- type: nauc_precision_at_5_max
value: 36.58403503282701
- type: nauc_precision_at_5_std
value: 25.437593302644462
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: .nan
- type: nauc_recall_at_100_max
value: .nan
- type: nauc_recall_at_100_std
value: .nan
- type: nauc_recall_at_10_diff1
value: 34.43489322486968
- type: nauc_recall_at_10_max
value: 23.186302881364337
- type: nauc_recall_at_10_std
value: 13.793599801223758
- type: nauc_recall_at_1_diff1
value: 57.67214955091278
- type: nauc_recall_at_1_max
value: 35.78220074855998
- type: nauc_recall_at_1_std
value: 3.4210250671003326
- type: nauc_recall_at_20_diff1
value: 32.55853898108042
- type: nauc_recall_at_20_max
value: 11.404195591059747
- type: nauc_recall_at_20_std
value: 43.56751367414731
- type: nauc_recall_at_3_diff1
value: 40.852371923000334
- type: nauc_recall_at_3_max
value: 44.53164299608149
- type: nauc_recall_at_3_std
value: 11.10902703119201
- type: nauc_recall_at_5_diff1
value: 26.165078486515746
- type: nauc_recall_at_5_max
value: 36.58403503282719
- type: nauc_recall_at_5_std
value: 25.437593302644736
- type: ndcg_at_1
value: 66.919
- type: ndcg_at_10
value: 81.722
- type: ndcg_at_100
value: 82.704
- type: ndcg_at_1000
value: 82.704
- type: ndcg_at_20
value: 82.44399999999999
- type: ndcg_at_3
value: 77.63
- type: ndcg_at_5
value: 80.314
- type: precision_at_1
value: 66.919
- type: precision_at_10
value: 9.596
- type: precision_at_100
value: 1
- type: precision_at_1000
value: 0.1
- type: precision_at_20
value: 4.936999999999999
- type: precision_at_3
value: 28.366999999999997
- type: precision_at_5
value: 18.333
- type: recall_at_1
value: 66.919
- type: recall_at_10
value: 95.96000000000001
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 98.737
- type: recall_at_3
value: 85.101
- type: recall_at_5
value: 91.667
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalBenchCorporateLobbying (default)
revision: f69691c650464e62546d7f2a4536f8f87c891e38
split: test
type: mteb/legalbench_corporate_lobbying
metrics:
- type: main_score
value: 95.275
- type: map_at_1
value: 90
- type: map_at_10
value: 93.896
- type: map_at_100
value: 93.896
- type: map_at_1000
value: 93.89999999999999
- type: map_at_20
value: 93.896
- type: map_at_3
value: 93.529
- type: map_at_5
value: 93.735
- type: mrr_at_1
value: 90
- type: mrr_at_10
value: 93.89577497665734
- type: mrr_at_100
value: 93.89577497665734
- type: mrr_at_1000
value: 93.89961722973894
- type: mrr_at_20
value: 93.89577497665734
- type: mrr_at_3
value: 93.52941176470588
- type: mrr_at_5
value: 93.73529411764707
- type: nauc_map_at_1000_diff1
value: 87.98954780116061
- type: nauc_map_at_1000_max
value: 48.73465559069755
- type: nauc_map_at_1000_std
value: -24.16685492630147
- type: nauc_map_at_100_diff1
value: 87.97183672739986
- type: nauc_map_at_100_max
value: 48.726283117594946
- type: nauc_map_at_100_std
value: -24.21564012563856
- type: nauc_map_at_10_diff1
value: 87.97183672739986
- type: nauc_map_at_10_max
value: 48.726283117594946
- type: nauc_map_at_10_std
value: -24.21564012563856
- type: nauc_map_at_1_diff1
value: 88.58268797715169
- type: nauc_map_at_1_max
value: 41.42912066787494
- type: nauc_map_at_1_std
value: -20.242214532871987
- type: nauc_map_at_20_diff1
value: 87.97183672739986
- type: nauc_map_at_20_max
value: 48.726283117594946
- type: nauc_map_at_20_std
value: -24.21564012563856
- type: nauc_map_at_3_diff1
value: 88.40647370059126
- type: nauc_map_at_3_max
value: 51.27040715275998
- type: nauc_map_at_3_std
value: -26.350691791868247
- type: nauc_map_at_5_diff1
value: 88.02546579403796
- type: nauc_map_at_5_max
value: 49.82239113694508
- type: nauc_map_at_5_std
value: -24.375227399253678
- type: nauc_mrr_at_1000_diff1
value: 87.98954780116061
- type: nauc_mrr_at_1000_max
value: 50.085159299027296
- type: nauc_mrr_at_1000_std
value: -23.851737394357862
- type: nauc_mrr_at_100_diff1
value: 87.97183672739986
- type: nauc_mrr_at_100_max
value: 50.075936762728254
- type: nauc_mrr_at_100_std
value: -23.90072094177412
- type: nauc_mrr_at_10_diff1
value: 87.97183672739986
- type: nauc_mrr_at_10_max
value: 50.075936762728254
- type: nauc_mrr_at_10_std
value: -23.90072094177412
- type: nauc_mrr_at_1_diff1
value: 88.58268797715169
- type: nauc_mrr_at_1_max
value: 43.076838578568626
- type: nauc_mrr_at_1_std
value: -19.857747020376795
- type: nauc_mrr_at_20_diff1
value: 87.97183672739986
- type: nauc_mrr_at_20_max
value: 50.075936762728254
- type: nauc_mrr_at_20_std
value: -23.90072094177412
- type: nauc_mrr_at_3_diff1
value: 88.40647370059126
- type: nauc_mrr_at_3_max
value: 52.54364372011418
- type: nauc_mrr_at_3_std
value: -26.053603259485598
- type: nauc_mrr_at_5_diff1
value: 88.02546579403796
- type: nauc_mrr_at_5_max
value: 51.13747115956445
- type: nauc_mrr_at_5_std
value: -24.068375393975824
- type: nauc_ndcg_at_1000_diff1
value: 87.66050817567788
- type: nauc_ndcg_at_1000_max
value: 49.54001381756884
- type: nauc_ndcg_at_1000_std
value: -24.35184631163463
- type: nauc_ndcg_at_100_diff1
value: 87.24416537653535
- type: nauc_ndcg_at_100_max
value: 49.57831808101997
- type: nauc_ndcg_at_100_std
value: -24.793463718624537
- type: nauc_ndcg_at_10_diff1
value: 87.24416537653535
- type: nauc_ndcg_at_10_max
value: 49.57831808101997
- type: nauc_ndcg_at_10_std
value: -24.793463718624537
- type: nauc_ndcg_at_1_diff1
value: 88.58268797715169
- type: nauc_ndcg_at_1_max
value: 41.42912066787494
- type: nauc_ndcg_at_1_std
value: -20.242214532871987
- type: nauc_ndcg_at_20_diff1
value: 87.24416537653535
- type: nauc_ndcg_at_20_max
value: 49.57831808101997
- type: nauc_ndcg_at_20_std
value: -24.793463718624537
- type: nauc_ndcg_at_3_diff1
value: 88.28539315614394
- type: nauc_ndcg_at_3_max
value: 55.661427932222765
- type: nauc_ndcg_at_3_std
value: -29.09686303480698
- type: nauc_ndcg_at_5_diff1
value: 87.44364795542245
- type: nauc_ndcg_at_5_max
value: 52.79959281779241
- type: nauc_ndcg_at_5_std
value: -24.76177349657667
- type: nauc_precision_at_1000_diff1
value: 100
- type: nauc_precision_at_1000_max
value: 100
- type: nauc_precision_at_1000_std
value: 100
- type: nauc_precision_at_100_diff1
value: 63.81886087768098
- type: nauc_precision_at_100_max
value: 56.1391223155907
- type: nauc_precision_at_100_std
value: -36.99813258637037
- type: nauc_precision_at_10_diff1
value: 63.81886087768457
- type: nauc_precision_at_10_max
value: 56.13912231559321
- type: nauc_precision_at_10_std
value: -36.99813258636605
- type: nauc_precision_at_1_diff1
value: 88.58268797715169
- type: nauc_precision_at_1_max
value: 41.42912066787494
- type: nauc_precision_at_1_std
value: -20.242214532871987
- type: nauc_precision_at_20_diff1
value: 63.81886087768457
- type: nauc_precision_at_20_max
value: 56.13912231559321
- type: nauc_precision_at_20_std
value: -36.99813258636605
- type: nauc_precision_at_3_diff1
value: 87.42089428363992
- type: nauc_precision_at_3_max
value: 85.71428571428666
- type: nauc_precision_at_3_std
value: -47.90953418404368
- type: nauc_precision_at_5_diff1
value: 81.13134142545934
- type: nauc_precision_at_5_max
value: 80.75007780890182
- type: nauc_precision_at_5_std
value: -25.715841892311385
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: 63.81886087768457
- type: nauc_recall_at_100_max
value: 56.13912231559266
- type: nauc_recall_at_100_std
value: -36.99813258636772
- type: nauc_recall_at_10_diff1
value: 63.81886087768457
- type: nauc_recall_at_10_max
value: 56.13912231559266
- type: nauc_recall_at_10_std
value: -36.99813258636772
- type: nauc_recall_at_1_diff1
value: 88.58268797715169
- type: nauc_recall_at_1_max
value: 41.42912066787494
- type: nauc_recall_at_1_std
value: -20.242214532871987
- type: nauc_recall_at_20_diff1
value: 63.81886087768457
- type: nauc_recall_at_20_max
value: 56.13912231559266
- type: nauc_recall_at_20_std
value: -36.99813258636772
- type: nauc_recall_at_3_diff1
value: 87.4208942836394
- type: nauc_recall_at_3_max
value: 85.71428571428564
- type: nauc_recall_at_3_std
value: -47.90953418404379
- type: nauc_recall_at_5_diff1
value: 81.1313414254593
- type: nauc_recall_at_5_max
value: 80.75007780890142
- type: nauc_recall_at_5_std
value: -25.715841892312174
- type: ndcg_at_1
value: 90
- type: ndcg_at_10
value: 95.275
- type: ndcg_at_100
value: 95.275
- type: ndcg_at_1000
value: 95.355
- type: ndcg_at_20
value: 95.275
- type: ndcg_at_3
value: 94.524
- type: ndcg_at_5
value: 94.891
- type: precision_at_1
value: 90
- type: precision_at_10
value: 9.940999999999999
- type: precision_at_100
value: 0.9939999999999999
- type: precision_at_1000
value: 0.1
- type: precision_at_20
value: 4.971
- type: precision_at_3
value: 32.451
- type: precision_at_5
value: 19.647000000000002
- type: recall_at_1
value: 90
- type: recall_at_10
value: 99.412
- type: recall_at_100
value: 99.412
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 99.412
- type: recall_at_3
value: 97.353
- type: recall_at_5
value: 98.235
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalQuAD (default)
revision: 37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e
split: test
type: mteb/LegalQuAD
metrics:
- type: main_score
value: 64.899
- type: map_at_1
value: 48.5
- type: map_at_10
value: 59.607
- type: map_at_100
value: 60.102
- type: map_at_1000
value: 60.141
- type: map_at_20
value: 59.836
- type: map_at_3
value: 57.416999999999994
- type: map_at_5
value: 58.342000000000006
- type: mrr_at_1
value: 48.5
- type: mrr_at_10
value: 59.60734126984125
- type: mrr_at_100
value: 60.10134959097208
- type: mrr_at_1000
value: 60.140317577579324
- type: mrr_at_20
value: 59.83595938375349
- type: mrr_at_3
value: 57.41666666666665
- type: mrr_at_5
value: 58.34166666666666
- type: nauc_map_at_1000_diff1
value: 62.80752643431633
- type: nauc_map_at_1000_max
value: 62.00607212479527
- type: nauc_map_at_1000_std
value: 17.541941429951855
- type: nauc_map_at_100_diff1
value: 62.807006564215705
- type: nauc_map_at_100_max
value: 62.025705691974885
- type: nauc_map_at_100_std
value: 17.56923712268851
- type: nauc_map_at_10_diff1
value: 62.51717633954077
- type: nauc_map_at_10_max
value: 61.97403063599849
- type: nauc_map_at_10_std
value: 17.634018830663045
- type: nauc_map_at_1_diff1
value: 65.73772773363197
- type: nauc_map_at_1_max
value: 59.189205309863524
- type: nauc_map_at_1_std
value: 14.461624107787843
- type: nauc_map_at_20_diff1
value: 62.72297267428931
- type: nauc_map_at_20_max
value: 62.09138256517757
- type: nauc_map_at_20_std
value: 17.751999039640335
- type: nauc_map_at_3_diff1
value: 63.4965467376001
- type: nauc_map_at_3_max
value: 62.20134238108197
- type: nauc_map_at_3_std
value: 16.0562228318107
- type: nauc_map_at_5_diff1
value: 63.596270852816886
- type: nauc_map_at_5_max
value: 62.370748922533515
- type: nauc_map_at_5_std
value: 16.199213024713476
- type: nauc_mrr_at_1000_diff1
value: 62.807542338240665
- type: nauc_mrr_at_1000_max
value: 62.00610157228559
- type: nauc_mrr_at_1000_std
value: 17.54188368114912
- type: nauc_mrr_at_100_diff1
value: 62.80702245597816
- type: nauc_mrr_at_100_max
value: 62.02573476935551
- type: nauc_mrr_at_100_std
value: 17.569178989249227
- type: nauc_mrr_at_10_diff1
value: 62.51717633954077
- type: nauc_mrr_at_10_max
value: 61.97403063599849
- type: nauc_mrr_at_10_std
value: 17.634018830663045
- type: nauc_mrr_at_1_diff1
value: 65.73772773363197
- type: nauc_mrr_at_1_max
value: 59.189205309863524
- type: nauc_mrr_at_1_std
value: 14.461624107787843
- type: nauc_mrr_at_20_diff1
value: 62.72297267428931
- type: nauc_mrr_at_20_max
value: 62.09138256517757
- type: nauc_mrr_at_20_std
value: 17.751999039640335
- type: nauc_mrr_at_3_diff1
value: 63.4965467376001
- type: nauc_mrr_at_3_max
value: 62.20134238108197
- type: nauc_mrr_at_3_std
value: 16.0562228318107
- type: nauc_mrr_at_5_diff1
value: 63.596270852816886
- type: nauc_mrr_at_5_max
value: 62.370748922533515
- type: nauc_mrr_at_5_std
value: 16.199213024713476
- type: nauc_ndcg_at_1000_diff1
value: 61.99027512405032
- type: nauc_ndcg_at_1000_max
value: 62.3794577316155
- type: nauc_ndcg_at_1000_std
value: 18.81209576316582
- type: nauc_ndcg_at_100_diff1
value: 61.981491536700695
- type: nauc_ndcg_at_100_max
value: 62.83920688186244
- type: nauc_ndcg_at_100_std
value: 19.50643109965116
- type: nauc_ndcg_at_10_diff1
value: 60.04817741489627
- type: nauc_ndcg_at_10_max
value: 62.4470034515535
- type: nauc_ndcg_at_10_std
value: 20.53817059813657
- type: nauc_ndcg_at_1_diff1
value: 65.73772773363197
- type: nauc_ndcg_at_1_max
value: 59.189205309863524
- type: nauc_ndcg_at_1_std
value: 14.461624107787843
- type: nauc_ndcg_at_20_diff1
value: 60.907316049398794
- type: nauc_ndcg_at_20_max
value: 62.99570450758024
- type: nauc_ndcg_at_20_std
value: 21.058986818468124
- type: nauc_ndcg_at_3_diff1
value: 62.73893976493435
- type: nauc_ndcg_at_3_max
value: 63.03331537097003
- type: nauc_ndcg_at_3_std
value: 16.64703608470413
- type: nauc_ndcg_at_5_diff1
value: 62.877505768846845
- type: nauc_ndcg_at_5_max
value: 63.383723263638544
- type: nauc_ndcg_at_5_std
value: 16.955089182466672
- type: nauc_precision_at_1000_diff1
value: 100
- type: nauc_precision_at_1000_max
value: 100
- type: nauc_precision_at_1000_std
value: 100
- type: nauc_precision_at_100_diff1
value: 61.49859943977586
- type: nauc_precision_at_100_max
value: 78.61811391223151
- type: nauc_precision_at_100_std
value: 43.9635854341738
- type: nauc_precision_at_10_diff1
value: 46.05874298601228
- type: nauc_precision_at_10_max
value: 64.15281767659134
- type: nauc_precision_at_10_std
value: 36.998952935806905
- type: nauc_precision_at_1_diff1
value: 65.73772773363197
- type: nauc_precision_at_1_max
value: 59.189205309863524
- type: nauc_precision_at_1_std
value: 14.461624107787843
- type: nauc_precision_at_20_diff1
value: 49.614383338756745
- type: nauc_precision_at_20_max
value: 68.69996745850949
- type: nauc_precision_at_20_std
value: 44.100227790432726
- type: nauc_precision_at_3_diff1
value: 60.181007322330856
- type: nauc_precision_at_3_max
value: 65.7700665904469
- type: nauc_precision_at_3_std
value: 18.6619072970213
- type: nauc_precision_at_5_diff1
value: 60.24398508980005
- type: nauc_precision_at_5_max
value: 67.09105872101463
- type: nauc_precision_at_5_std
value: 19.826693130657848
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: 61.498599439775816
- type: nauc_recall_at_100_max
value: 78.61811391223146
- type: nauc_recall_at_100_std
value: 43.96358543417379
- type: nauc_recall_at_10_diff1
value: 46.05874298601233
- type: nauc_recall_at_10_max
value: 64.1528176765914
- type: nauc_recall_at_10_std
value: 36.99895293580692
- type: nauc_recall_at_1_diff1
value: 65.73772773363197
- type: nauc_recall_at_1_max
value: 59.189205309863524
- type: nauc_recall_at_1_std
value: 14.461624107787843
- type: nauc_recall_at_20_diff1
value: 49.61438333875686
- type: nauc_recall_at_20_max
value: 68.69996745850956
- type: nauc_recall_at_20_std
value: 44.10022779043285
- type: nauc_recall_at_3_diff1
value: 60.181007322330935
- type: nauc_recall_at_3_max
value: 65.77006659044693
- type: nauc_recall_at_3_std
value: 18.661907297021287
- type: nauc_recall_at_5_diff1
value: 60.243985089800056
- type: nauc_recall_at_5_max
value: 67.09105872101472
- type: nauc_recall_at_5_std
value: 19.826693130657883
- type: ndcg_at_1
value: 48.5
- type: ndcg_at_10
value: 64.899
- type: ndcg_at_100
value: 67.634
- type: ndcg_at_1000
value: 68.345
- type: ndcg_at_20
value: 65.767
- type: ndcg_at_3
value: 60.268
- type: ndcg_at_5
value: 61.925
- type: precision_at_1
value: 48.5
- type: precision_at_10
value: 8.15
- type: precision_at_100
value: 0.95
- type: precision_at_1000
value: 0.1
- type: precision_at_20
value: 4.25
- type: precision_at_3
value: 22.833000000000002
- type: precision_at_5
value: 14.499999999999998
- type: recall_at_1
value: 48.5
- type: recall_at_10
value: 81.5
- type: recall_at_100
value: 95
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 85
- type: recall_at_3
value: 68.5
- type: recall_at_5
value: 72.5
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalSummarization (default)
revision: 3bb1a05c66872889662af04c5691c14489cebd72
split: test
type: mteb/legal_summarization
metrics:
- type: main_score
value: 68.37299999999999
- type: map_at_1
value: 50.583999999999996
- type: map_at_10
value: 62.287000000000006
- type: map_at_100
value: 63.537
- type: map_at_1000
value: 63.573
- type: map_at_20
value: 63.077000000000005
- type: map_at_3
value: 58.358
- type: map_at_5
value: 60.211000000000006
- type: mrr_at_1
value: 58.80281690140845
- type: mrr_at_10
value: 67.8209534987704
- type: mrr_at_100
value: 68.40289291033511
- type: mrr_at_1000
value: 68.41681870169752
- type: mrr_at_20
value: 68.25865150223059
- type: mrr_at_3
value: 65.66901408450703
- type: mrr_at_5
value: 66.84859154929578
- type: nauc_map_at_1000_diff1
value: 66.23490767864845
- type: nauc_map_at_1000_max
value: 56.62757337597366
- type: nauc_map_at_1000_std
value: 4.528549025111963
- type: nauc_map_at_100_diff1
value: 66.22940398014607
- type: nauc_map_at_100_max
value: 56.634349360676865
- type: nauc_map_at_100_std
value: 4.541880997700564
- type: nauc_map_at_10_diff1
value: 65.94549814528568
- type: nauc_map_at_10_max
value: 55.52461478236527
- type: nauc_map_at_10_std
value: 2.457230374531773
- type: nauc_map_at_1_diff1
value: 68.72392823338977
- type: nauc_map_at_1_max
value: 53.391642623028936
- type: nauc_map_at_1_std
value: 1.4283196945625762
- type: nauc_map_at_20_diff1
value: 66.20579936575423
- type: nauc_map_at_20_max
value: 56.531407372997954
- type: nauc_map_at_20_std
value: 4.117228263552818
- type: nauc_map_at_3_diff1
value: 65.34148569044554
- type: nauc_map_at_3_max
value: 52.23666020679251
- type: nauc_map_at_3_std
value: 0.6213942899425956
- type: nauc_map_at_5_diff1
value: 65.56848099295489
- type: nauc_map_at_5_max
value: 54.14599505704255
- type: nauc_map_at_5_std
value: 1.7510168090848297
- type: nauc_mrr_at_1000_diff1
value: 65.2248453530452
- type: nauc_mrr_at_1000_max
value: 56.26467474118066
- type: nauc_mrr_at_1000_std
value: 7.765895326925132
- type: nauc_mrr_at_100_diff1
value: 65.21070712758778
- type: nauc_mrr_at_100_max
value: 56.271177686981325
- type: nauc_mrr_at_100_std
value: 7.785345389803579
- type: nauc_mrr_at_10_diff1
value: 64.99745390900522
- type: nauc_mrr_at_10_max
value: 56.055839490543576
- type: nauc_mrr_at_10_std
value: 7.496513504421416
- type: nauc_mrr_at_1_diff1
value: 68.92907857149805
- type: nauc_mrr_at_1_max
value: 56.26107663193521
- type: nauc_mrr_at_1_std
value: 6.297369156935706
- type: nauc_mrr_at_20_diff1
value: 65.1678016982267
- type: nauc_mrr_at_20_max
value: 56.335972588669726
- type: nauc_mrr_at_20_std
value: 7.8243664682538485
- type: nauc_mrr_at_3_diff1
value: 64.83973635858855
- type: nauc_mrr_at_3_max
value: 55.5714129852856
- type: nauc_mrr_at_3_std
value: 6.613504325180008
- type: nauc_mrr_at_5_diff1
value: 64.74589091931088
- type: nauc_mrr_at_5_max
value: 55.88271351361698
- type: nauc_mrr_at_5_std
value: 7.1386788643876455
- type: nauc_ndcg_at_1000_diff1
value: 65.202639941499
- type: nauc_ndcg_at_1000_max
value: 57.564093890541756
- type: nauc_ndcg_at_1000_std
value: 7.258587810697105
- type: nauc_ndcg_at_100_diff1
value: 64.7813249495164
- type: nauc_ndcg_at_100_max
value: 57.79712787033168
- type: nauc_ndcg_at_100_std
value: 7.773895293143383
- type: nauc_ndcg_at_10_diff1
value: 64.32391711969152
- type: nauc_ndcg_at_10_max
value: 55.63793686530347
- type: nauc_ndcg_at_10_std
value: 3.4059961155442764
- type: nauc_ndcg_at_1_diff1
value: 68.92907857149805
- type: nauc_ndcg_at_1_max
value: 56.26107663193521
- type: nauc_ndcg_at_1_std
value: 6.297369156935706
- type: nauc_ndcg_at_20_diff1
value: 64.81574204816175
- type: nauc_ndcg_at_20_max
value: 58.2840773030739
- type: nauc_ndcg_at_20_std
value: 7.263179111057372
- type: nauc_ndcg_at_3_diff1
value: 63.912724511990696
- type: nauc_ndcg_at_3_max
value: 53.75803159488968
- type: nauc_ndcg_at_3_std
value: 3.1036435446695645
- type: nauc_ndcg_at_5_diff1
value: 63.79330356150732
- type: nauc_ndcg_at_5_max
value: 55.082744998876464
- type: nauc_ndcg_at_5_std
value: 3.6246823425115213
- type: nauc_precision_at_1000_diff1
value: -22.10580213313522
- type: nauc_precision_at_1000_max
value: -0.9118701386503028
- type: nauc_precision_at_1000_std
value: 27.333746721273283
- type: nauc_precision_at_100_diff1
value: -18.882895589911644
- type: nauc_precision_at_100_max
value: 4.113768286012285
- type: nauc_precision_at_100_std
value: 30.048337440426497
- type: nauc_precision_at_10_diff1
value: 3.248704005833223
- type: nauc_precision_at_10_max
value: 23.154996086400423
- type: nauc_precision_at_10_std
value: 20.251997611045912
- type: nauc_precision_at_1_diff1
value: 68.92907857149805
- type: nauc_precision_at_1_max
value: 56.26107663193521
- type: nauc_precision_at_1_std
value: 6.297369156935706
- type: nauc_precision_at_20_diff1
value: -7.13873585811708
- type: nauc_precision_at_20_max
value: 17.375565602388527
- type: nauc_precision_at_20_std
value: 31.317699557764485
- type: nauc_precision_at_3_diff1
value: 33.63751338671969
- type: nauc_precision_at_3_max
value: 38.476765888636756
- type: nauc_precision_at_3_std
value: 12.2723755464466
- type: nauc_precision_at_5_diff1
value: 20.649180734546427
- type: nauc_precision_at_5_max
value: 34.74830242946571
- type: nauc_precision_at_5_std
value: 16.90872320810353
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: 38.57539921023781
- type: nauc_recall_at_100_max
value: 63.42225430043433
- type: nauc_recall_at_100_std
value: 31.59021194191864
- type: nauc_recall_at_10_diff1
value: 53.672218603348455
- type: nauc_recall_at_10_max
value: 48.673710550594265
- type: nauc_recall_at_10_std
value: -2.566654931781578
- type: nauc_recall_at_1_diff1
value: 68.72392823338977
- type: nauc_recall_at_1_max
value: 53.391642623028936
- type: nauc_recall_at_1_std
value: 1.4283196945625762
- type: nauc_recall_at_20_diff1
value: 51.84453378101791
- type: nauc_recall_at_20_max
value: 62.44943925787231
- type: nauc_recall_at_20_std
value: 15.782443159610663
- type: nauc_recall_at_3_diff1
value: 56.20810589388257
- type: nauc_recall_at_3_max
value: 45.892032964653836
- type: nauc_recall_at_3_std
value: -2.820516274258717
- type: nauc_recall_at_5_diff1
value: 54.215941877653904
- type: nauc_recall_at_5_max
value: 48.20665726343471
- type: nauc_recall_at_5_std
value: -0.791029661993311
- type: ndcg_at_1
value: 58.803000000000004
- type: ndcg_at_10
value: 68.37299999999999
- type: ndcg_at_100
value: 72.446
- type: ndcg_at_1000
value: 72.992
- type: ndcg_at_20
value: 70.414
- type: ndcg_at_3
value: 62.839
- type: ndcg_at_5
value: 65.023
- type: precision_at_1
value: 58.803000000000004
- type: precision_at_10
value: 11.197
- type: precision_at_100
value: 1.4749999999999999
- type: precision_at_1000
value: 0.155
- type: precision_at_20
value: 6.356000000000001
- type: precision_at_3
value: 27.346999999999998
- type: precision_at_5
value: 18.521
- type: recall_at_1
value: 50.583999999999996
- type: recall_at_10
value: 80.833
- type: recall_at_100
value: 96.467
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 87.543
- type: recall_at_3
value: 65.515
- type: recall_at_5
value: 71.47200000000001
task:
type: Retrieval
tags:
- mteb
VectorStack AI | vstackai-law-1
This repository describes the steps of using the VectorStack AI's vstackai-law-1 model.
You can use the embedding model via our API or via private deployments in your private cloud.
Usage via private deployment
If you are interested in running this model in your own private cloud/ own hardware, contact us to learn more.
Usage via API
The following code snippet shows the steps required to generate embeddings via the API. Install the SDK via:
pip install -U vectorstackai
Get your free API key on www.vectorstack.ai
import vectorstackai
import numpy as np
api_key = "{API_KEY}" # Get your API key from www.vectorstack.ai
client = vectorstackai.Client(api_key=api_key)
# Documents related to law domain (e.g., court cases, consumer contracts, etc.)
documents = [
"The defendant was charged with violation of contract terms in the lease agreement signed on January 1, 2022.",
"This contract stipulates that the consumer has 30 days to return the product in case of any manufacturing defects.",
"In the case of Smith v. Johnson, the court ruled that the plaintiff had the right to claim damages under section 12 of the Consumer Protection Act."
]
# Get embeddings for the legal documents
doc_embeddings = client.embed(texts=documents, model='vstackai-law-1', is_query=False) # EmbeddingsObject(num_embeddings=3, embedding_dims=1536)
doc_embeddings = np.array(doc_embeddings.embeddings)
# Encode the query
query = "How many days does the consumer have to return the product?"
query_embedding = client.embed(
texts=[query],
model='vstackai-law-1',
is_query=True,
instruction='Represent the query for searching legal documents'
) # EmbeddingsObject(num_embeddings=1, embedding_dims=1536)
query_embedding = np.array(query_embedding.embeddings) # (1, 1536)
# To check if the embeddings work, you can compute similarity between the query and documents
similarities = np.dot(doc_embeddings, query_embedding.T)
print(similarities)
# array([[0.355],
# [0.772],
# [0.433]])