|
--- |
|
license: apache-2.0 |
|
model-index: |
|
- name: vstackai-law-1 |
|
results: |
|
- dataset: |
|
config: default |
|
name: MTEB AILACasedocs (default) |
|
revision: 4106e6bcc72e0698d714ea8b101355e3e238431a |
|
split: test |
|
type: mteb/AILA_casedocs |
|
metrics: |
|
- type: main_score |
|
value: 45.881 |
|
- type: map_at_1 |
|
value: 15.626000000000001 |
|
- type: map_at_10 |
|
value: 35.655 |
|
- type: map_at_100 |
|
value: 39.715 |
|
- type: map_at_1000 |
|
value: 39.889 |
|
- type: map_at_20 |
|
value: 37.102000000000004 |
|
- type: map_at_3 |
|
value: 27.089999999999996 |
|
- type: map_at_5 |
|
value: 31.025999999999996 |
|
- type: mrr_at_1 |
|
value: 46.0 |
|
- type: mrr_at_10 |
|
value: 54.604761904761915 |
|
- type: mrr_at_100 |
|
value: 55.25030966503531 |
|
- type: mrr_at_1000 |
|
value: 55.28445318355383 |
|
- type: mrr_at_20 |
|
value: 54.921136173767756 |
|
- type: mrr_at_3 |
|
value: 51.66666666666666 |
|
- type: mrr_at_5 |
|
value: 52.166666666666664 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 15.57495845891449 |
|
- type: nauc_map_at_1000_max |
|
value: 23.143422477463474 |
|
- type: nauc_map_at_1000_std |
|
value: -8.945561626866498 |
|
- type: nauc_map_at_100_diff1 |
|
value: 15.506630125824048 |
|
- type: nauc_map_at_100_max |
|
value: 23.282567495936846 |
|
- type: nauc_map_at_100_std |
|
value: -8.911449134456035 |
|
- type: nauc_map_at_10_diff1 |
|
value: 15.618163394870141 |
|
- type: nauc_map_at_10_max |
|
value: 22.07816596666465 |
|
- type: nauc_map_at_10_std |
|
value: -10.298462021115187 |
|
- type: nauc_map_at_1_diff1 |
|
value: 13.37923142053057 |
|
- type: nauc_map_at_1_max |
|
value: 13.483533976115272 |
|
- type: nauc_map_at_1_std |
|
value: -6.849119244118052 |
|
- type: nauc_map_at_20_diff1 |
|
value: 15.678610092128679 |
|
- type: nauc_map_at_20_max |
|
value: 23.01239046943845 |
|
- type: nauc_map_at_20_std |
|
value: -10.384062772234483 |
|
- type: nauc_map_at_3_diff1 |
|
value: 21.805983515218617 |
|
- type: nauc_map_at_3_max |
|
value: 15.777843249645866 |
|
- type: nauc_map_at_3_std |
|
value: -15.963631005395404 |
|
- type: nauc_map_at_5_diff1 |
|
value: 20.668846473212096 |
|
- type: nauc_map_at_5_max |
|
value: 21.465743982369055 |
|
- type: nauc_map_at_5_std |
|
value: -13.06765002103128 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 18.950498468308886 |
|
- type: nauc_mrr_at_1000_max |
|
value: 26.651641047762148 |
|
- type: nauc_mrr_at_1000_std |
|
value: -4.101624493270528 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 18.92883047034938 |
|
- type: nauc_mrr_at_100_max |
|
value: 26.722955290521792 |
|
- type: nauc_mrr_at_100_std |
|
value: -4.027052366530224 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 18.60683936270885 |
|
- type: nauc_mrr_at_10_max |
|
value: 26.16665005770204 |
|
- type: nauc_mrr_at_10_std |
|
value: -4.698063142209917 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 17.54487398491498 |
|
- type: nauc_mrr_at_1_max |
|
value: 26.099182675882364 |
|
- type: nauc_mrr_at_1_std |
|
value: -7.442642768915407 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 19.0367989604306 |
|
- type: nauc_mrr_at_20_max |
|
value: 26.630737835733974 |
|
- type: nauc_mrr_at_20_std |
|
value: -4.11566988702261 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 21.267024293259933 |
|
- type: nauc_mrr_at_3_max |
|
value: 26.69418345236406 |
|
- type: nauc_mrr_at_3_std |
|
value: -3.612150752894062 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 19.90001865151827 |
|
- type: nauc_mrr_at_5_max |
|
value: 27.66941220085612 |
|
- type: nauc_mrr_at_5_std |
|
value: -2.7974974728343893 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 15.901300620232577 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 25.612869768853642 |
|
- type: nauc_ndcg_at_1000_std |
|
value: -5.309417560619307 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 14.95954368378741 |
|
- type: nauc_ndcg_at_100_max |
|
value: 27.23572900207926 |
|
- type: nauc_ndcg_at_100_std |
|
value: -3.5899694438310528 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 14.060694457887331 |
|
- type: nauc_ndcg_at_10_max |
|
value: 23.459115385918828 |
|
- type: nauc_ndcg_at_10_std |
|
value: -7.547036607345263 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 17.54487398491498 |
|
- type: nauc_ndcg_at_1_max |
|
value: 26.099182675882364 |
|
- type: nauc_ndcg_at_1_std |
|
value: -7.442642768915407 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 14.406205372080914 |
|
- type: nauc_ndcg_at_20_max |
|
value: 26.249115297438518 |
|
- type: nauc_ndcg_at_20_std |
|
value: -6.498773153759981 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 15.967994683338851 |
|
- type: nauc_ndcg_at_3_max |
|
value: 23.71128286998367 |
|
- type: nauc_ndcg_at_3_std |
|
value: -7.166430376561868 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 16.371530698236622 |
|
- type: nauc_ndcg_at_5_max |
|
value: 25.725132293180273 |
|
- type: nauc_ndcg_at_5_std |
|
value: -3.8545166535972943 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: -10.41717795536071 |
|
- type: nauc_precision_at_1000_max |
|
value: 4.7029594790402625 |
|
- type: nauc_precision_at_1000_std |
|
value: 14.173763073581878 |
|
- type: nauc_precision_at_100_diff1 |
|
value: -11.074302280828615 |
|
- type: nauc_precision_at_100_max |
|
value: 8.244938186343832 |
|
- type: nauc_precision_at_100_std |
|
value: 15.285975835662343 |
|
- type: nauc_precision_at_10_diff1 |
|
value: -10.837400355480538 |
|
- type: nauc_precision_at_10_max |
|
value: 22.832192981934664 |
|
- type: nauc_precision_at_10_std |
|
value: 10.05034983325703 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 17.54487398491498 |
|
- type: nauc_precision_at_1_max |
|
value: 26.099182675882364 |
|
- type: nauc_precision_at_1_std |
|
value: -7.442642768915407 |
|
- type: nauc_precision_at_20_diff1 |
|
value: -9.411528844386767 |
|
- type: nauc_precision_at_20_max |
|
value: 22.63946047993228 |
|
- type: nauc_precision_at_20_std |
|
value: 10.267476763547872 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 12.089219490235562 |
|
- type: nauc_precision_at_3_max |
|
value: 21.867564149285098 |
|
- type: nauc_precision_at_3_std |
|
value: -7.900490900126483 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 6.088160593562308 |
|
- type: nauc_precision_at_5_max |
|
value: 25.24343016238549 |
|
- type: nauc_precision_at_5_std |
|
value: -0.2798984072447858 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: -5.301190251888983 |
|
- type: nauc_recall_at_100_max |
|
value: 61.552650883513884 |
|
- type: nauc_recall_at_100_std |
|
value: 41.63529646496176 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 3.4992492433968714 |
|
- type: nauc_recall_at_10_max |
|
value: 14.516376506894568 |
|
- type: nauc_recall_at_10_std |
|
value: -8.155441184017405 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 13.37923142053057 |
|
- type: nauc_recall_at_1_max |
|
value: 13.483533976115272 |
|
- type: nauc_recall_at_1_std |
|
value: -6.849119244118052 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 3.2751273323204697 |
|
- type: nauc_recall_at_20_max |
|
value: 22.900211937193326 |
|
- type: nauc_recall_at_20_std |
|
value: -1.1031026054527242 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 22.819746834360018 |
|
- type: nauc_recall_at_3_max |
|
value: 13.682536574839425 |
|
- type: nauc_recall_at_3_std |
|
value: -14.733108963214727 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 22.45051937631727 |
|
- type: nauc_recall_at_5_max |
|
value: 24.19540105383745 |
|
- type: nauc_recall_at_5_std |
|
value: -7.468653169210884 |
|
- type: ndcg_at_1 |
|
value: 46.0 |
|
- type: ndcg_at_10 |
|
value: 45.881 |
|
- type: ndcg_at_100 |
|
value: 57.25900000000001 |
|
- type: ndcg_at_1000 |
|
value: 58.528000000000006 |
|
- type: ndcg_at_20 |
|
value: 48.413000000000004 |
|
- type: ndcg_at_3 |
|
value: 42.709 |
|
- type: ndcg_at_5 |
|
value: 41.253 |
|
- type: precision_at_1 |
|
value: 46.0 |
|
- type: precision_at_10 |
|
value: 19.400000000000002 |
|
- type: precision_at_100 |
|
value: 3.66 |
|
- type: precision_at_1000 |
|
value: 0.38999999999999996 |
|
- type: precision_at_20 |
|
value: 11.3 |
|
- type: precision_at_3 |
|
value: 34.666999999999994 |
|
- type: precision_at_5 |
|
value: 26.8 |
|
- type: recall_at_1 |
|
value: 15.626000000000001 |
|
- type: recall_at_10 |
|
value: 53.722 |
|
- type: recall_at_100 |
|
value: 93.312 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 62.011 |
|
- type: recall_at_3 |
|
value: 30.44 |
|
- type: recall_at_5 |
|
value: 35.836 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB AILAStatutes (default) |
|
revision: ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e |
|
split: test |
|
type: mteb/AILA_statutes |
|
metrics: |
|
- type: main_score |
|
value: 47.122 |
|
- type: map_at_1 |
|
value: 11.4 |
|
- type: map_at_10 |
|
value: 30.796 |
|
- type: map_at_100 |
|
value: 37.822 |
|
- type: map_at_1000 |
|
value: 37.822 |
|
- type: map_at_20 |
|
value: 34.211000000000006 |
|
- type: map_at_3 |
|
value: 20.089000000000002 |
|
- type: map_at_5 |
|
value: 25.304 |
|
- type: mrr_at_1 |
|
value: 50.0 |
|
- type: mrr_at_10 |
|
value: 67.45714285714284 |
|
- type: mrr_at_100 |
|
value: 67.45714285714284 |
|
- type: mrr_at_1000 |
|
value: 67.45714285714284 |
|
- type: mrr_at_20 |
|
value: 67.45714285714284 |
|
- type: mrr_at_3 |
|
value: 63.66666666666666 |
|
- type: mrr_at_5 |
|
value: 66.06666666666666 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 13.231953724763489 |
|
- type: nauc_map_at_1000_max |
|
value: 36.25413964500182 |
|
- type: nauc_map_at_1000_std |
|
value: 16.12109134339587 |
|
- type: nauc_map_at_100_diff1 |
|
value: 13.231953724763489 |
|
- type: nauc_map_at_100_max |
|
value: 36.25413964500182 |
|
- type: nauc_map_at_100_std |
|
value: 16.12109134339587 |
|
- type: nauc_map_at_10_diff1 |
|
value: 17.94211109348458 |
|
- type: nauc_map_at_10_max |
|
value: 39.28669806176402 |
|
- type: nauc_map_at_10_std |
|
value: 13.594341769002869 |
|
- type: nauc_map_at_1_diff1 |
|
value: 30.100915555842434 |
|
- type: nauc_map_at_1_max |
|
value: 33.83683525374127 |
|
- type: nauc_map_at_1_std |
|
value: 20.814102357206625 |
|
- type: nauc_map_at_20_diff1 |
|
value: 13.652796877938734 |
|
- type: nauc_map_at_20_max |
|
value: 35.83818028836039 |
|
- type: nauc_map_at_20_std |
|
value: 14.37656196108266 |
|
- type: nauc_map_at_3_diff1 |
|
value: 34.486067078257975 |
|
- type: nauc_map_at_3_max |
|
value: 31.543224895382878 |
|
- type: nauc_map_at_3_std |
|
value: -5.527795564447535 |
|
- type: nauc_map_at_5_diff1 |
|
value: 26.851187948582773 |
|
- type: nauc_map_at_5_max |
|
value: 38.89484793235884 |
|
- type: nauc_map_at_5_std |
|
value: 3.4001045162742694 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 26.39698483562357 |
|
- type: nauc_mrr_at_1000_max |
|
value: 30.835723241593847 |
|
- type: nauc_mrr_at_1000_std |
|
value: 20.683462333907705 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 26.39698483562357 |
|
- type: nauc_mrr_at_100_max |
|
value: 30.835723241593847 |
|
- type: nauc_mrr_at_100_std |
|
value: 20.683462333907705 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 26.39698483562357 |
|
- type: nauc_mrr_at_10_max |
|
value: 30.835723241593847 |
|
- type: nauc_mrr_at_10_std |
|
value: 20.683462333907705 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 27.95469686875413 |
|
- type: nauc_mrr_at_1_max |
|
value: 31.818787475016645 |
|
- type: nauc_mrr_at_1_std |
|
value: 19.56295802798131 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 26.39698483562357 |
|
- type: nauc_mrr_at_20_max |
|
value: 30.835723241593847 |
|
- type: nauc_mrr_at_20_std |
|
value: 20.683462333907705 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 27.65080554514802 |
|
- type: nauc_mrr_at_3_max |
|
value: 32.553977955855814 |
|
- type: nauc_mrr_at_3_std |
|
value: 18.78114112837834 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 26.384655469886393 |
|
- type: nauc_mrr_at_5_max |
|
value: 31.664536690997192 |
|
- type: nauc_mrr_at_5_std |
|
value: 23.119445085922425 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 13.624306343440482 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 33.949287066404395 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 18.873031773538894 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 13.624306343440482 |
|
- type: nauc_ndcg_at_100_max |
|
value: 33.949287066404395 |
|
- type: nauc_ndcg_at_100_std |
|
value: 18.873031773538894 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 17.95882248371854 |
|
- type: nauc_ndcg_at_10_max |
|
value: 38.42611161111767 |
|
- type: nauc_ndcg_at_10_std |
|
value: 18.382289466996596 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 27.95469686875413 |
|
- type: nauc_ndcg_at_1_max |
|
value: 31.818787475016645 |
|
- type: nauc_ndcg_at_1_std |
|
value: 19.56295802798131 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 10.108403510160493 |
|
- type: nauc_ndcg_at_20_max |
|
value: 33.30645155135527 |
|
- type: nauc_ndcg_at_20_std |
|
value: 19.106968240050197 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 26.02063027004639 |
|
- type: nauc_ndcg_at_3_max |
|
value: 29.85933476459198 |
|
- type: nauc_ndcg_at_3_std |
|
value: 0.04135555148143812 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 24.51049558894743 |
|
- type: nauc_ndcg_at_5_max |
|
value: 39.030804792387535 |
|
- type: nauc_ndcg_at_5_std |
|
value: 10.626686520599742 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: -52.64814035601246 |
|
- type: nauc_precision_at_1000_max |
|
value: -13.856314374050852 |
|
- type: nauc_precision_at_1000_std |
|
value: 17.593896928340197 |
|
- type: nauc_precision_at_100_diff1 |
|
value: -52.648140356011965 |
|
- type: nauc_precision_at_100_max |
|
value: -13.856314374050674 |
|
- type: nauc_precision_at_100_std |
|
value: 17.593896928340225 |
|
- type: nauc_precision_at_10_diff1 |
|
value: -11.6303596329846 |
|
- type: nauc_precision_at_10_max |
|
value: 26.23949752159085 |
|
- type: nauc_precision_at_10_std |
|
value: 25.41101749534196 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 27.95469686875413 |
|
- type: nauc_precision_at_1_max |
|
value: 31.818787475016645 |
|
- type: nauc_precision_at_1_std |
|
value: 19.56295802798131 |
|
- type: nauc_precision_at_20_diff1 |
|
value: -26.535056035995154 |
|
- type: nauc_precision_at_20_max |
|
value: 11.922354543765811 |
|
- type: nauc_precision_at_20_std |
|
value: 23.095498544921597 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 21.2869202175386 |
|
- type: nauc_precision_at_3_max |
|
value: 31.7866363919357 |
|
- type: nauc_precision_at_3_std |
|
value: -3.858197066525092 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 1.8680405985613282 |
|
- type: nauc_precision_at_5_max |
|
value: 35.366076833199564 |
|
- type: nauc_precision_at_5_std |
|
value: 16.706786604164005 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_100_max |
|
value: .nan |
|
- type: nauc_recall_at_100_std |
|
value: .nan |
|
- type: nauc_recall_at_10_diff1 |
|
value: 8.112331861587329 |
|
- type: nauc_recall_at_10_max |
|
value: 34.89423965499535 |
|
- type: nauc_recall_at_10_std |
|
value: 21.6246021152069 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 30.100915555842434 |
|
- type: nauc_recall_at_1_max |
|
value: 33.83683525374127 |
|
- type: nauc_recall_at_1_std |
|
value: 20.814102357206625 |
|
- type: nauc_recall_at_20_diff1 |
|
value: -11.669499704119648 |
|
- type: nauc_recall_at_20_max |
|
value: 22.30570195122146 |
|
- type: nauc_recall_at_20_std |
|
value: 23.238122093323348 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 33.29080234285882 |
|
- type: nauc_recall_at_3_max |
|
value: 30.134100912744604 |
|
- type: nauc_recall_at_3_std |
|
value: -12.43448322513982 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 18.77021416083913 |
|
- type: nauc_recall_at_5_max |
|
value: 40.54971590909089 |
|
- type: nauc_recall_at_5_std |
|
value: 9.419471153846175 |
|
- type: ndcg_at_1 |
|
value: 50.0 |
|
- type: ndcg_at_10 |
|
value: 47.122 |
|
- type: ndcg_at_100 |
|
value: 63.287000000000006 |
|
- type: ndcg_at_1000 |
|
value: 63.287000000000006 |
|
- type: ndcg_at_20 |
|
value: 53.172 |
|
- type: ndcg_at_3 |
|
value: 39.36 |
|
- type: ndcg_at_5 |
|
value: 38.586 |
|
- type: precision_at_1 |
|
value: 50.0 |
|
- type: precision_at_10 |
|
value: 23.0 |
|
- type: precision_at_100 |
|
value: 4.34 |
|
- type: precision_at_1000 |
|
value: 0.434 |
|
- type: precision_at_20 |
|
value: 14.799999999999999 |
|
- type: precision_at_3 |
|
value: 35.333 |
|
- type: precision_at_5 |
|
value: 31.2 |
|
- type: recall_at_1 |
|
value: 11.4 |
|
- type: recall_at_10 |
|
value: 52.532999999999994 |
|
- type: recall_at_100 |
|
value: 100.0 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 67.0 |
|
- type: recall_at_3 |
|
value: 25.467000000000002 |
|
- type: recall_at_5 |
|
value: 36.467 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB GerDaLIRSmall (default) |
|
revision: 48327de6ee192e9610f3069789719788957c7abd |
|
split: test |
|
type: mteb/GerDaLIRSmall |
|
metrics: |
|
- type: main_score |
|
value: 50.041999999999994 |
|
- type: map_at_1 |
|
value: 32.192 |
|
- type: map_at_10 |
|
value: 44.156 |
|
- type: map_at_100 |
|
value: 45.08 |
|
- type: map_at_1000 |
|
value: 45.126 |
|
- type: map_at_20 |
|
value: 44.743 |
|
- type: map_at_3 |
|
value: 40.886 |
|
- type: map_at_5 |
|
value: 42.834 |
|
- type: mrr_at_1 |
|
value: 35.67108059506294 |
|
- type: mrr_at_10 |
|
value: 46.08966424562292 |
|
- type: mrr_at_100 |
|
value: 46.85163949774394 |
|
- type: mrr_at_1000 |
|
value: 46.88987506054722 |
|
- type: mrr_at_20 |
|
value: 46.56428992878071 |
|
- type: mrr_at_3 |
|
value: 43.33415072748076 |
|
- type: mrr_at_5 |
|
value: 44.98406081412419 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 47.08157584565528 |
|
- type: nauc_map_at_1000_max |
|
value: 43.206128601312486 |
|
- type: nauc_map_at_1000_std |
|
value: 4.745982286256461 |
|
- type: nauc_map_at_100_diff1 |
|
value: 47.07078139806274 |
|
- type: nauc_map_at_100_max |
|
value: 43.23099197560626 |
|
- type: nauc_map_at_100_std |
|
value: 4.781780686055857 |
|
- type: nauc_map_at_10_diff1 |
|
value: 47.06888215588044 |
|
- type: nauc_map_at_10_max |
|
value: 42.993085843512404 |
|
- type: nauc_map_at_10_std |
|
value: 4.137168566460313 |
|
- type: nauc_map_at_1_diff1 |
|
value: 50.99589527197208 |
|
- type: nauc_map_at_1_max |
|
value: 35.91682256310639 |
|
- type: nauc_map_at_1_std |
|
value: -1.167131810256268 |
|
- type: nauc_map_at_20_diff1 |
|
value: 47.04454709015284 |
|
- type: nauc_map_at_20_max |
|
value: 43.25308050978819 |
|
- type: nauc_map_at_20_std |
|
value: 4.655176251983949 |
|
- type: nauc_map_at_3_diff1 |
|
value: 47.477586403020254 |
|
- type: nauc_map_at_3_max |
|
value: 41.070269193209505 |
|
- type: nauc_map_at_3_std |
|
value: 1.8419013437125402 |
|
- type: nauc_map_at_5_diff1 |
|
value: 47.02373499346413 |
|
- type: nauc_map_at_5_max |
|
value: 42.16258581830696 |
|
- type: nauc_map_at_5_std |
|
value: 3.0623218224529687 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 47.80488822997101 |
|
- type: nauc_mrr_at_1000_max |
|
value: 44.30020606841032 |
|
- type: nauc_mrr_at_1000_std |
|
value: 5.636122646285231 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 47.79392263287222 |
|
- type: nauc_mrr_at_100_max |
|
value: 44.32088767447976 |
|
- type: nauc_mrr_at_100_std |
|
value: 5.668194634702436 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 47.76641281589811 |
|
- type: nauc_mrr_at_10_max |
|
value: 44.279337066083805 |
|
- type: nauc_mrr_at_10_std |
|
value: 5.324066003258172 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 51.827466333842494 |
|
- type: nauc_mrr_at_1_max |
|
value: 39.9726801468231 |
|
- type: nauc_mrr_at_1_std |
|
value: 0.7288717627709872 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 47.764880161368986 |
|
- type: nauc_mrr_at_20_max |
|
value: 44.37823842067007 |
|
- type: nauc_mrr_at_20_std |
|
value: 5.622281146435594 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 48.09440765276413 |
|
- type: nauc_mrr_at_3_max |
|
value: 43.08121834325808 |
|
- type: nauc_mrr_at_3_std |
|
value: 3.463004783984404 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 47.722106520188504 |
|
- type: nauc_mrr_at_5_max |
|
value: 43.75524931306371 |
|
- type: nauc_mrr_at_5_std |
|
value: 4.544429473258565 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 46.173190405196785 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 45.74626009789115 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 8.766536922367903 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 45.83562302519218 |
|
- type: nauc_ndcg_at_100_max |
|
value: 46.439699221515376 |
|
- type: nauc_ndcg_at_100_std |
|
value: 9.948770560612058 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 45.75120503110639 |
|
- type: nauc_ndcg_at_10_max |
|
value: 45.97030387145503 |
|
- type: nauc_ndcg_at_10_std |
|
value: 7.585101690722181 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 51.80241384384428 |
|
- type: nauc_ndcg_at_1_max |
|
value: 39.954443617615624 |
|
- type: nauc_ndcg_at_1_std |
|
value: 0.6967907136634796 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 45.653758645238895 |
|
- type: nauc_ndcg_at_20_max |
|
value: 46.71180118809166 |
|
- type: nauc_ndcg_at_20_std |
|
value: 9.250603495033369 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 46.451364520561924 |
|
- type: nauc_ndcg_at_3_max |
|
value: 42.65149578608251 |
|
- type: nauc_ndcg_at_3_std |
|
value: 3.18214445444501 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 45.70310380498345 |
|
- type: nauc_ndcg_at_5_max |
|
value: 44.210803603105866 |
|
- type: nauc_ndcg_at_5_std |
|
value: 5.164266026630885 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: 1.5935204591909151 |
|
- type: nauc_precision_at_1000_max |
|
value: 24.49290419273546 |
|
- type: nauc_precision_at_1000_std |
|
value: 23.419221041007578 |
|
- type: nauc_precision_at_100_diff1 |
|
value: 11.340541238515147 |
|
- type: nauc_precision_at_100_max |
|
value: 38.03551764407295 |
|
- type: nauc_precision_at_100_std |
|
value: 31.0523811487812 |
|
- type: nauc_precision_at_10_diff1 |
|
value: 25.73178507519107 |
|
- type: nauc_precision_at_10_max |
|
value: 46.868875006839545 |
|
- type: nauc_precision_at_10_std |
|
value: 20.062754134173872 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 51.80241384384428 |
|
- type: nauc_precision_at_1_max |
|
value: 39.954443617615624 |
|
- type: nauc_precision_at_1_std |
|
value: 0.6967907136634796 |
|
- type: nauc_precision_at_20_diff1 |
|
value: 20.43139551018563 |
|
- type: nauc_precision_at_20_max |
|
value: 46.00136497143788 |
|
- type: nauc_precision_at_20_std |
|
value: 26.43348589267883 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 36.477540595476235 |
|
- type: nauc_precision_at_3_max |
|
value: 45.27812276802484 |
|
- type: nauc_precision_at_3_std |
|
value: 7.988192977691942 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 30.711545494807886 |
|
- type: nauc_precision_at_5_max |
|
value: 45.84532014309742 |
|
- type: nauc_precision_at_5_std |
|
value: 12.810198584032126 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: 34.222675530936584 |
|
- type: nauc_recall_at_1000_max |
|
value: 64.84714200625044 |
|
- type: nauc_recall_at_1000_std |
|
value: 58.96182247732879 |
|
- type: nauc_recall_at_100_diff1 |
|
value: 35.36855555958951 |
|
- type: nauc_recall_at_100_max |
|
value: 62.91068693686705 |
|
- type: nauc_recall_at_100_std |
|
value: 46.54158522159893 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 39.32976276229237 |
|
- type: nauc_recall_at_10_max |
|
value: 53.86613555456697 |
|
- type: nauc_recall_at_10_std |
|
value: 18.601960247067957 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 50.99589527197208 |
|
- type: nauc_recall_at_1_max |
|
value: 35.91682256310639 |
|
- type: nauc_recall_at_1_std |
|
value: -1.167131810256268 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 37.57662491377409 |
|
- type: nauc_recall_at_20_max |
|
value: 58.543627483010816 |
|
- type: nauc_recall_at_20_std |
|
value: 28.511725371993336 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 42.4797724119925 |
|
- type: nauc_recall_at_3_max |
|
value: 43.96696921591431 |
|
- type: nauc_recall_at_3_std |
|
value: 4.855189723191562 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 40.20239865372907 |
|
- type: nauc_recall_at_5_max |
|
value: 47.80150483467517 |
|
- type: nauc_recall_at_5_std |
|
value: 9.903701080938268 |
|
- type: ndcg_at_1 |
|
value: 35.679 |
|
- type: ndcg_at_10 |
|
value: 50.041999999999994 |
|
- type: ndcg_at_100 |
|
value: 54.193000000000005 |
|
- type: ndcg_at_1000 |
|
value: 55.444 |
|
- type: ndcg_at_20 |
|
value: 51.961 |
|
- type: ndcg_at_3 |
|
value: 43.942 |
|
- type: ndcg_at_5 |
|
value: 47.121 |
|
- type: precision_at_1 |
|
value: 35.679 |
|
- type: precision_at_10 |
|
value: 7.733 |
|
- type: precision_at_100 |
|
value: 0.997 |
|
- type: precision_at_1000 |
|
value: 0.11100000000000002 |
|
- type: precision_at_20 |
|
value: 4.304 |
|
- type: precision_at_3 |
|
value: 19.165 |
|
- type: precision_at_5 |
|
value: 13.366 |
|
- type: recall_at_1 |
|
value: 32.192 |
|
- type: recall_at_10 |
|
value: 66.226 |
|
- type: recall_at_100 |
|
value: 84.985 |
|
- type: recall_at_1000 |
|
value: 94.76299999999999 |
|
- type: recall_at_20 |
|
value: 73.456 |
|
- type: recall_at_3 |
|
value: 50.071 |
|
- type: recall_at_5 |
|
value: 57.60999999999999 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB LeCaRDv2 (default) |
|
revision: b78e18688c3d012a33dc3676597c1d1b2243ce1c |
|
split: test |
|
type: mteb/LeCaRDv2 |
|
metrics: |
|
- type: main_score |
|
value: 75.955 |
|
- type: map_at_1 |
|
value: 4.099 |
|
- type: map_at_10 |
|
value: 27.639999999999997 |
|
- type: map_at_100 |
|
value: 63.919000000000004 |
|
- type: map_at_1000 |
|
value: 64.742 |
|
- type: map_at_20 |
|
value: 43.64 |
|
- type: map_at_3 |
|
value: 11.017000000000001 |
|
- type: map_at_5 |
|
value: 16.487 |
|
- type: mrr_at_1 |
|
value: 86.79245283018868 |
|
- type: mrr_at_10 |
|
value: 91.09613656783468 |
|
- type: mrr_at_100 |
|
value: 91.24099956647127 |
|
- type: mrr_at_1000 |
|
value: 91.24099956647127 |
|
- type: mrr_at_20 |
|
value: 91.24099956647127 |
|
- type: mrr_at_3 |
|
value: 90.67085953878407 |
|
- type: mrr_at_5 |
|
value: 90.7966457023061 |
|
- type: nauc_map_at_1000_diff1 |
|
value: -0.1165116760309431 |
|
- type: nauc_map_at_1000_max |
|
value: 36.09282039931682 |
|
- type: nauc_map_at_1000_std |
|
value: 16.717641031560422 |
|
- type: nauc_map_at_100_diff1 |
|
value: -0.1977916865120706 |
|
- type: nauc_map_at_100_max |
|
value: 35.95558832098128 |
|
- type: nauc_map_at_100_std |
|
value: 17.15965620332017 |
|
- type: nauc_map_at_10_diff1 |
|
value: 11.88383387650193 |
|
- type: nauc_map_at_10_max |
|
value: 19.323749519368256 |
|
- type: nauc_map_at_10_std |
|
value: -13.368970916127784 |
|
- type: nauc_map_at_1_diff1 |
|
value: 12.468762885123516 |
|
- type: nauc_map_at_1_max |
|
value: 5.597229875368963 |
|
- type: nauc_map_at_1_std |
|
value: -25.835685303037202 |
|
- type: nauc_map_at_20_diff1 |
|
value: 4.625997058170944 |
|
- type: nauc_map_at_20_max |
|
value: 23.42312594320493 |
|
- type: nauc_map_at_20_std |
|
value: -1.2842557193275415 |
|
- type: nauc_map_at_3_diff1 |
|
value: 16.457500025698952 |
|
- type: nauc_map_at_3_max |
|
value: 12.260630248199256 |
|
- type: nauc_map_at_3_std |
|
value: -23.60500260290979 |
|
- type: nauc_map_at_5_diff1 |
|
value: 15.924351493208256 |
|
- type: nauc_map_at_5_max |
|
value: 17.553142481774472 |
|
- type: nauc_map_at_5_std |
|
value: -17.604184344296943 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 23.24803044899622 |
|
- type: nauc_mrr_at_1000_max |
|
value: 67.6449236362031 |
|
- type: nauc_mrr_at_1000_std |
|
value: 18.966799318760344 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 23.24803044899622 |
|
- type: nauc_mrr_at_100_max |
|
value: 67.6449236362031 |
|
- type: nauc_mrr_at_100_std |
|
value: 18.966799318760344 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 22.7515823299183 |
|
- type: nauc_mrr_at_10_max |
|
value: 67.04946200016059 |
|
- type: nauc_mrr_at_10_std |
|
value: 19.187654338444936 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 29.056072093682022 |
|
- type: nauc_mrr_at_1_max |
|
value: 68.55895633768269 |
|
- type: nauc_mrr_at_1_std |
|
value: 15.092113581290862 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 23.24803044899622 |
|
- type: nauc_mrr_at_20_max |
|
value: 67.6449236362031 |
|
- type: nauc_mrr_at_20_std |
|
value: 18.966799318760344 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 23.287597081925508 |
|
- type: nauc_mrr_at_3_max |
|
value: 68.05169166666252 |
|
- type: nauc_mrr_at_3_std |
|
value: 19.785092034648617 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 23.11480929003818 |
|
- type: nauc_mrr_at_5_max |
|
value: 67.99383983419222 |
|
- type: nauc_mrr_at_5_std |
|
value: 19.052480486459373 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 1.5782017453966002 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 41.5302298842471 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 20.50865790647901 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: -2.1894348746609804 |
|
- type: nauc_ndcg_at_100_max |
|
value: 37.14267089909159 |
|
- type: nauc_ndcg_at_100_std |
|
value: 25.276579300026942 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 4.916325074632776 |
|
- type: nauc_ndcg_at_10_max |
|
value: 41.10158619113785 |
|
- type: nauc_ndcg_at_10_std |
|
value: 12.855454525343141 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 29.056072093682022 |
|
- type: nauc_ndcg_at_1_max |
|
value: 68.55895633768269 |
|
- type: nauc_ndcg_at_1_std |
|
value: 15.092113581290862 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: -0.7532516396275979 |
|
- type: nauc_ndcg_at_20_max |
|
value: 32.78457335101816 |
|
- type: nauc_ndcg_at_20_std |
|
value: 13.463927254088695 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 12.68114834388015 |
|
- type: nauc_ndcg_at_3_max |
|
value: 54.41470867032895 |
|
- type: nauc_ndcg_at_3_std |
|
value: 18.565606565635647 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 10.193606786989358 |
|
- type: nauc_ndcg_at_5_max |
|
value: 51.401706187544825 |
|
- type: nauc_ndcg_at_5_std |
|
value: 19.805481146848177 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: -11.29314155191534 |
|
- type: nauc_precision_at_1000_max |
|
value: 36.90077388824628 |
|
- type: nauc_precision_at_1000_std |
|
value: 42.40123140196757 |
|
- type: nauc_precision_at_100_diff1 |
|
value: -12.974354162087675 |
|
- type: nauc_precision_at_100_max |
|
value: 34.81416396301287 |
|
- type: nauc_precision_at_100_std |
|
value: 45.724028299929955 |
|
- type: nauc_precision_at_10_diff1 |
|
value: 0.25354300446169603 |
|
- type: nauc_precision_at_10_max |
|
value: 34.7650903227545 |
|
- type: nauc_precision_at_10_std |
|
value: 14.426822867124692 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 29.056072093682022 |
|
- type: nauc_precision_at_1_max |
|
value: 68.55895633768269 |
|
- type: nauc_precision_at_1_std |
|
value: 15.092113581290862 |
|
- type: nauc_precision_at_20_diff1 |
|
value: -4.398113098730461 |
|
- type: nauc_precision_at_20_max |
|
value: 29.644722534077783 |
|
- type: nauc_precision_at_20_std |
|
value: 20.498896408366953 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 8.511537696975939 |
|
- type: nauc_precision_at_3_max |
|
value: 51.02291732645718 |
|
- type: nauc_precision_at_3_std |
|
value: 19.018537614881883 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 5.968041335751909 |
|
- type: nauc_precision_at_5_max |
|
value: 47.08751216174023 |
|
- type: nauc_precision_at_5_std |
|
value: 21.422128104399597 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: -114.68537043092684 |
|
- type: nauc_recall_at_1000_max |
|
value: -17.258899842714303 |
|
- type: nauc_recall_at_1000_std |
|
value: 35.64737342608127 |
|
- type: nauc_recall_at_100_diff1 |
|
value: -21.918374369212724 |
|
- type: nauc_recall_at_100_max |
|
value: 9.323860457981256 |
|
- type: nauc_recall_at_100_std |
|
value: 41.88023148573863 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 6.872030552051965 |
|
- type: nauc_recall_at_10_max |
|
value: 3.87403864340295 |
|
- type: nauc_recall_at_10_std |
|
value: -21.003946521947647 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 12.468762885123516 |
|
- type: nauc_recall_at_1_max |
|
value: 5.597229875368963 |
|
- type: nauc_recall_at_1_std |
|
value: -25.835685303037202 |
|
- type: nauc_recall_at_20_diff1 |
|
value: -0.957514361288361 |
|
- type: nauc_recall_at_20_max |
|
value: 6.322474574744254 |
|
- type: nauc_recall_at_20_std |
|
value: -7.517845037342536 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 11.6795437402863 |
|
- type: nauc_recall_at_3_max |
|
value: 4.802113241170994 |
|
- type: nauc_recall_at_3_std |
|
value: -25.562583690610108 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 11.34475005606284 |
|
- type: nauc_recall_at_5_max |
|
value: 8.318487558973697 |
|
- type: nauc_recall_at_5_std |
|
value: -21.462212735159852 |
|
- type: ndcg_at_1 |
|
value: 86.792 |
|
- type: ndcg_at_10 |
|
value: 75.955 |
|
- type: ndcg_at_100 |
|
value: 83.532 |
|
- type: ndcg_at_1000 |
|
value: 86.085 |
|
- type: ndcg_at_20 |
|
value: 70.61699999999999 |
|
- type: ndcg_at_3 |
|
value: 83.672 |
|
- type: ndcg_at_5 |
|
value: 80.988 |
|
- type: precision_at_1 |
|
value: 86.792 |
|
- type: precision_at_10 |
|
value: 71.572 |
|
- type: precision_at_100 |
|
value: 22.962 |
|
- type: precision_at_1000 |
|
value: 2.448 |
|
- type: precision_at_20 |
|
value: 62.075 |
|
- type: precision_at_3 |
|
value: 83.019 |
|
- type: precision_at_5 |
|
value: 79.119 |
|
- type: recall_at_1 |
|
value: 4.099 |
|
- type: recall_at_10 |
|
value: 30.963 |
|
- type: recall_at_100 |
|
value: 93.66799999999999 |
|
- type: recall_at_1000 |
|
value: 99.91 |
|
- type: recall_at_20 |
|
value: 51.763999999999996 |
|
- type: recall_at_3 |
|
value: 11.644 |
|
- type: recall_at_5 |
|
value: 17.646 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB LegalBenchConsumerContractsQA (default) |
|
revision: b23590301ec94e8087e2850b21d43d4956b1cca9 |
|
split: test |
|
type: mteb/legalbench_consumer_contracts_qa |
|
metrics: |
|
- type: main_score |
|
value: 81.722 |
|
- type: map_at_1 |
|
value: 66.919 |
|
- type: map_at_10 |
|
value: 77.11099999999999 |
|
- type: map_at_100 |
|
value: 77.365 |
|
- type: map_at_1000 |
|
value: 77.365 |
|
- type: map_at_20 |
|
value: 77.32 |
|
- type: map_at_3 |
|
value: 75.042 |
|
- type: map_at_5 |
|
value: 76.519 |
|
- type: mrr_at_1 |
|
value: 67.17171717171718 |
|
- type: mrr_at_10 |
|
value: 77.23735369568702 |
|
- type: mrr_at_100 |
|
value: 77.49038149701025 |
|
- type: mrr_at_1000 |
|
value: 77.49038149701025 |
|
- type: mrr_at_20 |
|
value: 77.44484086340144 |
|
- type: mrr_at_3 |
|
value: 75.16835016835017 |
|
- type: mrr_at_5 |
|
value: 76.64562289562288 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 52.97349214688111 |
|
- type: nauc_map_at_1000_max |
|
value: 36.6960251962721 |
|
- type: nauc_map_at_1000_std |
|
value: 5.3061037068017995 |
|
- type: nauc_map_at_100_diff1 |
|
value: 52.97349214688111 |
|
- type: nauc_map_at_100_max |
|
value: 36.6960251962721 |
|
- type: nauc_map_at_100_std |
|
value: 5.3061037068017995 |
|
- type: nauc_map_at_10_diff1 |
|
value: 52.75678159019962 |
|
- type: nauc_map_at_10_max |
|
value: 36.540933731741696 |
|
- type: nauc_map_at_10_std |
|
value: 5.301224105047955 |
|
- type: nauc_map_at_1_diff1 |
|
value: 57.67214955091278 |
|
- type: nauc_map_at_1_max |
|
value: 35.78220074855998 |
|
- type: nauc_map_at_1_std |
|
value: 3.4210250671003326 |
|
- type: nauc_map_at_20_diff1 |
|
value: 52.932588393157765 |
|
- type: nauc_map_at_20_max |
|
value: 36.63003364305272 |
|
- type: nauc_map_at_20_std |
|
value: 5.379295465437984 |
|
- type: nauc_map_at_3_diff1 |
|
value: 52.480331430472546 |
|
- type: nauc_map_at_3_max |
|
value: 38.091587355109084 |
|
- type: nauc_map_at_3_std |
|
value: 5.195570015494146 |
|
- type: nauc_map_at_5_diff1 |
|
value: 51.82108883079516 |
|
- type: nauc_map_at_5_max |
|
value: 36.87971971720586 |
|
- type: nauc_map_at_5_std |
|
value: 6.1809111678690725 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 52.50999035933979 |
|
- type: nauc_mrr_at_1000_max |
|
value: 36.34762852524654 |
|
- type: nauc_mrr_at_1000_std |
|
value: 4.968842635399586 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 52.50999035933979 |
|
- type: nauc_mrr_at_100_max |
|
value: 36.34762852524654 |
|
- type: nauc_mrr_at_100_std |
|
value: 4.968842635399586 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 52.30508629262912 |
|
- type: nauc_mrr_at_10_max |
|
value: 36.209224098805336 |
|
- type: nauc_mrr_at_10_std |
|
value: 4.966404389655018 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 57.04712337185792 |
|
- type: nauc_mrr_at_1_max |
|
value: 35.30097575331179 |
|
- type: nauc_mrr_at_1_std |
|
value: 2.9296181705495568 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 52.4700212348044 |
|
- type: nauc_mrr_at_20_max |
|
value: 36.282215725584074 |
|
- type: nauc_mrr_at_20_std |
|
value: 5.043076008887179 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 52.06774147045164 |
|
- type: nauc_mrr_at_3_max |
|
value: 37.79424665327426 |
|
- type: nauc_mrr_at_3_std |
|
value: 4.890057068652614 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 51.37929937217122 |
|
- type: nauc_mrr_at_5_max |
|
value: 36.55880654826355 |
|
- type: nauc_mrr_at_5_std |
|
value: 5.858835123935254 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 52.35035912300684 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 36.68201866972285 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 5.7202049594831035 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 52.35035912300684 |
|
- type: nauc_ndcg_at_100_max |
|
value: 36.68201866972285 |
|
- type: nauc_ndcg_at_100_std |
|
value: 5.7202049594831035 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 51.28316055116551 |
|
- type: nauc_ndcg_at_10_max |
|
value: 35.87712251784529 |
|
- type: nauc_ndcg_at_10_std |
|
value: 6.021170340566541 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 57.67214955091278 |
|
- type: nauc_ndcg_at_1_max |
|
value: 35.78220074855998 |
|
- type: nauc_ndcg_at_1_std |
|
value: 3.4210250671003326 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 52.026541138524394 |
|
- type: nauc_ndcg_at_20_max |
|
value: 36.23499300677105 |
|
- type: nauc_ndcg_at_20_std |
|
value: 6.3316005762687695 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 50.317993712307285 |
|
- type: nauc_ndcg_at_3_max |
|
value: 39.27048927075592 |
|
- type: nauc_ndcg_at_3_std |
|
value: 6.266648429252955 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 48.69195472599095 |
|
- type: nauc_ndcg_at_5_max |
|
value: 36.920018592423126 |
|
- type: nauc_ndcg_at_5_std |
|
value: 8.451954024556523 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_precision_at_1000_max |
|
value: .nan |
|
- type: nauc_precision_at_1000_std |
|
value: .nan |
|
- type: nauc_precision_at_100_diff1 |
|
value: .nan |
|
- type: nauc_precision_at_100_max |
|
value: .nan |
|
- type: nauc_precision_at_100_std |
|
value: .nan |
|
- type: nauc_precision_at_10_diff1 |
|
value: 34.43489322486972 |
|
- type: nauc_precision_at_10_max |
|
value: 23.186302881363822 |
|
- type: nauc_precision_at_10_std |
|
value: 13.793599801222541 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 57.67214955091278 |
|
- type: nauc_precision_at_1_max |
|
value: 35.78220074855998 |
|
- type: nauc_precision_at_1_std |
|
value: 3.4210250671003326 |
|
- type: nauc_precision_at_20_diff1 |
|
value: 32.55853898108205 |
|
- type: nauc_precision_at_20_max |
|
value: 11.404195591060642 |
|
- type: nauc_precision_at_20_std |
|
value: 43.5675136741475 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 40.852371923000376 |
|
- type: nauc_precision_at_3_max |
|
value: 44.53164299608135 |
|
- type: nauc_precision_at_3_std |
|
value: 11.109027031191964 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 26.165078486515554 |
|
- type: nauc_precision_at_5_max |
|
value: 36.58403503282701 |
|
- type: nauc_precision_at_5_std |
|
value: 25.437593302644462 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_100_max |
|
value: .nan |
|
- type: nauc_recall_at_100_std |
|
value: .nan |
|
- type: nauc_recall_at_10_diff1 |
|
value: 34.43489322486968 |
|
- type: nauc_recall_at_10_max |
|
value: 23.186302881364337 |
|
- type: nauc_recall_at_10_std |
|
value: 13.793599801223758 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 57.67214955091278 |
|
- type: nauc_recall_at_1_max |
|
value: 35.78220074855998 |
|
- type: nauc_recall_at_1_std |
|
value: 3.4210250671003326 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 32.55853898108042 |
|
- type: nauc_recall_at_20_max |
|
value: 11.404195591059747 |
|
- type: nauc_recall_at_20_std |
|
value: 43.56751367414731 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 40.852371923000334 |
|
- type: nauc_recall_at_3_max |
|
value: 44.53164299608149 |
|
- type: nauc_recall_at_3_std |
|
value: 11.10902703119201 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 26.165078486515746 |
|
- type: nauc_recall_at_5_max |
|
value: 36.58403503282719 |
|
- type: nauc_recall_at_5_std |
|
value: 25.437593302644736 |
|
- type: ndcg_at_1 |
|
value: 66.919 |
|
- type: ndcg_at_10 |
|
value: 81.722 |
|
- type: ndcg_at_100 |
|
value: 82.704 |
|
- type: ndcg_at_1000 |
|
value: 82.704 |
|
- type: ndcg_at_20 |
|
value: 82.44399999999999 |
|
- type: ndcg_at_3 |
|
value: 77.63 |
|
- type: ndcg_at_5 |
|
value: 80.314 |
|
- type: precision_at_1 |
|
value: 66.919 |
|
- type: precision_at_10 |
|
value: 9.596 |
|
- type: precision_at_100 |
|
value: 1.0 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_20 |
|
value: 4.936999999999999 |
|
- type: precision_at_3 |
|
value: 28.366999999999997 |
|
- type: precision_at_5 |
|
value: 18.333 |
|
- type: recall_at_1 |
|
value: 66.919 |
|
- type: recall_at_10 |
|
value: 95.96000000000001 |
|
- type: recall_at_100 |
|
value: 100.0 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 98.737 |
|
- type: recall_at_3 |
|
value: 85.101 |
|
- type: recall_at_5 |
|
value: 91.667 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB LegalBenchCorporateLobbying (default) |
|
revision: f69691c650464e62546d7f2a4536f8f87c891e38 |
|
split: test |
|
type: mteb/legalbench_corporate_lobbying |
|
metrics: |
|
- type: main_score |
|
value: 95.275 |
|
- type: map_at_1 |
|
value: 90.0 |
|
- type: map_at_10 |
|
value: 93.896 |
|
- type: map_at_100 |
|
value: 93.896 |
|
- type: map_at_1000 |
|
value: 93.89999999999999 |
|
- type: map_at_20 |
|
value: 93.896 |
|
- type: map_at_3 |
|
value: 93.529 |
|
- type: map_at_5 |
|
value: 93.735 |
|
- type: mrr_at_1 |
|
value: 90.0 |
|
- type: mrr_at_10 |
|
value: 93.89577497665734 |
|
- type: mrr_at_100 |
|
value: 93.89577497665734 |
|
- type: mrr_at_1000 |
|
value: 93.89961722973894 |
|
- type: mrr_at_20 |
|
value: 93.89577497665734 |
|
- type: mrr_at_3 |
|
value: 93.52941176470588 |
|
- type: mrr_at_5 |
|
value: 93.73529411764707 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 87.98954780116061 |
|
- type: nauc_map_at_1000_max |
|
value: 48.73465559069755 |
|
- type: nauc_map_at_1000_std |
|
value: -24.16685492630147 |
|
- type: nauc_map_at_100_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_map_at_100_max |
|
value: 48.726283117594946 |
|
- type: nauc_map_at_100_std |
|
value: -24.21564012563856 |
|
- type: nauc_map_at_10_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_map_at_10_max |
|
value: 48.726283117594946 |
|
- type: nauc_map_at_10_std |
|
value: -24.21564012563856 |
|
- type: nauc_map_at_1_diff1 |
|
value: 88.58268797715169 |
|
- type: nauc_map_at_1_max |
|
value: 41.42912066787494 |
|
- type: nauc_map_at_1_std |
|
value: -20.242214532871987 |
|
- type: nauc_map_at_20_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_map_at_20_max |
|
value: 48.726283117594946 |
|
- type: nauc_map_at_20_std |
|
value: -24.21564012563856 |
|
- type: nauc_map_at_3_diff1 |
|
value: 88.40647370059126 |
|
- type: nauc_map_at_3_max |
|
value: 51.27040715275998 |
|
- type: nauc_map_at_3_std |
|
value: -26.350691791868247 |
|
- type: nauc_map_at_5_diff1 |
|
value: 88.02546579403796 |
|
- type: nauc_map_at_5_max |
|
value: 49.82239113694508 |
|
- type: nauc_map_at_5_std |
|
value: -24.375227399253678 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 87.98954780116061 |
|
- type: nauc_mrr_at_1000_max |
|
value: 50.085159299027296 |
|
- type: nauc_mrr_at_1000_std |
|
value: -23.851737394357862 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_mrr_at_100_max |
|
value: 50.075936762728254 |
|
- type: nauc_mrr_at_100_std |
|
value: -23.90072094177412 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_mrr_at_10_max |
|
value: 50.075936762728254 |
|
- type: nauc_mrr_at_10_std |
|
value: -23.90072094177412 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 88.58268797715169 |
|
- type: nauc_mrr_at_1_max |
|
value: 43.076838578568626 |
|
- type: nauc_mrr_at_1_std |
|
value: -19.857747020376795 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 87.97183672739986 |
|
- type: nauc_mrr_at_20_max |
|
value: 50.075936762728254 |
|
- type: nauc_mrr_at_20_std |
|
value: -23.90072094177412 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 88.40647370059126 |
|
- type: nauc_mrr_at_3_max |
|
value: 52.54364372011418 |
|
- type: nauc_mrr_at_3_std |
|
value: -26.053603259485598 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 88.02546579403796 |
|
- type: nauc_mrr_at_5_max |
|
value: 51.13747115956445 |
|
- type: nauc_mrr_at_5_std |
|
value: -24.068375393975824 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 87.66050817567788 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 49.54001381756884 |
|
- type: nauc_ndcg_at_1000_std |
|
value: -24.35184631163463 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 87.24416537653535 |
|
- type: nauc_ndcg_at_100_max |
|
value: 49.57831808101997 |
|
- type: nauc_ndcg_at_100_std |
|
value: -24.793463718624537 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 87.24416537653535 |
|
- type: nauc_ndcg_at_10_max |
|
value: 49.57831808101997 |
|
- type: nauc_ndcg_at_10_std |
|
value: -24.793463718624537 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 88.58268797715169 |
|
- type: nauc_ndcg_at_1_max |
|
value: 41.42912066787494 |
|
- type: nauc_ndcg_at_1_std |
|
value: -20.242214532871987 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 87.24416537653535 |
|
- type: nauc_ndcg_at_20_max |
|
value: 49.57831808101997 |
|
- type: nauc_ndcg_at_20_std |
|
value: -24.793463718624537 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 88.28539315614394 |
|
- type: nauc_ndcg_at_3_max |
|
value: 55.661427932222765 |
|
- type: nauc_ndcg_at_3_std |
|
value: -29.09686303480698 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 87.44364795542245 |
|
- type: nauc_ndcg_at_5_max |
|
value: 52.79959281779241 |
|
- type: nauc_ndcg_at_5_std |
|
value: -24.76177349657667 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: 100.0 |
|
- type: nauc_precision_at_1000_max |
|
value: 100.0 |
|
- type: nauc_precision_at_1000_std |
|
value: 100.0 |
|
- type: nauc_precision_at_100_diff1 |
|
value: 63.81886087768098 |
|
- type: nauc_precision_at_100_max |
|
value: 56.1391223155907 |
|
- type: nauc_precision_at_100_std |
|
value: -36.99813258637037 |
|
- type: nauc_precision_at_10_diff1 |
|
value: 63.81886087768457 |
|
- type: nauc_precision_at_10_max |
|
value: 56.13912231559321 |
|
- type: nauc_precision_at_10_std |
|
value: -36.99813258636605 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 88.58268797715169 |
|
- type: nauc_precision_at_1_max |
|
value: 41.42912066787494 |
|
- type: nauc_precision_at_1_std |
|
value: -20.242214532871987 |
|
- type: nauc_precision_at_20_diff1 |
|
value: 63.81886087768457 |
|
- type: nauc_precision_at_20_max |
|
value: 56.13912231559321 |
|
- type: nauc_precision_at_20_std |
|
value: -36.99813258636605 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 87.42089428363992 |
|
- type: nauc_precision_at_3_max |
|
value: 85.71428571428666 |
|
- type: nauc_precision_at_3_std |
|
value: -47.90953418404368 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 81.13134142545934 |
|
- type: nauc_precision_at_5_max |
|
value: 80.75007780890182 |
|
- type: nauc_precision_at_5_std |
|
value: -25.715841892311385 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: 63.81886087768457 |
|
- type: nauc_recall_at_100_max |
|
value: 56.13912231559266 |
|
- type: nauc_recall_at_100_std |
|
value: -36.99813258636772 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 63.81886087768457 |
|
- type: nauc_recall_at_10_max |
|
value: 56.13912231559266 |
|
- type: nauc_recall_at_10_std |
|
value: -36.99813258636772 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 88.58268797715169 |
|
- type: nauc_recall_at_1_max |
|
value: 41.42912066787494 |
|
- type: nauc_recall_at_1_std |
|
value: -20.242214532871987 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 63.81886087768457 |
|
- type: nauc_recall_at_20_max |
|
value: 56.13912231559266 |
|
- type: nauc_recall_at_20_std |
|
value: -36.99813258636772 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 87.4208942836394 |
|
- type: nauc_recall_at_3_max |
|
value: 85.71428571428564 |
|
- type: nauc_recall_at_3_std |
|
value: -47.90953418404379 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 81.1313414254593 |
|
- type: nauc_recall_at_5_max |
|
value: 80.75007780890142 |
|
- type: nauc_recall_at_5_std |
|
value: -25.715841892312174 |
|
- type: ndcg_at_1 |
|
value: 90.0 |
|
- type: ndcg_at_10 |
|
value: 95.275 |
|
- type: ndcg_at_100 |
|
value: 95.275 |
|
- type: ndcg_at_1000 |
|
value: 95.355 |
|
- type: ndcg_at_20 |
|
value: 95.275 |
|
- type: ndcg_at_3 |
|
value: 94.524 |
|
- type: ndcg_at_5 |
|
value: 94.891 |
|
- type: precision_at_1 |
|
value: 90.0 |
|
- type: precision_at_10 |
|
value: 9.940999999999999 |
|
- type: precision_at_100 |
|
value: 0.9939999999999999 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_20 |
|
value: 4.971 |
|
- type: precision_at_3 |
|
value: 32.451 |
|
- type: precision_at_5 |
|
value: 19.647000000000002 |
|
- type: recall_at_1 |
|
value: 90.0 |
|
- type: recall_at_10 |
|
value: 99.412 |
|
- type: recall_at_100 |
|
value: 99.412 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 99.412 |
|
- type: recall_at_3 |
|
value: 97.353 |
|
- type: recall_at_5 |
|
value: 98.235 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB LegalQuAD (default) |
|
revision: 37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e |
|
split: test |
|
type: mteb/LegalQuAD |
|
metrics: |
|
- type: main_score |
|
value: 64.899 |
|
- type: map_at_1 |
|
value: 48.5 |
|
- type: map_at_10 |
|
value: 59.607 |
|
- type: map_at_100 |
|
value: 60.102 |
|
- type: map_at_1000 |
|
value: 60.141 |
|
- type: map_at_20 |
|
value: 59.836 |
|
- type: map_at_3 |
|
value: 57.416999999999994 |
|
- type: map_at_5 |
|
value: 58.342000000000006 |
|
- type: mrr_at_1 |
|
value: 48.5 |
|
- type: mrr_at_10 |
|
value: 59.60734126984125 |
|
- type: mrr_at_100 |
|
value: 60.10134959097208 |
|
- type: mrr_at_1000 |
|
value: 60.140317577579324 |
|
- type: mrr_at_20 |
|
value: 59.83595938375349 |
|
- type: mrr_at_3 |
|
value: 57.41666666666665 |
|
- type: mrr_at_5 |
|
value: 58.34166666666666 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 62.80752643431633 |
|
- type: nauc_map_at_1000_max |
|
value: 62.00607212479527 |
|
- type: nauc_map_at_1000_std |
|
value: 17.541941429951855 |
|
- type: nauc_map_at_100_diff1 |
|
value: 62.807006564215705 |
|
- type: nauc_map_at_100_max |
|
value: 62.025705691974885 |
|
- type: nauc_map_at_100_std |
|
value: 17.56923712268851 |
|
- type: nauc_map_at_10_diff1 |
|
value: 62.51717633954077 |
|
- type: nauc_map_at_10_max |
|
value: 61.97403063599849 |
|
- type: nauc_map_at_10_std |
|
value: 17.634018830663045 |
|
- type: nauc_map_at_1_diff1 |
|
value: 65.73772773363197 |
|
- type: nauc_map_at_1_max |
|
value: 59.189205309863524 |
|
- type: nauc_map_at_1_std |
|
value: 14.461624107787843 |
|
- type: nauc_map_at_20_diff1 |
|
value: 62.72297267428931 |
|
- type: nauc_map_at_20_max |
|
value: 62.09138256517757 |
|
- type: nauc_map_at_20_std |
|
value: 17.751999039640335 |
|
- type: nauc_map_at_3_diff1 |
|
value: 63.4965467376001 |
|
- type: nauc_map_at_3_max |
|
value: 62.20134238108197 |
|
- type: nauc_map_at_3_std |
|
value: 16.0562228318107 |
|
- type: nauc_map_at_5_diff1 |
|
value: 63.596270852816886 |
|
- type: nauc_map_at_5_max |
|
value: 62.370748922533515 |
|
- type: nauc_map_at_5_std |
|
value: 16.199213024713476 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 62.807542338240665 |
|
- type: nauc_mrr_at_1000_max |
|
value: 62.00610157228559 |
|
- type: nauc_mrr_at_1000_std |
|
value: 17.54188368114912 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 62.80702245597816 |
|
- type: nauc_mrr_at_100_max |
|
value: 62.02573476935551 |
|
- type: nauc_mrr_at_100_std |
|
value: 17.569178989249227 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 62.51717633954077 |
|
- type: nauc_mrr_at_10_max |
|
value: 61.97403063599849 |
|
- type: nauc_mrr_at_10_std |
|
value: 17.634018830663045 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 65.73772773363197 |
|
- type: nauc_mrr_at_1_max |
|
value: 59.189205309863524 |
|
- type: nauc_mrr_at_1_std |
|
value: 14.461624107787843 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 62.72297267428931 |
|
- type: nauc_mrr_at_20_max |
|
value: 62.09138256517757 |
|
- type: nauc_mrr_at_20_std |
|
value: 17.751999039640335 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 63.4965467376001 |
|
- type: nauc_mrr_at_3_max |
|
value: 62.20134238108197 |
|
- type: nauc_mrr_at_3_std |
|
value: 16.0562228318107 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 63.596270852816886 |
|
- type: nauc_mrr_at_5_max |
|
value: 62.370748922533515 |
|
- type: nauc_mrr_at_5_std |
|
value: 16.199213024713476 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 61.99027512405032 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 62.3794577316155 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 18.81209576316582 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 61.981491536700695 |
|
- type: nauc_ndcg_at_100_max |
|
value: 62.83920688186244 |
|
- type: nauc_ndcg_at_100_std |
|
value: 19.50643109965116 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 60.04817741489627 |
|
- type: nauc_ndcg_at_10_max |
|
value: 62.4470034515535 |
|
- type: nauc_ndcg_at_10_std |
|
value: 20.53817059813657 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 65.73772773363197 |
|
- type: nauc_ndcg_at_1_max |
|
value: 59.189205309863524 |
|
- type: nauc_ndcg_at_1_std |
|
value: 14.461624107787843 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 60.907316049398794 |
|
- type: nauc_ndcg_at_20_max |
|
value: 62.99570450758024 |
|
- type: nauc_ndcg_at_20_std |
|
value: 21.058986818468124 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 62.73893976493435 |
|
- type: nauc_ndcg_at_3_max |
|
value: 63.03331537097003 |
|
- type: nauc_ndcg_at_3_std |
|
value: 16.64703608470413 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 62.877505768846845 |
|
- type: nauc_ndcg_at_5_max |
|
value: 63.383723263638544 |
|
- type: nauc_ndcg_at_5_std |
|
value: 16.955089182466672 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: 100.0 |
|
- type: nauc_precision_at_1000_max |
|
value: 100.0 |
|
- type: nauc_precision_at_1000_std |
|
value: 100.0 |
|
- type: nauc_precision_at_100_diff1 |
|
value: 61.49859943977586 |
|
- type: nauc_precision_at_100_max |
|
value: 78.61811391223151 |
|
- type: nauc_precision_at_100_std |
|
value: 43.9635854341738 |
|
- type: nauc_precision_at_10_diff1 |
|
value: 46.05874298601228 |
|
- type: nauc_precision_at_10_max |
|
value: 64.15281767659134 |
|
- type: nauc_precision_at_10_std |
|
value: 36.998952935806905 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 65.73772773363197 |
|
- type: nauc_precision_at_1_max |
|
value: 59.189205309863524 |
|
- type: nauc_precision_at_1_std |
|
value: 14.461624107787843 |
|
- type: nauc_precision_at_20_diff1 |
|
value: 49.614383338756745 |
|
- type: nauc_precision_at_20_max |
|
value: 68.69996745850949 |
|
- type: nauc_precision_at_20_std |
|
value: 44.100227790432726 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 60.181007322330856 |
|
- type: nauc_precision_at_3_max |
|
value: 65.7700665904469 |
|
- type: nauc_precision_at_3_std |
|
value: 18.6619072970213 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 60.24398508980005 |
|
- type: nauc_precision_at_5_max |
|
value: 67.09105872101463 |
|
- type: nauc_precision_at_5_std |
|
value: 19.826693130657848 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: 61.498599439775816 |
|
- type: nauc_recall_at_100_max |
|
value: 78.61811391223146 |
|
- type: nauc_recall_at_100_std |
|
value: 43.96358543417379 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 46.05874298601233 |
|
- type: nauc_recall_at_10_max |
|
value: 64.1528176765914 |
|
- type: nauc_recall_at_10_std |
|
value: 36.99895293580692 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 65.73772773363197 |
|
- type: nauc_recall_at_1_max |
|
value: 59.189205309863524 |
|
- type: nauc_recall_at_1_std |
|
value: 14.461624107787843 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 49.61438333875686 |
|
- type: nauc_recall_at_20_max |
|
value: 68.69996745850956 |
|
- type: nauc_recall_at_20_std |
|
value: 44.10022779043285 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 60.181007322330935 |
|
- type: nauc_recall_at_3_max |
|
value: 65.77006659044693 |
|
- type: nauc_recall_at_3_std |
|
value: 18.661907297021287 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 60.243985089800056 |
|
- type: nauc_recall_at_5_max |
|
value: 67.09105872101472 |
|
- type: nauc_recall_at_5_std |
|
value: 19.826693130657883 |
|
- type: ndcg_at_1 |
|
value: 48.5 |
|
- type: ndcg_at_10 |
|
value: 64.899 |
|
- type: ndcg_at_100 |
|
value: 67.634 |
|
- type: ndcg_at_1000 |
|
value: 68.345 |
|
- type: ndcg_at_20 |
|
value: 65.767 |
|
- type: ndcg_at_3 |
|
value: 60.268 |
|
- type: ndcg_at_5 |
|
value: 61.925 |
|
- type: precision_at_1 |
|
value: 48.5 |
|
- type: precision_at_10 |
|
value: 8.15 |
|
- type: precision_at_100 |
|
value: 0.95 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_20 |
|
value: 4.25 |
|
- type: precision_at_3 |
|
value: 22.833000000000002 |
|
- type: precision_at_5 |
|
value: 14.499999999999998 |
|
- type: recall_at_1 |
|
value: 48.5 |
|
- type: recall_at_10 |
|
value: 81.5 |
|
- type: recall_at_100 |
|
value: 95.0 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 85.0 |
|
- type: recall_at_3 |
|
value: 68.5 |
|
- type: recall_at_5 |
|
value: 72.5 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB LegalSummarization (default) |
|
revision: 3bb1a05c66872889662af04c5691c14489cebd72 |
|
split: test |
|
type: mteb/legal_summarization |
|
metrics: |
|
- type: main_score |
|
value: 68.37299999999999 |
|
- type: map_at_1 |
|
value: 50.583999999999996 |
|
- type: map_at_10 |
|
value: 62.287000000000006 |
|
- type: map_at_100 |
|
value: 63.537 |
|
- type: map_at_1000 |
|
value: 63.573 |
|
- type: map_at_20 |
|
value: 63.077000000000005 |
|
- type: map_at_3 |
|
value: 58.358 |
|
- type: map_at_5 |
|
value: 60.211000000000006 |
|
- type: mrr_at_1 |
|
value: 58.80281690140845 |
|
- type: mrr_at_10 |
|
value: 67.8209534987704 |
|
- type: mrr_at_100 |
|
value: 68.40289291033511 |
|
- type: mrr_at_1000 |
|
value: 68.41681870169752 |
|
- type: mrr_at_20 |
|
value: 68.25865150223059 |
|
- type: mrr_at_3 |
|
value: 65.66901408450703 |
|
- type: mrr_at_5 |
|
value: 66.84859154929578 |
|
- type: nauc_map_at_1000_diff1 |
|
value: 66.23490767864845 |
|
- type: nauc_map_at_1000_max |
|
value: 56.62757337597366 |
|
- type: nauc_map_at_1000_std |
|
value: 4.528549025111963 |
|
- type: nauc_map_at_100_diff1 |
|
value: 66.22940398014607 |
|
- type: nauc_map_at_100_max |
|
value: 56.634349360676865 |
|
- type: nauc_map_at_100_std |
|
value: 4.541880997700564 |
|
- type: nauc_map_at_10_diff1 |
|
value: 65.94549814528568 |
|
- type: nauc_map_at_10_max |
|
value: 55.52461478236527 |
|
- type: nauc_map_at_10_std |
|
value: 2.457230374531773 |
|
- type: nauc_map_at_1_diff1 |
|
value: 68.72392823338977 |
|
- type: nauc_map_at_1_max |
|
value: 53.391642623028936 |
|
- type: nauc_map_at_1_std |
|
value: 1.4283196945625762 |
|
- type: nauc_map_at_20_diff1 |
|
value: 66.20579936575423 |
|
- type: nauc_map_at_20_max |
|
value: 56.531407372997954 |
|
- type: nauc_map_at_20_std |
|
value: 4.117228263552818 |
|
- type: nauc_map_at_3_diff1 |
|
value: 65.34148569044554 |
|
- type: nauc_map_at_3_max |
|
value: 52.23666020679251 |
|
- type: nauc_map_at_3_std |
|
value: 0.6213942899425956 |
|
- type: nauc_map_at_5_diff1 |
|
value: 65.56848099295489 |
|
- type: nauc_map_at_5_max |
|
value: 54.14599505704255 |
|
- type: nauc_map_at_5_std |
|
value: 1.7510168090848297 |
|
- type: nauc_mrr_at_1000_diff1 |
|
value: 65.2248453530452 |
|
- type: nauc_mrr_at_1000_max |
|
value: 56.26467474118066 |
|
- type: nauc_mrr_at_1000_std |
|
value: 7.765895326925132 |
|
- type: nauc_mrr_at_100_diff1 |
|
value: 65.21070712758778 |
|
- type: nauc_mrr_at_100_max |
|
value: 56.271177686981325 |
|
- type: nauc_mrr_at_100_std |
|
value: 7.785345389803579 |
|
- type: nauc_mrr_at_10_diff1 |
|
value: 64.99745390900522 |
|
- type: nauc_mrr_at_10_max |
|
value: 56.055839490543576 |
|
- type: nauc_mrr_at_10_std |
|
value: 7.496513504421416 |
|
- type: nauc_mrr_at_1_diff1 |
|
value: 68.92907857149805 |
|
- type: nauc_mrr_at_1_max |
|
value: 56.26107663193521 |
|
- type: nauc_mrr_at_1_std |
|
value: 6.297369156935706 |
|
- type: nauc_mrr_at_20_diff1 |
|
value: 65.1678016982267 |
|
- type: nauc_mrr_at_20_max |
|
value: 56.335972588669726 |
|
- type: nauc_mrr_at_20_std |
|
value: 7.8243664682538485 |
|
- type: nauc_mrr_at_3_diff1 |
|
value: 64.83973635858855 |
|
- type: nauc_mrr_at_3_max |
|
value: 55.5714129852856 |
|
- type: nauc_mrr_at_3_std |
|
value: 6.613504325180008 |
|
- type: nauc_mrr_at_5_diff1 |
|
value: 64.74589091931088 |
|
- type: nauc_mrr_at_5_max |
|
value: 55.88271351361698 |
|
- type: nauc_mrr_at_5_std |
|
value: 7.1386788643876455 |
|
- type: nauc_ndcg_at_1000_diff1 |
|
value: 65.202639941499 |
|
- type: nauc_ndcg_at_1000_max |
|
value: 57.564093890541756 |
|
- type: nauc_ndcg_at_1000_std |
|
value: 7.258587810697105 |
|
- type: nauc_ndcg_at_100_diff1 |
|
value: 64.7813249495164 |
|
- type: nauc_ndcg_at_100_max |
|
value: 57.79712787033168 |
|
- type: nauc_ndcg_at_100_std |
|
value: 7.773895293143383 |
|
- type: nauc_ndcg_at_10_diff1 |
|
value: 64.32391711969152 |
|
- type: nauc_ndcg_at_10_max |
|
value: 55.63793686530347 |
|
- type: nauc_ndcg_at_10_std |
|
value: 3.4059961155442764 |
|
- type: nauc_ndcg_at_1_diff1 |
|
value: 68.92907857149805 |
|
- type: nauc_ndcg_at_1_max |
|
value: 56.26107663193521 |
|
- type: nauc_ndcg_at_1_std |
|
value: 6.297369156935706 |
|
- type: nauc_ndcg_at_20_diff1 |
|
value: 64.81574204816175 |
|
- type: nauc_ndcg_at_20_max |
|
value: 58.2840773030739 |
|
- type: nauc_ndcg_at_20_std |
|
value: 7.263179111057372 |
|
- type: nauc_ndcg_at_3_diff1 |
|
value: 63.912724511990696 |
|
- type: nauc_ndcg_at_3_max |
|
value: 53.75803159488968 |
|
- type: nauc_ndcg_at_3_std |
|
value: 3.1036435446695645 |
|
- type: nauc_ndcg_at_5_diff1 |
|
value: 63.79330356150732 |
|
- type: nauc_ndcg_at_5_max |
|
value: 55.082744998876464 |
|
- type: nauc_ndcg_at_5_std |
|
value: 3.6246823425115213 |
|
- type: nauc_precision_at_1000_diff1 |
|
value: -22.10580213313522 |
|
- type: nauc_precision_at_1000_max |
|
value: -0.9118701386503028 |
|
- type: nauc_precision_at_1000_std |
|
value: 27.333746721273283 |
|
- type: nauc_precision_at_100_diff1 |
|
value: -18.882895589911644 |
|
- type: nauc_precision_at_100_max |
|
value: 4.113768286012285 |
|
- type: nauc_precision_at_100_std |
|
value: 30.048337440426497 |
|
- type: nauc_precision_at_10_diff1 |
|
value: 3.248704005833223 |
|
- type: nauc_precision_at_10_max |
|
value: 23.154996086400423 |
|
- type: nauc_precision_at_10_std |
|
value: 20.251997611045912 |
|
- type: nauc_precision_at_1_diff1 |
|
value: 68.92907857149805 |
|
- type: nauc_precision_at_1_max |
|
value: 56.26107663193521 |
|
- type: nauc_precision_at_1_std |
|
value: 6.297369156935706 |
|
- type: nauc_precision_at_20_diff1 |
|
value: -7.13873585811708 |
|
- type: nauc_precision_at_20_max |
|
value: 17.375565602388527 |
|
- type: nauc_precision_at_20_std |
|
value: 31.317699557764485 |
|
- type: nauc_precision_at_3_diff1 |
|
value: 33.63751338671969 |
|
- type: nauc_precision_at_3_max |
|
value: 38.476765888636756 |
|
- type: nauc_precision_at_3_std |
|
value: 12.2723755464466 |
|
- type: nauc_precision_at_5_diff1 |
|
value: 20.649180734546427 |
|
- type: nauc_precision_at_5_max |
|
value: 34.74830242946571 |
|
- type: nauc_precision_at_5_std |
|
value: 16.90872320810353 |
|
- type: nauc_recall_at_1000_diff1 |
|
value: .nan |
|
- type: nauc_recall_at_1000_max |
|
value: .nan |
|
- type: nauc_recall_at_1000_std |
|
value: .nan |
|
- type: nauc_recall_at_100_diff1 |
|
value: 38.57539921023781 |
|
- type: nauc_recall_at_100_max |
|
value: 63.42225430043433 |
|
- type: nauc_recall_at_100_std |
|
value: 31.59021194191864 |
|
- type: nauc_recall_at_10_diff1 |
|
value: 53.672218603348455 |
|
- type: nauc_recall_at_10_max |
|
value: 48.673710550594265 |
|
- type: nauc_recall_at_10_std |
|
value: -2.566654931781578 |
|
- type: nauc_recall_at_1_diff1 |
|
value: 68.72392823338977 |
|
- type: nauc_recall_at_1_max |
|
value: 53.391642623028936 |
|
- type: nauc_recall_at_1_std |
|
value: 1.4283196945625762 |
|
- type: nauc_recall_at_20_diff1 |
|
value: 51.84453378101791 |
|
- type: nauc_recall_at_20_max |
|
value: 62.44943925787231 |
|
- type: nauc_recall_at_20_std |
|
value: 15.782443159610663 |
|
- type: nauc_recall_at_3_diff1 |
|
value: 56.20810589388257 |
|
- type: nauc_recall_at_3_max |
|
value: 45.892032964653836 |
|
- type: nauc_recall_at_3_std |
|
value: -2.820516274258717 |
|
- type: nauc_recall_at_5_diff1 |
|
value: 54.215941877653904 |
|
- type: nauc_recall_at_5_max |
|
value: 48.20665726343471 |
|
- type: nauc_recall_at_5_std |
|
value: -0.791029661993311 |
|
- type: ndcg_at_1 |
|
value: 58.803000000000004 |
|
- type: ndcg_at_10 |
|
value: 68.37299999999999 |
|
- type: ndcg_at_100 |
|
value: 72.446 |
|
- type: ndcg_at_1000 |
|
value: 72.992 |
|
- type: ndcg_at_20 |
|
value: 70.414 |
|
- type: ndcg_at_3 |
|
value: 62.839 |
|
- type: ndcg_at_5 |
|
value: 65.023 |
|
- type: precision_at_1 |
|
value: 58.803000000000004 |
|
- type: precision_at_10 |
|
value: 11.197 |
|
- type: precision_at_100 |
|
value: 1.4749999999999999 |
|
- type: precision_at_1000 |
|
value: 0.155 |
|
- type: precision_at_20 |
|
value: 6.356000000000001 |
|
- type: precision_at_3 |
|
value: 27.346999999999998 |
|
- type: precision_at_5 |
|
value: 18.521 |
|
- type: recall_at_1 |
|
value: 50.583999999999996 |
|
- type: recall_at_10 |
|
value: 80.833 |
|
- type: recall_at_100 |
|
value: 96.467 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_20 |
|
value: 87.543 |
|
- type: recall_at_3 |
|
value: 65.515 |
|
- type: recall_at_5 |
|
value: 71.47200000000001 |
|
task: |
|
type: Retrieval |
|
tags: |
|
- mteb |
|
--- |
|
|
|
# VectorStack AI | vstackai-law-1 |
|
|
|
This repository describes the steps of using the VectorStack AI's `vstackai-law-1` model. |
|
You can use the embedding model via our API or via private deployments in your private cloud. |
|
|
|
## Usage via private deployment |
|
If you are interested in running this model in your own private cloud/ own hardware, [contact us](https://www.vectorstack.ai/contact) to learn more. |
|
|
|
## Usage via API |
|
The following code snippet shows the steps required to generate embeddings via the API. Install the SDK via: |
|
``` |
|
pip install -U vectorstackai |
|
``` |
|
Get your free API key on www.vectorstack.ai |
|
|
|
```python |
|
import vectorstackai |
|
import numpy as np |
|
api_key = "{API_KEY}" # Get your API key from www.vectorstack.ai |
|
client = vectorstackai.Client(api_key=api_key) |
|
|
|
# Documents related to law domain (e.g., court cases, consumer contracts, etc.) |
|
documents = [ |
|
"The defendant was charged with violation of contract terms in the lease agreement signed on January 1, 2022.", |
|
"This contract stipulates that the consumer has 30 days to return the product in case of any manufacturing defects.", |
|
"In the case of Smith v. Johnson, the court ruled that the plaintiff had the right to claim damages under section 12 of the Consumer Protection Act." |
|
] |
|
|
|
# Get embeddings for the legal documents |
|
doc_embeddings = client.embed(texts=documents, model='vstackai-law-1', is_query=False) # EmbeddingsObject(num_embeddings=3, embedding_dims=1536) |
|
doc_embeddings = np.array(doc_embeddings.embeddings) |
|
|
|
# Encode the query |
|
query = "How many days does the consumer have to return the product?" |
|
query_embedding = client.embed( |
|
texts=[query], |
|
model='vstackai-law-1', |
|
is_query=True, |
|
instruction='Represent the query for searching legal documents' |
|
) # EmbeddingsObject(num_embeddings=1, embedding_dims=1536) |
|
query_embedding = np.array(query_embedding.embeddings) # (1, 1536) |
|
|
|
# To check if the embeddings work, you can compute similarity between the query and documents |
|
similarities = np.dot(doc_embeddings, query_embedding.T) |
|
print(similarities) |
|
# array([[0.355], |
|
# [0.772], |
|
# [0.433]]) |
|
``` |
