Add new SentenceTransformer model
Browse files- README.md +106 -98
- model.safetensors +1 -1
README.md
CHANGED
|
@@ -348,49 +348,49 @@ model-index:
|
|
| 348 |
type: mteb/AILA_casedocs
|
| 349 |
metrics:
|
| 350 |
- type: cosine_accuracy@1
|
| 351 |
-
value: 0.
|
| 352 |
name: Cosine Accuracy@1
|
| 353 |
- type: cosine_accuracy@3
|
| 354 |
-
value: 0.
|
| 355 |
name: Cosine Accuracy@3
|
| 356 |
- type: cosine_accuracy@5
|
| 357 |
value: 0.38
|
| 358 |
name: Cosine Accuracy@5
|
| 359 |
- type: cosine_accuracy@10
|
| 360 |
-
value: 0.
|
| 361 |
name: Cosine Accuracy@10
|
| 362 |
- type: cosine_precision@1
|
| 363 |
-
value: 0.
|
| 364 |
name: Cosine Precision@1
|
| 365 |
- type: cosine_precision@3
|
| 366 |
-
value: 0.
|
| 367 |
name: Cosine Precision@3
|
| 368 |
- type: cosine_precision@5
|
| 369 |
-
value: 0.
|
| 370 |
name: Cosine Precision@5
|
| 371 |
- type: cosine_precision@10
|
| 372 |
-
value: 0.
|
| 373 |
name: Cosine Precision@10
|
| 374 |
- type: cosine_recall@1
|
| 375 |
-
value: 0.
|
| 376 |
name: Cosine Recall@1
|
| 377 |
- type: cosine_recall@3
|
| 378 |
-
value: 0.
|
| 379 |
name: Cosine Recall@3
|
| 380 |
- type: cosine_recall@5
|
| 381 |
-
value: 0.
|
| 382 |
name: Cosine Recall@5
|
| 383 |
- type: cosine_recall@10
|
| 384 |
-
value: 0.
|
| 385 |
name: Cosine Recall@10
|
| 386 |
- type: cosine_ndcg@10
|
| 387 |
-
value: 0.
|
| 388 |
name: Cosine Ndcg@10
|
| 389 |
- type: cosine_mrr@10
|
| 390 |
-
value: 0.
|
| 391 |
name: Cosine Mrr@10
|
| 392 |
- type: cosine_map@100
|
| 393 |
-
value: 0.
|
| 394 |
name: Cosine Map@100
|
| 395 |
- task:
|
| 396 |
type: information-retrieval
|
|
@@ -400,49 +400,49 @@ model-index:
|
|
| 400 |
type: mteb/AILA_statutes
|
| 401 |
metrics:
|
| 402 |
- type: cosine_accuracy@1
|
| 403 |
-
value: 0.
|
| 404 |
name: Cosine Accuracy@1
|
| 405 |
- type: cosine_accuracy@3
|
| 406 |
-
value: 0.
|
| 407 |
name: Cosine Accuracy@3
|
| 408 |
- type: cosine_accuracy@5
|
| 409 |
-
value: 0.
|
| 410 |
name: Cosine Accuracy@5
|
| 411 |
- type: cosine_accuracy@10
|
| 412 |
value: 0.68
|
| 413 |
name: Cosine Accuracy@10
|
| 414 |
- type: cosine_precision@1
|
| 415 |
-
value: 0.
|
| 416 |
name: Cosine Precision@1
|
| 417 |
- type: cosine_precision@3
|
| 418 |
-
value: 0.
|
| 419 |
name: Cosine Precision@3
|
| 420 |
- type: cosine_precision@5
|
| 421 |
-
value: 0.
|
| 422 |
name: Cosine Precision@5
|
| 423 |
- type: cosine_precision@10
|
| 424 |
-
value: 0.
|
| 425 |
name: Cosine Precision@10
|
| 426 |
- type: cosine_recall@1
|
| 427 |
-
value: 0.
|
| 428 |
name: Cosine Recall@1
|
| 429 |
- type: cosine_recall@3
|
| 430 |
-
value: 0.
|
| 431 |
name: Cosine Recall@3
|
| 432 |
- type: cosine_recall@5
|
| 433 |
-
value: 0.
|
| 434 |
name: Cosine Recall@5
|
| 435 |
- type: cosine_recall@10
|
| 436 |
-
value: 0.
|
| 437 |
name: Cosine Recall@10
|
| 438 |
- type: cosine_ndcg@10
|
| 439 |
-
value: 0.
|
| 440 |
name: Cosine Ndcg@10
|
| 441 |
- type: cosine_mrr@10
|
| 442 |
-
value: 0.
|
| 443 |
name: Cosine Mrr@10
|
| 444 |
- type: cosine_map@100
|
| 445 |
-
value: 0.
|
| 446 |
name: Cosine Map@100
|
| 447 |
- task:
|
| 448 |
type: information-retrieval
|
|
@@ -452,49 +452,49 @@ model-index:
|
|
| 452 |
type: mteb/legalbench_consumer_contracts_qa
|
| 453 |
metrics:
|
| 454 |
- type: cosine_accuracy@1
|
| 455 |
-
value: 0.
|
| 456 |
name: Cosine Accuracy@1
|
| 457 |
- type: cosine_accuracy@3
|
| 458 |
-
value: 0.
|
| 459 |
name: Cosine Accuracy@3
|
| 460 |
- type: cosine_accuracy@5
|
| 461 |
-
value: 0.
|
| 462 |
name: Cosine Accuracy@5
|
| 463 |
- type: cosine_accuracy@10
|
| 464 |
-
value: 0.
|
| 465 |
name: Cosine Accuracy@10
|
| 466 |
- type: cosine_precision@1
|
| 467 |
-
value: 0.
|
| 468 |
name: Cosine Precision@1
|
| 469 |
- type: cosine_precision@3
|
| 470 |
-
value: 0.
|
| 471 |
name: Cosine Precision@3
|
| 472 |
- type: cosine_precision@5
|
| 473 |
-
value: 0.
|
| 474 |
name: Cosine Precision@5
|
| 475 |
- type: cosine_precision@10
|
| 476 |
-
value: 0.
|
| 477 |
name: Cosine Precision@10
|
| 478 |
- type: cosine_recall@1
|
| 479 |
-
value: 0.
|
| 480 |
name: Cosine Recall@1
|
| 481 |
- type: cosine_recall@3
|
| 482 |
-
value: 0.
|
| 483 |
name: Cosine Recall@3
|
| 484 |
- type: cosine_recall@5
|
| 485 |
-
value: 0.
|
| 486 |
name: Cosine Recall@5
|
| 487 |
- type: cosine_recall@10
|
| 488 |
-
value: 0.
|
| 489 |
name: Cosine Recall@10
|
| 490 |
- type: cosine_ndcg@10
|
| 491 |
-
value: 0.
|
| 492 |
name: Cosine Ndcg@10
|
| 493 |
- type: cosine_mrr@10
|
| 494 |
-
value: 0.
|
| 495 |
name: Cosine Mrr@10
|
| 496 |
- type: cosine_map@100
|
| 497 |
-
value: 0.
|
| 498 |
name: Cosine Map@100
|
| 499 |
- task:
|
| 500 |
type: information-retrieval
|
|
@@ -504,49 +504,49 @@ model-index:
|
|
| 504 |
type: mteb/legalbench_corporate_lobbying
|
| 505 |
metrics:
|
| 506 |
- type: cosine_accuracy@1
|
| 507 |
-
value: 0.
|
| 508 |
name: Cosine Accuracy@1
|
| 509 |
- type: cosine_accuracy@3
|
| 510 |
-
value: 0.
|
| 511 |
name: Cosine Accuracy@3
|
| 512 |
- type: cosine_accuracy@5
|
| 513 |
-
value: 0.
|
| 514 |
name: Cosine Accuracy@5
|
| 515 |
- type: cosine_accuracy@10
|
| 516 |
-
value: 0.
|
| 517 |
name: Cosine Accuracy@10
|
| 518 |
- type: cosine_precision@1
|
| 519 |
-
value: 0.
|
| 520 |
name: Cosine Precision@1
|
| 521 |
- type: cosine_precision@3
|
| 522 |
-
value: 0.
|
| 523 |
name: Cosine Precision@3
|
| 524 |
- type: cosine_precision@5
|
| 525 |
-
value: 0.
|
| 526 |
name: Cosine Precision@5
|
| 527 |
- type: cosine_precision@10
|
| 528 |
-
value: 0.
|
| 529 |
name: Cosine Precision@10
|
| 530 |
- type: cosine_recall@1
|
| 531 |
-
value: 0.
|
| 532 |
name: Cosine Recall@1
|
| 533 |
- type: cosine_recall@3
|
| 534 |
-
value: 0.
|
| 535 |
name: Cosine Recall@3
|
| 536 |
- type: cosine_recall@5
|
| 537 |
-
value: 0.
|
| 538 |
name: Cosine Recall@5
|
| 539 |
- type: cosine_recall@10
|
| 540 |
-
value: 0.
|
| 541 |
name: Cosine Recall@10
|
| 542 |
- type: cosine_ndcg@10
|
| 543 |
-
value: 0.
|
| 544 |
name: Cosine Ndcg@10
|
| 545 |
- type: cosine_mrr@10
|
| 546 |
-
value: 0.
|
| 547 |
name: Cosine Mrr@10
|
| 548 |
- type: cosine_map@100
|
| 549 |
-
value: 0.
|
| 550 |
name: Cosine Map@100
|
| 551 |
- task:
|
| 552 |
type: information-retrieval
|
|
@@ -556,49 +556,49 @@ model-index:
|
|
| 556 |
type: mteb/legal_summarization
|
| 557 |
metrics:
|
| 558 |
- type: cosine_accuracy@1
|
| 559 |
-
value: 0.
|
| 560 |
name: Cosine Accuracy@1
|
| 561 |
- type: cosine_accuracy@3
|
| 562 |
-
value: 0.
|
| 563 |
name: Cosine Accuracy@3
|
| 564 |
- type: cosine_accuracy@5
|
| 565 |
-
value: 0.
|
| 566 |
name: Cosine Accuracy@5
|
| 567 |
- type: cosine_accuracy@10
|
| 568 |
-
value: 0.
|
| 569 |
name: Cosine Accuracy@10
|
| 570 |
- type: cosine_precision@1
|
| 571 |
-
value: 0.
|
| 572 |
name: Cosine Precision@1
|
| 573 |
- type: cosine_precision@3
|
| 574 |
-
value: 0.
|
| 575 |
name: Cosine Precision@3
|
| 576 |
- type: cosine_precision@5
|
| 577 |
-
value: 0.
|
| 578 |
name: Cosine Precision@5
|
| 579 |
- type: cosine_precision@10
|
| 580 |
-
value: 0.
|
| 581 |
name: Cosine Precision@10
|
| 582 |
- type: cosine_recall@1
|
| 583 |
-
value: 0.
|
| 584 |
name: Cosine Recall@1
|
| 585 |
- type: cosine_recall@3
|
| 586 |
-
value: 0.
|
| 587 |
name: Cosine Recall@3
|
| 588 |
- type: cosine_recall@5
|
| 589 |
-
value: 0.
|
| 590 |
name: Cosine Recall@5
|
| 591 |
- type: cosine_recall@10
|
| 592 |
-
value: 0.
|
| 593 |
name: Cosine Recall@10
|
| 594 |
- type: cosine_ndcg@10
|
| 595 |
-
value: 0.
|
| 596 |
name: Cosine Ndcg@10
|
| 597 |
- type: cosine_mrr@10
|
| 598 |
-
value: 0.
|
| 599 |
name: Cosine Mrr@10
|
| 600 |
- type: cosine_map@100
|
| 601 |
-
value: 0.
|
| 602 |
name: Cosine Map@100
|
| 603 |
---
|
| 604 |
|
|
@@ -709,21 +709,21 @@ You can finetune this model on your own dataset.
|
|
| 709 |
|
| 710 |
| Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
|
| 711 |
|:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
|
| 712 |
-
| cosine_accuracy@1 | 0.
|
| 713 |
-
| cosine_accuracy@3 | 0.
|
| 714 |
-
| cosine_accuracy@5 | 0.38 | 0.
|
| 715 |
-
| cosine_accuracy@10 | 0.
|
| 716 |
-
| cosine_precision@1 | 0.
|
| 717 |
-
| cosine_precision@3 | 0.
|
| 718 |
-
| cosine_precision@5 | 0.
|
| 719 |
-
| cosine_precision@10 | 0.
|
| 720 |
-
| cosine_recall@1 | 0.
|
| 721 |
-
| cosine_recall@3 | 0.
|
| 722 |
-
| cosine_recall@5 | 0.
|
| 723 |
-
| cosine_recall@10 | 0.
|
| 724 |
-
| **cosine_ndcg@10** | **0.
|
| 725 |
-
| cosine_mrr@10 | 0.
|
| 726 |
-
| cosine_map@100 | 0.
|
| 727 |
|
| 728 |
<!--
|
| 729 |
## Bias, Risks and Limitations
|
|
@@ -939,7 +939,7 @@ You can finetune this model on your own dataset.
|
|
| 939 |
- `eval_strategy`: steps
|
| 940 |
- `per_device_train_batch_size`: 64
|
| 941 |
- `learning_rate`: 1e-06
|
| 942 |
-
- `num_train_epochs`:
|
| 943 |
- `warmup_ratio`: 0.1
|
| 944 |
- `fp16`: True
|
| 945 |
- `batch_sampler`: no_duplicates
|
|
@@ -964,7 +964,7 @@ You can finetune this model on your own dataset.
|
|
| 964 |
- `adam_beta2`: 0.999
|
| 965 |
- `adam_epsilon`: 1e-08
|
| 966 |
- `max_grad_norm`: 1.0
|
| 967 |
-
- `num_train_epochs`:
|
| 968 |
- `max_steps`: -1
|
| 969 |
- `lr_scheduler_type`: linear
|
| 970 |
- `lr_scheduler_kwargs`: {}
|
|
@@ -1067,14 +1067,22 @@ You can finetune this model on your own dataset.
|
|
| 1067 |
| Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
|
| 1068 |
|:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
|
| 1069 |
| 0 | 0 | - | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
|
| 1070 |
-
| 0.1196 | 100 | - | 0.
|
| 1071 |
-
| 0.2392 | 200 | - | 0.
|
| 1072 |
-
| 0.3589 | 300 | - | 0.
|
| 1073 |
-
| 0.4785 | 400 | - | 0.
|
| 1074 |
-
| 0.5981 | 500 | 4.
|
| 1075 |
-
| 0.7177 | 600 | - | 0.
|
| 1076 |
-
| 0.8373 | 700 | - | 0.
|
| 1077 |
-
| 0.9569 | 800 | - | 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1078 |
|
| 1079 |
|
| 1080 |
### Framework Versions
|
|
|
|
| 348 |
type: mteb/AILA_casedocs
|
| 349 |
metrics:
|
| 350 |
- type: cosine_accuracy@1
|
| 351 |
+
value: 0.32
|
| 352 |
name: Cosine Accuracy@1
|
| 353 |
- type: cosine_accuracy@3
|
| 354 |
+
value: 0.36
|
| 355 |
name: Cosine Accuracy@3
|
| 356 |
- type: cosine_accuracy@5
|
| 357 |
value: 0.38
|
| 358 |
name: Cosine Accuracy@5
|
| 359 |
- type: cosine_accuracy@10
|
| 360 |
+
value: 0.54
|
| 361 |
name: Cosine Accuracy@10
|
| 362 |
- type: cosine_precision@1
|
| 363 |
+
value: 0.32
|
| 364 |
name: Cosine Precision@1
|
| 365 |
- type: cosine_precision@3
|
| 366 |
+
value: 0.2
|
| 367 |
name: Cosine Precision@3
|
| 368 |
- type: cosine_precision@5
|
| 369 |
+
value: 0.14
|
| 370 |
name: Cosine Precision@5
|
| 371 |
- type: cosine_precision@10
|
| 372 |
+
value: 0.106
|
| 373 |
name: Cosine Precision@10
|
| 374 |
- type: cosine_recall@1
|
| 375 |
+
value: 0.10011421911421912
|
| 376 |
name: Cosine Recall@1
|
| 377 |
- type: cosine_recall@3
|
| 378 |
+
value: 0.173986013986014
|
| 379 |
name: Cosine Recall@3
|
| 380 |
- type: cosine_recall@5
|
| 381 |
+
value: 0.20522843822843825
|
| 382 |
name: Cosine Recall@5
|
| 383 |
- type: cosine_recall@10
|
| 384 |
+
value: 0.30841841491841493
|
| 385 |
name: Cosine Recall@10
|
| 386 |
- type: cosine_ndcg@10
|
| 387 |
+
value: 0.26821891394168423
|
| 388 |
name: Cosine Ndcg@10
|
| 389 |
- type: cosine_mrr@10
|
| 390 |
+
value: 0.36276984126984124
|
| 391 |
name: Cosine Mrr@10
|
| 392 |
- type: cosine_map@100
|
| 393 |
+
value: 0.23063570433429445
|
| 394 |
name: Cosine Map@100
|
| 395 |
- task:
|
| 396 |
type: information-retrieval
|
|
|
|
| 400 |
type: mteb/AILA_statutes
|
| 401 |
metrics:
|
| 402 |
- type: cosine_accuracy@1
|
| 403 |
+
value: 0.26
|
| 404 |
name: Cosine Accuracy@1
|
| 405 |
- type: cosine_accuracy@3
|
| 406 |
+
value: 0.42
|
| 407 |
name: Cosine Accuracy@3
|
| 408 |
- type: cosine_accuracy@5
|
| 409 |
+
value: 0.52
|
| 410 |
name: Cosine Accuracy@5
|
| 411 |
- type: cosine_accuracy@10
|
| 412 |
value: 0.68
|
| 413 |
name: Cosine Accuracy@10
|
| 414 |
- type: cosine_precision@1
|
| 415 |
+
value: 0.26
|
| 416 |
name: Cosine Precision@1
|
| 417 |
- type: cosine_precision@3
|
| 418 |
+
value: 0.15999999999999998
|
| 419 |
name: Cosine Precision@3
|
| 420 |
- type: cosine_precision@5
|
| 421 |
+
value: 0.132
|
| 422 |
name: Cosine Precision@5
|
| 423 |
- type: cosine_precision@10
|
| 424 |
+
value: 0.10800000000000001
|
| 425 |
name: Cosine Precision@10
|
| 426 |
- type: cosine_recall@1
|
| 427 |
+
value: 0.06499999999999999
|
| 428 |
name: Cosine Recall@1
|
| 429 |
- type: cosine_recall@3
|
| 430 |
+
value: 0.125
|
| 431 |
name: Cosine Recall@3
|
| 432 |
- type: cosine_recall@5
|
| 433 |
+
value: 0.16399999999999998
|
| 434 |
name: Cosine Recall@5
|
| 435 |
- type: cosine_recall@10
|
| 436 |
+
value: 0.256
|
| 437 |
name: Cosine Recall@10
|
| 438 |
- type: cosine_ndcg@10
|
| 439 |
+
value: 0.22688341584203112
|
| 440 |
name: Cosine Ndcg@10
|
| 441 |
- type: cosine_mrr@10
|
| 442 |
+
value: 0.3726904761904762
|
| 443 |
name: Cosine Mrr@10
|
| 444 |
- type: cosine_map@100
|
| 445 |
+
value: 0.19663910082742547
|
| 446 |
name: Cosine Map@100
|
| 447 |
- task:
|
| 448 |
type: information-retrieval
|
|
|
|
| 452 |
type: mteb/legalbench_consumer_contracts_qa
|
| 453 |
metrics:
|
| 454 |
- type: cosine_accuracy@1
|
| 455 |
+
value: 0.45202020202020204
|
| 456 |
name: Cosine Accuracy@1
|
| 457 |
- type: cosine_accuracy@3
|
| 458 |
+
value: 0.6843434343434344
|
| 459 |
name: Cosine Accuracy@3
|
| 460 |
- type: cosine_accuracy@5
|
| 461 |
+
value: 0.7878787878787878
|
| 462 |
name: Cosine Accuracy@5
|
| 463 |
- type: cosine_accuracy@10
|
| 464 |
+
value: 0.8661616161616161
|
| 465 |
name: Cosine Accuracy@10
|
| 466 |
- type: cosine_precision@1
|
| 467 |
+
value: 0.45202020202020204
|
| 468 |
name: Cosine Precision@1
|
| 469 |
- type: cosine_precision@3
|
| 470 |
+
value: 0.2281144781144781
|
| 471 |
name: Cosine Precision@3
|
| 472 |
- type: cosine_precision@5
|
| 473 |
+
value: 0.15757575757575756
|
| 474 |
name: Cosine Precision@5
|
| 475 |
- type: cosine_precision@10
|
| 476 |
+
value: 0.0866161616161616
|
| 477 |
name: Cosine Precision@10
|
| 478 |
- type: cosine_recall@1
|
| 479 |
+
value: 0.45202020202020204
|
| 480 |
name: Cosine Recall@1
|
| 481 |
- type: cosine_recall@3
|
| 482 |
+
value: 0.6843434343434344
|
| 483 |
name: Cosine Recall@3
|
| 484 |
- type: cosine_recall@5
|
| 485 |
+
value: 0.7878787878787878
|
| 486 |
name: Cosine Recall@5
|
| 487 |
- type: cosine_recall@10
|
| 488 |
+
value: 0.8661616161616161
|
| 489 |
name: Cosine Recall@10
|
| 490 |
- type: cosine_ndcg@10
|
| 491 |
+
value: 0.6554316826573866
|
| 492 |
name: Cosine Ndcg@10
|
| 493 |
- type: cosine_mrr@10
|
| 494 |
+
value: 0.5881553631553632
|
| 495 |
name: Cosine Mrr@10
|
| 496 |
- type: cosine_map@100
|
| 497 |
+
value: 0.5945229817141628
|
| 498 |
name: Cosine Map@100
|
| 499 |
- task:
|
| 500 |
type: information-retrieval
|
|
|
|
| 504 |
type: mteb/legalbench_corporate_lobbying
|
| 505 |
metrics:
|
| 506 |
- type: cosine_accuracy@1
|
| 507 |
+
value: 0.7382352941176471
|
| 508 |
name: Cosine Accuracy@1
|
| 509 |
- type: cosine_accuracy@3
|
| 510 |
+
value: 0.8941176470588236
|
| 511 |
name: Cosine Accuracy@3
|
| 512 |
- type: cosine_accuracy@5
|
| 513 |
+
value: 0.9294117647058824
|
| 514 |
name: Cosine Accuracy@5
|
| 515 |
- type: cosine_accuracy@10
|
| 516 |
+
value: 0.9647058823529412
|
| 517 |
name: Cosine Accuracy@10
|
| 518 |
- type: cosine_precision@1
|
| 519 |
+
value: 0.7382352941176471
|
| 520 |
name: Cosine Precision@1
|
| 521 |
- type: cosine_precision@3
|
| 522 |
+
value: 0.2980392156862745
|
| 523 |
name: Cosine Precision@3
|
| 524 |
- type: cosine_precision@5
|
| 525 |
+
value: 0.18588235294117644
|
| 526 |
name: Cosine Precision@5
|
| 527 |
- type: cosine_precision@10
|
| 528 |
+
value: 0.09647058823529411
|
| 529 |
name: Cosine Precision@10
|
| 530 |
- type: cosine_recall@1
|
| 531 |
+
value: 0.7382352941176471
|
| 532 |
name: Cosine Recall@1
|
| 533 |
- type: cosine_recall@3
|
| 534 |
+
value: 0.8941176470588236
|
| 535 |
name: Cosine Recall@3
|
| 536 |
- type: cosine_recall@5
|
| 537 |
+
value: 0.9294117647058824
|
| 538 |
name: Cosine Recall@5
|
| 539 |
- type: cosine_recall@10
|
| 540 |
+
value: 0.9647058823529412
|
| 541 |
name: Cosine Recall@10
|
| 542 |
- type: cosine_ndcg@10
|
| 543 |
+
value: 0.8568617986155895
|
| 544 |
name: Cosine Ndcg@10
|
| 545 |
- type: cosine_mrr@10
|
| 546 |
+
value: 0.8217670401493932
|
| 547 |
name: Cosine Mrr@10
|
| 548 |
- type: cosine_map@100
|
| 549 |
+
value: 0.822962290292461
|
| 550 |
name: Cosine Map@100
|
| 551 |
- task:
|
| 552 |
type: information-retrieval
|
|
|
|
| 556 |
type: mteb/legal_summarization
|
| 557 |
metrics:
|
| 558 |
- type: cosine_accuracy@1
|
| 559 |
+
value: 0.49295774647887325
|
| 560 |
name: Cosine Accuracy@1
|
| 561 |
- type: cosine_accuracy@3
|
| 562 |
+
value: 0.647887323943662
|
| 563 |
name: Cosine Accuracy@3
|
| 564 |
- type: cosine_accuracy@5
|
| 565 |
+
value: 0.7112676056338029
|
| 566 |
name: Cosine Accuracy@5
|
| 567 |
- type: cosine_accuracy@10
|
| 568 |
+
value: 0.795774647887324
|
| 569 |
name: Cosine Accuracy@10
|
| 570 |
- type: cosine_precision@1
|
| 571 |
+
value: 0.49295774647887325
|
| 572 |
name: Cosine Precision@1
|
| 573 |
- type: cosine_precision@3
|
| 574 |
+
value: 0.23708920187793425
|
| 575 |
name: Cosine Precision@3
|
| 576 |
- type: cosine_precision@5
|
| 577 |
+
value: 0.1640845070422535
|
| 578 |
name: Cosine Precision@5
|
| 579 |
- type: cosine_precision@10
|
| 580 |
+
value: 0.09859154929577464
|
| 581 |
name: Cosine Precision@10
|
| 582 |
- type: cosine_recall@1
|
| 583 |
+
value: 0.437591076763612
|
| 584 |
name: Cosine Recall@1
|
| 585 |
- type: cosine_recall@3
|
| 586 |
+
value: 0.570583729650631
|
| 587 |
name: Cosine Recall@3
|
| 588 |
- type: cosine_recall@5
|
| 589 |
+
value: 0.6355626181330407
|
| 590 |
name: Cosine Recall@5
|
| 591 |
- type: cosine_recall@10
|
| 592 |
+
value: 0.7263897780623133
|
| 593 |
name: Cosine Recall@10
|
| 594 |
- type: cosine_ndcg@10
|
| 595 |
+
value: 0.5978698544115026
|
| 596 |
name: Cosine Ndcg@10
|
| 597 |
- type: cosine_mrr@10
|
| 598 |
+
value: 0.5853118712273642
|
| 599 |
name: Cosine Mrr@10
|
| 600 |
- type: cosine_map@100
|
| 601 |
+
value: 0.555963840687379
|
| 602 |
name: Cosine Map@100
|
| 603 |
---
|
| 604 |
|
|
|
|
| 709 |
|
| 710 |
| Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
|
| 711 |
|:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
|
| 712 |
+
| cosine_accuracy@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
|
| 713 |
+
| cosine_accuracy@3 | 0.36 | 0.42 | 0.6843 | 0.8941 | 0.6479 |
|
| 714 |
+
| cosine_accuracy@5 | 0.38 | 0.52 | 0.7879 | 0.9294 | 0.7113 |
|
| 715 |
+
| cosine_accuracy@10 | 0.54 | 0.68 | 0.8662 | 0.9647 | 0.7958 |
|
| 716 |
+
| cosine_precision@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
|
| 717 |
+
| cosine_precision@3 | 0.2 | 0.16 | 0.2281 | 0.298 | 0.2371 |
|
| 718 |
+
| cosine_precision@5 | 0.14 | 0.132 | 0.1576 | 0.1859 | 0.1641 |
|
| 719 |
+
| cosine_precision@10 | 0.106 | 0.108 | 0.0866 | 0.0965 | 0.0986 |
|
| 720 |
+
| cosine_recall@1 | 0.1001 | 0.065 | 0.452 | 0.7382 | 0.4376 |
|
| 721 |
+
| cosine_recall@3 | 0.174 | 0.125 | 0.6843 | 0.8941 | 0.5706 |
|
| 722 |
+
| cosine_recall@5 | 0.2052 | 0.164 | 0.7879 | 0.9294 | 0.6356 |
|
| 723 |
+
| cosine_recall@10 | 0.3084 | 0.256 | 0.8662 | 0.9647 | 0.7264 |
|
| 724 |
+
| **cosine_ndcg@10** | **0.2682** | **0.2269** | **0.6554** | **0.8569** | **0.5979** |
|
| 725 |
+
| cosine_mrr@10 | 0.3628 | 0.3727 | 0.5882 | 0.8218 | 0.5853 |
|
| 726 |
+
| cosine_map@100 | 0.2306 | 0.1966 | 0.5945 | 0.823 | 0.556 |
|
| 727 |
|
| 728 |
<!--
|
| 729 |
## Bias, Risks and Limitations
|
|
|
|
| 939 |
- `eval_strategy`: steps
|
| 940 |
- `per_device_train_batch_size`: 64
|
| 941 |
- `learning_rate`: 1e-06
|
| 942 |
+
- `num_train_epochs`: 2
|
| 943 |
- `warmup_ratio`: 0.1
|
| 944 |
- `fp16`: True
|
| 945 |
- `batch_sampler`: no_duplicates
|
|
|
|
| 964 |
- `adam_beta2`: 0.999
|
| 965 |
- `adam_epsilon`: 1e-08
|
| 966 |
- `max_grad_norm`: 1.0
|
| 967 |
+
- `num_train_epochs`: 2
|
| 968 |
- `max_steps`: -1
|
| 969 |
- `lr_scheduler_type`: linear
|
| 970 |
- `lr_scheduler_kwargs`: {}
|
|
|
|
| 1067 |
| Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
|
| 1068 |
|:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
|
| 1069 |
| 0 | 0 | - | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
|
| 1070 |
+
| 0.1196 | 100 | - | 0.1936 | 0.2103 | 0.6553 | 0.8662 | 0.5969 |
|
| 1071 |
+
| 0.2392 | 200 | - | 0.2130 | 0.2135 | 0.6610 | 0.8652 | 0.5988 |
|
| 1072 |
+
| 0.3589 | 300 | - | 0.2148 | 0.2145 | 0.6626 | 0.8651 | 0.6051 |
|
| 1073 |
+
| 0.4785 | 400 | - | 0.2227 | 0.2089 | 0.6584 | 0.8618 | 0.6050 |
|
| 1074 |
+
| 0.5981 | 500 | 4.8514 | 0.2253 | 0.2046 | 0.6583 | 0.8558 | 0.6033 |
|
| 1075 |
+
| 0.7177 | 600 | - | 0.2301 | 0.2104 | 0.6585 | 0.8568 | 0.6003 |
|
| 1076 |
+
| 0.8373 | 700 | - | 0.2421 | 0.2037 | 0.6588 | 0.8592 | 0.6007 |
|
| 1077 |
+
| 0.9569 | 800 | - | 0.2560 | 0.2089 | 0.6589 | 0.8597 | 0.6013 |
|
| 1078 |
+
| 1.0766 | 900 | - | 0.2518 | 0.2122 | 0.6592 | 0.8616 | 0.6014 |
|
| 1079 |
+
| 1.1962 | 1000 | 3.2784 | 0.2638 | 0.2193 | 0.6598 | 0.8622 | 0.6010 |
|
| 1080 |
+
| 1.3158 | 1100 | - | 0.2660 | 0.2193 | 0.6601 | 0.8630 | 0.6008 |
|
| 1081 |
+
| 1.4354 | 1200 | - | 0.2617 | 0.2237 | 0.6594 | 0.8601 | 0.5990 |
|
| 1082 |
+
| 1.5550 | 1300 | - | 0.2656 | 0.2249 | 0.6571 | 0.8563 | 0.5990 |
|
| 1083 |
+
| 1.6746 | 1400 | - | 0.2648 | 0.2254 | 0.6568 | 0.8558 | 0.5992 |
|
| 1084 |
+
| 1.7943 | 1500 | 3.3186 | 0.2683 | 0.2283 | 0.6564 | 0.8557 | 0.5988 |
|
| 1085 |
+
| 1.9139 | 1600 | - | 0.2682 | 0.2269 | 0.6554 | 0.8569 | 0.5979 |
|
| 1086 |
|
| 1087 |
|
| 1088 |
### Framework Versions
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 90864192
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6e9b8047dac0268b40a558cc5a9b1d602366a655283a7049668101ace9062ef
|
| 3 |
size 90864192
|