Training in progress, step 7000
Browse files- model.safetensors +1 -1
- wandb/debug-internal.log +0 -0
- wandb/run-20231118_132020-qg88991p/files/output.log +1065 -0
- wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +1 -1
- wandb/run-20231118_132020-qg88991p/logs/debug-internal.log +0 -0
- wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb +2 -2
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 3219908024
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2833606f70f74be87bdb4bbe4cf8557b7e259740d4b73b3931cfcfa776d5dc23
|
3 |
size 3219908024
|
wandb/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231118_132020-qg88991p/files/output.log
CHANGED
@@ -6505,3 +6505,1068 @@ Reading metadata...: 1it [00:00, 6.69it/s]
|
|
6505 |
|
6506 |
|
6507 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6505 |
|
6506 |
|
6507 |
|
6508 |
+
|
6509 |
+
|
6510 |
+
|
6511 |
+
|
6512 |
+
|
6513 |
+
|
6514 |
+
|
6515 |
+
|
6516 |
+
|
6517 |
+
10%|ββββββββββββββββββββββββ | 6080/60000 [18:03:49<136:45:44, 9.13s/it]
|
6518 |
+
|
6519 |
+
|
6520 |
+
|
6521 |
+
|
6522 |
+
|
6523 |
+
|
6524 |
+
|
6525 |
+
|
6526 |
+
|
6527 |
+
|
6528 |
+
|
6529 |
+
|
6530 |
+
|
6531 |
+
|
6532 |
+
|
6533 |
+
|
6534 |
+
|
6535 |
+
|
6536 |
+
|
6537 |
+
10%|ββββββββββββββββββββββββ | 6099/60000 [18:06:46<138:57:15, 9.28s/it]
|
6538 |
+
|
6539 |
+
|
6540 |
+
|
6541 |
+
|
6542 |
+
|
6543 |
+
|
6544 |
+
|
6545 |
+
|
6546 |
+
|
6547 |
+
|
6548 |
+
|
6549 |
+
|
6550 |
+
|
6551 |
+
|
6552 |
+
|
6553 |
+
|
6554 |
+
|
6555 |
+
|
6556 |
+
|
6557 |
+
|
6558 |
+
|
6559 |
+
10%|ββββββββββββββββββββββββ | 6120/60000 [18:10:02<137:38:52, 9.20s/it]
|
6560 |
+
|
6561 |
+
|
6562 |
+
|
6563 |
+
|
6564 |
+
|
6565 |
+
|
6566 |
+
|
6567 |
+
|
6568 |
+
|
6569 |
+
|
6570 |
+
|
6571 |
+
|
6572 |
+
|
6573 |
+
|
6574 |
+
|
6575 |
+
|
6576 |
+
|
6577 |
+
|
6578 |
+
|
6579 |
+
|
6580 |
+
10%|βββββββββββββββββββββββββ | 6140/60000 [18:13:08<141:54:36, 9.49s/it]
|
6581 |
+
|
6582 |
+
|
6583 |
+
|
6584 |
+
|
6585 |
+
|
6586 |
+
|
6587 |
+
|
6588 |
+
|
6589 |
+
|
6590 |
+
|
6591 |
+
|
6592 |
+
|
6593 |
+
|
6594 |
+
|
6595 |
+
|
6596 |
+
|
6597 |
+
|
6598 |
+
|
6599 |
+
|
6600 |
+
10%|βββββββββββββββββββββββββ | 6159/60000 [18:16:10<139:33:45, 9.33s/it]
|
6601 |
+
|
6602 |
+
|
6603 |
+
|
6604 |
+
|
6605 |
+
|
6606 |
+
|
6607 |
+
|
6608 |
+
|
6609 |
+
|
6610 |
+
|
6611 |
+
|
6612 |
+
|
6613 |
+
|
6614 |
+
|
6615 |
+
|
6616 |
+
|
6617 |
+
|
6618 |
+
|
6619 |
+
|
6620 |
+
|
6621 |
+
10%|βββββββββββββββββββββββββ | 6179/60000 [18:19:14<140:41:45, 9.41s/it]
|
6622 |
+
|
6623 |
+
|
6624 |
+
|
6625 |
+
|
6626 |
+
|
6627 |
+
|
6628 |
+
|
6629 |
+
|
6630 |
+
|
6631 |
+
|
6632 |
+
|
6633 |
+
|
6634 |
+
|
6635 |
+
|
6636 |
+
|
6637 |
+
|
6638 |
+
|
6639 |
+
|
6640 |
+
|
6641 |
+
|
6642 |
+
10%|βββββββββββββββββββββββββ | 6199/60000 [18:22:21<142:31:31, 9.54s/it]
|
6643 |
+
|
6644 |
+
|
6645 |
+
|
6646 |
+
|
6647 |
+
|
6648 |
+
|
6649 |
+
|
6650 |
+
|
6651 |
+
|
6652 |
+
|
6653 |
+
|
6654 |
+
|
6655 |
+
|
6656 |
+
|
6657 |
+
|
6658 |
+
|
6659 |
+
|
6660 |
+
|
6661 |
+
|
6662 |
+
|
6663 |
+
|
6664 |
+
10%|βββββββββββββββββββββββββ | 6220/60000 [18:25:33<136:35:07, 9.14s/it]
|
6665 |
+
|
6666 |
+
|
6667 |
+
|
6668 |
+
|
6669 |
+
|
6670 |
+
|
6671 |
+
|
6672 |
+
|
6673 |
+
|
6674 |
+
|
6675 |
+
|
6676 |
+
|
6677 |
+
|
6678 |
+
|
6679 |
+
|
6680 |
+
|
6681 |
+
|
6682 |
+
|
6683 |
+
|
6684 |
+
10%|βββββββββββββββββββββββββ | 6239/60000 [18:28:35<146:24:14, 9.80s/it]
|
6685 |
+
|
6686 |
+
|
6687 |
+
|
6688 |
+
|
6689 |
+
|
6690 |
+
|
6691 |
+
|
6692 |
+
|
6693 |
+
|
6694 |
+
|
6695 |
+
|
6696 |
+
|
6697 |
+
|
6698 |
+
|
6699 |
+
|
6700 |
+
|
6701 |
+
|
6702 |
+
|
6703 |
+
|
6704 |
+
|
6705 |
+
|
6706 |
+
10%|βββββββββββββββββββββββββ | 6260/60000 [18:31:48<135:03:47, 9.05s/it]
|
6707 |
+
|
6708 |
+
|
6709 |
+
|
6710 |
+
|
6711 |
+
|
6712 |
+
|
6713 |
+
|
6714 |
+
|
6715 |
+
|
6716 |
+
|
6717 |
+
|
6718 |
+
|
6719 |
+
|
6720 |
+
|
6721 |
+
|
6722 |
+
|
6723 |
+
|
6724 |
+
|
6725 |
+
|
6726 |
+
|
6727 |
+
10%|βββββββββββββββββββββββββ | 6280/60000 [18:34:52<134:42:02, 9.03s/it]
|
6728 |
+
|
6729 |
+
|
6730 |
+
|
6731 |
+
|
6732 |
+
|
6733 |
+
|
6734 |
+
|
6735 |
+
|
6736 |
+
|
6737 |
+
|
6738 |
+
|
6739 |
+
|
6740 |
+
|
6741 |
+
|
6742 |
+
|
6743 |
+
|
6744 |
+
|
6745 |
+
|
6746 |
+
|
6747 |
+
|
6748 |
+
10%|βββββββββββββββββββββββββ | 6300/60000 [18:38:14<178:07:41, 11.94s/it]
|
6749 |
+
|
6750 |
+
|
6751 |
+
|
6752 |
+
|
6753 |
+
|
6754 |
+
|
6755 |
+
|
6756 |
+
|
6757 |
+
|
6758 |
+
|
6759 |
+
|
6760 |
+
|
6761 |
+
|
6762 |
+
|
6763 |
+
|
6764 |
+
|
6765 |
+
|
6766 |
+
|
6767 |
+
|
6768 |
+
11%|βββββββββββββββββββββββββ | 6319/60000 [18:41:12<136:30:04, 9.15s/it]
|
6769 |
+
|
6770 |
+
|
6771 |
+
|
6772 |
+
|
6773 |
+
|
6774 |
+
|
6775 |
+
|
6776 |
+
|
6777 |
+
|
6778 |
+
|
6779 |
+
|
6780 |
+
|
6781 |
+
|
6782 |
+
|
6783 |
+
|
6784 |
+
|
6785 |
+
|
6786 |
+
|
6787 |
+
|
6788 |
+
|
6789 |
+
11%|βββββββββββββββββββββββββ | 6339/60000 [18:44:56<137:07:25, 9.20s/it]
|
6790 |
+
|
6791 |
+
|
6792 |
+
|
6793 |
+
|
6794 |
+
|
6795 |
+
|
6796 |
+
|
6797 |
+
|
6798 |
+
|
6799 |
+
|
6800 |
+
|
6801 |
+
|
6802 |
+
|
6803 |
+
|
6804 |
+
|
6805 |
+
|
6806 |
+
|
6807 |
+
|
6808 |
+
|
6809 |
+
|
6810 |
+
|
6811 |
+
11%|βββββββββββββββββββββββββ | 6360/60000 [18:48:09<137:16:45, 9.21s/it]
|
6812 |
+
|
6813 |
+
|
6814 |
+
|
6815 |
+
|
6816 |
+
|
6817 |
+
|
6818 |
+
|
6819 |
+
|
6820 |
+
|
6821 |
+
|
6822 |
+
|
6823 |
+
|
6824 |
+
|
6825 |
+
|
6826 |
+
|
6827 |
+
|
6828 |
+
|
6829 |
+
|
6830 |
+
|
6831 |
+
|
6832 |
+
11%|βββββββββββββββββββββββββ | 6380/60000 [18:51:13<136:51:39, 9.19s/it]
|
6833 |
+
|
6834 |
+
|
6835 |
+
|
6836 |
+
|
6837 |
+
|
6838 |
+
|
6839 |
+
|
6840 |
+
|
6841 |
+
|
6842 |
+
|
6843 |
+
|
6844 |
+
|
6845 |
+
|
6846 |
+
|
6847 |
+
|
6848 |
+
|
6849 |
+
|
6850 |
+
|
6851 |
+
|
6852 |
+
|
6853 |
+
11%|ββββββββββββββββββββββββββ | 6400/60000 [18:54:18<137:26:33, 9.23s/it]
|
6854 |
+
|
6855 |
+
|
6856 |
+
|
6857 |
+
|
6858 |
+
|
6859 |
+
|
6860 |
+
|
6861 |
+
|
6862 |
+
|
6863 |
+
|
6864 |
+
|
6865 |
+
|
6866 |
+
|
6867 |
+
|
6868 |
+
|
6869 |
+
|
6870 |
+
|
6871 |
+
|
6872 |
+
|
6873 |
+
|
6874 |
+
11%|ββββββββββββββββββββββββββ | 6420/60000 [18:57:26<137:32:23, 9.24s/it]
|
6875 |
+
|
6876 |
+
|
6877 |
+
|
6878 |
+
|
6879 |
+
|
6880 |
+
|
6881 |
+
|
6882 |
+
|
6883 |
+
|
6884 |
+
|
6885 |
+
|
6886 |
+
|
6887 |
+
|
6888 |
+
|
6889 |
+
|
6890 |
+
|
6891 |
+
|
6892 |
+
|
6893 |
+
|
6894 |
+
|
6895 |
+
11%|ββββββββββββββββββββββββββ | 6440/60000 [19:00:28<135:19:39, 9.10s/it]
|
6896 |
+
|
6897 |
+
|
6898 |
+
|
6899 |
+
|
6900 |
+
|
6901 |
+
|
6902 |
+
|
6903 |
+
|
6904 |
+
|
6905 |
+
|
6906 |
+
|
6907 |
+
|
6908 |
+
|
6909 |
+
|
6910 |
+
|
6911 |
+
|
6912 |
+
|
6913 |
+
|
6914 |
+
|
6915 |
+
|
6916 |
+
11%|ββββββββββββββββββββββββββ | 6460/60000 [19:03:35<153:07:22, 10.30s/it]
|
6917 |
+
|
6918 |
+
|
6919 |
+
|
6920 |
+
|
6921 |
+
Reading metadata...: 2165it [00:00, 12982.32it/s] | 6464/60000 [19:04:18<155:13:19, 10.44s/it]
|
6922 |
+
|
6923 |
+
|
6924 |
+
|
6925 |
+
|
6926 |
+
|
6927 |
+
|
6928 |
+
|
6929 |
+
|
6930 |
+
|
6931 |
+
|
6932 |
+
|
6933 |
+
|
6934 |
+
|
6935 |
+
|
6936 |
+
|
6937 |
+
|
6938 |
+
11%|ββββββββββββββββββββββββββ | 6480/60000 [19:06:54<143:38:53, 9.66s/it]
|
6939 |
+
|
6940 |
+
|
6941 |
+
|
6942 |
+
|
6943 |
+
|
6944 |
+
|
6945 |
+
|
6946 |
+
|
6947 |
+
|
6948 |
+
|
6949 |
+
|
6950 |
+
|
6951 |
+
|
6952 |
+
|
6953 |
+
|
6954 |
+
|
6955 |
+
|
6956 |
+
|
6957 |
+
|
6958 |
+
|
6959 |
+
11%|ββββββββββββββββββββββββββ | 6500/60000 [19:10:07<148:01:49, 9.96s/it]
|
6960 |
+
|
6961 |
+
|
6962 |
+
|
6963 |
+
|
6964 |
+
|
6965 |
+
|
6966 |
+
|
6967 |
+
|
6968 |
+
|
6969 |
+
|
6970 |
+
|
6971 |
+
|
6972 |
+
|
6973 |
+
|
6974 |
+
|
6975 |
+
|
6976 |
+
|
6977 |
+
|
6978 |
+
|
6979 |
+
|
6980 |
+
11%|ββββββββββββββββββββββββββ | 6520/60000 [19:13:16<142:11:29, 9.57s/it]
|
6981 |
+
|
6982 |
+
|
6983 |
+
|
6984 |
+
|
6985 |
+
|
6986 |
+
|
6987 |
+
|
6988 |
+
|
6989 |
+
|
6990 |
+
|
6991 |
+
|
6992 |
+
|
6993 |
+
|
6994 |
+
|
6995 |
+
|
6996 |
+
|
6997 |
+
|
6998 |
+
|
6999 |
+
|
7000 |
+
|
7001 |
+
11%|ββββββββββββββββββββββββββ | 6540/60000 [19:16:23<143:35:52, 9.67s/it]
|
7002 |
+
|
7003 |
+
|
7004 |
+
|
7005 |
+
|
7006 |
+
|
7007 |
+
|
7008 |
+
|
7009 |
+
|
7010 |
+
|
7011 |
+
|
7012 |
+
|
7013 |
+
|
7014 |
+
|
7015 |
+
|
7016 |
+
|
7017 |
+
|
7018 |
+
|
7019 |
+
|
7020 |
+
|
7021 |
+
|
7022 |
+
11%|ββββββββββββββββββββββββββ | 6560/60000 [19:19:28<136:50:54, 9.22s/it]
|
7023 |
+
|
7024 |
+
|
7025 |
+
|
7026 |
+
|
7027 |
+
|
7028 |
+
|
7029 |
+
|
7030 |
+
|
7031 |
+
|
7032 |
+
|
7033 |
+
|
7034 |
+
|
7035 |
+
|
7036 |
+
|
7037 |
+
|
7038 |
+
|
7039 |
+
|
7040 |
+
|
7041 |
+
|
7042 |
+
|
7043 |
+
11%|ββββββββββββββββββββββββββ | 6580/60000 [19:22:33<138:23:59, 9.33s/it]
|
7044 |
+
|
7045 |
+
|
7046 |
+
|
7047 |
+
|
7048 |
+
|
7049 |
+
|
7050 |
+
|
7051 |
+
|
7052 |
+
|
7053 |
+
|
7054 |
+
|
7055 |
+
|
7056 |
+
|
7057 |
+
|
7058 |
+
|
7059 |
+
|
7060 |
+
|
7061 |
+
|
7062 |
+
|
7063 |
+
|
7064 |
+
11%|ββββββββββββοΏ½οΏ½βββββββββββββ | 6600/60000 [19:26:09<137:35:52, 9.28s/it]
|
7065 |
+
|
7066 |
+
|
7067 |
+
|
7068 |
+
|
7069 |
+
|
7070 |
+
|
7071 |
+
|
7072 |
+
|
7073 |
+
|
7074 |
+
|
7075 |
+
|
7076 |
+
|
7077 |
+
|
7078 |
+
|
7079 |
+
|
7080 |
+
|
7081 |
+
|
7082 |
+
|
7083 |
+
|
7084 |
+
|
7085 |
+
11%|ββββββββββββββββββββββββββ | 6620/60000 [19:29:13<135:27:06, 9.13s/it]
|
7086 |
+
|
7087 |
+
|
7088 |
+
|
7089 |
+
|
7090 |
+
|
7091 |
+
|
7092 |
+
|
7093 |
+
|
7094 |
+
|
7095 |
+
|
7096 |
+
|
7097 |
+
|
7098 |
+
|
7099 |
+
|
7100 |
+
|
7101 |
+
|
7102 |
+
|
7103 |
+
|
7104 |
+
|
7105 |
+
|
7106 |
+
11%|ββββββββββββββββββββββββββ | 6640/60000 [19:33:06<143:51:38, 9.71s/it]
|
7107 |
+
|
7108 |
+
|
7109 |
+
|
7110 |
+
|
7111 |
+
|
7112 |
+
|
7113 |
+
|
7114 |
+
|
7115 |
+
|
7116 |
+
|
7117 |
+
|
7118 |
+
|
7119 |
+
|
7120 |
+
|
7121 |
+
|
7122 |
+
|
7123 |
+
|
7124 |
+
|
7125 |
+
|
7126 |
+
11%|βββββββββββββββββββββββββββ | 6659/60000 [19:36:01<136:15:03, 9.20s/it]
|
7127 |
+
|
7128 |
+
|
7129 |
+
|
7130 |
+
|
7131 |
+
|
7132 |
+
|
7133 |
+
|
7134 |
+
|
7135 |
+
|
7136 |
+
|
7137 |
+
|
7138 |
+
|
7139 |
+
|
7140 |
+
|
7141 |
+
|
7142 |
+
|
7143 |
+
|
7144 |
+
|
7145 |
+
|
7146 |
+
|
7147 |
+
11%|βββββββββββββββββββββββββββ | 6679/60000 [19:39:12<144:17:09, 9.74s/it]
|
7148 |
+
|
7149 |
+
|
7150 |
+
|
7151 |
+
|
7152 |
+
|
7153 |
+
|
7154 |
+
|
7155 |
+
|
7156 |
+
|
7157 |
+
|
7158 |
+
|
7159 |
+
|
7160 |
+
|
7161 |
+
|
7162 |
+
|
7163 |
+
|
7164 |
+
|
7165 |
+
|
7166 |
+
|
7167 |
+
|
7168 |
+
11%|βββββββββββββββββββββββββββ | 6699/60000 [19:42:18<133:33:33, 9.02s/it]
|
7169 |
+
|
7170 |
+
|
7171 |
+
|
7172 |
+
|
7173 |
+
|
7174 |
+
|
7175 |
+
|
7176 |
+
|
7177 |
+
|
7178 |
+
|
7179 |
+
|
7180 |
+
|
7181 |
+
|
7182 |
+
|
7183 |
+
|
7184 |
+
|
7185 |
+
|
7186 |
+
|
7187 |
+
|
7188 |
+
|
7189 |
+
11%|βββββββββββββββββββββββββββ | 6719/60000 [19:45:22<137:27:06, 9.29s/it]
|
7190 |
+
|
7191 |
+
|
7192 |
+
|
7193 |
+
|
7194 |
+
|
7195 |
+
|
7196 |
+
|
7197 |
+
|
7198 |
+
|
7199 |
+
|
7200 |
+
|
7201 |
+
|
7202 |
+
|
7203 |
+
|
7204 |
+
|
7205 |
+
|
7206 |
+
|
7207 |
+
|
7208 |
+
|
7209 |
+
|
7210 |
+
|
7211 |
+
11%|βββββββββββββββββββββββββββ | 6740/60000 [19:48:49<143:07:45, 9.67s/it]
|
7212 |
+
|
7213 |
+
|
7214 |
+
|
7215 |
+
|
7216 |
+
|
7217 |
+
|
7218 |
+
|
7219 |
+
|
7220 |
+
|
7221 |
+
|
7222 |
+
|
7223 |
+
|
7224 |
+
|
7225 |
+
|
7226 |
+
|
7227 |
+
|
7228 |
+
|
7229 |
+
|
7230 |
+
|
7231 |
+
|
7232 |
+
11%|βββββββββββββββββββββββββββ | 6760/60000 [19:51:53<137:12:21, 9.28s/it]
|
7233 |
+
|
7234 |
+
|
7235 |
+
|
7236 |
+
|
7237 |
+
|
7238 |
+
|
7239 |
+
|
7240 |
+
|
7241 |
+
|
7242 |
+
|
7243 |
+
|
7244 |
+
|
7245 |
+
|
7246 |
+
|
7247 |
+
|
7248 |
+
|
7249 |
+
|
7250 |
+
|
7251 |
+
|
7252 |
+
11%|βββββββββββββββββββββββββββ | 6779/60000 [19:54:52<135:29:24, 9.16s/it]
|
7253 |
+
|
7254 |
+
|
7255 |
+
|
7256 |
+
|
7257 |
+
|
7258 |
+
|
7259 |
+
|
7260 |
+
|
7261 |
+
|
7262 |
+
|
7263 |
+
|
7264 |
+
|
7265 |
+
|
7266 |
+
|
7267 |
+
|
7268 |
+
|
7269 |
+
|
7270 |
+
|
7271 |
+
|
7272 |
+
|
7273 |
+
11%|βββββββββββββββββββββββββββ | 6799/60000 [19:57:56<135:05:53, 9.14s/it]
|
7274 |
+
|
7275 |
+
|
7276 |
+
|
7277 |
+
|
7278 |
+
|
7279 |
+
|
7280 |
+
|
7281 |
+
|
7282 |
+
|
7283 |
+
|
7284 |
+
|
7285 |
+
|
7286 |
+
|
7287 |
+
|
7288 |
+
|
7289 |
+
|
7290 |
+
|
7291 |
+
|
7292 |
+
|
7293 |
+
|
7294 |
+
11%|βββββββββββββββββββββββββββ | 6819/60000 [20:00:59<137:10:24, 9.29s/it]
|
7295 |
+
|
7296 |
+
|
7297 |
+
|
7298 |
+
|
7299 |
+
|
7300 |
+
|
7301 |
+
|
7302 |
+
|
7303 |
+
|
7304 |
+
|
7305 |
+
|
7306 |
+
|
7307 |
+
|
7308 |
+
|
7309 |
+
|
7310 |
+
|
7311 |
+
|
7312 |
+
|
7313 |
+
|
7314 |
+
|
7315 |
+
11%|βββββββββββββββββββββββββββ | 6839/60000 [20:04:03<133:52:03, 9.07s/it]
|
7316 |
+
|
7317 |
+
|
7318 |
+
|
7319 |
+
|
7320 |
+
|
7321 |
+
|
7322 |
+
|
7323 |
+
|
7324 |
+
|
7325 |
+
|
7326 |
+
|
7327 |
+
|
7328 |
+
|
7329 |
+
|
7330 |
+
|
7331 |
+
|
7332 |
+
|
7333 |
+
|
7334 |
+
|
7335 |
+
|
7336 |
+
|
7337 |
+
11%|βββββββββββββββββββββββββββ | 6860/60000 [20:07:22<134:15:04, 9.09s/it]
|
7338 |
+
|
7339 |
+
|
7340 |
+
|
7341 |
+
|
7342 |
+
|
7343 |
+
|
7344 |
+
|
7345 |
+
|
7346 |
+
|
7347 |
+
|
7348 |
+
|
7349 |
+
|
7350 |
+
|
7351 |
+
|
7352 |
+
|
7353 |
+
|
7354 |
+
|
7355 |
+
|
7356 |
+
|
7357 |
+
|
7358 |
+
11%|βββββββββββββββββββββββββββ | 6880/60000 [20:10:28<141:11:17, 9.57s/it]
|
7359 |
+
|
7360 |
+
|
7361 |
+
|
7362 |
+
|
7363 |
+
|
7364 |
+
|
7365 |
+
|
7366 |
+
|
7367 |
+
|
7368 |
+
|
7369 |
+
|
7370 |
+
|
7371 |
+
|
7372 |
+
|
7373 |
+
|
7374 |
+
|
7375 |
+
|
7376 |
+
|
7377 |
+
|
7378 |
+
Reading metadata...: 1650it [00:00, 4019.95it/s] | 6899/60000 [20:13:26<135:28:47, 9.18s/it]
|
7379 |
+
|
7380 |
+
12%|ββββββββββββββββββββββββββββ | 6900/60000 [20:13:37<143:28:17, 9.73s/it]
|
7381 |
+
|
7382 |
+
|
7383 |
+
|
7384 |
+
|
7385 |
+
|
7386 |
+
|
7387 |
+
|
7388 |
+
|
7389 |
+
|
7390 |
+
|
7391 |
+
|
7392 |
+
|
7393 |
+
|
7394 |
+
|
7395 |
+
|
7396 |
+
|
7397 |
+
|
7398 |
+
|
7399 |
+
|
7400 |
+
12%|ββββββββββββββββββββββββββββ | 6919/60000 [20:16:36<138:08:56, 9.37s/it]
|
7401 |
+
|
7402 |
+
|
7403 |
+
|
7404 |
+
|
7405 |
+
|
7406 |
+
|
7407 |
+
|
7408 |
+
|
7409 |
+
|
7410 |
+
|
7411 |
+
|
7412 |
+
|
7413 |
+
|
7414 |
+
|
7415 |
+
|
7416 |
+
|
7417 |
+
|
7418 |
+
|
7419 |
+
|
7420 |
+
|
7421 |
+
|
7422 |
+
12%|ββββββββββββββββββββββββββββ | 6940/60000 [20:19:57<155:15:25, 10.53s/it]
|
7423 |
+
|
7424 |
+
|
7425 |
+
|
7426 |
+
|
7427 |
+
|
7428 |
+
|
7429 |
+
|
7430 |
+
|
7431 |
+
|
7432 |
+
|
7433 |
+
|
7434 |
+
|
7435 |
+
|
7436 |
+
|
7437 |
+
|
7438 |
+
|
7439 |
+
|
7440 |
+
|
7441 |
+
|
7442 |
+
12%|ββββββββββββββββββββββββββββ | 6959/60000 [20:22:53<136:55:04, 9.29s/it]
|
7443 |
+
|
7444 |
+
|
7445 |
+
|
7446 |
+
|
7447 |
+
|
7448 |
+
|
7449 |
+
|
7450 |
+
|
7451 |
+
|
7452 |
+
|
7453 |
+
|
7454 |
+
|
7455 |
+
|
7456 |
+
|
7457 |
+
|
7458 |
+
|
7459 |
+
|
7460 |
+
|
7461 |
+
|
7462 |
+
|
7463 |
+
12%|ββββββββββββββββββββββββββββ | 6979/60000 [20:26:13<139:06:06, 9.44s/it]
|
7464 |
+
|
7465 |
+
|
7466 |
+
|
7467 |
+
|
7468 |
+
|
7469 |
+
|
7470 |
+
|
7471 |
+
|
7472 |
+
|
7473 |
+
|
7474 |
+
|
7475 |
+
|
7476 |
+
|
7477 |
+
|
7478 |
+
|
7479 |
+
|
7480 |
+
|
7481 |
+
|
7482 |
+
|
7483 |
+
|
7484 |
+
12%|ββββββββββββββββββββββββββββ | 7000/60000 [20:29:27<135:59:18, 9.24s/it][INFO|trainer.py:3173] 2023-11-19 09:49:51,427 >> ***** Running Evaluation *****
|
7485 |
+
[INFO|trainer.py:3177] 2023-11-19 09:49:51,427 >> Num examples: Unknown
|
7486 |
+
[INFO|trainer.py:3178] 2023-11-19 09:49:51,428 >> Batch size = 4
|
7487 |
+
Reading metadata...: 1704it [00:00, 10117.74it/s]
|
7488 |
+
Reading metadata...: 1it [00:00, 6.90it/s]
|
7489 |
+
|
7490 |
+
[INFO|trainer_utils.py:759] 2023-11-19 09:49:52,356 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message.
|
7491 |
+
12%|ββββββββββββββββββββββββββββ | 7000/60000 [20:40:36<135:59:18, 9.24s/it]
|
7492 |
+
12%|ββββββββββββββββββββββββββββ | 7000/60000 [20:40:36<135:59:18, 9.24s/it][INFO|trainer.py:2896] 2023-11-19 10:01:42,953 >> Saving model checkpoint to ./checkpoint-7000
|
7493 |
+
[INFO|configuration_utils.py:462] 2023-11-19 10:01:42,964 >> Configuration saved in ./checkpoint-7000/config.json
|
7494 |
+
[INFO|configuration_utils.py:568] 2023-11-19 10:01:42,971 >> Configuration saved in ./checkpoint-7000/generation_config.json
|
7495 |
+
[2023-11-19 10:02:35,205] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step7000 is about to be saved!
|
7496 |
+
[2023-11-19 10:02:35,238] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt
|
7497 |
+
[2023-11-19 10:02:35,238] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt...
|
7498 |
+
[INFO|modeling_utils.py:2194] 2023-11-19 10:02:35,175 >> Model weights saved in ./checkpoint-7000/pytorch_model.bin
|
7499 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 10:02:35,181 >> Feature extractor saved in ./checkpoint-7000/preprocessor_config.json
|
7500 |
+
[2023-11-19 10:02:56,297] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt.
|
7501 |
+
[2023-11-19 10:02:56,319] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
|
7502 |
+
[2023-11-19 10:04:10,359] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
|
7503 |
+
[2023-11-19 10:04:10,369] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt
|
7504 |
+
[2023-11-19 10:04:10,370] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7000 is ready now!
|
7505 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 10:05:15,419 >> Feature extractor saved in ./preprocessor_config.json
|
7506 |
+
|
7507 |
+
|
7508 |
+
|
7509 |
+
|
7510 |
+
|
7511 |
+
|
7512 |
+
|
7513 |
+
|
7514 |
+
|
7515 |
+
|
7516 |
+
|
7517 |
+
|
7518 |
+
|
7519 |
+
|
7520 |
+
|
7521 |
+
|
7522 |
+
|
7523 |
+
|
7524 |
+
|
7525 |
+
12%|ββββββββββββββββββββββββββββ | 7020/60000 [20:48:31<159:37:27, 10.85s/it]
|
7526 |
+
|
7527 |
+
|
7528 |
+
|
7529 |
+
|
7530 |
+
|
7531 |
+
|
7532 |
+
|
7533 |
+
|
7534 |
+
|
7535 |
+
|
7536 |
+
|
7537 |
+
|
7538 |
+
|
7539 |
+
|
7540 |
+
|
7541 |
+
|
7542 |
+
|
7543 |
+
|
7544 |
+
|
7545 |
+
|
7546 |
+
12%|ββββββββββββββββββββββββββββ | 7040/60000 [20:52:05<149:28:41, 10.16s/it]
|
7547 |
+
|
7548 |
+
|
7549 |
+
|
7550 |
+
|
7551 |
+
|
7552 |
+
|
7553 |
+
|
7554 |
+
|
7555 |
+
|
7556 |
+
|
7557 |
+
|
7558 |
+
|
7559 |
+
|
7560 |
+
|
7561 |
+
|
7562 |
+
|
7563 |
+
|
7564 |
+
|
7565 |
+
|
7566 |
+
12%|ββββββββββββββββββββββββββββ | 7059/60000 [20:55:20<151:14:07, 10.28s/it]
|
7567 |
+
|
7568 |
+
|
7569 |
+
|
7570 |
+
|
7571 |
+
|
7572 |
+
|
wandb/run-20231118_132020-qg88991p/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"train/loss": 0.
|
|
|
1 |
+
{"train/loss": 0.1191, "train/learning_rate": 2.6924237288135594e-06, "train/epoch": 0.12, "train/global_step": 7060, "_timestamp": 1700381754.2567582, "_runtime": 75333.34166812897, "_step": 359, "eval/loss": 0.1439208984375, "eval/wer": 8.660007538635508, "eval/runtime": 668.8365, "eval/samples_per_second": 2.548, "eval/steps_per_second": 0.637}
|
wandb/run-20231118_132020-qg88991p/logs/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:829e7e0b338083a1d991e646ecb5258a236916e87ee43a0b5f00c5dcaf0766fe
|
3 |
+
size 5909427
|