Training in progress, step 6000
Browse files- model.safetensors +1 -1
- wandb/debug-internal.log +0 -0
- wandb/run-20231118_132020-qg88991p/files/output.log +2151 -0
- wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +1 -1
- wandb/run-20231118_132020-qg88991p/logs/debug-internal.log +0 -0
- wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb +2 -2
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 3219908024
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:13e8bfb6cc0a5778a11cf6475ff56b532497650ddb8b982d4a541b76fea9f445
|
3 |
size 3219908024
|
wandb/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231118_132020-qg88991p/files/output.log
CHANGED
@@ -4354,3 +4354,2154 @@ Reading metadata...: 1it [00:00, 6.53it/s]
|
|
4354 |
7%|ββββββββββββββββ | 4060/60000 [12:20:57<155:35:39, 10.01s/it]
|
4355 |
|
4356 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4354 |
7%|ββββββββββββββββ | 4060/60000 [12:20:57<155:35:39, 10.01s/it]
|
4355 |
|
4356 |
|
4357 |
+
|
4358 |
+
|
4359 |
+
|
4360 |
+
|
4361 |
+
|
4362 |
+
|
4363 |
+
|
4364 |
+
|
4365 |
+
|
4366 |
+
|
4367 |
+
|
4368 |
+
|
4369 |
+
|
4370 |
+
|
4371 |
+
|
4372 |
+
|
4373 |
+
|
4374 |
+
|
4375 |
+
7%|ββββββββββββββββ | 4080/60000 [12:24:15<148:47:04, 9.58s/it]
|
4376 |
+
|
4377 |
+
|
4378 |
+
|
4379 |
+
|
4380 |
+
|
4381 |
+
|
4382 |
+
|
4383 |
+
|
4384 |
+
|
4385 |
+
|
4386 |
+
|
4387 |
+
|
4388 |
+
|
4389 |
+
|
4390 |
+
|
4391 |
+
|
4392 |
+
|
4393 |
+
|
4394 |
+
|
4395 |
+
7%|ββββββββββββββββ | 4099/60000 [12:27:10<142:22:51, 9.17s/it]
|
4396 |
+
|
4397 |
+
|
4398 |
+
|
4399 |
+
|
4400 |
+
|
4401 |
+
|
4402 |
+
|
4403 |
+
|
4404 |
+
|
4405 |
+
|
4406 |
+
|
4407 |
+
|
4408 |
+
|
4409 |
+
|
4410 |
+
|
4411 |
+
|
4412 |
+
|
4413 |
+
|
4414 |
+
|
4415 |
+
|
4416 |
+
7%|βββββββββββββββββ | 4119/60000 [12:30:18<149:47:00, 9.65s/it]
|
4417 |
+
|
4418 |
+
|
4419 |
+
|
4420 |
+
|
4421 |
+
|
4422 |
+
|
4423 |
+
|
4424 |
+
|
4425 |
+
|
4426 |
+
|
4427 |
+
|
4428 |
+
|
4429 |
+
|
4430 |
+
|
4431 |
+
|
4432 |
+
|
4433 |
+
|
4434 |
+
|
4435 |
+
|
4436 |
+
|
4437 |
+
7%|βββββββββββββββββ | 4139/60000 [12:33:21<142:13:05, 9.17s/it]
|
4438 |
+
|
4439 |
+
|
4440 |
+
|
4441 |
+
|
4442 |
+
|
4443 |
+
|
4444 |
+
|
4445 |
+
|
4446 |
+
|
4447 |
+
|
4448 |
+
|
4449 |
+
|
4450 |
+
|
4451 |
+
|
4452 |
+
|
4453 |
+
|
4454 |
+
|
4455 |
+
|
4456 |
+
|
4457 |
+
|
4458 |
+
7%|βββββββββββββββββ | 4159/60000 [12:36:24<141:09:26, 9.10s/it]
|
4459 |
+
|
4460 |
+
|
4461 |
+
|
4462 |
+
|
4463 |
+
|
4464 |
+
|
4465 |
+
|
4466 |
+
|
4467 |
+
|
4468 |
+
|
4469 |
+
|
4470 |
+
|
4471 |
+
|
4472 |
+
|
4473 |
+
|
4474 |
+
|
4475 |
+
|
4476 |
+
|
4477 |
+
|
4478 |
+
|
4479 |
+
|
4480 |
+
7%|βββββββββββββββββ | 4180/60000 [12:39:38<141:22:34, 9.12s/it]
|
4481 |
+
|
4482 |
+
|
4483 |
+
|
4484 |
+
|
4485 |
+
|
4486 |
+
|
4487 |
+
|
4488 |
+
|
4489 |
+
|
4490 |
+
|
4491 |
+
|
4492 |
+
|
4493 |
+
|
4494 |
+
|
4495 |
+
|
4496 |
+
|
4497 |
+
|
4498 |
+
|
4499 |
+
|
4500 |
+
7%|βββββββββββββββββ | 4199/60000 [12:42:32<143:07:23, 9.23s/it]
|
4501 |
+
|
4502 |
+
|
4503 |
+
|
4504 |
+
|
4505 |
+
|
4506 |
+
|
4507 |
+
|
4508 |
+
|
4509 |
+
|
4510 |
+
|
4511 |
+
|
4512 |
+
|
4513 |
+
|
4514 |
+
|
4515 |
+
|
4516 |
+
|
4517 |
+
|
4518 |
+
|
4519 |
+
|
4520 |
+
|
4521 |
+
|
4522 |
+
7%|βββββββββββββββββ | 4220/60000 [12:45:49<140:44:27, 9.08s/it]
|
4523 |
+
|
4524 |
+
|
4525 |
+
|
4526 |
+
|
4527 |
+
|
4528 |
+
|
4529 |
+
|
4530 |
+
|
4531 |
+
|
4532 |
+
|
4533 |
+
|
4534 |
+
|
4535 |
+
|
4536 |
+
|
4537 |
+
|
4538 |
+
|
4539 |
+
|
4540 |
+
|
4541 |
+
|
4542 |
+
|
4543 |
+
7%|βββββββββββββββββ | 4240/60000 [12:48:52<142:41:50, 9.21s/it]
|
4544 |
+
|
4545 |
+
|
4546 |
+
|
4547 |
+
|
4548 |
+
|
4549 |
+
|
4550 |
+
|
4551 |
+
|
4552 |
+
|
4553 |
+
|
4554 |
+
|
4555 |
+
|
4556 |
+
|
4557 |
+
|
4558 |
+
|
4559 |
+
|
4560 |
+
|
4561 |
+
|
4562 |
+
|
4563 |
+
|
4564 |
+
7%|βββββββββββββββββ | 4260/60000 [12:51:57<141:40:28, 9.15s/it]
|
4565 |
+
|
4566 |
+
|
4567 |
+
|
4568 |
+
|
4569 |
+
|
4570 |
+
|
4571 |
+
|
4572 |
+
|
4573 |
+
|
4574 |
+
|
4575 |
+
|
4576 |
+
|
4577 |
+
|
4578 |
+
|
4579 |
+
|
4580 |
+
|
4581 |
+
|
4582 |
+
|
4583 |
+
|
4584 |
+
7%|βββββββββββββββββ | 4279/60000 [12:54:51<143:00:35, 9.24s/it]
|
4585 |
+
|
4586 |
+
|
4587 |
+
|
4588 |
+
|
4589 |
+
|
4590 |
+
|
4591 |
+
|
4592 |
+
|
4593 |
+
|
4594 |
+
|
4595 |
+
|
4596 |
+
|
4597 |
+
|
4598 |
+
|
4599 |
+
|
4600 |
+
|
4601 |
+
|
4602 |
+
|
4603 |
+
|
4604 |
+
|
4605 |
+
|
4606 |
+
7%|βββββββββββββββββ | 4300/60000 [12:58:09<146:41:35, 9.48s/it]
|
4607 |
+
|
4608 |
+
|
4609 |
+
|
4610 |
+
|
4611 |
+
|
4612 |
+
|
4613 |
+
|
4614 |
+
|
4615 |
+
|
4616 |
+
|
4617 |
+
|
4618 |
+
|
4619 |
+
|
4620 |
+
|
4621 |
+
|
4622 |
+
|
4623 |
+
|
4624 |
+
|
4625 |
+
|
4626 |
+
7%|βββββββββββββββββ | 4319/60000 [13:01:17<143:17:07, 9.26s/it]
|
4627 |
+
|
4628 |
+
|
4629 |
+
|
4630 |
+
|
4631 |
+
|
4632 |
+
|
4633 |
+
|
4634 |
+
|
4635 |
+
|
4636 |
+
|
4637 |
+
|
4638 |
+
|
4639 |
+
|
4640 |
+
|
4641 |
+
|
4642 |
+
|
4643 |
+
|
4644 |
+
|
4645 |
+
|
4646 |
+
|
4647 |
+
7%|βββββββββββββββββ | 4339/60000 [13:04:19<140:12:50, 9.07s/it]
|
4648 |
+
|
4649 |
+
|
4650 |
+
|
4651 |
+
|
4652 |
+
|
4653 |
+
|
4654 |
+
|
4655 |
+
|
4656 |
+
|
4657 |
+
|
4658 |
+
|
4659 |
+
|
4660 |
+
|
4661 |
+
|
4662 |
+
|
4663 |
+
|
4664 |
+
|
4665 |
+
|
4666 |
+
|
4667 |
+
|
4668 |
+
|
4669 |
+
7%|ββββββββββββββββββ | 4360/60000 [13:07:32<141:53:59, 9.18s/it]
|
4670 |
+
|
4671 |
+
|
4672 |
+
|
4673 |
+
|
4674 |
+
|
4675 |
+
|
4676 |
+
|
4677 |
+
|
4678 |
+
|
4679 |
+
|
4680 |
+
|
4681 |
+
|
4682 |
+
|
4683 |
+
|
4684 |
+
|
4685 |
+
|
4686 |
+
|
4687 |
+
|
4688 |
+
|
4689 |
+
|
4690 |
+
7%|ββββββββββββββββββ | 4380/60000 [13:10:38<145:43:12, 9.43s/it]
|
4691 |
+
|
4692 |
+
|
4693 |
+
|
4694 |
+
|
4695 |
+
|
4696 |
+
|
4697 |
+
|
4698 |
+
|
4699 |
+
|
4700 |
+
|
4701 |
+
|
4702 |
+
|
4703 |
+
|
4704 |
+
|
4705 |
+
|
4706 |
+
|
4707 |
+
|
4708 |
+
|
4709 |
+
|
4710 |
+
7%|ββββββββββββββββββ | 4399/60000 [13:14:23<142:17:34, 9.21s/it]
|
4711 |
+
|
4712 |
+
|
4713 |
+
|
4714 |
+
|
4715 |
+
|
4716 |
+
|
4717 |
+
|
4718 |
+
|
4719 |
+
|
4720 |
+
|
4721 |
+
|
4722 |
+
|
4723 |
+
|
4724 |
+
|
4725 |
+
|
4726 |
+
|
4727 |
+
|
4728 |
+
|
4729 |
+
|
4730 |
+
|
4731 |
+
7%|ββββββββββββββββββ | 4419/60000 [13:17:25<140:58:10, 9.13s/it]
|
4732 |
+
|
4733 |
+
|
4734 |
+
|
4735 |
+
|
4736 |
+
|
4737 |
+
|
4738 |
+
|
4739 |
+
|
4740 |
+
|
4741 |
+
|
4742 |
+
|
4743 |
+
|
4744 |
+
|
4745 |
+
|
4746 |
+
|
4747 |
+
|
4748 |
+
|
4749 |
+
|
4750 |
+
|
4751 |
+
|
4752 |
+
7%|ββββββββββββββββββ | 4439/60000 [13:20:31<141:51:43, 9.19s/it]
|
4753 |
+
|
4754 |
+
|
4755 |
+
|
4756 |
+
|
4757 |
+
|
4758 |
+
|
4759 |
+
|
4760 |
+
|
4761 |
+
|
4762 |
+
|
4763 |
+
|
4764 |
+
|
4765 |
+
|
4766 |
+
|
4767 |
+
|
4768 |
+
|
4769 |
+
|
4770 |
+
|
4771 |
+
|
4772 |
+
|
4773 |
+
|
4774 |
+
7%|ββββββββββββββββββ | 4460/60000 [13:23:45<141:09:48, 9.15s/it]
|
4775 |
+
|
4776 |
+
|
4777 |
+
|
4778 |
+
|
4779 |
+
|
4780 |
+
|
4781 |
+
|
4782 |
+
|
4783 |
+
|
4784 |
+
|
4785 |
+
|
4786 |
+
|
4787 |
+
|
4788 |
+
|
4789 |
+
|
4790 |
+
|
4791 |
+
|
4792 |
+
|
4793 |
+
|
4794 |
+
|
4795 |
+
7%|ββββββββββββββββββ | 4480/60000 [13:27:06<177:08:56, 11.49s/it]
|
4796 |
+
|
4797 |
+
|
4798 |
+
|
4799 |
+
|
4800 |
+
|
4801 |
+
|
4802 |
+
|
4803 |
+
|
4804 |
+
|
4805 |
+
|
4806 |
+
|
4807 |
+
|
4808 |
+
|
4809 |
+
|
4810 |
+
|
4811 |
+
|
4812 |
+
|
4813 |
+
|
4814 |
+
|
4815 |
+
|
4816 |
+
8%|ββββββββββββββββββ | 4500/60000 [13:30:08<139:35:12, 9.05s/it]
|
4817 |
+
|
4818 |
+
|
4819 |
+
|
4820 |
+
|
4821 |
+
|
4822 |
+
|
4823 |
+
|
4824 |
+
|
4825 |
+
|
4826 |
+
|
4827 |
+
|
4828 |
+
|
4829 |
+
|
4830 |
+
|
4831 |
+
|
4832 |
+
|
4833 |
+
|
4834 |
+
|
4835 |
+
|
4836 |
+
|
4837 |
+
8%|ββββββββββββββββββ | 4520/60000 [13:33:36<140:19:47, 9.11s/it]
|
4838 |
+
|
4839 |
+
|
4840 |
+
|
4841 |
+
|
4842 |
+
|
4843 |
+
|
4844 |
+
|
4845 |
+
|
4846 |
+
|
4847 |
+
|
4848 |
+
|
4849 |
+
|
4850 |
+
|
4851 |
+
|
4852 |
+
|
4853 |
+
|
4854 |
+
|
4855 |
+
|
4856 |
+
|
4857 |
+
8%|ββββββββββββββββββ | 4539/60000 [13:36:30<140:32:06, 9.12s/it]
|
4858 |
+
|
4859 |
+
|
4860 |
+
|
4861 |
+
|
4862 |
+
|
4863 |
+
|
4864 |
+
|
4865 |
+
|
4866 |
+
|
4867 |
+
|
4868 |
+
|
4869 |
+
|
4870 |
+
|
4871 |
+
|
4872 |
+
|
4873 |
+
|
4874 |
+
|
4875 |
+
|
4876 |
+
|
4877 |
+
|
4878 |
+
8%|ββββββββββββββββββ | 4559/60000 [13:39:38<150:37:32, 9.78s/it]
|
4879 |
+
|
4880 |
+
|
4881 |
+
|
4882 |
+
|
4883 |
+
|
4884 |
+
|
4885 |
+
|
4886 |
+
|
4887 |
+
|
4888 |
+
|
4889 |
+
|
4890 |
+
|
4891 |
+
|
4892 |
+
|
4893 |
+
|
4894 |
+
|
4895 |
+
|
4896 |
+
|
4897 |
+
|
4898 |
+
|
4899 |
+
8%|ββββββββββββββββββ | 4579/60000 [13:42:45<144:33:59, 9.39s/it]
|
4900 |
+
|
4901 |
+
|
4902 |
+
|
4903 |
+
|
4904 |
+
|
4905 |
+
|
4906 |
+
|
4907 |
+
|
4908 |
+
|
4909 |
+
|
4910 |
+
|
4911 |
+
|
4912 |
+
|
4913 |
+
8%|ββββββββββββββββββ | 4592/60000 [13:44:38<123:38:09, 8.03s/it]
|
4914 |
+
|
4915 |
+
8%|ββββββββββββββββββ | 4593/60000 [13:44:44<112:47:52, 7.33s/it]
|
4916 |
+
|
4917 |
+
|
4918 |
+
|
4919 |
+
|
4920 |
+
|
4921 |
+
|
4922 |
+
|
4923 |
+
8%|ββββββββββββββββββ | 4600/60000 [13:45:48<138:21:37, 8.99s/it]
|
4924 |
+
|
4925 |
+
|
4926 |
+
|
4927 |
+
|
4928 |
+
|
4929 |
+
|
4930 |
+
|
4931 |
+
|
4932 |
+
|
4933 |
+
|
4934 |
+
|
4935 |
+
|
4936 |
+
|
4937 |
+
|
4938 |
+
|
4939 |
+
|
4940 |
+
|
4941 |
+
|
4942 |
+
|
4943 |
+
8%|βββββββββββββββββββ | 4619/60000 [13:48:41<138:11:57, 8.98s/it]
|
4944 |
+
|
4945 |
+
|
4946 |
+
|
4947 |
+
|
4948 |
+
|
4949 |
+
|
4950 |
+
|
4951 |
+
|
4952 |
+
|
4953 |
+
|
4954 |
+
|
4955 |
+
|
4956 |
+
|
4957 |
+
|
4958 |
+
|
4959 |
+
|
4960 |
+
|
4961 |
+
|
4962 |
+
|
4963 |
+
|
4964 |
+
8%|βββββββββββββββββββ | 4639/60000 [13:51:53<144:05:49, 9.37s/it]
|
4965 |
+
|
4966 |
+
|
4967 |
+
|
4968 |
+
|
4969 |
+
|
4970 |
+
|
4971 |
+
|
4972 |
+
|
4973 |
+
|
4974 |
+
|
4975 |
+
|
4976 |
+
|
4977 |
+
|
4978 |
+
|
4979 |
+
|
4980 |
+
|
4981 |
+
|
4982 |
+
|
4983 |
+
|
4984 |
+
|
4985 |
+
|
4986 |
+
8%|βββββββββββββββββββ | 4660/60000 [13:55:10<142:38:56, 9.28s/it]
|
4987 |
+
|
4988 |
+
|
4989 |
+
|
4990 |
+
|
4991 |
+
|
4992 |
+
|
4993 |
+
|
4994 |
+
|
4995 |
+
|
4996 |
+
|
4997 |
+
|
4998 |
+
|
4999 |
+
|
5000 |
+
|
5001 |
+
|
5002 |
+
|
5003 |
+
|
5004 |
+
|
5005 |
+
|
5006 |
+
|
5007 |
+
8%|βββββββββββββββββββ | 4680/60000 [13:58:16<142:21:32, 9.26s/it]
|
5008 |
+
|
5009 |
+
|
5010 |
+
|
5011 |
+
|
5012 |
+
|
5013 |
+
|
5014 |
+
|
5015 |
+
|
5016 |
+
|
5017 |
+
|
5018 |
+
|
5019 |
+
|
5020 |
+
|
5021 |
+
|
5022 |
+
|
5023 |
+
|
5024 |
+
|
5025 |
+
|
5026 |
+
|
5027 |
+
|
5028 |
+
8%|βββββββββββββββββββ | 4700/60000 [14:01:20<139:31:05, 9.08s/it]
|
5029 |
+
|
5030 |
+
|
5031 |
+
|
5032 |
+
|
5033 |
+
|
5034 |
+
|
5035 |
+
|
5036 |
+
|
5037 |
+
|
5038 |
+
|
5039 |
+
|
5040 |
+
|
5041 |
+
|
5042 |
+
|
5043 |
+
|
5044 |
+
|
5045 |
+
|
5046 |
+
|
5047 |
+
|
5048 |
+
|
5049 |
+
8%|βββββββββββββββββββ | 4720/60000 [14:04:23<140:58:48, 9.18s/it]
|
5050 |
+
|
5051 |
+
|
5052 |
+
|
5053 |
+
|
5054 |
+
|
5055 |
+
|
5056 |
+
|
5057 |
+
|
5058 |
+
|
5059 |
+
|
5060 |
+
|
5061 |
+
|
5062 |
+
|
5063 |
+
|
5064 |
+
|
5065 |
+
|
5066 |
+
|
5067 |
+
|
5068 |
+
|
5069 |
+
|
5070 |
+
8%|βββββββββββββββββββ | 4740/60000 [14:07:47<142:14:29, 9.27s/it]
|
5071 |
+
|
5072 |
+
|
5073 |
+
|
5074 |
+
|
5075 |
+
|
5076 |
+
|
5077 |
+
|
5078 |
+
|
5079 |
+
|
5080 |
+
|
5081 |
+
|
5082 |
+
|
5083 |
+
|
5084 |
+
|
5085 |
+
|
5086 |
+
|
5087 |
+
|
5088 |
+
|
5089 |
+
|
5090 |
+
|
5091 |
+
8%|βββββββββββββββββββ | 4760/60000 [14:10:50<141:40:34, 9.23s/it]
|
5092 |
+
|
5093 |
+
|
5094 |
+
|
5095 |
+
|
5096 |
+
|
5097 |
+
|
5098 |
+
|
5099 |
+
|
5100 |
+
|
5101 |
+
|
5102 |
+
|
5103 |
+
|
5104 |
+
|
5105 |
+
|
5106 |
+
|
5107 |
+
|
5108 |
+
|
5109 |
+
|
5110 |
+
|
5111 |
+
|
5112 |
+
8%|βββββββββββββββββββ | 4780/60000 [14:13:53<139:52:31, 9.12s/it]
|
5113 |
+
|
5114 |
+
|
5115 |
+
|
5116 |
+
|
5117 |
+
|
5118 |
+
|
5119 |
+
|
5120 |
+
|
5121 |
+
|
5122 |
+
|
5123 |
+
|
5124 |
+
|
5125 |
+
|
5126 |
+
|
5127 |
+
|
5128 |
+
|
5129 |
+
|
5130 |
+
|
5131 |
+
|
5132 |
+
|
5133 |
+
8%|βββββββββββββββββββ | 4800/60000 [14:16:55<138:27:32, 9.03s/it]
|
5134 |
+
|
5135 |
+
|
5136 |
+
|
5137 |
+
|
5138 |
+
|
5139 |
+
|
5140 |
+
|
5141 |
+
|
5142 |
+
|
5143 |
+
|
5144 |
+
|
5145 |
+
|
5146 |
+
|
5147 |
+
|
5148 |
+
|
5149 |
+
|
5150 |
+
|
5151 |
+
|
5152 |
+
|
5153 |
+
|
5154 |
+
8%|βββββββββββββββββββ | 4820/60000 [14:19:59<141:38:49, 9.24s/it]
|
5155 |
+
|
5156 |
+
|
5157 |
+
|
5158 |
+
|
5159 |
+
|
5160 |
+
|
5161 |
+
|
5162 |
+
|
5163 |
+
|
5164 |
+
|
5165 |
+
|
5166 |
+
|
5167 |
+
|
5168 |
+
|
5169 |
+
|
5170 |
+
|
5171 |
+
|
5172 |
+
|
5173 |
+
|
5174 |
+
|
5175 |
+
8%|βββββββββββββββββββ | 4840/60000 [14:23:13<154:09:08, 10.06s/it]
|
5176 |
+
|
5177 |
+
|
5178 |
+
|
5179 |
+
|
5180 |
+
|
5181 |
+
|
5182 |
+
|
5183 |
+
|
5184 |
+
|
5185 |
+
|
5186 |
+
|
5187 |
+
|
5188 |
+
|
5189 |
+
|
5190 |
+
|
5191 |
+
|
5192 |
+
|
5193 |
+
|
5194 |
+
|
5195 |
+
8%|βββββββββββββββββββ | 4859/60000 [14:26:14<141:10:51, 9.22s/it]
|
5196 |
+
|
5197 |
+
|
5198 |
+
|
5199 |
+
|
5200 |
+
|
5201 |
+
|
5202 |
+
|
5203 |
+
|
5204 |
+
|
5205 |
+
|
5206 |
+
|
5207 |
+
|
5208 |
+
|
5209 |
+
|
5210 |
+
|
5211 |
+
|
5212 |
+
|
5213 |
+
|
5214 |
+
|
5215 |
+
|
5216 |
+
|
5217 |
+
8%|ββββββββββββββββββββ | 4880/60000 [14:29:29<141:33:20, 9.25s/it]
|
5218 |
+
|
5219 |
+
|
5220 |
+
|
5221 |
+
|
5222 |
+
|
5223 |
+
|
5224 |
+
|
5225 |
+
|
5226 |
+
|
5227 |
+
|
5228 |
+
|
5229 |
+
|
5230 |
+
|
5231 |
+
|
5232 |
+
|
5233 |
+
|
5234 |
+
|
5235 |
+
|
5236 |
+
|
5237 |
+
8%|ββββββββββββββββββββ | 4899/60000 [14:32:25<138:47:05, 9.07s/it]
|
5238 |
+
|
5239 |
+
|
5240 |
+
|
5241 |
+
|
5242 |
+
|
5243 |
+
|
5244 |
+
|
5245 |
+
|
5246 |
+
|
5247 |
+
|
5248 |
+
|
5249 |
+
|
5250 |
+
|
5251 |
+
|
5252 |
+
|
5253 |
+
|
5254 |
+
|
5255 |
+
|
5256 |
+
|
5257 |
+
|
5258 |
+
8%|ββββββββββββββββββββ | 4919/60000 [14:35:34<142:20:01, 9.30s/it]
|
5259 |
+
|
5260 |
+
Reading metadata...: 1it [00:00, 2.55it/s]
|
5261 |
+
|
5262 |
+
|
5263 |
+
|
5264 |
+
|
5265 |
+
|
5266 |
+
|
5267 |
+
|
5268 |
+
|
5269 |
+
|
5270 |
+
|
5271 |
+
|
5272 |
+
|
5273 |
+
|
5274 |
+
|
5275 |
+
|
5276 |
+
|
5277 |
+
|
5278 |
+
|
5279 |
+
|
5280 |
+
|
5281 |
+
8%|ββββββββββββββββββββ | 4939/60000 [14:38:35<137:08:39, 8.97s/it]
|
5282 |
+
|
5283 |
+
|
5284 |
+
|
5285 |
+
|
5286 |
+
|
5287 |
+
|
5288 |
+
|
5289 |
+
|
5290 |
+
|
5291 |
+
|
5292 |
+
|
5293 |
+
|
5294 |
+
|
5295 |
+
|
5296 |
+
|
5297 |
+
|
5298 |
+
|
5299 |
+
|
5300 |
+
|
5301 |
+
|
5302 |
+
8%|ββββββββββββββββββββ | 4959/60000 [14:41:37<138:07:06, 9.03s/it]
|
5303 |
+
|
5304 |
+
|
5305 |
+
|
5306 |
+
|
5307 |
+
|
5308 |
+
|
5309 |
+
|
5310 |
+
|
5311 |
+
|
5312 |
+
|
5313 |
+
|
5314 |
+
|
5315 |
+
|
5316 |
+
|
5317 |
+
|
5318 |
+
|
5319 |
+
|
5320 |
+
|
5321 |
+
|
5322 |
+
|
5323 |
+
8%|ββββββββββββββββββββ | 4979/60000 [14:44:51<136:58:29, 8.96s/it]
|
5324 |
+
|
5325 |
+
|
5326 |
+
|
5327 |
+
|
5328 |
+
|
5329 |
+
|
5330 |
+
|
5331 |
+
|
5332 |
+
|
5333 |
+
|
5334 |
+
|
5335 |
+
|
5336 |
+
|
5337 |
+
|
5338 |
+
|
5339 |
+
|
5340 |
+
|
5341 |
+
|
5342 |
+
|
5343 |
+
|
5344 |
+
8%|ββββββββββββββββββββ | 4999/60000 [14:47:58<161:55:31, 10.60s/it]
|
5345 |
+
8%|ββββββββββββββββββββ | 5000/60000 [14:48:07<154:23:38, 10.11s/it][INFO|trainer.py:3173] 2023-11-19 04:08:31,335 >> ***** Running Evaluation *****
|
5346 |
+
[INFO|trainer.py:3177] 2023-11-19 04:08:31,336 >> Num examples: Unknown
|
5347 |
+
[INFO|trainer.py:3178] 2023-11-19 04:08:31,336 >> Batch size = 4
|
5348 |
+
Reading metadata...: 1704it [00:00, 10489.07it/s]
|
5349 |
+
[INFO|trainer_utils.py:759] 2023-11-19 04:08:32,633 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message.
|
5350 |
+
{'eval_loss': 0.13427734375, 'eval_wer': 8.876743309460988, 'eval_runtime': 590.5226, 'eval_samples_per_second': 2.886, 'eval_steps_per_second': 0.721, 'epoch': 0.08}
|
5351 |
+
8%|ββββββββββββββββββββ | 5000/60000 [14:57:58<154:23:38, 10.11s/it][INFO|trainer.py:2896] 2023-11-19 04:18:48,757 >> Saving model checkpoint to ./checkpoint-5000
|
5352 |
+
[INFO|configuration_utils.py:462] 2023-11-19 04:18:48,768 >> Configuration saved in ./checkpoint-5000/config.json
|
5353 |
+
[INFO|configuration_utils.py:568] 2023-11-19 04:18:48,773 >> Configuration saved in ./checkpoint-5000/generation_config.json
|
5354 |
+
[2023-11-19 04:19:36,452] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step5000 is about to be saved!
|
5355 |
+
[2023-11-19 04:19:36,477] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt
|
5356 |
+
[2023-11-19 04:19:36,477] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt...
|
5357 |
+
[INFO|modeling_utils.py:2194] 2023-11-19 04:19:36,434 >> Model weights saved in ./checkpoint-5000/pytorch_model.bin
|
5358 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 04:19:36,438 >> Feature extractor saved in ./checkpoint-5000/preprocessor_config.json
|
5359 |
+
[2023-11-19 04:19:45,104] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt.
|
5360 |
+
[2023-11-19 04:19:45,108] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
|
5361 |
+
[2023-11-19 04:20:11,141] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
|
5362 |
+
[2023-11-19 04:20:11,150] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt
|
5363 |
+
[2023-11-19 04:20:11,150] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step5000 is ready now!
|
5364 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 04:21:08,374 >> Feature extractor saved in ./preprocessor_config.json
|
5365 |
+
|
5366 |
+
|
5367 |
+
|
5368 |
+
|
5369 |
+
|
5370 |
+
|
5371 |
+
|
5372 |
+
|
5373 |
+
|
5374 |
+
|
5375 |
+
|
5376 |
+
|
5377 |
+
|
5378 |
+
|
5379 |
+
|
5380 |
+
|
5381 |
+
|
5382 |
+
|
5383 |
+
|
5384 |
+
8%|ββββββββββββββββββββ | 5020/60000 [15:03:55<151:26:59, 9.92s/it]
|
5385 |
+
|
5386 |
+
|
5387 |
+
|
5388 |
+
|
5389 |
+
|
5390 |
+
|
5391 |
+
|
5392 |
+
|
5393 |
+
|
5394 |
+
|
5395 |
+
|
5396 |
+
|
5397 |
+
|
5398 |
+
|
5399 |
+
|
5400 |
+
|
5401 |
+
|
5402 |
+
|
5403 |
+
|
5404 |
+
8%|ββββββββββββββββββββ | 5039/60000 [15:06:53<141:57:34, 9.30s/it]
|
5405 |
+
|
5406 |
+
|
5407 |
+
|
5408 |
+
|
5409 |
+
|
5410 |
+
|
5411 |
+
|
5412 |
+
|
5413 |
+
|
5414 |
+
|
5415 |
+
|
5416 |
+
|
5417 |
+
|
5418 |
+
|
5419 |
+
|
5420 |
+
|
5421 |
+
|
5422 |
+
|
5423 |
+
|
5424 |
+
|
5425 |
+
|
5426 |
+
8%|ββββββββββββββββββββ | 5060/60000 [15:10:10<144:48:47, 9.49s/it]
|
5427 |
+
|
5428 |
+
|
5429 |
+
|
5430 |
+
|
5431 |
+
|
5432 |
+
|
5433 |
+
|
5434 |
+
|
5435 |
+
|
5436 |
+
|
5437 |
+
|
5438 |
+
|
5439 |
+
|
5440 |
+
|
5441 |
+
|
5442 |
+
|
5443 |
+
|
5444 |
+
|
5445 |
+
|
5446 |
+
8%|ββββββββββββββββββββ | 5079/60000 [15:13:10<142:38:34, 9.35s/it]
|
5447 |
+
|
5448 |
+
|
5449 |
+
|
5450 |
+
|
5451 |
+
|
5452 |
+
|
5453 |
+
|
5454 |
+
|
5455 |
+
|
5456 |
+
|
5457 |
+
|
5458 |
+
|
5459 |
+
|
5460 |
+
|
5461 |
+
|
5462 |
+
|
5463 |
+
|
5464 |
+
|
5465 |
+
|
5466 |
+
|
5467 |
+
8%|ββββββββββββββββββββ | 5099/60000 [15:16:17<138:15:38, 9.07s/it]
|
5468 |
+
|
5469 |
+
|
5470 |
+
|
5471 |
+
|
5472 |
+
|
5473 |
+
|
5474 |
+
|
5475 |
+
|
5476 |
+
|
5477 |
+
|
5478 |
+
|
5479 |
+
|
5480 |
+
|
5481 |
+
|
5482 |
+
|
5483 |
+
|
5484 |
+
|
5485 |
+
|
5486 |
+
|
5487 |
+
|
5488 |
+
9%|βββββββββββββββββββββ | 5119/60000 [15:19:49<186:27:32, 12.23s/it]
|
5489 |
+
|
5490 |
+
|
5491 |
+
|
5492 |
+
|
5493 |
+
|
5494 |
+
|
5495 |
+
|
5496 |
+
|
5497 |
+
|
5498 |
+
|
5499 |
+
|
5500 |
+
|
5501 |
+
|
5502 |
+
|
5503 |
+
|
5504 |
+
|
5505 |
+
|
5506 |
+
|
5507 |
+
|
5508 |
+
|
5509 |
+
|
5510 |
+
9%|βββββββββββββββββββββ | 5140/60000 [15:23:05<140:57:48, 9.25s/it]
|
5511 |
+
|
5512 |
+
|
5513 |
+
|
5514 |
+
|
5515 |
+
|
5516 |
+
|
5517 |
+
|
5518 |
+
|
5519 |
+
|
5520 |
+
|
5521 |
+
|
5522 |
+
|
5523 |
+
|
5524 |
+
|
5525 |
+
|
5526 |
+
|
5527 |
+
|
5528 |
+
|
5529 |
+
|
5530 |
+
9%|βββββββββββββββββββββ | 5159/60000 [15:26:01<141:07:57, 9.26s/it]
|
5531 |
+
|
5532 |
+
|
5533 |
+
|
5534 |
+
|
5535 |
+
|
5536 |
+
Reading metadata...: 2165it [00:00, 13143.40it/s] | 5165/60000 [15:26:56<140:01:37, 9.19s/it]
|
5537 |
+
|
5538 |
+
|
5539 |
+
|
5540 |
+
|
5541 |
+
|
5542 |
+
|
5543 |
+
|
5544 |
+
|
5545 |
+
|
5546 |
+
|
5547 |
+
|
5548 |
+
|
5549 |
+
|
5550 |
+
|
5551 |
+
|
5552 |
+
9%|βββββββββββββββββββββ | 5180/60000 [15:29:22<147:19:27, 9.67s/it]
|
5553 |
+
|
5554 |
+
|
5555 |
+
|
5556 |
+
|
5557 |
+
9%|βββββββββββββββββββββ | 5184/60000 [15:29:55<126:22:47, 8.30s/it]
|
5558 |
+
|
5559 |
+
|
5560 |
+
|
5561 |
+
|
5562 |
+
|
5563 |
+
|
5564 |
+
|
5565 |
+
|
5566 |
+
|
5567 |
+
|
5568 |
+
|
5569 |
+
|
5570 |
+
|
5571 |
+
|
5572 |
+
|
5573 |
+
9%|βββββββββββββββββββββ | 5199/60000 [15:32:13<141:17:06, 9.28s/it]
|
5574 |
+
|
5575 |
+
|
5576 |
+
|
5577 |
+
|
5578 |
+
|
5579 |
+
|
5580 |
+
|
5581 |
+
|
5582 |
+
|
5583 |
+
|
5584 |
+
|
5585 |
+
|
5586 |
+
|
5587 |
+
|
5588 |
+
|
5589 |
+
|
5590 |
+
|
5591 |
+
|
5592 |
+
|
5593 |
+
|
5594 |
+
9%|βββββββββββββββββββββ | 5219/60000 [15:35:18<138:39:00, 9.11s/it]
|
5595 |
+
|
5596 |
+
|
5597 |
+
|
5598 |
+
|
5599 |
+
|
5600 |
+
|
5601 |
+
|
5602 |
+
|
5603 |
+
|
5604 |
+
|
5605 |
+
|
5606 |
+
|
5607 |
+
|
5608 |
+
|
5609 |
+
|
5610 |
+
|
5611 |
+
|
5612 |
+
|
5613 |
+
|
5614 |
+
|
5615 |
+
|
5616 |
+
9%|βββββββββββββββββββββ | 5240/60000 [15:38:44<142:33:38, 9.37s/it]
|
5617 |
+
|
5618 |
+
|
5619 |
+
|
5620 |
+
|
5621 |
+
|
5622 |
+
|
5623 |
+
|
5624 |
+
|
5625 |
+
|
5626 |
+
|
5627 |
+
|
5628 |
+
|
5629 |
+
|
5630 |
+
|
5631 |
+
|
5632 |
+
|
5633 |
+
|
5634 |
+
|
5635 |
+
|
5636 |
+
9%|βββββββββββββββββββββ | 5259/60000 [15:41:40<138:42:07, 9.12s/it]
|
5637 |
+
|
5638 |
+
|
5639 |
+
|
5640 |
+
|
5641 |
+
|
5642 |
+
|
5643 |
+
|
5644 |
+
|
5645 |
+
|
5646 |
+
|
5647 |
+
|
5648 |
+
|
5649 |
+
|
5650 |
+
|
5651 |
+
|
5652 |
+
|
5653 |
+
|
5654 |
+
|
5655 |
+
|
5656 |
+
|
5657 |
+
9%|βββββββββββββββββββββ | 5279/60000 [15:44:50<139:40:03, 9.19s/it]
|
5658 |
+
|
5659 |
+
|
5660 |
+
|
5661 |
+
|
5662 |
+
|
5663 |
+
|
5664 |
+
|
5665 |
+
|
5666 |
+
|
5667 |
+
|
5668 |
+
|
5669 |
+
|
5670 |
+
|
5671 |
+
|
5672 |
+
|
5673 |
+
|
5674 |
+
|
5675 |
+
|
5676 |
+
|
5677 |
+
|
5678 |
+
9%|βββββββββββββββββββββ | 5299/60000 [15:47:54<143:46:47, 9.46s/it]
|
5679 |
+
|
5680 |
+
|
5681 |
+
|
5682 |
+
|
5683 |
+
|
5684 |
+
|
5685 |
+
|
5686 |
+
|
5687 |
+
|
5688 |
+
|
5689 |
+
|
5690 |
+
|
5691 |
+
|
5692 |
+
|
5693 |
+
|
5694 |
+
|
5695 |
+
|
5696 |
+
|
5697 |
+
|
5698 |
+
|
5699 |
+
9%|βββββββββββββββββββββ | 5319/60000 [15:51:13<140:19:23, 9.24s/it]
|
5700 |
+
|
5701 |
+
|
5702 |
+
|
5703 |
+
|
5704 |
+
|
5705 |
+
|
5706 |
+
|
5707 |
+
|
5708 |
+
|
5709 |
+
|
5710 |
+
|
5711 |
+
|
5712 |
+
|
5713 |
+
|
5714 |
+
|
5715 |
+
|
5716 |
+
|
5717 |
+
|
5718 |
+
|
5719 |
+
|
5720 |
+
|
5721 |
+
9%|βββββββββββββββββββββ | 5340/60000 [15:55:01<143:07:48, 9.43s/it]
|
5722 |
+
|
5723 |
+
|
5724 |
+
|
5725 |
+
|
5726 |
+
|
5727 |
+
|
5728 |
+
|
5729 |
+
|
5730 |
+
|
5731 |
+
|
5732 |
+
|
5733 |
+
|
5734 |
+
|
5735 |
+
|
5736 |
+
|
5737 |
+
|
5738 |
+
|
5739 |
+
|
5740 |
+
|
5741 |
+
|
5742 |
+
9%|βββββββββββββββββββββ | 5360/60000 [15:58:12<142:34:52, 9.39s/it]
|
5743 |
+
|
5744 |
+
|
5745 |
+
|
5746 |
+
|
5747 |
+
|
5748 |
+
|
5749 |
+
|
5750 |
+
|
5751 |
+
|
5752 |
+
|
5753 |
+
|
5754 |
+
|
5755 |
+
|
5756 |
+
|
5757 |
+
|
5758 |
+
|
5759 |
+
|
5760 |
+
|
5761 |
+
|
5762 |
+
|
5763 |
+
9%|ββββββββββββββββββββββ | 5380/60000 [16:01:16<138:08:35, 9.11s/it]
|
5764 |
+
|
5765 |
+
|
5766 |
+
|
5767 |
+
|
5768 |
+
|
5769 |
+
|
5770 |
+
|
5771 |
+
|
5772 |
+
|
5773 |
+
|
5774 |
+
|
5775 |
+
|
5776 |
+
|
5777 |
+
|
5778 |
+
|
5779 |
+
|
5780 |
+
|
5781 |
+
|
5782 |
+
|
5783 |
+
|
5784 |
+
9%|ββββββββββββββββββββββ | 5400/60000 [16:04:22<144:24:15, 9.52s/it]
|
5785 |
+
|
5786 |
+
|
5787 |
+
|
5788 |
+
|
5789 |
+
|
5790 |
+
|
5791 |
+
|
5792 |
+
|
5793 |
+
|
5794 |
+
|
5795 |
+
|
5796 |
+
|
5797 |
+
|
5798 |
+
|
5799 |
+
|
5800 |
+
|
5801 |
+
|
5802 |
+
|
5803 |
+
|
5804 |
+
|
5805 |
+
9%|ββββββββββββββββββββββ | 5420/60000 [16:07:50<157:42:27, 10.40s/it]
|
5806 |
+
|
5807 |
+
|
5808 |
+
|
5809 |
+
|
5810 |
+
|
5811 |
+
|
5812 |
+
|
5813 |
+
|
5814 |
+
|
5815 |
+
|
5816 |
+
|
5817 |
+
|
5818 |
+
|
5819 |
+
|
5820 |
+
|
5821 |
+
|
5822 |
+
|
5823 |
+
|
5824 |
+
|
5825 |
+
|
5826 |
+
9%|ββββββββββββββββββββββ | 5440/60000 [16:11:02<156:41:23, 10.34s/it]
|
5827 |
+
|
5828 |
+
|
5829 |
+
|
5830 |
+
|
5831 |
+
|
5832 |
+
|
5833 |
+
|
5834 |
+
|
5835 |
+
|
5836 |
+
|
5837 |
+
|
5838 |
+
|
5839 |
+
|
5840 |
+
|
5841 |
+
|
5842 |
+
|
5843 |
+
|
5844 |
+
|
5845 |
+
|
5846 |
+
|
5847 |
+
9%|ββββββββββββββββββββββ | 5460/60000 [16:14:06<140:40:02, 9.28s/it]
|
5848 |
+
|
5849 |
+
|
5850 |
+
|
5851 |
+
|
5852 |
+
|
5853 |
+
|
5854 |
+
|
5855 |
+
|
5856 |
+
|
5857 |
+
|
5858 |
+
|
5859 |
+
|
5860 |
+
|
5861 |
+
|
5862 |
+
|
5863 |
+
|
5864 |
+
|
5865 |
+
|
5866 |
+
|
5867 |
+
|
5868 |
+
9%|ββββββββββββββββββββββ | 5480/60000 [16:17:07<135:30:01, 8.95s/it]
|
5869 |
+
|
5870 |
+
|
5871 |
+
|
5872 |
+
|
5873 |
+
|
5874 |
+
|
5875 |
+
|
5876 |
+
|
5877 |
+
|
5878 |
+
|
5879 |
+
|
5880 |
+
|
5881 |
+
|
5882 |
+
|
5883 |
+
|
5884 |
+
|
5885 |
+
|
5886 |
+
|
5887 |
+
|
5888 |
+
|
5889 |
+
9%|ββββββββββββββββββββββ | 5500/60000 [16:20:12<139:29:05, 9.21s/it]
|
5890 |
+
|
5891 |
+
|
5892 |
+
|
5893 |
+
|
5894 |
+
|
5895 |
+
|
5896 |
+
|
5897 |
+
|
5898 |
+
|
5899 |
+
|
5900 |
+
|
5901 |
+
|
5902 |
+
|
5903 |
+
|
5904 |
+
|
5905 |
+
|
5906 |
+
|
5907 |
+
|
5908 |
+
|
5909 |
+
|
5910 |
+
9%|ββββββββββββββββββββββ | 5520/60000 [16:23:17<139:41:46, 9.23s/it]
|
5911 |
+
|
5912 |
+
|
5913 |
+
|
5914 |
+
|
5915 |
+
|
5916 |
+
|
5917 |
+
|
5918 |
+
|
5919 |
+
|
5920 |
+
|
5921 |
+
|
5922 |
+
|
5923 |
+
|
5924 |
+
|
5925 |
+
|
5926 |
+
|
5927 |
+
|
5928 |
+
|
5929 |
+
|
5930 |
+
9%|ββββββββββββββββββββββ | 5539/60000 [16:26:15<138:57:22, 9.19s/it]
|
5931 |
+
|
5932 |
+
|
5933 |
+
|
5934 |
+
|
5935 |
+
|
5936 |
+
|
5937 |
+
|
5938 |
+
|
5939 |
+
|
5940 |
+
|
5941 |
+
|
5942 |
+
|
5943 |
+
|
5944 |
+
|
5945 |
+
|
5946 |
+
|
5947 |
+
|
5948 |
+
|
5949 |
+
|
5950 |
+
|
5951 |
+
|
5952 |
+
9%|ββββββββββββββββββββββ | 5560/60000 [16:29:26<136:37:37, 9.03s/it]
|
5953 |
+
|
5954 |
+
|
5955 |
+
|
5956 |
+
|
5957 |
+
|
5958 |
+
|
5959 |
+
|
5960 |
+
|
5961 |
+
|
5962 |
+
|
5963 |
+
|
5964 |
+
|
5965 |
+
|
5966 |
+
|
5967 |
+
|
5968 |
+
|
5969 |
+
|
5970 |
+
|
5971 |
+
|
5972 |
+
|
5973 |
+
9%|ββββββββββββββββββββββ | 5580/60000 [16:32:26<135:34:52, 8.97s/it]
|
5974 |
+
|
5975 |
+
|
5976 |
+
|
5977 |
+
|
5978 |
+
|
5979 |
+
|
5980 |
+
|
5981 |
+
|
5982 |
+
|
5983 |
+
|
5984 |
+
|
5985 |
+
|
5986 |
+
|
5987 |
+
|
5988 |
+
|
5989 |
+
|
5990 |
+
|
5991 |
+
|
5992 |
+
|
5993 |
+
|
5994 |
+
9%|ββββββββββββββββββββββ | 5600/60000 [16:35:27<136:14:29, 9.02s/it]
|
5995 |
+
|
5996 |
+
|
5997 |
+
|
5998 |
+
|
5999 |
+
|
6000 |
+
|
6001 |
+
|
6002 |
+
|
6003 |
+
|
6004 |
+
|
6005 |
+
|
6006 |
+
|
6007 |
+
|
6008 |
+
|
6009 |
+
|
6010 |
+
|
6011 |
+
|
6012 |
+
|
6013 |
+
|
6014 |
+
9%|ββββββββββββββββββββββ | 5619/60000 [16:38:24<144:47:43, 9.59s/it]
|
6015 |
+
|
6016 |
+
|
6017 |
+
|
6018 |
+
|
6019 |
+
|
6020 |
+
|
6021 |
+
|
6022 |
+
|
6023 |
+
|
6024 |
+
|
6025 |
+
|
6026 |
+
|
6027 |
+
|
6028 |
+
|
6029 |
+
|
6030 |
+
|
6031 |
+
|
6032 |
+
|
6033 |
+
|
6034 |
+
|
6035 |
+
9%|βββββββββββββββββββββββ | 5639/60000 [16:41:44<142:26:22, 9.43s/it]
|
6036 |
+
|
6037 |
+
|
6038 |
+
|
6039 |
+
|
6040 |
+
|
6041 |
+
|
6042 |
+
|
6043 |
+
|
6044 |
+
|
6045 |
+
|
6046 |
+
|
6047 |
+
|
6048 |
+
|
6049 |
+
|
6050 |
+
|
6051 |
+
|
6052 |
+
|
6053 |
+
|
6054 |
+
|
6055 |
+
|
6056 |
+
|
6057 |
+
9%|βββββββββββββββββββββββ | 5660/60000 [16:44:55<136:47:25, 9.06s/it]
|
6058 |
+
|
6059 |
+
|
6060 |
+
|
6061 |
+
|
6062 |
+
|
6063 |
+
|
6064 |
+
|
6065 |
+
|
6066 |
+
|
6067 |
+
|
6068 |
+
|
6069 |
+
|
6070 |
+
|
6071 |
+
|
6072 |
+
|
6073 |
+
|
6074 |
+
|
6075 |
+
|
6076 |
+
|
6077 |
+
|
6078 |
+
9%|βββββββββββββββββββββββ | 5680/60000 [16:48:15<161:38:48, 10.71s/it]
|
6079 |
+
|
6080 |
+
|
6081 |
+
|
6082 |
+
|
6083 |
+
|
6084 |
+
|
6085 |
+
|
6086 |
+
|
6087 |
+
|
6088 |
+
|
6089 |
+
|
6090 |
+
|
6091 |
+
|
6092 |
+
|
6093 |
+
|
6094 |
+
|
6095 |
+
|
6096 |
+
|
6097 |
+
|
6098 |
+
9%|βββββββββββββββββββββββ | 5699/60000 [16:51:10<139:03:35, 9.22s/it]
|
6099 |
+
|
6100 |
+
|
6101 |
+
|
6102 |
+
|
6103 |
+
|
6104 |
+
|
6105 |
+
|
6106 |
+
|
6107 |
+
|
6108 |
+
|
6109 |
+
|
6110 |
+
|
6111 |
+
|
6112 |
+
|
6113 |
+
|
6114 |
+
|
6115 |
+
|
6116 |
+
|
6117 |
+
|
6118 |
+
|
6119 |
+
|
6120 |
+
10%|βββββββββββββββββββββββ | 5720/60000 [16:54:27<135:50:07, 9.01s/it]
|
6121 |
+
|
6122 |
+
|
6123 |
+
|
6124 |
+
|
6125 |
+
|
6126 |
+
|
6127 |
+
|
6128 |
+
|
6129 |
+
|
6130 |
+
|
6131 |
+
|
6132 |
+
|
6133 |
+
|
6134 |
+
|
6135 |
+
|
6136 |
+
|
6137 |
+
|
6138 |
+
|
6139 |
+
|
6140 |
+
|
6141 |
+
10%|βββββββββββββββββββββββ | 5740/60000 [16:57:26<136:37:25, 9.06s/it]
|
6142 |
+
|
6143 |
+
|
6144 |
+
|
6145 |
+
|
6146 |
+
|
6147 |
+
|
6148 |
+
|
6149 |
+
|
6150 |
+
|
6151 |
+
|
6152 |
+
|
6153 |
+
|
6154 |
+
|
6155 |
+
|
6156 |
+
|
6157 |
+
|
6158 |
+
|
6159 |
+
|
6160 |
+
|
6161 |
+
|
6162 |
+
10%|βββββββββββββββββββββββ | 5760/60000 [17:00:30<139:16:32, 9.24s/it]
|
6163 |
+
|
6164 |
+
|
6165 |
+
|
6166 |
+
|
6167 |
+
|
6168 |
+
|
6169 |
+
|
6170 |
+
|
6171 |
+
|
6172 |
+
|
6173 |
+
|
6174 |
+
|
6175 |
+
|
6176 |
+
|
6177 |
+
|
6178 |
+
|
6179 |
+
|
6180 |
+
|
6181 |
+
|
6182 |
+
|
6183 |
+
10%|βββββββββββββββββββββββ | 5780/60000 [17:03:34<138:17:50, 9.18s/it]
|
6184 |
+
|
6185 |
+
|
6186 |
+
|
6187 |
+
|
6188 |
+
|
6189 |
+
|
6190 |
+
|
6191 |
+
|
6192 |
+
|
6193 |
+
|
6194 |
+
|
6195 |
+
|
6196 |
+
|
6197 |
+
|
6198 |
+
|
6199 |
+
|
6200 |
+
|
6201 |
+
|
6202 |
+
|
6203 |
+
|
6204 |
+
10%|βββββββββββββββββββββββ | 5800/60000 [17:06:42<138:04:03, 9.17s/it]
|
6205 |
+
|
6206 |
+
|
6207 |
+
|
6208 |
+
|
6209 |
+
|
6210 |
+
|
6211 |
+
|
6212 |
+
|
6213 |
+
|
6214 |
+
|
6215 |
+
|
6216 |
+
|
6217 |
+
|
6218 |
+
|
6219 |
+
|
6220 |
+
|
6221 |
+
|
6222 |
+
|
6223 |
+
|
6224 |
+
|
6225 |
+
10%|βββββββββββββββββββββββ | 5820/60000 [17:09:43<135:29:11, 9.00s/it]
|
6226 |
+
|
6227 |
+
|
6228 |
+
|
6229 |
+
|
6230 |
+
|
6231 |
+
|
6232 |
+
|
6233 |
+
|
6234 |
+
|
6235 |
+
|
6236 |
+
|
6237 |
+
|
6238 |
+
|
6239 |
+
|
6240 |
+
|
6241 |
+
|
6242 |
+
|
6243 |
+
|
6244 |
+
|
6245 |
+
10%|βββββββββββββββββββββββ | 5839/60000 [17:12:37<136:53:55, 9.10s/it]
|
6246 |
+
|
6247 |
+
|
6248 |
+
|
6249 |
+
|
6250 |
+
|
6251 |
+
|
6252 |
+
|
6253 |
+
|
6254 |
+
|
6255 |
+
|
6256 |
+
|
6257 |
+
|
6258 |
+
|
6259 |
+
|
6260 |
+
|
6261 |
+
|
6262 |
+
|
6263 |
+
|
6264 |
+
|
6265 |
+
|
6266 |
+
10%|βββββββββββββββββββββββ | 5859/60000 [17:16:20<154:49:23, 10.29s/it]
|
6267 |
+
|
6268 |
+
|
6269 |
+
|
6270 |
+
|
6271 |
+
|
6272 |
+
|
6273 |
+
|
6274 |
+
|
6275 |
+
|
6276 |
+
|
6277 |
+
|
6278 |
+
|
6279 |
+
|
6280 |
+
|
6281 |
+
|
6282 |
+
|
6283 |
+
|
6284 |
+
|
6285 |
+
|
6286 |
+
|
6287 |
+
10%|βββββββββββββββββββββββ | 5879/60000 [17:19:28<145:28:07, 9.68s/it]
|
6288 |
+
|
6289 |
+
|
6290 |
+
|
6291 |
+
|
6292 |
+
|
6293 |
+
|
6294 |
+
|
6295 |
+
|
6296 |
+
|
6297 |
+
|
6298 |
+
|
6299 |
+
|
6300 |
+
|
6301 |
+
|
6302 |
+
|
6303 |
+
|
6304 |
+
|
6305 |
+
|
6306 |
+
|
6307 |
+
|
6308 |
+
10%|ββββββββββββββββββββββββ | 5899/60000 [17:22:36<137:05:28, 9.12s/it]
|
6309 |
+
|
6310 |
+
|
6311 |
+
|
6312 |
+
|
6313 |
+
|
6314 |
+
|
6315 |
+
|
6316 |
+
|
6317 |
+
|
6318 |
+
|
6319 |
+
Reading metadata...: 1650it [00:00, 10583.89it/s] | 5910/60000 [17:24:15<135:54:39, 9.05s/it]
|
6320 |
+
|
6321 |
+
|
6322 |
+
|
6323 |
+
|
6324 |
+
|
6325 |
+
|
6326 |
+
|
6327 |
+
|
6328 |
+
|
6329 |
+
10%|ββββββββββββββββββββββββ | 5919/60000 [17:25:40<138:56:39, 9.25s/it]
|
6330 |
+
|
6331 |
+
|
6332 |
+
|
6333 |
+
|
6334 |
+
|
6335 |
+
|
6336 |
+
|
6337 |
+
|
6338 |
+
|
6339 |
+
|
6340 |
+
|
6341 |
+
|
6342 |
+
|
6343 |
+
|
6344 |
+
|
6345 |
+
|
6346 |
+
|
6347 |
+
|
6348 |
+
|
6349 |
+
|
6350 |
+
10%|ββββββββββββββββββββββββ | 5939/60000 [17:28:43<138:23:48, 9.22s/it]
|
6351 |
+
|
6352 |
+
|
6353 |
+
|
6354 |
+
|
6355 |
+
|
6356 |
+
|
6357 |
+
|
6358 |
+
|
6359 |
+
|
6360 |
+
|
6361 |
+
|
6362 |
+
|
6363 |
+
|
6364 |
+
|
6365 |
+
|
6366 |
+
|
6367 |
+
|
6368 |
+
|
6369 |
+
|
6370 |
+
|
6371 |
+
10%|ββββββββββββββββββββββββ | 5959/60000 [17:31:44<134:51:49, 8.98s/it]
|
6372 |
+
|
6373 |
+
|
6374 |
+
|
6375 |
+
|
6376 |
+
|
6377 |
+
|
6378 |
+
|
6379 |
+
|
6380 |
+
|
6381 |
+
|
6382 |
+
|
6383 |
+
|
6384 |
+
|
6385 |
+
|
6386 |
+
|
6387 |
+
|
6388 |
+
|
6389 |
+
|
6390 |
+
|
6391 |
+
|
6392 |
+
10%|ββββββββββββββββββββββββ | 5979/60000 [17:35:03<136:08:23, 9.07s/it]
|
6393 |
+
|
6394 |
+
|
6395 |
+
|
6396 |
+
|
6397 |
+
|
6398 |
+
|
6399 |
+
|
6400 |
+
|
6401 |
+
|
6402 |
+
|
6403 |
+
|
6404 |
+
|
6405 |
+
|
6406 |
+
|
6407 |
+
|
6408 |
+
|
6409 |
+
|
6410 |
+
|
6411 |
+
|
6412 |
+
|
6413 |
+
10%|ββββββββββββββββββββββββ | 6000/60000 [17:38:28<137:45:52, 9.18s/it][INFO|trainer.py:3173] 2023-11-19 06:58:51,907 >> ***** Running Evaluation *****
|
6414 |
+
[INFO|trainer.py:3177] 2023-11-19 06:58:51,911 >> Num examples: Unknown
|
6415 |
+
[INFO|trainer.py:3178] 2023-11-19 06:58:51,911 >> Batch size = 4
|
6416 |
+
Reading metadata...: 1704it [00:00, 9946.01it/s]
|
6417 |
+
Reading metadata...: 1it [00:00, 6.69it/s]
|
6418 |
+
|
6419 |
+
[INFO|trainer_utils.py:759] 2023-11-19 06:58:52,895 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message.
|
6420 |
+
{'eval_loss': 0.1368408203125, 'eval_wer': 8.876743309460988, 'eval_runtime': 591.8673, 'eval_samples_per_second': 2.879, 'eval_steps_per_second': 0.72, 'epoch': 0.1}
|
6421 |
+
10%|ββββββββββββββββββββββββ | 6000/60000 [17:48:20<137:45:52, 9.18s/it][INFO|trainer.py:2896] 2023-11-19 07:09:16,613 >> Saving model checkpoint to ./checkpoint-6000
|
6422 |
+
[INFO|configuration_utils.py:462] 2023-11-19 07:09:16,627 >> Configuration saved in ./checkpoint-6000/config.json
|
6423 |
+
[INFO|configuration_utils.py:568] 2023-11-19 07:09:16,632 >> Configuration saved in ./checkpoint-6000/generation_config.json
|
6424 |
+
[2023-11-19 07:10:00,067] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step6000 is about to be saved!
|
6425 |
+
[2023-11-19 07:10:00,107] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt
|
6426 |
+
[2023-11-19 07:10:00,108] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt...
|
6427 |
+
[INFO|modeling_utils.py:2194] 2023-11-19 07:10:00,022 >> Model weights saved in ./checkpoint-6000/pytorch_model.bin
|
6428 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 07:10:00,031 >> Feature extractor saved in ./checkpoint-6000/preprocessor_config.json
|
6429 |
+
[2023-11-19 07:10:08,388] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt.
|
6430 |
+
[2023-11-19 07:10:08,398] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
|
6431 |
+
[2023-11-19 07:10:29,571] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
|
6432 |
+
[2023-11-19 07:10:29,584] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt
|
6433 |
+
[2023-11-19 07:10:29,584] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step6000 is ready now!
|
6434 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-19 07:11:26,721 >> Feature extractor saved in ./preprocessor_config.json
|
6435 |
+
|
6436 |
+
|
6437 |
+
|
6438 |
+
|
6439 |
+
|
6440 |
+
|
6441 |
+
|
6442 |
+
|
6443 |
+
|
6444 |
+
|
6445 |
+
|
6446 |
+
|
6447 |
+
|
6448 |
+
|
6449 |
+
|
6450 |
+
|
6451 |
+
|
6452 |
+
|
6453 |
+
10%|ββββββββββββββββββββββββ | 6019/60000 [17:54:07<147:13:43, 9.82s/it]
|
6454 |
+
|
6455 |
+
|
6456 |
+
|
6457 |
+
|
6458 |
+
|
6459 |
+
|
6460 |
+
|
6461 |
+
|
6462 |
+
|
6463 |
+
|
6464 |
+
|
6465 |
+
|
6466 |
+
|
6467 |
+
|
6468 |
+
|
6469 |
+
|
6470 |
+
|
6471 |
+
|
6472 |
+
|
6473 |
+
|
6474 |
+
10%|ββββββββββββββββββββββββ | 6039/60000 [17:57:18<140:10:10, 9.35s/it]
|
6475 |
+
|
6476 |
+
|
6477 |
+
|
6478 |
+
|
6479 |
+
|
6480 |
+
|
6481 |
+
|
6482 |
+
|
6483 |
+
|
6484 |
+
|
6485 |
+
|
6486 |
+
|
6487 |
+
|
6488 |
+
|
6489 |
+
|
6490 |
+
|
6491 |
+
|
6492 |
+
|
6493 |
+
|
6494 |
+
|
6495 |
+
|
6496 |
+
10%|ββββββββββββββββββββββββ | 6060/60000 [18:00:41<148:15:47, 9.90s/it]
|
6497 |
+
|
6498 |
+
|
6499 |
+
|
6500 |
+
|
6501 |
+
|
6502 |
+
|
6503 |
+
|
6504 |
+
|
6505 |
+
|
6506 |
+
|
6507 |
+
|
wandb/run-20231118_132020-qg88991p/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"train/loss": 0.
|
|
|
1 |
+
{"train/loss": 0.0959, "train/learning_rate": 2.7432711864406777e-06, "train/epoch": 0.1, "train/global_step": 6060, "_timestamp": 1700371265.4985242, "_runtime": 64844.58343410492, "_step": 308, "eval/loss": 0.1368408203125, "eval/wer": 8.876743309460988, "eval/runtime": 591.8673, "eval/samples_per_second": 2.879, "eval/steps_per_second": 0.72}
|
wandb/run-20231118_132020-qg88991p/logs/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:45abb02b364f3abe2253240c15f9ce34b371ded1f82652adbb3b0ae2cfbbceb2
|
3 |
+
size 5049554
|