diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,175086 @@ + > Using CUDA: True + > Number of GPUs: 1 + + > Model has 28597969 parameters + + > EPOCH: 0/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 08:17:53)  + + --> STEP: 0/234 -- GLOBAL_STEP: 0 + | > current_lr: 0.00000 + | > step_time: 12.12650 (12.12650) + | > loader_time: 22.68740 (22.68738) + + + --> STEP: 5/234 -- GLOBAL_STEP: 5 + | > current_lr: 0.00000 + | > step_time: 2.68570 (3.99518) + | > loader_time: 0.36770 (0.09663) + + + --> STEP: 10/234 -- GLOBAL_STEP: 10 + | > loss: 3.89252 (3.89252) + | > log_mle: 0.98751 (0.98751) + | > loss_dur: 2.90501 (2.90501) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 0.00000 (0.00000) + | > current_lr: 0.00000 + | > step_time: 15.49750 (5.03919) + | > loader_time: 0.01310 (0.08928) + + + --> STEP: 15/234 -- GLOBAL_STEP: 15 + | > loss: 3.80741 (3.88777) + | > log_mle: 0.98324 (0.97882) + | > loss_dur: 2.82416 (2.90895) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.09069 (8.14044) + | > current_lr: 0.00000 + | > step_time: 1.72030 (4.84132) + | > loader_time: 0.06970 (0.07550) + + + --> STEP: 20/234 -- GLOBAL_STEP: 20 + | > loss: 3.79023 (3.88042) + | > log_mle: 0.96544 (0.97602) + | > loss_dur: 2.82479 (2.90440) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.14864 (9.23894) + | > current_lr: 0.00000 + | > step_time: 3.71410 (4.79257) + | > loader_time: 0.08250 (0.07422) + + + --> STEP: 25/234 -- GLOBAL_STEP: 25 + | > loss: 3.62421 (3.83168) + | > log_mle: 0.98632 (0.97616) + | > loss_dur: 2.63789 (2.85552) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.92090 (9.53036) + | > current_lr: 0.00000 + | > step_time: 7.70580 (5.33115) + | > loader_time: 0.18470 (0.07812) + + + --> STEP: 30/234 -- GLOBAL_STEP: 30 + | > loss: 3.65033 (3.80186) + | > log_mle: 0.96006 (0.97499) + | > loss_dur: 2.69027 (2.82688) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.10343 (9.69570) + | > current_lr: 0.00000 + | > step_time: 3.71830 (5.08438) + | > loader_time: 0.06840 (0.07302) + + + --> STEP: 35/234 -- GLOBAL_STEP: 35 + | > loss: 3.75461 (3.78836) + | > log_mle: 0.99428 (0.97451) + | > loss_dur: 2.76033 (2.81385) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.29777 (9.81111) + | > current_lr: 0.00000 + | > step_time: 4.01600 (5.01052) + | > loader_time: 0.09090 (0.07281) + + + --> STEP: 40/234 -- GLOBAL_STEP: 40 + | > loss: 3.66123 (3.77675) + | > log_mle: 0.98385 (0.97522) + | > loss_dur: 2.67738 (2.80153) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.03661 (9.88348) + | > current_lr: 0.00000 + | > step_time: 4.12150 (4.81746) + | > loader_time: 0.07870 (0.07092) + + + --> STEP: 45/234 -- GLOBAL_STEP: 45 + | > loss: 3.74496 (3.77078) + | > log_mle: 0.99601 (0.97622) + | > loss_dur: 2.74895 (2.79456) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.24856 (9.94610) + | > current_lr: 0.00000 + | > step_time: 2.34760 (4.69469) + | > loader_time: 0.00180 (0.06354) + + + --> STEP: 50/234 -- GLOBAL_STEP: 50 + | > loss: 3.65161 (3.74864) + | > log_mle: 0.96929 (0.97568) + | > loss_dur: 2.68232 (2.77296) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.24362 (9.96563) + | > current_lr: 0.00000 + | > step_time: 4.83070 (4.66013) + | > loader_time: 0.00220 (0.05931) + + + --> STEP: 55/234 -- GLOBAL_STEP: 55 + | > loss: 3.66479 (3.74145) + | > log_mle: 0.95748 (0.97528) + | > loss_dur: 2.70731 (2.76617) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.31771 (10.00253) + | > current_lr: 0.00000 + | > step_time: 5.71660 (4.60215) + | > loader_time: 0.07710 (0.05736) + + + --> STEP: 60/234 -- GLOBAL_STEP: 60 + | > loss: 3.61837 (3.73025) + | > log_mle: 0.96880 (0.97505) + | > loss_dur: 2.64957 (2.75520) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.17410 (10.02244) + | > current_lr: 0.00000 + | > step_time: 8.67650 (4.59970) + | > loader_time: 0.02310 (0.05847) + + + --> STEP: 65/234 -- GLOBAL_STEP: 65 + | > loss: 3.56958 (3.71938) + | > log_mle: 0.96754 (0.97402) + | > loss_dur: 2.60204 (2.74536) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.10173 (10.03388) + | > current_lr: 0.00000 + | > step_time: 2.18210 (4.51209) + | > loader_time: 0.00230 (0.05681) + + + --> STEP: 70/234 -- GLOBAL_STEP: 70 + | > loss: 3.59043 (3.71043) + | > log_mle: 0.95725 (0.97347) + | > loss_dur: 2.63318 (2.73695) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.24414 (10.04943) + | > current_lr: 0.00000 + | > step_time: 4.48890 (4.53161) + | > loader_time: 0.00210 (0.05423) + + + --> STEP: 75/234 -- GLOBAL_STEP: 75 + | > loss: 3.66206 (3.70179) + | > log_mle: 0.97437 (0.97336) + | > loss_dur: 2.68769 (2.72843) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.32133 (10.05898) + | > current_lr: 0.00000 + | > step_time: 1.37230 (4.48216) + | > loader_time: 0.00460 (0.05318) + + + --> STEP: 80/234 -- GLOBAL_STEP: 80 + | > loss: 3.56297 (3.69493) + | > log_mle: 0.97041 (0.97317) + | > loss_dur: 2.59257 (2.72176) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.14065 (10.07204) + | > current_lr: 0.00000 + | > step_time: 4.57680 (4.45310) + | > loader_time: 0.10030 (0.05701) + + + --> STEP: 85/234 -- GLOBAL_STEP: 85 + | > loss: 3.67567 (3.69054) + | > log_mle: 0.96974 (0.97281) + | > loss_dur: 2.70593 (2.71774) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.38055 (10.08566) + | > current_lr: 0.00000 + | > step_time: 3.61690 (4.39762) + | > loader_time: 0.08370 (0.05906) + + + --> STEP: 90/234 -- GLOBAL_STEP: 90 + | > loss: 3.53352 (3.68429) + | > log_mle: 0.97429 (0.97250) + | > loss_dur: 2.55922 (2.71179) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.18016 (10.09453) + | > current_lr: 0.00000 + | > step_time: 4.21120 (4.37698) + | > loader_time: 0.09530 (0.06294) + + + --> STEP: 95/234 -- GLOBAL_STEP: 95 + | > loss: 3.70433 (3.67936) + | > log_mle: 0.96200 (0.97201) + | > loss_dur: 2.74233 (2.70735) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.50799 (10.10311) + | > current_lr: 0.00000 + | > step_time: 5.71340 (4.36966) + | > loader_time: 0.08330 (0.06247) + + + --> STEP: 100/234 -- GLOBAL_STEP: 100 + | > loss: 3.70095 (3.67701) + | > log_mle: 0.96127 (0.97145) + | > loss_dur: 2.73968 (2.70556) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.51023 (10.11552) + | > current_lr: 0.00000 + | > step_time: 1.60450 (4.36639) + | > loader_time: 0.07830 (0.06311) + + + --> STEP: 105/234 -- GLOBAL_STEP: 105 + | > loss: 3.56347 (3.67445) + | > log_mle: 0.96341 (0.97127) + | > loss_dur: 2.60006 (2.70318) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.23943 (10.12643) + | > current_lr: 0.00000 + | > step_time: 6.37490 (4.33494) + | > loader_time: 0.01950 (0.06271) + + + --> STEP: 110/234 -- GLOBAL_STEP: 110 + | > loss: 3.67925 (3.67202) + | > log_mle: 0.97155 (0.97079) + | > loss_dur: 2.70770 (2.70123) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.36459 (10.13496) + | > current_lr: 0.00000 + | > step_time: 3.41930 (4.27813) + | > loader_time: 0.09830 (0.06442) + + + --> STEP: 115/234 -- GLOBAL_STEP: 115 + | > loss: 3.65084 (3.67121) + | > log_mle: 0.96439 (0.97080) + | > loss_dur: 2.68645 (2.70042) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.32566 (10.14546) + | > current_lr: 0.00000 + | > step_time: 5.31890 (4.31452) + | > loader_time: 0.09320 (0.06510) + + + --> STEP: 120/234 -- GLOBAL_STEP: 120 + | > loss: 3.67438 (3.67006) + | > log_mle: 0.95586 (0.97074) + | > loss_dur: 2.71852 (2.69933) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.46029 (10.15606) + | > current_lr: 0.00000 + | > step_time: 3.59370 (4.26644) + | > loader_time: 0.08420 (0.06529) + + + --> STEP: 125/234 -- GLOBAL_STEP: 125 + | > loss: 3.52294 (3.66452) + | > log_mle: 0.95841 (0.97065) + | > loss_dur: 2.56454 (2.69386) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.13009 (10.15816) + | > current_lr: 0.00000 + | > step_time: 4.29050 (4.24147) + | > loader_time: 0.00790 (0.06490) + + + --> STEP: 130/234 -- GLOBAL_STEP: 130 + | > loss: 3.59780 (3.66259) + | > log_mle: 0.95838 (0.97054) + | > loss_dur: 2.63942 (2.69205) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.24600 (10.16451) + | > current_lr: 0.00000 + | > step_time: 3.49520 (4.24548) + | > loader_time: 0.01140 (0.06265) + + + --> STEP: 135/234 -- GLOBAL_STEP: 135 + | > loss: 3.53816 (3.66060) + | > log_mle: 0.96425 (0.97054) + | > loss_dur: 2.57391 (2.69007) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.16919 (10.16975) + | > current_lr: 0.00000 + | > step_time: 6.70020 (4.28381) + | > loader_time: 0.20240 (0.06552) + + + --> STEP: 140/234 -- GLOBAL_STEP: 140 + | > loss: 3.56085 (3.65959) + | > log_mle: 0.97888 (0.97058) + | > loss_dur: 2.58197 (2.68901) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.16654 (10.17514) + | > current_lr: 0.00000 + | > step_time: 4.80260 (4.27210) + | > loader_time: 0.10340 (0.06678) + + + --> STEP: 145/234 -- GLOBAL_STEP: 145 + | > loss: 3.69205 (3.66029) + | > log_mle: 0.96380 (0.97063) + | > loss_dur: 2.72825 (2.68966) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.44256 (10.18533) + | > current_lr: 0.00000 + | > step_time: 9.01430 (4.34667) + | > loader_time: 0.08780 (0.06842) + + + --> STEP: 150/234 -- GLOBAL_STEP: 150 + | > loss: 3.67340 (3.65886) + | > log_mle: 0.96600 (0.97054) + | > loss_dur: 2.70740 (2.68833) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.38890 (10.19069) + | > current_lr: 0.00000 + | > step_time: 2.28290 (4.35371) + | > loader_time: 0.04250 (0.06774) + + + --> STEP: 155/234 -- GLOBAL_STEP: 155 + | > loss: 3.75594 (3.65907) + | > log_mle: 0.97315 (0.97050) + | > loss_dur: 2.78279 (2.68857) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.59728 (10.19852) + | > current_lr: 0.00000 + | > step_time: 8.40050 (4.36727) + | > loader_time: 0.08570 (0.06752) + + + --> STEP: 160/234 -- GLOBAL_STEP: 160 + | > loss: 3.62411 (3.65864) + | > log_mle: 0.96071 (0.97040) + | > loss_dur: 2.66339 (2.68825) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.36533 (10.20408) + | > current_lr: 0.00000 + | > step_time: 6.71070 (4.41646) + | > loader_time: 0.00500 (0.06900) + + + --> STEP: 165/234 -- GLOBAL_STEP: 165 + | > loss: 3.66169 (3.65842) + | > log_mle: 0.97288 (0.97030) + | > loss_dur: 2.68881 (2.68812) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.38824 (10.21008) + | > current_lr: 0.00000 + | > step_time: 3.00730 (4.46577) + | > loader_time: 0.09580 (0.07033) + + + --> STEP: 170/234 -- GLOBAL_STEP: 170 + | > loss: 3.74552 (3.66127) + | > log_mle: 0.97567 (0.97035) + | > loss_dur: 2.76985 (2.69092) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.54268 (10.22192) + | > current_lr: 0.00000 + | > step_time: 4.81210 (4.48051) + | > loader_time: 0.00500 (0.07000) + + + --> STEP: 175/234 -- GLOBAL_STEP: 175 + | > loss: 3.71189 (3.66269) + | > log_mle: 0.97022 (0.97049) + | > loss_dur: 2.74168 (2.69220) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.53447 (10.23027) + | > current_lr: 0.00000 + | > step_time: 7.97450 (4.52281) + | > loader_time: 0.09600 (0.07297) + + + --> STEP: 180/234 -- GLOBAL_STEP: 180 + | > loss: 3.71202 (3.66374) + | > log_mle: 0.97466 (0.97059) + | > loss_dur: 2.73735 (2.69316) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.50088 (10.23756) + | > current_lr: 0.00000 + | > step_time: 4.10850 (4.53548) + | > loader_time: 0.01170 (0.08082) + + + --> STEP: 185/234 -- GLOBAL_STEP: 185 + | > loss: 3.74099 (3.66528) + | > log_mle: 0.98614 (0.97074) + | > loss_dur: 2.75485 (2.69454) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.51828 (10.24570) + | > current_lr: 0.00000 + | > step_time: 4.19990 (4.59316) + | > loader_time: 0.09980 (0.08243) + + + --> STEP: 190/234 -- GLOBAL_STEP: 190 + | > loss: 3.74139 (3.66583) + | > log_mle: 0.96757 (0.97078) + | > loss_dur: 2.77382 (2.69505) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.67037 (10.25165) + | > current_lr: 0.00000 + | > step_time: 4.40930 (4.56925) + | > loader_time: 0.09540 (0.08189) + + + --> STEP: 195/234 -- GLOBAL_STEP: 195 + | > loss: 3.76995 (3.66721) + | > log_mle: 0.98740 (0.97102) + | > loss_dur: 2.78255 (2.69619) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.61740 (10.25908) + | > current_lr: 0.00000 + | > step_time: 3.39960 (4.57785) + | > loader_time: 0.10030 (0.08196) + + + --> STEP: 200/234 -- GLOBAL_STEP: 200 + | > loss: 3.69004 (3.66780) + | > log_mle: 0.97625 (0.97120) + | > loss_dur: 2.71380 (2.69660) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.44282 (10.26441) + | > current_lr: 0.00000 + | > step_time: 8.29260 (4.62730) + | > loader_time: 0.59840 (0.09167) + + + --> STEP: 205/234 -- GLOBAL_STEP: 205 + | > loss: 3.76238 (3.66807) + | > log_mle: 0.97884 (0.97116) + | > loss_dur: 2.78353 (2.69691) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.63696 (10.26939) + | > current_lr: 0.00000 + | > step_time: 11.19510 (4.70017) + | > loader_time: 0.09940 (0.09293) + + + --> STEP: 210/234 -- GLOBAL_STEP: 210 + | > loss: 3.74280 (3.66931) + | > log_mle: 0.97459 (0.97122) + | > loss_dur: 2.76821 (2.69809) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.61458 (10.27612) + | > current_lr: 0.00000 + | > step_time: 3.89900 (4.72778) + | > loader_time: 0.10200 (0.09350) + + + --> STEP: 215/234 -- GLOBAL_STEP: 215 + | > loss: 3.68990 (3.67123) + | > log_mle: 0.97842 (0.97126) + | > loss_dur: 2.71148 (2.69997) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.47798 (10.28435) + | > current_lr: 0.00000 + | > step_time: 4.72210 (4.72723) + | > loader_time: 0.07700 (0.09387) + + + --> STEP: 220/234 -- GLOBAL_STEP: 220 + | > loss: 3.79151 (3.67333) + | > log_mle: 0.97244 (0.97134) + | > loss_dur: 2.81907 (2.70199) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.72625 (10.29285) + | > current_lr: 0.00000 + | > step_time: 2.99970 (4.75163) + | > loader_time: 0.00730 (0.09361) + + + --> STEP: 225/234 -- GLOBAL_STEP: 225 + | > loss: 3.75772 (3.67558) + | > log_mle: 0.99050 (0.97146) + | > loss_dur: 2.76722 (2.70412) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.57346 (10.30107) + | > current_lr: 0.00000 + | > step_time: 0.33770 (4.66166) + | > loader_time: 0.00360 (0.09202) + + + --> STEP: 230/234 -- GLOBAL_STEP: 230 + | > loss: 4.00186 (3.67972) + | > log_mle: 0.97706 (0.97173) + | > loss_dur: 3.02479 (2.70800) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.08162 (10.31213) + | > current_lr: 0.00000 + | > step_time: 0.32310 (4.56903) + | > loader_time: 0.00400 (0.09011) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00340 (+0.00000) + | > avg_loss: 3.78599 (+0.00000) + | > avg_log_mle: 0.97523 (+0.00000) + | > avg_loss_dur: 2.81076 (+0.00000) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_234.pth + + > EPOCH: 1/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 08:36:59)  + + --> STEP: 1/234 -- GLOBAL_STEP: 235 + | > loss: 4.54797 (4.54797) + | > log_mle: 1.02217 (1.02217) + | > loss_dur: 3.52581 (3.52581) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.47464 (10.47464) + | > current_lr: 0.00000 + | > step_time: 2.89310 (2.89307) + | > loader_time: 0.01040 (0.01039) + + + --> STEP: 6/234 -- GLOBAL_STEP: 240 + | > loss: 4.22963 (4.37317) + | > log_mle: 0.99764 (1.01039) + | > loss_dur: 3.23199 (3.36278) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.39600 (10.39505) + | > current_lr: 0.00000 + | > step_time: 4.60740 (4.69125) + | > loader_time: 0.00420 (0.07832) + + + --> STEP: 11/234 -- GLOBAL_STEP: 245 + | > loss: 3.93222 (4.22207) + | > log_mle: 0.97651 (0.99486) + | > loss_dur: 2.95571 (3.22721) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.41586 (10.39186) + | > current_lr: 0.00000 + | > step_time: 5.69700 (4.86750) + | > loader_time: 0.00130 (0.11615) + + + --> STEP: 16/234 -- GLOBAL_STEP: 250 + | > loss: 3.65446 (4.09069) + | > log_mle: 0.96186 (0.98833) + | > loss_dur: 2.69260 (3.10236) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.86354 (10.27893) + | > current_lr: 0.00000 + | > step_time: 2.86210 (5.44376) + | > loader_time: 0.00170 (0.09746) + + + --> STEP: 21/234 -- GLOBAL_STEP: 255 + | > loss: 3.67940 (4.01678) + | > log_mle: 0.98498 (0.98536) + | > loss_dur: 2.69442 (3.03142) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.99676 (10.24216) + | > current_lr: 0.00000 + | > step_time: 3.12660 (5.21223) + | > loader_time: 0.00170 (0.08327) + + + --> STEP: 26/234 -- GLOBAL_STEP: 260 + | > loss: 3.68878 (3.95325) + | > log_mle: 0.97062 (0.98269) + | > loss_dur: 2.71816 (2.97055) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.22761 (10.20277) + | > current_lr: 0.00000 + | > step_time: 3.30040 (5.06381) + | > loader_time: 0.19300 (0.09676) + + + --> STEP: 31/234 -- GLOBAL_STEP: 265 + | > loss: 3.69974 (3.90312) + | > log_mle: 0.96883 (0.98049) + | > loss_dur: 2.73091 (2.92263) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.15373 (10.17175) + | > current_lr: 0.00000 + | > step_time: 2.39040 (4.87691) + | > loader_time: 0.09990 (0.09657) + + + --> STEP: 36/234 -- GLOBAL_STEP: 270 + | > loss: 3.74435 (3.87127) + | > log_mle: 0.96848 (0.97907) + | > loss_dur: 2.77587 (2.89220) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.37160 (10.17037) + | > current_lr: 0.00000 + | > step_time: 3.27340 (4.86239) + | > loader_time: 0.02340 (0.09020) + + + --> STEP: 41/234 -- GLOBAL_STEP: 275 + | > loss: 3.54510 (3.84209) + | > log_mle: 0.95597 (0.97845) + | > loss_dur: 2.58913 (2.86364) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.96366 (10.16058) + | > current_lr: 0.00000 + | > step_time: 5.69220 (4.81886) + | > loader_time: 0.01910 (0.09483) + + + --> STEP: 46/234 -- GLOBAL_STEP: 280 + | > loss: 3.57139 (3.82199) + | > log_mle: 0.98048 (0.97914) + | > loss_dur: 2.59091 (2.84286) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.05588 (10.15946) + | > current_lr: 0.00000 + | > step_time: 2.39470 (4.65434) + | > loader_time: 0.10040 (0.09069) + + + --> STEP: 51/234 -- GLOBAL_STEP: 285 + | > loss: 3.59734 (3.79561) + | > log_mle: 0.97548 (0.97809) + | > loss_dur: 2.62187 (2.81753) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.15595 (10.14616) + | > current_lr: 0.00000 + | > step_time: 4.89030 (4.46662) + | > loader_time: 0.07550 (0.08842) + + + --> STEP: 56/234 -- GLOBAL_STEP: 290 + | > loss: 3.64952 (3.78203) + | > log_mle: 0.97961 (0.97740) + | > loss_dur: 2.66990 (2.80463) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.17103 (10.15074) + | > current_lr: 0.00000 + | > step_time: 2.71750 (4.28044) + | > loader_time: 0.09110 (0.08551) + + + --> STEP: 61/234 -- GLOBAL_STEP: 295 + | > loss: 3.55838 (3.76493) + | > log_mle: 0.96257 (0.97655) + | > loss_dur: 2.59581 (2.78838) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.07468 (10.14420) + | > current_lr: 0.00000 + | > step_time: 2.38650 (4.25318) + | > loader_time: 0.09000 (0.08473) + + + --> STEP: 66/234 -- GLOBAL_STEP: 300 + | > loss: 3.55103 (3.75013) + | > log_mle: 0.95792 (0.97532) + | > loss_dur: 2.59311 (2.77481) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.14543 (10.14144) + | > current_lr: 0.00000 + | > step_time: 1.72200 (4.17658) + | > loader_time: 0.09050 (0.08229) + + + --> STEP: 71/234 -- GLOBAL_STEP: 305 + | > loss: 3.65205 (3.73664) + | > log_mle: 0.97534 (0.97483) + | > loss_dur: 2.67671 (2.76181) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.15074 (10.13312) + | > current_lr: 0.00000 + | > step_time: 7.00620 (4.17848) + | > loader_time: 0.10030 (0.08325) + + + --> STEP: 76/234 -- GLOBAL_STEP: 310 + | > loss: 3.56033 (3.72464) + | > log_mle: 0.97119 (0.97441) + | > loss_dur: 2.58913 (2.75023) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.06842 (10.13206) + | > current_lr: 0.00000 + | > step_time: 2.29620 (4.14700) + | > loader_time: 0.09760 (0.08158) + + + --> STEP: 81/234 -- GLOBAL_STEP: 315 + | > loss: 3.50744 (3.71225) + | > log_mle: 0.96302 (0.97391) + | > loss_dur: 2.54442 (2.73834) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.96769 (10.12743) + | > current_lr: 0.00000 + | > step_time: 1.56960 (4.07688) + | > loader_time: 0.59380 (0.08790) + + + --> STEP: 86/234 -- GLOBAL_STEP: 320 + | > loss: 3.62710 (3.70420) + | > log_mle: 0.97857 (0.97357) + | > loss_dur: 2.64853 (2.73063) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.19545 (10.12641) + | > current_lr: 0.00000 + | > step_time: 8.21690 (4.13533) + | > loader_time: 0.09880 (0.08638) + + + --> STEP: 91/234 -- GLOBAL_STEP: 325 + | > loss: 3.56991 (3.69484) + | > log_mle: 0.97207 (0.97303) + | > loss_dur: 2.59783 (2.72182) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.16259 (10.12550) + | > current_lr: 0.00000 + | > step_time: 1.99310 (4.10606) + | > loader_time: 0.10940 (0.08927) + + + --> STEP: 96/234 -- GLOBAL_STEP: 330 + | > loss: 3.53729 (3.68648) + | > log_mle: 0.96712 (0.97236) + | > loss_dur: 2.57017 (2.71412) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.03060 (10.12150) + | > current_lr: 0.00000 + | > step_time: 3.49380 (4.06002) + | > loader_time: 0.00840 (0.08818) + + + --> STEP: 101/234 -- GLOBAL_STEP: 335 + | > loss: 3.51805 (3.67956) + | > log_mle: 0.96794 (0.97171) + | > loss_dur: 2.55011 (2.70785) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.03348 (10.11964) + | > current_lr: 0.00000 + | > step_time: 1.69750 (4.03542) + | > loader_time: 0.10380 (0.08506) + + + --> STEP: 106/234 -- GLOBAL_STEP: 340 + | > loss: 3.70247 (3.67590) + | > log_mle: 0.95944 (0.97130) + | > loss_dur: 2.74303 (2.70459) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.47730 (10.12473) + | > current_lr: 0.00000 + | > step_time: 1.52500 (4.00673) + | > loader_time: 0.00380 (0.08407) + + + --> STEP: 111/234 -- GLOBAL_STEP: 345 + | > loss: 3.60388 (3.67013) + | > log_mle: 0.97277 (0.97085) + | > loss_dur: 2.63111 (2.69928) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.11405 (10.12294) + | > current_lr: 0.00000 + | > step_time: 2.98930 (4.01045) + | > loader_time: 0.13020 (0.08398) + + + --> STEP: 116/234 -- GLOBAL_STEP: 350 + | > loss: 3.59348 (3.66578) + | > log_mle: 0.97257 (0.97071) + | > loss_dur: 2.62091 (2.69507) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.21690 (10.12398) + | > current_lr: 0.00000 + | > step_time: 3.38910 (3.97579) + | > loader_time: 0.00270 (0.08301) + + + --> STEP: 121/234 -- GLOBAL_STEP: 355 + | > loss: 3.46228 (3.66119) + | > log_mle: 0.98272 (0.97060) + | > loss_dur: 2.47956 (2.69058) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.03049 (10.12732) + | > current_lr: 0.00000 + | > step_time: 4.21740 (3.99354) + | > loader_time: 0.09790 (0.08209) + + + --> STEP: 126/234 -- GLOBAL_STEP: 360 + | > loss: 3.59307 (3.65602) + | > log_mle: 0.96040 (0.97022) + | > loss_dur: 2.63267 (2.68580) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.15240 (10.12746) + | > current_lr: 0.00000 + | > step_time: 3.50510 (3.95670) + | > loader_time: 0.00780 (0.08372) + + + --> STEP: 131/234 -- GLOBAL_STEP: 365 + | > loss: 3.58239 (3.65182) + | > log_mle: 0.97405 (0.97011) + | > loss_dur: 2.60834 (2.68171) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.15259 (10.12653) + | > current_lr: 0.00000 + | > step_time: 3.30390 (3.91737) + | > loader_time: 0.00820 (0.08106) + + + --> STEP: 136/234 -- GLOBAL_STEP: 370 + | > loss: 3.73695 (3.64920) + | > log_mle: 0.96791 (0.96994) + | > loss_dur: 2.76904 (2.67925) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.44193 (10.12809) + | > current_lr: 0.00000 + | > step_time: 6.66360 (3.98214) + | > loader_time: 0.03780 (0.08071) + + + --> STEP: 141/234 -- GLOBAL_STEP: 375 + | > loss: 3.57217 (3.64675) + | > log_mle: 0.96549 (0.96986) + | > loss_dur: 2.60668 (2.67689) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.26330 (10.13085) + | > current_lr: 0.00000 + | > step_time: 2.61850 (3.95662) + | > loader_time: 0.09560 (0.07991) + + + --> STEP: 146/234 -- GLOBAL_STEP: 380 + | > loss: 3.58821 (3.64534) + | > log_mle: 0.96186 (0.96977) + | > loss_dur: 2.62636 (2.67557) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.17447 (10.13313) + | > current_lr: 0.00000 + | > step_time: 5.19810 (3.98433) + | > loader_time: 0.01050 (0.07970) + + + --> STEP: 151/234 -- GLOBAL_STEP: 385 + | > loss: 3.51284 (3.64162) + | > log_mle: 0.95920 (0.96956) + | > loss_dur: 2.55364 (2.67206) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.08703 (10.13315) + | > current_lr: 0.00000 + | > step_time: 7.20740 (4.03835) + | > loader_time: 0.30160 (0.07998) + + + --> STEP: 156/234 -- GLOBAL_STEP: 390 + | > loss: 3.58368 (3.64041) + | > log_mle: 0.96647 (0.96947) + | > loss_dur: 2.61721 (2.67094) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.26398 (10.13647) + | > current_lr: 0.00000 + | > step_time: 3.34430 (4.05298) + | > loader_time: 0.30300 (0.08141) + + + --> STEP: 161/234 -- GLOBAL_STEP: 395 + | > loss: 3.70591 (3.63903) + | > log_mle: 0.96689 (0.96928) + | > loss_dur: 2.73902 (2.66975) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.41602 (10.13871) + | > current_lr: 0.00000 + | > step_time: 4.90510 (4.03962) + | > loader_time: 0.18690 (0.08133) + + + --> STEP: 166/234 -- GLOBAL_STEP: 400 + | > loss: 3.59555 (3.63675) + | > log_mle: 0.95933 (0.96905) + | > loss_dur: 2.63621 (2.66770) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.28723 (10.13995) + | > current_lr: 0.00000 + | > step_time: 3.90390 (4.02762) + | > loader_time: 0.00410 (0.07956) + + + --> STEP: 171/234 -- GLOBAL_STEP: 405 + | > loss: 3.66265 (3.63745) + | > log_mle: 0.95961 (0.96901) + | > loss_dur: 2.70304 (2.66845) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.40615 (10.14597) + | > current_lr: 0.00000 + | > step_time: 3.28720 (4.08233) + | > loader_time: 0.00500 (0.07946) + + + --> STEP: 176/234 -- GLOBAL_STEP: 410 + | > loss: 3.59944 (3.63718) + | > log_mle: 0.96695 (0.96909) + | > loss_dur: 2.63248 (2.66808) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.22109 (10.15025) + | > current_lr: 0.00000 + | > step_time: 8.41310 (4.10908) + | > loader_time: 0.00390 (0.07831) + + + --> STEP: 181/234 -- GLOBAL_STEP: 415 + | > loss: 3.59685 (3.63713) + | > log_mle: 0.95933 (0.96905) + | > loss_dur: 2.63752 (2.66808) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.21314 (10.15494) + | > current_lr: 0.00000 + | > step_time: 7.08470 (4.17182) + | > loader_time: 0.19920 (0.07986) + + + --> STEP: 186/234 -- GLOBAL_STEP: 420 + | > loss: 3.65787 (3.63803) + | > log_mle: 0.96600 (0.96915) + | > loss_dur: 2.69186 (2.66888) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.34394 (10.16052) + | > current_lr: 0.00000 + | > step_time: 4.38450 (4.16245) + | > loader_time: 0.68750 (0.08324) + + + --> STEP: 191/234 -- GLOBAL_STEP: 425 + | > loss: 3.63172 (3.63758) + | > log_mle: 0.98033 (0.96918) + | > loss_dur: 2.65138 (2.66840) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.29934 (10.16387) + | > current_lr: 0.00000 + | > step_time: 8.21040 (4.24889) + | > loader_time: 0.00570 (0.08299) + + + --> STEP: 196/234 -- GLOBAL_STEP: 430 + | > loss: 3.66683 (3.63761) + | > log_mle: 0.98901 (0.96937) + | > loss_dur: 2.67782 (2.66824) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.32775 (10.16722) + | > current_lr: 0.00000 + | > step_time: 6.19070 (4.29353) + | > loader_time: 0.00950 (0.08704) + + + --> STEP: 201/234 -- GLOBAL_STEP: 435 + | > loss: 3.54032 (3.63651) + | > log_mle: 0.96040 (0.96931) + | > loss_dur: 2.57992 (2.66720) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.16702 (10.16908) + | > current_lr: 0.00000 + | > step_time: 3.18910 (4.33552) + | > loader_time: 0.09930 (0.08722) + + + --> STEP: 206/234 -- GLOBAL_STEP: 440 + | > loss: 3.61529 (3.63598) + | > log_mle: 0.96527 (0.96922) + | > loss_dur: 2.65001 (2.66676) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.25424 (10.17114) + | > current_lr: 0.00000 + | > step_time: 11.30920 (4.34186) + | > loader_time: 0.18880 (0.09241) + + + --> STEP: 211/234 -- GLOBAL_STEP: 445 + | > loss: 3.76962 (3.63685) + | > log_mle: 0.96134 (0.96918) + | > loss_dur: 2.80828 (2.66767) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.59607 (10.17595) + | > current_lr: 0.00000 + | > step_time: 10.28800 (4.41145) + | > loader_time: 0.01050 (0.09498) + + + --> STEP: 216/234 -- GLOBAL_STEP: 450 + | > loss: 3.62055 (3.63693) + | > log_mle: 0.97657 (0.96922) + | > loss_dur: 2.64398 (2.66772) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.23930 (10.17951) + | > current_lr: 0.00000 + | > step_time: 3.80470 (4.43991) + | > loader_time: 0.00510 (0.09433) + + + --> STEP: 221/234 -- GLOBAL_STEP: 455 + | > loss: 3.58704 (3.63705) + | > log_mle: 0.96649 (0.96916) + | > loss_dur: 2.62054 (2.66789) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.20738 (10.18288) + | > current_lr: 0.00000 + | > step_time: 4.60860 (4.44591) + | > loader_time: 0.06420 (0.09410) + + + --> STEP: 226/234 -- GLOBAL_STEP: 460 + | > loss: 3.71735 (3.63881) + | > log_mle: 0.97237 (0.96923) + | > loss_dur: 2.74498 (2.66959) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.53237 (10.18933) + | > current_lr: 0.00000 + | > step_time: 1.31270 (4.41883) + | > loader_time: 0.00350 (0.09291) + + + --> STEP: 231/234 -- GLOBAL_STEP: 465 + | > loss: 4.02210 (3.64332) + | > log_mle: 0.97028 (0.96940) + | > loss_dur: 3.05182 (2.67392) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.98862 (10.19989) + | > current_lr: 0.00000 + | > step_time: 0.29860 (4.32912) + | > loader_time: 0.00650 (0.09103) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.12514 (+1.12175) + | > avg_loss: 3.69989 (-0.08610) + | > avg_log_mle: 0.96963 (-0.00559) + | > avg_loss_dur: 2.73026 (-0.08051) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_468.pth + + > EPOCH: 2/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 08:55:15)  + + --> STEP: 2/234 -- GLOBAL_STEP: 470 + | > loss: 4.43751 (4.40113) + | > log_mle: 1.02703 (1.02242) + | > loss_dur: 3.41049 (3.37871) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.11447 (10.03425) + | > current_lr: 0.00000 + | > step_time: 2.60370 (2.85064) + | > loader_time: 0.00170 (2.95354) + + + --> STEP: 7/234 -- GLOBAL_STEP: 475 + | > loss: 4.00402 (4.27821) + | > log_mle: 0.97052 (1.00113) + | > loss_dur: 3.03350 (3.27707) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.23495 (10.23958) + | > current_lr: 0.00000 + | > step_time: 10.39040 (5.70530) + | > loader_time: 0.00640 (0.87364) + + + --> STEP: 12/234 -- GLOBAL_STEP: 480 + | > loss: 3.77223 (4.11043) + | > log_mle: 0.98534 (0.99021) + | > loss_dur: 2.78689 (3.12022) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.84475 (10.11991) + | > current_lr: 0.00000 + | > step_time: 5.78490 (5.95161) + | > loader_time: 0.20560 (0.55072) + + + --> STEP: 17/234 -- GLOBAL_STEP: 485 + | > loss: 3.75257 (3.98940) + | > log_mle: 0.98551 (0.98409) + | > loss_dur: 2.76706 (3.00531) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.00190 (10.01047) + | > current_lr: 0.00000 + | > step_time: 4.10070 (5.27148) + | > loader_time: 0.00310 (0.40647) + + + --> STEP: 22/234 -- GLOBAL_STEP: 490 + | > loss: 3.66122 (3.92293) + | > log_mle: 0.96447 (0.98020) + | > loss_dur: 2.69674 (2.94273) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.88670 (9.98299) + | > current_lr: 0.00000 + | > step_time: 1.80050 (4.83019) + | > loader_time: 0.01550 (0.32803) + + + --> STEP: 27/234 -- GLOBAL_STEP: 495 + | > loss: 3.53407 (3.85955) + | > log_mle: 0.96767 (0.97779) + | > loss_dur: 2.56640 (2.88176) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.76919 (9.93337) + | > current_lr: 0.00000 + | > step_time: 4.69280 (4.44156) + | > loader_time: 0.09830 (0.27459) + + + --> STEP: 32/234 -- GLOBAL_STEP: 500 + | > loss: 3.45556 (3.80620) + | > log_mle: 0.93521 (0.97461) + | > loss_dur: 2.52036 (2.83159) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.65656 (9.89160) + | > current_lr: 0.00000 + | > step_time: 5.90700 (4.47272) + | > loader_time: 0.09690 (0.24431) + + + --> STEP: 37/234 -- GLOBAL_STEP: 505 + | > loss: 3.47133 (3.77241) + | > log_mle: 0.95669 (0.97389) + | > loss_dur: 2.51463 (2.79852) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.61724 (9.87765) + | > current_lr: 0.00000 + | > step_time: 0.90540 (4.37974) + | > loader_time: 0.00450 (0.21401) + + + --> STEP: 42/234 -- GLOBAL_STEP: 510 + | > loss: 3.61568 (3.75145) + | > log_mle: 1.00132 (0.97429) + | > loss_dur: 2.61437 (2.77715) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.84549 (9.87957) + | > current_lr: 0.00000 + | > step_time: 1.27920 (3.98907) + | > loader_time: 0.00170 (0.18879) + + + --> STEP: 47/234 -- GLOBAL_STEP: 515 + | > loss: 3.44882 (3.72656) + | > log_mle: 0.96133 (0.97392) + | > loss_dur: 2.48749 (2.75264) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.69548 (9.87033) + | > current_lr: 0.00000 + | > step_time: 1.42510 (3.76458) + | > loader_time: 0.00160 (0.17255) + + + --> STEP: 52/234 -- GLOBAL_STEP: 520 + | > loss: 3.50005 (3.70136) + | > log_mle: 0.99027 (0.97337) + | > loss_dur: 2.50979 (2.72800) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.72752 (9.85431) + | > current_lr: 0.00000 + | > step_time: 2.40080 (3.57657) + | > loader_time: 0.00260 (0.15776) + + + --> STEP: 57/234 -- GLOBAL_STEP: 525 + | > loss: 3.55573 (3.68517) + | > log_mle: 0.97498 (0.97228) + | > loss_dur: 2.58075 (2.71289) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.86744 (9.84986) + | > current_lr: 0.00000 + | > step_time: 2.70710 (3.47110) + | > loader_time: 0.09940 (0.14776) + + + --> STEP: 62/234 -- GLOBAL_STEP: 530 + | > loss: 3.51329 (3.66743) + | > log_mle: 0.96088 (0.97110) + | > loss_dur: 2.55241 (2.69632) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.75804 (9.84236) + | > current_lr: 0.00000 + | > step_time: 2.78520 (3.35384) + | > loader_time: 0.00160 (0.13712) + + + --> STEP: 67/234 -- GLOBAL_STEP: 535 + | > loss: 3.40468 (3.65098) + | > log_mle: 0.95092 (0.96965) + | > loss_dur: 2.45375 (2.68133) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.58085 (9.83211) + | > current_lr: 0.00000 + | > step_time: 4.32990 (3.36808) + | > loader_time: 0.09820 (0.13149) + + + --> STEP: 72/234 -- GLOBAL_STEP: 540 + | > loss: 3.39280 (3.63824) + | > log_mle: 0.94667 (0.96902) + | > loss_dur: 2.44613 (2.66922) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.65873 (9.82773) + | > current_lr: 0.00000 + | > step_time: 1.48130 (3.27645) + | > loader_time: 0.00280 (0.12482) + + + --> STEP: 77/234 -- GLOBAL_STEP: 545 + | > loss: 3.48032 (3.62528) + | > log_mle: 0.95123 (0.96858) + | > loss_dur: 2.52909 (2.65670) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.83886 (9.81965) + | > current_lr: 0.00000 + | > step_time: 1.41960 (3.16387) + | > loader_time: 0.08650 (0.11996) + + + --> STEP: 82/234 -- GLOBAL_STEP: 550 + | > loss: 3.32461 (3.61093) + | > log_mle: 0.95858 (0.96807) + | > loss_dur: 2.36603 (2.64286) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.49128 (9.80585) + | > current_lr: 0.00000 + | > step_time: 1.51400 (3.14187) + | > loader_time: 0.08800 (0.11721) + + + --> STEP: 87/234 -- GLOBAL_STEP: 555 + | > loss: 3.39700 (3.60324) + | > log_mle: 0.94862 (0.96752) + | > loss_dur: 2.44838 (2.63572) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.65028 (9.80447) + | > current_lr: 0.00000 + | > step_time: 4.14900 (3.10798) + | > loader_time: 0.00250 (0.11165) + + + --> STEP: 92/234 -- GLOBAL_STEP: 560 + | > loss: 3.31105 (3.59081) + | > log_mle: 0.95551 (0.96696) + | > loss_dur: 2.35555 (2.62386) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.33220 (9.79139) + | > current_lr: 0.00000 + | > step_time: 2.71540 (3.04724) + | > loader_time: 0.09740 (0.10886) + + + --> STEP: 97/234 -- GLOBAL_STEP: 565 + | > loss: 3.36514 (3.58280) + | > log_mle: 0.94658 (0.96610) + | > loss_dur: 2.41856 (2.61670) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.48833 (9.78445) + | > current_lr: 0.00000 + | > step_time: 4.49130 (3.12747) + | > loader_time: 0.00980 (0.10644) + + + --> STEP: 102/234 -- GLOBAL_STEP: 570 + | > loss: 3.39045 (3.57456) + | > log_mle: 0.96247 (0.96551) + | > loss_dur: 2.42798 (2.60904) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.55687 (9.77476) + | > current_lr: 0.00000 + | > step_time: 3.90020 (3.14580) + | > loader_time: 0.40150 (0.10705) + + + --> STEP: 107/234 -- GLOBAL_STEP: 575 + | > loss: 3.40869 (3.56947) + | > log_mle: 0.94409 (0.96483) + | > loss_dur: 2.46460 (2.60464) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.70406 (9.77358) + | > current_lr: 0.00000 + | > step_time: 2.70420 (3.11830) + | > loader_time: 0.07790 (0.10366) + + + --> STEP: 112/234 -- GLOBAL_STEP: 580 + | > loss: 3.41386 (3.56407) + | > log_mle: 0.96160 (0.96444) + | > loss_dur: 2.45226 (2.59963) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.63348 (9.76957) + | > current_lr: 0.00000 + | > step_time: 1.39210 (3.07362) + | > loader_time: 0.09460 (0.10079) + + + --> STEP: 117/234 -- GLOBAL_STEP: 585 + | > loss: 3.40433 (3.55922) + | > log_mle: 0.95152 (0.96409) + | > loss_dur: 2.45280 (2.59512) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.66263 (9.76744) + | > current_lr: 0.00000 + | > step_time: 1.69250 (3.04547) + | > loader_time: 0.00210 (0.09731) + + + --> STEP: 122/234 -- GLOBAL_STEP: 590 + | > loss: 3.31247 (3.55256) + | > log_mle: 0.95158 (0.96388) + | > loss_dur: 2.36090 (2.58868) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.42564 (9.76152) + | > current_lr: 0.00000 + | > step_time: 1.90490 (3.06928) + | > loader_time: 0.00290 (0.09568) + + + --> STEP: 127/234 -- GLOBAL_STEP: 595 + | > loss: 3.45285 (3.54680) + | > log_mle: 0.95603 (0.96343) + | > loss_dur: 2.49682 (2.58337) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.73802 (9.75555) + | > current_lr: 0.00000 + | > step_time: 3.29950 (3.03785) + | > loader_time: 0.00330 (0.09286) + + + --> STEP: 132/234 -- GLOBAL_STEP: 600 + | > loss: 3.33980 (3.54017) + | > log_mle: 0.95437 (0.96318) + | > loss_dur: 2.38543 (2.57698) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.43497 (9.74605) + | > current_lr: 0.00000 + | > step_time: 3.31410 (3.03640) + | > loader_time: 0.19030 (0.09092) + + + --> STEP: 137/234 -- GLOBAL_STEP: 605 + | > loss: 3.50905 (3.53648) + | > log_mle: 0.96463 (0.96297) + | > loss_dur: 2.54442 (2.57350) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.75628 (9.74106) + | > current_lr: 0.00000 + | > step_time: 2.10010 (3.00772) + | > loader_time: 0.00240 (0.08905) + + + --> STEP: 142/234 -- GLOBAL_STEP: 610 + | > loss: 3.43543 (3.53199) + | > log_mle: 0.95669 (0.96271) + | > loss_dur: 2.47874 (2.56929) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.56038 (9.73454) + | > current_lr: 0.00000 + | > step_time: 1.94070 (2.98489) + | > loader_time: 0.00210 (0.08726) + + + --> STEP: 147/234 -- GLOBAL_STEP: 615 + | > loss: 3.38588 (3.52856) + | > log_mle: 0.95224 (0.96246) + | > loss_dur: 2.43364 (2.56610) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.54913 (9.72802) + | > current_lr: 0.00000 + | > step_time: 4.18900 (2.97179) + | > loader_time: 0.02130 (0.08520) + + + --> STEP: 152/234 -- GLOBAL_STEP: 620 + | > loss: 3.39094 (3.52386) + | > log_mle: 0.95157 (0.96212) + | > loss_dur: 2.43937 (2.56174) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.44447 (9.72035) + | > current_lr: 0.00000 + | > step_time: 4.30250 (2.97020) + | > loader_time: 0.00330 (0.08351) + + + --> STEP: 157/234 -- GLOBAL_STEP: 625 + | > loss: 3.41839 (3.52137) + | > log_mle: 0.94676 (0.96187) + | > loss_dur: 2.47163 (2.55951) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.52017 (9.71617) + | > current_lr: 0.00000 + | > step_time: 1.21220 (2.98070) + | > loader_time: 0.09680 (0.08214) + + + --> STEP: 162/234 -- GLOBAL_STEP: 630 + | > loss: 3.43255 (3.51929) + | > log_mle: 0.94525 (0.96153) + | > loss_dur: 2.48730 (2.55776) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.59980 (9.71234) + | > current_lr: 0.00000 + | > step_time: 6.41590 (2.99246) + | > loader_time: 0.19280 (0.08103) + + + --> STEP: 167/234 -- GLOBAL_STEP: 635 + | > loss: 3.42619 (3.51564) + | > log_mle: 0.96089 (0.96127) + | > loss_dur: 2.46530 (2.55438) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.43012 (9.70458) + | > current_lr: 0.00000 + | > step_time: 5.39910 (3.01856) + | > loader_time: 0.11810 (0.08045) + + + --> STEP: 172/234 -- GLOBAL_STEP: 640 + | > loss: 3.48726 (3.51486) + | > log_mle: 0.96986 (0.96112) + | > loss_dur: 2.51740 (2.55374) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.59608 (9.70239) + | > current_lr: 0.00000 + | > step_time: 2.80860 (3.03106) + | > loader_time: 0.00400 (0.07993) + + + --> STEP: 177/234 -- GLOBAL_STEP: 645 + | > loss: 3.44731 (3.51312) + | > log_mle: 0.96418 (0.96100) + | > loss_dur: 2.48313 (2.55212) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.47232 (9.69840) + | > current_lr: 0.00000 + | > step_time: 4.81180 (3.07666) + | > loader_time: 0.00220 (0.08103) + + + --> STEP: 182/234 -- GLOBAL_STEP: 650 + | > loss: 3.51415 (3.51094) + | > log_mle: 0.96422 (0.96080) + | > loss_dur: 2.54993 (2.55014) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.62613 (9.69250) + | > current_lr: 0.00000 + | > step_time: 2.00000 (3.15715) + | > loader_time: 0.01150 (0.08281) + + + --> STEP: 187/234 -- GLOBAL_STEP: 655 + | > loss: 3.39553 (3.50900) + | > log_mle: 0.95178 (0.96067) + | > loss_dur: 2.44374 (2.54834) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.40295 (9.68629) + | > current_lr: 0.00000 + | > step_time: 9.91150 (3.20713) + | > loader_time: 0.08690 (0.08412) + + + --> STEP: 192/234 -- GLOBAL_STEP: 660 + | > loss: 3.42118 (3.50647) + | > log_mle: 0.95016 (0.96053) + | > loss_dur: 2.47102 (2.54594) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.49513 (9.67909) + | > current_lr: 0.00000 + | > step_time: 7.10610 (3.31072) + | > loader_time: 0.19130 (0.08544) + + + --> STEP: 197/234 -- GLOBAL_STEP: 665 + | > loss: 3.34194 (3.50431) + | > log_mle: 0.94968 (0.96054) + | > loss_dur: 2.39227 (2.54377) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.28198 (9.67141) + | > current_lr: 0.00000 + | > step_time: 7.79530 (3.41055) + | > loader_time: 0.11010 (0.08721) + + + --> STEP: 202/234 -- GLOBAL_STEP: 670 + | > loss: 3.40237 (3.50135) + | > log_mle: 0.94519 (0.96030) + | > loss_dur: 2.45717 (2.54105) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.30075 (9.66247) + | > current_lr: 0.00000 + | > step_time: 4.89300 (3.43793) + | > loader_time: 0.20350 (0.08698) + + + --> STEP: 207/234 -- GLOBAL_STEP: 675 + | > loss: 3.41990 (3.49907) + | > log_mle: 0.94634 (0.96004) + | > loss_dur: 2.47356 (2.53903) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.37621 (9.65463) + | > current_lr: 0.00000 + | > step_time: 4.19970 (3.47536) + | > loader_time: 0.00440 (0.08671) + + + --> STEP: 212/234 -- GLOBAL_STEP: 680 + | > loss: 3.39534 (3.49778) + | > log_mle: 0.94717 (0.95983) + | > loss_dur: 2.44818 (2.53795) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.26890 (9.64790) + | > current_lr: 0.00000 + | > step_time: 9.30630 (3.58649) + | > loader_time: 0.17530 (0.08648) + + + --> STEP: 217/234 -- GLOBAL_STEP: 685 + | > loss: 3.42178 (3.49605) + | > log_mle: 0.95288 (0.95970) + | > loss_dur: 2.46889 (2.53635) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.31035 (9.64055) + | > current_lr: 0.00000 + | > step_time: 4.90050 (3.66836) + | > loader_time: 0.00570 (0.09051) + + + --> STEP: 222/234 -- GLOBAL_STEP: 690 + | > loss: 3.51183 (3.49487) + | > log_mle: 0.95970 (0.95949) + | > loss_dur: 2.55213 (2.53537) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.44471 (9.63393) + | > current_lr: 0.00000 + | > step_time: 0.44680 (3.62914) + | > loader_time: 0.00840 (0.08900) + + + --> STEP: 227/234 -- GLOBAL_STEP: 695 + | > loss: 3.43352 (3.49388) + | > log_mle: 0.95339 (0.95933) + | > loss_dur: 2.48014 (2.53455) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.30486 (9.62730) + | > current_lr: 0.00000 + | > step_time: 0.24930 (3.55449) + | > loader_time: 0.00620 (0.08715) + + + --> STEP: 232/234 -- GLOBAL_STEP: 700 + | > loss: 4.17182 (3.49951) + | > log_mle: 0.95581 (0.95930) + | > loss_dur: 3.21602 (2.54021) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.50192 (9.63138) + | > current_lr: 0.00000 + | > step_time: 0.38120 (3.48425) + | > loader_time: 0.02160 (0.08549) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.42562 (-0.69952) + | > avg_loss: 3.42067 (-0.27923) + | > avg_log_mle: 0.95211 (-0.01752) + | > avg_loss_dur: 2.46855 (-0.26171) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_702.pth + + > EPOCH: 3/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 09:10:09)  + + --> STEP: 3/234 -- GLOBAL_STEP: 705 + | > loss: 4.11104 (4.17956) + | > log_mle: 1.00135 (1.00607) + | > loss_dur: 3.10969 (3.17349) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.48408 (9.49396) + | > current_lr: 0.00000 + | > step_time: 1.69990 (3.79286) + | > loader_time: 0.00220 (0.06796) + + + --> STEP: 8/234 -- GLOBAL_STEP: 710 + | > loss: 3.98872 (4.10040) + | > log_mle: 0.94376 (0.98170) + | > loss_dur: 3.04497 (3.11871) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.65873 (9.54204) + | > current_lr: 0.00000 + | > step_time: 7.19220 (6.79829) + | > loader_time: 0.40590 (0.10110) + + + --> STEP: 13/234 -- GLOBAL_STEP: 715 + | > loss: 3.61225 (3.90770) + | > log_mle: 0.95437 (0.97442) + | > loss_dur: 2.65788 (2.93328) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.88304 (9.29424) + | > current_lr: 0.00000 + | > step_time: 6.88960 (6.18472) + | > loader_time: 0.00560 (0.08353) + + + --> STEP: 18/234 -- GLOBAL_STEP: 720 + | > loss: 3.42942 (3.79112) + | > log_mle: 0.94673 (0.96861) + | > loss_dur: 2.48269 (2.82251) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.71631 (9.14791) + | > current_lr: 0.00000 + | > step_time: 2.58960 (5.59392) + | > loader_time: 0.00370 (0.07806) + + + --> STEP: 23/234 -- GLOBAL_STEP: 725 + | > loss: 3.34770 (3.70851) + | > log_mle: 0.93548 (0.96461) + | > loss_dur: 2.41222 (2.74389) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.74464 (9.05918) + | > current_lr: 0.00000 + | > step_time: 2.71290 (5.27453) + | > loader_time: 0.00320 (0.07872) + + + --> STEP: 28/234 -- GLOBAL_STEP: 730 + | > loss: 3.28057 (3.64478) + | > log_mle: 0.94579 (0.96272) + | > loss_dur: 2.33478 (2.68206) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.53134 (8.97891) + | > current_lr: 0.00000 + | > step_time: 2.59890 (5.01538) + | > loader_time: 0.00240 (0.07842) + + + --> STEP: 33/234 -- GLOBAL_STEP: 735 + | > loss: 3.39071 (3.59282) + | > log_mle: 0.96580 (0.96020) + | > loss_dur: 2.42491 (2.63261) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.76190 (8.91541) + | > current_lr: 0.00000 + | > step_time: 5.29950 (5.01276) + | > loader_time: 0.00270 (0.08510) + + + --> STEP: 38/234 -- GLOBAL_STEP: 740 + | > loss: 3.35420 (3.55906) + | > log_mle: 0.95085 (0.95885) + | > loss_dur: 2.40335 (2.60021) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.68015 (8.88533) + | > current_lr: 0.00000 + | > step_time: 5.00560 (4.93015) + | > loader_time: 0.08510 (0.08885) + + + --> STEP: 43/234 -- GLOBAL_STEP: 745 + | > loss: 3.33484 (3.53569) + | > log_mle: 0.95362 (0.95908) + | > loss_dur: 2.38122 (2.57661) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.66925 (8.86384) + | > current_lr: 0.00000 + | > step_time: 2.90170 (4.81299) + | > loader_time: 0.08450 (0.09227) + + + --> STEP: 48/234 -- GLOBAL_STEP: 750 + | > loss: 3.26856 (3.50800) + | > log_mle: 0.95212 (0.95842) + | > loss_dur: 2.31645 (2.54959) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.47432 (8.83147) + | > current_lr: 0.00000 + | > step_time: 2.50810 (4.51175) + | > loader_time: 0.08340 (0.08625) + + + --> STEP: 53/234 -- GLOBAL_STEP: 755 + | > loss: 3.25218 (3.48349) + | > log_mle: 0.93136 (0.95725) + | > loss_dur: 2.32082 (2.52624) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.48844 (8.80401) + | > current_lr: 0.00000 + | > step_time: 1.38700 (4.26464) + | > loader_time: 0.00120 (0.07851) + + + --> STEP: 58/234 -- GLOBAL_STEP: 760 + | > loss: 3.22465 (3.46515) + | > log_mle: 0.94415 (0.95621) + | > loss_dur: 2.28050 (2.50893) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.46502 (8.78055) + | > current_lr: 0.00000 + | > step_time: 3.40540 (4.10764) + | > loader_time: 0.09230 (0.07491) + + + --> STEP: 63/234 -- GLOBAL_STEP: 765 + | > loss: 3.32940 (3.44933) + | > log_mle: 0.93365 (0.95471) + | > loss_dur: 2.39575 (2.49462) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.55043 (8.76014) + | > current_lr: 0.00000 + | > step_time: 3.39250 (3.99128) + | > loader_time: 0.00290 (0.07287) + + + --> STEP: 68/234 -- GLOBAL_STEP: 770 + | > loss: 3.24423 (3.42860) + | > log_mle: 0.94640 (0.95332) + | > loss_dur: 2.29783 (2.47529) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.46139 (8.72791) + | > current_lr: 0.00000 + | > step_time: 3.61520 (3.83350) + | > loader_time: 0.08780 (0.07121) + + + --> STEP: 73/234 -- GLOBAL_STEP: 775 + | > loss: 3.21378 (3.41341) + | > log_mle: 0.93656 (0.95238) + | > loss_dur: 2.27723 (2.46104) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.34008 (8.70518) + | > current_lr: 0.00000 + | > step_time: 4.30350 (3.82424) + | > loader_time: 0.17610 (0.07144) + + + --> STEP: 78/234 -- GLOBAL_STEP: 780 + | > loss: 3.20393 (3.39898) + | > log_mle: 0.95454 (0.95192) + | > loss_dur: 2.24939 (2.44706) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.33557 (8.67954) + | > current_lr: 0.00000 + | > step_time: 2.51660 (3.72490) + | > loader_time: 0.09000 (0.07144) + + + --> STEP: 83/234 -- GLOBAL_STEP: 785 + | > loss: 3.26829 (3.38507) + | > log_mle: 0.95041 (0.95114) + | > loss_dur: 2.31788 (2.43393) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.39217 (8.65468) + | > current_lr: 0.00000 + | > step_time: 1.29840 (3.66724) + | > loader_time: 0.00340 (0.07098) + + + --> STEP: 88/234 -- GLOBAL_STEP: 790 + | > loss: 3.15098 (3.37458) + | > log_mle: 0.93482 (0.95019) + | > loss_dur: 2.21616 (2.42440) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.16293 (8.63414) + | > current_lr: 0.00000 + | > step_time: 7.40660 (3.61611) + | > loader_time: 0.08890 (0.06982) + + + --> STEP: 93/234 -- GLOBAL_STEP: 795 + | > loss: 3.11997 (3.36135) + | > log_mle: 0.92870 (0.94931) + | > loss_dur: 2.19127 (2.41204) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.15180 (8.60679) + | > current_lr: 0.00000 + | > step_time: 2.41010 (3.53063) + | > loader_time: 0.00390 (0.06789) + + + --> STEP: 98/234 -- GLOBAL_STEP: 800 + | > loss: 3.18935 (3.35276) + | > log_mle: 0.92274 (0.94818) + | > loss_dur: 2.26661 (2.40458) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.24820 (8.58777) + | > current_lr: 0.00000 + | > step_time: 1.69180 (3.52210) + | > loader_time: 0.00250 (0.06639) + + + --> STEP: 103/234 -- GLOBAL_STEP: 805 + | > loss: 3.23220 (3.34499) + | > log_mle: 0.93673 (0.94751) + | > loss_dur: 2.29547 (2.39748) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.19419 (8.56766) + | > current_lr: 0.00000 + | > step_time: 0.99110 (3.47664) + | > loader_time: 0.00270 (0.06477) + + + --> STEP: 108/234 -- GLOBAL_STEP: 810 + | > loss: 3.10837 (3.33613) + | > log_mle: 0.92542 (0.94649) + | > loss_dur: 2.18295 (2.38964) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.97057 (8.54600) + | > current_lr: 0.00000 + | > step_time: 1.70180 (3.42683) + | > loader_time: 0.00190 (0.06349) + + + --> STEP: 113/234 -- GLOBAL_STEP: 815 + | > loss: 3.15599 (3.32908) + | > log_mle: 0.93903 (0.94596) + | > loss_dur: 2.21697 (2.38312) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.06581 (8.52542) + | > current_lr: 0.00000 + | > step_time: 1.27440 (3.38304) + | > loader_time: 0.00220 (0.06158) + + + --> STEP: 118/234 -- GLOBAL_STEP: 820 + | > loss: 3.18954 (3.32282) + | > log_mle: 0.93787 (0.94535) + | > loss_dur: 2.25168 (2.37748) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.03223 (8.50555) + | > current_lr: 0.00000 + | > step_time: 2.30530 (3.38484) + | > loader_time: 0.09670 (0.06138) + + + --> STEP: 123/234 -- GLOBAL_STEP: 825 + | > loss: 3.07509 (3.31394) + | > log_mle: 0.93354 (0.94484) + | > loss_dur: 2.14155 (2.36910) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.92748 (8.48242) + | > current_lr: 0.00000 + | > step_time: 2.40720 (3.33434) + | > loader_time: 0.08210 (0.06105) + + + --> STEP: 128/234 -- GLOBAL_STEP: 830 + | > loss: 3.05894 (3.30644) + | > log_mle: 0.93301 (0.94413) + | > loss_dur: 2.12594 (2.36231) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.76796 (8.46016) + | > current_lr: 0.00000 + | > step_time: 1.50910 (3.30423) + | > loader_time: 0.00280 (0.05953) + + + --> STEP: 133/234 -- GLOBAL_STEP: 835 + | > loss: 3.16347 (3.30007) + | > log_mle: 0.92659 (0.94356) + | > loss_dur: 2.23689 (2.35652) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.95565 (8.43970) + | > current_lr: 0.00000 + | > step_time: 1.84740 (3.30225) + | > loader_time: 0.08750 (0.05947) + + + --> STEP: 138/234 -- GLOBAL_STEP: 840 + | > loss: 3.12607 (3.29458) + | > log_mle: 0.92308 (0.94303) + | > loss_dur: 2.20299 (2.35155) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.88143 (8.41877) + | > current_lr: 0.00000 + | > step_time: 2.89750 (3.26840) + | > loader_time: 0.08680 (0.05876) + + + --> STEP: 143/234 -- GLOBAL_STEP: 845 + | > loss: 3.20603 (3.28957) + | > log_mle: 0.94187 (0.94262) + | > loss_dur: 2.26416 (2.34695) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.82370 (8.39682) + | > current_lr: 0.00000 + | > step_time: 3.99950 (3.32336) + | > loader_time: 0.00360 (0.05936) + + + --> STEP: 148/234 -- GLOBAL_STEP: 850 + | > loss: 2.98311 (3.28314) + | > log_mle: 0.92062 (0.94192) + | > loss_dur: 2.06249 (2.34122) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.51328 (8.37278) + | > current_lr: 0.00000 + | > step_time: 2.10470 (3.30457) + | > loader_time: 0.08550 (0.05891) + + + --> STEP: 153/234 -- GLOBAL_STEP: 855 + | > loss: 3.14472 (3.27790) + | > log_mle: 0.92302 (0.94129) + | > loss_dur: 2.22170 (2.33660) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.65654 (8.34912) + | > current_lr: 0.00000 + | > step_time: 2.60360 (3.30523) + | > loader_time: 0.07750 (0.05824) + + + --> STEP: 158/234 -- GLOBAL_STEP: 860 + | > loss: 3.10211 (3.27306) + | > log_mle: 0.92698 (0.94075) + | > loss_dur: 2.17513 (2.33232) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.58046 (8.32624) + | > current_lr: 0.00000 + | > step_time: 3.09860 (3.35661) + | > loader_time: 0.00400 (0.05937) + + + --> STEP: 163/234 -- GLOBAL_STEP: 865 + | > loss: 3.02495 (3.26883) + | > log_mle: 0.92443 (0.94008) + | > loss_dur: 2.10051 (2.32875) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.40143 (8.30452) + | > current_lr: 0.00000 + | > step_time: 2.41050 (3.33406) + | > loader_time: 0.08830 (0.05817) + + + --> STEP: 168/234 -- GLOBAL_STEP: 870 + | > loss: 3.17754 (3.26484) + | > log_mle: 0.91699 (0.93946) + | > loss_dur: 2.26055 (2.32538) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.57272 (8.28163) + | > current_lr: 0.00000 + | > step_time: 8.41330 (3.39874) + | > loader_time: 0.09850 (0.05858) + + + --> STEP: 173/234 -- GLOBAL_STEP: 875 + | > loss: 3.17723 (3.26224) + | > log_mle: 0.93099 (0.93906) + | > loss_dur: 2.24624 (2.32319) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.57810 (8.26021) + | > current_lr: 0.00000 + | > step_time: 4.99870 (3.42125) + | > loader_time: 0.00650 (0.05807) + + + --> STEP: 178/234 -- GLOBAL_STEP: 880 + | > loss: 3.06711 (3.25815) + | > log_mle: 0.91172 (0.93848) + | > loss_dur: 2.15539 (2.31967) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.27848 (8.23620) + | > current_lr: 0.00000 + | > step_time: 5.48450 (3.45657) + | > loader_time: 0.02230 (0.05812) + + + --> STEP: 183/234 -- GLOBAL_STEP: 885 + | > loss: 3.10046 (3.25469) + | > log_mle: 0.92039 (0.93799) + | > loss_dur: 2.18007 (2.31670) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.24102 (8.21224) + | > current_lr: 0.00000 + | > step_time: 9.49670 (3.54734) + | > loader_time: 0.10170 (0.05979) + + + --> STEP: 188/234 -- GLOBAL_STEP: 890 + | > loss: 3.06393 (3.25096) + | > log_mle: 0.92085 (0.93751) + | > loss_dur: 2.14308 (2.31346) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.09800 (8.18680) + | > current_lr: 0.00000 + | > step_time: 3.39510 (3.57917) + | > loader_time: 0.00310 (0.06080) + + + --> STEP: 193/234 -- GLOBAL_STEP: 895 + | > loss: 3.08820 (3.24694) + | > log_mle: 0.92226 (0.93703) + | > loss_dur: 2.16594 (2.30992) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.10791 (8.16129) + | > current_lr: 0.00000 + | > step_time: 7.08540 (3.65418) + | > loader_time: 0.02560 (0.06042) + + + --> STEP: 198/234 -- GLOBAL_STEP: 900 + | > loss: 3.12568 (3.24392) + | > log_mle: 0.92273 (0.93668) + | > loss_dur: 2.20295 (2.30724) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.24407 (8.13786) + | > current_lr: 0.00000 + | > step_time: 5.39880 (3.70530) + | > loader_time: 0.10310 (0.06142) + + + --> STEP: 203/234 -- GLOBAL_STEP: 905 + | > loss: 3.10953 (3.23946) + | > log_mle: 0.90943 (0.93601) + | > loss_dur: 2.20010 (2.30345) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.16964 (8.11108) + | > current_lr: 0.00000 + | > step_time: 5.70400 (3.71996) + | > loader_time: 0.18290 (0.06225) + + + --> STEP: 208/234 -- GLOBAL_STEP: 910 + | > loss: 3.14129 (3.23594) + | > log_mle: 0.91633 (0.93544) + | > loss_dur: 2.22496 (2.30051) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.11619 (8.08574) + | > current_lr: 0.00000 + | > step_time: 7.60100 (3.79060) + | > loader_time: 0.00490 (0.06272) + + + --> STEP: 213/234 -- GLOBAL_STEP: 915 + | > loss: 3.08469 (3.23358) + | > log_mle: 0.91208 (0.93483) + | > loss_dur: 2.17261 (2.29875) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.95555 (8.06242) + | > current_lr: 0.00000 + | > step_time: 7.20340 (3.85835) + | > loader_time: 0.38120 (0.06636) + + + --> STEP: 218/234 -- GLOBAL_STEP: 920 + | > loss: 3.14963 (3.23075) + | > log_mle: 0.90457 (0.93428) + | > loss_dur: 2.24505 (2.29647) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.11501 (8.03793) + | > current_lr: 0.00000 + | > step_time: 6.09760 (3.90551) + | > loader_time: 0.00340 (0.06984) + + + --> STEP: 223/234 -- GLOBAL_STEP: 925 + | > loss: 3.15231 (3.22862) + | > log_mle: 0.90127 (0.93367) + | > loss_dur: 2.25103 (2.29494) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.05822 (8.01450) + | > current_lr: 0.00000 + | > step_time: 0.40760 (3.86728) + | > loader_time: 0.07220 (0.06950) + + + --> STEP: 228/234 -- GLOBAL_STEP: 930 + | > loss: 3.21051 (3.22738) + | > log_mle: 0.90375 (0.93312) + | > loss_dur: 2.30676 (2.29426) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.08111 (7.99301) + | > current_lr: 0.00000 + | > step_time: 0.23960 (3.78772) + | > loader_time: 0.00370 (0.06804) + + + --> STEP: 233/234 -- GLOBAL_STEP: 935 + | > loss: 4.77007 (3.23862) + | > log_mle: 0.95996 (0.93290) + | > loss_dur: 3.81011 (2.30571) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.05522 (7.98743) + | > current_lr: 0.00000 + | > step_time: 0.21320 (3.71213) + | > loader_time: 0.00350 (0.06712) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27294 (-0.15269) + | > avg_loss: 3.06286 (-0.35780) + | > avg_log_mle: 0.91055 (-0.04157) + | > avg_loss_dur: 2.15231 (-0.31624) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_936.pth + + > EPOCH: 4/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 09:25:55)  + + --> STEP: 4/234 -- GLOBAL_STEP: 940 + | > loss: 4.02997 (3.95622) + | > log_mle: 0.95183 (0.96570) + | > loss_dur: 3.07814 (2.99052) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.18723 (8.06396) + | > current_lr: 0.00000 + | > step_time: 7.37560 (8.78840) + | > loader_time: 0.11610 (0.03387) + + + --> STEP: 9/234 -- GLOBAL_STEP: 945 + | > loss: 3.46828 (3.79735) + | > log_mle: 0.92780 (0.94413) + | > loss_dur: 2.54048 (2.85322) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.17595 (7.83632) + | > current_lr: 0.00000 + | > step_time: 9.49190 (7.06313) + | > loader_time: 0.10250 (0.04608) + + + --> STEP: 14/234 -- GLOBAL_STEP: 950 + | > loss: 3.27429 (3.63300) + | > log_mle: 0.90977 (0.93654) + | > loss_dur: 2.36452 (2.69646) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.94898 (7.58367) + | > current_lr: 0.00000 + | > step_time: 4.50210 (6.35631) + | > loader_time: 0.39980 (0.07791) + + + --> STEP: 19/234 -- GLOBAL_STEP: 955 + | > loss: 3.34211 (3.54056) + | > log_mle: 0.91962 (0.93198) + | > loss_dur: 2.42249 (2.60858) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.11130 (7.43500) + | > current_lr: 0.00000 + | > step_time: 2.40370 (5.19291) + | > loader_time: 0.08640 (0.06684) + + + --> STEP: 24/234 -- GLOBAL_STEP: 960 + | > loss: 3.24335 (3.45741) + | > log_mle: 0.91886 (0.92817) + | > loss_dur: 2.32449 (2.52924) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.88110 (7.28986) + | > current_lr: 0.00000 + | > step_time: 4.20100 (4.89037) + | > loader_time: 0.08460 (0.06805) + + + --> STEP: 29/234 -- GLOBAL_STEP: 965 + | > loss: 3.16691 (3.39721) + | > log_mle: 0.92026 (0.92603) + | > loss_dur: 2.24665 (2.47118) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.84607 (7.19120) + | > current_lr: 0.00000 + | > step_time: 3.21060 (4.76136) + | > loader_time: 0.09060 (0.07374) + + + --> STEP: 34/234 -- GLOBAL_STEP: 970 + | > loss: 3.13921 (3.35155) + | > log_mle: 0.90439 (0.92282) + | > loss_dur: 2.23482 (2.42872) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.75835 (7.11853) + | > current_lr: 0.00000 + | > step_time: 3.10010 (4.62547) + | > loader_time: 0.09860 (0.06844) + + + --> STEP: 39/234 -- GLOBAL_STEP: 975 + | > loss: 3.17205 (3.32132) + | > log_mle: 0.92782 (0.92168) + | > loss_dur: 2.24423 (2.39964) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.72494 (7.06673) + | > current_lr: 0.00000 + | > step_time: 2.70040 (4.52744) + | > loader_time: 0.09930 (0.07273) + + + --> STEP: 44/234 -- GLOBAL_STEP: 980 + | > loss: 3.01843 (3.29132) + | > log_mle: 0.90198 (0.92085) + | > loss_dur: 2.11645 (2.37047) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.49035 (7.00901) + | > current_lr: 0.00000 + | > step_time: 1.80730 (4.32195) + | > loader_time: 0.00200 (0.07571) + + + --> STEP: 49/234 -- GLOBAL_STEP: 985 + | > loss: 2.97615 (3.26386) + | > log_mle: 0.89152 (0.91952) + | > loss_dur: 2.08463 (2.34434) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.52436 (6.96315) + | > current_lr: 0.00000 + | > step_time: 2.99660 (4.10601) + | > loader_time: 0.00660 (0.06831) + + + --> STEP: 54/234 -- GLOBAL_STEP: 990 + | > loss: 3.01461 (3.24163) + | > log_mle: 0.89733 (0.91814) + | > loss_dur: 2.11728 (2.32349) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.43641 (6.92455) + | > current_lr: 0.00000 + | > step_time: 1.82070 (3.97074) + | > loader_time: 0.00250 (0.06389) + + + --> STEP: 59/234 -- GLOBAL_STEP: 995 + | > loss: 2.90493 (3.22098) + | > log_mle: 0.88914 (0.91660) + | > loss_dur: 2.01579 (2.30438) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.33091 (6.88493) + | > current_lr: 0.00000 + | > step_time: 2.09840 (3.82571) + | > loader_time: 0.00640 (0.06034) + + + --> STEP: 64/234 -- GLOBAL_STEP: 1000 + | > loss: 2.91856 (3.20458) + | > log_mle: 0.88662 (0.91472) + | > loss_dur: 2.03194 (2.28986) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.39778 (6.85427) + | > current_lr: 0.00000 + | > step_time: 2.48760 (3.75603) + | > loader_time: 0.00190 (0.05728) + + + --> STEP: 69/234 -- GLOBAL_STEP: 1005 + | > loss: 2.95551 (3.18667) + | > log_mle: 0.90948 (0.91341) + | > loss_dur: 2.04602 (2.27326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.34186 (6.81883) + | > current_lr: 0.00000 + | > step_time: 1.49140 (3.64168) + | > loader_time: 0.07850 (0.05832) + + + --> STEP: 74/234 -- GLOBAL_STEP: 1010 + | > loss: 2.91095 (3.17216) + | > log_mle: 0.90616 (0.91210) + | > loss_dur: 2.00478 (2.26007) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.27061 (6.78961) + | > current_lr: 0.00000 + | > step_time: 1.89080 (3.53270) + | > loader_time: 0.00240 (0.05456) + + + --> STEP: 79/234 -- GLOBAL_STEP: 1015 + | > loss: 2.93730 (3.15872) + | > log_mle: 0.88312 (0.91087) + | > loss_dur: 2.05418 (2.24785) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.27088 (6.76137) + | > current_lr: 0.00000 + | > step_time: 7.38980 (3.49326) + | > loader_time: 0.09580 (0.05570) + + + --> STEP: 84/234 -- GLOBAL_STEP: 1020 + | > loss: 3.04078 (3.14631) + | > log_mle: 0.87564 (0.90958) + | > loss_dur: 2.16513 (2.23673) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.51082 (6.73380) + | > current_lr: 0.00000 + | > step_time: 4.29410 (3.51970) + | > loader_time: 0.00420 (0.05275) + + + --> STEP: 89/234 -- GLOBAL_STEP: 1025 + | > loss: 2.91483 (3.13471) + | > log_mle: 0.87644 (0.90825) + | > loss_dur: 2.03839 (2.22646) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.20432 (6.70718) + | > current_lr: 0.00000 + | > step_time: 1.60630 (3.44136) + | > loader_time: 0.00390 (0.05118) + + + --> STEP: 94/234 -- GLOBAL_STEP: 1030 + | > loss: 2.98723 (3.12228) + | > log_mle: 0.87427 (0.90693) + | > loss_dur: 2.11296 (2.21535) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.30748 (6.67875) + | > current_lr: 0.00000 + | > step_time: 1.60050 (3.41196) + | > loader_time: 0.00210 (0.05030) + + + --> STEP: 99/234 -- GLOBAL_STEP: 1035 + | > loss: 2.90611 (3.11315) + | > log_mle: 0.87959 (0.90550) + | > loss_dur: 2.02651 (2.20765) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.06774 (6.65300) + | > current_lr: 0.00000 + | > step_time: 1.85280 (3.36631) + | > loader_time: 0.00130 (0.04797) + + + --> STEP: 104/234 -- GLOBAL_STEP: 1040 + | > loss: 2.98507 (3.10633) + | > log_mle: 0.87135 (0.90431) + | > loss_dur: 2.11371 (2.20202) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.23673 (6.63116) + | > current_lr: 0.00000 + | > step_time: 2.70640 (3.31986) + | > loader_time: 0.10470 (0.04802) + + + --> STEP: 109/234 -- GLOBAL_STEP: 1045 + | > loss: 2.93610 (3.09852) + | > log_mle: 0.87335 (0.90296) + | > loss_dur: 2.06275 (2.19557) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.05259 (6.60877) + | > current_lr: 0.00000 + | > step_time: 2.31320 (3.30871) + | > loader_time: 0.00290 (0.04754) + + + --> STEP: 114/234 -- GLOBAL_STEP: 1050 + | > loss: 2.91556 (3.09196) + | > log_mle: 0.87496 (0.90201) + | > loss_dur: 2.04060 (2.18995) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.08211 (6.58760) + | > current_lr: 0.00000 + | > step_time: 4.92120 (3.34373) + | > loader_time: 0.19000 (0.04891) + + + --> STEP: 119/234 -- GLOBAL_STEP: 1055 + | > loss: 2.90909 (3.08579) + | > log_mle: 0.88299 (0.90108) + | > loss_dur: 2.02610 (2.18471) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.99039 (6.56583) + | > current_lr: 0.00000 + | > step_time: 2.31420 (3.34546) + | > loader_time: 0.19340 (0.04992) + + + --> STEP: 124/234 -- GLOBAL_STEP: 1060 + | > loss: 2.90326 (3.07737) + | > log_mle: 0.87134 (0.90009) + | > loss_dur: 2.03192 (2.17728) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.99223 (6.54174) + | > current_lr: 0.00000 + | > step_time: 1.87090 (3.28494) + | > loader_time: 0.02380 (0.05008) + + + --> STEP: 129/234 -- GLOBAL_STEP: 1065 + | > loss: 2.94069 (3.07097) + | > log_mle: 0.87896 (0.89902) + | > loss_dur: 2.06172 (2.17195) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.02789 (6.51989) + | > current_lr: 0.00000 + | > step_time: 4.40630 (3.28473) + | > loader_time: 0.00280 (0.04970) + + + --> STEP: 134/234 -- GLOBAL_STEP: 1070 + | > loss: 2.96541 (3.06552) + | > log_mle: 0.87602 (0.89801) + | > loss_dur: 2.08939 (2.16751) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.01771 (6.49914) + | > current_lr: 0.00000 + | > step_time: 2.66010 (3.27291) + | > loader_time: 0.08710 (0.04990) + + + --> STEP: 139/234 -- GLOBAL_STEP: 1075 + | > loss: 2.95226 (3.06102) + | > log_mle: 0.86484 (0.89699) + | > loss_dur: 2.08742 (2.16403) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.94558 (6.47984) + | > current_lr: 0.00000 + | > step_time: 2.00680 (3.23769) + | > loader_time: 0.00280 (0.04879) + + + --> STEP: 144/234 -- GLOBAL_STEP: 1080 + | > loss: 2.92466 (3.05644) + | > log_mle: 0.86593 (0.89615) + | > loss_dur: 2.05872 (2.16029) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.95968 (6.46042) + | > current_lr: 0.00000 + | > step_time: 5.30590 (3.37521) + | > loader_time: 0.69980 (0.05536) + + + --> STEP: 149/234 -- GLOBAL_STEP: 1085 + | > loss: 2.99204 (3.05168) + | > log_mle: 0.86857 (0.89506) + | > loss_dur: 2.12347 (2.15662) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.00866 (6.44136) + | > current_lr: 0.00000 + | > step_time: 4.38790 (3.35326) + | > loader_time: 0.00580 (0.05418) + + + --> STEP: 154/234 -- GLOBAL_STEP: 1090 + | > loss: 2.94945 (3.04797) + | > log_mle: 0.86648 (0.89399) + | > loss_dur: 2.08297 (2.15398) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.93233 (6.42355) + | > current_lr: 0.00000 + | > step_time: 2.29990 (3.31919) + | > loader_time: 0.00380 (0.05356) + + + --> STEP: 159/234 -- GLOBAL_STEP: 1095 + | > loss: 3.03572 (3.04516) + | > log_mle: 0.86078 (0.89299) + | > loss_dur: 2.17494 (2.15217) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.02459 (6.40720) + | > current_lr: 0.00000 + | > step_time: 5.31450 (3.35341) + | > loader_time: 0.09920 (0.05440) + + + --> STEP: 164/234 -- GLOBAL_STEP: 1100 + | > loss: 2.96434 (3.04193) + | > log_mle: 0.85299 (0.89188) + | > loss_dur: 2.11135 (2.15005) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.90052 (6.39045) + | > current_lr: 0.00000 + | > step_time: 1.79980 (3.33927) + | > loader_time: 0.00300 (0.05488) + + + --> STEP: 169/234 -- GLOBAL_STEP: 1105 + | > loss: 3.03979 (3.04008) + | > log_mle: 0.86247 (0.89090) + | > loss_dur: 2.17732 (2.14918) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.99245 (6.37547) + | > current_lr: 0.00000 + | > step_time: 7.40070 (3.38186) + | > loader_time: 0.10660 (0.05681) + + + --> STEP: 174/234 -- GLOBAL_STEP: 1110 + | > loss: 2.93107 (3.03865) + | > log_mle: 0.85036 (0.88996) + | > loss_dur: 2.08071 (2.14869) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.80806 (6.36125) + | > current_lr: 0.00000 + | > step_time: 4.58550 (3.40080) + | > loader_time: 0.10930 (0.05695) + + + --> STEP: 179/234 -- GLOBAL_STEP: 1115 + | > loss: 2.99509 (3.03683) + | > log_mle: 0.85451 (0.88897) + | > loss_dur: 2.14058 (2.14786) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.85455 (6.34710) + | > current_lr: 0.00000 + | > step_time: 13.28940 (3.51141) + | > loader_time: 0.10710 (0.05989) + + + --> STEP: 184/234 -- GLOBAL_STEP: 1120 + | > loss: 2.99883 (3.03563) + | > log_mle: 0.85261 (0.88803) + | > loss_dur: 2.14622 (2.14760) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.86883 (6.33413) + | > current_lr: 0.00000 + | > step_time: 3.70170 (3.50607) + | > loader_time: 0.00270 (0.05884) + + + --> STEP: 189/234 -- GLOBAL_STEP: 1125 + | > loss: 2.95931 (3.03391) + | > log_mle: 0.85292 (0.88713) + | > loss_dur: 2.10639 (2.14678) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.84638 (6.32042) + | > current_lr: 0.00000 + | > step_time: 5.79820 (3.58156) + | > loader_time: 0.00670 (0.05838) + + + --> STEP: 194/234 -- GLOBAL_STEP: 1130 + | > loss: 2.96761 (3.03172) + | > log_mle: 0.84862 (0.88623) + | > loss_dur: 2.11898 (2.14549) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.78554 (6.30633) + | > current_lr: 0.00000 + | > step_time: 4.89720 (3.62278) + | > loader_time: 0.00890 (0.05701) + + + --> STEP: 199/234 -- GLOBAL_STEP: 1135 + | > loss: 2.90887 (3.03026) + | > log_mle: 0.84003 (0.88539) + | > loss_dur: 2.06884 (2.14487) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.74021 (6.29443) + | > current_lr: 0.00000 + | > step_time: 9.60560 (3.67860) + | > loader_time: 0.08740 (0.05796) + + + --> STEP: 204/234 -- GLOBAL_STEP: 1140 + | > loss: 2.95356 (3.02802) + | > log_mle: 0.84241 (0.88436) + | > loss_dur: 2.11115 (2.14365) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.76252 (6.28109) + | > current_lr: 0.00000 + | > step_time: 0.99550 (3.69083) + | > loader_time: 0.00350 (0.05745) + + + --> STEP: 209/234 -- GLOBAL_STEP: 1145 + | > loss: 2.96042 (3.02679) + | > log_mle: 0.84684 (0.88340) + | > loss_dur: 2.11359 (2.14339) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.77440 (6.26975) + | > current_lr: 0.00000 + | > step_time: 4.80110 (3.68726) + | > loader_time: 0.09820 (0.05755) + + + --> STEP: 214/234 -- GLOBAL_STEP: 1150 + | > loss: 2.98772 (3.02658) + | > log_mle: 0.83933 (0.88235) + | > loss_dur: 2.14840 (2.14423) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.83913 (6.26027) + | > current_lr: 0.00000 + | > step_time: 8.50690 (3.73499) + | > loader_time: 0.09880 (0.05757) + + + --> STEP: 219/234 -- GLOBAL_STEP: 1155 + | > loss: 3.00138 (3.02572) + | > log_mle: 0.83324 (0.88135) + | > loss_dur: 2.16814 (2.14438) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.77809 (6.24958) + | > current_lr: 0.00000 + | > step_time: 3.90620 (3.79766) + | > loader_time: 0.08750 (0.05804) + + + --> STEP: 224/234 -- GLOBAL_STEP: 1160 + | > loss: 3.09093 (3.02601) + | > log_mle: 0.83097 (0.88033) + | > loss_dur: 2.25996 (2.14567) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.94228 (6.24107) + | > current_lr: 0.00000 + | > step_time: 0.22300 (3.76455) + | > loader_time: 0.00320 (0.05725) + + + --> STEP: 229/234 -- GLOBAL_STEP: 1165 + | > loss: 3.21655 (3.02711) + | > log_mle: 0.86716 (0.87953) + | > loss_dur: 2.34939 (2.14758) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.00506 (6.23369) + | > current_lr: 0.00000 + | > step_time: 0.23560 (3.68757) + | > loader_time: 0.00350 (0.05608) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22565 (-0.04729) + | > avg_loss: 2.94092 (-0.12195) + | > avg_log_mle: 0.83959 (-0.07096) + | > avg_loss_dur: 2.10133 (-0.05099) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_1170.pth + + > EPOCH: 5/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 09:41:22)  + + --> STEP: 0/234 -- GLOBAL_STEP: 1170 + | > loss: 4.02418 (4.02418) + | > log_mle: 0.95504 (0.95504) + | > loss_dur: 3.06914 (3.06914) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.76906 (7.76906) + | > current_lr: 0.00000 + | > step_time: 7.79440 (7.79435) + | > loader_time: 25.44240 (25.44243) + + + --> STEP: 5/234 -- GLOBAL_STEP: 1175 + | > loss: 3.74391 (3.76817) + | > log_mle: 0.86895 (0.89515) + | > loss_dur: 2.87496 (2.87302) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.23570 (7.20213) + | > current_lr: 0.00000 + | > step_time: 8.50410 (5.06225) + | > loader_time: 0.00260 (0.07946) + + + --> STEP: 10/234 -- GLOBAL_STEP: 1180 + | > loss: 3.20729 (3.58962) + | > log_mle: 0.86500 (0.87902) + | > loss_dur: 2.34230 (2.71060) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.23746 (6.87763) + | > current_lr: 0.00000 + | > step_time: 2.10660 (4.50351) + | > loader_time: 0.00120 (0.07213) + + + --> STEP: 15/234 -- GLOBAL_STEP: 1185 + | > loss: 3.12158 (3.44660) + | > log_mle: 0.85794 (0.87202) + | > loss_dur: 2.26363 (2.57458) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.01978 (6.62651) + | > current_lr: 0.00000 + | > step_time: 7.01750 (4.87838) + | > loader_time: 0.09720 (0.08838) + + + --> STEP: 20/234 -- GLOBAL_STEP: 1190 + | > loss: 3.04894 (3.35348) + | > log_mle: 0.84750 (0.86729) + | > loss_dur: 2.20143 (2.48619) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.92831 (6.45505) + | > current_lr: 0.00000 + | > step_time: 3.71000 (4.56348) + | > loader_time: 0.00450 (0.07583) + + + --> STEP: 25/234 -- GLOBAL_STEP: 1195 + | > loss: 2.98661 (3.27754) + | > log_mle: 0.85871 (0.86415) + | > loss_dur: 2.12790 (2.41339) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.79862 (6.32211) + | > current_lr: 0.00000 + | > step_time: 4.00580 (4.67123) + | > loader_time: 0.00330 (0.07252) + + + --> STEP: 30/234 -- GLOBAL_STEP: 1200 + | > loss: 2.90319 (3.21676) + | > log_mle: 0.83423 (0.86091) + | > loss_dur: 2.06895 (2.35585) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.78008 (6.22111) + | > current_lr: 0.00000 + | > step_time: 6.39460 (4.63255) + | > loader_time: 0.10430 (0.07347) + + + --> STEP: 35/234 -- GLOBAL_STEP: 1205 + | > loss: 2.97277 (3.17915) + | > log_mle: 0.85625 (0.85828) + | > loss_dur: 2.11651 (2.32087) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.81264 (6.15763) + | > current_lr: 0.00000 + | > step_time: 1.50950 (4.61990) + | > loader_time: 0.00450 (0.07378) + + + --> STEP: 40/234 -- GLOBAL_STEP: 1210 + | > loss: 2.92950 (3.15460) + | > log_mle: 0.84898 (0.85627) + | > loss_dur: 2.08053 (2.29833) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.72280 (6.11985) + | > current_lr: 0.00000 + | > step_time: 1.49320 (4.35454) + | > loader_time: 0.00740 (0.06697) + + + --> STEP: 45/234 -- GLOBAL_STEP: 1215 + | > loss: 2.99551 (3.13189) + | > log_mle: 0.85181 (0.85523) + | > loss_dur: 2.14370 (2.27665) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.82031 (6.07839) + | > current_lr: 0.00000 + | > step_time: 3.19050 (4.12040) + | > loader_time: 0.00270 (0.06542) + + + --> STEP: 50/234 -- GLOBAL_STEP: 1220 + | > loss: 2.87250 (3.10232) + | > log_mle: 0.83275 (0.85304) + | > loss_dur: 2.03975 (2.24928) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.69415 (6.03109) + | > current_lr: 0.00000 + | > step_time: 2.81010 (3.90190) + | > loader_time: 0.00260 (0.06093) + + + --> STEP: 55/234 -- GLOBAL_STEP: 1225 + | > loss: 2.84409 (3.08088) + | > log_mle: 0.81984 (0.85116) + | > loss_dur: 2.02425 (2.22972) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.57707 (5.99418) + | > current_lr: 0.00000 + | > step_time: 1.49650 (3.69960) + | > loader_time: 0.01940 (0.05747) + + + --> STEP: 60/234 -- GLOBAL_STEP: 1230 + | > loss: 2.87483 (3.06327) + | > log_mle: 0.82197 (0.84943) + | > loss_dur: 2.05286 (2.21384) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.68577 (5.96384) + | > current_lr: 0.00000 + | > step_time: 1.79930 (3.60019) + | > loader_time: 0.17860 (0.06065) + + + --> STEP: 65/234 -- GLOBAL_STEP: 1235 + | > loss: 2.75675 (3.04715) + | > log_mle: 0.82459 (0.84745) + | > loss_dur: 1.93216 (2.19970) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.44026 (5.93557) + | > current_lr: 0.00000 + | > step_time: 1.38950 (3.49371) + | > loader_time: 0.00210 (0.05751) + + + --> STEP: 70/234 -- GLOBAL_STEP: 1240 + | > loss: 2.86333 (3.03162) + | > log_mle: 0.81916 (0.84587) + | > loss_dur: 2.04417 (2.18575) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.69625 (5.91065) + | > current_lr: 0.00000 + | > step_time: 4.56190 (3.41704) + | > loader_time: 0.00210 (0.05587) + + + --> STEP: 75/234 -- GLOBAL_STEP: 1245 + | > loss: 2.83436 (3.01758) + | > log_mle: 0.82390 (0.84440) + | > loss_dur: 2.01046 (2.17318) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.57194 (5.88684) + | > current_lr: 0.00000 + | > step_time: 1.70680 (3.35918) + | > loader_time: 0.00180 (0.05231) + + + --> STEP: 80/234 -- GLOBAL_STEP: 1250 + | > loss: 2.73041 (3.00409) + | > log_mle: 0.82080 (0.84282) + | > loss_dur: 1.90962 (2.16127) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.32918 (5.86411) + | > current_lr: 0.00000 + | > step_time: 5.69940 (3.32523) + | > loader_time: 0.10260 (0.05195) + + + --> STEP: 85/234 -- GLOBAL_STEP: 1255 + | > loss: 2.83058 (2.99252) + | > log_mle: 0.81588 (0.84122) + | > loss_dur: 2.01470 (2.15130) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.58905 (5.84566) + | > current_lr: 0.00000 + | > step_time: 1.79840 (3.23562) + | > loader_time: 0.00230 (0.05077) + + + --> STEP: 90/234 -- GLOBAL_STEP: 1260 + | > loss: 2.71256 (2.98053) + | > log_mle: 0.81503 (0.83962) + | > loss_dur: 1.89753 (2.14091) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.39590 (5.82648) + | > current_lr: 0.00000 + | > step_time: 3.99850 (3.19935) + | > loader_time: 0.00130 (0.04904) + + + --> STEP: 95/234 -- GLOBAL_STEP: 1265 + | > loss: 2.87900 (2.97042) + | > log_mle: 0.80439 (0.83791) + | > loss_dur: 2.07461 (2.13252) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.59376 (5.80916) + | > current_lr: 0.00000 + | > step_time: 2.13520 (3.19249) + | > loader_time: 0.00320 (0.04834) + + + --> STEP: 100/234 -- GLOBAL_STEP: 1270 + | > loss: 2.85139 (2.96126) + | > log_mle: 0.80318 (0.83626) + | > loss_dur: 2.04821 (2.12500) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.60232 (5.79247) + | > current_lr: 0.00000 + | > step_time: 1.51070 (3.20601) + | > loader_time: 0.08680 (0.05068) + + + --> STEP: 105/234 -- GLOBAL_STEP: 1275 + | > loss: 2.76564 (2.95385) + | > log_mle: 0.80471 (0.83480) + | > loss_dur: 1.96093 (2.11906) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.49051 (5.77985) + | > current_lr: 0.00000 + | > step_time: 3.30130 (3.17584) + | > loader_time: 0.08340 (0.05104) + + + --> STEP: 110/234 -- GLOBAL_STEP: 1280 + | > loss: 2.81813 (2.94653) + | > log_mle: 0.80476 (0.83327) + | > loss_dur: 2.01336 (2.11326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.53607 (5.76660) + | > current_lr: 0.00000 + | > step_time: 3.51240 (3.14172) + | > loader_time: 0.08570 (0.05187) + + + --> STEP: 115/234 -- GLOBAL_STEP: 1285 + | > loss: 2.87341 (2.94042) + | > log_mle: 0.79810 (0.83196) + | > loss_dur: 2.07531 (2.10846) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.57994 (5.75449) + | > current_lr: 0.00000 + | > step_time: 2.20780 (3.16811) + | > loader_time: 0.05530 (0.05498) + + + --> STEP: 120/234 -- GLOBAL_STEP: 1290 + | > loss: 2.81299 (2.93378) + | > log_mle: 0.78961 (0.83070) + | > loss_dur: 2.02339 (2.10308) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.57240 (5.74219) + | > current_lr: 0.00000 + | > step_time: 2.50930 (3.11605) + | > loader_time: 0.00190 (0.05281) + + + --> STEP: 125/234 -- GLOBAL_STEP: 1295 + | > loss: 2.68637 (2.92423) + | > log_mle: 0.79116 (0.82949) + | > loss_dur: 1.89521 (2.09474) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.26343 (5.72549) + | > current_lr: 0.00000 + | > step_time: 2.58830 (3.14967) + | > loader_time: 0.00530 (0.05248) + + + --> STEP: 130/234 -- GLOBAL_STEP: 1300 + | > loss: 2.75925 (2.91797) + | > log_mle: 0.78824 (0.82812) + | > loss_dur: 1.97101 (2.08985) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.43409 (5.71343) + | > current_lr: 0.00000 + | > step_time: 2.40050 (3.10538) + | > loader_time: 0.00390 (0.05126) + + + --> STEP: 135/234 -- GLOBAL_STEP: 1305 + | > loss: 2.70343 (2.91210) + | > log_mle: 0.78966 (0.82685) + | > loss_dur: 1.91378 (2.08525) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.37894 (5.70252) + | > current_lr: 0.00000 + | > step_time: 2.10710 (3.09726) + | > loader_time: 0.08430 (0.05135) + + + --> STEP: 140/234 -- GLOBAL_STEP: 1310 + | > loss: 2.74778 (2.90778) + | > log_mle: 0.79958 (0.82562) + | > loss_dur: 1.94819 (2.08216) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.40268 (5.69318) + | > current_lr: 0.00000 + | > step_time: 2.18840 (3.11899) + | > loader_time: 0.00390 (0.05098) + + + --> STEP: 145/234 -- GLOBAL_STEP: 1315 + | > loss: 2.81570 (2.90349) + | > log_mle: 0.78276 (0.82438) + | > loss_dur: 2.03294 (2.07911) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.46649 (5.68395) + | > current_lr: 0.00000 + | > step_time: 8.32240 (3.19890) + | > loader_time: 0.29370 (0.05421) + + + --> STEP: 150/234 -- GLOBAL_STEP: 1320 + | > loss: 2.81325 (2.89795) + | > log_mle: 0.78174 (0.82302) + | > loss_dur: 2.03151 (2.07493) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.46453 (5.67331) + | > current_lr: 0.00000 + | > step_time: 1.82270 (3.21822) + | > loader_time: 0.08170 (0.05492) + + + --> STEP: 155/234 -- GLOBAL_STEP: 1325 + | > loss: 2.83498 (2.89372) + | > log_mle: 0.78112 (0.82169) + | > loss_dur: 2.05386 (2.07203) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.47956 (5.66373) + | > current_lr: 0.00000 + | > step_time: 1.49320 (3.21293) + | > loader_time: 0.00510 (0.05384) + + + --> STEP: 160/234 -- GLOBAL_STEP: 1330 + | > loss: 2.74174 (2.89001) + | > log_mle: 0.77424 (0.82038) + | > loss_dur: 1.96750 (2.06963) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.33444 (5.65546) + | > current_lr: 0.00000 + | > step_time: 2.20060 (3.21126) + | > loader_time: 0.07140 (0.05330) + + + --> STEP: 165/234 -- GLOBAL_STEP: 1335 + | > loss: 2.75574 (2.88579) + | > log_mle: 0.77620 (0.81903) + | > loss_dur: 1.97953 (2.06676) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.31941 (5.64600) + | > current_lr: 0.00000 + | > step_time: 3.80540 (3.18500) + | > loader_time: 0.10300 (0.05294) + + + --> STEP: 170/234 -- GLOBAL_STEP: 1340 + | > loss: 2.82478 (2.88408) + | > log_mle: 0.77438 (0.81777) + | > loss_dur: 2.05041 (2.06631) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.39920 (5.64073) + | > current_lr: 0.00000 + | > step_time: 5.90470 (3.27394) + | > loader_time: 0.00570 (0.05365) + + + --> STEP: 175/234 -- GLOBAL_STEP: 1345 + | > loss: 2.77406 (2.88112) + | > log_mle: 0.77104 (0.81650) + | > loss_dur: 2.00303 (2.06463) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.35909 (5.63321) + | > current_lr: 0.00000 + | > step_time: 5.51540 (3.30680) + | > loader_time: 0.18070 (0.05539) + + + --> STEP: 180/234 -- GLOBAL_STEP: 1350 + | > loss: 2.76230 (2.87800) + | > log_mle: 0.76963 (0.81521) + | > loss_dur: 1.99268 (2.06279) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.34395 (5.62536) + | > current_lr: 0.00000 + | > step_time: 7.98610 (3.34165) + | > loader_time: 0.01740 (0.05497) + + + --> STEP: 185/234 -- GLOBAL_STEP: 1355 + | > loss: 2.83203 (2.87590) + | > log_mle: 0.77860 (0.81406) + | > loss_dur: 2.05343 (2.06184) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.40000 (5.61894) + | > current_lr: 0.00000 + | > step_time: 7.19660 (3.35926) + | > loader_time: 0.10060 (0.05612) + + + --> STEP: 190/234 -- GLOBAL_STEP: 1360 + | > loss: 2.74277 (2.87262) + | > log_mle: 0.76445 (0.81282) + | > loss_dur: 1.97832 (2.05980) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.32210 (5.61117) + | > current_lr: 0.00000 + | > step_time: 3.21890 (3.44096) + | > loader_time: 0.00300 (0.05574) + + + --> STEP: 195/234 -- GLOBAL_STEP: 1365 + | > loss: 2.83284 (2.87008) + | > log_mle: 0.77182 (0.81170) + | > loss_dur: 2.06101 (2.05838) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.42802 (5.60430) + | > current_lr: 0.00000 + | > step_time: 3.40030 (3.44905) + | > loader_time: 0.08500 (0.05628) + + + --> STEP: 200/234 -- GLOBAL_STEP: 1370 + | > loss: 2.74429 (2.86680) + | > log_mle: 0.76553 (0.81055) + | > loss_dur: 1.97876 (2.05625) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.26822 (5.59666) + | > current_lr: 0.00000 + | > step_time: 3.29470 (3.50279) + | > loader_time: 0.00420 (0.05675) + + + --> STEP: 205/234 -- GLOBAL_STEP: 1375 + | > loss: 2.78005 (2.86345) + | > log_mle: 0.76260 (0.80927) + | > loss_dur: 2.01746 (2.05418) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.36512 (5.58922) + | > current_lr: 0.00000 + | > step_time: 3.38870 (3.54514) + | > loader_time: 0.11880 (0.05697) + + + --> STEP: 210/234 -- GLOBAL_STEP: 1380 + | > loss: 2.77083 (2.86083) + | > log_mle: 0.75335 (0.80800) + | > loss_dur: 2.01748 (2.05283) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.32049 (5.58275) + | > current_lr: 0.00000 + | > step_time: 2.51430 (3.57511) + | > loader_time: 0.08270 (0.05689) + + + --> STEP: 215/234 -- GLOBAL_STEP: 1385 + | > loss: 2.73927 (2.85869) + | > log_mle: 0.75491 (0.80669) + | > loss_dur: 1.98435 (2.05200) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.29571 (5.57704) + | > current_lr: 0.00000 + | > step_time: 5.89920 (3.65750) + | > loader_time: 0.30610 (0.06770) + + + --> STEP: 220/234 -- GLOBAL_STEP: 1390 + | > loss: 2.79213 (2.85654) + | > log_mle: 0.74931 (0.80538) + | > loss_dur: 2.04282 (2.05116) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.37758 (5.57128) + | > current_lr: 0.00000 + | > step_time: 1.70650 (3.63631) + | > loader_time: 0.00370 (0.06703) + + + --> STEP: 225/234 -- GLOBAL_STEP: 1395 + | > loss: 2.78470 (2.85514) + | > log_mle: 0.75724 (0.80417) + | > loss_dur: 2.02746 (2.05098) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.36067 (5.56687) + | > current_lr: 0.00000 + | > step_time: 1.70150 (3.61989) + | > loader_time: 0.00470 (0.06639) + + + --> STEP: 230/234 -- GLOBAL_STEP: 1400 + | > loss: 3.02457 (2.85552) + | > log_mle: 0.73925 (0.80300) + | > loss_dur: 2.28532 (2.05252) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.65345 (5.56429) + | > current_lr: 0.00000 + | > step_time: 0.26200 (3.56068) + | > loader_time: 0.00430 (0.06503) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.97471 (+0.74907) + | > avg_loss: 2.68134 (-0.25957) + | > avg_log_mle: 0.75120 (-0.08839) + | > avg_loss_dur: 1.93014 (-0.17118) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_1404.pth + + > EPOCH: 6/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 09:56:43)  + + --> STEP: 1/234 -- GLOBAL_STEP: 1405 + | > loss: 3.49179 (3.49179) + | > log_mle: 0.81891 (0.81891) + | > loss_dur: 2.67288 (2.67288) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.63468 (6.63468) + | > current_lr: 0.00000 + | > step_time: 4.51340 (4.51336) + | > loader_time: 4.68850 (4.68845) + + + --> STEP: 6/234 -- GLOBAL_STEP: 1410 + | > loss: 3.31150 (3.41971) + | > log_mle: 0.79965 (0.81185) + | > loss_dur: 2.51184 (2.60786) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.29585 (6.42861) + | > current_lr: 0.00000 + | > step_time: 4.31090 (5.69057) + | > loader_time: 0.00290 (0.86536) + + + --> STEP: 11/234 -- GLOBAL_STEP: 1415 + | > loss: 2.91819 (3.24003) + | > log_mle: 0.78229 (0.79679) + | > loss_dur: 2.13589 (2.44324) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.60711 (6.09985) + | > current_lr: 0.00000 + | > step_time: 4.08460 (5.27682) + | > loader_time: 0.00100 (0.48128) + + + --> STEP: 16/234 -- GLOBAL_STEP: 1420 + | > loss: 2.74258 (3.11219) + | > log_mle: 0.76219 (0.78917) + | > loss_dur: 1.98039 (2.32302) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.20339 (5.88407) + | > current_lr: 0.00000 + | > step_time: 2.99890 (4.97939) + | > loader_time: 0.00300 (0.34366) + + + --> STEP: 21/234 -- GLOBAL_STEP: 1425 + | > loss: 2.70050 (3.04129) + | > log_mle: 0.77730 (0.78572) + | > loss_dur: 1.92321 (2.25557) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.20260 (5.76669) + | > current_lr: 0.00000 + | > step_time: 1.20260 (5.18542) + | > loader_time: 0.00170 (0.28044) + + + --> STEP: 26/234 -- GLOBAL_STEP: 1430 + | > loss: 2.69177 (2.97615) + | > log_mle: 0.76957 (0.78202) + | > loss_dur: 1.92220 (2.19413) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.18585 (5.65973) + | > current_lr: 0.00000 + | > step_time: 4.68230 (4.91760) + | > loader_time: 0.00120 (0.23374) + + + --> STEP: 31/234 -- GLOBAL_STEP: 1435 + | > loss: 2.72309 (2.92612) + | > log_mle: 0.75424 (0.77809) + | > loss_dur: 1.96885 (2.14803) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.25594 (5.58622) + | > current_lr: 0.00000 + | > step_time: 5.20260 (4.79864) + | > loader_time: 0.00190 (0.20240) + + + --> STEP: 36/234 -- GLOBAL_STEP: 1440 + | > loss: 2.71816 (2.89260) + | > log_mle: 0.75354 (0.77517) + | > loss_dur: 1.96462 (2.11743) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.26862 (5.53379) + | > current_lr: 0.00000 + | > step_time: 3.29730 (4.78786) + | > loader_time: 0.10070 (0.19856) + + + --> STEP: 41/234 -- GLOBAL_STEP: 1445 + | > loss: 2.63377 (2.86367) + | > log_mle: 0.74211 (0.77241) + | > loss_dur: 1.89166 (2.09127) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.16793 (5.49268) + | > current_lr: 0.00000 + | > step_time: 7.09210 (4.68235) + | > loader_time: 0.09810 (0.17937) + + + --> STEP: 46/234 -- GLOBAL_STEP: 1450 + | > loss: 2.56840 (2.83953) + | > log_mle: 0.74932 (0.77119) + | > loss_dur: 1.81909 (2.06834) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.06706 (5.45537) + | > current_lr: 0.00000 + | > step_time: 1.29470 (4.36684) + | > loader_time: 0.00510 (0.16211) + + + --> STEP: 51/234 -- GLOBAL_STEP: 1455 + | > loss: 2.55097 (2.81157) + | > log_mle: 0.74684 (0.76842) + | > loss_dur: 1.80413 (2.04315) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.02346 (5.41610) + | > current_lr: 0.00000 + | > step_time: 1.29020 (4.14668) + | > loader_time: 0.00170 (0.14813) + + + --> STEP: 56/234 -- GLOBAL_STEP: 1460 + | > loss: 2.64030 (2.79244) + | > log_mle: 0.74223 (0.76606) + | > loss_dur: 1.89807 (2.02638) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.14479 (5.38946) + | > current_lr: 0.00000 + | > step_time: 1.62960 (3.96631) + | > loader_time: 0.00210 (0.13517) + + + --> STEP: 61/234 -- GLOBAL_STEP: 1465 + | > loss: 2.53976 (2.77158) + | > log_mle: 0.73570 (0.76376) + | > loss_dur: 1.80406 (2.00783) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.03571 (5.35867) + | > current_lr: 0.00000 + | > step_time: 3.82470 (3.80121) + | > loader_time: 0.09330 (0.12713) + + + --> STEP: 66/234 -- GLOBAL_STEP: 1470 + | > loss: 2.47160 (2.75465) + | > log_mle: 0.73370 (0.76148) + | > loss_dur: 1.73789 (1.99318) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.92071 (5.33590) + | > current_lr: 0.00000 + | > step_time: 1.08170 (3.63000) + | > loader_time: 0.00200 (0.11766) + + + --> STEP: 71/234 -- GLOBAL_STEP: 1475 + | > loss: 2.57386 (2.73931) + | > log_mle: 0.73550 (0.75960) + | > loss_dur: 1.83835 (1.97971) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.93332 (5.31290) + | > current_lr: 0.00000 + | > step_time: 1.82060 (3.52834) + | > loader_time: 0.00260 (0.11200) + + + --> STEP: 76/234 -- GLOBAL_STEP: 1480 + | > loss: 2.53387 (2.72297) + | > log_mle: 0.72724 (0.75759) + | > loss_dur: 1.80663 (1.96538) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.04674 (5.29132) + | > current_lr: 0.00000 + | > step_time: 1.38910 (3.43857) + | > loader_time: 0.00340 (0.10676) + + + --> STEP: 81/234 -- GLOBAL_STEP: 1485 + | > loss: 2.46482 (2.70746) + | > log_mle: 0.71857 (0.75551) + | > loss_dur: 1.74625 (1.95195) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.92876 (5.26902) + | > current_lr: 0.00000 + | > step_time: 2.59520 (3.34726) + | > loader_time: 0.00340 (0.10133) + + + --> STEP: 86/234 -- GLOBAL_STEP: 1490 + | > loss: 2.52806 (2.69574) + | > log_mle: 0.72713 (0.75361) + | > loss_dur: 1.80094 (1.94213) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.00305 (5.25210) + | > current_lr: 0.00000 + | > step_time: 3.21460 (3.31432) + | > loader_time: 0.19640 (0.09890) + + + --> STEP: 91/234 -- GLOBAL_STEP: 1495 + | > loss: 2.42967 (2.68145) + | > log_mle: 0.71942 (0.75149) + | > loss_dur: 1.71025 (1.92995) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.79845 (5.23089) + | > current_lr: 0.00000 + | > step_time: 2.30000 (3.26174) + | > loader_time: 0.08020 (0.09657) + + + --> STEP: 96/234 -- GLOBAL_STEP: 1500 + | > loss: 2.47078 (2.67075) + | > log_mle: 0.71479 (0.74934) + | > loss_dur: 1.75600 (1.92141) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.91944 (5.21595) + | > current_lr: 0.00000 + | > step_time: 2.21270 (3.19944) + | > loader_time: 0.18990 (0.09543) + + + --> STEP: 101/234 -- GLOBAL_STEP: 1505 + | > loss: 2.45592 (2.66026) + | > log_mle: 0.70505 (0.74725) + | > loss_dur: 1.75087 (1.91300) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.85661 (5.19933) + | > current_lr: 0.00000 + | > step_time: 1.38570 (3.16941) + | > loader_time: 0.00180 (0.09253) + + + --> STEP: 106/234 -- GLOBAL_STEP: 1510 + | > loss: 2.50389 (2.65122) + | > log_mle: 0.70721 (0.74539) + | > loss_dur: 1.79669 (1.90583) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.87242 (5.18473) + | > current_lr: 0.00000 + | > step_time: 1.70830 (3.10872) + | > loader_time: 0.00350 (0.08984) + + + --> STEP: 111/234 -- GLOBAL_STEP: 1515 + | > loss: 2.51389 (2.64241) + | > log_mle: 0.70707 (0.74352) + | > loss_dur: 1.80681 (1.89889) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.94573 (5.17118) + | > current_lr: 0.00000 + | > step_time: 1.11820 (3.08624) + | > loader_time: 0.00360 (0.08927) + + + --> STEP: 116/234 -- GLOBAL_STEP: 1520 + | > loss: 2.44122 (2.63396) + | > log_mle: 0.70909 (0.74185) + | > loss_dur: 1.73213 (1.89211) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.80087 (5.15747) + | > current_lr: 0.00000 + | > step_time: 3.40730 (3.05081) + | > loader_time: 0.00270 (0.08706) + + + --> STEP: 121/234 -- GLOBAL_STEP: 1525 + | > loss: 2.31812 (2.62454) + | > log_mle: 0.71508 (0.74029) + | > loss_dur: 1.60304 (1.88425) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.57971 (5.14174) + | > current_lr: 0.00000 + | > step_time: 1.89900 (3.02069) + | > loader_time: 0.02180 (0.08602) + + + --> STEP: 126/234 -- GLOBAL_STEP: 1530 + | > loss: 2.47083 (2.61474) + | > log_mle: 0.68908 (0.73848) + | > loss_dur: 1.78175 (1.87626) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.93582 (5.12638) + | > current_lr: 0.00000 + | > step_time: 2.50860 (2.99618) + | > loader_time: 0.00930 (0.08347) + + + --> STEP: 131/234 -- GLOBAL_STEP: 1535 + | > loss: 2.41219 (2.60633) + | > log_mle: 0.69231 (0.73673) + | > loss_dur: 1.71988 (1.86961) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.75278 (5.11191) + | > current_lr: 0.00000 + | > step_time: 3.69550 (2.99467) + | > loader_time: 0.00590 (0.08043) + + + --> STEP: 136/234 -- GLOBAL_STEP: 1540 + | > loss: 2.50669 (2.59949) + | > log_mle: 0.68183 (0.73498) + | > loss_dur: 1.82486 (1.86451) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.95357 (5.10138) + | > current_lr: 0.00000 + | > step_time: 4.30400 (2.98528) + | > loader_time: 0.09250 (0.07944) + + + --> STEP: 141/234 -- GLOBAL_STEP: 1545 + | > loss: 2.39168 (2.59296) + | > log_mle: 0.68464 (0.73337) + | > loss_dur: 1.70704 (1.85959) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.76498 (5.09112) + | > current_lr: 0.00000 + | > step_time: 3.40890 (2.98159) + | > loader_time: 0.00370 (0.07858) + + + --> STEP: 146/234 -- GLOBAL_STEP: 1550 + | > loss: 2.40973 (2.58734) + | > log_mle: 0.67866 (0.73172) + | > loss_dur: 1.73107 (1.85563) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.70389 (5.08098) + | > current_lr: 0.00000 + | > step_time: 4.71140 (3.04669) + | > loader_time: 0.08840 (0.07774) + + + --> STEP: 151/234 -- GLOBAL_STEP: 1555 + | > loss: 2.33408 (2.57952) + | > log_mle: 0.67789 (0.72998) + | > loss_dur: 1.65619 (1.84954) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.65455 (5.06797) + | > current_lr: 0.00000 + | > step_time: 6.41140 (3.09862) + | > loader_time: 0.19440 (0.07853) + + + --> STEP: 156/234 -- GLOBAL_STEP: 1560 + | > loss: 2.38489 (2.57398) + | > log_mle: 0.67188 (0.72824) + | > loss_dur: 1.71301 (1.84574) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.79798 (5.05816) + | > current_lr: 0.00000 + | > step_time: 5.59920 (3.13271) + | > loader_time: 0.11110 (0.07813) + + + --> STEP: 161/234 -- GLOBAL_STEP: 1565 + | > loss: 2.45453 (2.56895) + | > log_mle: 0.66736 (0.72652) + | > loss_dur: 1.78716 (1.84242) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.82101 (5.04889) + | > current_lr: 0.00000 + | > step_time: 5.50780 (3.14298) + | > loader_time: 0.09550 (0.07807) + + + --> STEP: 166/234 -- GLOBAL_STEP: 1570 + | > loss: 2.36044 (2.56239) + | > log_mle: 0.66866 (0.72478) + | > loss_dur: 1.69177 (1.83761) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.69867 (5.03747) + | > current_lr: 0.00000 + | > step_time: 1.70680 (3.16296) + | > loader_time: 0.00430 (0.07764) + + + --> STEP: 171/234 -- GLOBAL_STEP: 1575 + | > loss: 2.40405 (2.55881) + | > log_mle: 0.65857 (0.72307) + | > loss_dur: 1.74548 (1.83573) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.76805 (5.03114) + | > current_lr: 0.00000 + | > step_time: 2.30930 (3.15831) + | > loader_time: 0.00310 (0.07717) + + + --> STEP: 176/234 -- GLOBAL_STEP: 1580 + | > loss: 2.34054 (2.55354) + | > log_mle: 0.65667 (0.72138) + | > loss_dur: 1.68388 (1.83216) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.60450 (5.02087) + | > current_lr: 0.00000 + | > step_time: 2.30440 (3.19923) + | > loader_time: 0.01130 (0.08078) + + + --> STEP: 181/234 -- GLOBAL_STEP: 1585 + | > loss: 2.36511 (2.54870) + | > log_mle: 0.66005 (0.71966) + | > loss_dur: 1.70506 (1.82904) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.71865 (5.01182) + | > current_lr: 0.00000 + | > step_time: 7.31110 (3.21393) + | > loader_time: 0.19370 (0.08219) + + + --> STEP: 186/234 -- GLOBAL_STEP: 1590 + | > loss: 2.38496 (2.54480) + | > log_mle: 0.65767 (0.71810) + | > loss_dur: 1.72728 (1.82670) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.71905 (5.00401) + | > current_lr: 0.00000 + | > step_time: 3.69790 (3.18778) + | > loader_time: 0.08990 (0.08153) + + + --> STEP: 191/234 -- GLOBAL_STEP: 1595 + | > loss: 2.37149 (2.53968) + | > log_mle: 0.66665 (0.71650) + | > loss_dur: 1.70484 (1.82318) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.68209 (4.99472) + | > current_lr: 0.00000 + | > step_time: 2.69490 (3.22001) + | > loader_time: 0.00330 (0.08041) + + + --> STEP: 196/234 -- GLOBAL_STEP: 1600 + | > loss: 2.38276 (2.53533) + | > log_mle: 0.67175 (0.71497) + | > loss_dur: 1.71101 (1.82036) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.66814 (4.98633) + | > current_lr: 0.00000 + | > step_time: 6.21170 (3.27052) + | > loader_time: 0.00410 (0.07898) + + + --> STEP: 201/234 -- GLOBAL_STEP: 1605 + | > loss: 2.26052 (2.52957) + | > log_mle: 0.64510 (0.71328) + | > loss_dur: 1.61542 (1.81628) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.48812 (4.97552) + | > current_lr: 0.00000 + | > step_time: 11.10110 (3.41805) + | > loader_time: 0.10640 (0.07812) + + + --> STEP: 206/234 -- GLOBAL_STEP: 1610 + | > loss: 2.31263 (2.52480) + | > log_mle: 0.63650 (0.71155) + | > loss_dur: 1.67613 (1.81325) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.62306 (4.96722) + | > current_lr: 0.00000 + | > step_time: 3.50270 (3.46297) + | > loader_time: 0.08370 (0.07984) + + + --> STEP: 211/234 -- GLOBAL_STEP: 1615 + | > loss: 2.42775 (2.52096) + | > log_mle: 0.62988 (0.70982) + | > loss_dur: 1.79788 (1.81114) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.77770 (4.96017) + | > current_lr: 0.00000 + | > step_time: 7.91250 (3.54814) + | > loader_time: 0.08380 (0.07931) + + + --> STEP: 216/234 -- GLOBAL_STEP: 1620 + | > loss: 2.31156 (2.51627) + | > log_mle: 0.63349 (0.70807) + | > loss_dur: 1.67808 (1.80820) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.55439 (4.95123) + | > current_lr: 0.00000 + | > step_time: 12.90570 (3.60264) + | > loader_time: 0.19990 (0.07972) + + + --> STEP: 221/234 -- GLOBAL_STEP: 1625 + | > loss: 2.30655 (2.51205) + | > log_mle: 0.63742 (0.70632) + | > loss_dur: 1.66914 (1.80573) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.57263 (4.94355) + | > current_lr: 0.00000 + | > step_time: 3.60030 (3.62488) + | > loader_time: 0.00740 (0.07880) + + + --> STEP: 226/234 -- GLOBAL_STEP: 1630 + | > loss: 2.34277 (2.50873) + | > log_mle: 0.62249 (0.70459) + | > loss_dur: 1.72029 (1.80414) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.65382 (4.93738) + | > current_lr: 0.00000 + | > step_time: 0.23990 (3.57885) + | > loader_time: 0.00420 (0.07754) + + + --> STEP: 231/234 -- GLOBAL_STEP: 1635 + | > loss: 2.62102 (2.50822) + | > log_mle: 0.61186 (0.70291) + | > loss_dur: 2.00917 (1.80531) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.05848 (4.93537) + | > current_lr: 0.00000 + | > step_time: 0.26890 (3.50682) + | > loader_time: 0.00480 (0.07595) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27202 (-0.70269) + | > avg_loss: 2.24330 (-0.43804) + | > avg_log_mle: 0.63055 (-0.12066) + | > avg_loss_dur: 1.61275 (-0.31739) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_1638.pth + + > EPOCH: 7/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 10:11:32)  + + --> STEP: 2/234 -- GLOBAL_STEP: 1640 + | > loss: 3.02142 (3.00550) + | > log_mle: 0.74543 (0.72989) + | > loss_dur: 2.27599 (2.27561) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.52570 (5.59650) + | > current_lr: 0.00000 + | > step_time: 1.32290 (1.75902) + | > loader_time: 4.99670 (2.54396) + + + --> STEP: 7/234 -- GLOBAL_STEP: 1645 + | > loss: 2.62635 (2.87561) + | > log_mle: 0.66453 (0.69906) + | > loss_dur: 1.96182 (2.17655) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.96925 (5.40219) + | > current_lr: 0.00000 + | > step_time: 2.40760 (2.82979) + | > loader_time: 0.00200 (1.28760) + + + --> STEP: 12/234 -- GLOBAL_STEP: 1650 + | > loss: 2.40538 (2.72027) + | > log_mle: 0.67758 (0.68777) + | > loss_dur: 1.72780 (2.03250) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.65965 (5.13628) + | > current_lr: 0.00000 + | > step_time: 8.70090 (3.50147) + | > loader_time: 0.00280 (0.95898) + + + --> STEP: 17/234 -- GLOBAL_STEP: 1655 + | > loss: 2.42140 (2.62321) + | > log_mle: 0.67238 (0.67940) + | > loss_dur: 1.74902 (1.94381) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.66388 (4.97339) + | > current_lr: 0.00000 + | > step_time: 2.80910 (3.40139) + | > loader_time: 0.00210 (0.67803) + + + --> STEP: 22/234 -- GLOBAL_STEP: 1660 + | > loss: 2.29042 (2.55991) + | > log_mle: 0.64907 (0.67447) + | > loss_dur: 1.64134 (1.88544) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.48509 (4.87803) + | > current_lr: 0.00000 + | > step_time: 3.11700 (4.17124) + | > loader_time: 0.08180 (0.62719) + + + --> STEP: 27/234 -- GLOBAL_STEP: 1665 + | > loss: 2.18012 (2.50445) + | > log_mle: 0.64089 (0.67030) + | > loss_dur: 1.53923 (1.83415) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.30329 (4.79273) + | > current_lr: 0.00000 + | > step_time: 1.50040 (4.50613) + | > loader_time: 0.08380 (0.52534) + + + --> STEP: 32/234 -- GLOBAL_STEP: 1670 + | > loss: 2.15368 (2.46129) + | > log_mle: 0.62125 (0.66523) + | > loss_dur: 1.53243 (1.79606) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.35944 (4.73655) + | > current_lr: 0.00000 + | > step_time: 4.39760 (4.33406) + | > loader_time: 0.00560 (0.44933) + + + --> STEP: 37/234 -- GLOBAL_STEP: 1675 + | > loss: 2.15974 (2.43335) + | > log_mle: 0.63186 (0.66227) + | > loss_dur: 1.52787 (1.77107) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.40174 (4.70321) + | > current_lr: 0.00000 + | > step_time: 1.62630 (4.21221) + | > loader_time: 0.18460 (0.40427) + + + --> STEP: 42/234 -- GLOBAL_STEP: 1680 + | > loss: 2.29505 (2.41099) + | > log_mle: 0.67555 (0.66009) + | > loss_dur: 1.61950 (1.75090) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.57884 (4.67652) + | > current_lr: 0.00000 + | > step_time: 2.02390 (3.92701) + | > loader_time: 0.07850 (0.35819) + + + --> STEP: 47/234 -- GLOBAL_STEP: 1685 + | > loss: 2.11127 (2.38693) + | > log_mle: 0.62185 (0.65706) + | > loss_dur: 1.48942 (1.72987) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.30534 (4.64694) + | > current_lr: 0.00000 + | > step_time: 1.72990 (3.70404) + | > loader_time: 0.00160 (0.32212) + + + --> STEP: 52/234 -- GLOBAL_STEP: 1690 + | > loss: 2.16646 (2.36288) + | > log_mle: 0.64305 (0.65426) + | > loss_dur: 1.52342 (1.70862) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.40778 (4.61535) + | > current_lr: 0.00000 + | > step_time: 1.70100 (3.48432) + | > loader_time: 0.00210 (0.29135) + + + --> STEP: 57/234 -- GLOBAL_STEP: 1695 + | > loss: 2.13646 (2.34446) + | > log_mle: 0.63816 (0.65138) + | > loss_dur: 1.49829 (1.69308) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.28217 (4.59085) + | > current_lr: 0.00000 + | > step_time: 4.50070 (3.37330) + | > loader_time: 0.07060 (0.26874) + + + --> STEP: 62/234 -- GLOBAL_STEP: 1700 + | > loss: 2.20067 (2.32641) + | > log_mle: 0.62335 (0.64846) + | > loss_dur: 1.57732 (1.67795) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.46919 (4.56989) + | > current_lr: 0.00000 + | > step_time: 3.52210 (3.32214) + | > loader_time: 0.08330 (0.24863) + + + --> STEP: 67/234 -- GLOBAL_STEP: 1705 + | > loss: 2.08001 (2.30888) + | > log_mle: 0.60503 (0.64563) + | > loss_dur: 1.47499 (1.66326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.27513 (4.54729) + | > current_lr: 0.00000 + | > step_time: 1.60640 (3.26056) + | > loader_time: 0.00240 (0.23290) + + + --> STEP: 72/234 -- GLOBAL_STEP: 1710 + | > loss: 2.02935 (2.29313) + | > log_mle: 0.60711 (0.64340) + | > loss_dur: 1.42224 (1.64973) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.18611 (4.52469) + | > current_lr: 0.00000 + | > step_time: 1.73550 (3.17197) + | > loader_time: 0.00240 (0.21697) + + + --> STEP: 77/234 -- GLOBAL_STEP: 1715 + | > loss: 2.04595 (2.27817) + | > log_mle: 0.59721 (0.64089) + | > loss_dur: 1.44875 (1.63728) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.16666 (4.50581) + | > current_lr: 0.00000 + | > step_time: 1.80530 (3.07255) + | > loader_time: 0.09240 (0.20421) + + + --> STEP: 82/234 -- GLOBAL_STEP: 1720 + | > loss: 1.97533 (2.26279) + | > log_mle: 0.59972 (0.63851) + | > loss_dur: 1.37561 (1.62428) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.13078 (4.48598) + | > current_lr: 0.00000 + | > step_time: 5.48750 (3.14014) + | > loader_time: 0.10710 (0.19780) + + + --> STEP: 87/234 -- GLOBAL_STEP: 1725 + | > loss: 2.00805 (2.25198) + | > log_mle: 0.59538 (0.63624) + | > loss_dur: 1.41267 (1.61574) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.18433 (4.47271) + | > current_lr: 0.00000 + | > step_time: 0.78710 (3.07493) + | > loader_time: 0.00290 (0.18869) + + + --> STEP: 92/234 -- GLOBAL_STEP: 1730 + | > loss: 1.98427 (2.23788) + | > log_mle: 0.58285 (0.63362) + | > loss_dur: 1.40142 (1.60426) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.13456 (4.45292) + | > current_lr: 0.00000 + | > step_time: 1.40430 (2.98971) + | > loader_time: 0.00220 (0.17944) + + + --> STEP: 97/234 -- GLOBAL_STEP: 1735 + | > loss: 1.98308 (2.22671) + | > log_mle: 0.58298 (0.63106) + | > loss_dur: 1.40009 (1.59565) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.06354 (4.43729) + | > current_lr: 0.00000 + | > step_time: 1.61500 (2.94378) + | > loader_time: 0.08850 (0.17123) + + + --> STEP: 102/234 -- GLOBAL_STEP: 1740 + | > loss: 1.96333 (2.21614) + | > log_mle: 0.59021 (0.62872) + | > loss_dur: 1.37312 (1.58742) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.00440 (4.42133) + | > current_lr: 0.00000 + | > step_time: 1.90410 (2.92505) + | > loader_time: 0.08570 (0.16567) + + + --> STEP: 107/234 -- GLOBAL_STEP: 1745 + | > loss: 1.96218 (2.20716) + | > log_mle: 0.57267 (0.62636) + | > loss_dur: 1.38952 (1.58080) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.06589 (4.40813) + | > current_lr: 0.00000 + | > step_time: 1.59460 (2.89977) + | > loader_time: 0.09690 (0.15896) + + + --> STEP: 112/234 -- GLOBAL_STEP: 1750 + | > loss: 1.98780 (2.19833) + | > log_mle: 0.58125 (0.62422) + | > loss_dur: 1.40655 (1.57412) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.11040 (4.39478) + | > current_lr: 0.00000 + | > step_time: 1.45900 (2.89011) + | > loader_time: 0.00250 (0.15353) + + + --> STEP: 117/234 -- GLOBAL_STEP: 1755 + | > loss: 1.93796 (2.18956) + | > log_mle: 0.57059 (0.62214) + | > loss_dur: 1.36737 (1.56743) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.01977 (4.38177) + | > current_lr: 0.00000 + | > step_time: 2.39400 (2.83595) + | > loader_time: 0.00280 (0.14909) + + + --> STEP: 122/234 -- GLOBAL_STEP: 1760 + | > loss: 1.92302 (2.18024) + | > log_mle: 0.57047 (0.62027) + | > loss_dur: 1.35256 (1.55996) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.97149 (4.36706) + | > current_lr: 0.00000 + | > step_time: 2.60790 (2.80009) + | > loader_time: 0.00430 (0.14365) + + + --> STEP: 127/234 -- GLOBAL_STEP: 1765 + | > loss: 1.96753 (2.17093) + | > log_mle: 0.56371 (0.61809) + | > loss_dur: 1.40383 (1.55285) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.04310 (4.35241) + | > current_lr: 0.00000 + | > step_time: 1.39980 (2.76326) + | > loader_time: 0.00520 (0.13887) + + + --> STEP: 132/234 -- GLOBAL_STEP: 1770 + | > loss: 1.91394 (2.16216) + | > log_mle: 0.56458 (0.61595) + | > loss_dur: 1.34936 (1.54621) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.92480 (4.33835) + | > current_lr: 0.00000 + | > step_time: 2.40840 (2.74277) + | > loader_time: 0.07600 (0.13571) + + + --> STEP: 137/234 -- GLOBAL_STEP: 1775 + | > loss: 2.02556 (2.15564) + | > log_mle: 0.56542 (0.61379) + | > loss_dur: 1.46014 (1.54184) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.18709 (4.32918) + | > current_lr: 0.00000 + | > step_time: 2.59410 (2.75589) + | > loader_time: 0.00660 (0.13091) + + + --> STEP: 142/234 -- GLOBAL_STEP: 1780 + | > loss: 1.97026 (2.14858) + | > log_mle: 0.55605 (0.61177) + | > loss_dur: 1.41421 (1.53681) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.02483 (4.31837) + | > current_lr: 0.00000 + | > step_time: 1.38620 (2.74693) + | > loader_time: 0.00400 (0.12884) + + + --> STEP: 147/234 -- GLOBAL_STEP: 1785 + | > loss: 1.88553 (2.14222) + | > log_mle: 0.54823 (0.60961) + | > loss_dur: 1.33729 (1.53261) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.91699 (4.30829) + | > current_lr: 0.00000 + | > step_time: 1.97110 (2.73389) + | > loader_time: 0.00450 (0.12573) + + + --> STEP: 152/234 -- GLOBAL_STEP: 1790 + | > loss: 1.93770 (2.13493) + | > log_mle: 0.53696 (0.60740) + | > loss_dur: 1.40074 (1.52753) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.96197 (4.29675) + | > current_lr: 0.00000 + | > step_time: 6.79320 (2.73787) + | > loader_time: 0.09500 (0.12293) + + + --> STEP: 157/234 -- GLOBAL_STEP: 1795 + | > loss: 1.95313 (2.12925) + | > log_mle: 0.54144 (0.60520) + | > loss_dur: 1.41169 (1.52405) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.97613 (4.28745) + | > current_lr: 0.00000 + | > step_time: 5.39960 (2.75841) + | > loader_time: 0.10190 (0.12219) + + + --> STEP: 162/234 -- GLOBAL_STEP: 1800 + | > loss: 1.91099 (2.12363) + | > log_mle: 0.53264 (0.60298) + | > loss_dur: 1.37835 (1.52064) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.89929 (4.27791) + | > current_lr: 0.00000 + | > step_time: 3.79480 (2.73532) + | > loader_time: 0.00330 (0.12010) + + + --> STEP: 167/234 -- GLOBAL_STEP: 1805 + | > loss: 1.97480 (2.11750) + | > log_mle: 0.52460 (0.60077) + | > loss_dur: 1.45020 (1.51673) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.05920 (4.26774) + | > current_lr: 0.00000 + | > step_time: 4.30400 (2.81521) + | > loader_time: 0.07650 (0.11920) + + + --> STEP: 172/234 -- GLOBAL_STEP: 1810 + | > loss: 1.96001 (2.11306) + | > log_mle: 0.52766 (0.59854) + | > loss_dur: 1.43235 (1.51452) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.98067 (4.26027) + | > current_lr: 0.00000 + | > step_time: 1.31380 (2.80618) + | > loader_time: 0.00280 (0.11680) + + + --> STEP: 177/234 -- GLOBAL_STEP: 1815 + | > loss: 1.95651 (2.10771) + | > log_mle: 0.52985 (0.59631) + | > loss_dur: 1.42666 (1.51140) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.98987 (4.25135) + | > current_lr: 0.00000 + | > step_time: 3.18090 (2.79504) + | > loader_time: 0.01080 (0.11475) + + + --> STEP: 182/234 -- GLOBAL_STEP: 1820 + | > loss: 1.99324 (2.10290) + | > log_mle: 0.51900 (0.59403) + | > loss_dur: 1.47423 (1.50887) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.03781 (4.24307) + | > current_lr: 0.00000 + | > step_time: 9.98900 (2.86033) + | > loader_time: 0.20270 (0.11445) + + + --> STEP: 187/234 -- GLOBAL_STEP: 1825 + | > loss: 1.86731 (2.09827) + | > log_mle: 0.50807 (0.59192) + | > loss_dur: 1.35924 (1.50635) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.78831 (4.23503) + | > current_lr: 0.00000 + | > step_time: 8.29080 (2.96086) + | > loader_time: 0.00410 (0.11356) + + + --> STEP: 192/234 -- GLOBAL_STEP: 1830 + | > loss: 1.90131 (2.09325) + | > log_mle: 0.49990 (0.58977) + | > loss_dur: 1.40141 (1.50348) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.94060 (4.22655) + | > current_lr: 0.00000 + | > step_time: 3.50840 (2.99316) + | > loader_time: 0.06890 (0.11215) + + + --> STEP: 197/234 -- GLOBAL_STEP: 1835 + | > loss: 1.85707 (2.08875) + | > log_mle: 0.49852 (0.58773) + | > loss_dur: 1.35855 (1.50102) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.83961 (4.21846) + | > current_lr: 0.00000 + | > step_time: 6.28410 (3.05628) + | > loader_time: 0.00750 (0.11029) + + + --> STEP: 202/234 -- GLOBAL_STEP: 1840 + | > loss: 1.87501 (2.08339) + | > log_mle: 0.47825 (0.58549) + | > loss_dur: 1.39676 (1.49790) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.85394 (4.20863) + | > current_lr: 0.00000 + | > step_time: 11.30660 (3.12571) + | > loader_time: 0.07310 (0.10991) + + + --> STEP: 207/234 -- GLOBAL_STEP: 1845 + | > loss: 1.86933 (2.07861) + | > log_mle: 0.48011 (0.58324) + | > loss_dur: 1.38922 (1.49537) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.82022 (4.20033) + | > current_lr: 0.00000 + | > step_time: 2.61440 (3.15060) + | > loader_time: 0.00510 (0.10826) + + + --> STEP: 212/234 -- GLOBAL_STEP: 1850 + | > loss: 1.89494 (2.07476) + | > log_mle: 0.48001 (0.58093) + | > loss_dur: 1.41493 (1.49382) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.88277 (4.19372) + | > current_lr: 0.00000 + | > step_time: 3.73340 (3.18341) + | > loader_time: 0.09360 (0.10770) + + + --> STEP: 217/234 -- GLOBAL_STEP: 1855 + | > loss: 1.88741 (2.07020) + | > log_mle: 0.47406 (0.57859) + | > loss_dur: 1.41335 (1.49161) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.83063 (4.18517) + | > current_lr: 0.00000 + | > step_time: 3.48840 (3.19385) + | > loader_time: 0.10310 (0.10653) + + + --> STEP: 222/234 -- GLOBAL_STEP: 1860 + | > loss: 1.93975 (2.06635) + | > log_mle: 0.47381 (0.57625) + | > loss_dur: 1.46594 (1.49011) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.92668 (4.17815) + | > current_lr: 0.00000 + | > step_time: 3.30030 (3.21869) + | > loader_time: 0.08390 (0.10633) + + + --> STEP: 227/234 -- GLOBAL_STEP: 1865 + | > loss: 1.88970 (2.06279) + | > log_mle: 0.47465 (0.57389) + | > loss_dur: 1.41505 (1.48890) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.85165 (4.17218) + | > current_lr: 0.00000 + | > step_time: 0.24150 (3.15900) + | > loader_time: 0.00470 (0.10448) + + + --> STEP: 232/234 -- GLOBAL_STEP: 1870 + | > loss: 2.44651 (2.06437) + | > log_mle: 0.43611 (0.57134) + | > loss_dur: 2.01040 (1.49302) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.79065 (4.17450) + | > current_lr: 0.00000 + | > step_time: 0.33830 (3.09672) + | > loader_time: 0.00500 (0.10233) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.24481 (-0.02721) + | > avg_loss: 1.83859 (-0.40471) + | > avg_log_mle: 0.48906 (-0.14149) + | > avg_loss_dur: 1.34953 (-0.26323) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_1872.pth + + > EPOCH: 8/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 10:24:50)  + + --> STEP: 3/234 -- GLOBAL_STEP: 1875 + | > loss: 2.34760 (2.48377) + | > log_mle: 0.58595 (0.60559) + | > loss_dur: 1.76165 (1.87818) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.45410 (4.72913) + | > current_lr: 0.00000 + | > step_time: 8.21220 (7.84493) + | > loader_time: 0.00390 (1.43531) + + + --> STEP: 8/234 -- GLOBAL_STEP: 1880 + | > loss: 2.28259 (2.41318) + | > log_mle: 0.54380 (0.57671) + | > loss_dur: 1.73879 (1.83648) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.35253 (4.60855) + | > current_lr: 0.00000 + | > step_time: 10.13750 (6.20144) + | > loader_time: 0.08820 (0.59300) + + + --> STEP: 13/234 -- GLOBAL_STEP: 1885 + | > loss: 2.05073 (2.27906) + | > log_mle: 0.55269 (0.56943) + | > loss_dur: 1.49804 (1.70963) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.95372 (4.37502) + | > current_lr: 0.00000 + | > step_time: 4.79800 (5.91349) + | > loader_time: 0.00110 (0.38014) + + + --> STEP: 18/234 -- GLOBAL_STEP: 1890 + | > loss: 1.99512 (2.20145) + | > log_mle: 0.54210 (0.56268) + | > loss_dur: 1.45302 (1.63876) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.87141 (4.24177) + | > current_lr: 0.00000 + | > step_time: 0.81990 (4.70162) + | > loader_time: 0.00310 (0.28478) + + + --> STEP: 23/234 -- GLOBAL_STEP: 1895 + | > loss: 1.85344 (2.14429) + | > log_mle: 0.52907 (0.55817) + | > loss_dur: 1.32437 (1.58612) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.68568 (4.15179) + | > current_lr: 0.00000 + | > step_time: 2.90520 (4.04383) + | > loader_time: 0.09750 (0.23102) + + + --> STEP: 28/234 -- GLOBAL_STEP: 1900 + | > loss: 1.86719 (2.09795) + | > log_mle: 0.53668 (0.55497) + | > loss_dur: 1.33052 (1.54298) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.76729 (4.08242) + | > current_lr: 0.00000 + | > step_time: 1.56520 (3.86081) + | > loader_time: 0.00130 (0.19970) + + + --> STEP: 33/234 -- GLOBAL_STEP: 1905 + | > loss: 1.91831 (2.06272) + | > log_mle: 0.54639 (0.55101) + | > loss_dur: 1.37191 (1.51171) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.84115 (4.03676) + | > current_lr: 0.00000 + | > step_time: 1.82390 (3.48416) + | > loader_time: 0.08600 (0.17499) + + + --> STEP: 38/234 -- GLOBAL_STEP: 1910 + | > loss: 1.88083 (2.03857) + | > log_mle: 0.51635 (0.54780) + | > loss_dur: 1.36448 (1.49078) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.86672 (4.01209) + | > current_lr: 0.00000 + | > step_time: 1.62890 (3.20949) + | > loader_time: 0.08330 (0.15436) + + + --> STEP: 43/234 -- GLOBAL_STEP: 1915 + | > loss: 1.83956 (2.01960) + | > log_mle: 0.52101 (0.54622) + | > loss_dur: 1.31855 (1.47338) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.69566 (3.98455) + | > current_lr: 0.00000 + | > step_time: 1.60590 (3.03595) + | > loader_time: 0.00330 (0.14285) + + + --> STEP: 48/234 -- GLOBAL_STEP: 1920 + | > loss: 1.79295 (1.99863) + | > log_mle: 0.52174 (0.54373) + | > loss_dur: 1.27120 (1.45490) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.72757 (3.95619) + | > current_lr: 0.00000 + | > step_time: 1.72090 (2.94369) + | > loader_time: 0.09150 (0.13358) + + + --> STEP: 53/234 -- GLOBAL_STEP: 1925 + | > loss: 1.80151 (1.97959) + | > log_mle: 0.51406 (0.54151) + | > loss_dur: 1.28745 (1.43809) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.63009 (3.92778) + | > current_lr: 0.00000 + | > step_time: 1.21200 (2.83553) + | > loader_time: 0.08200 (0.12446) + + + --> STEP: 58/234 -- GLOBAL_STEP: 1930 + | > loss: 1.80773 (1.96507) + | > log_mle: 0.51917 (0.53937) + | > loss_dur: 1.28856 (1.42570) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.75098 (3.90762) + | > current_lr: 0.00000 + | > step_time: 1.60360 (2.74135) + | > loader_time: 0.00330 (0.11673) + + + --> STEP: 63/234 -- GLOBAL_STEP: 1935 + | > loss: 1.87688 (1.95200) + | > log_mle: 0.50676 (0.53676) + | > loss_dur: 1.37011 (1.41524) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.88212 (3.89171) + | > current_lr: 0.00000 + | > step_time: 1.69720 (2.67404) + | > loader_time: 0.00230 (0.10789) + + + --> STEP: 68/234 -- GLOBAL_STEP: 1940 + | > loss: 1.76552 (1.93656) + | > log_mle: 0.51331 (0.53473) + | > loss_dur: 1.25222 (1.40183) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.60129 (3.86763) + | > current_lr: 0.00000 + | > step_time: 2.05200 (2.61644) + | > loader_time: 0.10530 (0.10411) + + + --> STEP: 73/234 -- GLOBAL_STEP: 1945 + | > loss: 1.76241 (1.92431) + | > log_mle: 0.49924 (0.53290) + | > loss_dur: 1.26318 (1.39141) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.56302 (3.84657) + | > current_lr: 0.00000 + | > step_time: 3.00700 (2.59417) + | > loader_time: 0.00300 (0.09732) + + + --> STEP: 78/234 -- GLOBAL_STEP: 1950 + | > loss: 1.72181 (1.91165) + | > log_mle: 0.51682 (0.53115) + | > loss_dur: 1.20499 (1.38049) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.50449 (3.82816) + | > current_lr: 0.00000 + | > step_time: 1.18770 (2.53927) + | > loader_time: 0.00220 (0.09457) + + + --> STEP: 83/234 -- GLOBAL_STEP: 1955 + | > loss: 1.78323 (1.90049) + | > log_mle: 0.50556 (0.52931) + | > loss_dur: 1.27767 (1.37118) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.62536 (3.81123) + | > current_lr: 0.00000 + | > step_time: 1.06770 (2.50291) + | > loader_time: 0.00250 (0.08924) + + + --> STEP: 88/234 -- GLOBAL_STEP: 1960 + | > loss: 1.69119 (1.89107) + | > log_mle: 0.48198 (0.52729) + | > loss_dur: 1.20922 (1.36378) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.42644 (3.79531) + | > current_lr: 0.00000 + | > step_time: 1.81550 (2.48987) + | > loader_time: 0.08940 (0.08719) + + + --> STEP: 93/234 -- GLOBAL_STEP: 1965 + | > loss: 1.65848 (1.87959) + | > log_mle: 0.48063 (0.52511) + | > loss_dur: 1.17784 (1.35448) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.40281 (3.77606) + | > current_lr: 0.00000 + | > step_time: 1.48250 (2.48078) + | > loader_time: 0.00240 (0.08470) + + + --> STEP: 98/234 -- GLOBAL_STEP: 1970 + | > loss: 1.72316 (1.87162) + | > log_mle: 0.49647 (0.52301) + | > loss_dur: 1.22669 (1.34861) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.49087 (3.76270) + | > current_lr: 0.00000 + | > step_time: 2.68630 (2.45128) + | > loader_time: 0.00840 (0.08145) + + + --> STEP: 103/234 -- GLOBAL_STEP: 1975 + | > loss: 1.75304 (1.86382) + | > log_mle: 0.47949 (0.52092) + | > loss_dur: 1.27355 (1.34289) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.56212 (3.74855) + | > current_lr: 0.00000 + | > step_time: 2.80050 (2.42282) + | > loader_time: 0.00410 (0.07913) + + + --> STEP: 108/234 -- GLOBAL_STEP: 1980 + | > loss: 1.69947 (1.85669) + | > log_mle: 0.47680 (0.51888) + | > loss_dur: 1.22266 (1.33781) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.50763 (3.73530) + | > current_lr: 0.00000 + | > step_time: 2.16480 (2.40028) + | > loader_time: 0.09350 (0.07731) + + + --> STEP: 113/234 -- GLOBAL_STEP: 1985 + | > loss: 1.69938 (1.85014) + | > log_mle: 0.48169 (0.51711) + | > loss_dur: 1.21769 (1.33303) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.46701 (3.72290) + | > current_lr: 0.00000 + | > step_time: 2.60640 (2.38013) + | > loader_time: 0.00350 (0.07405) + + + --> STEP: 118/234 -- GLOBAL_STEP: 1990 + | > loss: 1.72574 (1.84386) + | > log_mle: 0.48255 (0.51540) + | > loss_dur: 1.24319 (1.32845) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.46994 (3.71053) + | > current_lr: 0.00000 + | > step_time: 1.80240 (2.38609) + | > loader_time: 0.08450 (0.07322) + + + --> STEP: 123/234 -- GLOBAL_STEP: 1995 + | > loss: 1.62783 (1.83600) + | > log_mle: 0.48775 (0.51404) + | > loss_dur: 1.14008 (1.32196) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.23327 (3.69456) + | > current_lr: 0.00000 + | > step_time: 2.58840 (2.38317) + | > loader_time: 0.00320 (0.07104) + + + --> STEP: 128/234 -- GLOBAL_STEP: 2000 + | > loss: 1.62058 (1.82910) + | > log_mle: 0.47241 (0.51209) + | > loss_dur: 1.14816 (1.31701) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.22851 (3.68091) + | > current_lr: 0.00000 + | > step_time: 1.51530 (2.40533) + | > loader_time: 0.00440 (0.07049) + + + --> STEP: 133/234 -- GLOBAL_STEP: 2005 + | > loss: 1.69871 (1.82325) + | > log_mle: 0.45932 (0.51016) + | > loss_dur: 1.23939 (1.31308) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.41245 (3.66921) + | > current_lr: 0.00000 + | > step_time: 3.18920 (2.39963) + | > loader_time: 0.09720 (0.06869) + + + --> STEP: 138/234 -- GLOBAL_STEP: 2010 + | > loss: 1.67991 (1.81848) + | > log_mle: 0.47269 (0.50840) + | > loss_dur: 1.20723 (1.31008) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.39072 (3.65974) + | > current_lr: 0.00000 + | > step_time: 2.90650 (2.38516) + | > loader_time: 0.00360 (0.06895) + + + --> STEP: 143/234 -- GLOBAL_STEP: 2015 + | > loss: 1.72877 (1.81376) + | > log_mle: 0.45051 (0.50655) + | > loss_dur: 1.27826 (1.30721) + | > amp_scaler: 32768.00000 (16956.86713) + | > grad_norm: 3.48261 (3.65071) + | > current_lr: 0.00000 + | > step_time: 1.50350 (2.38999) + | > loader_time: 0.00300 (0.06730) + + + --> STEP: 148/234 -- GLOBAL_STEP: 2020 + | > loss: 1.55304 (1.80798) + | > log_mle: 0.44996 (0.50451) + | > loss_dur: 1.10308 (1.30348) + | > amp_scaler: 32768.00000 (17491.02703) + | > grad_norm: 3.15439 (3.63957) + | > current_lr: 0.00000 + | > step_time: 1.80210 (2.38410) + | > loader_time: 0.08900 (0.06573) + + + --> STEP: 153/234 -- GLOBAL_STEP: 2025 + | > loss: 1.66270 (1.80340) + | > log_mle: 0.43038 (0.50236) + | > loss_dur: 1.23232 (1.30105) + | > amp_scaler: 32768.00000 (17990.27451) + | > grad_norm: 3.32135 (3.63030) + | > current_lr: 0.00000 + | > step_time: 6.99840 (2.41414) + | > loader_time: 0.19790 (0.06604) + + + --> STEP: 158/234 -- GLOBAL_STEP: 2030 + | > loss: 1.63831 (1.79920) + | > log_mle: 0.44307 (0.50037) + | > loss_dur: 1.19523 (1.29882) + | > amp_scaler: 32768.00000 (18457.92405) + | > grad_norm: 3.25400 (3.62141) + | > current_lr: 0.00000 + | > step_time: 3.40780 (2.40736) + | > loader_time: 0.08520 (0.06512) + + + --> STEP: 163/234 -- GLOBAL_STEP: 2035 + | > loss: 1.60700 (1.79513) + | > log_mle: 0.44497 (0.49831) + | > loss_dur: 1.16203 (1.29682) + | > amp_scaler: 32768.00000 (18896.88344) + | > grad_norm: 3.20282 (3.61251) + | > current_lr: 0.00000 + | > step_time: 2.88070 (2.45618) + | > loader_time: 0.00250 (0.06384) + + + --> STEP: 168/234 -- GLOBAL_STEP: 2040 + | > loss: 1.69912 (1.79128) + | > log_mle: 0.42546 (0.49618) + | > loss_dur: 1.27366 (1.29510) + | > amp_scaler: 32768.00000 (19309.71429) + | > grad_norm: 3.36987 (3.60439) + | > current_lr: 0.00000 + | > step_time: 2.70800 (2.52536) + | > loader_time: 0.09640 (0.06436) + + + --> STEP: 173/234 -- GLOBAL_STEP: 2045 + | > loss: 1.67702 (1.78834) + | > log_mle: 0.42764 (0.49404) + | > loss_dur: 1.24938 (1.29430) + | > amp_scaler: 32768.00000 (19698.68208) + | > grad_norm: 3.39007 (3.59844) + | > current_lr: 0.00000 + | > step_time: 4.70080 (2.57090) + | > loader_time: 0.10220 (0.06531) + + + --> STEP: 178/234 -- GLOBAL_STEP: 2050 + | > loss: 1.61995 (1.78442) + | > log_mle: 0.39746 (0.49176) + | > loss_dur: 1.22249 (1.29267) + | > amp_scaler: 32768.00000 (20065.79775) + | > grad_norm: 3.27613 (3.59100) + | > current_lr: 0.00000 + | > step_time: 10.21990 (2.67858) + | > loader_time: 0.38530 (0.06727) + + + --> STEP: 183/234 -- GLOBAL_STEP: 2055 + | > loss: 1.66894 (1.78153) + | > log_mle: 0.40881 (0.48966) + | > loss_dur: 1.26013 (1.29187) + | > amp_scaler: 32768.00000 (20412.85246) + | > grad_norm: 3.40515 (3.58564) + | > current_lr: 0.00000 + | > step_time: 1.49860 (2.70432) + | > loader_time: 0.00410 (0.06765) + + + --> STEP: 188/234 -- GLOBAL_STEP: 2060 + | > loss: 1.64566 (1.77839) + | > log_mle: 0.40296 (0.48762) + | > loss_dur: 1.24270 (1.29076) + | > amp_scaler: 32768.00000 (20741.44681) + | > grad_norm: 3.28298 (3.57911) + | > current_lr: 0.00000 + | > step_time: 1.70080 (2.75649) + | > loader_time: 0.01250 (0.06841) + + + --> STEP: 193/234 -- GLOBAL_STEP: 2065 + | > loss: 1.64887 (1.77510) + | > log_mle: 0.41059 (0.48564) + | > loss_dur: 1.23827 (1.28945) + | > amp_scaler: 32768.00000 (21053.01554) + | > grad_norm: 3.31706 (3.57276) + | > current_lr: 0.00000 + | > step_time: 7.50460 (2.87322) + | > loader_time: 0.19450 (0.06979) + + + --> STEP: 198/234 -- GLOBAL_STEP: 2070 + | > loss: 1.63729 (1.77211) + | > log_mle: 0.40429 (0.48370) + | > loss_dur: 1.23300 (1.28841) + | > amp_scaler: 32768.00000 (21348.84848) + | > grad_norm: 3.28081 (3.56679) + | > current_lr: 0.00000 + | > step_time: 5.49380 (2.91888) + | > loader_time: 0.00450 (0.06907) + + + --> STEP: 203/234 -- GLOBAL_STEP: 2075 + | > loss: 1.65528 (1.76863) + | > log_mle: 0.41488 (0.48166) + | > loss_dur: 1.24040 (1.28697) + | > amp_scaler: 32768.00000 (21630.10837) + | > grad_norm: 3.32743 (3.55979) + | > current_lr: 0.00000 + | > step_time: 2.10350 (2.89975) + | > loader_time: 0.08880 (0.06849) + + + --> STEP: 208/234 -- GLOBAL_STEP: 2080 + | > loss: 1.67018 (1.76564) + | > log_mle: 0.39446 (0.47945) + | > loss_dur: 1.27571 (1.28619) + | > amp_scaler: 32768.00000 (21897.84615) + | > grad_norm: 3.39730 (3.55470) + | > current_lr: 0.00000 + | > step_time: 9.49940 (2.91562) + | > loader_time: 0.09760 (0.06868) + + + --> STEP: 213/234 -- GLOBAL_STEP: 2085 + | > loss: 1.63856 (1.76309) + | > log_mle: 0.37308 (0.47709) + | > loss_dur: 1.26547 (1.28599) + | > amp_scaler: 32768.00000 (22153.01408) + | > grad_norm: 3.29945 (3.55001) + | > current_lr: 0.00000 + | > step_time: 7.90190 (2.95800) + | > loader_time: 0.11230 (0.06851) + + + --> STEP: 218/234 -- GLOBAL_STEP: 2090 + | > loss: 1.65480 (1.76023) + | > log_mle: 0.37976 (0.47485) + | > loss_dur: 1.27505 (1.28538) + | > amp_scaler: 32768.00000 (22396.47706) + | > grad_norm: 3.38867 (3.54527) + | > current_lr: 0.00000 + | > step_time: 4.60530 (3.00581) + | > loader_time: 0.09440 (0.06869) + + + --> STEP: 223/234 -- GLOBAL_STEP: 2095 + | > loss: 1.64430 (1.75780) + | > log_mle: 0.36497 (0.47249) + | > loss_dur: 1.27933 (1.28531) + | > amp_scaler: 32768.00000 (22629.02242) + | > grad_norm: 3.36144 (3.54210) + | > current_lr: 0.00000 + | > step_time: 0.25940 (2.98725) + | > loader_time: 0.00430 (0.06763) + + + --> STEP: 228/234 -- GLOBAL_STEP: 2100 + | > loss: 1.70424 (1.75580) + | > log_mle: 0.36394 (0.47014) + | > loss_dur: 1.34030 (1.28566) + | > amp_scaler: 32768.00000 (22851.36842) + | > grad_norm: 3.49940 (3.53900) + | > current_lr: 0.00000 + | > step_time: 0.24680 (2.92717) + | > loader_time: 0.00370 (0.06623) + + + --> STEP: 233/234 -- GLOBAL_STEP: 2105 + | > loss: 3.02020 (1.76380) + | > log_mle: 0.41925 (0.46767) + | > loss_dur: 2.60095 (1.29612) + | > amp_scaler: 16384.00000 (22993.85408) + | > grad_norm: 0.00000 (3.52868) + | > current_lr: 0.00000 + | > step_time: 0.15740 (2.86991) + | > loader_time: 0.00330 (0.06521) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.40585 (+1.16103) + | > avg_loss: 1.61580 (-0.22279) + | > avg_log_mle: 0.40566 (-0.08340) + | > avg_loss_dur: 1.21014 (-0.13939) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_2106.pth + + > EPOCH: 9/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 10:37:17)  + + --> STEP: 4/234 -- GLOBAL_STEP: 2110 + | > loss: 2.29271 (2.24419) + | > log_mle: 0.50913 (0.53337) + | > loss_dur: 1.78358 (1.71082) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.17925 (4.18286) + | > current_lr: 0.00000 + | > step_time: 2.19530 (2.97561) + | > loader_time: 0.01060 (0.00442) + + + --> STEP: 9/234 -- GLOBAL_STEP: 2115 + | > loss: 1.91077 (2.13472) + | > log_mle: 0.48651 (0.51120) + | > loss_dur: 1.42426 (1.62352) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.60305 (4.02497) + | > current_lr: 0.00000 + | > step_time: 8.12430 (5.58322) + | > loader_time: 0.09850 (0.02304) + + + --> STEP: 14/234 -- GLOBAL_STEP: 2120 + | > loss: 1.79424 (2.02727) + | > log_mle: 0.48308 (0.50619) + | > loss_dur: 1.31116 (1.52109) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.42966 (3.84538) + | > current_lr: 0.00000 + | > step_time: 3.19130 (5.17765) + | > loader_time: 0.00720 (0.03629) + + + --> STEP: 19/234 -- GLOBAL_STEP: 2125 + | > loss: 1.84940 (1.96804) + | > log_mle: 0.49436 (0.50193) + | > loss_dur: 1.35504 (1.46611) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.51619 (3.74373) + | > current_lr: 0.00000 + | > step_time: 4.49040 (5.04680) + | > loader_time: 0.10740 (0.04814) + + + --> STEP: 24/234 -- GLOBAL_STEP: 2130 + | > loss: 1.79880 (1.91866) + | > log_mle: 0.48703 (0.49793) + | > loss_dur: 1.31178 (1.42073) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.50355 (3.66610) + | > current_lr: 0.00000 + | > step_time: 6.00110 (5.05502) + | > loader_time: 0.10590 (0.05491) + + + --> STEP: 29/234 -- GLOBAL_STEP: 2135 + | > loss: 1.70048 (1.87703) + | > log_mle: 0.48737 (0.49526) + | > loss_dur: 1.21311 (1.38177) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.31189 (3.59729) + | > current_lr: 0.00000 + | > step_time: 0.95250 (4.61441) + | > loader_time: 0.00130 (0.05476) + + + --> STEP: 34/234 -- GLOBAL_STEP: 2140 + | > loss: 1.71344 (1.84731) + | > log_mle: 0.46756 (0.49134) + | > loss_dur: 1.24587 (1.35597) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.41332 (3.55983) + | > current_lr: 0.00000 + | > step_time: 5.28240 (4.42634) + | > loader_time: 0.00130 (0.05307) + + + --> STEP: 39/234 -- GLOBAL_STEP: 2145 + | > loss: 1.72724 (1.82661) + | > log_mle: 0.47528 (0.48872) + | > loss_dur: 1.25196 (1.33789) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.45582 (3.53941) + | > current_lr: 0.00000 + | > step_time: 3.89950 (4.21312) + | > loader_time: 0.09420 (0.05322) + + + --> STEP: 44/234 -- GLOBAL_STEP: 2150 + | > loss: 1.64971 (1.80981) + | > log_mle: 0.46977 (0.48730) + | > loss_dur: 1.17994 (1.32252) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.29232 (3.51865) + | > current_lr: 0.00000 + | > step_time: 1.45430 (3.93915) + | > loader_time: 0.00210 (0.05099) + + + --> STEP: 49/234 -- GLOBAL_STEP: 2155 + | > loss: 1.58436 (1.79095) + | > log_mle: 0.45131 (0.48484) + | > loss_dur: 1.13305 (1.30610) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.22929 (3.49561) + | > current_lr: 0.00000 + | > step_time: 1.17310 (3.70651) + | > loader_time: 0.00140 (0.04925) + + + --> STEP: 54/234 -- GLOBAL_STEP: 2160 + | > loss: 1.62364 (1.77542) + | > log_mle: 0.45576 (0.48314) + | > loss_dur: 1.16788 (1.29228) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.27662 (3.47231) + | > current_lr: 0.00000 + | > step_time: 2.29540 (3.52535) + | > loader_time: 0.00220 (0.04489) + + + --> STEP: 59/234 -- GLOBAL_STEP: 2165 + | > loss: 1.54334 (1.76202) + | > log_mle: 0.44561 (0.48121) + | > loss_dur: 1.09773 (1.28081) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.11024 (3.45278) + | > current_lr: 0.00000 + | > step_time: 3.30540 (3.39541) + | > loader_time: 0.19670 (0.04596) + + + --> STEP: 64/234 -- GLOBAL_STEP: 2170 + | > loss: 1.55075 (1.75133) + | > log_mle: 0.45760 (0.47904) + | > loss_dur: 1.09315 (1.27229) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.10416 (3.43968) + | > current_lr: 0.00000 + | > step_time: 1.59530 (3.26438) + | > loader_time: 0.00210 (0.04406) + + + --> STEP: 69/234 -- GLOBAL_STEP: 2175 + | > loss: 1.60210 (1.73886) + | > log_mle: 0.47004 (0.47758) + | > loss_dur: 1.13206 (1.26128) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.12654 (3.41880) + | > current_lr: 0.00000 + | > step_time: 1.28580 (3.14653) + | > loader_time: 0.00260 (0.04231) + + + --> STEP: 74/234 -- GLOBAL_STEP: 2180 + | > loss: 1.51765 (1.72735) + | > log_mle: 0.45804 (0.47582) + | > loss_dur: 1.05961 (1.25153) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.11952 (3.40092) + | > current_lr: 0.00000 + | > step_time: 3.40910 (3.05726) + | > loader_time: 0.08730 (0.04189) + + + --> STEP: 79/234 -- GLOBAL_STEP: 2185 + | > loss: 1.57357 (1.71720) + | > log_mle: 0.44201 (0.47410) + | > loss_dur: 1.13156 (1.24309) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.15927 (3.38510) + | > current_lr: 0.00000 + | > step_time: 2.20440 (2.97465) + | > loader_time: 0.00220 (0.03942) + + + --> STEP: 84/234 -- GLOBAL_STEP: 2190 + | > loss: 1.58731 (1.70789) + | > log_mle: 0.43810 (0.47251) + | > loss_dur: 1.14920 (1.23538) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.15670 (3.36978) + | > current_lr: 0.00000 + | > step_time: 1.59090 (2.91321) + | > loader_time: 0.00290 (0.03818) + + + --> STEP: 89/234 -- GLOBAL_STEP: 2195 + | > loss: 1.51177 (1.69900) + | > log_mle: 0.42967 (0.47055) + | > loss_dur: 1.08210 (1.22845) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03740 (3.35519) + | > current_lr: 0.00000 + | > step_time: 3.10780 (2.85973) + | > loader_time: 0.00290 (0.03619) + + + --> STEP: 94/234 -- GLOBAL_STEP: 2200 + | > loss: 1.57300 (1.68958) + | > log_mle: 0.41470 (0.46828) + | > loss_dur: 1.15830 (1.22129) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.16324 (3.33983) + | > current_lr: 0.00000 + | > step_time: 2.90600 (2.81955) + | > loader_time: 0.00280 (0.03442) + + + --> STEP: 99/234 -- GLOBAL_STEP: 2205 + | > loss: 1.53940 (1.68255) + | > log_mle: 0.40816 (0.46615) + | > loss_dur: 1.13125 (1.21640) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.13353 (3.32920) + | > current_lr: 0.00000 + | > step_time: 1.69550 (2.77297) + | > loader_time: 0.00330 (0.03368) + + + --> STEP: 104/234 -- GLOBAL_STEP: 2210 + | > loss: 1.56356 (1.67601) + | > log_mle: 0.40555 (0.46398) + | > loss_dur: 1.15801 (1.21203) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.12963 (3.31712) + | > current_lr: 0.00000 + | > step_time: 2.70000 (2.73202) + | > loader_time: 0.00370 (0.03464) + + + --> STEP: 109/234 -- GLOBAL_STEP: 2215 + | > loss: 1.53605 (1.66955) + | > log_mle: 0.41899 (0.46210) + | > loss_dur: 1.11706 (1.20746) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.97738 (3.30445) + | > current_lr: 0.00000 + | > step_time: 1.44890 (2.69541) + | > loader_time: 0.00250 (0.03320) + + + --> STEP: 114/234 -- GLOBAL_STEP: 2220 + | > loss: 1.52823 (1.66391) + | > log_mle: 0.42120 (0.46025) + | > loss_dur: 1.10703 (1.20366) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03520 (3.29476) + | > current_lr: 0.00000 + | > step_time: 6.01420 (2.69752) + | > loader_time: 0.29310 (0.03605) + + + --> STEP: 119/234 -- GLOBAL_STEP: 2225 + | > loss: 1.51141 (1.65855) + | > log_mle: 0.42249 (0.45856) + | > loss_dur: 1.08892 (1.19999) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00737 (3.28414) + | > current_lr: 0.00000 + | > step_time: 3.19630 (2.68855) + | > loader_time: 0.10300 (0.03692) + + + --> STEP: 124/234 -- GLOBAL_STEP: 2230 + | > loss: 1.50606 (1.65186) + | > log_mle: 0.40698 (0.45715) + | > loss_dur: 1.09908 (1.19471) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.01714 (3.27136) + | > current_lr: 0.00000 + | > step_time: 2.40560 (2.75291) + | > loader_time: 0.08700 (0.03923) + + + --> STEP: 129/234 -- GLOBAL_STEP: 2235 + | > loss: 1.50857 (1.64600) + | > log_mle: 0.41125 (0.45517) + | > loss_dur: 1.09732 (1.19084) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.04329 (3.26044) + | > current_lr: 0.00000 + | > step_time: 3.69880 (2.73158) + | > loader_time: 0.00510 (0.03862) + + + --> STEP: 134/234 -- GLOBAL_STEP: 2240 + | > loss: 1.52831 (1.64121) + | > log_mle: 0.40183 (0.45312) + | > loss_dur: 1.12647 (1.18808) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.04410 (3.25147) + | > current_lr: 0.00000 + | > step_time: 1.89440 (2.68740) + | > loader_time: 0.00430 (0.03786) + + + --> STEP: 139/234 -- GLOBAL_STEP: 2245 + | > loss: 1.50969 (1.63703) + | > log_mle: 0.37465 (0.45113) + | > loss_dur: 1.13504 (1.18590) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.10049 (3.24484) + | > current_lr: 0.00000 + | > step_time: 2.62040 (2.68669) + | > loader_time: 0.19470 (0.03993) + + + --> STEP: 144/234 -- GLOBAL_STEP: 2250 + | > loss: 1.48496 (1.63270) + | > log_mle: 0.38237 (0.44921) + | > loss_dur: 1.10259 (1.18349) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.97522 (3.23729) + | > current_lr: 0.00000 + | > step_time: 2.90820 (2.75460) + | > loader_time: 0.08380 (0.04126) + + + --> STEP: 149/234 -- GLOBAL_STEP: 2255 + | > loss: 1.48997 (1.62779) + | > log_mle: 0.36983 (0.44695) + | > loss_dur: 1.12014 (1.18083) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00728 (3.22849) + | > current_lr: 0.00000 + | > step_time: 1.61590 (2.71164) + | > loader_time: 0.00310 (0.03998) + + + --> STEP: 154/234 -- GLOBAL_STEP: 2260 + | > loss: 1.49766 (1.62370) + | > log_mle: 0.37881 (0.44465) + | > loss_dur: 1.11884 (1.17905) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.01829 (3.22117) + | > current_lr: 0.00000 + | > step_time: 1.78870 (2.69977) + | > loader_time: 0.00270 (0.03920) + + + --> STEP: 159/234 -- GLOBAL_STEP: 2265 + | > loss: 1.56460 (1.62046) + | > log_mle: 0.37193 (0.44245) + | > loss_dur: 1.19266 (1.17802) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.14110 (3.21572) + | > current_lr: 0.00000 + | > step_time: 2.10260 (2.71469) + | > loader_time: 0.08790 (0.03918) + + + --> STEP: 164/234 -- GLOBAL_STEP: 2270 + | > loss: 1.49032 (1.61639) + | > log_mle: 0.36671 (0.44020) + | > loss_dur: 1.12360 (1.17619) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03657 (3.20933) + | > current_lr: 0.00000 + | > step_time: 3.00060 (2.70687) + | > loader_time: 0.00350 (0.03810) + + + --> STEP: 169/234 -- GLOBAL_STEP: 2275 + | > loss: 1.54607 (1.61338) + | > log_mle: 0.36918 (0.43791) + | > loss_dur: 1.17689 (1.17547) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.09762 (3.20398) + | > current_lr: 0.00000 + | > step_time: 2.39720 (2.69550) + | > loader_time: 0.00190 (0.03765) + + + --> STEP: 174/234 -- GLOBAL_STEP: 2280 + | > loss: 1.46433 (1.61035) + | > log_mle: 0.33357 (0.43533) + | > loss_dur: 1.13076 (1.17502) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00276 (3.19955) + | > current_lr: 0.00000 + | > step_time: 1.45570 (2.68782) + | > loader_time: 0.00280 (0.03752) + + + --> STEP: 179/234 -- GLOBAL_STEP: 2285 + | > loss: 1.50573 (1.60710) + | > log_mle: 0.34041 (0.43287) + | > loss_dur: 1.16532 (1.17423) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.05455 (3.19486) + | > current_lr: 0.00000 + | > step_time: 4.30060 (2.70672) + | > loader_time: 0.10410 (0.03906) + + + --> STEP: 184/234 -- GLOBAL_STEP: 2290 + | > loss: 1.49582 (1.60443) + | > log_mle: 0.34974 (0.43062) + | > loss_dur: 1.14608 (1.17381) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.96152 (3.19128) + | > current_lr: 0.00000 + | > step_time: 1.68790 (2.72443) + | > loader_time: 0.00370 (0.04014) + + + --> STEP: 189/234 -- GLOBAL_STEP: 2295 + | > loss: 1.50039 (1.60162) + | > log_mle: 0.35129 (0.42836) + | > loss_dur: 1.14910 (1.17326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.04863 (3.18838) + | > current_lr: 0.00000 + | > step_time: 4.20340 (2.80486) + | > loader_time: 0.06610 (0.04144) + + + --> STEP: 194/234 -- GLOBAL_STEP: 2300 + | > loss: 1.49981 (1.59851) + | > log_mle: 0.32980 (0.42605) + | > loss_dur: 1.17001 (1.17246) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.11959 (3.18444) + | > current_lr: 0.00000 + | > step_time: 2.80300 (2.79753) + | > loader_time: 0.00340 (0.04239) + + + --> STEP: 199/234 -- GLOBAL_STEP: 2305 + | > loss: 1.45184 (1.59570) + | > log_mle: 0.32613 (0.42389) + | > loss_dur: 1.12572 (1.17181) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.01325 (3.18007) + | > current_lr: 0.00000 + | > step_time: 2.70730 (2.85700) + | > loader_time: 0.10040 (0.04376) + + + --> STEP: 204/234 -- GLOBAL_STEP: 2310 + | > loss: 1.45714 (1.59267) + | > log_mle: 0.31648 (0.42163) + | > loss_dur: 1.14066 (1.17105) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.97440 (3.17526) + | > current_lr: 0.00000 + | > step_time: 7.70050 (2.95265) + | > loader_time: 0.09380 (0.04607) + + + --> STEP: 209/234 -- GLOBAL_STEP: 2315 + | > loss: 1.47130 (1.59007) + | > log_mle: 0.33085 (0.41925) + | > loss_dur: 1.14045 (1.17082) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.07139 (3.17298) + | > current_lr: 0.00000 + | > step_time: 7.90170 (3.04464) + | > loader_time: 0.19090 (0.04884) + + + --> STEP: 214/234 -- GLOBAL_STEP: 2320 + | > loss: 1.47021 (1.58774) + | > log_mle: 0.31584 (0.41655) + | > loss_dur: 1.15437 (1.17119) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.09267 (3.17226) + | > current_lr: 0.00000 + | > step_time: 4.40850 (3.09703) + | > loader_time: 0.09690 (0.05130) + + + --> STEP: 219/234 -- GLOBAL_STEP: 2325 + | > loss: 1.45898 (1.58508) + | > log_mle: 0.27282 (0.41387) + | > loss_dur: 1.18616 (1.17121) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.05269 (3.17012) + | > current_lr: 0.00000 + | > step_time: 3.50880 (3.14647) + | > loader_time: 0.09540 (0.05478) + + + --> STEP: 224/234 -- GLOBAL_STEP: 2330 + | > loss: 1.51368 (1.58313) + | > log_mle: 0.29227 (0.41132) + | > loss_dur: 1.22141 (1.17180) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.12716 (3.16812) + | > current_lr: 0.00000 + | > step_time: 0.24480 (3.10124) + | > loader_time: 0.00580 (0.05405) + + + --> STEP: 229/234 -- GLOBAL_STEP: 2335 + | > loss: 1.63134 (1.58173) + | > log_mle: 0.30167 (0.40869) + | > loss_dur: 1.32967 (1.17304) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.38747 (3.16939) + | > current_lr: 0.00000 + | > step_time: 0.24780 (3.03894) + | > loader_time: 0.00430 (0.05295) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40340 (-1.00245) + | > avg_loss: 1.46028 (-0.15552) + | > avg_log_mle: 0.34331 (-0.06235) + | > avg_loss_dur: 1.11696 (-0.09318) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_2340.pth + + > EPOCH: 10/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 10:50:09)  + + --> STEP: 0/234 -- GLOBAL_STEP: 2340 + | > loss: 2.20794 (2.20794) + | > log_mle: 0.50912 (0.50912) + | > loss_dur: 1.69882 (1.69882) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.49437 (4.49437) + | > current_lr: 0.00000 + | > step_time: 3.41340 (3.41341) + | > loader_time: 19.57610 (19.57608) + + + --> STEP: 5/234 -- GLOBAL_STEP: 2345 + | > loss: 2.03003 (2.07176) + | > log_mle: 0.46375 (0.48389) + | > loss_dur: 1.56628 (1.58786) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.86868 (3.92356) + | > current_lr: 0.00000 + | > step_time: 5.30450 (5.68440) + | > loader_time: 0.09210 (0.07396) + + + --> STEP: 10/234 -- GLOBAL_STEP: 2350 + | > loss: 1.69790 (1.95024) + | > log_mle: 0.45998 (0.46812) + | > loss_dur: 1.23791 (1.48213) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.29755 (3.72908) + | > current_lr: 0.00000 + | > step_time: 4.97740 (5.58191) + | > loader_time: 0.11990 (0.08876) + + + --> STEP: 15/234 -- GLOBAL_STEP: 2355 + | > loss: 1.67190 (1.86269) + | > log_mle: 0.44594 (0.46316) + | > loss_dur: 1.22596 (1.39953) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.25549 (3.57388) + | > current_lr: 0.00000 + | > step_time: 6.20220 (5.54988) + | > loader_time: 0.00370 (0.08656) + + + --> STEP: 20/234 -- GLOBAL_STEP: 2360 + | > loss: 1.62602 (1.81319) + | > log_mle: 0.44200 (0.45962) + | > loss_dur: 1.18402 (1.35358) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.27969 (3.49482) + | > current_lr: 0.00000 + | > step_time: 3.49840 (5.45880) + | > loader_time: 0.08950 (0.09327) + + + --> STEP: 25/234 -- GLOBAL_STEP: 2365 + | > loss: 1.54404 (1.76729) + | > log_mle: 0.44534 (0.45616) + | > loss_dur: 1.09870 (1.31114) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.98589 (3.41392) + | > current_lr: 0.00000 + | > step_time: 3.80630 (5.27911) + | > loader_time: 0.00430 (0.08237) + + + --> STEP: 30/234 -- GLOBAL_STEP: 2370 + | > loss: 1.50979 (1.73074) + | > log_mle: 0.42464 (0.45287) + | > loss_dur: 1.08514 (1.27787) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.05742 (3.35375) + | > current_lr: 0.00000 + | > step_time: 11.10930 (5.19922) + | > loader_time: 0.10490 (0.07894) + + + --> STEP: 35/234 -- GLOBAL_STEP: 2375 + | > loss: 1.55563 (1.70646) + | > log_mle: 0.43710 (0.44955) + | > loss_dur: 1.11853 (1.25691) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.34790 (3.32518) + | > current_lr: 0.00000 + | > step_time: 1.51610 (4.88293) + | > loader_time: 0.00230 (0.07367) + + + --> STEP: 40/234 -- GLOBAL_STEP: 2380 + | > loss: 1.55641 (1.68774) + | > log_mle: 0.44202 (0.44710) + | > loss_dur: 1.11439 (1.24063) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.07875 (3.30277) + | > current_lr: 0.00000 + | > step_time: 1.07740 (4.42475) + | > loader_time: 0.00210 (0.06555) + + + --> STEP: 45/234 -- GLOBAL_STEP: 2385 + | > loss: 1.56854 (1.67276) + | > log_mle: 0.43137 (0.44542) + | > loss_dur: 1.13716 (1.22734) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.14908 (3.28096) + | > current_lr: 0.00000 + | > step_time: 3.10080 (4.14616) + | > loader_time: 0.09080 (0.06230) + + + --> STEP: 50/234 -- GLOBAL_STEP: 2390 + | > loss: 1.49540 (1.65329) + | > log_mle: 0.42562 (0.44304) + | > loss_dur: 1.06978 (1.21025) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.94012 (3.24989) + | > current_lr: 0.00000 + | > step_time: 1.39050 (3.90379) + | > loader_time: 0.00240 (0.05824) + + + --> STEP: 55/234 -- GLOBAL_STEP: 2395 + | > loss: 1.47010 (1.63826) + | > log_mle: 0.40781 (0.44111) + | > loss_dur: 1.06230 (1.19715) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.92389 (3.22436) + | > current_lr: 0.00000 + | > step_time: 3.27690 (3.72594) + | > loader_time: 0.00350 (0.05507) + + + --> STEP: 60/234 -- GLOBAL_STEP: 2400 + | > loss: 1.47077 (1.62536) + | > log_mle: 0.40166 (0.43915) + | > loss_dur: 1.06912 (1.18621) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03702 (3.20521) + | > current_lr: 0.00000 + | > step_time: 2.20990 (3.58387) + | > loader_time: 0.08740 (0.05660) + + + --> STEP: 65/234 -- GLOBAL_STEP: 2405 + | > loss: 1.44318 (1.61494) + | > log_mle: 0.41515 (0.43728) + | > loss_dur: 1.02803 (1.17766) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.91333 (3.19007) + | > current_lr: 0.00000 + | > step_time: 1.20570 (3.40667) + | > loader_time: 0.00270 (0.05244) + + + --> STEP: 70/234 -- GLOBAL_STEP: 2410 + | > loss: 1.44036 (1.60336) + | > log_mle: 0.41183 (0.43588) + | > loss_dur: 1.02853 (1.16748) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.85665 (3.16807) + | > current_lr: 0.00000 + | > step_time: 2.11950 (3.29639) + | > loader_time: 0.10690 (0.05034) + + + --> STEP: 75/234 -- GLOBAL_STEP: 2415 + | > loss: 1.44564 (1.59262) + | > log_mle: 0.41032 (0.43410) + | > loss_dur: 1.03532 (1.15852) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.91694 (3.14924) + | > current_lr: 0.00000 + | > step_time: 2.49780 (3.19058) + | > loader_time: 0.08500 (0.05060) + + + --> STEP: 80/234 -- GLOBAL_STEP: 2420 + | > loss: 1.42652 (1.58248) + | > log_mle: 0.41359 (0.43253) + | > loss_dur: 1.01294 (1.14995) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.85353 (3.13154) + | > current_lr: 0.00000 + | > step_time: 1.81300 (3.09870) + | > loader_time: 0.00300 (0.04859) + + + --> STEP: 85/234 -- GLOBAL_STEP: 2425 + | > loss: 1.45902 (1.57407) + | > log_mle: 0.40628 (0.43092) + | > loss_dur: 1.05275 (1.14315) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.98570 (3.11871) + | > current_lr: 0.00000 + | > step_time: 1.29660 (3.09377) + | > loader_time: 0.00250 (0.04696) + + + --> STEP: 90/234 -- GLOBAL_STEP: 2430 + | > loss: 1.35661 (1.56436) + | > log_mle: 0.39285 (0.42877) + | > loss_dur: 0.96376 (1.13559) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.71146 (3.10109) + | > current_lr: 0.00000 + | > step_time: 2.69760 (3.04797) + | > loader_time: 0.00240 (0.04541) + + + --> STEP: 95/234 -- GLOBAL_STEP: 2435 + | > loss: 1.43090 (1.55577) + | > log_mle: 0.35764 (0.42601) + | > loss_dur: 1.07326 (1.12976) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.96243 (3.08797) + | > current_lr: 0.00000 + | > step_time: 1.29990 (2.97321) + | > loader_time: 0.00300 (0.04501) + + + --> STEP: 100/234 -- GLOBAL_STEP: 2440 + | > loss: 1.42192 (1.54894) + | > log_mle: 0.38070 (0.42399) + | > loss_dur: 1.04123 (1.12495) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.78488 (3.07514) + | > current_lr: 0.00000 + | > step_time: 1.91290 (2.93861) + | > loader_time: 0.08760 (0.04482) + + + --> STEP: 105/234 -- GLOBAL_STEP: 2445 + | > loss: 1.38599 (1.54216) + | > log_mle: 0.39250 (0.42175) + | > loss_dur: 0.99349 (1.12040) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76795 (3.06357) + | > current_lr: 0.00000 + | > step_time: 2.80890 (2.87796) + | > loader_time: 0.00260 (0.04355) + + + --> STEP: 110/234 -- GLOBAL_STEP: 2450 + | > loss: 1.42798 (1.53636) + | > log_mle: 0.38395 (0.41972) + | > loss_dur: 1.04403 (1.11664) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.86819 (3.05230) + | > current_lr: 0.00000 + | > step_time: 2.01370 (2.83114) + | > loader_time: 0.00410 (0.04170) + + + --> STEP: 115/234 -- GLOBAL_STEP: 2455 + | > loss: 1.45283 (1.53098) + | > log_mle: 0.37189 (0.41758) + | > loss_dur: 1.08094 (1.11341) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.92099 (3.04242) + | > current_lr: 0.00000 + | > step_time: 3.22260 (2.81955) + | > loader_time: 0.09730 (0.04235) + + + --> STEP: 120/234 -- GLOBAL_STEP: 2460 + | > loss: 1.38621 (1.52514) + | > log_mle: 0.35103 (0.41560) + | > loss_dur: 1.03518 (1.10954) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.88239 (3.03253) + | > current_lr: 0.00000 + | > step_time: 1.70120 (2.77633) + | > loader_time: 0.09060 (0.04342) + + + --> STEP: 125/234 -- GLOBAL_STEP: 2465 + | > loss: 1.35175 (1.51835) + | > log_mle: 0.36002 (0.41422) + | > loss_dur: 0.99173 (1.10413) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.67987 (3.01869) + | > current_lr: 0.00000 + | > step_time: 0.99210 (2.76688) + | > loader_time: 0.00780 (0.04185) + + + --> STEP: 130/234 -- GLOBAL_STEP: 2470 + | > loss: 1.38365 (1.51297) + | > log_mle: 0.35128 (0.41199) + | > loss_dur: 1.03238 (1.10098) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.84118 (3.00981) + | > current_lr: 0.00000 + | > step_time: 2.40050 (2.72579) + | > loader_time: 0.00330 (0.04098) + + + --> STEP: 135/234 -- GLOBAL_STEP: 2475 + | > loss: 1.34348 (1.50791) + | > log_mle: 0.37464 (0.40991) + | > loss_dur: 0.96884 (1.09800) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.71403 (3.00166) + | > current_lr: 0.00000 + | > step_time: 1.80550 (2.70708) + | > loader_time: 0.00250 (0.04021) + + + --> STEP: 140/234 -- GLOBAL_STEP: 2480 + | > loss: 1.36299 (1.50398) + | > log_mle: 0.37488 (0.40772) + | > loss_dur: 0.98811 (1.09626) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.70384 (3.00044) + | > current_lr: 0.00000 + | > step_time: 1.88450 (2.69654) + | > loader_time: 0.00180 (0.04024) + + + --> STEP: 145/234 -- GLOBAL_STEP: 2485 + | > loss: 1.39455 (1.49974) + | > log_mle: 0.32328 (0.40520) + | > loss_dur: 1.07127 (1.09454) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.09059 (3.00014) + | > current_lr: 0.00000 + | > step_time: 2.10740 (2.68164) + | > loader_time: 0.00330 (0.04004) + + + --> STEP: 150/234 -- GLOBAL_STEP: 2490 + | > loss: 1.40326 (1.49478) + | > log_mle: 0.33283 (0.40274) + | > loss_dur: 1.07043 (1.09205) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.10023 (2.99408) + | > current_lr: 0.00000 + | > step_time: 1.91310 (2.65885) + | > loader_time: 0.00350 (0.03937) + + + --> STEP: 155/234 -- GLOBAL_STEP: 2495 + | > loss: 1.40430 (1.49061) + | > log_mle: 0.30769 (0.39997) + | > loss_dur: 1.09661 (1.09063) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.09268 (2.99021) + | > current_lr: 0.00000 + | > step_time: 2.61200 (2.65321) + | > loader_time: 0.08490 (0.04050) + + + --> STEP: 160/234 -- GLOBAL_STEP: 2500 + | > loss: 1.33287 (1.48677) + | > log_mle: 0.29912 (0.39746) + | > loss_dur: 1.03375 (1.08932) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.84641 (2.98720) + | > current_lr: 0.00000 + | > step_time: 3.47530 (2.65640) + | > loader_time: 0.00550 (0.04067) + + + --> STEP: 165/234 -- GLOBAL_STEP: 2505 + | > loss: 1.35081 (1.48290) + | > log_mle: 0.30317 (0.39498) + | > loss_dur: 1.04764 (1.08792) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.79084 (2.98239) + | > current_lr: 0.00000 + | > step_time: 2.90580 (2.68335) + | > loader_time: 0.00350 (0.04127) + + + --> STEP: 170/234 -- GLOBAL_STEP: 2510 + | > loss: 1.40562 (1.48013) + | > log_mle: 0.29152 (0.39235) + | > loss_dur: 1.11411 (1.08778) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.89364 (2.97904) + | > current_lr: 0.00000 + | > step_time: 3.69970 (2.68441) + | > loader_time: 0.39250 (0.04299) + + + --> STEP: 175/234 -- GLOBAL_STEP: 2515 + | > loss: 1.35268 (1.47662) + | > log_mle: 0.29415 (0.38941) + | > loss_dur: 1.05853 (1.08721) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.87953 (2.97696) + | > current_lr: 0.00000 + | > step_time: 1.38910 (2.68011) + | > loader_time: 0.07800 (0.04336) + + + --> STEP: 180/234 -- GLOBAL_STEP: 2520 + | > loss: 1.36917 (1.47335) + | > log_mle: 0.28712 (0.38660) + | > loss_dur: 1.08205 (1.08675) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03030 (2.97453) + | > current_lr: 0.00000 + | > step_time: 2.29890 (2.67458) + | > loader_time: 0.00300 (0.04314) + + + --> STEP: 185/234 -- GLOBAL_STEP: 2525 + | > loss: 1.38757 (1.47076) + | > log_mle: 0.28683 (0.38405) + | > loss_dur: 1.10074 (1.08671) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.89976 (2.97249) + | > current_lr: 0.00000 + | > step_time: 3.70840 (2.70998) + | > loader_time: 0.00450 (0.04401) + + + --> STEP: 190/234 -- GLOBAL_STEP: 2530 + | > loss: 1.33232 (1.46753) + | > log_mle: 0.28127 (0.38145) + | > loss_dur: 1.05105 (1.08608) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.83491 (2.96916) + | > current_lr: 0.00000 + | > step_time: 1.81180 (2.72304) + | > loader_time: 0.09320 (0.04443) + + + --> STEP: 195/234 -- GLOBAL_STEP: 2535 + | > loss: 1.39290 (1.46467) + | > log_mle: 0.28148 (0.37882) + | > loss_dur: 1.11142 (1.08586) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.14411 (2.97046) + | > current_lr: 0.00000 + | > step_time: 9.41660 (2.79445) + | > loader_time: 0.19290 (0.04642) + + + --> STEP: 200/234 -- GLOBAL_STEP: 2540 + | > loss: 1.36205 (1.46165) + | > log_mle: 0.27824 (0.37634) + | > loss_dur: 1.08381 (1.08532) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.11100 (2.97050) + | > current_lr: 0.00000 + | > step_time: 3.20360 (2.83113) + | > loader_time: 0.00400 (0.04822) + + + --> STEP: 205/234 -- GLOBAL_STEP: 2545 + | > loss: 1.37415 (1.45870) + | > log_mle: 0.27725 (0.37380) + | > loss_dur: 1.09690 (1.08490) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.49175 (2.97552) + | > current_lr: 0.00000 + | > step_time: 3.60620 (2.86891) + | > loader_time: 0.09370 (0.04987) + + + --> STEP: 210/234 -- GLOBAL_STEP: 2550 + | > loss: 1.34769 (1.45584) + | > log_mle: 0.23817 (0.37092) + | > loss_dur: 1.10952 (1.08492) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.32285 (2.98663) + | > current_lr: 0.00000 + | > step_time: 4.00450 (2.91719) + | > loader_time: 0.00380 (0.04924) + + + --> STEP: 215/234 -- GLOBAL_STEP: 2555 + | > loss: 1.33665 (1.45327) + | > log_mle: 0.26028 (0.36793) + | > loss_dur: 1.07637 (1.08534) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.91243 (2.98912) + | > current_lr: 0.00000 + | > step_time: 7.80950 (3.02012) + | > loader_time: 0.18880 (0.05355) + + + --> STEP: 220/234 -- GLOBAL_STEP: 2560 + | > loss: 1.34304 (1.45048) + | > log_mle: 0.23315 (0.36477) + | > loss_dur: 1.10989 (1.08571) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.90026 (2.99274) + | > current_lr: 0.00000 + | > step_time: 3.60240 (3.04286) + | > loader_time: 0.00510 (0.05374) + + + --> STEP: 225/234 -- GLOBAL_STEP: 2565 + | > loss: 1.31539 (1.44826) + | > log_mle: 0.20703 (0.36177) + | > loss_dur: 1.10836 (1.08649) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.96920 (2.99738) + | > current_lr: 0.00000 + | > step_time: 0.25310 (3.00452) + | > loader_time: 0.00460 (0.05305) + + + --> STEP: 230/234 -- GLOBAL_STEP: 2570 + | > loss: 1.45777 (1.44721) + | > log_mle: 0.17120 (0.35855) + | > loss_dur: 1.28657 (1.08865) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.60275 (3.00776) + | > current_lr: 0.00000 + | > step_time: 0.25090 (2.94471) + | > loader_time: 0.00350 (0.05197) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17809 (-0.22531) + | > avg_loss: 1.32870 (-0.13158) + | > avg_log_mle: 0.29134 (-0.05198) + | > avg_loss_dur: 1.03736 (-0.07960) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_2574.pth + + > EPOCH: 11/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 11:02:43)  + + --> STEP: 1/234 -- GLOBAL_STEP: 2575 + | > loss: 2.01064 (2.01064) + | > log_mle: 0.45982 (0.45982) + | > loss_dur: 1.55082 (1.55082) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.96519 (3.96519) + | > current_lr: 0.00000 + | > step_time: 2.60940 (2.60938) + | > loader_time: 0.09520 (0.09519) + + + --> STEP: 6/234 -- GLOBAL_STEP: 2580 + | > loss: 1.86030 (1.94259) + | > log_mle: 0.43883 (0.44764) + | > loss_dur: 1.42147 (1.49495) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.68093 (3.74348) + | > current_lr: 0.00000 + | > step_time: 1.93590 (4.36547) + | > loader_time: 0.00190 (0.05159) + + + --> STEP: 11/234 -- GLOBAL_STEP: 2585 + | > loss: 1.60448 (1.82216) + | > log_mle: 0.43003 (0.43396) + | > loss_dur: 1.17445 (1.38820) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.13965 (3.52814) + | > current_lr: 0.00000 + | > step_time: 2.69940 (4.22903) + | > loader_time: 0.00260 (0.06460) + + + --> STEP: 16/234 -- GLOBAL_STEP: 2590 + | > loss: 1.51180 (1.74225) + | > log_mle: 0.40644 (0.42843) + | > loss_dur: 1.10536 (1.31382) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.99031 (3.39138) + | > current_lr: 0.00000 + | > step_time: 2.69960 (4.32303) + | > loader_time: 0.00550 (0.06282) + + + --> STEP: 21/234 -- GLOBAL_STEP: 2595 + | > loss: 1.47232 (1.69749) + | > log_mle: 0.41977 (0.42636) + | > loss_dur: 1.05255 (1.27113) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.91810 (3.31142) + | > current_lr: 0.00000 + | > step_time: 3.22410 (4.59422) + | > loader_time: 0.09320 (0.06329) + + + --> STEP: 26/234 -- GLOBAL_STEP: 2600 + | > loss: 1.45812 (1.65653) + | > log_mle: 0.40879 (0.42277) + | > loss_dur: 1.04933 (1.23376) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.75408 (3.23337) + | > current_lr: 0.00000 + | > step_time: 2.68570 (4.32998) + | > loader_time: 0.00130 (0.06322) + + + --> STEP: 31/234 -- GLOBAL_STEP: 2605 + | > loss: 1.46667 (1.62091) + | > log_mle: 0.39444 (0.41931) + | > loss_dur: 1.07223 (1.20160) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.93850 (3.17823) + | > current_lr: 0.00000 + | > step_time: 3.43140 (4.32845) + | > loader_time: 0.09240 (0.06043) + + + --> STEP: 36/234 -- GLOBAL_STEP: 2610 + | > loss: 1.48928 (1.59797) + | > log_mle: 0.39714 (0.41626) + | > loss_dur: 1.09214 (1.18170) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.01412 (3.18093) + | > current_lr: 0.00000 + | > step_time: 6.29430 (4.46786) + | > loader_time: 0.10950 (0.06315) + + + --> STEP: 41/234 -- GLOBAL_STEP: 2615 + | > loss: 1.41263 (1.57780) + | > log_mle: 0.38518 (0.41379) + | > loss_dur: 1.02744 (1.16401) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.87121 (3.15142) + | > current_lr: 0.00000 + | > step_time: 3.31870 (4.30279) + | > loader_time: 0.07490 (0.06399) + + + --> STEP: 46/234 -- GLOBAL_STEP: 2620 + | > loss: 1.37731 (1.56189) + | > log_mle: 0.39260 (0.41245) + | > loss_dur: 0.98471 (1.14944) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76832 (3.12224) + | > current_lr: 0.00000 + | > step_time: 1.88650 (4.02643) + | > loader_time: 0.01980 (0.06158) + + + --> STEP: 51/234 -- GLOBAL_STEP: 2625 + | > loss: 1.38622 (1.54324) + | > log_mle: 0.40143 (0.41058) + | > loss_dur: 0.98479 (1.13266) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.74465 (3.08739) + | > current_lr: 0.00000 + | > step_time: 4.49550 (3.99478) + | > loader_time: 0.11790 (0.06492) + + + --> STEP: 56/234 -- GLOBAL_STEP: 2630 + | > loss: 1.43134 (1.52960) + | > log_mle: 0.38952 (0.40861) + | > loss_dur: 1.04182 (1.12099) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.85876 (3.06020) + | > current_lr: 0.00000 + | > step_time: 1.29090 (3.85367) + | > loader_time: 0.00200 (0.06079) + + + --> STEP: 61/234 -- GLOBAL_STEP: 2635 + | > loss: 1.37521 (1.51581) + | > log_mle: 0.39388 (0.40683) + | > loss_dur: 0.98133 (1.10897) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.72219 (3.03520) + | > current_lr: 0.00000 + | > step_time: 1.71010 (3.68862) + | > loader_time: 0.08390 (0.05734) + + + --> STEP: 66/234 -- GLOBAL_STEP: 2640 + | > loss: 1.33438 (1.50451) + | > log_mle: 0.39597 (0.40512) + | > loss_dur: 0.93841 (1.09939) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.59533 (3.01644) + | > current_lr: 0.00000 + | > step_time: 1.09430 (3.50939) + | > loader_time: 0.00230 (0.05315) + + + --> STEP: 71/234 -- GLOBAL_STEP: 2645 + | > loss: 1.39333 (1.49430) + | > log_mle: 0.37106 (0.40350) + | > loss_dur: 1.02226 (1.09080) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.82516 (2.99522) + | > current_lr: 0.00000 + | > step_time: 2.00800 (3.43001) + | > loader_time: 0.00270 (0.05111) + + + --> STEP: 76/234 -- GLOBAL_STEP: 2650 + | > loss: 1.33262 (1.48312) + | > log_mle: 0.37788 (0.40185) + | > loss_dur: 0.95474 (1.08127) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.66354 (2.97286) + | > current_lr: 0.00000 + | > step_time: 1.11220 (3.30327) + | > loader_time: 0.00220 (0.05146) + + + --> STEP: 81/234 -- GLOBAL_STEP: 2655 + | > loss: 1.30726 (1.47293) + | > log_mle: 0.36391 (0.40022) + | > loss_dur: 0.94335 (1.07272) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.70339 (2.95440) + | > current_lr: 0.00000 + | > step_time: 2.18890 (3.22530) + | > loader_time: 0.00320 (0.04949) + + + --> STEP: 86/234 -- GLOBAL_STEP: 2660 + | > loss: 1.35721 (1.46517) + | > log_mle: 0.36839 (0.39872) + | > loss_dur: 0.98882 (1.06645) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.85667 (2.94486) + | > current_lr: 0.00000 + | > step_time: 2.61730 (3.14500) + | > loader_time: 0.00210 (0.04869) + + + --> STEP: 91/234 -- GLOBAL_STEP: 2665 + | > loss: 1.29849 (1.45498) + | > log_mle: 0.36651 (0.39647) + | > loss_dur: 0.93198 (1.05851) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.79433 (2.93264) + | > current_lr: 0.00000 + | > step_time: 1.38850 (3.10841) + | > loader_time: 0.00230 (0.04709) + + + --> STEP: 96/234 -- GLOBAL_STEP: 2670 + | > loss: 1.32853 (1.44685) + | > log_mle: 0.36643 (0.39350) + | > loss_dur: 0.96209 (1.05335) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.64896 (2.92738) + | > current_lr: 0.00000 + | > step_time: 2.10960 (3.04476) + | > loader_time: 0.09540 (0.04765) + + + --> STEP: 101/234 -- GLOBAL_STEP: 2675 + | > loss: 1.29116 (1.43960) + | > log_mle: 0.33425 (0.39110) + | > loss_dur: 0.95691 (1.04850) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76762 (2.92110) + | > current_lr: 0.00000 + | > step_time: 2.60390 (3.00777) + | > loader_time: 0.06620 (0.04771) + + + --> STEP: 106/234 -- GLOBAL_STEP: 2680 + | > loss: 1.35631 (1.43360) + | > log_mle: 0.33869 (0.38877) + | > loss_dur: 1.01762 (1.04483) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.30597 (2.91934) + | > current_lr: 0.00000 + | > step_time: 1.69520 (2.98081) + | > loader_time: 0.00160 (0.04735) + + + --> STEP: 111/234 -- GLOBAL_STEP: 2685 + | > loss: 1.31751 (1.42750) + | > log_mle: 0.31589 (0.38642) + | > loss_dur: 1.00162 (1.04108) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.70988 (2.91852) + | > current_lr: 0.00000 + | > step_time: 1.58300 (2.94040) + | > loader_time: 0.00260 (0.04600) + + + --> STEP: 116/234 -- GLOBAL_STEP: 2690 + | > loss: 1.28072 (1.42165) + | > log_mle: 0.33132 (0.38422) + | > loss_dur: 0.94940 (1.03742) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.94045 (2.91376) + | > current_lr: 0.00000 + | > step_time: 2.41170 (2.92576) + | > loader_time: 0.08090 (0.04645) + + + --> STEP: 121/234 -- GLOBAL_STEP: 2695 + | > loss: 1.26604 (1.41568) + | > log_mle: 0.38113 (0.38250) + | > loss_dur: 0.88491 (1.03317) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.43078 (2.90214) + | > current_lr: 0.00000 + | > step_time: 1.49980 (2.93146) + | > loader_time: 0.07880 (0.04738) + + + --> STEP: 126/234 -- GLOBAL_STEP: 2700 + | > loss: 1.28537 (1.40931) + | > log_mle: 0.29978 (0.38041) + | > loss_dur: 0.98559 (1.02890) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.69177 (2.88986) + | > current_lr: 0.00000 + | > step_time: 11.19170 (2.99222) + | > loader_time: 0.08930 (0.04699) + + + --> STEP: 131/234 -- GLOBAL_STEP: 2705 + | > loss: 1.25110 (1.40364) + | > log_mle: 0.28703 (0.37788) + | > loss_dur: 0.96407 (1.02576) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.33755 (2.88849) + | > current_lr: 0.00000 + | > step_time: 2.70280 (2.96266) + | > loader_time: 0.00310 (0.04601) + + + --> STEP: 136/234 -- GLOBAL_STEP: 2710 + | > loss: 1.28941 (1.39886) + | > log_mle: 0.25872 (0.37543) + | > loss_dur: 1.03070 (1.02342) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.01801 (2.89584) + | > current_lr: 0.00000 + | > step_time: 2.49310 (2.91372) + | > loader_time: 0.00280 (0.04459) + + + --> STEP: 141/234 -- GLOBAL_STEP: 2715 + | > loss: 1.24911 (1.39455) + | > log_mle: 0.30223 (0.37332) + | > loss_dur: 0.94688 (1.02123) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.23828 (2.91604) + | > current_lr: 0.00000 + | > step_time: 1.61400 (2.87982) + | > loader_time: 0.08860 (0.04486) + + + --> STEP: 146/234 -- GLOBAL_STEP: 2720 + | > loss: 1.26180 (1.39024) + | > log_mle: 0.27547 (0.37029) + | > loss_dur: 0.98633 (1.01995) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.33712 (2.93739) + | > current_lr: 0.00000 + | > step_time: 3.49840 (2.86761) + | > loader_time: 0.07660 (0.04566) + + + --> STEP: 151/234 -- GLOBAL_STEP: 2725 + | > loss: 1.22571 (1.38494) + | > log_mle: 0.29107 (0.36765) + | > loss_dur: 0.93465 (1.01729) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.55171 (2.93475) + | > current_lr: 0.00000 + | > step_time: 1.81400 (2.85767) + | > loader_time: 0.08460 (0.04592) + + + --> STEP: 156/234 -- GLOBAL_STEP: 2730 + | > loss: 1.23229 (1.38063) + | > log_mle: 0.27456 (0.36442) + | > loss_dur: 0.95773 (1.01621) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.80409 (2.93255) + | > current_lr: 0.00000 + | > step_time: 4.21330 (2.84831) + | > loader_time: 0.10260 (0.04556) + + + --> STEP: 161/234 -- GLOBAL_STEP: 2735 + | > loss: 1.28187 (1.37701) + | > log_mle: 0.26230 (0.36155) + | > loss_dur: 1.01957 (1.01546) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76587 (2.92798) + | > current_lr: 0.00000 + | > step_time: 2.51110 (2.81021) + | > loader_time: 0.19510 (0.04647) + + + --> STEP: 166/234 -- GLOBAL_STEP: 2740 + | > loss: 1.24331 (1.37277) + | > log_mle: 0.28573 (0.35895) + | > loss_dur: 0.95758 (1.01382) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.72648 (2.92646) + | > current_lr: 0.00000 + | > step_time: 4.40490 (2.85635) + | > loader_time: 0.10500 (0.04739) + + + --> STEP: 171/234 -- GLOBAL_STEP: 2745 + | > loss: 1.23493 (1.36994) + | > log_mle: 0.23168 (0.35572) + | > loss_dur: 1.00325 (1.01422) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.33439 (2.95979) + | > current_lr: 0.00000 + | > step_time: 3.60710 (2.85990) + | > loader_time: 0.09660 (0.04786) + + + --> STEP: 176/234 -- GLOBAL_STEP: 2750 + | > loss: 1.23539 (1.36622) + | > log_mle: 0.24482 (0.35249) + | > loss_dur: 0.99058 (1.01372) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.94712 (2.96464) + | > current_lr: 0.00000 + | > step_time: 1.82550 (2.84425) + | > loader_time: 0.09230 (0.04867) + + + --> STEP: 181/234 -- GLOBAL_STEP: 2755 + | > loss: 1.26344 (1.36296) + | > log_mle: 0.27615 (0.34954) + | > loss_dur: 0.98729 (1.01342) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.14828 (2.97260) + | > current_lr: 0.00000 + | > step_time: 1.81090 (2.83546) + | > loader_time: 0.08370 (0.04834) + + + --> STEP: 186/234 -- GLOBAL_STEP: 2760 + | > loss: 1.26375 (1.36017) + | > log_mle: 0.25660 (0.34658) + | > loss_dur: 1.00715 (1.01359) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.96240 (2.97673) + | > current_lr: 0.00000 + | > step_time: 5.69740 (2.87643) + | > loader_time: 0.00910 (0.04920) + + + --> STEP: 191/234 -- GLOBAL_STEP: 2765 + | > loss: 1.25326 (1.35676) + | > log_mle: 0.25257 (0.34368) + | > loss_dur: 1.00069 (1.01308) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.30759 (3.00164) + | > current_lr: 0.00000 + | > step_time: 3.87260 (2.88269) + | > loader_time: 0.51290 (0.05315) + + + --> STEP: 196/234 -- GLOBAL_STEP: 2770 + | > loss: 1.27669 (1.35395) + | > log_mle: 0.26045 (0.34079) + | > loss_dur: 1.01624 (1.01316) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.75739 (3.06548) + | > current_lr: 0.00000 + | > step_time: 3.89100 (2.91418) + | > loader_time: 0.00590 (0.05543) + + + --> STEP: 201/234 -- GLOBAL_STEP: 2775 + | > loss: 1.22550 (1.35051) + | > log_mle: 0.26070 (0.33804) + | > loss_dur: 0.96481 (1.01247) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.35333 (3.09255) + | > current_lr: 0.00000 + | > step_time: 2.21280 (2.90995) + | > loader_time: 0.18890 (0.05594) + + + --> STEP: 206/234 -- GLOBAL_STEP: 2780 + | > loss: 1.20479 (1.34729) + | > log_mle: 0.20780 (0.33499) + | > loss_dur: 0.99699 (1.01230) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.90762 (3.12859) + | > current_lr: 0.00000 + | > step_time: 4.71150 (2.94383) + | > loader_time: 0.18900 (0.05648) + + + --> STEP: 211/234 -- GLOBAL_STEP: 2785 + | > loss: 1.26059 (1.34445) + | > log_mle: 0.16597 (0.33162) + | > loss_dur: 1.09462 (1.01282) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.43594 (3.19389) + | > current_lr: 0.00000 + | > step_time: 3.70510 (2.98267) + | > loader_time: 0.08580 (0.05750) + + + --> STEP: 216/234 -- GLOBAL_STEP: 2790 + | > loss: 1.19375 (1.34144) + | > log_mle: 0.17516 (0.32840) + | > loss_dur: 1.01859 (1.01304) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.66569 (3.23305) + | > current_lr: 0.00000 + | > step_time: 6.29800 (3.00428) + | > loader_time: 0.28800 (0.05982) + + + --> STEP: 221/234 -- GLOBAL_STEP: 2795 + | > loss: 1.22744 (1.33866) + | > log_mle: 0.21581 (0.32511) + | > loss_dur: 1.01163 (1.01355) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.70885 (3.26276) + | > current_lr: 0.00000 + | > step_time: 4.00850 (3.03221) + | > loader_time: 0.00500 (0.05896) + + + --> STEP: 226/234 -- GLOBAL_STEP: 2800 + | > loss: 1.21232 (1.33616) + | > log_mle: 0.15723 (0.32156) + | > loss_dur: 1.05509 (1.01459) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.33539 (3.27214) + | > current_lr: 0.00000 + | > step_time: 0.40320 (3.00716) + | > loader_time: 0.00480 (0.05817) + + + --> STEP: 231/234 -- GLOBAL_STEP: 2805 + | > loss: 1.38845 (1.33556) + | > log_mle: 0.11558 (0.31783) + | > loss_dur: 1.27287 (1.01773) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.10657 (3.28877) + | > current_lr: 0.00000 + | > step_time: 0.26720 (2.94765) + | > loader_time: 0.00480 (0.05700) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.78081 (+0.60272) + | > avg_loss: 1.22975 (-0.09895) + | > avg_log_mle: 0.25239 (-0.03894) + | > avg_loss_dur: 0.97736 (-0.06001) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_2808.pth + + > EPOCH: 12/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 11:15:33)  + + --> STEP: 2/234 -- GLOBAL_STEP: 2810 + | > loss: 1.91135 (1.89802) + | > log_mle: 0.45646 (0.44569) + | > loss_dur: 1.45488 (1.45233) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.44816 (3.64182) + | > current_lr: 0.00000 + | > step_time: 1.80300 (2.34801) + | > loader_time: 0.00160 (0.00176) + + + --> STEP: 7/234 -- GLOBAL_STEP: 2815 + | > loss: 1.65163 (1.80940) + | > log_mle: 0.38737 (0.41628) + | > loss_dur: 1.26426 (1.39311) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.18991 (3.57415) + | > current_lr: 0.00000 + | > step_time: 5.49080 (4.38213) + | > loader_time: 0.20090 (0.07224) + + + --> STEP: 12/234 -- GLOBAL_STEP: 2820 + | > loss: 1.47644 (1.70087) + | > log_mle: 0.39861 (0.40719) + | > loss_dur: 1.07783 (1.29368) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.89796 (3.37179) + | > current_lr: 0.00000 + | > step_time: 0.88350 (4.15004) + | > loader_time: 0.00170 (0.09165) + + + --> STEP: 17/234 -- GLOBAL_STEP: 2825 + | > loss: 1.51043 (1.63850) + | > log_mle: 0.40370 (0.40283) + | > loss_dur: 1.10673 (1.23567) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.98323 (3.23217) + | > current_lr: 0.00000 + | > step_time: 0.78440 (3.30046) + | > loader_time: 0.00160 (0.07028) + + + --> STEP: 22/234 -- GLOBAL_STEP: 2830 + | > loss: 1.38044 (1.59205) + | > log_mle: 0.37628 (0.39987) + | > loss_dur: 1.00415 (1.19218) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.73557 (3.13745) + | > current_lr: 0.00000 + | > step_time: 6.01310 (3.31373) + | > loader_time: 0.09650 (0.06802) + + + --> STEP: 27/234 -- GLOBAL_STEP: 2835 + | > loss: 1.33201 (1.55274) + | > log_mle: 0.37314 (0.39660) + | > loss_dur: 0.95888 (1.15614) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.58658 (3.05125) + | > current_lr: 0.00000 + | > step_time: 8.89070 (3.53248) + | > loader_time: 0.09880 (0.05988) + + + --> STEP: 32/234 -- GLOBAL_STEP: 2840 + | > loss: 1.30914 (1.52036) + | > log_mle: 0.35551 (0.39297) + | > loss_dur: 0.95363 (1.12739) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.59007 (2.98864) + | > current_lr: 0.00000 + | > step_time: 3.29670 (3.60874) + | > loader_time: 0.00310 (0.05392) + + + --> STEP: 37/234 -- GLOBAL_STEP: 2845 + | > loss: 1.28598 (1.49879) + | > log_mle: 0.36730 (0.39044) + | > loss_dur: 0.91868 (1.10835) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.56652 (3.03736) + | > current_lr: 0.00000 + | > step_time: 3.11350 (3.65444) + | > loader_time: 0.08870 (0.05356) + + + --> STEP: 42/234 -- GLOBAL_STEP: 2850 + | > loss: 1.40034 (1.48232) + | > log_mle: 0.40312 (0.38906) + | > loss_dur: 0.99722 (1.09326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.80523 (3.00282) + | > current_lr: 0.00000 + | > step_time: 4.79690 (3.71977) + | > loader_time: 0.00380 (0.04990) + + + --> STEP: 47/234 -- GLOBAL_STEP: 2855 + | > loss: 1.26190 (1.46387) + | > log_mle: 0.36739 (0.38691) + | > loss_dur: 0.89451 (1.07696) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.54506 (2.96214) + | > current_lr: 0.00000 + | > step_time: 0.80720 (3.52029) + | > loader_time: 0.00200 (0.04484) + + + --> STEP: 52/234 -- GLOBAL_STEP: 2860 + | > loss: 1.30901 (1.44728) + | > log_mle: 0.38512 (0.38560) + | > loss_dur: 0.92389 (1.06168) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.62924 (2.92578) + | > current_lr: 0.00000 + | > step_time: 2.89920 (3.31733) + | > loader_time: 0.09840 (0.04423) + + + --> STEP: 57/234 -- GLOBAL_STEP: 2865 + | > loss: 1.31048 (1.43416) + | > log_mle: 0.37994 (0.38362) + | > loss_dur: 0.93054 (1.05054) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.57739 (2.89316) + | > current_lr: 0.00000 + | > step_time: 1.90030 (3.14929) + | > loader_time: 0.00150 (0.04052) + + + --> STEP: 62/234 -- GLOBAL_STEP: 2870 + | > loss: 1.29715 (1.42087) + | > log_mle: 0.34036 (0.38131) + | > loss_dur: 0.95678 (1.03956) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.73890 (2.86794) + | > current_lr: 0.00000 + | > step_time: 2.32580 (3.02726) + | > loader_time: 0.00300 (0.03743) + + + --> STEP: 67/234 -- GLOBAL_STEP: 2875 + | > loss: 1.23930 (1.40890) + | > log_mle: 0.34342 (0.37975) + | > loss_dur: 0.89589 (1.02914) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.65271 (2.84258) + | > current_lr: 0.00000 + | > step_time: 1.13090 (2.93303) + | > loader_time: 0.00230 (0.03777) + + + --> STEP: 72/234 -- GLOBAL_STEP: 2880 + | > loss: 1.22355 (1.39908) + | > log_mle: 0.35381 (0.37832) + | > loss_dur: 0.86974 (1.02076) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.41688 (2.82737) + | > current_lr: 0.00000 + | > step_time: 2.12460 (2.85719) + | > loader_time: 0.09590 (0.03679) + + + --> STEP: 77/234 -- GLOBAL_STEP: 2885 + | > loss: 1.24232 (1.38871) + | > log_mle: 0.33986 (0.37657) + | > loss_dur: 0.90246 (1.01213) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.45971 (2.80599) + | > current_lr: 0.00000 + | > step_time: 2.49400 (2.79336) + | > loader_time: 0.00210 (0.03457) + + + --> STEP: 82/234 -- GLOBAL_STEP: 2890 + | > loss: 1.18515 (1.37861) + | > log_mle: 0.36105 (0.37529) + | > loss_dur: 0.82410 (1.00332) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.33114 (2.78385) + | > current_lr: 0.00000 + | > step_time: 1.75190 (2.72522) + | > loader_time: 0.00210 (0.03364) + + + --> STEP: 87/234 -- GLOBAL_STEP: 2895 + | > loss: 1.20088 (1.37145) + | > log_mle: 0.34192 (0.37361) + | > loss_dur: 0.85896 (0.99784) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.42813 (2.77024) + | > current_lr: 0.00000 + | > step_time: 1.82170 (2.67626) + | > loader_time: 0.00250 (0.03407) + + + --> STEP: 92/234 -- GLOBAL_STEP: 2900 + | > loss: 1.19030 (1.36173) + | > log_mle: 0.31966 (0.37108) + | > loss_dur: 0.87064 (0.99066) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.49522 (2.75345) + | > current_lr: 0.00000 + | > step_time: 3.02920 (2.65639) + | > loader_time: 0.00380 (0.03705) + + + --> STEP: 97/234 -- GLOBAL_STEP: 2905 + | > loss: 1.18792 (1.35382) + | > log_mle: 0.32188 (0.36796) + | > loss_dur: 0.86604 (0.98586) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.45080 (2.74209) + | > current_lr: 0.00000 + | > step_time: 2.20390 (2.64139) + | > loader_time: 0.00280 (0.03626) + + + --> STEP: 102/234 -- GLOBAL_STEP: 2910 + | > loss: 1.20373 (1.34713) + | > log_mle: 0.33841 (0.36563) + | > loss_dur: 0.86531 (0.98151) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.36753 (2.73260) + | > current_lr: 0.00000 + | > step_time: 1.89890 (2.59634) + | > loader_time: 0.00330 (0.03699) + + + --> STEP: 107/234 -- GLOBAL_STEP: 2915 + | > loss: 1.16834 (1.34106) + | > log_mle: 0.30426 (0.36285) + | > loss_dur: 0.86408 (0.97821) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.50452 (2.72566) + | > current_lr: 0.00000 + | > step_time: 1.80300 (2.54908) + | > loader_time: 0.08540 (0.03753) + + + --> STEP: 112/234 -- GLOBAL_STEP: 2920 + | > loss: 1.19297 (1.33528) + | > log_mle: 0.30418 (0.36037) + | > loss_dur: 0.88879 (0.97491) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.06386 (2.72745) + | > current_lr: 0.00000 + | > step_time: 2.50840 (2.54255) + | > loader_time: 0.09030 (0.03994) + + + --> STEP: 117/234 -- GLOBAL_STEP: 2925 + | > loss: 1.16210 (1.32946) + | > log_mle: 0.30438 (0.35804) + | > loss_dur: 0.85772 (0.97142) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.44564 (2.72007) + | > current_lr: 0.00000 + | > step_time: 1.47200 (2.54160) + | > loader_time: 0.00230 (0.03985) + + + --> STEP: 122/234 -- GLOBAL_STEP: 2930 + | > loss: 1.15546 (1.32368) + | > log_mle: 0.31338 (0.35633) + | > loss_dur: 0.84208 (0.96735) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.31743 (2.70706) + | > current_lr: 0.00000 + | > step_time: 2.91870 (2.51730) + | > loader_time: 0.08730 (0.04049) + + + --> STEP: 127/234 -- GLOBAL_STEP: 2935 + | > loss: 1.18922 (1.31782) + | > log_mle: 0.28537 (0.35390) + | > loss_dur: 0.90384 (0.96392) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.51906 (2.69960) + | > current_lr: 0.00000 + | > step_time: 0.89760 (2.53256) + | > loader_time: 0.00230 (0.04037) + + + --> STEP: 132/234 -- GLOBAL_STEP: 2940 + | > loss: 1.16758 (1.31218) + | > log_mle: 0.29645 (0.35127) + | > loss_dur: 0.87112 (0.96091) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.29049 (2.71021) + | > current_lr: 0.00000 + | > step_time: 2.28900 (2.50485) + | > loader_time: 0.00310 (0.03896) + + + --> STEP: 137/234 -- GLOBAL_STEP: 2945 + | > loss: 1.24272 (1.30802) + | > log_mle: 0.29626 (0.34864) + | > loss_dur: 0.94646 (0.95938) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.98074 (2.73295) + | > current_lr: 0.00000 + | > step_time: 2.77440 (2.47613) + | > loader_time: 0.02530 (0.03843) + + + --> STEP: 142/234 -- GLOBAL_STEP: 2950 + | > loss: 1.18521 (1.30346) + | > log_mle: 0.27799 (0.34624) + | > loss_dur: 0.90723 (0.95722) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.90675 (2.79427) + | > current_lr: 0.00000 + | > step_time: 2.29750 (2.45794) + | > loader_time: 0.00340 (0.03770) + + + --> STEP: 147/234 -- GLOBAL_STEP: 2955 + | > loss: 1.13950 (1.29868) + | > log_mle: 0.27172 (0.34289) + | > loss_dur: 0.86778 (0.95579) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.67606 (2.82924) + | > current_lr: 0.00000 + | > step_time: 3.00340 (2.45007) + | > loader_time: 0.09880 (0.03785) + + + --> STEP: 152/234 -- GLOBAL_STEP: 2960 + | > loss: 1.15532 (1.29355) + | > log_mle: 0.23228 (0.33978) + | > loss_dur: 0.92304 (0.95376) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.06266 (2.82820) + | > current_lr: 0.00000 + | > step_time: 4.09310 (2.46025) + | > loader_time: 0.10660 (0.03797) + + + --> STEP: 157/234 -- GLOBAL_STEP: 2965 + | > loss: 1.19058 (1.28935) + | > log_mle: 0.26182 (0.33645) + | > loss_dur: 0.92876 (0.95290) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.98214 (2.82647) + | > current_lr: 0.00000 + | > step_time: 11.00750 (2.57412) + | > loader_time: 0.10140 (0.03999) + + + --> STEP: 162/234 -- GLOBAL_STEP: 2970 + | > loss: 1.16099 (1.28550) + | > log_mle: 0.23911 (0.33320) + | > loss_dur: 0.92189 (0.95230) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.60518 (2.83650) + | > current_lr: 0.00000 + | > step_time: 2.40600 (2.56934) + | > loader_time: 0.00430 (0.03936) + + + --> STEP: 167/234 -- GLOBAL_STEP: 2975 + | > loss: 1.16898 (1.28137) + | > log_mle: 0.19707 (0.33017) + | > loss_dur: 0.97191 (0.95120) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.41553 (2.86583) + | > current_lr: 0.00000 + | > step_time: 1.09790 (2.54984) + | > loader_time: 0.08470 (0.03973) + + + --> STEP: 172/234 -- GLOBAL_STEP: 2980 + | > loss: 1.16108 (1.27829) + | > log_mle: 0.20165 (0.32669) + | > loss_dur: 0.95943 (0.95160) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.86413 (2.89093) + | > current_lr: 0.00000 + | > step_time: 5.49770 (2.58324) + | > loader_time: 0.00420 (0.03957) + + + --> STEP: 177/234 -- GLOBAL_STEP: 2985 + | > loss: 1.18279 (1.27451) + | > log_mle: 0.22792 (0.32337) + | > loss_dur: 0.95487 (0.95115) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.03747 (2.92727) + | > current_lr: 0.00000 + | > step_time: 1.98620 (2.63232) + | > loader_time: 0.00590 (0.03970) + + + --> STEP: 182/234 -- GLOBAL_STEP: 2990 + | > loss: 1.18488 (1.27127) + | > log_mle: 0.19614 (0.32003) + | > loss_dur: 0.98873 (0.95124) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.85859 (2.95080) + | > current_lr: 0.00000 + | > step_time: 1.98640 (2.65494) + | > loader_time: 0.10340 (0.04072) + + + --> STEP: 187/234 -- GLOBAL_STEP: 2995 + | > loss: 1.11523 (1.26803) + | > log_mle: 0.18960 (0.31679) + | > loss_dur: 0.92563 (0.95124) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.73315 (2.97302) + | > current_lr: 0.00000 + | > step_time: 3.69190 (2.73528) + | > loader_time: 0.09350 (0.04335) + + + --> STEP: 192/234 -- GLOBAL_STEP: 3000 + | > loss: 1.11716 (1.26454) + | > log_mle: 0.17311 (0.31360) + | > loss_dur: 0.94405 (0.95093) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.51725 (3.01614) + | > current_lr: 0.00000 + | > step_time: 1.50450 (2.81586) + | > loader_time: 0.00340 (0.04574) + + + --> STEP: 197/234 -- GLOBAL_STEP: 3005 + | > loss: 1.09770 (1.26148) + | > log_mle: 0.18727 (0.31056) + | > loss_dur: 0.91044 (0.95092) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.94247 (3.06767) + | > current_lr: 0.00000 + | > step_time: 5.49290 (2.81017) + | > loader_time: 0.09700 (0.04612) + + + --> STEP: 202/234 -- GLOBAL_STEP: 3010 + | > loss: 1.09108 (1.25802) + | > log_mle: 0.12936 (0.30735) + | > loss_dur: 0.96172 (0.95068) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.68541 (3.08899) + | > current_lr: 0.00000 + | > step_time: 4.59640 (2.81987) + | > loader_time: 0.00310 (0.04630) + + + --> STEP: 207/234 -- GLOBAL_STEP: 3015 + | > loss: 1.09102 (1.25482) + | > log_mle: 0.13915 (0.30414) + | > loss_dur: 0.95187 (0.95068) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.92292 (3.10830) + | > current_lr: 0.00000 + | > step_time: 3.28400 (2.85901) + | > loader_time: 0.11120 (0.04813) + + + --> STEP: 212/234 -- GLOBAL_STEP: 3020 + | > loss: 1.12990 (1.25198) + | > log_mle: 0.15094 (0.30058) + | > loss_dur: 0.97896 (0.95140) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.78125 (3.15004) + | > current_lr: 0.00000 + | > step_time: 4.30410 (2.92464) + | > loader_time: 0.00510 (0.04806) + + + --> STEP: 217/234 -- GLOBAL_STEP: 3025 + | > loss: 1.11792 (1.24883) + | > log_mle: 0.13832 (0.29708) + | > loss_dur: 0.97960 (0.95174) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.75807 (3.28740) + | > current_lr: 0.00000 + | > step_time: 4.89330 (2.97095) + | > loader_time: 0.00900 (0.04892) + + + --> STEP: 222/234 -- GLOBAL_STEP: 3030 + | > loss: 1.16332 (1.24617) + | > log_mle: 0.13212 (0.29358) + | > loss_dur: 1.03120 (0.95259) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.61423 (3.35302) + | > current_lr: 0.00000 + | > step_time: 2.99570 (2.96220) + | > loader_time: 0.00300 (0.04792) + + + --> STEP: 227/234 -- GLOBAL_STEP: 3035 + | > loss: 1.12403 (1.24337) + | > log_mle: 0.14084 (0.28985) + | > loss_dur: 0.98319 (0.95353) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.81571 (3.37729) + | > current_lr: 0.00000 + | > step_time: 0.25180 (2.91911) + | > loader_time: 0.00330 (0.04699) + + + --> STEP: 232/234 -- GLOBAL_STEP: 3040 + | > loss: 1.47506 (1.24403) + | > log_mle: 0.01617 (0.28532) + | > loss_dur: 1.45889 (0.95871) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.38580 (3.42590) + | > current_lr: 0.00000 + | > step_time: 0.33120 (2.86212) + | > loader_time: 0.00590 (0.04607) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12865 (-0.65216) + | > avg_loss: 1.14729 (-0.08246) + | > avg_log_mle: 0.21827 (-0.03412) + | > avg_loss_dur: 0.92902 (-0.04834) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_3042.pth + + > EPOCH: 13/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 11:27:58)  + + --> STEP: 3/234 -- GLOBAL_STEP: 3045 + | > loss: 1.66933 (1.76338) + | > log_mle: 0.39296 (0.41445) + | > loss_dur: 1.27637 (1.34893) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.22959 (3.41708) + | > current_lr: 0.00000 + | > step_time: 6.39560 (3.86293) + | > loader_time: 0.00450 (0.02940) + + + --> STEP: 8/234 -- GLOBAL_STEP: 3050 + | > loss: 1.60968 (1.71431) + | > log_mle: 0.36592 (0.39209) + | > loss_dur: 1.24376 (1.32222) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.20270 (3.42815) + | > current_lr: 0.00000 + | > step_time: 3.40200 (5.37186) + | > loader_time: 0.09990 (0.04831) + + + --> STEP: 13/234 -- GLOBAL_STEP: 3055 + | > loss: 1.45588 (1.60941) + | > log_mle: 0.37871 (0.38603) + | > loss_dur: 1.07718 (1.22338) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.74098 (3.22612) + | > current_lr: 0.00000 + | > step_time: 3.88320 (3.96593) + | > loader_time: 0.10360 (0.04403) + + + --> STEP: 18/234 -- GLOBAL_STEP: 3060 + | > loss: 1.37012 (1.54868) + | > log_mle: 0.37419 (0.38195) + | > loss_dur: 0.99592 (1.16673) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.63186 (3.07984) + | > current_lr: 0.00000 + | > step_time: 2.19530 (3.47060) + | > loader_time: 0.08740 (0.04199) + + + --> STEP: 23/234 -- GLOBAL_STEP: 3065 + | > loss: 1.29176 (1.50405) + | > log_mle: 0.35787 (0.37869) + | > loss_dur: 0.93389 (1.12535) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.52813 (2.97940) + | > current_lr: 0.00000 + | > step_time: 2.40770 (3.00398) + | > loader_time: 0.00280 (0.03749) + + + --> STEP: 28/234 -- GLOBAL_STEP: 3070 + | > loss: 1.28974 (1.46816) + | > log_mle: 0.36049 (0.37578) + | > loss_dur: 0.92925 (1.09239) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.63884 (2.90269) + | > current_lr: 0.00000 + | > step_time: 2.21380 (3.01126) + | > loader_time: 0.08540 (0.04733) + + + --> STEP: 33/234 -- GLOBAL_STEP: 3075 + | > loss: 1.30506 (1.43822) + | > log_mle: 0.36841 (0.37256) + | > loss_dur: 0.93665 (1.06566) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.55371 (2.83701) + | > current_lr: 0.00000 + | > step_time: 1.00930 (2.91594) + | > loader_time: 0.00200 (0.04553) + + + --> STEP: 38/234 -- GLOBAL_STEP: 3080 + | > loss: 1.25894 (1.41624) + | > log_mle: 0.34806 (0.36950) + | > loss_dur: 0.91088 (1.04674) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.54718 (2.91488) + | > current_lr: 0.00000 + | > step_time: 2.53750 (2.78763) + | > loader_time: 0.08470 (0.04415) + + + --> STEP: 43/234 -- GLOBAL_STEP: 3085 + | > loss: 1.24588 (1.39992) + | > log_mle: 0.34344 (0.36806) + | > loss_dur: 0.90244 (1.03185) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.52557 (2.87338) + | > current_lr: 0.00000 + | > step_time: 3.80240 (2.67685) + | > loader_time: 0.19520 (0.04577) + + + --> STEP: 48/234 -- GLOBAL_STEP: 3090 + | > loss: 1.21331 (1.38157) + | > log_mle: 0.35475 (0.36629) + | > loss_dur: 0.85856 (1.01528) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.41572 (2.82783) + | > current_lr: 0.00000 + | > step_time: 1.10160 (2.54758) + | > loader_time: 0.00190 (0.04337) + + + --> STEP: 53/234 -- GLOBAL_STEP: 3095 + | > loss: 1.21173 (1.36569) + | > log_mle: 0.33800 (0.36475) + | > loss_dur: 0.87373 (1.00094) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.32516 (2.78410) + | > current_lr: 0.00000 + | > step_time: 1.58700 (2.44361) + | > loader_time: 0.00290 (0.04128) + + + --> STEP: 58/234 -- GLOBAL_STEP: 3100 + | > loss: 1.19997 (1.35288) + | > log_mle: 0.35340 (0.36314) + | > loss_dur: 0.84657 (0.98974) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.34385 (2.74985) + | > current_lr: 0.00000 + | > step_time: 1.10480 (2.44881) + | > loader_time: 0.00200 (0.04088) + + + --> STEP: 63/234 -- GLOBAL_STEP: 3105 + | > loss: 1.26497 (1.34095) + | > log_mle: 0.33805 (0.36061) + | > loss_dur: 0.92692 (0.98033) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.45382 (2.73502) + | > current_lr: 0.00000 + | > step_time: 1.49370 (2.39332) + | > loader_time: 0.00290 (0.03921) + + + --> STEP: 68/234 -- GLOBAL_STEP: 3110 + | > loss: 1.19961 (1.32841) + | > log_mle: 0.34456 (0.35925) + | > loss_dur: 0.85505 (0.96915) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.36517 (2.70428) + | > current_lr: 0.00000 + | > step_time: 2.11100 (2.38941) + | > loader_time: 0.09340 (0.03790) + + + --> STEP: 73/234 -- GLOBAL_STEP: 3115 + | > loss: 1.19007 (1.31884) + | > log_mle: 0.32719 (0.35759) + | > loss_dur: 0.86288 (0.96124) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.36270 (2.68827) + | > current_lr: 0.00000 + | > step_time: 1.37710 (2.33304) + | > loader_time: 0.00380 (0.03701) + + + --> STEP: 78/234 -- GLOBAL_STEP: 3120 + | > loss: 1.15834 (1.30863) + | > log_mle: 0.34658 (0.35612) + | > loss_dur: 0.81176 (0.95251) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.34746 (2.66683) + | > current_lr: 0.00000 + | > step_time: 1.82040 (2.29458) + | > loader_time: 0.00280 (0.03717) + + + --> STEP: 83/234 -- GLOBAL_STEP: 3125 + | > loss: 1.21458 (1.29983) + | > log_mle: 0.32959 (0.35465) + | > loss_dur: 0.88499 (0.94518) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.60673 (2.65258) + | > current_lr: 0.00000 + | > step_time: 1.20660 (2.26363) + | > loader_time: 0.00290 (0.03518) + + + --> STEP: 88/234 -- GLOBAL_STEP: 3130 + | > loss: 1.11707 (1.29192) + | > log_mle: 0.29365 (0.35256) + | > loss_dur: 0.82342 (0.93935) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76862 (2.65138) + | > current_lr: 0.00000 + | > step_time: 1.41260 (2.25021) + | > loader_time: 0.00320 (0.03433) + + + --> STEP: 93/234 -- GLOBAL_STEP: 3135 + | > loss: 1.08730 (1.28231) + | > log_mle: 0.28202 (0.34985) + | > loss_dur: 0.80529 (0.93246) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.72707 (2.64389) + | > current_lr: 0.00000 + | > step_time: 2.09960 (2.24773) + | > loader_time: 0.00300 (0.03463) + + + --> STEP: 98/234 -- GLOBAL_STEP: 3140 + | > loss: 1.17006 (1.27554) + | > log_mle: 0.33255 (0.34713) + | > loss_dur: 0.83750 (0.92841) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.29263 (2.63606) + | > current_lr: 0.00000 + | > step_time: 2.09460 (2.21801) + | > loader_time: 0.00300 (0.03552) + + + --> STEP: 103/234 -- GLOBAL_STEP: 3145 + | > loss: 1.14460 (1.26876) + | > log_mle: 0.27157 (0.34411) + | > loss_dur: 0.87303 (0.92466) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.86086 (2.65992) + | > current_lr: 0.00000 + | > step_time: 2.49030 (2.22196) + | > loader_time: 0.00310 (0.03399) + + + --> STEP: 108/234 -- GLOBAL_STEP: 3150 + | > loss: 1.13425 (1.26266) + | > log_mle: 0.30611 (0.34156) + | > loss_dur: 0.82813 (0.92110) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.33295 (2.68364) + | > current_lr: 0.00000 + | > step_time: 1.59240 (2.21057) + | > loader_time: 0.00350 (0.03320) + + + --> STEP: 113/234 -- GLOBAL_STEP: 3155 + | > loss: 1.11100 (1.25692) + | > log_mle: 0.27513 (0.33869) + | > loss_dur: 0.83587 (0.91823) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.76429 (2.71374) + | > current_lr: 0.00000 + | > step_time: 3.59840 (2.26258) + | > loader_time: 0.10170 (0.03516) + + + --> STEP: 118/234 -- GLOBAL_STEP: 3160 + | > loss: 1.15837 (1.25169) + | > log_mle: 0.29576 (0.33644) + | > loss_dur: 0.86260 (0.91525) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.89216 (2.71643) + | > current_lr: 0.00000 + | > step_time: 2.41340 (2.26256) + | > loader_time: 0.02020 (0.03618) + + + --> STEP: 123/234 -- GLOBAL_STEP: 3165 + | > loss: 1.10961 (1.24579) + | > log_mle: 0.31978 (0.33487) + | > loss_dur: 0.78983 (0.91092) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.15847 (2.71018) + | > current_lr: 0.00000 + | > step_time: 1.90400 (2.23885) + | > loader_time: 0.07600 (0.03606) + + + --> STEP: 128/234 -- GLOBAL_STEP: 3170 + | > loss: 1.07994 (1.23994) + | > log_mle: 0.27220 (0.33196) + | > loss_dur: 0.80775 (0.90798) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.78192 (2.72466) + | > current_lr: 0.00000 + | > step_time: 1.00480 (2.22985) + | > loader_time: 0.00220 (0.03620) + + + --> STEP: 133/234 -- GLOBAL_STEP: 3175 + | > loss: 1.13511 (1.23484) + | > log_mle: 0.26206 (0.32912) + | > loss_dur: 0.87305 (0.90571) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.05777 (2.76485) + | > current_lr: 0.00000 + | > step_time: 2.19730 (2.22312) + | > loader_time: 0.00370 (0.03496) + + + --> STEP: 138/234 -- GLOBAL_STEP: 3180 + | > loss: 1.11852 (1.23067) + | > log_mle: 0.28674 (0.32653) + | > loss_dur: 0.83178 (0.90414) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.99130 (2.77556) + | > current_lr: 0.00000 + | > step_time: 2.50210 (2.24125) + | > loader_time: 0.00860 (0.03580) + + + --> STEP: 143/234 -- GLOBAL_STEP: 3185 + | > loss: 1.10686 (1.22604) + | > log_mle: 0.20458 (0.32338) + | > loss_dur: 0.90228 (0.90266) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.93376 (2.79615) + | > current_lr: 0.00000 + | > step_time: 2.00270 (2.25655) + | > loader_time: 0.19470 (0.03726) + + + --> STEP: 148/234 -- GLOBAL_STEP: 3190 + | > loss: 1.01587 (1.22064) + | > log_mle: 0.24788 (0.32011) + | > loss_dur: 0.76799 (0.90053) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.67505 (2.83101) + | > current_lr: 0.00000 + | > step_time: 1.90770 (2.25673) + | > loader_time: 0.10160 (0.03680) + + + --> STEP: 153/234 -- GLOBAL_STEP: 3195 + | > loss: 1.06327 (1.21590) + | > log_mle: 0.17910 (0.31637) + | > loss_dur: 0.88417 (0.89953) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.10823 (2.85468) + | > current_lr: 0.00000 + | > step_time: 1.80190 (2.29618) + | > loader_time: 0.00370 (0.03891) + + + --> STEP: 158/234 -- GLOBAL_STEP: 3200 + | > loss: 1.06604 (1.21173) + | > log_mle: 0.21363 (0.31305) + | > loss_dur: 0.85241 (0.89867) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.30500 (2.88006) + | > current_lr: 0.00000 + | > step_time: 4.60860 (2.31666) + | > loader_time: 0.08400 (0.03882) + + + --> STEP: 163/234 -- GLOBAL_STEP: 3205 + | > loss: 1.04831 (1.20773) + | > log_mle: 0.23038 (0.30974) + | > loss_dur: 0.81793 (0.89799) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.12440 (2.94256) + | > current_lr: 0.00000 + | > step_time: 4.00620 (2.38435) + | > loader_time: 0.09950 (0.04240) + + + --> STEP: 168/234 -- GLOBAL_STEP: 3210 + | > loss: 1.12826 (1.20408) + | > log_mle: 0.19434 (0.30632) + | > loss_dur: 0.93392 (0.89776) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.90943 (2.97752) + | > current_lr: 0.00000 + | > step_time: 1.70390 (2.38784) + | > loader_time: 0.00390 (0.04134) + + + --> STEP: 173/234 -- GLOBAL_STEP: 3215 + | > loss: 1.08797 (1.20076) + | > log_mle: 0.19160 (0.30263) + | > loss_dur: 0.89637 (0.89813) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.04055 (3.03929) + | > current_lr: 0.00000 + | > step_time: 1.90020 (2.42229) + | > loader_time: 0.09210 (0.04252) + + + --> STEP: 178/234 -- GLOBAL_STEP: 3220 + | > loss: 1.03322 (1.19668) + | > log_mle: 0.14758 (0.29890) + | > loss_dur: 0.88564 (0.89778) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.90408 (3.19320) + | > current_lr: 0.00000 + | > step_time: 0.87430 (2.42723) + | > loader_time: 0.00300 (0.04230) + + + --> STEP: 183/234 -- GLOBAL_STEP: 3225 + | > loss: 1.07244 (1.19360) + | > log_mle: 0.15548 (0.29548) + | > loss_dur: 0.91696 (0.89812) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.67405 (3.34366) + | > current_lr: 0.00000 + | > step_time: 2.10020 (2.46273) + | > loader_time: 0.11540 (0.04286) + + + --> STEP: 188/234 -- GLOBAL_STEP: 3230 + | > loss: 1.05073 (1.19031) + | > log_mle: 0.14655 (0.29203) + | > loss_dur: 0.90418 (0.89828) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.50580 (3.42819) + | > current_lr: 0.00000 + | > step_time: 2.70760 (2.46486) + | > loader_time: 0.00400 (0.04487) + + + --> STEP: 193/234 -- GLOBAL_STEP: 3235 + | > loss: 1.04705 (1.18691) + | > log_mle: 0.15036 (0.28874) + | > loss_dur: 0.89670 (0.89817) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.04981 (3.52359) + | > current_lr: 0.00000 + | > step_time: 9.53630 (2.56915) + | > loader_time: 0.09910 (0.04527) + + + --> STEP: 198/234 -- GLOBAL_STEP: 3240 + | > loss: 1.05164 (1.18388) + | > log_mle: 0.15286 (0.28556) + | > loss_dur: 0.89877 (0.89831) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.35540 (3.58729) + | > current_lr: 0.00000 + | > step_time: 2.99800 (2.60869) + | > loader_time: 0.00570 (0.04524) + + + --> STEP: 203/234 -- GLOBAL_STEP: 3245 + | > loss: 1.07926 (1.18057) + | > log_mle: 0.18812 (0.28241) + | > loss_dur: 0.89114 (0.89816) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.94647 (3.65557) + | > current_lr: 0.00000 + | > step_time: 7.30270 (2.69375) + | > loader_time: 0.00390 (0.04616) + + + --> STEP: 208/234 -- GLOBAL_STEP: 3250 + | > loss: 1.06586 (1.17726) + | > log_mle: 0.14105 (0.27888) + | > loss_dur: 0.92481 (0.89838) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.07970 (3.77097) + | > current_lr: 0.00000 + | > step_time: 6.70470 (2.73332) + | > loader_time: 0.08660 (0.04598) + + + --> STEP: 213/234 -- GLOBAL_STEP: 3255 + | > loss: 1.02349 (1.17420) + | > log_mle: 0.10443 (0.27503) + | > loss_dur: 0.91906 (0.89917) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.38652 (3.83660) + | > current_lr: 0.00000 + | > step_time: 8.59230 (2.79789) + | > loader_time: 0.10340 (0.04706) + + + --> STEP: 218/234 -- GLOBAL_STEP: 3260 + | > loss: 1.05714 (1.17115) + | > log_mle: 0.12788 (0.27152) + | > loss_dur: 0.92926 (0.89963) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.12870 (3.93434) + | > current_lr: 0.00000 + | > step_time: 3.69200 (2.88384) + | > loader_time: 0.01050 (0.04914) + + + --> STEP: 223/234 -- GLOBAL_STEP: 3265 + | > loss: 1.03456 (1.16839) + | > log_mle: 0.10146 (0.26779) + | > loss_dur: 0.93310 (0.90060) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.16758 (3.99645) + | > current_lr: 0.00000 + | > step_time: 2.61270 (2.87806) + | > loader_time: 0.09030 (0.04891) + + + --> STEP: 228/234 -- GLOBAL_STEP: 3270 + | > loss: 1.08129 (1.16575) + | > log_mle: 0.09744 (0.26393) + | > loss_dur: 0.98386 (0.90182) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.04561 (4.10828) + | > current_lr: 0.00000 + | > step_time: 0.25480 (2.85671) + | > loader_time: 0.00360 (0.04830) + + + --> STEP: 233/234 -- GLOBAL_STEP: 3275 + | > loss: 2.12617 (1.17064) + | > log_mle: 0.14734 (0.25949) + | > loss_dur: 1.97884 (0.91115) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.69633 (4.26209) + | > current_lr: 0.00000 + | > step_time: 0.19690 (2.80120) + | > loader_time: 0.00310 (0.04735) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.74353 (+0.61488) + | > avg_loss: 1.07583 (-0.07146) + | > avg_log_mle: 0.18901 (-0.02926) + | > avg_loss_dur: 0.88682 (-0.04220) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_3276.pth + + > EPOCH: 14/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 11:40:16)  + + --> STEP: 4/234 -- GLOBAL_STEP: 3280 + | > loss: 1.76086 (1.70983) + | > log_mle: 0.35568 (0.38539) + | > loss_dur: 1.40518 (1.32443) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.96261 (3.52329) + | > current_lr: 0.00000 + | > step_time: 7.00500 (4.50628) + | > loader_time: 0.00250 (0.07099) + + + --> STEP: 9/234 -- GLOBAL_STEP: 3285 + | > loss: 1.41943 (1.61637) + | > log_mle: 0.34630 (0.36999) + | > loss_dur: 1.07312 (1.24638) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.94031 (3.29829) + | > current_lr: 0.00000 + | > step_time: 4.70720 (5.77030) + | > loader_time: 0.19090 (0.08672) + + + --> STEP: 14/234 -- GLOBAL_STEP: 3290 + | > loss: 1.30915 (1.51836) + | > log_mle: 0.34917 (0.36567) + | > loss_dur: 0.95998 (1.15269) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.63651 (3.08244) + | > current_lr: 0.00000 + | > step_time: 0.86460 (4.16817) + | > loader_time: 0.00140 (0.06769) + + + --> STEP: 19/234 -- GLOBAL_STEP: 3295 + | > loss: 1.37632 (1.46783) + | > log_mle: 0.35884 (0.36280) + | > loss_dur: 1.01749 (1.10502) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.66675 (2.95135) + | > current_lr: 0.00000 + | > step_time: 0.87080 (3.31226) + | > loader_time: 0.00140 (0.05452) + + + --> STEP: 24/234 -- GLOBAL_STEP: 3300 + | > loss: 1.32883 (1.42513) + | > log_mle: 0.35047 (0.35959) + | > loss_dur: 0.97836 (1.06554) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.64297 (2.85002) + | > current_lr: 0.00000 + | > step_time: 1.09170 (2.85924) + | > loader_time: 0.00130 (0.04362) + + + --> STEP: 29/234 -- GLOBAL_STEP: 3305 + | > loss: 1.23663 (1.38874) + | > log_mle: 0.35247 (0.35704) + | > loss_dur: 0.88416 (1.03170) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.39660 (2.76634) + | > current_lr: 0.00000 + | > step_time: 1.04450 (2.58333) + | > loader_time: 0.00180 (0.03965) + + + --> STEP: 34/234 -- GLOBAL_STEP: 3310 + | > loss: 1.22795 (1.36165) + | > log_mle: 0.33069 (0.35337) + | > loss_dur: 0.89726 (1.00828) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.44851 (2.70443) + | > current_lr: 0.00000 + | > step_time: 1.10790 (2.41607) + | > loader_time: 0.00270 (0.03415) + + + --> STEP: 39/234 -- GLOBAL_STEP: 3315 + | > loss: 1.23125 (1.34131) + | > log_mle: 0.33695 (0.35067) + | > loss_dur: 0.89429 (0.99064) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.66121 (2.86063) + | > current_lr: 0.00000 + | > step_time: 1.20230 (2.33651) + | > loader_time: 0.00200 (0.03006) + + + --> STEP: 44/234 -- GLOBAL_STEP: 3320 + | > loss: 1.17996 (1.32453) + | > log_mle: 0.33088 (0.34919) + | > loss_dur: 0.84909 (0.97534) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.23581 (2.80148) + | > current_lr: 0.00000 + | > step_time: 2.01500 (2.24753) + | > loader_time: 0.00240 (0.02686) + + + --> STEP: 49/234 -- GLOBAL_STEP: 3325 + | > loss: 1.12753 (1.30635) + | > log_mle: 0.32129 (0.34735) + | > loss_dur: 0.80625 (0.95900) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.22778 (2.74761) + | > current_lr: 0.00000 + | > step_time: 1.50330 (2.16437) + | > loader_time: 0.00200 (0.02639) + + + --> STEP: 54/234 -- GLOBAL_STEP: 3330 + | > loss: 1.14865 (1.29229) + | > log_mle: 0.32235 (0.34600) + | > loss_dur: 0.82630 (0.94630) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.26636 (2.70087) + | > current_lr: 0.00000 + | > step_time: 1.08650 (2.08927) + | > loader_time: 0.00220 (0.02414) + + + --> STEP: 59/234 -- GLOBAL_STEP: 3335 + | > loss: 1.07323 (1.27899) + | > log_mle: 0.30524 (0.34425) + | > loss_dur: 0.76800 (0.93474) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.20681 (2.66141) + | > current_lr: 0.00000 + | > step_time: 3.53270 (2.10628) + | > loader_time: 0.00240 (0.02524) + + + --> STEP: 64/234 -- GLOBAL_STEP: 3340 + | > loss: 1.07916 (1.26778) + | > log_mle: 0.33125 (0.34224) + | > loss_dur: 0.74791 (0.92554) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.07575 (2.63784) + | > current_lr: 0.00000 + | > step_time: 1.90440 (2.09547) + | > loader_time: 0.08830 (0.02482) + + + --> STEP: 69/234 -- GLOBAL_STEP: 3345 + | > loss: 1.15735 (1.25732) + | > log_mle: 0.34087 (0.34111) + | > loss_dur: 0.81648 (0.91620) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.14764 (2.60738) + | > current_lr: 0.00000 + | > step_time: 1.90120 (2.14016) + | > loader_time: 0.00210 (0.02711) + + + --> STEP: 74/234 -- GLOBAL_STEP: 3350 + | > loss: 1.05515 (1.24720) + | > log_mle: 0.32072 (0.33921) + | > loss_dur: 0.73444 (0.90799) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.22830 (2.60516) + | > current_lr: 0.00000 + | > step_time: 1.14760 (2.11168) + | > loader_time: 0.05900 (0.02657) + + + --> STEP: 79/234 -- GLOBAL_STEP: 3355 + | > loss: 1.11082 (1.23815) + | > log_mle: 0.31178 (0.33772) + | > loss_dur: 0.79903 (0.90043) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.22928 (2.58210) + | > current_lr: 0.00000 + | > step_time: 2.09240 (2.10807) + | > loader_time: 0.00370 (0.02620) + + + --> STEP: 84/234 -- GLOBAL_STEP: 3360 + | > loss: 1.12550 (1.22987) + | > log_mle: 0.30476 (0.33624) + | > loss_dur: 0.82074 (0.89363) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.31896 (2.57128) + | > current_lr: 0.00000 + | > step_time: 1.87960 (2.10047) + | > loader_time: 0.00220 (0.02575) + + + --> STEP: 89/234 -- GLOBAL_STEP: 3365 + | > loss: 1.05113 (1.22169) + | > log_mle: 0.28635 (0.33397) + | > loss_dur: 0.76478 (0.88771) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.46246 (2.57259) + | > current_lr: 0.00000 + | > step_time: 3.99310 (2.14442) + | > loader_time: 0.10700 (0.02862) + + + --> STEP: 94/234 -- GLOBAL_STEP: 3370 + | > loss: 1.08440 (1.21285) + | > log_mle: 0.25765 (0.33097) + | > loss_dur: 0.82675 (0.88188) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.50621 (2.58086) + | > current_lr: 0.00000 + | > step_time: 0.98160 (2.11682) + | > loader_time: 0.00220 (0.02895) + + + --> STEP: 99/234 -- GLOBAL_STEP: 3375 + | > loss: 1.04012 (1.20593) + | > log_mle: 0.23671 (0.32805) + | > loss_dur: 0.80341 (0.87788) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.86914 (2.62984) + | > current_lr: 0.00000 + | > step_time: 1.71190 (2.10496) + | > loader_time: 0.08870 (0.03204) + + + --> STEP: 104/234 -- GLOBAL_STEP: 3380 + | > loss: 1.06721 (1.19966) + | > log_mle: 0.23367 (0.32500) + | > loss_dur: 0.83354 (0.87467) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.54904 (2.64663) + | > current_lr: 0.00000 + | > step_time: 1.70290 (2.07449) + | > loader_time: 0.01910 (0.03077) + + + --> STEP: 109/234 -- GLOBAL_STEP: 3385 + | > loss: 1.07280 (1.19397) + | > log_mle: 0.25521 (0.32267) + | > loss_dur: 0.81758 (0.87130) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.19310 (2.68350) + | > current_lr: 0.00000 + | > step_time: 1.01480 (2.06125) + | > loader_time: 0.00310 (0.02963) + + + --> STEP: 114/234 -- GLOBAL_STEP: 3390 + | > loss: 1.05733 (1.18837) + | > log_mle: 0.26921 (0.31989) + | > loss_dur: 0.78812 (0.86848) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.77434 (2.70666) + | > current_lr: 0.00000 + | > step_time: 4.02260 (2.11046) + | > loader_time: 0.00300 (0.02850) + + + --> STEP: 119/234 -- GLOBAL_STEP: 3395 + | > loss: 1.05572 (1.18340) + | > log_mle: 0.26822 (0.31764) + | > loss_dur: 0.78750 (0.86576) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.44158 (2.70880) + | > current_lr: 0.00000 + | > step_time: 1.15850 (2.08750) + | > loader_time: 0.00390 (0.02819) + + + --> STEP: 124/234 -- GLOBAL_STEP: 3400 + | > loss: 1.03209 (1.17772) + | > log_mle: 0.24707 (0.31590) + | > loss_dur: 0.78502 (0.86181) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00369 (2.70450) + | > current_lr: 0.00000 + | > step_time: 2.10750 (2.07495) + | > loader_time: 0.00330 (0.02802) + + + --> STEP: 129/234 -- GLOBAL_STEP: 3405 + | > loss: 1.04379 (1.17220) + | > log_mle: 0.25333 (0.31302) + | > loss_dur: 0.79045 (0.85918) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.67785 (2.70765) + | > current_lr: 0.00000 + | > step_time: 1.31060 (2.07156) + | > loader_time: 0.00260 (0.02949) + + + --> STEP: 134/234 -- GLOBAL_STEP: 3410 + | > loss: 1.04682 (1.16730) + | > log_mle: 0.22673 (0.30994) + | > loss_dur: 0.82009 (0.85736) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.42359 (2.72591) + | > current_lr: 0.00000 + | > step_time: 1.29960 (2.06873) + | > loader_time: 0.08850 (0.02914) + + + --> STEP: 139/234 -- GLOBAL_STEP: 3415 + | > loss: 1.01332 (1.16312) + | > log_mle: 0.18170 (0.30698) + | > loss_dur: 0.83162 (0.85614) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.15123 (2.78116) + | > current_lr: 0.00000 + | > step_time: 1.60550 (2.05402) + | > loader_time: 0.00340 (0.02879) + + + --> STEP: 144/234 -- GLOBAL_STEP: 3420 + | > loss: 1.00531 (1.15856) + | > log_mle: 0.19560 (0.30390) + | > loss_dur: 0.80971 (0.85465) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.69615 (2.80679) + | > current_lr: 0.00000 + | > step_time: 4.62250 (2.11081) + | > loader_time: 0.08550 (0.03100) + + + --> STEP: 149/234 -- GLOBAL_STEP: 3425 + | > loss: 0.99533 (1.15324) + | > log_mle: 0.16813 (0.30039) + | > loss_dur: 0.82720 (0.85285) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.89697 (2.83690) + | > current_lr: 0.00000 + | > step_time: 2.40550 (2.13213) + | > loader_time: 0.00370 (0.03181) + + + --> STEP: 154/234 -- GLOBAL_STEP: 3430 + | > loss: 1.01433 (1.14878) + | > log_mle: 0.19365 (0.29674) + | > loss_dur: 0.82068 (0.85204) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.40401 (2.90939) + | > current_lr: 0.00000 + | > step_time: 1.79790 (2.12584) + | > loader_time: 0.00360 (0.03327) + + + --> STEP: 159/234 -- GLOBAL_STEP: 3435 + | > loss: 1.06005 (1.14496) + | > log_mle: 0.18391 (0.29329) + | > loss_dur: 0.87614 (0.85167) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.24569 (2.98395) + | > current_lr: 0.00000 + | > step_time: 2.89510 (2.14378) + | > loader_time: 0.00350 (0.03338) + + + --> STEP: 164/234 -- GLOBAL_STEP: 3440 + | > loss: 1.00684 (1.14071) + | > log_mle: 0.18034 (0.28988) + | > loss_dur: 0.82651 (0.85083) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03751 (3.03229) + | > current_lr: 0.00000 + | > step_time: 3.40180 (2.15395) + | > loader_time: 0.19590 (0.03473) + + + --> STEP: 169/234 -- GLOBAL_STEP: 3445 + | > loss: 1.05315 (1.13747) + | > log_mle: 0.18459 (0.28642) + | > loss_dur: 0.86857 (0.85104) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.82136 (3.08794) + | > current_lr: 0.00000 + | > step_time: 1.89660 (2.22337) + | > loader_time: 0.00390 (0.03440) + + + --> STEP: 174/234 -- GLOBAL_STEP: 3450 + | > loss: 0.95509 (1.13356) + | > log_mle: 0.11884 (0.28226) + | > loss_dur: 0.83625 (0.85130) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.29376 (3.15900) + | > current_lr: 0.00000 + | > step_time: 5.70320 (2.23364) + | > loader_time: 0.09750 (0.03465) + + + --> STEP: 179/234 -- GLOBAL_STEP: 3455 + | > loss: 1.00596 (1.12993) + | > log_mle: 0.13473 (0.27852) + | > loss_dur: 0.87123 (0.85141) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.24072 (3.22810) + | > current_lr: 0.00000 + | > step_time: 4.80460 (2.35267) + | > loader_time: 0.09990 (0.03597) + + + --> STEP: 184/234 -- GLOBAL_STEP: 3460 + | > loss: 1.00509 (1.12684) + | > log_mle: 0.14867 (0.27509) + | > loss_dur: 0.85642 (0.85175) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.61708 (3.31658) + | > current_lr: 0.00000 + | > step_time: 7.50780 (2.43361) + | > loader_time: 0.19340 (0.03672) + + + --> STEP: 189/234 -- GLOBAL_STEP: 3465 + | > loss: 1.00742 (1.12354) + | > log_mle: 0.15559 (0.27160) + | > loss_dur: 0.85183 (0.85194) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.67890 (3.41204) + | > current_lr: 0.00000 + | > step_time: 5.30720 (2.48312) + | > loader_time: 0.08940 (0.03821) + + + --> STEP: 194/234 -- GLOBAL_STEP: 3470 + | > loss: 0.99767 (1.12009) + | > log_mle: 0.12608 (0.26806) + | > loss_dur: 0.87159 (0.85203) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.51102 (3.45969) + | > current_lr: 0.00000 + | > step_time: 5.49590 (2.53252) + | > loader_time: 0.09590 (0.03929) + + + --> STEP: 199/234 -- GLOBAL_STEP: 3475 + | > loss: 0.95856 (1.11689) + | > log_mle: 0.11941 (0.26477) + | > loss_dur: 0.83915 (0.85212) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.73326 (3.50930) + | > current_lr: 0.00000 + | > step_time: 2.89500 (2.64004) + | > loader_time: 0.09480 (0.04481) + + + --> STEP: 204/234 -- GLOBAL_STEP: 3480 + | > loss: 0.95512 (1.11356) + | > log_mle: 0.10150 (0.26145) + | > loss_dur: 0.85362 (0.85211) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.23776 (3.53871) + | > current_lr: 0.00000 + | > step_time: 3.29240 (2.67044) + | > loader_time: 0.10940 (0.04611) + + + --> STEP: 209/234 -- GLOBAL_STEP: 3485 + | > loss: 0.98804 (1.11041) + | > log_mle: 0.13098 (0.25795) + | > loss_dur: 0.85706 (0.85246) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.32185 (3.60227) + | > current_lr: 0.00000 + | > step_time: 2.80020 (2.70224) + | > loader_time: 0.00450 (0.04872) + + + --> STEP: 214/234 -- GLOBAL_STEP: 3490 + | > loss: 0.97668 (1.10729) + | > log_mle: 0.10680 (0.25388) + | > loss_dur: 0.86988 (0.85341) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.75844 (3.70516) + | > current_lr: 0.00000 + | > step_time: 6.20090 (2.72498) + | > loader_time: 0.08290 (0.04929) + + + --> STEP: 219/234 -- GLOBAL_STEP: 3495 + | > loss: 0.94504 (1.10413) + | > log_mle: 0.03769 (0.24997) + | > loss_dur: 0.90735 (0.85416) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.99126 (3.80866) + | > current_lr: 0.00000 + | > step_time: 6.80550 (2.78889) + | > loader_time: 0.00290 (0.07140) + + + --> STEP: 224/234 -- GLOBAL_STEP: 3500 + | > loss: 1.00057 (1.10159) + | > log_mle: 0.07238 (0.24631) + | > loss_dur: 0.92819 (0.85528) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.56089 (3.89319) + | > current_lr: 0.00000 + | > step_time: 1.40610 (2.81022) + | > loader_time: 10.10170 (0.11618) + + + --> STEP: 229/234 -- GLOBAL_STEP: 3505 + | > loss: 1.07377 (1.09923) + | > log_mle: 0.06104 (0.24231) + | > loss_dur: 1.01274 (0.85692) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.34086 (3.98253) + | > current_lr: 0.00000 + | > step_time: 0.25440 (2.76929) + | > loader_time: 0.00270 (0.11372) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.35624 (-0.38729) + | > avg_loss: 1.01904 (-0.05678) + | > avg_log_mle: 0.16852 (-0.02049) + | > avg_loss_dur: 0.85053 (-0.03629) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_3510.pth + + > EPOCH: 15/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 11:52:22)  + + --> STEP: 0/234 -- GLOBAL_STEP: 3510 + | > loss: 1.69435 (1.69435) + | > log_mle: 0.37349 (0.37349) + | > loss_dur: 1.32086 (1.32086) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.13844 (4.13844) + | > current_lr: 0.00000 + | > step_time: 9.91190 (9.91189) + | > loader_time: 8.88510 (8.88512) + + + --> STEP: 5/234 -- GLOBAL_STEP: 3515 + | > loss: 1.57504 (1.62778) + | > log_mle: 0.35625 (0.36583) + | > loss_dur: 1.21879 (1.26195) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.57544 (3.51286) + | > current_lr: 0.00000 + | > step_time: 2.18980 (4.63564) + | > loader_time: 0.00200 (2.28032) + + + --> STEP: 10/234 -- GLOBAL_STEP: 3520 + | > loss: 1.28532 (1.51864) + | > log_mle: 0.33769 (0.35113) + | > loss_dur: 0.94763 (1.16751) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.66129 (3.17483) + | > current_lr: 0.00000 + | > step_time: 4.39820 (5.51021) + | > loader_time: 0.00310 (1.18687) + + + --> STEP: 15/234 -- GLOBAL_STEP: 3525 + | > loss: 1.26718 (1.43678) + | > log_mle: 0.33521 (0.34754) + | > loss_dur: 0.93198 (1.08924) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.50980 (2.94841) + | > current_lr: 0.00000 + | > step_time: 4.00270 (5.09333) + | > loader_time: 0.00130 (0.79848) + + + --> STEP: 20/234 -- GLOBAL_STEP: 3530 + | > loss: 1.20780 (1.38886) + | > log_mle: 0.33491 (0.34504) + | > loss_dur: 0.87289 (1.04383) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.26675 (2.81227) + | > current_lr: 0.00000 + | > step_time: 3.90750 (4.81731) + | > loader_time: 0.19300 (0.62380) + + + --> STEP: 25/234 -- GLOBAL_STEP: 3535 + | > loss: 1.15257 (1.34770) + | > log_mle: 0.33346 (0.34204) + | > loss_dur: 0.81911 (1.00565) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.15401 (2.70597) + | > current_lr: 0.00000 + | > step_time: 3.09870 (4.65436) + | > loader_time: 0.09860 (0.50720) + + + --> STEP: 30/234 -- GLOBAL_STEP: 3540 + | > loss: 1.09548 (1.31309) + | > log_mle: 0.31306 (0.33897) + | > loss_dur: 0.78242 (0.97412) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.10512 (2.61973) + | > current_lr: 0.00000 + | > step_time: 8.70100 (4.74584) + | > loader_time: 0.29350 (0.43916) + + + --> STEP: 35/234 -- GLOBAL_STEP: 3545 + | > loss: 1.12987 (1.28924) + | > log_mle: 0.31651 (0.33561) + | > loss_dur: 0.81337 (0.95363) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.27722 (2.73420) + | > current_lr: 0.00000 + | > step_time: 5.13050 (4.57152) + | > loader_time: 0.07500 (0.37922) + + + --> STEP: 40/234 -- GLOBAL_STEP: 3550 + | > loss: 1.14168 (1.26981) + | > log_mle: 0.33175 (0.33340) + | > loss_dur: 0.80993 (0.93641) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.17392 (2.67508) + | > current_lr: 0.00000 + | > step_time: 1.29230 (4.49532) + | > loader_time: 0.00300 (0.33690) + + + --> STEP: 45/234 -- GLOBAL_STEP: 3555 + | > loss: 1.14166 (1.25421) + | > log_mle: 0.31788 (0.33160) + | > loss_dur: 0.82378 (0.92260) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.31049 (2.62330) + | > current_lr: 0.00000 + | > step_time: 4.00340 (4.25648) + | > loader_time: 0.10980 (0.30559) + + + --> STEP: 50/234 -- GLOBAL_STEP: 3560 + | > loss: 1.09730 (1.23664) + | > log_mle: 0.31583 (0.32980) + | > loss_dur: 0.78147 (0.90684) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.01763 (2.56738) + | > current_lr: 0.00000 + | > step_time: 1.41910 (4.05494) + | > loader_time: 0.09320 (0.28259) + + + --> STEP: 55/234 -- GLOBAL_STEP: 3565 + | > loss: 1.07669 (1.22315) + | > log_mle: 0.29701 (0.32816) + | > loss_dur: 0.77968 (0.89499) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.01701 (2.52541) + | > current_lr: 0.00000 + | > step_time: 4.48270 (3.86209) + | > loader_time: 0.11150 (0.25908) + + + --> STEP: 60/234 -- GLOBAL_STEP: 3570 + | > loss: 1.04694 (1.21049) + | > log_mle: 0.29069 (0.32639) + | > loss_dur: 0.75625 (0.88410) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.25139 (2.49355) + | > current_lr: 0.00000 + | > step_time: 2.12410 (3.68780) + | > loader_time: 0.00680 (0.23777) + + + --> STEP: 65/234 -- GLOBAL_STEP: 3575 + | > loss: 1.05000 (1.20016) + | > log_mle: 0.31025 (0.32475) + | > loss_dur: 0.73975 (0.87541) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.03126 (2.48156) + | > current_lr: 0.00000 + | > step_time: 1.80240 (3.51736) + | > loader_time: 0.00250 (0.22127) + + + --> STEP: 70/234 -- GLOBAL_STEP: 3580 + | > loss: 1.05462 (1.19031) + | > log_mle: 0.29970 (0.32352) + | > loss_dur: 0.75492 (0.86679) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.59270 (2.46302) + | > current_lr: 0.00000 + | > step_time: 3.20410 (3.51794) + | > loader_time: 0.07540 (0.20923) + + + --> STEP: 75/234 -- GLOBAL_STEP: 3585 + | > loss: 1.04440 (1.18049) + | > log_mle: 0.29966 (0.32165) + | > loss_dur: 0.74474 (0.85884) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.41317 (2.47641) + | > current_lr: 0.00000 + | > step_time: 2.32040 (3.46908) + | > loader_time: 0.09100 (0.19799) + + + --> STEP: 80/234 -- GLOBAL_STEP: 3590 + | > loss: 1.04211 (1.17194) + | > log_mle: 0.30844 (0.32031) + | > loss_dur: 0.73366 (0.85163) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.98370 (2.45701) + | > current_lr: 0.00000 + | > step_time: 1.39830 (3.35943) + | > loader_time: 0.00270 (0.18691) + + + --> STEP: 85/234 -- GLOBAL_STEP: 3595 + | > loss: 1.05360 (1.16425) + | > log_mle: 0.29793 (0.31873) + | > loss_dur: 0.75567 (0.84552) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.17475 (2.45109) + | > current_lr: 0.00000 + | > step_time: 6.29130 (3.35439) + | > loader_time: 0.19140 (0.17931) + + + --> STEP: 90/234 -- GLOBAL_STEP: 3600 + | > loss: 0.97540 (1.15559) + | > log_mle: 0.27006 (0.31618) + | > loss_dur: 0.70535 (0.83941) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.74344 (2.47181) + | > current_lr: 0.00000 + | > step_time: 1.60510 (3.28709) + | > loader_time: 0.09760 (0.17256) + + + --> STEP: 95/234 -- GLOBAL_STEP: 3605 + | > loss: 1.00534 (1.14747) + | > log_mle: 0.20946 (0.31255) + | > loss_dur: 0.79588 (0.83492) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.55857 (2.49860) + | > current_lr: 0.00000 + | > step_time: 4.41290 (3.28375) + | > loader_time: 0.18250 (0.16828) + + + --> STEP: 100/234 -- GLOBAL_STEP: 3610 + | > loss: 1.03080 (1.14114) + | > log_mle: 0.25971 (0.31015) + | > loss_dur: 0.77108 (0.83099) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.28524 (2.52840) + | > current_lr: 0.00000 + | > step_time: 2.20070 (3.23588) + | > loader_time: 0.18130 (0.16627) + + + --> STEP: 105/234 -- GLOBAL_STEP: 3615 + | > loss: 1.01572 (1.13481) + | > log_mle: 0.28495 (0.30731) + | > loss_dur: 0.73077 (0.82749) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.03946 (2.55747) + | > current_lr: 0.00000 + | > step_time: 2.00180 (3.19910) + | > loader_time: 0.00400 (0.15929) + + + --> STEP: 110/234 -- GLOBAL_STEP: 3620 + | > loss: 1.02782 (1.12949) + | > log_mle: 0.26106 (0.30477) + | > loss_dur: 0.76675 (0.82472) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.07035 (2.64290) + | > current_lr: 0.00000 + | > step_time: 2.11050 (3.22516) + | > loader_time: 0.08120 (0.15638) + + + --> STEP: 115/234 -- GLOBAL_STEP: 3625 + | > loss: 1.05229 (1.12423) + | > log_mle: 0.24981 (0.30185) + | > loss_dur: 0.80247 (0.82239) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.83887 (2.70722) + | > current_lr: 0.00000 + | > step_time: 2.80470 (3.18512) + | > loader_time: 0.07590 (0.15111) + + + --> STEP: 120/234 -- GLOBAL_STEP: 3630 + | > loss: 0.97217 (1.11873) + | > log_mle: 0.21282 (0.29929) + | > loss_dur: 0.75935 (0.81944) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.43665 (2.80937) + | > current_lr: 0.00000 + | > step_time: 4.10150 (3.27258) + | > loader_time: 0.00650 (0.14894) + + + --> STEP: 125/234 -- GLOBAL_STEP: 3635 + | > loss: 0.94962 (1.11319) + | > log_mle: 0.22443 (0.29767) + | > loss_dur: 0.72518 (0.81552) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.39793 (2.85119) + | > current_lr: 0.00000 + | > step_time: 2.70040 (3.29272) + | > loader_time: 0.06530 (0.14526) + + + --> STEP: 130/234 -- GLOBAL_STEP: 3640 + | > loss: 0.97610 (1.10804) + | > log_mle: 0.21366 (0.29467) + | > loss_dur: 0.76244 (0.81337) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.68829 (2.93718) + | > current_lr: 0.00000 + | > step_time: 0.92280 (3.32157) + | > loader_time: 0.07550 (0.14315) + + + --> STEP: 135/234 -- GLOBAL_STEP: 3645 + | > loss: 0.97216 (1.10317) + | > log_mle: 0.25879 (0.29188) + | > loss_dur: 0.71337 (0.81129) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.72707 (3.00574) + | > current_lr: 0.00000 + | > step_time: 4.00090 (3.31936) + | > loader_time: 0.09940 (0.14119) + + + --> STEP: 140/234 -- GLOBAL_STEP: 3650 + | > loss: 0.98426 (1.09904) + | > log_mle: 0.24286 (0.28875) + | > loss_dur: 0.74140 (0.81029) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.21502 (3.08637) + | > current_lr: 0.00000 + | > step_time: 3.40790 (3.29801) + | > loader_time: 0.09150 (0.13763) + + + --> STEP: 145/234 -- GLOBAL_STEP: 3655 + | > loss: 0.96724 (1.09439) + | > log_mle: 0.16919 (0.28508) + | > loss_dur: 0.79805 (0.80931) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.97271 (3.16513) + | > current_lr: 0.00000 + | > step_time: 2.71430 (3.31831) + | > loader_time: 0.08340 (0.13509) + + + --> STEP: 150/234 -- GLOBAL_STEP: 3660 + | > loss: 0.99540 (1.08946) + | > log_mle: 0.18408 (0.28164) + | > loss_dur: 0.81133 (0.80782) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.70050 (3.31529) + | > current_lr: 0.00000 + | > step_time: 1.90250 (3.31297) + | > loader_time: 0.00300 (0.13271) + + + --> STEP: 155/234 -- GLOBAL_STEP: 3665 + | > loss: 0.96501 (1.08483) + | > log_mle: 0.13769 (0.27763) + | > loss_dur: 0.82732 (0.80721) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.85376 (3.41903) + | > current_lr: 0.00000 + | > step_time: 2.99030 (3.39249) + | > loader_time: 0.10580 (0.13240) + + + --> STEP: 160/234 -- GLOBAL_STEP: 3670 + | > loss: 0.91481 (1.08074) + | > log_mle: 0.13642 (0.27412) + | > loss_dur: 0.77839 (0.80662) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.77410 (3.49856) + | > current_lr: 0.00000 + | > step_time: 1.11960 (3.39308) + | > loader_time: 0.06810 (0.13220) + + + --> STEP: 165/234 -- GLOBAL_STEP: 3675 + | > loss: 0.94516 (1.07676) + | > log_mle: 0.14224 (0.27071) + | > loss_dur: 0.80292 (0.80605) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.71606 (3.55907) + | > current_lr: 0.00000 + | > step_time: 3.40230 (3.38950) + | > loader_time: 0.00650 (0.13070) + + + --> STEP: 170/234 -- GLOBAL_STEP: 3680 + | > loss: 0.96600 (1.07365) + | > log_mle: 0.12152 (0.26711) + | > loss_dur: 0.84449 (0.80653) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.19738 (3.62380) + | > current_lr: 0.00000 + | > step_time: 4.09100 (3.43450) + | > loader_time: 0.00530 (0.12901) + + + --> STEP: 175/234 -- GLOBAL_STEP: 3685 + | > loss: 0.93374 (1.06955) + | > log_mle: 0.13305 (0.26297) + | > loss_dur: 0.80069 (0.80657) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.67716 (3.71004) + | > current_lr: 0.00000 + | > step_time: 6.50430 (3.48725) + | > loader_time: 0.07740 (0.12674) + + + --> STEP: 180/234 -- GLOBAL_STEP: 3690 + | > loss: 0.94006 (1.06590) + | > log_mle: 0.12363 (0.25914) + | > loss_dur: 0.81643 (0.80676) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.30470 (3.75424) + | > current_lr: 0.00000 + | > step_time: 1.49850 (3.60418) + | > loader_time: 0.00410 (0.12387) + + + --> STEP: 185/234 -- GLOBAL_STEP: 3695 + | > loss: 0.94925 (1.06283) + | > log_mle: 0.11278 (0.25559) + | > loss_dur: 0.83647 (0.80724) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.60005 (3.82422) + | > current_lr: 0.00000 + | > step_time: 3.71560 (3.59991) + | > loader_time: 0.10130 (0.12208) + + + --> STEP: 190/234 -- GLOBAL_STEP: 3700 + | > loss: 0.91005 (1.05925) + | > log_mle: 0.11942 (0.25210) + | > loss_dur: 0.79063 (0.80715) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.75932 (3.89924) + | > current_lr: 0.00000 + | > step_time: 4.20810 (3.60451) + | > loader_time: 0.00520 (0.12002) + + + --> STEP: 195/234 -- GLOBAL_STEP: 3705 + | > loss: 0.96547 (1.05603) + | > log_mle: 0.11581 (0.24850) + | > loss_dur: 0.84967 (0.80753) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.66580 (3.99655) + | > current_lr: 0.00000 + | > step_time: 3.50010 (3.66799) + | > loader_time: 0.19440 (0.12004) + + + --> STEP: 200/234 -- GLOBAL_STEP: 3710 + | > loss: 0.93650 (1.05272) + | > log_mle: 0.11140 (0.24516) + | > loss_dur: 0.82510 (0.80756) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.37882 (4.07356) + | > current_lr: 0.00000 + | > step_time: 3.50980 (3.73193) + | > loader_time: 0.08720 (0.12050) + + + --> STEP: 205/234 -- GLOBAL_STEP: 3715 + | > loss: 0.94619 (1.04945) + | > log_mle: 0.11341 (0.24181) + | > loss_dur: 0.83278 (0.80764) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.38823 (4.10148) + | > current_lr: 0.00000 + | > step_time: 9.89710 (3.79128) + | > loader_time: 0.20300 (0.11955) + + + --> STEP: 210/234 -- GLOBAL_STEP: 3720 + | > loss: 0.89789 (1.04604) + | > log_mle: 0.05639 (0.23799) + | > loss_dur: 0.84151 (0.80804) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.44681 (4.17399) + | > current_lr: 0.00000 + | > step_time: 8.19460 (3.82088) + | > loader_time: 0.09300 (0.11910) + + + --> STEP: 215/234 -- GLOBAL_STEP: 3725 + | > loss: 0.91109 (1.04285) + | > log_mle: 0.09566 (0.23406) + | > loss_dur: 0.81543 (0.80879) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.81673 (4.22615) + | > current_lr: 0.00000 + | > step_time: 10.28460 (3.91522) + | > loader_time: 0.20770 (0.11955) + + + --> STEP: 220/234 -- GLOBAL_STEP: 3730 + | > loss: 0.90661 (1.03958) + | > log_mle: 0.06045 (0.22994) + | > loss_dur: 0.84616 (0.80965) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.82975 (4.32069) + | > current_lr: 0.00000 + | > step_time: 2.50680 (3.90852) + | > loader_time: 0.08440 (0.11946) + + + --> STEP: 225/234 -- GLOBAL_STEP: 3735 + | > loss: 0.86731 (1.03674) + | > log_mle: 0.01693 (0.22605) + | > loss_dur: 0.85038 (0.81070) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.08818 (4.49637) + | > current_lr: 0.00000 + | > step_time: 0.23960 (3.83450) + | > loader_time: 0.00380 (0.11688) + + + --> STEP: 230/234 -- GLOBAL_STEP: 3740 + | > loss: 0.96963 (1.03466) + | > log_mle: -0.02482 (0.22182) + | > loss_dur: 0.99445 (0.81283) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.19831 (4.63566) + | > current_lr: 0.00000 + | > step_time: 0.25210 (3.75643) + | > loader_time: 0.00490 (0.11442) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40210 (+0.04585) + | > avg_loss: 0.94989 (-0.06915) + | > avg_log_mle: 0.14429 (-0.02423) + | > avg_loss_dur: 0.80561 (-0.04492) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_3744.pth + + > EPOCH: 16/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 12:08:19)  + + --> STEP: 1/234 -- GLOBAL_STEP: 3745 + | > loss: 1.53190 (1.53190) + | > log_mle: 0.36426 (0.36426) + | > loss_dur: 1.16764 (1.16764) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.22038 (3.22038) + | > current_lr: 0.00000 + | > step_time: 3.61130 (3.61132) + | > loader_time: 0.00230 (0.00233) + + + --> STEP: 6/234 -- GLOBAL_STEP: 3750 + | > loss: 1.40473 (1.50044) + | > log_mle: 0.33935 (0.34703) + | > loss_dur: 1.06538 (1.15341) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.89653 (3.18656) + | > current_lr: 0.00000 + | > step_time: 6.10600 (7.28781) + | > loader_time: 0.09630 (0.08297) + + + --> STEP: 11/234 -- GLOBAL_STEP: 3755 + | > loss: 1.19988 (1.40157) + | > log_mle: 0.33301 (0.33389) + | > loss_dur: 0.86688 (1.06768) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.32683 (2.93509) + | > current_lr: 0.00000 + | > step_time: 1.21320 (6.03966) + | > loader_time: 0.08370 (0.06967) + + + --> STEP: 16/234 -- GLOBAL_STEP: 3760 + | > loss: 1.14190 (1.33625) + | > log_mle: 0.30876 (0.32921) + | > loss_dur: 0.83314 (1.00704) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.13867 (2.73357) + | > current_lr: 0.00000 + | > step_time: 3.40400 (5.66543) + | > loader_time: 0.08580 (0.08372) + + + --> STEP: 21/234 -- GLOBAL_STEP: 3765 + | > loss: 1.12121 (1.29674) + | > log_mle: 0.32447 (0.32807) + | > loss_dur: 0.79674 (0.96868) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.09939 (2.62376) + | > current_lr: 0.00000 + | > step_time: 1.30800 (5.13653) + | > loader_time: 0.00290 (0.06915) + + + --> STEP: 26/234 -- GLOBAL_STEP: 3770 + | > loss: 1.09694 (1.26052) + | > log_mle: 0.31059 (0.32485) + | > loss_dur: 0.78635 (0.93568) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.00408 (2.53031) + | > current_lr: 0.00000 + | > step_time: 1.30030 (4.46874) + | > loader_time: 0.01370 (0.05700) + + + --> STEP: 31/234 -- GLOBAL_STEP: 3775 + | > loss: 1.09860 (1.23049) + | > log_mle: 0.29907 (0.32169) + | > loss_dur: 0.79953 (0.90880) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.14985 (2.45378) + | > current_lr: 0.00000 + | > step_time: 2.40980 (4.00665) + | > loader_time: 0.00180 (0.05041) + + + --> STEP: 36/234 -- GLOBAL_STEP: 3780 + | > loss: 1.09636 (1.20977) + | > log_mle: 0.29902 (0.31860) + | > loss_dur: 0.79734 (0.89117) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.49724 (2.60188) + | > current_lr: 0.00000 + | > step_time: 3.70930 (3.90358) + | > loader_time: 0.00240 (0.04641) + + + --> STEP: 41/234 -- GLOBAL_STEP: 3785 + | > loss: 1.03337 (1.19088) + | > log_mle: 0.29002 (0.31638) + | > loss_dur: 0.74334 (0.87450) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.05567 (2.54104) + | > current_lr: 0.00000 + | > step_time: 2.51620 (3.64600) + | > loader_time: 0.08420 (0.04536) + + + --> STEP: 46/234 -- GLOBAL_STEP: 3790 + | > loss: 1.02677 (1.17720) + | > log_mle: 0.29879 (0.31497) + | > loss_dur: 0.72799 (0.86224) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.92430 (2.48782) + | > current_lr: 0.00000 + | > step_time: 1.34900 (3.55208) + | > loader_time: 0.00210 (0.04665) + + + --> STEP: 51/234 -- GLOBAL_STEP: 3795 + | > loss: 1.01800 (1.16094) + | > log_mle: 0.31057 (0.31356) + | > loss_dur: 0.70743 (0.84738) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.04452 (2.43805) + | > current_lr: 0.00000 + | > step_time: 3.78650 (3.43933) + | > loader_time: 0.10970 (0.04443) + + + --> STEP: 56/234 -- GLOBAL_STEP: 3800 + | > loss: 1.06158 (1.14923) + | > log_mle: 0.29494 (0.31174) + | > loss_dur: 0.76664 (0.83749) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.04774 (2.39775) + | > current_lr: 0.00000 + | > step_time: 2.10790 (3.40378) + | > loader_time: 0.07760 (0.04202) + + + --> STEP: 61/234 -- GLOBAL_STEP: 3805 + | > loss: 1.02123 (1.13722) + | > log_mle: 0.30166 (0.31017) + | > loss_dur: 0.71957 (0.82705) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.93393 (2.36484) + | > current_lr: 0.00000 + | > step_time: 3.50450 (3.34125) + | > loader_time: 0.10420 (0.04228) + + + --> STEP: 66/234 -- GLOBAL_STEP: 3810 + | > loss: 1.00053 (1.12757) + | > log_mle: 0.30298 (0.30862) + | > loss_dur: 0.69755 (0.81895) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.81578 (2.34792) + | > current_lr: 0.00000 + | > step_time: 2.40080 (3.29450) + | > loader_time: 0.17950 (0.04309) + + + --> STEP: 71/234 -- GLOBAL_STEP: 3815 + | > loss: 1.03062 (1.11930) + | > log_mle: 0.26502 (0.30692) + | > loss_dur: 0.76560 (0.81238) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.24243 (2.36449) + | > current_lr: 0.00000 + | > step_time: 1.39480 (3.17239) + | > loader_time: 0.00290 (0.04137) + + + --> STEP: 76/234 -- GLOBAL_STEP: 3820 + | > loss: 1.00031 (1.10983) + | > log_mle: 0.28533 (0.30537) + | > loss_dur: 0.71498 (0.80445) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.94356 (2.34869) + | > current_lr: 0.00000 + | > step_time: 1.10540 (3.10525) + | > loader_time: 0.08320 (0.03989) + + + --> STEP: 81/234 -- GLOBAL_STEP: 3825 + | > loss: 0.97113 (1.10179) + | > log_mle: 0.26659 (0.30385) + | > loss_dur: 0.70455 (0.79794) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.57171 (2.33265) + | > current_lr: 0.00000 + | > step_time: 4.10390 (3.09924) + | > loader_time: 0.00650 (0.04075) + + + --> STEP: 86/234 -- GLOBAL_STEP: 3830 + | > loss: 0.99800 (1.09512) + | > log_mle: 0.26692 (0.30232) + | > loss_dur: 0.73108 (0.79280) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.59297 (2.32555) + | > current_lr: 0.00000 + | > step_time: 1.90340 (3.03943) + | > loader_time: 0.00240 (0.03855) + + + --> STEP: 91/234 -- GLOBAL_STEP: 3835 + | > loss: 0.96559 (1.08672) + | > log_mle: 0.26795 (0.29977) + | > loss_dur: 0.69764 (0.78695) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.33886 (2.34881) + | > current_lr: 0.00000 + | > step_time: 1.21040 (2.97567) + | > loader_time: 0.00350 (0.03844) + + + --> STEP: 96/234 -- GLOBAL_STEP: 3840 + | > loss: 0.98109 (1.07914) + | > log_mle: 0.27062 (0.29616) + | > loss_dur: 0.71047 (0.78298) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.28310 (2.39197) + | > current_lr: 0.00000 + | > step_time: 2.79840 (2.98117) + | > loader_time: 0.00610 (0.03846) + + + --> STEP: 101/234 -- GLOBAL_STEP: 3845 + | > loss: 0.93735 (1.07286) + | > log_mle: 0.22408 (0.29332) + | > loss_dur: 0.71328 (0.77955) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.74063 (2.45530) + | > current_lr: 0.00000 + | > step_time: 1.49650 (2.98504) + | > loader_time: 0.00300 (0.03668) + + + --> STEP: 106/234 -- GLOBAL_STEP: 3850 + | > loss: 1.00240 (1.06778) + | > log_mle: 0.22904 (0.29056) + | > loss_dur: 0.77336 (0.77722) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.99319 (2.55652) + | > current_lr: 0.00000 + | > step_time: 1.49270 (2.94713) + | > loader_time: 0.00230 (0.03671) + + + --> STEP: 111/234 -- GLOBAL_STEP: 3855 + | > loss: 0.95794 (1.06243) + | > log_mle: 0.19587 (0.28772) + | > loss_dur: 0.76208 (0.77472) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.36880 (2.62052) + | > current_lr: 0.00000 + | > step_time: 1.00230 (2.92909) + | > loader_time: 0.00480 (0.03769) + + + --> STEP: 116/234 -- GLOBAL_STEP: 3860 + | > loss: 0.95014 (1.05734) + | > log_mle: 0.21961 (0.28502) + | > loss_dur: 0.73053 (0.77232) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.11224 (2.69816) + | > current_lr: 0.00000 + | > step_time: 5.70780 (2.90369) + | > loader_time: 0.00290 (0.03675) + + + --> STEP: 121/234 -- GLOBAL_STEP: 3865 + | > loss: 0.95371 (1.05224) + | > log_mle: 0.28631 (0.28300) + | > loss_dur: 0.66740 (0.76925) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.98868 (2.73235) + | > current_lr: 0.00000 + | > step_time: 5.17980 (2.91242) + | > loader_time: 0.09500 (0.03674) + + + --> STEP: 126/234 -- GLOBAL_STEP: 3870 + | > loss: 0.92689 (1.04677) + | > log_mle: 0.17860 (0.28050) + | > loss_dur: 0.74828 (0.76627) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.90586 (2.80096) + | > current_lr: 0.00000 + | > step_time: 2.68590 (2.88411) + | > loader_time: 0.00240 (0.03606) + + + --> STEP: 131/234 -- GLOBAL_STEP: 3875 + | > loss: 0.88675 (1.04154) + | > log_mle: 0.15813 (0.27733) + | > loss_dur: 0.72862 (0.76421) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.38347 (2.91162) + | > current_lr: 0.00000 + | > step_time: 2.69860 (2.87419) + | > loader_time: 0.08610 (0.03675) + + + --> STEP: 136/234 -- GLOBAL_STEP: 3880 + | > loss: 0.90076 (1.03704) + | > log_mle: 0.11942 (0.27428) + | > loss_dur: 0.78135 (0.76276) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.28636 (2.99436) + | > current_lr: 0.00000 + | > step_time: 2.80460 (2.91221) + | > loader_time: 0.00360 (0.03887) + + + --> STEP: 141/234 -- GLOBAL_STEP: 3885 + | > loss: 0.90848 (1.03320) + | > log_mle: 0.18288 (0.27162) + | > loss_dur: 0.72560 (0.76158) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.55583 (3.10564) + | > current_lr: 0.00000 + | > step_time: 3.20140 (2.99752) + | > loader_time: 0.00340 (0.04390) + + + --> STEP: 146/234 -- GLOBAL_STEP: 3890 + | > loss: 0.89761 (1.02877) + | > log_mle: 0.14573 (0.26767) + | > loss_dur: 0.75188 (0.76110) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.94184 (3.26223) + | > current_lr: 0.00000 + | > step_time: 6.60400 (3.06767) + | > loader_time: 0.10070 (0.04519) + + + --> STEP: 151/234 -- GLOBAL_STEP: 3895 + | > loss: 0.87093 (1.02389) + | > log_mle: 0.16820 (0.26437) + | > loss_dur: 0.70273 (0.75952) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.49503 (3.34404) + | > current_lr: 0.00000 + | > step_time: 3.49950 (3.07130) + | > loader_time: 0.00300 (0.04458) + + + --> STEP: 156/234 -- GLOBAL_STEP: 3900 + | > loss: 0.86379 (1.01933) + | > log_mle: 0.14367 (0.26017) + | > loss_dur: 0.72012 (0.75916) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.98403 (3.42413) + | > current_lr: 0.00000 + | > step_time: 2.79230 (3.09648) + | > loader_time: 0.00180 (0.04508) + + + --> STEP: 161/234 -- GLOBAL_STEP: 3905 + | > loss: 0.90673 (1.01556) + | > log_mle: 0.13192 (0.25655) + | > loss_dur: 0.77481 (0.75901) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.10107 (3.54139) + | > current_lr: 0.00000 + | > step_time: 4.58980 (3.12836) + | > loader_time: 0.09920 (0.04545) + + + --> STEP: 166/234 -- GLOBAL_STEP: 3910 + | > loss: 0.89572 (1.01155) + | > log_mle: 0.16769 (0.25333) + | > loss_dur: 0.72802 (0.75822) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.97627 (3.60291) + | > current_lr: 0.00000 + | > step_time: 2.30900 (3.15223) + | > loader_time: 0.08770 (0.04651) + + + --> STEP: 171/234 -- GLOBAL_STEP: 3915 + | > loss: 0.84865 (1.00828) + | > log_mle: 0.08963 (0.24924) + | > loss_dur: 0.75902 (0.75904) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.24553 (3.73359) + | > current_lr: 0.00000 + | > step_time: 1.21410 (3.12433) + | > loader_time: 0.00420 (0.04679) + + + --> STEP: 176/234 -- GLOBAL_STEP: 3920 + | > loss: 0.86443 (1.00431) + | > log_mle: 0.11502 (0.24520) + | > loss_dur: 0.74942 (0.75911) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.34107 (3.82589) + | > current_lr: 0.00000 + | > step_time: 3.01420 (3.11630) + | > loader_time: 0.10040 (0.04819) + + + --> STEP: 181/234 -- GLOBAL_STEP: 3925 + | > loss: 0.90442 (1.00095) + | > log_mle: 0.15608 (0.24156) + | > loss_dur: 0.74834 (0.75939) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.79171 (3.90064) + | > current_lr: 0.00000 + | > step_time: 2.39910 (3.13970) + | > loader_time: 0.00500 (0.04847) + + + --> STEP: 186/234 -- GLOBAL_STEP: 3930 + | > loss: 0.89032 (0.99784) + | > log_mle: 0.12887 (0.23782) + | > loss_dur: 0.76145 (0.76002) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.26603 (3.99125) + | > current_lr: 0.00000 + | > step_time: 8.30690 (3.17862) + | > loader_time: 0.00770 (0.04825) + + + --> STEP: 191/234 -- GLOBAL_STEP: 3935 + | > loss: 0.89444 (0.99437) + | > log_mle: 0.12410 (0.23427) + | > loss_dur: 0.77034 (0.76010) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.76677 (4.06139) + | > current_lr: 0.00000 + | > step_time: 5.80350 (3.23271) + | > loader_time: 0.00300 (0.04932) + + + --> STEP: 196/234 -- GLOBAL_STEP: 3940 + | > loss: 0.90869 (0.99120) + | > log_mle: 0.12748 (0.23063) + | > loss_dur: 0.78122 (0.76057) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.32393 (4.07608) + | > current_lr: 0.00000 + | > step_time: 4.70400 (3.32262) + | > loader_time: 0.00250 (0.05014) + + + --> STEP: 201/234 -- GLOBAL_STEP: 3945 + | > loss: 0.87312 (0.98774) + | > log_mle: 0.13913 (0.22731) + | > loss_dur: 0.73399 (0.76044) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.92864 (4.18231) + | > current_lr: 0.00000 + | > step_time: 3.48630 (3.40870) + | > loader_time: 0.00310 (0.05063) + + + --> STEP: 206/234 -- GLOBAL_STEP: 3950 + | > loss: 0.82724 (0.98433) + | > log_mle: 0.07119 (0.22361) + | > loss_dur: 0.75605 (0.76072) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.18502 (4.26229) + | > current_lr: 0.00000 + | > step_time: 14.68930 (3.50715) + | > loader_time: 0.08880 (0.05214) + + + --> STEP: 211/234 -- GLOBAL_STEP: 3955 + | > loss: 0.86445 (0.98109) + | > log_mle: 0.01512 (0.21948) + | > loss_dur: 0.84933 (0.76160) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.35639 (4.34042) + | > current_lr: 0.00000 + | > step_time: 7.10350 (3.58186) + | > loader_time: 0.12080 (0.05242) + + + --> STEP: 216/234 -- GLOBAL_STEP: 3960 + | > loss: 0.82503 (0.97773) + | > log_mle: 0.02913 (0.21557) + | > loss_dur: 0.79590 (0.76215) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.08231 (4.47025) + | > current_lr: 0.00000 + | > step_time: 3.50350 (3.62168) + | > loader_time: 0.19830 (0.05355) + + + --> STEP: 221/234 -- GLOBAL_STEP: 3965 + | > loss: 0.87122 (0.97469) + | > log_mle: 0.08394 (0.21166) + | > loss_dur: 0.78728 (0.76303) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.14352 (4.61007) + | > current_lr: 0.00000 + | > step_time: 1.59870 (3.65924) + | > loader_time: 0.00290 (0.05859) + + + --> STEP: 226/234 -- GLOBAL_STEP: 3970 + | > loss: 0.82252 (0.97166) + | > log_mle: 0.01155 (0.20740) + | > loss_dur: 0.81097 (0.76426) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.65584 (4.73461) + | > current_lr: 0.00000 + | > step_time: 0.23470 (3.58823) + | > loader_time: 0.00390 (0.05737) + + + --> STEP: 231/234 -- GLOBAL_STEP: 3975 + | > loss: 0.95339 (0.97009) + | > log_mle: -0.03580 (0.20294) + | > loss_dur: 0.98918 (0.76715) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.56798 (4.90129) + | > current_lr: 0.00000 + | > step_time: 0.27200 (3.51609) + | > loader_time: 0.00350 (0.05621) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.20764 (-0.19445) + | > avg_loss: 0.89012 (-0.05978) + | > avg_log_mle: 0.12360 (-0.02068) + | > avg_loss_dur: 0.76651 (-0.03910) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_3978.pth + + > EPOCH: 17/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 12:23:21)  + + --> STEP: 2/234 -- GLOBAL_STEP: 3980 + | > loss: 1.50266 (1.47841) + | > log_mle: 0.36397 (0.35601) + | > loss_dur: 1.13869 (1.12240) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.87770 (2.94937) + | > current_lr: 0.00000 + | > step_time: 2.79650 (3.65811) + | > loader_time: 0.00080 (3.29788) + + + --> STEP: 7/234 -- GLOBAL_STEP: 3985 + | > loss: 1.27250 (1.39114) + | > log_mle: 0.29775 (0.32647) + | > loss_dur: 0.97474 (1.06466) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.57230 (3.13870) + | > current_lr: 0.00000 + | > step_time: 7.09440 (3.97663) + | > loader_time: 0.00160 (1.93630) + + + --> STEP: 12/234 -- GLOBAL_STEP: 3990 + | > loss: 1.14686 (1.30676) + | > log_mle: 0.30848 (0.31735) + | > loss_dur: 0.83837 (0.98941) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.15864 (2.84737) + | > current_lr: 0.00000 + | > step_time: 3.67670 (5.13580) + | > loader_time: 0.31780 (1.22242) + + + --> STEP: 17/234 -- GLOBAL_STEP: 3995 + | > loss: 1.12953 (1.25236) + | > log_mle: 0.31431 (0.31377) + | > loss_dur: 0.81522 (0.93860) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.20508 (2.66181) + | > current_lr: 0.00000 + | > step_time: 7.80770 (5.39617) + | > loader_time: 0.38410 (0.90098) + + + --> STEP: 22/234 -- GLOBAL_STEP: 4000 + | > loss: 1.03714 (1.21560) + | > log_mle: 0.28912 (0.31168) + | > loss_dur: 0.74802 (0.90392) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.16880 (2.54219) + | > current_lr: 0.00000 + | > step_time: 2.48810 (4.95500) + | > loader_time: 0.00290 (0.70042) + + + --> STEP: 27/234 -- GLOBAL_STEP: 4005 + | > loss: 1.00561 (1.18305) + | > log_mle: 0.28588 (0.30874) + | > loss_dur: 0.71973 (0.87431) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.88599 (2.43961) + | > current_lr: 0.00000 + | > step_time: 2.99340 (4.63642) + | > loader_time: 0.10900 (0.58517) + + + --> STEP: 32/234 -- GLOBAL_STEP: 4010 + | > loss: 0.96473 (1.15550) + | > log_mle: 0.26866 (0.30534) + | > loss_dur: 0.69607 (0.85016) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.87664 (2.35888) + | > current_lr: 0.00000 + | > step_time: 2.69250 (4.75162) + | > loader_time: 0.10400 (0.50702) + + + --> STEP: 37/234 -- GLOBAL_STEP: 4015 + | > loss: 0.95064 (1.13680) + | > log_mle: 0.28068 (0.30280) + | > loss_dur: 0.66996 (0.83400) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.83863 (2.50177) + | > current_lr: 0.00000 + | > step_time: 2.72500 (4.53665) + | > loader_time: 0.08130 (0.44354) + + + --> STEP: 42/234 -- GLOBAL_STEP: 4020 + | > loss: 1.04837 (1.12257) + | > log_mle: 0.31087 (0.30145) + | > loss_dur: 0.73751 (0.82112) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.95987 (2.43776) + | > current_lr: 0.00000 + | > step_time: 2.11570 (4.20151) + | > loader_time: 0.08240 (0.39464) + + + --> STEP: 47/234 -- GLOBAL_STEP: 4025 + | > loss: 0.93366 (1.10766) + | > log_mle: 0.28478 (0.29949) + | > loss_dur: 0.64888 (0.80816) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.81921 (2.37963) + | > current_lr: 0.00000 + | > step_time: 5.80980 (4.05714) + | > loader_time: 0.00170 (0.35651) + + + --> STEP: 52/234 -- GLOBAL_STEP: 4030 + | > loss: 0.97193 (1.09339) + | > log_mle: 0.29737 (0.29844) + | > loss_dur: 0.67456 (0.79495) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.98172 (2.33094) + | > current_lr: 0.00000 + | > step_time: 0.90030 (3.86491) + | > loader_time: 0.00170 (0.32403) + + + --> STEP: 57/234 -- GLOBAL_STEP: 4035 + | > loss: 0.97051 (1.08250) + | > log_mle: 0.29688 (0.29665) + | > loss_dur: 0.67364 (0.78585) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.82476 (2.28661) + | > current_lr: 0.00000 + | > step_time: 1.98760 (3.68885) + | > loader_time: 0.00210 (0.29974) + + + --> STEP: 62/234 -- GLOBAL_STEP: 4040 + | > loss: 0.95724 (1.07137) + | > log_mle: 0.24643 (0.29434) + | > loss_dur: 0.71081 (0.77703) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.99979 (2.27218) + | > current_lr: 0.00000 + | > step_time: 2.34850 (3.56312) + | > loader_time: 0.19030 (0.27885) + + + --> STEP: 67/234 -- GLOBAL_STEP: 4045 + | > loss: 0.91682 (1.06169) + | > log_mle: 0.25577 (0.29302) + | > loss_dur: 0.66104 (0.76866) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00444 (2.25660) + | > current_lr: 0.00000 + | > step_time: 2.70160 (3.52071) + | > loader_time: 0.00180 (0.25969) + + + --> STEP: 72/234 -- GLOBAL_STEP: 4050 + | > loss: 0.90408 (1.05382) + | > log_mle: 0.27090 (0.29165) + | > loss_dur: 0.63318 (0.76217) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.70743 (2.26496) + | > current_lr: 0.00000 + | > step_time: 3.78590 (3.46940) + | > loader_time: 0.09450 (0.24943) + + + --> STEP: 77/234 -- GLOBAL_STEP: 4055 + | > loss: 0.91808 (1.04530) + | > log_mle: 0.25492 (0.28997) + | > loss_dur: 0.66316 (0.75532) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.94418 (2.26273) + | > current_lr: 0.00000 + | > step_time: 6.40320 (3.44061) + | > loader_time: 0.10520 (0.23705) + + + --> STEP: 82/234 -- GLOBAL_STEP: 4060 + | > loss: 0.88662 (1.03736) + | > log_mle: 0.27793 (0.28882) + | > loss_dur: 0.60868 (0.74854) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.68140 (2.24582) + | > current_lr: 0.00000 + | > step_time: 3.42110 (3.43248) + | > loader_time: 0.09980 (0.22927) + + + --> STEP: 87/234 -- GLOBAL_STEP: 4065 + | > loss: 0.89889 (1.03110) + | > log_mle: 0.25533 (0.28706) + | > loss_dur: 0.64356 (0.74404) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.60451 (2.25156) + | > current_lr: 0.00000 + | > step_time: 0.90020 (3.34655) + | > loader_time: 0.00250 (0.21838) + + + --> STEP: 92/234 -- GLOBAL_STEP: 4070 + | > loss: 0.87458 (1.02298) + | > log_mle: 0.22766 (0.28426) + | > loss_dur: 0.64691 (0.73872) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.03093 (2.27358) + | > current_lr: 0.00000 + | > step_time: 1.19890 (3.31394) + | > loader_time: 0.00330 (0.20862) + + + --> STEP: 97/234 -- GLOBAL_STEP: 4075 + | > loss: 0.88007 (1.01579) + | > log_mle: 0.23004 (0.28070) + | > loss_dur: 0.65003 (0.73509) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.94261 (2.32149) + | > current_lr: 0.00000 + | > step_time: 1.38410 (3.24654) + | > loader_time: 0.00250 (0.19908) + + + --> STEP: 102/234 -- GLOBAL_STEP: 4080 + | > loss: 0.90093 (1.00985) + | > log_mle: 0.25022 (0.27809) + | > loss_dur: 0.65070 (0.73176) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.24916 (2.36641) + | > current_lr: 0.00000 + | > step_time: 1.88490 (3.23932) + | > loader_time: 0.00230 (0.19106) + + + --> STEP: 107/234 -- GLOBAL_STEP: 4085 + | > loss: 0.85874 (1.00456) + | > log_mle: 0.20943 (0.27494) + | > loss_dur: 0.64931 (0.72962) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.85128 (2.45032) + | > current_lr: 0.00000 + | > step_time: 4.30710 (3.21971) + | > loader_time: 0.18320 (0.18477) + + + --> STEP: 112/234 -- GLOBAL_STEP: 4090 + | > loss: 0.86885 (0.99954) + | > log_mle: 0.20563 (0.27209) + | > loss_dur: 0.66322 (0.72746) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.83755 (2.53178) + | > current_lr: 0.00000 + | > step_time: 4.60120 (3.23763) + | > loader_time: 0.10640 (0.17763) + + + --> STEP: 117/234 -- GLOBAL_STEP: 4095 + | > loss: 0.85217 (0.99456) + | > log_mle: 0.21152 (0.26945) + | > loss_dur: 0.64065 (0.72511) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.24592 (2.60699) + | > current_lr: 0.00000 + | > step_time: 2.41760 (3.22188) + | > loader_time: 0.07750 (0.17385) + + + --> STEP: 122/234 -- GLOBAL_STEP: 4100 + | > loss: 0.86119 (0.98992) + | > log_mle: 0.22196 (0.26753) + | > loss_dur: 0.63923 (0.72239) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.75497 (2.62315) + | > current_lr: 0.00000 + | > step_time: 2.71550 (3.17780) + | > loader_time: 0.00240 (0.16799) + + + --> STEP: 127/234 -- GLOBAL_STEP: 4105 + | > loss: 0.86408 (0.98472) + | > log_mle: 0.18596 (0.26475) + | > loss_dur: 0.67812 (0.71997) + | > amp_scaler: 32768.00000 (16513.00787) + | > grad_norm: 4.19986 (2.68153) + | > current_lr: 0.00000 + | > step_time: 1.69960 (3.18802) + | > loader_time: 0.08620 (0.16218) + + + --> STEP: 132/234 -- GLOBAL_STEP: 4110 + | > loss: 0.84666 (0.97963) + | > log_mle: 0.20109 (0.26169) + | > loss_dur: 0.64556 (0.71793) + | > amp_scaler: 32768.00000 (17128.72727) + | > grad_norm: 3.68715 (2.72529) + | > current_lr: 0.00000 + | > step_time: 2.90110 (3.15728) + | > loader_time: 0.00820 (0.15627) + + + --> STEP: 137/234 -- GLOBAL_STEP: 4115 + | > loss: 0.91305 (0.97565) + | > log_mle: 0.19694 (0.25860) + | > loss_dur: 0.71611 (0.71705) + | > amp_scaler: 32768.00000 (17699.50365) + | > grad_norm: 3.08575 (2.78357) + | > current_lr: 0.00000 + | > step_time: 2.99370 (3.14997) + | > loader_time: 0.49860 (0.15567) + + + --> STEP: 142/234 -- GLOBAL_STEP: 4120 + | > loss: 0.86002 (0.97151) + | > log_mle: 0.17565 (0.25577) + | > loss_dur: 0.68437 (0.71574) + | > amp_scaler: 32768.00000 (18230.08451) + | > grad_norm: 5.05602 (2.86823) + | > current_lr: 0.00000 + | > step_time: 5.51570 (3.19687) + | > loader_time: 0.09740 (0.15425) + + + --> STEP: 147/234 -- GLOBAL_STEP: 4125 + | > loss: 0.82192 (0.96682) + | > log_mle: 0.17265 (0.25179) + | > loss_dur: 0.64927 (0.71503) + | > amp_scaler: 32768.00000 (18724.57143) + | > grad_norm: 6.92473 (3.02299) + | > current_lr: 0.00000 + | > step_time: 2.08980 (3.22889) + | > loader_time: 0.01890 (0.15185) + + + --> STEP: 152/234 -- GLOBAL_STEP: 4130 + | > loss: 0.82171 (0.96206) + | > log_mle: 0.12046 (0.24813) + | > loss_dur: 0.70125 (0.71393) + | > amp_scaler: 32768.00000 (19186.52632) + | > grad_norm: 5.37659 (3.11313) + | > current_lr: 0.00000 + | > step_time: 8.28410 (3.26934) + | > loader_time: 0.00410 (0.14892) + + + --> STEP: 157/234 -- GLOBAL_STEP: 4135 + | > loss: 0.86569 (0.95778) + | > log_mle: 0.15971 (0.24415) + | > loss_dur: 0.70598 (0.71362) + | > amp_scaler: 32768.00000 (19619.05732) + | > grad_norm: 5.05941 (3.23438) + | > current_lr: 0.00000 + | > step_time: 2.50950 (3.34677) + | > loader_time: 0.08140 (0.14794) + + + --> STEP: 162/234 -- GLOBAL_STEP: 4140 + | > loss: 0.82785 (0.95378) + | > log_mle: 0.13040 (0.24031) + | > loss_dur: 0.69745 (0.71346) + | > amp_scaler: 32768.00000 (20024.88889) + | > grad_norm: 7.15409 (3.34323) + | > current_lr: 0.00000 + | > step_time: 3.39350 (3.35580) + | > loader_time: 0.00460 (0.14459) + + + --> STEP: 167/234 -- GLOBAL_STEP: 4145 + | > loss: 0.81126 (0.94972) + | > log_mle: 0.07879 (0.23678) + | > loss_dur: 0.73247 (0.71294) + | > amp_scaler: 32768.00000 (20406.41916) + | > grad_norm: 11.33472 (3.45352) + | > current_lr: 0.00000 + | > step_time: 7.60020 (3.39905) + | > loader_time: 0.10900 (0.14280) + + + --> STEP: 172/234 -- GLOBAL_STEP: 4150 + | > loss: 0.82686 (0.94641) + | > log_mle: 0.08526 (0.23270) + | > loss_dur: 0.74160 (0.71371) + | > amp_scaler: 32768.00000 (20765.76744) + | > grad_norm: 9.09225 (3.56792) + | > current_lr: 0.00000 + | > step_time: 2.60440 (3.46216) + | > loader_time: 0.00340 (0.14136) + + + --> STEP: 177/234 -- GLOBAL_STEP: 4155 + | > loss: 0.85059 (0.94260) + | > log_mle: 0.11534 (0.22880) + | > loss_dur: 0.73525 (0.71379) + | > amp_scaler: 32768.00000 (21104.81356) + | > grad_norm: 6.83840 (3.67672) + | > current_lr: 0.00000 + | > step_time: 4.89550 (3.52476) + | > loader_time: 0.00340 (0.13859) + + + --> STEP: 182/234 -- GLOBAL_STEP: 4160 + | > loss: 0.83097 (0.93918) + | > log_mle: 0.07821 (0.22493) + | > loss_dur: 0.75276 (0.71425) + | > amp_scaler: 32768.00000 (21425.23077) + | > grad_norm: 7.50191 (3.75586) + | > current_lr: 0.00000 + | > step_time: 6.50810 (3.60768) + | > loader_time: 0.09960 (0.13740) + + + --> STEP: 187/234 -- GLOBAL_STEP: 4165 + | > loss: 0.78613 (0.93579) + | > log_mle: 0.07498 (0.22115) + | > loss_dur: 0.71115 (0.71465) + | > amp_scaler: 32768.00000 (21728.51337) + | > grad_norm: 8.26161 (3.83826) + | > current_lr: 0.00000 + | > step_time: 3.69270 (3.59836) + | > loader_time: 0.01100 (0.13424) + + + --> STEP: 192/234 -- GLOBAL_STEP: 4170 + | > loss: 0.77896 (0.93228) + | > log_mle: 0.05551 (0.21748) + | > loss_dur: 0.72346 (0.71480) + | > amp_scaler: 32768.00000 (22016.00000) + | > grad_norm: 5.65535 (3.93672) + | > current_lr: 0.00000 + | > step_time: 2.71180 (3.66039) + | > loader_time: 0.06350 (0.13426) + + + --> STEP: 197/234 -- GLOBAL_STEP: 4175 + | > loss: 0.77118 (0.92909) + | > log_mle: 0.07217 (0.21393) + | > loss_dur: 0.69901 (0.71516) + | > amp_scaler: 32768.00000 (22288.89340) + | > grad_norm: 8.61538 (4.00874) + | > current_lr: 0.00000 + | > step_time: 9.90730 (3.74562) + | > loader_time: 0.07390 (0.13228) + + + --> STEP: 202/234 -- GLOBAL_STEP: 4180 + | > loss: 0.74387 (0.92543) + | > log_mle: 0.00563 (0.21024) + | > loss_dur: 0.73823 (0.71520) + | > amp_scaler: 32768.00000 (22548.27723) + | > grad_norm: 11.12428 (4.10347) + | > current_lr: 0.00000 + | > step_time: 7.00330 (3.78358) + | > loader_time: 0.19960 (0.13205) + + + --> STEP: 207/234 -- GLOBAL_STEP: 4185 + | > loss: 0.75524 (0.92202) + | > log_mle: 0.01909 (0.20659) + | > loss_dur: 0.73614 (0.71543) + | > amp_scaler: 32768.00000 (22795.13043) + | > grad_norm: 14.34364 (4.25467) + | > current_lr: 0.00000 + | > step_time: 8.60900 (3.79474) + | > loader_time: 0.00360 (0.13176) + + + --> STEP: 212/234 -- GLOBAL_STEP: 4190 + | > loss: 0.78186 (0.91889) + | > log_mle: 0.03124 (0.20250) + | > loss_dur: 0.75062 (0.71639) + | > amp_scaler: 32768.00000 (23030.33962) + | > grad_norm: 9.09087 (4.38592) + | > current_lr: 0.00000 + | > step_time: 7.21290 (3.84118) + | > loader_time: 0.07820 (0.13095) + + + --> STEP: 217/234 -- GLOBAL_STEP: 4195 + | > loss: 0.76822 (0.91541) + | > log_mle: 0.01626 (0.19849) + | > loss_dur: 0.75197 (0.71692) + | > amp_scaler: 32768.00000 (23254.70968) + | > grad_norm: 6.53632 (4.46747) + | > current_lr: 0.00000 + | > step_time: 3.79530 (3.86362) + | > loader_time: 0.10710 (0.13036) + + + --> STEP: 222/234 -- GLOBAL_STEP: 4200 + | > loss: 0.80385 (0.91245) + | > log_mle: 0.01025 (0.19452) + | > loss_dur: 0.79360 (0.71793) + | > amp_scaler: 32768.00000 (23468.97297) + | > grad_norm: 7.03531 (4.59992) + | > current_lr: 0.00000 + | > step_time: 2.19860 (3.83369) + | > loader_time: 0.00390 (0.12788) + + + --> STEP: 227/234 -- GLOBAL_STEP: 4205 + | > loss: 0.77473 (0.90921) + | > log_mle: 0.02490 (0.19028) + | > loss_dur: 0.74983 (0.71893) + | > amp_scaler: 32768.00000 (23673.79736) + | > grad_norm: 19.44108 (4.80321) + | > current_lr: 0.00000 + | > step_time: 0.24320 (3.77858) + | > loader_time: 0.00260 (0.12592) + + + --> STEP: 232/234 -- GLOBAL_STEP: 4210 + | > loss: 1.02232 (0.90855) + | > log_mle: -0.12333 (0.18514) + | > loss_dur: 1.14566 (0.72341) + | > amp_scaler: 32768.00000 (23869.79310) + | > grad_norm: 16.42076 (4.98949) + | > current_lr: 0.00000 + | > step_time: 0.33070 (3.70317) + | > loader_time: 0.00550 (0.12330) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02296 (-0.18468) + | > avg_loss: 0.84109 (-0.04903) + | > avg_log_mle: 0.11377 (-0.00983) + | > avg_loss_dur: 0.72732 (-0.03919) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_4212.pth + + > EPOCH: 18/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 12:39:16)  + + --> STEP: 3/234 -- GLOBAL_STEP: 4215 + | > loss: 1.25468 (1.35435) + | > log_mle: 0.31001 (0.33095) + | > loss_dur: 0.94467 (1.02341) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.48030 (2.62398) + | > current_lr: 0.00000 + | > step_time: 4.39260 (5.52329) + | > loader_time: 0.09680 (0.43009) + + + --> STEP: 8/234 -- GLOBAL_STEP: 4220 + | > loss: 1.21899 (1.29855) + | > log_mle: 0.28086 (0.30784) + | > loss_dur: 0.93812 (0.99071) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.47600 (2.78004) + | > current_lr: 0.00000 + | > step_time: 7.00280 (4.88387) + | > loader_time: 0.00540 (0.18917) + + + --> STEP: 13/234 -- GLOBAL_STEP: 4225 + | > loss: 1.09507 (1.21565) + | > log_mle: 0.29972 (0.30227) + | > loss_dur: 0.79536 (0.91338) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.18919 (2.56657) + | > current_lr: 0.00000 + | > step_time: 5.79220 (5.48867) + | > loader_time: 0.00180 (0.12494) + + + --> STEP: 18/234 -- GLOBAL_STEP: 4230 + | > loss: 1.05473 (1.16919) + | > log_mle: 0.29286 (0.29871) + | > loss_dur: 0.76188 (0.87048) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.92969 (2.41614) + | > current_lr: 0.00000 + | > step_time: 3.89270 (5.56398) + | > loader_time: 0.00140 (0.09091) + + + --> STEP: 23/234 -- GLOBAL_STEP: 4235 + | > loss: 0.96981 (1.13455) + | > log_mle: 0.27916 (0.29624) + | > loss_dur: 0.69065 (0.83831) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.83290 (2.31479) + | > current_lr: 0.00000 + | > step_time: 8.10510 (5.35148) + | > loader_time: 0.00120 (0.08282) + + + --> STEP: 28/234 -- GLOBAL_STEP: 4240 + | > loss: 0.95241 (1.10637) + | > log_mle: 0.28119 (0.29374) + | > loss_dur: 0.67121 (0.81263) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.84449 (2.23493) + | > current_lr: 0.00000 + | > step_time: 1.35910 (5.00256) + | > loader_time: 0.00180 (0.07879) + + + --> STEP: 33/234 -- GLOBAL_STEP: 4245 + | > loss: 0.98473 (1.08368) + | > log_mle: 0.28708 (0.29080) + | > loss_dur: 0.69765 (0.79289) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.87067 (2.16934) + | > current_lr: 0.00000 + | > step_time: 1.58360 (4.45223) + | > loader_time: 0.00300 (0.06721) + + + --> STEP: 38/234 -- GLOBAL_STEP: 4250 + | > loss: 0.93909 (1.06599) + | > log_mle: 0.26858 (0.28791) + | > loss_dur: 0.67051 (0.77808) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.76204 (2.29368) + | > current_lr: 0.00000 + | > step_time: 2.07650 (4.29224) + | > loader_time: 0.00140 (0.06379) + + + --> STEP: 43/234 -- GLOBAL_STEP: 4255 + | > loss: 0.93369 (1.05326) + | > log_mle: 0.26396 (0.28657) + | > loss_dur: 0.66973 (0.76670) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.79873 (2.24457) + | > current_lr: 0.00000 + | > step_time: 1.04650 (3.99523) + | > loader_time: 0.00170 (0.05886) + + + --> STEP: 48/234 -- GLOBAL_STEP: 4260 + | > loss: 0.89811 (1.03936) + | > log_mle: 0.27471 (0.28498) + | > loss_dur: 0.62340 (0.75438) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.73295 (2.19538) + | > current_lr: 0.00000 + | > step_time: 2.09640 (3.86210) + | > loader_time: 0.09470 (0.05894) + + + --> STEP: 53/234 -- GLOBAL_STEP: 4265 + | > loss: 0.90110 (1.02667) + | > log_mle: 0.25625 (0.28364) + | > loss_dur: 0.64485 (0.74303) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.69470 (2.15031) + | > current_lr: 0.00000 + | > step_time: 1.83510 (3.72106) + | > loader_time: 0.08870 (0.05895) + + + --> STEP: 58/234 -- GLOBAL_STEP: 4270 + | > loss: 0.90642 (1.01702) + | > log_mle: 0.27595 (0.28228) + | > loss_dur: 0.63046 (0.73474) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.69113 (2.11440) + | > current_lr: 0.00000 + | > step_time: 2.01840 (3.57402) + | > loader_time: 0.00270 (0.05712) + + + --> STEP: 63/234 -- GLOBAL_STEP: 4275 + | > loss: 0.95192 (1.00737) + | > log_mle: 0.25753 (0.27971) + | > loss_dur: 0.69439 (0.72766) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.91578 (2.10801) + | > current_lr: 0.00000 + | > step_time: 3.18850 (3.44774) + | > loader_time: 0.00180 (0.05430) + + + --> STEP: 68/234 -- GLOBAL_STEP: 4280 + | > loss: 0.89861 (0.99800) + | > log_mle: 0.26704 (0.27858) + | > loss_dur: 0.63157 (0.71941) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.99284 (2.08834) + | > current_lr: 0.00000 + | > step_time: 2.09330 (3.42822) + | > loader_time: 0.00230 (0.05328) + + + --> STEP: 73/234 -- GLOBAL_STEP: 4285 + | > loss: 0.89419 (0.99109) + | > log_mle: 0.24580 (0.27695) + | > loss_dur: 0.64838 (0.71414) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.97802 (2.09922) + | > current_lr: 0.00000 + | > step_time: 2.30210 (3.37681) + | > loader_time: 0.00250 (0.05245) + + + --> STEP: 78/234 -- GLOBAL_STEP: 4290 + | > loss: 0.87261 (0.98306) + | > log_mle: 0.26661 (0.27558) + | > loss_dur: 0.60600 (0.70748) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.19419 (2.08943) + | > current_lr: 0.00000 + | > step_time: 4.09920 (3.33053) + | > loader_time: 0.08380 (0.05135) + + + --> STEP: 83/234 -- GLOBAL_STEP: 4295 + | > loss: 0.88093 (0.97581) + | > log_mle: 0.24496 (0.27419) + | > loss_dur: 0.63597 (0.70162) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.81834 (2.08800) + | > current_lr: 0.00000 + | > step_time: 2.30300 (3.31905) + | > loader_time: 0.00340 (0.04848) + + + --> STEP: 88/234 -- GLOBAL_STEP: 4300 + | > loss: 0.82720 (0.96926) + | > log_mle: 0.20756 (0.27204) + | > loss_dur: 0.61964 (0.69722) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.02561 (2.10112) + | > current_lr: 0.00000 + | > step_time: 2.70250 (3.30337) + | > loader_time: 0.00360 (0.04900) + + + --> STEP: 93/234 -- GLOBAL_STEP: 4305 + | > loss: 0.78100 (0.96116) + | > log_mle: 0.19436 (0.26916) + | > loss_dur: 0.58664 (0.69200) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.65126 (2.14423) + | > current_lr: 0.00000 + | > step_time: 2.60930 (3.23238) + | > loader_time: 0.08530 (0.04850) + + + --> STEP: 98/234 -- GLOBAL_STEP: 4310 + | > loss: 0.87143 (0.95528) + | > log_mle: 0.25373 (0.26622) + | > loss_dur: 0.61770 (0.68906) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.82674 (2.18478) + | > current_lr: 0.00000 + | > step_time: 2.60280 (3.23394) + | > loader_time: 0.08540 (0.04983) + + + --> STEP: 103/234 -- GLOBAL_STEP: 4315 + | > loss: 0.83626 (0.94910) + | > log_mle: 0.18210 (0.26290) + | > loss_dur: 0.65415 (0.68620) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.89761 (2.28256) + | > current_lr: 0.00000 + | > step_time: 6.68710 (3.23056) + | > loader_time: 0.10160 (0.04955) + + + --> STEP: 108/234 -- GLOBAL_STEP: 4320 + | > loss: 0.83870 (0.94409) + | > log_mle: 0.22601 (0.26019) + | > loss_dur: 0.61269 (0.68390) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.70231 (2.34098) + | > current_lr: 0.00000 + | > step_time: 3.28950 (3.25427) + | > loader_time: 0.09350 (0.05242) + + + --> STEP: 113/234 -- GLOBAL_STEP: 4325 + | > loss: 0.81542 (0.93904) + | > log_mle: 0.18538 (0.25699) + | > loss_dur: 0.63005 (0.68205) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.96095 (2.51411) + | > current_lr: 0.00000 + | > step_time: 5.08400 (3.30226) + | > loader_time: 0.10510 (0.05436) + + + --> STEP: 118/234 -- GLOBAL_STEP: 4330 + | > loss: 0.85716 (0.93460) + | > log_mle: 0.21032 (0.25459) + | > loss_dur: 0.64684 (0.68001) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.12834 (2.59355) + | > current_lr: 0.00000 + | > step_time: 7.88030 (3.29921) + | > loader_time: 0.10930 (0.05387) + + + --> STEP: 123/234 -- GLOBAL_STEP: 4335 + | > loss: 0.82039 (0.92985) + | > log_mle: 0.23850 (0.25293) + | > loss_dur: 0.58189 (0.67692) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.94748 (2.69772) + | > current_lr: 0.00000 + | > step_time: 5.63000 (3.30993) + | > loader_time: 0.00570 (0.05261) + + + --> STEP: 128/234 -- GLOBAL_STEP: 4340 + | > loss: 0.79167 (0.92456) + | > log_mle: 0.18396 (0.24971) + | > loss_dur: 0.60770 (0.67486) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.09735 (2.81652) + | > current_lr: 0.00000 + | > step_time: 2.07910 (3.29530) + | > loader_time: 0.01520 (0.05144) + + + --> STEP: 133/234 -- GLOBAL_STEP: 4345 + | > loss: 0.82063 (0.92000) + | > log_mle: 0.17119 (0.24657) + | > loss_dur: 0.64944 (0.67342) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.12130 (2.89606) + | > current_lr: 0.00000 + | > step_time: 0.97130 (3.24075) + | > loader_time: 0.00180 (0.05148) + + + --> STEP: 138/234 -- GLOBAL_STEP: 4350 + | > loss: 0.83031 (0.91631) + | > log_mle: 0.20189 (0.24367) + | > loss_dur: 0.62842 (0.67263) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.82464 (2.93459) + | > current_lr: 0.00000 + | > step_time: 3.00770 (3.20079) + | > loader_time: 0.00210 (0.05088) + + + --> STEP: 143/234 -- GLOBAL_STEP: 4355 + | > loss: 0.78321 (0.91189) + | > log_mle: 0.10062 (0.24009) + | > loss_dur: 0.68259 (0.67180) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.35396 (3.00817) + | > current_lr: 0.00000 + | > step_time: 2.12140 (3.16115) + | > loader_time: 0.08440 (0.05155) + + + --> STEP: 148/234 -- GLOBAL_STEP: 4360 + | > loss: 0.73517 (0.90695) + | > log_mle: 0.15885 (0.23646) + | > loss_dur: 0.57632 (0.67049) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.95923 (3.09906) + | > current_lr: 0.00000 + | > step_time: 4.20020 (3.16685) + | > loader_time: 0.08690 (0.05113) + + + --> STEP: 153/234 -- GLOBAL_STEP: 4365 + | > loss: 0.74301 (0.90241) + | > log_mle: 0.07547 (0.23223) + | > loss_dur: 0.66754 (0.67018) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 13.02338 (3.24141) + | > current_lr: 0.00000 + | > step_time: 2.91210 (3.21969) + | > loader_time: 0.08290 (0.05130) + + + --> STEP: 158/234 -- GLOBAL_STEP: 4370 + | > loss: 0.76510 (0.89825) + | > log_mle: 0.11570 (0.22849) + | > loss_dur: 0.64941 (0.66977) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.42435 (3.41878) + | > current_lr: 0.00000 + | > step_time: 7.29700 (3.32595) + | > loader_time: 0.09870 (0.05348) + + + --> STEP: 163/234 -- GLOBAL_STEP: 4375 + | > loss: 0.75915 (0.89428) + | > log_mle: 0.13649 (0.22474) + | > loss_dur: 0.62266 (0.66954) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.55341 (3.56187) + | > current_lr: 0.00000 + | > step_time: 5.89790 (3.34659) + | > loader_time: 0.00450 (0.05422) + + + --> STEP: 168/234 -- GLOBAL_STEP: 4380 + | > loss: 0.81474 (0.89060) + | > log_mle: 0.09809 (0.22094) + | > loss_dur: 0.71665 (0.66966) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.26261 (3.64133) + | > current_lr: 0.00000 + | > step_time: 4.30710 (3.40000) + | > loader_time: 0.00590 (0.05552) + + + --> STEP: 173/234 -- GLOBAL_STEP: 4385 + | > loss: 0.76318 (0.88691) + | > log_mle: 0.09060 (0.21677) + | > loss_dur: 0.67258 (0.67015) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.38039 (3.72501) + | > current_lr: 0.00000 + | > step_time: 4.69670 (3.41281) + | > loader_time: 0.00500 (0.05852) + + + --> STEP: 178/234 -- GLOBAL_STEP: 4390 + | > loss: 0.72859 (0.88286) + | > log_mle: 0.04375 (0.21257) + | > loss_dur: 0.68485 (0.67029) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.86217 (3.80938) + | > current_lr: 0.00000 + | > step_time: 2.07400 (3.45671) + | > loader_time: 0.00380 (0.05856) + + + --> STEP: 183/234 -- GLOBAL_STEP: 4395 + | > loss: 0.75197 (0.87958) + | > log_mle: 0.04934 (0.20869) + | > loss_dur: 0.70263 (0.67089) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.68860 (3.91001) + | > current_lr: 0.00000 + | > step_time: 4.51080 (3.47092) + | > loader_time: 0.18400 (0.06023) + + + --> STEP: 188/234 -- GLOBAL_STEP: 4400 + | > loss: 0.72867 (0.87604) + | > log_mle: 0.04140 (0.20482) + | > loss_dur: 0.68728 (0.67123) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 13.31417 (4.05625) + | > current_lr: 0.00000 + | > step_time: 4.41950 (3.51327) + | > loader_time: 0.08370 (0.06077) + + + --> STEP: 193/234 -- GLOBAL_STEP: 4405 + | > loss: 0.72003 (0.87250) + | > log_mle: 0.04150 (0.20110) + | > loss_dur: 0.67853 (0.67140) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.75286 (4.17798) + | > current_lr: 0.00000 + | > step_time: 5.29540 (3.62064) + | > loader_time: 0.01050 (0.06218) + + + --> STEP: 198/234 -- GLOBAL_STEP: 4410 + | > loss: 0.73866 (0.86933) + | > log_mle: 0.04791 (0.19753) + | > loss_dur: 0.69076 (0.67181) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.18719 (4.27484) + | > current_lr: 0.00000 + | > step_time: 5.59850 (3.63372) + | > loader_time: 0.09290 (0.06262) + + + --> STEP: 203/234 -- GLOBAL_STEP: 4415 + | > loss: 0.76128 (0.86585) + | > log_mle: 0.09015 (0.19402) + | > loss_dur: 0.67113 (0.67183) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.29509 (4.38136) + | > current_lr: 0.00000 + | > step_time: 5.21070 (3.72989) + | > loader_time: 0.09390 (0.06406) + + + --> STEP: 208/234 -- GLOBAL_STEP: 4420 + | > loss: 0.73760 (0.86234) + | > log_mle: 0.03523 (0.19007) + | > loss_dur: 0.70237 (0.67227) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 11.81239 (4.54194) + | > current_lr: 0.00000 + | > step_time: 1.89420 (3.77105) + | > loader_time: 0.00310 (0.06434) + + + --> STEP: 213/234 -- GLOBAL_STEP: 4425 + | > loss: 0.69763 (0.85898) + | > log_mle: -0.00384 (0.18576) + | > loss_dur: 0.70147 (0.67321) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 13.46325 (4.74207) + | > current_lr: 0.00000 + | > step_time: 12.58890 (3.83054) + | > loader_time: 0.19380 (0.06556) + + + --> STEP: 218/234 -- GLOBAL_STEP: 4430 + | > loss: 0.73286 (0.85570) + | > log_mle: 0.02486 (0.18186) + | > loss_dur: 0.70800 (0.67383) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.17972 (4.88178) + | > current_lr: 0.00000 + | > step_time: 10.19910 (3.90234) + | > loader_time: 0.10090 (0.06633) + + + --> STEP: 223/234 -- GLOBAL_STEP: 4435 + | > loss: 0.70223 (0.85264) + | > log_mle: -0.00631 (0.17774) + | > loss_dur: 0.70855 (0.67490) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 11.51525 (5.05445) + | > current_lr: 0.00000 + | > step_time: 0.24310 (3.86606) + | > loader_time: 0.00500 (0.06536) + + + --> STEP: 228/234 -- GLOBAL_STEP: 4440 + | > loss: 0.74010 (0.84949) + | > log_mle: -0.00957 (0.17344) + | > loss_dur: 0.74966 (0.67605) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.78967 (5.24132) + | > current_lr: 0.00000 + | > step_time: 0.25300 (3.78664) + | > loader_time: 0.00330 (0.06400) + + + --> STEP: 233/234 -- GLOBAL_STEP: 4445 + | > loss: 1.57800 (0.85220) + | > log_mle: 0.03492 (0.16843) + | > loss_dur: 1.54308 (0.68377) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.81414 (5.42461) + | > current_lr: 0.00000 + | > step_time: 0.18740 (3.71096) + | > loader_time: 0.00580 (0.06274) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17647 (+0.15350) + | > avg_loss: 0.78502 (-0.05607) + | > avg_log_mle: 0.09960 (-0.01417) + | > avg_loss_dur: 0.68542 (-0.04189) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_4446.pth + + > EPOCH: 19/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 12:55:06)  + + --> STEP: 4/234 -- GLOBAL_STEP: 4450 + | > loss: 1.31209 (1.28912) + | > log_mle: 0.27405 (0.30558) + | > loss_dur: 1.03804 (0.98354) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.13571 (2.59897) + | > current_lr: 0.00000 + | > step_time: 1.99910 (2.79723) + | > loader_time: 0.00410 (2.75093) + + + --> STEP: 9/234 -- GLOBAL_STEP: 4455 + | > loss: 1.07333 (1.21378) + | > log_mle: 0.26783 (0.29062) + | > loss_dur: 0.80550 (0.92316) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.41273 (2.53642) + | > current_lr: 0.00000 + | > step_time: 6.31800 (5.74522) + | > loader_time: 0.08950 (1.26380) + + + --> STEP: 14/234 -- GLOBAL_STEP: 4460 + | > loss: 0.99552 (1.14000) + | > log_mle: 0.27222 (0.28684) + | > loss_dur: 0.72330 (0.85317) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.07393 (2.37339) + | > current_lr: 0.00000 + | > step_time: 4.50760 (5.87978) + | > loader_time: 0.00410 (0.84709) + + + --> STEP: 19/234 -- GLOBAL_STEP: 4465 + | > loss: 1.03676 (1.10375) + | > log_mle: 0.28198 (0.28422) + | > loss_dur: 0.75478 (0.81952) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.96578 (2.24134) + | > current_lr: 0.00000 + | > step_time: 1.68900 (5.59989) + | > loader_time: 0.09530 (0.64418) + + + --> STEP: 24/234 -- GLOBAL_STEP: 4470 + | > loss: 0.97711 (1.06969) + | > log_mle: 0.27296 (0.28159) + | > loss_dur: 0.70415 (0.78810) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.88203 (2.14997) + | > current_lr: 0.00000 + | > step_time: 2.09740 (5.25095) + | > loader_time: 0.00120 (0.51387) + + + --> STEP: 29/234 -- GLOBAL_STEP: 4475 + | > loss: 0.93316 (1.04251) + | > log_mle: 0.27753 (0.27943) + | > loss_dur: 0.65564 (0.76308) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.57931 (2.06569) + | > current_lr: 0.00000 + | > step_time: 4.70950 (4.97419) + | > loader_time: 0.00190 (0.43160) + + + --> STEP: 34/234 -- GLOBAL_STEP: 4480 + | > loss: 0.91452 (1.02106) + | > log_mle: 0.25563 (0.27597) + | > loss_dur: 0.65889 (0.74510) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.72310 (2.00855) + | > current_lr: 0.00000 + | > step_time: 9.34640 (5.02425) + | > loader_time: 0.10160 (0.37966) + + + --> STEP: 39/234 -- GLOBAL_STEP: 4485 + | > loss: 0.92111 (1.00430) + | > log_mle: 0.25885 (0.27329) + | > loss_dur: 0.66225 (0.73101) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.76685 (2.09579) + | > current_lr: 0.00000 + | > step_time: 2.11220 (4.62871) + | > loader_time: 0.08480 (0.33557) + + + --> STEP: 44/234 -- GLOBAL_STEP: 4490 + | > loss: 0.88223 (0.99128) + | > log_mle: 0.25495 (0.27197) + | > loss_dur: 0.62728 (0.71931) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.66580 (2.04852) + | > current_lr: 0.00000 + | > step_time: 1.49780 (4.27273) + | > loader_time: 0.00310 (0.29936) + + + --> STEP: 49/234 -- GLOBAL_STEP: 4495 + | > loss: 0.82883 (0.97754) + | > log_mle: 0.24659 (0.27031) + | > loss_dur: 0.58224 (0.70722) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.53313 (2.00865) + | > current_lr: 0.00000 + | > step_time: 2.31440 (4.08856) + | > loader_time: 0.08710 (0.27079) + + + --> STEP: 54/234 -- GLOBAL_STEP: 4500 + | > loss: 0.86040 (0.96654) + | > log_mle: 0.24783 (0.26909) + | > loss_dur: 0.61258 (0.69746) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.57965 (1.97068) + | > current_lr: 0.00000 + | > step_time: 2.69520 (3.92466) + | > loader_time: 0.10000 (0.25097) + + + --> STEP: 59/234 -- GLOBAL_STEP: 4505 + | > loss: 0.78224 (0.95684) + | > log_mle: 0.22879 (0.26749) + | > loss_dur: 0.55345 (0.68936) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.61326 (1.93912) + | > current_lr: 0.00000 + | > step_time: 1.01060 (3.76709) + | > loader_time: 0.00270 (0.23147) + + + --> STEP: 64/234 -- GLOBAL_STEP: 4510 + | > loss: 0.80926 (0.94829) + | > log_mle: 0.25846 (0.26545) + | > loss_dur: 0.55080 (0.68285) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.40168 (1.92708) + | > current_lr: 0.00000 + | > step_time: 2.50310 (3.65034) + | > loader_time: 0.00410 (0.21455) + + + --> STEP: 69/234 -- GLOBAL_STEP: 4515 + | > loss: 0.88360 (0.94078) + | > log_mle: 0.26672 (0.26447) + | > loss_dur: 0.61688 (0.67631) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.47873 (1.90735) + | > current_lr: 0.00000 + | > step_time: 1.50770 (3.52373) + | > loader_time: 0.00240 (0.20315) + + + --> STEP: 74/234 -- GLOBAL_STEP: 4520 + | > loss: 0.78494 (0.93303) + | > log_mle: 0.24373 (0.26254) + | > loss_dur: 0.54121 (0.67049) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.97387 (1.91941) + | > current_lr: 0.00000 + | > step_time: 2.61150 (3.47505) + | > loader_time: 0.00820 (0.19087) + + + --> STEP: 79/234 -- GLOBAL_STEP: 4525 + | > loss: 0.82386 (0.92584) + | > log_mle: 0.23717 (0.26111) + | > loss_dur: 0.58669 (0.66473) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.61359 (1.91083) + | > current_lr: 0.00000 + | > step_time: 1.31020 (3.38240) + | > loader_time: 0.00340 (0.18023) + + + --> STEP: 84/234 -- GLOBAL_STEP: 4530 + | > loss: 0.82574 (0.91939) + | > log_mle: 0.22922 (0.25967) + | > loss_dur: 0.59652 (0.65972) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.75949 (1.92559) + | > current_lr: 0.00000 + | > step_time: 1.50440 (3.32901) + | > loader_time: 0.00460 (0.17174) + + + --> STEP: 89/234 -- GLOBAL_STEP: 4535 + | > loss: 0.76521 (0.91266) + | > log_mle: 0.20827 (0.25732) + | > loss_dur: 0.55694 (0.65534) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.94229 (1.96452) + | > current_lr: 0.00000 + | > step_time: 5.18120 (3.39564) + | > loader_time: 0.11100 (0.16651) + + + --> STEP: 94/234 -- GLOBAL_STEP: 4540 + | > loss: 0.78016 (0.90518) + | > log_mle: 0.17531 (0.25412) + | > loss_dur: 0.60485 (0.65106) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.73684 (2.07301) + | > current_lr: 0.00000 + | > step_time: 2.70680 (3.36722) + | > loader_time: 0.00260 (0.15899) + + + --> STEP: 99/234 -- GLOBAL_STEP: 4545 + | > loss: 0.74876 (0.89896) + | > log_mle: 0.15024 (0.25096) + | > loss_dur: 0.59852 (0.64801) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.11941 (2.17859) + | > current_lr: 0.00000 + | > step_time: 6.40920 (3.42067) + | > loader_time: 0.19620 (0.15575) + + + --> STEP: 104/234 -- GLOBAL_STEP: 4550 + | > loss: 0.77230 (0.89337) + | > log_mle: 0.14581 (0.24763) + | > loss_dur: 0.62649 (0.64575) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.82286 (2.28697) + | > current_lr: 0.00000 + | > step_time: 4.20760 (3.42652) + | > loader_time: 0.10100 (0.15554) + + + --> STEP: 109/234 -- GLOBAL_STEP: 4555 + | > loss: 0.78472 (0.88876) + | > log_mle: 0.16927 (0.24516) + | > loss_dur: 0.61545 (0.64360) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.02170 (2.36177) + | > current_lr: 0.00000 + | > step_time: 2.09450 (3.45754) + | > loader_time: 0.00290 (0.15111) + + + --> STEP: 114/234 -- GLOBAL_STEP: 4560 + | > loss: 0.77020 (0.88373) + | > log_mle: 0.18862 (0.24213) + | > loss_dur: 0.58158 (0.64160) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.20034 (2.45032) + | > current_lr: 0.00000 + | > step_time: 1.50310 (3.40512) + | > loader_time: 0.00240 (0.14620) + + + --> STEP: 119/234 -- GLOBAL_STEP: 4565 + | > loss: 0.77093 (0.87974) + | > log_mle: 0.18693 (0.23973) + | > loss_dur: 0.58400 (0.64001) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.17939 (2.52837) + | > current_lr: 0.00000 + | > step_time: 1.69960 (3.34174) + | > loader_time: 0.00380 (0.14017) + + + --> STEP: 124/234 -- GLOBAL_STEP: 4570 + | > loss: 0.74004 (0.87511) + | > log_mle: 0.16237 (0.23787) + | > loss_dur: 0.57766 (0.63724) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.53417 (2.59261) + | > current_lr: 0.00000 + | > step_time: 1.70180 (3.32650) + | > loader_time: 0.00600 (0.13660) + + + --> STEP: 129/234 -- GLOBAL_STEP: 4575 + | > loss: 0.76555 (0.87011) + | > log_mle: 0.17268 (0.23472) + | > loss_dur: 0.59286 (0.63539) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.62597 (2.64801) + | > current_lr: 0.00000 + | > step_time: 2.90430 (3.28613) + | > loader_time: 0.10540 (0.13223) + + + --> STEP: 134/234 -- GLOBAL_STEP: 4580 + | > loss: 0.76605 (0.86560) + | > log_mle: 0.13732 (0.23131) + | > loss_dur: 0.62874 (0.63429) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.77235 (2.73776) + | > current_lr: 0.00000 + | > step_time: 1.10070 (3.23218) + | > loader_time: 0.08480 (0.13083) + + + --> STEP: 139/234 -- GLOBAL_STEP: 4585 + | > loss: 0.70069 (0.86158) + | > log_mle: 0.08829 (0.22807) + | > loss_dur: 0.61240 (0.63351) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.12071 (2.88551) + | > current_lr: 0.00000 + | > step_time: 2.39300 (3.20075) + | > loader_time: 0.00310 (0.12748) + + + --> STEP: 144/234 -- GLOBAL_STEP: 4590 + | > loss: 0.71004 (0.85722) + | > log_mle: 0.10740 (0.22464) + | > loss_dur: 0.60264 (0.63258) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.17864 (3.02798) + | > current_lr: 0.00000 + | > step_time: 5.18360 (3.20295) + | > loader_time: 0.10650 (0.12492) + + + --> STEP: 149/234 -- GLOBAL_STEP: 4595 + | > loss: 0.69996 (0.85238) + | > log_mle: 0.07361 (0.22080) + | > loss_dur: 0.62635 (0.63159) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.84226 (3.16729) + | > current_lr: 0.00000 + | > step_time: 1.79530 (3.23145) + | > loader_time: 0.08120 (0.12214) + + + --> STEP: 154/234 -- GLOBAL_STEP: 4600 + | > loss: 0.71172 (0.84791) + | > log_mle: 0.10388 (0.21676) + | > loss_dur: 0.60784 (0.63115) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.31109 (3.34223) + | > current_lr: 0.00000 + | > step_time: 2.38270 (3.19131) + | > loader_time: 0.09440 (0.12047) + + + --> STEP: 159/234 -- GLOBAL_STEP: 4605 + | > loss: 0.74445 (0.84392) + | > log_mle: 0.09017 (0.21291) + | > loss_dur: 0.65429 (0.63101) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.96214 (3.50325) + | > current_lr: 0.00000 + | > step_time: 1.80170 (3.22565) + | > loader_time: 0.06490 (0.11899) + + + --> STEP: 164/234 -- GLOBAL_STEP: 4610 + | > loss: 0.69794 (0.83967) + | > log_mle: 0.09010 (0.20917) + | > loss_dur: 0.60784 (0.63050) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.06794 (3.67146) + | > current_lr: 0.00000 + | > step_time: 1.61820 (3.17918) + | > loader_time: 0.09620 (0.11664) + + + --> STEP: 169/234 -- GLOBAL_STEP: 4615 + | > loss: 0.73460 (0.83623) + | > log_mle: 0.09486 (0.20537) + | > loss_dur: 0.63974 (0.63087) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.15066 (3.74519) + | > current_lr: 0.00000 + | > step_time: 1.21090 (3.17763) + | > loader_time: 0.09460 (0.11454) + + + --> STEP: 174/234 -- GLOBAL_STEP: 4620 + | > loss: 0.65546 (0.83209) + | > log_mle: 0.02077 (0.20074) + | > loss_dur: 0.63469 (0.63136) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 14.41288 (3.88177) + | > current_lr: 0.00000 + | > step_time: 2.09300 (3.17007) + | > loader_time: 0.01910 (0.11235) + + + --> STEP: 179/234 -- GLOBAL_STEP: 4625 + | > loss: 0.69413 (0.82838) + | > log_mle: 0.03813 (0.19663) + | > loss_dur: 0.65600 (0.63175) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.51540 (4.06355) + | > current_lr: 0.00000 + | > step_time: 1.00010 (3.15828) + | > loader_time: 0.03910 (0.11094) + + + --> STEP: 184/234 -- GLOBAL_STEP: 4630 + | > loss: 0.69685 (0.82510) + | > log_mle: 0.05389 (0.19284) + | > loss_dur: 0.64296 (0.63225) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.28143 (4.19642) + | > current_lr: 0.00000 + | > step_time: 2.51990 (3.15776) + | > loader_time: 0.00420 (0.10894) + + + --> STEP: 189/234 -- GLOBAL_STEP: 4635 + | > loss: 0.69957 (0.82154) + | > log_mle: 0.06133 (0.18898) + | > loss_dur: 0.63824 (0.63255) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.14302 (4.30738) + | > current_lr: 0.00000 + | > step_time: 4.59770 (3.18156) + | > loader_time: 0.00630 (0.10882) + + + --> STEP: 194/234 -- GLOBAL_STEP: 4640 + | > loss: 0.67521 (0.81784) + | > log_mle: 0.02943 (0.18508) + | > loss_dur: 0.64577 (0.63276) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.09049 (4.39256) + | > current_lr: 0.00000 + | > step_time: 3.07090 (3.17308) + | > loader_time: 0.29420 (0.10860) + + + --> STEP: 199/234 -- GLOBAL_STEP: 4645 + | > loss: 0.64993 (0.81453) + | > log_mle: 0.02296 (0.18146) + | > loss_dur: 0.62698 (0.63307) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.44300 (4.50445) + | > current_lr: 0.00000 + | > step_time: 4.80360 (3.20868) + | > loader_time: 0.02480 (0.10732) + + + --> STEP: 204/234 -- GLOBAL_STEP: 4650 + | > loss: 0.64618 (0.81111) + | > log_mle: 0.00126 (0.17784) + | > loss_dur: 0.64492 (0.63327) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.16177 (4.54987) + | > current_lr: 0.00000 + | > step_time: 3.69320 (3.20245) + | > loader_time: 0.02120 (0.10578) + + + --> STEP: 209/234 -- GLOBAL_STEP: 4655 + | > loss: 0.68848 (0.80777) + | > log_mle: 0.03608 (0.17402) + | > loss_dur: 0.65240 (0.63375) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.60918 (4.62687) + | > current_lr: 0.00000 + | > step_time: 3.32140 (3.22993) + | > loader_time: 0.09170 (0.10564) + + + --> STEP: 214/234 -- GLOBAL_STEP: 4660 + | > loss: 0.65256 (0.80425) + | > log_mle: 0.00841 (0.16955) + | > loss_dur: 0.64414 (0.63470) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.52861 (4.73569) + | > current_lr: 0.00000 + | > step_time: 4.20100 (3.26623) + | > loader_time: 0.08980 (0.10536) + + + --> STEP: 219/234 -- GLOBAL_STEP: 4665 + | > loss: 0.61957 (0.80071) + | > log_mle: -0.06784 (0.16525) + | > loss_dur: 0.68741 (0.63546) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.22015 (4.83053) + | > current_lr: 0.00000 + | > step_time: 2.29030 (3.29377) + | > loader_time: 0.00930 (0.10353) + + + --> STEP: 224/234 -- GLOBAL_STEP: 4670 + | > loss: 0.66925 (0.79771) + | > log_mle: -0.03119 (0.16124) + | > loss_dur: 0.70044 (0.63647) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.92868 (4.91428) + | > current_lr: 0.00000 + | > step_time: 0.25950 (3.24939) + | > loader_time: 0.00500 (0.10133) + + + --> STEP: 229/234 -- GLOBAL_STEP: 4675 + | > loss: 0.73254 (0.79475) + | > log_mle: -0.04705 (0.15681) + | > loss_dur: 0.77958 (0.63793) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.56522 (5.02470) + | > current_lr: 0.00000 + | > step_time: 0.26680 (3.18410) + | > loader_time: 0.00340 (0.09920) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.51050 (+0.33403) + | > avg_loss: 0.73311 (-0.05191) + | > avg_log_mle: 0.08716 (-0.01244) + | > avg_loss_dur: 0.64595 (-0.03947) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_4680.pth + + > EPOCH: 20/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 13:08:36)  + + --> STEP: 0/234 -- GLOBAL_STEP: 4680 + | > loss: 1.20580 (1.20580) + | > log_mle: 0.28608 (0.28608) + | > loss_dur: 0.91973 (0.91973) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.78324 (2.78324) + | > current_lr: 0.00000 + | > step_time: 4.09130 (4.09125) + | > loader_time: 15.54890 (15.54894) + + + --> STEP: 5/234 -- GLOBAL_STEP: 4685 + | > loss: 1.14450 (1.20186) + | > log_mle: 0.27880 (0.28809) + | > loss_dur: 0.86570 (0.91377) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.01568 (2.60045) + | > current_lr: 0.00000 + | > step_time: 4.63300 (3.30722) + | > loader_time: 0.08400 (0.03891) + + + --> STEP: 10/234 -- GLOBAL_STEP: 4690 + | > loss: 0.95785 (1.12292) + | > log_mle: 0.26108 (0.27450) + | > loss_dur: 0.69677 (0.84842) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.42209 (2.43565) + | > current_lr: 0.00000 + | > step_time: 6.30950 (4.29522) + | > loader_time: 0.00190 (0.03487) + + + --> STEP: 15/234 -- GLOBAL_STEP: 4695 + | > loss: 0.95752 (1.06553) + | > log_mle: 0.26001 (0.27147) + | > loss_dur: 0.69751 (0.79405) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.67508 (2.23280) + | > current_lr: 0.00000 + | > step_time: 6.29010 (4.26929) + | > loader_time: 0.00120 (0.09396) + + + --> STEP: 20/234 -- GLOBAL_STEP: 4700 + | > loss: 0.90062 (1.03287) + | > log_mle: 0.26314 (0.26952) + | > loss_dur: 0.63749 (0.76335) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.68961 (2.10946) + | > current_lr: 0.00000 + | > step_time: 7.90090 (4.66701) + | > loader_time: 0.10070 (0.09048) + + + --> STEP: 25/234 -- GLOBAL_STEP: 4705 + | > loss: 0.85701 (1.00296) + | > log_mle: 0.26272 (0.26719) + | > loss_dur: 0.59428 (0.73576) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.57037 (2.00951) + | > current_lr: 0.00000 + | > step_time: 5.49800 (4.81145) + | > loader_time: 0.01330 (0.08442) + + + --> STEP: 30/234 -- GLOBAL_STEP: 4710 + | > loss: 0.82065 (0.97808) + | > log_mle: 0.24034 (0.26453) + | > loss_dur: 0.58031 (0.71356) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.52566 (1.92813) + | > current_lr: 0.00000 + | > step_time: 3.69360 (4.69911) + | > loader_time: 0.00470 (0.07677) + + + --> STEP: 35/234 -- GLOBAL_STEP: 4715 + | > loss: 0.84681 (0.95999) + | > log_mle: 0.23746 (0.26129) + | > loss_dur: 0.60934 (0.69871) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.45871 (1.99166) + | > current_lr: 0.00000 + | > step_time: 3.89740 (4.71016) + | > loader_time: 0.00110 (0.08014) + + + --> STEP: 40/234 -- GLOBAL_STEP: 4720 + | > loss: 0.86998 (0.94622) + | > log_mle: 0.25839 (0.25925) + | > loss_dur: 0.61158 (0.68697) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.51294 (1.94359) + | > current_lr: 0.00000 + | > step_time: 7.64660 (4.73139) + | > loader_time: 0.10480 (0.08175) + + + --> STEP: 45/234 -- GLOBAL_STEP: 4725 + | > loss: 0.85012 (0.93457) + | > log_mle: 0.24283 (0.25764) + | > loss_dur: 0.60729 (0.67694) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.89210 (1.90369) + | > current_lr: 0.00000 + | > step_time: 3.41730 (4.48048) + | > loader_time: 0.08770 (0.07493) + + + --> STEP: 50/234 -- GLOBAL_STEP: 4730 + | > loss: 0.81219 (0.92150) + | > log_mle: 0.24584 (0.25615) + | > loss_dur: 0.56635 (0.66535) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.37703 (1.85290) + | > current_lr: 0.00000 + | > step_time: 2.19820 (4.29998) + | > loader_time: 0.00190 (0.06771) + + + --> STEP: 55/234 -- GLOBAL_STEP: 4735 + | > loss: 0.79599 (0.91079) + | > log_mle: 0.22583 (0.25465) + | > loss_dur: 0.57016 (0.65614) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.48799 (1.82217) + | > current_lr: 0.00000 + | > step_time: 4.80440 (4.10765) + | > loader_time: 0.02210 (0.06506) + + + --> STEP: 60/234 -- GLOBAL_STEP: 4740 + | > loss: 0.77176 (0.90175) + | > log_mle: 0.21615 (0.25302) + | > loss_dur: 0.55561 (0.64873) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.60869 (1.79279) + | > current_lr: 0.00000 + | > step_time: 1.20440 (3.99701) + | > loader_time: 0.00280 (0.05988) + + + --> STEP: 65/234 -- GLOBAL_STEP: 4745 + | > loss: 0.79280 (0.89382) + | > log_mle: 0.23941 (0.25142) + | > loss_dur: 0.55339 (0.64240) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.42741 (1.79210) + | > current_lr: 0.00000 + | > step_time: 1.38920 (3.84965) + | > loader_time: 0.00210 (0.05968) + + + --> STEP: 70/234 -- GLOBAL_STEP: 4750 + | > loss: 0.78285 (0.88671) + | > log_mle: 0.22607 (0.25034) + | > loss_dur: 0.55678 (0.63637) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.36892 (1.81288) + | > current_lr: 0.00000 + | > step_time: 2.89710 (3.76464) + | > loader_time: 0.00430 (0.06219) + + + --> STEP: 75/234 -- GLOBAL_STEP: 4755 + | > loss: 0.77109 (0.87934) + | > log_mle: 0.22647 (0.24847) + | > loss_dur: 0.54462 (0.63087) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.01133 (1.83288) + | > current_lr: 0.00000 + | > step_time: 2.83410 (3.78661) + | > loader_time: 0.00760 (0.06205) + + + --> STEP: 80/234 -- GLOBAL_STEP: 4760 + | > loss: 0.76935 (0.87258) + | > log_mle: 0.23767 (0.24721) + | > loss_dur: 0.53168 (0.62537) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.35567 (1.81961) + | > current_lr: 0.00000 + | > step_time: 3.38420 (3.68594) + | > loader_time: 0.00320 (0.05961) + + + --> STEP: 85/234 -- GLOBAL_STEP: 4765 + | > loss: 0.77553 (0.86664) + | > log_mle: 0.22432 (0.24562) + | > loss_dur: 0.55121 (0.62102) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.53586 (1.82813) + | > current_lr: 0.00000 + | > step_time: 0.99940 (3.61545) + | > loader_time: 0.07830 (0.05931) + + + --> STEP: 90/234 -- GLOBAL_STEP: 4770 + | > loss: 0.71659 (0.85954) + | > log_mle: 0.19389 (0.24295) + | > loss_dur: 0.52270 (0.61659) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.11106 (1.88885) + | > current_lr: 0.00000 + | > step_time: 3.89190 (3.56449) + | > loader_time: 0.01040 (0.05632) + + + --> STEP: 95/234 -- GLOBAL_STEP: 4775 + | > loss: 0.71845 (0.85251) + | > log_mle: 0.12554 (0.23903) + | > loss_dur: 0.59291 (0.61348) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.48023 (2.01946) + | > current_lr: 0.00000 + | > step_time: 4.80910 (3.57076) + | > loader_time: 0.00350 (0.05544) + + + --> STEP: 100/234 -- GLOBAL_STEP: 4780 + | > loss: 0.75310 (0.84691) + | > log_mle: 0.18305 (0.23650) + | > loss_dur: 0.57006 (0.61042) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.26109 (2.08282) + | > current_lr: 0.00000 + | > step_time: 1.00670 (3.55825) + | > loader_time: 0.00230 (0.05470) + + + --> STEP: 105/234 -- GLOBAL_STEP: 4785 + | > loss: 0.74357 (0.84155) + | > log_mle: 0.21384 (0.23349) + | > loss_dur: 0.52973 (0.60807) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.83549 (2.16570) + | > current_lr: 0.00000 + | > step_time: 4.70260 (3.54799) + | > loader_time: 0.10110 (0.05395) + + + --> STEP: 110/234 -- GLOBAL_STEP: 4790 + | > loss: 0.74927 (0.83703) + | > log_mle: 0.18539 (0.23076) + | > loss_dur: 0.56388 (0.60627) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.23654 (2.31788) + | > current_lr: 0.00000 + | > step_time: 5.68840 (3.53133) + | > loader_time: 0.10610 (0.05462) + + + --> STEP: 115/234 -- GLOBAL_STEP: 4795 + | > loss: 0.75673 (0.83225) + | > log_mle: 0.17164 (0.22758) + | > loss_dur: 0.58509 (0.60467) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.89514 (2.44266) + | > current_lr: 0.00000 + | > step_time: 2.12550 (3.46604) + | > loader_time: 0.08830 (0.05383) + + + --> STEP: 120/234 -- GLOBAL_STEP: 4800 + | > loss: 0.68498 (0.82769) + | > log_mle: 0.12925 (0.22484) + | > loss_dur: 0.55572 (0.60285) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.70863 (2.53673) + | > current_lr: 0.00000 + | > step_time: 2.52180 (3.46845) + | > loader_time: 0.08280 (0.05540) + + + --> STEP: 125/234 -- GLOBAL_STEP: 4805 + | > loss: 0.69510 (0.82341) + | > log_mle: 0.14403 (0.22312) + | > loss_dur: 0.55107 (0.60029) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.39821 (2.61841) + | > current_lr: 0.00000 + | > step_time: 5.22510 (3.48197) + | > loader_time: 0.19520 (0.05567) + + + --> STEP: 130/234 -- GLOBAL_STEP: 4810 + | > loss: 0.70252 (0.81888) + | > log_mle: 0.13258 (0.21989) + | > loss_dur: 0.56995 (0.59899) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.26369 (2.72056) + | > current_lr: 0.00000 + | > step_time: 2.01570 (3.44132) + | > loader_time: 0.00670 (0.05434) + + + --> STEP: 135/234 -- GLOBAL_STEP: 4815 + | > loss: 0.71409 (0.81457) + | > log_mle: 0.18503 (0.21687) + | > loss_dur: 0.52905 (0.59769) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.50278 (2.90072) + | > current_lr: 0.00000 + | > step_time: 3.89340 (3.45917) + | > loader_time: 0.00320 (0.05429) + + + --> STEP: 140/234 -- GLOBAL_STEP: 4820 + | > loss: 0.71738 (0.81070) + | > log_mle: 0.16293 (0.21346) + | > loss_dur: 0.55445 (0.59724) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 4.62648 (3.09813) + | > current_lr: 0.00000 + | > step_time: 6.20150 (3.46918) + | > loader_time: 0.00370 (0.05313) + + + --> STEP: 145/234 -- GLOBAL_STEP: 4825 + | > loss: 0.67316 (0.80608) + | > log_mle: 0.08348 (0.20945) + | > loss_dur: 0.58968 (0.59663) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.09891 (3.28346) + | > current_lr: 0.00000 + | > step_time: 2.78500 (3.54219) + | > loader_time: 0.00590 (0.05266) + + + --> STEP: 150/234 -- GLOBAL_STEP: 4830 + | > loss: 0.70469 (0.80149) + | > log_mle: 0.09942 (0.20571) + | > loss_dur: 0.60528 (0.59578) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.45841 (3.41296) + | > current_lr: 0.00000 + | > step_time: 2.19830 (3.54428) + | > loader_time: 0.08860 (0.05271) + + + --> STEP: 155/234 -- GLOBAL_STEP: 4835 + | > loss: 0.65421 (0.79676) + | > log_mle: 0.04488 (0.20128) + | > loss_dur: 0.60933 (0.59548) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 11.22688 (3.57512) + | > current_lr: 0.00000 + | > step_time: 7.31200 (3.61519) + | > loader_time: 0.00490 (0.05295) + + + --> STEP: 160/234 -- GLOBAL_STEP: 4840 + | > loss: 0.61853 (0.79266) + | > log_mle: 0.04612 (0.19743) + | > loss_dur: 0.57241 (0.59524) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.33344 (3.71673) + | > current_lr: 0.00000 + | > step_time: 4.29620 (3.65046) + | > loader_time: 0.10520 (0.05439) + + + --> STEP: 165/234 -- GLOBAL_STEP: 4845 + | > loss: 0.65032 (0.78871) + | > log_mle: 0.05093 (0.19370) + | > loss_dur: 0.59939 (0.59502) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.85115 (3.83735) + | > current_lr: 0.00000 + | > step_time: 3.30040 (3.64130) + | > loader_time: 0.08450 (0.05483) + + + --> STEP: 170/234 -- GLOBAL_STEP: 4850 + | > loss: 0.65935 (0.78529) + | > log_mle: 0.02619 (0.18975) + | > loss_dur: 0.63317 (0.59554) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.24514 (4.00082) + | > current_lr: 0.00000 + | > step_time: 4.11170 (3.67913) + | > loader_time: 0.09300 (0.05538) + + + --> STEP: 175/234 -- GLOBAL_STEP: 4855 + | > loss: 0.64670 (0.78096) + | > log_mle: 0.04364 (0.18521) + | > loss_dur: 0.60306 (0.59575) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.95162 (4.21537) + | > current_lr: 0.00000 + | > step_time: 2.39490 (3.69795) + | > loader_time: 0.17390 (0.05699) + + + --> STEP: 180/234 -- GLOBAL_STEP: 4860 + | > loss: 0.63820 (0.77721) + | > log_mle: 0.03212 (0.18101) + | > loss_dur: 0.60608 (0.59621) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.82429 (4.30450) + | > current_lr: 0.00000 + | > step_time: 4.00520 (3.74973) + | > loader_time: 0.08620 (0.05858) + + + --> STEP: 185/234 -- GLOBAL_STEP: 4865 + | > loss: 0.65595 (0.77398) + | > log_mle: 0.01714 (0.17710) + | > loss_dur: 0.63881 (0.59688) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.69023 (4.42938) + | > current_lr: 0.00000 + | > step_time: 2.01740 (3.75820) + | > loader_time: 0.06690 (0.05806) + + + --> STEP: 190/234 -- GLOBAL_STEP: 4870 + | > loss: 0.62012 (0.77021) + | > log_mle: 0.02967 (0.17329) + | > loss_dur: 0.59045 (0.59692) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 14.31369 (4.57729) + | > current_lr: 0.00000 + | > step_time: 1.67120 (3.80572) + | > loader_time: 0.00320 (0.05764) + + + --> STEP: 195/234 -- GLOBAL_STEP: 4875 + | > loss: 0.65479 (0.76666) + | > log_mle: 0.02592 (0.16937) + | > loss_dur: 0.62888 (0.59729) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 14.45292 (4.82366) + | > current_lr: 0.00000 + | > step_time: 4.40910 (3.81732) + | > loader_time: 0.08600 (0.05865) + + + --> STEP: 200/234 -- GLOBAL_STEP: 4880 + | > loss: 0.63529 (0.76328) + | > log_mle: 0.01999 (0.16572) + | > loss_dur: 0.61530 (0.59756) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.52349 (5.03647) + | > current_lr: 0.00000 + | > step_time: 6.60740 (3.87138) + | > loader_time: 0.18720 (0.05960) + + + --> STEP: 205/234 -- GLOBAL_STEP: 4885 + | > loss: 0.64066 (0.75986) + | > log_mle: 0.02246 (0.16208) + | > loss_dur: 0.61820 (0.59777) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.26353 (5.15943) + | > current_lr: 0.00000 + | > step_time: 8.80910 (3.92764) + | > loader_time: 0.05250 (0.06083) + + + --> STEP: 210/234 -- GLOBAL_STEP: 4890 + | > loss: 0.59065 (0.75619) + | > log_mle: -0.04178 (0.15793) + | > loss_dur: 0.63243 (0.59826) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 11.80491 (5.28703) + | > current_lr: 0.00000 + | > step_time: 3.31140 (3.93532) + | > loader_time: 0.09130 (0.06126) + + + --> STEP: 215/234 -- GLOBAL_STEP: 4895 + | > loss: 0.61023 (0.75270) + | > log_mle: 0.00489 (0.15364) + | > loss_dur: 0.60534 (0.59905) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.22812 (5.48040) + | > current_lr: 0.00000 + | > step_time: 11.30570 (3.99865) + | > loader_time: 0.08930 (0.06432) + + + --> STEP: 220/234 -- GLOBAL_STEP: 4900 + | > loss: 0.59595 (0.74912) + | > log_mle: -0.03618 (0.14915) + | > loss_dur: 0.63213 (0.59997) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 9.83236 (5.64911) + | > current_lr: 0.00000 + | > step_time: 2.01210 (4.04096) + | > loader_time: 0.08220 (0.06549) + + + --> STEP: 225/234 -- GLOBAL_STEP: 4905 + | > loss: 0.55264 (0.74594) + | > log_mle: -0.08701 (0.14490) + | > loss_dur: 0.63965 (0.60104) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 15.75383 (5.81707) + | > current_lr: 0.00000 + | > step_time: 0.24320 (3.97256) + | > loader_time: 0.00300 (0.06451) + + + --> STEP: 230/234 -- GLOBAL_STEP: 4910 + | > loss: 0.63090 (0.74317) + | > log_mle: -0.12417 (0.14033) + | > loss_dur: 0.75507 (0.60284) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 20.43930 (6.06382) + | > current_lr: 0.00000 + | > step_time: 0.25420 (3.89168) + | > loader_time: 0.00350 (0.06319) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.99949 (+1.48899) + | > avg_loss: 0.66982 (-0.06329) + | > avg_log_mle: 0.06065 (-0.02651) + | > avg_loss_dur: 0.60917 (-0.03678) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_4914.pth + + > EPOCH: 21/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 13:24:59)  + + --> STEP: 1/234 -- GLOBAL_STEP: 4915 + | > loss: 1.16697 (1.16697) + | > log_mle: 0.28942 (0.28942) + | > loss_dur: 0.87754 (0.87754) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.19049 (2.19049) + | > current_lr: 0.00001 + | > step_time: 1.49730 (1.49727) + | > loader_time: 0.00350 (0.00349) + + + --> STEP: 6/234 -- GLOBAL_STEP: 4920 + | > loss: 1.04326 (1.12092) + | > log_mle: 0.26580 (0.27181) + | > loss_dur: 0.77747 (0.84912) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.13185 (2.36614) + | > current_lr: 0.00001 + | > step_time: 9.01450 (3.65356) + | > loader_time: 0.09380 (0.03561) + + + --> STEP: 11/234 -- GLOBAL_STEP: 4925 + | > loss: 0.88044 (1.04699) + | > log_mle: 0.26011 (0.25954) + | > loss_dur: 0.62033 (0.78745) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.61139 (2.19296) + | > current_lr: 0.00001 + | > step_time: 3.48880 (5.54610) + | > loader_time: 0.00370 (0.06688) + + + --> STEP: 16/234 -- GLOBAL_STEP: 4930 + | > loss: 0.85793 (0.99908) + | > log_mle: 0.23745 (0.25565) + | > loss_dur: 0.62049 (0.74342) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.59157 (2.03104) + | > current_lr: 0.00001 + | > step_time: 10.60850 (5.63897) + | > loader_time: 0.59520 (0.09575) + + + --> STEP: 21/234 -- GLOBAL_STEP: 4935 + | > loss: 0.83996 (0.97026) + | > log_mle: 0.25624 (0.25514) + | > loss_dur: 0.58371 (0.71512) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.60214 (1.92671) + | > current_lr: 0.00001 + | > step_time: 3.08200 (5.11626) + | > loader_time: 0.00150 (0.09056) + + + --> STEP: 26/234 -- GLOBAL_STEP: 4940 + | > loss: 0.84407 (0.94477) + | > log_mle: 0.23977 (0.25246) + | > loss_dur: 0.60430 (0.69231) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.36330 (1.82575) + | > current_lr: 0.00001 + | > step_time: 2.29790 (5.01383) + | > loader_time: 0.00120 (0.08513) + + + --> STEP: 31/234 -- GLOBAL_STEP: 4945 + | > loss: 0.83275 (0.92378) + | > log_mle: 0.22811 (0.24972) + | > loss_dur: 0.60464 (0.67406) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.44644 (1.74941) + | > current_lr: 0.00001 + | > step_time: 3.61890 (5.09913) + | > loader_time: 0.07780 (0.08063) + + + --> STEP: 36/234 -- GLOBAL_STEP: 4950 + | > loss: 0.82455 (0.90851) + | > log_mle: 0.22780 (0.24676) + | > loss_dur: 0.59674 (0.66175) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.98254 (1.85635) + | > current_lr: 0.00001 + | > step_time: 6.19790 (5.03596) + | > loader_time: 0.10030 (0.07793) + + + --> STEP: 41/234 -- GLOBAL_STEP: 4955 + | > loss: 0.78183 (0.89452) + | > log_mle: 0.22242 (0.24479) + | > loss_dur: 0.55941 (0.64973) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.32757 (1.80023) + | > current_lr: 0.00001 + | > step_time: 6.39780 (4.76090) + | > loader_time: 0.11010 (0.07760) + + + --> STEP: 46/234 -- GLOBAL_STEP: 4960 + | > loss: 0.77415 (0.88429) + | > log_mle: 0.22976 (0.24351) + | > loss_dur: 0.54439 (0.64078) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.32931 (1.76279) + | > current_lr: 0.00001 + | > step_time: 6.88920 (4.78986) + | > loader_time: 0.10220 (0.08791) + + + --> STEP: 51/234 -- GLOBAL_STEP: 4965 + | > loss: 0.77921 (0.87217) + | > log_mle: 0.24256 (0.24240) + | > loss_dur: 0.53665 (0.62977) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.37472 (1.71479) + | > current_lr: 0.00001 + | > step_time: 2.69540 (4.57645) + | > loader_time: 0.00300 (0.08186) + + + --> STEP: 56/234 -- GLOBAL_STEP: 4970 + | > loss: 0.79579 (0.86322) + | > log_mle: 0.22592 (0.24068) + | > loss_dur: 0.56987 (0.62254) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.40949 (1.68740) + | > current_lr: 0.00001 + | > step_time: 2.60360 (4.36702) + | > loader_time: 0.00180 (0.07999) + + + --> STEP: 61/234 -- GLOBAL_STEP: 4975 + | > loss: 0.76595 (0.85417) + | > log_mle: 0.23222 (0.23922) + | > loss_dur: 0.53373 (0.61495) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.24363 (1.65772) + | > current_lr: 0.00001 + | > step_time: 4.32240 (4.23731) + | > loader_time: 0.00290 (0.07662) + + + --> STEP: 66/234 -- GLOBAL_STEP: 4980 + | > loss: 0.75655 (0.84696) + | > log_mle: 0.23475 (0.23772) + | > loss_dur: 0.52180 (0.60924) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.15211 (1.64893) + | > current_lr: 0.00001 + | > step_time: 1.40120 (4.14494) + | > loader_time: 0.07550 (0.07219) + + + --> STEP: 71/234 -- GLOBAL_STEP: 4985 + | > loss: 0.75999 (0.84076) + | > log_mle: 0.19333 (0.23610) + | > loss_dur: 0.56666 (0.60466) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 3.06163 (1.67856) + | > current_lr: 0.00001 + | > step_time: 2.40200 (4.00680) + | > loader_time: 0.08120 (0.06962) + + + --> STEP: 76/234 -- GLOBAL_STEP: 4990 + | > loss: 0.73929 (0.83319) + | > log_mle: 0.21542 (0.23459) + | > loss_dur: 0.52388 (0.59860) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 1.41094 (1.67244) + | > current_lr: 0.00001 + | > step_time: 3.21470 (3.95697) + | > loader_time: 0.00180 (0.06879) + + + --> STEP: 81/234 -- GLOBAL_STEP: 4995 + | > loss: 0.71292 (0.82652) + | > log_mle: 0.19628 (0.23316) + | > loss_dur: 0.51665 (0.59336) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.47669 (1.66750) + | > current_lr: 0.00001 + | > step_time: 4.80680 (3.99578) + | > loader_time: 0.02140 (0.06721) + + + --> STEP: 86/234 -- GLOBAL_STEP: 5000 + | > loss: 0.73457 (0.82088) + | > log_mle: 0.19553 (0.23161) + | > loss_dur: 0.53904 (0.58927) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.73806 (1.68050) + | > current_lr: 0.00001 + | > step_time: 1.10420 (3.87041) + | > loader_time: 0.08270 (0.06457) + + + --> STEP: 91/234 -- GLOBAL_STEP: 5005 + | > loss: 0.72407 (0.81395) + | > log_mle: 0.19533 (0.22901) + | > loss_dur: 0.52874 (0.58495) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.12178 (1.75636) + | > current_lr: 0.00001 + | > step_time: 2.59130 (3.79200) + | > loader_time: 0.00320 (0.06224) + + + --> STEP: 96/234 -- GLOBAL_STEP: 5010 + | > loss: 0.72109 (0.80700) + | > log_mle: 0.19932 (0.22515) + | > loss_dur: 0.52177 (0.58185) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.27350 (1.87769) + | > current_lr: 0.00001 + | > step_time: 5.19220 (3.72898) + | > loader_time: 0.10340 (0.06097) + + + --> STEP: 101/234 -- GLOBAL_STEP: 5015 + | > loss: 0.68105 (0.80121) + | > log_mle: 0.14918 (0.22219) + | > loss_dur: 0.53187 (0.57902) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 5.39885 (2.00627) + | > current_lr: 0.00001 + | > step_time: 1.21920 (3.66024) + | > loader_time: 0.00230 (0.06071) + + + --> STEP: 106/234 -- GLOBAL_STEP: 5020 + | > loss: 0.73503 (0.79664) + | > log_mle: 0.15267 (0.21924) + | > loss_dur: 0.58236 (0.57740) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.47933 (2.08742) + | > current_lr: 0.00001 + | > step_time: 1.39420 (3.59327) + | > loader_time: 0.00490 (0.05878) + + + --> STEP: 111/234 -- GLOBAL_STEP: 5025 + | > loss: 0.68401 (0.79170) + | > log_mle: 0.11643 (0.21620) + | > loss_dur: 0.56759 (0.57550) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.97989 (2.23098) + | > current_lr: 0.00001 + | > step_time: 6.41060 (3.63687) + | > loader_time: 0.09130 (0.05779) + + + --> STEP: 116/234 -- GLOBAL_STEP: 5030 + | > loss: 0.70657 (0.78721) + | > log_mle: 0.14526 (0.21332) + | > loss_dur: 0.56131 (0.57389) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.30880 (2.38533) + | > current_lr: 0.00001 + | > step_time: 3.89240 (3.61364) + | > loader_time: 0.00260 (0.05706) + + + --> STEP: 121/234 -- GLOBAL_STEP: 5035 + | > loss: 0.72648 (0.78303) + | > log_mle: 0.21606 (0.21121) + | > loss_dur: 0.51042 (0.57182) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 2.26498 (2.48728) + | > current_lr: 0.00001 + | > step_time: 2.28800 (3.64874) + | > loader_time: 0.00310 (0.05720) + + + --> STEP: 126/234 -- GLOBAL_STEP: 5040 + | > loss: 0.63814 (0.77823) + | > log_mle: 0.09796 (0.20855) + | > loss_dur: 0.54018 (0.56968) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.72672 (2.61536) + | > current_lr: 0.00001 + | > step_time: 3.69790 (3.58963) + | > loader_time: 0.00330 (0.05579) + + + --> STEP: 131/234 -- GLOBAL_STEP: 5045 + | > loss: 0.62403 (0.77356) + | > log_mle: 0.07369 (0.20514) + | > loss_dur: 0.55034 (0.56842) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.02162 (2.70275) + | > current_lr: 0.00001 + | > step_time: 1.89350 (3.55068) + | > loader_time: 0.00210 (0.05512) + + + --> STEP: 136/234 -- GLOBAL_STEP: 5050 + | > loss: 0.62736 (0.76918) + | > log_mle: 0.03503 (0.20184) + | > loss_dur: 0.59232 (0.56734) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.71860 (2.79141) + | > current_lr: 0.00001 + | > step_time: 1.89780 (3.52689) + | > loader_time: 0.00430 (0.05387) + + + --> STEP: 141/234 -- GLOBAL_STEP: 5055 + | > loss: 0.63615 (0.76543) + | > log_mle: 0.10220 (0.19892) + | > loss_dur: 0.53395 (0.56651) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.03701 (2.90712) + | > current_lr: 0.00001 + | > step_time: 5.40580 (3.51417) + | > loader_time: 0.07900 (0.05431) + + + --> STEP: 146/234 -- GLOBAL_STEP: 5060 + | > loss: 0.63560 (0.76089) + | > log_mle: 0.06404 (0.19466) + | > loss_dur: 0.57157 (0.56623) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.99066 (3.13991) + | > current_lr: 0.00001 + | > step_time: 2.61110 (3.53714) + | > loader_time: 0.09240 (0.05720) + + + --> STEP: 151/234 -- GLOBAL_STEP: 5065 + | > loss: 0.62298 (0.75616) + | > log_mle: 0.08739 (0.19108) + | > loss_dur: 0.53559 (0.56508) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.67749 (3.25984) + | > current_lr: 0.00001 + | > step_time: 6.70070 (3.57024) + | > loader_time: 0.08950 (0.05664) + + + --> STEP: 156/234 -- GLOBAL_STEP: 5070 + | > loss: 0.59101 (0.75123) + | > log_mle: 0.05984 (0.18647) + | > loss_dur: 0.53117 (0.56476) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.25047 (3.43802) + | > current_lr: 0.00001 + | > step_time: 4.49370 (3.64106) + | > loader_time: 0.40310 (0.05873) + + + --> STEP: 161/234 -- GLOBAL_STEP: 5075 + | > loss: 0.62798 (0.74724) + | > log_mle: 0.04575 (0.18252) + | > loss_dur: 0.58223 (0.56473) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.35965 (3.56680) + | > current_lr: 0.00001 + | > step_time: 3.11240 (3.61074) + | > loader_time: 0.06700 (0.05742) + + + --> STEP: 166/234 -- GLOBAL_STEP: 5080 + | > loss: 0.61980 (0.74329) + | > log_mle: 0.08599 (0.17904) + | > loss_dur: 0.53380 (0.56425) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.29428 (3.68148) + | > current_lr: 0.00001 + | > step_time: 3.00530 (3.66047) + | > loader_time: 0.09760 (0.06119) + + + --> STEP: 171/234 -- GLOBAL_STEP: 5085 + | > loss: 0.56178 (0.73955) + | > log_mle: 0.00151 (0.17458) + | > loss_dur: 0.56027 (0.56497) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 10.12703 (3.88063) + | > current_lr: 0.00001 + | > step_time: 3.79130 (3.69503) + | > loader_time: 0.40440 (0.06333) + + + --> STEP: 176/234 -- GLOBAL_STEP: 5090 + | > loss: 0.58957 (0.73531) + | > log_mle: 0.02885 (0.17018) + | > loss_dur: 0.56072 (0.56512) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 8.09669 (4.13010) + | > current_lr: 0.00001 + | > step_time: 3.28880 (3.75714) + | > loader_time: 0.00490 (0.06434) + + + --> STEP: 181/234 -- GLOBAL_STEP: 5095 + | > loss: 0.63719 (0.73183) + | > log_mle: 0.07539 (0.16622) + | > loss_dur: 0.56179 (0.56561) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.11929 (4.24948) + | > current_lr: 0.00001 + | > step_time: 4.78240 (3.79477) + | > loader_time: 0.10530 (0.06467) + + + --> STEP: 186/234 -- GLOBAL_STEP: 5100 + | > loss: 0.60894 (0.72837) + | > log_mle: 0.04559 (0.16213) + | > loss_dur: 0.56335 (0.56624) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 12.14041 (4.42064) + | > current_lr: 0.00001 + | > step_time: 8.29380 (3.82429) + | > loader_time: 0.09260 (0.06556) + + + --> STEP: 191/234 -- GLOBAL_STEP: 5105 + | > loss: 0.60985 (0.72460) + | > log_mle: 0.03820 (0.15829) + | > loss_dur: 0.57165 (0.56632) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.08396 (4.57027) + | > current_lr: 0.00001 + | > step_time: 6.38640 (3.90219) + | > loader_time: 0.10990 (0.06751) + + + --> STEP: 196/234 -- GLOBAL_STEP: 5110 + | > loss: 0.63108 (0.72114) + | > log_mle: 0.04071 (0.15434) + | > loss_dur: 0.59037 (0.56680) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 7.58041 (4.72135) + | > current_lr: 0.00001 + | > step_time: 7.59890 (3.95880) + | > loader_time: 0.00360 (0.06771) + + + --> STEP: 201/234 -- GLOBAL_STEP: 5115 + | > loss: 0.61608 (0.71757) + | > log_mle: 0.05685 (0.15074) + | > loss_dur: 0.55923 (0.56683) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 6.68941 (4.84504) + | > current_lr: 0.00001 + | > step_time: 15.69820 (4.07019) + | > loader_time: 0.40030 (0.07050) + + + --> STEP: 206/234 -- GLOBAL_STEP: 5120 + | > loss: 0.54830 (0.71389) + | > log_mle: -0.01687 (0.14675) + | > loss_dur: 0.56518 (0.56714) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 16.63022 (5.08158) + | > current_lr: 0.00001 + | > step_time: 5.20020 (4.10484) + | > loader_time: 0.00280 (0.07189) + + + --> STEP: 211/234 -- GLOBAL_STEP: 5125 + | > loss: 0.56538 (0.71040) + | > log_mle: -0.07764 (0.14231) + | > loss_dur: 0.64302 (0.56809) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 18.07763 (5.31417) + | > current_lr: 0.00001 + | > step_time: 11.50780 (4.15402) + | > loader_time: 0.07540 (0.07348) + + + --> STEP: 216/234 -- GLOBAL_STEP: 5130 + | > loss: 0.53774 (0.70678) + | > log_mle: -0.06414 (0.13809) + | > loss_dur: 0.60188 (0.56869) + | > amp_scaler: 32768.00000 (32768.00000) + | > grad_norm: 15.17494 (5.55097) + | > current_lr: 0.00001 + | > step_time: 5.69890 (4.20479) + | > loader_time: 0.20070 (0.07482) + + + --> STEP: 221/234 -- GLOBAL_STEP: 5135 + | > loss: 0.59032 (0.70334) + | > log_mle: -0.00225 (0.13388) + | > loss_dur: 0.59257 (0.56947) + | > amp_scaler: 16384.00000 (32545.59276) + | > grad_norm: 10.83694 (5.66516) + | > current_lr: 0.00001 + | > step_time: 1.89660 (4.19112) + | > loader_time: 0.00430 (0.07582) + + + --> STEP: 226/234 -- GLOBAL_STEP: 5140 + | > loss: 0.53003 (0.69982) + | > log_mle: -0.08269 (0.12926) + | > loss_dur: 0.61272 (0.57056) + | > amp_scaler: 16384.00000 (32188.03540) + | > grad_norm: 15.85405 (5.87700) + | > current_lr: 0.00001 + | > step_time: 0.24290 (4.10942) + | > loader_time: 0.00410 (0.07423) + + + --> STEP: 231/234 -- GLOBAL_STEP: 5145 + | > loss: 0.62027 (0.69742) + | > log_mle: -0.13361 (0.12443) + | > loss_dur: 0.75388 (0.57299) + | > amp_scaler: 16384.00000 (31845.95671) + | > grad_norm: 14.33134 (6.05518) + | > current_lr: 0.00001 + | > step_time: 0.27880 (4.02618) + | > loader_time: 0.00360 (0.07270) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03190 (-1.96759) + | > avg_loss: 0.62503 (-0.04479) + | > avg_log_mle: 0.04727 (-0.01338) + | > avg_loss_dur: 0.57776 (-0.03141) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_5148.pth + + > EPOCH: 22/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 13:42:10)  + + --> STEP: 2/234 -- GLOBAL_STEP: 5150 + | > loss: 1.15672 (1.13626) + | > log_mle: 0.28863 (0.28191) + | > loss_dur: 0.86810 (0.85435) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.97904 (2.01905) + | > current_lr: 0.00001 + | > step_time: 6.40460 (5.99687) + | > loader_time: 0.18920 (0.09570) + + + --> STEP: 7/234 -- GLOBAL_STEP: 5155 + | > loss: 0.95650 (1.04835) + | > log_mle: 0.22819 (0.25348) + | > loss_dur: 0.72831 (0.79487) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.91759 (2.31909) + | > current_lr: 0.00001 + | > step_time: 2.69950 (5.18464) + | > loader_time: 0.00270 (0.05708) + + + --> STEP: 12/234 -- GLOBAL_STEP: 5160 + | > loss: 0.84539 (0.97848) + | > log_mle: 0.23571 (0.24462) + | > loss_dur: 0.60968 (0.73387) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.66534 (2.10961) + | > current_lr: 0.00001 + | > step_time: 6.68530 (5.68300) + | > loader_time: 0.08290 (0.06653) + + + --> STEP: 17/234 -- GLOBAL_STEP: 5165 + | > loss: 0.84785 (0.93971) + | > log_mle: 0.24450 (0.24174) + | > loss_dur: 0.60335 (0.69797) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.46996 (1.93398) + | > current_lr: 0.00001 + | > step_time: 1.80770 (5.98285) + | > loader_time: 0.00150 (0.05337) + + + --> STEP: 22/234 -- GLOBAL_STEP: 5170 + | > loss: 0.78862 (0.91243) + | > log_mle: 0.21910 (0.24013) + | > loss_dur: 0.56951 (0.67230) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.30700 (1.81550) + | > current_lr: 0.00001 + | > step_time: 1.86740 (4.97686) + | > loader_time: 0.00150 (0.04515) + + + --> STEP: 27/234 -- GLOBAL_STEP: 5175 + | > loss: 0.74122 (0.88842) + | > log_mle: 0.21789 (0.23772) + | > loss_dur: 0.52333 (0.65070) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.24669 (1.71412) + | > current_lr: 0.00001 + | > step_time: 2.00330 (4.64436) + | > loader_time: 0.00460 (0.04469) + + + --> STEP: 32/234 -- GLOBAL_STEP: 5180 + | > loss: 0.72772 (0.86806) + | > log_mle: 0.20056 (0.23469) + | > loss_dur: 0.52715 (0.63338) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.26246 (1.63800) + | > current_lr: 0.00001 + | > step_time: 1.49050 (4.16890) + | > loader_time: 0.00140 (0.03801) + + + --> STEP: 37/234 -- GLOBAL_STEP: 5185 + | > loss: 0.70050 (0.85372) + | > log_mle: 0.21368 (0.23238) + | > loss_dur: 0.48682 (0.62134) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.29566 (1.71725) + | > current_lr: 0.00001 + | > step_time: 1.39930 (3.88705) + | > loader_time: 0.00270 (0.03748) + + + --> STEP: 42/234 -- GLOBAL_STEP: 5190 + | > loss: 0.76829 (0.84299) + | > log_mle: 0.24143 (0.23119) + | > loss_dur: 0.52686 (0.61180) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.31085 (1.66531) + | > current_lr: 0.00001 + | > step_time: 3.39230 (3.74314) + | > loader_time: 0.00130 (0.03760) + + + --> STEP: 47/234 -- GLOBAL_STEP: 5195 + | > loss: 0.71427 (0.83236) + | > log_mle: 0.21761 (0.22937) + | > loss_dur: 0.49666 (0.60299) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.10986 (1.63214) + | > current_lr: 0.00001 + | > step_time: 2.09810 (3.55298) + | > loader_time: 0.00180 (0.03776) + + + --> STEP: 52/234 -- GLOBAL_STEP: 5200 + | > loss: 0.72863 (0.82138) + | > log_mle: 0.22681 (0.22848) + | > loss_dur: 0.50183 (0.59290) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.73481 (1.60067) + | > current_lr: 0.00001 + | > step_time: 2.70040 (3.39800) + | > loader_time: 0.00250 (0.03595) + + + --> STEP: 57/234 -- GLOBAL_STEP: 5205 + | > loss: 0.74824 (0.81351) + | > log_mle: 0.22784 (0.22683) + | > loss_dur: 0.52040 (0.58668) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.12413 (1.56847) + | > current_lr: 0.00001 + | > step_time: 2.89700 (3.33653) + | > loader_time: 0.00570 (0.03606) + + + --> STEP: 62/234 -- GLOBAL_STEP: 5210 + | > loss: 0.69962 (0.80438) + | > log_mle: 0.17166 (0.22447) + | > loss_dur: 0.52796 (0.57991) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.61058 (1.56532) + | > current_lr: 0.00001 + | > step_time: 2.09990 (3.29523) + | > loader_time: 0.00400 (0.03473) + + + --> STEP: 67/234 -- GLOBAL_STEP: 5215 + | > loss: 0.67537 (0.79724) + | > log_mle: 0.18554 (0.22323) + | > loss_dur: 0.48983 (0.57400) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.09892 (1.55036) + | > current_lr: 0.00001 + | > step_time: 2.59670 (3.31535) + | > loader_time: 0.00520 (0.03390) + + + --> STEP: 72/234 -- GLOBAL_STEP: 5220 + | > loss: 0.67128 (0.79133) + | > log_mle: 0.20267 (0.22188) + | > loss_dur: 0.46862 (0.56944) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.98679 (1.57397) + | > current_lr: 0.00001 + | > step_time: 4.09800 (3.24772) + | > loader_time: 0.09030 (0.03498) + + + --> STEP: 77/234 -- GLOBAL_STEP: 5225 + | > loss: 0.68558 (0.78459) + | > log_mle: 0.18663 (0.22019) + | > loss_dur: 0.49895 (0.56439) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.64278 (1.61204) + | > current_lr: 0.00001 + | > step_time: 2.59240 (3.23685) + | > loader_time: 0.11240 (0.03629) + + + --> STEP: 82/234 -- GLOBAL_STEP: 5230 + | > loss: 0.67978 (0.77835) + | > log_mle: 0.20855 (0.21907) + | > loss_dur: 0.47124 (0.55928) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.06346 (1.62687) + | > current_lr: 0.00001 + | > step_time: 4.29740 (3.19003) + | > loader_time: 0.00360 (0.03537) + + + --> STEP: 87/234 -- GLOBAL_STEP: 5235 + | > loss: 0.67903 (0.77316) + | > log_mle: 0.18689 (0.21729) + | > loss_dur: 0.49214 (0.55586) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.91456 (1.68765) + | > current_lr: 0.00001 + | > step_time: 1.26410 (3.21615) + | > loader_time: 0.00140 (0.03644) + + + --> STEP: 92/234 -- GLOBAL_STEP: 5240 + | > loss: 0.64149 (0.76648) + | > log_mle: 0.15471 (0.21440) + | > loss_dur: 0.48678 (0.55208) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.07083 (1.81419) + | > current_lr: 0.00001 + | > step_time: 2.91230 (3.17872) + | > loader_time: 0.00320 (0.03773) + + + --> STEP: 97/234 -- GLOBAL_STEP: 5245 + | > loss: 0.63793 (0.76005) + | > log_mle: 0.15804 (0.21064) + | > loss_dur: 0.47989 (0.54941) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.25327 (2.00588) + | > current_lr: 0.00001 + | > step_time: 4.08360 (3.28548) + | > loader_time: 0.01600 (0.04110) + + + --> STEP: 102/234 -- GLOBAL_STEP: 5250 + | > loss: 0.65768 (0.75474) + | > log_mle: 0.18041 (0.20792) + | > loss_dur: 0.47727 (0.54683) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.63146 (2.12707) + | > current_lr: 0.00001 + | > step_time: 3.00480 (3.26092) + | > loader_time: 0.00510 (0.04101) + + + --> STEP: 107/234 -- GLOBAL_STEP: 5255 + | > loss: 0.62876 (0.74995) + | > log_mle: 0.13336 (0.20453) + | > loss_dur: 0.49541 (0.54542) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.94562 (2.22884) + | > current_lr: 0.00001 + | > step_time: 2.80750 (3.24214) + | > loader_time: 0.08650 (0.04003) + + + --> STEP: 112/234 -- GLOBAL_STEP: 5260 + | > loss: 0.63671 (0.74540) + | > log_mle: 0.12963 (0.20146) + | > loss_dur: 0.50708 (0.54394) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.60187 (2.34716) + | > current_lr: 0.00001 + | > step_time: 1.61090 (3.18678) + | > loader_time: 0.08200 (0.04153) + + + --> STEP: 117/234 -- GLOBAL_STEP: 5265 + | > loss: 0.62026 (0.74118) + | > log_mle: 0.13989 (0.19866) + | > loss_dur: 0.48037 (0.54252) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.06502 (2.49761) + | > current_lr: 0.00001 + | > step_time: 2.19520 (3.14963) + | > loader_time: 0.00220 (0.04179) + + + --> STEP: 122/234 -- GLOBAL_STEP: 5270 + | > loss: 0.63061 (0.73737) + | > log_mle: 0.15017 (0.19665) + | > loss_dur: 0.48044 (0.54073) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.90516 (2.57783) + | > current_lr: 0.00001 + | > step_time: 6.70930 (3.16746) + | > loader_time: 0.07580 (0.04223) + + + --> STEP: 127/234 -- GLOBAL_STEP: 5275 + | > loss: 0.61530 (0.73277) + | > log_mle: 0.10881 (0.19366) + | > loss_dur: 0.50649 (0.53911) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.24306 (2.66083) + | > current_lr: 0.00001 + | > step_time: 1.41290 (3.13509) + | > loader_time: 0.07540 (0.04341) + + + --> STEP: 132/234 -- GLOBAL_STEP: 5280 + | > loss: 0.60419 (0.72818) + | > log_mle: 0.12546 (0.19040) + | > loss_dur: 0.47874 (0.53778) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.39122 (2.74547) + | > current_lr: 0.00001 + | > step_time: 2.40720 (3.11791) + | > loader_time: 0.08610 (0.04469) + + + --> STEP: 137/234 -- GLOBAL_STEP: 5285 + | > loss: 0.66158 (0.72443) + | > log_mle: 0.11945 (0.18705) + | > loss_dur: 0.54212 (0.53738) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.84340 (2.87970) + | > current_lr: 0.00001 + | > step_time: 2.20700 (3.10001) + | > loader_time: 0.00380 (0.04382) + + + --> STEP: 142/234 -- GLOBAL_STEP: 5290 + | > loss: 0.61383 (0.72045) + | > log_mle: 0.09692 (0.18398) + | > loss_dur: 0.51690 (0.53647) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.94097 (2.99797) + | > current_lr: 0.00001 + | > step_time: 1.80840 (3.07092) + | > loader_time: 0.00280 (0.04432) + + + --> STEP: 147/234 -- GLOBAL_STEP: 5295 + | > loss: 0.58672 (0.71586) + | > log_mle: 0.09573 (0.17969) + | > loss_dur: 0.49098 (0.53617) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.93786 (3.14631) + | > current_lr: 0.00001 + | > step_time: 2.30560 (3.03909) + | > loader_time: 0.09540 (0.04355) + + + --> STEP: 152/234 -- GLOBAL_STEP: 5300 + | > loss: 0.56217 (0.71112) + | > log_mle: 0.03662 (0.17573) + | > loss_dur: 0.52556 (0.53539) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.73939 (3.29488) + | > current_lr: 0.00001 + | > step_time: 3.80520 (3.05353) + | > loader_time: 0.09250 (0.04300) + + + --> STEP: 157/234 -- GLOBAL_STEP: 5305 + | > loss: 0.60900 (0.70664) + | > log_mle: 0.08116 (0.17140) + | > loss_dur: 0.52784 (0.53524) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.90483 (3.47134) + | > current_lr: 0.00001 + | > step_time: 9.38530 (3.13012) + | > loader_time: 0.21800 (0.04369) + + + --> STEP: 162/234 -- GLOBAL_STEP: 5310 + | > loss: 0.57968 (0.70243) + | > log_mle: 0.04924 (0.16724) + | > loss_dur: 0.53044 (0.53519) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.36500 (3.62693) + | > current_lr: 0.00001 + | > step_time: 4.11260 (3.14109) + | > loader_time: 0.08370 (0.04650) + + + --> STEP: 167/234 -- GLOBAL_STEP: 5315 + | > loss: 0.54458 (0.69834) + | > log_mle: -0.00839 (0.16342) + | > loss_dur: 0.55297 (0.53492) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.46100 (3.74691) + | > current_lr: 0.00001 + | > step_time: 3.31120 (3.18365) + | > loader_time: 0.39470 (0.05212) + + + --> STEP: 172/234 -- GLOBAL_STEP: 5320 + | > loss: 0.55143 (0.69458) + | > log_mle: -0.00209 (0.15900) + | > loss_dur: 0.55353 (0.53558) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.08110 (3.95567) + | > current_lr: 0.00001 + | > step_time: 5.38960 (3.23075) + | > loader_time: 0.00230 (0.05289) + + + --> STEP: 177/234 -- GLOBAL_STEP: 5325 + | > loss: 0.58710 (0.69057) + | > log_mle: 0.03187 (0.15479) + | > loss_dur: 0.55522 (0.53578) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.19248 (4.15121) + | > current_lr: 0.00001 + | > step_time: 3.49860 (3.27563) + | > loader_time: 0.00440 (0.05311) + + + --> STEP: 182/234 -- GLOBAL_STEP: 5330 + | > loss: 0.56972 (0.68701) + | > log_mle: -0.00832 (0.15063) + | > loss_dur: 0.57804 (0.53638) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.60426 (4.36587) + | > current_lr: 0.00001 + | > step_time: 5.59650 (3.35713) + | > loader_time: 0.00790 (0.05347) + + + --> STEP: 187/234 -- GLOBAL_STEP: 5335 + | > loss: 0.52898 (0.68338) + | > log_mle: -0.01058 (0.14654) + | > loss_dur: 0.53956 (0.53685) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.83078 (4.54497) + | > current_lr: 0.00001 + | > step_time: 6.09350 (3.39892) + | > loader_time: 0.01290 (0.05262) + + + --> STEP: 192/234 -- GLOBAL_STEP: 5340 + | > loss: 0.51192 (0.67966) + | > log_mle: -0.03132 (0.14258) + | > loss_dur: 0.54324 (0.53708) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.52847 (4.66384) + | > current_lr: 0.00001 + | > step_time: 3.62380 (3.43031) + | > loader_time: 0.00450 (0.05232) + + + --> STEP: 197/234 -- GLOBAL_STEP: 5345 + | > loss: 0.50456 (0.67609) + | > log_mle: -0.01246 (0.13872) + | > loss_dur: 0.51702 (0.53737) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.88488 (4.80325) + | > current_lr: 0.00001 + | > step_time: 2.59420 (3.48374) + | > loader_time: 0.00460 (0.05261) + + + --> STEP: 202/234 -- GLOBAL_STEP: 5350 + | > loss: 0.48424 (0.67256) + | > log_mle: -0.08361 (0.13476) + | > loss_dur: 0.56785 (0.53781) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.39932 (4.97982) + | > current_lr: 0.00001 + | > step_time: 5.29090 (3.51727) + | > loader_time: 0.07580 (0.05224) + + + --> STEP: 207/234 -- GLOBAL_STEP: 5355 + | > loss: 0.49669 (0.66903) + | > log_mle: -0.07132 (0.13082) + | > loss_dur: 0.56801 (0.53821) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.06992 (5.15286) + | > current_lr: 0.00001 + | > step_time: 2.58260 (3.55410) + | > loader_time: 0.00500 (0.05234) + + + --> STEP: 212/234 -- GLOBAL_STEP: 5360 + | > loss: 0.52646 (0.66558) + | > log_mle: -0.05878 (0.12641) + | > loss_dur: 0.58525 (0.53917) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.47025 (5.34513) + | > current_lr: 0.00001 + | > step_time: 6.60510 (3.62085) + | > loader_time: 0.08440 (0.05840) + + + --> STEP: 217/234 -- GLOBAL_STEP: 5365 + | > loss: 0.50270 (0.66188) + | > log_mle: -0.07480 (0.12209) + | > loss_dur: 0.57751 (0.53979) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.48913 (5.43730) + | > current_lr: 0.00001 + | > step_time: 3.39070 (3.70570) + | > loader_time: 0.30190 (0.06241) + + + --> STEP: 222/234 -- GLOBAL_STEP: 5370 + | > loss: 0.53090 (0.65854) + | > log_mle: -0.08082 (0.11783) + | > loss_dur: 0.61172 (0.54071) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.65073 (5.57612) + | > current_lr: 0.00001 + | > step_time: 7.30170 (3.75203) + | > loader_time: 0.00300 (0.06241) + + + --> STEP: 227/234 -- GLOBAL_STEP: 5375 + | > loss: 0.51016 (0.65495) + | > log_mle: -0.06475 (0.11328) + | > loss_dur: 0.57491 (0.54167) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.53637 (5.84002) + | > current_lr: 0.00001 + | > step_time: 0.24230 (3.71598) + | > loader_time: 0.00470 (0.06150) + + + --> STEP: 232/234 -- GLOBAL_STEP: 5380 + | > loss: 0.67652 (0.65314) + | > log_mle: -0.22494 (0.10775) + | > loss_dur: 0.90146 (0.54539) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 20.72729 (6.10171) + | > current_lr: 0.00001 + | > step_time: 0.33030 (3.64170) + | > loader_time: 0.01200 (0.06029) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12756 (+0.09566) + | > avg_loss: 0.58486 (-0.04017) + | > avg_log_mle: 0.03449 (-0.01278) + | > avg_loss_dur: 0.55037 (-0.02739) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_5382.pth + + > EPOCH: 23/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 13:57:44)  + + --> STEP: 3/234 -- GLOBAL_STEP: 5385 + | > loss: 0.96689 (1.04296) + | > log_mle: 0.23780 (0.25715) + | > loss_dur: 0.72909 (0.78581) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.69696 (1.84178) + | > current_lr: 0.00001 + | > step_time: 4.00290 (3.86401) + | > loader_time: 0.00140 (0.00220) + + + --> STEP: 8/234 -- GLOBAL_STEP: 5390 + | > loss: 0.91883 (0.98411) + | > log_mle: 0.21163 (0.23581) + | > loss_dur: 0.70720 (0.74830) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.67462 (2.39782) + | > current_lr: 0.00001 + | > step_time: 4.70120 (3.97754) + | > loader_time: 0.59540 (0.11226) + + + --> STEP: 13/234 -- GLOBAL_STEP: 5395 + | > loss: 0.82142 (0.91968) + | > log_mle: 0.23065 (0.23057) + | > loss_dur: 0.59076 (0.68911) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.41324 (2.18323) + | > current_lr: 0.00001 + | > step_time: 6.00470 (4.79589) + | > loader_time: 0.00340 (0.08425) + + + --> STEP: 18/234 -- GLOBAL_STEP: 5400 + | > loss: 0.79372 (0.88508) + | > log_mle: 0.22151 (0.22740) + | > loss_dur: 0.57221 (0.65768) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.31443 (1.95336) + | > current_lr: 0.00001 + | > step_time: 4.70310 (5.12987) + | > loader_time: 0.10060 (0.09470) + + + --> STEP: 23/234 -- GLOBAL_STEP: 5405 + | > loss: 0.73301 (0.85846) + | > log_mle: 0.21076 (0.22552) + | > loss_dur: 0.52224 (0.63294) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.08889 (1.80282) + | > current_lr: 0.00001 + | > step_time: 3.90560 (4.96929) + | > loader_time: 0.00150 (0.08273) + + + --> STEP: 28/234 -- GLOBAL_STEP: 5410 + | > loss: 0.71266 (0.83692) + | > log_mle: 0.21311 (0.22348) + | > loss_dur: 0.49955 (0.61345) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.21198 (1.69089) + | > current_lr: 0.00001 + | > step_time: 3.47950 (4.72365) + | > loader_time: 0.00170 (0.07104) + + + --> STEP: 33/234 -- GLOBAL_STEP: 5415 + | > loss: 0.74904 (0.82079) + | > log_mle: 0.21906 (0.22075) + | > loss_dur: 0.52998 (0.60004) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.13532 (1.60145) + | > current_lr: 0.00001 + | > step_time: 4.19340 (4.53115) + | > loader_time: 0.00170 (0.06668) + + + --> STEP: 38/234 -- GLOBAL_STEP: 5420 + | > loss: 0.70718 (0.80696) + | > log_mle: 0.19910 (0.21794) + | > loss_dur: 0.50808 (0.58902) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.19471 (1.65900) + | > current_lr: 0.00001 + | > step_time: 8.40020 (4.68753) + | > loader_time: 0.00400 (0.07383) + + + --> STEP: 43/234 -- GLOBAL_STEP: 5425 + | > loss: 0.69458 (0.79672) + | > log_mle: 0.19532 (0.21673) + | > loss_dur: 0.49926 (0.57999) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.58467 (1.61702) + | > current_lr: 0.00001 + | > step_time: 2.10970 (4.33075) + | > loader_time: 0.00220 (0.06751) + + + --> STEP: 48/234 -- GLOBAL_STEP: 5430 + | > loss: 0.66358 (0.78582) + | > log_mle: 0.20540 (0.21524) + | > loss_dur: 0.45818 (0.57058) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.08769 (1.57499) + | > current_lr: 0.00001 + | > step_time: 0.78850 (4.11732) + | > loader_time: 0.00120 (0.06257) + + + --> STEP: 53/234 -- GLOBAL_STEP: 5435 + | > loss: 0.69009 (0.77589) + | > log_mle: 0.18692 (0.21407) + | > loss_dur: 0.50318 (0.56182) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.12545 (1.53614) + | > current_lr: 0.00001 + | > step_time: 3.41330 (4.01794) + | > loader_time: 0.09060 (0.06180) + + + --> STEP: 58/234 -- GLOBAL_STEP: 5440 + | > loss: 0.68679 (0.76854) + | > log_mle: 0.20701 (0.21283) + | > loss_dur: 0.47978 (0.55572) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.07404 (1.50323) + | > current_lr: 0.00001 + | > step_time: 4.09700 (3.95131) + | > loader_time: 0.18770 (0.06152) + + + --> STEP: 63/234 -- GLOBAL_STEP: 5445 + | > loss: 0.70991 (0.76016) + | > log_mle: 0.18829 (0.21019) + | > loss_dur: 0.52163 (0.54997) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.56541 (1.51713) + | > current_lr: 0.00001 + | > step_time: 1.20950 (3.78533) + | > loader_time: 0.00140 (0.05934) + + + --> STEP: 68/234 -- GLOBAL_STEP: 5450 + | > loss: 0.68657 (0.75291) + | > log_mle: 0.19958 (0.20919) + | > loss_dur: 0.48699 (0.54372) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.75784 (1.52402) + | > current_lr: 0.00001 + | > step_time: 2.97130 (3.72110) + | > loader_time: 0.00250 (0.05605) + + + --> STEP: 73/234 -- GLOBAL_STEP: 5455 + | > loss: 0.67120 (0.74710) + | > log_mle: 0.17511 (0.20757) + | > loss_dur: 0.49609 (0.53954) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.83015 (1.57799) + | > current_lr: 0.00001 + | > step_time: 2.76050 (3.63316) + | > loader_time: 0.00220 (0.05250) + + + --> STEP: 78/234 -- GLOBAL_STEP: 5460 + | > loss: 0.65357 (0.74048) + | > log_mle: 0.19644 (0.20617) + | > loss_dur: 0.45713 (0.53431) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.66722 (1.61710) + | > current_lr: 0.00001 + | > step_time: 2.28950 (3.53637) + | > loader_time: 0.00270 (0.05241) + + + --> STEP: 83/234 -- GLOBAL_STEP: 5465 + | > loss: 0.63854 (0.73412) + | > log_mle: 0.17262 (0.20477) + | > loss_dur: 0.46592 (0.52934) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.16196 (1.63990) + | > current_lr: 0.00001 + | > step_time: 2.47610 (3.47868) + | > loader_time: 0.41380 (0.05755) + + + --> STEP: 88/234 -- GLOBAL_STEP: 5470 + | > loss: 0.60824 (0.72851) + | > log_mle: 0.13458 (0.20259) + | > loss_dur: 0.47366 (0.52592) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.99550 (1.66660) + | > current_lr: 0.00001 + | > step_time: 9.99910 (3.57087) + | > loader_time: 0.09700 (0.05761) + + + --> STEP: 93/234 -- GLOBAL_STEP: 5475 + | > loss: 0.58869 (0.72212) + | > log_mle: 0.12050 (0.19956) + | > loss_dur: 0.46819 (0.52256) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.87949 (1.77691) + | > current_lr: 0.00001 + | > step_time: 1.90290 (3.54647) + | > loader_time: 0.00310 (0.05786) + + + --> STEP: 98/234 -- GLOBAL_STEP: 5480 + | > loss: 0.64003 (0.71651) + | > log_mle: 0.18505 (0.19647) + | > loss_dur: 0.45498 (0.52004) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.50657 (1.88545) + | > current_lr: 0.00001 + | > step_time: 5.69660 (3.52988) + | > loader_time: 0.19930 (0.05873) + + + --> STEP: 103/234 -- GLOBAL_STEP: 5485 + | > loss: 0.61677 (0.71099) + | > log_mle: 0.10587 (0.19298) + | > loss_dur: 0.51090 (0.51801) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.99127 (2.01729) + | > current_lr: 0.00001 + | > step_time: 2.40820 (3.46134) + | > loader_time: 0.07110 (0.05750) + + + --> STEP: 108/234 -- GLOBAL_STEP: 5490 + | > loss: 0.61443 (0.70626) + | > log_mle: 0.15455 (0.19008) + | > loss_dur: 0.45988 (0.51618) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.50674 (2.10768) + | > current_lr: 0.00001 + | > step_time: 5.30810 (3.45563) + | > loader_time: 0.09850 (0.05746) + + + --> STEP: 113/234 -- GLOBAL_STEP: 5495 + | > loss: 0.58177 (0.70144) + | > log_mle: 0.10771 (0.18663) + | > loss_dur: 0.47406 (0.51482) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.83501 (2.29042) + | > current_lr: 0.00001 + | > step_time: 2.10730 (3.44085) + | > loader_time: 0.00300 (0.05924) + + + --> STEP: 118/234 -- GLOBAL_STEP: 5500 + | > loss: 0.63243 (0.69759) + | > log_mle: 0.13832 (0.18411) + | > loss_dur: 0.49411 (0.51348) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.47780 (2.40278) + | > current_lr: 0.00001 + | > step_time: 1.54030 (3.41863) + | > loader_time: 0.00690 (0.05927) + + + --> STEP: 123/234 -- GLOBAL_STEP: 5505 + | > loss: 0.61766 (0.69373) + | > log_mle: 0.16444 (0.18234) + | > loss_dur: 0.45322 (0.51139) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.91125 (2.46168) + | > current_lr: 0.00001 + | > step_time: 1.70500 (3.36659) + | > loader_time: 0.00820 (0.05840) + + + --> STEP: 128/234 -- GLOBAL_STEP: 5510 + | > loss: 0.57429 (0.68909) + | > log_mle: 0.10917 (0.17890) + | > loss_dur: 0.46512 (0.51020) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.94802 (2.60439) + | > current_lr: 0.00001 + | > step_time: 2.59890 (3.34212) + | > loader_time: 0.09340 (0.05842) + + + --> STEP: 133/234 -- GLOBAL_STEP: 5515 + | > loss: 0.59006 (0.68465) + | > log_mle: 0.09460 (0.17556) + | > loss_dur: 0.49546 (0.50909) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.46784 (2.80774) + | > current_lr: 0.00001 + | > step_time: 1.90560 (3.29878) + | > loader_time: 0.08630 (0.05885) + + + --> STEP: 138/234 -- GLOBAL_STEP: 5520 + | > loss: 0.59076 (0.68095) + | > log_mle: 0.12822 (0.17247) + | > loss_dur: 0.46254 (0.50848) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.61542 (2.97666) + | > current_lr: 0.00001 + | > step_time: 3.20390 (3.27790) + | > loader_time: 0.00360 (0.05827) + + + --> STEP: 143/234 -- GLOBAL_STEP: 5525 + | > loss: 0.55641 (0.67668) + | > log_mle: 0.01872 (0.16863) + | > loss_dur: 0.53768 (0.50806) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.78878 (3.17628) + | > current_lr: 0.00001 + | > step_time: 2.31490 (3.26918) + | > loader_time: 0.00470 (0.05829) + + + --> STEP: 148/234 -- GLOBAL_STEP: 5530 + | > loss: 0.52229 (0.67209) + | > log_mle: 0.08228 (0.16475) + | > loss_dur: 0.44001 (0.50733) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.02384 (3.34100) + | > current_lr: 0.00001 + | > step_time: 1.50700 (3.23928) + | > loader_time: 0.08730 (0.05867) + + + --> STEP: 153/234 -- GLOBAL_STEP: 5535 + | > loss: 0.49909 (0.66739) + | > log_mle: -0.00937 (0.16021) + | > loss_dur: 0.50847 (0.50718) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.93877 (3.54194) + | > current_lr: 0.00001 + | > step_time: 5.21300 (3.25067) + | > loader_time: 0.09480 (0.05801) + + + --> STEP: 158/234 -- GLOBAL_STEP: 5540 + | > loss: 0.53340 (0.66323) + | > log_mle: 0.03577 (0.15618) + | > loss_dur: 0.49763 (0.50705) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.71493 (3.71060) + | > current_lr: 0.00001 + | > step_time: 6.00660 (3.28533) + | > loader_time: 0.28780 (0.05913) + + + --> STEP: 163/234 -- GLOBAL_STEP: 5545 + | > loss: 0.53977 (0.65912) + | > log_mle: 0.05827 (0.15216) + | > loss_dur: 0.48150 (0.50695) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.50054 (3.84779) + | > current_lr: 0.00001 + | > step_time: 3.89650 (3.33198) + | > loader_time: 0.01790 (0.05876) + + + --> STEP: 168/234 -- GLOBAL_STEP: 5550 + | > loss: 0.56902 (0.65523) + | > log_mle: 0.01790 (0.14811) + | > loss_dur: 0.55112 (0.50713) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.66768 (4.01805) + | > current_lr: 0.00001 + | > step_time: 5.78960 (3.35837) + | > loader_time: 0.40360 (0.06049) + + + --> STEP: 173/234 -- GLOBAL_STEP: 5555 + | > loss: 0.52543 (0.65126) + | > log_mle: 0.00887 (0.14364) + | > loss_dur: 0.51656 (0.50762) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.31000 (4.20166) + | > current_lr: 0.00001 + | > step_time: 5.60440 (3.40919) + | > loader_time: 0.09880 (0.06066) + + + --> STEP: 178/234 -- GLOBAL_STEP: 5560 + | > loss: 0.49415 (0.64711) + | > log_mle: -0.04027 (0.13915) + | > loss_dur: 0.53442 (0.50796) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.95019 (4.36477) + | > current_lr: 0.00001 + | > step_time: 5.18950 (3.47788) + | > loader_time: 0.00500 (0.06399) + + + --> STEP: 183/234 -- GLOBAL_STEP: 5565 + | > loss: 0.50924 (0.64375) + | > log_mle: -0.03499 (0.13502) + | > loss_dur: 0.54423 (0.50873) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.78569 (4.58267) + | > current_lr: 0.00001 + | > step_time: 5.47450 (3.49797) + | > loader_time: 0.00310 (0.06436) + + + --> STEP: 188/234 -- GLOBAL_STEP: 5570 + | > loss: 0.49009 (0.64006) + | > log_mle: -0.04394 (0.13087) + | > loss_dur: 0.53404 (0.50918) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.82726 (4.73291) + | > current_lr: 0.00001 + | > step_time: 1.87490 (3.52886) + | > loader_time: 0.01900 (0.06483) + + + --> STEP: 193/234 -- GLOBAL_STEP: 5575 + | > loss: 0.47692 (0.63626) + | > log_mle: -0.04638 (0.12689) + | > loss_dur: 0.52330 (0.50937) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.49904 (4.84961) + | > current_lr: 0.00001 + | > step_time: 12.28020 (3.62820) + | > loader_time: 0.08600 (0.06565) + + + --> STEP: 198/234 -- GLOBAL_STEP: 5580 + | > loss: 0.49621 (0.63285) + | > log_mle: -0.03814 (0.12309) + | > loss_dur: 0.53435 (0.50976) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.78086 (5.00889) + | > current_lr: 0.00001 + | > step_time: 4.69920 (3.67545) + | > loader_time: 0.19460 (0.06647) + + + --> STEP: 203/234 -- GLOBAL_STEP: 5585 + | > loss: 0.51663 (0.62942) + | > log_mle: 0.00783 (0.11933) + | > loss_dur: 0.50881 (0.51010) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.22826 (5.14336) + | > current_lr: 0.00001 + | > step_time: 6.91070 (3.76179) + | > loader_time: 0.09480 (0.06974) + + + --> STEP: 208/234 -- GLOBAL_STEP: 5590 + | > loss: 0.48105 (0.62571) + | > log_mle: -0.05218 (0.11509) + | > loss_dur: 0.53323 (0.51062) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.51699 (5.33156) + | > current_lr: 0.00001 + | > step_time: 2.19830 (3.79918) + | > loader_time: 0.10110 (0.06915) + + + --> STEP: 213/234 -- GLOBAL_STEP: 5595 + | > loss: 0.44768 (0.62195) + | > log_mle: -0.09232 (0.11049) + | > loss_dur: 0.53999 (0.51147) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.44210 (5.58228) + | > current_lr: 0.00001 + | > step_time: 2.60530 (3.81846) + | > loader_time: 0.00230 (0.06943) + + + --> STEP: 218/234 -- GLOBAL_STEP: 5600 + | > loss: 0.47834 (0.61839) + | > log_mle: -0.06123 (0.10632) + | > loss_dur: 0.53958 (0.51208) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.82226 (5.83003) + | > current_lr: 0.00001 + | > step_time: 8.19890 (3.87414) + | > loader_time: 0.01130 (0.07099) + + + --> STEP: 223/234 -- GLOBAL_STEP: 5605 + | > loss: 0.45550 (0.61500) + | > log_mle: -0.09433 (0.10192) + | > loss_dur: 0.54983 (0.51308) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.88535 (6.09371) + | > current_lr: 0.00001 + | > step_time: 0.45100 (3.86145) + | > loader_time: 0.00750 (0.07169) + + + --> STEP: 228/234 -- GLOBAL_STEP: 5610 + | > loss: 0.47977 (0.61148) + | > log_mle: -0.09851 (0.09732) + | > loss_dur: 0.57827 (0.51415) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.09948 (6.34371) + | > current_lr: 0.00001 + | > step_time: 0.23930 (3.78214) + | > loader_time: 0.00720 (0.07021) + + + --> STEP: 233/234 -- GLOBAL_STEP: 5615 + | > loss: 1.16807 (0.61258) + | > log_mle: -0.05553 (0.09196) + | > loss_dur: 1.22361 (0.52062) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.44621 (6.69424) + | > current_lr: 0.00001 + | > step_time: 0.19320 (3.70686) + | > loader_time: 0.00280 (0.06880) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27066 (+0.14311) + | > avg_loss: 0.53588 (-0.04898) + | > avg_log_mle: 0.01466 (-0.01983) + | > avg_loss_dur: 0.52122 (-0.02915) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_5616.pth + + > EPOCH: 24/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 14:13:41)  + + --> STEP: 4/234 -- GLOBAL_STEP: 5620 + | > loss: 0.97254 (0.99087) + | > log_mle: 0.20057 (0.23214) + | > loss_dur: 0.77197 (0.75872) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.21462 (2.12836) + | > current_lr: 0.00001 + | > step_time: 5.70970 (5.17931) + | > loader_time: 0.08810 (0.04411) + + + --> STEP: 9/234 -- GLOBAL_STEP: 5625 + | > loss: 0.80032 (0.91482) + | > log_mle: 0.19445 (0.21827) + | > loss_dur: 0.60588 (0.69654) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.27729 (2.04675) + | > current_lr: 0.00001 + | > step_time: 5.99820 (6.23369) + | > loader_time: 0.29260 (0.06623) + + + --> STEP: 14/234 -- GLOBAL_STEP: 5630 + | > loss: 0.73897 (0.85800) + | > log_mle: 0.20155 (0.21491) + | > loss_dur: 0.53742 (0.64309) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.60870 (1.85836) + | > current_lr: 0.00001 + | > step_time: 5.89860 (6.42169) + | > loader_time: 0.09100 (0.07761) + + + --> STEP: 19/234 -- GLOBAL_STEP: 5635 + | > loss: 0.75444 (0.82822) + | > log_mle: 0.21191 (0.21276) + | > loss_dur: 0.54253 (0.61545) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.45034 (1.71223) + | > current_lr: 0.00001 + | > step_time: 2.31950 (5.49041) + | > loader_time: 0.17680 (0.08124) + + + --> STEP: 24/234 -- GLOBAL_STEP: 5640 + | > loss: 0.73722 (0.80479) + | > log_mle: 0.20393 (0.21071) + | > loss_dur: 0.53328 (0.59408) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.25180 (1.59705) + | > current_lr: 0.00001 + | > step_time: 4.11250 (5.08598) + | > loader_time: 0.08380 (0.07077) + + + --> STEP: 29/234 -- GLOBAL_STEP: 5645 + | > loss: 0.71736 (0.78354) + | > log_mle: 0.20919 (0.20904) + | > loss_dur: 0.50817 (0.57450) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.99059 (1.50976) + | > current_lr: 0.00001 + | > step_time: 4.00490 (5.25891) + | > loader_time: 0.08510 (0.07059) + + + --> STEP: 34/234 -- GLOBAL_STEP: 5650 + | > loss: 0.67382 (0.76735) + | > log_mle: 0.18822 (0.20585) + | > loss_dur: 0.48559 (0.56150) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.37605 (1.46107) + | > current_lr: 0.00001 + | > step_time: 2.11210 (5.11216) + | > loader_time: 0.00530 (0.06683) + + + --> STEP: 39/234 -- GLOBAL_STEP: 5655 + | > loss: 0.68698 (0.75494) + | > log_mle: 0.18829 (0.20329) + | > loss_dur: 0.49869 (0.55165) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.40740 (1.55630) + | > current_lr: 0.00001 + | > step_time: 3.10970 (5.02021) + | > loader_time: 0.09470 (0.09752) + + + --> STEP: 44/234 -- GLOBAL_STEP: 5660 + | > loss: 0.65764 (0.74475) + | > log_mle: 0.18629 (0.20215) + | > loss_dur: 0.47135 (0.54260) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.08788 (1.51378) + | > current_lr: 0.00001 + | > step_time: 0.98400 (4.95821) + | > loader_time: 0.00200 (0.09137) + + + --> STEP: 49/234 -- GLOBAL_STEP: 5665 + | > loss: 0.61552 (0.73440) + | > log_mle: 0.18024 (0.20066) + | > loss_dur: 0.43528 (0.53374) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.03693 (1.48155) + | > current_lr: 0.00001 + | > step_time: 3.80980 (4.69523) + | > loader_time: 0.01020 (0.08454) + + + --> STEP: 54/234 -- GLOBAL_STEP: 5670 + | > loss: 0.63292 (0.72563) + | > log_mle: 0.18029 (0.19962) + | > loss_dur: 0.45263 (0.52602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.98939 (1.45778) + | > current_lr: 0.00001 + | > step_time: 1.88810 (4.43440) + | > loader_time: 0.00150 (0.07692) + + + --> STEP: 59/234 -- GLOBAL_STEP: 5675 + | > loss: 0.57619 (0.71773) + | > log_mle: 0.16075 (0.19813) + | > loss_dur: 0.41544 (0.51960) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.52379 (1.43683) + | > current_lr: 0.00001 + | > step_time: 2.32170 (4.23228) + | > loader_time: 0.08300 (0.07493) + + + --> STEP: 64/234 -- GLOBAL_STEP: 5680 + | > loss: 0.60025 (0.70997) + | > log_mle: 0.19108 (0.19608) + | > loss_dur: 0.40917 (0.51390) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.87473 (1.47172) + | > current_lr: 0.00001 + | > step_time: 2.08740 (4.06068) + | > loader_time: 0.00530 (0.07095) + + + --> STEP: 69/234 -- GLOBAL_STEP: 5685 + | > loss: 0.66560 (0.70418) + | > log_mle: 0.20038 (0.19526) + | > loss_dur: 0.46523 (0.50892) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.05266 (1.50357) + | > current_lr: 0.00001 + | > step_time: 1.39160 (3.92845) + | > loader_time: 0.00540 (0.06863) + + + --> STEP: 74/234 -- GLOBAL_STEP: 5690 + | > loss: 0.58686 (0.69796) + | > log_mle: 0.17573 (0.19333) + | > loss_dur: 0.41113 (0.50463) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.96688 (1.57838) + | > current_lr: 0.00001 + | > step_time: 1.29120 (3.83632) + | > loader_time: 0.09000 (0.06905) + + + --> STEP: 79/234 -- GLOBAL_STEP: 5695 + | > loss: 0.59875 (0.69250) + | > log_mle: 0.16884 (0.19193) + | > loss_dur: 0.42990 (0.50057) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.34990 (1.59675) + | > current_lr: 0.00001 + | > step_time: 1.70920 (3.71749) + | > loader_time: 0.29060 (0.06854) + + + --> STEP: 84/234 -- GLOBAL_STEP: 5700 + | > loss: 0.59819 (0.68675) + | > log_mle: 0.16156 (0.19051) + | > loss_dur: 0.43664 (0.49625) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.35339 (1.62490) + | > current_lr: 0.00001 + | > step_time: 2.70710 (3.61661) + | > loader_time: 0.00580 (0.06676) + + + --> STEP: 89/234 -- GLOBAL_STEP: 5705 + | > loss: 0.56304 (0.68115) + | > log_mle: 0.13835 (0.18811) + | > loss_dur: 0.42469 (0.49304) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.08156 (1.65928) + | > current_lr: 0.00001 + | > step_time: 2.40110 (3.57962) + | > loader_time: 0.00200 (0.06411) + + + --> STEP: 94/234 -- GLOBAL_STEP: 5710 + | > loss: 0.55495 (0.67484) + | > log_mle: 0.10102 (0.18475) + | > loss_dur: 0.45393 (0.49009) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.22468 (1.83059) + | > current_lr: 0.00001 + | > step_time: 2.59950 (3.51595) + | > loader_time: 0.07500 (0.06163) + + + --> STEP: 99/234 -- GLOBAL_STEP: 5715 + | > loss: 0.53515 (0.66944) + | > log_mle: 0.07593 (0.18151) + | > loss_dur: 0.45922 (0.48793) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.39159 (2.04500) + | > current_lr: 0.00001 + | > step_time: 2.40380 (3.46836) + | > loader_time: 0.09300 (0.06189) + + + --> STEP: 104/234 -- GLOBAL_STEP: 5720 + | > loss: 0.53781 (0.66461) + | > log_mle: 0.07133 (0.17806) + | > loss_dur: 0.46647 (0.48655) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.88759 (2.20946) + | > current_lr: 0.00001 + | > step_time: 2.33030 (3.51555) + | > loader_time: 0.08730 (0.06633) + + + --> STEP: 109/234 -- GLOBAL_STEP: 5725 + | > loss: 0.57004 (0.66031) + | > log_mle: 0.09463 (0.17545) + | > loss_dur: 0.47541 (0.48486) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.07982 (2.41839) + | > current_lr: 0.00001 + | > step_time: 1.90360 (3.47903) + | > loader_time: 0.08100 (0.06551) + + + --> STEP: 114/234 -- GLOBAL_STEP: 5730 + | > loss: 0.55218 (0.65573) + | > log_mle: 0.11665 (0.17221) + | > loss_dur: 0.43553 (0.48352) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.56745 (2.55298) + | > current_lr: 0.00001 + | > step_time: 3.38970 (3.47892) + | > loader_time: 0.00300 (0.06362) + + + --> STEP: 119/234 -- GLOBAL_STEP: 5735 + | > loss: 0.56232 (0.65227) + | > log_mle: 0.11381 (0.16969) + | > loss_dur: 0.44851 (0.48257) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.14863 (2.64364) + | > current_lr: 0.00001 + | > step_time: 1.07180 (3.50676) + | > loader_time: 0.00230 (0.06335) + + + --> STEP: 124/234 -- GLOBAL_STEP: 5740 + | > loss: 0.52917 (0.64831) + | > log_mle: 0.08880 (0.16773) + | > loss_dur: 0.44037 (0.48057) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.61574 (2.75291) + | > current_lr: 0.00001 + | > step_time: 3.50060 (3.48576) + | > loader_time: 0.08550 (0.06299) + + + --> STEP: 129/234 -- GLOBAL_STEP: 5745 + | > loss: 0.54959 (0.64394) + | > log_mle: 0.10142 (0.16440) + | > loss_dur: 0.44816 (0.47954) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.75578 (2.90584) + | > current_lr: 0.00001 + | > step_time: 6.32340 (3.51316) + | > loader_time: 0.19980 (0.06718) + + + --> STEP: 134/234 -- GLOBAL_STEP: 5750 + | > loss: 0.53594 (0.63961) + | > log_mle: 0.06094 (0.16078) + | > loss_dur: 0.47499 (0.47883) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.89871 (3.14071) + | > current_lr: 0.00001 + | > step_time: 2.80700 (3.46188) + | > loader_time: 0.00320 (0.06540) + + + --> STEP: 139/234 -- GLOBAL_STEP: 5755 + | > loss: 0.49570 (0.63581) + | > log_mle: 0.00937 (0.15734) + | > loss_dur: 0.48632 (0.47846) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.27003 (3.33732) + | > current_lr: 0.00001 + | > step_time: 2.80330 (3.52402) + | > loader_time: 0.09430 (0.06530) + + + --> STEP: 144/234 -- GLOBAL_STEP: 5760 + | > loss: 0.49253 (0.63173) + | > log_mle: 0.02804 (0.15369) + | > loss_dur: 0.46449 (0.47804) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.52828 (3.52633) + | > current_lr: 0.00001 + | > step_time: 2.11340 (3.53708) + | > loader_time: 0.07010 (0.06500) + + + --> STEP: 149/234 -- GLOBAL_STEP: 5765 + | > loss: 0.46753 (0.62714) + | > log_mle: -0.00947 (0.14960) + | > loss_dur: 0.47700 (0.47753) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.91166 (3.70205) + | > current_lr: 0.00001 + | > step_time: 1.52120 (3.54993) + | > loader_time: 0.00440 (0.06418) + + + --> STEP: 154/234 -- GLOBAL_STEP: 5770 + | > loss: 0.49252 (0.62275) + | > log_mle: 0.02577 (0.14530) + | > loss_dur: 0.46675 (0.47745) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.36320 (3.93626) + | > current_lr: 0.00001 + | > step_time: 4.10890 (3.52826) + | > loader_time: 0.08570 (0.06391) + + + --> STEP: 159/234 -- GLOBAL_STEP: 5775 + | > loss: 0.50386 (0.61876) + | > log_mle: 0.00953 (0.14119) + | > loss_dur: 0.49433 (0.47757) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.23040 (4.15594) + | > current_lr: 0.00001 + | > step_time: 5.90230 (3.63511) + | > loader_time: 0.00320 (0.06800) + + + --> STEP: 164/234 -- GLOBAL_STEP: 5780 + | > loss: 0.48273 (0.61466) + | > log_mle: 0.01195 (0.13719) + | > loss_dur: 0.47078 (0.47746) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.97786 (4.31620) + | > current_lr: 0.00001 + | > step_time: 2.40960 (3.62122) + | > loader_time: 0.00470 (0.06779) + + + --> STEP: 169/234 -- GLOBAL_STEP: 5785 + | > loss: 0.51400 (0.61104) + | > log_mle: 0.01591 (0.13316) + | > loss_dur: 0.49809 (0.47788) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.72302 (4.41957) + | > current_lr: 0.00001 + | > step_time: 6.31310 (3.65452) + | > loader_time: 0.17920 (0.06803) + + + --> STEP: 174/234 -- GLOBAL_STEP: 5790 + | > loss: 0.42657 (0.60667) + | > log_mle: -0.06371 (0.12822) + | > loss_dur: 0.49028 (0.47844) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.39953 (4.59866) + | > current_lr: 0.00001 + | > step_time: 1.70040 (3.65068) + | > loader_time: 0.08800 (0.06766) + + + --> STEP: 179/234 -- GLOBAL_STEP: 5795 + | > loss: 0.46271 (0.60282) + | > log_mle: -0.04466 (0.12384) + | > loss_dur: 0.50738 (0.47899) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.20555 (4.80257) + | > current_lr: 0.00001 + | > step_time: 2.71640 (3.68350) + | > loader_time: 0.09180 (0.06790) + + + --> STEP: 184/234 -- GLOBAL_STEP: 5800 + | > loss: 0.46710 (0.59949) + | > log_mle: -0.02978 (0.11977) + | > loss_dur: 0.49688 (0.47972) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.91840 (4.95218) + | > current_lr: 0.00001 + | > step_time: 1.88820 (3.70285) + | > loader_time: 0.02070 (0.06779) + + + --> STEP: 189/234 -- GLOBAL_STEP: 5805 + | > loss: 0.46758 (0.59591) + | > log_mle: -0.02152 (0.11566) + | > loss_dur: 0.48910 (0.48025) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.08239 (5.11828) + | > current_lr: 0.00001 + | > step_time: 2.10540 (3.67381) + | > loader_time: 0.06560 (0.06736) + + + --> STEP: 194/234 -- GLOBAL_STEP: 5810 + | > loss: 0.45054 (0.59207) + | > log_mle: -0.05325 (0.11151) + | > loss_dur: 0.50379 (0.48056) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.53030 (5.32607) + | > current_lr: 0.00001 + | > step_time: 9.81140 (3.74510) + | > loader_time: 0.02230 (0.06743) + + + --> STEP: 199/234 -- GLOBAL_STEP: 5815 + | > loss: 0.43745 (0.58868) + | > log_mle: -0.06212 (0.10765) + | > loss_dur: 0.49957 (0.48103) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.87503 (5.50503) + | > current_lr: 0.00001 + | > step_time: 1.87910 (3.70802) + | > loader_time: 0.02020 (0.06720) + + + --> STEP: 204/234 -- GLOBAL_STEP: 5820 + | > loss: 0.43204 (0.58529) + | > log_mle: -0.08387 (0.10380) + | > loss_dur: 0.51591 (0.48149) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.14959 (5.68100) + | > current_lr: 0.00001 + | > step_time: 6.30630 (3.74428) + | > loader_time: 0.18470 (0.06872) + + + --> STEP: 209/234 -- GLOBAL_STEP: 5825 + | > loss: 0.45480 (0.58188) + | > log_mle: -0.04732 (0.09976) + | > loss_dur: 0.50211 (0.48212) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.69100 (5.86562) + | > current_lr: 0.00001 + | > step_time: 4.63990 (3.72960) + | > loader_time: 0.09870 (0.06979) + + + --> STEP: 214/234 -- GLOBAL_STEP: 5830 + | > loss: 0.43239 (0.57811) + | > log_mle: -0.07664 (0.09501) + | > loss_dur: 0.50903 (0.48310) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.95256 (6.07470) + | > current_lr: 0.00001 + | > step_time: 1.89630 (3.73364) + | > loader_time: 0.09570 (0.06998) + + + --> STEP: 219/234 -- GLOBAL_STEP: 5835 + | > loss: 0.38509 (0.57442) + | > log_mle: -0.15436 (0.09048) + | > loss_dur: 0.53945 (0.48393) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.21463 (6.28746) + | > current_lr: 0.00001 + | > step_time: 1.50520 (3.68852) + | > loader_time: 0.00310 (0.06920) + + + --> STEP: 224/234 -- GLOBAL_STEP: 5840 + | > loss: 0.42075 (0.57125) + | > log_mle: -0.11727 (0.08626) + | > loss_dur: 0.53801 (0.48500) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.21537 (6.49771) + | > current_lr: 0.00001 + | > step_time: 2.10970 (3.64497) + | > loader_time: 0.05870 (0.06801) + + + --> STEP: 229/234 -- GLOBAL_STEP: 5845 + | > loss: 0.47164 (0.56795) + | > log_mle: -0.13550 (0.08159) + | > loss_dur: 0.60714 (0.48636) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.71657 (6.74948) + | > current_lr: 0.00001 + | > step_time: 0.25980 (3.58217) + | > loader_time: 0.00490 (0.06697) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.34770 (+0.07704) + | > avg_loss: 0.49670 (-0.03918) + | > avg_log_mle: 0.00122 (-0.01344) + | > avg_loss_dur: 0.49548 (-0.02574) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_5850.pth + + > EPOCH: 25/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 14:28:43)  + + --> STEP: 0/234 -- GLOBAL_STEP: 5850 + | > loss: 0.91985 (0.91985) + | > log_mle: 0.20646 (0.20646) + | > loss_dur: 0.71339 (0.71339) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.19390 (2.19390) + | > current_lr: 0.00001 + | > step_time: 5.75960 (5.75961) + | > loader_time: 21.54970 (21.54972) + + + --> STEP: 5/234 -- GLOBAL_STEP: 5855 + | > loss: 0.82806 (0.90055) + | > log_mle: 0.20441 (0.21529) + | > loss_dur: 0.62365 (0.68526) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.37189 (2.65176) + | > current_lr: 0.00001 + | > step_time: 2.31530 (1.96420) + | > loader_time: 0.00800 (0.02078) + + + --> STEP: 10/234 -- GLOBAL_STEP: 5860 + | > loss: 0.67239 (0.83365) + | > log_mle: 0.18700 (0.20195) + | > loss_dur: 0.48539 (0.63171) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.54752 (2.22989) + | > current_lr: 0.00001 + | > step_time: 2.29700 (2.97341) + | > loader_time: 0.00460 (0.04250) + + + --> STEP: 15/234 -- GLOBAL_STEP: 5865 + | > loss: 0.72040 (0.79086) + | > log_mle: 0.18865 (0.19951) + | > loss_dur: 0.53176 (0.59135) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.13186 (1.92154) + | > current_lr: 0.00001 + | > step_time: 10.00020 (3.50386) + | > loader_time: 0.00110 (0.05014) + + + --> STEP: 20/234 -- GLOBAL_STEP: 5870 + | > loss: 0.65489 (0.76601) + | > log_mle: 0.19327 (0.19795) + | > loss_dur: 0.46162 (0.56806) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.07861 (1.73920) + | > current_lr: 0.00001 + | > step_time: 2.90510 (3.32404) + | > loader_time: 0.00330 (0.06884) + + + --> STEP: 25/234 -- GLOBAL_STEP: 5875 + | > loss: 0.63973 (0.74417) + | > log_mle: 0.19475 (0.19615) + | > loss_dur: 0.44498 (0.54802) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.05550 (1.61119) + | > current_lr: 0.00001 + | > step_time: 0.88640 (3.07363) + | > loader_time: 0.00390 (0.05558) + + + --> STEP: 30/234 -- GLOBAL_STEP: 5880 + | > loss: 0.61711 (0.72586) + | > log_mle: 0.16928 (0.19378) + | > loss_dur: 0.44782 (0.53209) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.91875 (1.51109) + | > current_lr: 0.00001 + | > step_time: 6.31510 (3.19568) + | > loader_time: 0.09590 (0.05306) + + + --> STEP: 35/234 -- GLOBAL_STEP: 5885 + | > loss: 0.62899 (0.71202) + | > log_mle: 0.16827 (0.19082) + | > loss_dur: 0.46072 (0.52120) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.96610 (1.57127) + | > current_lr: 0.00001 + | > step_time: 5.91040 (3.29691) + | > loader_time: 0.08120 (0.05574) + + + --> STEP: 40/234 -- GLOBAL_STEP: 5890 + | > loss: 0.65683 (0.70180) + | > log_mle: 0.18918 (0.18891) + | > loss_dur: 0.46766 (0.51289) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.12773 (1.52024) + | > current_lr: 0.00001 + | > step_time: 1.23990 (3.07702) + | > loader_time: 0.09890 (0.05551) + + + --> STEP: 45/234 -- GLOBAL_STEP: 5895 + | > loss: 0.62493 (0.69254) + | > log_mle: 0.16963 (0.18736) + | > loss_dur: 0.45530 (0.50519) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.67447 (1.49432) + | > current_lr: 0.00001 + | > step_time: 0.90140 (2.91114) + | > loader_time: 0.00480 (0.05548) + + + --> STEP: 50/234 -- GLOBAL_STEP: 5900 + | > loss: 0.59178 (0.68176) + | > log_mle: 0.17748 (0.18603) + | > loss_dur: 0.41430 (0.49572) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.93717 (1.44055) + | > current_lr: 0.00001 + | > step_time: 1.22010 (2.78036) + | > loader_time: 0.00210 (0.05320) + + + --> STEP: 55/234 -- GLOBAL_STEP: 5905 + | > loss: 0.57427 (0.67382) + | > log_mle: 0.15784 (0.18468) + | > loss_dur: 0.41643 (0.48914) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.11725 (1.41619) + | > current_lr: 0.00001 + | > step_time: 2.40800 (2.74399) + | > loader_time: 0.00250 (0.05581) + + + --> STEP: 60/234 -- GLOBAL_STEP: 5910 + | > loss: 0.56834 (0.66692) + | > log_mle: 0.14583 (0.18305) + | > loss_dur: 0.42251 (0.48387) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.61680 (1.40099) + | > current_lr: 0.00001 + | > step_time: 3.48990 (2.75218) + | > loader_time: 0.00280 (0.05483) + + + --> STEP: 65/234 -- GLOBAL_STEP: 5915 + | > loss: 0.58591 (0.66072) + | > log_mle: 0.17045 (0.18143) + | > loss_dur: 0.41546 (0.47929) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.97038 (1.39977) + | > current_lr: 0.00001 + | > step_time: 6.24280 (2.79630) + | > loader_time: 0.00280 (0.05368) + + + --> STEP: 70/234 -- GLOBAL_STEP: 5920 + | > loss: 0.58847 (0.65566) + | > log_mle: 0.15581 (0.18042) + | > loss_dur: 0.43266 (0.47524) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.19561 (1.43996) + | > current_lr: 0.00001 + | > step_time: 3.81050 (2.77366) + | > loader_time: 0.08630 (0.05376) + + + --> STEP: 75/234 -- GLOBAL_STEP: 5925 + | > loss: 0.57553 (0.64963) + | > log_mle: 0.15672 (0.17852) + | > loss_dur: 0.41882 (0.47112) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.87913 (1.53394) + | > current_lr: 0.00001 + | > step_time: 4.51870 (2.84932) + | > loader_time: 0.00250 (0.05410) + + + --> STEP: 80/234 -- GLOBAL_STEP: 5930 + | > loss: 0.55655 (0.64426) + | > log_mle: 0.16964 (0.17731) + | > loss_dur: 0.38691 (0.46695) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.99538 (1.53472) + | > current_lr: 0.00001 + | > step_time: 3.08370 (2.80520) + | > loader_time: 0.00280 (0.05274) + + + --> STEP: 85/234 -- GLOBAL_STEP: 5935 + | > loss: 0.56085 (0.63892) + | > log_mle: 0.15427 (0.17570) + | > loss_dur: 0.40658 (0.46322) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.20814 (1.55778) + | > current_lr: 0.00001 + | > step_time: 1.40010 (2.82390) + | > loader_time: 0.08550 (0.05494) + + + --> STEP: 90/234 -- GLOBAL_STEP: 5940 + | > loss: 0.52285 (0.63361) + | > log_mle: 0.12371 (0.17300) + | > loss_dur: 0.39914 (0.46061) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.24654 (1.66928) + | > current_lr: 0.00001 + | > step_time: 1.91410 (2.80204) + | > loader_time: 0.00370 (0.05401) + + + --> STEP: 95/234 -- GLOBAL_STEP: 5945 + | > loss: 0.49504 (0.62760) + | > log_mle: 0.05014 (0.16893) + | > loss_dur: 0.44491 (0.45867) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.21128 (1.87794) + | > current_lr: 0.00001 + | > step_time: 1.28360 (2.73554) + | > loader_time: 0.00220 (0.05238) + + + --> STEP: 100/234 -- GLOBAL_STEP: 5950 + | > loss: 0.53177 (0.62297) + | > log_mle: 0.11292 (0.16639) + | > loss_dur: 0.41884 (0.45658) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.29794 (1.98574) + | > current_lr: 0.00001 + | > step_time: 2.30570 (2.74270) + | > loader_time: 0.00530 (0.05078) + + + --> STEP: 105/234 -- GLOBAL_STEP: 5955 + | > loss: 0.56065 (0.61858) + | > log_mle: 0.14312 (0.16327) + | > loss_dur: 0.41753 (0.45531) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.56007 (2.12497) + | > current_lr: 0.00001 + | > step_time: 2.29990 (2.73311) + | > loader_time: 0.00530 (0.04937) + + + --> STEP: 110/234 -- GLOBAL_STEP: 5960 + | > loss: 0.54289 (0.61453) + | > log_mle: 0.11380 (0.16040) + | > loss_dur: 0.42909 (0.45412) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.64452 (2.32417) + | > current_lr: 0.00001 + | > step_time: 5.30410 (2.79610) + | > loader_time: 0.08780 (0.05137) + + + --> STEP: 115/234 -- GLOBAL_STEP: 5965 + | > loss: 0.54168 (0.61024) + | > log_mle: 0.09950 (0.15708) + | > loss_dur: 0.44218 (0.45316) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.97908 (2.48085) + | > current_lr: 0.00001 + | > step_time: 2.91610 (2.85275) + | > loader_time: 0.08790 (0.05336) + + + --> STEP: 120/234 -- GLOBAL_STEP: 5970 + | > loss: 0.47262 (0.60641) + | > log_mle: 0.05329 (0.15421) + | > loss_dur: 0.41933 (0.45220) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.76121 (2.61030) + | > current_lr: 0.00001 + | > step_time: 1.61280 (2.82671) + | > loader_time: 0.00330 (0.05125) + + + --> STEP: 125/234 -- GLOBAL_STEP: 5975 + | > loss: 0.50656 (0.60300) + | > log_mle: 0.06989 (0.15244) + | > loss_dur: 0.43667 (0.45056) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.87633 (2.71193) + | > current_lr: 0.00001 + | > step_time: 7.19320 (2.86651) + | > loader_time: 0.11490 (0.05159) + + + --> STEP: 130/234 -- GLOBAL_STEP: 5980 + | > loss: 0.48661 (0.59896) + | > log_mle: 0.05801 (0.14908) + | > loss_dur: 0.42859 (0.44988) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.83382 (2.92382) + | > current_lr: 0.00001 + | > step_time: 0.76940 (2.84630) + | > loader_time: 0.00220 (0.05111) + + + --> STEP: 135/234 -- GLOBAL_STEP: 5985 + | > loss: 0.51334 (0.59509) + | > log_mle: 0.11597 (0.14591) + | > loss_dur: 0.39737 (0.44918) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.17212 (3.05336) + | > current_lr: 0.00001 + | > step_time: 1.88640 (2.83909) + | > loader_time: 0.00210 (0.04942) + + + --> STEP: 140/234 -- GLOBAL_STEP: 5990 + | > loss: 0.51171 (0.59149) + | > log_mle: 0.08785 (0.14230) + | > loss_dur: 0.42385 (0.44919) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.04488 (3.30333) + | > current_lr: 0.00001 + | > step_time: 3.90640 (2.88941) + | > loader_time: 0.10710 (0.04922) + + + --> STEP: 145/234 -- GLOBAL_STEP: 5995 + | > loss: 0.44790 (0.58725) + | > log_mle: 0.00503 (0.13810) + | > loss_dur: 0.44287 (0.44915) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.36550 (3.57277) + | > current_lr: 0.00001 + | > step_time: 4.08890 (2.92395) + | > loader_time: 0.10590 (0.04854) + + + --> STEP: 150/234 -- GLOBAL_STEP: 6000 + | > loss: 0.47679 (0.58306) + | > log_mle: 0.02243 (0.13417) + | > loss_dur: 0.45436 (0.44889) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.30249 (3.74772) + | > current_lr: 0.00001 + | > step_time: 4.59670 (2.96227) + | > loader_time: 0.19710 (0.05133) + + + --> STEP: 155/234 -- GLOBAL_STEP: 6005 + | > loss: 0.43249 (0.57860) + | > log_mle: -0.03590 (0.12951) + | > loss_dur: 0.46839 (0.44909) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.72978 (4.02772) + | > current_lr: 0.00001 + | > step_time: 3.58730 (3.09632) + | > loader_time: 0.00220 (0.05410) + + + --> STEP: 160/234 -- GLOBAL_STEP: 6010 + | > loss: 0.41604 (0.57444) + | > log_mle: -0.03291 (0.12544) + | > loss_dur: 0.44896 (0.44900) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.99846 (4.25913) + | > current_lr: 0.00001 + | > step_time: 6.53040 (3.19892) + | > loader_time: 0.19730 (0.05494) + + + --> STEP: 165/234 -- GLOBAL_STEP: 6015 + | > loss: 0.43453 (0.57057) + | > log_mle: -0.02733 (0.12154) + | > loss_dur: 0.46186 (0.44903) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.97666 (4.45642) + | > current_lr: 0.00001 + | > step_time: 3.30210 (3.16596) + | > loader_time: 0.00710 (0.05462) + + + --> STEP: 170/234 -- GLOBAL_STEP: 6020 + | > loss: 0.43267 (0.56711) + | > log_mle: -0.05629 (0.11738) + | > loss_dur: 0.48896 (0.44972) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.39268 (4.66401) + | > current_lr: 0.00001 + | > step_time: 2.09710 (3.13577) + | > loader_time: 0.00280 (0.05464) + + + --> STEP: 175/234 -- GLOBAL_STEP: 6025 + | > loss: 0.43041 (0.56278) + | > log_mle: -0.03485 (0.11262) + | > loss_dur: 0.46527 (0.45016) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.86244 (4.89465) + | > current_lr: 0.00001 + | > step_time: 2.38210 (3.16386) + | > loader_time: 0.00730 (0.05692) + + + --> STEP: 180/234 -- GLOBAL_STEP: 6030 + | > loss: 0.43567 (0.55923) + | > log_mle: -0.04717 (0.10823) + | > loss_dur: 0.48283 (0.45100) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.67099 (5.12696) + | > current_lr: 0.00001 + | > step_time: 3.69150 (3.18434) + | > loader_time: 0.00580 (0.05653) + + + --> STEP: 185/234 -- GLOBAL_STEP: 6035 + | > loss: 0.43448 (0.55615) + | > log_mle: -0.06426 (0.10414) + | > loss_dur: 0.49874 (0.45201) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.74442 (5.36135) + | > current_lr: 0.00001 + | > step_time: 4.69650 (3.19559) + | > loader_time: 0.10620 (0.05757) + + + --> STEP: 190/234 -- GLOBAL_STEP: 6040 + | > loss: 0.42084 (0.55278) + | > log_mle: -0.05055 (0.10014) + | > loss_dur: 0.47139 (0.45264) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.44869 (5.59850) + | > current_lr: 0.00001 + | > step_time: 2.10350 (3.21168) + | > loader_time: 0.08810 (0.05958) + + + --> STEP: 195/234 -- GLOBAL_STEP: 6045 + | > loss: 0.43463 (0.54921) + | > log_mle: -0.05599 (0.09601) + | > loss_dur: 0.49062 (0.45320) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.59420 (5.83632) + | > current_lr: 0.00001 + | > step_time: 3.31260 (3.29867) + | > loader_time: 0.09580 (0.06059) + + + --> STEP: 200/234 -- GLOBAL_STEP: 6050 + | > loss: 0.42244 (0.54590) + | > log_mle: -0.06081 (0.09217) + | > loss_dur: 0.48325 (0.45373) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.08648 (6.03851) + | > current_lr: 0.00001 + | > step_time: 10.89360 (3.43455) + | > loader_time: 0.09520 (0.06058) + + + --> STEP: 205/234 -- GLOBAL_STEP: 6055 + | > loss: 0.42865 (0.54266) + | > log_mle: -0.05494 (0.08839) + | > loss_dur: 0.48360 (0.45427) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.83961 (6.25125) + | > current_lr: 0.00001 + | > step_time: 5.01190 (3.47881) + | > loader_time: 0.08500 (0.06147) + + + --> STEP: 210/234 -- GLOBAL_STEP: 6060 + | > loss: 0.37593 (0.53906) + | > log_mle: -0.12333 (0.08406) + | > loss_dur: 0.49927 (0.45500) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.02935 (6.54013) + | > current_lr: 0.00001 + | > step_time: 2.79660 (3.53071) + | > loader_time: 0.00270 (0.06376) + + + --> STEP: 215/234 -- GLOBAL_STEP: 6065 + | > loss: 0.40282 (0.53552) + | > log_mle: -0.07557 (0.07959) + | > loss_dur: 0.47839 (0.45593) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.79311 (6.78268) + | > current_lr: 0.00001 + | > step_time: 5.20330 (3.61098) + | > loader_time: 0.10230 (0.06462) + + + --> STEP: 220/234 -- GLOBAL_STEP: 6070 + | > loss: 0.38535 (0.53198) + | > log_mle: -0.11816 (0.07490) + | > loss_dur: 0.50351 (0.45708) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.44577 (7.02756) + | > current_lr: 0.00001 + | > step_time: 3.38530 (3.67539) + | > loader_time: 0.00800 (0.06490) + + + --> STEP: 225/234 -- GLOBAL_STEP: 6075 + | > loss: 0.33718 (0.52869) + | > log_mle: -0.17239 (0.07046) + | > loss_dur: 0.50957 (0.45824) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.47303 (7.29265) + | > current_lr: 0.00001 + | > step_time: 0.24630 (3.60675) + | > loader_time: 0.00330 (0.06354) + + + --> STEP: 230/234 -- GLOBAL_STEP: 6080 + | > loss: 0.39752 (0.52572) + | > log_mle: -0.21312 (0.06563) + | > loss_dur: 0.61064 (0.46009) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.92931 (7.52853) + | > current_lr: 0.00001 + | > step_time: 0.26120 (3.53368) + | > loader_time: 0.00440 (0.06224) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12983 (-0.21786) + | > avg_loss: 0.46314 (-0.03356) + | > avg_log_mle: -0.00589 (-0.00711) + | > avg_loss_dur: 0.46904 (-0.02644) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_6084.pth + + > EPOCH: 26/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 14:43:53)  + + --> STEP: 1/234 -- GLOBAL_STEP: 6085 + | > loss: 0.89697 (0.89697) + | > log_mle: 0.21475 (0.21475) + | > loss_dur: 0.68221 (0.68221) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.72876 (1.72876) + | > current_lr: 0.00001 + | > step_time: 1.59680 (1.59677) + | > loader_time: 0.00250 (0.00253) + + + --> STEP: 6/234 -- GLOBAL_STEP: 6090 + | > loss: 0.75949 (0.83247) + | > log_mle: 0.19634 (0.20009) + | > loss_dur: 0.56315 (0.63238) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.60707 (2.02520) + | > current_lr: 0.00001 + | > step_time: 15.41370 (5.86605) + | > loader_time: 0.19180 (0.13067) + + + --> STEP: 11/234 -- GLOBAL_STEP: 6095 + | > loss: 0.68213 (0.77180) + | > log_mle: 0.18919 (0.18813) + | > loss_dur: 0.49294 (0.58367) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.11239 (1.79455) + | > current_lr: 0.00001 + | > step_time: 1.39900 (4.88236) + | > loader_time: 0.00660 (0.11648) + + + --> STEP: 16/234 -- GLOBAL_STEP: 6100 + | > loss: 0.62150 (0.73251) + | > log_mle: 0.16730 (0.18455) + | > loss_dur: 0.45420 (0.54796) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.06564 (1.62220) + | > current_lr: 0.00001 + | > step_time: 6.59430 (4.48794) + | > loader_time: 0.00620 (0.09198) + + + --> STEP: 21/234 -- GLOBAL_STEP: 6105 + | > loss: 0.60515 (0.71141) + | > log_mle: 0.18815 (0.18435) + | > loss_dur: 0.41700 (0.52706) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.25126 (1.51450) + | > current_lr: 0.00001 + | > step_time: 7.10690 (4.34913) + | > loader_time: 0.09330 (0.08400) + + + --> STEP: 26/234 -- GLOBAL_STEP: 6110 + | > loss: 0.61021 (0.69133) + | > log_mle: 0.17078 (0.18199) + | > loss_dur: 0.43943 (0.50934) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.09102 (1.43351) + | > current_lr: 0.00001 + | > step_time: 4.21270 (4.19780) + | > loader_time: 0.09890 (0.07616) + + + --> STEP: 31/234 -- GLOBAL_STEP: 6115 + | > loss: 0.63174 (0.67667) + | > log_mle: 0.16020 (0.17944) + | > loss_dur: 0.47154 (0.49723) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.06880 (1.36366) + | > current_lr: 0.00001 + | > step_time: 9.89830 (4.15620) + | > loader_time: 0.00740 (0.06998) + + + --> STEP: 36/234 -- GLOBAL_STEP: 6120 + | > loss: 0.59415 (0.66500) + | > log_mle: 0.15804 (0.17662) + | > loss_dur: 0.43611 (0.48838) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.51459 (1.43346) + | > current_lr: 0.00001 + | > step_time: 3.39240 (4.16057) + | > loader_time: 0.00160 (0.07322) + + + --> STEP: 41/234 -- GLOBAL_STEP: 6125 + | > loss: 0.56392 (0.65507) + | > log_mle: 0.15446 (0.17476) + | > loss_dur: 0.40946 (0.48030) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.06652 (1.38891) + | > current_lr: 0.00001 + | > step_time: 2.09730 (3.88022) + | > loader_time: 0.00430 (0.06657) + + + --> STEP: 46/234 -- GLOBAL_STEP: 6130 + | > loss: 0.56971 (0.64629) + | > log_mle: 0.16066 (0.17344) + | > loss_dur: 0.40905 (0.47285) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.98887 (1.37149) + | > current_lr: 0.00001 + | > step_time: 1.59660 (3.71924) + | > loader_time: 0.00200 (0.06532) + + + --> STEP: 51/234 -- GLOBAL_STEP: 6135 + | > loss: 0.56442 (0.63673) + | > log_mle: 0.17416 (0.17248) + | > loss_dur: 0.39026 (0.46425) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.22845 (1.33001) + | > current_lr: 0.00001 + | > step_time: 4.21850 (3.62591) + | > loader_time: 0.19560 (0.06793) + + + --> STEP: 56/234 -- GLOBAL_STEP: 6140 + | > loss: 0.58049 (0.62973) + | > log_mle: 0.15686 (0.17086) + | > loss_dur: 0.42364 (0.45887) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.21403 (1.31196) + | > current_lr: 0.00001 + | > step_time: 1.71280 (3.50990) + | > loader_time: 0.00280 (0.06488) + + + --> STEP: 61/234 -- GLOBAL_STEP: 6145 + | > loss: 0.55651 (0.62281) + | > log_mle: 0.16149 (0.16939) + | > loss_dur: 0.39502 (0.45343) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.98217 (1.29538) + | > current_lr: 0.00001 + | > step_time: 1.39260 (3.36780) + | > loader_time: 0.00250 (0.06428) + + + --> STEP: 66/234 -- GLOBAL_STEP: 6150 + | > loss: 0.54877 (0.61685) + | > log_mle: 0.16608 (0.16787) + | > loss_dur: 0.38269 (0.44898) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.99223 (1.29667) + | > current_lr: 0.00001 + | > step_time: 2.60800 (3.28265) + | > loader_time: 0.19420 (0.06666) + + + --> STEP: 71/234 -- GLOBAL_STEP: 6155 + | > loss: 0.53161 (0.61190) + | > log_mle: 0.12107 (0.16625) + | > loss_dur: 0.41054 (0.44564) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.37171 (1.42933) + | > current_lr: 0.00001 + | > step_time: 1.80720 (3.23212) + | > loader_time: 0.00260 (0.06585) + + + --> STEP: 76/234 -- GLOBAL_STEP: 6160 + | > loss: 0.54037 (0.60641) + | > log_mle: 0.14493 (0.16473) + | > loss_dur: 0.39544 (0.44169) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.73378 (1.46821) + | > current_lr: 0.00001 + | > step_time: 1.88840 (3.13269) + | > loader_time: 0.00350 (0.06476) + + + --> STEP: 81/234 -- GLOBAL_STEP: 6165 + | > loss: 0.51459 (0.60121) + | > log_mle: 0.12580 (0.16331) + | > loss_dur: 0.38879 (0.43790) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.94231 (1.49703) + | > current_lr: 0.00001 + | > step_time: 3.68610 (3.17499) + | > loader_time: 0.10280 (0.06543) + + + --> STEP: 86/234 -- GLOBAL_STEP: 6170 + | > loss: 0.53303 (0.59696) + | > log_mle: 0.12601 (0.16174) + | > loss_dur: 0.40702 (0.43522) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.07552 (1.57118) + | > current_lr: 0.00001 + | > step_time: 0.88880 (3.09939) + | > loader_time: 0.00300 (0.06273) + + + --> STEP: 91/234 -- GLOBAL_STEP: 6175 + | > loss: 0.53698 (0.59193) + | > log_mle: 0.12481 (0.15911) + | > loss_dur: 0.41217 (0.43281) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.27370 (1.68754) + | > current_lr: 0.00001 + | > step_time: 1.00000 (3.02761) + | > loader_time: 0.00350 (0.06029) + + + --> STEP: 96/234 -- GLOBAL_STEP: 6180 + | > loss: 0.51910 (0.58615) + | > log_mle: 0.12962 (0.15517) + | > loss_dur: 0.38948 (0.43098) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.88926 (1.90465) + | > current_lr: 0.00001 + | > step_time: 1.97940 (2.96790) + | > loader_time: 0.00160 (0.05887) + + + --> STEP: 101/234 -- GLOBAL_STEP: 6185 + | > loss: 0.49249 (0.58160) + | > log_mle: 0.07802 (0.15215) + | > loss_dur: 0.41446 (0.42945) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.90036 (2.09512) + | > current_lr: 0.00001 + | > step_time: 1.57830 (2.93363) + | > loader_time: 0.00950 (0.05801) + + + --> STEP: 106/234 -- GLOBAL_STEP: 6190 + | > loss: 0.51777 (0.57787) + | > log_mle: 0.07932 (0.14911) + | > loss_dur: 0.43845 (0.42876) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.04925 (2.25893) + | > current_lr: 0.00001 + | > step_time: 2.23350 (2.90312) + | > loader_time: 0.00210 (0.05610) + + + --> STEP: 111/234 -- GLOBAL_STEP: 6195 + | > loss: 0.47359 (0.57359) + | > log_mle: 0.04063 (0.14595) + | > loss_dur: 0.43296 (0.42764) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.10821 (2.43748) + | > current_lr: 0.00001 + | > step_time: 3.01460 (2.85735) + | > loader_time: 0.08370 (0.05440) + + + --> STEP: 116/234 -- GLOBAL_STEP: 6200 + | > loss: 0.50479 (0.56981) + | > log_mle: 0.07091 (0.14293) + | > loss_dur: 0.43387 (0.42688) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.41380 (2.62256) + | > current_lr: 0.00001 + | > step_time: 2.49060 (2.85203) + | > loader_time: 0.00150 (0.05219) + + + --> STEP: 121/234 -- GLOBAL_STEP: 6205 + | > loss: 0.53427 (0.56655) + | > log_mle: 0.14665 (0.14072) + | > loss_dur: 0.38762 (0.42583) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.57656 (2.71956) + | > current_lr: 0.00001 + | > step_time: 2.91000 (2.82686) + | > loader_time: 0.10750 (0.05244) + + + --> STEP: 126/234 -- GLOBAL_STEP: 6210 + | > loss: 0.45686 (0.56268) + | > log_mle: 0.02324 (0.13795) + | > loss_dur: 0.43362 (0.42473) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.37478 (2.92120) + | > current_lr: 0.00001 + | > step_time: 1.41490 (2.83011) + | > loader_time: 0.00380 (0.05124) + + + --> STEP: 131/234 -- GLOBAL_STEP: 6215 + | > loss: 0.42176 (0.55857) + | > log_mle: -0.00173 (0.13446) + | > loss_dur: 0.42350 (0.42411) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.65772 (3.21422) + | > current_lr: 0.00001 + | > step_time: 2.01000 (2.80233) + | > loader_time: 0.08650 (0.05008) + + + --> STEP: 136/234 -- GLOBAL_STEP: 6220 + | > loss: 0.43498 (0.55502) + | > log_mle: -0.03945 (0.13106) + | > loss_dur: 0.47443 (0.42396) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 29.09883 (3.54976) + | > current_lr: 0.00001 + | > step_time: 3.40400 (2.83195) + | > loader_time: 0.07570 (0.05157) + + + --> STEP: 141/234 -- GLOBAL_STEP: 6225 + | > loss: 0.44026 (0.55171) + | > log_mle: 0.02850 (0.12802) + | > loss_dur: 0.41176 (0.42369) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.34031 (3.73264) + | > current_lr: 0.00001 + | > step_time: 1.80500 (2.84580) + | > loader_time: 0.00240 (0.05179) + + + --> STEP: 146/234 -- GLOBAL_STEP: 6230 + | > loss: 0.44397 (0.54774) + | > log_mle: -0.01255 (0.12355) + | > loss_dur: 0.45652 (0.42419) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.20501 (4.00101) + | > current_lr: 0.00001 + | > step_time: 6.09740 (2.84354) + | > loader_time: 0.29430 (0.05271) + + + --> STEP: 151/234 -- GLOBAL_STEP: 6235 + | > loss: 0.42667 (0.54377) + | > log_mle: 0.01259 (0.11984) + | > loss_dur: 0.41409 (0.42393) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.35469 (4.11751) + | > current_lr: 0.00001 + | > step_time: 3.90850 (2.97594) + | > loader_time: 0.00230 (0.05682) + + + --> STEP: 156/234 -- GLOBAL_STEP: 6240 + | > loss: 0.39485 (0.53921) + | > log_mle: -0.01646 (0.11501) + | > loss_dur: 0.41131 (0.42420) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.45779 (4.32869) + | > current_lr: 0.00001 + | > step_time: 2.89720 (3.04329) + | > loader_time: 0.00580 (0.06342) + + + --> STEP: 161/234 -- GLOBAL_STEP: 6245 + | > loss: 0.41606 (0.53544) + | > log_mle: -0.03349 (0.11086) + | > loss_dur: 0.44955 (0.42458) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.23213 (4.49870) + | > current_lr: 0.00001 + | > step_time: 3.39890 (3.04259) + | > loader_time: 0.00230 (0.06447) + + + --> STEP: 166/234 -- GLOBAL_STEP: 6250 + | > loss: 0.42913 (0.53176) + | > log_mle: 0.01121 (0.10723) + | > loss_dur: 0.41791 (0.42453) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.15453 (4.67815) + | > current_lr: 0.00001 + | > step_time: 2.30960 (3.13426) + | > loader_time: 0.18300 (0.06590) + + + --> STEP: 171/234 -- GLOBAL_STEP: 6255 + | > loss: 0.35700 (0.52800) + | > log_mle: -0.07915 (0.10255) + | > loss_dur: 0.43615 (0.42545) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.19192 (4.84451) + | > current_lr: 0.00001 + | > step_time: 2.20590 (3.10249) + | > loader_time: 0.00470 (0.06499) + + + --> STEP: 176/234 -- GLOBAL_STEP: 6260 + | > loss: 0.39371 (0.52406) + | > log_mle: -0.04915 (0.09795) + | > loss_dur: 0.44286 (0.42611) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.52514 (5.01785) + | > current_lr: 0.00001 + | > step_time: 5.20000 (3.13944) + | > loader_time: 0.11000 (0.06534) + + + --> STEP: 181/234 -- GLOBAL_STEP: 6265 + | > loss: 0.44663 (0.52075) + | > log_mle: -0.00111 (0.09381) + | > loss_dur: 0.44774 (0.42693) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.13865 (5.18469) + | > current_lr: 0.00001 + | > step_time: 3.71510 (3.19148) + | > loader_time: 0.19360 (0.06683) + + + --> STEP: 186/234 -- GLOBAL_STEP: 6270 + | > loss: 0.41558 (0.51752) + | > log_mle: -0.03227 (0.08954) + | > loss_dur: 0.44785 (0.42798) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.33845 (5.40544) + | > current_lr: 0.00001 + | > step_time: 10.29400 (3.27184) + | > loader_time: 0.00350 (0.06555) + + + --> STEP: 191/234 -- GLOBAL_STEP: 6275 + | > loss: 0.39704 (0.51400) + | > log_mle: -0.03925 (0.08550) + | > loss_dur: 0.43629 (0.42850) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.90931 (5.61630) + | > current_lr: 0.00001 + | > step_time: 1.89800 (3.35328) + | > loader_time: 0.19470 (0.06736) + + + --> STEP: 196/234 -- GLOBAL_STEP: 6280 + | > loss: 0.41350 (0.51065) + | > log_mle: -0.03955 (0.08137) + | > loss_dur: 0.45306 (0.42928) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.52205 (5.86839) + | > current_lr: 0.00001 + | > step_time: 10.49780 (3.44227) + | > loader_time: 0.01150 (0.06768) + + + --> STEP: 201/234 -- GLOBAL_STEP: 6285 + | > loss: 0.41974 (0.50753) + | > log_mle: -0.02034 (0.07763) + | > loss_dur: 0.44009 (0.42990) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.86742 (6.12774) + | > current_lr: 0.00001 + | > step_time: 4.70480 (3.49725) + | > loader_time: 0.00470 (0.06959) + + + --> STEP: 206/234 -- GLOBAL_STEP: 6290 + | > loss: 0.35155 (0.50411) + | > log_mle: -0.09807 (0.07347) + | > loss_dur: 0.44962 (0.43065) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.27813 (6.38970) + | > current_lr: 0.00001 + | > step_time: 5.81240 (3.55907) + | > loader_time: 0.18330 (0.06972) + + + --> STEP: 211/234 -- GLOBAL_STEP: 6295 + | > loss: 0.33997 (0.50055) + | > log_mle: -0.16161 (0.06882) + | > loss_dur: 0.50159 (0.43173) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.27163 (6.64503) + | > current_lr: 0.00001 + | > step_time: 5.29880 (3.66463) + | > loader_time: 0.00600 (0.06957) + + + --> STEP: 216/234 -- GLOBAL_STEP: 6300 + | > loss: 0.33532 (0.49705) + | > log_mle: -0.14796 (0.06441) + | > loss_dur: 0.48328 (0.43264) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 21.72283 (6.87798) + | > current_lr: 0.00001 + | > step_time: 6.39650 (3.71368) + | > loader_time: 0.00590 (0.06978) + + + --> STEP: 221/234 -- GLOBAL_STEP: 6305 + | > loss: 0.38675 (0.49371) + | > log_mle: -0.08178 (0.06002) + | > loss_dur: 0.46853 (0.43369) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.21405 (7.14362) + | > current_lr: 0.00001 + | > step_time: 2.38550 (3.73270) + | > loader_time: 0.00250 (0.06921) + + + --> STEP: 226/234 -- GLOBAL_STEP: 6310 + | > loss: 0.31863 (0.49019) + | > log_mle: -0.16673 (0.05519) + | > loss_dur: 0.48536 (0.43500) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.92456 (7.38733) + | > current_lr: 0.00001 + | > step_time: 0.23800 (3.66569) + | > loader_time: 0.00350 (0.06813) + + + --> STEP: 231/234 -- GLOBAL_STEP: 6315 + | > loss: 0.39231 (0.48755) + | > log_mle: -0.21869 (0.05014) + | > loss_dur: 0.61100 (0.43741) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.81231 (7.61233) + | > current_lr: 0.00001 + | > step_time: 0.27590 (3.59191) + | > loader_time: 0.00380 (0.06673) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.44963 (+0.31979) + | > avg_loss: 0.42029 (-0.04286) + | > avg_log_mle: -0.02682 (-0.02093) + | > avg_loss_dur: 0.44711 (-0.02192) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_6318.pth + + > EPOCH: 27/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 14:59:16)  + + --> STEP: 2/234 -- GLOBAL_STEP: 6320 + | > loss: 0.88640 (0.87497) + | > log_mle: 0.21485 (0.20753) + | > loss_dur: 0.67155 (0.66744) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.45492 (1.52602) + | > current_lr: 0.00001 + | > step_time: 6.28110 (11.34723) + | > loader_time: 0.00090 (0.04762) + + + --> STEP: 7/234 -- GLOBAL_STEP: 6325 + | > loss: 0.68503 (0.76701) + | > log_mle: 0.15715 (0.18163) + | > loss_dur: 0.52788 (0.58537) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.51751 (2.08478) + | > current_lr: 0.00001 + | > step_time: 4.21120 (6.34512) + | > loader_time: 0.00190 (0.04342) + + + --> STEP: 12/234 -- GLOBAL_STEP: 6330 + | > loss: 0.58308 (0.71064) + | > log_mle: 0.16500 (0.17295) + | > loss_dur: 0.41808 (0.53768) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.24190 (1.81139) + | > current_lr: 0.00001 + | > step_time: 2.40750 (5.21876) + | > loader_time: 0.00150 (0.03397) + + + --> STEP: 17/234 -- GLOBAL_STEP: 6335 + | > loss: 0.60850 (0.68141) + | > log_mle: 0.17541 (0.17049) + | > loss_dur: 0.43309 (0.51091) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.30027 (1.64851) + | > current_lr: 0.00001 + | > step_time: 3.50820 (4.89653) + | > loader_time: 0.00300 (0.03050) + + + --> STEP: 22/234 -- GLOBAL_STEP: 6340 + | > loss: 0.55143 (0.66016) + | > log_mle: 0.14882 (0.16909) + | > loss_dur: 0.40261 (0.49106) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.25001 (1.53927) + | > current_lr: 0.00001 + | > step_time: 3.39930 (4.82557) + | > loader_time: 0.00250 (0.03301) + + + --> STEP: 27/234 -- GLOBAL_STEP: 6345 + | > loss: 0.54559 (0.64268) + | > log_mle: 0.14814 (0.16706) + | > loss_dur: 0.39744 (0.47562) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.01196 (1.45642) + | > current_lr: 0.00001 + | > step_time: 3.60780 (4.68905) + | > loader_time: 0.00350 (0.04093) + + + --> STEP: 32/234 -- GLOBAL_STEP: 6350 + | > loss: 0.51861 (0.62910) + | > log_mle: 0.13135 (0.16417) + | > loss_dur: 0.38726 (0.46493) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.01548 (1.38745) + | > current_lr: 0.00001 + | > step_time: 7.79890 (5.01843) + | > loader_time: 0.08550 (0.04650) + + + --> STEP: 37/234 -- GLOBAL_STEP: 6355 + | > loss: 0.51905 (0.61915) + | > log_mle: 0.14644 (0.16203) + | > loss_dur: 0.37261 (0.45712) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.84368 (1.49622) + | > current_lr: 0.00001 + | > step_time: 0.89620 (4.55339) + | > loader_time: 0.00160 (0.04048) + + + --> STEP: 42/234 -- GLOBAL_STEP: 6360 + | > loss: 0.55824 (0.61246) + | > log_mle: 0.17076 (0.16092) + | > loss_dur: 0.38748 (0.45154) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.90694 (1.44677) + | > current_lr: 0.00001 + | > step_time: 2.10120 (4.21833) + | > loader_time: 0.00260 (0.03597) + + + --> STEP: 47/234 -- GLOBAL_STEP: 6365 + | > loss: 0.51953 (0.60385) + | > log_mle: 0.14924 (0.15916) + | > loss_dur: 0.37030 (0.44469) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.81838 (1.42689) + | > current_lr: 0.00001 + | > step_time: 3.06020 (4.03858) + | > loader_time: 0.10870 (0.03841) + + + --> STEP: 52/234 -- GLOBAL_STEP: 6370 + | > loss: 0.53644 (0.59546) + | > log_mle: 0.15830 (0.15845) + | > loss_dur: 0.37814 (0.43701) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.50590 (1.38717) + | > current_lr: 0.00001 + | > step_time: 1.67980 (3.86587) + | > loader_time: 0.00210 (0.03499) + + + --> STEP: 57/234 -- GLOBAL_STEP: 6375 + | > loss: 0.53564 (0.58939) + | > log_mle: 0.15830 (0.15689) + | > loss_dur: 0.37734 (0.43250) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.82010 (1.35267) + | > current_lr: 0.00001 + | > step_time: 1.00370 (3.70052) + | > loader_time: 0.00530 (0.03374) + + + --> STEP: 62/234 -- GLOBAL_STEP: 6380 + | > loss: 0.49365 (0.58193) + | > log_mle: 0.09990 (0.15449) + | > loss_dur: 0.39375 (0.42744) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.06075 (1.38235) + | > current_lr: 0.00001 + | > step_time: 1.30520 (3.63408) + | > loader_time: 0.00260 (0.03269) + + + --> STEP: 67/234 -- GLOBAL_STEP: 6385 + | > loss: 0.47005 (0.57590) + | > log_mle: 0.11682 (0.15336) + | > loss_dur: 0.35323 (0.42253) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.50806 (1.41668) + | > current_lr: 0.00001 + | > step_time: 5.80200 (3.60053) + | > loader_time: 0.10070 (0.03332) + + + --> STEP: 72/234 -- GLOBAL_STEP: 6390 + | > loss: 0.49941 (0.57191) + | > log_mle: 0.13363 (0.15209) + | > loss_dur: 0.36577 (0.41981) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.71975 (1.52041) + | > current_lr: 0.00001 + | > step_time: 2.60180 (3.50919) + | > loader_time: 0.08820 (0.03570) + + + --> STEP: 77/234 -- GLOBAL_STEP: 6395 + | > loss: 0.49033 (0.56695) + | > log_mle: 0.11856 (0.15041) + | > loss_dur: 0.37177 (0.41654) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.12580 (1.55843) + | > current_lr: 0.00001 + | > step_time: 4.59520 (3.46708) + | > loader_time: 0.00380 (0.03561) + + + --> STEP: 82/234 -- GLOBAL_STEP: 6400 + | > loss: 0.48453 (0.56207) + | > log_mle: 0.13822 (0.14931) + | > loss_dur: 0.34631 (0.41276) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.78275 (1.58125) + | > current_lr: 0.00001 + | > step_time: 1.79980 (3.35452) + | > loader_time: 0.00140 (0.03559) + + + --> STEP: 87/234 -- GLOBAL_STEP: 6405 + | > loss: 0.48157 (0.55787) + | > log_mle: 0.11762 (0.14755) + | > loss_dur: 0.36395 (0.41033) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.42936 (1.67391) + | > current_lr: 0.00001 + | > step_time: 4.20720 (3.34793) + | > loader_time: 0.00680 (0.03707) + + + --> STEP: 92/234 -- GLOBAL_STEP: 6410 + | > loss: 0.45281 (0.55308) + | > log_mle: 0.08239 (0.14457) + | > loss_dur: 0.37043 (0.40851) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.67958 (1.85953) + | > current_lr: 0.00001 + | > step_time: 1.89650 (3.39448) + | > loader_time: 0.00450 (0.03922) + + + --> STEP: 97/234 -- GLOBAL_STEP: 6415 + | > loss: 0.46029 (0.54780) + | > log_mle: 0.08800 (0.14071) + | > loss_dur: 0.37229 (0.40709) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.30358 (2.13021) + | > current_lr: 0.00001 + | > step_time: 1.18860 (3.34819) + | > loader_time: 0.00400 (0.03743) + + + --> STEP: 102/234 -- GLOBAL_STEP: 6420 + | > loss: 0.47590 (0.54371) + | > log_mle: 0.11010 (0.13797) + | > loss_dur: 0.36580 (0.40574) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.15211 (2.34357) + | > current_lr: 0.00001 + | > step_time: 1.37110 (3.28308) + | > loader_time: 0.00290 (0.03729) + + + --> STEP: 107/234 -- GLOBAL_STEP: 6425 + | > loss: 0.43711 (0.53990) + | > log_mle: 0.06179 (0.13447) + | > loss_dur: 0.37532 (0.40543) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.98337 (2.53059) + | > current_lr: 0.00001 + | > step_time: 4.69450 (3.23783) + | > loader_time: 0.00500 (0.03571) + + + --> STEP: 112/234 -- GLOBAL_STEP: 6430 + | > loss: 0.45009 (0.53600) + | > log_mle: 0.05712 (0.13129) + | > loss_dur: 0.39297 (0.40470) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.53930 (2.69587) + | > current_lr: 0.00001 + | > step_time: 1.87470 (3.20850) + | > loader_time: 0.00240 (0.03507) + + + --> STEP: 117/234 -- GLOBAL_STEP: 6435 + | > loss: 0.45928 (0.53268) + | > log_mle: 0.06823 (0.12843) + | > loss_dur: 0.39105 (0.40425) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.13609 (2.89289) + | > current_lr: 0.00001 + | > step_time: 1.70040 (3.13208) + | > loader_time: 0.08370 (0.03644) + + + --> STEP: 122/234 -- GLOBAL_STEP: 6440 + | > loss: 0.45418 (0.52955) + | > log_mle: 0.07849 (0.12636) + | > loss_dur: 0.37569 (0.40319) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.62333 (3.01531) + | > current_lr: 0.00001 + | > step_time: 4.21330 (3.10459) + | > loader_time: 0.08580 (0.03717) + + + --> STEP: 127/234 -- GLOBAL_STEP: 6445 + | > loss: 0.43055 (0.52587) + | > log_mle: 0.03501 (0.12324) + | > loss_dur: 0.39554 (0.40263) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.17673 (3.13820) + | > current_lr: 0.00001 + | > step_time: 2.80740 (3.05881) + | > loader_time: 0.08540 (0.03713) + + + --> STEP: 132/234 -- GLOBAL_STEP: 6450 + | > loss: 0.43943 (0.52208) + | > log_mle: 0.05209 (0.11986) + | > loss_dur: 0.38734 (0.40222) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.38857 (3.27889) + | > current_lr: 0.00001 + | > step_time: 1.31110 (3.06662) + | > loader_time: 0.00270 (0.03777) + + + --> STEP: 137/234 -- GLOBAL_STEP: 6455 + | > loss: 0.45732 (0.51904) + | > log_mle: 0.04517 (0.11637) + | > loss_dur: 0.41216 (0.40266) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.87367 (3.42853) + | > current_lr: 0.00001 + | > step_time: 1.90950 (3.04737) + | > loader_time: 0.08380 (0.03827) + + + --> STEP: 142/234 -- GLOBAL_STEP: 6460 + | > loss: 0.41602 (0.51547) + | > log_mle: 0.02366 (0.11318) + | > loss_dur: 0.39235 (0.40229) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.40395 (3.59060) + | > current_lr: 0.00001 + | > step_time: 5.50930 (3.07725) + | > loader_time: 0.08770 (0.03906) + + + --> STEP: 147/234 -- GLOBAL_STEP: 6465 + | > loss: 0.40739 (0.51151) + | > log_mle: 0.02178 (0.10869) + | > loss_dur: 0.38561 (0.40282) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.17091 (3.80044) + | > current_lr: 0.00001 + | > step_time: 4.00160 (3.06104) + | > loader_time: 0.00290 (0.03838) + + + --> STEP: 152/234 -- GLOBAL_STEP: 6470 + | > loss: 0.38725 (0.50755) + | > log_mle: -0.04043 (0.10458) + | > loss_dur: 0.42768 (0.40298) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.06496 (3.98896) + | > current_lr: 0.00001 + | > step_time: 3.80110 (3.04246) + | > loader_time: 0.19320 (0.03970) + + + --> STEP: 157/234 -- GLOBAL_STEP: 6475 + | > loss: 0.41019 (0.50338) + | > log_mle: 0.00515 (0.10006) + | > loss_dur: 0.40504 (0.40332) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.95961 (4.27026) + | > current_lr: 0.00001 + | > step_time: 1.30120 (3.07558) + | > loader_time: 0.18040 (0.04027) + + + --> STEP: 162/234 -- GLOBAL_STEP: 6480 + | > loss: 0.37756 (0.49944) + | > log_mle: -0.02796 (0.09573) + | > loss_dur: 0.40552 (0.40371) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.98115 (4.47663) + | > current_lr: 0.00001 + | > step_time: 0.99860 (3.06566) + | > loader_time: 0.09570 (0.04202) + + + --> STEP: 167/234 -- GLOBAL_STEP: 6485 + | > loss: 0.35079 (0.49568) + | > log_mle: -0.08706 (0.09175) + | > loss_dur: 0.43786 (0.40393) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.70099 (4.62426) + | > current_lr: 0.00001 + | > step_time: 1.40130 (3.05245) + | > loader_time: 0.00230 (0.04190) + + + --> STEP: 172/234 -- GLOBAL_STEP: 6490 + | > loss: 0.34764 (0.49198) + | > log_mle: -0.08203 (0.08714) + | > loss_dur: 0.42966 (0.40484) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.50202 (4.80044) + | > current_lr: 0.00001 + | > step_time: 2.89140 (3.07416) + | > loader_time: 0.00300 (0.04287) + + + --> STEP: 177/234 -- GLOBAL_STEP: 6495 + | > loss: 0.40062 (0.48837) + | > log_mle: -0.04795 (0.08274) + | > loss_dur: 0.44856 (0.40563) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.49477 (4.97049) + | > current_lr: 0.00001 + | > step_time: 6.59820 (3.14218) + | > loader_time: 0.00630 (0.04332) + + + --> STEP: 182/234 -- GLOBAL_STEP: 6500 + | > loss: 0.37374 (0.48500) + | > log_mle: -0.08783 (0.07841) + | > loss_dur: 0.46157 (0.40659) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.55001 (5.23626) + | > current_lr: 0.00001 + | > step_time: 2.19750 (3.18215) + | > loader_time: 0.20600 (0.04434) + + + --> STEP: 187/234 -- GLOBAL_STEP: 6505 + | > loss: 0.32193 (0.48152) + | > log_mle: -0.09002 (0.07416) + | > loss_dur: 0.41194 (0.40736) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.35381 (5.45666) + | > current_lr: 0.00001 + | > step_time: 1.99320 (3.17464) + | > loader_time: 0.09830 (0.04619) + + + --> STEP: 192/234 -- GLOBAL_STEP: 6510 + | > loss: 0.32180 (0.47804) + | > log_mle: -0.11024 (0.07004) + | > loss_dur: 0.43205 (0.40800) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.30379 (5.66204) + | > current_lr: 0.00001 + | > step_time: 4.89490 (3.21225) + | > loader_time: 0.39930 (0.04959) + + + --> STEP: 197/234 -- GLOBAL_STEP: 6515 + | > loss: 0.33088 (0.47483) + | > log_mle: -0.08980 (0.06603) + | > loss_dur: 0.42068 (0.40880) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.94848 (5.87620) + | > current_lr: 0.00001 + | > step_time: 3.39180 (3.30283) + | > loader_time: 0.00540 (0.04936) + + + --> STEP: 202/234 -- GLOBAL_STEP: 6520 + | > loss: 0.29140 (0.47162) + | > log_mle: -0.16467 (0.06191) + | > loss_dur: 0.45606 (0.40971) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.40837 (6.09522) + | > current_lr: 0.00001 + | > step_time: 5.00630 (3.31564) + | > loader_time: 0.08630 (0.04864) + + + --> STEP: 207/234 -- GLOBAL_STEP: 6525 + | > loss: 0.30847 (0.46829) + | > log_mle: -0.15281 (0.05781) + | > loss_dur: 0.46128 (0.41048) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.68184 (6.28867) + | > current_lr: 0.00001 + | > step_time: 7.59850 (3.38926) + | > loader_time: 0.19580 (0.04940) + + + --> STEP: 212/234 -- GLOBAL_STEP: 6530 + | > loss: 0.32502 (0.46484) + | > log_mle: -0.13758 (0.05324) + | > loss_dur: 0.46260 (0.41160) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.64742 (6.50410) + | > current_lr: 0.00001 + | > step_time: 9.90040 (3.47492) + | > loader_time: 0.39420 (0.05150) + + + --> STEP: 217/234 -- GLOBAL_STEP: 6535 + | > loss: 0.31633 (0.46122) + | > log_mle: -0.15539 (0.04876) + | > loss_dur: 0.47172 (0.41246) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.75686 (6.68640) + | > current_lr: 0.00001 + | > step_time: 2.10040 (3.55018) + | > loader_time: 0.00390 (0.05217) + + + --> STEP: 222/234 -- GLOBAL_STEP: 6540 + | > loss: 0.32382 (0.45795) + | > log_mle: -0.16112 (0.04435) + | > loss_dur: 0.48494 (0.41360) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.16422 (6.92725) + | > current_lr: 0.00001 + | > step_time: 4.09500 (3.56709) + | > loader_time: 0.00320 (0.05112) + + + --> STEP: 227/234 -- GLOBAL_STEP: 6545 + | > loss: 0.32784 (0.45440) + | > log_mle: -0.14540 (0.03960) + | > loss_dur: 0.47324 (0.41480) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.95868 (7.16336) + | > current_lr: 0.00001 + | > step_time: 0.25720 (3.54733) + | > loader_time: 0.00370 (0.05009) + + + --> STEP: 232/234 -- GLOBAL_STEP: 6550 + | > loss: 0.45050 (0.45238) + | > log_mle: -0.30926 (0.03385) + | > loss_dur: 0.75976 (0.41853) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 30.16382 (7.49968) + | > current_lr: 0.00001 + | > step_time: 0.33580 (3.47689) + | > loader_time: 0.00440 (0.04909) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04123 (-0.40840) + | > avg_loss: 0.38820 (-0.03208) + | > avg_log_mle: -0.03391 (-0.00708) + | > avg_loss_dur: 0.42211 (-0.02500) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_6552.pth + + > EPOCH: 28/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 15:14:25)  + + --> STEP: 3/234 -- GLOBAL_STEP: 6555 + | > loss: 0.66613 (0.75542) + | > log_mle: 0.16735 (0.18470) + | > loss_dur: 0.49878 (0.57072) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.50331 (1.49862) + | > current_lr: 0.00001 + | > step_time: 3.91140 (5.00824) + | > loader_time: 0.00440 (0.86820) + + + --> STEP: 8/234 -- GLOBAL_STEP: 6560 + | > loss: 0.62310 (0.69927) + | > log_mle: 0.13994 (0.16424) + | > loss_dur: 0.48317 (0.53503) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.57385 (2.06903) + | > current_lr: 0.00001 + | > step_time: 5.70570 (4.42078) + | > loader_time: 0.00770 (0.37146) + + + --> STEP: 13/234 -- GLOBAL_STEP: 6565 + | > loss: 0.58910 (0.65165) + | > log_mle: 0.16118 (0.15928) + | > loss_dur: 0.42793 (0.49238) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.49540 (2.02207) + | > current_lr: 0.00001 + | > step_time: 4.59130 (4.92620) + | > loader_time: 0.00130 (0.25154) + + + --> STEP: 18/234 -- GLOBAL_STEP: 6570 + | > loss: 0.55780 (0.63022) + | > log_mle: 0.14986 (0.15631) + | > loss_dur: 0.40794 (0.47391) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.56509 (1.84404) + | > current_lr: 0.00001 + | > step_time: 3.40260 (5.21956) + | > loader_time: 0.00300 (0.20299) + + + --> STEP: 23/234 -- GLOBAL_STEP: 6575 + | > loss: 0.51789 (0.61041) + | > log_mle: 0.14105 (0.15464) + | > loss_dur: 0.37684 (0.45576) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.15077 (1.71824) + | > current_lr: 0.00001 + | > step_time: 4.51570 (5.24927) + | > loader_time: 0.19350 (0.16816) + + + --> STEP: 28/234 -- GLOBAL_STEP: 6580 + | > loss: 0.52009 (0.59537) + | > log_mle: 0.14325 (0.15286) + | > loss_dur: 0.37684 (0.44251) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.01484 (1.61721) + | > current_lr: 0.00001 + | > step_time: 2.80370 (4.91976) + | > loader_time: 0.00190 (0.14866) + + + --> STEP: 33/234 -- GLOBAL_STEP: 6585 + | > loss: 0.55495 (0.58469) + | > log_mle: 0.15071 (0.15033) + | > loss_dur: 0.40425 (0.43436) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.88354 (1.52397) + | > current_lr: 0.00001 + | > step_time: 2.98590 (4.95120) + | > loader_time: 0.10450 (0.13570) + + + --> STEP: 38/234 -- GLOBAL_STEP: 6590 + | > loss: 0.53743 (0.57575) + | > log_mle: 0.12930 (0.14774) + | > loss_dur: 0.40813 (0.42801) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.94237 (1.62760) + | > current_lr: 0.00001 + | > step_time: 2.21700 (4.68539) + | > loader_time: 0.00220 (0.12329) + + + --> STEP: 43/234 -- GLOBAL_STEP: 6595 + | > loss: 0.49259 (0.56873) + | > log_mle: 0.12686 (0.14661) + | > loss_dur: 0.36572 (0.42212) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.98328 (1.58922) + | > current_lr: 0.00001 + | > step_time: 1.39100 (4.36907) + | > loader_time: 0.00170 (0.11334) + + + --> STEP: 48/234 -- GLOBAL_STEP: 6600 + | > loss: 0.48519 (0.56097) + | > log_mle: 0.13575 (0.14512) + | > loss_dur: 0.34944 (0.41585) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.85744 (1.54393) + | > current_lr: 0.00001 + | > step_time: 2.92460 (4.13903) + | > loader_time: 0.08190 (0.10526) + + + --> STEP: 53/234 -- GLOBAL_STEP: 6605 + | > loss: 0.48750 (0.55369) + | > log_mle: 0.11756 (0.14406) + | > loss_dur: 0.36994 (0.40962) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.04483 (1.49946) + | > current_lr: 0.00001 + | > step_time: 1.61070 (3.93907) + | > loader_time: 0.08280 (0.09706) + + + --> STEP: 58/234 -- GLOBAL_STEP: 6610 + | > loss: 0.48132 (0.54877) + | > log_mle: 0.13726 (0.14291) + | > loss_dur: 0.34406 (0.40585) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.46197 (1.46427) + | > current_lr: 0.00001 + | > step_time: 2.59040 (3.77213) + | > loader_time: 0.00170 (0.08889) + + + --> STEP: 63/234 -- GLOBAL_STEP: 6615 + | > loss: 0.49055 (0.54233) + | > log_mle: 0.11921 (0.14028) + | > loss_dur: 0.37134 (0.40205) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.73391 (1.48869) + | > current_lr: 0.00001 + | > step_time: 3.10410 (3.64367) + | > loader_time: 0.00270 (0.08213) + + + --> STEP: 68/234 -- GLOBAL_STEP: 6620 + | > loss: 0.48574 (0.53710) + | > log_mle: 0.12910 (0.13930) + | > loss_dur: 0.35663 (0.39781) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.19161 (1.51098) + | > current_lr: 0.00001 + | > step_time: 1.88160 (3.58720) + | > loader_time: 0.00200 (0.08026) + + + --> STEP: 73/234 -- GLOBAL_STEP: 6625 + | > loss: 0.47105 (0.53329) + | > log_mle: 0.10357 (0.13769) + | > loss_dur: 0.36747 (0.39560) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.58539 (1.66727) + | > current_lr: 0.00001 + | > step_time: 3.60650 (3.50470) + | > loader_time: 0.00260 (0.08009) + + + --> STEP: 78/234 -- GLOBAL_STEP: 6630 + | > loss: 0.47577 (0.52901) + | > log_mle: 0.12880 (0.13637) + | > loss_dur: 0.34697 (0.39264) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.37193 (1.72553) + | > current_lr: 0.00001 + | > step_time: 4.18930 (3.48805) + | > loader_time: 0.10200 (0.07988) + + + --> STEP: 83/234 -- GLOBAL_STEP: 6635 + | > loss: 0.47375 (0.52446) + | > log_mle: 0.10332 (0.13498) + | > loss_dur: 0.37043 (0.38948) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.75742 (1.78546) + | > current_lr: 0.00001 + | > step_time: 2.81240 (3.48901) + | > loader_time: 0.08410 (0.07964) + + + --> STEP: 88/234 -- GLOBAL_STEP: 6640 + | > loss: 0.41602 (0.52009) + | > log_mle: 0.06361 (0.13282) + | > loss_dur: 0.35241 (0.38726) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.51864 (1.88505) + | > current_lr: 0.00001 + | > step_time: 2.21360 (3.43613) + | > loader_time: 0.08760 (0.07623) + + + --> STEP: 93/234 -- GLOBAL_STEP: 6645 + | > loss: 0.42290 (0.51554) + | > log_mle: 0.04989 (0.12978) + | > loss_dur: 0.37301 (0.38576) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.72492 (2.09867) + | > current_lr: 0.00001 + | > step_time: 1.38820 (3.42976) + | > loader_time: 0.00350 (0.07517) + + + --> STEP: 98/234 -- GLOBAL_STEP: 6650 + | > loss: 0.46982 (0.51106) + | > log_mle: 0.11700 (0.12668) + | > loss_dur: 0.35282 (0.38438) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.05367 (2.29263) + | > current_lr: 0.00001 + | > step_time: 2.41180 (3.33749) + | > loader_time: 0.00340 (0.07147) + + + --> STEP: 103/234 -- GLOBAL_STEP: 6655 + | > loss: 0.43460 (0.50672) + | > log_mle: 0.03369 (0.12313) + | > loss_dur: 0.40091 (0.38359) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.17525 (2.53151) + | > current_lr: 0.00001 + | > step_time: 3.60620 (3.30279) + | > loader_time: 0.08670 (0.07125) + + + --> STEP: 108/234 -- GLOBAL_STEP: 6660 + | > loss: 0.41066 (0.50294) + | > log_mle: 0.08316 (0.12014) + | > loss_dur: 0.32751 (0.38280) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.44316 (2.68619) + | > current_lr: 0.00001 + | > step_time: 2.29970 (3.25286) + | > loader_time: 0.00270 (0.07047) + + + --> STEP: 113/234 -- GLOBAL_STEP: 6665 + | > loss: 0.39972 (0.49908) + | > log_mle: 0.03519 (0.11659) + | > loss_dur: 0.36453 (0.38249) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.47634 (2.96753) + | > current_lr: 0.00001 + | > step_time: 1.79520 (3.18172) + | > loader_time: 0.00480 (0.06890) + + + --> STEP: 118/234 -- GLOBAL_STEP: 6670 + | > loss: 0.44330 (0.49635) + | > log_mle: 0.06617 (0.11401) + | > loss_dur: 0.37713 (0.38233) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.83874 (3.16943) + | > current_lr: 0.00001 + | > step_time: 2.79860 (3.14945) + | > loader_time: 0.10460 (0.06855) + + + --> STEP: 123/234 -- GLOBAL_STEP: 6675 + | > loss: 0.42262 (0.49322) + | > log_mle: 0.09357 (0.11216) + | > loss_dur: 0.32905 (0.38106) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.26791 (3.25868) + | > current_lr: 0.00001 + | > step_time: 3.69210 (3.13293) + | > loader_time: 0.00280 (0.06590) + + + --> STEP: 128/234 -- GLOBAL_STEP: 6680 + | > loss: 0.39762 (0.48957) + | > log_mle: 0.03655 (0.10863) + | > loss_dur: 0.36107 (0.38094) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.83604 (3.47463) + | > current_lr: 0.00001 + | > step_time: 1.51470 (3.09019) + | > loader_time: 0.08570 (0.06411) + + + --> STEP: 133/234 -- GLOBAL_STEP: 6685 + | > loss: 0.41123 (0.48590) + | > log_mle: 0.02087 (0.10517) + | > loss_dur: 0.39036 (0.38073) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.49692 (3.63041) + | > current_lr: 0.00001 + | > step_time: 1.39400 (3.04563) + | > loader_time: 0.09480 (0.06250) + + + --> STEP: 138/234 -- GLOBAL_STEP: 6690 + | > loss: 0.39972 (0.48300) + | > log_mle: 0.05578 (0.10197) + | > loss_dur: 0.34394 (0.38103) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.74273 (3.75233) + | > current_lr: 0.00001 + | > step_time: 5.10730 (3.06495) + | > loader_time: 0.00340 (0.06170) + + + --> STEP: 143/234 -- GLOBAL_STEP: 6695 + | > loss: 0.35618 (0.47922) + | > log_mle: -0.05688 (0.09801) + | > loss_dur: 0.41306 (0.38121) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.17506 (3.93347) + | > current_lr: 0.00001 + | > step_time: 2.59620 (3.05087) + | > loader_time: 0.00800 (0.06092) + + + --> STEP: 148/234 -- GLOBAL_STEP: 6700 + | > loss: 0.35641 (0.47535) + | > log_mle: 0.00914 (0.09402) + | > loss_dur: 0.34727 (0.38132) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.19298 (4.10035) + | > current_lr: 0.00001 + | > step_time: 2.29570 (3.05460) + | > loader_time: 0.00500 (0.06025) + + + --> STEP: 153/234 -- GLOBAL_STEP: 6705 + | > loss: 0.31768 (0.47114) + | > log_mle: -0.08743 (0.08933) + | > loss_dur: 0.40511 (0.38182) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.41319 (4.38004) + | > current_lr: 0.00001 + | > step_time: 1.79440 (3.08683) + | > loader_time: 0.00240 (0.06036) + + + --> STEP: 158/234 -- GLOBAL_STEP: 6710 + | > loss: 0.35139 (0.46730) + | > log_mle: -0.03943 (0.08515) + | > loss_dur: 0.39082 (0.38216) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.61724 (4.59164) + | > current_lr: 0.00001 + | > step_time: 1.79100 (3.05635) + | > loader_time: 0.00510 (0.05914) + + + --> STEP: 163/234 -- GLOBAL_STEP: 6715 + | > loss: 0.36660 (0.46349) + | > log_mle: -0.01651 (0.08100) + | > loss_dur: 0.38311 (0.38249) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.89733 (4.81768) + | > current_lr: 0.00001 + | > step_time: 7.00620 (3.04617) + | > loader_time: 0.08520 (0.05845) + + + --> STEP: 168/234 -- GLOBAL_STEP: 6720 + | > loss: 0.36818 (0.46000) + | > log_mle: -0.05766 (0.07682) + | > loss_dur: 0.42583 (0.38319) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.85817 (5.02998) + | > current_lr: 0.00001 + | > step_time: 5.29190 (3.07697) + | > loader_time: 0.00320 (0.06205) + + + --> STEP: 173/234 -- GLOBAL_STEP: 6725 + | > loss: 0.34349 (0.45628) + | > log_mle: -0.06667 (0.07219) + | > loss_dur: 0.41016 (0.38409) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.64526 (5.26767) + | > current_lr: 0.00001 + | > step_time: 2.31090 (3.06738) + | > loader_time: 0.00330 (0.06234) + + + --> STEP: 178/234 -- GLOBAL_STEP: 6730 + | > loss: 0.30027 (0.45243) + | > log_mle: -0.11964 (0.06753) + | > loss_dur: 0.41991 (0.38490) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.05388 (5.47958) + | > current_lr: 0.00001 + | > step_time: 2.50230 (3.08844) + | > loader_time: 0.10050 (0.06238) + + + --> STEP: 183/234 -- GLOBAL_STEP: 6735 + | > loss: 0.31861 (0.44928) + | > log_mle: -0.11286 (0.06326) + | > loss_dur: 0.43146 (0.38602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.77500 (5.69161) + | > current_lr: 0.00001 + | > step_time: 3.09390 (3.17838) + | > loader_time: 0.00550 (0.06338) + + + --> STEP: 188/234 -- GLOBAL_STEP: 6740 + | > loss: 0.30050 (0.44580) + | > log_mle: -0.12262 (0.05898) + | > loss_dur: 0.42312 (0.38682) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.20726 (5.87036) + | > current_lr: 0.00001 + | > step_time: 4.89800 (3.23164) + | > loader_time: 0.10140 (0.06382) + + + --> STEP: 193/234 -- GLOBAL_STEP: 6745 + | > loss: 0.31118 (0.44235) + | > log_mle: -0.12310 (0.05486) + | > loss_dur: 0.43428 (0.38749) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.46712 (6.05861) + | > current_lr: 0.00001 + | > step_time: 8.58800 (3.31633) + | > loader_time: 0.10500 (0.06527) + + + --> STEP: 198/234 -- GLOBAL_STEP: 6750 + | > loss: 0.30321 (0.43914) + | > log_mle: -0.11479 (0.05091) + | > loss_dur: 0.41799 (0.38822) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.51225 (6.21842) + | > current_lr: 0.00001 + | > step_time: 8.09940 (3.38504) + | > loader_time: 0.00900 (0.06678) + + + --> STEP: 203/234 -- GLOBAL_STEP: 6755 + | > loss: 0.33977 (0.43607) + | > log_mle: -0.06556 (0.04707) + | > loss_dur: 0.40533 (0.38900) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.68707 (6.42891) + | > current_lr: 0.00001 + | > step_time: 7.39000 (3.40713) + | > loader_time: 0.49520 (0.06853) + + + --> STEP: 208/234 -- GLOBAL_STEP: 6760 + | > loss: 0.30938 (0.43275) + | > log_mle: -0.12871 (0.04270) + | > loss_dur: 0.43808 (0.39005) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.26116 (6.64585) + | > current_lr: 0.00001 + | > step_time: 7.09100 (3.46224) + | > loader_time: 0.00230 (0.06970) + + + --> STEP: 213/234 -- GLOBAL_STEP: 6765 + | > loss: 0.26321 (0.42915) + | > log_mle: -0.17130 (0.03793) + | > loss_dur: 0.43451 (0.39121) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.71885 (6.85243) + | > current_lr: 0.00001 + | > step_time: 3.29920 (3.56823) + | > loader_time: 0.18770 (0.06989) + + + --> STEP: 218/234 -- GLOBAL_STEP: 6770 + | > loss: 0.30119 (0.42587) + | > log_mle: -0.13839 (0.03362) + | > loss_dur: 0.43958 (0.39225) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.74066 (7.09512) + | > current_lr: 0.00001 + | > step_time: 7.29940 (3.61351) + | > loader_time: 0.09450 (0.06881) + + + --> STEP: 223/234 -- GLOBAL_STEP: 6775 + | > loss: 0.26035 (0.42247) + | > log_mle: -0.17460 (0.02906) + | > loss_dur: 0.43495 (0.39341) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.24995 (7.35480) + | > current_lr: 0.00001 + | > step_time: 0.26160 (3.59910) + | > loader_time: 0.00450 (0.06813) + + + --> STEP: 228/234 -- GLOBAL_STEP: 6780 + | > loss: 0.29306 (0.41905) + | > log_mle: -0.17772 (0.02432) + | > loss_dur: 0.47078 (0.39474) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.26881 (7.63043) + | > current_lr: 0.00001 + | > step_time: 0.24070 (3.52546) + | > loader_time: 0.00340 (0.06671) + + + --> STEP: 233/234 -- GLOBAL_STEP: 6785 + | > loss: 0.95248 (0.41979) + | > log_mle: -0.13414 (0.01876) + | > loss_dur: 1.08662 (0.40103) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.97413 (8.01476) + | > current_lr: 0.00001 + | > step_time: 0.20080 (3.45552) + | > loader_time: 0.00260 (0.06536) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.54909 (+0.50787) + | > avg_loss: 0.34718 (-0.04102) + | > avg_log_mle: -0.05333 (-0.01942) + | > avg_loss_dur: 0.40051 (-0.02161) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_6786.pth + + > EPOCH: 29/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 15:29:15)  + + --> STEP: 4/234 -- GLOBAL_STEP: 6790 + | > loss: 0.67707 (0.71476) + | > log_mle: 0.13314 (0.16157) + | > loss_dur: 0.54393 (0.55319) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.17368 (2.10550) + | > current_lr: 0.00001 + | > step_time: 7.39090 (4.75180) + | > loader_time: 0.09740 (0.09264) + + + --> STEP: 9/234 -- GLOBAL_STEP: 6795 + | > loss: 0.57949 (0.64999) + | > log_mle: 0.12396 (0.14767) + | > loss_dur: 0.45553 (0.50233) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.61153 (1.80757) + | > current_lr: 0.00001 + | > step_time: 3.90390 (5.09188) + | > loader_time: 0.00290 (0.06277) + + + --> STEP: 14/234 -- GLOBAL_STEP: 6800 + | > loss: 0.54561 (0.60905) + | > log_mle: 0.13022 (0.14444) + | > loss_dur: 0.41539 (0.46461) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.26902 (1.74276) + | > current_lr: 0.00001 + | > step_time: 6.30140 (5.47994) + | > loader_time: 0.09800 (0.09671) + + + --> STEP: 19/234 -- GLOBAL_STEP: 6805 + | > loss: 0.55079 (0.58903) + | > log_mle: 0.14129 (0.14255) + | > loss_dur: 0.40950 (0.44648) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.08354 (1.57844) + | > current_lr: 0.00001 + | > step_time: 5.50870 (5.28185) + | > loader_time: 0.00420 (0.08109) + + + --> STEP: 24/234 -- GLOBAL_STEP: 6810 + | > loss: 0.50470 (0.56967) + | > log_mle: 0.13539 (0.14073) + | > loss_dur: 0.36931 (0.42894) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.15060 (1.47137) + | > current_lr: 0.00001 + | > step_time: 1.80060 (5.06888) + | > loader_time: 0.00140 (0.07551) + + + --> STEP: 29/234 -- GLOBAL_STEP: 6815 + | > loss: 0.50343 (0.55414) + | > log_mle: 0.14017 (0.13925) + | > loss_dur: 0.36327 (0.41489) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.18561 (1.40473) + | > current_lr: 0.00001 + | > step_time: 5.40520 (5.03659) + | > loader_time: 0.00780 (0.07316) + + + --> STEP: 34/234 -- GLOBAL_STEP: 6820 + | > loss: 0.49423 (0.54436) + | > log_mle: 0.12081 (0.13622) + | > loss_dur: 0.37343 (0.40814) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.37704 (1.35476) + | > current_lr: 0.00001 + | > step_time: 2.78970 (4.88433) + | > loader_time: 0.00290 (0.06834) + + + --> STEP: 39/234 -- GLOBAL_STEP: 6825 + | > loss: 0.49910 (0.53664) + | > log_mle: 0.11745 (0.13367) + | > loss_dur: 0.38166 (0.40298) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.57323 (1.49889) + | > current_lr: 0.00001 + | > step_time: 0.79820 (4.40183) + | > loader_time: 0.00210 (0.06178) + + + --> STEP: 44/234 -- GLOBAL_STEP: 6830 + | > loss: 0.45990 (0.52955) + | > log_mle: 0.11893 (0.13268) + | > loss_dur: 0.34098 (0.39687) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.04805 (1.45220) + | > current_lr: 0.00001 + | > step_time: 1.20330 (4.11272) + | > loader_time: 0.00300 (0.05667) + + + --> STEP: 49/234 -- GLOBAL_STEP: 6835 + | > loss: 0.43853 (0.52251) + | > log_mle: 0.11176 (0.13111) + | > loss_dur: 0.32678 (0.39140) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.79388 (1.42825) + | > current_lr: 0.00001 + | > step_time: 3.68170 (3.97252) + | > loader_time: 0.01110 (0.05487) + + + --> STEP: 54/234 -- GLOBAL_STEP: 6840 + | > loss: 0.44996 (0.51628) + | > log_mle: 0.11163 (0.13022) + | > loss_dur: 0.33833 (0.38606) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.94246 (1.41387) + | > current_lr: 0.00001 + | > step_time: 1.90040 (3.76898) + | > loader_time: 0.00290 (0.04998) + + + --> STEP: 59/234 -- GLOBAL_STEP: 6845 + | > loss: 0.40410 (0.51085) + | > log_mle: 0.09163 (0.12880) + | > loss_dur: 0.31246 (0.38205) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.30358 (1.41559) + | > current_lr: 0.00001 + | > step_time: 2.98610 (3.67880) + | > loader_time: 0.00460 (0.05034) + + + --> STEP: 64/234 -- GLOBAL_STEP: 6850 + | > loss: 0.43326 (0.50520) + | > log_mle: 0.12272 (0.12674) + | > loss_dur: 0.31054 (0.37846) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.79672 (1.44611) + | > current_lr: 0.00001 + | > step_time: 4.01360 (3.56383) + | > loader_time: 0.09560 (0.04933) + + + --> STEP: 69/234 -- GLOBAL_STEP: 6855 + | > loss: 0.47413 (0.50098) + | > log_mle: 0.13510 (0.12600) + | > loss_dur: 0.33904 (0.37498) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.42483 (1.49904) + | > current_lr: 0.00001 + | > step_time: 1.72420 (3.47630) + | > loader_time: 0.00230 (0.04709) + + + --> STEP: 74/234 -- GLOBAL_STEP: 6860 + | > loss: 0.41158 (0.49665) + | > log_mle: 0.10804 (0.12408) + | > loss_dur: 0.30354 (0.37257) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.07089 (1.69800) + | > current_lr: 0.00001 + | > step_time: 1.40170 (3.40324) + | > loader_time: 0.00270 (0.04540) + + + --> STEP: 79/234 -- GLOBAL_STEP: 6865 + | > loss: 0.41414 (0.49264) + | > log_mle: 0.09980 (0.12271) + | > loss_dur: 0.31434 (0.36992) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.76285 (1.74411) + | > current_lr: 0.00001 + | > step_time: 1.60580 (3.31930) + | > loader_time: 0.00260 (0.04373) + + + --> STEP: 84/234 -- GLOBAL_STEP: 6870 + | > loss: 0.42673 (0.48832) + | > log_mle: 0.09378 (0.12129) + | > loss_dur: 0.33295 (0.36703) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.92303 (1.81645) + | > current_lr: 0.00001 + | > step_time: 1.81150 (3.25204) + | > loader_time: 0.09500 (0.04240) + + + --> STEP: 89/234 -- GLOBAL_STEP: 6875 + | > loss: 0.40174 (0.48408) + | > log_mle: 0.06821 (0.11889) + | > loss_dur: 0.33354 (0.36519) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.81230 (1.95235) + | > current_lr: 0.00001 + | > step_time: 2.00050 (3.20749) + | > loader_time: 0.09320 (0.04222) + + + --> STEP: 94/234 -- GLOBAL_STEP: 6880 + | > loss: 0.37266 (0.47934) + | > log_mle: 0.03035 (0.11547) + | > loss_dur: 0.34231 (0.36388) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.60147 (2.18516) + | > current_lr: 0.00001 + | > step_time: 3.21680 (3.18598) + | > loader_time: 0.09210 (0.04415) + + + --> STEP: 99/234 -- GLOBAL_STEP: 6885 + | > loss: 0.34955 (0.47497) + | > log_mle: 0.00430 (0.11214) + | > loss_dur: 0.34525 (0.36282) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.61579 (2.40616) + | > current_lr: 0.00001 + | > step_time: 1.18550 (3.12414) + | > loader_time: 0.00370 (0.04289) + + + --> STEP: 104/234 -- GLOBAL_STEP: 6890 + | > loss: 0.34471 (0.47074) + | > log_mle: -0.00173 (0.10862) + | > loss_dur: 0.34644 (0.36213) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.15012 (2.61129) + | > current_lr: 0.00001 + | > step_time: 2.19940 (3.09704) + | > loader_time: 0.10540 (0.04352) + + + --> STEP: 109/234 -- GLOBAL_STEP: 6895 + | > loss: 0.39328 (0.46747) + | > log_mle: 0.02472 (0.10595) + | > loss_dur: 0.36856 (0.36152) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.73409 (2.86072) + | > current_lr: 0.00001 + | > step_time: 2.31930 (3.21020) + | > loader_time: 0.08840 (0.04703) + + + --> STEP: 114/234 -- GLOBAL_STEP: 6900 + | > loss: 0.38250 (0.46372) + | > log_mle: 0.04520 (0.10263) + | > loss_dur: 0.33730 (0.36109) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.70628 (3.09086) + | > current_lr: 0.00001 + | > step_time: 2.89020 (3.17997) + | > loader_time: 0.00210 (0.04661) + + + --> STEP: 119/234 -- GLOBAL_STEP: 6905 + | > loss: 0.39094 (0.46113) + | > log_mle: 0.04312 (0.10007) + | > loss_dur: 0.34783 (0.36106) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.91976 (3.24082) + | > current_lr: 0.00001 + | > step_time: 1.08100 (3.16227) + | > loader_time: 0.00220 (0.04551) + + + --> STEP: 124/234 -- GLOBAL_STEP: 6910 + | > loss: 0.35939 (0.45796) + | > log_mle: 0.01665 (0.09804) + | > loss_dur: 0.34275 (0.35991) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.21537 (3.32841) + | > current_lr: 0.00001 + | > step_time: 1.90000 (3.12166) + | > loader_time: 0.00290 (0.04527) + + + --> STEP: 129/234 -- GLOBAL_STEP: 6915 + | > loss: 0.38627 (0.45459) + | > log_mle: 0.02987 (0.09463) + | > loss_dur: 0.35640 (0.35996) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.85548 (3.50447) + | > current_lr: 0.00001 + | > step_time: 2.81150 (3.07366) + | > loader_time: 0.00680 (0.04505) + + + --> STEP: 134/234 -- GLOBAL_STEP: 6920 + | > loss: 0.37840 (0.45092) + | > log_mle: -0.01334 (0.09087) + | > loss_dur: 0.39174 (0.36005) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.46685 (3.69734) + | > current_lr: 0.00001 + | > step_time: 3.91460 (3.10175) + | > loader_time: 0.00610 (0.04490) + + + --> STEP: 139/234 -- GLOBAL_STEP: 6925 + | > loss: 0.30882 (0.44763) + | > log_mle: -0.06541 (0.08731) + | > loss_dur: 0.37423 (0.36031) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.86128 (3.86595) + | > current_lr: 0.00001 + | > step_time: 2.90310 (3.07892) + | > loader_time: 0.08880 (0.04519) + + + --> STEP: 144/234 -- GLOBAL_STEP: 6930 + | > loss: 0.32589 (0.44411) + | > log_mle: -0.04538 (0.08356) + | > loss_dur: 0.37126 (0.36055) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.94861 (4.09317) + | > current_lr: 0.00001 + | > step_time: 3.49470 (3.08257) + | > loader_time: 0.09490 (0.04685) + + + --> STEP: 149/234 -- GLOBAL_STEP: 6935 + | > loss: 0.30862 (0.44018) + | > log_mle: -0.08163 (0.07940) + | > loss_dur: 0.39025 (0.36079) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.21556 (4.35709) + | > current_lr: 0.00001 + | > step_time: 2.39960 (3.09525) + | > loader_time: 0.00410 (0.04839) + + + --> STEP: 154/234 -- GLOBAL_STEP: 6940 + | > loss: 0.32133 (0.43630) + | > log_mle: -0.04899 (0.07496) + | > loss_dur: 0.37032 (0.36134) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.92015 (4.60019) + | > current_lr: 0.00001 + | > step_time: 4.21970 (3.08534) + | > loader_time: 0.00420 (0.04849) + + + --> STEP: 159/234 -- GLOBAL_STEP: 6945 + | > loss: 0.31903 (0.43247) + | > log_mle: -0.06492 (0.07074) + | > loss_dur: 0.38395 (0.36173) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.20188 (4.88373) + | > current_lr: 0.00001 + | > step_time: 4.40970 (3.07710) + | > loader_time: 0.09740 (0.04880) + + + --> STEP: 164/234 -- GLOBAL_STEP: 6950 + | > loss: 0.30717 (0.42875) + | > log_mle: -0.06155 (0.06665) + | > loss_dur: 0.36872 (0.36210) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.08634 (5.05997) + | > current_lr: 0.00001 + | > step_time: 2.90140 (3.08829) + | > loader_time: 0.00320 (0.04967) + + + --> STEP: 169/234 -- GLOBAL_STEP: 6955 + | > loss: 0.34591 (0.42555) + | > log_mle: -0.05510 (0.06254) + | > loss_dur: 0.40102 (0.36301) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.91535 (5.28925) + | > current_lr: 0.00001 + | > step_time: 4.50290 (3.12607) + | > loader_time: 0.19160 (0.05267) + + + --> STEP: 174/234 -- GLOBAL_STEP: 6960 + | > loss: 0.25570 (0.42140) + | > log_mle: -0.13786 (0.05748) + | > loss_dur: 0.39357 (0.36392) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.02967 (5.59138) + | > current_lr: 0.00001 + | > step_time: 4.30200 (3.20973) + | > loader_time: 0.00410 (0.05407) + + + --> STEP: 179/234 -- GLOBAL_STEP: 6965 + | > loss: 0.28070 (0.41785) + | > log_mle: -0.11998 (0.05298) + | > loss_dur: 0.40069 (0.36487) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.61059 (5.75660) + | > current_lr: 0.00001 + | > step_time: 4.59240 (3.24004) + | > loader_time: 0.30260 (0.05488) + + + --> STEP: 184/234 -- GLOBAL_STEP: 6970 + | > loss: 0.30076 (0.41486) + | > log_mle: -0.10192 (0.04884) + | > loss_dur: 0.40267 (0.36602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.06023 (5.93844) + | > current_lr: 0.00001 + | > step_time: 7.89950 (3.32485) + | > loader_time: 0.20610 (0.05666) + + + --> STEP: 189/234 -- GLOBAL_STEP: 6975 + | > loss: 0.29729 (0.41144) + | > log_mle: -0.09764 (0.04460) + | > loss_dur: 0.39493 (0.36683) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.79195 (6.10771) + | > current_lr: 0.00001 + | > step_time: 2.60520 (3.40876) + | > loader_time: 0.09940 (0.06086) + + + --> STEP: 194/234 -- GLOBAL_STEP: 6980 + | > loss: 0.26924 (0.40800) + | > log_mle: -0.12799 (0.04036) + | > loss_dur: 0.39723 (0.36764) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.99934 (6.31591) + | > current_lr: 0.00001 + | > step_time: 7.41760 (3.46653) + | > loader_time: 0.08990 (0.06180) + + + --> STEP: 199/234 -- GLOBAL_STEP: 6985 + | > loss: 0.26873 (0.40497) + | > log_mle: -0.13796 (0.03641) + | > loss_dur: 0.40669 (0.36856) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.57857 (6.60258) + | > current_lr: 0.00001 + | > step_time: 3.79350 (3.50311) + | > loader_time: 0.00540 (0.06176) + + + --> STEP: 204/234 -- GLOBAL_STEP: 6990 + | > loss: 0.26333 (0.40201) + | > log_mle: -0.16145 (0.03249) + | > loss_dur: 0.42478 (0.36952) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 22.24393 (6.92200) + | > current_lr: 0.00001 + | > step_time: 5.19110 (3.51775) + | > loader_time: 0.00290 (0.06117) + + + --> STEP: 209/234 -- GLOBAL_STEP: 6995 + | > loss: 0.28035 (0.39879) + | > log_mle: -0.12267 (0.02834) + | > loss_dur: 0.40302 (0.37045) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.61111 (7.19067) + | > current_lr: 0.00001 + | > step_time: 1.99150 (3.52978) + | > loader_time: 0.00390 (0.06123) + + + --> STEP: 214/234 -- GLOBAL_STEP: 7000 + | > loss: 0.24902 (0.39516) + | > log_mle: -0.15233 (0.02347) + | > loss_dur: 0.40134 (0.37169) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.81673 (7.49770) + | > current_lr: 0.00001 + | > step_time: 3.00550 (3.58660) + | > loader_time: 0.08530 (0.06151) + + + --> STEP: 219/234 -- GLOBAL_STEP: 7005 + | > loss: 0.19495 (0.39159) + | > log_mle: -0.23204 (0.01882) + | > loss_dur: 0.42699 (0.37277) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 30.08044 (7.76910) + | > current_lr: 0.00001 + | > step_time: 2.99730 (3.61349) + | > loader_time: 0.00350 (0.06197) + + + --> STEP: 224/234 -- GLOBAL_STEP: 7010 + | > loss: 0.23430 (0.38840) + | > log_mle: -0.19423 (0.01447) + | > loss_dur: 0.42854 (0.37393) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.69324 (8.00653) + | > current_lr: 0.00001 + | > step_time: 2.99200 (3.59148) + | > loader_time: 0.00450 (0.06322) + + + --> STEP: 229/234 -- GLOBAL_STEP: 7015 + | > loss: 0.29294 (0.38521) + | > log_mle: -0.21192 (0.00967) + | > loss_dur: 0.50486 (0.37554) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 31.11171 (8.30229) + | > current_lr: 0.00001 + | > step_time: 1.32740 (3.58918) + | > loader_time: 0.08740 (0.06311) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.22827 (+0.67918) + | > avg_loss: 0.32545 (-0.02173) + | > avg_log_mle: -0.05450 (-0.00118) + | > avg_loss_dur: 0.37995 (-0.02055) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_7020.pth + + > EPOCH: 30/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 15:44:30)  + + --> STEP: 0/234 -- GLOBAL_STEP: 7020 + | > loss: 0.72746 (0.72746) + | > log_mle: 0.13249 (0.13249) + | > loss_dur: 0.59497 (0.59497) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.10218 (2.10218) + | > current_lr: 0.00001 + | > step_time: 17.30090 (17.30093) + | > loader_time: 2.75310 (2.75310) + + + --> STEP: 5/234 -- GLOBAL_STEP: 7025 + | > loss: 0.57446 (0.65030) + | > log_mle: 0.13442 (0.14570) + | > loss_dur: 0.44004 (0.50460) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.33732 (3.48186) + | > current_lr: 0.00001 + | > step_time: 4.90010 (6.94388) + | > loader_time: 0.19430 (0.38285) + + + --> STEP: 10/234 -- GLOBAL_STEP: 7030 + | > loss: 0.46305 (0.58866) + | > log_mle: 0.11707 (0.13288) + | > loss_dur: 0.34599 (0.45578) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.93527 (2.60811) + | > current_lr: 0.00001 + | > step_time: 2.60680 (5.65568) + | > loader_time: 0.00280 (0.22858) + + + --> STEP: 15/234 -- GLOBAL_STEP: 7035 + | > loss: 0.50966 (0.55859) + | > log_mle: 0.12042 (0.13063) + | > loss_dur: 0.38924 (0.42796) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.09022 (2.14566) + | > current_lr: 0.00001 + | > step_time: 1.78760 (4.65580) + | > loader_time: 0.00160 (0.15291) + + + --> STEP: 20/234 -- GLOBAL_STEP: 7040 + | > loss: 0.46266 (0.54092) + | > log_mle: 0.12563 (0.12924) + | > loss_dur: 0.33703 (0.41167) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.15658 (1.90519) + | > current_lr: 0.00001 + | > step_time: 1.89530 (3.88408) + | > loader_time: 0.00130 (0.11512) + + + --> STEP: 25/234 -- GLOBAL_STEP: 7045 + | > loss: 0.45857 (0.52607) + | > log_mle: 0.12866 (0.12769) + | > loss_dur: 0.32991 (0.39837) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.14028 (1.74606) + | > current_lr: 0.00001 + | > step_time: 1.10450 (3.39834) + | > loader_time: 0.00250 (0.09259) + + + --> STEP: 30/234 -- GLOBAL_STEP: 7050 + | > loss: 0.44223 (0.51298) + | > log_mle: 0.10012 (0.12538) + | > loss_dur: 0.34211 (0.38760) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.02381 (1.62784) + | > current_lr: 0.00001 + | > step_time: 2.41470 (3.29978) + | > loader_time: 0.08820 (0.08042) + + + --> STEP: 35/234 -- GLOBAL_STEP: 7055 + | > loss: 0.43104 (0.50473) + | > log_mle: 0.10028 (0.12266) + | > loss_dur: 0.33076 (0.38207) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.43069 (1.74864) + | > current_lr: 0.00001 + | > step_time: 3.90510 (3.46014) + | > loader_time: 0.08830 (0.08331) + + + --> STEP: 40/234 -- GLOBAL_STEP: 7060 + | > loss: 0.46769 (0.49861) + | > log_mle: 0.12246 (0.12087) + | > loss_dur: 0.34523 (0.37774) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.29917 (1.70042) + | > current_lr: 0.00001 + | > step_time: 2.14190 (3.57943) + | > loader_time: 0.00290 (0.08711) + + + --> STEP: 45/234 -- GLOBAL_STEP: 7065 + | > loss: 0.44051 (0.49115) + | > log_mle: 0.09902 (0.11940) + | > loss_dur: 0.34148 (0.37175) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.48303 (1.66853) + | > current_lr: 0.00001 + | > step_time: 1.68800 (3.33278) + | > loader_time: 0.10040 (0.07983) + + + --> STEP: 50/234 -- GLOBAL_STEP: 7070 + | > loss: 0.42034 (0.48481) + | > log_mle: 0.11162 (0.11819) + | > loss_dur: 0.30872 (0.36663) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.89378 (1.60747) + | > current_lr: 0.00001 + | > step_time: 1.11750 (3.21441) + | > loader_time: 0.00150 (0.07739) + + + --> STEP: 55/234 -- GLOBAL_STEP: 7075 + | > loss: 0.41267 (0.47949) + | > log_mle: 0.09178 (0.11695) + | > loss_dur: 0.32089 (0.36254) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.57269 (1.61453) + | > current_lr: 0.00001 + | > step_time: 2.03470 (3.08026) + | > loader_time: 0.00310 (0.07056) + + + --> STEP: 60/234 -- GLOBAL_STEP: 7080 + | > loss: 0.40820 (0.47493) + | > log_mle: 0.07861 (0.11537) + | > loss_dur: 0.32959 (0.35956) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.12229 (1.62089) + | > current_lr: 0.00001 + | > step_time: 2.69660 (2.98101) + | > loader_time: 0.00380 (0.06638) + + + --> STEP: 65/234 -- GLOBAL_STEP: 7085 + | > loss: 0.40408 (0.47005) + | > log_mle: 0.10260 (0.11377) + | > loss_dur: 0.30148 (0.35628) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.18050 (1.63055) + | > current_lr: 0.00001 + | > step_time: 1.29490 (2.93156) + | > loader_time: 0.00230 (0.06407) + + + --> STEP: 70/234 -- GLOBAL_STEP: 7090 + | > loss: 0.42433 (0.46646) + | > log_mle: 0.08948 (0.11284) + | > loss_dur: 0.33485 (0.35362) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.91640 (1.69970) + | > current_lr: 0.00001 + | > step_time: 1.09980 (2.82273) + | > loader_time: 0.00360 (0.05969) + + + --> STEP: 75/234 -- GLOBAL_STEP: 7095 + | > loss: 0.41130 (0.46206) + | > log_mle: 0.08852 (0.11091) + | > loss_dur: 0.32277 (0.35115) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.08326 (1.81362) + | > current_lr: 0.00001 + | > step_time: 1.49890 (2.75880) + | > loader_time: 0.00350 (0.05687) + + + --> STEP: 80/234 -- GLOBAL_STEP: 7100 + | > loss: 0.38625 (0.45820) + | > log_mle: 0.10251 (0.10972) + | > loss_dur: 0.28373 (0.34847) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.92423 (1.82120) + | > current_lr: 0.00001 + | > step_time: 1.40320 (2.74410) + | > loader_time: 0.00240 (0.05667) + + + --> STEP: 85/234 -- GLOBAL_STEP: 7105 + | > loss: 0.39383 (0.45422) + | > log_mle: 0.08703 (0.10812) + | > loss_dur: 0.30680 (0.34610) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.21255 (1.88155) + | > current_lr: 0.00001 + | > step_time: 1.02450 (2.71192) + | > loader_time: 0.07510 (0.05559) + + + --> STEP: 90/234 -- GLOBAL_STEP: 7110 + | > loss: 0.37568 (0.45002) + | > log_mle: 0.05547 (0.10541) + | > loss_dur: 0.32022 (0.34461) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.64396 (2.05708) + | > current_lr: 0.00001 + | > step_time: 1.39580 (2.65037) + | > loader_time: 0.09740 (0.05765) + + + --> STEP: 95/234 -- GLOBAL_STEP: 7115 + | > loss: 0.33024 (0.44527) + | > log_mle: -0.02064 (0.10126) + | > loss_dur: 0.35088 (0.34400) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.42681 (2.40130) + | > current_lr: 0.00001 + | > step_time: 2.31450 (2.62443) + | > loader_time: 0.00250 (0.05492) + + + --> STEP: 100/234 -- GLOBAL_STEP: 7120 + | > loss: 0.37838 (0.44194) + | > log_mle: 0.04404 (0.09868) + | > loss_dur: 0.33434 (0.34326) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.56563 (2.52360) + | > current_lr: 0.00001 + | > step_time: 2.21720 (2.59453) + | > loader_time: 0.10230 (0.05470) + + + --> STEP: 105/234 -- GLOBAL_STEP: 7125 + | > loss: 0.39816 (0.43848) + | > log_mle: 0.07597 (0.09550) + | > loss_dur: 0.32220 (0.34298) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.85859 (2.68662) + | > current_lr: 0.00001 + | > step_time: 0.97590 (2.55930) + | > loader_time: 0.00240 (0.05464) + + + --> STEP: 110/234 -- GLOBAL_STEP: 7130 + | > loss: 0.37076 (0.43532) + | > log_mle: 0.04403 (0.09256) + | > loss_dur: 0.32673 (0.34276) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.33784 (2.89307) + | > current_lr: 0.00001 + | > step_time: 2.20350 (2.53010) + | > loader_time: 0.01670 (0.05328) + + + --> STEP: 115/234 -- GLOBAL_STEP: 7135 + | > loss: 0.38692 (0.43173) + | > log_mle: 0.03153 (0.08915) + | > loss_dur: 0.35539 (0.34258) + | > amp_scaler: 32768.00000 (16811.40870) + | > grad_norm: 5.50037 (3.07676) + | > current_lr: 0.00001 + | > step_time: 1.67780 (2.49834) + | > loader_time: 0.01100 (0.05203) + + + --> STEP: 120/234 -- GLOBAL_STEP: 7140 + | > loss: 0.31745 (0.42879) + | > log_mle: -0.01667 (0.08623) + | > loss_dur: 0.33412 (0.34257) + | > amp_scaler: 32768.00000 (17476.26667) + | > grad_norm: 7.73098 (3.26211) + | > current_lr: 0.00001 + | > step_time: 3.00550 (2.49129) + | > loader_time: 0.09770 (0.05154) + + + --> STEP: 125/234 -- GLOBAL_STEP: 7145 + | > loss: 0.34688 (0.42619) + | > log_mle: -0.00125 (0.08438) + | > loss_dur: 0.34813 (0.34181) + | > amp_scaler: 32768.00000 (18087.93600) + | > grad_norm: 9.50367 (3.37630) + | > current_lr: 0.00001 + | > step_time: 2.76910 (2.47501) + | > loader_time: 0.00360 (0.05111) + + + --> STEP: 130/234 -- GLOBAL_STEP: 7150 + | > loss: 0.33088 (0.42284) + | > log_mle: -0.01218 (0.08096) + | > loss_dur: 0.34306 (0.34188) + | > amp_scaler: 32768.00000 (18652.55385) + | > grad_norm: 12.14561 (3.60299) + | > current_lr: 0.00001 + | > step_time: 1.51620 (2.44626) + | > loader_time: 0.00340 (0.05000) + + + --> STEP: 135/234 -- GLOBAL_STEP: 7155 + | > loss: 0.37148 (0.41976) + | > log_mle: 0.04786 (0.07770) + | > loss_dur: 0.32362 (0.34206) + | > amp_scaler: 32768.00000 (19175.34815) + | > grad_norm: 4.77319 (3.82646) + | > current_lr: 0.00001 + | > step_time: 1.48670 (2.45744) + | > loader_time: 0.00340 (0.04998) + + + --> STEP: 140/234 -- GLOBAL_STEP: 7160 + | > loss: 0.35597 (0.41652) + | > log_mle: 0.01811 (0.07396) + | > loss_dur: 0.33786 (0.34255) + | > amp_scaler: 32768.00000 (19660.80000) + | > grad_norm: 8.61064 (4.07083) + | > current_lr: 0.00001 + | > step_time: 3.31710 (2.45898) + | > loader_time: 0.00270 (0.04961) + + + --> STEP: 145/234 -- GLOBAL_STEP: 7165 + | > loss: 0.29110 (0.41275) + | > log_mle: -0.06619 (0.06966) + | > loss_dur: 0.35729 (0.34309) + | > amp_scaler: 32768.00000 (20112.77241) + | > grad_norm: 11.67060 (4.33444) + | > current_lr: 0.00001 + | > step_time: 2.77540 (2.45262) + | > loader_time: 0.02270 (0.04930) + + + --> STEP: 150/234 -- GLOBAL_STEP: 7170 + | > loss: 0.32453 (0.40915) + | > log_mle: -0.04904 (0.06564) + | > loss_dur: 0.37356 (0.34351) + | > amp_scaler: 32768.00000 (20534.61333) + | > grad_norm: 11.57149 (4.58884) + | > current_lr: 0.00001 + | > step_time: 1.29900 (2.43642) + | > loader_time: 0.00340 (0.04897) + + + --> STEP: 155/234 -- GLOBAL_STEP: 7175 + | > loss: 0.26410 (0.40492) + | > log_mle: -0.10945 (0.06082) + | > loss_dur: 0.37355 (0.34411) + | > amp_scaler: 32768.00000 (20929.23871) + | > grad_norm: 16.87396 (4.84695) + | > current_lr: 0.00001 + | > step_time: 0.80630 (2.42823) + | > loader_time: 0.00520 (0.04921) + + + --> STEP: 160/234 -- GLOBAL_STEP: 7180 + | > loss: 0.25846 (0.40121) + | > log_mle: -0.10592 (0.05664) + | > loss_dur: 0.36438 (0.34458) + | > amp_scaler: 32768.00000 (21299.20000) + | > grad_norm: 10.79121 (5.05319) + | > current_lr: 0.00001 + | > step_time: 1.09590 (2.40486) + | > loader_time: 0.00360 (0.04936) + + + --> STEP: 165/234 -- GLOBAL_STEP: 7185 + | > loss: 0.27445 (0.39769) + | > log_mle: -0.10025 (0.05260) + | > loss_dur: 0.37470 (0.34509) + | > amp_scaler: 32768.00000 (21646.73939) + | > grad_norm: 13.49286 (5.23479) + | > current_lr: 0.00001 + | > step_time: 2.00800 (2.38171) + | > loader_time: 0.10210 (0.04907) + + + --> STEP: 170/234 -- GLOBAL_STEP: 7190 + | > loss: 0.26861 (0.39444) + | > log_mle: -0.13128 (0.04833) + | > loss_dur: 0.39990 (0.34611) + | > amp_scaler: 32768.00000 (21973.83529) + | > grad_norm: 20.92566 (5.54170) + | > current_lr: 0.00001 + | > step_time: 3.52420 (2.40035) + | > loader_time: 0.09080 (0.04911) + + + --> STEP: 175/234 -- GLOBAL_STEP: 7195 + | > loss: 0.26563 (0.39034) + | > log_mle: -0.10974 (0.04343) + | > loss_dur: 0.37537 (0.34690) + | > amp_scaler: 16384.00000 (21814.12571) + | > grad_norm: 13.42699 (5.80940) + | > current_lr: 0.00001 + | > step_time: 1.58640 (2.40034) + | > loader_time: 0.09890 (0.04945) + + + --> STEP: 180/234 -- GLOBAL_STEP: 7200 + | > loss: 0.25944 (0.38684) + | > log_mle: -0.12035 (0.03890) + | > loss_dur: 0.37979 (0.34794) + | > amp_scaler: 16384.00000 (21663.28889) + | > grad_norm: 17.86260 (6.15978) + | > current_lr: 0.00001 + | > step_time: 1.50770 (2.39704) + | > loader_time: 0.09500 (0.05022) + + + --> STEP: 185/234 -- GLOBAL_STEP: 7205 + | > loss: 0.25544 (0.38381) + | > log_mle: -0.13873 (0.03469) + | > loss_dur: 0.39417 (0.34912) + | > amp_scaler: 16384.00000 (21520.60541) + | > grad_norm: 19.57680 (6.41577) + | > current_lr: 0.00001 + | > step_time: 2.60180 (2.40515) + | > loader_time: 0.00380 (0.04995) + + + --> STEP: 190/234 -- GLOBAL_STEP: 7210 + | > loss: 0.24475 (0.38043) + | > log_mle: -0.12417 (0.03055) + | > loss_dur: 0.36892 (0.34988) + | > amp_scaler: 16384.00000 (21385.43158) + | > grad_norm: 18.09517 (6.60681) + | > current_lr: 0.00001 + | > step_time: 3.70620 (2.42442) + | > loader_time: 0.17580 (0.05101) + + + --> STEP: 195/234 -- GLOBAL_STEP: 7215 + | > loss: 0.28099 (0.37716) + | > log_mle: -0.12922 (0.02630) + | > loss_dur: 0.41021 (0.35086) + | > amp_scaler: 16384.00000 (21257.18974) + | > grad_norm: 17.04849 (6.76930) + | > current_lr: 0.00001 + | > step_time: 1.99540 (2.42485) + | > loader_time: 0.08330 (0.05112) + + + --> STEP: 200/234 -- GLOBAL_STEP: 7220 + | > loss: 0.26446 (0.37405) + | > log_mle: -0.13596 (0.02234) + | > loss_dur: 0.40042 (0.35171) + | > amp_scaler: 16384.00000 (21135.36000) + | > grad_norm: 12.32782 (6.97688) + | > current_lr: 0.00001 + | > step_time: 3.09050 (2.45973) + | > loader_time: 0.00510 (0.05178) + + + --> STEP: 205/234 -- GLOBAL_STEP: 7225 + | > loss: 0.26093 (0.37120) + | > log_mle: -0.13099 (0.01845) + | > loss_dur: 0.39193 (0.35276) + | > amp_scaler: 16384.00000 (21019.47317) + | > grad_norm: 13.18419 (7.17307) + | > current_lr: 0.00001 + | > step_time: 1.59880 (2.45293) + | > loader_time: 0.01750 (0.05203) + + + --> STEP: 210/234 -- GLOBAL_STEP: 7230 + | > loss: 0.20707 (0.36775) + | > log_mle: -0.20212 (0.01396) + | > loss_dur: 0.40919 (0.35379) + | > amp_scaler: 16384.00000 (20909.10476) + | > grad_norm: 20.94552 (7.45113) + | > current_lr: 0.00001 + | > step_time: 1.69290 (2.46630) + | > loader_time: 0.00460 (0.05186) + + + --> STEP: 215/234 -- GLOBAL_STEP: 7235 + | > loss: 0.23183 (0.36421) + | > log_mle: -0.15120 (0.00935) + | > loss_dur: 0.38302 (0.35486) + | > amp_scaler: 16384.00000 (20803.86977) + | > grad_norm: 13.98517 (7.67680) + | > current_lr: 0.00001 + | > step_time: 3.08860 (2.46522) + | > loader_time: 0.00470 (0.05202) + + + --> STEP: 220/234 -- GLOBAL_STEP: 7240 + | > loss: 0.21025 (0.36066) + | > log_mle: -0.19382 (0.00452) + | > loss_dur: 0.40407 (0.35614) + | > amp_scaler: 16384.00000 (20703.41818) + | > grad_norm: 18.35943 (7.92477) + | > current_lr: 0.00001 + | > step_time: 3.90470 (2.49875) + | > loader_time: 0.00500 (0.05303) + + + --> STEP: 225/234 -- GLOBAL_STEP: 7245 + | > loss: 0.16818 (0.35728) + | > log_mle: -0.24959 (-0.00006) + | > loss_dur: 0.41777 (0.35734) + | > amp_scaler: 16384.00000 (20607.43111) + | > grad_norm: 16.01251 (8.11723) + | > current_lr: 0.00001 + | > step_time: 0.23600 (2.46076) + | > loader_time: 0.00390 (0.05233) + + + --> STEP: 230/234 -- GLOBAL_STEP: 7250 + | > loss: 0.22300 (0.35439) + | > log_mle: -0.29049 (-0.00502) + | > loss_dur: 0.51349 (0.35941) + | > amp_scaler: 16384.00000 (20515.61739) + | > grad_norm: 27.97662 (8.40734) + | > current_lr: 0.00001 + | > step_time: 0.26020 (2.41263) + | > loader_time: 0.00450 (0.05130) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.52424 (-0.70403) + | > avg_loss: 0.29708 (-0.02838) + | > avg_log_mle: -0.07038 (-0.01588) + | > avg_loss_dur: 0.36746 (-0.01250) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_7254.pth + + > EPOCH: 31/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 15:54:56)  + + --> STEP: 1/234 -- GLOBAL_STEP: 7255 + | > loss: 0.66484 (0.66484) + | > log_mle: 0.14441 (0.14441) + | > loss_dur: 0.52043 (0.52043) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.59161 (1.59161) + | > current_lr: 0.00001 + | > step_time: 2.69740 (2.69738) + | > loader_time: 0.89180 (0.89179) + + + --> STEP: 6/234 -- GLOBAL_STEP: 7260 + | > loss: 0.51972 (0.59117) + | > log_mle: 0.12778 (0.13122) + | > loss_dur: 0.39194 (0.45995) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.48739 (2.12257) + | > current_lr: 0.00001 + | > step_time: 4.82440 (4.17355) + | > loader_time: 0.09460 (0.21390) + + + --> STEP: 11/234 -- GLOBAL_STEP: 7265 + | > loss: 0.49305 (0.54283) + | > log_mle: 0.12013 (0.11918) + | > loss_dur: 0.37293 (0.42365) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.10367 (1.90458) + | > current_lr: 0.00001 + | > step_time: 9.59680 (4.85715) + | > loader_time: 0.09290 (0.12752) + + + --> STEP: 16/234 -- GLOBAL_STEP: 7270 + | > loss: 0.42980 (0.51652) + | > log_mle: 0.10054 (0.11590) + | > loss_dur: 0.32925 (0.40062) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.00240 (1.68807) + | > current_lr: 0.00001 + | > step_time: 11.20840 (5.56025) + | > loader_time: 0.39270 (0.13558) + + + --> STEP: 21/234 -- GLOBAL_STEP: 7275 + | > loss: 0.44694 (0.50414) + | > log_mle: 0.11975 (0.11589) + | > loss_dur: 0.32719 (0.38826) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.37629 (1.58820) + | > current_lr: 0.00001 + | > step_time: 2.02230 (4.68428) + | > loader_time: 0.00370 (0.10877) + + + --> STEP: 26/234 -- GLOBAL_STEP: 7280 + | > loss: 0.41397 (0.48881) + | > log_mle: 0.10364 (0.11376) + | > loss_dur: 0.31033 (0.37505) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.01721 (1.49822) + | > current_lr: 0.00001 + | > step_time: 3.78610 (4.16735) + | > loader_time: 0.00360 (0.09464) + + + --> STEP: 31/234 -- GLOBAL_STEP: 7285 + | > loss: 0.44531 (0.47797) + | > log_mle: 0.09403 (0.11130) + | > loss_dur: 0.35128 (0.36667) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.28947 (1.44273) + | > current_lr: 0.00001 + | > step_time: 4.19830 (4.38474) + | > loader_time: 0.00190 (0.08612) + + + --> STEP: 36/234 -- GLOBAL_STEP: 7290 + | > loss: 0.42858 (0.46978) + | > log_mle: 0.08950 (0.10863) + | > loss_dur: 0.33907 (0.36114) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.61896 (1.69474) + | > current_lr: 0.00001 + | > step_time: 5.28940 (4.42585) + | > loader_time: 0.10170 (0.08426) + + + --> STEP: 41/234 -- GLOBAL_STEP: 7295 + | > loss: 0.41065 (0.46459) + | > log_mle: 0.08957 (0.10700) + | > loss_dur: 0.32108 (0.35760) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.00027 (1.64106) + | > current_lr: 0.00001 + | > step_time: 1.89670 (4.17859) + | > loader_time: 0.00240 (0.08039) + + + --> STEP: 46/234 -- GLOBAL_STEP: 7300 + | > loss: 0.40040 (0.45827) + | > log_mle: 0.09337 (0.10576) + | > loss_dur: 0.30703 (0.35251) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.16931 (1.61974) + | > current_lr: 0.00001 + | > step_time: 2.19800 (3.95210) + | > loader_time: 0.00430 (0.07408) + + + --> STEP: 51/234 -- GLOBAL_STEP: 7305 + | > loss: 0.40478 (0.45232) + | > log_mle: 0.10904 (0.10497) + | > loss_dur: 0.29574 (0.34735) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.31165 (1.57318) + | > current_lr: 0.00001 + | > step_time: 1.35800 (3.76311) + | > loader_time: 0.00210 (0.07194) + + + --> STEP: 56/234 -- GLOBAL_STEP: 7310 + | > loss: 0.41643 (0.44768) + | > log_mle: 0.08922 (0.10346) + | > loss_dur: 0.32721 (0.34422) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.97956 (1.54862) + | > current_lr: 0.00001 + | > step_time: 1.40030 (3.57296) + | > loader_time: 0.00230 (0.06575) + + + --> STEP: 61/234 -- GLOBAL_STEP: 7315 + | > loss: 0.38538 (0.44263) + | > log_mle: 0.09432 (0.10204) + | > loss_dur: 0.29106 (0.34059) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.90129 (1.53865) + | > current_lr: 0.00001 + | > step_time: 3.69430 (3.45666) + | > loader_time: 0.00530 (0.06198) + + + --> STEP: 66/234 -- GLOBAL_STEP: 7320 + | > loss: 0.39669 (0.43830) + | > log_mle: 0.10075 (0.10058) + | > loss_dur: 0.29595 (0.33772) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.26928 (1.56247) + | > current_lr: 0.00001 + | > step_time: 1.87570 (3.38699) + | > loader_time: 0.00270 (0.06322) + + + --> STEP: 71/234 -- GLOBAL_STEP: 7325 + | > loss: 0.37972 (0.43487) + | > log_mle: 0.05404 (0.09909) + | > loss_dur: 0.32569 (0.33578) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.43763 (1.78250) + | > current_lr: 0.00001 + | > step_time: 2.60640 (3.34595) + | > loader_time: 0.00260 (0.06258) + + + --> STEP: 76/234 -- GLOBAL_STEP: 7330 + | > loss: 0.37121 (0.43120) + | > log_mle: 0.07666 (0.09754) + | > loss_dur: 0.29455 (0.33366) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.04700 (1.82469) + | > current_lr: 0.00001 + | > step_time: 5.01500 (3.30778) + | > loader_time: 0.08880 (0.06102) + + + --> STEP: 81/234 -- GLOBAL_STEP: 7335 + | > loss: 0.34551 (0.42684) + | > log_mle: 0.05893 (0.09616) + | > loss_dur: 0.28659 (0.33069) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.59374 (1.87581) + | > current_lr: 0.00001 + | > step_time: 2.39380 (3.27513) + | > loader_time: 0.00340 (0.05962) + + + --> STEP: 86/234 -- GLOBAL_STEP: 7340 + | > loss: 0.37710 (0.42349) + | > log_mle: 0.05793 (0.09458) + | > loss_dur: 0.31917 (0.32890) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.56734 (1.94320) + | > current_lr: 0.00001 + | > step_time: 1.40790 (3.21298) + | > loader_time: 0.08780 (0.05837) + + + --> STEP: 91/234 -- GLOBAL_STEP: 7345 + | > loss: 0.39400 (0.41992) + | > log_mle: 0.05676 (0.09194) + | > loss_dur: 0.33724 (0.32798) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.38117 (2.18057) + | > current_lr: 0.00001 + | > step_time: 3.21520 (3.26415) + | > loader_time: 0.08730 (0.05935) + + + --> STEP: 96/234 -- GLOBAL_STEP: 7350 + | > loss: 0.37017 (0.41495) + | > log_mle: 0.06198 (0.08791) + | > loss_dur: 0.30818 (0.32704) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.88488 (2.51181) + | > current_lr: 0.00001 + | > step_time: 3.59730 (3.28879) + | > loader_time: 0.00850 (0.05727) + + + --> STEP: 101/234 -- GLOBAL_STEP: 7355 + | > loss: 0.31473 (0.41107) + | > log_mle: 0.00918 (0.08487) + | > loss_dur: 0.30555 (0.32620) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.88956 (2.74701) + | > current_lr: 0.00001 + | > step_time: 4.01530 (3.25534) + | > loader_time: 0.09940 (0.05654) + + + --> STEP: 106/234 -- GLOBAL_STEP: 7360 + | > loss: 0.38454 (0.40807) + | > log_mle: 0.00914 (0.08174) + | > loss_dur: 0.37540 (0.32632) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.51838 (2.95744) + | > current_lr: 0.00001 + | > step_time: 2.69950 (3.19049) + | > loader_time: 0.00700 (0.05484) + + + --> STEP: 111/234 -- GLOBAL_STEP: 7365 + | > loss: 0.31712 (0.40458) + | > log_mle: -0.02576 (0.07856) + | > loss_dur: 0.34288 (0.32602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.37230 (3.31018) + | > current_lr: 0.00001 + | > step_time: 3.18750 (3.14552) + | > loader_time: 0.00750 (0.05352) + + + --> STEP: 116/234 -- GLOBAL_STEP: 7370 + | > loss: 0.34565 (0.40173) + | > log_mle: 0.00202 (0.07549) + | > loss_dur: 0.34363 (0.32624) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.01996 (3.56187) + | > current_lr: 0.00001 + | > step_time: 3.20850 (3.15219) + | > loader_time: 0.09330 (0.05440) + + + --> STEP: 121/234 -- GLOBAL_STEP: 7375 + | > loss: 0.38665 (0.39920) + | > log_mle: 0.07899 (0.07323) + | > loss_dur: 0.30766 (0.32597) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.33894 (3.69192) + | > current_lr: 0.00001 + | > step_time: 2.81550 (3.12115) + | > loader_time: 0.19230 (0.05469) + + + --> STEP: 126/234 -- GLOBAL_STEP: 7380 + | > loss: 0.29942 (0.39623) + | > log_mle: -0.04688 (0.07036) + | > loss_dur: 0.34629 (0.32586) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.01254 (3.83954) + | > current_lr: 0.00001 + | > step_time: 1.71240 (3.09093) + | > loader_time: 0.00530 (0.05322) + + + --> STEP: 131/234 -- GLOBAL_STEP: 7385 + | > loss: 0.26908 (0.39277) + | > log_mle: -0.07519 (0.06675) + | > loss_dur: 0.34428 (0.32602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.81148 (4.00540) + | > current_lr: 0.00001 + | > step_time: 3.39840 (3.07986) + | > loader_time: 0.08600 (0.05400) + + + --> STEP: 136/234 -- GLOBAL_STEP: 7390 + | > loss: 0.27126 (0.38983) + | > log_mle: -0.11870 (0.06321) + | > loss_dur: 0.38996 (0.32662) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.44039 (4.22885) + | > current_lr: 0.00001 + | > step_time: 4.39760 (3.10777) + | > loader_time: 0.10840 (0.05432) + + + --> STEP: 141/234 -- GLOBAL_STEP: 7395 + | > loss: 0.28886 (0.38691) + | > log_mle: -0.04100 (0.06007) + | > loss_dur: 0.32986 (0.32684) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.21403 (4.36778) + | > current_lr: 0.00001 + | > step_time: 5.00230 (3.13452) + | > loader_time: 0.00320 (0.05461) + + + --> STEP: 146/234 -- GLOBAL_STEP: 7400 + | > loss: 0.27990 (0.38313) + | > log_mle: -0.08458 (0.05550) + | > loss_dur: 0.36449 (0.32763) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.71772 (4.61154) + | > current_lr: 0.00001 + | > step_time: 1.07980 (3.08950) + | > loader_time: 0.00260 (0.05336) + + + --> STEP: 151/234 -- GLOBAL_STEP: 7405 + | > loss: 0.27467 (0.37954) + | > log_mle: -0.05715 (0.05172) + | > loss_dur: 0.33182 (0.32782) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.77350 (4.92578) + | > current_lr: 0.00001 + | > step_time: 2.68490 (3.05353) + | > loader_time: 0.00180 (0.05333) + + + --> STEP: 156/234 -- GLOBAL_STEP: 7410 + | > loss: 0.23861 (0.37532) + | > log_mle: -0.08912 (0.04677) + | > loss_dur: 0.32773 (0.32855) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.00986 (5.29102) + | > current_lr: 0.00001 + | > step_time: 3.20280 (3.02622) + | > loader_time: 0.09770 (0.05237) + + + --> STEP: 161/234 -- GLOBAL_STEP: 7415 + | > loss: 0.25028 (0.37176) + | > log_mle: -0.10469 (0.04251) + | > loss_dur: 0.35498 (0.32925) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 20.31916 (5.53865) + | > current_lr: 0.00001 + | > step_time: 5.69450 (3.15703) + | > loader_time: 0.10770 (0.05396) + + + --> STEP: 166/234 -- GLOBAL_STEP: 7420 + | > loss: 0.26740 (0.36844) + | > log_mle: -0.05862 (0.03882) + | > loss_dur: 0.32602 (0.32962) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.53891 (5.79333) + | > current_lr: 0.00001 + | > step_time: 4.40730 (3.16606) + | > loader_time: 0.00630 (0.05367) + + + --> STEP: 171/234 -- GLOBAL_STEP: 7425 + | > loss: 0.19151 (0.36484) + | > log_mle: -0.14997 (0.03405) + | > loss_dur: 0.34148 (0.33080) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.67690 (6.12041) + | > current_lr: 0.00001 + | > step_time: 3.70360 (3.25264) + | > loader_time: 0.00330 (0.05610) + + + --> STEP: 176/234 -- GLOBAL_STEP: 7430 + | > loss: 0.23532 (0.36096) + | > log_mle: -0.12228 (0.02936) + | > loss_dur: 0.35760 (0.33160) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.73973 (6.40798) + | > current_lr: 0.00001 + | > step_time: 3.00280 (3.30754) + | > loader_time: 0.18980 (0.05827) + + + --> STEP: 181/234 -- GLOBAL_STEP: 7435 + | > loss: 0.29009 (0.35782) + | > log_mle: -0.07082 (0.02515) + | > loss_dur: 0.36091 (0.33268) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.11933 (6.73028) + | > current_lr: 0.00001 + | > step_time: 2.10550 (3.34104) + | > loader_time: 0.00340 (0.05941) + + + --> STEP: 186/234 -- GLOBAL_STEP: 7440 + | > loss: 0.26944 (0.35467) + | > log_mle: -0.10284 (0.02080) + | > loss_dur: 0.37228 (0.33387) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.48629 (7.05887) + | > current_lr: 0.00001 + | > step_time: 1.69190 (3.35922) + | > loader_time: 0.00370 (0.06154) + + + --> STEP: 191/234 -- GLOBAL_STEP: 7445 + | > loss: 0.25047 (0.35137) + | > log_mle: -0.11060 (0.01667) + | > loss_dur: 0.36107 (0.33470) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.71652 (7.36101) + | > current_lr: 0.00001 + | > step_time: 4.49500 (3.34671) + | > loader_time: 0.10570 (0.06060) + + + --> STEP: 196/234 -- GLOBAL_STEP: 7450 + | > loss: 0.27008 (0.34826) + | > log_mle: -0.10869 (0.01247) + | > loss_dur: 0.37877 (0.33578) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.70354 (7.69814) + | > current_lr: 0.00001 + | > step_time: 4.40400 (3.35374) + | > loader_time: 0.09530 (0.06056) + + + --> STEP: 201/234 -- GLOBAL_STEP: 7455 + | > loss: 0.28817 (0.34542) + | > log_mle: -0.08959 (0.00866) + | > loss_dur: 0.37777 (0.33676) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.12284 (7.94606) + | > current_lr: 0.00001 + | > step_time: 3.90410 (3.41319) + | > loader_time: 0.09160 (0.06235) + + + --> STEP: 206/234 -- GLOBAL_STEP: 7460 + | > loss: 0.20602 (0.34226) + | > log_mle: -0.17182 (0.00440) + | > loss_dur: 0.37785 (0.33786) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.32088 (8.16976) + | > current_lr: 0.00001 + | > step_time: 3.81380 (3.47301) + | > loader_time: 0.00570 (0.06341) + + + --> STEP: 211/234 -- GLOBAL_STEP: 7465 + | > loss: 0.17419 (0.33883) + | > log_mle: -0.23634 (-0.00034) + | > loss_dur: 0.41053 (0.33918) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.72041 (8.38633) + | > current_lr: 0.00001 + | > step_time: 5.30380 (3.52105) + | > loader_time: 0.39130 (0.06704) + + + --> STEP: 216/234 -- GLOBAL_STEP: 7470 + | > loss: 0.16850 (0.33542) + | > log_mle: -0.22198 (-0.00483) + | > loss_dur: 0.39048 (0.34026) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 25.04965 (8.71534) + | > current_lr: 0.00001 + | > step_time: 5.79650 (3.64089) + | > loader_time: 0.08920 (0.06649) + + + --> STEP: 221/234 -- GLOBAL_STEP: 7475 + | > loss: 0.22747 (0.33215) + | > log_mle: -0.15311 (-0.00931) + | > loss_dur: 0.38057 (0.34145) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.80722 (8.98694) + | > current_lr: 0.00001 + | > step_time: 3.89660 (3.67113) + | > loader_time: 0.00700 (0.06542) + + + --> STEP: 226/234 -- GLOBAL_STEP: 7480 + | > loss: 0.15915 (0.32858) + | > log_mle: -0.24012 (-0.01423) + | > loss_dur: 0.39927 (0.34281) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.52626 (9.29800) + | > current_lr: 0.00001 + | > step_time: 0.23530 (3.60086) + | > loader_time: 0.00260 (0.06404) + + + --> STEP: 231/234 -- GLOBAL_STEP: 7485 + | > loss: 0.23174 (0.32601) + | > log_mle: -0.29356 (-0.01937) + | > loss_dur: 0.52530 (0.34538) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.98016 (9.60515) + | > current_lr: 0.00001 + | > step_time: 0.28750 (3.52870) + | > loader_time: 0.00350 (0.06274) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.48222 (-0.04202) + | > avg_loss: 0.26171 (-0.03537) + | > avg_log_mle: -0.09081 (-0.02043) + | > avg_loss_dur: 0.35251 (-0.01494) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_7488.pth + + > EPOCH: 32/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 16:10:00)  + + --> STEP: 2/234 -- GLOBAL_STEP: 7490 + | > loss: 0.62370 (0.61299) + | > log_mle: 0.14734 (0.13944) + | > loss_dur: 0.47637 (0.47356) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.34408 (1.48931) + | > current_lr: 0.00001 + | > step_time: 4.90520 (3.75213) + | > loader_time: 0.00110 (0.89867) + + + --> STEP: 7/234 -- GLOBAL_STEP: 7495 + | > loss: 0.46504 (0.53092) + | > log_mle: 0.08946 (0.11414) + | > loss_dur: 0.37558 (0.41678) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.43177 (1.94230) + | > current_lr: 0.00001 + | > step_time: 6.20710 (4.27570) + | > loader_time: 0.00160 (0.34067) + + + --> STEP: 12/234 -- GLOBAL_STEP: 7500 + | > loss: 0.41949 (0.49022) + | > log_mle: 0.09963 (0.10575) + | > loss_dur: 0.31986 (0.38446) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.37189 (1.80267) + | > current_lr: 0.00001 + | > step_time: 3.50040 (4.23576) + | > loader_time: 0.00800 (0.21646) + + + --> STEP: 17/234 -- GLOBAL_STEP: 7505 + | > loss: 0.42466 (0.47481) + | > log_mle: 0.11031 (0.10364) + | > loss_dur: 0.31435 (0.37117) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.49333 (1.67104) + | > current_lr: 0.00001 + | > step_time: 7.30640 (5.07152) + | > loader_time: 0.00400 (0.16488) + + + --> STEP: 22/234 -- GLOBAL_STEP: 7510 + | > loss: 0.37720 (0.46157) + | > log_mle: 0.08293 (0.10233) + | > loss_dur: 0.29426 (0.35923) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.19748 (1.58715) + | > current_lr: 0.00001 + | > step_time: 4.38880 (5.06849) + | > loader_time: 0.00100 (0.14582) + + + --> STEP: 27/234 -- GLOBAL_STEP: 7515 + | > loss: 0.37884 (0.44864) + | > log_mle: 0.08169 (0.10053) + | > loss_dur: 0.29715 (0.34812) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.21410 (1.52373) + | > current_lr: 0.00001 + | > step_time: 1.04050 (4.65992) + | > loader_time: 0.00110 (0.12287) + + + --> STEP: 32/234 -- GLOBAL_STEP: 7520 + | > loss: 0.38279 (0.44016) + | > log_mle: 0.06465 (0.09772) + | > loss_dur: 0.31814 (0.34244) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.13210 (1.46656) + | > current_lr: 0.00001 + | > step_time: 3.00200 (4.26530) + | > loader_time: 0.00300 (0.10700) + + + --> STEP: 37/234 -- GLOBAL_STEP: 7525 + | > loss: 0.37375 (0.43358) + | > log_mle: 0.08201 (0.09578) + | > loss_dur: 0.29173 (0.33779) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.19564 (1.69865) + | > current_lr: 0.00001 + | > step_time: 2.59500 (4.49697) + | > loader_time: 0.00350 (0.10041) + + + --> STEP: 42/234 -- GLOBAL_STEP: 7530 + | > loss: 0.40544 (0.43007) + | > log_mle: 0.10562 (0.09477) + | > loss_dur: 0.29981 (0.33530) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.04203 (1.67494) + | > current_lr: 0.00001 + | > step_time: 2.91070 (4.25280) + | > loader_time: 0.09330 (0.09099) + + + --> STEP: 47/234 -- GLOBAL_STEP: 7535 + | > loss: 0.38900 (0.42399) + | > log_mle: 0.08284 (0.09303) + | > loss_dur: 0.30616 (0.33095) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.19945 (1.66711) + | > current_lr: 0.00001 + | > step_time: 1.64240 (3.98002) + | > loader_time: 0.00190 (0.08315) + + + --> STEP: 52/234 -- GLOBAL_STEP: 7540 + | > loss: 0.37685 (0.41855) + | > log_mle: 0.09480 (0.09252) + | > loss_dur: 0.28205 (0.32603) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.79703 (1.65521) + | > current_lr: 0.00001 + | > step_time: 1.16630 (3.77643) + | > loader_time: 0.00140 (0.07545) + + + --> STEP: 57/234 -- GLOBAL_STEP: 7545 + | > loss: 0.40585 (0.41494) + | > log_mle: 0.09344 (0.09104) + | > loss_dur: 0.31242 (0.32391) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.15651 (1.61724) + | > current_lr: 0.00001 + | > step_time: 1.93770 (3.64297) + | > loader_time: 0.00200 (0.07058) + + + --> STEP: 62/234 -- GLOBAL_STEP: 7550 + | > loss: 0.35578 (0.40924) + | > log_mle: 0.03403 (0.08867) + | > loss_dur: 0.32176 (0.32057) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.71924 (1.64799) + | > current_lr: 0.00001 + | > step_time: 1.39800 (3.50165) + | > loader_time: 0.00210 (0.06658) + + + --> STEP: 67/234 -- GLOBAL_STEP: 7555 + | > loss: 0.33661 (0.40546) + | > log_mle: 0.05164 (0.08757) + | > loss_dur: 0.28497 (0.31789) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.70034 (1.68910) + | > current_lr: 0.00001 + | > step_time: 1.43510 (3.37735) + | > loader_time: 0.00230 (0.06179) + + + --> STEP: 72/234 -- GLOBAL_STEP: 7560 + | > loss: 0.37054 (0.40277) + | > log_mle: 0.06901 (0.08638) + | > loss_dur: 0.30153 (0.31639) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.16982 (1.90795) + | > current_lr: 0.00001 + | > step_time: 2.29740 (3.28661) + | > loader_time: 0.00380 (0.05881) + + + --> STEP: 77/234 -- GLOBAL_STEP: 7565 + | > loss: 0.34124 (0.39929) + | > log_mle: 0.05384 (0.08472) + | > loss_dur: 0.28740 (0.31458) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.69437 (1.97528) + | > current_lr: 0.00001 + | > step_time: 3.50670 (3.22509) + | > loader_time: 0.00260 (0.05624) + + + --> STEP: 82/234 -- GLOBAL_STEP: 7570 + | > loss: 0.34426 (0.39577) + | > log_mle: 0.07166 (0.08365) + | > loss_dur: 0.27260 (0.31212) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.14762 (2.02763) + | > current_lr: 0.00001 + | > step_time: 1.69700 (3.20065) + | > loader_time: 0.19040 (0.05974) + + + --> STEP: 87/234 -- GLOBAL_STEP: 7575 + | > loss: 0.33619 (0.39263) + | > log_mle: 0.05265 (0.08193) + | > loss_dur: 0.28354 (0.31071) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.70321 (2.11655) + | > current_lr: 0.00001 + | > step_time: 2.30180 (3.13541) + | > loader_time: 0.08740 (0.06025) + + + --> STEP: 92/234 -- GLOBAL_STEP: 7580 + | > loss: 0.29091 (0.38859) + | > log_mle: 0.01663 (0.07891) + | > loss_dur: 0.27428 (0.30968) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.60670 (2.26376) + | > current_lr: 0.00001 + | > step_time: 2.10020 (3.08588) + | > loader_time: 0.08280 (0.05800) + + + --> STEP: 97/234 -- GLOBAL_STEP: 7585 + | > loss: 0.29342 (0.38387) + | > log_mle: 0.02145 (0.07499) + | > loss_dur: 0.27197 (0.30887) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.07540 (2.51156) + | > current_lr: 0.00001 + | > step_time: 3.00010 (3.05467) + | > loader_time: 0.00390 (0.05785) + + + --> STEP: 102/234 -- GLOBAL_STEP: 7590 + | > loss: 0.33360 (0.38077) + | > log_mle: 0.04388 (0.07222) + | > loss_dur: 0.28971 (0.30855) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.14570 (2.74762) + | > current_lr: 0.00001 + | > step_time: 3.99160 (3.03910) + | > loader_time: 0.00350 (0.05599) + + + --> STEP: 107/234 -- GLOBAL_STEP: 7595 + | > loss: 0.29689 (0.37769) + | > log_mle: -0.00452 (0.06873) + | > loss_dur: 0.30141 (0.30896) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.11269 (3.00562) + | > current_lr: 0.00001 + | > step_time: 2.05110 (2.99481) + | > loader_time: 0.00270 (0.05353) + + + --> STEP: 112/234 -- GLOBAL_STEP: 7600 + | > loss: 0.30410 (0.37437) + | > log_mle: -0.00900 (0.06554) + | > loss_dur: 0.31310 (0.30883) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.17892 (3.23895) + | > current_lr: 0.00001 + | > step_time: 3.49650 (2.96038) + | > loader_time: 0.00280 (0.05127) + + + --> STEP: 117/234 -- GLOBAL_STEP: 7605 + | > loss: 0.31433 (0.37170) + | > log_mle: 0.00044 (0.06262) + | > loss_dur: 0.31389 (0.30909) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.06147 (3.46723) + | > current_lr: 0.00001 + | > step_time: 5.59570 (2.95200) + | > loader_time: 0.10510 (0.05225) + + + --> STEP: 122/234 -- GLOBAL_STEP: 7610 + | > loss: 0.31182 (0.36957) + | > log_mle: 0.01319 (0.06052) + | > loss_dur: 0.29862 (0.30905) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.49844 (3.54897) + | > current_lr: 0.00001 + | > step_time: 2.39030 (2.94396) + | > loader_time: 0.00420 (0.05175) + + + --> STEP: 127/234 -- GLOBAL_STEP: 7615 + | > loss: 0.28159 (0.36625) + | > log_mle: -0.03180 (0.05735) + | > loss_dur: 0.31339 (0.30890) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.79049 (3.69470) + | > current_lr: 0.00001 + | > step_time: 1.69130 (2.92087) + | > loader_time: 0.00280 (0.05207) + + + --> STEP: 132/234 -- GLOBAL_STEP: 7620 + | > loss: 0.27900 (0.36302) + | > log_mle: -0.01498 (0.05392) + | > loss_dur: 0.29398 (0.30910) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.01793 (3.87723) + | > current_lr: 0.00001 + | > step_time: 2.78000 (2.92245) + | > loader_time: 0.00160 (0.05171) + + + --> STEP: 137/234 -- GLOBAL_STEP: 7625 + | > loss: 0.30815 (0.36040) + | > log_mle: -0.02277 (0.05037) + | > loss_dur: 0.33092 (0.31003) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.93995 (4.11591) + | > current_lr: 0.00001 + | > step_time: 1.70410 (2.90323) + | > loader_time: 0.00440 (0.05052) + + + --> STEP: 142/234 -- GLOBAL_STEP: 7630 + | > loss: 0.27353 (0.35732) + | > log_mle: -0.04325 (0.04713) + | > loss_dur: 0.31678 (0.31019) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.13578 (4.32321) + | > current_lr: 0.00001 + | > step_time: 3.02900 (2.87879) + | > loader_time: 0.07970 (0.05055) + + + --> STEP: 147/234 -- GLOBAL_STEP: 7635 + | > loss: 0.26986 (0.35377) + | > log_mle: -0.04471 (0.04260) + | > loss_dur: 0.31456 (0.31117) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.35817 (4.57570) + | > current_lr: 0.00001 + | > step_time: 1.59450 (2.84883) + | > loader_time: 0.09340 (0.05019) + + + --> STEP: 152/234 -- GLOBAL_STEP: 7640 + | > loss: 0.23811 (0.35018) + | > log_mle: -0.10893 (0.03842) + | > loss_dur: 0.34703 (0.31176) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.21812 (4.83117) + | > current_lr: 0.00001 + | > step_time: 2.22110 (2.84356) + | > loader_time: 0.00390 (0.04917) + + + --> STEP: 157/234 -- GLOBAL_STEP: 7645 + | > loss: 0.27098 (0.34631) + | > log_mle: -0.06332 (0.03381) + | > loss_dur: 0.33431 (0.31250) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.08046 (5.17410) + | > current_lr: 0.00001 + | > step_time: 4.30480 (2.83442) + | > loader_time: 0.09490 (0.04895) + + + --> STEP: 162/234 -- GLOBAL_STEP: 7650 + | > loss: 0.22609 (0.34271) + | > log_mle: -0.09696 (0.02937) + | > loss_dur: 0.32305 (0.31334) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.49030 (5.45063) + | > current_lr: 0.00001 + | > step_time: 4.80500 (2.83169) + | > loader_time: 0.09700 (0.04919) + + + --> STEP: 167/234 -- GLOBAL_STEP: 7655 + | > loss: 0.18860 (0.33919) + | > log_mle: -0.15906 (0.02531) + | > loss_dur: 0.34766 (0.31388) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.11541 (5.65478) + | > current_lr: 0.00001 + | > step_time: 4.20750 (2.86320) + | > loader_time: 0.11390 (0.04906) + + + --> STEP: 172/234 -- GLOBAL_STEP: 7660 + | > loss: 0.20502 (0.33582) + | > log_mle: -0.15141 (0.02060) + | > loss_dur: 0.35643 (0.31521) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.15452 (5.90015) + | > current_lr: 0.00001 + | > step_time: 2.10450 (2.87765) + | > loader_time: 0.09140 (0.04984) + + + --> STEP: 177/234 -- GLOBAL_STEP: 7665 + | > loss: 0.22636 (0.33216) + | > log_mle: -0.11792 (0.01614) + | > loss_dur: 0.34428 (0.31602) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.45456 (6.14899) + | > current_lr: 0.00001 + | > step_time: 4.30520 (2.89077) + | > loader_time: 0.00260 (0.05005) + + + --> STEP: 182/234 -- GLOBAL_STEP: 7670 + | > loss: 0.22525 (0.32901) + | > log_mle: -0.15844 (0.01171) + | > loss_dur: 0.38369 (0.31730) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.00008 (6.36743) + | > current_lr: 0.00001 + | > step_time: 3.10170 (2.90698) + | > loader_time: 0.19620 (0.05195) + + + --> STEP: 187/234 -- GLOBAL_STEP: 7675 + | > loss: 0.19207 (0.32590) + | > log_mle: -0.15811 (0.00738) + | > loss_dur: 0.35017 (0.31852) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 24.52634 (6.67074) + | > current_lr: 0.00001 + | > step_time: 4.20120 (2.95665) + | > loader_time: 0.10090 (0.05415) + + + --> STEP: 192/234 -- GLOBAL_STEP: 7680 + | > loss: 0.16836 (0.32254) + | > log_mle: -0.17986 (0.00319) + | > loss_dur: 0.34822 (0.31934) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.32700 (6.95444) + | > current_lr: 0.00001 + | > step_time: 12.39720 (3.07405) + | > loader_time: 0.01450 (0.05473) + + + --> STEP: 197/234 -- GLOBAL_STEP: 7685 + | > loss: 0.18492 (0.31960) + | > log_mle: -0.15735 (-0.00089) + | > loss_dur: 0.34227 (0.32049) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 20.28746 (7.20989) + | > current_lr: 0.00001 + | > step_time: 8.00350 (3.17948) + | > loader_time: 0.09090 (0.05774) + + + --> STEP: 202/234 -- GLOBAL_STEP: 7690 + | > loss: 0.14009 (0.31668) + | > log_mle: -0.23516 (-0.00506) + | > loss_dur: 0.37525 (0.32175) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.77450 (7.52346) + | > current_lr: 0.00001 + | > step_time: 5.89680 (3.24322) + | > loader_time: 0.00600 (0.05741) + + + --> STEP: 207/234 -- GLOBAL_STEP: 7695 + | > loss: 0.16133 (0.31363) + | > log_mle: -0.22407 (-0.00925) + | > loss_dur: 0.38539 (0.32287) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.40143 (7.76186) + | > current_lr: 0.00001 + | > step_time: 2.19520 (3.25866) + | > loader_time: 0.00450 (0.05758) + + + --> STEP: 212/234 -- GLOBAL_STEP: 7700 + | > loss: 0.16994 (0.31038) + | > log_mle: -0.20913 (-0.01390) + | > loss_dur: 0.37907 (0.32428) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 21.21166 (8.08607) + | > current_lr: 0.00001 + | > step_time: 9.50100 (3.30300) + | > loader_time: 0.00540 (0.05720) + + + --> STEP: 217/234 -- GLOBAL_STEP: 7705 + | > loss: 0.15434 (0.30690) + | > log_mle: -0.22697 (-0.01847) + | > loss_dur: 0.38131 (0.32537) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.59307 (8.35855) + | > current_lr: 0.00001 + | > step_time: 5.59250 (3.41026) + | > loader_time: 0.00970 (0.05872) + + + --> STEP: 222/234 -- GLOBAL_STEP: 7710 + | > loss: 0.17311 (0.30378) + | > log_mle: -0.23174 (-0.02297) + | > loss_dur: 0.40484 (0.32675) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 28.69006 (8.63835) + | > current_lr: 0.00001 + | > step_time: 0.23930 (3.38441) + | > loader_time: 0.00320 (0.05749) + + + --> STEP: 227/234 -- GLOBAL_STEP: 7715 + | > loss: 0.17107 (0.30031) + | > log_mle: -0.21750 (-0.02781) + | > loss_dur: 0.38856 (0.32812) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.09249 (8.93865) + | > current_lr: 0.00001 + | > step_time: 0.25760 (3.31522) + | > loader_time: 0.00350 (0.05630) + + + --> STEP: 232/234 -- GLOBAL_STEP: 7720 + | > loss: 0.29002 (0.29834) + | > log_mle: -0.38474 (-0.03367) + | > loss_dur: 0.67476 (0.33201) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 34.98093 (9.40014) + | > current_lr: 0.00001 + | > step_time: 0.32260 (3.24960) + | > loader_time: 0.00460 (0.05517) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02663 (-0.45559) + | > avg_loss: 0.24604 (-0.01566) + | > avg_log_mle: -0.09289 (-0.00208) + | > avg_loss_dur: 0.33893 (-0.01358) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_7722.pth + + > EPOCH: 33/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 16:23:52)  + + --> STEP: 3/234 -- GLOBAL_STEP: 7725 + | > loss: 0.46293 (0.52677) + | > log_mle: 0.10218 (0.11805) + | > loss_dur: 0.36075 (0.40872) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.41480 (1.46822) + | > current_lr: 0.00001 + | > step_time: 3.30740 (7.13819) + | > loader_time: 0.00780 (0.26620) + + + --> STEP: 8/234 -- GLOBAL_STEP: 7730 + | > loss: 0.39768 (0.47346) + | > log_mle: 0.07482 (0.09822) + | > loss_dur: 0.32286 (0.37524) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.39430 (2.35946) + | > current_lr: 0.00001 + | > step_time: 3.61850 (3.77748) + | > loader_time: 0.09140 (0.13239) + + + --> STEP: 13/234 -- GLOBAL_STEP: 7735 + | > loss: 0.42595 (0.44806) + | > log_mle: 0.09543 (0.09341) + | > loss_dur: 0.33052 (0.35465) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.61150 (2.17003) + | > current_lr: 0.00001 + | > step_time: 1.20530 (3.65497) + | > loader_time: 0.07300 (0.09622) + + + --> STEP: 18/234 -- GLOBAL_STEP: 7740 + | > loss: 0.38984 (0.43500) + | > log_mle: 0.08417 (0.09081) + | > loss_dur: 0.30567 (0.34420) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.38216 (1.97976) + | > current_lr: 0.00001 + | > step_time: 2.43100 (3.25985) + | > loader_time: 0.08450 (0.07456) + + + --> STEP: 23/234 -- GLOBAL_STEP: 7745 + | > loss: 0.35694 (0.42298) + | > log_mle: 0.07575 (0.08921) + | > loss_dur: 0.28119 (0.33377) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.38921 (1.83661) + | > current_lr: 0.00001 + | > step_time: 2.18170 (2.97015) + | > loader_time: 0.00260 (0.05880) + + + --> STEP: 28/234 -- GLOBAL_STEP: 7750 + | > loss: 0.35494 (0.41091) + | > log_mle: 0.07914 (0.08761) + | > loss_dur: 0.27581 (0.32330) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.13348 (1.73098) + | > current_lr: 0.00001 + | > step_time: 2.91500 (3.24701) + | > loader_time: 0.08520 (0.05490) + + + --> STEP: 33/234 -- GLOBAL_STEP: 7755 + | > loss: 0.38373 (0.40427) + | > log_mle: 0.08746 (0.08525) + | > loss_dur: 0.29627 (0.31902) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.29414 (1.65915) + | > current_lr: 0.00001 + | > step_time: 4.50650 (3.46208) + | > loader_time: 0.00300 (0.05257) + + + --> STEP: 38/234 -- GLOBAL_STEP: 7760 + | > loss: 0.39110 (0.39920) + | > log_mle: 0.06538 (0.08282) + | > loss_dur: 0.32571 (0.31638) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.97072 (1.89594) + | > current_lr: 0.00001 + | > step_time: 1.19750 (3.52187) + | > loader_time: 0.00190 (0.04620) + + + --> STEP: 43/234 -- GLOBAL_STEP: 7765 + | > loss: 0.35445 (0.39686) + | > log_mle: 0.06334 (0.08184) + | > loss_dur: 0.29111 (0.31502) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.46729 (1.84545) + | > current_lr: 0.00001 + | > step_time: 1.19820 (3.35375) + | > loader_time: 0.00200 (0.04323) + + + --> STEP: 48/234 -- GLOBAL_STEP: 7770 + | > loss: 0.33725 (0.39156) + | > log_mle: 0.07306 (0.08036) + | > loss_dur: 0.26419 (0.31119) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.22223 (1.81755) + | > current_lr: 0.00001 + | > step_time: 0.99760 (3.18652) + | > loader_time: 0.08200 (0.04415) + + + --> STEP: 53/234 -- GLOBAL_STEP: 7775 + | > loss: 0.34232 (0.38710) + | > log_mle: 0.05433 (0.07951) + | > loss_dur: 0.28799 (0.30759) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.04833 (1.78159) + | > current_lr: 0.00001 + | > step_time: 2.00460 (3.12211) + | > loader_time: 0.07650 (0.04329) + + + --> STEP: 58/234 -- GLOBAL_STEP: 7780 + | > loss: 0.34479 (0.38435) + | > log_mle: 0.07205 (0.07840) + | > loss_dur: 0.27273 (0.30595) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.31646 (1.72798) + | > current_lr: 0.00001 + | > step_time: 1.71330 (2.99653) + | > loader_time: 0.08280 (0.04424) + + + --> STEP: 63/234 -- GLOBAL_STEP: 7785 + | > loss: 0.34058 (0.37949) + | > log_mle: 0.05528 (0.07581) + | > loss_dur: 0.28529 (0.30367) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.53009 (1.76722) + | > current_lr: 0.00001 + | > step_time: 3.61620 (3.02225) + | > loader_time: 0.08400 (0.04233) + + + --> STEP: 68/234 -- GLOBAL_STEP: 7790 + | > loss: 0.33589 (0.37606) + | > log_mle: 0.06640 (0.07496) + | > loss_dur: 0.26949 (0.30110) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.32083 (1.88313) + | > current_lr: 0.00001 + | > step_time: 1.16770 (2.93394) + | > loader_time: 0.00180 (0.04063) + + + --> STEP: 73/234 -- GLOBAL_STEP: 7795 + | > loss: 0.32070 (0.37388) + | > log_mle: 0.03935 (0.07343) + | > loss_dur: 0.28135 (0.30045) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.72112 (2.02718) + | > current_lr: 0.00001 + | > step_time: 1.30470 (2.88934) + | > loader_time: 0.00310 (0.03917) + + + --> STEP: 78/234 -- GLOBAL_STEP: 7800 + | > loss: 0.32701 (0.37061) + | > log_mle: 0.06563 (0.07217) + | > loss_dur: 0.26138 (0.29844) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.71640 (2.09251) + | > current_lr: 0.00001 + | > step_time: 1.96370 (2.86761) + | > loader_time: 0.00190 (0.03682) + + + --> STEP: 83/234 -- GLOBAL_STEP: 7805 + | > loss: 0.32595 (0.36745) + | > log_mle: 0.03809 (0.07079) + | > loss_dur: 0.28785 (0.29667) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.98374 (2.12403) + | > current_lr: 0.00001 + | > step_time: 2.20190 (2.84064) + | > loader_time: 0.19780 (0.03815) + + + --> STEP: 88/234 -- GLOBAL_STEP: 7810 + | > loss: 0.27650 (0.36417) + | > log_mle: -0.00081 (0.06866) + | > loss_dur: 0.27730 (0.29551) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.81248 (2.21906) + | > current_lr: 0.00001 + | > step_time: 2.89160 (2.83507) + | > loader_time: 0.10040 (0.03990) + + + --> STEP: 93/234 -- GLOBAL_STEP: 7815 + | > loss: 0.27104 (0.36048) + | > log_mle: -0.01498 (0.06558) + | > loss_dur: 0.28602 (0.29490) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.61766 (2.41564) + | > current_lr: 0.00001 + | > step_time: 1.80560 (2.78939) + | > loader_time: 0.07560 (0.03982) + + + --> STEP: 98/234 -- GLOBAL_STEP: 7820 + | > loss: 0.35327 (0.35722) + | > log_mle: 0.05492 (0.06247) + | > loss_dur: 0.29834 (0.29475) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.22253 (2.65607) + | > current_lr: 0.00001 + | > step_time: 2.91020 (2.76271) + | > loader_time: 0.00310 (0.03873) + + + --> STEP: 103/234 -- GLOBAL_STEP: 7825 + | > loss: 0.28734 (0.35379) + | > log_mle: -0.03131 (0.05892) + | > loss_dur: 0.31865 (0.29487) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.59884 (3.00726) + | > current_lr: 0.00001 + | > step_time: 3.21450 (2.72269) + | > loader_time: 0.08530 (0.03930) + + + --> STEP: 108/234 -- GLOBAL_STEP: 7830 + | > loss: 0.27743 (0.35063) + | > log_mle: 0.01779 (0.05593) + | > loss_dur: 0.25964 (0.29470) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.95675 (3.25042) + | > current_lr: 0.00001 + | > step_time: 1.89230 (2.70891) + | > loader_time: 0.00260 (0.03944) + + + --> STEP: 113/234 -- GLOBAL_STEP: 7835 + | > loss: 0.27403 (0.34755) + | > log_mle: -0.03193 (0.05234) + | > loss_dur: 0.30595 (0.29521) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.81937 (3.54819) + | > current_lr: 0.00001 + | > step_time: 1.70570 (2.75655) + | > loader_time: 0.09090 (0.04180) + + + --> STEP: 118/234 -- GLOBAL_STEP: 7840 + | > loss: 0.29532 (0.34527) + | > log_mle: 0.00145 (0.04971) + | > loss_dur: 0.29388 (0.29555) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.51847 (3.67342) + | > current_lr: 0.00001 + | > step_time: 2.50710 (2.73980) + | > loader_time: 0.00370 (0.04247) + + + --> STEP: 123/234 -- GLOBAL_STEP: 7845 + | > loss: 0.29499 (0.34306) + | > log_mle: 0.02936 (0.04789) + | > loss_dur: 0.26563 (0.29517) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.50242 (3.80044) + | > current_lr: 0.00001 + | > step_time: 1.50410 (2.68628) + | > loader_time: 0.00400 (0.04149) + + + --> STEP: 128/234 -- GLOBAL_STEP: 7850 + | > loss: 0.26864 (0.33976) + | > log_mle: -0.02877 (0.04432) + | > loss_dur: 0.29740 (0.29544) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.70428 (4.09607) + | > current_lr: 0.00001 + | > step_time: 4.80880 (2.74099) + | > loader_time: 0.19040 (0.04360) + + + --> STEP: 133/234 -- GLOBAL_STEP: 7855 + | > loss: 0.27005 (0.33654) + | > log_mle: -0.04421 (0.04083) + | > loss_dur: 0.31426 (0.29571) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.27918 (4.37724) + | > current_lr: 0.00001 + | > step_time: 6.89550 (2.86656) + | > loader_time: 0.19760 (0.04566) + + + --> STEP: 138/234 -- GLOBAL_STEP: 7860 + | > loss: 0.27612 (0.33397) + | > log_mle: -0.00834 (0.03761) + | > loss_dur: 0.28446 (0.29636) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.82985 (4.60636) + | > current_lr: 0.00001 + | > step_time: 10.39350 (2.92579) + | > loader_time: 0.10550 (0.04681) + + + --> STEP: 143/234 -- GLOBAL_STEP: 7865 + | > loss: 0.22394 (0.33086) + | > log_mle: -0.12376 (0.03361) + | > loss_dur: 0.34770 (0.29725) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.32012 (4.96847) + | > current_lr: 0.00001 + | > step_time: 4.79950 (2.94605) + | > loader_time: 0.10760 (0.04792) + + + --> STEP: 148/234 -- GLOBAL_STEP: 7870 + | > loss: 0.23374 (0.32746) + | > log_mle: -0.05712 (0.02959) + | > loss_dur: 0.29086 (0.29787) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.38251 (5.18192) + | > current_lr: 0.00001 + | > step_time: 2.69790 (2.98648) + | > loader_time: 0.07120 (0.04999) + + + --> STEP: 153/234 -- GLOBAL_STEP: 7875 + | > loss: 0.18085 (0.32361) + | > log_mle: -0.15725 (0.02480) + | > loss_dur: 0.33810 (0.29880) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.87245 (5.44520) + | > current_lr: 0.00001 + | > step_time: 4.59860 (3.06730) + | > loader_time: 0.19770 (0.05084) + + + --> STEP: 158/234 -- GLOBAL_STEP: 7880 + | > loss: 0.22388 (0.32011) + | > log_mle: -0.10537 (0.02056) + | > loss_dur: 0.32925 (0.29955) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.37062 (5.72267) + | > current_lr: 0.00001 + | > step_time: 2.50070 (3.08233) + | > loader_time: 0.00350 (0.05103) + + + --> STEP: 163/234 -- GLOBAL_STEP: 7885 + | > loss: 0.23582 (0.31665) + | > log_mle: -0.08276 (0.01636) + | > loss_dur: 0.31857 (0.30029) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.58202 (6.12407) + | > current_lr: 0.00001 + | > step_time: 3.30490 (3.08790) + | > loader_time: 0.09190 (0.05122) + + + --> STEP: 168/234 -- GLOBAL_STEP: 7890 + | > loss: 0.23666 (0.31334) + | > log_mle: -0.12351 (0.01212) + | > loss_dur: 0.36018 (0.30122) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.07598 (6.43199) + | > current_lr: 0.00001 + | > step_time: 6.19230 (3.14709) + | > loader_time: 0.01160 (0.05153) + + + --> STEP: 173/234 -- GLOBAL_STEP: 7895 + | > loss: 0.19553 (0.30978) + | > log_mle: -0.13287 (0.00742) + | > loss_dur: 0.32839 (0.30237) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.09641 (6.77379) + | > current_lr: 0.00001 + | > step_time: 5.50840 (3.19674) + | > loader_time: 0.18620 (0.05403) + + + --> STEP: 178/234 -- GLOBAL_STEP: 7900 + | > loss: 0.16800 (0.30618) + | > log_mle: -0.18701 (0.00271) + | > loss_dur: 0.35501 (0.30347) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.81876 (7.14694) + | > current_lr: 0.00001 + | > step_time: 5.70770 (3.20309) + | > loader_time: 0.09400 (0.05412) + + + --> STEP: 183/234 -- GLOBAL_STEP: 7905 + | > loss: 0.17815 (0.30323) + | > log_mle: -0.18346 (-0.00164) + | > loss_dur: 0.36161 (0.30487) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.56077 (7.40857) + | > current_lr: 0.00001 + | > step_time: 5.41020 (3.26424) + | > loader_time: 0.09000 (0.05524) + + + --> STEP: 188/234 -- GLOBAL_STEP: 7910 + | > loss: 0.15298 (0.30001) + | > log_mle: -0.19229 (-0.00599) + | > loss_dur: 0.34527 (0.30601) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.41965 (7.63335) + | > current_lr: 0.00001 + | > step_time: 12.90380 (3.40201) + | > loader_time: 0.50100 (0.06532) + + + --> STEP: 193/234 -- GLOBAL_STEP: 7915 + | > loss: 0.17609 (0.29689) + | > log_mle: -0.19239 (-0.01017) + | > loss_dur: 0.36848 (0.30705) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.98710 (7.94485) + | > current_lr: 0.00001 + | > step_time: 1.80160 (3.41646) + | > loader_time: 0.00380 (0.06522) + + + --> STEP: 198/234 -- GLOBAL_STEP: 7920 + | > loss: 0.17428 (0.29405) + | > log_mle: -0.18155 (-0.01414) + | > loss_dur: 0.35583 (0.30819) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.47153 (8.23413) + | > current_lr: 0.00001 + | > step_time: 5.20700 (3.51464) + | > loader_time: 0.10560 (0.06570) + + + --> STEP: 203/234 -- GLOBAL_STEP: 7925 + | > loss: 0.21153 (0.29137) + | > log_mle: -0.13244 (-0.01804) + | > loss_dur: 0.34397 (0.30941) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.97299 (8.50920) + | > current_lr: 0.00001 + | > step_time: 2.70300 (3.54936) + | > loader_time: 0.00360 (0.06597) + + + --> STEP: 208/234 -- GLOBAL_STEP: 7930 + | > loss: 0.16831 (0.28822) + | > log_mle: -0.19588 (-0.02247) + | > loss_dur: 0.36419 (0.31069) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 20.65250 (8.80953) + | > current_lr: 0.00001 + | > step_time: 3.19270 (3.57939) + | > loader_time: 0.10180 (0.06722) + + + --> STEP: 213/234 -- GLOBAL_STEP: 7935 + | > loss: 0.13853 (0.28491) + | > log_mle: -0.23855 (-0.02728) + | > loss_dur: 0.37707 (0.31219) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 24.22144 (9.19662) + | > current_lr: 0.00001 + | > step_time: 6.89250 (3.60953) + | > loader_time: 0.11050 (0.06755) + + + --> STEP: 218/234 -- GLOBAL_STEP: 7940 + | > loss: 0.16679 (0.28171) + | > log_mle: -0.20291 (-0.03163) + | > loss_dur: 0.36970 (0.31334) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 35.70182 (9.58936) + | > current_lr: 0.00001 + | > step_time: 6.10580 (3.65887) + | > loader_time: 0.09580 (0.06828) + + + --> STEP: 223/234 -- GLOBAL_STEP: 7945 + | > loss: 0.12803 (0.27852) + | > log_mle: -0.24080 (-0.03621) + | > loss_dur: 0.36882 (0.31473) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 25.03711 (9.98347) + | > current_lr: 0.00001 + | > step_time: 2.60110 (3.65796) + | > loader_time: 0.00460 (0.06727) + + + --> STEP: 228/234 -- GLOBAL_STEP: 7950 + | > loss: 0.15428 (0.27520) + | > log_mle: -0.24492 (-0.04099) + | > loss_dur: 0.39920 (0.31619) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.90832 (10.47001) + | > current_lr: 0.00001 + | > step_time: 0.25300 (3.58932) + | > loader_time: 0.00380 (0.06621) + + + --> STEP: 233/234 -- GLOBAL_STEP: 7955 + | > loss: 0.76441 (0.27586) + | > log_mle: -0.20011 (-0.04661) + | > loss_dur: 0.96452 (0.32247) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.48435 (10.91967) + | > current_lr: 0.00001 + | > step_time: 0.21160 (3.51833) + | > loader_time: 0.00340 (0.06488) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.05056 (+1.02393) + | > avg_loss: 0.20900 (-0.03704) + | > avg_log_mle: -0.11662 (-0.02373) + | > avg_loss_dur: 0.32562 (-0.01331) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_7956.pth + + > EPOCH: 34/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 16:39:17)  + + --> STEP: 4/234 -- GLOBAL_STEP: 7960 + | > loss: 0.50794 (0.49616) + | > log_mle: 0.07097 (0.09638) + | > loss_dur: 0.43697 (0.39978) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.37449 (3.25076) + | > current_lr: 0.00001 + | > step_time: 3.71310 (3.82933) + | > loader_time: 0.08180 (0.06710) + + + --> STEP: 9/234 -- GLOBAL_STEP: 7965 + | > loss: 0.39463 (0.43813) + | > log_mle: 0.06009 (0.08261) + | > loss_dur: 0.33454 (0.35552) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.99441 (2.45963) + | > current_lr: 0.00001 + | > step_time: 1.49730 (5.07059) + | > loader_time: 0.00110 (0.05167) + + + --> STEP: 14/234 -- GLOBAL_STEP: 7970 + | > loss: 0.37646 (0.41142) + | > log_mle: 0.06487 (0.07939) + | > loss_dur: 0.31159 (0.33203) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.45324 (2.15241) + | > current_lr: 0.00001 + | > step_time: 5.61230 (4.90537) + | > loader_time: 0.08470 (0.06587) + + + --> STEP: 19/234 -- GLOBAL_STEP: 7975 + | > loss: 0.38603 (0.40038) + | > log_mle: 0.07660 (0.07773) + | > loss_dur: 0.30943 (0.32265) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.17929 (1.94418) + | > current_lr: 0.00001 + | > step_time: 1.09920 (4.44060) + | > loader_time: 0.00100 (0.05387) + + + --> STEP: 24/234 -- GLOBAL_STEP: 7980 + | > loss: 0.32711 (0.38753) + | > log_mle: 0.07211 (0.07610) + | > loss_dur: 0.25500 (0.31143) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.23587 (1.82116) + | > current_lr: 0.00001 + | > step_time: 0.80650 (3.80214) + | > loader_time: 0.00110 (0.04580) + + + --> STEP: 29/234 -- GLOBAL_STEP: 7985 + | > loss: 0.34243 (0.37882) + | > log_mle: 0.07630 (0.07485) + | > loss_dur: 0.26613 (0.30397) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.14283 (1.70409) + | > current_lr: 0.00001 + | > step_time: 2.89120 (3.51021) + | > loader_time: 0.10540 (0.04518) + + + --> STEP: 34/234 -- GLOBAL_STEP: 7990 + | > loss: 0.37127 (0.37454) + | > log_mle: 0.05760 (0.07214) + | > loss_dur: 0.31367 (0.30240) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.55188 (1.64612) + | > current_lr: 0.00001 + | > step_time: 1.78900 (3.50017) + | > loader_time: 0.00380 (0.05002) + + + --> STEP: 39/234 -- GLOBAL_STEP: 7995 + | > loss: 0.35316 (0.36990) + | > log_mle: 0.05470 (0.06981) + | > loss_dur: 0.29846 (0.30010) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.26999 (1.83244) + | > current_lr: 0.00001 + | > step_time: 2.70900 (3.39042) + | > loader_time: 0.08490 (0.05013) + + + --> STEP: 44/234 -- GLOBAL_STEP: 8000 + | > loss: 0.31049 (0.36584) + | > log_mle: 0.05669 (0.06902) + | > loss_dur: 0.25380 (0.29682) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.40078 (1.77161) + | > current_lr: 0.00001 + | > step_time: 1.19930 (3.35926) + | > loader_time: 0.00210 (0.04925) + + + --> STEP: 49/234 -- GLOBAL_STEP: 8005 + | > loss: 0.29380 (0.36069) + | > log_mle: 0.04939 (0.06750) + | > loss_dur: 0.24441 (0.29319) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.98398 (1.73631) + | > current_lr: 0.00001 + | > step_time: 1.20200 (3.17735) + | > loader_time: 0.00200 (0.04458) + + + --> STEP: 54/234 -- GLOBAL_STEP: 8010 + | > loss: 0.31189 (0.35721) + | > log_mle: 0.04905 (0.06674) + | > loss_dur: 0.26284 (0.29047) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.26186 (1.72527) + | > current_lr: 0.00001 + | > step_time: 2.01120 (3.15382) + | > loader_time: 0.09050 (0.04243) + + + --> STEP: 59/234 -- GLOBAL_STEP: 8015 + | > loss: 0.26901 (0.35412) + | > log_mle: 0.02976 (0.06543) + | > loss_dur: 0.23925 (0.28869) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.52278 (1.70865) + | > current_lr: 0.00001 + | > step_time: 1.89730 (3.00483) + | > loader_time: 0.00340 (0.04172) + + + --> STEP: 64/234 -- GLOBAL_STEP: 8020 + | > loss: 0.32206 (0.35092) + | > log_mle: 0.06098 (0.06345) + | > loss_dur: 0.26108 (0.28747) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 0.84979 (1.74621) + | > current_lr: 0.00001 + | > step_time: 1.29940 (2.90708) + | > loader_time: 0.10610 (0.04163) + + + --> STEP: 69/234 -- GLOBAL_STEP: 8025 + | > loss: 0.35123 (0.34810) + | > log_mle: 0.07332 (0.06280) + | > loss_dur: 0.27791 (0.28530) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.37885 (1.81951) + | > current_lr: 0.00001 + | > step_time: 2.51250 (2.83396) + | > loader_time: 0.09220 (0.04148) + + + --> STEP: 74/234 -- GLOBAL_STEP: 8030 + | > loss: 0.29170 (0.34517) + | > log_mle: 0.04663 (0.06100) + | > loss_dur: 0.24507 (0.28416) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.62468 (2.11403) + | > current_lr: 0.00001 + | > step_time: 2.89310 (2.82913) + | > loader_time: 0.00460 (0.04244) + + + --> STEP: 79/234 -- GLOBAL_STEP: 8035 + | > loss: 0.30103 (0.34275) + | > log_mle: 0.03725 (0.05970) + | > loss_dur: 0.26377 (0.28306) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.61131 (2.16050) + | > current_lr: 0.00001 + | > step_time: 1.81910 (2.79931) + | > loader_time: 0.00190 (0.03993) + + + --> STEP: 84/234 -- GLOBAL_STEP: 8040 + | > loss: 0.28145 (0.33970) + | > log_mle: 0.03147 (0.05833) + | > loss_dur: 0.24998 (0.28137) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.58675 (2.20996) + | > current_lr: 0.00001 + | > step_time: 1.50660 (2.73286) + | > loader_time: 0.08520 (0.03969) + + + --> STEP: 89/234 -- GLOBAL_STEP: 8045 + | > loss: 0.28213 (0.33673) + | > log_mle: 0.00634 (0.05600) + | > loss_dur: 0.27579 (0.28073) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.41787 (2.31103) + | > current_lr: 0.00001 + | > step_time: 1.20210 (2.76264) + | > loader_time: 0.08050 (0.04071) + + + --> STEP: 94/234 -- GLOBAL_STEP: 8050 + | > loss: 0.25381 (0.33316) + | > log_mle: -0.03229 (0.05260) + | > loss_dur: 0.28610 (0.28055) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 5.51778 (2.50191) + | > current_lr: 0.00001 + | > step_time: 1.11320 (2.80609) + | > loader_time: 0.07040 (0.04136) + + + --> STEP: 99/234 -- GLOBAL_STEP: 8055 + | > loss: 0.23362 (0.33002) + | > log_mle: -0.05757 (0.04931) + | > loss_dur: 0.29119 (0.28071) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.73791 (2.77791) + | > current_lr: 0.00001 + | > step_time: 1.79690 (2.80568) + | > loader_time: 0.00250 (0.03943) + + + --> STEP: 104/234 -- GLOBAL_STEP: 8060 + | > loss: 0.22296 (0.32695) + | > log_mle: -0.06301 (0.04580) + | > loss_dur: 0.28596 (0.28115) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.29613 (3.05720) + | > current_lr: 0.00001 + | > step_time: 1.71280 (2.77114) + | > loader_time: 0.00390 (0.04002) + + + --> STEP: 109/234 -- GLOBAL_STEP: 8065 + | > loss: 0.27203 (0.32471) + | > log_mle: -0.03753 (0.04318) + | > loss_dur: 0.30956 (0.28153) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.01770 (3.43262) + | > current_lr: 0.00001 + | > step_time: 4.79400 (2.79090) + | > loader_time: 0.10440 (0.04090) + + + --> STEP: 114/234 -- GLOBAL_STEP: 8070 + | > loss: 0.25657 (0.32193) + | > log_mle: -0.01809 (0.03989) + | > loss_dur: 0.27466 (0.28204) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.04958 (3.80918) + | > current_lr: 0.00001 + | > step_time: 1.28690 (2.86285) + | > loader_time: 0.00250 (0.04410) + + + --> STEP: 119/234 -- GLOBAL_STEP: 8075 + | > loss: 0.28217 (0.32024) + | > log_mle: -0.01894 (0.03733) + | > loss_dur: 0.30111 (0.28290) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.16890 (4.00421) + | > current_lr: 0.00001 + | > step_time: 0.90630 (2.80248) + | > loader_time: 0.08380 (0.04305) + + + --> STEP: 124/234 -- GLOBAL_STEP: 8080 + | > loss: 0.23238 (0.31796) + | > log_mle: -0.04507 (0.03535) + | > loss_dur: 0.27745 (0.28261) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.53470 (4.13330) + | > current_lr: 0.00001 + | > step_time: 2.49760 (2.80953) + | > loader_time: 0.00550 (0.04226) + + + --> STEP: 129/234 -- GLOBAL_STEP: 8085 + | > loss: 0.25997 (0.31513) + | > log_mle: -0.03041 (0.03197) + | > loss_dur: 0.29038 (0.28316) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.72678 (4.37546) + | > current_lr: 0.00001 + | > step_time: 0.89290 (2.78975) + | > loader_time: 0.00400 (0.04140) + + + --> STEP: 134/234 -- GLOBAL_STEP: 8090 + | > loss: 0.25112 (0.31205) + | > log_mle: -0.07553 (0.02821) + | > loss_dur: 0.32665 (0.28385) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.21052 (4.60209) + | > current_lr: 0.00001 + | > step_time: 2.79360 (2.78278) + | > loader_time: 0.00390 (0.04060) + + + --> STEP: 139/234 -- GLOBAL_STEP: 8095 + | > loss: 0.17849 (0.30924) + | > log_mle: -0.12977 (0.02467) + | > loss_dur: 0.30826 (0.28458) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.95541 (4.84974) + | > current_lr: 0.00001 + | > step_time: 5.39320 (2.79204) + | > loader_time: 0.00370 (0.04133) + + + --> STEP: 144/234 -- GLOBAL_STEP: 8100 + | > loss: 0.21794 (0.30648) + | > log_mle: -0.10787 (0.02091) + | > loss_dur: 0.32581 (0.28557) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.91218 (5.12394) + | > current_lr: 0.00001 + | > step_time: 2.70080 (2.76730) + | > loader_time: 0.00160 (0.04060) + + + --> STEP: 149/234 -- GLOBAL_STEP: 8105 + | > loss: 0.18336 (0.30307) + | > log_mle: -0.14585 (0.01672) + | > loss_dur: 0.32921 (0.28634) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.21921 (5.39535) + | > current_lr: 0.00001 + | > step_time: 3.10690 (2.79482) + | > loader_time: 0.08370 (0.04222) + + + --> STEP: 154/234 -- GLOBAL_STEP: 8110 + | > loss: 0.19973 (0.29944) + | > log_mle: -0.11312 (0.01223) + | > loss_dur: 0.31285 (0.28721) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 12.18455 (5.63600) + | > current_lr: 0.00001 + | > step_time: 3.70470 (2.82231) + | > loader_time: 0.00300 (0.04270) + + + --> STEP: 159/234 -- GLOBAL_STEP: 8115 + | > loss: 0.19525 (0.29612) + | > log_mle: -0.13051 (0.00792) + | > loss_dur: 0.32576 (0.28820) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.54345 (5.90500) + | > current_lr: 0.00001 + | > step_time: 4.69960 (2.90459) + | > loader_time: 0.00690 (0.04456) + + + --> STEP: 164/234 -- GLOBAL_STEP: 8120 + | > loss: 0.17834 (0.29274) + | > log_mle: -0.12422 (0.00381) + | > loss_dur: 0.30256 (0.28893) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.16099 (6.23805) + | > current_lr: 0.00001 + | > step_time: 3.09750 (2.92705) + | > loader_time: 0.00340 (0.04384) + + + --> STEP: 169/234 -- GLOBAL_STEP: 8125 + | > loss: 0.21589 (0.28977) + | > log_mle: -0.11864 (-0.00035) + | > loss_dur: 0.33453 (0.29012) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.60927 (6.57868) + | > current_lr: 0.00001 + | > step_time: 5.80320 (2.99244) + | > loader_time: 0.09540 (0.04386) + + + --> STEP: 174/234 -- GLOBAL_STEP: 8130 + | > loss: 0.12397 (0.28583) + | > log_mle: -0.20048 (-0.00544) + | > loss_dur: 0.32445 (0.29127) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 28.21421 (7.09361) + | > current_lr: 0.00001 + | > step_time: 7.69360 (3.07123) + | > loader_time: 0.09120 (0.04545) + + + --> STEP: 179/234 -- GLOBAL_STEP: 8135 + | > loss: 0.15213 (0.28257) + | > log_mle: -0.18677 (-0.01001) + | > loss_dur: 0.33890 (0.29257) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.85066 (7.36933) + | > current_lr: 0.00001 + | > step_time: 7.50740 (3.12913) + | > loader_time: 0.09160 (0.04537) + + + --> STEP: 184/234 -- GLOBAL_STEP: 8140 + | > loss: 0.17116 (0.27980) + | > log_mle: -0.16660 (-0.01419) + | > loss_dur: 0.33776 (0.29399) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.79826 (7.58960) + | > current_lr: 0.00001 + | > step_time: 5.90570 (3.20450) + | > loader_time: 0.09650 (0.04850) + + + --> STEP: 189/234 -- GLOBAL_STEP: 8145 + | > loss: 0.17865 (0.27688) + | > log_mle: -0.16253 (-0.01846) + | > loss_dur: 0.34119 (0.29534) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.26678 (7.87670) + | > current_lr: 0.00001 + | > step_time: 7.00120 (3.26680) + | > loader_time: 0.10200 (0.05566) + + + --> STEP: 194/234 -- GLOBAL_STEP: 8150 + | > loss: 0.14214 (0.27374) + | > log_mle: -0.19285 (-0.02275) + | > loss_dur: 0.33500 (0.29649) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.38089 (8.14953) + | > current_lr: 0.00001 + | > step_time: 5.23970 (3.29784) + | > loader_time: 0.00430 (0.05671) + + + --> STEP: 199/234 -- GLOBAL_STEP: 8155 + | > loss: 0.14668 (0.27095) + | > log_mle: -0.20341 (-0.02674) + | > loss_dur: 0.35009 (0.29769) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.21156 (8.36174) + | > current_lr: 0.00001 + | > step_time: 1.98640 (3.28047) + | > loader_time: 0.00280 (0.05623) + + + --> STEP: 204/234 -- GLOBAL_STEP: 8160 + | > loss: 0.15011 (0.26838) + | > log_mle: -0.22486 (-0.03070) + | > loss_dur: 0.37497 (0.29908) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 23.47692 (8.67862) + | > current_lr: 0.00001 + | > step_time: 4.00000 (3.30443) + | > loader_time: 0.09380 (0.05592) + + + --> STEP: 209/234 -- GLOBAL_STEP: 8165 + | > loss: 0.16482 (0.26541) + | > log_mle: -0.18770 (-0.03490) + | > loss_dur: 0.35252 (0.30030) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.54908 (9.03226) + | > current_lr: 0.00001 + | > step_time: 5.01190 (3.32788) + | > loader_time: 0.08730 (0.05725) + + + --> STEP: 214/234 -- GLOBAL_STEP: 8170 + | > loss: 0.11701 (0.26189) + | > log_mle: -0.21645 (-0.03980) + | > loss_dur: 0.33346 (0.30169) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 34.14098 (9.51607) + | > current_lr: 0.00001 + | > step_time: 3.89490 (3.41697) + | > loader_time: 0.00430 (0.05698) + + + --> STEP: 219/234 -- GLOBAL_STEP: 8175 + | > loss: 0.07049 (0.25855) + | > log_mle: -0.29888 (-0.04448) + | > loss_dur: 0.36937 (0.30303) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 26.02464 (9.90655) + | > current_lr: 0.00001 + | > step_time: 4.50190 (3.46542) + | > loader_time: 0.29950 (0.05714) + + + --> STEP: 224/234 -- GLOBAL_STEP: 8180 + | > loss: 0.10109 (0.25562) + | > log_mle: -0.26015 (-0.04885) + | > loss_dur: 0.36124 (0.30448) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 22.65747 (10.30757) + | > current_lr: 0.00001 + | > step_time: 0.65610 (3.44869) + | > loader_time: 0.09000 (0.05635) + + + --> STEP: 229/234 -- GLOBAL_STEP: 8185 + | > loss: 0.16107 (0.25266) + | > log_mle: -0.27870 (-0.05369) + | > loss_dur: 0.43978 (0.30635) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 24.77052 (10.68755) + | > current_lr: 0.00001 + | > step_time: 0.25460 (3.37878) + | > loader_time: 0.00420 (0.05524) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.35155 (-0.69901) + | > avg_loss: 0.19866 (-0.01034) + | > avg_log_mle: -0.12092 (-0.00430) + | > avg_loss_dur: 0.31958 (-0.00604) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_8190.pth + + > EPOCH: 35/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 16:53:50)  + + --> STEP: 0/234 -- GLOBAL_STEP: 8190 + | > loss: 0.49365 (0.49365) + | > log_mle: 0.06361 (0.06361) + | > loss_dur: 0.43005 (0.43005) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.12999 (2.12999) + | > current_lr: 0.00001 + | > step_time: 6.18550 (6.18548) + | > loader_time: 14.34300 (14.34301) + + + --> STEP: 5/234 -- GLOBAL_STEP: 8195 + | > loss: 0.37317 (0.43597) + | > log_mle: 0.06911 (0.08121) + | > loss_dur: 0.30405 (0.35476) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.95823 (2.66386) + | > current_lr: 0.00001 + | > step_time: 9.29400 (8.57742) + | > loader_time: 0.10770 (0.09505) + + + --> STEP: 10/234 -- GLOBAL_STEP: 8200 + | > loss: 0.34629 (0.40155) + | > log_mle: 0.05296 (0.06906) + | > loss_dur: 0.29333 (0.33249) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.00593 (2.35514) + | > current_lr: 0.00001 + | > step_time: 1.40150 (5.54892) + | > loader_time: 0.00120 (0.05756) + + + --> STEP: 15/234 -- GLOBAL_STEP: 8205 + | > loss: 0.36380 (0.38306) + | > log_mle: 0.05837 (0.06713) + | > loss_dur: 0.30543 (0.31593) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.42565 (2.06278) + | > current_lr: 0.00001 + | > step_time: 4.10050 (4.31534) + | > loader_time: 0.19700 (0.05771) + + + --> STEP: 20/234 -- GLOBAL_STEP: 8210 + | > loss: 0.33741 (0.37335) + | > log_mle: 0.06454 (0.06636) + | > loss_dur: 0.27287 (0.30699) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.29341 (1.91416) + | > current_lr: 0.00001 + | > step_time: 2.58410 (3.63892) + | > loader_time: 0.00120 (0.04842) + + + --> STEP: 25/234 -- GLOBAL_STEP: 8215 + | > loss: 0.33117 (0.36188) + | > log_mle: 0.06847 (0.06516) + | > loss_dur: 0.26270 (0.29672) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.40709 (1.82478) + | > current_lr: 0.00001 + | > step_time: 2.11490 (3.36402) + | > loader_time: 0.09620 (0.04679) + + + --> STEP: 30/234 -- GLOBAL_STEP: 8220 + | > loss: 0.31916 (0.35317) + | > log_mle: 0.03912 (0.06300) + | > loss_dur: 0.28004 (0.29017) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.30139 (1.71990) + | > current_lr: 0.00001 + | > step_time: 6.42000 (3.58492) + | > loader_time: 0.09080 (0.05158) + + + --> STEP: 35/234 -- GLOBAL_STEP: 8225 + | > loss: 0.30438 (0.34917) + | > log_mle: 0.04031 (0.06054) + | > loss_dur: 0.26407 (0.28863) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.56901 (1.93437) + | > current_lr: 0.00001 + | > step_time: 5.90420 (3.69347) + | > loader_time: 0.09030 (0.05239) + + + --> STEP: 40/234 -- GLOBAL_STEP: 8230 + | > loss: 0.36713 (0.34620) + | > log_mle: 0.06173 (0.05892) + | > loss_dur: 0.30540 (0.28728) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.75451 (1.90715) + | > current_lr: 0.00001 + | > step_time: 0.89360 (3.55380) + | > loader_time: 0.00200 (0.04875) + + + --> STEP: 45/234 -- GLOBAL_STEP: 8235 + | > loss: 0.29574 (0.34142) + | > log_mle: 0.03521 (0.05762) + | > loss_dur: 0.26053 (0.28380) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.31806 (1.86066) + | > current_lr: 0.00001 + | > step_time: 2.00210 (3.33034) + | > loader_time: 0.00250 (0.04706) + + + --> STEP: 50/234 -- GLOBAL_STEP: 8240 + | > loss: 0.30058 (0.33699) + | > log_mle: 0.05319 (0.05656) + | > loss_dur: 0.24739 (0.28043) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.13886 (1.79971) + | > current_lr: 0.00001 + | > step_time: 1.64330 (3.19544) + | > loader_time: 0.08310 (0.04608) + + + --> STEP: 55/234 -- GLOBAL_STEP: 8245 + | > loss: 0.28616 (0.33328) + | > log_mle: 0.03291 (0.05554) + | > loss_dur: 0.25325 (0.27774) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.51226 (1.78462) + | > current_lr: 0.00001 + | > step_time: 1.80470 (3.10574) + | > loader_time: 0.00220 (0.04402) + + + --> STEP: 60/234 -- GLOBAL_STEP: 8250 + | > loss: 0.26695 (0.33047) + | > log_mle: 0.01789 (0.05401) + | > loss_dur: 0.24906 (0.27646) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 2.05930 (1.77523) + | > current_lr: 0.00001 + | > step_time: 4.71310 (3.07046) + | > loader_time: 0.19350 (0.04546) + + + --> STEP: 65/234 -- GLOBAL_STEP: 8255 + | > loss: 0.28606 (0.32793) + | > log_mle: 0.04256 (0.05249) + | > loss_dur: 0.24350 (0.27544) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.43663 (1.82106) + | > current_lr: 0.00001 + | > step_time: 1.54830 (3.00336) + | > loader_time: 0.00190 (0.04607) + + + --> STEP: 70/234 -- GLOBAL_STEP: 8260 + | > loss: 0.29639 (0.32579) + | > log_mle: 0.03064 (0.05169) + | > loss_dur: 0.26575 (0.27410) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.54025 (1.95266) + | > current_lr: 0.00001 + | > step_time: 1.60910 (2.93640) + | > loader_time: 0.08260 (0.04517) + + + --> STEP: 75/234 -- GLOBAL_STEP: 8265 + | > loss: 0.30013 (0.32329) + | > log_mle: 0.02800 (0.04988) + | > loss_dur: 0.27213 (0.27341) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 3.84745 (2.13394) + | > current_lr: 0.00001 + | > step_time: 1.18960 (2.85100) + | > loader_time: 0.00250 (0.04346) + + + --> STEP: 80/234 -- GLOBAL_STEP: 8270 + | > loss: 0.27425 (0.32029) + | > log_mle: 0.04423 (0.04880) + | > loss_dur: 0.23001 (0.27150) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.28441 (2.16510) + | > current_lr: 0.00001 + | > step_time: 2.90020 (2.84134) + | > loader_time: 0.10360 (0.04352) + + + --> STEP: 85/234 -- GLOBAL_STEP: 8275 + | > loss: 0.27112 (0.31755) + | > log_mle: 0.02806 (0.04728) + | > loss_dur: 0.24306 (0.27027) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.78418 (2.25379) + | > current_lr: 0.00001 + | > step_time: 1.09960 (2.86708) + | > loader_time: 0.00490 (0.04445) + + + --> STEP: 90/234 -- GLOBAL_STEP: 8280 + | > loss: 0.25314 (0.31447) + | > log_mle: -0.00292 (0.04466) + | > loss_dur: 0.25606 (0.26980) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 10.75037 (2.53778) + | > current_lr: 0.00001 + | > step_time: 2.81320 (2.79550) + | > loader_time: 0.00210 (0.04212) + + + --> STEP: 95/234 -- GLOBAL_STEP: 8285 + | > loss: 0.21009 (0.31049) + | > log_mle: -0.08084 (0.04059) + | > loss_dur: 0.29093 (0.26990) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 13.87619 (2.93166) + | > current_lr: 0.00001 + | > step_time: 1.28940 (2.72193) + | > loader_time: 0.09620 (0.04271) + + + --> STEP: 100/234 -- GLOBAL_STEP: 8290 + | > loss: 0.25600 (0.30807) + | > log_mle: -0.01553 (0.03809) + | > loss_dur: 0.27153 (0.26998) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.20798 (3.09482) + | > current_lr: 0.00001 + | > step_time: 2.50630 (2.71286) + | > loader_time: 0.09340 (0.04328) + + + --> STEP: 105/234 -- GLOBAL_STEP: 8295 + | > loss: 0.27708 (0.30513) + | > log_mle: 0.01687 (0.03493) + | > loss_dur: 0.26022 (0.27020) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 1.99737 (3.27334) + | > current_lr: 0.00001 + | > step_time: 3.21440 (2.77462) + | > loader_time: 0.08590 (0.04558) + + + --> STEP: 110/234 -- GLOBAL_STEP: 8300 + | > loss: 0.26161 (0.30279) + | > log_mle: -0.01369 (0.03202) + | > loss_dur: 0.27530 (0.27077) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 7.74649 (3.56489) + | > current_lr: 0.00001 + | > step_time: 2.29140 (2.83855) + | > loader_time: 0.10010 (0.04889) + + + --> STEP: 115/234 -- GLOBAL_STEP: 8305 + | > loss: 0.27426 (0.30030) + | > log_mle: -0.02931 (0.02862) + | > loss_dur: 0.30356 (0.27168) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 6.16044 (3.79386) + | > current_lr: 0.00001 + | > step_time: 0.73860 (2.81858) + | > loader_time: 0.00270 (0.04993) + + + --> STEP: 120/234 -- GLOBAL_STEP: 8310 + | > loss: 0.20015 (0.29811) + | > log_mle: -0.07717 (0.02570) + | > loss_dur: 0.27732 (0.27241) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.10260 (3.93949) + | > current_lr: 0.00001 + | > step_time: 1.17600 (2.80521) + | > loader_time: 0.00160 (0.04864) + + + --> STEP: 125/234 -- GLOBAL_STEP: 8315 + | > loss: 0.24186 (0.29654) + | > log_mle: -0.06330 (0.02385) + | > loss_dur: 0.30516 (0.27269) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.67723 (4.04910) + | > current_lr: 0.00001 + | > step_time: 6.68010 (2.82159) + | > loader_time: 0.20080 (0.05068) + + + --> STEP: 130/234 -- GLOBAL_STEP: 8320 + | > loss: 0.20909 (0.29351) + | > log_mle: -0.07453 (0.02040) + | > loss_dur: 0.28362 (0.27311) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 8.56729 (4.20292) + | > current_lr: 0.00001 + | > step_time: 1.51210 (2.78237) + | > loader_time: 0.08900 (0.05012) + + + --> STEP: 135/234 -- GLOBAL_STEP: 8325 + | > loss: 0.25265 (0.29084) + | > log_mle: -0.01342 (0.01711) + | > loss_dur: 0.26606 (0.27372) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 4.90319 (4.38506) + | > current_lr: 0.00001 + | > step_time: 1.82130 (2.74447) + | > loader_time: 0.00370 (0.04901) + + + --> STEP: 140/234 -- GLOBAL_STEP: 8330 + | > loss: 0.25021 (0.28807) + | > log_mle: -0.04225 (0.01339) + | > loss_dur: 0.29246 (0.27468) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 9.93552 (4.69620) + | > current_lr: 0.00001 + | > step_time: 1.89490 (2.74294) + | > loader_time: 0.00200 (0.04739) + + + --> STEP: 145/234 -- GLOBAL_STEP: 8335 + | > loss: 0.18417 (0.28512) + | > log_mle: -0.12592 (0.00912) + | > loss_dur: 0.31009 (0.27600) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.47701 (5.15786) + | > current_lr: 0.00001 + | > step_time: 2.52200 (2.74029) + | > loader_time: 0.00280 (0.04590) + + + --> STEP: 150/234 -- GLOBAL_STEP: 8340 + | > loss: 0.19452 (0.28196) + | > log_mle: -0.11073 (0.00509) + | > loss_dur: 0.30525 (0.27687) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 11.30664 (5.44602) + | > current_lr: 0.00001 + | > step_time: 6.20020 (2.76877) + | > loader_time: 0.17240 (0.04681) + + + --> STEP: 155/234 -- GLOBAL_STEP: 8345 + | > loss: 0.15240 (0.27823) + | > log_mle: -0.17258 (0.00025) + | > loss_dur: 0.32499 (0.27797) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 17.40562 (5.82010) + | > current_lr: 0.00001 + | > step_time: 2.80390 (2.75763) + | > loader_time: 0.00370 (0.04655) + + + --> STEP: 160/234 -- GLOBAL_STEP: 8350 + | > loss: 0.15894 (0.27514) + | > log_mle: -0.16616 (-0.00396) + | > loss_dur: 0.32510 (0.27910) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.28652 (6.06658) + | > current_lr: 0.00001 + | > step_time: 2.01160 (2.73055) + | > loader_time: 0.00320 (0.04667) + + + --> STEP: 165/234 -- GLOBAL_STEP: 8355 + | > loss: 0.17241 (0.27202) + | > log_mle: -0.16198 (-0.00803) + | > loss_dur: 0.33439 (0.28005) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.88482 (6.31244) + | > current_lr: 0.00001 + | > step_time: 4.59690 (2.71829) + | > loader_time: 0.10300 (0.04597) + + + --> STEP: 170/234 -- GLOBAL_STEP: 8360 + | > loss: 0.14612 (0.26902) + | > log_mle: -0.19523 (-0.01236) + | > loss_dur: 0.34135 (0.28138) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 15.68201 (6.51483) + | > current_lr: 0.00001 + | > step_time: 2.30910 (2.72085) + | > loader_time: 0.10090 (0.04692) + + + --> STEP: 175/234 -- GLOBAL_STEP: 8365 + | > loss: 0.17089 (0.26529) + | > log_mle: -0.16868 (-0.01728) + | > loss_dur: 0.33957 (0.28257) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 21.14795 (6.87372) + | > current_lr: 0.00001 + | > step_time: 4.00010 (2.73002) + | > loader_time: 0.00460 (0.04666) + + + --> STEP: 180/234 -- GLOBAL_STEP: 8370 + | > loss: 0.15352 (0.26211) + | > log_mle: -0.18084 (-0.02185) + | > loss_dur: 0.33436 (0.28396) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.98400 (7.22866) + | > current_lr: 0.00001 + | > step_time: 1.70670 (2.80259) + | > loader_time: 0.08880 (0.04595) + + + --> STEP: 185/234 -- GLOBAL_STEP: 8375 + | > loss: 0.14284 (0.25941) + | > log_mle: -0.20211 (-0.02610) + | > loss_dur: 0.34496 (0.28552) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.30166 (7.43879) + | > current_lr: 0.00001 + | > step_time: 4.22090 (2.87245) + | > loader_time: 0.09480 (0.04729) + + + --> STEP: 190/234 -- GLOBAL_STEP: 8380 + | > loss: 0.14666 (0.25646) + | > log_mle: -0.18706 (-0.03027) + | > loss_dur: 0.33371 (0.28673) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 14.93938 (7.64829) + | > current_lr: 0.00001 + | > step_time: 9.40610 (2.93329) + | > loader_time: 0.69840 (0.05330) + + + --> STEP: 195/234 -- GLOBAL_STEP: 8385 + | > loss: 0.16160 (0.25342) + | > log_mle: -0.19285 (-0.03454) + | > loss_dur: 0.35445 (0.28796) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 16.42424 (7.97650) + | > current_lr: 0.00001 + | > step_time: 9.38990 (3.01453) + | > loader_time: 0.09960 (0.05455) + + + --> STEP: 200/234 -- GLOBAL_STEP: 8390 + | > loss: 0.14709 (0.25061) + | > log_mle: -0.19761 (-0.03850) + | > loss_dur: 0.34471 (0.28912) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 24.94622 (8.33725) + | > current_lr: 0.00001 + | > step_time: 6.10560 (3.10568) + | > loader_time: 0.00390 (0.05531) + + + --> STEP: 205/234 -- GLOBAL_STEP: 8395 + | > loss: 0.15262 (0.24808) + | > log_mle: -0.19102 (-0.04239) + | > loss_dur: 0.34364 (0.29046) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 18.42600 (8.69360) + | > current_lr: 0.00001 + | > step_time: 7.48820 (3.18507) + | > loader_time: 0.00720 (0.05829) + + + --> STEP: 210/234 -- GLOBAL_STEP: 8400 + | > loss: 0.09778 (0.24498) + | > log_mle: -0.26379 (-0.04689) + | > loss_dur: 0.36156 (0.29187) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 24.26179 (9.02634) + | > current_lr: 0.00001 + | > step_time: 6.69990 (3.26924) + | > loader_time: 0.30560 (0.06320) + + + --> STEP: 215/234 -- GLOBAL_STEP: 8405 + | > loss: 0.11902 (0.24177) + | > log_mle: -0.21328 (-0.05152) + | > loss_dur: 0.33231 (0.29328) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 19.30552 (9.45424) + | > current_lr: 0.00001 + | > step_time: 5.50020 (3.36303) + | > loader_time: 0.19080 (0.06622) + + + --> STEP: 220/234 -- GLOBAL_STEP: 8410 + | > loss: 0.09514 (0.23834) + | > log_mle: -0.25802 (-0.05639) + | > loss_dur: 0.35317 (0.29473) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 20.37299 (9.82311) + | > current_lr: 0.00001 + | > step_time: 1.79060 (3.37609) + | > loader_time: 0.00520 (0.06614) + + + --> STEP: 225/234 -- GLOBAL_STEP: 8415 + | > loss: 0.06092 (0.23529) + | > log_mle: -0.30943 (-0.06097) + | > loss_dur: 0.37034 (0.29627) + | > amp_scaler: 16384.00000 (16384.00000) + | > grad_norm: 39.83560 (10.23483) + | > current_lr: 0.00001 + | > step_time: 0.24760 (3.32661) + | > loader_time: 0.00420 (0.06477) + + + --> STEP: 230/234 -- GLOBAL_STEP: 8420 + | > loss: 0.09312 (0.23253) + | > log_mle: -0.35428 (-0.06594) + | > loss_dur: 0.44739 (0.29847) + | > amp_scaler: 8192.00000 (16348.38261) + | > grad_norm: 0.00000 (10.54215) + | > current_lr: 0.00001 + | > step_time: 0.21260 (3.25947) + | > loader_time: 0.00390 (0.06345) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55173 (+0.20018) + | > avg_loss: 0.18350 (-0.01516) + | > avg_log_mle: -0.12865 (-0.00773) + | > avg_loss_dur: 0.31215 (-0.00743) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_8424.pth + + > EPOCH: 36/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 17:07:46)  + + --> STEP: 1/234 -- GLOBAL_STEP: 8425 + | > loss: 0.42682 (0.42682) + | > log_mle: 0.08080 (0.08080) + | > loss_dur: 0.34602 (0.34602) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.59803 (1.59803) + | > current_lr: 0.00001 + | > step_time: 8.20820 (8.20821) + | > loader_time: 0.18940 (0.18939) + + + --> STEP: 6/234 -- GLOBAL_STEP: 8430 + | > loss: 0.36466 (0.40550) + | > log_mle: 0.06700 (0.06915) + | > loss_dur: 0.29766 (0.33634) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.47265 (2.01593) + | > current_lr: 0.00001 + | > step_time: 9.99330 (7.36512) + | > loader_time: 0.10630 (0.42922) + + + --> STEP: 11/234 -- GLOBAL_STEP: 8435 + | > loss: 0.31559 (0.36863) + | > log_mle: 0.05950 (0.05772) + | > loss_dur: 0.25610 (0.31091) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.53951 (1.92353) + | > current_lr: 0.00001 + | > step_time: 2.12670 (6.29119) + | > loader_time: 0.00280 (0.27871) + + + --> STEP: 16/234 -- GLOBAL_STEP: 8440 + | > loss: 0.33084 (0.35795) + | > log_mle: 0.04308 (0.05500) + | > loss_dur: 0.28776 (0.30295) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.52828 (1.79556) + | > current_lr: 0.00001 + | > step_time: 0.95890 (4.66432) + | > loader_time: 0.00200 (0.19741) + + + --> STEP: 21/234 -- GLOBAL_STEP: 8445 + | > loss: 0.32179 (0.34893) + | > log_mle: 0.06011 (0.05521) + | > loss_dur: 0.26168 (0.29372) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36720 (1.75034) + | > current_lr: 0.00001 + | > step_time: 1.20320 (3.86219) + | > loader_time: 0.00180 (0.15084) + + + --> STEP: 26/234 -- GLOBAL_STEP: 8450 + | > loss: 0.29022 (0.33760) + | > log_mle: 0.04462 (0.05346) + | > loss_dur: 0.24560 (0.28413) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.16569 (1.67120) + | > current_lr: 0.00001 + | > step_time: 1.20220 (3.41082) + | > loader_time: 0.08580 (0.12547) + + + --> STEP: 31/234 -- GLOBAL_STEP: 8455 + | > loss: 0.32486 (0.33103) + | > log_mle: 0.03629 (0.05123) + | > loss_dur: 0.28858 (0.27980) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.80611 (1.64427) + | > current_lr: 0.00001 + | > step_time: 1.30320 (3.15162) + | > loader_time: 0.00250 (0.10826) + + + --> STEP: 36/234 -- GLOBAL_STEP: 8460 + | > loss: 0.30068 (0.32583) + | > log_mle: 0.03181 (0.04885) + | > loss_dur: 0.26888 (0.27698) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.57576 (1.87689) + | > current_lr: 0.00001 + | > step_time: 1.36630 (2.91953) + | > loader_time: 0.00230 (0.09596) + + + --> STEP: 41/234 -- GLOBAL_STEP: 8465 + | > loss: 0.27307 (0.32250) + | > log_mle: 0.03400 (0.04748) + | > loss_dur: 0.23907 (0.27501) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.32123 (1.88671) + | > current_lr: 0.00001 + | > step_time: 1.59600 (2.83920) + | > loader_time: 0.00190 (0.08452) + + + --> STEP: 46/234 -- GLOBAL_STEP: 8470 + | > loss: 0.28660 (0.31846) + | > log_mle: 0.03410 (0.04633) + | > loss_dur: 0.25250 (0.27214) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.51775 (1.87762) + | > current_lr: 0.00001 + | > step_time: 2.80230 (2.76252) + | > loader_time: 0.00800 (0.07739) + + + --> STEP: 51/234 -- GLOBAL_STEP: 8475 + | > loss: 0.28601 (0.31501) + | > log_mle: 0.05128 (0.04565) + | > loss_dur: 0.23473 (0.26936) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10614 (1.83688) + | > current_lr: 0.00001 + | > step_time: 1.77910 (2.66398) + | > loader_time: 0.00230 (0.07312) + + + --> STEP: 56/234 -- GLOBAL_STEP: 8480 + | > loss: 0.29373 (0.31249) + | > log_mle: 0.02965 (0.04426) + | > loss_dur: 0.26408 (0.26823) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.25979 (1.81244) + | > current_lr: 0.00001 + | > step_time: 2.10400 (2.57737) + | > loader_time: 0.09300 (0.07006) + + + --> STEP: 61/234 -- GLOBAL_STEP: 8485 + | > loss: 0.28201 (0.30947) + | > log_mle: 0.03649 (0.04293) + | > loss_dur: 0.24552 (0.26654) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.12915 (1.81147) + | > current_lr: 0.00001 + | > step_time: 2.10150 (2.52048) + | > loader_time: 0.00310 (0.06598) + + + --> STEP: 66/234 -- GLOBAL_STEP: 8490 + | > loss: 0.27283 (0.30729) + | > log_mle: 0.04196 (0.04156) + | > loss_dur: 0.23088 (0.26573) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.50786 (1.86665) + | > current_lr: 0.00001 + | > step_time: 1.38560 (2.43412) + | > loader_time: 0.00220 (0.06250) + + + --> STEP: 71/234 -- GLOBAL_STEP: 8495 + | > loss: 0.26786 (0.30508) + | > log_mle: -0.00333 (0.04017) + | > loss_dur: 0.27120 (0.26491) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.03812 (2.27068) + | > current_lr: 0.00001 + | > step_time: 1.82790 (2.37393) + | > loader_time: 0.00250 (0.05919) + + + --> STEP: 76/234 -- GLOBAL_STEP: 8500 + | > loss: 0.27233 (0.30256) + | > log_mle: 0.01854 (0.03872) + | > loss_dur: 0.25379 (0.26384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.92567 (2.30697) + | > current_lr: 0.00001 + | > step_time: 1.71510 (2.34258) + | > loader_time: 0.09580 (0.06000) + + + --> STEP: 81/234 -- GLOBAL_STEP: 8505 + | > loss: 0.24309 (0.29977) + | > log_mle: 0.00138 (0.03748) + | > loss_dur: 0.24172 (0.26229) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.21307 (2.35868) + | > current_lr: 0.00001 + | > step_time: 2.19840 (2.30675) + | > loader_time: 0.08480 (0.05953) + + + --> STEP: 86/234 -- GLOBAL_STEP: 8510 + | > loss: 0.25442 (0.29771) + | > log_mle: 0.00022 (0.03598) + | > loss_dur: 0.25420 (0.26173) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.40081 (2.45954) + | > current_lr: 0.00001 + | > step_time: 1.92140 (2.28423) + | > loader_time: 0.09050 (0.05934) + + + --> STEP: 91/234 -- GLOBAL_STEP: 8515 + | > loss: 0.27187 (0.29491) + | > log_mle: -0.00254 (0.03336) + | > loss_dur: 0.27441 (0.26156) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.60661 (2.63338) + | > current_lr: 0.00001 + | > step_time: 2.99750 (2.28472) + | > loader_time: 0.10090 (0.05927) + + + --> STEP: 96/234 -- GLOBAL_STEP: 8520 + | > loss: 0.25488 (0.29107) + | > log_mle: 0.00474 (0.02939) + | > loss_dur: 0.25014 (0.26168) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.69202 (2.98357) + | > current_lr: 0.00001 + | > step_time: 1.29790 (2.25627) + | > loader_time: 0.10380 (0.05941) + + + --> STEP: 101/234 -- GLOBAL_STEP: 8525 + | > loss: 0.22065 (0.28865) + | > log_mle: -0.04801 (0.02648) + | > loss_dur: 0.26866 (0.26216) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.76703 (3.39839) + | > current_lr: 0.00001 + | > step_time: 1.52870 (2.24817) + | > loader_time: 0.00240 (0.05736) + + + --> STEP: 106/234 -- GLOBAL_STEP: 8530 + | > loss: 0.25331 (0.28641) + | > log_mle: -0.04645 (0.02349) + | > loss_dur: 0.29976 (0.26292) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.84360 (3.77270) + | > current_lr: 0.00001 + | > step_time: 1.38470 (2.23826) + | > loader_time: 0.01030 (0.05746) + + + --> STEP: 111/234 -- GLOBAL_STEP: 8535 + | > loss: 0.22450 (0.28382) + | > log_mle: -0.08482 (0.02034) + | > loss_dur: 0.30933 (0.26348) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.07491 (4.08133) + | > current_lr: 0.00001 + | > step_time: 1.20280 (2.22252) + | > loader_time: 0.00280 (0.05757) + + + --> STEP: 116/234 -- GLOBAL_STEP: 8540 + | > loss: 0.24158 (0.28166) + | > log_mle: -0.05609 (0.01730) + | > loss_dur: 0.29767 (0.26436) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.09112 (4.29781) + | > current_lr: 0.00001 + | > step_time: 2.59490 (2.20285) + | > loader_time: 0.00770 (0.05664) + + + --> STEP: 121/234 -- GLOBAL_STEP: 8545 + | > loss: 0.29461 (0.27980) + | > log_mle: 0.02247 (0.01511) + | > loss_dur: 0.27214 (0.26469) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.92077 (4.42538) + | > current_lr: 0.00001 + | > step_time: 1.90720 (2.17354) + | > loader_time: 0.09570 (0.05518) + + + --> STEP: 126/234 -- GLOBAL_STEP: 8550 + | > loss: 0.18285 (0.27732) + | > log_mle: -0.10331 (0.01231) + | > loss_dur: 0.28615 (0.26501) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.61397 (4.62662) + | > current_lr: 0.00001 + | > step_time: 1.00750 (2.14851) + | > loader_time: 0.00250 (0.05520) + + + --> STEP: 131/234 -- GLOBAL_STEP: 8555 + | > loss: 0.15578 (0.27429) + | > log_mle: -0.13250 (0.00874) + | > loss_dur: 0.28828 (0.26555) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.60172 (4.86765) + | > current_lr: 0.00001 + | > step_time: 0.97430 (2.12490) + | > loader_time: 0.00260 (0.05406) + + + --> STEP: 136/234 -- GLOBAL_STEP: 8560 + | > loss: 0.16774 (0.27177) + | > log_mle: -0.17459 (0.00524) + | > loss_dur: 0.34234 (0.26653) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.35790 (5.10276) + | > current_lr: 0.00001 + | > step_time: 1.08980 (2.12093) + | > loader_time: 0.00320 (0.05288) + + + --> STEP: 141/234 -- GLOBAL_STEP: 8565 + | > loss: 0.19843 (0.26945) + | > log_mle: -0.09870 (0.00215) + | > loss_dur: 0.29713 (0.26730) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.27852 (5.34418) + | > current_lr: 0.00001 + | > step_time: 2.27590 (2.12341) + | > loader_time: 0.00250 (0.05237) + + + --> STEP: 146/234 -- GLOBAL_STEP: 8570 + | > loss: 0.16619 (0.26630) + | > log_mle: -0.14309 (-0.00242) + | > loss_dur: 0.30928 (0.26872) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.14499 (5.68275) + | > current_lr: 0.00001 + | > step_time: 2.01590 (2.12474) + | > loader_time: 0.09370 (0.05181) + + + --> STEP: 151/234 -- GLOBAL_STEP: 8575 + | > loss: 0.16886 (0.26319) + | > log_mle: -0.11430 (-0.00621) + | > loss_dur: 0.28316 (0.26940) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.49204 (5.88710) + | > current_lr: 0.00001 + | > step_time: 2.59810 (2.12203) + | > loader_time: 0.09780 (0.05317) + + + --> STEP: 156/234 -- GLOBAL_STEP: 8580 + | > loss: 0.14620 (0.25934) + | > log_mle: -0.14760 (-0.01120) + | > loss_dur: 0.29380 (0.27053) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.34002 (6.20341) + | > current_lr: 0.00001 + | > step_time: 3.10180 (2.11684) + | > loader_time: 0.00460 (0.05262) + + + --> STEP: 161/234 -- GLOBAL_STEP: 8585 + | > loss: 0.12922 (0.25625) + | > log_mle: -0.16518 (-0.01545) + | > loss_dur: 0.29440 (0.27170) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.81366 (6.50663) + | > current_lr: 0.00001 + | > step_time: 3.12010 (2.11120) + | > loader_time: 0.07690 (0.05210) + + + --> STEP: 166/234 -- GLOBAL_STEP: 8590 + | > loss: 0.15971 (0.25342) + | > log_mle: -0.11610 (-0.01916) + | > loss_dur: 0.27581 (0.27258) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.03687 (6.76325) + | > current_lr: 0.00001 + | > step_time: 2.02310 (2.10569) + | > loader_time: 0.00270 (0.05227) + + + --> STEP: 171/234 -- GLOBAL_STEP: 8595 + | > loss: 0.08927 (0.25000) + | > log_mle: -0.20924 (-0.02397) + | > loss_dur: 0.29851 (0.27397) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.29312 (7.16658) + | > current_lr: 0.00001 + | > step_time: 1.50130 (2.10075) + | > loader_time: 0.07920 (0.05223) + + + --> STEP: 176/234 -- GLOBAL_STEP: 8600 + | > loss: 0.13058 (0.24648) + | > log_mle: -0.17915 (-0.02867) + | > loss_dur: 0.30973 (0.27515) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.86079 (7.56559) + | > current_lr: 0.00001 + | > step_time: 7.49610 (2.11890) + | > loader_time: 0.09730 (0.05231) + + + --> STEP: 181/234 -- GLOBAL_STEP: 8605 + | > loss: 0.19296 (0.24378) + | > log_mle: -0.12832 (-0.03290) + | > loss_dur: 0.32127 (0.27669) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.24606 (7.88470) + | > current_lr: 0.00001 + | > step_time: 5.30600 (2.17743) + | > loader_time: 0.07950 (0.05252) + + + --> STEP: 186/234 -- GLOBAL_STEP: 8610 + | > loss: 0.17567 (0.24110) + | > log_mle: -0.16069 (-0.03728) + | > loss_dur: 0.33637 (0.27838) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.18149 (8.17963) + | > current_lr: 0.00001 + | > step_time: 1.57090 (2.19689) + | > loader_time: 0.02160 (0.05271) + + + --> STEP: 191/234 -- GLOBAL_STEP: 8615 + | > loss: 0.14129 (0.23813) + | > log_mle: -0.16801 (-0.04145) + | > loss_dur: 0.30930 (0.27958) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.20726 (8.44997) + | > current_lr: 0.00001 + | > step_time: 4.89330 (2.22964) + | > loader_time: 0.10850 (0.05378) + + + --> STEP: 196/234 -- GLOBAL_STEP: 8620 + | > loss: 0.16027 (0.23528) + | > log_mle: -0.16897 (-0.04569) + | > loss_dur: 0.32924 (0.28097) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.24451 (8.75858) + | > current_lr: 0.00001 + | > step_time: 1.60310 (2.22559) + | > loader_time: 0.06970 (0.05399) + + + --> STEP: 201/234 -- GLOBAL_STEP: 8625 + | > loss: 0.18346 (0.23279) + | > log_mle: -0.14601 (-0.04950) + | > loss_dur: 0.32947 (0.28229) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.08753 (9.15139) + | > current_lr: 0.00001 + | > step_time: 1.19820 (2.20514) + | > loader_time: 0.09130 (0.05449) + + + --> STEP: 206/234 -- GLOBAL_STEP: 8630 + | > loss: 0.09787 (0.22983) + | > log_mle: -0.23078 (-0.05375) + | > loss_dur: 0.32865 (0.28359) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.60281 (9.53867) + | > current_lr: 0.00001 + | > step_time: 3.52030 (2.21838) + | > loader_time: 0.20320 (0.05546) + + + --> STEP: 211/234 -- GLOBAL_STEP: 8635 + | > loss: 0.07362 (0.22666) + | > log_mle: -0.29376 (-0.05851) + | > loss_dur: 0.36738 (0.28517) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.48927 (9.96035) + | > current_lr: 0.00001 + | > step_time: 3.81210 (2.25214) + | > loader_time: 0.40500 (0.05697) + + + --> STEP: 216/234 -- GLOBAL_STEP: 8640 + | > loss: 0.07617 (0.22352) + | > log_mle: -0.28191 (-0.06303) + | > loss_dur: 0.35808 (0.28655) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.84349 (10.31042) + | > current_lr: 0.00001 + | > step_time: 3.51230 (2.28021) + | > loader_time: 0.08830 (0.05702) + + + --> STEP: 221/234 -- GLOBAL_STEP: 8645 + | > loss: 0.12685 (0.22051) + | > log_mle: -0.21202 (-0.06753) + | > loss_dur: 0.33887 (0.28804) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.10113 (10.56988) + | > current_lr: 0.00001 + | > step_time: 3.19000 (2.36564) + | > loader_time: 0.00460 (0.05721) + + + --> STEP: 226/234 -- GLOBAL_STEP: 8650 + | > loss: 0.05516 (0.21719) + | > log_mle: -0.29877 (-0.07248) + | > loss_dur: 0.35393 (0.28967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.93688 (10.98585) + | > current_lr: 0.00001 + | > step_time: 0.23590 (2.32575) + | > loader_time: 0.00460 (0.05676) + + + --> STEP: 231/234 -- GLOBAL_STEP: 8655 + | > loss: 0.13934 (0.21486) + | > log_mle: -0.35485 (-0.07765) + | > loss_dur: 0.49419 (0.29251) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.52386 (11.38173) + | > current_lr: 0.00001 + | > step_time: 0.29040 (2.28094) + | > loader_time: 0.00490 (0.05562) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.27475 (+0.72301) + | > avg_loss: 0.15559 (-0.02791) + | > avg_log_mle: -0.15098 (-0.02233) + | > avg_loss_dur: 0.30657 (-0.00558) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_8658.pth + + > EPOCH: 37/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 17:17:38)  + + --> STEP: 2/234 -- GLOBAL_STEP: 8660 + | > loss: 0.45948 (0.43218) + | > log_mle: 0.08577 (0.07727) + | > loss_dur: 0.37372 (0.35491) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.49584 (1.54411) + | > current_lr: 0.00001 + | > step_time: 4.08780 (3.99430) + | > loader_time: 1.51260 (0.80477) + + + --> STEP: 7/234 -- GLOBAL_STEP: 8665 + | > loss: 0.32522 (0.37497) + | > log_mle: 0.03091 (0.05435) + | > loss_dur: 0.29431 (0.32061) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.80863 (2.42792) + | > current_lr: 0.00001 + | > step_time: 5.88800 (6.19859) + | > loader_time: 0.00090 (0.26938) + + + --> STEP: 12/234 -- GLOBAL_STEP: 8670 + | > loss: 0.28884 (0.34649) + | > log_mle: 0.04185 (0.04661) + | > loss_dur: 0.24699 (0.29988) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.45783 (2.27240) + | > current_lr: 0.00001 + | > step_time: 3.79510 (5.47236) + | > loader_time: 0.00160 (0.15810) + + + --> STEP: 17/234 -- GLOBAL_STEP: 8675 + | > loss: 0.31604 (0.33832) + | > log_mle: 0.05277 (0.04500) + | > loss_dur: 0.26326 (0.29332) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.69404 (2.09150) + | > current_lr: 0.00001 + | > step_time: 4.00210 (4.85660) + | > loader_time: 0.19650 (0.12830) + + + --> STEP: 22/234 -- GLOBAL_STEP: 8680 + | > loss: 0.26298 (0.32860) + | > log_mle: 0.02675 (0.04392) + | > loss_dur: 0.23623 (0.28468) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.21608 (1.93326) + | > current_lr: 0.00001 + | > step_time: 0.72710 (4.18835) + | > loader_time: 0.00200 (0.11205) + + + --> STEP: 27/234 -- GLOBAL_STEP: 8685 + | > loss: 0.25360 (0.31711) + | > log_mle: 0.02489 (0.04244) + | > loss_dur: 0.22871 (0.27467) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.42428 (1.82375) + | > current_lr: 0.00001 + | > step_time: 1.16320 (3.68580) + | > loader_time: 0.00180 (0.09841) + + + --> STEP: 32/234 -- GLOBAL_STEP: 8690 + | > loss: 0.25090 (0.31036) + | > log_mle: 0.00895 (0.03998) + | > loss_dur: 0.24195 (0.27039) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.41038 (1.74618) + | > current_lr: 0.00001 + | > step_time: 3.11640 (3.56349) + | > loader_time: 0.08200 (0.08855) + + + --> STEP: 37/234 -- GLOBAL_STEP: 8695 + | > loss: 0.25218 (0.30660) + | > log_mle: 0.02928 (0.03845) + | > loss_dur: 0.22290 (0.26814) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.57733 (2.03263) + | > current_lr: 0.00001 + | > step_time: 1.40390 (3.44220) + | > loader_time: 0.00270 (0.08421) + + + --> STEP: 42/234 -- GLOBAL_STEP: 8700 + | > loss: 0.32610 (0.30492) + | > log_mle: 0.04898 (0.03763) + | > loss_dur: 0.27713 (0.26729) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.40453 (2.03789) + | > current_lr: 0.00001 + | > step_time: 2.29130 (3.34570) + | > loader_time: 0.00230 (0.07691) + + + --> STEP: 47/234 -- GLOBAL_STEP: 8705 + | > loss: 0.28586 (0.30118) + | > log_mle: 0.02658 (0.03604) + | > loss_dur: 0.25928 (0.26515) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.40811 (2.04414) + | > current_lr: 0.00001 + | > step_time: 1.69040 (3.20230) + | > loader_time: 0.00300 (0.07461) + + + --> STEP: 52/234 -- GLOBAL_STEP: 8710 + | > loss: 0.29681 (0.29792) + | > log_mle: 0.04032 (0.03574) + | > loss_dur: 0.25650 (0.26218) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.05375 (2.03190) + | > current_lr: 0.00001 + | > step_time: 1.70800 (3.04679) + | > loader_time: 0.00610 (0.07092) + + + --> STEP: 57/234 -- GLOBAL_STEP: 8715 + | > loss: 0.28550 (0.29565) + | > log_mle: 0.03768 (0.03434) + | > loss_dur: 0.24782 (0.26131) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.51864 (1.98399) + | > current_lr: 0.00001 + | > step_time: 2.09000 (3.03743) + | > loader_time: 0.00160 (0.06504) + + + --> STEP: 62/234 -- GLOBAL_STEP: 8720 + | > loss: 0.25679 (0.29179) + | > log_mle: -0.02188 (0.03205) + | > loss_dur: 0.27867 (0.25974) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.17878 (2.09225) + | > current_lr: 0.00001 + | > step_time: 2.92710 (2.97732) + | > loader_time: 0.09370 (0.06422) + + + --> STEP: 67/234 -- GLOBAL_STEP: 8725 + | > loss: 0.23492 (0.28944) + | > log_mle: -0.00353 (0.03103) + | > loss_dur: 0.23846 (0.25842) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.31095 (2.11957) + | > current_lr: 0.00001 + | > step_time: 1.75570 (2.90159) + | > loader_time: 0.00290 (0.06092) + + + --> STEP: 72/234 -- GLOBAL_STEP: 8730 + | > loss: 0.26032 (0.28795) + | > log_mle: 0.01433 (0.02996) + | > loss_dur: 0.24599 (0.25798) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.57998 (2.33826) + | > current_lr: 0.00001 + | > step_time: 3.10890 (2.85651) + | > loader_time: 0.08740 (0.05916) + + + --> STEP: 77/234 -- GLOBAL_STEP: 8735 + | > loss: 0.23732 (0.28514) + | > log_mle: -0.00024 (0.02835) + | > loss_dur: 0.23756 (0.25679) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.37422 (2.40320) + | > current_lr: 0.00001 + | > step_time: 2.20940 (2.85664) + | > loader_time: 0.00240 (0.05678) + + + --> STEP: 82/234 -- GLOBAL_STEP: 8740 + | > loss: 0.24432 (0.28275) + | > log_mle: 0.01535 (0.02736) + | > loss_dur: 0.22896 (0.25539) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.40839 (2.46076) + | > current_lr: 0.00001 + | > step_time: 2.30990 (2.80783) + | > loader_time: 0.09310 (0.05555) + + + --> STEP: 87/234 -- GLOBAL_STEP: 8745 + | > loss: 0.26082 (0.28047) + | > log_mle: -0.00114 (0.02573) + | > loss_dur: 0.26196 (0.25474) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.11891 (2.60932) + | > current_lr: 0.00001 + | > step_time: 4.10560 (2.77488) + | > loader_time: 0.19200 (0.05569) + + + --> STEP: 92/234 -- GLOBAL_STEP: 8750 + | > loss: 0.20156 (0.27740) + | > log_mle: -0.03828 (0.02278) + | > loss_dur: 0.23985 (0.25462) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.05844 (2.86061) + | > current_lr: 0.00001 + | > step_time: 1.80020 (2.79121) + | > loader_time: 0.00200 (0.05282) + + + --> STEP: 97/234 -- GLOBAL_STEP: 8755 + | > loss: 0.21288 (0.27369) + | > log_mle: -0.03296 (0.01895) + | > loss_dur: 0.24584 (0.25474) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.69323 (3.24381) + | > current_lr: 0.00001 + | > step_time: 4.19900 (2.79168) + | > loader_time: 0.00300 (0.05024) + + + --> STEP: 102/234 -- GLOBAL_STEP: 8760 + | > loss: 0.24568 (0.27119) + | > log_mle: -0.01089 (0.01626) + | > loss_dur: 0.25656 (0.25493) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.28858 (3.47344) + | > current_lr: 0.00001 + | > step_time: 2.21400 (2.75602) + | > loader_time: 0.00320 (0.05035) + + + --> STEP: 107/234 -- GLOBAL_STEP: 8765 + | > loss: 0.20781 (0.26836) + | > log_mle: -0.06037 (0.01277) + | > loss_dur: 0.26818 (0.25559) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.30074 (3.77053) + | > current_lr: 0.00001 + | > step_time: 1.20680 (2.71677) + | > loader_time: 0.00230 (0.04985) + + + --> STEP: 112/234 -- GLOBAL_STEP: 8770 + | > loss: 0.21676 (0.26596) + | > log_mle: -0.06420 (0.00962) + | > loss_dur: 0.28095 (0.25634) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.87804 (4.08463) + | > current_lr: 0.00001 + | > step_time: 2.89880 (2.68832) + | > loader_time: 0.08640 (0.04852) + + + --> STEP: 117/234 -- GLOBAL_STEP: 8775 + | > loss: 0.20591 (0.26383) + | > log_mle: -0.05594 (0.00669) + | > loss_dur: 0.26185 (0.25715) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.80326 (4.29849) + | > current_lr: 0.00001 + | > step_time: 3.31130 (2.68382) + | > loader_time: 0.08670 (0.04859) + + + --> STEP: 122/234 -- GLOBAL_STEP: 8780 + | > loss: 0.22490 (0.26227) + | > log_mle: -0.04267 (0.00461) + | > loss_dur: 0.26757 (0.25766) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.52573 (4.37598) + | > current_lr: 0.00001 + | > step_time: 2.90320 (2.66810) + | > loader_time: 0.08370 (0.04822) + + + --> STEP: 127/234 -- GLOBAL_STEP: 8785 + | > loss: 0.18428 (0.25957) + | > log_mle: -0.08708 (0.00147) + | > loss_dur: 0.27136 (0.25810) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.67235 (4.60860) + | > current_lr: 0.00001 + | > step_time: 2.00340 (2.63568) + | > loader_time: 0.00370 (0.04710) + + + --> STEP: 132/234 -- GLOBAL_STEP: 8790 + | > loss: 0.19256 (0.25698) + | > log_mle: -0.06874 (-0.00190) + | > loss_dur: 0.26129 (0.25888) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.59727 (4.93604) + | > current_lr: 0.00001 + | > step_time: 2.69650 (2.63413) + | > loader_time: 0.00420 (0.04610) + + + --> STEP: 137/234 -- GLOBAL_STEP: 8795 + | > loss: 0.21483 (0.25467) + | > log_mle: -0.07947 (-0.00544) + | > loss_dur: 0.29430 (0.26011) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.31614 (5.14587) + | > current_lr: 0.00001 + | > step_time: 2.40100 (2.62281) + | > loader_time: 0.08540 (0.04626) + + + --> STEP: 142/234 -- GLOBAL_STEP: 8800 + | > loss: 0.18133 (0.25233) + | > log_mle: -0.09740 (-0.00861) + | > loss_dur: 0.27873 (0.26094) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.67942 (5.38609) + | > current_lr: 0.00001 + | > step_time: 2.60750 (2.64660) + | > loader_time: 0.08480 (0.04533) + + + --> STEP: 147/234 -- GLOBAL_STEP: 8805 + | > loss: 0.17055 (0.24906) + | > log_mle: -0.10005 (-0.01312) + | > loss_dur: 0.27060 (0.26218) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.40906 (5.71443) + | > current_lr: 0.00001 + | > step_time: 1.80510 (2.64992) + | > loader_time: 0.00270 (0.04646) + + + --> STEP: 152/234 -- GLOBAL_STEP: 8810 + | > loss: 0.14791 (0.24584) + | > log_mle: -0.16488 (-0.01728) + | > loss_dur: 0.31278 (0.26312) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.23350 (5.95396) + | > current_lr: 0.00001 + | > step_time: 3.29270 (2.65972) + | > loader_time: 0.00340 (0.04803) + + + --> STEP: 157/234 -- GLOBAL_STEP: 8815 + | > loss: 0.17553 (0.24227) + | > log_mle: -0.11991 (-0.02194) + | > loss_dur: 0.29544 (0.26421) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.92370 (6.28105) + | > current_lr: 0.00001 + | > step_time: 1.59320 (2.71775) + | > loader_time: 0.00370 (0.04905) + + + --> STEP: 162/234 -- GLOBAL_STEP: 8820 + | > loss: 0.13517 (0.23897) + | > log_mle: -0.15160 (-0.02638) + | > loss_dur: 0.28677 (0.26535) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.77582 (6.53421) + | > current_lr: 0.00001 + | > step_time: 4.00780 (2.70311) + | > loader_time: 0.19240 (0.04929) + + + --> STEP: 167/234 -- GLOBAL_STEP: 8825 + | > loss: 0.09796 (0.23596) + | > log_mle: -0.21562 (-0.03042) + | > loss_dur: 0.31358 (0.26638) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.22930 (6.84194) + | > current_lr: 0.00001 + | > step_time: 2.61340 (2.69729) + | > loader_time: 0.07450 (0.04834) + + + --> STEP: 172/234 -- GLOBAL_STEP: 8830 + | > loss: 0.11693 (0.23287) + | > log_mle: -0.20824 (-0.03512) + | > loss_dur: 0.32517 (0.26799) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.65240 (7.28722) + | > current_lr: 0.00001 + | > step_time: 3.58800 (2.69211) + | > loader_time: 0.00270 (0.04902) + + + --> STEP: 177/234 -- GLOBAL_STEP: 8835 + | > loss: 0.13712 (0.22965) + | > log_mle: -0.17412 (-0.03959) + | > loss_dur: 0.31124 (0.26924) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.87074 (7.62276) + | > current_lr: 0.00001 + | > step_time: 6.90290 (2.71888) + | > loader_time: 0.10100 (0.05105) + + + --> STEP: 182/234 -- GLOBAL_STEP: 8840 + | > loss: 0.12696 (0.22691) + | > log_mle: -0.21449 (-0.04402) + | > loss_dur: 0.34145 (0.27093) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.96094 (7.92042) + | > current_lr: 0.00001 + | > step_time: 4.49660 (2.73691) + | > loader_time: 0.09450 (0.05116) + + + --> STEP: 187/234 -- GLOBAL_STEP: 8845 + | > loss: 0.09660 (0.22412) + | > log_mle: -0.21622 (-0.04837) + | > loss_dur: 0.31282 (0.27249) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.15564 (8.19020) + | > current_lr: 0.00001 + | > step_time: 4.20830 (2.75737) + | > loader_time: 0.00410 (0.05090) + + + --> STEP: 192/234 -- GLOBAL_STEP: 8850 + | > loss: 0.07605 (0.22108) + | > log_mle: -0.23835 (-0.05263) + | > loss_dur: 0.31440 (0.27371) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.96811 (8.42706) + | > current_lr: 0.00001 + | > step_time: 10.79850 (2.85324) + | > loader_time: 0.19750 (0.05070) + + + --> STEP: 197/234 -- GLOBAL_STEP: 8855 + | > loss: 0.09566 (0.21843) + | > log_mle: -0.21363 (-0.05672) + | > loss_dur: 0.30930 (0.27515) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.42856 (8.76661) + | > current_lr: 0.00001 + | > step_time: 3.59130 (2.90519) + | > loader_time: 0.19630 (0.05231) + + + --> STEP: 202/234 -- GLOBAL_STEP: 8860 + | > loss: 0.03847 (0.21567) + | > log_mle: -0.29266 (-0.06092) + | > loss_dur: 0.33113 (0.27658) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.84427 (9.07092) + | > current_lr: 0.00001 + | > step_time: 2.09180 (2.93377) + | > loader_time: 0.00460 (0.05251) + + + --> STEP: 207/234 -- GLOBAL_STEP: 8865 + | > loss: 0.06425 (0.21283) + | > log_mle: -0.27984 (-0.06509) + | > loss_dur: 0.34410 (0.27792) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.31559 (9.38044) + | > current_lr: 0.00001 + | > step_time: 6.70280 (3.00298) + | > loader_time: 0.00290 (0.05316) + + + --> STEP: 212/234 -- GLOBAL_STEP: 8870 + | > loss: 0.08524 (0.20978) + | > log_mle: -0.26460 (-0.06977) + | > loss_dur: 0.34984 (0.27955) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.49524 (9.74876) + | > current_lr: 0.00001 + | > step_time: 7.79450 (3.09707) + | > loader_time: 0.09140 (0.05333) + + + --> STEP: 217/234 -- GLOBAL_STEP: 8875 + | > loss: 0.06123 (0.20643) + | > log_mle: -0.28319 (-0.07436) + | > loss_dur: 0.34443 (0.28078) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.16564 (10.06910) + | > current_lr: 0.00001 + | > step_time: 5.09610 (3.13904) + | > loader_time: 0.00440 (0.05402) + + + --> STEP: 222/234 -- GLOBAL_STEP: 8880 + | > loss: 0.08432 (0.20350) + | > log_mle: -0.28854 (-0.07885) + | > loss_dur: 0.37286 (0.28235) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.33035 (10.40984) + | > current_lr: 0.00001 + | > step_time: 2.69860 (3.14279) + | > loader_time: 0.00500 (0.05403) + + + --> STEP: 227/234 -- GLOBAL_STEP: 8885 + | > loss: 0.08902 (0.20021) + | > log_mle: -0.27072 (-0.08369) + | > loss_dur: 0.35975 (0.28391) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.03964 (10.84247) + | > current_lr: 0.00001 + | > step_time: 0.24420 (3.08629) + | > loader_time: 0.00400 (0.05294) + + + --> STEP: 232/234 -- GLOBAL_STEP: 8890 + | > loss: 0.19850 (0.19843) + | > log_mle: -0.44163 (-0.08956) + | > loss_dur: 0.64014 (0.28799) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.80679 (11.40506) + | > current_lr: 0.00001 + | > step_time: 0.33150 (3.02577) + | > loader_time: 0.00440 (0.05189) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.97606 (-0.29868) + | > avg_loss: 0.17424 (+0.01865) + | > avg_log_mle: -0.13366 (+0.01732) + | > avg_loss_dur: 0.30790 (+0.00133) + + + > EPOCH: 38/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 17:30:28)  + + --> STEP: 3/234 -- GLOBAL_STEP: 8895 + | > loss: 0.34772 (0.39339) + | > log_mle: 0.04523 (0.05937) + | > loss_dur: 0.30249 (0.33402) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.42312 (1.52288) + | > current_lr: 0.00001 + | > step_time: 2.70000 (3.09805) + | > loader_time: 0.00140 (1.00109) + + + --> STEP: 8/234 -- GLOBAL_STEP: 8900 + | > loss: 0.29189 (0.34050) + | > log_mle: 0.02044 (0.04076) + | > loss_dur: 0.27144 (0.29974) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.73556 (2.10058) + | > current_lr: 0.00001 + | > step_time: 2.69770 (5.47577) + | > loader_time: 0.08970 (0.42315) + + + --> STEP: 13/234 -- GLOBAL_STEP: 8905 + | > loss: 0.29992 (0.32057) + | > log_mle: 0.03961 (0.03639) + | > loss_dur: 0.26031 (0.28418) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.87287 (2.05470) + | > current_lr: 0.00001 + | > step_time: 6.39850 (5.40935) + | > loader_time: 0.19260 (0.30642) + + + --> STEP: 18/234 -- GLOBAL_STEP: 8910 + | > loss: 0.27845 (0.31337) + | > log_mle: 0.02800 (0.03430) + | > loss_dur: 0.25046 (0.27907) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.38640 (1.94237) + | > current_lr: 0.00001 + | > step_time: 2.10200 (4.53715) + | > loader_time: 0.00230 (0.22759) + + + --> STEP: 23/234 -- GLOBAL_STEP: 8915 + | > loss: 0.24561 (0.30582) + | > log_mle: 0.02049 (0.03309) + | > loss_dur: 0.22512 (0.27273) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.12624 (1.83329) + | > current_lr: 0.00001 + | > step_time: 2.29650 (4.17811) + | > loader_time: 0.00140 (0.18590) + + + --> STEP: 28/234 -- GLOBAL_STEP: 8920 + | > loss: 0.23166 (0.29719) + | > log_mle: 0.02575 (0.03199) + | > loss_dur: 0.20591 (0.26519) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.18681 (1.76153) + | > current_lr: 0.00001 + | > step_time: 2.99890 (4.17782) + | > loader_time: 0.00310 (0.15711) + + + --> STEP: 33/234 -- GLOBAL_STEP: 8925 + | > loss: 0.29285 (0.29224) + | > log_mle: 0.03501 (0.02997) + | > loss_dur: 0.25783 (0.26227) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.31674 (1.71450) + | > current_lr: 0.00001 + | > step_time: 8.81130 (4.14562) + | > loader_time: 0.18320 (0.14439) + + + --> STEP: 38/234 -- GLOBAL_STEP: 8930 + | > loss: 0.27009 (0.28897) + | > log_mle: 0.01207 (0.02784) + | > loss_dur: 0.25801 (0.26113) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.27469 (1.96640) + | > current_lr: 0.00001 + | > step_time: 1.67250 (3.85490) + | > loader_time: 0.00210 (0.13027) + + + --> STEP: 43/234 -- GLOBAL_STEP: 8935 + | > loss: 0.24967 (0.28665) + | > log_mle: 0.01152 (0.02712) + | > loss_dur: 0.23815 (0.25953) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.82053 (1.95151) + | > current_lr: 0.00001 + | > step_time: 2.69390 (3.65796) + | > loader_time: 0.00280 (0.11772) + + + --> STEP: 48/234 -- GLOBAL_STEP: 8940 + | > loss: 0.23203 (0.28217) + | > log_mle: 0.02141 (0.02578) + | > loss_dur: 0.21062 (0.25638) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.65890 (1.93160) + | > current_lr: 0.00001 + | > step_time: 3.20630 (3.60191) + | > loader_time: 0.08850 (0.11133) + + + --> STEP: 53/234 -- GLOBAL_STEP: 8945 + | > loss: 0.25472 (0.27969) + | > log_mle: 0.00200 (0.02515) + | > loss_dur: 0.25272 (0.25454) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.51248 (1.91549) + | > current_lr: 0.00001 + | > step_time: 2.41490 (3.42277) + | > loader_time: 0.00200 (0.10269) + + + --> STEP: 58/234 -- GLOBAL_STEP: 8950 + | > loss: 0.24987 (0.27773) + | > log_mle: 0.01919 (0.02415) + | > loss_dur: 0.23069 (0.25358) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.77236 (1.87948) + | > current_lr: 0.00001 + | > step_time: 3.20040 (3.36100) + | > loader_time: 0.00280 (0.09998) + + + --> STEP: 63/234 -- GLOBAL_STEP: 8955 + | > loss: 0.26147 (0.27450) + | > log_mle: 0.00150 (0.02167) + | > loss_dur: 0.25997 (0.25283) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.66776 (1.96105) + | > current_lr: 0.00001 + | > step_time: 3.91630 (3.31376) + | > loader_time: 0.09920 (0.09497) + + + --> STEP: 68/234 -- GLOBAL_STEP: 8960 + | > loss: 0.23556 (0.27173) + | > log_mle: 0.01312 (0.02088) + | > loss_dur: 0.22244 (0.25085) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.47609 (2.02059) + | > current_lr: 0.00001 + | > step_time: 1.99770 (3.26287) + | > loader_time: 0.00230 (0.09503) + + + --> STEP: 73/234 -- GLOBAL_STEP: 8965 + | > loss: 0.22223 (0.26997) + | > log_mle: -0.01435 (0.01948) + | > loss_dur: 0.23658 (0.25050) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.86775 (2.31997) + | > current_lr: 0.00001 + | > step_time: 1.60060 (3.19765) + | > loader_time: 0.08690 (0.09245) + + + --> STEP: 78/234 -- GLOBAL_STEP: 8970 + | > loss: 0.26349 (0.26781) + | > log_mle: 0.01315 (0.01826) + | > loss_dur: 0.25034 (0.24955) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.56212 (2.42380) + | > current_lr: 0.00001 + | > step_time: 1.88880 (3.12490) + | > loader_time: 0.00150 (0.08784) + + + --> STEP: 83/234 -- GLOBAL_STEP: 8975 + | > loss: 0.23848 (0.26523) + | > log_mle: -0.01500 (0.01694) + | > loss_dur: 0.25347 (0.24828) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.23075 (2.49842) + | > current_lr: 0.00001 + | > step_time: 1.29290 (3.08818) + | > loader_time: 0.00350 (0.08499) + + + --> STEP: 88/234 -- GLOBAL_STEP: 8980 + | > loss: 0.19391 (0.26283) + | > log_mle: -0.05326 (0.01494) + | > loss_dur: 0.24717 (0.24789) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.45007 (2.75925) + | > current_lr: 0.00001 + | > step_time: 7.09770 (3.12100) + | > loader_time: 0.08370 (0.08314) + + + --> STEP: 93/234 -- GLOBAL_STEP: 8985 + | > loss: 0.19593 (0.25983) + | > log_mle: -0.06462 (0.01199) + | > loss_dur: 0.26055 (0.24784) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.31256 (3.18131) + | > current_lr: 0.00001 + | > step_time: 4.40050 (3.16613) + | > loader_time: 0.10200 (0.08197) + + + --> STEP: 98/234 -- GLOBAL_STEP: 8990 + | > loss: 0.25957 (0.25711) + | > log_mle: 0.00350 (0.00898) + | > loss_dur: 0.25606 (0.24813) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.37901 (3.43754) + | > current_lr: 0.00001 + | > step_time: 1.80180 (3.17705) + | > loader_time: 0.00310 (0.07992) + + + --> STEP: 103/234 -- GLOBAL_STEP: 8995 + | > loss: 0.18976 (0.25451) + | > log_mle: -0.08568 (0.00546) + | > loss_dur: 0.27545 (0.24905) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.67377 (3.72231) + | > current_lr: 0.00001 + | > step_time: 2.51890 (3.13459) + | > loader_time: 0.08140 (0.07781) + + + --> STEP: 108/234 -- GLOBAL_STEP: 9000 + | > loss: 0.20512 (0.25207) + | > log_mle: -0.03469 (0.00250) + | > loss_dur: 0.23981 (0.24957) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.85411 (3.90607) + | > current_lr: 0.00001 + | > step_time: 3.19510 (3.13685) + | > loader_time: 0.09470 (0.07766) + + + --> STEP: 113/234 -- GLOBAL_STEP: 9005 + | > loss: 0.18277 (0.24959) + | > log_mle: -0.08331 (-0.00105) + | > loss_dur: 0.26608 (0.25064) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.06636 (4.10460) + | > current_lr: 0.00001 + | > step_time: 1.69180 (3.17851) + | > loader_time: 0.00300 (0.07695) + + + --> STEP: 118/234 -- GLOBAL_STEP: 9010 + | > loss: 0.20694 (0.24793) + | > log_mle: -0.05057 (-0.00366) + | > loss_dur: 0.25751 (0.25159) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.23675 (4.23843) + | > current_lr: 0.00001 + | > step_time: 1.00380 (3.15923) + | > loader_time: 0.00520 (0.07621) + + + --> STEP: 123/234 -- GLOBAL_STEP: 9015 + | > loss: 0.22029 (0.24639) + | > log_mle: -0.02308 (-0.00543) + | > loss_dur: 0.24337 (0.25183) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.93229 (4.35990) + | > current_lr: 0.00001 + | > step_time: 5.29970 (3.16444) + | > loader_time: 0.18930 (0.07834) + + + --> STEP: 128/234 -- GLOBAL_STEP: 9020 + | > loss: 0.18181 (0.24371) + | > log_mle: -0.08153 (-0.00895) + | > loss_dur: 0.26334 (0.25266) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.22085 (4.66668) + | > current_lr: 0.00001 + | > step_time: 4.91490 (3.16908) + | > loader_time: 0.08670 (0.07750) + + + --> STEP: 133/234 -- GLOBAL_STEP: 9025 + | > loss: 0.18568 (0.24113) + | > log_mle: -0.09756 (-0.01238) + | > loss_dur: 0.28324 (0.25351) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.56430 (4.98573) + | > current_lr: 0.00001 + | > step_time: 1.30300 (3.15540) + | > loader_time: 0.00510 (0.07605) + + + --> STEP: 138/234 -- GLOBAL_STEP: 9030 + | > loss: 0.19897 (0.23906) + | > log_mle: -0.05909 (-0.01556) + | > loss_dur: 0.25806 (0.25462) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.91934 (5.25381) + | > current_lr: 0.00001 + | > step_time: 2.80660 (3.17883) + | > loader_time: 0.09560 (0.07618) + + + --> STEP: 143/234 -- GLOBAL_STEP: 9035 + | > loss: 0.13913 (0.23634) + | > log_mle: -0.17716 (-0.01950) + | > loss_dur: 0.31629 (0.25584) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.92861 (5.67250) + | > current_lr: 0.00001 + | > step_time: 1.50370 (3.15450) + | > loader_time: 0.00440 (0.07550) + + + --> STEP: 148/234 -- GLOBAL_STEP: 9040 + | > loss: 0.14529 (0.23341) + | > log_mle: -0.11068 (-0.02348) + | > loss_dur: 0.25597 (0.25689) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.36279 (5.99412) + | > current_lr: 0.00001 + | > step_time: 3.50170 (3.14383) + | > loader_time: 0.00590 (0.07309) + + + --> STEP: 153/234 -- GLOBAL_STEP: 9045 + | > loss: 0.08565 (0.22993) + | > log_mle: -0.21161 (-0.02826) + | > loss_dur: 0.29726 (0.25818) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.24263 (6.36548) + | > current_lr: 0.00001 + | > step_time: 1.68980 (3.17435) + | > loader_time: 0.00590 (0.07333) + + + --> STEP: 158/234 -- GLOBAL_STEP: 9050 + | > loss: 0.13566 (0.22666) + | > log_mle: -0.16002 (-0.03253) + | > loss_dur: 0.29568 (0.25919) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.43475 (6.68257) + | > current_lr: 0.00001 + | > step_time: 1.61460 (3.17410) + | > loader_time: 0.07860 (0.07328) + + + --> STEP: 163/234 -- GLOBAL_STEP: 9055 + | > loss: 0.15583 (0.22361) + | > log_mle: -0.13795 (-0.03678) + | > loss_dur: 0.29378 (0.26040) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.50313 (7.00514) + | > current_lr: 0.00001 + | > step_time: 5.30810 (3.18433) + | > loader_time: 0.08900 (0.07340) + + + --> STEP: 168/234 -- GLOBAL_STEP: 9060 + | > loss: 0.14973 (0.22060) + | > log_mle: -0.17866 (-0.04104) + | > loss_dur: 0.32839 (0.26165) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.54949 (7.38357) + | > current_lr: 0.00001 + | > step_time: 4.00070 (3.23777) + | > loader_time: 0.00440 (0.07303) + + + --> STEP: 173/234 -- GLOBAL_STEP: 9065 + | > loss: 0.11645 (0.21740) + | > log_mle: -0.18554 (-0.04578) + | > loss_dur: 0.30199 (0.26317) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.48526 (7.81995) + | > current_lr: 0.00001 + | > step_time: 8.90050 (3.30779) + | > loader_time: 0.11540 (0.07319) + + + --> STEP: 178/234 -- GLOBAL_STEP: 9070 + | > loss: 0.08888 (0.21398) + | > log_mle: -0.24047 (-0.05051) + | > loss_dur: 0.32934 (0.26449) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.13828 (8.22058) + | > current_lr: 0.00001 + | > step_time: 6.90900 (3.36655) + | > loader_time: 0.17860 (0.07334) + + + --> STEP: 183/234 -- GLOBAL_STEP: 9075 + | > loss: 0.09307 (0.21127) + | > log_mle: -0.23522 (-0.05487) + | > loss_dur: 0.32830 (0.26614) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.00089 (8.62677) + | > current_lr: 0.00001 + | > step_time: 3.20530 (3.38944) + | > loader_time: 0.00430 (0.07294) + + + --> STEP: 188/234 -- GLOBAL_STEP: 9080 + | > loss: 0.07857 (0.20840) + | > log_mle: -0.24715 (-0.05925) + | > loss_dur: 0.32572 (0.26765) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.21964 (8.91138) + | > current_lr: 0.00001 + | > step_time: 7.31520 (3.52404) + | > loader_time: 0.07680 (0.07297) + + + --> STEP: 193/234 -- GLOBAL_STEP: 9085 + | > loss: 0.09509 (0.20551) + | > log_mle: -0.24565 (-0.06343) + | > loss_dur: 0.34074 (0.26894) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.47749 (9.30507) + | > current_lr: 0.00001 + | > step_time: 6.89710 (3.53534) + | > loader_time: 0.09530 (0.07300) + + + --> STEP: 198/234 -- GLOBAL_STEP: 9090 + | > loss: 0.08618 (0.20271) + | > log_mle: -0.23574 (-0.06741) + | > loss_dur: 0.32192 (0.27011) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.53475 (9.74792) + | > current_lr: 0.00001 + | > step_time: 3.90970 (3.59920) + | > loader_time: 0.19200 (0.07357) + + + --> STEP: 203/234 -- GLOBAL_STEP: 9095 + | > loss: 0.12173 (0.20024) + | > log_mle: -0.18523 (-0.07131) + | > loss_dur: 0.30696 (0.27156) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.60806 (10.12243) + | > current_lr: 0.00001 + | > step_time: 2.71490 (3.64070) + | > loader_time: 0.08930 (0.07908) + + + --> STEP: 208/234 -- GLOBAL_STEP: 9100 + | > loss: 0.08915 (0.19728) + | > log_mle: -0.24921 (-0.07576) + | > loss_dur: 0.33836 (0.27304) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.52165 (10.49850) + | > current_lr: 0.00001 + | > step_time: 3.60000 (3.63395) + | > loader_time: 0.00740 (0.07771) + + + --> STEP: 213/234 -- GLOBAL_STEP: 9105 + | > loss: 0.05713 (0.19415) + | > log_mle: -0.29165 (-0.08058) + | > loss_dur: 0.34879 (0.27473) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.29193 (10.92257) + | > current_lr: 0.00001 + | > step_time: 4.19650 (3.69043) + | > loader_time: 0.00520 (0.07827) + + + --> STEP: 218/234 -- GLOBAL_STEP: 9110 + | > loss: 0.07358 (0.19106) + | > log_mle: -0.25940 (-0.08498) + | > loss_dur: 0.33298 (0.27604) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.00308 (11.24068) + | > current_lr: 0.00001 + | > step_time: 6.09420 (3.77545) + | > loader_time: 0.09750 (0.07923) + + + --> STEP: 223/234 -- GLOBAL_STEP: 9115 + | > loss: 0.04796 (0.18802) + | > log_mle: -0.29650 (-0.08960) + | > loss_dur: 0.34446 (0.27762) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.74391 (11.60785) + | > current_lr: 0.00001 + | > step_time: 0.22420 (3.73064) + | > loader_time: 0.00520 (0.07795) + + + --> STEP: 228/234 -- GLOBAL_STEP: 9120 + | > loss: 0.07250 (0.18486) + | > log_mle: -0.29746 (-0.09441) + | > loss_dur: 0.36997 (0.27928) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.25003 (12.03469) + | > current_lr: 0.00001 + | > step_time: 0.24480 (3.65405) + | > loader_time: 0.00390 (0.07632) + + + --> STEP: 233/234 -- GLOBAL_STEP: 9125 + | > loss: 0.75889 (0.18604) + | > log_mle: -0.25413 (-0.10007) + | > loss_dur: 1.01303 (0.28611) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.39737 (12.49405) + | > current_lr: 0.00001 + | > step_time: 0.18850 (3.58140) + | > loader_time: 0.00290 (0.07478) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05296 (-0.92310) + | > avg_loss: 0.13197 (-0.04227) + | > avg_log_mle: -0.17219 (-0.03853) + | > avg_loss_dur: 0.30416 (-0.00374) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_9126.pth + + > EPOCH: 39/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 17:45:53)  + + --> STEP: 4/234 -- GLOBAL_STEP: 9130 + | > loss: 0.34979 (0.36945) + | > log_mle: 0.01718 (0.04111) + | > loss_dur: 0.33261 (0.32834) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.57295 (3.29864) + | > current_lr: 0.00001 + | > step_time: 12.51190 (5.74953) + | > loader_time: 0.08550 (0.02334) + + + --> STEP: 9/234 -- GLOBAL_STEP: 9135 + | > loss: 0.28982 (0.31989) + | > log_mle: 0.00612 (0.02803) + | > loss_dur: 0.28370 (0.29186) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.92926 (2.66670) + | > current_lr: 0.00001 + | > step_time: 7.91350 (5.84198) + | > loader_time: 0.29400 (0.08811) + + + --> STEP: 14/234 -- GLOBAL_STEP: 9140 + | > loss: 0.31140 (0.30558) + | > log_mle: 0.01335 (0.02519) + | > loss_dur: 0.29804 (0.28039) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.14568 (2.49292) + | > current_lr: 0.00001 + | > step_time: 1.90710 (5.88453) + | > loader_time: 0.00220 (0.08390) + + + --> STEP: 19/234 -- GLOBAL_STEP: 9145 + | > loss: 0.31469 (0.29796) + | > log_mle: 0.02487 (0.02433) + | > loss_dur: 0.28982 (0.27363) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.58006 (2.28938) + | > current_lr: 0.00001 + | > step_time: 1.59670 (5.33033) + | > loader_time: 0.00130 (0.06696) + + + --> STEP: 24/234 -- GLOBAL_STEP: 9150 + | > loss: 0.26816 (0.28899) + | > log_mle: 0.02218 (0.02316) + | > loss_dur: 0.24598 (0.26583) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.76668 (2.14550) + | > current_lr: 0.00001 + | > step_time: 3.28670 (5.27414) + | > loader_time: 0.00440 (0.06524) + + + --> STEP: 29/234 -- GLOBAL_STEP: 9155 + | > loss: 0.24291 (0.28202) + | > log_mle: 0.02355 (0.02216) + | > loss_dur: 0.21936 (0.25986) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.24143 (2.02794) + | > current_lr: 0.00001 + | > step_time: 4.90810 (5.29898) + | > loader_time: 0.00580 (0.06537) + + + --> STEP: 34/234 -- GLOBAL_STEP: 9160 + | > loss: 0.28064 (0.27840) + | > log_mle: 0.00800 (0.01969) + | > loss_dur: 0.27264 (0.25871) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.97333 (1.96859) + | > current_lr: 0.00001 + | > step_time: 1.11260 (4.69656) + | > loader_time: 0.08370 (0.05856) + + + --> STEP: 39/234 -- GLOBAL_STEP: 9165 + | > loss: 0.23843 (0.27394) + | > log_mle: 0.00177 (0.01749) + | > loss_dur: 0.23666 (0.25644) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.95107 (2.21483) + | > current_lr: 0.00001 + | > step_time: 2.18960 (4.37579) + | > loader_time: 0.00170 (0.05622) + + + --> STEP: 44/234 -- GLOBAL_STEP: 9170 + | > loss: 0.24129 (0.27195) + | > log_mle: 0.00690 (0.01691) + | > loss_dur: 0.23439 (0.25504) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.62688 (2.15367) + | > current_lr: 0.00001 + | > step_time: 1.41040 (4.04644) + | > loader_time: 0.00170 (0.05207) + + + --> STEP: 49/234 -- GLOBAL_STEP: 9175 + | > loss: 0.21132 (0.26771) + | > log_mle: -0.00005 (0.01549) + | > loss_dur: 0.21137 (0.25222) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.26929 (2.10211) + | > current_lr: 0.00001 + | > step_time: 2.51420 (3.83352) + | > loader_time: 0.00950 (0.05099) + + + --> STEP: 54/234 -- GLOBAL_STEP: 9180 + | > loss: 0.22358 (0.26521) + | > log_mle: -0.00235 (0.01493) + | > loss_dur: 0.22593 (0.25028) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.40890 (2.07982) + | > current_lr: 0.00001 + | > step_time: 1.41210 (3.66559) + | > loader_time: 0.08850 (0.04807) + + + --> STEP: 59/234 -- GLOBAL_STEP: 9185 + | > loss: 0.19365 (0.26273) + | > log_mle: -0.02008 (0.01372) + | > loss_dur: 0.21373 (0.24901) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.78846 (2.05999) + | > current_lr: 0.00001 + | > step_time: 2.09380 (3.54080) + | > loader_time: 0.19740 (0.05047) + + + --> STEP: 64/234 -- GLOBAL_STEP: 9190 + | > loss: 0.22022 (0.26047) + | > log_mle: 0.01120 (0.01186) + | > loss_dur: 0.20902 (0.24861) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.33778 (2.17417) + | > current_lr: 0.00001 + | > step_time: 2.39470 (3.44687) + | > loader_time: 0.00240 (0.04676) + + + --> STEP: 69/234 -- GLOBAL_STEP: 9195 + | > loss: 0.26599 (0.25839) + | > log_mle: 0.02346 (0.01130) + | > loss_dur: 0.24253 (0.24709) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.60997 (2.23253) + | > current_lr: 0.00001 + | > step_time: 3.09430 (3.33755) + | > loader_time: 0.10030 (0.04858) + + + --> STEP: 74/234 -- GLOBAL_STEP: 9200 + | > loss: 0.20633 (0.25568) + | > log_mle: -0.00323 (0.00959) + | > loss_dur: 0.20956 (0.24609) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.62399 (2.52233) + | > current_lr: 0.00001 + | > step_time: 1.31090 (3.26349) + | > loader_time: 0.08280 (0.04768) + + + --> STEP: 79/234 -- GLOBAL_STEP: 9205 + | > loss: 0.21279 (0.25355) + | > log_mle: -0.01260 (0.00838) + | > loss_dur: 0.22539 (0.24517) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.03539 (2.57579) + | > current_lr: 0.00001 + | > step_time: 2.08700 (3.22112) + | > loader_time: 0.00230 (0.04695) + + + --> STEP: 84/234 -- GLOBAL_STEP: 9210 + | > loss: 0.21860 (0.25163) + | > log_mle: -0.01607 (0.00715) + | > loss_dur: 0.23466 (0.24449) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.93030 (2.70577) + | > current_lr: 0.00001 + | > step_time: 2.99170 (3.19952) + | > loader_time: 0.00380 (0.04439) + + + --> STEP: 89/234 -- GLOBAL_STEP: 9215 + | > loss: 0.19016 (0.24906) + | > log_mle: -0.04310 (0.00493) + | > loss_dur: 0.23326 (0.24413) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.63051 (2.87994) + | > current_lr: 0.00001 + | > step_time: 1.59660 (3.14096) + | > loader_time: 0.09510 (0.04397) + + + --> STEP: 94/234 -- GLOBAL_STEP: 9220 + | > loss: 0.16368 (0.24564) + | > log_mle: -0.07964 (0.00161) + | > loss_dur: 0.24332 (0.24403) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.52891 (3.16248) + | > current_lr: 0.00001 + | > step_time: 1.99340 (3.10568) + | > loader_time: 0.00290 (0.04377) + + + --> STEP: 99/234 -- GLOBAL_STEP: 9225 + | > loss: 0.16319 (0.24323) + | > log_mle: -0.10662 (-0.00159) + | > loss_dur: 0.26981 (0.24482) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.19574 (3.45080) + | > current_lr: 0.00001 + | > step_time: 1.89770 (3.03278) + | > loader_time: 0.00220 (0.04256) + + + --> STEP: 104/234 -- GLOBAL_STEP: 9230 + | > loss: 0.13779 (0.24034) + | > log_mle: -0.11533 (-0.00510) + | > loss_dur: 0.25312 (0.24544) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.68355 (3.66612) + | > current_lr: 0.00001 + | > step_time: 4.70150 (3.05551) + | > loader_time: 0.09560 (0.04484) + + + --> STEP: 109/234 -- GLOBAL_STEP: 9235 + | > loss: 0.19253 (0.23841) + | > log_mle: -0.08955 (-0.00772) + | > loss_dur: 0.28207 (0.24613) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.30398 (3.88624) + | > current_lr: 0.00001 + | > step_time: 1.99110 (3.01780) + | > loader_time: 0.00540 (0.04293) + + + --> STEP: 114/234 -- GLOBAL_STEP: 9240 + | > loss: 0.17840 (0.23587) + | > log_mle: -0.06887 (-0.01100) + | > loss_dur: 0.24727 (0.24686) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.24448 (4.17764) + | > current_lr: 0.00001 + | > step_time: 1.59490 (2.98645) + | > loader_time: 0.00320 (0.04270) + + + --> STEP: 119/234 -- GLOBAL_STEP: 9245 + | > loss: 0.19739 (0.23410) + | > log_mle: -0.06909 (-0.01354) + | > loss_dur: 0.26648 (0.24764) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.87932 (4.37828) + | > current_lr: 0.00001 + | > step_time: 3.80470 (2.99481) + | > loader_time: 0.08030 (0.04240) + + + --> STEP: 124/234 -- GLOBAL_STEP: 9250 + | > loss: 0.15853 (0.23225) + | > log_mle: -0.09484 (-0.01548) + | > loss_dur: 0.25337 (0.24773) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.25933 (4.59290) + | > current_lr: 0.00001 + | > step_time: 1.40410 (2.96116) + | > loader_time: 0.00200 (0.04217) + + + --> STEP: 129/234 -- GLOBAL_STEP: 9255 + | > loss: 0.18196 (0.22963) + | > log_mle: -0.08057 (-0.01885) + | > loss_dur: 0.26253 (0.24848) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.04289 (4.83545) + | > current_lr: 0.00001 + | > step_time: 2.70540 (2.94189) + | > loader_time: 0.08610 (0.04203) + + + --> STEP: 134/234 -- GLOBAL_STEP: 9260 + | > loss: 0.17932 (0.22689) + | > log_mle: -0.12563 (-0.02259) + | > loss_dur: 0.30495 (0.24948) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.04049 (5.18802) + | > current_lr: 0.00001 + | > step_time: 1.58450 (2.91116) + | > loader_time: 0.00270 (0.04184) + + + --> STEP: 139/234 -- GLOBAL_STEP: 9265 + | > loss: 0.09708 (0.22435) + | > log_mle: -0.18188 (-0.02613) + | > loss_dur: 0.27896 (0.25048) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.86287 (5.51737) + | > current_lr: 0.00001 + | > step_time: 1.31620 (2.87195) + | > loader_time: 0.08520 (0.04108) + + + --> STEP: 144/234 -- GLOBAL_STEP: 9270 + | > loss: 0.14413 (0.22198) + | > log_mle: -0.16075 (-0.02989) + | > loss_dur: 0.30488 (0.25187) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.07127 (5.81627) + | > current_lr: 0.00001 + | > step_time: 3.20040 (2.88121) + | > loader_time: 0.00310 (0.04100) + + + --> STEP: 149/234 -- GLOBAL_STEP: 9275 + | > loss: 0.10363 (0.21876) + | > log_mle: -0.19913 (-0.03409) + | > loss_dur: 0.30276 (0.25284) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.17840 (6.10803) + | > current_lr: 0.00001 + | > step_time: 9.00440 (2.94234) + | > loader_time: 0.30170 (0.04288) + + + --> STEP: 154/234 -- GLOBAL_STEP: 9280 + | > loss: 0.11672 (0.21540) + | > log_mle: -0.16371 (-0.03858) + | > loss_dur: 0.28043 (0.25399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.13934 (6.47800) + | > current_lr: 0.00001 + | > step_time: 2.81410 (3.07873) + | > loader_time: 0.09570 (0.04403) + + + --> STEP: 159/234 -- GLOBAL_STEP: 9285 + | > loss: 0.12231 (0.21239) + | > log_mle: -0.18204 (-0.04290) + | > loss_dur: 0.30435 (0.25529) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.03496 (6.79911) + | > current_lr: 0.00001 + | > step_time: 3.80470 (3.10154) + | > loader_time: 0.00710 (0.04391) + + + --> STEP: 164/234 -- GLOBAL_STEP: 9290 + | > loss: 0.10908 (0.20933) + | > log_mle: -0.17489 (-0.04704) + | > loss_dur: 0.28398 (0.25637) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.90092 (7.11231) + | > current_lr: 0.00001 + | > step_time: 9.30500 (3.13645) + | > loader_time: 0.09710 (0.04474) + + + --> STEP: 169/234 -- GLOBAL_STEP: 9295 + | > loss: 0.13841 (0.20645) + | > log_mle: -0.16977 (-0.05122) + | > loss_dur: 0.30818 (0.25767) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.70938 (7.34314) + | > current_lr: 0.00001 + | > step_time: 1.70640 (3.17514) + | > loader_time: 0.00320 (0.04533) + + + --> STEP: 174/234 -- GLOBAL_STEP: 9300 + | > loss: 0.04661 (0.20267) + | > log_mle: -0.25339 (-0.05635) + | > loss_dur: 0.30000 (0.25902) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.03138 (7.76517) + | > current_lr: 0.00001 + | > step_time: 2.00130 (3.17824) + | > loader_time: 0.08630 (0.04518) + + + --> STEP: 179/234 -- GLOBAL_STEP: 9305 + | > loss: 0.08380 (0.19958) + | > log_mle: -0.23636 (-0.06090) + | > loss_dur: 0.32016 (0.26048) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.67562 (8.15296) + | > current_lr: 0.00001 + | > step_time: 2.17970 (3.18103) + | > loader_time: 0.00290 (0.04444) + + + --> STEP: 184/234 -- GLOBAL_STEP: 9310 + | > loss: 0.10841 (0.19710) + | > log_mle: -0.21579 (-0.06506) + | > loss_dur: 0.32419 (0.26216) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.33216 (8.55031) + | > current_lr: 0.00001 + | > step_time: 7.30190 (3.23486) + | > loader_time: 0.20420 (0.04709) + + + --> STEP: 189/234 -- GLOBAL_STEP: 9315 + | > loss: 0.10040 (0.19430) + | > log_mle: -0.21382 (-0.06934) + | > loss_dur: 0.31423 (0.26364) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.41643 (8.94533) + | > current_lr: 0.00001 + | > step_time: 4.59640 (3.26887) + | > loader_time: 0.00320 (0.04700) + + + --> STEP: 194/234 -- GLOBAL_STEP: 9320 + | > loss: 0.06443 (0.19131) + | > log_mle: -0.24443 (-0.07364) + | > loss_dur: 0.30886 (0.26495) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.72751 (9.32403) + | > current_lr: 0.00001 + | > step_time: 2.71480 (3.32250) + | > loader_time: 0.28660 (0.04872) + + + --> STEP: 199/234 -- GLOBAL_STEP: 9325 + | > loss: 0.07541 (0.18865) + | > log_mle: -0.25383 (-0.07765) + | > loss_dur: 0.32924 (0.26629) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.84823 (9.60746) + | > current_lr: 0.00001 + | > step_time: 8.71080 (3.37268) + | > loader_time: 0.00420 (0.05296) + + + --> STEP: 204/234 -- GLOBAL_STEP: 9330 + | > loss: 0.06623 (0.18611) + | > log_mle: -0.27787 (-0.08163) + | > loss_dur: 0.34410 (0.26774) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.58789 (9.96083) + | > current_lr: 0.00001 + | > step_time: 3.68920 (3.37581) + | > loader_time: 0.09610 (0.05360) + + + --> STEP: 209/234 -- GLOBAL_STEP: 9335 + | > loss: 0.08331 (0.18319) + | > log_mle: -0.24004 (-0.08586) + | > loss_dur: 0.32335 (0.26905) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.82056 (10.24350) + | > current_lr: 0.00001 + | > step_time: 4.80630 (3.37887) + | > loader_time: 0.09330 (0.05364) + + + --> STEP: 214/234 -- GLOBAL_STEP: 9340 + | > loss: 0.03354 (0.17985) + | > log_mle: -0.26972 (-0.09080) + | > loss_dur: 0.30325 (0.27065) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.67025 (10.63947) + | > current_lr: 0.00001 + | > step_time: 4.80510 (3.49920) + | > loader_time: 0.40420 (0.05737) + + + --> STEP: 219/234 -- GLOBAL_STEP: 9345 + | > loss: -0.00565 (0.17655) + | > log_mle: -0.35166 (-0.09552) + | > loss_dur: 0.34601 (0.27207) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.90662 (11.00753) + | > current_lr: 0.00001 + | > step_time: 3.40150 (3.57140) + | > loader_time: 0.09440 (0.06010) + + + --> STEP: 224/234 -- GLOBAL_STEP: 9350 + | > loss: 0.02740 (0.17364) + | > log_mle: -0.31246 (-0.09995) + | > loss_dur: 0.33986 (0.27359) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.31271 (11.29171) + | > current_lr: 0.00001 + | > step_time: 1.39710 (3.56976) + | > loader_time: 0.00440 (0.05966) + + + --> STEP: 229/234 -- GLOBAL_STEP: 9355 + | > loss: 0.07893 (0.17075) + | > log_mle: -0.33058 (-0.10480) + | > loss_dur: 0.40951 (0.27555) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.24906 (11.75926) + | > current_lr: 0.00001 + | > step_time: 0.24230 (3.49941) + | > loader_time: 0.00260 (0.05843) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.25721 (+0.20425) + | > avg_loss: 0.12123 (-0.01074) + | > avg_log_mle: -0.17891 (-0.00672) + | > avg_loss_dur: 0.30014 (-0.00402) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_9360.pth + + > EPOCH: 40/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 18:00:44)  + + --> STEP: 0/234 -- GLOBAL_STEP: 9360 + | > loss: 0.32348 (0.32348) + | > log_mle: 0.00767 (0.00767) + | > loss_dur: 0.31580 (0.31580) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.35889 (2.35889) + | > current_lr: 0.00001 + | > step_time: 3.58860 (3.58856) + | > loader_time: 2.86450 (2.86445) + + + --> STEP: 5/234 -- GLOBAL_STEP: 9365 + | > loss: 0.23711 (0.32845) + | > log_mle: 0.01530 (0.02777) + | > loss_dur: 0.22182 (0.30068) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30570 (2.36081) + | > current_lr: 0.00001 + | > step_time: 4.10660 (4.56061) + | > loader_time: 0.09100 (1.95709) + + + --> STEP: 10/234 -- GLOBAL_STEP: 9370 + | > loss: 0.22711 (0.29463) + | > log_mle: 0.00174 (0.01669) + | > loss_dur: 0.22537 (0.27794) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.77028 (2.48108) + | > current_lr: 0.00001 + | > step_time: 4.50590 (5.06024) + | > loader_time: 0.00500 (0.98059) + + + --> STEP: 15/234 -- GLOBAL_STEP: 9375 + | > loss: 0.28511 (0.28476) + | > log_mle: 0.00872 (0.01531) + | > loss_dur: 0.27639 (0.26945) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.75130 (2.28583) + | > current_lr: 0.00001 + | > step_time: 3.20540 (5.21352) + | > loader_time: 0.00400 (0.67311) + + + --> STEP: 20/234 -- GLOBAL_STEP: 9380 + | > loss: 0.25125 (0.27689) + | > log_mle: 0.01326 (0.01494) + | > loss_dur: 0.23799 (0.26195) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.56695 (2.14573) + | > current_lr: 0.00001 + | > step_time: 7.49390 (5.85955) + | > loader_time: 0.10680 (0.54035) + + + --> STEP: 25/234 -- GLOBAL_STEP: 9385 + | > loss: 0.26854 (0.26895) + | > log_mle: 0.01929 (0.01413) + | > loss_dur: 0.24925 (0.25482) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.56087 (2.03967) + | > current_lr: 0.00001 + | > step_time: 3.59420 (5.52387) + | > loader_time: 0.00450 (0.44421) + + + --> STEP: 30/234 -- GLOBAL_STEP: 9390 + | > loss: 0.22251 (0.26140) + | > log_mle: -0.01080 (0.01222) + | > loss_dur: 0.23331 (0.24918) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.48215 (1.94964) + | > current_lr: 0.00001 + | > step_time: 3.20730 (5.33745) + | > loader_time: 0.00170 (0.37990) + + + --> STEP: 35/234 -- GLOBAL_STEP: 9395 + | > loss: 0.21935 (0.25849) + | > log_mle: -0.00844 (0.01009) + | > loss_dur: 0.22779 (0.24840) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.18599 (2.32933) + | > current_lr: 0.00001 + | > step_time: 4.70430 (5.25842) + | > loader_time: 0.10010 (0.34213) + + + --> STEP: 40/234 -- GLOBAL_STEP: 9400 + | > loss: 0.26358 (0.25573) + | > log_mle: 0.01293 (0.00872) + | > loss_dur: 0.25065 (0.24701) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.62091 (2.37001) + | > current_lr: 0.00001 + | > step_time: 2.80900 (4.92411) + | > loader_time: 0.08700 (0.30182) + + + --> STEP: 45/234 -- GLOBAL_STEP: 9405 + | > loss: 0.20769 (0.25249) + | > log_mle: -0.01698 (0.00754) + | > loss_dur: 0.22467 (0.24495) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.60112 (2.32481) + | > current_lr: 0.00001 + | > step_time: 3.89970 (4.82853) + | > loader_time: 0.19310 (0.27927) + + + --> STEP: 50/234 -- GLOBAL_STEP: 9410 + | > loss: 0.22441 (0.24912) + | > log_mle: 0.00476 (0.00661) + | > loss_dur: 0.21965 (0.24251) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.31823 (2.25079) + | > current_lr: 0.00001 + | > step_time: 2.27900 (4.54000) + | > loader_time: 0.00200 (0.25331) + + + --> STEP: 55/234 -- GLOBAL_STEP: 9415 + | > loss: 0.22378 (0.24684) + | > log_mle: -0.01551 (0.00568) + | > loss_dur: 0.23929 (0.24116) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.75475 (2.22241) + | > current_lr: 0.00001 + | > step_time: 1.18990 (4.25199) + | > loader_time: 0.00300 (0.23048) + + + --> STEP: 60/234 -- GLOBAL_STEP: 9420 + | > loss: 0.19283 (0.24414) + | > log_mle: -0.03113 (0.00426) + | > loss_dur: 0.22396 (0.23989) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.34478 (2.20233) + | > current_lr: 0.00001 + | > step_time: 1.27600 (4.08746) + | > loader_time: 0.00210 (0.21307) + + + --> STEP: 65/234 -- GLOBAL_STEP: 9425 + | > loss: 0.21197 (0.24229) + | > log_mle: -0.00616 (0.00283) + | > loss_dur: 0.21813 (0.23946) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.58316 (2.24668) + | > current_lr: 0.00001 + | > step_time: 2.50900 (3.93360) + | > loader_time: 0.00370 (0.20173) + + + --> STEP: 70/234 -- GLOBAL_STEP: 9430 + | > loss: 0.22162 (0.24050) + | > log_mle: -0.01576 (0.00213) + | > loss_dur: 0.23739 (0.23837) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.14910 (2.46186) + | > current_lr: 0.00001 + | > step_time: 1.80360 (3.83419) + | > loader_time: 0.00190 (0.19000) + + + --> STEP: 75/234 -- GLOBAL_STEP: 9435 + | > loss: 0.22218 (0.23803) + | > log_mle: -0.02231 (0.00034) + | > loss_dur: 0.24449 (0.23769) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.70763 (2.66150) + | > current_lr: 0.00001 + | > step_time: 2.49710 (3.70532) + | > loader_time: 0.00320 (0.17754) + + + --> STEP: 80/234 -- GLOBAL_STEP: 9440 + | > loss: 0.19621 (0.23601) + | > log_mle: -0.00406 (-0.00066) + | > loss_dur: 0.20026 (0.23667) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.59433 (2.68268) + | > current_lr: 0.00001 + | > step_time: 1.17310 (3.60860) + | > loader_time: 0.00270 (0.17124) + + + --> STEP: 85/234 -- GLOBAL_STEP: 9445 + | > loss: 0.19879 (0.23388) + | > log_mle: -0.01948 (-0.00212) + | > loss_dur: 0.21826 (0.23599) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.22842 (2.73782) + | > current_lr: 0.00001 + | > step_time: 1.60100 (3.52788) + | > loader_time: 0.08730 (0.16336) + + + --> STEP: 90/234 -- GLOBAL_STEP: 9450 + | > loss: 0.18909 (0.23126) + | > log_mle: -0.05120 (-0.00463) + | > loss_dur: 0.24028 (0.23589) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.44899 (3.04612) + | > current_lr: 0.00001 + | > step_time: 1.79650 (3.47757) + | > loader_time: 0.00260 (0.15637) + + + --> STEP: 95/234 -- GLOBAL_STEP: 9455 + | > loss: 0.13153 (0.22763) + | > log_mle: -0.12862 (-0.00867) + | > loss_dur: 0.26015 (0.23630) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.98018 (3.50628) + | > current_lr: 0.00001 + | > step_time: 2.50760 (3.44008) + | > loader_time: 0.00250 (0.15021) + + + --> STEP: 100/234 -- GLOBAL_STEP: 9460 + | > loss: 0.19265 (0.22581) + | > log_mle: -0.06244 (-0.01108) + | > loss_dur: 0.25509 (0.23689) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.76768 (3.78259) + | > current_lr: 0.00001 + | > step_time: 2.10760 (3.40211) + | > loader_time: 0.00340 (0.14462) + + + --> STEP: 105/234 -- GLOBAL_STEP: 9465 + | > loss: 0.19506 (0.22315) + | > log_mle: -0.03106 (-0.01421) + | > loss_dur: 0.22612 (0.23735) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.35496 (4.04342) + | > current_lr: 0.00001 + | > step_time: 2.54910 (3.39114) + | > loader_time: 0.07870 (0.13858) + + + --> STEP: 110/234 -- GLOBAL_STEP: 9470 + | > loss: 0.17082 (0.22118) + | > log_mle: -0.06288 (-0.01707) + | > loss_dur: 0.23370 (0.23825) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.87405 (4.30786) + | > current_lr: 0.00001 + | > step_time: 3.59230 (3.33460) + | > loader_time: 0.00210 (0.13318) + + + --> STEP: 115/234 -- GLOBAL_STEP: 9475 + | > loss: 0.20039 (0.21912) + | > log_mle: -0.07720 (-0.02042) + | > loss_dur: 0.27759 (0.23954) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.40977 (4.53361) + | > current_lr: 0.00001 + | > step_time: 1.71460 (3.31372) + | > loader_time: 0.08470 (0.12826) + + + --> STEP: 120/234 -- GLOBAL_STEP: 9480 + | > loss: 0.13390 (0.21719) + | > log_mle: -0.12428 (-0.02330) + | > loss_dur: 0.25818 (0.24048) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.64646 (4.68058) + | > current_lr: 0.00001 + | > step_time: 1.90450 (3.26913) + | > loader_time: 0.08270 (0.12375) + + + --> STEP: 125/234 -- GLOBAL_STEP: 9485 + | > loss: 0.18516 (0.21580) + | > log_mle: -0.11225 (-0.02510) + | > loss_dur: 0.29741 (0.24089) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.55262 (4.75622) + | > current_lr: 0.00001 + | > step_time: 1.99810 (3.29437) + | > loader_time: 0.09660 (0.12256) + + + --> STEP: 130/234 -- GLOBAL_STEP: 9490 + | > loss: 0.14566 (0.21305) + | > log_mle: -0.12269 (-0.02850) + | > loss_dur: 0.26835 (0.24155) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.68604 (4.93365) + | > current_lr: 0.00001 + | > step_time: 1.91330 (3.25128) + | > loader_time: 0.00210 (0.11860) + + + --> STEP: 135/234 -- GLOBAL_STEP: 9495 + | > loss: 0.17700 (0.21057) + | > log_mle: -0.06048 (-0.03176) + | > loss_dur: 0.23748 (0.24233) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.48800 (5.12798) + | > current_lr: 0.00001 + | > step_time: 1.90710 (3.23676) + | > loader_time: 0.00280 (0.11581) + + + --> STEP: 140/234 -- GLOBAL_STEP: 9500 + | > loss: 0.17819 (0.20815) + | > log_mle: -0.08954 (-0.03547) + | > loss_dur: 0.26773 (0.24362) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.06696 (5.50916) + | > current_lr: 0.00001 + | > step_time: 2.79740 (3.24188) + | > loader_time: 0.09550 (0.11573) + + + --> STEP: 145/234 -- GLOBAL_STEP: 9505 + | > loss: 0.11502 (0.20544) + | > log_mle: -0.17332 (-0.03970) + | > loss_dur: 0.28834 (0.24514) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.99680 (6.19438) + | > current_lr: 0.00001 + | > step_time: 1.40250 (3.24726) + | > loader_time: 0.00260 (0.11325) + + + --> STEP: 150/234 -- GLOBAL_STEP: 9510 + | > loss: 0.13498 (0.20268) + | > log_mle: -0.15988 (-0.04371) + | > loss_dur: 0.29486 (0.24639) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.44892 (6.55192) + | > current_lr: 0.00001 + | > step_time: 5.21260 (3.23046) + | > loader_time: 0.09980 (0.11023) + + + --> STEP: 155/234 -- GLOBAL_STEP: 9515 + | > loss: 0.07748 (0.19902) + | > log_mle: -0.22085 (-0.04856) + | > loss_dur: 0.29833 (0.24758) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.17429 (6.86161) + | > current_lr: 0.00001 + | > step_time: 2.60240 (3.20881) + | > loader_time: 0.10750 (0.10849) + + + --> STEP: 160/234 -- GLOBAL_STEP: 9520 + | > loss: 0.07833 (0.19618) + | > log_mle: -0.21367 (-0.05274) + | > loss_dur: 0.29200 (0.24893) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.34525 (7.24709) + | > current_lr: 0.00001 + | > step_time: 7.51160 (3.22484) + | > loader_time: 0.08580 (0.10624) + + + --> STEP: 165/234 -- GLOBAL_STEP: 9525 + | > loss: 0.10152 (0.19333) + | > log_mle: -0.21206 (-0.05680) + | > loss_dur: 0.31358 (0.25013) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.97839 (7.56242) + | > current_lr: 0.00001 + | > step_time: 2.10460 (3.25924) + | > loader_time: 0.09060 (0.10544) + + + --> STEP: 170/234 -- GLOBAL_STEP: 9530 + | > loss: 0.07926 (0.19052) + | > log_mle: -0.24275 (-0.06111) + | > loss_dur: 0.32201 (0.25163) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.80460 (7.92883) + | > current_lr: 0.00001 + | > step_time: 7.61690 (3.30523) + | > loader_time: 0.10020 (0.10410) + + + --> STEP: 175/234 -- GLOBAL_STEP: 9535 + | > loss: 0.09589 (0.18706) + | > log_mle: -0.21936 (-0.06601) + | > loss_dur: 0.31524 (0.25307) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.28218 (8.41742) + | > current_lr: 0.00001 + | > step_time: 2.19530 (3.29081) + | > loader_time: 0.00470 (0.10124) + + + --> STEP: 180/234 -- GLOBAL_STEP: 9540 + | > loss: 0.08432 (0.18412) + | > log_mle: -0.22889 (-0.07056) + | > loss_dur: 0.31321 (0.25468) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.46367 (8.86057) + | > current_lr: 0.00001 + | > step_time: 2.71100 (3.27889) + | > loader_time: 0.00800 (0.09961) + + + --> STEP: 185/234 -- GLOBAL_STEP: 9545 + | > loss: 0.08549 (0.18174) + | > log_mle: -0.24834 (-0.07476) + | > loss_dur: 0.33384 (0.25650) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.54452 (9.34745) + | > current_lr: 0.00001 + | > step_time: 2.19840 (3.29460) + | > loader_time: 0.00270 (0.09835) + + + --> STEP: 190/234 -- GLOBAL_STEP: 9550 + | > loss: 0.07513 (0.17902) + | > log_mle: -0.23597 (-0.07894) + | > loss_dur: 0.31110 (0.25796) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.15025 (9.72181) + | > current_lr: 0.00001 + | > step_time: 6.18400 (3.35058) + | > loader_time: 0.00810 (0.09733) + + + --> STEP: 195/234 -- GLOBAL_STEP: 9555 + | > loss: 0.07927 (0.17607) + | > log_mle: -0.24191 (-0.08324) + | > loss_dur: 0.32118 (0.25930) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.32323 (10.12695) + | > current_lr: 0.00001 + | > step_time: 5.19850 (3.38421) + | > loader_time: 0.00360 (0.09724) + + + --> STEP: 200/234 -- GLOBAL_STEP: 9560 + | > loss: 0.08134 (0.17343) + | > log_mle: -0.24773 (-0.08724) + | > loss_dur: 0.32906 (0.26067) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.40078 (10.45950) + | > current_lr: 0.00001 + | > step_time: 3.60920 (3.43020) + | > loader_time: 0.19390 (0.09685) + + + --> STEP: 205/234 -- GLOBAL_STEP: 9565 + | > loss: 0.06918 (0.17095) + | > log_mle: -0.23885 (-0.09112) + | > loss_dur: 0.30803 (0.26207) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.18920 (10.84537) + | > current_lr: 0.00001 + | > step_time: 9.50360 (3.51195) + | > loader_time: 0.20740 (0.09655) + + + --> STEP: 210/234 -- GLOBAL_STEP: 9570 + | > loss: 0.02331 (0.16789) + | > log_mle: -0.31225 (-0.09565) + | > loss_dur: 0.33556 (0.26354) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.04284 (11.26824) + | > current_lr: 0.00001 + | > step_time: 9.01340 (3.55462) + | > loader_time: 0.19520 (0.09652) + + + --> STEP: 215/234 -- GLOBAL_STEP: 9575 + | > loss: 0.04296 (0.16474) + | > log_mle: -0.26194 (-0.10027) + | > loss_dur: 0.30490 (0.26501) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.22273 (11.90568) + | > current_lr: 0.00001 + | > step_time: 8.99350 (3.66373) + | > loader_time: 0.10530 (0.09663) + + + --> STEP: 220/234 -- GLOBAL_STEP: 9580 + | > loss: 0.02977 (0.16152) + | > log_mle: -0.30468 (-0.10512) + | > loss_dur: 0.33445 (0.26664) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.43063 (12.53096) + | > current_lr: 0.00001 + | > step_time: 4.19240 (3.68549) + | > loader_time: 0.00340 (0.09673) + + + --> STEP: 225/234 -- GLOBAL_STEP: 9585 + | > loss: -0.00933 (0.15861) + | > log_mle: -0.36139 (-0.10971) + | > loss_dur: 0.35206 (0.26832) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.81686 (13.08896) + | > current_lr: 0.00001 + | > step_time: 0.23240 (3.61437) + | > loader_time: 0.00260 (0.09466) + + + --> STEP: 230/234 -- GLOBAL_STEP: 9590 + | > loss: 0.01995 (0.15598) + | > log_mle: -0.40132 (-0.11467) + | > loss_dur: 0.42127 (0.27065) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.26738 (13.74807) + | > current_lr: 0.00001 + | > step_time: 0.24620 (3.54110) + | > loader_time: 0.00470 (0.09269) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.62935 (+0.37213) + | > avg_loss: 0.11680 (-0.00443) + | > avg_log_mle: -0.18127 (-0.00236) + | > avg_loss_dur: 0.29806 (-0.00208) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_9594.pth + + > EPOCH: 41/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 18:15:37)  + + --> STEP: 1/234 -- GLOBAL_STEP: 9595 + | > loss: 0.34447 (0.34447) + | > log_mle: 0.02817 (0.02817) + | > loss_dur: 0.31631 (0.31631) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.76631 (1.76631) + | > current_lr: 0.00001 + | > step_time: 11.50500 (11.50501) + | > loader_time: 0.09410 (0.09407) + + + --> STEP: 6/234 -- GLOBAL_STEP: 9600 + | > loss: 0.27492 (0.31367) + | > log_mle: 0.01736 (0.01844) + | > loss_dur: 0.25756 (0.29524) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.76341 (2.79580) + | > current_lr: 0.00001 + | > step_time: 4.99870 (5.90279) + | > loader_time: 0.09930 (0.06635) + + + --> STEP: 11/234 -- GLOBAL_STEP: 9605 + | > loss: 0.25182 (0.27954) + | > log_mle: 0.01003 (0.00764) + | > loss_dur: 0.24178 (0.27190) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.68035 (2.67555) + | > current_lr: 0.00001 + | > step_time: 4.31280 (5.57837) + | > loader_time: 0.09470 (0.08039) + + + --> STEP: 16/234 -- GLOBAL_STEP: 9610 + | > loss: 0.24040 (0.27066) + | > log_mle: -0.00386 (0.00562) + | > loss_dur: 0.24426 (0.26504) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41265 (2.49614) + | > current_lr: 0.00001 + | > step_time: 2.08630 (4.82207) + | > loader_time: 0.00300 (0.06156) + + + --> STEP: 21/234 -- GLOBAL_STEP: 9615 + | > loss: 0.23277 (0.26266) + | > log_mle: 0.01193 (0.00626) + | > loss_dur: 0.22084 (0.25640) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10958 (2.34723) + | > current_lr: 0.00001 + | > step_time: 4.59300 (4.93996) + | > loader_time: 0.00090 (0.06624) + + + --> STEP: 26/234 -- GLOBAL_STEP: 9620 + | > loss: 0.22865 (0.25396) + | > log_mle: -0.00354 (0.00487) + | > loss_dur: 0.23219 (0.24909) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.71704 (2.21381) + | > current_lr: 0.00001 + | > step_time: 3.59150 (4.33760) + | > loader_time: 0.00290 (0.05415) + + + --> STEP: 31/234 -- GLOBAL_STEP: 9625 + | > loss: 0.23529 (0.24761) + | > log_mle: -0.01081 (0.00285) + | > loss_dur: 0.24610 (0.24476) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.59321 (2.11081) + | > current_lr: 0.00001 + | > step_time: 4.59620 (4.03873) + | > loader_time: 0.00450 (0.04585) + + + --> STEP: 36/234 -- GLOBAL_STEP: 9630 + | > loss: 0.24021 (0.24459) + | > log_mle: -0.01615 (0.00075) + | > loss_dur: 0.25636 (0.24384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.76975 (2.34273) + | > current_lr: 0.00001 + | > step_time: 5.89520 (4.00775) + | > loader_time: 0.00130 (0.04263) + + + --> STEP: 41/234 -- GLOBAL_STEP: 9635 + | > loss: 0.19364 (0.24107) + | > log_mle: -0.01160 (-0.00041) + | > loss_dur: 0.20524 (0.24147) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.84667 (2.33063) + | > current_lr: 0.00001 + | > step_time: 3.69540 (3.91625) + | > loader_time: 0.00410 (0.04020) + + + --> STEP: 46/234 -- GLOBAL_STEP: 9640 + | > loss: 0.22946 (0.23909) + | > log_mle: -0.01283 (-0.00146) + | > loss_dur: 0.24229 (0.24055) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.57821 (2.30136) + | > current_lr: 0.00001 + | > step_time: 1.52750 (3.76871) + | > loader_time: 0.08340 (0.04022) + + + --> STEP: 51/234 -- GLOBAL_STEP: 9645 + | > loss: 0.20813 (0.23578) + | > log_mle: 0.00421 (-0.00196) + | > loss_dur: 0.20391 (0.23774) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.43808 (2.26151) + | > current_lr: 0.00001 + | > step_time: 1.71830 (3.59266) + | > loader_time: 0.00270 (0.03861) + + + --> STEP: 56/234 -- GLOBAL_STEP: 9650 + | > loss: 0.24018 (0.23449) + | > log_mle: -0.01615 (-0.00319) + | > loss_dur: 0.25633 (0.23768) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.76186 (2.22991) + | > current_lr: 0.00001 + | > step_time: 2.62090 (3.49560) + | > loader_time: 0.08540 (0.04043) + + + --> STEP: 61/234 -- GLOBAL_STEP: 9655 + | > loss: 0.19802 (0.23108) + | > log_mle: -0.01045 (-0.00440) + | > loss_dur: 0.20847 (0.23548) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.53807 (2.22445) + | > current_lr: 0.00001 + | > step_time: 1.97750 (3.38553) + | > loader_time: 0.00180 (0.03879) + + + --> STEP: 66/234 -- GLOBAL_STEP: 9660 + | > loss: 0.21350 (0.22953) + | > log_mle: -0.00461 (-0.00568) + | > loss_dur: 0.21811 (0.23520) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.63876 (2.29534) + | > current_lr: 0.00001 + | > step_time: 1.38390 (3.28570) + | > loader_time: 0.00250 (0.03750) + + + --> STEP: 71/234 -- GLOBAL_STEP: 9665 + | > loss: 0.19253 (0.22752) + | > log_mle: -0.05135 (-0.00702) + | > loss_dur: 0.24388 (0.23453) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.85821 (2.59809) + | > current_lr: 0.00001 + | > step_time: 2.01220 (3.24492) + | > loader_time: 0.08780 (0.03631) + + + --> STEP: 76/234 -- GLOBAL_STEP: 9670 + | > loss: 0.19924 (0.22527) + | > log_mle: -0.02931 (-0.00842) + | > loss_dur: 0.22854 (0.23370) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.15278 (2.68316) + | > current_lr: 0.00001 + | > step_time: 0.99360 (3.29470) + | > loader_time: 0.09720 (0.04036) + + + --> STEP: 81/234 -- GLOBAL_STEP: 9675 + | > loss: 0.18150 (0.22293) + | > log_mle: -0.04341 (-0.00954) + | > loss_dur: 0.22491 (0.23247) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.85416 (2.80093) + | > current_lr: 0.00001 + | > step_time: 1.21340 (3.26841) + | > loader_time: 0.00390 (0.04135) + + + --> STEP: 86/234 -- GLOBAL_STEP: 9680 + | > loss: 0.17533 (0.22096) + | > log_mle: -0.04273 (-0.01093) + | > loss_dur: 0.21806 (0.23189) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.64151 (2.91870) + | > current_lr: 0.00001 + | > step_time: 3.22030 (3.19928) + | > loader_time: 0.00210 (0.03909) + + + --> STEP: 91/234 -- GLOBAL_STEP: 9685 + | > loss: 0.19380 (0.21856) + | > log_mle: -0.04743 (-0.01343) + | > loss_dur: 0.24123 (0.23199) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.45795 (3.22616) + | > current_lr: 0.00001 + | > step_time: 1.50200 (3.14901) + | > loader_time: 0.00160 (0.03802) + + + --> STEP: 96/234 -- GLOBAL_STEP: 9690 + | > loss: 0.18746 (0.21521) + | > log_mle: -0.04143 (-0.01733) + | > loss_dur: 0.22889 (0.23255) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.34420 (3.62281) + | > current_lr: 0.00001 + | > step_time: 2.37640 (3.10363) + | > loader_time: 0.00490 (0.03810) + + + --> STEP: 101/234 -- GLOBAL_STEP: 9695 + | > loss: 0.16641 (0.21309) + | > log_mle: -0.09459 (-0.02020) + | > loss_dur: 0.26100 (0.23329) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.30905 (3.96983) + | > current_lr: 0.00001 + | > step_time: 2.54370 (3.06886) + | > loader_time: 0.00250 (0.03641) + + + --> STEP: 106/234 -- GLOBAL_STEP: 9700 + | > loss: 0.18526 (0.21081) + | > log_mle: -0.09473 (-0.02325) + | > loss_dur: 0.27999 (0.23406) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.65395 (4.22801) + | > current_lr: 0.00001 + | > step_time: 2.92010 (3.04506) + | > loader_time: 0.08700 (0.03902) + + + --> STEP: 111/234 -- GLOBAL_STEP: 9705 + | > loss: 0.15538 (0.20861) + | > log_mle: -0.13102 (-0.02639) + | > loss_dur: 0.28641 (0.23500) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.75504 (4.54564) + | > current_lr: 0.00001 + | > step_time: 1.42190 (2.99601) + | > loader_time: 0.00300 (0.03740) + + + --> STEP: 116/234 -- GLOBAL_STEP: 9710 + | > loss: 0.17939 (0.20686) + | > log_mle: -0.10261 (-0.02942) + | > loss_dur: 0.28200 (0.23628) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.99662 (4.78916) + | > current_lr: 0.00001 + | > step_time: 3.09510 (2.99453) + | > loader_time: 0.00340 (0.03829) + + + --> STEP: 121/234 -- GLOBAL_STEP: 9715 + | > loss: 0.23009 (0.20530) + | > log_mle: -0.02296 (-0.03160) + | > loss_dur: 0.25305 (0.23689) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.72769 (4.91348) + | > current_lr: 0.00001 + | > step_time: 2.95440 (3.06121) + | > loader_time: 0.00430 (0.04171) + + + --> STEP: 126/234 -- GLOBAL_STEP: 9720 + | > loss: 0.11609 (0.20298) + | > log_mle: -0.14905 (-0.03439) + | > loss_dur: 0.26515 (0.23736) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.29869 (5.13382) + | > current_lr: 0.00001 + | > step_time: 2.28610 (2.99862) + | > loader_time: 0.00190 (0.04139) + + + --> STEP: 131/234 -- GLOBAL_STEP: 9725 + | > loss: 0.11379 (0.20043) + | > log_mle: -0.17994 (-0.03796) + | > loss_dur: 0.29373 (0.23839) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.14151 (5.37169) + | > current_lr: 0.00001 + | > step_time: 1.78270 (2.96111) + | > loader_time: 0.00320 (0.04138) + + + --> STEP: 136/234 -- GLOBAL_STEP: 9730 + | > loss: 0.08174 (0.19787) + | > log_mle: -0.22403 (-0.04145) + | > loss_dur: 0.30577 (0.23932) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.78719 (5.65452) + | > current_lr: 0.00001 + | > step_time: 2.79940 (2.96554) + | > loader_time: 0.08810 (0.04125) + + + --> STEP: 141/234 -- GLOBAL_STEP: 9735 + | > loss: 0.14385 (0.19593) + | > log_mle: -0.14383 (-0.04454) + | > loss_dur: 0.28768 (0.24047) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.79572 (5.92469) + | > current_lr: 0.00001 + | > step_time: 6.48760 (3.05234) + | > loader_time: 0.11000 (0.04416) + + + --> STEP: 146/234 -- GLOBAL_STEP: 9740 + | > loss: 0.09885 (0.19285) + | > log_mle: -0.19056 (-0.04908) + | > loss_dur: 0.28942 (0.24193) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.06300 (6.38071) + | > current_lr: 0.00001 + | > step_time: 4.10000 (3.04586) + | > loader_time: 0.00280 (0.04526) + + + --> STEP: 151/234 -- GLOBAL_STEP: 9745 + | > loss: 0.09256 (0.18997) + | > log_mle: -0.16086 (-0.05285) + | > loss_dur: 0.25342 (0.24283) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.74166 (6.68877) + | > current_lr: 0.00001 + | > step_time: 2.00000 (3.04622) + | > loader_time: 0.00260 (0.04526) + + + --> STEP: 156/234 -- GLOBAL_STEP: 9750 + | > loss: 0.07737 (0.18631) + | > log_mle: -0.19348 (-0.05786) + | > loss_dur: 0.27085 (0.24417) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.59173 (7.14232) + | > current_lr: 0.00001 + | > step_time: 3.00160 (3.02772) + | > loader_time: 0.00270 (0.04452) + + + --> STEP: 161/234 -- GLOBAL_STEP: 9755 + | > loss: 0.07379 (0.18339) + | > log_mle: -0.21171 (-0.06211) + | > loss_dur: 0.28550 (0.24549) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.49116 (7.53341) + | > current_lr: 0.00001 + | > step_time: 2.17220 (3.05119) + | > loader_time: 0.00320 (0.04542) + + + --> STEP: 166/234 -- GLOBAL_STEP: 9760 + | > loss: 0.10505 (0.18074) + | > log_mle: -0.16214 (-0.06581) + | > loss_dur: 0.26718 (0.24654) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.35735 (7.83335) + | > current_lr: 0.00001 + | > step_time: 1.91770 (3.03234) + | > loader_time: 0.00360 (0.04468) + + + --> STEP: 171/234 -- GLOBAL_STEP: 9765 + | > loss: 0.02907 (0.17752) + | > log_mle: -0.25633 (-0.07063) + | > loss_dur: 0.28539 (0.24815) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.67145 (8.29297) + | > current_lr: 0.00001 + | > step_time: 2.78820 (3.05237) + | > loader_time: 0.11370 (0.04519) + + + --> STEP: 176/234 -- GLOBAL_STEP: 9770 + | > loss: 0.07708 (0.17429) + | > log_mle: -0.22501 (-0.07533) + | > loss_dur: 0.30210 (0.24963) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.68213 (8.75042) + | > current_lr: 0.00001 + | > step_time: 4.19890 (3.06123) + | > loader_time: 0.09960 (0.04655) + + + --> STEP: 181/234 -- GLOBAL_STEP: 9775 + | > loss: 0.12796 (0.17171) + | > log_mle: -0.17402 (-0.07957) + | > loss_dur: 0.30198 (0.25128) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.10535 (9.08603) + | > current_lr: 0.00001 + | > step_time: 5.20460 (3.07891) + | > loader_time: 0.20340 (0.04648) + + + --> STEP: 186/234 -- GLOBAL_STEP: 9780 + | > loss: 0.11023 (0.16915) + | > log_mle: -0.20803 (-0.08398) + | > loss_dur: 0.31826 (0.25312) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.10567 (9.39674) + | > current_lr: 0.00001 + | > step_time: 3.71030 (3.20379) + | > loader_time: 0.09440 (0.04838) + + + --> STEP: 191/234 -- GLOBAL_STEP: 9785 + | > loss: 0.08040 (0.16630) + | > log_mle: -0.21439 (-0.08814) + | > loss_dur: 0.29478 (0.25444) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.74414 (9.77865) + | > current_lr: 0.00001 + | > step_time: 3.40640 (3.25935) + | > loader_time: 0.09520 (0.04863) + + + --> STEP: 196/234 -- GLOBAL_STEP: 9790 + | > loss: 0.08992 (0.16345) + | > log_mle: -0.21714 (-0.09240) + | > loss_dur: 0.30706 (0.25585) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.38221 (10.08619) + | > current_lr: 0.00001 + | > step_time: 7.41150 (3.26864) + | > loader_time: 0.08350 (0.04918) + + + --> STEP: 201/234 -- GLOBAL_STEP: 9795 + | > loss: 0.12321 (0.16102) + | > log_mle: -0.19325 (-0.09624) + | > loss_dur: 0.31646 (0.25726) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.90951 (10.37870) + | > current_lr: 0.00001 + | > step_time: 2.49980 (3.38291) + | > loader_time: 0.20200 (0.05002) + + + --> STEP: 206/234 -- GLOBAL_STEP: 9800 + | > loss: 0.03220 (0.15811) + | > log_mle: -0.27956 (-0.10053) + | > loss_dur: 0.31176 (0.25865) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.47153 (10.69154) + | > current_lr: 0.00001 + | > step_time: 13.50850 (3.47425) + | > loader_time: 0.19500 (0.05065) + + + --> STEP: 211/234 -- GLOBAL_STEP: 9805 + | > loss: 0.00962 (0.15502) + | > log_mle: -0.34362 (-0.10533) + | > loss_dur: 0.35324 (0.26035) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.99782 (11.03526) + | > current_lr: 0.00001 + | > step_time: 6.90690 (3.51380) + | > loader_time: 0.08430 (0.05175) + + + --> STEP: 216/234 -- GLOBAL_STEP: 9810 + | > loss: -0.00278 (0.15188) + | > log_mle: -0.33133 (-0.10990) + | > loss_dur: 0.32855 (0.26178) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.35064 (11.35348) + | > current_lr: 0.00001 + | > step_time: 5.39790 (3.57976) + | > loader_time: 0.30230 (0.05336) + + + --> STEP: 221/234 -- GLOBAL_STEP: 9815 + | > loss: 0.05575 (0.14884) + | > log_mle: -0.25861 (-0.11443) + | > loss_dur: 0.31436 (0.26327) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.73821 (11.62575) + | > current_lr: 0.00001 + | > step_time: 2.59530 (3.58893) + | > loader_time: 0.00390 (0.05666) + + + --> STEP: 226/234 -- GLOBAL_STEP: 9820 + | > loss: -0.01013 (0.14566) + | > log_mle: -0.34671 (-0.11940) + | > loss_dur: 0.33658 (0.26506) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.79537 (12.05810) + | > current_lr: 0.00001 + | > step_time: 0.24180 (3.53907) + | > loader_time: 0.00330 (0.05549) + + + --> STEP: 231/234 -- GLOBAL_STEP: 9825 + | > loss: 0.07613 (0.14340) + | > log_mle: -0.40216 (-0.12460) + | > loss_dur: 0.47828 (0.26800) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.78293 (12.50293) + | > current_lr: 0.00001 + | > step_time: 0.28780 (3.46806) + | > loader_time: 0.00460 (0.05438) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55582 (-0.07353) + | > avg_loss: 0.10997 (-0.00683) + | > avg_log_mle: -0.18441 (-0.00314) + | > avg_loss_dur: 0.29437 (-0.00369) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_9828.pth + + > EPOCH: 42/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 18:30:43)  + + --> STEP: 2/234 -- GLOBAL_STEP: 9830 + | > loss: 0.37559 (0.32260) + | > log_mle: 0.03592 (0.02701) + | > loss_dur: 0.33967 (0.29559) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.74849 (1.74344) + | > current_lr: 0.00001 + | > step_time: 6.69280 (4.59716) + | > loader_time: 0.00070 (0.00328) + + + --> STEP: 7/234 -- GLOBAL_STEP: 9835 + | > loss: 0.22941 (0.27862) + | > log_mle: -0.01601 (0.00573) + | > loss_dur: 0.24542 (0.27289) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.47209 (2.51359) + | > current_lr: 0.00001 + | > step_time: 2.01430 (6.19814) + | > loader_time: 0.11150 (0.04417) + + + --> STEP: 12/234 -- GLOBAL_STEP: 9840 + | > loss: 0.23096 (0.25826) + | > log_mle: -0.00415 (-0.00105) + | > loss_dur: 0.23511 (0.25931) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30130 (2.56600) + | > current_lr: 0.00001 + | > step_time: 1.09130 (4.12847) + | > loader_time: 0.00100 (0.02638) + + + --> STEP: 17/234 -- GLOBAL_STEP: 9845 + | > loss: 0.24472 (0.25237) + | > log_mle: 0.00642 (-0.00209) + | > loss_dur: 0.23830 (0.25446) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.29953 (2.49320) + | > current_lr: 0.00001 + | > step_time: 1.30280 (3.33109) + | > loader_time: 0.00260 (0.02387) + + + --> STEP: 22/234 -- GLOBAL_STEP: 9850 + | > loss: 0.18132 (0.24441) + | > log_mle: -0.01713 (-0.00277) + | > loss_dur: 0.19845 (0.24717) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.85331 (2.35996) + | > current_lr: 0.00001 + | > step_time: 1.85570 (2.94114) + | > loader_time: 0.00190 (0.01954) + + + --> STEP: 27/234 -- GLOBAL_STEP: 9855 + | > loss: 0.17656 (0.23759) + | > log_mle: -0.02050 (-0.00403) + | > loss_dur: 0.19706 (0.24162) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.84847 (2.25370) + | > current_lr: 0.00001 + | > step_time: 5.00440 (2.78894) + | > loader_time: 0.09200 (0.02755) + + + --> STEP: 32/234 -- GLOBAL_STEP: 9860 + | > loss: 0.17753 (0.23139) + | > log_mle: -0.03573 (-0.00628) + | > loss_dur: 0.21326 (0.23767) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.51933 (2.19379) + | > current_lr: 0.00001 + | > step_time: 2.01770 (2.63802) + | > loader_time: 0.00320 (0.02678) + + + --> STEP: 37/234 -- GLOBAL_STEP: 9865 + | > loss: 0.18104 (0.22914) + | > log_mle: -0.01576 (-0.00769) + | > loss_dur: 0.19680 (0.23683) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.69966 (2.37047) + | > current_lr: 0.00001 + | > step_time: 0.98890 (2.59459) + | > loader_time: 0.00310 (0.02866) + + + --> STEP: 42/234 -- GLOBAL_STEP: 9870 + | > loss: 0.24468 (0.22866) + | > log_mle: 0.00267 (-0.00839) + | > loss_dur: 0.24202 (0.23705) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.03020 (2.38782) + | > current_lr: 0.00001 + | > step_time: 1.60070 (2.49033) + | > loader_time: 0.08660 (0.02758) + + + --> STEP: 47/234 -- GLOBAL_STEP: 9875 + | > loss: 0.20345 (0.22527) + | > log_mle: -0.01820 (-0.00993) + | > loss_dur: 0.22165 (0.23520) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.31420 (2.38954) + | > current_lr: 0.00001 + | > step_time: 1.54130 (2.46242) + | > loader_time: 0.00200 (0.02911) + + + --> STEP: 52/234 -- GLOBAL_STEP: 9880 + | > loss: 0.22100 (0.22220) + | > log_mle: -0.00496 (-0.01004) + | > loss_dur: 0.22596 (0.23224) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.65495 (2.38705) + | > current_lr: 0.00001 + | > step_time: 1.28490 (2.37812) + | > loader_time: 0.00310 (0.02982) + + + --> STEP: 57/234 -- GLOBAL_STEP: 9885 + | > loss: 0.21815 (0.22089) + | > log_mle: -0.00652 (-0.01128) + | > loss_dur: 0.22467 (0.23217) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.84062 (2.35270) + | > current_lr: 0.00001 + | > step_time: 2.62330 (2.29555) + | > loader_time: 0.07730 (0.03013) + + + --> STEP: 62/234 -- GLOBAL_STEP: 9890 + | > loss: 0.18235 (0.21758) + | > log_mle: -0.06323 (-0.01342) + | > loss_dur: 0.24559 (0.23100) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.39301 (2.53430) + | > current_lr: 0.00001 + | > step_time: 1.10010 (2.26812) + | > loader_time: 0.00270 (0.03314) + + + --> STEP: 67/234 -- GLOBAL_STEP: 9895 + | > loss: 0.17508 (0.21595) + | > log_mle: -0.04683 (-0.01432) + | > loss_dur: 0.22191 (0.23028) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.59428 (2.54963) + | > current_lr: 0.00001 + | > step_time: 1.79770 (2.22550) + | > loader_time: 0.09510 (0.03340) + + + --> STEP: 72/234 -- GLOBAL_STEP: 9900 + | > loss: 0.19246 (0.21432) + | > log_mle: -0.02914 (-0.01535) + | > loss_dur: 0.22160 (0.22967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.89120 (2.77501) + | > current_lr: 0.00001 + | > step_time: 2.59530 (2.22366) + | > loader_time: 0.01070 (0.03266) + + + --> STEP: 77/234 -- GLOBAL_STEP: 9905 + | > loss: 0.16542 (0.21205) + | > log_mle: -0.04291 (-0.01690) + | > loss_dur: 0.20833 (0.22895) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.94938 (2.86022) + | > current_lr: 0.00001 + | > step_time: 3.22830 (2.21564) + | > loader_time: 0.08950 (0.03309) + + + --> STEP: 82/234 -- GLOBAL_STEP: 9910 + | > loss: 0.17458 (0.21039) + | > log_mle: -0.02873 (-0.01781) + | > loss_dur: 0.20331 (0.22820) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.87374 (2.89931) + | > current_lr: 0.00001 + | > step_time: 1.59580 (2.19490) + | > loader_time: 0.00170 (0.03319) + + + --> STEP: 87/234 -- GLOBAL_STEP: 9915 + | > loss: 0.18826 (0.20862) + | > log_mle: -0.04445 (-0.01937) + | > loss_dur: 0.23271 (0.22799) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.07598 (3.03770) + | > current_lr: 0.00001 + | > step_time: 2.48100 (2.15843) + | > loader_time: 0.02060 (0.03252) + + + --> STEP: 92/234 -- GLOBAL_STEP: 9920 + | > loss: 0.13303 (0.20572) + | > log_mle: -0.08283 (-0.02228) + | > loss_dur: 0.21586 (0.22800) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.59450 (3.32158) + | > current_lr: 0.00001 + | > step_time: 4.58020 (2.16040) + | > loader_time: 0.09950 (0.03284) + + + --> STEP: 97/234 -- GLOBAL_STEP: 9925 + | > loss: 0.15829 (0.20283) + | > log_mle: -0.07645 (-0.02602) + | > loss_dur: 0.23474 (0.22886) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.17188 (3.73460) + | > current_lr: 0.00001 + | > step_time: 2.21150 (2.12253) + | > loader_time: 0.10320 (0.03405) + + + --> STEP: 102/234 -- GLOBAL_STEP: 9930 + | > loss: 0.18901 (0.20096) + | > log_mle: -0.05475 (-0.02865) + | > loss_dur: 0.24376 (0.22962) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.43571 (3.99705) + | > current_lr: 0.00001 + | > step_time: 1.98390 (2.13747) + | > loader_time: 0.00230 (0.03588) + + + --> STEP: 107/234 -- GLOBAL_STEP: 9935 + | > loss: 0.14982 (0.19845) + | > log_mle: -0.10295 (-0.03209) + | > loss_dur: 0.25277 (0.23054) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.07214 (4.42096) + | > current_lr: 0.00001 + | > step_time: 0.71830 (2.10884) + | > loader_time: 0.07350 (0.03641) + + + --> STEP: 112/234 -- GLOBAL_STEP: 9940 + | > loss: 0.15293 (0.19634) + | > log_mle: -0.10816 (-0.03521) + | > loss_dur: 0.26109 (0.23155) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.97000 (4.73838) + | > current_lr: 0.00001 + | > step_time: 1.89660 (2.13143) + | > loader_time: 0.00330 (0.03671) + + + --> STEP: 117/234 -- GLOBAL_STEP: 9945 + | > loss: 0.15430 (0.19472) + | > log_mle: -0.09891 (-0.03810) + | > loss_dur: 0.25321 (0.23283) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.95335 (4.96550) + | > current_lr: 0.00001 + | > step_time: 1.15610 (2.14524) + | > loader_time: 0.00200 (0.03667) + + + --> STEP: 122/234 -- GLOBAL_STEP: 9950 + | > loss: 0.14621 (0.19319) + | > log_mle: -0.08588 (-0.04012) + | > loss_dur: 0.23209 (0.23332) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.46874 (5.08467) + | > current_lr: 0.00001 + | > step_time: 2.83910 (2.12535) + | > loader_time: 0.09510 (0.03659) + + + --> STEP: 127/234 -- GLOBAL_STEP: 9955 + | > loss: 0.12953 (0.19079) + | > log_mle: -0.13047 (-0.04320) + | > loss_dur: 0.26000 (0.23399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.31838 (5.37603) + | > current_lr: 0.00001 + | > step_time: 1.51760 (2.11328) + | > loader_time: 0.07480 (0.03595) + + + --> STEP: 132/234 -- GLOBAL_STEP: 9960 + | > loss: 0.12479 (0.18835) + | > log_mle: -0.11385 (-0.04654) + | > loss_dur: 0.23864 (0.23489) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.67616 (5.66824) + | > current_lr: 0.00001 + | > step_time: 3.58510 (2.10482) + | > loader_time: 0.02260 (0.03575) + + + --> STEP: 137/234 -- GLOBAL_STEP: 9965 + | > loss: 0.16031 (0.18628) + | > log_mle: -0.12284 (-0.05005) + | > loss_dur: 0.28314 (0.23633) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.05141 (5.95192) + | > current_lr: 0.00001 + | > step_time: 2.03700 (2.09105) + | > loader_time: 0.07620 (0.03584) + + + --> STEP: 142/234 -- GLOBAL_STEP: 9970 + | > loss: 0.11899 (0.18406) + | > log_mle: -0.14193 (-0.05325) + | > loss_dur: 0.26092 (0.23731) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.48217 (6.18403) + | > current_lr: 0.00001 + | > step_time: 1.58260 (2.09624) + | > loader_time: 0.00370 (0.03607) + + + --> STEP: 147/234 -- GLOBAL_STEP: 9975 + | > loss: 0.11429 (0.18110) + | > log_mle: -0.14390 (-0.05774) + | > loss_dur: 0.25819 (0.23884) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.57455 (6.58851) + | > current_lr: 0.00001 + | > step_time: 1.00870 (2.09224) + | > loader_time: 0.00270 (0.03621) + + + --> STEP: 152/234 -- GLOBAL_STEP: 9980 + | > loss: 0.08469 (0.17799) + | > log_mle: -0.20973 (-0.06191) + | > loss_dur: 0.29442 (0.23991) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.21003 (6.86745) + | > current_lr: 0.00001 + | > step_time: 2.38480 (2.09181) + | > loader_time: 0.01110 (0.03697) + + + --> STEP: 157/234 -- GLOBAL_STEP: 9985 + | > loss: 0.12026 (0.17459) + | > log_mle: -0.16492 (-0.06659) + | > loss_dur: 0.28518 (0.24118) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.60254 (7.23554) + | > current_lr: 0.00001 + | > step_time: 2.82510 (2.11126) + | > loader_time: 0.09470 (0.03761) + + + --> STEP: 162/234 -- GLOBAL_STEP: 9990 + | > loss: 0.06448 (0.17130) + | > log_mle: -0.19706 (-0.07102) + | > loss_dur: 0.26154 (0.24232) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.46894 (7.60187) + | > current_lr: 0.00001 + | > step_time: 4.49450 (2.12328) + | > loader_time: 0.01670 (0.03780) + + + --> STEP: 167/234 -- GLOBAL_STEP: 9995 + | > loss: 0.03166 (0.16841) + | > log_mle: -0.25904 (-0.07506) + | > loss_dur: 0.29070 (0.24347) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.58296 (8.05100) + | > current_lr: 0.00001 + | > step_time: 1.48700 (2.13393) + | > loader_time: 0.19520 (0.03904) + + + --> STEP: 172/234 -- GLOBAL_STEP: 10000 + | > loss: 0.05633 (0.16547) + | > log_mle: -0.25462 (-0.07978) + | > loss_dur: 0.31095 (0.24525) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.09937 (8.57805) + | > current_lr: 0.00001 + | > step_time: 6.08500 (2.20027) + | > loader_time: 0.10650 (0.04024) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_10000.pth + + --> STEP: 177/234 -- GLOBAL_STEP: 10005 + | > loss: 0.06267 (0.16236) + | > log_mle: -0.21972 (-0.08426) + | > loss_dur: 0.28239 (0.24662) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.80050 (8.96639) + | > current_lr: 0.00001 + | > step_time: 3.98350 (2.22384) + | > loader_time: 0.10830 (0.04001) + + + --> STEP: 182/234 -- GLOBAL_STEP: 10010 + | > loss: 0.06729 (0.15972) + | > log_mle: -0.25960 (-0.08870) + | > loss_dur: 0.32689 (0.24842) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.30154 (9.41471) + | > current_lr: 0.00001 + | > step_time: 3.61450 (2.30068) + | > loader_time: 0.08700 (0.04002) + + + --> STEP: 187/234 -- GLOBAL_STEP: 10015 + | > loss: 0.03920 (0.15702) + | > log_mle: -0.25989 (-0.09305) + | > loss_dur: 0.29909 (0.25006) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.95256 (9.88180) + | > current_lr: 0.00001 + | > step_time: 8.69860 (2.44178) + | > loader_time: 0.29840 (0.04275) + + + --> STEP: 192/234 -- GLOBAL_STEP: 10020 + | > loss: 0.00919 (0.15393) + | > log_mle: -0.28209 (-0.09729) + | > loss_dur: 0.29129 (0.25122) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.62809 (10.35424) + | > current_lr: 0.00001 + | > step_time: 9.21450 (2.52677) + | > loader_time: 0.09280 (0.04622) + + + --> STEP: 197/234 -- GLOBAL_STEP: 10025 + | > loss: 0.01787 (0.15116) + | > log_mle: -0.25804 (-0.10139) + | > loss_dur: 0.27591 (0.25256) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.39719 (10.71279) + | > current_lr: 0.00001 + | > step_time: 8.49780 (2.58599) + | > loader_time: 0.00480 (0.04606) + + + --> STEP: 202/234 -- GLOBAL_STEP: 10030 + | > loss: -0.02077 (0.14848) + | > log_mle: -0.33768 (-0.10560) + | > loss_dur: 0.31690 (0.25408) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.08649 (11.04389) + | > current_lr: 0.00001 + | > step_time: 3.09100 (2.68875) + | > loader_time: 0.29330 (0.04793) + + + --> STEP: 207/234 -- GLOBAL_STEP: 10035 + | > loss: -0.00662 (0.14564) + | > log_mle: -0.32479 (-0.10978) + | > loss_dur: 0.31817 (0.25541) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.81287 (11.35655) + | > current_lr: 0.00001 + | > step_time: 3.80540 (2.73633) + | > loader_time: 0.09940 (0.04859) + + + --> STEP: 212/234 -- GLOBAL_STEP: 10040 + | > loss: 0.01720 (0.14265) + | > log_mle: -0.31112 (-0.11447) + | > loss_dur: 0.32833 (0.25712) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.15895 (11.73730) + | > current_lr: 0.00001 + | > step_time: 9.29260 (2.83544) + | > loader_time: 0.00490 (0.04793) + + + --> STEP: 217/234 -- GLOBAL_STEP: 10045 + | > loss: 0.00097 (0.13945) + | > log_mle: -0.32547 (-0.11905) + | > loss_dur: 0.32644 (0.25851) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.52713 (12.17347) + | > current_lr: 0.00001 + | > step_time: 3.59040 (2.88485) + | > loader_time: 0.59500 (0.05015) + + + --> STEP: 222/234 -- GLOBAL_STEP: 10050 + | > loss: 0.01911 (0.13657) + | > log_mle: -0.33508 (-0.12355) + | > loss_dur: 0.35420 (0.26012) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.49422 (12.59148) + | > current_lr: 0.00001 + | > step_time: 2.88760 (2.92918) + | > loader_time: 0.00770 (0.05394) + + + --> STEP: 227/234 -- GLOBAL_STEP: 10055 + | > loss: 0.02134 (0.13339) + | > log_mle: -0.31784 (-0.12842) + | > loss_dur: 0.33918 (0.26182) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.82609 (12.95302) + | > current_lr: 0.00001 + | > step_time: 0.24600 (2.87310) + | > loader_time: 0.00420 (0.05284) + + + --> STEP: 232/234 -- GLOBAL_STEP: 10060 + | > loss: 0.12213 (0.13151) + | > log_mle: -0.48834 (-0.13434) + | > loss_dur: 0.61047 (0.26585) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.06366 (13.42442) + | > current_lr: 0.00001 + | > step_time: 0.32320 (2.81692) + | > loader_time: 0.00390 (0.05178) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.52950 (-0.02632) + | > avg_loss: 0.10198 (-0.00798) + | > avg_log_mle: -0.19047 (-0.00606) + | > avg_loss_dur: 0.29245 (-0.00192) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_10062.pth + + > EPOCH: 43/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 18:43:11)  + + --> STEP: 3/234 -- GLOBAL_STEP: 10065 + | > loss: 0.30783 (0.31227) + | > log_mle: -0.00186 (0.01181) + | > loss_dur: 0.30969 (0.30046) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.79701 (1.92921) + | > current_lr: 0.00001 + | > step_time: 4.19140 (4.96250) + | > loader_time: 0.00970 (0.00639) + + + --> STEP: 8/234 -- GLOBAL_STEP: 10070 + | > loss: 0.23293 (0.25877) + | > log_mle: -0.02510 (-0.00558) + | > loss_dur: 0.25803 (0.26435) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.40477 (2.93372) + | > current_lr: 0.00001 + | > step_time: 4.10490 (5.61083) + | > loader_time: 0.00370 (0.01587) + + + --> STEP: 13/234 -- GLOBAL_STEP: 10075 + | > loss: 0.23015 (0.24272) + | > log_mle: -0.00553 (-0.00946) + | > loss_dur: 0.23568 (0.25218) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.29697 (2.88069) + | > current_lr: 0.00001 + | > step_time: 1.50980 (4.78330) + | > loader_time: 0.00160 (0.01843) + + + --> STEP: 18/234 -- GLOBAL_STEP: 10080 + | > loss: 0.18730 (0.23452) + | > log_mle: -0.01648 (-0.01087) + | > loss_dur: 0.20378 (0.24539) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.73489 (2.73214) + | > current_lr: 0.00001 + | > step_time: 1.78990 (4.66957) + | > loader_time: 0.00180 (0.02571) + + + --> STEP: 23/234 -- GLOBAL_STEP: 10085 + | > loss: 0.20014 (0.22904) + | > log_mle: -0.02324 (-0.01169) + | > loss_dur: 0.22338 (0.24073) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36977 (2.62626) + | > current_lr: 0.00001 + | > step_time: 4.30190 (4.37217) + | > loader_time: 0.00140 (0.02803) + + + --> STEP: 28/234 -- GLOBAL_STEP: 10090 + | > loss: 0.17196 (0.22295) + | > log_mle: -0.01698 (-0.01254) + | > loss_dur: 0.18895 (0.23548) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.94001 (2.52746) + | > current_lr: 0.00001 + | > step_time: 3.00730 (4.53175) + | > loader_time: 0.09200 (0.05793) + + + --> STEP: 33/234 -- GLOBAL_STEP: 10095 + | > loss: 0.22487 (0.21892) + | > log_mle: -0.00720 (-0.01434) + | > loss_dur: 0.23207 (0.23326) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.01736 (2.45128) + | > current_lr: 0.00001 + | > step_time: 1.18700 (4.43902) + | > loader_time: 0.00350 (0.06004) + + + --> STEP: 38/234 -- GLOBAL_STEP: 10100 + | > loss: 0.21667 (0.21671) + | > log_mle: -0.03029 (-0.01625) + | > loss_dur: 0.24696 (0.23297) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.00118 (2.67594) + | > current_lr: 0.00001 + | > step_time: 1.43450 (4.07288) + | > loader_time: 0.00150 (0.05464) + + + --> STEP: 43/234 -- GLOBAL_STEP: 10105 + | > loss: 0.18476 (0.21505) + | > log_mle: -0.03046 (-0.01679) + | > loss_dur: 0.21522 (0.23183) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.76470 (2.67627) + | > current_lr: 0.00001 + | > step_time: 1.38920 (3.79638) + | > loader_time: 0.00250 (0.05043) + + + --> STEP: 48/234 -- GLOBAL_STEP: 10110 + | > loss: 0.16840 (0.21123) + | > log_mle: -0.02105 (-0.01801) + | > loss_dur: 0.18944 (0.22924) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.42138 (2.63052) + | > current_lr: 0.00001 + | > step_time: 1.31780 (3.63485) + | > loader_time: 0.00340 (0.04549) + + + --> STEP: 53/234 -- GLOBAL_STEP: 10115 + | > loss: 0.20778 (0.20922) + | > log_mle: -0.04040 (-0.01847) + | > loss_dur: 0.24818 (0.22769) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.96244 (2.58644) + | > current_lr: 0.00001 + | > step_time: 1.11520 (3.44232) + | > loader_time: 0.07500 (0.04431) + + + --> STEP: 58/234 -- GLOBAL_STEP: 10120 + | > loss: 0.19851 (0.20784) + | > log_mle: -0.02373 (-0.01938) + | > loss_dur: 0.22224 (0.22722) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.86239 (2.51957) + | > current_lr: 0.00001 + | > step_time: 1.30850 (3.29721) + | > loader_time: 0.00280 (0.04215) + + + --> STEP: 63/234 -- GLOBAL_STEP: 10125 + | > loss: 0.19086 (0.20459) + | > log_mle: -0.03997 (-0.02176) + | > loss_dur: 0.23083 (0.22635) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.20601 (2.59810) + | > current_lr: 0.00001 + | > step_time: 1.39370 (3.16740) + | > loader_time: 0.00240 (0.04162) + + + --> STEP: 68/234 -- GLOBAL_STEP: 10130 + | > loss: 0.18414 (0.20242) + | > log_mle: -0.02908 (-0.02244) + | > loss_dur: 0.21322 (0.22486) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.47694 (2.68370) + | > current_lr: 0.00001 + | > step_time: 1.69490 (3.03287) + | > loader_time: 0.00210 (0.04262) + + + --> STEP: 73/234 -- GLOBAL_STEP: 10135 + | > loss: 0.15553 (0.20102) + | > log_mle: -0.05521 (-0.02363) + | > loss_dur: 0.21073 (0.22465) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.10810 (3.43783) + | > current_lr: 0.00001 + | > step_time: 1.86870 (2.96190) + | > loader_time: 0.00250 (0.04234) + + + --> STEP: 78/234 -- GLOBAL_STEP: 10140 + | > loss: 0.19913 (0.19924) + | > log_mle: -0.02834 (-0.02474) + | > loss_dur: 0.22746 (0.22398) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.83003 (3.50381) + | > current_lr: 0.00001 + | > step_time: 1.17640 (2.87728) + | > loader_time: 0.00250 (0.04095) + + + --> STEP: 83/234 -- GLOBAL_STEP: 10145 + | > loss: 0.17506 (0.19729) + | > log_mle: -0.05665 (-0.02595) + | > loss_dur: 0.23171 (0.22324) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.53102 (3.56149) + | > current_lr: 0.00001 + | > step_time: 1.61220 (2.81038) + | > loader_time: 0.00330 (0.03868) + + + --> STEP: 88/234 -- GLOBAL_STEP: 10150 + | > loss: 0.12604 (0.19529) + | > log_mle: -0.09576 (-0.02789) + | > loss_dur: 0.22179 (0.22317) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.51716 (3.69584) + | > current_lr: 0.00001 + | > step_time: 1.50840 (2.77804) + | > loader_time: 0.09020 (0.03856) + + + --> STEP: 93/234 -- GLOBAL_STEP: 10155 + | > loss: 0.13285 (0.19206) + | > log_mle: -0.10906 (-0.03083) + | > loss_dur: 0.24191 (0.22289) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.62416 (3.98433) + | > current_lr: 0.00001 + | > step_time: 1.60100 (2.76862) + | > loader_time: 0.00360 (0.03858) + + + --> STEP: 98/234 -- GLOBAL_STEP: 10160 + | > loss: 0.20634 (0.18992) + | > log_mle: -0.03664 (-0.03376) + | > loss_dur: 0.24298 (0.22367) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.04708 (4.23595) + | > current_lr: 0.00001 + | > step_time: 1.31490 (2.76112) + | > loader_time: 0.07810 (0.03841) + + + --> STEP: 103/234 -- GLOBAL_STEP: 10165 + | > loss: 0.12045 (0.18726) + | > log_mle: -0.12563 (-0.03719) + | > loss_dur: 0.24608 (0.22445) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.85285 (4.57599) + | > current_lr: 0.00001 + | > step_time: 2.19300 (2.73577) + | > loader_time: 0.00260 (0.03739) + + + --> STEP: 108/234 -- GLOBAL_STEP: 10170 + | > loss: 0.15880 (0.18538) + | > log_mle: -0.07727 (-0.04009) + | > loss_dur: 0.23606 (0.22547) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.56567 (4.81730) + | > current_lr: 0.00001 + | > step_time: 1.58610 (2.71848) + | > loader_time: 0.00180 (0.03807) + + + --> STEP: 113/234 -- GLOBAL_STEP: 10175 + | > loss: 0.12592 (0.18317) + | > log_mle: -0.12425 (-0.04358) + | > loss_dur: 0.25017 (0.22675) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.04101 (5.15054) + | > current_lr: 0.00001 + | > step_time: 1.81310 (2.71775) + | > loader_time: 0.00310 (0.03747) + + + --> STEP: 118/234 -- GLOBAL_STEP: 10180 + | > loss: 0.14223 (0.18153) + | > log_mle: -0.09328 (-0.04616) + | > loss_dur: 0.23551 (0.22769) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.11440 (5.31780) + | > current_lr: 0.00001 + | > step_time: 1.60970 (2.68359) + | > loader_time: 0.08570 (0.03673) + + + --> STEP: 123/234 -- GLOBAL_STEP: 10185 + | > loss: 0.15239 (0.18021) + | > log_mle: -0.06508 (-0.04790) + | > loss_dur: 0.21746 (0.22811) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.08838 (5.41079) + | > current_lr: 0.00001 + | > step_time: 1.40100 (2.66662) + | > loader_time: 0.00320 (0.03540) + + + --> STEP: 128/234 -- GLOBAL_STEP: 10190 + | > loss: 0.10752 (0.17747) + | > log_mle: -0.12209 (-0.05143) + | > loss_dur: 0.22961 (0.22891) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.13161 (5.65773) + | > current_lr: 0.00001 + | > step_time: 3.99780 (2.71887) + | > loader_time: 0.10640 (0.03640) + + + --> STEP: 133/234 -- GLOBAL_STEP: 10195 + | > loss: 0.12609 (0.17503) + | > log_mle: -0.14101 (-0.05493) + | > loss_dur: 0.26710 (0.22996) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.89293 (5.88552) + | > current_lr: 0.00001 + | > step_time: 1.68460 (2.71204) + | > loader_time: 0.00240 (0.03645) + + + --> STEP: 138/234 -- GLOBAL_STEP: 10200 + | > loss: 0.14053 (0.17307) + | > log_mle: -0.10361 (-0.05816) + | > loss_dur: 0.24413 (0.23122) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.83533 (6.06506) + | > current_lr: 0.00001 + | > step_time: 4.80780 (2.70450) + | > loader_time: 0.08330 (0.03717) + + + --> STEP: 143/234 -- GLOBAL_STEP: 10205 + | > loss: 0.08549 (0.17056) + | > log_mle: -0.21977 (-0.06211) + | > loss_dur: 0.30526 (0.23267) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.98001 (6.48131) + | > current_lr: 0.00001 + | > step_time: 2.89730 (2.73303) + | > loader_time: 0.00210 (0.03918) + + + --> STEP: 148/234 -- GLOBAL_STEP: 10210 + | > loss: 0.09399 (0.16775) + | > log_mle: -0.15139 (-0.06609) + | > loss_dur: 0.24538 (0.23384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.73526 (6.78127) + | > current_lr: 0.00001 + | > step_time: 3.41210 (2.83135) + | > loader_time: 0.78740 (0.04795) + + + --> STEP: 153/234 -- GLOBAL_STEP: 10215 + | > loss: 0.02069 (0.16422) + | > log_mle: -0.25616 (-0.07088) + | > loss_dur: 0.27686 (0.23510) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.16086 (7.13540) + | > current_lr: 0.00001 + | > step_time: 6.49170 (2.96302) + | > loader_time: 0.20260 (0.05022) + + + --> STEP: 158/234 -- GLOBAL_STEP: 10220 + | > loss: 0.08280 (0.16127) + | > log_mle: -0.20316 (-0.07515) + | > loss_dur: 0.28596 (0.23642) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.97093 (7.55785) + | > current_lr: 0.00001 + | > step_time: 2.20030 (2.93070) + | > loader_time: 0.10420 (0.05098) + + + --> STEP: 163/234 -- GLOBAL_STEP: 10225 + | > loss: 0.10070 (0.15815) + | > log_mle: -0.17958 (-0.07939) + | > loss_dur: 0.28028 (0.23754) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.91976 (7.90282) + | > current_lr: 0.00001 + | > step_time: 1.41410 (3.07343) + | > loader_time: 0.08730 (0.05113) + + + --> STEP: 168/234 -- GLOBAL_STEP: 10230 + | > loss: 0.08592 (0.15529) + | > log_mle: -0.22123 (-0.08365) + | > loss_dur: 0.30715 (0.23894) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.36527 (8.28588) + | > current_lr: 0.00001 + | > step_time: 4.10630 (3.07376) + | > loader_time: 0.00290 (0.05065) + + + --> STEP: 173/234 -- GLOBAL_STEP: 10235 + | > loss: 0.04389 (0.15215) + | > log_mle: -0.22837 (-0.08838) + | > loss_dur: 0.27226 (0.24053) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.53998 (8.64766) + | > current_lr: 0.00001 + | > step_time: 7.78850 (3.14892) + | > loader_time: 0.30300 (0.05216) + + + --> STEP: 178/234 -- GLOBAL_STEP: 10240 + | > loss: 0.03357 (0.14899) + | > log_mle: -0.28394 (-0.09312) + | > loss_dur: 0.31751 (0.24211) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.99586 (9.02376) + | > current_lr: 0.00001 + | > step_time: 2.19050 (3.12036) + | > loader_time: 0.00300 (0.05125) + + + --> STEP: 183/234 -- GLOBAL_STEP: 10245 + | > loss: 0.02722 (0.14648) + | > log_mle: -0.27762 (-0.09743) + | > loss_dur: 0.30484 (0.24391) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.27364 (9.59283) + | > current_lr: 0.00001 + | > step_time: 4.60500 (3.14888) + | > loader_time: 0.10300 (0.05050) + + + --> STEP: 188/234 -- GLOBAL_STEP: 10250 + | > loss: 0.01995 (0.14381) + | > log_mle: -0.28918 (-0.10175) + | > loss_dur: 0.30913 (0.24556) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.20690 (10.07014) + | > current_lr: 0.00001 + | > step_time: 3.90250 (3.19974) + | > loader_time: 0.00740 (0.05024) + + + --> STEP: 193/234 -- GLOBAL_STEP: 10255 + | > loss: 0.02948 (0.14104) + | > log_mle: -0.28804 (-0.10593) + | > loss_dur: 0.31752 (0.24698) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.44686 (10.50300) + | > current_lr: 0.00001 + | > step_time: 7.71030 (3.22948) + | > loader_time: 0.18370 (0.05287) + + + --> STEP: 198/234 -- GLOBAL_STEP: 10260 + | > loss: 0.02172 (0.13831) + | > log_mle: -0.27738 (-0.10993) + | > loss_dur: 0.29910 (0.24824) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.85994 (10.83349) + | > current_lr: 0.00001 + | > step_time: 2.89840 (3.32911) + | > loader_time: 0.08690 (0.05308) + + + --> STEP: 203/234 -- GLOBAL_STEP: 10265 + | > loss: 0.06462 (0.13582) + | > log_mle: -0.22719 (-0.11383) + | > loss_dur: 0.29181 (0.24965) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.19175 (11.18431) + | > current_lr: 0.00001 + | > step_time: 3.09970 (3.38323) + | > loader_time: 0.09440 (0.05516) + + + --> STEP: 208/234 -- GLOBAL_STEP: 10270 + | > loss: 0.02661 (0.13290) + | > log_mle: -0.29200 (-0.11829) + | > loss_dur: 0.31861 (0.25118) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.43517 (11.65611) + | > current_lr: 0.00001 + | > step_time: 5.31270 (3.38660) + | > loader_time: 0.09600 (0.05479) + + + --> STEP: 213/234 -- GLOBAL_STEP: 10275 + | > loss: -0.00669 (0.12978) + | > log_mle: -0.33723 (-0.12315) + | > loss_dur: 0.33054 (0.25294) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.40228 (12.10098) + | > current_lr: 0.00001 + | > step_time: 5.09410 (3.45680) + | > loader_time: 0.00450 (0.05408) + + + --> STEP: 218/234 -- GLOBAL_STEP: 10280 + | > loss: 0.02371 (0.12669) + | > log_mle: -0.29927 (-0.12755) + | > loss_dur: 0.32298 (0.25425) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.50932 (12.49235) + | > current_lr: 0.00001 + | > step_time: 5.10680 (3.47895) + | > loader_time: 0.09610 (0.05335) + + + --> STEP: 223/234 -- GLOBAL_STEP: 10285 + | > loss: -0.01656 (0.12370) + | > log_mle: -0.33903 (-0.13218) + | > loss_dur: 0.32247 (0.25588) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.31342 (13.04030) + | > current_lr: 0.00001 + | > step_time: 2.49710 (3.47267) + | > loader_time: 0.00550 (0.05302) + + + --> STEP: 228/234 -- GLOBAL_STEP: 10290 + | > loss: 0.02127 (0.12064) + | > log_mle: -0.34181 (-0.13702) + | > loss_dur: 0.36307 (0.25767) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.37337 (13.48593) + | > current_lr: 0.00001 + | > step_time: 0.29960 (3.40568) + | > loader_time: 0.00460 (0.05193) + + + --> STEP: 233/234 -- GLOBAL_STEP: 10295 + | > loss: 0.64901 (0.12160) + | > log_mle: -0.29658 (-0.14269) + | > loss_dur: 0.94559 (0.26430) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.79666 (14.07063) + | > current_lr: 0.00001 + | > step_time: 0.19360 (3.33847) + | > loader_time: 0.00350 (0.05093) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12315 (-0.40635) + | > avg_loss: 0.07668 (-0.02530) + | > avg_log_mle: -0.21329 (-0.02282) + | > avg_loss_dur: 0.28997 (-0.00248) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_10296.pth + + > EPOCH: 44/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 18:57:33)  + + --> STEP: 4/234 -- GLOBAL_STEP: 10300 + | > loss: 0.29480 (0.29664) + | > log_mle: -0.02821 (-0.00465) + | > loss_dur: 0.32301 (0.30129) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.29801 (2.96520) + | > current_lr: 0.00001 + | > step_time: 11.79590 (6.90113) + | > loader_time: 0.90450 (0.29858) + + + --> STEP: 9/234 -- GLOBAL_STEP: 10305 + | > loss: 0.20741 (0.24822) + | > log_mle: -0.03681 (-0.01660) + | > loss_dur: 0.24422 (0.26483) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.46869 (2.66765) + | > current_lr: 0.00001 + | > step_time: 7.00860 (5.40337) + | > loader_time: 0.00190 (0.15499) + + + --> STEP: 14/234 -- GLOBAL_STEP: 10310 + | > loss: 0.22825 (0.23401) + | > log_mle: -0.02960 (-0.01861) + | > loss_dur: 0.25784 (0.25262) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.06783 (2.58980) + | > current_lr: 0.00001 + | > step_time: 1.48770 (3.97000) + | > loader_time: 0.00510 (0.10049) + + + --> STEP: 19/234 -- GLOBAL_STEP: 10315 + | > loss: 0.23923 (0.22769) + | > log_mle: -0.01644 (-0.01886) + | > loss_dur: 0.25567 (0.24655) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.25767 (2.51030) + | > current_lr: 0.00001 + | > step_time: 9.19600 (4.30376) + | > loader_time: 0.10420 (0.08537) + + + --> STEP: 24/234 -- GLOBAL_STEP: 10320 + | > loss: 0.19466 (0.21851) + | > log_mle: -0.01951 (-0.01966) + | > loss_dur: 0.21417 (0.23817) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.01709 (2.40517) + | > current_lr: 0.00001 + | > step_time: 1.89740 (3.68469) + | > loader_time: 0.00170 (0.07124) + + + --> STEP: 29/234 -- GLOBAL_STEP: 10325 + | > loss: 0.17636 (0.21175) + | > log_mle: -0.01799 (-0.02030) + | > loss_dur: 0.19435 (0.23206) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.79853 (2.33643) + | > current_lr: 0.00001 + | > step_time: 1.20170 (3.31466) + | > loader_time: 0.00330 (0.06249) + + + --> STEP: 34/234 -- GLOBAL_STEP: 10330 + | > loss: 0.21677 (0.20868) + | > log_mle: -0.03237 (-0.02248) + | > loss_dur: 0.24914 (0.23116) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.17942 (2.28874) + | > current_lr: 0.00001 + | > step_time: 2.00750 (3.06903) + | > loader_time: 0.00100 (0.05361) + + + --> STEP: 39/234 -- GLOBAL_STEP: 10335 + | > loss: 0.18838 (0.20540) + | > log_mle: -0.03762 (-0.02438) + | > loss_dur: 0.22599 (0.22978) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.33154 (2.52120) + | > current_lr: 0.00001 + | > step_time: 1.69020 (2.83385) + | > loader_time: 0.00200 (0.04897) + + + --> STEP: 44/234 -- GLOBAL_STEP: 10340 + | > loss: 0.16769 (0.20369) + | > log_mle: -0.03227 (-0.02471) + | > loss_dur: 0.19997 (0.22840) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.35426 (2.49660) + | > current_lr: 0.00001 + | > step_time: 1.50130 (2.66504) + | > loader_time: 0.00340 (0.04371) + + + --> STEP: 49/234 -- GLOBAL_STEP: 10345 + | > loss: 0.14961 (0.19994) + | > log_mle: -0.03887 (-0.02597) + | > loss_dur: 0.18848 (0.22591) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.26509 (2.47373) + | > current_lr: 0.00001 + | > step_time: 1.58650 (2.56276) + | > loader_time: 0.00250 (0.03950) + + + --> STEP: 54/234 -- GLOBAL_STEP: 10350 + | > loss: 0.16957 (0.19855) + | > log_mle: -0.04268 (-0.02642) + | > loss_dur: 0.21225 (0.22497) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.62357 (2.44149) + | > current_lr: 0.00001 + | > step_time: 0.91210 (2.54042) + | > loader_time: 0.00250 (0.04113) + + + --> STEP: 59/234 -- GLOBAL_STEP: 10355 + | > loss: 0.13277 (0.19680) + | > log_mle: -0.05903 (-0.02752) + | > loss_dur: 0.19180 (0.22433) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.75905 (2.43397) + | > current_lr: 0.00001 + | > step_time: 0.99850 (2.53909) + | > loader_time: 0.00320 (0.03921) + + + --> STEP: 64/234 -- GLOBAL_STEP: 10360 + | > loss: 0.16732 (0.19423) + | > log_mle: -0.02887 (-0.02931) + | > loss_dur: 0.19619 (0.22354) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.42272 (2.51944) + | > current_lr: 0.00001 + | > step_time: 1.98230 (2.47814) + | > loader_time: 0.00610 (0.03776) + + + --> STEP: 69/234 -- GLOBAL_STEP: 10365 + | > loss: 0.20531 (0.19285) + | > log_mle: -0.01743 (-0.02982) + | > loss_dur: 0.22274 (0.22268) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.13157 (2.56328) + | > current_lr: 0.00001 + | > step_time: 1.20450 (2.40866) + | > loader_time: 0.07470 (0.03624) + + + --> STEP: 74/234 -- GLOBAL_STEP: 10370 + | > loss: 0.15746 (0.19086) + | > log_mle: -0.04290 (-0.03144) + | > loss_dur: 0.20036 (0.22230) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.04144 (2.97468) + | > current_lr: 0.00001 + | > step_time: 1.25970 (2.36759) + | > loader_time: 0.00220 (0.03398) + + + --> STEP: 79/234 -- GLOBAL_STEP: 10375 + | > loss: 0.15660 (0.18947) + | > log_mle: -0.05212 (-0.03259) + | > loss_dur: 0.20872 (0.22206) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.94880 (3.05173) + | > current_lr: 0.00001 + | > step_time: 1.90400 (2.31509) + | > loader_time: 0.00250 (0.03307) + + + --> STEP: 84/234 -- GLOBAL_STEP: 10380 + | > loss: 0.15495 (0.18750) + | > log_mle: -0.05541 (-0.03379) + | > loss_dur: 0.21037 (0.22129) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.08782 (3.13957) + | > current_lr: 0.00001 + | > step_time: 4.12500 (2.30170) + | > loader_time: 0.11130 (0.03471) + + + --> STEP: 89/234 -- GLOBAL_STEP: 10385 + | > loss: 0.13579 (0.18508) + | > log_mle: -0.08319 (-0.03598) + | > loss_dur: 0.21898 (0.22105) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.47084 (3.26330) + | > current_lr: 0.00001 + | > step_time: 1.70970 (2.28540) + | > loader_time: 0.08560 (0.03478) + + + --> STEP: 94/234 -- GLOBAL_STEP: 10390 + | > loss: 0.11532 (0.18187) + | > log_mle: -0.11844 (-0.03926) + | > loss_dur: 0.23376 (0.22113) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.45353 (3.53222) + | > current_lr: 0.00001 + | > step_time: 3.50330 (2.32566) + | > loader_time: 0.00750 (0.03579) + + + --> STEP: 99/234 -- GLOBAL_STEP: 10395 + | > loss: 0.10311 (0.17950) + | > log_mle: -0.14426 (-0.04238) + | > loss_dur: 0.24737 (0.22188) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.73433 (3.91750) + | > current_lr: 0.00001 + | > step_time: 1.59300 (2.31323) + | > loader_time: 0.00190 (0.03584) + + + --> STEP: 104/234 -- GLOBAL_STEP: 10400 + | > loss: 0.08556 (0.17692) + | > log_mle: -0.15294 (-0.04577) + | > loss_dur: 0.23849 (0.22269) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.30260 (4.31429) + | > current_lr: 0.00001 + | > step_time: 1.60330 (2.29929) + | > loader_time: 0.00270 (0.03770) + + + --> STEP: 109/234 -- GLOBAL_STEP: 10405 + | > loss: 0.15057 (0.17529) + | > log_mle: -0.12745 (-0.04834) + | > loss_dur: 0.27802 (0.22364) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.61629 (4.65814) + | > current_lr: 0.00001 + | > step_time: 1.26960 (2.28803) + | > loader_time: 0.00280 (0.03689) + + + --> STEP: 114/234 -- GLOBAL_STEP: 10410 + | > loss: 0.12882 (0.17292) + | > log_mle: -0.10888 (-0.05156) + | > loss_dur: 0.23770 (0.22448) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.78980 (4.99134) + | > current_lr: 0.00001 + | > step_time: 4.04680 (2.33297) + | > loader_time: 0.18790 (0.03855) + + + --> STEP: 119/234 -- GLOBAL_STEP: 10415 + | > loss: 0.13557 (0.17133) + | > log_mle: -0.10781 (-0.05407) + | > loss_dur: 0.24338 (0.22541) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.29484 (5.18338) + | > current_lr: 0.00001 + | > step_time: 3.20260 (2.35385) + | > loader_time: 0.09980 (0.03880) + + + --> STEP: 124/234 -- GLOBAL_STEP: 10420 + | > loss: 0.10553 (0.16982) + | > log_mle: -0.13619 (-0.05602) + | > loss_dur: 0.24171 (0.22584) + | > amp_scaler: 16384.00000 (8258.06452) + | > grad_norm: 11.35643 (5.29274) + | > current_lr: 0.00001 + | > step_time: 2.19440 (2.37591) + | > loader_time: 0.00260 (0.04019) + + + --> STEP: 129/234 -- GLOBAL_STEP: 10425 + | > loss: 0.12220 (0.16738) + | > log_mle: -0.11957 (-0.05938) + | > loss_dur: 0.24176 (0.22676) + | > amp_scaler: 16384.00000 (8573.02326) + | > grad_norm: 11.88256 (5.56293) + | > current_lr: 0.00001 + | > step_time: 4.78820 (2.41972) + | > loader_time: 0.00390 (0.04006) + + + --> STEP: 134/234 -- GLOBAL_STEP: 10430 + | > loss: 0.11261 (0.16486) + | > log_mle: -0.16741 (-0.06313) + | > loss_dur: 0.28002 (0.22799) + | > amp_scaler: 16384.00000 (8864.47761) + | > grad_norm: 14.39933 (5.91131) + | > current_lr: 0.00001 + | > step_time: 4.90010 (2.43451) + | > loader_time: 0.18690 (0.04069) + + + --> STEP: 139/234 -- GLOBAL_STEP: 10435 + | > loss: 0.03937 (0.16232) + | > log_mle: -0.22169 (-0.06668) + | > loss_dur: 0.26106 (0.22900) + | > amp_scaler: 16384.00000 (9134.96403) + | > grad_norm: 28.14657 (6.29689) + | > current_lr: 0.00001 + | > step_time: 4.90610 (2.44421) + | > loader_time: 0.09400 (0.04133) + + + --> STEP: 144/234 -- GLOBAL_STEP: 10440 + | > loss: 0.07928 (0.15996) + | > log_mle: -0.19971 (-0.07043) + | > loss_dur: 0.27899 (0.23039) + | > amp_scaler: 16384.00000 (9386.66667) + | > grad_norm: 15.94285 (6.63682) + | > current_lr: 0.00001 + | > step_time: 5.21610 (2.59154) + | > loader_time: 0.08820 (0.04379) + + + --> STEP: 149/234 -- GLOBAL_STEP: 10445 + | > loss: 0.03943 (0.15691) + | > log_mle: -0.23824 (-0.07461) + | > loss_dur: 0.27767 (0.23152) + | > amp_scaler: 16384.00000 (9621.47651) + | > grad_norm: 22.38758 (7.01405) + | > current_lr: 0.00001 + | > step_time: 7.89250 (2.69534) + | > loader_time: 0.00530 (0.04480) + + + --> STEP: 154/234 -- GLOBAL_STEP: 10450 + | > loss: 0.05830 (0.15365) + | > log_mle: -0.20448 (-0.07910) + | > loss_dur: 0.26278 (0.23275) + | > amp_scaler: 16384.00000 (9841.03896) + | > grad_norm: 18.86421 (7.48805) + | > current_lr: 0.00001 + | > step_time: 12.51650 (2.80546) + | > loader_time: 0.28690 (0.04974) + + + --> STEP: 159/234 -- GLOBAL_STEP: 10455 + | > loss: 0.05915 (0.15081) + | > log_mle: -0.22205 (-0.08341) + | > loss_dur: 0.28119 (0.23422) + | > amp_scaler: 16384.00000 (10046.79245) + | > grad_norm: 17.57255 (7.88664) + | > current_lr: 0.00001 + | > step_time: 3.31830 (2.88777) + | > loader_time: 0.08940 (0.05174) + + + --> STEP: 164/234 -- GLOBAL_STEP: 10460 + | > loss: 0.05813 (0.14791) + | > log_mle: -0.21620 (-0.08756) + | > loss_dur: 0.27433 (0.23548) + | > amp_scaler: 16384.00000 (10240.00000) + | > grad_norm: 20.49428 (8.24018) + | > current_lr: 0.00001 + | > step_time: 2.68740 (2.93432) + | > loader_time: 0.00780 (0.05275) + + + --> STEP: 169/234 -- GLOBAL_STEP: 10465 + | > loss: 0.08877 (0.14526) + | > log_mle: -0.21011 (-0.09175) + | > loss_dur: 0.29887 (0.23701) + | > amp_scaler: 16384.00000 (10421.77515) + | > grad_norm: 20.03201 (8.59399) + | > current_lr: 0.00001 + | > step_time: 12.70450 (3.05474) + | > loader_time: 0.09080 (0.05458) + + + --> STEP: 174/234 -- GLOBAL_STEP: 10470 + | > loss: -0.00673 (0.14162) + | > log_mle: -0.29512 (-0.09691) + | > loss_dur: 0.28839 (0.23853) + | > amp_scaler: 16384.00000 (10593.10345) + | > grad_norm: 26.86292 (9.02225) + | > current_lr: 0.00001 + | > step_time: 7.40770 (3.14811) + | > loader_time: 0.10480 (0.05699) + + + --> STEP: 179/234 -- GLOBAL_STEP: 10475 + | > loss: 0.03126 (0.13872) + | > log_mle: -0.27996 (-0.10149) + | > loss_dur: 0.31122 (0.24022) + | > amp_scaler: 16384.00000 (10754.86034) + | > grad_norm: 26.07356 (9.47060) + | > current_lr: 0.00001 + | > step_time: 3.71050 (3.14951) + | > loader_time: 0.00440 (0.05604) + + + --> STEP: 184/234 -- GLOBAL_STEP: 10480 + | > loss: 0.04853 (0.13614) + | > log_mle: -0.25702 (-0.10567) + | > loss_dur: 0.30555 (0.24182) + | > amp_scaler: 16384.00000 (10907.82609) + | > grad_norm: 24.14628 (9.86981) + | > current_lr: 0.00001 + | > step_time: 4.80970 (3.17972) + | > loader_time: 0.00330 (0.05564) + + + --> STEP: 189/234 -- GLOBAL_STEP: 10485 + | > loss: 0.04567 (0.13351) + | > log_mle: -0.25492 (-0.10998) + | > loss_dur: 0.30060 (0.24349) + | > amp_scaler: 16384.00000 (11052.69841) + | > grad_norm: 19.88244 (10.24544) + | > current_lr: 0.00001 + | > step_time: 4.09830 (3.20460) + | > loader_time: 0.00560 (0.05725) + + + --> STEP: 194/234 -- GLOBAL_STEP: 10490 + | > loss: 0.00873 (0.13049) + | > log_mle: -0.28394 (-0.11430) + | > loss_dur: 0.29267 (0.24479) + | > amp_scaler: 16384.00000 (11190.10309) + | > grad_norm: 38.58171 (10.65903) + | > current_lr: 0.00001 + | > step_time: 2.79200 (3.19585) + | > loader_time: 0.00410 (0.05841) + + + --> STEP: 199/234 -- GLOBAL_STEP: 10495 + | > loss: 0.00485 (0.12776) + | > log_mle: -0.29456 (-0.11830) + | > loss_dur: 0.29941 (0.24606) + | > amp_scaler: 16384.00000 (11320.60302) + | > grad_norm: 25.32513 (11.04057) + | > current_lr: 0.00001 + | > step_time: 2.20660 (3.17491) + | > loader_time: 0.10120 (0.05888) + + + --> STEP: 204/234 -- GLOBAL_STEP: 10500 + | > loss: 0.01148 (0.12532) + | > log_mle: -0.32178 (-0.12231) + | > loss_dur: 0.33326 (0.24763) + | > amp_scaler: 16384.00000 (11444.70588) + | > grad_norm: 21.43480 (11.33906) + | > current_lr: 0.00001 + | > step_time: 3.40180 (3.18393) + | > loader_time: 0.08930 (0.06027) + + + --> STEP: 209/234 -- GLOBAL_STEP: 10505 + | > loss: 0.02722 (0.12253) + | > log_mle: -0.28205 (-0.12654) + | > loss_dur: 0.30927 (0.24906) + | > amp_scaler: 16384.00000 (11562.87081) + | > grad_norm: 24.53471 (11.74560) + | > current_lr: 0.00001 + | > step_time: 4.60680 (3.22266) + | > loader_time: 0.00980 (0.06030) + + + --> STEP: 214/234 -- GLOBAL_STEP: 10510 + | > loss: -0.01821 (0.11922) + | > log_mle: -0.31208 (-0.13150) + | > loss_dur: 0.29387 (0.25072) + | > amp_scaler: 16384.00000 (11675.51402) + | > grad_norm: 27.05002 (12.16971) + | > current_lr: 0.00001 + | > step_time: 8.70030 (3.30574) + | > loader_time: 0.11260 (0.06124) + + + --> STEP: 219/234 -- GLOBAL_STEP: 10515 + | > loss: -0.06425 (0.11596) + | > log_mle: -0.39180 (-0.13623) + | > loss_dur: 0.32755 (0.25219) + | > amp_scaler: 16384.00000 (11783.01370) + | > grad_norm: 35.05234 (12.60790) + | > current_lr: 0.00001 + | > step_time: 1.92480 (3.37278) + | > loader_time: 0.29190 (0.06393) + + + --> STEP: 224/234 -- GLOBAL_STEP: 10520 + | > loss: -0.02933 (0.11315) + | > log_mle: -0.35425 (-0.14066) + | > loss_dur: 0.32492 (0.25381) + | > amp_scaler: 16384.00000 (11885.71429) + | > grad_norm: 33.79273 (13.01495) + | > current_lr: 0.00001 + | > step_time: 0.23690 (3.30933) + | > loader_time: 0.00370 (0.06257) + + + --> STEP: 229/234 -- GLOBAL_STEP: 10525 + | > loss: 0.02696 (0.11035) + | > log_mle: -0.37314 (-0.14554) + | > loss_dur: 0.40010 (0.25588) + | > amp_scaler: 16384.00000 (11983.93013) + | > grad_norm: 43.62575 (13.50103) + | > current_lr: 0.00001 + | > step_time: 0.26200 (3.24260) + | > loader_time: 0.00610 (0.06129) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.01720 (-0.10595) + | > avg_loss: 0.09481 (+0.01813) + | > avg_log_mle: -0.18991 (+0.02338) + | > avg_loss_dur: 0.28471 (-0.00525) + + + > EPOCH: 45/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 19:11:15)  + + --> STEP: 0/234 -- GLOBAL_STEP: 10530 + | > loss: 0.26349 (0.26349) + | > log_mle: -0.03921 (-0.03921) + | > loss_dur: 0.30270 (0.30270) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.14751 (3.14751) + | > current_lr: 0.00001 + | > step_time: 8.49470 (8.49468) + | > loader_time: 15.88150 (15.88145) + + + --> STEP: 5/234 -- GLOBAL_STEP: 10535 + | > loss: 0.19675 (0.26999) + | > log_mle: -0.02764 (-0.01528) + | > loss_dur: 0.22439 (0.28527) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41084 (2.60921) + | > current_lr: 0.00001 + | > step_time: 2.92170 (6.88270) + | > loader_time: 0.00120 (0.07789) + + + --> STEP: 10/234 -- GLOBAL_STEP: 10540 + | > loss: 0.18050 (0.23413) + | > log_mle: -0.03995 (-0.02553) + | > loss_dur: 0.22045 (0.25966) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.13103 (2.73031) + | > current_lr: 0.00001 + | > step_time: 3.09560 (5.76956) + | > loader_time: 0.09310 (0.06798) + + + --> STEP: 15/234 -- GLOBAL_STEP: 10545 + | > loss: 0.19890 (0.22411) + | > log_mle: -0.03155 (-0.02633) + | > loss_dur: 0.23045 (0.25044) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.21203 (2.63770) + | > current_lr: 0.00001 + | > step_time: 1.11360 (4.95220) + | > loader_time: 0.00120 (0.05773) + + + --> STEP: 20/234 -- GLOBAL_STEP: 10550 + | > loss: 0.19284 (0.21435) + | > log_mle: -0.02543 (-0.02619) + | > loss_dur: 0.21827 (0.24055) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.91621 (2.55329) + | > current_lr: 0.00001 + | > step_time: 5.29200 (4.39603) + | > loader_time: 0.00540 (0.04399) + + + --> STEP: 25/234 -- GLOBAL_STEP: 10555 + | > loss: 0.21515 (0.20774) + | > log_mle: -0.01874 (-0.02664) + | > loss_dur: 0.23389 (0.23438) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30670 (2.46055) + | > current_lr: 0.00001 + | > step_time: 0.91000 (3.77272) + | > loader_time: 0.00110 (0.03837) + + + --> STEP: 30/234 -- GLOBAL_STEP: 10560 + | > loss: 0.15679 (0.20075) + | > log_mle: -0.04921 (-0.02830) + | > loss_dur: 0.20600 (0.22905) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.06747 (2.37766) + | > current_lr: 0.00001 + | > step_time: 1.11030 (3.38026) + | > loader_time: 0.00240 (0.03233) + + + --> STEP: 35/234 -- GLOBAL_STEP: 10565 + | > loss: 0.16912 (0.19815) + | > log_mle: -0.04768 (-0.03016) + | > loss_dur: 0.21680 (0.22831) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.57061 (2.68440) + | > current_lr: 0.00001 + | > step_time: 3.98320 (3.43389) + | > loader_time: 0.00650 (0.03309) + + + --> STEP: 40/234 -- GLOBAL_STEP: 10570 + | > loss: 0.20945 (0.19637) + | > log_mle: -0.02652 (-0.03135) + | > loss_dur: 0.23597 (0.22772) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.40394 (2.67925) + | > current_lr: 0.00001 + | > step_time: 1.70410 (3.25193) + | > loader_time: 0.08610 (0.03343) + + + --> STEP: 45/234 -- GLOBAL_STEP: 10575 + | > loss: 0.15211 (0.19309) + | > log_mle: -0.05807 (-0.03236) + | > loss_dur: 0.21018 (0.22545) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.73847 (2.63507) + | > current_lr: 0.00001 + | > step_time: 0.90660 (3.09592) + | > loader_time: 0.00330 (0.03361) + + + --> STEP: 50/234 -- GLOBAL_STEP: 10580 + | > loss: 0.16977 (0.19010) + | > log_mle: -0.03164 (-0.03303) + | > loss_dur: 0.20141 (0.22312) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.82224 (2.56821) + | > current_lr: 0.00001 + | > step_time: 2.00770 (2.92287) + | > loader_time: 0.08140 (0.03205) + + + --> STEP: 55/234 -- GLOBAL_STEP: 10585 + | > loss: 0.16097 (0.18846) + | > log_mle: -0.05306 (-0.03381) + | > loss_dur: 0.21402 (0.22227) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.14124 (2.53513) + | > current_lr: 0.00001 + | > step_time: 1.22320 (2.78772) + | > loader_time: 0.09890 (0.03266) + + + --> STEP: 60/234 -- GLOBAL_STEP: 10590 + | > loss: 0.13349 (0.18635) + | > log_mle: -0.06875 (-0.03509) + | > loss_dur: 0.20224 (0.22144) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.19691 (2.53508) + | > current_lr: 0.00001 + | > step_time: 1.59730 (2.66800) + | > loader_time: 0.00200 (0.03147) + + + --> STEP: 65/234 -- GLOBAL_STEP: 10595 + | > loss: 0.16691 (0.18449) + | > log_mle: -0.04454 (-0.03640) + | > loss_dur: 0.21145 (0.22089) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10282 (2.64945) + | > current_lr: 0.00001 + | > step_time: 1.40080 (2.57319) + | > loader_time: 0.09020 (0.03182) + + + --> STEP: 70/234 -- GLOBAL_STEP: 10600 + | > loss: 0.17039 (0.18312) + | > log_mle: -0.05251 (-0.03698) + | > loss_dur: 0.22289 (0.22010) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.22780 (2.82653) + | > current_lr: 0.00001 + | > step_time: 1.80420 (2.48947) + | > loader_time: 0.08710 (0.03094) + + + --> STEP: 75/234 -- GLOBAL_STEP: 10605 + | > loss: 0.15940 (0.18122) + | > log_mle: -0.06063 (-0.03866) + | > loss_dur: 0.22003 (0.21988) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.84736 (3.14550) + | > current_lr: 0.00001 + | > step_time: 1.53350 (2.43429) + | > loader_time: 0.00220 (0.03012) + + + --> STEP: 80/234 -- GLOBAL_STEP: 10610 + | > loss: 0.15174 (0.17949) + | > log_mle: -0.04043 (-0.03954) + | > loss_dur: 0.19216 (0.21904) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30764 (3.16066) + | > current_lr: 0.00001 + | > step_time: 1.31220 (2.37425) + | > loader_time: 0.08450 (0.03046) + + + --> STEP: 85/234 -- GLOBAL_STEP: 10615 + | > loss: 0.15118 (0.17758) + | > log_mle: -0.05643 (-0.04091) + | > loss_dur: 0.20761 (0.21849) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.48955 (3.25560) + | > current_lr: 0.00001 + | > step_time: 2.29560 (2.33111) + | > loader_time: 0.00230 (0.03074) + + + --> STEP: 90/234 -- GLOBAL_STEP: 10620 + | > loss: 0.12558 (0.17483) + | > log_mle: -0.08876 (-0.04335) + | > loss_dur: 0.21434 (0.21818) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.58727 (3.60411) + | > current_lr: 0.00001 + | > step_time: 2.50360 (2.29063) + | > loader_time: 0.00360 (0.03021) + + + --> STEP: 95/234 -- GLOBAL_STEP: 10625 + | > loss: 0.08708 (0.17146) + | > log_mle: -0.16651 (-0.04732) + | > loss_dur: 0.25360 (0.21878) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.06250 (4.06927) + | > current_lr: 0.00001 + | > step_time: 1.40590 (2.26161) + | > loader_time: 0.00480 (0.02968) + + + --> STEP: 100/234 -- GLOBAL_STEP: 10630 + | > loss: 0.12419 (0.16964) + | > log_mle: -0.10073 (-0.04966) + | > loss_dur: 0.22492 (0.21930) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.98024 (4.26626) + | > current_lr: 0.00001 + | > step_time: 2.31010 (2.26563) + | > loader_time: 0.00380 (0.02836) + + + --> STEP: 105/234 -- GLOBAL_STEP: 10635 + | > loss: 0.13785 (0.16731) + | > log_mle: -0.06918 (-0.05272) + | > loss_dur: 0.20703 (0.22003) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.84757 (4.58553) + | > current_lr: 0.00001 + | > step_time: 1.29440 (2.23792) + | > loader_time: 0.00350 (0.02794) + + + --> STEP: 110/234 -- GLOBAL_STEP: 10640 + | > loss: 0.10973 (0.16554) + | > log_mle: -0.09911 (-0.05552) + | > loss_dur: 0.20884 (0.22106) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.88776 (4.92336) + | > current_lr: 0.00001 + | > step_time: 2.02870 (2.20806) + | > loader_time: 0.00320 (0.02842) + + + --> STEP: 115/234 -- GLOBAL_STEP: 10645 + | > loss: 0.13564 (0.16342) + | > log_mle: -0.11601 (-0.05883) + | > loss_dur: 0.25165 (0.22226) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.00816 (5.21459) + | > current_lr: 0.00001 + | > step_time: 2.41810 (2.20493) + | > loader_time: 0.00590 (0.02879) + + + --> STEP: 120/234 -- GLOBAL_STEP: 10650 + | > loss: 0.08897 (0.16160) + | > log_mle: -0.16166 (-0.06168) + | > loss_dur: 0.25063 (0.22328) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.44668 (5.44903) + | > current_lr: 0.00001 + | > step_time: 1.21300 (2.17803) + | > loader_time: 0.08840 (0.02912) + + + --> STEP: 125/234 -- GLOBAL_STEP: 10655 + | > loss: 0.10141 (0.16002) + | > log_mle: -0.15077 (-0.06345) + | > loss_dur: 0.25218 (0.22347) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.54790 (5.56470) + | > current_lr: 0.00001 + | > step_time: 1.30400 (2.15235) + | > loader_time: 0.00270 (0.02878) + + + --> STEP: 130/234 -- GLOBAL_STEP: 10660 + | > loss: 0.09418 (0.15751) + | > log_mle: -0.16040 (-0.06681) + | > loss_dur: 0.25458 (0.22432) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.88595 (5.80390) + | > current_lr: 0.00001 + | > step_time: 1.68410 (2.15116) + | > loader_time: 0.00560 (0.02851) + + + --> STEP: 135/234 -- GLOBAL_STEP: 10665 + | > loss: 0.11647 (0.15536) + | > log_mle: -0.09712 (-0.07005) + | > loss_dur: 0.21359 (0.22541) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.88274 (6.04121) + | > current_lr: 0.00001 + | > step_time: 1.40700 (2.13002) + | > loader_time: 0.00310 (0.02822) + + + --> STEP: 140/234 -- GLOBAL_STEP: 10670 + | > loss: 0.12208 (0.15292) + | > log_mle: -0.12764 (-0.07376) + | > loss_dur: 0.24972 (0.22668) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.48349 (6.37930) + | > current_lr: 0.00001 + | > step_time: 1.66270 (2.13441) + | > loader_time: 0.00250 (0.02796) + + + --> STEP: 145/234 -- GLOBAL_STEP: 10675 + | > loss: 0.04666 (0.15012) + | > log_mle: -0.21207 (-0.07802) + | > loss_dur: 0.25873 (0.22814) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.60542 (6.76940) + | > current_lr: 0.00001 + | > step_time: 2.30480 (2.12482) + | > loader_time: 0.00310 (0.02711) + + + --> STEP: 150/234 -- GLOBAL_STEP: 10680 + | > loss: 0.08038 (0.14729) + | > log_mle: -0.19839 (-0.08203) + | > loss_dur: 0.27878 (0.22933) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.74459 (7.09046) + | > current_lr: 0.00001 + | > step_time: 1.36440 (2.12789) + | > loader_time: 0.00220 (0.02637) + + + --> STEP: 155/234 -- GLOBAL_STEP: 10685 + | > loss: 0.02779 (0.14371) + | > log_mle: -0.25969 (-0.08688) + | > loss_dur: 0.28748 (0.23059) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.92084 (7.46238) + | > current_lr: 0.00001 + | > step_time: 2.50920 (2.14834) + | > loader_time: 0.09360 (0.02687) + + + --> STEP: 160/234 -- GLOBAL_STEP: 10690 + | > loss: 0.02393 (0.14082) + | > log_mle: -0.25161 (-0.09109) + | > loss_dur: 0.27554 (0.23191) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.81357 (7.92610) + | > current_lr: 0.00001 + | > step_time: 1.10660 (2.15007) + | > loader_time: 0.00360 (0.02731) + + + --> STEP: 165/234 -- GLOBAL_STEP: 10695 + | > loss: 0.05018 (0.13798) + | > log_mle: -0.25031 (-0.09514) + | > loss_dur: 0.30049 (0.23312) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.60989 (8.34981) + | > current_lr: 0.00001 + | > step_time: 1.30040 (2.14816) + | > loader_time: 0.00790 (0.02761) + + + --> STEP: 170/234 -- GLOBAL_STEP: 10700 + | > loss: 0.02596 (0.13524) + | > log_mle: -0.28394 (-0.09948) + | > loss_dur: 0.30991 (0.23471) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.08951 (8.67077) + | > current_lr: 0.00001 + | > step_time: 1.80940 (2.15407) + | > loader_time: 0.08860 (0.02889) + + + --> STEP: 175/234 -- GLOBAL_STEP: 10705 + | > loss: 0.02968 (0.13154) + | > log_mle: -0.25879 (-0.10446) + | > loss_dur: 0.28846 (0.23600) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.11228 (9.04707) + | > current_lr: 0.00001 + | > step_time: 3.59910 (2.18156) + | > loader_time: 0.18600 (0.03139) + + + --> STEP: 180/234 -- GLOBAL_STEP: 10710 + | > loss: 0.01754 (0.12854) + | > log_mle: -0.26981 (-0.10908) + | > loss_dur: 0.28735 (0.23761) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.48520 (9.40900) + | > current_lr: 0.00001 + | > step_time: 9.11220 (2.26552) + | > loader_time: 0.08760 (0.03203) + + + --> STEP: 185/234 -- GLOBAL_STEP: 10715 + | > loss: 0.02918 (0.12605) + | > log_mle: -0.28754 (-0.11333) + | > loss_dur: 0.31671 (0.23937) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.68750 (9.86668) + | > current_lr: 0.00001 + | > step_time: 7.80710 (2.33959) + | > loader_time: 0.00700 (0.03431) + + + --> STEP: 190/234 -- GLOBAL_STEP: 10720 + | > loss: 0.01242 (0.12328) + | > log_mle: -0.27448 (-0.11752) + | > loss_dur: 0.28690 (0.24080) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.67717 (10.22587) + | > current_lr: 0.00001 + | > step_time: 5.18540 (2.42692) + | > loader_time: 0.00340 (0.03507) + + + --> STEP: 195/234 -- GLOBAL_STEP: 10725 + | > loss: 0.02858 (0.12031) + | > log_mle: -0.28124 (-0.12182) + | > loss_dur: 0.30982 (0.24213) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.70199 (10.79266) + | > current_lr: 0.00001 + | > step_time: 2.89910 (2.49135) + | > loader_time: 0.00310 (0.03616) + + + --> STEP: 200/234 -- GLOBAL_STEP: 10730 + | > loss: 0.03103 (0.11764) + | > log_mle: -0.28855 (-0.12581) + | > loss_dur: 0.31958 (0.24346) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.22698 (11.21016) + | > current_lr: 0.00001 + | > step_time: 1.70620 (2.50169) + | > loader_time: 0.00430 (0.03719) + + + --> STEP: 205/234 -- GLOBAL_STEP: 10735 + | > loss: 0.00307 (0.11503) + | > log_mle: -0.27836 (-0.12970) + | > loss_dur: 0.28144 (0.24473) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.46953 (11.65787) + | > current_lr: 0.00001 + | > step_time: 2.70850 (2.50110) + | > loader_time: 0.19150 (0.03782) + + + --> STEP: 210/234 -- GLOBAL_STEP: 10740 + | > loss: -0.03251 (0.11206) + | > log_mle: -0.35331 (-0.13425) + | > loss_dur: 0.32080 (0.24631) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.20975 (12.09729) + | > current_lr: 0.00001 + | > step_time: 3.20450 (2.52529) + | > loader_time: 0.08930 (0.03794) + + + --> STEP: 215/234 -- GLOBAL_STEP: 10745 + | > loss: -0.00968 (0.10893) + | > log_mle: -0.30117 (-0.13892) + | > loss_dur: 0.29148 (0.24785) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.99119 (12.62220) + | > current_lr: 0.00001 + | > step_time: 3.40590 (2.58245) + | > loader_time: 0.00380 (0.03762) + + + --> STEP: 220/234 -- GLOBAL_STEP: 10750 + | > loss: -0.03028 (0.10562) + | > log_mle: -0.34638 (-0.14382) + | > loss_dur: 0.31610 (0.24943) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.06528 (13.11706) + | > current_lr: 0.00001 + | > step_time: 7.89290 (2.64777) + | > loader_time: 0.09740 (0.04260) + + + --> STEP: 225/234 -- GLOBAL_STEP: 10755 + | > loss: -0.06643 (0.10272) + | > log_mle: -0.40271 (-0.14847) + | > loss_dur: 0.33628 (0.25119) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.67534 (13.50563) + | > current_lr: 0.00001 + | > step_time: 0.23730 (2.62065) + | > loader_time: 0.00480 (0.04177) + + + --> STEP: 230/234 -- GLOBAL_STEP: 10760 + | > loss: -0.03322 (0.10019) + | > log_mle: -0.44299 (-0.15345) + | > loss_dur: 0.40977 (0.25364) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.47261 (14.21637) + | > current_lr: 0.00001 + | > step_time: 0.25280 (2.56893) + | > loader_time: 0.00440 (0.04095) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07792 (+0.06072) + | > avg_loss: 0.05533 (-0.03948) + | > avg_log_mle: -0.22867 (-0.03876) + | > avg_loss_dur: 0.28400 (-0.00072) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_10764.pth + + > EPOCH: 46/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 19:22:19)  + + --> STEP: 1/234 -- GLOBAL_STEP: 10765 + | > loss: 0.23602 (0.23602) + | > log_mle: -0.01290 (-0.01290) + | > loss_dur: 0.24892 (0.24892) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.94840 (1.94840) + | > current_lr: 0.00001 + | > step_time: 3.50850 (3.50852) + | > loader_time: 0.00150 (0.00151) + + + --> STEP: 6/234 -- GLOBAL_STEP: 10770 + | > loss: 0.21542 (0.25209) + | > log_mle: -0.02177 (-0.02192) + | > loss_dur: 0.23720 (0.27401) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.33348 (2.70322) + | > current_lr: 0.00001 + | > step_time: 7.60500 (5.83911) + | > loader_time: 0.19580 (0.08323) + + + --> STEP: 11/234 -- GLOBAL_STEP: 10775 + | > loss: 0.19629 (0.22039) + | > log_mle: -0.02951 (-0.03169) + | > loss_dur: 0.22580 (0.25208) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.88277 (2.77583) + | > current_lr: 0.00001 + | > step_time: 0.77090 (4.23182) + | > loader_time: 0.00120 (0.07026) + + + --> STEP: 16/234 -- GLOBAL_STEP: 10780 + | > loss: 0.17900 (0.20862) + | > log_mle: -0.04071 (-0.03326) + | > loss_dur: 0.21971 (0.24189) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.66926 (2.65332) + | > current_lr: 0.00001 + | > step_time: 1.07330 (3.23111) + | > loader_time: 0.00210 (0.04880) + + + --> STEP: 21/234 -- GLOBAL_STEP: 10785 + | > loss: 0.17162 (0.20343) + | > log_mle: -0.02493 (-0.03237) + | > loss_dur: 0.19655 (0.23580) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.52354 (2.55392) + | > current_lr: 0.00001 + | > step_time: 1.11550 (2.72601) + | > loader_time: 0.09160 (0.04185) + + + --> STEP: 26/234 -- GLOBAL_STEP: 10790 + | > loss: 0.17366 (0.19628) + | > log_mle: -0.03952 (-0.03339) + | > loss_dur: 0.21318 (0.22967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.35849 (2.48193) + | > current_lr: 0.00001 + | > step_time: 1.64510 (2.46074) + | > loader_time: 0.00190 (0.03414) + + + --> STEP: 31/234 -- GLOBAL_STEP: 10795 + | > loss: 0.19970 (0.19182) + | > log_mle: -0.04703 (-0.03515) + | > loss_dur: 0.24672 (0.22698) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36633 (2.40043) + | > current_lr: 0.00001 + | > step_time: 1.10320 (2.28117) + | > loader_time: 0.00270 (0.02897) + + + --> STEP: 36/234 -- GLOBAL_STEP: 10800 + | > loss: 0.17422 (0.18859) + | > log_mle: -0.05302 (-0.03713) + | > loss_dur: 0.22724 (0.22572) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.15355 (2.58378) + | > current_lr: 0.00001 + | > step_time: 1.50950 (2.13875) + | > loader_time: 0.00190 (0.02523) + + + --> STEP: 41/234 -- GLOBAL_STEP: 10805 + | > loss: 0.12854 (0.18551) + | > log_mle: -0.04694 (-0.03811) + | > loss_dur: 0.17548 (0.22362) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.80522 (2.54998) + | > current_lr: 0.00001 + | > step_time: 1.25420 (2.04575) + | > loader_time: 0.00200 (0.02244) + + + --> STEP: 46/234 -- GLOBAL_STEP: 10810 + | > loss: 0.16933 (0.18323) + | > log_mle: -0.04924 (-0.03911) + | > loss_dur: 0.21858 (0.22234) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.92130 (2.52576) + | > current_lr: 0.00001 + | > step_time: 1.46870 (1.96897) + | > loader_time: 0.00190 (0.02027) + + + --> STEP: 51/234 -- GLOBAL_STEP: 10815 + | > loss: 0.15256 (0.18020) + | > log_mle: -0.03216 (-0.03941) + | > loss_dur: 0.18472 (0.21960) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.06684 (2.44901) + | > current_lr: 0.00001 + | > step_time: 1.02860 (1.92095) + | > loader_time: 0.00170 (0.02007) + + + --> STEP: 56/234 -- GLOBAL_STEP: 10820 + | > loss: 0.18620 (0.17862) + | > log_mle: -0.05318 (-0.04057) + | > loss_dur: 0.23938 (0.21919) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10085 (2.42272) + | > current_lr: 0.00001 + | > step_time: 1.66620 (1.86887) + | > loader_time: 0.00210 (0.01847) + + + --> STEP: 61/234 -- GLOBAL_STEP: 10825 + | > loss: 0.15929 (0.17590) + | > log_mle: -0.04744 (-0.04174) + | > loss_dur: 0.20673 (0.21764) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.97614 (2.41947) + | > current_lr: 0.00001 + | > step_time: 1.29930 (1.87108) + | > loader_time: 0.00200 (0.01851) + + + --> STEP: 66/234 -- GLOBAL_STEP: 10830 + | > loss: 0.15859 (0.17487) + | > log_mle: -0.03998 (-0.04289) + | > loss_dur: 0.19857 (0.21776) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.23542 (2.55180) + | > current_lr: 0.00001 + | > step_time: 1.50160 (1.86529) + | > loader_time: 0.00230 (0.01728) + + + --> STEP: 71/234 -- GLOBAL_STEP: 10835 + | > loss: 0.13830 (0.17342) + | > log_mle: -0.08538 (-0.04408) + | > loss_dur: 0.22367 (0.21750) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.95450 (2.97502) + | > current_lr: 0.00001 + | > step_time: 1.30240 (1.83487) + | > loader_time: 0.00290 (0.01626) + + + --> STEP: 76/234 -- GLOBAL_STEP: 10840 + | > loss: 0.15312 (0.17178) + | > log_mle: -0.06533 (-0.04539) + | > loss_dur: 0.21845 (0.21717) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.67648 (3.11040) + | > current_lr: 0.00001 + | > step_time: 1.80570 (1.84441) + | > loader_time: 0.09240 (0.01761) + + + --> STEP: 81/234 -- GLOBAL_STEP: 10845 + | > loss: 0.12937 (0.16970) + | > log_mle: -0.07810 (-0.04639) + | > loss_dur: 0.20746 (0.21609) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.45260 (3.22547) + | > current_lr: 0.00001 + | > step_time: 1.28960 (1.84754) + | > loader_time: 0.00240 (0.01771) + + + --> STEP: 86/234 -- GLOBAL_STEP: 10850 + | > loss: 0.13500 (0.16829) + | > log_mle: -0.07890 (-0.04770) + | > loss_dur: 0.21390 (0.21599) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.11675 (3.33822) + | > current_lr: 0.00001 + | > step_time: 2.62060 (1.85909) + | > loader_time: 0.00300 (0.01895) + + + --> STEP: 91/234 -- GLOBAL_STEP: 10855 + | > loss: 0.14244 (0.16597) + | > log_mle: -0.08507 (-0.05013) + | > loss_dur: 0.22751 (0.21610) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.17474 (3.59808) + | > current_lr: 0.00001 + | > step_time: 1.79130 (1.84664) + | > loader_time: 0.09640 (0.02001) + + + --> STEP: 96/234 -- GLOBAL_STEP: 10860 + | > loss: 0.13371 (0.16243) + | > log_mle: -0.07646 (-0.05397) + | > loss_dur: 0.21017 (0.21640) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.89968 (3.97735) + | > current_lr: 0.00001 + | > step_time: 3.50600 (1.86105) + | > loader_time: 0.09130 (0.02089) + + + --> STEP: 101/234 -- GLOBAL_STEP: 10865 + | > loss: 0.10682 (0.16050) + | > log_mle: -0.13143 (-0.05685) + | > loss_dur: 0.23826 (0.21735) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.24256 (4.22520) + | > current_lr: 0.00001 + | > step_time: 1.49490 (1.88271) + | > loader_time: 0.00250 (0.01999) + + + --> STEP: 106/234 -- GLOBAL_STEP: 10870 + | > loss: 0.12984 (0.15836) + | > log_mle: -0.12923 (-0.05986) + | > loss_dur: 0.25907 (0.21822) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.72901 (4.60256) + | > current_lr: 0.00001 + | > step_time: 4.19740 (1.89757) + | > loader_time: 0.00350 (0.01999) + + + --> STEP: 111/234 -- GLOBAL_STEP: 10875 + | > loss: 0.09685 (0.15641) + | > log_mle: -0.16551 (-0.06291) + | > loss_dur: 0.26236 (0.21932) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.71592 (5.04219) + | > current_lr: 0.00001 + | > step_time: 1.60140 (1.91951) + | > loader_time: 0.09670 (0.02329) + + + --> STEP: 116/234 -- GLOBAL_STEP: 10880 + | > loss: 0.12432 (0.15467) + | > log_mle: -0.13836 (-0.06589) + | > loss_dur: 0.26268 (0.22055) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.54239 (5.41425) + | > current_lr: 0.00001 + | > step_time: 1.31150 (1.91007) + | > loader_time: 0.09060 (0.02316) + + + --> STEP: 121/234 -- GLOBAL_STEP: 10885 + | > loss: 0.16794 (0.15313) + | > log_mle: -0.05920 (-0.06804) + | > loss_dur: 0.22714 (0.22117) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.05036 (5.56820) + | > current_lr: 0.00001 + | > step_time: 2.18760 (1.90893) + | > loader_time: 0.00390 (0.02306) + + + --> STEP: 126/234 -- GLOBAL_STEP: 10890 + | > loss: 0.06790 (0.15073) + | > log_mle: -0.18457 (-0.07081) + | > loss_dur: 0.25247 (0.22154) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.27816 (5.76499) + | > current_lr: 0.00001 + | > step_time: 2.91860 (1.94540) + | > loader_time: 0.00310 (0.02528) + + + --> STEP: 131/234 -- GLOBAL_STEP: 10895 + | > loss: 0.05596 (0.14813) + | > log_mle: -0.21688 (-0.07437) + | > loss_dur: 0.27284 (0.22249) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.87773 (5.99205) + | > current_lr: 0.00001 + | > step_time: 3.00260 (1.96107) + | > loader_time: 0.00860 (0.02449) + + + --> STEP: 136/234 -- GLOBAL_STEP: 10900 + | > loss: 0.03379 (0.14574) + | > log_mle: -0.25912 (-0.07784) + | > loss_dur: 0.29291 (0.22358) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.29951 (6.42959) + | > current_lr: 0.00001 + | > step_time: 2.20520 (1.97343) + | > loader_time: 0.08320 (0.02490) + + + --> STEP: 141/234 -- GLOBAL_STEP: 10905 + | > loss: 0.09015 (0.14391) + | > log_mle: -0.17924 (-0.08091) + | > loss_dur: 0.26939 (0.22482) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.93985 (6.69845) + | > current_lr: 0.00001 + | > step_time: 1.20720 (1.96391) + | > loader_time: 0.09070 (0.02594) + + + --> STEP: 146/234 -- GLOBAL_STEP: 10910 + | > loss: 0.04228 (0.14094) + | > log_mle: -0.22513 (-0.08545) + | > loss_dur: 0.26740 (0.22639) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.10334 (7.23206) + | > current_lr: 0.00001 + | > step_time: 2.50700 (1.98087) + | > loader_time: 0.08510 (0.02629) + + + --> STEP: 151/234 -- GLOBAL_STEP: 10915 + | > loss: 0.04352 (0.13808) + | > log_mle: -0.19559 (-0.08919) + | > loss_dur: 0.23911 (0.22726) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.59245 (7.63825) + | > current_lr: 0.00001 + | > step_time: 2.50430 (1.99076) + | > loader_time: 0.00350 (0.02663) + + + --> STEP: 156/234 -- GLOBAL_STEP: 10920 + | > loss: 0.02660 (0.13437) + | > log_mle: -0.23200 (-0.09421) + | > loss_dur: 0.25860 (0.22858) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.79472 (8.08631) + | > current_lr: 0.00001 + | > step_time: 11.19840 (2.09539) + | > loader_time: 0.00380 (0.02844) + + + --> STEP: 161/234 -- GLOBAL_STEP: 10925 + | > loss: 0.01777 (0.13157) + | > log_mle: -0.24948 (-0.09847) + | > loss_dur: 0.26725 (0.23004) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.99384 (8.53588) + | > current_lr: 0.00001 + | > step_time: 2.70900 (2.09063) + | > loader_time: 0.00390 (0.02830) + + + --> STEP: 166/234 -- GLOBAL_STEP: 10930 + | > loss: 0.05534 (0.12906) + | > log_mle: -0.19744 (-0.10216) + | > loss_dur: 0.25278 (0.23122) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.90060 (8.94235) + | > current_lr: 0.00001 + | > step_time: 2.58280 (2.11251) + | > loader_time: 0.00290 (0.03040) + + + --> STEP: 171/234 -- GLOBAL_STEP: 10935 + | > loss: -0.02258 (0.12582) + | > log_mle: -0.29344 (-0.10699) + | > loss_dur: 0.27086 (0.23281) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.54416 (9.40420) + | > current_lr: 0.00001 + | > step_time: 2.80620 (2.12160) + | > loader_time: 0.00270 (0.03064) + + + --> STEP: 176/234 -- GLOBAL_STEP: 10940 + | > loss: 0.02156 (0.12262) + | > log_mle: -0.26383 (-0.11170) + | > loss_dur: 0.28539 (0.23432) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.42194 (9.89432) + | > current_lr: 0.00001 + | > step_time: 3.53300 (2.13832) + | > loader_time: 0.00330 (0.03082) + + + --> STEP: 181/234 -- GLOBAL_STEP: 10945 + | > loss: 0.07821 (0.12004) + | > log_mle: -0.20896 (-0.11595) + | > loss_dur: 0.28717 (0.23599) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.38251 (10.31442) + | > current_lr: 0.00001 + | > step_time: 5.59760 (2.15666) + | > loader_time: 0.18400 (0.03107) + + + --> STEP: 186/234 -- GLOBAL_STEP: 10950 + | > loss: 0.06519 (0.11755) + | > log_mle: -0.24345 (-0.12033) + | > loss_dur: 0.30864 (0.23788) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.79763 (10.84865) + | > current_lr: 0.00001 + | > step_time: 4.30280 (2.16975) + | > loader_time: 0.19300 (0.03138) + + + --> STEP: 191/234 -- GLOBAL_STEP: 10955 + | > loss: 0.03126 (0.11464) + | > log_mle: -0.25340 (-0.12452) + | > loss_dur: 0.28466 (0.23916) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.55969 (11.25328) + | > current_lr: 0.00001 + | > step_time: 3.59020 (2.19298) + | > loader_time: 0.00450 (0.03122) + + + --> STEP: 196/234 -- GLOBAL_STEP: 10960 + | > loss: 0.03879 (0.11179) + | > log_mle: -0.25344 (-0.12880) + | > loss_dur: 0.29223 (0.24059) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.09848 (11.67035) + | > current_lr: 0.00001 + | > step_time: 2.39330 (2.23509) + | > loader_time: 0.00450 (0.03192) + + + --> STEP: 201/234 -- GLOBAL_STEP: 10965 + | > loss: 0.06664 (0.10926) + | > log_mle: -0.22950 (-0.13266) + | > loss_dur: 0.29614 (0.24192) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.93933 (12.01141) + | > current_lr: 0.00001 + | > step_time: 3.69160 (2.29934) + | > loader_time: 0.00580 (0.03270) + + + --> STEP: 206/234 -- GLOBAL_STEP: 10970 + | > loss: -0.02794 (0.10627) + | > log_mle: -0.31772 (-0.13697) + | > loss_dur: 0.28978 (0.24324) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.54863 (12.37914) + | > current_lr: 0.00001 + | > step_time: 6.90310 (2.37806) + | > loader_time: 0.08940 (0.03536) + + + --> STEP: 211/234 -- GLOBAL_STEP: 10975 + | > loss: -0.04861 (0.10311) + | > log_mle: -0.37990 (-0.14177) + | > loss_dur: 0.33129 (0.24488) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.69489 (12.95010) + | > current_lr: 0.00001 + | > step_time: 9.39830 (2.41603) + | > loader_time: 0.17960 (0.03775) + + + --> STEP: 216/234 -- GLOBAL_STEP: 10980 + | > loss: -0.05542 (0.09991) + | > log_mle: -0.36710 (-0.14633) + | > loss_dur: 0.31168 (0.24623) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.94374 (13.46748) + | > current_lr: 0.00001 + | > step_time: 6.29390 (2.46602) + | > loader_time: 0.10360 (0.03783) + + + --> STEP: 221/234 -- GLOBAL_STEP: 10985 + | > loss: 0.00870 (0.09688) + | > log_mle: -0.29644 (-0.15087) + | > loss_dur: 0.30514 (0.24775) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.04783 (13.77834) + | > current_lr: 0.00001 + | > step_time: 4.18620 (2.54956) + | > loader_time: 0.19710 (0.04056) + + + --> STEP: 226/234 -- GLOBAL_STEP: 10990 + | > loss: -0.06043 (0.09365) + | > log_mle: -0.38238 (-0.15586) + | > loss_dur: 0.32194 (0.24951) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.03307 (14.27079) + | > current_lr: 0.00001 + | > step_time: 0.23530 (2.51670) + | > loader_time: 0.00330 (0.04012) + + + --> STEP: 231/234 -- GLOBAL_STEP: 10995 + | > loss: 0.02317 (0.09145) + | > log_mle: -0.43870 (-0.16106) + | > loss_dur: 0.46187 (0.25251) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.67445 (14.79826) + | > current_lr: 0.00001 + | > step_time: 0.27450 (2.46776) + | > loader_time: 0.00450 (0.03935) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.87620 (+0.79828) + | > avg_loss: 0.05948 (+0.00415) + | > avg_log_mle: -0.22157 (+0.00710) + | > avg_loss_dur: 0.28105 (-0.00295) + + + > EPOCH: 47/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 19:33:11)  + + --> STEP: 2/234 -- GLOBAL_STEP: 11000 + | > loss: 0.30143 (0.26193) + | > log_mle: -0.00362 (-0.01271) + | > loss_dur: 0.30505 (0.27464) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.81839 (2.04354) + | > current_lr: 0.00001 + | > step_time: 2.39720 (2.39561) + | > loader_time: 0.00110 (0.29992) + + + --> STEP: 7/234 -- GLOBAL_STEP: 11005 + | > loss: 0.15646 (0.22302) + | > log_mle: -0.05363 (-0.03270) + | > loss_dur: 0.21008 (0.25573) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.51622 (2.83841) + | > current_lr: 0.00001 + | > step_time: 1.41540 (6.55897) + | > loader_time: 0.00270 (0.57215) + + + --> STEP: 12/234 -- GLOBAL_STEP: 11010 + | > loss: 0.16611 (0.20061) + | > log_mle: -0.03945 (-0.03881) + | > loss_dur: 0.20555 (0.23942) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.59609 (3.23447) + | > current_lr: 0.00001 + | > step_time: 3.30000 (4.64633) + | > loader_time: 0.00230 (0.34360) + + + --> STEP: 17/234 -- GLOBAL_STEP: 11015 + | > loss: 0.18604 (0.19408) + | > log_mle: -0.02866 (-0.03926) + | > loss_dur: 0.21471 (0.23334) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.48886 (3.07273) + | > current_lr: 0.00001 + | > step_time: 1.40970 (3.62671) + | > loader_time: 0.00170 (0.24759) + + + --> STEP: 22/234 -- GLOBAL_STEP: 11020 + | > loss: 0.13717 (0.18619) + | > log_mle: -0.05255 (-0.03954) + | > loss_dur: 0.18971 (0.22574) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.35217 (2.92874) + | > current_lr: 0.00001 + | > step_time: 1.49730 (3.12549) + | > loader_time: 0.00210 (0.19170) + + + --> STEP: 27/234 -- GLOBAL_STEP: 11025 + | > loss: 0.14022 (0.18262) + | > log_mle: -0.05550 (-0.04050) + | > loss_dur: 0.19573 (0.22313) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.23429 (2.79937) + | > current_lr: 0.00001 + | > step_time: 0.99500 (2.82269) + | > loader_time: 0.00230 (0.15958) + + + --> STEP: 32/234 -- GLOBAL_STEP: 11030 + | > loss: 0.12868 (0.17601) + | > log_mle: -0.07069 (-0.04251) + | > loss_dur: 0.19938 (0.21852) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.33978 (2.70934) + | > current_lr: 0.00001 + | > step_time: 1.39080 (2.61253) + | > loader_time: 0.00260 (0.13768) + + + --> STEP: 37/234 -- GLOBAL_STEP: 11035 + | > loss: 0.14469 (0.17341) + | > log_mle: -0.04970 (-0.04367) + | > loss_dur: 0.19439 (0.21709) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.08793 (2.86818) + | > current_lr: 0.00001 + | > step_time: 2.20850 (2.49982) + | > loader_time: 0.00240 (0.12205) + + + --> STEP: 42/234 -- GLOBAL_STEP: 11040 + | > loss: 0.19883 (0.17319) + | > log_mle: -0.03229 (-0.04413) + | > loss_dur: 0.23113 (0.21733) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.85530 (2.88255) + | > current_lr: 0.00001 + | > step_time: 0.89940 (2.35642) + | > loader_time: 0.00190 (0.10782) + + + --> STEP: 47/234 -- GLOBAL_STEP: 11045 + | > loss: 0.13907 (0.16953) + | > log_mle: -0.05342 (-0.04556) + | > loss_dur: 0.19248 (0.21508) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.14821 (2.90863) + | > current_lr: 0.00001 + | > step_time: 1.80900 (2.30420) + | > loader_time: 0.09750 (0.10632) + + + --> STEP: 52/234 -- GLOBAL_STEP: 11050 + | > loss: 0.16986 (0.16729) + | > log_mle: -0.04025 (-0.04559) + | > loss_dur: 0.21011 (0.21288) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.45739 (2.86559) + | > current_lr: 0.00001 + | > step_time: 1.21500 (2.19683) + | > loader_time: 0.02240 (0.09668) + + + --> STEP: 57/234 -- GLOBAL_STEP: 11055 + | > loss: 0.17452 (0.16623) + | > log_mle: -0.04114 (-0.04676) + | > loss_dur: 0.21566 (0.21299) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.83330 (2.79501) + | > current_lr: 0.00001 + | > step_time: 1.90790 (2.12972) + | > loader_time: 0.00290 (0.08842) + + + --> STEP: 62/234 -- GLOBAL_STEP: 11060 + | > loss: 0.14679 (0.16324) + | > log_mle: -0.09642 (-0.04880) + | > loss_dur: 0.24321 (0.21204) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.37825 (2.93242) + | > current_lr: 0.00001 + | > step_time: 1.70320 (2.06435) + | > loader_time: 0.00300 (0.08281) + + + --> STEP: 67/234 -- GLOBAL_STEP: 11065 + | > loss: 0.13292 (0.16215) + | > log_mle: -0.08114 (-0.04963) + | > loss_dur: 0.21406 (0.21178) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.14886 (2.94987) + | > current_lr: 0.00001 + | > step_time: 1.30980 (2.04305) + | > loader_time: 0.00250 (0.07684) + + + --> STEP: 72/234 -- GLOBAL_STEP: 11070 + | > loss: 0.15928 (0.16175) + | > log_mle: -0.06289 (-0.05051) + | > loss_dur: 0.22216 (0.21226) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.81515 (3.16672) + | > current_lr: 0.00001 + | > step_time: 1.29090 (2.00940) + | > loader_time: 0.00270 (0.07420) + + + --> STEP: 77/234 -- GLOBAL_STEP: 11075 + | > loss: 0.11393 (0.15947) + | > log_mle: -0.07612 (-0.05197) + | > loss_dur: 0.19005 (0.21145) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.10014 (3.28649) + | > current_lr: 0.00001 + | > step_time: 1.30560 (1.97640) + | > loader_time: 0.08410 (0.07266) + + + --> STEP: 82/234 -- GLOBAL_STEP: 11080 + | > loss: 0.13658 (0.15800) + | > log_mle: -0.06239 (-0.05277) + | > loss_dur: 0.19898 (0.21077) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30483 (3.34333) + | > current_lr: 0.00001 + | > step_time: 1.54350 (1.95186) + | > loader_time: 0.00340 (0.07041) + + + --> STEP: 87/234 -- GLOBAL_STEP: 11085 + | > loss: 0.15105 (0.15693) + | > log_mle: -0.07740 (-0.05424) + | > loss_dur: 0.22845 (0.21116) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.05068 (3.48974) + | > current_lr: 0.00001 + | > step_time: 1.50060 (1.93503) + | > loader_time: 0.08250 (0.06747) + + + --> STEP: 92/234 -- GLOBAL_STEP: 11090 + | > loss: 0.07622 (0.15390) + | > log_mle: -0.11608 (-0.05707) + | > loss_dur: 0.19230 (0.21097) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.19428 (3.76549) + | > current_lr: 0.00001 + | > step_time: 3.21210 (1.92351) + | > loader_time: 0.08500 (0.06671) + + + --> STEP: 97/234 -- GLOBAL_STEP: 11095 + | > loss: 0.11054 (0.15112) + | > log_mle: -0.10911 (-0.06076) + | > loss_dur: 0.21965 (0.21189) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.31683 (4.18501) + | > current_lr: 0.00001 + | > step_time: 1.59730 (1.90990) + | > loader_time: 0.00290 (0.06425) + + + --> STEP: 102/234 -- GLOBAL_STEP: 11100 + | > loss: 0.14565 (0.14987) + | > log_mle: -0.08824 (-0.06334) + | > loss_dur: 0.23389 (0.21321) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.64422 (4.40354) + | > current_lr: 0.00001 + | > step_time: 2.49490 (1.89657) + | > loader_time: 0.00320 (0.06202) + + + --> STEP: 107/234 -- GLOBAL_STEP: 11105 + | > loss: 0.09530 (0.14748) + | > log_mle: -0.13453 (-0.06672) + | > loss_dur: 0.22984 (0.21420) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.86132 (4.71655) + | > current_lr: 0.00001 + | > step_time: 1.30080 (1.90523) + | > loader_time: 0.00350 (0.06176) + + + --> STEP: 112/234 -- GLOBAL_STEP: 11110 + | > loss: 0.10922 (0.14618) + | > log_mle: -0.14028 (-0.06965) + | > loss_dur: 0.24950 (0.21583) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.71101 (5.10677) + | > current_lr: 0.00001 + | > step_time: 1.08100 (1.90684) + | > loader_time: 0.00340 (0.06074) + + + --> STEP: 117/234 -- GLOBAL_STEP: 11115 + | > loss: 0.11468 (0.14464) + | > log_mle: -0.12831 (-0.07241) + | > loss_dur: 0.24298 (0.21704) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.81797 (5.43171) + | > current_lr: 0.00001 + | > step_time: 1.28430 (1.89602) + | > loader_time: 0.00260 (0.05907) + + + --> STEP: 122/234 -- GLOBAL_STEP: 11120 + | > loss: 0.09832 (0.14331) + | > log_mle: -0.11762 (-0.07426) + | > loss_dur: 0.21595 (0.21756) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.41567 (5.67042) + | > current_lr: 0.00001 + | > step_time: 1.69650 (1.92927) + | > loader_time: 0.00460 (0.06058) + + + --> STEP: 127/234 -- GLOBAL_STEP: 11125 + | > loss: 0.07948 (0.14109) + | > log_mle: -0.16385 (-0.07726) + | > loss_dur: 0.24333 (0.21834) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.73638 (6.00844) + | > current_lr: 0.00001 + | > step_time: 1.40430 (1.91249) + | > loader_time: 0.00360 (0.05833) + + + --> STEP: 132/234 -- GLOBAL_STEP: 11130 + | > loss: 0.07560 (0.13850) + | > log_mle: -0.14452 (-0.08056) + | > loss_dur: 0.22011 (0.21906) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.64052 (6.35491) + | > current_lr: 0.00001 + | > step_time: 1.89890 (1.90509) + | > loader_time: 0.00310 (0.05623) + + + --> STEP: 137/234 -- GLOBAL_STEP: 11135 + | > loss: 0.10475 (0.13647) + | > log_mle: -0.15654 (-0.08406) + | > loss_dur: 0.26129 (0.22053) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.51694 (6.68919) + | > current_lr: 0.00001 + | > step_time: 3.86680 (1.91367) + | > loader_time: 0.00250 (0.05557) + + + --> STEP: 142/234 -- GLOBAL_STEP: 11140 + | > loss: 0.06844 (0.13432) + | > log_mle: -0.17456 (-0.08722) + | > loss_dur: 0.24300 (0.22154) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.95697 (6.99471) + | > current_lr: 0.00001 + | > step_time: 0.91390 (1.90298) + | > loader_time: 0.00280 (0.05431) + + + --> STEP: 147/234 -- GLOBAL_STEP: 11145 + | > loss: 0.06918 (0.13138) + | > log_mle: -0.17478 (-0.09172) + | > loss_dur: 0.24396 (0.22310) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.50277 (7.49542) + | > current_lr: 0.00001 + | > step_time: 1.99850 (1.91295) + | > loader_time: 0.08740 (0.05315) + + + --> STEP: 152/234 -- GLOBAL_STEP: 11150 + | > loss: 0.03781 (0.12834) + | > log_mle: -0.24391 (-0.09585) + | > loss_dur: 0.28172 (0.22419) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.64047 (7.97741) + | > current_lr: 0.00001 + | > step_time: 1.40790 (1.92035) + | > loader_time: 0.00500 (0.05208) + + + --> STEP: 157/234 -- GLOBAL_STEP: 11155 + | > loss: 0.07065 (0.12511) + | > log_mle: -0.19654 (-0.10047) + | > loss_dur: 0.26718 (0.22558) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.89969 (8.61362) + | > current_lr: 0.00001 + | > step_time: 6.41330 (1.96764) + | > loader_time: 0.09060 (0.05222) + + + --> STEP: 162/234 -- GLOBAL_STEP: 11160 + | > loss: 0.01390 (0.12191) + | > log_mle: -0.23081 (-0.10489) + | > loss_dur: 0.24470 (0.22680) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.89481 (9.10514) + | > current_lr: 0.00001 + | > step_time: 3.10090 (2.07426) + | > loader_time: 0.00670 (0.05626) + + + --> STEP: 167/234 -- GLOBAL_STEP: 11165 + | > loss: -0.02808 (0.11919) + | > log_mle: -0.29637 (-0.10894) + | > loss_dur: 0.26829 (0.22813) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.36546 (9.51719) + | > current_lr: 0.00001 + | > step_time: 5.09880 (2.11581) + | > loader_time: 0.19240 (0.05736) + + + --> STEP: 172/234 -- GLOBAL_STEP: 11170 + | > loss: 0.01069 (0.11625) + | > log_mle: -0.28852 (-0.11367) + | > loss_dur: 0.29921 (0.22992) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.47072 (10.02793) + | > current_lr: 0.00001 + | > step_time: 1.90050 (2.14375) + | > loader_time: 0.00280 (0.05756) + + + --> STEP: 177/234 -- GLOBAL_STEP: 11175 + | > loss: 0.02641 (0.11326) + | > log_mle: -0.25321 (-0.11815) + | > loss_dur: 0.27962 (0.23140) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.55464 (10.46766) + | > current_lr: 0.00001 + | > step_time: 0.76510 (2.15926) + | > loader_time: 0.00250 (0.05769) + + + --> STEP: 182/234 -- GLOBAL_STEP: 11180 + | > loss: 0.02023 (0.11046) + | > log_mle: -0.29322 (-0.12260) + | > loss_dur: 0.31345 (0.23306) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.77623 (10.89343) + | > current_lr: 0.00001 + | > step_time: 3.61390 (2.17106) + | > loader_time: 0.09040 (0.05764) + + + --> STEP: 187/234 -- GLOBAL_STEP: 11185 + | > loss: -0.00619 (0.10784) + | > log_mle: -0.29444 (-0.12697) + | > loss_dur: 0.28825 (0.23480) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.58645 (11.34555) + | > current_lr: 0.00001 + | > step_time: 1.68280 (2.18286) + | > loader_time: 0.00330 (0.05662) + + + --> STEP: 192/234 -- GLOBAL_STEP: 11190 + | > loss: -0.03199 (0.10483) + | > log_mle: -0.31545 (-0.13122) + | > loss_dur: 0.28346 (0.23605) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.01749 (11.79878) + | > current_lr: 0.00001 + | > step_time: 3.10030 (2.20423) + | > loader_time: 0.00350 (0.05724) + + + --> STEP: 197/234 -- GLOBAL_STEP: 11195 + | > loss: -0.01948 (0.10212) + | > log_mle: -0.29193 (-0.13532) + | > loss_dur: 0.27246 (0.23744) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.50132 (12.25902) + | > current_lr: 0.00001 + | > step_time: 2.69630 (2.23562) + | > loader_time: 0.00320 (0.05718) + + + --> STEP: 202/234 -- GLOBAL_STEP: 11200 + | > loss: -0.06993 (0.09940) + | > log_mle: -0.37246 (-0.13953) + | > loss_dur: 0.30253 (0.23894) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.60443 (12.63823) + | > current_lr: 0.00001 + | > step_time: 3.41420 (2.26806) + | > loader_time: 0.29130 (0.05866) + + + --> STEP: 207/234 -- GLOBAL_STEP: 11205 + | > loss: -0.05539 (0.09664) + | > log_mle: -0.35948 (-0.14375) + | > loss_dur: 0.30409 (0.24038) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.15077 (12.97942) + | > current_lr: 0.00001 + | > step_time: 1.99230 (2.33892) + | > loader_time: 0.09130 (0.06050) + + + --> STEP: 212/234 -- GLOBAL_STEP: 11210 + | > loss: -0.03399 (0.09364) + | > log_mle: -0.34543 (-0.14845) + | > loss_dur: 0.31144 (0.24209) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.30400 (13.40154) + | > current_lr: 0.00001 + | > step_time: 2.30830 (2.40215) + | > loader_time: 0.09020 (0.06141) + + + --> STEP: 217/234 -- GLOBAL_STEP: 11215 + | > loss: -0.04860 (0.09043) + | > log_mle: -0.36239 (-0.15307) + | > loss_dur: 0.31379 (0.24350) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.81266 (13.79121) + | > current_lr: 0.00001 + | > step_time: 2.00130 (2.47676) + | > loader_time: 0.08530 (0.06180) + + + --> STEP: 222/234 -- GLOBAL_STEP: 11220 + | > loss: -0.04041 (0.08745) + | > log_mle: -0.37164 (-0.15763) + | > loss_dur: 0.33123 (0.24508) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.85523 (14.13059) + | > current_lr: 0.00001 + | > step_time: 0.57700 (2.47405) + | > loader_time: 0.00340 (0.06164) + + + --> STEP: 227/234 -- GLOBAL_STEP: 11225 + | > loss: -0.02724 (0.08425) + | > log_mle: -0.35235 (-0.16250) + | > loss_dur: 0.32511 (0.24675) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.99984 (14.65236) + | > current_lr: 0.00001 + | > step_time: 0.24600 (2.42484) + | > loader_time: 0.00350 (0.06037) + + + --> STEP: 232/234 -- GLOBAL_STEP: 11230 + | > loss: 0.07812 (0.08252) + | > log_mle: -0.52016 (-0.16835) + | > loss_dur: 0.59829 (0.25087) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 69.11949 (15.60290) + | > current_lr: 0.00001 + | > step_time: 0.34620 (2.37863) + | > loader_time: 0.00570 (0.05917) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.90240 (+0.02620) + | > avg_loss: 0.04018 (-0.01930) + | > avg_log_mle: -0.23723 (-0.01567) + | > avg_loss_dur: 0.27741 (-0.00363) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_11232.pth + + > EPOCH: 48/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 19:43:41)  + + --> STEP: 3/234 -- GLOBAL_STEP: 11235 + | > loss: 0.27763 (0.26331) + | > log_mle: -0.03797 (-0.02554) + | > loss_dur: 0.31560 (0.28885) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10399 (2.09877) + | > current_lr: 0.00001 + | > step_time: 4.38780 (7.79779) + | > loader_time: 0.10700 (0.03767) + + + --> STEP: 8/234 -- GLOBAL_STEP: 11240 + | > loss: 0.13796 (0.21525) + | > log_mle: -0.05932 (-0.04161) + | > loss_dur: 0.19729 (0.25686) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.47229 (3.00458) + | > current_lr: 0.00001 + | > step_time: 3.01380 (4.62598) + | > loader_time: 0.09170 (0.03833) + + + --> STEP: 13/234 -- GLOBAL_STEP: 11245 + | > loss: 0.20209 (0.19799) + | > log_mle: -0.04081 (-0.04507) + | > loss_dur: 0.24289 (0.24306) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.54987 (3.09810) + | > current_lr: 0.00001 + | > step_time: 5.90940 (5.74812) + | > loader_time: 0.00120 (0.06030) + + + --> STEP: 18/234 -- GLOBAL_STEP: 11250 + | > loss: 0.13559 (0.18705) + | > log_mle: -0.05054 (-0.04609) + | > loss_dur: 0.18613 (0.23314) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.26570 (2.90391) + | > current_lr: 0.00001 + | > step_time: 0.72270 (4.81411) + | > loader_time: 0.08700 (0.05958) + + + --> STEP: 23/234 -- GLOBAL_STEP: 11255 + | > loss: 0.14235 (0.17955) + | > log_mle: -0.05594 (-0.04646) + | > loss_dur: 0.19829 (0.22601) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10347 (2.80664) + | > current_lr: 0.00001 + | > step_time: 4.50760 (4.32407) + | > loader_time: 0.39570 (0.06424) + + + --> STEP: 28/234 -- GLOBAL_STEP: 11260 + | > loss: 0.13479 (0.17395) + | > log_mle: -0.04990 (-0.04699) + | > loss_dur: 0.18470 (0.22094) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.13855 (2.70066) + | > current_lr: 0.00001 + | > step_time: 0.94060 (3.90543) + | > loader_time: 0.00220 (0.05945) + + + --> STEP: 33/234 -- GLOBAL_STEP: 11265 + | > loss: 0.17409 (0.16970) + | > log_mle: -0.04086 (-0.04865) + | > loss_dur: 0.21495 (0.21835) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.76288 (2.59064) + | > current_lr: 0.00001 + | > step_time: 1.01840 (3.47177) + | > loader_time: 0.00200 (0.05296) + + + --> STEP: 38/234 -- GLOBAL_STEP: 11270 + | > loss: 0.17483 (0.16795) + | > log_mle: -0.06305 (-0.05036) + | > loss_dur: 0.23788 (0.21831) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.25972 (2.81116) + | > current_lr: 0.00001 + | > step_time: 0.79320 (3.16019) + | > loader_time: 0.00350 (0.04629) + + + --> STEP: 43/234 -- GLOBAL_STEP: 11275 + | > loss: 0.14081 (0.16644) + | > log_mle: -0.06399 (-0.05084) + | > loss_dur: 0.20479 (0.21728) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.80710 (2.77654) + | > current_lr: 0.00001 + | > step_time: 2.00160 (2.95503) + | > loader_time: 0.00200 (0.04308) + + + --> STEP: 48/234 -- GLOBAL_STEP: 11280 + | > loss: 0.12480 (0.16363) + | > log_mle: -0.05336 (-0.05198) + | > loss_dur: 0.17816 (0.21561) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.05517 (2.72428) + | > current_lr: 0.00001 + | > step_time: 1.40590 (2.78870) + | > loader_time: 0.09100 (0.04238) + + + --> STEP: 53/234 -- GLOBAL_STEP: 11285 + | > loss: 0.15594 (0.16188) + | > log_mle: -0.07160 (-0.05230) + | > loss_dur: 0.22754 (0.21418) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.40155 (2.68321) + | > current_lr: 0.00001 + | > step_time: 1.13060 (2.67492) + | > loader_time: 0.00210 (0.04020) + + + --> STEP: 58/234 -- GLOBAL_STEP: 11290 + | > loss: 0.13966 (0.16087) + | > log_mle: -0.05612 (-0.05311) + | > loss_dur: 0.19578 (0.21398) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.71586 (2.63217) + | > current_lr: 0.00001 + | > step_time: 3.49600 (2.59118) + | > loader_time: 0.00280 (0.03837) + + + --> STEP: 63/234 -- GLOBAL_STEP: 11295 + | > loss: 0.15609 (0.15804) + | > log_mle: -0.07148 (-0.05534) + | > loss_dur: 0.22757 (0.21338) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.95501 (2.79114) + | > current_lr: 0.00001 + | > step_time: 2.50100 (2.52637) + | > loader_time: 0.00340 (0.03695) + + + --> STEP: 68/234 -- GLOBAL_STEP: 11300 + | > loss: 0.14354 (0.15624) + | > log_mle: -0.06240 (-0.05593) + | > loss_dur: 0.20594 (0.21217) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.67721 (2.82362) + | > current_lr: 0.00001 + | > step_time: 1.20720 (2.46379) + | > loader_time: 0.08490 (0.03562) + + + --> STEP: 73/234 -- GLOBAL_STEP: 11305 + | > loss: 0.11791 (0.15506) + | > log_mle: -0.08802 (-0.05714) + | > loss_dur: 0.20593 (0.21220) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.10786 (3.09132) + | > current_lr: 0.00001 + | > step_time: 1.59770 (2.38368) + | > loader_time: 0.09750 (0.03464) + + + --> STEP: 78/234 -- GLOBAL_STEP: 11310 + | > loss: 0.14952 (0.15366) + | > log_mle: -0.05839 (-0.05810) + | > loss_dur: 0.20791 (0.21176) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.71468 (3.24236) + | > current_lr: 0.00001 + | > step_time: 1.29940 (2.31907) + | > loader_time: 0.08380 (0.03360) + + + --> STEP: 83/234 -- GLOBAL_STEP: 11315 + | > loss: 0.12343 (0.15186) + | > log_mle: -0.08954 (-0.05919) + | > loss_dur: 0.21297 (0.21105) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.05943 (3.36452) + | > current_lr: 0.00001 + | > step_time: 1.50980 (2.29164) + | > loader_time: 0.00270 (0.03275) + + + --> STEP: 88/234 -- GLOBAL_STEP: 11320 + | > loss: 0.08402 (0.15013) + | > log_mle: -0.12705 (-0.06100) + | > loss_dur: 0.21107 (0.21113) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.54177 (3.52318) + | > current_lr: 0.00001 + | > step_time: 2.69540 (2.29893) + | > loader_time: 0.00480 (0.03231) + + + --> STEP: 93/234 -- GLOBAL_STEP: 11325 + | > loss: 0.08497 (0.14719) + | > log_mle: -0.14110 (-0.06388) + | > loss_dur: 0.22607 (0.21107) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.65310 (3.80832) + | > current_lr: 0.00001 + | > step_time: 5.90190 (2.36701) + | > loader_time: 0.10500 (0.03576) + + + --> STEP: 98/234 -- GLOBAL_STEP: 11330 + | > loss: 0.15896 (0.14509) + | > log_mle: -0.06659 (-0.06672) + | > loss_dur: 0.22555 (0.21181) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.08240 (4.05308) + | > current_lr: 0.00001 + | > step_time: 1.59680 (2.32756) + | > loader_time: 0.00280 (0.03493) + + + --> STEP: 103/234 -- GLOBAL_STEP: 11335 + | > loss: 0.07569 (0.14281) + | > log_mle: -0.15917 (-0.07008) + | > loss_dur: 0.23486 (0.21289) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.13345 (4.42086) + | > current_lr: 0.00001 + | > step_time: 2.15210 (2.30360) + | > loader_time: 0.00180 (0.03339) + + + --> STEP: 108/234 -- GLOBAL_STEP: 11340 + | > loss: 0.10888 (0.14082) + | > log_mle: -0.10919 (-0.07298) + | > loss_dur: 0.21807 (0.21379) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.56978 (4.69601) + | > current_lr: 0.00001 + | > step_time: 1.71260 (2.27749) + | > loader_time: 0.08320 (0.03272) + + + --> STEP: 113/234 -- GLOBAL_STEP: 11345 + | > loss: 0.07497 (0.13846) + | > log_mle: -0.15458 (-0.07641) + | > loss_dur: 0.22955 (0.21487) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.61052 (5.10366) + | > current_lr: 0.00001 + | > step_time: 1.60480 (2.25292) + | > loader_time: 0.00300 (0.03144) + + + --> STEP: 118/234 -- GLOBAL_STEP: 11350 + | > loss: 0.09446 (0.13673) + | > log_mle: -0.12603 (-0.07898) + | > loss_dur: 0.22050 (0.21571) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.93906 (5.30506) + | > current_lr: 0.00001 + | > step_time: 3.61810 (2.26721) + | > loader_time: 0.10610 (0.03193) + + + --> STEP: 123/234 -- GLOBAL_STEP: 11355 + | > loss: 0.10935 (0.13534) + | > log_mle: -0.09780 (-0.08070) + | > loss_dur: 0.20715 (0.21603) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.66377 (5.41986) + | > current_lr: 0.00001 + | > step_time: 1.09260 (2.22917) + | > loader_time: 0.00310 (0.03075) + + + --> STEP: 128/234 -- GLOBAL_STEP: 11360 + | > loss: 0.06043 (0.13265) + | > log_mle: -0.15480 (-0.08421) + | > loss_dur: 0.21523 (0.21687) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.58580 (5.73833) + | > current_lr: 0.00001 + | > step_time: 2.37020 (2.20748) + | > loader_time: 0.08600 (0.03097) + + + --> STEP: 133/234 -- GLOBAL_STEP: 11365 + | > loss: 0.08064 (0.13030) + | > log_mle: -0.17271 (-0.08765) + | > loss_dur: 0.25335 (0.21795) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.37191 (6.18452) + | > current_lr: 0.00001 + | > step_time: 1.40930 (2.18033) + | > loader_time: 0.08590 (0.03055) + + + --> STEP: 138/234 -- GLOBAL_STEP: 11370 + | > loss: 0.08991 (0.12846) + | > log_mle: -0.13573 (-0.09084) + | > loss_dur: 0.22564 (0.21930) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.28280 (6.47286) + | > current_lr: 0.00001 + | > step_time: 2.28750 (2.17028) + | > loader_time: 0.00370 (0.03018) + + + --> STEP: 143/234 -- GLOBAL_STEP: 11375 + | > loss: 0.04491 (0.12597) + | > log_mle: -0.25236 (-0.09477) + | > loss_dur: 0.29727 (0.22073) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.99673 (6.93936) + | > current_lr: 0.00001 + | > step_time: 1.80630 (2.16324) + | > loader_time: 0.00390 (0.02982) + + + --> STEP: 148/234 -- GLOBAL_STEP: 11380 + | > loss: 0.05446 (0.12326) + | > log_mle: -0.18360 (-0.09872) + | > loss_dur: 0.23806 (0.22198) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.37704 (7.40941) + | > current_lr: 0.00001 + | > step_time: 1.29760 (2.14556) + | > loader_time: 0.09090 (0.03008) + + + --> STEP: 153/234 -- GLOBAL_STEP: 11385 + | > loss: -0.01877 (0.11993) + | > log_mle: -0.28944 (-0.10351) + | > loss_dur: 0.27067 (0.22344) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.25698 (7.87110) + | > current_lr: 0.00001 + | > step_time: 1.41130 (2.14162) + | > loader_time: 0.09780 (0.03095) + + + --> STEP: 158/234 -- GLOBAL_STEP: 11390 + | > loss: 0.03635 (0.11696) + | > log_mle: -0.23558 (-0.10778) + | > loss_dur: 0.27193 (0.22473) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.53174 (8.28151) + | > current_lr: 0.00001 + | > step_time: 2.30880 (2.13533) + | > loader_time: 0.08820 (0.03060) + + + --> STEP: 163/234 -- GLOBAL_STEP: 11395 + | > loss: 0.05671 (0.11388) + | > log_mle: -0.21199 (-0.11200) + | > loss_dur: 0.26870 (0.22587) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.51654 (8.67849) + | > current_lr: 0.00001 + | > step_time: 3.80630 (2.16147) + | > loader_time: 0.09500 (0.03191) + + + --> STEP: 168/234 -- GLOBAL_STEP: 11400 + | > loss: 0.04521 (0.11096) + | > log_mle: -0.25581 (-0.11628) + | > loss_dur: 0.30102 (0.22724) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.99667 (9.03667) + | > current_lr: 0.00001 + | > step_time: 2.91950 (2.16793) + | > loader_time: 0.00360 (0.03222) + + + --> STEP: 173/234 -- GLOBAL_STEP: 11405 + | > loss: -0.00119 (0.10776) + | > log_mle: -0.26026 (-0.12098) + | > loss_dur: 0.25907 (0.22874) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.91272 (9.67405) + | > current_lr: 0.00001 + | > step_time: 4.51660 (2.16993) + | > loader_time: 0.00380 (0.03144) + + + --> STEP: 178/234 -- GLOBAL_STEP: 11410 + | > loss: -0.02088 (0.10465) + | > log_mle: -0.31585 (-0.12568) + | > loss_dur: 0.29497 (0.23033) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.73291 (10.20060) + | > current_lr: 0.00001 + | > step_time: 4.09670 (2.17873) + | > loader_time: 0.00400 (0.03105) + + + --> STEP: 183/234 -- GLOBAL_STEP: 11415 + | > loss: -0.01105 (0.10203) + | > log_mle: -0.30753 (-0.12997) + | > loss_dur: 0.29649 (0.23200) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.98682 (10.92780) + | > current_lr: 0.00001 + | > step_time: 2.39820 (2.17494) + | > loader_time: 0.00320 (0.03078) + + + --> STEP: 188/234 -- GLOBAL_STEP: 11420 + | > loss: -0.03232 (0.09927) + | > log_mle: -0.32103 (-0.13432) + | > loss_dur: 0.28872 (0.23359) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.80182 (11.44003) + | > current_lr: 0.00001 + | > step_time: 2.30070 (2.19699) + | > loader_time: 0.00320 (0.03159) + + + --> STEP: 193/234 -- GLOBAL_STEP: 11425 + | > loss: -0.02814 (0.09638) + | > log_mle: -0.32112 (-0.13851) + | > loss_dur: 0.29299 (0.23489) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.26701 (11.97371) + | > current_lr: 0.00001 + | > step_time: 1.79860 (2.23699) + | > loader_time: 0.07990 (0.03232) + + + --> STEP: 198/234 -- GLOBAL_STEP: 11430 + | > loss: -0.02397 (0.09362) + | > log_mle: -0.30955 (-0.14250) + | > loss_dur: 0.28558 (0.23612) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.99860 (12.40005) + | > current_lr: 0.00001 + | > step_time: 2.00420 (2.24824) + | > loader_time: 0.08950 (0.03339) + + + --> STEP: 203/234 -- GLOBAL_STEP: 11435 + | > loss: 0.01530 (0.09111) + | > log_mle: -0.25973 (-0.14639) + | > loss_dur: 0.27502 (0.23750) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.28263 (12.89293) + | > current_lr: 0.00001 + | > step_time: 5.39950 (2.26379) + | > loader_time: 0.20580 (0.03498) + + + --> STEP: 208/234 -- GLOBAL_STEP: 11440 + | > loss: -0.02065 (0.08815) + | > log_mle: -0.32437 (-0.15085) + | > loss_dur: 0.30371 (0.23900) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.75277 (13.26694) + | > current_lr: 0.00001 + | > step_time: 2.99830 (2.31277) + | > loader_time: 0.00420 (0.03935) + + + --> STEP: 213/234 -- GLOBAL_STEP: 11445 + | > loss: -0.05682 (0.08497) + | > log_mle: -0.36929 (-0.15572) + | > loss_dur: 0.31247 (0.24069) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.54200 (13.66268) + | > current_lr: 0.00001 + | > step_time: 2.19230 (2.40265) + | > loader_time: 0.00430 (0.04125) + + + --> STEP: 218/234 -- GLOBAL_STEP: 11450 + | > loss: -0.01423 (0.08198) + | > log_mle: -0.33366 (-0.16014) + | > loss_dur: 0.31943 (0.24212) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.73468 (14.04763) + | > current_lr: 0.00001 + | > step_time: 9.18670 (2.51041) + | > loader_time: 0.00400 (0.04173) + + + --> STEP: 223/234 -- GLOBAL_STEP: 11455 + | > loss: -0.05485 (0.07900) + | > log_mle: -0.37039 (-0.16479) + | > loss_dur: 0.31554 (0.24378) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.91562 (14.58856) + | > current_lr: 0.00001 + | > step_time: 0.24030 (2.49653) + | > loader_time: 0.00340 (0.04175) + + + --> STEP: 228/234 -- GLOBAL_STEP: 11460 + | > loss: -0.03533 (0.07596) + | > log_mle: -0.37300 (-0.16961) + | > loss_dur: 0.33767 (0.24557) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.49067 (15.12914) + | > current_lr: 0.00001 + | > step_time: 0.25420 (2.44729) + | > loader_time: 0.00570 (0.04094) + + + --> STEP: 233/234 -- GLOBAL_STEP: 11465 + | > loss: 0.63625 (0.07700) + | > log_mle: -0.32931 (-0.17530) + | > loss_dur: 0.96556 (0.25230) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.38371 (15.77044) + | > current_lr: 0.00001 + | > step_time: 0.19380 (2.40070) + | > loader_time: 0.00340 (0.04015) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05126 (-0.85114) + | > avg_loss: 0.03099 (-0.00919) + | > avg_log_mle: -0.24803 (-0.01080) + | > avg_loss_dur: 0.27902 (+0.00161) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_11466.pth + + > EPOCH: 49/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 19:54:16)  + + --> STEP: 4/234 -- GLOBAL_STEP: 11470 + | > loss: 0.22019 (0.24111) + | > log_mle: -0.06172 (-0.03916) + | > loss_dur: 0.28191 (0.28028) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.69809 (3.29755) + | > current_lr: 0.00001 + | > step_time: 5.91030 (4.02418) + | > loader_time: 0.19660 (0.05274) + + + --> STEP: 9/234 -- GLOBAL_STEP: 11475 + | > loss: 0.15816 (0.19553) + | > log_mle: -0.06868 (-0.05019) + | > loss_dur: 0.22684 (0.24572) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.87663 (3.16141) + | > current_lr: 0.00001 + | > step_time: 1.91840 (4.24665) + | > loader_time: 0.08790 (0.07525) + + + --> STEP: 14/234 -- GLOBAL_STEP: 11480 + | > loss: 0.15839 (0.18010) + | > log_mle: -0.06158 (-0.05159) + | > loss_dur: 0.21997 (0.23169) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.40429 (3.05573) + | > current_lr: 0.00001 + | > step_time: 2.00390 (3.61042) + | > loader_time: 0.08450 (0.07346) + + + --> STEP: 19/234 -- GLOBAL_STEP: 11485 + | > loss: 0.15882 (0.17062) + | > log_mle: -0.04823 (-0.05156) + | > loss_dur: 0.20705 (0.22218) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.90634 (2.96146) + | > current_lr: 0.00001 + | > step_time: 1.00260 (2.99356) + | > loader_time: 0.00130 (0.05921) + + + --> STEP: 24/234 -- GLOBAL_STEP: 11490 + | > loss: 0.13638 (0.16395) + | > log_mle: -0.05159 (-0.05202) + | > loss_dur: 0.18797 (0.21598) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10287 (2.85249) + | > current_lr: 0.00001 + | > step_time: 1.14600 (2.61693) + | > loader_time: 0.00210 (0.04725) + + + --> STEP: 29/234 -- GLOBAL_STEP: 11495 + | > loss: 0.12842 (0.16019) + | > log_mle: -0.04842 (-0.05244) + | > loss_dur: 0.17684 (0.21263) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.10505 (2.77112) + | > current_lr: 0.00001 + | > step_time: 2.27220 (2.41968) + | > loader_time: 0.00180 (0.04228) + + + --> STEP: 34/234 -- GLOBAL_STEP: 11500 + | > loss: 0.17046 (0.15816) + | > log_mle: -0.06212 (-0.05431) + | > loss_dur: 0.23257 (0.21247) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.81750 (2.69987) + | > current_lr: 0.00001 + | > step_time: 1.17450 (2.23301) + | > loader_time: 0.00180 (0.03640) + + + --> STEP: 39/234 -- GLOBAL_STEP: 11505 + | > loss: 0.14215 (0.15571) + | > log_mle: -0.06815 (-0.05598) + | > loss_dur: 0.21030 (0.21170) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.84915 (2.96217) + | > current_lr: 0.00001 + | > step_time: 1.15300 (2.08597) + | > loader_time: 0.00200 (0.03203) + + + --> STEP: 44/234 -- GLOBAL_STEP: 11510 + | > loss: 0.11028 (0.15460) + | > log_mle: -0.06253 (-0.05621) + | > loss_dur: 0.17281 (0.21081) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.15882 (2.92237) + | > current_lr: 0.00001 + | > step_time: 1.60700 (2.00330) + | > loader_time: 0.08460 (0.03055) + + + --> STEP: 49/234 -- GLOBAL_STEP: 11515 + | > loss: 0.11165 (0.15127) + | > log_mle: -0.06901 (-0.05741) + | > loss_dur: 0.18066 (0.20867) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.04238 (2.87816) + | > current_lr: 0.00001 + | > step_time: 1.17720 (1.92906) + | > loader_time: 0.00190 (0.02770) + + + --> STEP: 54/234 -- GLOBAL_STEP: 11520 + | > loss: 0.12015 (0.15017) + | > log_mle: -0.07288 (-0.05777) + | > loss_dur: 0.19303 (0.20794) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.77867 (2.81857) + | > current_lr: 0.00001 + | > step_time: 1.37430 (1.89200) + | > loader_time: 0.00180 (0.02533) + + + --> STEP: 59/234 -- GLOBAL_STEP: 11525 + | > loss: 0.08115 (0.14878) + | > log_mle: -0.08908 (-0.05878) + | > loss_dur: 0.17023 (0.20756) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.62950 (2.78644) + | > current_lr: 0.00001 + | > step_time: 1.51290 (1.87895) + | > loader_time: 0.00210 (0.02489) + + + --> STEP: 64/234 -- GLOBAL_STEP: 11530 + | > loss: 0.12459 (0.14708) + | > log_mle: -0.05954 (-0.06050) + | > loss_dur: 0.18412 (0.20758) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.53401 (2.89579) + | > current_lr: 0.00001 + | > step_time: 1.30700 (1.83988) + | > loader_time: 0.00170 (0.02313) + + + --> STEP: 69/234 -- GLOBAL_STEP: 11535 + | > loss: 0.14673 (0.14604) + | > log_mle: -0.04590 (-0.06089) + | > loss_dur: 0.19262 (0.20693) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.44406 (2.94299) + | > current_lr: 0.00001 + | > step_time: 1.79610 (1.83809) + | > loader_time: 0.00320 (0.02290) + + + --> STEP: 74/234 -- GLOBAL_STEP: 11540 + | > loss: 0.09685 (0.14418) + | > log_mle: -0.07411 (-0.06240) + | > loss_dur: 0.17096 (0.20657) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.06865 (3.40206) + | > current_lr: 0.00001 + | > step_time: 1.14910 (1.83331) + | > loader_time: 0.08420 (0.02488) + + + --> STEP: 79/234 -- GLOBAL_STEP: 11545 + | > loss: 0.10972 (0.14313) + | > log_mle: -0.08241 (-0.06351) + | > loss_dur: 0.19213 (0.20664) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.51204 (3.43114) + | > current_lr: 0.00001 + | > step_time: 1.61080 (1.80163) + | > loader_time: 0.08340 (0.02553) + + + --> STEP: 84/234 -- GLOBAL_STEP: 11550 + | > loss: 0.11800 (0.14148) + | > log_mle: -0.08469 (-0.06465) + | > loss_dur: 0.20269 (0.20612) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.04550 (3.51108) + | > current_lr: 0.00001 + | > step_time: 1.59620 (1.80974) + | > loader_time: 0.00370 (0.02617) + + + --> STEP: 89/234 -- GLOBAL_STEP: 11555 + | > loss: 0.09517 (0.13919) + | > log_mle: -0.11140 (-0.06676) + | > loss_dur: 0.20657 (0.20594) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.51337 (3.67286) + | > current_lr: 0.00001 + | > step_time: 1.30210 (1.79814) + | > loader_time: 0.00270 (0.02486) + + + --> STEP: 94/234 -- GLOBAL_STEP: 11560 + | > loss: 0.08056 (0.13680) + | > log_mle: -0.14829 (-0.06996) + | > loss_dur: 0.22885 (0.20676) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.89858 (3.96227) + | > current_lr: 0.00001 + | > step_time: 2.01830 (1.80273) + | > loader_time: 0.09370 (0.02474) + + + --> STEP: 99/234 -- GLOBAL_STEP: 11565 + | > loss: 0.05420 (0.13466) + | > log_mle: -0.17489 (-0.07306) + | > loss_dur: 0.22909 (0.20772) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.39182 (4.34357) + | > current_lr: 0.00001 + | > step_time: 1.30440 (1.77888) + | > loader_time: 0.08840 (0.02556) + + + --> STEP: 104/234 -- GLOBAL_STEP: 11570 + | > loss: 0.06066 (0.13249) + | > log_mle: -0.18395 (-0.07645) + | > loss_dur: 0.24461 (0.20894) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.89439 (4.72235) + | > current_lr: 0.00001 + | > step_time: 1.48000 (1.77812) + | > loader_time: 0.00190 (0.02620) + + + --> STEP: 109/234 -- GLOBAL_STEP: 11575 + | > loss: 0.10071 (0.13089) + | > log_mle: -0.15947 (-0.07901) + | > loss_dur: 0.26018 (0.20989) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.25986 (4.95550) + | > current_lr: 0.00001 + | > step_time: 1.39630 (1.78253) + | > loader_time: 0.00340 (0.02515) + + + --> STEP: 114/234 -- GLOBAL_STEP: 11580 + | > loss: 0.07730 (0.12850) + | > log_mle: -0.13894 (-0.08219) + | > loss_dur: 0.21624 (0.21069) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.13578 (5.34236) + | > current_lr: 0.00001 + | > step_time: 1.77850 (1.77881) + | > loader_time: 0.00230 (0.02417) + + + --> STEP: 119/234 -- GLOBAL_STEP: 11585 + | > loss: 0.09196 (0.12717) + | > log_mle: -0.13770 (-0.08467) + | > loss_dur: 0.22966 (0.21183) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.59484 (5.56999) + | > current_lr: 0.00001 + | > step_time: 3.00230 (1.78660) + | > loader_time: 0.00310 (0.02396) + + + --> STEP: 124/234 -- GLOBAL_STEP: 11590 + | > loss: 0.06052 (0.12558) + | > log_mle: -0.16609 (-0.08655) + | > loss_dur: 0.22661 (0.21213) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.83269 (5.72559) + | > current_lr: 0.00001 + | > step_time: 1.38500 (1.78534) + | > loader_time: 0.00340 (0.02312) + + + --> STEP: 129/234 -- GLOBAL_STEP: 11595 + | > loss: 0.09793 (0.12348) + | > log_mle: -0.14940 (-0.08986) + | > loss_dur: 0.24733 (0.21334) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.58514 (6.01244) + | > current_lr: 0.00001 + | > step_time: 2.09130 (1.78760) + | > loader_time: 0.00480 (0.02300) + + + --> STEP: 134/234 -- GLOBAL_STEP: 11600 + | > loss: 0.06384 (0.12114) + | > log_mle: -0.19767 (-0.09359) + | > loss_dur: 0.26150 (0.21473) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.28445 (6.40080) + | > current_lr: 0.00001 + | > step_time: 1.99740 (1.79472) + | > loader_time: 0.08900 (0.02423) + + + --> STEP: 139/234 -- GLOBAL_STEP: 11605 + | > loss: -0.00244 (0.11894) + | > log_mle: -0.25208 (-0.09712) + | > loss_dur: 0.24964 (0.21606) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.37875 (6.75164) + | > current_lr: 0.00001 + | > step_time: 1.90320 (1.78781) + | > loader_time: 0.00550 (0.02471) + + + --> STEP: 144/234 -- GLOBAL_STEP: 11610 + | > loss: 0.03152 (0.11673) + | > log_mle: -0.22895 (-0.10083) + | > loss_dur: 0.26047 (0.21756) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.77758 (7.19174) + | > current_lr: 0.00001 + | > step_time: 2.09540 (1.79396) + | > loader_time: 0.00350 (0.02515) + + + --> STEP: 149/234 -- GLOBAL_STEP: 11615 + | > loss: -0.00302 (0.11372) + | > log_mle: -0.26725 (-0.10497) + | > loss_dur: 0.26423 (0.21869) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.19925 (7.75703) + | > current_lr: 0.00001 + | > step_time: 1.50120 (1.82029) + | > loader_time: 0.08700 (0.02552) + + + --> STEP: 154/234 -- GLOBAL_STEP: 11620 + | > loss: 0.01680 (0.11052) + | > log_mle: -0.23491 (-0.10945) + | > loss_dur: 0.25171 (0.21997) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.04848 (8.29530) + | > current_lr: 0.00001 + | > step_time: 2.60030 (1.83010) + | > loader_time: 0.20210 (0.02667) + + + --> STEP: 159/234 -- GLOBAL_STEP: 11625 + | > loss: 0.01012 (0.10760) + | > log_mle: -0.24997 (-0.11372) + | > loss_dur: 0.26008 (0.22132) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.32179 (8.84605) + | > current_lr: 0.00001 + | > step_time: 3.09740 (1.85559) + | > loader_time: 0.08840 (0.02816) + + + --> STEP: 164/234 -- GLOBAL_STEP: 11630 + | > loss: 0.02689 (0.10462) + | > log_mle: -0.24591 (-0.11784) + | > loss_dur: 0.27280 (0.22246) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.12997 (9.36192) + | > current_lr: 0.00001 + | > step_time: 2.20590 (1.86906) + | > loader_time: 0.19390 (0.03143) + + + --> STEP: 169/234 -- GLOBAL_STEP: 11635 + | > loss: 0.03863 (0.10198) + | > log_mle: -0.24101 (-0.12204) + | > loss_dur: 0.27964 (0.22402) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.77890 (9.74685) + | > current_lr: 0.00001 + | > step_time: 2.19150 (1.87764) + | > loader_time: 0.00210 (0.03155) + + + --> STEP: 174/234 -- GLOBAL_STEP: 11640 + | > loss: -0.05432 (0.09828) + | > log_mle: -0.32402 (-0.12720) + | > loss_dur: 0.26970 (0.22548) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.16293 (10.31717) + | > current_lr: 0.00001 + | > step_time: 3.69310 (1.90331) + | > loader_time: 0.10230 (0.03180) + + + --> STEP: 179/234 -- GLOBAL_STEP: 11645 + | > loss: -0.01610 (0.09536) + | > log_mle: -0.31102 (-0.13179) + | > loss_dur: 0.29493 (0.22715) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.07449 (10.83769) + | > current_lr: 0.00001 + | > step_time: 2.50170 (1.90768) + | > loader_time: 0.10290 (0.03158) + + + --> STEP: 184/234 -- GLOBAL_STEP: 11650 + | > loss: -0.00056 (0.09277) + | > log_mle: -0.28800 (-0.13597) + | > loss_dur: 0.28743 (0.22874) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.78384 (11.27756) + | > current_lr: 0.00001 + | > step_time: 4.20980 (1.95645) + | > loader_time: 0.08400 (0.03216) + + + --> STEP: 189/234 -- GLOBAL_STEP: 11655 + | > loss: 0.00265 (0.09007) + | > log_mle: -0.28354 (-0.14028) + | > loss_dur: 0.28619 (0.23035) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.81677 (11.71041) + | > current_lr: 0.00001 + | > step_time: 4.80140 (2.07658) + | > loader_time: 0.10050 (0.03299) + + + --> STEP: 194/234 -- GLOBAL_STEP: 11660 + | > loss: -0.03149 (0.08706) + | > log_mle: -0.31388 (-0.14460) + | > loss_dur: 0.28239 (0.23166) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.36347 (12.20079) + | > current_lr: 0.00001 + | > step_time: 4.09830 (2.12301) + | > loader_time: 0.00630 (0.03326) + + + --> STEP: 199/234 -- GLOBAL_STEP: 11665 + | > loss: -0.03846 (0.08445) + | > log_mle: -0.32341 (-0.14859) + | > loss_dur: 0.28496 (0.23304) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.68100 (12.73660) + | > current_lr: 0.00001 + | > step_time: 2.69360 (2.13038) + | > loader_time: 0.00420 (0.03384) + + + --> STEP: 204/234 -- GLOBAL_STEP: 11670 + | > loss: -0.03575 (0.08205) + | > log_mle: -0.34863 (-0.15257) + | > loss_dur: 0.31287 (0.23462) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 53.67744 (13.26471) + | > current_lr: 0.00001 + | > step_time: 3.59610 (2.19587) + | > loader_time: 0.09910 (0.03592) + + + --> STEP: 209/234 -- GLOBAL_STEP: 11675 + | > loss: -0.02801 (0.07916) + | > log_mle: -0.31234 (-0.15682) + | > loss_dur: 0.28433 (0.23598) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.93779 (13.72348) + | > current_lr: 0.00001 + | > step_time: 7.10180 (2.24341) + | > loader_time: 0.20140 (0.03691) + + + --> STEP: 214/234 -- GLOBAL_STEP: 11680 + | > loss: -0.06096 (0.07583) + | > log_mle: -0.34259 (-0.16181) + | > loss_dur: 0.28163 (0.23764) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.72562 (14.20437) + | > current_lr: 0.00001 + | > step_time: 5.19000 (2.29907) + | > loader_time: 0.09800 (0.03731) + + + --> STEP: 219/234 -- GLOBAL_STEP: 11685 + | > loss: -0.11423 (0.07265) + | > log_mle: -0.42454 (-0.16653) + | > loss_dur: 0.31030 (0.23918) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.19186 (14.87105) + | > current_lr: 0.00001 + | > step_time: 2.00110 (2.35261) + | > loader_time: 0.09060 (0.04005) + + + --> STEP: 224/234 -- GLOBAL_STEP: 11690 + | > loss: -0.07431 (0.06990) + | > log_mle: -0.38629 (-0.17099) + | > loss_dur: 0.31198 (0.24089) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.88884 (15.30388) + | > current_lr: 0.00001 + | > step_time: 0.23680 (2.32521) + | > loader_time: 0.00390 (0.03927) + + + --> STEP: 229/234 -- GLOBAL_STEP: 11695 + | > loss: -0.02086 (0.06721) + | > log_mle: -0.40496 (-0.17584) + | > loss_dur: 0.38410 (0.24305) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.55362 (16.05289) + | > current_lr: 0.00001 + | > step_time: 0.24930 (2.27983) + | > loader_time: 0.00490 (0.03850) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32479 (+0.27354) + | > avg_loss: 0.02212 (-0.00887) + | > avg_log_mle: -0.25279 (-0.00476) + | > avg_loss_dur: 0.27491 (-0.00411) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_11700.pth + + > EPOCH: 50/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:04:11)  + + --> STEP: 0/234 -- GLOBAL_STEP: 11700 + | > loss: 0.24760 (0.24760) + | > log_mle: -0.07573 (-0.07573) + | > loss_dur: 0.32333 (0.32333) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.58242 (3.58242) + | > current_lr: 0.00001 + | > step_time: 10.39540 (10.39543) + | > loader_time: 21.63470 (21.63469) + + + --> STEP: 5/234 -- GLOBAL_STEP: 11705 + | > loss: 0.14937 (0.21752) + | > log_mle: -0.05919 (-0.04750) + | > loss_dur: 0.20855 (0.26502) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.63244 (3.44306) + | > current_lr: 0.00001 + | > step_time: 1.39440 (1.67915) + | > loader_time: 0.00150 (0.02294) + + + --> STEP: 10/234 -- GLOBAL_STEP: 11710 + | > loss: 0.13413 (0.18818) + | > log_mle: -0.06942 (-0.05698) + | > loss_dur: 0.20354 (0.24516) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.13101 (3.23256) + | > current_lr: 0.00001 + | > step_time: 2.75510 (4.05638) + | > loader_time: 0.19540 (0.09960) + + + --> STEP: 15/234 -- GLOBAL_STEP: 11715 + | > loss: 0.16045 (0.17905) + | > log_mle: -0.05925 (-0.05736) + | > loss_dur: 0.21969 (0.23641) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41648 (3.05445) + | > current_lr: 0.00001 + | > step_time: 0.84800 (2.94369) + | > loader_time: 0.00120 (0.06686) + + + --> STEP: 20/234 -- GLOBAL_STEP: 11720 + | > loss: 0.15270 (0.17066) + | > log_mle: -0.05480 (-0.05678) + | > loss_dur: 0.20750 (0.22744) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36886 (2.97907) + | > current_lr: 0.00001 + | > step_time: 0.75160 (2.47658) + | > loader_time: 0.00140 (0.05559) + + + --> STEP: 25/234 -- GLOBAL_STEP: 11725 + | > loss: 0.16391 (0.16250) + | > log_mle: -0.04828 (-0.05693) + | > loss_dur: 0.21219 (0.21943) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.80697 (2.94515) + | > current_lr: 0.00001 + | > step_time: 1.04420 (2.19001) + | > loader_time: 0.00210 (0.04475) + + + --> STEP: 30/234 -- GLOBAL_STEP: 11730 + | > loss: 0.10753 (0.15669) + | > log_mle: -0.07735 (-0.05837) + | > loss_dur: 0.18488 (0.21506) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.62130 (2.89699) + | > current_lr: 0.00001 + | > step_time: 1.17340 (2.03387) + | > loader_time: 0.00170 (0.03766) + + + --> STEP: 35/234 -- GLOBAL_STEP: 11735 + | > loss: 0.12971 (0.15510) + | > log_mle: -0.07691 (-0.06009) + | > loss_dur: 0.20662 (0.21518) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.25224 (3.03282) + | > current_lr: 0.00001 + | > step_time: 1.82460 (2.06498) + | > loader_time: 0.00300 (0.03511) + + + --> STEP: 40/234 -- GLOBAL_STEP: 11740 + | > loss: 0.17461 (0.15320) + | > log_mle: -0.05580 (-0.06119) + | > loss_dur: 0.23040 (0.21439) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.76855 (3.03446) + | > current_lr: 0.00001 + | > step_time: 1.09180 (1.96085) + | > loader_time: 0.00200 (0.03096) + + + --> STEP: 45/234 -- GLOBAL_STEP: 11745 + | > loss: 0.11794 (0.15028) + | > log_mle: -0.08794 (-0.06215) + | > loss_dur: 0.20588 (0.21243) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.19941 (2.98579) + | > current_lr: 0.00001 + | > step_time: 1.16620 (1.88633) + | > loader_time: 0.00200 (0.02988) + + + --> STEP: 50/234 -- GLOBAL_STEP: 11750 + | > loss: 0.13746 (0.14721) + | > log_mle: -0.06071 (-0.06268) + | > loss_dur: 0.19816 (0.20988) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.64956 (2.91562) + | > current_lr: 0.00001 + | > step_time: 1.61060 (1.84519) + | > loader_time: 0.00280 (0.03080) + + + --> STEP: 55/234 -- GLOBAL_STEP: 11755 + | > loss: 0.11982 (0.14506) + | > log_mle: -0.08116 (-0.06340) + | > loss_dur: 0.20098 (0.20846) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.69111 (2.87983) + | > current_lr: 0.00001 + | > step_time: 1.11330 (1.79904) + | > loader_time: 0.08250 (0.02967) + + + --> STEP: 60/234 -- GLOBAL_STEP: 11760 + | > loss: 0.08933 (0.14285) + | > log_mle: -0.09925 (-0.06467) + | > loss_dur: 0.18858 (0.20753) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.12492 (2.86831) + | > current_lr: 0.00001 + | > step_time: 1.99920 (1.86259) + | > loader_time: 0.00210 (0.03040) + + + --> STEP: 65/234 -- GLOBAL_STEP: 11765 + | > loss: 0.12072 (0.14155) + | > log_mle: -0.07369 (-0.06589) + | > loss_dur: 0.19440 (0.20744) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.54762 (3.04572) + | > current_lr: 0.00001 + | > step_time: 1.40410 (1.82496) + | > loader_time: 0.00420 (0.02825) + + + --> STEP: 70/234 -- GLOBAL_STEP: 11770 + | > loss: 0.12351 (0.14083) + | > log_mle: -0.07790 (-0.06633) + | > loss_dur: 0.20142 (0.20716) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.35642 (3.48928) + | > current_lr: 0.00001 + | > step_time: 1.11570 (1.78880) + | > loader_time: 0.08610 (0.02763) + + + --> STEP: 75/234 -- GLOBAL_STEP: 11775 + | > loss: 0.11421 (0.13911) + | > log_mle: -0.08896 (-0.06794) + | > loss_dur: 0.20317 (0.20704) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.01101 (3.78776) + | > current_lr: 0.00001 + | > step_time: 1.05710 (1.75172) + | > loader_time: 0.00220 (0.02596) + + + --> STEP: 80/234 -- GLOBAL_STEP: 11780 + | > loss: 0.11131 (0.13763) + | > log_mle: -0.06847 (-0.06877) + | > loss_dur: 0.17978 (0.20640) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.25452 (3.80769) + | > current_lr: 0.00001 + | > step_time: 1.27720 (1.74728) + | > loader_time: 0.00350 (0.02454) + + + --> STEP: 85/234 -- GLOBAL_STEP: 11785 + | > loss: 0.10953 (0.13586) + | > log_mle: -0.08493 (-0.07008) + | > loss_dur: 0.19446 (0.20594) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.64976 (3.88404) + | > current_lr: 0.00001 + | > step_time: 1.18260 (1.74092) + | > loader_time: 0.00210 (0.02425) + + + --> STEP: 90/234 -- GLOBAL_STEP: 11790 + | > loss: 0.09276 (0.13361) + | > log_mle: -0.11702 (-0.07251) + | > loss_dur: 0.20978 (0.20612) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.90714 (4.14501) + | > current_lr: 0.00001 + | > step_time: 1.21320 (1.72989) + | > loader_time: 0.08480 (0.02399) + + + --> STEP: 95/234 -- GLOBAL_STEP: 11795 + | > loss: 0.04767 (0.13029) + | > log_mle: -0.19737 (-0.07649) + | > loss_dur: 0.24503 (0.20677) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.30759 (4.56949) + | > current_lr: 0.00001 + | > step_time: 2.60120 (1.72945) + | > loader_time: 0.00220 (0.02378) + + + --> STEP: 100/234 -- GLOBAL_STEP: 11800 + | > loss: 0.08821 (0.12858) + | > log_mle: -0.12870 (-0.07877) + | > loss_dur: 0.21691 (0.20735) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.56588 (4.77800) + | > current_lr: 0.00001 + | > step_time: 1.69840 (1.71590) + | > loader_time: 0.00170 (0.02352) + + + --> STEP: 105/234 -- GLOBAL_STEP: 11805 + | > loss: 0.09768 (0.12640) + | > log_mle: -0.09701 (-0.08180) + | > loss_dur: 0.19469 (0.20820) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.22763 (5.11230) + | > current_lr: 0.00001 + | > step_time: 2.45840 (1.70714) + | > loader_time: 0.00450 (0.02258) + + + --> STEP: 110/234 -- GLOBAL_STEP: 11810 + | > loss: 0.07602 (0.12491) + | > log_mle: -0.12483 (-0.08450) + | > loss_dur: 0.20086 (0.20940) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.61705 (6.26636) + | > current_lr: 0.00001 + | > step_time: 1.20650 (1.69637) + | > loader_time: 0.08730 (0.02244) + + + --> STEP: 115/234 -- GLOBAL_STEP: 11815 + | > loss: 0.09937 (0.12298) + | > log_mle: -0.14352 (-0.08770) + | > loss_dur: 0.24289 (0.21068) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.41356 (6.71890) + | > current_lr: 0.00001 + | > step_time: 2.40310 (1.70016) + | > loader_time: 0.00320 (0.02306) + + + --> STEP: 120/234 -- GLOBAL_STEP: 11820 + | > loss: 0.05291 (0.12115) + | > log_mle: -0.18901 (-0.09051) + | > loss_dur: 0.24192 (0.21165) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.25419 (6.99700) + | > current_lr: 0.00001 + | > step_time: 2.08510 (1.74592) + | > loader_time: 0.00340 (0.02618) + + + --> STEP: 125/234 -- GLOBAL_STEP: 11825 + | > loss: 0.07969 (0.11978) + | > log_mle: -0.17701 (-0.09223) + | > loss_dur: 0.25671 (0.21202) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.25988 (7.11028) + | > current_lr: 0.00001 + | > step_time: 2.60040 (1.75439) + | > loader_time: 0.07700 (0.02800) + + + --> STEP: 130/234 -- GLOBAL_STEP: 11830 + | > loss: 0.03968 (0.11726) + | > log_mle: -0.18746 (-0.09555) + | > loss_dur: 0.22715 (0.21282) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.33811 (7.43326) + | > current_lr: 0.00001 + | > step_time: 4.31070 (1.77611) + | > loader_time: 0.09030 (0.02840) + + + --> STEP: 135/234 -- GLOBAL_STEP: 11835 + | > loss: 0.08049 (0.11506) + | > log_mle: -0.12572 (-0.09877) + | > loss_dur: 0.20621 (0.21384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.24926 (7.73356) + | > current_lr: 0.00001 + | > step_time: 1.58600 (1.77327) + | > loader_time: 0.00240 (0.02885) + + + --> STEP: 140/234 -- GLOBAL_STEP: 11840 + | > loss: 0.08661 (0.11269) + | > log_mle: -0.15581 (-0.10250) + | > loss_dur: 0.24242 (0.21519) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.79524 (8.05457) + | > current_lr: 0.00001 + | > step_time: 2.40040 (1.78863) + | > loader_time: 0.00290 (0.02918) + + + --> STEP: 145/234 -- GLOBAL_STEP: 11845 + | > loss: 0.00861 (0.10993) + | > log_mle: -0.24150 (-0.10677) + | > loss_dur: 0.25011 (0.21670) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.80091 (8.47207) + | > current_lr: 0.00001 + | > step_time: 5.59930 (1.85507) + | > loader_time: 0.00390 (0.02910) + + + --> STEP: 150/234 -- GLOBAL_STEP: 11850 + | > loss: 0.02847 (0.10697) + | > log_mle: -0.22659 (-0.11078) + | > loss_dur: 0.25506 (0.21775) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.67027 (8.79500) + | > current_lr: 0.00001 + | > step_time: 1.78760 (1.84861) + | > loader_time: 0.00290 (0.02882) + + + --> STEP: 155/234 -- GLOBAL_STEP: 11855 + | > loss: -0.01808 (0.10335) + | > log_mle: -0.28868 (-0.11564) + | > loss_dur: 0.27060 (0.21899) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.92383 (9.22623) + | > current_lr: 0.00001 + | > step_time: 1.58780 (1.91168) + | > loader_time: 0.00250 (0.03099) + + + --> STEP: 160/234 -- GLOBAL_STEP: 11860 + | > loss: -0.01489 (0.10050) + | > log_mle: -0.28212 (-0.11988) + | > loss_dur: 0.26723 (0.22038) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.76042 (9.61420) + | > current_lr: 0.00001 + | > step_time: 8.91390 (1.97389) + | > loader_time: 0.18910 (0.03242) + + + --> STEP: 165/234 -- GLOBAL_STEP: 11865 + | > loss: 0.01344 (0.09771) + | > log_mle: -0.27740 (-0.12394) + | > loss_dur: 0.29084 (0.22165) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.94943 (9.99120) + | > current_lr: 0.00001 + | > step_time: 1.88120 (1.97341) + | > loader_time: 0.01970 (0.03209) + + + --> STEP: 170/234 -- GLOBAL_STEP: 11870 + | > loss: -0.01749 (0.09497) + | > log_mle: -0.31322 (-0.12827) + | > loss_dur: 0.29573 (0.22324) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.35288 (10.46360) + | > current_lr: 0.00001 + | > step_time: 1.59810 (1.97427) + | > loader_time: 0.00350 (0.03261) + + + --> STEP: 175/234 -- GLOBAL_STEP: 11875 + | > loss: -0.00769 (0.09139) + | > log_mle: -0.28784 (-0.13326) + | > loss_dur: 0.28015 (0.22465) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.01446 (10.88744) + | > current_lr: 0.00001 + | > step_time: 3.30750 (1.99114) + | > loader_time: 0.10460 (0.03375) + + + --> STEP: 180/234 -- GLOBAL_STEP: 11880 + | > loss: -0.01499 (0.08844) + | > log_mle: -0.29879 (-0.13789) + | > loss_dur: 0.28380 (0.22633) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.79820 (11.31199) + | > current_lr: 0.00001 + | > step_time: 5.80410 (2.02250) + | > loader_time: 0.09960 (0.03400) + + + --> STEP: 185/234 -- GLOBAL_STEP: 11885 + | > loss: -0.02149 (0.08586) + | > log_mle: -0.31804 (-0.14215) + | > loss_dur: 0.29655 (0.22802) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.85984 (11.79792) + | > current_lr: 0.00001 + | > step_time: 5.10950 (2.05769) + | > loader_time: 0.00290 (0.03516) + + + --> STEP: 190/234 -- GLOBAL_STEP: 11890 + | > loss: -0.02310 (0.08317) + | > log_mle: -0.30332 (-0.14635) + | > loss_dur: 0.28022 (0.22951) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.12008 (12.23324) + | > current_lr: 0.00001 + | > step_time: 1.01390 (2.07116) + | > loader_time: 0.05870 (0.03747) + + + --> STEP: 195/234 -- GLOBAL_STEP: 11895 + | > loss: -0.01200 (0.08015) + | > log_mle: -0.31161 (-0.15068) + | > loss_dur: 0.29961 (0.23083) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.22824 (12.70785) + | > current_lr: 0.00001 + | > step_time: 7.38620 (2.11441) + | > loader_time: 0.09550 (0.03937) + + + --> STEP: 200/234 -- GLOBAL_STEP: 11900 + | > loss: -0.00462 (0.07762) + | > log_mle: -0.31739 (-0.15468) + | > loss_dur: 0.31277 (0.23230) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.31857 (13.24528) + | > current_lr: 0.00001 + | > step_time: 4.90680 (2.17902) + | > loader_time: 0.19020 (0.04079) + + + --> STEP: 205/234 -- GLOBAL_STEP: 11905 + | > loss: -0.03267 (0.07511) + | > log_mle: -0.30843 (-0.15858) + | > loss_dur: 0.27575 (0.23369) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.73798 (13.66583) + | > current_lr: 0.00001 + | > step_time: 6.10200 (2.24543) + | > loader_time: 0.00340 (0.04084) + + + --> STEP: 210/234 -- GLOBAL_STEP: 11910 + | > loss: -0.08036 (0.07199) + | > log_mle: -0.38224 (-0.16314) + | > loss_dur: 0.30189 (0.23513) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.41599 (14.18207) + | > current_lr: 0.00001 + | > step_time: 9.99510 (2.29674) + | > loader_time: 0.00680 (0.04127) + + + --> STEP: 215/234 -- GLOBAL_STEP: 11915 + | > loss: -0.05197 (0.06878) + | > log_mle: -0.33199 (-0.16786) + | > loss_dur: 0.28003 (0.23664) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.10037 (14.66325) + | > current_lr: 0.00001 + | > step_time: 7.20190 (2.34653) + | > loader_time: 0.08300 (0.04202) + + + --> STEP: 220/234 -- GLOBAL_STEP: 11920 + | > loss: -0.06711 (0.06547) + | > log_mle: -0.37551 (-0.17278) + | > loss_dur: 0.30840 (0.23825) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.59649 (15.23747) + | > current_lr: 0.00001 + | > step_time: 3.61580 (2.42418) + | > loader_time: 0.09800 (0.04378) + + + --> STEP: 225/234 -- GLOBAL_STEP: 11925 + | > loss: -0.11325 (0.06247) + | > log_mle: -0.43129 (-0.17743) + | > loss_dur: 0.31805 (0.23990) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 48.65241 (15.79829) + | > current_lr: 0.00001 + | > step_time: 0.23320 (2.39565) + | > loader_time: 0.00330 (0.04289) + + + --> STEP: 230/234 -- GLOBAL_STEP: 11930 + | > loss: -0.07156 (0.05987) + | > log_mle: -0.47443 (-0.18248) + | > loss_dur: 0.40286 (0.24236) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.50590 (16.32495) + | > current_lr: 0.00001 + | > step_time: 0.25870 (2.34896) + | > loader_time: 0.00510 (0.04205) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.13058 (-0.19422) + | > avg_loss: 0.04184 (+0.01972) + | > avg_log_mle: -0.23280 (+0.01999) + | > avg_loss_dur: 0.27465 (-0.00027) + + + > EPOCH: 51/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:14:35)  + + --> STEP: 1/234 -- GLOBAL_STEP: 11935 + | > loss: 0.17968 (0.17968) + | > log_mle: -0.04696 (-0.04696) + | > loss_dur: 0.22664 (0.22664) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.52495 (2.52495) + | > current_lr: 0.00001 + | > step_time: 5.00600 (5.00604) + | > loader_time: 0.00130 (0.00133) + + + --> STEP: 6/234 -- GLOBAL_STEP: 11940 + | > loss: 0.18629 (0.20338) + | > log_mle: -0.05090 (-0.05295) + | > loss_dur: 0.23720 (0.25633) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.68387 (3.53979) + | > current_lr: 0.00001 + | > step_time: 0.52270 (5.95812) + | > loader_time: 0.00090 (0.05005) + + + --> STEP: 11/234 -- GLOBAL_STEP: 11945 + | > loss: 0.15533 (0.17270) + | > log_mle: -0.05727 (-0.06201) + | > loss_dur: 0.21261 (0.23470) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.47521 (3.60985) + | > current_lr: 0.00001 + | > step_time: 1.30500 (3.66124) + | > loader_time: 0.00110 (0.03470) + + + --> STEP: 16/234 -- GLOBAL_STEP: 11950 + | > loss: 0.12635 (0.16661) + | > log_mle: -0.06767 (-0.06298) + | > loss_dur: 0.19402 (0.22959) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.27000 (3.38278) + | > current_lr: 0.00001 + | > step_time: 0.80110 (2.94061) + | > loader_time: 0.00110 (0.02445) + + + --> STEP: 21/234 -- GLOBAL_STEP: 11955 + | > loss: 0.14228 (0.15975) + | > log_mle: -0.05347 (-0.06172) + | > loss_dur: 0.19575 (0.22147) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.65407 (3.20348) + | > current_lr: 0.00001 + | > step_time: 0.99100 (2.46766) + | > loader_time: 0.00310 (0.02304) + + + --> STEP: 26/234 -- GLOBAL_STEP: 11960 + | > loss: 0.13479 (0.15415) + | > log_mle: -0.06952 (-0.06250) + | > loss_dur: 0.20431 (0.21665) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.24354 (3.04352) + | > current_lr: 0.00001 + | > step_time: 1.50550 (2.24957) + | > loader_time: 0.08300 (0.02530) + + + --> STEP: 31/234 -- GLOBAL_STEP: 11965 + | > loss: 0.13477 (0.14834) + | > log_mle: -0.07498 (-0.06407) + | > loss_dur: 0.20974 (0.21241) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.40864 (2.92147) + | > current_lr: 0.00001 + | > step_time: 3.50100 (2.53902) + | > loader_time: 0.20180 (0.03695) + + + --> STEP: 36/234 -- GLOBAL_STEP: 11970 + | > loss: 0.13206 (0.14605) + | > log_mle: -0.08106 (-0.06591) + | > loss_dur: 0.21313 (0.21196) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.79150 (3.06257) + | > current_lr: 0.00001 + | > step_time: 1.60020 (2.72578) + | > loader_time: 0.00180 (0.03732) + + + --> STEP: 41/234 -- GLOBAL_STEP: 11975 + | > loss: 0.10440 (0.14460) + | > log_mle: -0.07330 (-0.06669) + | > loss_dur: 0.17771 (0.21129) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.86605 (3.04828) + | > current_lr: 0.00001 + | > step_time: 1.49690 (2.57811) + | > loader_time: 0.00210 (0.03505) + + + --> STEP: 46/234 -- GLOBAL_STEP: 11980 + | > loss: 0.13370 (0.14254) + | > log_mle: -0.07690 (-0.06766) + | > loss_dur: 0.21060 (0.21020) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.56872 (3.04925) + | > current_lr: 0.00001 + | > step_time: 1.07750 (2.44964) + | > loader_time: 0.00240 (0.03148) + + + --> STEP: 51/234 -- GLOBAL_STEP: 11985 + | > loss: 0.11866 (0.13976) + | > log_mle: -0.06076 (-0.06783) + | > loss_dur: 0.17942 (0.20759) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.34759 (2.97267) + | > current_lr: 0.00001 + | > step_time: 1.31380 (2.32935) + | > loader_time: 0.00380 (0.02861) + + + --> STEP: 56/234 -- GLOBAL_STEP: 11990 + | > loss: 0.14095 (0.13820) + | > log_mle: -0.08081 (-0.06892) + | > loss_dur: 0.22177 (0.20712) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.28086 (2.92405) + | > current_lr: 0.00001 + | > step_time: 1.11440 (2.30166) + | > loader_time: 0.00220 (0.02743) + + + --> STEP: 61/234 -- GLOBAL_STEP: 11995 + | > loss: 0.11138 (0.13557) + | > log_mle: -0.07609 (-0.07007) + | > loss_dur: 0.18747 (0.20564) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.01137 (2.91364) + | > current_lr: 0.00001 + | > step_time: 2.49320 (2.27221) + | > loader_time: 0.00280 (0.02665) + + + --> STEP: 66/234 -- GLOBAL_STEP: 12000 + | > loss: 0.13013 (0.13452) + | > log_mle: -0.06645 (-0.07105) + | > loss_dur: 0.19658 (0.20558) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.94205 (3.10199) + | > current_lr: 0.00001 + | > step_time: 1.60000 (2.32262) + | > loader_time: 0.08670 (0.02751) + + + --> STEP: 71/234 -- GLOBAL_STEP: 12005 + | > loss: 0.10249 (0.13319) + | > log_mle: -0.11321 (-0.07215) + | > loss_dur: 0.21569 (0.20534) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.86154 (3.45803) + | > current_lr: 0.00001 + | > step_time: 1.59940 (2.31945) + | > loader_time: 0.00210 (0.02575) + + + --> STEP: 76/234 -- GLOBAL_STEP: 12010 + | > loss: 0.11929 (0.13191) + | > log_mle: -0.09274 (-0.07336) + | > loss_dur: 0.21203 (0.20527) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.40795 (3.60531) + | > current_lr: 0.00001 + | > step_time: 2.20900 (2.31422) + | > loader_time: 0.00160 (0.02783) + + + --> STEP: 81/234 -- GLOBAL_STEP: 12015 + | > loss: 0.08972 (0.12995) + | > log_mle: -0.10475 (-0.07424) + | > loss_dur: 0.19447 (0.20418) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.74332 (3.69071) + | > current_lr: 0.00001 + | > step_time: 2.61180 (2.28382) + | > loader_time: 0.07880 (0.02822) + + + --> STEP: 86/234 -- GLOBAL_STEP: 12020 + | > loss: 0.10666 (0.12879) + | > log_mle: -0.10498 (-0.07546) + | > loss_dur: 0.21164 (0.20425) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.53379 (3.79300) + | > current_lr: 0.00001 + | > step_time: 1.49910 (2.24547) + | > loader_time: 0.00300 (0.02672) + + + --> STEP: 91/234 -- GLOBAL_STEP: 12025 + | > loss: 0.10602 (0.12677) + | > log_mle: -0.11165 (-0.07785) + | > loss_dur: 0.21766 (0.20461) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.27736 (4.06356) + | > current_lr: 0.00001 + | > step_time: 3.09650 (2.24199) + | > loader_time: 0.00280 (0.02733) + + + --> STEP: 96/234 -- GLOBAL_STEP: 12030 + | > loss: 0.10766 (0.12367) + | > log_mle: -0.10234 (-0.08160) + | > loss_dur: 0.21001 (0.20527) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.69202 (4.58149) + | > current_lr: 0.00001 + | > step_time: 3.60740 (2.23996) + | > loader_time: 0.00350 (0.02707) + + + --> STEP: 101/234 -- GLOBAL_STEP: 12035 + | > loss: 0.07508 (0.12186) + | > log_mle: -0.15457 (-0.08430) + | > loss_dur: 0.22965 (0.20616) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.85721 (5.13336) + | > current_lr: 0.00001 + | > step_time: 1.45750 (2.22862) + | > loader_time: 0.00240 (0.02589) + + + --> STEP: 106/234 -- GLOBAL_STEP: 12040 + | > loss: 0.09644 (0.12012) + | > log_mle: -0.15683 (-0.08725) + | > loss_dur: 0.25327 (0.20737) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.55040 (5.46956) + | > current_lr: 0.00001 + | > step_time: 1.59480 (2.20727) + | > loader_time: 0.00350 (0.02559) + + + --> STEP: 111/234 -- GLOBAL_STEP: 12045 + | > loss: 0.07076 (0.11850) + | > log_mle: -0.19376 (-0.09030) + | > loss_dur: 0.26452 (0.20880) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.13574 (5.82092) + | > current_lr: 0.00001 + | > step_time: 1.88920 (2.19819) + | > loader_time: 0.00270 (0.02678) + + + --> STEP: 116/234 -- GLOBAL_STEP: 12050 + | > loss: 0.08972 (0.11672) + | > log_mle: -0.16516 (-0.09327) + | > loss_dur: 0.25488 (0.20999) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.00991 (6.17964) + | > current_lr: 0.00001 + | > step_time: 1.38760 (2.16479) + | > loader_time: 0.00190 (0.02645) + + + --> STEP: 121/234 -- GLOBAL_STEP: 12055 + | > loss: 0.12317 (0.11511) + | > log_mle: -0.08487 (-0.09538) + | > loss_dur: 0.20804 (0.21048) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.03008 (6.33309) + | > current_lr: 0.00001 + | > step_time: 2.64830 (2.15431) + | > loader_time: 0.00280 (0.02617) + + + --> STEP: 126/234 -- GLOBAL_STEP: 12060 + | > loss: 0.03170 (0.11291) + | > log_mle: -0.21111 (-0.09809) + | > loss_dur: 0.24280 (0.21100) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.40031 (6.57895) + | > current_lr: 0.00001 + | > step_time: 2.71320 (2.13689) + | > loader_time: 0.08600 (0.02664) + + + --> STEP: 131/234 -- GLOBAL_STEP: 12065 + | > loss: 0.01453 (0.11055) + | > log_mle: -0.24336 (-0.10162) + | > loss_dur: 0.25789 (0.21217) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.70729 (6.88737) + | > current_lr: 0.00001 + | > step_time: 2.11370 (2.14402) + | > loader_time: 0.08660 (0.02638) + + + --> STEP: 136/234 -- GLOBAL_STEP: 12070 + | > loss: 0.00237 (0.10830) + | > log_mle: -0.28649 (-0.10509) + | > loss_dur: 0.28886 (0.21339) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.34274 (7.21495) + | > current_lr: 0.00001 + | > step_time: 1.20990 (2.15721) + | > loader_time: 0.00250 (0.02559) + + + --> STEP: 141/234 -- GLOBAL_STEP: 12075 + | > loss: 0.04941 (0.10634) + | > log_mle: -0.20413 (-0.10813) + | > loss_dur: 0.25353 (0.21447) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.40888 (7.67870) + | > current_lr: 0.00001 + | > step_time: 1.30920 (2.14878) + | > loader_time: 0.00410 (0.02547) + + + --> STEP: 146/234 -- GLOBAL_STEP: 12080 + | > loss: 0.00848 (0.10347) + | > log_mle: -0.25289 (-0.11265) + | > loss_dur: 0.26137 (0.21612) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.90462 (8.24385) + | > current_lr: 0.00001 + | > step_time: 3.01510 (2.14538) + | > loader_time: 0.08810 (0.02587) + + + --> STEP: 151/234 -- GLOBAL_STEP: 12085 + | > loss: 0.00711 (0.10065) + | > log_mle: -0.22170 (-0.11637) + | > loss_dur: 0.22881 (0.21702) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.16879 (8.67853) + | > current_lr: 0.00001 + | > step_time: 3.78980 (2.17028) + | > loader_time: 0.00350 (0.02626) + + + --> STEP: 156/234 -- GLOBAL_STEP: 12090 + | > loss: -0.00098 (0.09718) + | > log_mle: -0.25830 (-0.12136) + | > loss_dur: 0.25732 (0.21855) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.64504 (9.34056) + | > current_lr: 0.00001 + | > step_time: 1.36450 (2.16855) + | > loader_time: 0.00290 (0.02606) + + + --> STEP: 161/234 -- GLOBAL_STEP: 12095 + | > loss: -0.02047 (0.09420) + | > log_mle: -0.27435 (-0.12561) + | > loss_dur: 0.25388 (0.21980) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.01858 (9.87160) + | > current_lr: 0.00001 + | > step_time: 2.40510 (2.20687) + | > loader_time: 0.00380 (0.02763) + + + --> STEP: 166/234 -- GLOBAL_STEP: 12100 + | > loss: 0.01435 (0.09158) + | > log_mle: -0.22408 (-0.12930) + | > loss_dur: 0.23844 (0.22088) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.49519 (10.37197) + | > current_lr: 0.00001 + | > step_time: 3.01120 (2.22548) + | > loader_time: 0.00490 (0.02812) + + + --> STEP: 171/234 -- GLOBAL_STEP: 12105 + | > loss: -0.05967 (0.08831) + | > log_mle: -0.31774 (-0.13414) + | > loss_dur: 0.25807 (0.22245) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.83601 (10.98955) + | > current_lr: 0.00001 + | > step_time: 4.59980 (2.26866) + | > loader_time: 0.20280 (0.03084) + + + --> STEP: 176/234 -- GLOBAL_STEP: 12110 + | > loss: -0.01993 (0.08501) + | > log_mle: -0.28692 (-0.13886) + | > loss_dur: 0.26699 (0.22387) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.08178 (11.61898) + | > current_lr: 0.00001 + | > step_time: 2.19320 (2.29489) + | > loader_time: 0.00430 (0.03064) + + + --> STEP: 181/234 -- GLOBAL_STEP: 12115 + | > loss: 0.03719 (0.08245) + | > log_mle: -0.23604 (-0.14308) + | > loss_dur: 0.27323 (0.22553) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.69134 (12.19476) + | > current_lr: 0.00001 + | > step_time: 2.61220 (2.30614) + | > loader_time: 0.00350 (0.03187) + + + --> STEP: 186/234 -- GLOBAL_STEP: 12120 + | > loss: 0.02918 (0.07980) + | > log_mle: -0.26952 (-0.14748) + | > loss_dur: 0.29870 (0.22728) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.39461 (12.77319) + | > current_lr: 0.00001 + | > step_time: 4.40330 (2.31735) + | > loader_time: 0.09820 (0.03299) + + + --> STEP: 191/234 -- GLOBAL_STEP: 12125 + | > loss: -0.01719 (0.07686) + | > log_mle: -0.28197 (-0.15166) + | > loss_dur: 0.26478 (0.22852) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.10445 (13.27739) + | > current_lr: 0.00001 + | > step_time: 2.40450 (2.35402) + | > loader_time: 0.08490 (0.03466) + + + --> STEP: 196/234 -- GLOBAL_STEP: 12130 + | > loss: 0.00473 (0.07401) + | > log_mle: -0.28109 (-0.15594) + | > loss_dur: 0.28582 (0.22996) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.27866 (13.79519) + | > current_lr: 0.00001 + | > step_time: 3.49690 (2.39902) + | > loader_time: 0.00490 (0.03696) + + + --> STEP: 201/234 -- GLOBAL_STEP: 12135 + | > loss: 0.02622 (0.07155) + | > log_mle: -0.25580 (-0.15981) + | > loss_dur: 0.28202 (0.23137) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.15927 (14.16911) + | > current_lr: 0.00001 + | > step_time: 4.51770 (2.46650) + | > loader_time: 0.08260 (0.03835) + + + --> STEP: 206/234 -- GLOBAL_STEP: 12140 + | > loss: -0.05765 (0.06855) + | > log_mle: -0.34228 (-0.16411) + | > loss_dur: 0.28463 (0.23266) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.54118 (14.63845) + | > current_lr: 0.00001 + | > step_time: 3.30570 (2.57070) + | > loader_time: 0.08980 (0.03889) + + + --> STEP: 211/234 -- GLOBAL_STEP: 12145 + | > loss: -0.08449 (0.06543) + | > log_mle: -0.40770 (-0.16891) + | > loss_dur: 0.32321 (0.23434) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.34003 (15.19350) + | > current_lr: 0.00001 + | > step_time: 4.20380 (2.63023) + | > loader_time: 0.08870 (0.04119) + + + --> STEP: 216/234 -- GLOBAL_STEP: 12150 + | > loss: -0.08723 (0.06230) + | > log_mle: -0.39604 (-0.17351) + | > loss_dur: 0.30881 (0.23581) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.26838 (15.63996) + | > current_lr: 0.00001 + | > step_time: 3.59890 (2.72529) + | > loader_time: 0.10190 (0.04255) + + + --> STEP: 221/234 -- GLOBAL_STEP: 12155 + | > loss: -0.03108 (0.05922) + | > log_mle: -0.32367 (-0.17808) + | > loss_dur: 0.29259 (0.23730) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.32029 (16.02770) + | > current_lr: 0.00001 + | > step_time: 1.12020 (2.77238) + | > loader_time: 0.00350 (0.04169) + + + --> STEP: 226/234 -- GLOBAL_STEP: 12160 + | > loss: -0.09855 (0.05592) + | > log_mle: -0.41442 (-0.18314) + | > loss_dur: 0.31587 (0.23907) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.84022 (16.45236) + | > current_lr: 0.00001 + | > step_time: 0.24440 (2.71638) + | > loader_time: 0.00450 (0.04084) + + + --> STEP: 231/234 -- GLOBAL_STEP: 12165 + | > loss: -0.01955 (0.05364) + | > log_mle: -0.47065 (-0.18842) + | > loss_dur: 0.45110 (0.24206) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.34879 (16.87191) + | > current_lr: 0.00001 + | > step_time: 0.27530 (2.66305) + | > loader_time: 0.00500 (0.04005) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67661 (+0.54603) + | > avg_loss: 0.02013 (-0.02171) + | > avg_log_mle: -0.25154 (-0.01874) + | > avg_loss_dur: 0.27167 (-0.00297) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_12168.pth + + > EPOCH: 52/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:26:04)  + + --> STEP: 2/234 -- GLOBAL_STEP: 12170 + | > loss: 0.26703 (0.22264) + | > log_mle: -0.03458 (-0.04262) + | > loss_dur: 0.30162 (0.26526) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.33616 (2.28884) + | > current_lr: 0.00001 + | > step_time: 9.39630 (8.45053) + | > loader_time: 0.10150 (0.14813) + + + --> STEP: 7/234 -- GLOBAL_STEP: 12175 + | > loss: 0.14041 (0.19198) + | > log_mle: -0.08110 (-0.06172) + | > loss_dur: 0.22151 (0.25370) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.01525 (3.46119) + | > current_lr: 0.00001 + | > step_time: 2.51480 (4.91598) + | > loader_time: 0.09780 (0.08518) + + + --> STEP: 12/234 -- GLOBAL_STEP: 12180 + | > loss: 0.14340 (0.16621) + | > log_mle: -0.06817 (-0.06745) + | > loss_dur: 0.21157 (0.23366) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.70341 (3.43193) + | > current_lr: 0.00001 + | > step_time: 1.90340 (4.55327) + | > loader_time: 0.00180 (0.11522) + + + --> STEP: 17/234 -- GLOBAL_STEP: 12185 + | > loss: 0.14769 (0.15881) + | > log_mle: -0.05563 (-0.06742) + | > loss_dur: 0.20331 (0.22624) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.59334 (3.29693) + | > current_lr: 0.00001 + | > step_time: 1.06730 (3.74835) + | > loader_time: 0.00110 (0.08174) + + + --> STEP: 22/234 -- GLOBAL_STEP: 12190 + | > loss: 0.09867 (0.15230) + | > log_mle: -0.07786 (-0.06733) + | > loss_dur: 0.17652 (0.21963) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.94134 (3.24239) + | > current_lr: 0.00001 + | > step_time: 0.95330 (3.21562) + | > loader_time: 0.00180 (0.06738) + + + --> STEP: 27/234 -- GLOBAL_STEP: 12195 + | > loss: 0.11678 (0.14721) + | > log_mle: -0.08252 (-0.06800) + | > loss_dur: 0.19930 (0.21521) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.57275 (3.11466) + | > current_lr: 0.00001 + | > step_time: 1.82920 (2.88362) + | > loader_time: 0.10720 (0.06281) + + + --> STEP: 32/234 -- GLOBAL_STEP: 12200 + | > loss: 0.08931 (0.14135) + | > log_mle: -0.09689 (-0.06977) + | > loss_dur: 0.18620 (0.21112) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.77037 (3.02498) + | > current_lr: 0.00001 + | > step_time: 1.01180 (2.62107) + | > loader_time: 0.00210 (0.05326) + + + --> STEP: 37/234 -- GLOBAL_STEP: 12205 + | > loss: 0.09347 (0.13918) + | > log_mle: -0.07443 (-0.07072) + | > loss_dur: 0.16790 (0.20990) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.46147 (3.38531) + | > current_lr: 0.00001 + | > step_time: 1.78170 (2.48491) + | > loader_time: 0.00210 (0.04634) + + + --> STEP: 42/234 -- GLOBAL_STEP: 12210 + | > loss: 0.14143 (0.13879) + | > log_mle: -0.06038 (-0.07111) + | > loss_dur: 0.20181 (0.20991) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.32759 (3.33854) + | > current_lr: 0.00001 + | > step_time: 1.15380 (2.36539) + | > loader_time: 0.00310 (0.04331) + + + --> STEP: 47/234 -- GLOBAL_STEP: 12215 + | > loss: 0.11978 (0.13574) + | > log_mle: -0.07880 (-0.07244) + | > loss_dur: 0.19858 (0.20819) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.58612 (3.31466) + | > current_lr: 0.00001 + | > step_time: 1.53740 (2.26083) + | > loader_time: 0.00210 (0.04288) + + + --> STEP: 52/234 -- GLOBAL_STEP: 12220 + | > loss: 0.13679 (0.13342) + | > log_mle: -0.06672 (-0.07235) + | > loss_dur: 0.20351 (0.20577) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.17593 (3.22721) + | > current_lr: 0.00001 + | > step_time: 1.53150 (2.18962) + | > loader_time: 0.00210 (0.04411) + + + --> STEP: 57/234 -- GLOBAL_STEP: 12225 + | > loss: 0.13446 (0.13260) + | > log_mle: -0.06621 (-0.07340) + | > loss_dur: 0.20067 (0.20600) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.17551 (3.13981) + | > current_lr: 0.00001 + | > step_time: 2.51570 (2.15182) + | > loader_time: 0.08450 (0.04505) + + + --> STEP: 62/234 -- GLOBAL_STEP: 12230 + | > loss: 0.11365 (0.12937) + | > log_mle: -0.12263 (-0.07543) + | > loss_dur: 0.23628 (0.20479) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.37623 (3.26659) + | > current_lr: 0.00001 + | > step_time: 1.62760 (2.12572) + | > loader_time: 0.00210 (0.04294) + + + --> STEP: 67/234 -- GLOBAL_STEP: 12235 + | > loss: 0.09125 (0.12812) + | > log_mle: -0.10514 (-0.07613) + | > loss_dur: 0.19639 (0.20425) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.31406 (3.28001) + | > current_lr: 0.00001 + | > step_time: 2.28450 (2.07937) + | > loader_time: 0.10630 (0.04303) + + + --> STEP: 72/234 -- GLOBAL_STEP: 12240 + | > loss: 0.11008 (0.12769) + | > log_mle: -0.08704 (-0.07683) + | > loss_dur: 0.19712 (0.20452) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.67685 (3.65333) + | > current_lr: 0.00001 + | > step_time: 1.40340 (2.04082) + | > loader_time: 0.00290 (0.04278) + + + --> STEP: 77/234 -- GLOBAL_STEP: 12245 + | > loss: 0.08153 (0.12557) + | > log_mle: -0.10005 (-0.07819) + | > loss_dur: 0.18158 (0.20376) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.37193 (3.72492) + | > current_lr: 0.00001 + | > step_time: 1.90130 (1.99631) + | > loader_time: 0.08550 (0.04122) + + + --> STEP: 82/234 -- GLOBAL_STEP: 12250 + | > loss: 0.10315 (0.12396) + | > log_mle: -0.08722 (-0.07891) + | > loss_dur: 0.19037 (0.20287) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.58611 (3.78434) + | > current_lr: 0.00001 + | > step_time: 1.30880 (1.96988) + | > loader_time: 0.00280 (0.03887) + + + --> STEP: 87/234 -- GLOBAL_STEP: 12255 + | > loss: 0.09519 (0.12247) + | > log_mle: -0.10205 (-0.08032) + | > loss_dur: 0.19724 (0.20280) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.78450 (3.90133) + | > current_lr: 0.00001 + | > step_time: 1.20870 (1.93483) + | > loader_time: 0.08540 (0.03873) + + + --> STEP: 92/234 -- GLOBAL_STEP: 12260 + | > loss: 0.03941 (0.11957) + | > log_mle: -0.14175 (-0.08311) + | > loss_dur: 0.18116 (0.20268) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.98314 (4.16337) + | > current_lr: 0.00001 + | > step_time: 1.91570 (1.92759) + | > loader_time: 0.08660 (0.03954) + + + --> STEP: 97/234 -- GLOBAL_STEP: 12265 + | > loss: 0.07304 (0.11663) + | > log_mle: -0.13444 (-0.08675) + | > loss_dur: 0.20748 (0.20338) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.55983 (4.57527) + | > current_lr: 0.00001 + | > step_time: 1.50180 (1.92811) + | > loader_time: 0.00300 (0.03851) + + + --> STEP: 102/234 -- GLOBAL_STEP: 12270 + | > loss: 0.10437 (0.11520) + | > log_mle: -0.11278 (-0.08923) + | > loss_dur: 0.21715 (0.20443) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.75283 (4.93644) + | > current_lr: 0.00001 + | > step_time: 1.20420 (1.91887) + | > loader_time: 0.00240 (0.03678) + + + --> STEP: 107/234 -- GLOBAL_STEP: 12275 + | > loss: 0.04500 (0.11283) + | > log_mle: -0.16293 (-0.09265) + | > loss_dur: 0.20793 (0.20548) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.29604 (5.29126) + | > current_lr: 0.00001 + | > step_time: 2.60610 (1.94516) + | > loader_time: 0.00250 (0.03519) + + + --> STEP: 112/234 -- GLOBAL_STEP: 12280 + | > loss: 0.06413 (0.11119) + | > log_mle: -0.16650 (-0.09567) + | > loss_dur: 0.23063 (0.20685) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.59149 (5.65885) + | > current_lr: 0.00001 + | > step_time: 1.70480 (1.95270) + | > loader_time: 0.00280 (0.03461) + + + --> STEP: 117/234 -- GLOBAL_STEP: 12285 + | > loss: 0.06156 (0.10931) + | > log_mle: -0.15933 (-0.09853) + | > loss_dur: 0.22089 (0.20784) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.69603 (5.91417) + | > current_lr: 0.00001 + | > step_time: 1.48320 (1.94079) + | > loader_time: 0.00240 (0.03325) + + + --> STEP: 122/234 -- GLOBAL_STEP: 12290 + | > loss: 0.06630 (0.10787) + | > log_mle: -0.14346 (-0.10045) + | > loss_dur: 0.20975 (0.20832) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.38430 (6.07915) + | > current_lr: 0.00001 + | > step_time: 3.20080 (1.96221) + | > loader_time: 0.00330 (0.03338) + + + --> STEP: 127/234 -- GLOBAL_STEP: 12295 + | > loss: 0.04128 (0.10560) + | > log_mle: -0.19216 (-0.10349) + | > loss_dur: 0.23344 (0.20909) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.86706 (6.37984) + | > current_lr: 0.00001 + | > step_time: 1.41080 (1.96311) + | > loader_time: 0.00470 (0.03283) + + + --> STEP: 132/234 -- GLOBAL_STEP: 12300 + | > loss: 0.03584 (0.10299) + | > log_mle: -0.17313 (-0.10684) + | > loss_dur: 0.20897 (0.20982) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.19527 (6.66927) + | > current_lr: 0.00001 + | > step_time: 1.76640 (1.94837) + | > loader_time: 0.00280 (0.03233) + + + --> STEP: 137/234 -- GLOBAL_STEP: 12305 + | > loss: 0.07877 (0.10095) + | > log_mle: -0.18264 (-0.11033) + | > loss_dur: 0.26141 (0.21128) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.59858 (7.06616) + | > current_lr: 0.00001 + | > step_time: 2.60850 (1.94586) + | > loader_time: 0.00310 (0.03191) + + + --> STEP: 142/234 -- GLOBAL_STEP: 12310 + | > loss: 0.02276 (0.09871) + | > log_mle: -0.19956 (-0.11344) + | > loss_dur: 0.22232 (0.21215) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.83492 (7.48170) + | > current_lr: 0.00001 + | > step_time: 1.59930 (1.93992) + | > loader_time: 0.00300 (0.03209) + + + --> STEP: 147/234 -- GLOBAL_STEP: 12315 + | > loss: 0.03673 (0.09594) + | > log_mle: -0.20081 (-0.11790) + | > loss_dur: 0.23754 (0.21384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.16705 (8.21345) + | > current_lr: 0.00001 + | > step_time: 2.61330 (1.94567) + | > loader_time: 0.08650 (0.03335) + + + --> STEP: 152/234 -- GLOBAL_STEP: 12320 + | > loss: -0.00263 (0.09295) + | > log_mle: -0.27037 (-0.12204) + | > loss_dur: 0.26774 (0.21499) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.82339 (8.66110) + | > current_lr: 0.00001 + | > step_time: 2.31460 (1.95093) + | > loader_time: 0.00350 (0.03345) + + + --> STEP: 157/234 -- GLOBAL_STEP: 12325 + | > loss: 0.02604 (0.08944) + | > log_mle: -0.22437 (-0.12673) + | > loss_dur: 0.25041 (0.21617) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.05693 (9.07834) + | > current_lr: 0.00001 + | > step_time: 1.30330 (1.95583) + | > loader_time: 0.07730 (0.03402) + + + --> STEP: 162/234 -- GLOBAL_STEP: 12330 + | > loss: -0.01588 (0.08616) + | > log_mle: -0.25633 (-0.13116) + | > loss_dur: 0.24045 (0.21732) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.56074 (9.54390) + | > current_lr: 0.00001 + | > step_time: 1.49070 (1.94297) + | > loader_time: 0.00450 (0.03408) + + + --> STEP: 167/234 -- GLOBAL_STEP: 12335 + | > loss: -0.06132 (0.08337) + | > log_mle: -0.32049 (-0.13520) + | > loss_dur: 0.25917 (0.21857) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.95895 (10.08621) + | > current_lr: 0.00001 + | > step_time: 3.10490 (1.98434) + | > loader_time: 0.08880 (0.03593) + + + --> STEP: 172/234 -- GLOBAL_STEP: 12340 + | > loss: -0.02703 (0.08043) + | > log_mle: -0.31423 (-0.13993) + | > loss_dur: 0.28720 (0.22035) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.90261 (10.70261) + | > current_lr: 0.00001 + | > step_time: 1.60320 (1.98704) + | > loader_time: 0.08310 (0.03653) + + + --> STEP: 177/234 -- GLOBAL_STEP: 12345 + | > loss: -0.00637 (0.07739) + | > log_mle: -0.27712 (-0.14438) + | > loss_dur: 0.27075 (0.22177) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.78644 (11.18836) + | > current_lr: 0.00001 + | > step_time: 1.99840 (2.00731) + | > loader_time: 0.08540 (0.03913) + + + --> STEP: 182/234 -- GLOBAL_STEP: 12350 + | > loss: -0.01788 (0.07468) + | > log_mle: -0.32010 (-0.14881) + | > loss_dur: 0.30221 (0.22349) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.22684 (11.69700) + | > current_lr: 0.00001 + | > step_time: 4.79270 (2.04281) + | > loader_time: 0.19490 (0.04227) + + + --> STEP: 187/234 -- GLOBAL_STEP: 12355 + | > loss: -0.04566 (0.07199) + | > log_mle: -0.32354 (-0.15320) + | > loss_dur: 0.27789 (0.22519) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.29659 (12.12350) + | > current_lr: 0.00001 + | > step_time: 2.10860 (2.08229) + | > loader_time: 0.10200 (0.04227) + + + --> STEP: 192/234 -- GLOBAL_STEP: 12360 + | > loss: -0.07998 (0.06888) + | > log_mle: -0.34410 (-0.15747) + | > loss_dur: 0.26412 (0.22635) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.89289 (12.68918) + | > current_lr: 0.00001 + | > step_time: 2.19170 (2.11304) + | > loader_time: 0.00360 (0.04381) + + + --> STEP: 197/234 -- GLOBAL_STEP: 12365 + | > loss: -0.06398 (0.06614) + | > log_mle: -0.31838 (-0.16161) + | > loss_dur: 0.25440 (0.22775) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.67920 (13.05575) + | > current_lr: 0.00001 + | > step_time: 2.78610 (2.14105) + | > loader_time: 0.00470 (0.04472) + + + --> STEP: 202/234 -- GLOBAL_STEP: 12370 + | > loss: -0.10978 (0.06345) + | > log_mle: -0.39860 (-0.16584) + | > loss_dur: 0.28882 (0.22929) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.40439 (13.53288) + | > current_lr: 0.00001 + | > step_time: 3.59610 (2.18109) + | > loader_time: 0.00470 (0.04557) + + + --> STEP: 207/234 -- GLOBAL_STEP: 12375 + | > loss: -0.08779 (0.06058) + | > log_mle: -0.38582 (-0.17005) + | > loss_dur: 0.29804 (0.23063) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.73719 (14.00098) + | > current_lr: 0.00001 + | > step_time: 6.80910 (2.29173) + | > loader_time: 0.19030 (0.04630) + + + --> STEP: 212/234 -- GLOBAL_STEP: 12380 + | > loss: -0.06525 (0.05757) + | > log_mle: -0.37038 (-0.17476) + | > loss_dur: 0.30513 (0.23233) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 46.24348 (14.59260) + | > current_lr: 0.00001 + | > step_time: 4.19140 (2.38529) + | > loader_time: 0.00600 (0.04618) + + + --> STEP: 217/234 -- GLOBAL_STEP: 12385 + | > loss: -0.08461 (0.05443) + | > log_mle: -0.38690 (-0.17938) + | > loss_dur: 0.30228 (0.23381) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.12314 (15.16870) + | > current_lr: 0.00001 + | > step_time: 10.70430 (2.45606) + | > loader_time: 0.10040 (0.04748) + + + --> STEP: 222/234 -- GLOBAL_STEP: 12390 + | > loss: -0.06784 (0.05158) + | > log_mle: -0.39840 (-0.18394) + | > loss_dur: 0.33056 (0.23551) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.99227 (15.59608) + | > current_lr: 0.00001 + | > step_time: 0.23560 (2.44267) + | > loader_time: 0.00450 (0.04821) + + + --> STEP: 227/234 -- GLOBAL_STEP: 12395 + | > loss: -0.06379 (0.04840) + | > log_mle: -0.37850 (-0.18885) + | > loss_dur: 0.31471 (0.23725) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.66630 (16.06273) + | > current_lr: 0.00001 + | > step_time: 0.24220 (2.39405) + | > loader_time: 0.00480 (0.04725) + + + --> STEP: 232/234 -- GLOBAL_STEP: 12400 + | > loss: 0.02944 (0.04654) + | > log_mle: -0.55130 (-0.19477) + | > loss_dur: 0.58074 (0.24131) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 68.42005 (17.08164) + | > current_lr: 0.00001 + | > step_time: 0.32650 (2.34843) + | > loader_time: 0.00540 (0.04632) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.09899 (-0.57763) + | > avg_loss: 0.01187 (-0.00826) + | > avg_log_mle: -0.25763 (-0.00609) + | > avg_loss_dur: 0.26950 (-0.00217) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_12402.pth + + > EPOCH: 53/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:36:23)  + + --> STEP: 3/234 -- GLOBAL_STEP: 12405 + | > loss: 0.22199 (0.20824) + | > log_mle: -0.06554 (-0.05389) + | > loss_dur: 0.28753 (0.26213) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.21284 (2.16340) + | > current_lr: 0.00001 + | > step_time: 8.59950 (6.90269) + | > loader_time: 0.00160 (0.03240) + + + --> STEP: 8/234 -- GLOBAL_STEP: 12410 + | > loss: 0.12569 (0.17330) + | > log_mle: -0.08447 (-0.06831) + | > loss_dur: 0.21016 (0.24161) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.99857 (3.53442) + | > current_lr: 0.00001 + | > step_time: 4.59900 (6.05490) + | > loader_time: 0.19040 (0.13226) + + + --> STEP: 13/234 -- GLOBAL_STEP: 12415 + | > loss: 0.15372 (0.15837) + | > log_mle: -0.06680 (-0.07122) + | > loss_dur: 0.22052 (0.22958) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.07000 (3.52491) + | > current_lr: 0.00001 + | > step_time: 2.68760 (4.90987) + | > loader_time: 0.00900 (0.09850) + + + --> STEP: 18/234 -- GLOBAL_STEP: 12420 + | > loss: 0.10196 (0.14736) + | > log_mle: -0.07502 (-0.07189) + | > loss_dur: 0.17698 (0.21925) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.33597 (3.38529) + | > current_lr: 0.00001 + | > step_time: 1.70320 (4.39705) + | > loader_time: 0.00600 (0.08597) + + + --> STEP: 23/234 -- GLOBAL_STEP: 12425 + | > loss: 0.09823 (0.14133) + | > log_mle: -0.08090 (-0.07211) + | > loss_dur: 0.17913 (0.21344) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.16660 (3.25877) + | > current_lr: 0.00001 + | > step_time: 0.94460 (3.72668) + | > loader_time: 0.00120 (0.06774) + + + --> STEP: 28/234 -- GLOBAL_STEP: 12430 + | > loss: 0.09398 (0.13644) + | > log_mle: -0.07450 (-0.07247) + | > loss_dur: 0.16847 (0.20891) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36568 (3.15515) + | > current_lr: 0.00001 + | > step_time: 1.81080 (3.29315) + | > loader_time: 0.00240 (0.05897) + + + --> STEP: 33/234 -- GLOBAL_STEP: 12435 + | > loss: 0.13353 (0.13251) + | > log_mle: -0.06629 (-0.07393) + | > loss_dur: 0.19982 (0.20644) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.13497 (3.04443) + | > current_lr: 0.00001 + | > step_time: 1.18350 (2.99699) + | > loader_time: 0.00200 (0.05287) + + + --> STEP: 38/234 -- GLOBAL_STEP: 12440 + | > loss: 0.12995 (0.13068) + | > log_mle: -0.08643 (-0.07541) + | > loss_dur: 0.21638 (0.20608) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.23327 (3.20225) + | > current_lr: 0.00001 + | > step_time: 1.47590 (2.76607) + | > loader_time: 0.00220 (0.04833) + + + --> STEP: 43/234 -- GLOBAL_STEP: 12445 + | > loss: 0.09259 (0.12950) + | > log_mle: -0.08801 (-0.07567) + | > loss_dur: 0.18060 (0.20517) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.54614 (3.27604) + | > current_lr: 0.00001 + | > step_time: 1.13390 (2.57141) + | > loader_time: 0.00170 (0.04298) + + + --> STEP: 48/234 -- GLOBAL_STEP: 12450 + | > loss: 0.09848 (0.12674) + | > log_mle: -0.07630 (-0.07663) + | > loss_dur: 0.17477 (0.20337) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.43161 (3.27572) + | > current_lr: 0.00001 + | > step_time: 1.19250 (2.44691) + | > loader_time: 0.00180 (0.03873) + + + --> STEP: 53/234 -- GLOBAL_STEP: 12455 + | > loss: 0.12153 (0.12557) + | > log_mle: -0.09588 (-0.07689) + | > loss_dur: 0.21741 (0.20246) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.86932 (3.21177) + | > current_lr: 0.00001 + | > step_time: 1.07920 (2.31939) + | > loader_time: 0.00220 (0.03528) + + + --> STEP: 58/234 -- GLOBAL_STEP: 12460 + | > loss: 0.10646 (0.12468) + | > log_mle: -0.08069 (-0.07763) + | > loss_dur: 0.18715 (0.20231) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.92092 (3.14820) + | > current_lr: 0.00001 + | > step_time: 1.31640 (2.21946) + | > loader_time: 0.00220 (0.03523) + + + --> STEP: 63/234 -- GLOBAL_STEP: 12465 + | > loss: 0.12186 (0.12189) + | > log_mle: -0.09542 (-0.07981) + | > loss_dur: 0.21728 (0.20171) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.35246 (3.33921) + | > current_lr: 0.00001 + | > step_time: 1.31780 (2.17620) + | > loader_time: 0.00280 (0.03263) + + + --> STEP: 68/234 -- GLOBAL_STEP: 12470 + | > loss: 0.12174 (0.12076) + | > log_mle: -0.08572 (-0.08032) + | > loss_dur: 0.20747 (0.20109) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.96736 (3.36856) + | > current_lr: 0.00001 + | > step_time: 1.11780 (2.11137) + | > loader_time: 0.00240 (0.03173) + + + --> STEP: 73/234 -- GLOBAL_STEP: 12475 + | > loss: 0.09780 (0.11982) + | > log_mle: -0.11076 (-0.08135) + | > loss_dur: 0.20856 (0.20117) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.20663 (3.75348) + | > current_lr: 0.00001 + | > step_time: 1.59900 (2.07767) + | > loader_time: 0.00260 (0.03365) + + + --> STEP: 78/234 -- GLOBAL_STEP: 12480 + | > loss: 0.11710 (0.11826) + | > log_mle: -0.08344 (-0.08231) + | > loss_dur: 0.20054 (0.20057) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.09239 (3.87703) + | > current_lr: 0.00001 + | > step_time: 1.40100 (2.04666) + | > loader_time: 0.00250 (0.03280) + + + --> STEP: 83/234 -- GLOBAL_STEP: 12485 + | > loss: 0.10190 (0.11650) + | > log_mle: -0.11471 (-0.08338) + | > loss_dur: 0.21660 (0.19987) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.69568 (3.99753) + | > current_lr: 0.00001 + | > step_time: 1.56510 (2.05088) + | > loader_time: 0.00220 (0.03098) + + + --> STEP: 88/234 -- GLOBAL_STEP: 12490 + | > loss: 0.05978 (0.11484) + | > log_mle: -0.14997 (-0.08510) + | > loss_dur: 0.20974 (0.19994) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.78065 (4.19470) + | > current_lr: 0.00001 + | > step_time: 2.28620 (2.03552) + | > loader_time: 0.00240 (0.03133) + + + --> STEP: 93/234 -- GLOBAL_STEP: 12495 + | > loss: 0.06083 (0.11241) + | > log_mle: -0.16232 (-0.08791) + | > loss_dur: 0.22315 (0.20032) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.58506 (4.50138) + | > current_lr: 0.00001 + | > step_time: 1.40130 (2.01852) + | > loader_time: 0.00300 (0.03059) + + + --> STEP: 98/234 -- GLOBAL_STEP: 12500 + | > loss: 0.11809 (0.11026) + | > log_mle: -0.09145 (-0.09071) + | > loss_dur: 0.20953 (0.20097) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.35088 (4.74362) + | > current_lr: 0.00001 + | > step_time: 1.89700 (2.00528) + | > loader_time: 0.00200 (0.02919) + + + --> STEP: 103/234 -- GLOBAL_STEP: 12505 + | > loss: 0.04588 (0.10791) + | > log_mle: -0.18408 (-0.09410) + | > loss_dur: 0.22996 (0.20201) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.82801 (5.04487) + | > current_lr: 0.00001 + | > step_time: 1.76030 (1.99655) + | > loader_time: 0.00290 (0.02875) + + + --> STEP: 108/234 -- GLOBAL_STEP: 12510 + | > loss: 0.09079 (0.10622) + | > log_mle: -0.13282 (-0.09695) + | > loss_dur: 0.22361 (0.20317) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.30794 (5.34284) + | > current_lr: 0.00001 + | > step_time: 1.09390 (1.96492) + | > loader_time: 0.00250 (0.02753) + + + --> STEP: 113/234 -- GLOBAL_STEP: 12515 + | > loss: 0.05622 (0.10426) + | > log_mle: -0.17911 (-0.10034) + | > loss_dur: 0.23532 (0.20460) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.65706 (5.88075) + | > current_lr: 0.00001 + | > step_time: 1.50040 (1.97701) + | > loader_time: 0.00780 (0.02799) + + + --> STEP: 118/234 -- GLOBAL_STEP: 12520 + | > loss: 0.07923 (0.10288) + | > log_mle: -0.14733 (-0.10289) + | > loss_dur: 0.22656 (0.20577) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.69870 (6.09366) + | > current_lr: 0.00001 + | > step_time: 1.98450 (1.97829) + | > loader_time: 0.00230 (0.02696) + + + --> STEP: 123/234 -- GLOBAL_STEP: 12525 + | > loss: 0.08197 (0.10162) + | > log_mle: -0.12073 (-0.10459) + | > loss_dur: 0.20270 (0.20622) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.92762 (6.22191) + | > current_lr: 0.00001 + | > step_time: 1.29860 (1.97002) + | > loader_time: 0.00380 (0.02668) + + + --> STEP: 128/234 -- GLOBAL_STEP: 12530 + | > loss: 0.03851 (0.09907) + | > log_mle: -0.17922 (-0.10809) + | > loss_dur: 0.21772 (0.20715) + | > amp_scaler: 16384.00000 (8384.00000) + | > grad_norm: 12.00149 (6.56592) + | > current_lr: 0.00001 + | > step_time: 3.10980 (1.97960) + | > loader_time: 0.08710 (0.02708) + + + --> STEP: 133/234 -- GLOBAL_STEP: 12535 + | > loss: 0.05134 (0.09681) + | > log_mle: -0.19664 (-0.11154) + | > loss_dur: 0.24798 (0.20835) + | > amp_scaler: 16384.00000 (8684.75188) + | > grad_norm: 19.37964 (6.91630) + | > current_lr: 0.00001 + | > step_time: 2.80490 (2.03157) + | > loader_time: 0.29540 (0.02901) + + + --> STEP: 138/234 -- GLOBAL_STEP: 12540 + | > loss: 0.06132 (0.09497) + | > log_mle: -0.15796 (-0.11469) + | > loss_dur: 0.21928 (0.20965) + | > amp_scaler: 16384.00000 (8963.71014) + | > grad_norm: 12.83048 (7.37909) + | > current_lr: 0.00001 + | > step_time: 1.68530 (2.04290) + | > loader_time: 0.00390 (0.02981) + + + --> STEP: 143/234 -- GLOBAL_STEP: 12545 + | > loss: 0.00753 (0.09238) + | > log_mle: -0.27595 (-0.11858) + | > loss_dur: 0.28348 (0.21096) + | > amp_scaler: 16384.00000 (9223.16084) + | > grad_norm: 35.40081 (8.00717) + | > current_lr: 0.00001 + | > step_time: 1.19860 (2.03626) + | > loader_time: 0.00290 (0.03018) + + + --> STEP: 148/234 -- GLOBAL_STEP: 12550 + | > loss: 0.01756 (0.08962) + | > log_mle: -0.20564 (-0.12251) + | > loss_dur: 0.22320 (0.21214) + | > amp_scaler: 16384.00000 (9465.08108) + | > grad_norm: 25.35040 (8.59663) + | > current_lr: 0.00001 + | > step_time: 2.20150 (2.03783) + | > loader_time: 0.00580 (0.02990) + + + --> STEP: 153/234 -- GLOBAL_STEP: 12555 + | > loss: -0.05700 (0.08618) + | > log_mle: -0.31364 (-0.12730) + | > loss_dur: 0.25665 (0.21348) + | > amp_scaler: 16384.00000 (9691.18954) + | > grad_norm: 33.83033 (9.18896) + | > current_lr: 0.00001 + | > step_time: 3.40590 (2.05827) + | > loader_time: 0.09310 (0.03133) + + + --> STEP: 158/234 -- GLOBAL_STEP: 12560 + | > loss: 0.01139 (0.08328) + | > log_mle: -0.25920 (-0.13157) + | > loss_dur: 0.27059 (0.21485) + | > amp_scaler: 16384.00000 (9902.98734) + | > grad_norm: 26.54273 (9.66683) + | > current_lr: 0.00001 + | > step_time: 9.51080 (2.12683) + | > loader_time: 0.09960 (0.03156) + + + --> STEP: 163/234 -- GLOBAL_STEP: 12565 + | > loss: 0.02110 (0.08015) + | > log_mle: -0.23508 (-0.13579) + | > loss_dur: 0.25618 (0.21594) + | > amp_scaler: 16384.00000 (10101.79141) + | > grad_norm: 25.14602 (10.20713) + | > current_lr: 0.00001 + | > step_time: 1.90840 (2.16168) + | > loader_time: 0.08360 (0.03540) + + + --> STEP: 168/234 -- GLOBAL_STEP: 12570 + | > loss: 0.00699 (0.07736) + | > log_mle: -0.27936 (-0.14005) + | > loss_dur: 0.28635 (0.21741) + | > amp_scaler: 16384.00000 (10288.76190) + | > grad_norm: 25.48586 (10.76329) + | > current_lr: 0.00001 + | > step_time: 1.19530 (2.17436) + | > loader_time: 0.10800 (0.03608) + + + --> STEP: 173/234 -- GLOBAL_STEP: 12575 + | > loss: -0.02742 (0.07418) + | > log_mle: -0.28622 (-0.14481) + | > loss_dur: 0.25880 (0.21899) + | > amp_scaler: 8192.00000 (10322.86705) + | > grad_norm: 25.27938 (11.16959) + | > current_lr: 0.00001 + | > step_time: 2.69980 (2.15857) + | > loader_time: 0.19500 (0.03717) + + + --> STEP: 178/234 -- GLOBAL_STEP: 12580 + | > loss: -0.04653 (0.07101) + | > log_mle: -0.34023 (-0.14955) + | > loss_dur: 0.29370 (0.22056) + | > amp_scaler: 8192.00000 (10263.01124) + | > grad_norm: 41.34897 (11.83460) + | > current_lr: 0.00001 + | > step_time: 2.20690 (2.15697) + | > loader_time: 0.08800 (0.03884) + + + --> STEP: 183/234 -- GLOBAL_STEP: 12585 + | > loss: -0.05256 (0.06823) + | > log_mle: -0.33665 (-0.15394) + | > loss_dur: 0.28410 (0.22217) + | > amp_scaler: 8192.00000 (10206.42623) + | > grad_norm: 35.39553 (12.37460) + | > current_lr: 0.00001 + | > step_time: 3.40370 (2.14898) + | > loader_time: 0.00360 (0.03834) + + + --> STEP: 188/234 -- GLOBAL_STEP: 12590 + | > loss: -0.06481 (0.06543) + | > log_mle: -0.34897 (-0.15837) + | > loss_dur: 0.28416 (0.22379) + | > amp_scaler: 8192.00000 (10152.85106) + | > grad_norm: 29.56192 (12.80780) + | > current_lr: 0.00001 + | > step_time: 2.10070 (2.19873) + | > loader_time: 0.07690 (0.03936) + + + --> STEP: 193/234 -- GLOBAL_STEP: 12595 + | > loss: -0.05643 (0.06244) + | > log_mle: -0.34747 (-0.16259) + | > loss_dur: 0.29104 (0.22503) + | > amp_scaler: 8192.00000 (10102.05181) + | > grad_norm: 34.83155 (13.29989) + | > current_lr: 0.00001 + | > step_time: 1.80090 (2.22520) + | > loader_time: 0.01210 (0.03937) + + + --> STEP: 198/234 -- GLOBAL_STEP: 12600 + | > loss: -0.05872 (0.05972) + | > log_mle: -0.33569 (-0.16663) + | > loss_dur: 0.27696 (0.22635) + | > amp_scaler: 8192.00000 (10053.81818) + | > grad_norm: 32.97104 (13.75000) + | > current_lr: 0.00001 + | > step_time: 2.11360 (2.23533) + | > loader_time: 0.08900 (0.04131) + + + --> STEP: 203/234 -- GLOBAL_STEP: 12605 + | > loss: -0.01453 (0.05728) + | > log_mle: -0.28487 (-0.17054) + | > loss_dur: 0.27034 (0.22782) + | > amp_scaler: 8192.00000 (10007.96059) + | > grad_norm: 23.83158 (14.20222) + | > current_lr: 0.00001 + | > step_time: 10.00290 (2.31038) + | > loader_time: 0.09930 (0.04224) + + + --> STEP: 208/234 -- GLOBAL_STEP: 12610 + | > loss: -0.04719 (0.05434) + | > log_mle: -0.35233 (-0.17504) + | > loss_dur: 0.30514 (0.22938) + | > amp_scaler: 8192.00000 (9964.30769) + | > grad_norm: 31.58165 (14.74604) + | > current_lr: 0.00001 + | > step_time: 3.80520 (2.35483) + | > loader_time: 0.00410 (0.04228) + + + --> STEP: 213/234 -- GLOBAL_STEP: 12615 + | > loss: -0.09223 (0.05115) + | > log_mle: -0.39282 (-0.17990) + | > loss_dur: 0.30059 (0.23105) + | > amp_scaler: 8192.00000 (9922.70423) + | > grad_norm: 53.39168 (15.43989) + | > current_lr: 0.00001 + | > step_time: 2.89590 (2.38449) + | > loader_time: 0.00460 (0.04502) + + + --> STEP: 218/234 -- GLOBAL_STEP: 12620 + | > loss: -0.05868 (0.04815) + | > log_mle: -0.35722 (-0.18431) + | > loss_dur: 0.29853 (0.23245) + | > amp_scaler: 8192.00000 (9883.00917) + | > grad_norm: 44.51612 (16.11453) + | > current_lr: 0.00001 + | > step_time: 10.18980 (2.46550) + | > loader_time: 0.00300 (0.04627) + + + --> STEP: 223/234 -- GLOBAL_STEP: 12625 + | > loss: -0.09262 (0.04513) + | > log_mle: -0.39840 (-0.18900) + | > loss_dur: 0.30578 (0.23412) + | > amp_scaler: 8192.00000 (9845.09417) + | > grad_norm: 43.68185 (16.72035) + | > current_lr: 0.00001 + | > step_time: 0.22920 (2.44905) + | > loader_time: 0.00400 (0.04613) + + + --> STEP: 228/234 -- GLOBAL_STEP: 12630 + | > loss: -0.06772 (0.04204) + | > log_mle: -0.40124 (-0.19388) + | > loss_dur: 0.33352 (0.23591) + | > amp_scaler: 8192.00000 (9808.84211) + | > grad_norm: 38.04324 (17.26754) + | > current_lr: 0.00001 + | > step_time: 0.24050 (2.40058) + | > loader_time: 0.00400 (0.04520) + + + --> STEP: 233/234 -- GLOBAL_STEP: 12635 + | > loss: 0.58304 (0.04299) + | > log_mle: -0.35804 (-0.19965) + | > loss_dur: 0.94108 (0.24264) + | > amp_scaler: 8192.00000 (9774.14592) + | > grad_norm: 34.64688 (17.87253) + | > current_lr: 0.00001 + | > step_time: 0.18980 (2.35475) + | > loader_time: 0.00260 (0.04434) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00404 (-0.09495) + | > avg_loss: 0.01230 (+0.00043) + | > avg_log_mle: -0.25470 (+0.00293) + | > avg_loss_dur: 0.26700 (-0.00250) + + + > EPOCH: 54/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:46:48)  + + --> STEP: 4/234 -- GLOBAL_STEP: 12640 + | > loss: 0.16220 (0.20084) + | > log_mle: -0.08601 (-0.06529) + | > loss_dur: 0.24821 (0.26614) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.71103 (4.17246) + | > current_lr: 0.00001 + | > step_time: 1.80140 (2.90348) + | > loader_time: 0.00250 (0.00253) + + + --> STEP: 9/234 -- GLOBAL_STEP: 12645 + | > loss: 0.11960 (0.16511) + | > log_mle: -0.09300 (-0.07529) + | > loss_dur: 0.21260 (0.24040) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.30111 (3.59805) + | > current_lr: 0.00001 + | > step_time: 3.40390 (3.13456) + | > loader_time: 0.09830 (0.06809) + + + --> STEP: 14/234 -- GLOBAL_STEP: 12650 + | > loss: 0.11974 (0.14893) + | > log_mle: -0.08680 (-0.07666) + | > loss_dur: 0.20655 (0.22559) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36985 (3.43193) + | > current_lr: 0.00001 + | > step_time: 9.71000 (4.06638) + | > loader_time: 0.10170 (0.05821) + + + --> STEP: 19/234 -- GLOBAL_STEP: 12655 + | > loss: 0.12893 (0.14091) + | > log_mle: -0.07201 (-0.07622) + | > loss_dur: 0.20094 (0.21713) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.21148 (3.38838) + | > current_lr: 0.00001 + | > step_time: 3.70160 (4.49075) + | > loader_time: 0.00130 (0.05265) + + + --> STEP: 24/234 -- GLOBAL_STEP: 12660 + | > loss: 0.11456 (0.13514) + | > log_mle: -0.07454 (-0.07649) + | > loss_dur: 0.18910 (0.21163) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.68525 (3.26717) + | > current_lr: 0.00001 + | > step_time: 0.80190 (3.80126) + | > loader_time: 0.08540 (0.05247) + + + --> STEP: 29/234 -- GLOBAL_STEP: 12665 + | > loss: 0.09160 (0.13041) + | > log_mle: -0.07307 (-0.07684) + | > loss_dur: 0.16466 (0.20725) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.30327 (3.13922) + | > current_lr: 0.00001 + | > step_time: 0.95950 (3.37310) + | > loader_time: 0.00210 (0.04627) + + + --> STEP: 34/234 -- GLOBAL_STEP: 12670 + | > loss: 0.13447 (0.12788) + | > log_mle: -0.08501 (-0.07864) + | > loss_dur: 0.21948 (0.20652) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.14851 (3.05742) + | > current_lr: 0.00001 + | > step_time: 0.68750 (3.02310) + | > loader_time: 0.00170 (0.03992) + + + --> STEP: 39/234 -- GLOBAL_STEP: 12675 + | > loss: 0.10946 (0.12504) + | > log_mle: -0.09214 (-0.08026) + | > loss_dur: 0.20160 (0.20530) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.78409 (3.24485) + | > current_lr: 0.00001 + | > step_time: 1.13190 (2.77887) + | > loader_time: 0.00170 (0.03699) + + + --> STEP: 44/234 -- GLOBAL_STEP: 12680 + | > loss: 0.10106 (0.12414) + | > log_mle: -0.08458 (-0.08037) + | > loss_dur: 0.18564 (0.20452) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.42858 (3.17669) + | > current_lr: 0.00001 + | > step_time: 1.09770 (2.59993) + | > loader_time: 0.00320 (0.03308) + + + --> STEP: 49/234 -- GLOBAL_STEP: 12685 + | > loss: 0.08367 (0.12103) + | > log_mle: -0.09197 (-0.08148) + | > loss_dur: 0.17564 (0.20251) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.55795 (3.14810) + | > current_lr: 0.00001 + | > step_time: 1.27150 (2.48854) + | > loader_time: 0.00200 (0.02992) + + + --> STEP: 54/234 -- GLOBAL_STEP: 12690 + | > loss: 0.09263 (0.12014) + | > log_mle: -0.09698 (-0.08179) + | > loss_dur: 0.18960 (0.20192) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.97481 (3.10602) + | > current_lr: 0.00001 + | > step_time: 1.67080 (2.40056) + | > loader_time: 0.00220 (0.02894) + + + --> STEP: 59/234 -- GLOBAL_STEP: 12695 + | > loss: 0.05711 (0.11895) + | > log_mle: -0.11224 (-0.08270) + | > loss_dur: 0.16935 (0.20165) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.89501 (3.11184) + | > current_lr: 0.00001 + | > step_time: 1.30590 (2.37006) + | > loader_time: 0.00220 (0.02999) + + + --> STEP: 64/234 -- GLOBAL_STEP: 12700 + | > loss: 0.09144 (0.11687) + | > log_mle: -0.08217 (-0.08430) + | > loss_dur: 0.17360 (0.20117) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.95527 (3.29185) + | > current_lr: 0.00001 + | > step_time: 1.15990 (2.33487) + | > loader_time: 0.00370 (0.02912) + + + --> STEP: 69/234 -- GLOBAL_STEP: 12705 + | > loss: 0.13180 (0.11575) + | > log_mle: -0.06978 (-0.08461) + | > loss_dur: 0.20158 (0.20036) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.08081 (3.39377) + | > current_lr: 0.00001 + | > step_time: 1.16610 (2.26008) + | > loader_time: 0.00340 (0.02721) + + + --> STEP: 74/234 -- GLOBAL_STEP: 12710 + | > loss: 0.07298 (0.11411) + | > log_mle: -0.09719 (-0.08612) + | > loss_dur: 0.17018 (0.20023) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.24676 (3.76550) + | > current_lr: 0.00001 + | > step_time: 1.16720 (2.20646) + | > loader_time: 0.00240 (0.02553) + + + --> STEP: 79/234 -- GLOBAL_STEP: 12715 + | > loss: 0.07684 (0.11243) + | > log_mle: -0.10607 (-0.08723) + | > loss_dur: 0.18291 (0.19967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.61732 (3.77042) + | > current_lr: 0.00001 + | > step_time: 1.91820 (2.17946) + | > loader_time: 0.00190 (0.02512) + + + --> STEP: 84/234 -- GLOBAL_STEP: 12720 + | > loss: 0.09564 (0.11094) + | > log_mle: -0.10583 (-0.08827) + | > loss_dur: 0.20147 (0.19921) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.12017 (3.90486) + | > current_lr: 0.00001 + | > step_time: 2.21000 (2.15157) + | > loader_time: 0.09560 (0.02574) + + + --> STEP: 89/234 -- GLOBAL_STEP: 12725 + | > loss: 0.05688 (0.10888) + | > log_mle: -0.13348 (-0.09027) + | > loss_dur: 0.19035 (0.19915) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.60125 (4.12068) + | > current_lr: 0.00001 + | > step_time: 1.30210 (2.13837) + | > loader_time: 0.08600 (0.02638) + + + --> STEP: 94/234 -- GLOBAL_STEP: 12730 + | > loss: 0.04915 (0.10627) + | > log_mle: -0.16816 (-0.09340) + | > loss_dur: 0.21731 (0.19967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.62042 (4.49615) + | > current_lr: 0.00001 + | > step_time: 1.60380 (2.11429) + | > loader_time: 0.00330 (0.02689) + + + --> STEP: 99/234 -- GLOBAL_STEP: 12735 + | > loss: 0.03485 (0.10404) + | > log_mle: -0.19744 (-0.09643) + | > loss_dur: 0.23230 (0.20047) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.41135 (4.89427) + | > current_lr: 0.00001 + | > step_time: 1.40620 (2.11573) + | > loader_time: 0.09410 (0.02900) + + + --> STEP: 104/234 -- GLOBAL_STEP: 12740 + | > loss: 0.02724 (0.10186) + | > log_mle: -0.20673 (-0.09980) + | > loss_dur: 0.23397 (0.20165) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.97086 (5.28864) + | > current_lr: 0.00001 + | > step_time: 1.57050 (2.10721) + | > loader_time: 0.00230 (0.02854) + + + --> STEP: 109/234 -- GLOBAL_STEP: 12745 + | > loss: 0.08770 (0.10055) + | > log_mle: -0.18015 (-0.10229) + | > loss_dur: 0.26785 (0.20284) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.94774 (5.66858) + | > current_lr: 0.00001 + | > step_time: 1.98330 (2.07618) + | > loader_time: 0.00260 (0.02736) + + + --> STEP: 114/234 -- GLOBAL_STEP: 12750 + | > loss: 0.04612 (0.09806) + | > log_mle: -0.16235 (-0.10548) + | > loss_dur: 0.20847 (0.20354) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.49420 (6.06601) + | > current_lr: 0.00001 + | > step_time: 1.31220 (2.06812) + | > loader_time: 0.08840 (0.02776) + + + --> STEP: 119/234 -- GLOBAL_STEP: 12755 + | > loss: 0.07187 (0.09690) + | > log_mle: -0.16078 (-0.10795) + | > loss_dur: 0.23265 (0.20485) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.29870 (6.27768) + | > current_lr: 0.00001 + | > step_time: 2.53690 (2.06635) + | > loader_time: 0.00240 (0.02740) + + + --> STEP: 124/234 -- GLOBAL_STEP: 12760 + | > loss: 0.04076 (0.09550) + | > log_mle: -0.18665 (-0.10980) + | > loss_dur: 0.22742 (0.20531) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.66255 (6.46461) + | > current_lr: 0.00001 + | > step_time: 1.61160 (2.03536) + | > loader_time: 0.00200 (0.02777) + + + --> STEP: 129/234 -- GLOBAL_STEP: 12765 + | > loss: 0.05563 (0.09323) + | > log_mle: -0.17210 (-0.11311) + | > loss_dur: 0.22774 (0.20634) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.37184 (6.87423) + | > current_lr: 0.00001 + | > step_time: 1.61310 (2.01837) + | > loader_time: 0.08410 (0.02743) + + + --> STEP: 134/234 -- GLOBAL_STEP: 12770 + | > loss: 0.04479 (0.09075) + | > log_mle: -0.21858 (-0.11682) + | > loss_dur: 0.26337 (0.20757) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.08141 (7.45546) + | > current_lr: 0.00001 + | > step_time: 2.70060 (2.00358) + | > loader_time: 0.00280 (0.02792) + + + --> STEP: 139/234 -- GLOBAL_STEP: 12775 + | > loss: -0.03878 (0.08830) + | > log_mle: -0.27704 (-0.12032) + | > loss_dur: 0.23826 (0.20862) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.81650 (7.84911) + | > current_lr: 0.00001 + | > step_time: 1.59020 (2.01630) + | > loader_time: 0.00340 (0.02901) + + + --> STEP: 144/234 -- GLOBAL_STEP: 12780 + | > loss: 0.00997 (0.08608) + | > log_mle: -0.25350 (-0.12404) + | > loss_dur: 0.26347 (0.21011) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.64140 (8.24707) + | > current_lr: 0.00001 + | > step_time: 1.81100 (2.02553) + | > loader_time: 0.00370 (0.02982) + + + --> STEP: 149/234 -- GLOBAL_STEP: 12785 + | > loss: -0.03479 (0.08295) + | > log_mle: -0.29150 (-0.12818) + | > loss_dur: 0.25671 (0.21113) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.57606 (8.75268) + | > current_lr: 0.00001 + | > step_time: 2.50190 (2.02473) + | > loader_time: 0.00810 (0.03007) + + + --> STEP: 154/234 -- GLOBAL_STEP: 12790 + | > loss: -0.01631 (0.07975) + | > log_mle: -0.25721 (-0.13265) + | > loss_dur: 0.24090 (0.21240) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.39011 (9.37641) + | > current_lr: 0.00001 + | > step_time: 4.61250 (2.04277) + | > loader_time: 0.19230 (0.03044) + + + --> STEP: 159/234 -- GLOBAL_STEP: 12795 + | > loss: -0.02349 (0.07677) + | > log_mle: -0.27650 (-0.13696) + | > loss_dur: 0.25301 (0.21373) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.19165 (9.91042) + | > current_lr: 0.00001 + | > step_time: 2.80170 (2.09563) + | > loader_time: 0.09820 (0.03319) + + + --> STEP: 164/234 -- GLOBAL_STEP: 12800 + | > loss: -0.00807 (0.07386) + | > log_mle: -0.26886 (-0.14111) + | > loss_dur: 0.26079 (0.21497) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.45341 (10.39953) + | > current_lr: 0.00001 + | > step_time: 1.82280 (2.13440) + | > loader_time: 0.19350 (0.03632) + + + --> STEP: 169/234 -- GLOBAL_STEP: 12805 + | > loss: 0.00836 (0.07124) + | > log_mle: -0.26405 (-0.14530) + | > loss_dur: 0.27241 (0.21654) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.61792 (11.09987) + | > current_lr: 0.00001 + | > step_time: 1.71130 (2.12580) + | > loader_time: 0.08570 (0.03583) + + + --> STEP: 174/234 -- GLOBAL_STEP: 12810 + | > loss: -0.08718 (0.06756) + | > log_mle: -0.34677 (-0.15045) + | > loss_dur: 0.25959 (0.21801) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.30285 (11.80480) + | > current_lr: 0.00001 + | > step_time: 4.19690 (2.13137) + | > loader_time: 0.00450 (0.03600) + + + --> STEP: 179/234 -- GLOBAL_STEP: 12815 + | > loss: -0.04834 (0.06470) + | > log_mle: -0.33565 (-0.15506) + | > loss_dur: 0.28731 (0.21976) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.06181 (12.34806) + | > current_lr: 0.00001 + | > step_time: 2.78710 (2.14707) + | > loader_time: 0.00310 (0.03557) + + + --> STEP: 184/234 -- GLOBAL_STEP: 12820 + | > loss: -0.03846 (0.06208) + | > log_mle: -0.31094 (-0.15927) + | > loss_dur: 0.27248 (0.22134) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.30628 (12.79246) + | > current_lr: 0.00001 + | > step_time: 2.40150 (2.14892) + | > loader_time: 0.00310 (0.03627) + + + --> STEP: 189/234 -- GLOBAL_STEP: 12825 + | > loss: -0.03415 (0.05936) + | > log_mle: -0.31051 (-0.16359) + | > loss_dur: 0.27636 (0.22294) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.83957 (13.38783) + | > current_lr: 0.00001 + | > step_time: 5.20190 (2.21170) + | > loader_time: 0.00320 (0.03848) + + + --> STEP: 194/234 -- GLOBAL_STEP: 12830 + | > loss: -0.06815 (0.05619) + | > log_mle: -0.33721 (-0.16787) + | > loss_dur: 0.26907 (0.22407) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.66727 (14.12859) + | > current_lr: 0.00001 + | > step_time: 1.67420 (2.23913) + | > loader_time: 0.00240 (0.03843) + + + --> STEP: 199/234 -- GLOBAL_STEP: 12835 + | > loss: -0.06395 (0.05359) + | > log_mle: -0.34400 (-0.17185) + | > loss_dur: 0.28006 (0.22543) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 48.10367 (14.85818) + | > current_lr: 0.00001 + | > step_time: 2.00560 (2.25526) + | > loader_time: 0.08570 (0.03851) + + + --> STEP: 204/234 -- GLOBAL_STEP: 12840 + | > loss: -0.05717 (0.05116) + | > log_mle: -0.37286 (-0.17582) + | > loss_dur: 0.31569 (0.22698) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.49380 (15.46712) + | > current_lr: 0.00001 + | > step_time: 5.40440 (2.32251) + | > loader_time: 0.19400 (0.04091) + + + --> STEP: 209/234 -- GLOBAL_STEP: 12845 + | > loss: -0.05252 (0.04831) + | > log_mle: -0.33500 (-0.18005) + | > loss_dur: 0.28248 (0.22836) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.38433 (16.02559) + | > current_lr: 0.00001 + | > step_time: 4.89390 (2.41240) + | > loader_time: 0.00470 (0.04380) + + + --> STEP: 214/234 -- GLOBAL_STEP: 12850 + | > loss: -0.09256 (0.04497) + | > log_mle: -0.36758 (-0.18506) + | > loss_dur: 0.27503 (0.23003) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.89106 (16.52246) + | > current_lr: 0.00001 + | > step_time: 3.78820 (2.46226) + | > loader_time: 0.10540 (0.04548) + + + --> STEP: 219/234 -- GLOBAL_STEP: 12855 + | > loss: -0.14376 (0.04175) + | > log_mle: -0.44674 (-0.18981) + | > loss_dur: 0.30298 (0.23155) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.42596 (17.16937) + | > current_lr: 0.00001 + | > step_time: 3.98980 (2.49279) + | > loader_time: 0.00420 (0.04631) + + + --> STEP: 224/234 -- GLOBAL_STEP: 12860 + | > loss: -0.10437 (0.03886) + | > log_mle: -0.41051 (-0.19429) + | > loss_dur: 0.30614 (0.23315) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.01117 (17.57330) + | > current_lr: 0.00001 + | > step_time: 1.80240 (2.49787) + | > loader_time: 0.09000 (0.04683) + + + --> STEP: 229/234 -- GLOBAL_STEP: 12865 + | > loss: -0.05901 (0.03601) + | > log_mle: -0.43080 (-0.19920) + | > loss_dur: 0.37178 (0.23521) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.02781 (18.19751) + | > current_lr: 0.00001 + | > step_time: 0.25680 (2.46973) + | > loader_time: 0.00630 (0.04630) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.84971 (+0.84567) + | > avg_loss: -0.00161 (-0.01391) + | > avg_log_mle: -0.26533 (-0.01063) + | > avg_loss_dur: 0.26372 (-0.00328) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_12870.pth + + > EPOCH: 55/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 20:57:34)  + + --> STEP: 0/234 -- GLOBAL_STEP: 12870 + | > loss: 0.18716 (0.18716) + | > log_mle: -0.10473 (-0.10473) + | > loss_dur: 0.29189 (0.29189) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.09702 (3.09702) + | > current_lr: 0.00001 + | > step_time: 6.01140 (6.01144) + | > loader_time: 15.66060 (15.66064) + + + --> STEP: 5/234 -- GLOBAL_STEP: 12875 + | > loss: 0.12792 (0.18583) + | > log_mle: -0.08430 (-0.07286) + | > loss_dur: 0.21222 (0.25869) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.95334 (4.14245) + | > current_lr: 0.00001 + | > step_time: 3.00880 (5.10014) + | > loader_time: 0.19330 (0.20043) + + + --> STEP: 10/234 -- GLOBAL_STEP: 12880 + | > loss: 0.09325 (0.15357) + | > log_mle: -0.09358 (-0.08158) + | > loss_dur: 0.18683 (0.23515) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.27205 (3.83730) + | > current_lr: 0.00001 + | > step_time: 2.16460 (4.68632) + | > loader_time: 0.00230 (0.13089) + + + --> STEP: 15/234 -- GLOBAL_STEP: 12885 + | > loss: 0.11142 (0.14216) + | > log_mle: -0.08282 (-0.08138) + | > loss_dur: 0.19425 (0.22354) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.63711 (3.72715) + | > current_lr: 0.00001 + | > step_time: 1.15300 (3.43652) + | > loader_time: 0.00100 (0.09918) + + + --> STEP: 20/234 -- GLOBAL_STEP: 12890 + | > loss: 0.11888 (0.13536) + | > log_mle: -0.07759 (-0.08052) + | > loss_dur: 0.19648 (0.21589) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41509 (3.60656) + | > current_lr: 0.00001 + | > step_time: 0.90310 (2.81664) + | > loader_time: 0.00180 (0.07494) + + + --> STEP: 25/234 -- GLOBAL_STEP: 12895 + | > loss: 0.13284 (0.13018) + | > log_mle: -0.07002 (-0.08047) + | > loss_dur: 0.20285 (0.21065) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.14573 (3.46209) + | > current_lr: 0.00001 + | > step_time: 2.09560 (2.50578) + | > loader_time: 0.00220 (0.06042) + + + --> STEP: 30/234 -- GLOBAL_STEP: 12900 + | > loss: 0.07929 (0.12389) + | > log_mle: -0.09958 (-0.08177) + | > loss_dur: 0.17887 (0.20566) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.62138 (3.30422) + | > current_lr: 0.00001 + | > step_time: 1.22820 (2.43989) + | > loader_time: 0.00200 (0.06225) + + + --> STEP: 35/234 -- GLOBAL_STEP: 12905 + | > loss: 0.09845 (0.12187) + | > log_mle: -0.09955 (-0.08332) + | > loss_dur: 0.19800 (0.20520) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.36792 (3.37034) + | > current_lr: 0.00001 + | > step_time: 2.21090 (2.30063) + | > loader_time: 0.00260 (0.05618) + + + --> STEP: 40/234 -- GLOBAL_STEP: 12910 + | > loss: 0.13986 (0.12038) + | > log_mle: -0.07727 (-0.08432) + | > loss_dur: 0.21713 (0.20471) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.39779 (3.37178) + | > current_lr: 0.00001 + | > step_time: 1.16160 (2.17194) + | > loader_time: 0.00200 (0.04959) + + + --> STEP: 45/234 -- GLOBAL_STEP: 12915 + | > loss: 0.09277 (0.11838) + | > log_mle: -0.11126 (-0.08518) + | > loss_dur: 0.20403 (0.20356) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.97877 (3.36574) + | > current_lr: 0.00001 + | > step_time: 1.29040 (2.09361) + | > loader_time: 0.00220 (0.04434) + + + --> STEP: 50/234 -- GLOBAL_STEP: 12920 + | > loss: 0.11985 (0.11574) + | > log_mle: -0.08180 (-0.08563) + | > loss_dur: 0.20165 (0.20137) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.37731 (3.29648) + | > current_lr: 0.00001 + | > step_time: 1.71950 (2.04005) + | > loader_time: 0.09310 (0.04194) + + + --> STEP: 55/234 -- GLOBAL_STEP: 12925 + | > loss: 0.09185 (0.11432) + | > log_mle: -0.10379 (-0.08635) + | > loss_dur: 0.19564 (0.20067) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.89491 (3.22418) + | > current_lr: 0.00001 + | > step_time: 1.37270 (1.99494) + | > loader_time: 0.00230 (0.03988) + + + --> STEP: 60/234 -- GLOBAL_STEP: 12930 + | > loss: 0.04997 (0.11209) + | > log_mle: -0.12018 (-0.08751) + | > loss_dur: 0.17014 (0.19961) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.76443 (3.24210) + | > current_lr: 0.00001 + | > step_time: 1.11680 (1.95202) + | > loader_time: 0.08360 (0.03956) + + + --> STEP: 65/234 -- GLOBAL_STEP: 12935 + | > loss: 0.07779 (0.11074) + | > log_mle: -0.09349 (-0.08860) + | > loss_dur: 0.17127 (0.19934) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.33483 (3.42665) + | > current_lr: 0.00001 + | > step_time: 1.23360 (1.93036) + | > loader_time: 0.09590 (0.03949) + + + --> STEP: 70/234 -- GLOBAL_STEP: 12940 + | > loss: 0.08812 (0.11013) + | > log_mle: -0.10179 (-0.08896) + | > loss_dur: 0.18990 (0.19908) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.86837 (3.62110) + | > current_lr: 0.00001 + | > step_time: 1.20950 (1.88457) + | > loader_time: 0.07490 (0.03905) + + + --> STEP: 75/234 -- GLOBAL_STEP: 12945 + | > loss: 0.09118 (0.10871) + | > log_mle: -0.11118 (-0.09046) + | > loss_dur: 0.20236 (0.19918) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.42073 (3.94089) + | > current_lr: 0.00001 + | > step_time: 2.11190 (1.85494) + | > loader_time: 0.08350 (0.03771) + + + --> STEP: 80/234 -- GLOBAL_STEP: 12950 + | > loss: 0.08382 (0.10722) + | > log_mle: -0.09007 (-0.09124) + | > loss_dur: 0.17389 (0.19846) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.75601 (3.95184) + | > current_lr: 0.00001 + | > step_time: 1.18720 (1.82643) + | > loader_time: 0.00240 (0.03654) + + + --> STEP: 85/234 -- GLOBAL_STEP: 12955 + | > loss: 0.08801 (0.10579) + | > log_mle: -0.10627 (-0.09250) + | > loss_dur: 0.19428 (0.19828) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.63569 (4.02723) + | > current_lr: 0.00001 + | > step_time: 1.98620 (1.82184) + | > loader_time: 0.00370 (0.03550) + + + --> STEP: 90/234 -- GLOBAL_STEP: 12960 + | > loss: 0.07223 (0.10362) + | > log_mle: -0.14010 (-0.09487) + | > loss_dur: 0.21233 (0.19849) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.67954 (4.32895) + | > current_lr: 0.00001 + | > step_time: 2.30570 (1.84216) + | > loader_time: 0.00310 (0.03555) + + + --> STEP: 95/234 -- GLOBAL_STEP: 12965 + | > loss: 0.01452 (0.10062) + | > log_mle: -0.21783 (-0.09878) + | > loss_dur: 0.23234 (0.19940) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.54388 (4.85731) + | > current_lr: 0.00001 + | > step_time: 1.40180 (1.84302) + | > loader_time: 0.00290 (0.03467) + + + --> STEP: 100/234 -- GLOBAL_STEP: 12970 + | > loss: 0.05715 (0.09888) + | > log_mle: -0.14995 (-0.10103) + | > loss_dur: 0.20710 (0.19992) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.54974 (5.08523) + | > current_lr: 0.00001 + | > step_time: 2.49050 (1.83436) + | > loader_time: 0.00320 (0.03396) + + + --> STEP: 105/234 -- GLOBAL_STEP: 12975 + | > loss: 0.07077 (0.09686) + | > log_mle: -0.11980 (-0.10407) + | > loss_dur: 0.19057 (0.20093) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.79241 (5.45445) + | > current_lr: 0.00001 + | > step_time: 1.00710 (1.81279) + | > loader_time: 0.08510 (0.03404) + + + --> STEP: 110/234 -- GLOBAL_STEP: 12980 + | > loss: 0.04824 (0.09521) + | > log_mle: -0.14955 (-0.10687) + | > loss_dur: 0.19779 (0.20208) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.34464 (5.78308) + | > current_lr: 0.00001 + | > step_time: 1.70190 (1.81952) + | > loader_time: 0.08780 (0.03588) + + + --> STEP: 115/234 -- GLOBAL_STEP: 12985 + | > loss: 0.06617 (0.09315) + | > log_mle: -0.16650 (-0.11018) + | > loss_dur: 0.23266 (0.20332) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.25658 (6.10844) + | > current_lr: 0.00001 + | > step_time: 2.11160 (1.81455) + | > loader_time: 0.00250 (0.03591) + + + --> STEP: 120/234 -- GLOBAL_STEP: 12990 + | > loss: 0.01972 (0.09137) + | > log_mle: -0.21182 (-0.11297) + | > loss_dur: 0.23154 (0.20434) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.27281 (6.41500) + | > current_lr: 0.00001 + | > step_time: 1.40620 (1.84385) + | > loader_time: 0.00390 (0.03522) + + + --> STEP: 125/234 -- GLOBAL_STEP: 12995 + | > loss: 0.04587 (0.09017) + | > log_mle: -0.19833 (-0.11465) + | > loss_dur: 0.24420 (0.20481) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.27314 (6.74962) + | > current_lr: 0.00001 + | > step_time: 1.90700 (1.83584) + | > loader_time: 0.19320 (0.03628) + + + --> STEP: 130/234 -- GLOBAL_STEP: 13000 + | > loss: 0.02436 (0.08787) + | > log_mle: -0.21146 (-0.11794) + | > loss_dur: 0.23583 (0.20581) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.61941 (7.18217) + | > current_lr: 0.00001 + | > step_time: 3.79470 (1.85763) + | > loader_time: 0.00340 (0.03639) + + + --> STEP: 135/234 -- GLOBAL_STEP: 13005 + | > loss: 0.06211 (0.08567) + | > log_mle: -0.14789 (-0.12117) + | > loss_dur: 0.21000 (0.20685) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.58848 (7.52804) + | > current_lr: 0.00001 + | > step_time: 4.90560 (1.87505) + | > loader_time: 0.00400 (0.03647) + + + --> STEP: 140/234 -- GLOBAL_STEP: 13010 + | > loss: 0.05960 (0.08321) + | > log_mle: -0.17702 (-0.12487) + | > loss_dur: 0.23662 (0.20808) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.14127 (7.99990) + | > current_lr: 0.00001 + | > step_time: 1.30970 (1.93171) + | > loader_time: 0.09500 (0.03800) + + + --> STEP: 145/234 -- GLOBAL_STEP: 13015 + | > loss: -0.01412 (0.08054) + | > log_mle: -0.26297 (-0.12914) + | > loss_dur: 0.24885 (0.20967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.89290 (8.58369) + | > current_lr: 0.00001 + | > step_time: 1.09520 (1.94307) + | > loader_time: 0.00380 (0.03749) + + + --> STEP: 150/234 -- GLOBAL_STEP: 13020 + | > loss: 0.00312 (0.07768) + | > log_mle: -0.24787 (-0.13312) + | > loss_dur: 0.25099 (0.21080) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.37323 (9.17758) + | > current_lr: 0.00001 + | > step_time: 1.90630 (1.93632) + | > loader_time: 0.00260 (0.03686) + + + --> STEP: 155/234 -- GLOBAL_STEP: 13025 + | > loss: -0.03851 (0.07429) + | > log_mle: -0.31007 (-0.13798) + | > loss_dur: 0.27155 (0.21227) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.79733 (9.71443) + | > current_lr: 0.00001 + | > step_time: 2.61130 (1.95663) + | > loader_time: 0.00310 (0.03700) + + + --> STEP: 160/234 -- GLOBAL_STEP: 13030 + | > loss: -0.04716 (0.07131) + | > log_mle: -0.30432 (-0.14222) + | > loss_dur: 0.25715 (0.21353) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.45339 (10.13565) + | > current_lr: 0.00001 + | > step_time: 1.18990 (1.95407) + | > loader_time: 0.00320 (0.03715) + + + --> STEP: 165/234 -- GLOBAL_STEP: 13035 + | > loss: -0.02397 (0.06854) + | > log_mle: -0.30142 (-0.14628) + | > loss_dur: 0.27746 (0.21482) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.37273 (10.64519) + | > current_lr: 0.00001 + | > step_time: 3.51710 (1.97659) + | > loader_time: 0.29130 (0.03835) + + + --> STEP: 170/234 -- GLOBAL_STEP: 13040 + | > loss: -0.04933 (0.06578) + | > log_mle: -0.33522 (-0.15063) + | > loss_dur: 0.28589 (0.21641) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.26881 (11.15871) + | > current_lr: 0.00001 + | > step_time: 2.41570 (2.01492) + | > loader_time: 0.08600 (0.03897) + + + --> STEP: 175/234 -- GLOBAL_STEP: 13045 + | > loss: -0.02979 (0.06234) + | > log_mle: -0.31065 (-0.15561) + | > loss_dur: 0.28086 (0.21795) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.54205 (11.70956) + | > current_lr: 0.00001 + | > step_time: 1.79710 (2.02464) + | > loader_time: 0.08810 (0.03946) + + + --> STEP: 180/234 -- GLOBAL_STEP: 13050 + | > loss: -0.04597 (0.05934) + | > log_mle: -0.32089 (-0.16024) + | > loss_dur: 0.27493 (0.21958) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.26661 (12.21057) + | > current_lr: 0.00001 + | > step_time: 1.81120 (2.04674) + | > loader_time: 0.08990 (0.03950) + + + --> STEP: 185/234 -- GLOBAL_STEP: 13055 + | > loss: -0.04354 (0.05669) + | > log_mle: -0.34118 (-0.16453) + | > loss_dur: 0.29764 (0.22123) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.07197 (12.68302) + | > current_lr: 0.00001 + | > step_time: 4.99460 (2.10868) + | > loader_time: 0.00580 (0.04067) + + + --> STEP: 190/234 -- GLOBAL_STEP: 13060 + | > loss: -0.05584 (0.05394) + | > log_mle: -0.32423 (-0.16873) + | > loss_dur: 0.26839 (0.22268) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.93666 (13.12733) + | > current_lr: 0.00001 + | > step_time: 5.09690 (2.11910) + | > loader_time: 0.09920 (0.04062) + + + --> STEP: 195/234 -- GLOBAL_STEP: 13065 + | > loss: -0.03328 (0.05093) + | > log_mle: -0.33002 (-0.17306) + | > loss_dur: 0.29675 (0.22399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 70.74415 (13.93676) + | > current_lr: 0.00001 + | > step_time: 8.91530 (2.18603) + | > loader_time: 0.28950 (0.04312) + + + --> STEP: 200/234 -- GLOBAL_STEP: 13070 + | > loss: -0.04073 (0.04830) + | > log_mle: -0.34090 (-0.17705) + | > loss_dur: 0.30017 (0.22535) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.07374 (14.63398) + | > current_lr: 0.00001 + | > step_time: 5.00390 (2.24197) + | > loader_time: 0.00400 (0.04460) + + + --> STEP: 205/234 -- GLOBAL_STEP: 13075 + | > loss: -0.05642 (0.04586) + | > log_mle: -0.33040 (-0.18093) + | > loss_dur: 0.27397 (0.22678) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.35231 (15.20871) + | > current_lr: 0.00001 + | > step_time: 8.59260 (2.36433) + | > loader_time: 0.11000 (0.04705) + + + --> STEP: 210/234 -- GLOBAL_STEP: 13080 + | > loss: -0.10831 (0.04274) + | > log_mle: -0.40494 (-0.18551) + | > loss_dur: 0.29662 (0.22825) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.13158 (15.69005) + | > current_lr: 0.00001 + | > step_time: 2.10380 (2.41195) + | > loader_time: 0.08870 (0.04862) + + + --> STEP: 215/234 -- GLOBAL_STEP: 13085 + | > loss: -0.07596 (0.03957) + | > log_mle: -0.35441 (-0.19020) + | > loss_dur: 0.27845 (0.22977) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.27883 (16.45282) + | > current_lr: 0.00001 + | > step_time: 8.11260 (2.52983) + | > loader_time: 0.09440 (0.05025) + + + --> STEP: 220/234 -- GLOBAL_STEP: 13090 + | > loss: -0.10671 (0.03618) + | > log_mle: -0.39922 (-0.19515) + | > loss_dur: 0.29250 (0.23133) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.21098 (17.07597) + | > current_lr: 0.00001 + | > step_time: 1.59480 (2.54864) + | > loader_time: 0.00450 (0.04923) + + + --> STEP: 225/234 -- GLOBAL_STEP: 13095 + | > loss: -0.13938 (0.03314) + | > log_mle: -0.45391 (-0.19981) + | > loss_dur: 0.31453 (0.23296) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.96172 (17.60239) + | > current_lr: 0.00001 + | > step_time: 0.23340 (2.50334) + | > loader_time: 0.00440 (0.04822) + + + --> STEP: 230/234 -- GLOBAL_STEP: 13100 + | > loss: -0.10222 (0.03058) + | > log_mle: -0.49924 (-0.20488) + | > loss_dur: 0.39702 (0.23547) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 65.49183 (18.20153) + | > current_lr: 0.00001 + | > step_time: 0.25670 (2.45433) + | > loader_time: 0.00370 (0.04727) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.47900 (-0.37071) + | > avg_loss: 0.00947 (+0.01108) + | > avg_log_mle: -0.25193 (+0.01340) + | > avg_loss_dur: 0.26140 (-0.00232) + + + > EPOCH: 56/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 21:08:12)  + + --> STEP: 1/234 -- GLOBAL_STEP: 13105 + | > loss: 0.13118 (0.13118) + | > log_mle: -0.07393 (-0.07393) + | > loss_dur: 0.20510 (0.20510) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.34037 (2.34037) + | > current_lr: 0.00001 + | > step_time: 6.80030 (6.80025) + | > loader_time: 0.09470 (0.09465) + + + --> STEP: 6/234 -- GLOBAL_STEP: 13110 + | > loss: 0.16668 (0.17171) + | > log_mle: -0.07284 (-0.07655) + | > loss_dur: 0.23952 (0.24826) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.22722 (4.90924) + | > current_lr: 0.00001 + | > step_time: 2.80060 (4.86766) + | > loader_time: 0.09330 (0.06700) + + + --> STEP: 11/234 -- GLOBAL_STEP: 13115 + | > loss: 0.11162 (0.13995) + | > log_mle: -0.08152 (-0.08531) + | > loss_dur: 0.19314 (0.22527) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.22755 (4.20792) + | > current_lr: 0.00001 + | > step_time: 1.02320 (3.54054) + | > loader_time: 0.00320 (0.07939) + + + --> STEP: 16/234 -- GLOBAL_STEP: 13120 + | > loss: 0.08611 (0.13079) + | > log_mle: -0.08855 (-0.08589) + | > loss_dur: 0.17466 (0.21668) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.36695 (3.92671) + | > current_lr: 0.00001 + | > step_time: 1.20650 (2.83147) + | > loader_time: 0.00180 (0.05529) + + + --> STEP: 21/234 -- GLOBAL_STEP: 13125 + | > loss: 0.10219 (0.12686) + | > log_mle: -0.07508 (-0.08428) + | > loss_dur: 0.17727 (0.21115) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.02085 (3.67284) + | > current_lr: 0.00001 + | > step_time: 1.29560 (2.44609) + | > loader_time: 0.00210 (0.04739) + + + --> STEP: 26/234 -- GLOBAL_STEP: 13130 + | > loss: 0.10769 (0.12167) + | > log_mle: -0.08996 (-0.08480) + | > loss_dur: 0.19765 (0.20646) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.38151 (3.45419) + | > current_lr: 0.00001 + | > step_time: 1.19070 (2.30874) + | > loader_time: 0.00180 (0.03865) + + + --> STEP: 31/234 -- GLOBAL_STEP: 13135 + | > loss: 0.13417 (0.11786) + | > log_mle: -0.09590 (-0.08613) + | > loss_dur: 0.23007 (0.20399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.65638 (3.29544) + | > current_lr: 0.00001 + | > step_time: 1.90860 (2.19008) + | > loader_time: 0.00250 (0.03276) + + + --> STEP: 36/234 -- GLOBAL_STEP: 13140 + | > loss: 0.09344 (0.11470) + | > log_mle: -0.10248 (-0.08778) + | > loss_dur: 0.19592 (0.20248) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.70436 (3.41206) + | > current_lr: 0.00001 + | > step_time: 4.31130 (2.27529) + | > loader_time: 0.09960 (0.03569) + + + --> STEP: 41/234 -- GLOBAL_STEP: 13145 + | > loss: 0.08886 (0.11273) + | > log_mle: -0.09295 (-0.08839) + | > loss_dur: 0.18181 (0.20111) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.37521 (3.38775) + | > current_lr: 0.00001 + | > step_time: 1.76220 (2.20170) + | > loader_time: 0.00140 (0.03163) + + + --> STEP: 46/234 -- GLOBAL_STEP: 13150 + | > loss: 0.09458 (0.11066) + | > log_mle: -0.09698 (-0.08924) + | > loss_dur: 0.19156 (0.19991) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.02673 (3.43429) + | > current_lr: 0.00001 + | > step_time: 2.40930 (2.12446) + | > loader_time: 0.08300 (0.03018) + + + --> STEP: 51/234 -- GLOBAL_STEP: 13155 + | > loss: 0.09261 (0.10835) + | > log_mle: -0.08088 (-0.08927) + | > loss_dur: 0.17349 (0.19762) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.28020 (3.35492) + | > current_lr: 0.00001 + | > step_time: 2.19920 (2.12124) + | > loader_time: 0.00150 (0.02910) + + + --> STEP: 56/234 -- GLOBAL_STEP: 13160 + | > loss: 0.12120 (0.10762) + | > log_mle: -0.10058 (-0.09027) + | > loss_dur: 0.22178 (0.19789) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.48574 (3.29233) + | > current_lr: 0.00001 + | > step_time: 1.17510 (2.06342) + | > loader_time: 0.00220 (0.02671) + + + --> STEP: 61/234 -- GLOBAL_STEP: 13165 + | > loss: 0.06919 (0.10475) + | > log_mle: -0.09693 (-0.09133) + | > loss_dur: 0.16612 (0.19608) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.79288 (3.30256) + | > current_lr: 0.00001 + | > step_time: 2.50340 (2.01861) + | > loader_time: 0.08600 (0.02880) + + + --> STEP: 66/234 -- GLOBAL_STEP: 13170 + | > loss: 0.09251 (0.10391) + | > log_mle: -0.08668 (-0.09225) + | > loss_dur: 0.17919 (0.19616) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.02096 (3.44565) + | > current_lr: 0.00001 + | > step_time: 1.70380 (1.99895) + | > loader_time: 0.00220 (0.03246) + + + --> STEP: 71/234 -- GLOBAL_STEP: 13175 + | > loss: 0.08057 (0.10319) + | > log_mle: -0.13120 (-0.09325) + | > loss_dur: 0.21177 (0.19644) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.76605 (3.94487) + | > current_lr: 0.00001 + | > step_time: 1.97790 (2.00846) + | > loader_time: 0.00230 (0.03036) + + + --> STEP: 76/234 -- GLOBAL_STEP: 13180 + | > loss: 0.09011 (0.10213) + | > log_mle: -0.11317 (-0.09441) + | > loss_dur: 0.20327 (0.19654) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.47485 (4.00486) + | > current_lr: 0.00001 + | > step_time: 2.10290 (2.07007) + | > loader_time: 0.00230 (0.02973) + + + --> STEP: 81/234 -- GLOBAL_STEP: 13185 + | > loss: 0.06430 (0.10056) + | > log_mle: -0.12415 (-0.09525) + | > loss_dur: 0.18846 (0.19581) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.60250 (4.08827) + | > current_lr: 0.00001 + | > step_time: 2.11660 (2.06171) + | > loader_time: 0.00200 (0.02804) + + + --> STEP: 86/234 -- GLOBAL_STEP: 13190 + | > loss: 0.06688 (0.09932) + | > log_mle: -0.12603 (-0.09643) + | > loss_dur: 0.19292 (0.19575) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.97855 (4.21331) + | > current_lr: 0.00001 + | > step_time: 1.80120 (2.03316) + | > loader_time: 0.00290 (0.02657) + + + --> STEP: 91/234 -- GLOBAL_STEP: 13195 + | > loss: 0.08646 (0.09745) + | > log_mle: -0.13226 (-0.09880) + | > loss_dur: 0.21872 (0.19625) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.57049 (4.42555) + | > current_lr: 0.00001 + | > step_time: 2.80970 (2.03447) + | > loader_time: 0.08300 (0.02716) + + + --> STEP: 96/234 -- GLOBAL_STEP: 13200 + | > loss: 0.08369 (0.09432) + | > log_mle: -0.12244 (-0.10251) + | > loss_dur: 0.20612 (0.19683) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.01304 (4.92728) + | > current_lr: 0.00001 + | > step_time: 3.70250 (2.05729) + | > loader_time: 0.00330 (0.02695) + + + --> STEP: 101/234 -- GLOBAL_STEP: 13205 + | > loss: 0.04107 (0.09257) + | > log_mle: -0.17770 (-0.10521) + | > loss_dur: 0.21878 (0.19778) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.17600 (5.35088) + | > current_lr: 0.00001 + | > step_time: 4.80270 (2.06832) + | > loader_time: 0.00280 (0.02577) + + + --> STEP: 106/234 -- GLOBAL_STEP: 13210 + | > loss: 0.06780 (0.09086) + | > log_mle: -0.17633 (-0.10814) + | > loss_dur: 0.24413 (0.19899) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.26912 (5.74622) + | > current_lr: 0.00001 + | > step_time: 2.39990 (2.07945) + | > loader_time: 0.00280 (0.02548) + + + --> STEP: 111/234 -- GLOBAL_STEP: 13215 + | > loss: 0.04626 (0.08931) + | > log_mle: -0.21267 (-0.11110) + | > loss_dur: 0.25893 (0.20041) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.30490 (6.24307) + | > current_lr: 0.00001 + | > step_time: 2.59870 (2.08227) + | > loader_time: 0.00970 (0.02609) + + + --> STEP: 116/234 -- GLOBAL_STEP: 13220 + | > loss: 0.06934 (0.08760) + | > log_mle: -0.18535 (-0.11400) + | > loss_dur: 0.25469 (0.20160) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.01053 (6.62245) + | > current_lr: 0.00001 + | > step_time: 3.20050 (2.12277) + | > loader_time: 0.10260 (0.02746) + + + --> STEP: 121/234 -- GLOBAL_STEP: 13225 + | > loss: 0.10223 (0.08625) + | > log_mle: -0.10457 (-0.11608) + | > loss_dur: 0.20680 (0.20233) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.53940 (6.79405) + | > current_lr: 0.00001 + | > step_time: 1.40310 (2.10239) + | > loader_time: 0.08400 (0.02773) + + + --> STEP: 126/234 -- GLOBAL_STEP: 13230 + | > loss: 0.00518 (0.08420) + | > log_mle: -0.22906 (-0.11876) + | > loss_dur: 0.23424 (0.20296) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.04746 (7.05753) + | > current_lr: 0.00001 + | > step_time: 2.29840 (2.07857) + | > loader_time: 0.00560 (0.02799) + + + --> STEP: 131/234 -- GLOBAL_STEP: 13235 + | > loss: -0.02210 (0.08171) + | > log_mle: -0.26543 (-0.12224) + | > loss_dur: 0.24333 (0.20395) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.69348 (7.53067) + | > current_lr: 0.00001 + | > step_time: 1.26800 (2.06343) + | > loader_time: 0.00250 (0.02759) + + + --> STEP: 136/234 -- GLOBAL_STEP: 13240 + | > loss: -0.03811 (0.07940) + | > log_mle: -0.30735 (-0.12569) + | > loss_dur: 0.26924 (0.20509) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.45394 (8.05696) + | > current_lr: 0.00001 + | > step_time: 1.69880 (2.06790) + | > loader_time: 0.00330 (0.02936) + + + --> STEP: 141/234 -- GLOBAL_STEP: 13245 + | > loss: 0.02556 (0.07748) + | > log_mle: -0.22607 (-0.12874) + | > loss_dur: 0.25163 (0.20622) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.25493 (8.34921) + | > current_lr: 0.00001 + | > step_time: 1.41520 (2.10460) + | > loader_time: 0.09600 (0.03123) + + + --> STEP: 146/234 -- GLOBAL_STEP: 13250 + | > loss: -0.02207 (0.07463) + | > log_mle: -0.27367 (-0.13328) + | > loss_dur: 0.25160 (0.20791) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.03705 (8.94260) + | > current_lr: 0.00001 + | > step_time: 0.89890 (2.11273) + | > loader_time: 0.00310 (0.03224) + + + --> STEP: 151/234 -- GLOBAL_STEP: 13255 + | > loss: -0.01389 (0.07178) + | > log_mle: -0.24314 (-0.13703) + | > loss_dur: 0.22925 (0.20880) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.82562 (9.26514) + | > current_lr: 0.00001 + | > step_time: 2.40580 (2.10233) + | > loader_time: 0.10610 (0.03197) + + + --> STEP: 156/234 -- GLOBAL_STEP: 13260 + | > loss: -0.02800 (0.06822) + | > log_mle: -0.27846 (-0.14207) + | > loss_dur: 0.25046 (0.21029) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.31960 (9.85223) + | > current_lr: 0.00001 + | > step_time: 1.89470 (2.10114) + | > loader_time: 0.00590 (0.03216) + + + --> STEP: 161/234 -- GLOBAL_STEP: 13265 + | > loss: -0.04961 (0.06522) + | > log_mle: -0.29851 (-0.14638) + | > loss_dur: 0.24891 (0.21160) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.98250 (10.30252) + | > current_lr: 0.00001 + | > step_time: 6.60020 (2.19879) + | > loader_time: 0.29310 (0.03607) + + + --> STEP: 166/234 -- GLOBAL_STEP: 13270 + | > loss: -0.01164 (0.06264) + | > log_mle: -0.24621 (-0.15012) + | > loss_dur: 0.23457 (0.21275) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.11445 (10.67810) + | > current_lr: 0.00001 + | > step_time: 2.79110 (2.28549) + | > loader_time: 0.00510 (0.03511) + + + --> STEP: 171/234 -- GLOBAL_STEP: 13275 + | > loss: -0.08734 (0.05936) + | > log_mle: -0.33998 (-0.15496) + | > loss_dur: 0.25264 (0.21432) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.58925 (11.39527) + | > current_lr: 0.00001 + | > step_time: 3.40400 (2.30703) + | > loader_time: 0.19570 (0.03581) + + + --> STEP: 176/234 -- GLOBAL_STEP: 13280 + | > loss: -0.04787 (0.05608) + | > log_mle: -0.31016 (-0.15973) + | > loss_dur: 0.26229 (0.21581) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.93230 (11.95620) + | > current_lr: 0.00001 + | > step_time: 1.00260 (2.33996) + | > loader_time: 0.00370 (0.03629) + + + --> STEP: 181/234 -- GLOBAL_STEP: 13285 + | > loss: -0.00220 (0.05347) + | > log_mle: -0.25803 (-0.16401) + | > loss_dur: 0.25583 (0.21748) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.08991 (12.40840) + | > current_lr: 0.00001 + | > step_time: 1.70950 (2.34403) + | > loader_time: 0.08890 (0.03681) + + + --> STEP: 186/234 -- GLOBAL_STEP: 13290 + | > loss: -0.00090 (0.05081) + | > log_mle: -0.29153 (-0.16844) + | > loss_dur: 0.29063 (0.21925) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.21294 (13.03265) + | > current_lr: 0.00001 + | > step_time: 3.20250 (2.41654) + | > loader_time: 0.00340 (0.04016) + + + --> STEP: 191/234 -- GLOBAL_STEP: 13295 + | > loss: -0.05081 (0.04784) + | > log_mle: -0.30308 (-0.17264) + | > loss_dur: 0.25227 (0.22047) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.30546 (13.68955) + | > current_lr: 0.00001 + | > step_time: 6.09440 (2.52660) + | > loader_time: 0.00470 (0.04115) + + + --> STEP: 196/234 -- GLOBAL_STEP: 13300 + | > loss: -0.03653 (0.04498) + | > log_mle: -0.30529 (-0.17696) + | > loss_dur: 0.26876 (0.22194) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.79391 (14.23285) + | > current_lr: 0.00001 + | > step_time: 13.29830 (2.58885) + | > loader_time: 0.19730 (0.04250) + + + --> STEP: 201/234 -- GLOBAL_STEP: 13305 + | > loss: 0.00292 (0.04255) + | > log_mle: -0.27489 (-0.18081) + | > loss_dur: 0.27781 (0.22336) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.19431 (14.84698) + | > current_lr: 0.00001 + | > step_time: 6.60410 (2.64784) + | > loader_time: 0.18960 (0.04425) + + + --> STEP: 206/234 -- GLOBAL_STEP: 13310 + | > loss: -0.08202 (0.03965) + | > log_mle: -0.36484 (-0.18510) + | > loss_dur: 0.28281 (0.22475) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 49.74221 (15.51880) + | > current_lr: 0.00001 + | > step_time: 3.09540 (2.68549) + | > loader_time: 0.00410 (0.04375) + + + --> STEP: 211/234 -- GLOBAL_STEP: 13315 + | > loss: -0.12098 (0.03649) + | > log_mle: -0.43220 (-0.18995) + | > loss_dur: 0.31122 (0.22644) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.87984 (16.07785) + | > current_lr: 0.00001 + | > step_time: 5.00640 (2.77592) + | > loader_time: 0.19110 (0.05362) + + + --> STEP: 216/234 -- GLOBAL_STEP: 13320 + | > loss: -0.11388 (0.03343) + | > log_mle: -0.41866 (-0.19455) + | > loss_dur: 0.30479 (0.22797) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.37920 (16.78509) + | > current_lr: 0.00001 + | > step_time: 3.20170 (2.81769) + | > loader_time: 0.00440 (0.05367) + + + --> STEP: 221/234 -- GLOBAL_STEP: 13325 + | > loss: -0.06931 (0.03036) + | > log_mle: -0.34692 (-0.19914) + | > loss_dur: 0.27761 (0.22951) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.79053 (17.26033) + | > current_lr: 0.00001 + | > step_time: 1.03480 (2.86580) + | > loader_time: 0.09060 (0.05383) + + + --> STEP: 226/234 -- GLOBAL_STEP: 13330 + | > loss: -0.12608 (0.02708) + | > log_mle: -0.43497 (-0.20421) + | > loss_dur: 0.30889 (0.23128) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.65479 (17.83285) + | > current_lr: 0.00001 + | > step_time: 0.23960 (2.80767) + | > loader_time: 0.00660 (0.05274) + + + --> STEP: 231/234 -- GLOBAL_STEP: 13335 + | > loss: -0.04778 (0.02480) + | > log_mle: -0.49197 (-0.20949) + | > loss_dur: 0.44418 (0.23429) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.32990 (18.51514) + | > current_lr: 0.00001 + | > step_time: 0.27460 (2.75272) + | > loader_time: 0.00540 (0.05169) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.09573 (-0.38327) + | > avg_loss: -0.01713 (-0.02659) + | > avg_log_mle: -0.27667 (-0.02474) + | > avg_loss_dur: 0.25954 (-0.00185) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_13338.pth + + > EPOCH: 57/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 21:20:20)  + + --> STEP: 2/234 -- GLOBAL_STEP: 13340 + | > loss: 0.25597 (0.19276) + | > log_mle: -0.05608 (-0.06613) + | > loss_dur: 0.31205 (0.25888) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.45786 (3.25333) + | > current_lr: 0.00001 + | > step_time: 4.89440 (9.04339) + | > loader_time: 0.00210 (0.05139) + + + --> STEP: 7/234 -- GLOBAL_STEP: 13345 + | > loss: 0.10769 (0.15910) + | > log_mle: -0.10256 (-0.08355) + | > loss_dur: 0.21025 (0.24265) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.83291 (3.88283) + | > current_lr: 0.00001 + | > step_time: 5.58890 (5.69813) + | > loader_time: 0.00290 (0.04321) + + + --> STEP: 12/234 -- GLOBAL_STEP: 13350 + | > loss: 0.11020 (0.13583) + | > log_mle: -0.08991 (-0.08876) + | > loss_dur: 0.20011 (0.22459) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.65897 (4.06211) + | > current_lr: 0.00001 + | > step_time: 3.10280 (4.97309) + | > loader_time: 0.00270 (0.06779) + + + --> STEP: 17/234 -- GLOBAL_STEP: 13355 + | > loss: 0.11835 (0.12911) + | > log_mle: -0.07543 (-0.08850) + | > loss_dur: 0.19378 (0.21761) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.87518 (3.75305) + | > current_lr: 0.00001 + | > step_time: 1.68960 (4.74005) + | > loader_time: 0.00300 (0.06679) + + + --> STEP: 22/234 -- GLOBAL_STEP: 13360 + | > loss: 0.06800 (0.12255) + | > log_mle: -0.09965 (-0.08821) + | > loss_dur: 0.16765 (0.21076) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.97794 (3.64545) + | > current_lr: 0.00001 + | > step_time: 4.90110 (4.54478) + | > loader_time: 0.00160 (0.06081) + + + --> STEP: 27/234 -- GLOBAL_STEP: 13365 + | > loss: 0.08048 (0.11706) + | > log_mle: -0.10249 (-0.08877) + | > loss_dur: 0.18298 (0.20583) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.84265 (3.47044) + | > current_lr: 0.00001 + | > step_time: 9.70810 (4.73391) + | > loader_time: 0.00160 (0.05962) + + + --> STEP: 32/234 -- GLOBAL_STEP: 13370 + | > loss: 0.07575 (0.11153) + | > log_mle: -0.11488 (-0.09038) + | > loss_dur: 0.19063 (0.20192) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.80417 (3.35134) + | > current_lr: 0.00001 + | > step_time: 5.29300 (4.49167) + | > loader_time: 0.10330 (0.05689) + + + --> STEP: 37/234 -- GLOBAL_STEP: 13375 + | > loss: 0.06964 (0.10881) + | > log_mle: -0.09480 (-0.09131) + | > loss_dur: 0.16444 (0.20011) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.66681 (3.50460) + | > current_lr: 0.00001 + | > step_time: 1.11570 (4.27691) + | > loader_time: 0.00200 (0.05479) + + + --> STEP: 42/234 -- GLOBAL_STEP: 13380 + | > loss: 0.12046 (0.10805) + | > log_mle: -0.08030 (-0.09162) + | > loss_dur: 0.20076 (0.19967) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.77221 (3.44511) + | > current_lr: 0.00001 + | > step_time: 1.19150 (3.90928) + | > loader_time: 0.00300 (0.04855) + + + --> STEP: 47/234 -- GLOBAL_STEP: 13385 + | > loss: 0.08919 (0.10568) + | > log_mle: -0.09844 (-0.09285) + | > loss_dur: 0.18763 (0.19853) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.23936 (3.47174) + | > current_lr: 0.00001 + | > step_time: 1.58970 (3.66521) + | > loader_time: 0.00200 (0.04536) + + + --> STEP: 52/234 -- GLOBAL_STEP: 13390 + | > loss: 0.10450 (0.10425) + | > log_mle: -0.08721 (-0.09266) + | > loss_dur: 0.19171 (0.19691) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.10174 (3.43859) + | > current_lr: 0.00001 + | > step_time: 1.57300 (3.47559) + | > loader_time: 0.00200 (0.04292) + + + --> STEP: 57/234 -- GLOBAL_STEP: 13395 + | > loss: 0.10977 (0.10349) + | > log_mle: -0.08688 (-0.09368) + | > loss_dur: 0.19665 (0.19718) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.25162 (3.34954) + | > current_lr: 0.00001 + | > step_time: 1.49510 (3.30361) + | > loader_time: 0.00220 (0.04406) + + + --> STEP: 62/234 -- GLOBAL_STEP: 13400 + | > loss: 0.08662 (0.10046) + | > log_mle: -0.14006 (-0.09558) + | > loss_dur: 0.22668 (0.19604) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.72097 (3.60005) + | > current_lr: 0.00001 + | > step_time: 2.13430 (3.21044) + | > loader_time: 0.07450 (0.04341) + + + --> STEP: 67/234 -- GLOBAL_STEP: 13405 + | > loss: 0.07001 (0.09941) + | > log_mle: -0.12197 (-0.09614) + | > loss_dur: 0.19198 (0.19554) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.62971 (4.12252) + | > current_lr: 0.00001 + | > step_time: 3.41130 (3.11673) + | > loader_time: 0.00270 (0.04159) + + + --> STEP: 72/234 -- GLOBAL_STEP: 13410 + | > loss: 0.10788 (0.09897) + | > log_mle: -0.10547 (-0.09672) + | > loss_dur: 0.21335 (0.19569) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.82531 (4.42058) + | > current_lr: 0.00001 + | > step_time: 2.29590 (3.01819) + | > loader_time: 0.00310 (0.03893) + + + --> STEP: 77/234 -- GLOBAL_STEP: 13415 + | > loss: 0.04936 (0.09719) + | > log_mle: -0.11922 (-0.09802) + | > loss_dur: 0.16859 (0.19521) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.11470 (4.57240) + | > current_lr: 0.00001 + | > step_time: 1.60310 (2.92248) + | > loader_time: 0.00290 (0.03660) + + + --> STEP: 82/234 -- GLOBAL_STEP: 13420 + | > loss: 0.07469 (0.09589) + | > log_mle: -0.10683 (-0.09870) + | > loss_dur: 0.18153 (0.19459) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.54867 (4.57379) + | > current_lr: 0.00001 + | > step_time: 3.49490 (2.86650) + | > loader_time: 0.00340 (0.03456) + + + --> STEP: 87/234 -- GLOBAL_STEP: 13425 + | > loss: 0.08604 (0.09484) + | > log_mle: -0.12193 (-0.10004) + | > loss_dur: 0.20797 (0.19487) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.77326 (4.68668) + | > current_lr: 0.00001 + | > step_time: 1.41100 (2.80783) + | > loader_time: 0.08730 (0.03475) + + + --> STEP: 92/234 -- GLOBAL_STEP: 13430 + | > loss: 0.02417 (0.09227) + | > log_mle: -0.16078 (-0.10278) + | > loss_dur: 0.18494 (0.19505) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.65222 (4.92074) + | > current_lr: 0.00001 + | > step_time: 1.58540 (2.74193) + | > loader_time: 0.01070 (0.03309) + + + --> STEP: 97/234 -- GLOBAL_STEP: 13435 + | > loss: 0.04981 (0.08965) + | > log_mle: -0.15313 (-0.10634) + | > loss_dur: 0.20294 (0.19599) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.37083 (5.38274) + | > current_lr: 0.00001 + | > step_time: 1.29610 (2.68930) + | > loader_time: 0.00210 (0.03329) + + + --> STEP: 102/234 -- GLOBAL_STEP: 13440 + | > loss: 0.08275 (0.08811) + | > log_mle: -0.13247 (-0.10882) + | > loss_dur: 0.21522 (0.19693) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.64815 (5.64769) + | > current_lr: 0.00001 + | > step_time: 0.91870 (2.66258) + | > loader_time: 0.00220 (0.03262) + + + --> STEP: 107/234 -- GLOBAL_STEP: 13445 + | > loss: 0.03534 (0.08582) + | > log_mle: -0.18098 (-0.11224) + | > loss_dur: 0.21632 (0.19806) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.80518 (5.98443) + | > current_lr: 0.00001 + | > step_time: 3.50350 (2.62701) + | > loader_time: 0.00310 (0.03123) + + + --> STEP: 112/234 -- GLOBAL_STEP: 13450 + | > loss: 0.03998 (0.08426) + | > log_mle: -0.18572 (-0.11528) + | > loss_dur: 0.22569 (0.19954) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.91461 (6.33384) + | > current_lr: 0.00001 + | > step_time: 1.69330 (2.57660) + | > loader_time: 0.00320 (0.03070) + + + --> STEP: 117/234 -- GLOBAL_STEP: 13455 + | > loss: 0.03403 (0.08250) + | > log_mle: -0.17956 (-0.11814) + | > loss_dur: 0.21359 (0.20065) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.22106 (6.69554) + | > current_lr: 0.00001 + | > step_time: 3.00930 (2.54435) + | > loader_time: 0.09030 (0.03179) + + + --> STEP: 122/234 -- GLOBAL_STEP: 13460 + | > loss: 0.04484 (0.08135) + | > log_mle: -0.16258 (-0.12007) + | > loss_dur: 0.20742 (0.20143) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.51882 (6.82842) + | > current_lr: 0.00001 + | > step_time: 1.20260 (2.50018) + | > loader_time: 0.09980 (0.03201) + + + --> STEP: 127/234 -- GLOBAL_STEP: 13465 + | > loss: 0.02610 (0.07924) + | > log_mle: -0.20814 (-0.12304) + | > loss_dur: 0.23424 (0.20228) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.18391 (7.37491) + | > current_lr: 0.00001 + | > step_time: 2.00110 (2.47953) + | > loader_time: 0.08640 (0.03220) + + + --> STEP: 132/234 -- GLOBAL_STEP: 13470 + | > loss: 0.00609 (0.07679) + | > log_mle: -0.19208 (-0.12637) + | > loss_dur: 0.19817 (0.20316) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.92633 (7.75588) + | > current_lr: 0.00001 + | > step_time: 2.71130 (2.47064) + | > loader_time: 0.00410 (0.03238) + + + --> STEP: 137/234 -- GLOBAL_STEP: 13475 + | > loss: 0.04705 (0.07481) + | > log_mle: -0.20321 (-0.12987) + | > loss_dur: 0.25026 (0.20468) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.64513 (8.19369) + | > current_lr: 0.00001 + | > step_time: 2.19400 (2.45867) + | > loader_time: 0.00290 (0.03333) + + + --> STEP: 142/234 -- GLOBAL_STEP: 13480 + | > loss: 0.01411 (0.07268) + | > log_mle: -0.21726 (-0.13298) + | > loss_dur: 0.23137 (0.20566) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.75436 (8.67341) + | > current_lr: 0.00001 + | > step_time: 1.39770 (2.43967) + | > loader_time: 0.08560 (0.03348) + + + --> STEP: 147/234 -- GLOBAL_STEP: 13485 + | > loss: 0.01239 (0.06978) + | > log_mle: -0.22042 (-0.13745) + | > loss_dur: 0.23281 (0.20724) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.78646 (9.32155) + | > current_lr: 0.00001 + | > step_time: 1.51610 (2.41548) + | > loader_time: 0.08380 (0.03358) + + + --> STEP: 152/234 -- GLOBAL_STEP: 13490 + | > loss: -0.02563 (0.06683) + | > log_mle: -0.28873 (-0.14158) + | > loss_dur: 0.26309 (0.20841) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.95486 (9.90590) + | > current_lr: 0.00001 + | > step_time: 1.68450 (2.40974) + | > loader_time: 0.00490 (0.03436) + + + --> STEP: 157/234 -- GLOBAL_STEP: 13495 + | > loss: 0.01010 (0.06352) + | > log_mle: -0.24347 (-0.14622) + | > loss_dur: 0.25356 (0.20974) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.49615 (10.67420) + | > current_lr: 0.00001 + | > step_time: 2.77890 (2.41004) + | > loader_time: 0.00370 (0.03510) + + + --> STEP: 162/234 -- GLOBAL_STEP: 13500 + | > loss: -0.04418 (0.06022) + | > log_mle: -0.27712 (-0.15069) + | > loss_dur: 0.23294 (0.21092) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.84213 (11.22081) + | > current_lr: 0.00001 + | > step_time: 3.20550 (2.43245) + | > loader_time: 0.00360 (0.03524) + + + --> STEP: 167/234 -- GLOBAL_STEP: 13505 + | > loss: -0.09018 (0.05743) + | > log_mle: -0.34351 (-0.15475) + | > loss_dur: 0.25334 (0.21218) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.82031 (11.80093) + | > current_lr: 0.00001 + | > step_time: 2.39290 (2.42121) + | > loader_time: 0.00780 (0.03492) + + + --> STEP: 172/234 -- GLOBAL_STEP: 13510 + | > loss: -0.05527 (0.05438) + | > log_mle: -0.33822 (-0.15953) + | > loss_dur: 0.28295 (0.21391) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.05900 (12.45209) + | > current_lr: 0.00001 + | > step_time: 2.48770 (2.44621) + | > loader_time: 0.00320 (0.03722) + + + --> STEP: 177/234 -- GLOBAL_STEP: 13515 + | > loss: -0.03157 (0.05134) + | > log_mle: -0.30013 (-0.16404) + | > loss_dur: 0.26857 (0.21539) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.70567 (12.98091) + | > current_lr: 0.00001 + | > step_time: 1.59670 (2.47724) + | > loader_time: 0.00610 (0.03829) + + + --> STEP: 182/234 -- GLOBAL_STEP: 13520 + | > loss: -0.04770 (0.04869) + | > log_mle: -0.34200 (-0.16851) + | > loss_dur: 0.29429 (0.21720) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.51077 (13.48125) + | > current_lr: 0.00001 + | > step_time: 6.01480 (2.51212) + | > loader_time: 0.08520 (0.03990) + + + --> STEP: 187/234 -- GLOBAL_STEP: 13525 + | > loss: -0.07226 (0.04602) + | > log_mle: -0.33794 (-0.17285) + | > loss_dur: 0.26568 (0.21887) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.22633 (14.15247) + | > current_lr: 0.00001 + | > step_time: 3.39730 (2.59996) + | > loader_time: 0.19570 (0.04148) + + + --> STEP: 192/234 -- GLOBAL_STEP: 13530 + | > loss: -0.10031 (0.04295) + | > log_mle: -0.36375 (-0.17712) + | > loss_dur: 0.26345 (0.22007) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.28590 (14.76800) + | > current_lr: 0.00001 + | > step_time: 4.30920 (2.62967) + | > loader_time: 0.09520 (0.04193) + + + --> STEP: 197/234 -- GLOBAL_STEP: 13535 + | > loss: -0.08323 (0.04020) + | > log_mle: -0.34069 (-0.18125) + | > loss_dur: 0.25746 (0.22145) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.32833 (15.35164) + | > current_lr: 0.00001 + | > step_time: 4.90970 (2.70720) + | > loader_time: 0.19210 (0.04491) + + + --> STEP: 202/234 -- GLOBAL_STEP: 13540 + | > loss: -0.13042 (0.03759) + | > log_mle: -0.41887 (-0.18547) + | > loss_dur: 0.28845 (0.22306) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 48.76180 (15.88558) + | > current_lr: 0.00001 + | > step_time: 7.68990 (2.84504) + | > loader_time: 0.20110 (0.04739) + + + --> STEP: 207/234 -- GLOBAL_STEP: 13545 + | > loss: -0.12321 (0.03475) + | > log_mle: -0.40710 (-0.18968) + | > loss_dur: 0.28390 (0.22443) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.31493 (16.37857) + | > current_lr: 0.00001 + | > step_time: 8.39970 (2.87168) + | > loader_time: 0.10180 (0.04717) + + + --> STEP: 212/234 -- GLOBAL_STEP: 13550 + | > loss: -0.09814 (0.03160) + | > log_mle: -0.39160 (-0.19443) + | > loss_dur: 0.29347 (0.22603) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 53.94717 (16.97069) + | > current_lr: 0.00001 + | > step_time: 5.50980 (2.95161) + | > loader_time: 0.09730 (0.04880) + + + --> STEP: 217/234 -- GLOBAL_STEP: 13555 + | > loss: -0.10827 (0.02843) + | > log_mle: -0.40691 (-0.19908) + | > loss_dur: 0.29864 (0.22750) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 60.66978 (17.57389) + | > current_lr: 0.00001 + | > step_time: 9.09620 (3.01313) + | > loader_time: 0.30280 (0.05092) + + + --> STEP: 222/234 -- GLOBAL_STEP: 13560 + | > loss: -0.08748 (0.02556) + | > log_mle: -0.41482 (-0.20359) + | > loss_dur: 0.32734 (0.22915) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 57.04537 (18.36521) + | > current_lr: 0.00001 + | > step_time: 4.01030 (3.02055) + | > loader_time: 0.00350 (0.04986) + + + --> STEP: 227/234 -- GLOBAL_STEP: 13565 + | > loss: -0.08847 (0.02240) + | > log_mle: -0.39618 (-0.20847) + | > loss_dur: 0.30771 (0.23087) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.64212 (19.21107) + | > current_lr: 0.00001 + | > step_time: 0.24890 (2.97907) + | > loader_time: 0.00440 (0.04926) + + + --> STEP: 232/234 -- GLOBAL_STEP: 13570 + | > loss: 0.00623 (0.02051) + | > log_mle: -0.57770 (-0.21447) + | > loss_dur: 0.58394 (0.23498) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 51.81219 (19.95366) + | > current_lr: 0.00001 + | > step_time: 0.33710 (2.92083) + | > loader_time: 0.00580 (0.04830) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12794 (+0.03221) + | > avg_loss: -0.01674 (+0.00038) + | > avg_log_mle: -0.27538 (+0.00129) + | > avg_loss_dur: 0.25864 (-0.00091) + + + > EPOCH: 58/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 21:33:03)  + + --> STEP: 3/234 -- GLOBAL_STEP: 13575 + | > loss: 0.19949 (0.17976) + | > log_mle: -0.08767 (-0.07680) + | > loss_dur: 0.28716 (0.25655) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41044 (2.32541) + | > current_lr: 0.00001 + | > step_time: 1.40240 (2.06805) + | > loader_time: 0.00350 (0.00343) + + + --> STEP: 8/234 -- GLOBAL_STEP: 13580 + | > loss: 0.11311 (0.14906) + | > log_mle: -0.10495 (-0.09018) + | > loss_dur: 0.21806 (0.23924) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.65671 (3.46386) + | > current_lr: 0.00001 + | > step_time: 13.39560 (5.37911) + | > loader_time: 0.59980 (0.08787) + + + --> STEP: 13/234 -- GLOBAL_STEP: 13585 + | > loss: 0.12168 (0.12989) + | > log_mle: -0.08748 (-0.09240) + | > loss_dur: 0.20916 (0.22229) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.36260 (4.01812) + | > current_lr: 0.00001 + | > step_time: 2.20580 (5.16487) + | > loader_time: 0.00230 (0.08435) + + + --> STEP: 18/234 -- GLOBAL_STEP: 13590 + | > loss: 0.09708 (0.12088) + | > log_mle: -0.09534 (-0.09259) + | > loss_dur: 0.19243 (0.21346) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.55046 (3.82402) + | > current_lr: 0.00001 + | > step_time: 5.09370 (4.71319) + | > loader_time: 0.59680 (0.10008) + + + --> STEP: 23/234 -- GLOBAL_STEP: 13595 + | > loss: 0.06691 (0.11273) + | > log_mle: -0.10069 (-0.09248) + | > loss_dur: 0.16760 (0.20521) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.53466 (3.68696) + | > current_lr: 0.00001 + | > step_time: 6.20420 (4.50632) + | > loader_time: 0.39800 (0.11764) + + + --> STEP: 28/234 -- GLOBAL_STEP: 13600 + | > loss: 0.06839 (0.10941) + | > log_mle: -0.09320 (-0.09266) + | > loss_dur: 0.16158 (0.20207) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.34918 (3.54526) + | > current_lr: 0.00001 + | > step_time: 7.39280 (4.60517) + | > loader_time: 0.00680 (0.10679) + + + --> STEP: 33/234 -- GLOBAL_STEP: 13605 + | > loss: 0.12854 (0.10690) + | > log_mle: -0.08571 (-0.09397) + | > loss_dur: 0.21425 (0.20086) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.29315 (3.41013) + | > current_lr: 0.00001 + | > step_time: 1.10170 (4.41416) + | > loader_time: 0.00230 (0.14122) + + + --> STEP: 38/234 -- GLOBAL_STEP: 13610 + | > loss: 0.11710 (0.10525) + | > log_mle: -0.10645 (-0.09534) + | > loss_dur: 0.22355 (0.20060) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.21410 (3.61545) + | > current_lr: 0.00001 + | > step_time: 3.38930 (4.06415) + | > loader_time: 0.00400 (0.12488) + + + --> STEP: 43/234 -- GLOBAL_STEP: 13615 + | > loss: 0.07017 (0.10406) + | > log_mle: -0.10702 (-0.09559) + | > loss_dur: 0.17719 (0.19965) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.46349 (3.63625) + | > current_lr: 0.00001 + | > step_time: 2.81000 (3.82687) + | > loader_time: 0.09340 (0.11276) + + + --> STEP: 48/234 -- GLOBAL_STEP: 13620 + | > loss: 0.07680 (0.10150) + | > log_mle: -0.09349 (-0.09642) + | > loss_dur: 0.17029 (0.19792) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.03889 (3.64316) + | > current_lr: 0.00001 + | > step_time: 1.79660 (3.63479) + | > loader_time: 0.00120 (0.10297) + + + --> STEP: 53/234 -- GLOBAL_STEP: 13625 + | > loss: 0.10196 (0.10042) + | > log_mle: -0.11454 (-0.09664) + | > loss_dur: 0.21650 (0.19706) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.72524 (3.56541) + | > current_lr: 0.00001 + | > step_time: 2.50700 (3.50325) + | > loader_time: 0.08210 (0.09814) + + + --> STEP: 58/234 -- GLOBAL_STEP: 13630 + | > loss: 0.07523 (0.09923) + | > log_mle: -0.09901 (-0.09728) + | > loss_dur: 0.17424 (0.19651) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.05431 (3.48810) + | > current_lr: 0.00001 + | > step_time: 0.78310 (3.33886) + | > loader_time: 0.00220 (0.09160) + + + --> STEP: 63/234 -- GLOBAL_STEP: 13635 + | > loss: 0.10251 (0.09659) + | > log_mle: -0.11357 (-0.09941) + | > loss_dur: 0.21608 (0.19600) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.89339 (3.69951) + | > current_lr: 0.00001 + | > step_time: 1.30050 (3.25369) + | > loader_time: 0.09040 (0.08736) + + + --> STEP: 68/234 -- GLOBAL_STEP: 13640 + | > loss: 0.07912 (0.09510) + | > log_mle: -0.10558 (-0.09988) + | > loss_dur: 0.18470 (0.19499) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.30045 (3.72473) + | > current_lr: 0.00001 + | > step_time: 2.10470 (3.23078) + | > loader_time: 0.09700 (0.08676) + + + --> STEP: 73/234 -- GLOBAL_STEP: 13645 + | > loss: 0.06724 (0.09475) + | > log_mle: -0.12977 (-0.10091) + | > loss_dur: 0.19700 (0.19566) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.23788 (4.05623) + | > current_lr: 0.00001 + | > step_time: 2.28540 (3.13518) + | > loader_time: 0.00230 (0.08197) + + + --> STEP: 78/234 -- GLOBAL_STEP: 13650 + | > loss: 0.09972 (0.09373) + | > log_mle: -0.10197 (-0.10181) + | > loss_dur: 0.20169 (0.19554) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.52373 (4.17795) + | > current_lr: 0.00001 + | > step_time: 1.70110 (3.05645) + | > loader_time: 0.00310 (0.08019) + + + --> STEP: 83/234 -- GLOBAL_STEP: 13655 + | > loss: 0.09383 (0.09245) + | > log_mle: -0.13232 (-0.10282) + | > loss_dur: 0.22615 (0.19528) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.89890 (4.29193) + | > current_lr: 0.00001 + | > step_time: 1.50410 (2.98825) + | > loader_time: 0.08580 (0.07863) + + + --> STEP: 88/234 -- GLOBAL_STEP: 13660 + | > loss: 0.02555 (0.09081) + | > log_mle: -0.16922 (-0.10453) + | > loss_dur: 0.19477 (0.19534) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.68133 (4.45318) + | > current_lr: 0.00001 + | > step_time: 2.09720 (2.94014) + | > loader_time: 0.08880 (0.07531) + + + --> STEP: 93/234 -- GLOBAL_STEP: 13665 + | > loss: 0.02929 (0.08842) + | > log_mle: -0.18062 (-0.10730) + | > loss_dur: 0.20991 (0.19572) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.32487 (4.80332) + | > current_lr: 0.00001 + | > step_time: 4.12110 (2.90674) + | > loader_time: 0.11030 (0.07363) + + + --> STEP: 98/234 -- GLOBAL_STEP: 13670 + | > loss: 0.09261 (0.08627) + | > log_mle: -0.10941 (-0.11005) + | > loss_dur: 0.20202 (0.19632) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.48344 (5.15897) + | > current_lr: 0.00001 + | > step_time: 3.40150 (2.87415) + | > loader_time: 0.19510 (0.07196) + + + --> STEP: 103/234 -- GLOBAL_STEP: 13675 + | > loss: 0.01978 (0.08401) + | > log_mle: -0.20271 (-0.11338) + | > loss_dur: 0.22249 (0.19739) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.42851 (5.65508) + | > current_lr: 0.00001 + | > step_time: 1.50760 (2.82507) + | > loader_time: 0.09140 (0.07029) + + + --> STEP: 108/234 -- GLOBAL_STEP: 13680 + | > loss: 0.05515 (0.08203) + | > log_mle: -0.15244 (-0.11623) + | > loss_dur: 0.20759 (0.19826) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.26424 (5.97382) + | > current_lr: 0.00001 + | > step_time: 3.68870 (2.82484) + | > loader_time: 0.00470 (0.06886) + + + --> STEP: 113/234 -- GLOBAL_STEP: 13685 + | > loss: 0.01869 (0.07983) + | > log_mle: -0.19875 (-0.11964) + | > loss_dur: 0.21744 (0.19947) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.09461 (6.43254) + | > current_lr: 0.00001 + | > step_time: 1.81580 (2.76891) + | > loader_time: 0.00250 (0.06597) + + + --> STEP: 118/234 -- GLOBAL_STEP: 13690 + | > loss: 0.04397 (0.07832) + | > log_mle: -0.16572 (-0.12217) + | > loss_dur: 0.20969 (0.20050) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.90767 (6.65032) + | > current_lr: 0.00001 + | > step_time: 3.19640 (2.73378) + | > loader_time: 0.00310 (0.06402) + + + --> STEP: 123/234 -- GLOBAL_STEP: 13695 + | > loss: 0.05354 (0.07718) + | > log_mle: -0.13975 (-0.12382) + | > loss_dur: 0.19330 (0.20100) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.86907 (6.80601) + | > current_lr: 0.00001 + | > step_time: 3.81520 (2.73244) + | > loader_time: 0.09390 (0.06227) + + + --> STEP: 128/234 -- GLOBAL_STEP: 13700 + | > loss: 0.00915 (0.07470) + | > log_mle: -0.19645 (-0.12724) + | > loss_dur: 0.20561 (0.20194) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.43213 (7.26347) + | > current_lr: 0.00001 + | > step_time: 2.70690 (2.72566) + | > loader_time: 0.08260 (0.06192) + + + --> STEP: 133/234 -- GLOBAL_STEP: 13705 + | > loss: 0.02614 (0.07234) + | > log_mle: -0.21651 (-0.13068) + | > loss_dur: 0.24265 (0.20302) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.13586 (7.62518) + | > current_lr: 0.00001 + | > step_time: 2.51530 (2.70693) + | > loader_time: 0.00630 (0.06097) + + + --> STEP: 138/234 -- GLOBAL_STEP: 13710 + | > loss: 0.03509 (0.07052) + | > log_mle: -0.17807 (-0.13387) + | > loss_dur: 0.21316 (0.20440) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.03362 (7.91709) + | > current_lr: 0.00001 + | > step_time: 2.71270 (2.68500) + | > loader_time: 0.00300 (0.05945) + + + --> STEP: 143/234 -- GLOBAL_STEP: 13715 + | > loss: -0.01393 (0.06809) + | > log_mle: -0.29638 (-0.13782) + | > loss_dur: 0.28245 (0.20591) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.37555 (8.45504) + | > current_lr: 0.00001 + | > step_time: 2.29980 (2.65614) + | > loader_time: 0.00290 (0.05748) + + + --> STEP: 148/234 -- GLOBAL_STEP: 13720 + | > loss: -0.00715 (0.06539) + | > log_mle: -0.22525 (-0.14175) + | > loss_dur: 0.21810 (0.20715) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.52782 (9.03902) + | > current_lr: 0.00001 + | > step_time: 4.51460 (2.66257) + | > loader_time: 0.09640 (0.05803) + + + --> STEP: 153/234 -- GLOBAL_STEP: 13725 + | > loss: -0.07817 (0.06186) + | > log_mle: -0.33479 (-0.14658) + | > loss_dur: 0.25662 (0.20843) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.77490 (9.64165) + | > current_lr: 0.00001 + | > step_time: 2.69710 (2.66595) + | > loader_time: 0.09680 (0.05866) + + + --> STEP: 158/234 -- GLOBAL_STEP: 13730 + | > loss: -0.03115 (0.05874) + | > log_mle: -0.28148 (-0.15088) + | > loss_dur: 0.25032 (0.20963) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.79261 (10.14729) + | > current_lr: 0.00001 + | > step_time: 2.22450 (2.67866) + | > loader_time: 0.08380 (0.05871) + + + --> STEP: 163/234 -- GLOBAL_STEP: 13735 + | > loss: -0.01979 (0.05557) + | > log_mle: -0.25564 (-0.15516) + | > loss_dur: 0.23585 (0.21073) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.82337 (10.56933) + | > current_lr: 0.00001 + | > step_time: 2.20200 (2.64972) + | > loader_time: 0.00370 (0.05755) + + + --> STEP: 168/234 -- GLOBAL_STEP: 13740 + | > loss: -0.01485 (0.05267) + | > log_mle: -0.30037 (-0.15949) + | > loss_dur: 0.28552 (0.21215) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.30734 (11.10108) + | > current_lr: 0.00001 + | > step_time: 6.00520 (2.75139) + | > loader_time: 0.10760 (0.05829) + + + --> STEP: 173/234 -- GLOBAL_STEP: 13745 + | > loss: -0.05657 (0.04946) + | > log_mle: -0.30447 (-0.16425) + | > loss_dur: 0.24789 (0.21371) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.99341 (11.84784) + | > current_lr: 0.00001 + | > step_time: 2.91210 (2.75004) + | > loader_time: 0.08730 (0.05926) + + + --> STEP: 178/234 -- GLOBAL_STEP: 13750 + | > loss: -0.06738 (0.04645) + | > log_mle: -0.35460 (-0.16899) + | > loss_dur: 0.28722 (0.21544) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.73702 (12.65455) + | > current_lr: 0.00001 + | > step_time: 1.49580 (2.77355) + | > loader_time: 0.00270 (0.05869) + + + --> STEP: 183/234 -- GLOBAL_STEP: 13755 + | > loss: -0.07699 (0.04374) + | > log_mle: -0.35368 (-0.17334) + | > loss_dur: 0.27669 (0.21707) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.87502 (13.39500) + | > current_lr: 0.00001 + | > step_time: 6.78610 (2.85220) + | > loader_time: 0.11150 (0.05881) + + + --> STEP: 188/234 -- GLOBAL_STEP: 13760 + | > loss: -0.09658 (0.04093) + | > log_mle: -0.36899 (-0.17775) + | > loss_dur: 0.27241 (0.21868) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.56073 (13.97138) + | > current_lr: 0.00001 + | > step_time: 2.30400 (2.84806) + | > loader_time: 0.09690 (0.05886) + + + --> STEP: 193/234 -- GLOBAL_STEP: 13765 + | > loss: -0.08569 (0.03787) + | > log_mle: -0.36946 (-0.18203) + | > loss_dur: 0.28377 (0.21990) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.00972 (14.40501) + | > current_lr: 0.00001 + | > step_time: 2.10920 (2.88827) + | > loader_time: 0.08670 (0.06039) + + + --> STEP: 198/234 -- GLOBAL_STEP: 13770 + | > loss: -0.08320 (0.03517) + | > log_mle: -0.35630 (-0.18608) + | > loss_dur: 0.27310 (0.22125) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.82405 (14.97120) + | > current_lr: 0.00001 + | > step_time: 4.01060 (2.89574) + | > loader_time: 0.00280 (0.05989) + + + --> STEP: 203/234 -- GLOBAL_STEP: 13775 + | > loss: -0.04317 (0.03271) + | > log_mle: -0.30377 (-0.19001) + | > loss_dur: 0.26060 (0.22272) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.64375 (15.48812) + | > current_lr: 0.00001 + | > step_time: 11.90490 (3.01271) + | > loader_time: 0.19900 (0.06339) + + + --> STEP: 208/234 -- GLOBAL_STEP: 13780 + | > loss: -0.08072 (0.02975) + | > log_mle: -0.36691 (-0.19449) + | > loss_dur: 0.28619 (0.22424) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.55891 (16.08544) + | > current_lr: 0.00001 + | > step_time: 3.21030 (2.99848) + | > loader_time: 0.09450 (0.06322) + + + --> STEP: 213/234 -- GLOBAL_STEP: 13785 + | > loss: -0.11277 (0.02654) + | > log_mle: -0.41449 (-0.19937) + | > loss_dur: 0.30172 (0.22591) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.47329 (16.76100) + | > current_lr: 0.00001 + | > step_time: 3.09800 (3.06567) + | > loader_time: 0.01150 (0.06562) + + + --> STEP: 218/234 -- GLOBAL_STEP: 13790 + | > loss: -0.09279 (0.02343) + | > log_mle: -0.37969 (-0.20384) + | > loss_dur: 0.28690 (0.22727) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.49343 (17.27105) + | > current_lr: 0.00001 + | > step_time: 6.29380 (3.07888) + | > loader_time: 0.00490 (0.06463) + + + --> STEP: 223/234 -- GLOBAL_STEP: 13795 + | > loss: -0.11920 (0.02026) + | > log_mle: -0.41894 (-0.20857) + | > loss_dur: 0.29974 (0.22883) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.78569 (17.80805) + | > current_lr: 0.00001 + | > step_time: 3.20120 (3.10479) + | > loader_time: 0.00920 (0.06406) + + + --> STEP: 228/234 -- GLOBAL_STEP: 13800 + | > loss: -0.08993 (0.01729) + | > log_mle: -0.42024 (-0.21344) + | > loss_dur: 0.33031 (0.23072) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.89197 (18.56619) + | > current_lr: 0.00001 + | > step_time: 0.24930 (3.06616) + | > loader_time: 0.00330 (0.06344) + + + --> STEP: 233/234 -- GLOBAL_STEP: 13805 + | > loss: 0.55094 (0.01819) + | > log_mle: -0.37604 (-0.21920) + | > loss_dur: 0.92698 (0.23739) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.39772 (19.40978) + | > current_lr: 0.00001 + | > step_time: 0.19300 (3.00622) + | > loader_time: 0.00340 (0.06228) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15093 (+0.02298) + | > avg_loss: -0.02136 (-0.00462) + | > avg_log_mle: -0.27901 (-0.00363) + | > avg_loss_dur: 0.25765 (-0.00099) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_13806.pth + + > EPOCH: 59/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 21:46:10)  + + --> STEP: 4/234 -- GLOBAL_STEP: 13810 + | > loss: 0.16710 (0.17521) + | > log_mle: -0.10698 (-0.08720) + | > loss_dur: 0.27408 (0.26241) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.46280 (3.68925) + | > current_lr: 0.00001 + | > step_time: 1.30650 (7.07173) + | > loader_time: 0.00110 (0.00355) + + + --> STEP: 9/234 -- GLOBAL_STEP: 13815 + | > loss: 0.09909 (0.13867) + | > log_mle: -0.11333 (-0.09610) + | > loss_dur: 0.21241 (0.23477) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.60614 (3.49420) + | > current_lr: 0.00001 + | > step_time: 3.19800 (4.65752) + | > loader_time: 0.00230 (0.01368) + + + --> STEP: 14/234 -- GLOBAL_STEP: 13820 + | > loss: 0.10223 (0.12521) + | > log_mle: -0.10561 (-0.09698) + | > loss_dur: 0.20784 (0.22219) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.95245 (3.59901) + | > current_lr: 0.00001 + | > step_time: 1.50840 (4.48754) + | > loader_time: 0.08450 (0.04857) + + + --> STEP: 19/234 -- GLOBAL_STEP: 13825 + | > loss: 0.10974 (0.11716) + | > log_mle: -0.09056 (-0.09625) + | > loss_dur: 0.20031 (0.21340) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.05618 (3.49799) + | > current_lr: 0.00001 + | > step_time: 1.30550 (3.98080) + | > loader_time: 0.00240 (0.04074) + + + --> STEP: 24/234 -- GLOBAL_STEP: 13830 + | > loss: 0.09000 (0.10989) + | > log_mle: -0.09294 (-0.09618) + | > loss_dur: 0.18294 (0.20607) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.72946 (3.43709) + | > current_lr: 0.00001 + | > step_time: 1.29820 (3.42058) + | > loader_time: 0.00270 (0.03617) + + + --> STEP: 29/234 -- GLOBAL_STEP: 13835 + | > loss: 0.08250 (0.10504) + | > log_mle: -0.09094 (-0.09623) + | > loss_dur: 0.17343 (0.20127) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.81342 (3.33775) + | > current_lr: 0.00001 + | > step_time: 1.57410 (3.12582) + | > loader_time: 0.09540 (0.03346) + + + --> STEP: 34/234 -- GLOBAL_STEP: 13840 + | > loss: 0.09980 (0.10227) + | > log_mle: -0.10225 (-0.09785) + | > loss_dur: 0.20205 (0.20012) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.33336 (3.27015) + | > current_lr: 0.00001 + | > step_time: 1.25410 (2.85064) + | > loader_time: 0.00180 (0.02886) + + + --> STEP: 39/234 -- GLOBAL_STEP: 13845 + | > loss: 0.09521 (0.09999) + | > log_mle: -0.11056 (-0.09933) + | > loss_dur: 0.20578 (0.19932) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.88656 (3.67373) + | > current_lr: 0.00001 + | > step_time: 1.21470 (2.67732) + | > loader_time: 0.08760 (0.02975) + + + --> STEP: 44/234 -- GLOBAL_STEP: 13850 + | > loss: 0.05605 (0.09875) + | > log_mle: -0.10205 (-0.09928) + | > loss_dur: 0.15810 (0.19803) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.73744 (3.64040) + | > current_lr: 0.00001 + | > step_time: 2.29200 (2.58176) + | > loader_time: 0.00220 (0.02845) + + + --> STEP: 49/234 -- GLOBAL_STEP: 13855 + | > loss: 0.05380 (0.09569) + | > log_mle: -0.10898 (-0.10023) + | > loss_dur: 0.16278 (0.19592) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.33809 (3.63946) + | > current_lr: 0.00001 + | > step_time: 1.67860 (2.47861) + | > loader_time: 0.00230 (0.02577) + + + --> STEP: 54/234 -- GLOBAL_STEP: 13860 + | > loss: 0.05661 (0.09457) + | > log_mle: -0.11410 (-0.10045) + | > loss_dur: 0.17071 (0.19502) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.36969 (3.57578) + | > current_lr: 0.00001 + | > step_time: 1.30440 (2.39108) + | > loader_time: 0.08560 (0.02513) + + + --> STEP: 59/234 -- GLOBAL_STEP: 13865 + | > loss: 0.03659 (0.09310) + | > log_mle: -0.12968 (-0.10126) + | > loss_dur: 0.16627 (0.19436) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.58368 (3.53609) + | > current_lr: 0.00001 + | > step_time: 1.43620 (2.31946) + | > loader_time: 0.00220 (0.02457) + + + --> STEP: 64/234 -- GLOBAL_STEP: 13870 + | > loss: 0.06861 (0.09113) + | > log_mle: -0.09883 (-0.10275) + | > loss_dur: 0.16745 (0.19388) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.97023 (3.73710) + | > current_lr: 0.00001 + | > step_time: 1.29310 (2.24463) + | > loader_time: 0.00410 (0.02286) + + + --> STEP: 69/234 -- GLOBAL_STEP: 13875 + | > loss: 0.10280 (0.09030) + | > log_mle: -0.08748 (-0.10299) + | > loss_dur: 0.19028 (0.19329) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.00734 (3.75836) + | > current_lr: 0.00001 + | > step_time: 1.44130 (2.20052) + | > loader_time: 0.00220 (0.02137) + + + --> STEP: 74/234 -- GLOBAL_STEP: 13880 + | > loss: 0.05951 (0.08895) + | > log_mle: -0.11280 (-0.10432) + | > loss_dur: 0.17231 (0.19328) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.46456 (4.19954) + | > current_lr: 0.00001 + | > step_time: 3.60800 (2.21935) + | > loader_time: 0.08920 (0.02247) + + + --> STEP: 79/234 -- GLOBAL_STEP: 13885 + | > loss: 0.05018 (0.08776) + | > log_mle: -0.12123 (-0.10532) + | > loss_dur: 0.17141 (0.19308) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.48537 (4.22790) + | > current_lr: 0.00001 + | > step_time: 1.38650 (2.19368) + | > loader_time: 0.00350 (0.02222) + + + --> STEP: 84/234 -- GLOBAL_STEP: 13890 + | > loss: 0.08103 (0.08664) + | > log_mle: -0.12177 (-0.10631) + | > loss_dur: 0.20281 (0.19296) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.99679 (4.35627) + | > current_lr: 0.00001 + | > step_time: 2.24280 (2.17461) + | > loader_time: 0.00310 (0.02204) + + + --> STEP: 89/234 -- GLOBAL_STEP: 13895 + | > loss: 0.04733 (0.08468) + | > log_mle: -0.15128 (-0.10829) + | > loss_dur: 0.19860 (0.19297) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.60530 (4.54130) + | > current_lr: 0.00001 + | > step_time: 1.90050 (2.15185) + | > loader_time: 0.08370 (0.02281) + + + --> STEP: 94/234 -- GLOBAL_STEP: 13900 + | > loss: 0.03023 (0.08211) + | > log_mle: -0.18568 (-0.11142) + | > loss_dur: 0.21591 (0.19353) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.58699 (4.83905) + | > current_lr: 0.00001 + | > step_time: 1.59030 (2.15042) + | > loader_time: 0.00220 (0.02448) + + + --> STEP: 99/234 -- GLOBAL_STEP: 13905 + | > loss: 0.00701 (0.08005) + | > log_mle: -0.21622 (-0.11441) + | > loss_dur: 0.22323 (0.19446) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.54104 (5.17083) + | > current_lr: 0.00001 + | > step_time: 3.20760 (2.15615) + | > loader_time: 0.00270 (0.02339) + + + --> STEP: 104/234 -- GLOBAL_STEP: 13910 + | > loss: 0.00029 (0.07781) + | > log_mle: -0.22518 (-0.11775) + | > loss_dur: 0.22547 (0.19556) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.96512 (5.59350) + | > current_lr: 0.00001 + | > step_time: 1.36110 (2.13963) + | > loader_time: 0.00220 (0.02339) + + + --> STEP: 109/234 -- GLOBAL_STEP: 13915 + | > loss: 0.05542 (0.07657) + | > log_mle: -0.19776 (-0.12021) + | > loss_dur: 0.25318 (0.19678) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.87951 (6.11403) + | > current_lr: 0.00001 + | > step_time: 2.70590 (2.14639) + | > loader_time: 0.00260 (0.02245) + + + --> STEP: 114/234 -- GLOBAL_STEP: 13920 + | > loss: 0.02919 (0.07450) + | > log_mle: -0.17988 (-0.12337) + | > loss_dur: 0.20908 (0.19787) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.62869 (6.46438) + | > current_lr: 0.00001 + | > step_time: 1.20030 (2.12677) + | > loader_time: 0.00370 (0.02233) + + + --> STEP: 119/234 -- GLOBAL_STEP: 13925 + | > loss: 0.03091 (0.07323) + | > log_mle: -0.17698 (-0.12581) + | > loss_dur: 0.20789 (0.19904) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.29022 (6.76190) + | > current_lr: 0.00001 + | > step_time: 2.68920 (2.14649) + | > loader_time: 0.00420 (0.02226) + + + --> STEP: 124/234 -- GLOBAL_STEP: 13930 + | > loss: 0.00673 (0.07189) + | > log_mle: -0.20633 (-0.12765) + | > loss_dur: 0.21306 (0.19954) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.55730 (6.96091) + | > current_lr: 0.00001 + | > step_time: 2.26540 (2.14308) + | > loader_time: 0.00240 (0.02434) + + + --> STEP: 129/234 -- GLOBAL_STEP: 13935 + | > loss: 0.03154 (0.06959) + | > log_mle: -0.18986 (-0.13093) + | > loss_dur: 0.22139 (0.20052) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.65821 (7.36303) + | > current_lr: 0.00001 + | > step_time: 2.10630 (2.11825) + | > loader_time: 0.10230 (0.02621) + + + --> STEP: 134/234 -- GLOBAL_STEP: 13940 + | > loss: 0.00782 (0.06713) + | > log_mle: -0.24061 (-0.13470) + | > loss_dur: 0.24844 (0.20183) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.49765 (7.76895) + | > current_lr: 0.00001 + | > step_time: 2.10290 (2.11217) + | > loader_time: 0.00490 (0.02604) + + + --> STEP: 139/234 -- GLOBAL_STEP: 13945 + | > loss: -0.05727 (0.06476) + | > log_mle: -0.29473 (-0.13825) + | > loss_dur: 0.23746 (0.20301) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.08743 (8.25427) + | > current_lr: 0.00001 + | > step_time: 1.06670 (2.11799) + | > loader_time: 0.00260 (0.02522) + + + --> STEP: 144/234 -- GLOBAL_STEP: 13950 + | > loss: -0.01111 (0.06267) + | > log_mle: -0.27158 (-0.14194) + | > loss_dur: 0.26047 (0.20461) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.04652 (8.82966) + | > current_lr: 0.00001 + | > step_time: 1.90080 (2.12517) + | > loader_time: 0.00300 (0.02558) + + + --> STEP: 149/234 -- GLOBAL_STEP: 13955 + | > loss: -0.05816 (0.05964) + | > log_mle: -0.31229 (-0.14610) + | > loss_dur: 0.25414 (0.20574) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.04148 (9.29940) + | > current_lr: 0.00001 + | > step_time: 2.70220 (2.17197) + | > loader_time: 0.08920 (0.02601) + + + --> STEP: 154/234 -- GLOBAL_STEP: 13960 + | > loss: -0.04196 (0.05637) + | > log_mle: -0.27403 (-0.15057) + | > loss_dur: 0.23206 (0.20693) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.09442 (10.00405) + | > current_lr: 0.00001 + | > step_time: 3.90970 (2.17153) + | > loader_time: 0.08190 (0.02632) + + + --> STEP: 159/234 -- GLOBAL_STEP: 13965 + | > loss: -0.04665 (0.05344) + | > log_mle: -0.28997 (-0.15483) + | > loss_dur: 0.24332 (0.20827) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.42296 (11.02143) + | > current_lr: 0.00001 + | > step_time: 4.10190 (2.18137) + | > loader_time: 0.00390 (0.02605) + + + --> STEP: 164/234 -- GLOBAL_STEP: 13970 + | > loss: -0.03565 (0.05055) + | > log_mle: -0.28718 (-0.15892) + | > loss_dur: 0.25153 (0.20947) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.24967 (11.63543) + | > current_lr: 0.00001 + | > step_time: 6.40670 (2.23801) + | > loader_time: 0.00400 (0.02540) + + + --> STEP: 169/234 -- GLOBAL_STEP: 13975 + | > loss: -0.01867 (0.04778) + | > log_mle: -0.28098 (-0.16314) + | > loss_dur: 0.26231 (0.21092) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.28699 (12.13502) + | > current_lr: 0.00001 + | > step_time: 2.99950 (2.32249) + | > loader_time: 0.08170 (0.02760) + + + --> STEP: 174/234 -- GLOBAL_STEP: 13980 + | > loss: -0.10119 (0.04418) + | > log_mle: -0.36983 (-0.16838) + | > loss_dur: 0.26864 (0.21256) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.17114 (12.68841) + | > current_lr: 0.00001 + | > step_time: 1.48470 (2.32328) + | > loader_time: 0.00200 (0.02749) + + + --> STEP: 179/234 -- GLOBAL_STEP: 13985 + | > loss: -0.07677 (0.04114) + | > log_mle: -0.35665 (-0.17306) + | > loss_dur: 0.27988 (0.21420) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.02793 (13.15067) + | > current_lr: 0.00001 + | > step_time: 1.50040 (2.35077) + | > loader_time: 0.00400 (0.02790) + + + --> STEP: 184/234 -- GLOBAL_STEP: 13990 + | > loss: -0.05393 (0.03844) + | > log_mle: -0.33128 (-0.17733) + | > loss_dur: 0.27736 (0.21577) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.88427 (13.55326) + | > current_lr: 0.00001 + | > step_time: 2.91310 (2.38809) + | > loader_time: 0.08840 (0.03069) + + + --> STEP: 189/234 -- GLOBAL_STEP: 13995 + | > loss: -0.05611 (0.03561) + | > log_mle: -0.33045 (-0.18174) + | > loss_dur: 0.27434 (0.21735) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.20421 (14.02183) + | > current_lr: 0.00001 + | > step_time: 4.89960 (2.39795) + | > loader_time: 0.10450 (0.03139) + + + --> STEP: 194/234 -- GLOBAL_STEP: 14000 + | > loss: -0.08525 (0.03257) + | > log_mle: -0.35462 (-0.18599) + | > loss_dur: 0.26937 (0.21856) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.67982 (15.21502) + | > current_lr: 0.00001 + | > step_time: 1.59130 (2.40937) + | > loader_time: 0.00480 (0.03202) + + + --> STEP: 199/234 -- GLOBAL_STEP: 14005 + | > loss: -0.08756 (0.03014) + | > log_mle: -0.35742 (-0.18988) + | > loss_dur: 0.26986 (0.22002) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 53.75337 (16.10178) + | > current_lr: 0.00001 + | > step_time: 7.10380 (2.49160) + | > loader_time: 0.10810 (0.03466) + + + --> STEP: 204/234 -- GLOBAL_STEP: 14010 + | > loss: -0.09304 (0.02779) + | > log_mle: -0.38964 (-0.19383) + | > loss_dur: 0.29660 (0.22161) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 51.39719 (16.73155) + | > current_lr: 0.00001 + | > step_time: 3.90650 (2.54826) + | > loader_time: 0.00490 (0.03715) + + + --> STEP: 209/234 -- GLOBAL_STEP: 14015 + | > loss: -0.07616 (0.02491) + | > log_mle: -0.35423 (-0.19807) + | > loss_dur: 0.27807 (0.22298) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 37.55764 (17.28731) + | > current_lr: 0.00001 + | > step_time: 4.70980 (2.62383) + | > loader_time: 0.18910 (0.03868) + + + --> STEP: 214/234 -- GLOBAL_STEP: 14020 + | > loss: -0.11386 (0.02151) + | > log_mle: -0.38606 (-0.20309) + | > loss_dur: 0.27220 (0.22460) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.72416 (17.87991) + | > current_lr: 0.00001 + | > step_time: 4.40090 (2.65375) + | > loader_time: 0.00330 (0.03920) + + + --> STEP: 219/234 -- GLOBAL_STEP: 14025 + | > loss: -0.17145 (0.01829) + | > log_mle: -0.47145 (-0.20792) + | > loss_dur: 0.30000 (0.22621) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.25794 (18.32689) + | > current_lr: 0.00001 + | > step_time: 4.50790 (2.71922) + | > loader_time: 0.00490 (0.04109) + + + --> STEP: 224/234 -- GLOBAL_STEP: 14030 + | > loss: -0.13066 (0.01531) + | > log_mle: -0.43267 (-0.21249) + | > loss_dur: 0.30201 (0.22779) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.96357 (18.73465) + | > current_lr: 0.00001 + | > step_time: 0.22630 (2.67286) + | > loader_time: 0.00370 (0.04064) + + + --> STEP: 229/234 -- GLOBAL_STEP: 14035 + | > loss: -0.08189 (0.01249) + | > log_mle: -0.44914 (-0.21745) + | > loss_dur: 0.36725 (0.22994) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 80.13230 (19.47479) + | > current_lr: 0.00001 + | > step_time: 0.24510 (2.61961) + | > loader_time: 0.00470 (0.03984) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.35442 (+0.20349) + | > avg_loss: -0.02217 (-0.00081) + | > avg_log_mle: -0.28026 (-0.00125) + | > avg_loss_dur: 0.25808 (+0.00044) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_14040.pth + + > EPOCH: 60/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 21:57:23)  + + --> STEP: 0/234 -- GLOBAL_STEP: 14040 + | > loss: 0.13825 (0.13825) + | > log_mle: -0.12909 (-0.12909) + | > loss_dur: 0.26734 (0.26734) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.54286 (4.54286) + | > current_lr: 0.00001 + | > step_time: 3.79750 (3.79749) + | > loader_time: 17.22590 (17.22593) + + + --> STEP: 5/234 -- GLOBAL_STEP: 14045 + | > loss: 0.10401 (0.14544) + | > log_mle: -0.10370 (-0.09286) + | > loss_dur: 0.20771 (0.23831) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.99275 (5.21748) + | > current_lr: 0.00001 + | > step_time: 1.90190 (5.28206) + | > loader_time: 0.00370 (0.23520) + + + --> STEP: 10/234 -- GLOBAL_STEP: 14050 + | > loss: 0.07232 (0.11950) + | > log_mle: -0.11092 (-0.10049) + | > loss_dur: 0.18324 (0.21998) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.00446 (4.94942) + | > current_lr: 0.00001 + | > step_time: 3.10650 (4.36083) + | > loader_time: 0.00250 (0.11893) + + + --> STEP: 15/234 -- GLOBAL_STEP: 14055 + | > loss: 0.09516 (0.11105) + | > log_mle: -0.10194 (-0.10040) + | > loss_dur: 0.19710 (0.21146) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.30940 (4.43372) + | > current_lr: 0.00001 + | > step_time: 2.70540 (4.13228) + | > loader_time: 0.00180 (0.08696) + + + --> STEP: 20/234 -- GLOBAL_STEP: 14060 + | > loss: 0.08668 (0.10565) + | > log_mle: -0.09453 (-0.09915) + | > loss_dur: 0.18121 (0.20481) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.26805 (4.21578) + | > current_lr: 0.00001 + | > step_time: 2.50210 (3.73336) + | > loader_time: 0.09620 (0.07567) + + + --> STEP: 25/234 -- GLOBAL_STEP: 14065 + | > loss: 0.10469 (0.10112) + | > log_mle: -0.08869 (-0.09880) + | > loss_dur: 0.19339 (0.19991) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.48725 (4.08938) + | > current_lr: 0.00001 + | > step_time: 1.86370 (3.30059) + | > loader_time: 0.00220 (0.06491) + + + --> STEP: 30/234 -- GLOBAL_STEP: 14070 + | > loss: 0.05812 (0.09578) + | > log_mle: -0.11801 (-0.09999) + | > loss_dur: 0.17612 (0.19577) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.86221 (3.87985) + | > current_lr: 0.00001 + | > step_time: 1.59040 (2.99420) + | > loader_time: 0.00250 (0.05996) + + + --> STEP: 35/234 -- GLOBAL_STEP: 14075 + | > loss: 0.06927 (0.09441) + | > log_mle: -0.11578 (-0.10141) + | > loss_dur: 0.18505 (0.19582) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.17086 (4.02608) + | > current_lr: 0.00001 + | > step_time: 1.18210 (3.03688) + | > loader_time: 0.00140 (0.05671) + + + --> STEP: 40/234 -- GLOBAL_STEP: 14080 + | > loss: 0.12084 (0.09405) + | > log_mle: -0.09456 (-0.10226) + | > loss_dur: 0.21540 (0.19631) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.02035 (4.03196) + | > current_lr: 0.00001 + | > step_time: 1.01930 (2.79214) + | > loader_time: 0.00220 (0.04991) + + + --> STEP: 45/234 -- GLOBAL_STEP: 14085 + | > loss: 0.06615 (0.09201) + | > log_mle: -0.12924 (-0.10300) + | > loss_dur: 0.19539 (0.19501) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.99973 (4.01791) + | > current_lr: 0.00001 + | > step_time: 1.97070 (2.62628) + | > loader_time: 0.00330 (0.04463) + + + --> STEP: 50/234 -- GLOBAL_STEP: 14090 + | > loss: 0.09210 (0.09006) + | > log_mle: -0.09914 (-0.10332) + | > loss_dur: 0.19124 (0.19338) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.19072 (3.89947) + | > current_lr: 0.00001 + | > step_time: 1.19750 (2.52161) + | > loader_time: 0.00290 (0.04042) + + + --> STEP: 55/234 -- GLOBAL_STEP: 14095 + | > loss: 0.07036 (0.08890) + | > log_mle: -0.11972 (-0.10395) + | > loss_dur: 0.19008 (0.19285) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.76209 (3.79696) + | > current_lr: 0.00001 + | > step_time: 1.22630 (2.45248) + | > loader_time: 0.00250 (0.03851) + + + --> STEP: 60/234 -- GLOBAL_STEP: 14100 + | > loss: 0.03875 (0.08683) + | > log_mle: -0.13868 (-0.10511) + | > loss_dur: 0.17743 (0.19194) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.15527 (3.77145) + | > current_lr: 0.00001 + | > step_time: 3.41800 (2.43636) + | > loader_time: 0.00250 (0.03550) + + + --> STEP: 65/234 -- GLOBAL_STEP: 14105 + | > loss: 0.06615 (0.08571) + | > log_mle: -0.11103 (-0.10621) + | > loss_dur: 0.17718 (0.19192) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.79105 (3.86093) + | > current_lr: 0.00001 + | > step_time: 1.58200 (2.38296) + | > loader_time: 0.00190 (0.03295) + + + --> STEP: 70/234 -- GLOBAL_STEP: 14110 + | > loss: 0.06909 (0.08512) + | > log_mle: -0.11701 (-0.10656) + | > loss_dur: 0.18610 (0.19168) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.89434 (4.05032) + | > current_lr: 0.00001 + | > step_time: 1.05120 (2.30042) + | > loader_time: 0.00240 (0.03074) + + + --> STEP: 75/234 -- GLOBAL_STEP: 14115 + | > loss: 0.06685 (0.08395) + | > log_mle: -0.12830 (-0.10800) + | > loss_dur: 0.19516 (0.19195) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.91758 (4.35188) + | > current_lr: 0.00001 + | > step_time: 1.61310 (2.26961) + | > loader_time: 0.08970 (0.03114) + + + --> STEP: 80/234 -- GLOBAL_STEP: 14120 + | > loss: 0.05441 (0.08237) + | > log_mle: -0.10610 (-0.10867) + | > loss_dur: 0.16051 (0.19104) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.72536 (4.37786) + | > current_lr: 0.00001 + | > step_time: 2.58190 (2.25715) + | > loader_time: 0.00230 (0.03029) + + + --> STEP: 85/234 -- GLOBAL_STEP: 14125 + | > loss: 0.05839 (0.08106) + | > log_mle: -0.12303 (-0.10979) + | > loss_dur: 0.18142 (0.19085) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.68934 (4.56180) + | > current_lr: 0.00001 + | > step_time: 3.02010 (2.28104) + | > loader_time: 0.08450 (0.03308) + + + --> STEP: 90/234 -- GLOBAL_STEP: 14130 + | > loss: 0.05597 (0.07918) + | > log_mle: -0.15479 (-0.11203) + | > loss_dur: 0.21076 (0.19121) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.91008 (4.89457) + | > current_lr: 0.00001 + | > step_time: 2.29520 (2.24907) + | > loader_time: 0.00240 (0.03321) + + + --> STEP: 95/234 -- GLOBAL_STEP: 14135 + | > loss: -0.02093 (0.07625) + | > log_mle: -0.23420 (-0.11586) + | > loss_dur: 0.21327 (0.19211) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.14395 (5.41628) + | > current_lr: 0.00001 + | > step_time: 1.19150 (2.25918) + | > loader_time: 0.00400 (0.03471) + + + --> STEP: 100/234 -- GLOBAL_STEP: 14140 + | > loss: 0.03674 (0.07443) + | > log_mle: -0.16618 (-0.11807) + | > loss_dur: 0.20293 (0.19249) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.48476 (5.70084) + | > current_lr: 0.00001 + | > step_time: 1.29230 (2.25930) + | > loader_time: 0.00590 (0.03585) + + + --> STEP: 105/234 -- GLOBAL_STEP: 14145 + | > loss: 0.04400 (0.07249) + | > log_mle: -0.13786 (-0.12114) + | > loss_dur: 0.18186 (0.19363) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.55807 (5.99718) + | > current_lr: 0.00001 + | > step_time: 1.40900 (2.22840) + | > loader_time: 0.00230 (0.03508) + + + --> STEP: 110/234 -- GLOBAL_STEP: 14150 + | > loss: 0.02321 (0.07102) + | > log_mle: -0.16528 (-0.12393) + | > loss_dur: 0.18850 (0.19495) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.60840 (6.25314) + | > current_lr: 0.00001 + | > step_time: 3.21180 (2.22267) + | > loader_time: 0.00320 (0.03362) + + + --> STEP: 115/234 -- GLOBAL_STEP: 14155 + | > loss: 0.05303 (0.06906) + | > log_mle: -0.18255 (-0.12723) + | > loss_dur: 0.23558 (0.19630) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.42106 (6.58476) + | > current_lr: 0.00001 + | > step_time: 0.52210 (2.21779) + | > loader_time: 0.00320 (0.03301) + + + --> STEP: 120/234 -- GLOBAL_STEP: 14160 + | > loss: 0.00257 (0.06745) + | > log_mle: -0.22952 (-0.13005) + | > loss_dur: 0.23209 (0.19750) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.75148 (6.86826) + | > current_lr: 0.00001 + | > step_time: 4.10510 (2.23725) + | > loader_time: 0.00330 (0.03242) + + + --> STEP: 125/234 -- GLOBAL_STEP: 14165 + | > loss: 0.02346 (0.06621) + | > log_mle: -0.21657 (-0.13173) + | > loss_dur: 0.24003 (0.19794) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.83387 (7.16426) + | > current_lr: 0.00001 + | > step_time: 1.40370 (2.23881) + | > loader_time: 0.00370 (0.03345) + + + --> STEP: 130/234 -- GLOBAL_STEP: 14170 + | > loss: -0.00835 (0.06396) + | > log_mle: -0.22777 (-0.13506) + | > loss_dur: 0.21942 (0.19903) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.81692 (7.63325) + | > current_lr: 0.00001 + | > step_time: 1.41540 (2.22206) + | > loader_time: 0.08550 (0.03356) + + + --> STEP: 135/234 -- GLOBAL_STEP: 14175 + | > loss: 0.04243 (0.06174) + | > log_mle: -0.16319 (-0.13832) + | > loss_dur: 0.20563 (0.20006) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.19641 (7.98714) + | > current_lr: 0.00001 + | > step_time: 3.40230 (2.21896) + | > loader_time: 0.00560 (0.03306) + + + --> STEP: 140/234 -- GLOBAL_STEP: 14180 + | > loss: 0.04252 (0.05943) + | > log_mle: -0.19407 (-0.14206) + | > loss_dur: 0.23659 (0.20150) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.09669 (8.42721) + | > current_lr: 0.00001 + | > step_time: 1.40620 (2.22704) + | > loader_time: 0.00300 (0.03460) + + + --> STEP: 145/234 -- GLOBAL_STEP: 14185 + | > loss: -0.03424 (0.05682) + | > log_mle: -0.28145 (-0.14636) + | > loss_dur: 0.24721 (0.20318) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.60325 (8.94376) + | > current_lr: 0.00001 + | > step_time: 3.49740 (2.25252) + | > loader_time: 0.00330 (0.03577) + + + --> STEP: 150/234 -- GLOBAL_STEP: 14190 + | > loss: -0.02331 (0.05398) + | > log_mle: -0.26740 (-0.15038) + | > loss_dur: 0.24409 (0.20436) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.87263 (9.43821) + | > current_lr: 0.00001 + | > step_time: 2.51510 (2.25078) + | > loader_time: 0.08530 (0.03638) + + + --> STEP: 155/234 -- GLOBAL_STEP: 14195 + | > loss: -0.05455 (0.05052) + | > log_mle: -0.32616 (-0.15526) + | > loss_dur: 0.27162 (0.20578) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.34076 (10.14508) + | > current_lr: 0.00001 + | > step_time: 2.49960 (2.27294) + | > loader_time: 0.08760 (0.03697) + + + --> STEP: 160/234 -- GLOBAL_STEP: 14200 + | > loss: -0.06651 (0.04771) + | > log_mle: -0.32216 (-0.15946) + | > loss_dur: 0.25565 (0.20717) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.76446 (10.86657) + | > current_lr: 0.00001 + | > step_time: 11.69910 (2.34443) + | > loader_time: 0.09350 (0.03816) + + + --> STEP: 165/234 -- GLOBAL_STEP: 14205 + | > loss: -0.04975 (0.04490) + | > log_mle: -0.32195 (-0.16357) + | > loss_dur: 0.27220 (0.20847) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.87115 (11.38942) + | > current_lr: 0.00001 + | > step_time: 1.89630 (2.38202) + | > loader_time: 0.00320 (0.03757) + + + --> STEP: 170/234 -- GLOBAL_STEP: 14210 + | > loss: -0.07115 (0.04201) + | > log_mle: -0.35605 (-0.16798) + | > loss_dur: 0.28489 (0.21000) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.12550 (11.87949) + | > current_lr: 0.00001 + | > step_time: 2.00720 (2.36907) + | > loader_time: 0.08440 (0.03762) + + + --> STEP: 175/234 -- GLOBAL_STEP: 14215 + | > loss: -0.05074 (0.03856) + | > log_mle: -0.32860 (-0.17300) + | > loss_dur: 0.27786 (0.21157) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.58528 (12.52513) + | > current_lr: 0.00001 + | > step_time: 2.00600 (2.37227) + | > loader_time: 0.08350 (0.03754) + + + --> STEP: 180/234 -- GLOBAL_STEP: 14220 + | > loss: -0.07536 (0.03553) + | > log_mle: -0.34024 (-0.17766) + | > loss_dur: 0.26488 (0.21319) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.48511 (13.15693) + | > current_lr: 0.00001 + | > step_time: 1.50160 (2.44590) + | > loader_time: 0.08280 (0.03899) + + + --> STEP: 185/234 -- GLOBAL_STEP: 14225 + | > loss: -0.07104 (0.03281) + | > log_mle: -0.36259 (-0.18198) + | > loss_dur: 0.29155 (0.21478) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.54314 (13.76394) + | > current_lr: 0.00001 + | > step_time: 4.59480 (2.49173) + | > loader_time: 0.09940 (0.03999) + + + --> STEP: 190/234 -- GLOBAL_STEP: 14230 + | > loss: -0.08406 (0.03005) + | > log_mle: -0.34190 (-0.18618) + | > loss_dur: 0.25784 (0.21624) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.84708 (14.46496) + | > current_lr: 0.00001 + | > step_time: 1.50850 (2.57242) + | > loader_time: 0.08500 (0.04255) + + + --> STEP: 195/234 -- GLOBAL_STEP: 14235 + | > loss: -0.06916 (0.02704) + | > log_mle: -0.35400 (-0.19054) + | > loss_dur: 0.28484 (0.21758) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.08200 (15.17376) + | > current_lr: 0.00001 + | > step_time: 8.81450 (2.65233) + | > loader_time: 0.59740 (0.04598) + + + --> STEP: 200/234 -- GLOBAL_STEP: 14240 + | > loss: -0.06471 (0.02448) + | > log_mle: -0.35989 (-0.19458) + | > loss_dur: 0.29518 (0.21906) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.42706 (15.87019) + | > current_lr: 0.00001 + | > step_time: 0.74330 (2.63895) + | > loader_time: 0.00400 (0.04664) + + + --> STEP: 205/234 -- GLOBAL_STEP: 14245 + | > loss: -0.08061 (0.02199) + | > log_mle: -0.34860 (-0.19850) + | > loss_dur: 0.26799 (0.22048) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.02628 (16.48460) + | > current_lr: 0.00001 + | > step_time: 4.39350 (2.66341) + | > loader_time: 0.00600 (0.04695) + + + --> STEP: 210/234 -- GLOBAL_STEP: 14250 + | > loss: -0.13119 (0.01889) + | > log_mle: -0.42198 (-0.20310) + | > loss_dur: 0.29079 (0.22199) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 54.07807 (17.11922) + | > current_lr: 0.00001 + | > step_time: 3.51600 (2.76495) + | > loader_time: 0.09250 (0.04814) + + + --> STEP: 215/234 -- GLOBAL_STEP: 14255 + | > loss: -0.09410 (0.01567) + | > log_mle: -0.36987 (-0.20783) + | > loss_dur: 0.27577 (0.22349) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 60.09662 (17.83932) + | > current_lr: 0.00001 + | > step_time: 4.00560 (2.82029) + | > loader_time: 0.10050 (0.04986) + + + --> STEP: 220/234 -- GLOBAL_STEP: 14260 + | > loss: -0.12024 (0.01232) + | > log_mle: -0.41597 (-0.21276) + | > loss_dur: 0.29573 (0.22508) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 62.05601 (18.66322) + | > current_lr: 0.00001 + | > step_time: 6.11090 (2.86121) + | > loader_time: 0.08530 (0.05276) + + + --> STEP: 225/234 -- GLOBAL_STEP: 14265 + | > loss: -0.16416 (0.00932) + | > log_mle: -0.47438 (-0.21743) + | > loss_dur: 0.31023 (0.22675) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 58.28550 (19.41490) + | > current_lr: 0.00001 + | > step_time: 2.09260 (2.85045) + | > loader_time: 0.00390 (0.05205) + + + --> STEP: 230/234 -- GLOBAL_STEP: 14270 + | > loss: -0.12874 (0.00666) + | > log_mle: -0.51989 (-0.22253) + | > loss_dur: 0.39114 (0.22919) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 55.90023 (20.14543) + | > current_lr: 0.00001 + | > step_time: 0.25120 (2.79377) + | > loader_time: 0.00450 (0.05102) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15534 (-0.19908) + | > avg_loss: -0.03554 (-0.01337) + | > avg_log_mle: -0.29296 (-0.01271) + | > avg_loss_dur: 0.25742 (-0.00066) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_14274.pth + + > EPOCH: 61/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 22:09:22)  + + --> STEP: 1/234 -- GLOBAL_STEP: 14275 + | > loss: 0.11558 (0.11558) + | > log_mle: -0.09381 (-0.09381) + | > loss_dur: 0.20939 (0.20939) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.20886 (3.20886) + | > current_lr: 0.00002 + | > step_time: 7.71000 (7.70997) + | > loader_time: 0.80400 (0.80399) + + + --> STEP: 6/234 -- GLOBAL_STEP: 14280 + | > loss: 0.11529 (0.13206) + | > log_mle: -0.09145 (-0.09636) + | > loss_dur: 0.20674 (0.22841) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.85822 (4.54821) + | > current_lr: 0.00002 + | > step_time: 1.80230 (3.80310) + | > loader_time: 0.00130 (2.26615) + + + --> STEP: 11/234 -- GLOBAL_STEP: 14285 + | > loss: 0.06821 (0.11058) + | > log_mle: -0.09930 (-0.10422) + | > loss_dur: 0.16751 (0.21480) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.33083 (4.19122) + | > current_lr: 0.00002 + | > step_time: 4.20420 (3.57254) + | > loader_time: 0.09450 (1.24593) + + + --> STEP: 16/234 -- GLOBAL_STEP: 14290 + | > loss: 0.06764 (0.10415) + | > log_mle: -0.10606 (-0.10448) + | > loss_dur: 0.17370 (0.20863) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.20126 (4.10118) + | > current_lr: 0.00002 + | > step_time: 9.70280 (4.41371) + | > loader_time: 0.00430 (0.87508) + + + --> STEP: 21/234 -- GLOBAL_STEP: 14295 + | > loss: 0.08385 (0.10191) + | > log_mle: -0.09065 (-0.10241) + | > loss_dur: 0.17450 (0.20431) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.24480 (4.06105) + | > current_lr: 0.00002 + | > step_time: 1.79300 (4.15813) + | > loader_time: 0.00350 (0.67647) + + + --> STEP: 26/234 -- GLOBAL_STEP: 14300 + | > loss: 0.06999 (0.09566) + | > log_mle: -0.10677 (-0.10269) + | > loss_dur: 0.17676 (0.19835) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.89186 (3.85563) + | > current_lr: 0.00002 + | > step_time: 5.49050 (4.15855) + | > loader_time: 0.00530 (0.55345) + + + --> STEP: 31/234 -- GLOBAL_STEP: 14305 + | > loss: 0.10002 (0.09143) + | > log_mle: -0.11336 (-0.10397) + | > loss_dur: 0.21337 (0.19539) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.65847 (3.67114) + | > current_lr: 0.00002 + | > step_time: 7.11220 (4.43713) + | > loader_time: 0.00440 (0.47388) + + + --> STEP: 36/234 -- GLOBAL_STEP: 14310 + | > loss: 0.08208 (0.08939) + | > log_mle: -0.11931 (-0.10542) + | > loss_dur: 0.20138 (0.19481) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.81649 (4.02975) + | > current_lr: 0.00002 + | > step_time: 1.70490 (4.06162) + | > loader_time: 0.08070 (0.41286) + + + --> STEP: 41/234 -- GLOBAL_STEP: 14315 + | > loss: 0.05891 (0.08804) + | > log_mle: -0.10872 (-0.10592) + | > loss_dur: 0.16763 (0.19396) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.37879 (3.91989) + | > current_lr: 0.00002 + | > step_time: 0.60650 (3.80064) + | > loader_time: 0.00150 (0.36965) + + + --> STEP: 46/234 -- GLOBAL_STEP: 14320 + | > loss: 0.07706 (0.08635) + | > log_mle: -0.11464 (-0.10674) + | > loss_dur: 0.19170 (0.19309) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.27689 (3.89016) + | > current_lr: 0.00002 + | > step_time: 1.28510 (3.55032) + | > loader_time: 0.00180 (0.33158) + + + --> STEP: 51/234 -- GLOBAL_STEP: 14325 + | > loss: 0.06073 (0.08393) + | > log_mle: -0.09944 (-0.10667) + | > loss_dur: 0.16018 (0.19060) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.54713 (3.80140) + | > current_lr: 0.00002 + | > step_time: 1.65540 (3.36395) + | > loader_time: 0.00240 (0.29930) + + + --> STEP: 56/234 -- GLOBAL_STEP: 14330 + | > loss: 0.10279 (0.08342) + | > log_mle: -0.11722 (-0.10761) + | > loss_dur: 0.22001 (0.19103) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.83097 (3.71473) + | > current_lr: 0.00002 + | > step_time: 1.81580 (3.17479) + | > loader_time: 0.07820 (0.27411) + + + --> STEP: 61/234 -- GLOBAL_STEP: 14335 + | > loss: 0.05973 (0.08077) + | > log_mle: -0.11310 (-0.10860) + | > loss_dur: 0.17283 (0.18937) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.63712 (3.69579) + | > current_lr: 0.00002 + | > step_time: 3.49960 (3.09650) + | > loader_time: 0.00250 (0.25504) + + + --> STEP: 66/234 -- GLOBAL_STEP: 14340 + | > loss: 0.08527 (0.08012) + | > log_mle: -0.10342 (-0.10945) + | > loss_dur: 0.18869 (0.18957) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.63683 (3.82535) + | > current_lr: 0.00002 + | > step_time: 2.39180 (3.00926) + | > loader_time: 0.00330 (0.23723) + + + --> STEP: 71/234 -- GLOBAL_STEP: 14345 + | > loss: 0.06193 (0.07916) + | > log_mle: -0.14313 (-0.11032) + | > loss_dur: 0.20506 (0.18948) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.46934 (4.56152) + | > current_lr: 0.00002 + | > step_time: 1.29630 (2.91466) + | > loader_time: 0.00280 (0.22698) + + + --> STEP: 76/234 -- GLOBAL_STEP: 14350 + | > loss: 0.06421 (0.07825) + | > log_mle: -0.12831 (-0.11137) + | > loss_dur: 0.19252 (0.18963) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.75236 (4.63484) + | > current_lr: 0.00002 + | > step_time: 1.27140 (2.83040) + | > loader_time: 0.00220 (0.21224) + + + --> STEP: 81/234 -- GLOBAL_STEP: 14355 + | > loss: 0.04293 (0.07669) + | > log_mle: -0.14114 (-0.11214) + | > loss_dur: 0.18407 (0.18883) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.61244 (4.68242) + | > current_lr: 0.00002 + | > step_time: 1.39290 (2.76161) + | > loader_time: 0.00190 (0.20036) + + + --> STEP: 86/234 -- GLOBAL_STEP: 14360 + | > loss: 0.05042 (0.07549) + | > log_mle: -0.14200 (-0.11327) + | > loss_dur: 0.19242 (0.18876) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.40325 (4.81867) + | > current_lr: 0.00002 + | > step_time: 2.39730 (2.70592) + | > loader_time: 0.00280 (0.19087) + + + --> STEP: 91/234 -- GLOBAL_STEP: 14365 + | > loss: 0.06514 (0.07406) + | > log_mle: -0.14998 (-0.11556) + | > loss_dur: 0.21511 (0.18963) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.57055 (5.03296) + | > current_lr: 0.00002 + | > step_time: 1.29940 (2.65055) + | > loader_time: 0.00280 (0.18153) + + + --> STEP: 96/234 -- GLOBAL_STEP: 14370 + | > loss: 0.06319 (0.07130) + | > log_mle: -0.13957 (-0.11928) + | > loss_dur: 0.20276 (0.19058) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.19314 (5.43116) + | > current_lr: 0.00002 + | > step_time: 1.54070 (2.61420) + | > loader_time: 0.00450 (0.17225) + + + --> STEP: 101/234 -- GLOBAL_STEP: 14375 + | > loss: 0.03221 (0.06948) + | > log_mle: -0.19510 (-0.12198) + | > loss_dur: 0.22730 (0.19145) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.06516 (5.80973) + | > current_lr: 0.00002 + | > step_time: 2.10250 (2.57544) + | > loader_time: 0.00310 (0.16471) + + + --> STEP: 106/234 -- GLOBAL_STEP: 14380 + | > loss: 0.04284 (0.06778) + | > log_mle: -0.19567 (-0.12496) + | > loss_dur: 0.23851 (0.19273) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.56653 (6.19752) + | > current_lr: 0.00002 + | > step_time: 1.16430 (2.54691) + | > loader_time: 0.00270 (0.15892) + + + --> STEP: 111/234 -- GLOBAL_STEP: 14385 + | > loss: 0.00899 (0.06604) + | > log_mle: -0.23192 (-0.12801) + | > loss_dur: 0.24091 (0.19405) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.15252 (6.58708) + | > current_lr: 0.00002 + | > step_time: 1.69720 (2.51767) + | > loader_time: 0.00260 (0.15429) + + + --> STEP: 116/234 -- GLOBAL_STEP: 14390 + | > loss: 0.04720 (0.06466) + | > log_mle: -0.20425 (-0.13099) + | > loss_dur: 0.25144 (0.19564) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.77059 (6.87816) + | > current_lr: 0.00002 + | > step_time: 4.29430 (2.51097) + | > loader_time: 0.09720 (0.15013) + + + --> STEP: 121/234 -- GLOBAL_STEP: 14395 + | > loss: 0.07503 (0.06323) + | > log_mle: -0.12167 (-0.13306) + | > loss_dur: 0.19670 (0.19629) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.58915 (7.08956) + | > current_lr: 0.00002 + | > step_time: 1.33410 (2.50063) + | > loader_time: 0.00260 (0.14542) + + + --> STEP: 126/234 -- GLOBAL_STEP: 14400 + | > loss: -0.01871 (0.06106) + | > log_mle: -0.24799 (-0.13575) + | > loss_dur: 0.22928 (0.19680) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.16488 (7.41349) + | > current_lr: 0.00002 + | > step_time: 6.61170 (2.52295) + | > loader_time: 0.08660 (0.14251) + + + --> STEP: 131/234 -- GLOBAL_STEP: 14405 + | > loss: -0.03545 (0.05872) + | > log_mle: -0.28380 (-0.13927) + | > loss_dur: 0.24835 (0.19799) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.27747 (7.82621) + | > current_lr: 0.00002 + | > step_time: 2.72280 (2.50606) + | > loader_time: 0.00710 (0.13723) + + + --> STEP: 136/234 -- GLOBAL_STEP: 14410 + | > loss: -0.06135 (0.05650) + | > log_mle: -0.32844 (-0.14279) + | > loss_dur: 0.26709 (0.19929) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.58832 (8.14592) + | > current_lr: 0.00002 + | > step_time: 2.50550 (2.49973) + | > loader_time: 0.09810 (0.13362) + + + --> STEP: 141/234 -- GLOBAL_STEP: 14415 + | > loss: 0.00180 (0.05467) + | > log_mle: -0.24390 (-0.14587) + | > loss_dur: 0.24570 (0.20053) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.83302 (8.53479) + | > current_lr: 0.00002 + | > step_time: 2.08130 (2.47133) + | > loader_time: 0.00420 (0.12899) + + + --> STEP: 146/234 -- GLOBAL_STEP: 14420 + | > loss: -0.04475 (0.05200) + | > log_mle: -0.29080 (-0.15037) + | > loss_dur: 0.24605 (0.20237) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.71801 (9.49899) + | > current_lr: 0.00002 + | > step_time: 2.19790 (2.44433) + | > loader_time: 0.00310 (0.12591) + + + --> STEP: 151/234 -- GLOBAL_STEP: 14425 + | > loss: -0.03684 (0.04916) + | > log_mle: -0.25975 (-0.15410) + | > loss_dur: 0.22292 (0.20326) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.15527 (10.03745) + | > current_lr: 0.00002 + | > step_time: 5.40550 (2.54621) + | > loader_time: 0.09290 (0.13341) + + + --> STEP: 156/234 -- GLOBAL_STEP: 14430 + | > loss: -0.06102 (0.04551) + | > log_mle: -0.29730 (-0.15917) + | > loss_dur: 0.23628 (0.20468) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.16164 (10.84016) + | > current_lr: 0.00002 + | > step_time: 2.19560 (2.54130) + | > loader_time: 0.00270 (0.13048) + + + --> STEP: 161/234 -- GLOBAL_STEP: 14435 + | > loss: -0.07193 (0.04257) + | > log_mle: -0.31697 (-0.16351) + | > loss_dur: 0.24504 (0.20608) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.16408 (11.35575) + | > current_lr: 0.00002 + | > step_time: 3.09820 (2.52554) + | > loader_time: 0.00420 (0.12654) + + + --> STEP: 166/234 -- GLOBAL_STEP: 14440 + | > loss: -0.03571 (0.04008) + | > log_mle: -0.26312 (-0.16721) + | > loss_dur: 0.22741 (0.20729) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.91488 (12.01632) + | > current_lr: 0.00002 + | > step_time: 2.49290 (2.54576) + | > loader_time: 0.00260 (0.12566) + + + --> STEP: 171/234 -- GLOBAL_STEP: 14445 + | > loss: -0.11054 (0.03682) + | > log_mle: -0.35948 (-0.17208) + | > loss_dur: 0.24893 (0.20891) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.10836 (12.77615) + | > current_lr: 0.00002 + | > step_time: 2.60190 (2.56905) + | > loader_time: 0.00410 (0.12370) + + + --> STEP: 176/234 -- GLOBAL_STEP: 14450 + | > loss: -0.07139 (0.03357) + | > log_mle: -0.32962 (-0.17688) + | > loss_dur: 0.25823 (0.21045) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.55234 (13.43549) + | > current_lr: 0.00002 + | > step_time: 3.29300 (2.64262) + | > loader_time: 0.09580 (0.12365) + + + --> STEP: 181/234 -- GLOBAL_STEP: 14455 + | > loss: -0.02476 (0.03102) + | > log_mle: -0.27330 (-0.18114) + | > loss_dur: 0.24854 (0.21216) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.65436 (14.07163) + | > current_lr: 0.00002 + | > step_time: 9.99840 (2.69520) + | > loader_time: 0.19870 (0.12338) + + + --> STEP: 186/234 -- GLOBAL_STEP: 14460 + | > loss: -0.02060 (0.02837) + | > log_mle: -0.31000 (-0.18557) + | > loss_dur: 0.28940 (0.21393) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.28407 (14.66749) + | > current_lr: 0.00002 + | > step_time: 2.80330 (2.77334) + | > loader_time: 0.00320 (0.12185) + + + --> STEP: 191/234 -- GLOBAL_STEP: 14465 + | > loss: -0.07191 (0.02539) + | > log_mle: -0.32095 (-0.18981) + | > loss_dur: 0.24904 (0.21519) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 34.83768 (15.16663) + | > current_lr: 0.00002 + | > step_time: 2.30780 (2.76630) + | > loader_time: 0.00340 (0.11922) + + + --> STEP: 196/234 -- GLOBAL_STEP: 14470 + | > loss: -0.04811 (0.02248) + | > log_mle: -0.32013 (-0.19413) + | > loss_dur: 0.27202 (0.21660) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.56459 (15.78594) + | > current_lr: 0.00002 + | > step_time: 3.80410 (2.81149) + | > loader_time: 0.09840 (0.11870) + + + --> STEP: 201/234 -- GLOBAL_STEP: 14475 + | > loss: -0.01461 (0.02005) + | > log_mle: -0.29404 (-0.19802) + | > loss_dur: 0.27943 (0.21807) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.50813 (16.23511) + | > current_lr: 0.00002 + | > step_time: 6.79520 (2.82154) + | > loader_time: 0.00350 (0.11668) + + + --> STEP: 206/234 -- GLOBAL_STEP: 14480 + | > loss: -0.10871 (0.01713) + | > log_mle: -0.38615 (-0.20237) + | > loss_dur: 0.27744 (0.21950) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.38787 (16.81975) + | > current_lr: 0.00002 + | > step_time: 1.99590 (2.85704) + | > loader_time: 0.00890 (0.11613) + + + --> STEP: 211/234 -- GLOBAL_STEP: 14485 + | > loss: -0.14503 (0.01387) + | > log_mle: -0.45061 (-0.20722) + | > loss_dur: 0.30558 (0.22109) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 53.45991 (17.54500) + | > current_lr: 0.00002 + | > step_time: 4.90290 (2.87132) + | > loader_time: 0.10330 (0.11398) + + + --> STEP: 216/234 -- GLOBAL_STEP: 14490 + | > loss: -0.14514 (0.01061) + | > log_mle: -0.44075 (-0.21190) + | > loss_dur: 0.29561 (0.22251) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.32149 (18.00433) + | > current_lr: 0.00002 + | > step_time: 3.29780 (2.99463) + | > loader_time: 0.00400 (0.11283) + + + --> STEP: 221/234 -- GLOBAL_STEP: 14495 + | > loss: -0.08705 (0.00755) + | > log_mle: -0.36389 (-0.21651) + | > loss_dur: 0.27683 (0.22405) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.51225 (18.64429) + | > current_lr: 0.00002 + | > step_time: 2.71400 (3.02151) + | > loader_time: 0.09750 (0.11167) + + + --> STEP: 226/234 -- GLOBAL_STEP: 14500 + | > loss: -0.15087 (0.00424) + | > log_mle: -0.45151 (-0.22162) + | > loss_dur: 0.30065 (0.22586) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 63.29332 (19.25918) + | > current_lr: 0.00002 + | > step_time: 0.24410 (2.96625) + | > loader_time: 0.00440 (0.10964) + + + --> STEP: 231/234 -- GLOBAL_STEP: 14505 + | > loss: -0.06857 (0.00194) + | > log_mle: -0.51000 (-0.22692) + | > loss_dur: 0.44143 (0.22886) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 54.61249 (20.10972) + | > current_lr: 0.00002 + | > step_time: 0.27370 (2.90754) + | > loader_time: 0.00460 (0.10736) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.27191 (+1.11657) + | > avg_loss: -0.03349 (+0.00206) + | > avg_log_mle: -0.28891 (+0.00405) + | > avg_loss_dur: 0.25542 (-0.00200) + + + > EPOCH: 62/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 22:21:58)  + + --> STEP: 2/234 -- GLOBAL_STEP: 14510 + | > loss: 0.19280 (0.14786) + | > log_mle: -0.07698 (-0.08702) + | > loss_dur: 0.26978 (0.23488) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.75371 (3.15548) + | > current_lr: 0.00002 + | > step_time: 14.20120 (10.44697) + | > loader_time: 0.09900 (0.09836) + + + --> STEP: 7/234 -- GLOBAL_STEP: 14515 + | > loss: 0.08479 (0.12323) + | > log_mle: -0.11856 (-0.10196) + | > loss_dur: 0.20334 (0.22519) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.19165 (4.26790) + | > current_lr: 0.00002 + | > step_time: 1.70880 (5.75824) + | > loader_time: 0.00160 (0.06884) + + + --> STEP: 12/234 -- GLOBAL_STEP: 14520 + | > loss: 0.08951 (0.10836) + | > log_mle: -0.10783 (-0.10669) + | > loss_dur: 0.19734 (0.21505) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.65172 (4.29847) + | > current_lr: 0.00002 + | > step_time: 1.20840 (4.59328) + | > loader_time: 0.00110 (0.04109) + + + --> STEP: 17/234 -- GLOBAL_STEP: 14525 + | > loss: 0.10164 (0.10212) + | > log_mle: -0.09246 (-0.10593) + | > loss_dur: 0.19410 (0.20805) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.24322 (4.11863) + | > current_lr: 0.00002 + | > step_time: 3.08800 (3.99900) + | > loader_time: 0.00090 (0.02963) + + + --> STEP: 22/234 -- GLOBAL_STEP: 14530 + | > loss: 0.06828 (0.09630) + | > log_mle: -0.11616 (-0.10547) + | > loss_dur: 0.18445 (0.20177) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.22842 (4.02072) + | > current_lr: 0.00002 + | > step_time: 1.21540 (3.59874) + | > loader_time: 0.07540 (0.02668) + + + --> STEP: 27/234 -- GLOBAL_STEP: 14535 + | > loss: 0.05515 (0.09239) + | > log_mle: -0.11893 (-0.10576) + | > loss_dur: 0.17408 (0.19816) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.24684 (3.86034) + | > current_lr: 0.00002 + | > step_time: 4.70110 (3.75105) + | > loader_time: 0.00350 (0.02604) + + + --> STEP: 32/234 -- GLOBAL_STEP: 14540 + | > loss: 0.03547 (0.08754) + | > log_mle: -0.13150 (-0.10726) + | > loss_dur: 0.16697 (0.19480) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.49483 (3.73440) + | > current_lr: 0.00002 + | > step_time: 0.72070 (3.79477) + | > loader_time: 0.00190 (0.02827) + + + --> STEP: 37/234 -- GLOBAL_STEP: 14545 + | > loss: 0.05537 (0.08602) + | > log_mle: -0.11068 (-0.10794) + | > loss_dur: 0.16605 (0.19396) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.86799 (3.97819) + | > current_lr: 0.00002 + | > step_time: 1.11320 (3.54112) + | > loader_time: 0.00260 (0.03199) + + + --> STEP: 42/234 -- GLOBAL_STEP: 14550 + | > loss: 0.10097 (0.08622) + | > log_mle: -0.09713 (-0.10809) + | > loss_dur: 0.19809 (0.19430) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.67885 (4.08448) + | > current_lr: 0.00002 + | > step_time: 0.68130 (3.25955) + | > loader_time: 0.00160 (0.02842) + + + --> STEP: 47/234 -- GLOBAL_STEP: 14555 + | > loss: 0.06617 (0.08364) + | > log_mle: -0.11274 (-0.10922) + | > loss_dur: 0.17891 (0.19286) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.25084 (4.11555) + | > current_lr: 0.00002 + | > step_time: 1.88830 (3.07324) + | > loader_time: 0.00230 (0.02568) + + + --> STEP: 52/234 -- GLOBAL_STEP: 14560 + | > loss: 0.08396 (0.08181) + | > log_mle: -0.10518 (-0.10901) + | > loss_dur: 0.18914 (0.19082) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.37849 (4.01622) + | > current_lr: 0.00002 + | > step_time: 1.00670 (2.93744) + | > loader_time: 0.08310 (0.02669) + + + --> STEP: 57/234 -- GLOBAL_STEP: 14565 + | > loss: 0.08092 (0.08119) + | > log_mle: -0.10273 (-0.10995) + | > loss_dur: 0.18365 (0.19114) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.43453 (3.90055) + | > current_lr: 0.00002 + | > step_time: 1.70070 (2.79874) + | > loader_time: 0.00570 (0.02749) + + + --> STEP: 62/234 -- GLOBAL_STEP: 14570 + | > loss: 0.07145 (0.07828) + | > log_mle: -0.15095 (-0.11178) + | > loss_dur: 0.22240 (0.19006) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 63.33957 (4.84952) + | > current_lr: 0.00002 + | > step_time: 1.92570 (2.71503) + | > loader_time: 0.18310 (0.02837) + + + --> STEP: 67/234 -- GLOBAL_STEP: 14575 + | > loss: 0.04154 (0.07690) + | > log_mle: -0.13972 (-0.11231) + | > loss_dur: 0.18126 (0.18921) + | > amp_scaler: 16384.00000 (8558.80597) + | > grad_norm: 9.55301 (4.83120) + | > current_lr: 0.00002 + | > step_time: 2.99870 (2.67065) + | > loader_time: 0.00230 (0.03018) + + + --> STEP: 72/234 -- GLOBAL_STEP: 14580 + | > loss: 0.07204 (0.07671) + | > log_mle: -0.12022 (-0.11296) + | > loss_dur: 0.19226 (0.18967) + | > amp_scaler: 16384.00000 (9102.22222) + | > grad_norm: 2.51126 (5.04303) + | > current_lr: 0.00002 + | > step_time: 2.14210 (2.62708) + | > loader_time: 0.00250 (0.02942) + + + --> STEP: 77/234 -- GLOBAL_STEP: 14585 + | > loss: 0.03741 (0.07519) + | > log_mle: -0.13505 (-0.11424) + | > loss_dur: 0.17246 (0.18943) + | > amp_scaler: 16384.00000 (9575.06494) + | > grad_norm: 3.80457 (5.08793) + | > current_lr: 0.00002 + | > step_time: 1.69880 (2.55810) + | > loader_time: 0.08760 (0.02981) + + + --> STEP: 82/234 -- GLOBAL_STEP: 14590 + | > loss: 0.05108 (0.07390) + | > log_mle: -0.12243 (-0.11483) + | > loss_dur: 0.17350 (0.18872) + | > amp_scaler: 16384.00000 (9990.24390) + | > grad_norm: 3.65421 (5.12042) + | > current_lr: 0.00002 + | > step_time: 2.00410 (2.56064) + | > loader_time: 0.09170 (0.03146) + + + --> STEP: 87/234 -- GLOBAL_STEP: 14595 + | > loss: 0.05779 (0.07267) + | > log_mle: -0.13530 (-0.11606) + | > loss_dur: 0.19309 (0.18874) + | > amp_scaler: 16384.00000 (10357.70115) + | > grad_norm: 5.88774 (5.23820) + | > current_lr: 0.00002 + | > step_time: 2.70360 (2.53439) + | > loader_time: 0.09840 (0.03195) + + + --> STEP: 92/234 -- GLOBAL_STEP: 14600 + | > loss: 0.00919 (0.07034) + | > log_mle: -0.17617 (-0.11878) + | > loss_dur: 0.18536 (0.18912) + | > amp_scaler: 16384.00000 (10685.21739) + | > grad_norm: 9.52252 (5.45500) + | > current_lr: 0.00002 + | > step_time: 2.22310 (2.48038) + | > loader_time: 0.08770 (0.03131) + + + --> STEP: 97/234 -- GLOBAL_STEP: 14605 + | > loss: 0.02665 (0.06772) + | > log_mle: -0.16739 (-0.12232) + | > loss_dur: 0.19404 (0.19004) + | > amp_scaler: 16384.00000 (10978.96907) + | > grad_norm: 11.09460 (5.86089) + | > current_lr: 0.00002 + | > step_time: 2.18850 (2.46027) + | > loader_time: 0.00220 (0.03058) + + + --> STEP: 102/234 -- GLOBAL_STEP: 14610 + | > loss: 0.05584 (0.06599) + | > log_mle: -0.14825 (-0.12476) + | > loss_dur: 0.20409 (0.19075) + | > amp_scaler: 16384.00000 (11243.92157) + | > grad_norm: 7.07943 (6.18421) + | > current_lr: 0.00002 + | > step_time: 2.20590 (2.43596) + | > loader_time: 0.00230 (0.03088) + + + --> STEP: 107/234 -- GLOBAL_STEP: 14615 + | > loss: 0.01425 (0.06389) + | > log_mle: -0.19230 (-0.12805) + | > loss_dur: 0.20655 (0.19194) + | > amp_scaler: 16384.00000 (11484.11215) + | > grad_norm: 16.85472 (6.72198) + | > current_lr: 0.00002 + | > step_time: 1.84410 (2.40767) + | > loader_time: 0.00190 (0.03035) + + + --> STEP: 112/234 -- GLOBAL_STEP: 14620 + | > loss: 0.02539 (0.06236) + | > log_mle: -0.19963 (-0.13097) + | > loss_dur: 0.22502 (0.19334) + | > amp_scaler: 16384.00000 (11702.85714) + | > grad_norm: 13.94606 (7.18591) + | > current_lr: 0.00002 + | > step_time: 2.98980 (2.39629) + | > loader_time: 0.00300 (0.02911) + + + --> STEP: 117/234 -- GLOBAL_STEP: 14625 + | > loss: 0.01013 (0.06077) + | > log_mle: -0.19437 (-0.13382) + | > loss_dur: 0.20450 (0.19460) + | > amp_scaler: 16384.00000 (11902.90598) + | > grad_norm: 13.00497 (7.45340) + | > current_lr: 0.00002 + | > step_time: 2.00010 (2.39617) + | > loader_time: 0.00410 (0.02948) + + + --> STEP: 122/234 -- GLOBAL_STEP: 14630 + | > loss: 0.01822 (0.05977) + | > log_mle: -0.17761 (-0.13571) + | > loss_dur: 0.19583 (0.19548) + | > amp_scaler: 16384.00000 (12086.55738) + | > grad_norm: 9.78355 (7.59791) + | > current_lr: 0.00002 + | > step_time: 1.79390 (2.40122) + | > loader_time: 0.00160 (0.02918) + + + --> STEP: 127/234 -- GLOBAL_STEP: 14635 + | > loss: 0.01198 (0.05768) + | > log_mle: -0.22665 (-0.13872) + | > loss_dur: 0.23863 (0.19640) + | > amp_scaler: 16384.00000 (12255.74803) + | > grad_norm: 14.03742 (7.88635) + | > current_lr: 0.00002 + | > step_time: 2.59040 (2.40245) + | > loader_time: 0.00740 (0.02888) + + + --> STEP: 132/234 -- GLOBAL_STEP: 14640 + | > loss: -0.00705 (0.05526) + | > log_mle: -0.20771 (-0.14205) + | > loss_dur: 0.20067 (0.19730) + | > amp_scaler: 16384.00000 (12412.12121) + | > grad_norm: 15.43394 (8.26448) + | > current_lr: 0.00002 + | > step_time: 2.09900 (2.38251) + | > loader_time: 0.09490 (0.02931) + + + --> STEP: 137/234 -- GLOBAL_STEP: 14645 + | > loss: 0.03410 (0.05348) + | > log_mle: -0.21865 (-0.14556) + | > loss_dur: 0.25275 (0.19904) + | > amp_scaler: 16384.00000 (12557.08029) + | > grad_norm: 17.15948 (8.69753) + | > current_lr: 0.00002 + | > step_time: 3.09340 (2.37955) + | > loader_time: 0.00450 (0.02966) + + + --> STEP: 142/234 -- GLOBAL_STEP: 14650 + | > loss: -0.00987 (0.05140) + | > log_mle: -0.23409 (-0.14867) + | > loss_dur: 0.22422 (0.20007) + | > amp_scaler: 16384.00000 (12691.83099) + | > grad_norm: 20.52941 (9.12622) + | > current_lr: 0.00002 + | > step_time: 1.29750 (2.40048) + | > loader_time: 0.10060 (0.03230) + + + --> STEP: 147/234 -- GLOBAL_STEP: 14655 + | > loss: -0.00622 (0.04865) + | > log_mle: -0.23606 (-0.15315) + | > loss_dur: 0.22984 (0.20179) + | > amp_scaler: 8192.00000 (12538.77551) + | > grad_norm: 23.79371 (9.69351) + | > current_lr: 0.00002 + | > step_time: 5.29830 (2.42568) + | > loader_time: 0.10840 (0.03255) + + + --> STEP: 152/234 -- GLOBAL_STEP: 14660 + | > loss: -0.05083 (0.04572) + | > log_mle: -0.30486 (-0.15727) + | > loss_dur: 0.25403 (0.20299) + | > amp_scaler: 8192.00000 (12395.78947) + | > grad_norm: 37.50430 (10.48028) + | > current_lr: 0.00002 + | > step_time: 2.49470 (2.40895) + | > loader_time: 0.00410 (0.03212) + + + --> STEP: 157/234 -- GLOBAL_STEP: 14665 + | > loss: -0.01326 (0.04243) + | > log_mle: -0.25859 (-0.16194) + | > loss_dur: 0.24533 (0.20437) + | > amp_scaler: 8192.00000 (12261.91083) + | > grad_norm: 26.20033 (11.26017) + | > current_lr: 0.00002 + | > step_time: 1.08550 (2.41234) + | > loader_time: 0.00310 (0.03126) + + + --> STEP: 162/234 -- GLOBAL_STEP: 14670 + | > loss: -0.06198 (0.03922) + | > log_mle: -0.29363 (-0.16641) + | > loss_dur: 0.23166 (0.20563) + | > amp_scaler: 8192.00000 (12136.29630) + | > grad_norm: 27.19498 (11.91952) + | > current_lr: 0.00002 + | > step_time: 3.10370 (2.43066) + | > loader_time: 0.00290 (0.03196) + + + --> STEP: 167/234 -- GLOBAL_STEP: 14675 + | > loss: -0.11744 (0.03635) + | > log_mle: -0.36237 (-0.17049) + | > loss_dur: 0.24493 (0.20685) + | > amp_scaler: 8192.00000 (12018.20359) + | > grad_norm: 36.73734 (12.66198) + | > current_lr: 0.00002 + | > step_time: 1.69700 (2.41655) + | > loader_time: 0.09070 (0.03276) + + + --> STEP: 172/234 -- GLOBAL_STEP: 14680 + | > loss: -0.07782 (0.03339) + | > log_mle: -0.35331 (-0.17531) + | > loss_dur: 0.27549 (0.20870) + | > amp_scaler: 8192.00000 (11906.97674) + | > grad_norm: 52.54789 (13.39767) + | > current_lr: 0.00002 + | > step_time: 1.69730 (2.42358) + | > loader_time: 0.00440 (0.03245) + + + --> STEP: 177/234 -- GLOBAL_STEP: 14685 + | > loss: -0.04919 (0.03030) + | > log_mle: -0.31603 (-0.17987) + | > loss_dur: 0.26685 (0.21018) + | > amp_scaler: 8192.00000 (11802.03390) + | > grad_norm: 31.68932 (13.90392) + | > current_lr: 0.00002 + | > step_time: 4.80340 (2.47443) + | > loader_time: 0.00480 (0.03373) + + + --> STEP: 182/234 -- GLOBAL_STEP: 14690 + | > loss: -0.07506 (0.02749) + | > log_mle: -0.35919 (-0.18441) + | > loss_dur: 0.28413 (0.21190) + | > amp_scaler: 8192.00000 (11702.85714) + | > grad_norm: 37.41082 (14.49082) + | > current_lr: 0.00002 + | > step_time: 3.91120 (2.48517) + | > loader_time: 0.19490 (0.03492) + + + --> STEP: 187/234 -- GLOBAL_STEP: 14695 + | > loss: -0.09119 (0.02479) + | > log_mle: -0.35777 (-0.18883) + | > loss_dur: 0.26658 (0.21363) + | > amp_scaler: 8192.00000 (11608.98396) + | > grad_norm: 34.39302 (15.01043) + | > current_lr: 0.00002 + | > step_time: 3.30770 (2.53694) + | > loader_time: 0.00540 (0.03669) + + + --> STEP: 192/234 -- GLOBAL_STEP: 14700 + | > loss: -0.12509 (0.02173) + | > log_mle: -0.38146 (-0.19314) + | > loss_dur: 0.25637 (0.21487) + | > amp_scaler: 8192.00000 (11520.00000) + | > grad_norm: 41.92150 (15.75260) + | > current_lr: 0.00002 + | > step_time: 4.40160 (2.61676) + | > loader_time: 0.09750 (0.04044) + + + --> STEP: 197/234 -- GLOBAL_STEP: 14705 + | > loss: -0.10436 (0.01896) + | > log_mle: -0.35522 (-0.19733) + | > loss_dur: 0.25085 (0.21629) + | > amp_scaler: 8192.00000 (11435.53299) + | > grad_norm: 50.88973 (16.29136) + | > current_lr: 0.00002 + | > step_time: 1.78890 (2.66199) + | > loader_time: 0.00450 (0.04241) + + + --> STEP: 202/234 -- GLOBAL_STEP: 14710 + | > loss: -0.15607 (0.01623) + | > log_mle: -0.43730 (-0.20159) + | > loss_dur: 0.28123 (0.21782) + | > amp_scaler: 8192.00000 (11355.24752) + | > grad_norm: 47.15623 (16.85632) + | > current_lr: 0.00002 + | > step_time: 15.50690 (2.78581) + | > loader_time: 0.00530 (0.04326) + + + --> STEP: 207/234 -- GLOBAL_STEP: 14715 + | > loss: -0.13622 (0.01337) + | > log_mle: -0.42524 (-0.20586) + | > loss_dur: 0.28903 (0.21923) + | > amp_scaler: 8192.00000 (11278.84058) + | > grad_norm: 50.83336 (17.34940) + | > current_lr: 0.00002 + | > step_time: 10.20670 (2.85094) + | > loader_time: 0.29310 (0.04512) + + + --> STEP: 212/234 -- GLOBAL_STEP: 14720 + | > loss: -0.11956 (0.01023) + | > log_mle: -0.41120 (-0.21067) + | > loss_dur: 0.29164 (0.22090) + | > amp_scaler: 8192.00000 (11206.03774) + | > grad_norm: 41.07837 (17.92033) + | > current_lr: 0.00002 + | > step_time: 4.30290 (2.87762) + | > loader_time: 0.00570 (0.04534) + + + --> STEP: 217/234 -- GLOBAL_STEP: 14725 + | > loss: -0.13456 (0.00695) + | > log_mle: -0.42404 (-0.21535) + | > loss_dur: 0.28948 (0.22231) + | > amp_scaler: 8192.00000 (11136.58986) + | > grad_norm: 53.27179 (18.61207) + | > current_lr: 0.00002 + | > step_time: 6.19490 (2.95772) + | > loader_time: 0.09010 (0.04564) + + + --> STEP: 222/234 -- GLOBAL_STEP: 14730 + | > loss: -0.12326 (0.00396) + | > log_mle: -0.43774 (-0.21997) + | > loss_dur: 0.31448 (0.22393) + | > amp_scaler: 8192.00000 (11070.27027) + | > grad_norm: 52.23263 (19.29585) + | > current_lr: 0.00002 + | > step_time: 3.38930 (2.98019) + | > loader_time: 0.00950 (0.04563) + + + --> STEP: 227/234 -- GLOBAL_STEP: 14735 + | > loss: -0.09728 (0.00074) + | > log_mle: -0.41472 (-0.22493) + | > loss_dur: 0.31743 (0.22568) + | > amp_scaler: 8192.00000 (11006.87225) + | > grad_norm: 46.78538 (19.98086) + | > current_lr: 0.00002 + | > step_time: 0.24740 (2.94259) + | > loader_time: 0.00460 (0.04472) + + + --> STEP: 232/234 -- GLOBAL_STEP: 14740 + | > loss: -0.03247 (-0.00131) + | > log_mle: -0.59584 (-0.23103) + | > loss_dur: 0.56337 (0.22971) + | > amp_scaler: 8192.00000 (10946.20690) + | > grad_norm: 71.16146 (20.77449) + | > current_lr: 0.00002 + | > step_time: 0.32600 (2.88511) + | > loader_time: 0.09210 (0.04422) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04799 (-1.22391) + | > avg_loss: 0.00427 (+0.03776) + | > avg_log_mle: -0.25255 (+0.03636) + | > avg_loss_dur: 0.25682 (+0.00140) + + + > EPOCH: 63/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 22:34:22)  + + --> STEP: 3/234 -- GLOBAL_STEP: 14745 + | > loss: 0.15590 (0.14349) + | > log_mle: -0.10614 (-0.09600) + | > loss_dur: 0.26204 (0.23949) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.61330 (3.03166) + | > current_lr: 0.00002 + | > step_time: 3.79570 (9.13078) + | > loader_time: 0.00410 (3.43206) + + + --> STEP: 8/234 -- GLOBAL_STEP: 14750 + | > loss: 0.08167 (0.11702) + | > log_mle: -0.12052 (-0.10762) + | > loss_dur: 0.20219 (0.22464) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.40189 (4.44349) + | > current_lr: 0.00002 + | > step_time: 1.91780 (5.79074) + | > loader_time: 0.09120 (1.32258) + + + --> STEP: 13/234 -- GLOBAL_STEP: 14755 + | > loss: 0.10296 (0.10452) + | > log_mle: -0.10415 (-0.10976) + | > loss_dur: 0.20711 (0.21428) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.04606 (4.72512) + | > current_lr: 0.00002 + | > step_time: 1.07640 (4.02955) + | > loader_time: 0.00130 (0.81506) + + + --> STEP: 18/234 -- GLOBAL_STEP: 14760 + | > loss: 0.06971 (0.09689) + | > log_mle: -0.11033 (-0.10954) + | > loss_dur: 0.18003 (0.20642) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.44576 (4.43080) + | > current_lr: 0.00002 + | > step_time: 0.73760 (3.19388) + | > loader_time: 0.00140 (0.59338) + + + --> STEP: 23/234 -- GLOBAL_STEP: 14765 + | > loss: 0.05693 (0.09111) + | > log_mle: -0.11526 (-0.10907) + | > loss_dur: 0.17219 (0.20018) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.57046 (4.24580) + | > current_lr: 0.00002 + | > step_time: 1.36960 (2.81319) + | > loader_time: 0.07810 (0.46821) + + + --> STEP: 28/234 -- GLOBAL_STEP: 14770 + | > loss: 0.04224 (0.08664) + | > log_mle: -0.10768 (-0.10893) + | > loss_dur: 0.14992 (0.19557) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.78632 (4.04633) + | > current_lr: 0.00002 + | > step_time: 2.90200 (2.66917) + | > loader_time: 0.00260 (0.38802) + + + --> STEP: 33/234 -- GLOBAL_STEP: 14775 + | > loss: 0.08150 (0.08325) + | > log_mle: -0.10120 (-0.11009) + | > loss_dur: 0.18269 (0.19334) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.76172 (3.90085) + | > current_lr: 0.00002 + | > step_time: 2.70410 (2.56188) + | > loader_time: 0.00320 (0.34103) + + + --> STEP: 38/234 -- GLOBAL_STEP: 14780 + | > loss: 0.10198 (0.08152) + | > log_mle: -0.12136 (-0.11140) + | > loss_dur: 0.22335 (0.19292) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.31183 (3.99618) + | > current_lr: 0.00002 + | > step_time: 0.86550 (2.64080) + | > loader_time: 0.00200 (0.30396) + + + --> STEP: 43/234 -- GLOBAL_STEP: 14785 + | > loss: 0.06251 (0.08111) + | > log_mle: -0.12304 (-0.11150) + | > loss_dur: 0.18555 (0.19261) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.36216 (3.98472) + | > current_lr: 0.00002 + | > step_time: 2.09450 (2.50324) + | > loader_time: 0.00200 (0.26888) + + + --> STEP: 48/234 -- GLOBAL_STEP: 14790 + | > loss: 0.04357 (0.07856) + | > log_mle: -0.10752 (-0.11228) + | > loss_dur: 0.15109 (0.19085) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.98414 (3.93586) + | > current_lr: 0.00002 + | > step_time: 1.13820 (2.41907) + | > loader_time: 0.00270 (0.24284) + + + --> STEP: 53/234 -- GLOBAL_STEP: 14795 + | > loss: 0.08071 (0.07730) + | > log_mle: -0.12941 (-0.11239) + | > loss_dur: 0.21012 (0.18969) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.82012 (3.85248) + | > current_lr: 0.00002 + | > step_time: 1.28630 (2.34111) + | > loader_time: 0.00190 (0.22010) + + + --> STEP: 58/234 -- GLOBAL_STEP: 14800 + | > loss: 0.05668 (0.07643) + | > log_mle: -0.11336 (-0.11294) + | > loss_dur: 0.17004 (0.18937) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.37295 (3.76152) + | > current_lr: 0.00002 + | > step_time: 2.68810 (2.30044) + | > loader_time: 0.00290 (0.20135) + + + --> STEP: 63/234 -- GLOBAL_STEP: 14805 + | > loss: 0.07867 (0.07412) + | > log_mle: -0.12701 (-0.11492) + | > loss_dur: 0.20567 (0.18904) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.62329 (3.96970) + | > current_lr: 0.00002 + | > step_time: 1.04830 (2.29562) + | > loader_time: 0.00220 (0.18815) + + + --> STEP: 68/234 -- GLOBAL_STEP: 14810 + | > loss: 0.07712 (0.07310) + | > log_mle: -0.11854 (-0.11527) + | > loss_dur: 0.19567 (0.18837) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.87324 (4.03905) + | > current_lr: 0.00002 + | > step_time: 2.40510 (2.25811) + | > loader_time: 0.00300 (0.17582) + + + --> STEP: 73/234 -- GLOBAL_STEP: 14815 + | > loss: 0.05522 (0.07312) + | > log_mle: -0.14430 (-0.11616) + | > loss_dur: 0.19951 (0.18928) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.16187 (4.32416) + | > current_lr: 0.00002 + | > step_time: 1.19620 (2.24056) + | > loader_time: 0.00370 (0.16752) + + + --> STEP: 78/234 -- GLOBAL_STEP: 14820 + | > loss: 0.07145 (0.07182) + | > log_mle: -0.11656 (-0.11702) + | > loss_dur: 0.18801 (0.18884) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.94084 (4.43502) + | > current_lr: 0.00002 + | > step_time: 2.70490 (2.21433) + | > loader_time: 0.08530 (0.15799) + + + --> STEP: 83/234 -- GLOBAL_STEP: 14825 + | > loss: 0.04973 (0.07003) + | > log_mle: -0.14807 (-0.11800) + | > loss_dur: 0.19780 (0.18802) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.99117 (4.50381) + | > current_lr: 0.00002 + | > step_time: 1.43500 (2.16626) + | > loader_time: 0.00340 (0.15064) + + + --> STEP: 88/234 -- GLOBAL_STEP: 14830 + | > loss: 0.01593 (0.06843) + | > log_mle: -0.18191 (-0.11962) + | > loss_dur: 0.19784 (0.18805) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.25322 (4.69516) + | > current_lr: 0.00002 + | > step_time: 1.72160 (2.15362) + | > loader_time: 0.08500 (0.14421) + + + --> STEP: 93/234 -- GLOBAL_STEP: 14835 + | > loss: 0.01542 (0.06620) + | > log_mle: -0.19620 (-0.12238) + | > loss_dur: 0.21162 (0.18858) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.45367 (5.02331) + | > current_lr: 0.00002 + | > step_time: 1.50170 (2.15303) + | > loader_time: 0.00460 (0.14046) + + + --> STEP: 98/234 -- GLOBAL_STEP: 14840 + | > loss: 0.07202 (0.06410) + | > log_mle: -0.12311 (-0.12509) + | > loss_dur: 0.19512 (0.18919) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.48273 (5.32471) + | > current_lr: 0.00002 + | > step_time: 2.20310 (2.13309) + | > loader_time: 0.08340 (0.13680) + + + --> STEP: 103/234 -- GLOBAL_STEP: 14845 + | > loss: 0.02180 (0.06185) + | > log_mle: -0.21646 (-0.12844) + | > loss_dur: 0.23826 (0.19028) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.74007 (5.71473) + | > current_lr: 0.00002 + | > step_time: 1.71270 (2.12403) + | > loader_time: 0.00270 (0.13033) + + + --> STEP: 108/234 -- GLOBAL_STEP: 14850 + | > loss: 0.04531 (0.06006) + | > log_mle: -0.16708 (-0.13129) + | > loss_dur: 0.21239 (0.19135) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.77258 (6.04075) + | > current_lr: 0.00002 + | > step_time: 1.09550 (2.10824) + | > loader_time: 0.00260 (0.12588) + + + --> STEP: 113/234 -- GLOBAL_STEP: 14855 + | > loss: 0.01010 (0.05821) + | > log_mle: -0.21301 (-0.13463) + | > loss_dur: 0.22311 (0.19284) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.61586 (6.66833) + | > current_lr: 0.00002 + | > step_time: 2.50560 (2.12726) + | > loader_time: 0.08600 (0.12193) + + + --> STEP: 118/234 -- GLOBAL_STEP: 14860 + | > loss: 0.02335 (0.05670) + | > log_mle: -0.18240 (-0.13717) + | > loss_dur: 0.20575 (0.19386) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.64145 (6.92267) + | > current_lr: 0.00002 + | > step_time: 1.20610 (2.12179) + | > loader_time: 0.00270 (0.11844) + + + --> STEP: 123/234 -- GLOBAL_STEP: 14865 + | > loss: 0.03284 (0.05539) + | > log_mle: -0.15422 (-0.13883) + | > loss_dur: 0.18706 (0.19422) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.21130 (7.00510) + | > current_lr: 0.00002 + | > step_time: 2.09800 (2.10551) + | > loader_time: 0.00330 (0.11443) + + + --> STEP: 128/234 -- GLOBAL_STEP: 14870 + | > loss: -0.01068 (0.05283) + | > log_mle: -0.20991 (-0.14227) + | > loss_dur: 0.19923 (0.19510) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.51473 (7.47426) + | > current_lr: 0.00002 + | > step_time: 2.29230 (2.13973) + | > loader_time: 0.19630 (0.11537) + + + --> STEP: 133/234 -- GLOBAL_STEP: 14875 + | > loss: 0.00548 (0.05062) + | > log_mle: -0.23251 (-0.14569) + | > loss_dur: 0.23799 (0.19631) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.62562 (8.03355) + | > current_lr: 0.00002 + | > step_time: 4.11430 (2.13915) + | > loader_time: 0.18160 (0.11324) + + + --> STEP: 138/234 -- GLOBAL_STEP: 14880 + | > loss: 0.01990 (0.04877) + | > log_mle: -0.19200 (-0.14888) + | > loss_dur: 0.21190 (0.19765) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.17783 (8.51145) + | > current_lr: 0.00002 + | > step_time: 1.99630 (2.12803) + | > loader_time: 0.00380 (0.11134) + + + --> STEP: 143/234 -- GLOBAL_STEP: 14885 + | > loss: -0.04219 (0.04628) + | > log_mle: -0.31271 (-0.15282) + | > loss_dur: 0.27052 (0.19910) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.61576 (9.13748) + | > current_lr: 0.00002 + | > step_time: 1.70980 (2.20142) + | > loader_time: 0.08960 (0.11497) + + + --> STEP: 148/234 -- GLOBAL_STEP: 14890 + | > loss: -0.02472 (0.04358) + | > log_mle: -0.23995 (-0.15677) + | > loss_dur: 0.21522 (0.20034) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.41769 (9.72046) + | > current_lr: 0.00002 + | > step_time: 3.30460 (2.19076) + | > loader_time: 0.09340 (0.11235) + + + --> STEP: 153/234 -- GLOBAL_STEP: 14895 + | > loss: -0.10396 (0.04013) + | > log_mle: -0.34863 (-0.16158) + | > loss_dur: 0.24468 (0.20172) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 63.63403 (10.51228) + | > current_lr: 0.00002 + | > step_time: 1.40230 (2.18598) + | > loader_time: 0.00360 (0.10983) + + + --> STEP: 158/234 -- GLOBAL_STEP: 14900 + | > loss: -0.04196 (0.03722) + | > log_mle: -0.29442 (-0.16587) + | > loss_dur: 0.25246 (0.20309) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.59795 (11.06436) + | > current_lr: 0.00002 + | > step_time: 3.93570 (2.23286) + | > loader_time: 0.09260 (0.10826) + + + --> STEP: 163/234 -- GLOBAL_STEP: 14905 + | > loss: -0.03376 (0.03413) + | > log_mle: -0.27054 (-0.17014) + | > loss_dur: 0.23678 (0.20427) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.72576 (11.66439) + | > current_lr: 0.00002 + | > step_time: 2.59670 (2.21863) + | > loader_time: 0.00370 (0.10551) + + + --> STEP: 168/234 -- GLOBAL_STEP: 14910 + | > loss: -0.04671 (0.03120) + | > log_mle: -0.31636 (-0.17447) + | > loss_dur: 0.26965 (0.20567) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 36.99296 (12.32594) + | > current_lr: 0.00002 + | > step_time: 1.80090 (2.23767) + | > loader_time: 0.08690 (0.10506) + + + --> STEP: 173/234 -- GLOBAL_STEP: 14915 + | > loss: -0.06672 (0.02815) + | > log_mle: -0.32056 (-0.17921) + | > loss_dur: 0.25385 (0.20736) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.61788 (13.18712) + | > current_lr: 0.00002 + | > step_time: 1.70770 (2.26244) + | > loader_time: 0.00280 (0.10422) + + + --> STEP: 178/234 -- GLOBAL_STEP: 14920 + | > loss: -0.09185 (0.02513) + | > log_mle: -0.37878 (-0.18400) + | > loss_dur: 0.28693 (0.20913) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.97320 (13.91940) + | > current_lr: 0.00002 + | > step_time: 3.89720 (2.26760) + | > loader_time: 0.10350 (0.10337) + + + --> STEP: 183/234 -- GLOBAL_STEP: 14925 + | > loss: -0.10699 (0.02235) + | > log_mle: -0.37430 (-0.18844) + | > loss_dur: 0.26731 (0.21079) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.96359 (14.54164) + | > current_lr: 0.00002 + | > step_time: 1.39110 (2.27177) + | > loader_time: 0.00410 (0.10221) + + + --> STEP: 188/234 -- GLOBAL_STEP: 14930 + | > loss: -0.10962 (0.01951) + | > log_mle: -0.38490 (-0.19288) + | > loss_dur: 0.27528 (0.21239) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.85390 (15.12059) + | > current_lr: 0.00002 + | > step_time: 3.30260 (2.28526) + | > loader_time: 0.19780 (0.10160) + + + --> STEP: 193/234 -- GLOBAL_STEP: 14935 + | > loss: -0.10998 (0.01645) + | > log_mle: -0.38503 (-0.19717) + | > loss_dur: 0.27505 (0.21362) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.56408 (15.67417) + | > current_lr: 0.00002 + | > step_time: 4.09840 (2.29135) + | > loader_time: 0.10870 (0.10054) + + + --> STEP: 198/234 -- GLOBAL_STEP: 14940 + | > loss: -0.09804 (0.01372) + | > log_mle: -0.37361 (-0.20127) + | > loss_dur: 0.27558 (0.21499) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 49.89905 (16.26251) + | > current_lr: 0.00002 + | > step_time: 5.29970 (2.30928) + | > loader_time: 0.00410 (0.09953) + + + --> STEP: 203/234 -- GLOBAL_STEP: 14945 + | > loss: -0.06123 (0.01132) + | > log_mle: -0.32028 (-0.20524) + | > loss_dur: 0.25905 (0.21656) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.89648 (16.76739) + | > current_lr: 0.00002 + | > step_time: 3.48380 (2.37604) + | > loader_time: 0.10740 (0.09862) + + + --> STEP: 208/234 -- GLOBAL_STEP: 14950 + | > loss: -0.10646 (0.00832) + | > log_mle: -0.38993 (-0.20981) + | > loss_dur: 0.28346 (0.21813) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.72298 (17.35767) + | > current_lr: 0.00002 + | > step_time: 3.89290 (2.40546) + | > loader_time: 0.09560 (0.10376) + + + --> STEP: 213/234 -- GLOBAL_STEP: 14955 + | > loss: -0.13781 (0.00511) + | > log_mle: -0.42991 (-0.21473) + | > loss_dur: 0.29211 (0.21984) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 75.49467 (18.30779) + | > current_lr: 0.00002 + | > step_time: 10.20120 (2.46821) + | > loader_time: 0.10330 (0.10319) + + + --> STEP: 218/234 -- GLOBAL_STEP: 14960 + | > loss: -0.10798 (0.00210) + | > log_mle: -0.39438 (-0.21920) + | > loss_dur: 0.28641 (0.22129) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 59.67511 (19.06257) + | > current_lr: 0.00002 + | > step_time: 2.38940 (2.56627) + | > loader_time: 0.00630 (0.11175) + + + --> STEP: 223/234 -- GLOBAL_STEP: 14965 + | > loss: -0.14157 (-0.00095) + | > log_mle: -0.43627 (-0.22392) + | > loss_dur: 0.29471 (0.22297) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.69465 (19.80725) + | > current_lr: 0.00002 + | > step_time: 0.22840 (2.53092) + | > loader_time: 0.00300 (0.10969) + + + --> STEP: 228/234 -- GLOBAL_STEP: 14970 + | > loss: -0.11487 (-0.00400) + | > log_mle: -0.43843 (-0.22887) + | > loss_dur: 0.32357 (0.22487) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.62365 (20.46417) + | > current_lr: 0.00002 + | > step_time: 0.24110 (2.48066) + | > loader_time: 0.00490 (0.10737) + + + --> STEP: 233/234 -- GLOBAL_STEP: 14975 + | > loss: 0.53677 (-0.00310) + | > log_mle: -0.38819 (-0.23465) + | > loss_dur: 0.92497 (0.23155) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 69.39133 (21.73850) + | > current_lr: 0.00002 + | > step_time: 0.19230 (2.43312) + | > loader_time: 0.00330 (0.10516) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11981 (+0.07182) + | > avg_loss: -0.03430 (-0.03857) + | > avg_log_mle: -0.29277 (-0.04021) + | > avg_loss_dur: 0.25847 (+0.00165) + + + > EPOCH: 64/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 22:45:03)  + + --> STEP: 4/234 -- GLOBAL_STEP: 14980 + | > loss: 0.12979 (0.14451) + | > log_mle: -0.11845 (-0.10323) + | > loss_dur: 0.24824 (0.24775) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.92519 (4.80118) + | > current_lr: 0.00002 + | > step_time: 11.29480 (8.57964) + | > loader_time: 0.09870 (0.06763) + + + --> STEP: 9/234 -- GLOBAL_STEP: 14985 + | > loss: 0.08234 (0.11343) + | > log_mle: -0.12920 (-0.11177) + | > loss_dur: 0.21154 (0.22520) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.48520 (4.53175) + | > current_lr: 0.00002 + | > step_time: 1.58750 (5.26461) + | > loader_time: 0.09790 (0.05166) + + + --> STEP: 14/234 -- GLOBAL_STEP: 14990 + | > loss: 0.07015 (0.09986) + | > log_mle: -0.12020 (-0.11269) + | > loss_dur: 0.19036 (0.21255) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.92446 (4.35990) + | > current_lr: 0.00002 + | > step_time: 1.29570 (4.00773) + | > loader_time: 0.08170 (0.05904) + + + --> STEP: 19/234 -- GLOBAL_STEP: 14995 + | > loss: 0.08417 (0.09262) + | > log_mle: -0.10617 (-0.11174) + | > loss_dur: 0.19034 (0.20436) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.80008 (4.12443) + | > current_lr: 0.00002 + | > step_time: 6.10060 (3.94358) + | > loader_time: 0.10660 (0.06258) + + + --> STEP: 24/234 -- GLOBAL_STEP: 15000 + | > loss: 0.06745 (0.08639) + | > log_mle: -0.10613 (-0.11138) + | > loss_dur: 0.17358 (0.19777) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.29102 (4.02092) + | > current_lr: 0.00002 + | > step_time: 4.39120 (4.03071) + | > loader_time: 0.00890 (0.05776) + + + --> STEP: 29/234 -- GLOBAL_STEP: 15005 + | > loss: 0.05791 (0.08259) + | > log_mle: -0.10657 (-0.11140) + | > loss_dur: 0.16448 (0.19399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.00023 (3.82230) + | > current_lr: 0.00002 + | > step_time: 0.80330 (3.64520) + | > loader_time: 0.00190 (0.05099) + + + --> STEP: 34/234 -- GLOBAL_STEP: 15010 + | > loss: 0.07121 (0.07965) + | > log_mle: -0.11652 (-0.11291) + | > loss_dur: 0.18773 (0.19256) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.40355 (3.71648) + | > current_lr: 0.00002 + | > step_time: 1.81250 (3.30015) + | > loader_time: 0.08910 (0.04892) + + + --> STEP: 39/234 -- GLOBAL_STEP: 15015 + | > loss: 0.06498 (0.07761) + | > log_mle: -0.12516 (-0.11441) + | > loss_dur: 0.19014 (0.19203) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.32102 (3.91994) + | > current_lr: 0.00002 + | > step_time: 1.06550 (3.07030) + | > loader_time: 0.00320 (0.04297) + + + --> STEP: 44/234 -- GLOBAL_STEP: 15020 + | > loss: 0.05129 (0.07642) + | > log_mle: -0.11578 (-0.11432) + | > loss_dur: 0.16708 (0.19075) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.64284 (3.86235) + | > current_lr: 0.00002 + | > step_time: 2.01520 (2.93465) + | > loader_time: 0.00440 (0.03836) + + + --> STEP: 49/234 -- GLOBAL_STEP: 15025 + | > loss: 0.02888 (0.07372) + | > log_mle: -0.12491 (-0.11525) + | > loss_dur: 0.15379 (0.18897) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.28243 (3.90739) + | > current_lr: 0.00002 + | > step_time: 1.47840 (2.77499) + | > loader_time: 0.00290 (0.03678) + + + --> STEP: 54/234 -- GLOBAL_STEP: 15030 + | > loss: 0.04060 (0.07276) + | > log_mle: -0.12898 (-0.11547) + | > loss_dur: 0.16957 (0.18822) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.44545 (3.81819) + | > current_lr: 0.00002 + | > step_time: 1.72990 (2.68059) + | > loader_time: 0.00210 (0.03517) + + + --> STEP: 59/234 -- GLOBAL_STEP: 15035 + | > loss: 0.00852 (0.07102) + | > log_mle: -0.14556 (-0.11630) + | > loss_dur: 0.15408 (0.18732) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.11823 (3.77063) + | > current_lr: 0.00002 + | > step_time: 1.11000 (2.57882) + | > loader_time: 0.00270 (0.03240) + + + --> STEP: 64/234 -- GLOBAL_STEP: 15040 + | > loss: 0.04801 (0.06938) + | > log_mle: -0.11400 (-0.11775) + | > loss_dur: 0.16201 (0.18714) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.67161 (4.01129) + | > current_lr: 0.00002 + | > step_time: 1.40320 (2.54144) + | > loader_time: 0.08690 (0.03407) + + + --> STEP: 69/234 -- GLOBAL_STEP: 15045 + | > loss: 0.07635 (0.06828) + | > log_mle: -0.10096 (-0.11794) + | > loss_dur: 0.17730 (0.18621) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.89099 (4.03398) + | > current_lr: 0.00002 + | > step_time: 2.72200 (2.52706) + | > loader_time: 0.00220 (0.03323) + + + --> STEP: 74/234 -- GLOBAL_STEP: 15050 + | > loss: 0.02877 (0.06755) + | > log_mle: -0.12649 (-0.11914) + | > loss_dur: 0.15526 (0.18670) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.50584 (4.47581) + | > current_lr: 0.00002 + | > step_time: 2.30400 (2.50220) + | > loader_time: 0.00200 (0.03500) + + + --> STEP: 79/234 -- GLOBAL_STEP: 15055 + | > loss: 0.03411 (0.06618) + | > log_mle: -0.13607 (-0.12012) + | > loss_dur: 0.17018 (0.18630) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.13649 (4.48456) + | > current_lr: 0.00002 + | > step_time: 1.89810 (2.45171) + | > loader_time: 0.00300 (0.03394) + + + --> STEP: 84/234 -- GLOBAL_STEP: 15060 + | > loss: 0.05436 (0.06480) + | > log_mle: -0.13565 (-0.12107) + | > loss_dur: 0.19001 (0.18587) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.78611 (4.58883) + | > current_lr: 0.00002 + | > step_time: 1.80900 (2.46757) + | > loader_time: 0.00750 (0.03446) + + + --> STEP: 89/234 -- GLOBAL_STEP: 15065 + | > loss: 0.02006 (0.06300) + | > log_mle: -0.16495 (-0.12301) + | > loss_dur: 0.18501 (0.18601) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.16983 (4.83384) + | > current_lr: 0.00002 + | > step_time: 2.30380 (2.44251) + | > loader_time: 0.00340 (0.03267) + + + --> STEP: 94/234 -- GLOBAL_STEP: 15070 + | > loss: -0.00498 (0.06062) + | > log_mle: -0.19904 (-0.12608) + | > loss_dur: 0.19406 (0.18670) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.79251 (5.28530) + | > current_lr: 0.00002 + | > step_time: 2.19880 (2.41022) + | > loader_time: 0.08310 (0.03374) + + + --> STEP: 99/234 -- GLOBAL_STEP: 15075 + | > loss: -0.02364 (0.05845) + | > log_mle: -0.23019 (-0.12902) + | > loss_dur: 0.20655 (0.18747) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.17645 (5.74335) + | > current_lr: 0.00002 + | > step_time: 2.19740 (2.39862) + | > loader_time: 0.00290 (0.03390) + + + --> STEP: 104/234 -- GLOBAL_STEP: 15080 + | > loss: -0.03408 (0.05634) + | > log_mle: -0.24216 (-0.13238) + | > loss_dur: 0.20808 (0.18872) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.26252 (6.10886) + | > current_lr: 0.00002 + | > step_time: 1.27050 (2.35548) + | > loader_time: 0.00230 (0.03369) + + + --> STEP: 109/234 -- GLOBAL_STEP: 15085 + | > loss: 0.03766 (0.05510) + | > log_mle: -0.21170 (-0.13485) + | > loss_dur: 0.24936 (0.18995) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.23339 (6.49086) + | > current_lr: 0.00002 + | > step_time: 1.59500 (2.34620) + | > loader_time: 0.00240 (0.03425) + + + --> STEP: 114/234 -- GLOBAL_STEP: 15090 + | > loss: 0.00682 (0.05283) + | > log_mle: -0.19375 (-0.13800) + | > loss_dur: 0.20056 (0.19082) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.00347 (6.97116) + | > current_lr: 0.00002 + | > step_time: 0.79060 (2.33795) + | > loader_time: 0.00270 (0.03431) + + + --> STEP: 119/234 -- GLOBAL_STEP: 15095 + | > loss: 0.01462 (0.05164) + | > log_mle: -0.19161 (-0.14045) + | > loss_dur: 0.20623 (0.19208) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.45179 (7.27429) + | > current_lr: 0.00002 + | > step_time: 2.00240 (2.29718) + | > loader_time: 0.00360 (0.03496) + + + --> STEP: 124/234 -- GLOBAL_STEP: 15100 + | > loss: -0.01030 (0.05023) + | > log_mle: -0.21706 (-0.14225) + | > loss_dur: 0.20676 (0.19248) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.66192 (7.47702) + | > current_lr: 0.00002 + | > step_time: 2.39240 (2.29817) + | > loader_time: 0.00240 (0.03369) + + + --> STEP: 129/234 -- GLOBAL_STEP: 15105 + | > loss: 0.02093 (0.04816) + | > log_mle: -0.20211 (-0.14547) + | > loss_dur: 0.22303 (0.19363) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.39312 (8.19654) + | > current_lr: 0.00002 + | > step_time: 2.89030 (2.30366) + | > loader_time: 0.00400 (0.03377) + + + --> STEP: 134/234 -- GLOBAL_STEP: 15110 + | > loss: -0.00384 (0.04584) + | > log_mle: -0.25423 (-0.14922) + | > loss_dur: 0.25039 (0.19506) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 22.01595 (8.73793) + | > current_lr: 0.00002 + | > step_time: 1.57720 (2.28749) + | > loader_time: 0.00280 (0.03415) + + + --> STEP: 139/234 -- GLOBAL_STEP: 15115 + | > loss: -0.08004 (0.04367) + | > log_mle: -0.30921 (-0.15273) + | > loss_dur: 0.22917 (0.19640) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.93860 (9.25758) + | > current_lr: 0.00002 + | > step_time: 2.10280 (2.29953) + | > loader_time: 0.00210 (0.03500) + + + --> STEP: 144/234 -- GLOBAL_STEP: 15120 + | > loss: -0.03234 (0.04161) + | > log_mle: -0.28443 (-0.15644) + | > loss_dur: 0.25209 (0.19804) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.27024 (9.82592) + | > current_lr: 0.00002 + | > step_time: 1.79630 (2.28408) + | > loader_time: 0.09960 (0.03527) + + + --> STEP: 149/234 -- GLOBAL_STEP: 15125 + | > loss: -0.07486 (0.03869) + | > log_mle: -0.32417 (-0.16058) + | > loss_dur: 0.24931 (0.19927) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.26499 (10.51350) + | > current_lr: 0.00002 + | > step_time: 1.59430 (2.29206) + | > loader_time: 0.00260 (0.03530) + + + --> STEP: 154/234 -- GLOBAL_STEP: 15130 + | > loss: -0.05645 (0.03542) + | > log_mle: -0.29066 (-0.16511) + | > loss_dur: 0.23420 (0.20053) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.07191 (11.18105) + | > current_lr: 0.00002 + | > step_time: 5.61120 (2.33061) + | > loader_time: 0.01270 (0.03611) + + + --> STEP: 159/234 -- GLOBAL_STEP: 15135 + | > loss: -0.07276 (0.03240) + | > log_mle: -0.31071 (-0.16950) + | > loss_dur: 0.23795 (0.20190) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.89048 (11.79807) + | > current_lr: 0.00002 + | > step_time: 4.29800 (2.34785) + | > loader_time: 0.00330 (0.03570) + + + --> STEP: 164/234 -- GLOBAL_STEP: 15140 + | > loss: -0.05702 (0.02938) + | > log_mle: -0.30457 (-0.17369) + | > loss_dur: 0.24755 (0.20307) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.97353 (12.37228) + | > current_lr: 0.00002 + | > step_time: 2.31500 (2.34052) + | > loader_time: 0.08560 (0.03619) + + + --> STEP: 169/234 -- GLOBAL_STEP: 15145 + | > loss: -0.03117 (0.02664) + | > log_mle: -0.29484 (-0.17794) + | > loss_dur: 0.26367 (0.20459) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.49806 (12.99597) + | > current_lr: 0.00002 + | > step_time: 4.49630 (2.36546) + | > loader_time: 0.09930 (0.03837) + + + --> STEP: 174/234 -- GLOBAL_STEP: 15150 + | > loss: -0.12227 (0.02305) + | > log_mle: -0.38234 (-0.18315) + | > loss_dur: 0.26007 (0.20619) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.54476 (13.80847) + | > current_lr: 0.00002 + | > step_time: 6.91110 (2.44345) + | > loader_time: 0.19590 (0.03909) + + + --> STEP: 179/234 -- GLOBAL_STEP: 15155 + | > loss: -0.09333 (0.02016) + | > log_mle: -0.37001 (-0.18784) + | > loss_dur: 0.27668 (0.20801) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 55.42070 (14.39867) + | > current_lr: 0.00002 + | > step_time: 14.60520 (2.53167) + | > loader_time: 0.00580 (0.03813) + + + --> STEP: 184/234 -- GLOBAL_STEP: 15160 + | > loss: -0.08706 (0.01755) + | > log_mle: -0.34745 (-0.19204) + | > loss_dur: 0.26039 (0.20959) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.45134 (15.05802) + | > current_lr: 0.00002 + | > step_time: 3.60850 (2.58578) + | > loader_time: 0.09580 (0.03983) + + + --> STEP: 189/234 -- GLOBAL_STEP: 15165 + | > loss: -0.07673 (0.01487) + | > log_mle: -0.33863 (-0.19634) + | > loss_dur: 0.26190 (0.21121) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 58.32700 (15.92599) + | > current_lr: 0.00002 + | > step_time: 6.10360 (2.63273) + | > loader_time: 0.00730 (0.04094) + + + --> STEP: 194/234 -- GLOBAL_STEP: 15170 + | > loss: -0.10864 (0.01178) + | > log_mle: -0.37022 (-0.20063) + | > loss_dur: 0.26158 (0.21241) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 46.59646 (16.79617) + | > current_lr: 0.00002 + | > step_time: 2.50340 (2.67317) + | > loader_time: 0.08440 (0.04179) + + + --> STEP: 199/234 -- GLOBAL_STEP: 15175 + | > loss: -0.11410 (0.00917) + | > log_mle: -0.37842 (-0.20466) + | > loss_dur: 0.26432 (0.21383) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 61.73935 (17.54558) + | > current_lr: 0.00002 + | > step_time: 9.20250 (2.78688) + | > loader_time: 1.29420 (0.04836) + + + --> STEP: 204/234 -- GLOBAL_STEP: 15180 + | > loss: -0.10468 (0.00690) + | > log_mle: -0.40602 (-0.20862) + | > loss_dur: 0.30134 (0.21553) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 55.79601 (18.23840) + | > current_lr: 0.00002 + | > step_time: 2.60380 (2.84600) + | > loader_time: 0.00360 (0.04922) + + + --> STEP: 209/234 -- GLOBAL_STEP: 15185 + | > loss: -0.09956 (0.00408) + | > log_mle: -0.36972 (-0.21291) + | > loss_dur: 0.27016 (0.21698) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 49.00073 (18.94126) + | > current_lr: 0.00002 + | > step_time: 8.40220 (2.89328) + | > loader_time: 0.19920 (0.05093) + + + --> STEP: 214/234 -- GLOBAL_STEP: 15190 + | > loss: -0.13364 (0.00074) + | > log_mle: -0.39978 (-0.21793) + | > loss_dur: 0.26613 (0.21867) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.85994 (19.77516) + | > current_lr: 0.00002 + | > step_time: 11.59000 (3.03917) + | > loader_time: 0.00230 (0.05395) + + + --> STEP: 219/234 -- GLOBAL_STEP: 15195 + | > loss: -0.19439 (-0.00255) + | > log_mle: -0.48840 (-0.22277) + | > loss_dur: 0.29402 (0.22022) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 48.52382 (20.46430) + | > current_lr: 0.00002 + | > step_time: 3.09390 (3.10626) + | > loader_time: 0.08790 (0.05588) + + + --> STEP: 224/234 -- GLOBAL_STEP: 15200 + | > loss: -0.14457 (-0.00539) + | > log_mle: -0.44443 (-0.22727) + | > loss_dur: 0.29986 (0.22188) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 60.19154 (21.20143) + | > current_lr: 0.00002 + | > step_time: 0.23970 (3.04234) + | > loader_time: 0.00460 (0.05472) + + + --> STEP: 229/234 -- GLOBAL_STEP: 15205 + | > loss: -0.10929 (-0.00828) + | > log_mle: -0.46751 (-0.23224) + | > loss_dur: 0.35822 (0.22396) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 59.42417 (22.00254) + | > current_lr: 0.00002 + | > step_time: 0.24940 (2.98120) + | > loader_time: 0.00510 (0.05363) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.10096 (-0.01886) + | > avg_loss: -0.03808 (-0.00378) + | > avg_log_mle: -0.29357 (-0.00080) + | > avg_loss_dur: 0.25549 (-0.00298) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_15210.pth + + > EPOCH: 65/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 22:57:48)  + + --> STEP: 0/234 -- GLOBAL_STEP: 15210 + | > loss: 0.09425 (0.09425) + | > log_mle: -0.15026 (-0.15026) + | > loss_dur: 0.24451 (0.24451) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.11816 (5.11816) + | > current_lr: 0.00002 + | > step_time: 12.80960 (12.80962) + | > loader_time: 7.90760 (7.90759) + + + --> STEP: 5/234 -- GLOBAL_STEP: 15215 + | > loss: 0.06794 (0.11173) + | > log_mle: -0.11872 (-0.10966) + | > loss_dur: 0.18666 (0.22140) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.14994 (5.51421) + | > current_lr: 0.00002 + | > step_time: 1.26000 (4.89454) + | > loader_time: 0.00210 (0.06253) + + + --> STEP: 10/234 -- GLOBAL_STEP: 15220 + | > loss: 0.05641 (0.09058) + | > log_mle: -0.12323 (-0.11601) + | > loss_dur: 0.17964 (0.20659) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.08856 (5.35758) + | > current_lr: 0.00002 + | > step_time: 1.00450 (2.86243) + | > loader_time: 0.00100 (0.03223) + + + --> STEP: 15/234 -- GLOBAL_STEP: 15225 + | > loss: 0.06204 (0.08373) + | > log_mle: -0.11593 (-0.11578) + | > loss_dur: 0.17797 (0.19951) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.06593 (5.02121) + | > current_lr: 0.00002 + | > step_time: 1.18730 (2.38491) + | > loader_time: 0.00100 (0.02206) + + + --> STEP: 20/234 -- GLOBAL_STEP: 15230 + | > loss: 0.06457 (0.07893) + | > log_mle: -0.10862 (-0.11439) + | > loss_dur: 0.17319 (0.19332) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.02272 (4.66639) + | > current_lr: 0.00002 + | > step_time: 0.88110 (2.23551) + | > loader_time: 0.00170 (0.02136) + + + --> STEP: 25/234 -- GLOBAL_STEP: 15235 + | > loss: 0.07995 (0.07555) + | > log_mle: -0.10301 (-0.11393) + | > loss_dur: 0.18296 (0.18948) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.27450 (4.38695) + | > current_lr: 0.00002 + | > step_time: 1.62010 (2.07279) + | > loader_time: 0.00250 (0.02075) + + + --> STEP: 30/234 -- GLOBAL_STEP: 15240 + | > loss: 0.02438 (0.07058) + | > log_mle: -0.13236 (-0.11504) + | > loss_dur: 0.15674 (0.18562) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.89798 (4.12328) + | > current_lr: 0.00002 + | > step_time: 1.07630 (1.95237) + | > loader_time: 0.00160 (0.01763) + + + --> STEP: 35/234 -- GLOBAL_STEP: 15245 + | > loss: 0.05078 (0.06925) + | > log_mle: -0.13159 (-0.11635) + | > loss_dur: 0.18237 (0.18560) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.15634 (4.23855) + | > current_lr: 0.00002 + | > step_time: 2.80320 (1.92967) + | > loader_time: 0.08680 (0.01784) + + + --> STEP: 40/234 -- GLOBAL_STEP: 15250 + | > loss: 0.09471 (0.06889) + | > log_mle: -0.10929 (-0.11714) + | > loss_dur: 0.20400 (0.18603) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.48128 (4.18971) + | > current_lr: 0.00002 + | > step_time: 1.92370 (1.94147) + | > loader_time: 0.00910 (0.01811) + + + --> STEP: 45/234 -- GLOBAL_STEP: 15255 + | > loss: 0.03490 (0.06746) + | > log_mle: -0.14414 (-0.11784) + | > loss_dur: 0.17904 (0.18530) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.82880 (4.10874) + | > current_lr: 0.00002 + | > step_time: 1.31610 (1.92041) + | > loader_time: 0.00290 (0.01643) + + + --> STEP: 50/234 -- GLOBAL_STEP: 15260 + | > loss: 0.07371 (0.06588) + | > log_mle: -0.11278 (-0.11806) + | > loss_dur: 0.18649 (0.18394) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.12382 (3.98590) + | > current_lr: 0.00002 + | > step_time: 1.74720 (1.93489) + | > loader_time: 0.00200 (0.01679) + + + --> STEP: 55/234 -- GLOBAL_STEP: 15265 + | > loss: 0.04705 (0.06502) + | > log_mle: -0.13258 (-0.11861) + | > loss_dur: 0.17963 (0.18363) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.37036 (3.90953) + | > current_lr: 0.00002 + | > step_time: 1.10560 (1.87772) + | > loader_time: 0.08380 (0.01841) + + + --> STEP: 60/234 -- GLOBAL_STEP: 15270 + | > loss: 0.02664 (0.06410) + | > log_mle: -0.15190 (-0.11965) + | > loss_dur: 0.17854 (0.18375) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.21981 (3.91182) + | > current_lr: 0.00002 + | > step_time: 1.30740 (1.84829) + | > loader_time: 0.00310 (0.01990) + + + --> STEP: 65/234 -- GLOBAL_STEP: 15275 + | > loss: 0.04278 (0.06314) + | > log_mle: -0.12403 (-0.12058) + | > loss_dur: 0.16681 (0.18372) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.03003 (4.08498) + | > current_lr: 0.00002 + | > step_time: 1.51470 (1.83503) + | > loader_time: 0.09210 (0.01996) + + + --> STEP: 70/234 -- GLOBAL_STEP: 15280 + | > loss: 0.05711 (0.06257) + | > log_mle: -0.12936 (-0.12082) + | > loss_dur: 0.18647 (0.18340) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.65742 (4.27873) + | > current_lr: 0.00002 + | > step_time: 1.71170 (1.85087) + | > loader_time: 0.00210 (0.02028) + + + --> STEP: 75/234 -- GLOBAL_STEP: 15285 + | > loss: 0.04892 (0.06186) + | > log_mle: -0.14188 (-0.12212) + | > loss_dur: 0.19079 (0.18399) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.21935 (4.56848) + | > current_lr: 0.00002 + | > step_time: 1.68540 (1.83364) + | > loader_time: 0.00280 (0.01909) + + + --> STEP: 80/234 -- GLOBAL_STEP: 15290 + | > loss: 0.03700 (0.06075) + | > log_mle: -0.11808 (-0.12271) + | > loss_dur: 0.15508 (0.18346) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.39552 (4.57976) + | > current_lr: 0.00002 + | > step_time: 2.52970 (1.84544) + | > loader_time: 0.00290 (0.01806) + + + --> STEP: 85/234 -- GLOBAL_STEP: 15295 + | > loss: 0.03957 (0.05974) + | > log_mle: -0.13622 (-0.12379) + | > loss_dur: 0.17580 (0.18353) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.94335 (4.72866) + | > current_lr: 0.00002 + | > step_time: 1.80100 (1.83270) + | > loader_time: 0.00380 (0.01714) + + + --> STEP: 90/234 -- GLOBAL_STEP: 15300 + | > loss: 0.02956 (0.05785) + | > log_mle: -0.16855 (-0.12599) + | > loss_dur: 0.19811 (0.18384) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.63758 (5.04951) + | > current_lr: 0.00002 + | > step_time: 2.00210 (1.83185) + | > loader_time: 0.00230 (0.01733) + + + --> STEP: 95/234 -- GLOBAL_STEP: 15305 + | > loss: -0.02325 (0.05504) + | > log_mle: -0.24605 (-0.12979) + | > loss_dur: 0.22280 (0.18483) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.91598 (5.64424) + | > current_lr: 0.00002 + | > step_time: 3.19870 (1.90218) + | > loader_time: 0.00170 (0.01751) + + + --> STEP: 100/234 -- GLOBAL_STEP: 15310 + | > loss: 0.01976 (0.05347) + | > log_mle: -0.17823 (-0.13192) + | > loss_dur: 0.19800 (0.18539) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.27609 (5.87538) + | > current_lr: 0.00002 + | > step_time: 0.89530 (1.90992) + | > loader_time: 0.00300 (0.01867) + + + --> STEP: 105/234 -- GLOBAL_STEP: 15315 + | > loss: 0.02267 (0.05151) + | > log_mle: -0.15175 (-0.13494) + | > loss_dur: 0.17442 (0.18645) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.48555 (6.22972) + | > current_lr: 0.00002 + | > step_time: 2.50410 (1.91776) + | > loader_time: 0.00370 (0.01794) + + + --> STEP: 110/234 -- GLOBAL_STEP: 15320 + | > loss: 0.00467 (0.05006) + | > log_mle: -0.18029 (-0.13770) + | > loss_dur: 0.18496 (0.18776) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.77005 (6.59012) + | > current_lr: 0.00002 + | > step_time: 1.19800 (1.90992) + | > loader_time: 0.00320 (0.01726) + + + --> STEP: 115/234 -- GLOBAL_STEP: 15325 + | > loss: 0.02922 (0.04824) + | > log_mle: -0.19687 (-0.14100) + | > loss_dur: 0.22609 (0.18923) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 12.36954 (6.97254) + | > current_lr: 0.00002 + | > step_time: 1.69220 (1.93902) + | > loader_time: 0.00470 (0.01830) + + + --> STEP: 120/234 -- GLOBAL_STEP: 15330 + | > loss: -0.00723 (0.04700) + | > log_mle: -0.24269 (-0.14380) + | > loss_dur: 0.23546 (0.19080) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.27035 (7.31374) + | > current_lr: 0.00002 + | > step_time: 2.30390 (1.94081) + | > loader_time: 0.00350 (0.01918) + + + --> STEP: 125/234 -- GLOBAL_STEP: 15335 + | > loss: -0.00008 (0.04569) + | > log_mle: -0.23108 (-0.14549) + | > loss_dur: 0.23100 (0.19118) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.86796 (7.48130) + | > current_lr: 0.00002 + | > step_time: 3.57270 (1.96290) + | > loader_time: 0.09300 (0.01924) + + + --> STEP: 130/234 -- GLOBAL_STEP: 15340 + | > loss: -0.00422 (0.04355) + | > log_mle: -0.23904 (-0.14877) + | > loss_dur: 0.23483 (0.19232) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.79887 (7.94960) + | > current_lr: 0.00002 + | > step_time: 1.31350 (1.95472) + | > loader_time: 0.00340 (0.01992) + + + --> STEP: 135/234 -- GLOBAL_STEP: 15345 + | > loss: 0.02097 (0.04151) + | > log_mle: -0.17655 (-0.15202) + | > loss_dur: 0.19752 (0.19352) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.35021 (8.34584) + | > current_lr: 0.00002 + | > step_time: 2.78520 (1.98533) + | > loader_time: 0.00430 (0.01992) + + + --> STEP: 140/234 -- GLOBAL_STEP: 15350 + | > loss: 0.01888 (0.03918) + | > log_mle: -0.20619 (-0.15574) + | > loss_dur: 0.22507 (0.19493) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.81327 (8.84244) + | > current_lr: 0.00002 + | > step_time: 1.41310 (1.97812) + | > loader_time: 0.08670 (0.02198) + + + --> STEP: 145/234 -- GLOBAL_STEP: 15355 + | > loss: -0.05787 (0.03673) + | > log_mle: -0.29614 (-0.16001) + | > loss_dur: 0.23828 (0.19674) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.79260 (9.53846) + | > current_lr: 0.00002 + | > step_time: 4.20880 (1.99023) + | > loader_time: 0.19210 (0.02431) + + + --> STEP: 150/234 -- GLOBAL_STEP: 15360 + | > loss: -0.04081 (0.03385) + | > log_mle: -0.28207 (-0.16407) + | > loss_dur: 0.24126 (0.19793) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 23.96594 (10.03845) + | > current_lr: 0.00002 + | > step_time: 1.90350 (1.98136) + | > loader_time: 0.00290 (0.02418) + + + --> STEP: 155/234 -- GLOBAL_STEP: 15365 + | > loss: -0.08431 (0.03038) + | > log_mle: -0.34288 (-0.16898) + | > loss_dur: 0.25857 (0.19936) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.99257 (10.75055) + | > current_lr: 0.00002 + | > step_time: 2.19380 (1.98191) + | > loader_time: 0.00320 (0.02511) + + + --> STEP: 160/234 -- GLOBAL_STEP: 15370 + | > loss: -0.08580 (0.02746) + | > log_mle: -0.33732 (-0.17327) + | > loss_dur: 0.25152 (0.20073) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.30739 (11.36739) + | > current_lr: 0.00002 + | > step_time: 2.38480 (1.98821) + | > loader_time: 0.00300 (0.02597) + + + --> STEP: 165/234 -- GLOBAL_STEP: 15375 + | > loss: -0.05575 (0.02472) + | > log_mle: -0.33408 (-0.17739) + | > loss_dur: 0.27833 (0.20211) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 38.32404 (11.89541) + | > current_lr: 0.00002 + | > step_time: 4.80140 (2.01344) + | > loader_time: 0.50340 (0.02885) + + + --> STEP: 170/234 -- GLOBAL_STEP: 15380 + | > loss: -0.08626 (0.02191) + | > log_mle: -0.36546 (-0.18175) + | > loss_dur: 0.27920 (0.20366) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 51.68247 (12.66721) + | > current_lr: 0.00002 + | > step_time: 4.89090 (2.10139) + | > loader_time: 0.00460 (0.02979) + + + --> STEP: 175/234 -- GLOBAL_STEP: 15385 + | > loss: -0.06627 (0.01853) + | > log_mle: -0.34392 (-0.18676) + | > loss_dur: 0.27766 (0.20530) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.69341 (13.50551) + | > current_lr: 0.00002 + | > step_time: 10.89700 (2.17751) + | > loader_time: 0.29930 (0.03223) + + + --> STEP: 180/234 -- GLOBAL_STEP: 15390 + | > loss: -0.09628 (0.01562) + | > log_mle: -0.35612 (-0.19149) + | > loss_dur: 0.25984 (0.20710) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 31.26563 (14.07908) + | > current_lr: 0.00002 + | > step_time: 10.00290 (2.24710) + | > loader_time: 0.00570 (0.03354) + + + --> STEP: 185/234 -- GLOBAL_STEP: 15395 + | > loss: -0.09358 (0.01302) + | > log_mle: -0.37754 (-0.19584) + | > loss_dur: 0.28396 (0.20886) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.94166 (14.67534) + | > current_lr: 0.00002 + | > step_time: 1.58850 (2.33706) + | > loader_time: 0.00540 (0.03485) + + + --> STEP: 190/234 -- GLOBAL_STEP: 15400 + | > loss: -0.10325 (0.01022) + | > log_mle: -0.35983 (-0.20014) + | > loss_dur: 0.25659 (0.21036) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.94366 (15.15293) + | > current_lr: 0.00002 + | > step_time: 2.67720 (2.36235) + | > loader_time: 0.00260 (0.03592) + + + --> STEP: 195/234 -- GLOBAL_STEP: 15405 + | > loss: -0.08035 (0.00719) + | > log_mle: -0.36762 (-0.20461) + | > loss_dur: 0.28727 (0.21179) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 47.87812 (15.70131) + | > current_lr: 0.00002 + | > step_time: 0.69380 (2.39151) + | > loader_time: 0.00290 (0.03757) + + + --> STEP: 200/234 -- GLOBAL_STEP: 15410 + | > loss: -0.08405 (0.00452) + | > log_mle: -0.37474 (-0.20870) + | > loss_dur: 0.29069 (0.21322) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.43716 (16.32991) + | > current_lr: 0.00002 + | > step_time: 8.79610 (2.43562) + | > loader_time: 0.11300 (0.03954) + + + --> STEP: 205/234 -- GLOBAL_STEP: 15415 + | > loss: -0.09311 (0.00214) + | > log_mle: -0.36334 (-0.21265) + | > loss_dur: 0.27023 (0.21479) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.62482 (16.90378) + | > current_lr: 0.00002 + | > step_time: 2.79830 (2.43620) + | > loader_time: 0.11000 (0.04008) + + + --> STEP: 210/234 -- GLOBAL_STEP: 15420 + | > loss: -0.15215 (-0.00095) + | > log_mle: -0.43866 (-0.21732) + | > loss_dur: 0.28650 (0.21637) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 54.15396 (17.52825) + | > current_lr: 0.00002 + | > step_time: 10.40030 (2.55154) + | > loader_time: 0.10610 (0.04156) + + + --> STEP: 215/234 -- GLOBAL_STEP: 15425 + | > loss: -0.11665 (-0.00422) + | > log_mle: -0.38837 (-0.22210) + | > loss_dur: 0.27172 (0.21788) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 42.56789 (18.19561) + | > current_lr: 0.00002 + | > step_time: 6.00350 (2.62296) + | > loader_time: 0.10760 (0.04242) + + + --> STEP: 220/234 -- GLOBAL_STEP: 15430 + | > loss: -0.14139 (-0.00756) + | > log_mle: -0.43658 (-0.22713) + | > loss_dur: 0.29519 (0.21956) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 46.41969 (19.00491) + | > current_lr: 0.00002 + | > step_time: 4.00790 (2.62930) + | > loader_time: 0.08500 (0.04312) + + + --> STEP: 225/234 -- GLOBAL_STEP: 15435 + | > loss: -0.19224 (-0.01056) + | > log_mle: -0.48914 (-0.23185) + | > loss_dur: 0.29689 (0.22129) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 73.27628 (19.90054) + | > current_lr: 0.00002 + | > step_time: 2.50290 (2.62078) + | > loader_time: 0.00480 (0.04300) + + + --> STEP: 230/234 -- GLOBAL_STEP: 15440 + | > loss: -0.14799 (-0.01321) + | > log_mle: -0.53092 (-0.23692) + | > loss_dur: 0.38294 (0.22371) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 82.24831 (20.91583) + | > current_lr: 0.00002 + | > step_time: 0.24960 (2.57549) + | > loader_time: 0.00350 (0.04216) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32247 (+0.22152) + | > avg_loss: -0.06014 (-0.02206) + | > avg_log_mle: -0.31422 (-0.02065) + | > avg_loss_dur: 0.25407 (-0.00141) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_15444.pth + + > EPOCH: 66/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 23:08:53)  + + --> STEP: 1/234 -- GLOBAL_STEP: 15445 + | > loss: 0.05712 (0.05712) + | > log_mle: -0.11084 (-0.11084) + | > loss_dur: 0.16796 (0.16796) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.67324 (3.67324) + | > current_lr: 0.00002 + | > step_time: 9.30690 (9.30686) + | > loader_time: 14.80250 (14.80254) + + + --> STEP: 6/234 -- GLOBAL_STEP: 15450 + | > loss: 0.10673 (0.10921) + | > log_mle: -0.10726 (-0.11192) + | > loss_dur: 0.21398 (0.22112) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.36154 (4.94653) + | > current_lr: 0.00002 + | > step_time: 1.69880 (2.88536) + | > loader_time: 0.00250 (2.46842) + + + --> STEP: 11/234 -- GLOBAL_STEP: 15455 + | > loss: 0.06494 (0.08998) + | > log_mle: -0.11215 (-0.11857) + | > loss_dur: 0.17709 (0.20856) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.91474 (5.05458) + | > current_lr: 0.00002 + | > step_time: 3.49380 (4.11880) + | > loader_time: 0.00200 (1.36489) + + + --> STEP: 16/234 -- GLOBAL_STEP: 15460 + | > loss: 0.04607 (0.08411) + | > log_mle: -0.11821 (-0.11870) + | > loss_dur: 0.16428 (0.20282) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.07734 (4.66661) + | > current_lr: 0.00002 + | > step_time: 1.59730 (5.03071) + | > loader_time: 0.00240 (0.97638) + + + --> STEP: 21/234 -- GLOBAL_STEP: 15465 + | > loss: 0.07231 (0.08030) + | > log_mle: -0.10360 (-0.11658) + | > loss_dur: 0.17590 (0.19688) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.83435 (4.40960) + | > current_lr: 0.00002 + | > step_time: 3.40290 (4.54712) + | > loader_time: 0.10000 (0.76199) + + + --> STEP: 26/234 -- GLOBAL_STEP: 15470 + | > loss: 0.06425 (0.07531) + | > log_mle: -0.12208 (-0.11693) + | > loss_dur: 0.18632 (0.19225) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.85216 (4.11604) + | > current_lr: 0.00002 + | > step_time: 1.60810 (3.94588) + | > loader_time: 0.00240 (0.61592) + + + --> STEP: 31/234 -- GLOBAL_STEP: 15475 + | > loss: 0.10126 (0.07109) + | > log_mle: -0.12680 (-0.11807) + | > loss_dur: 0.22805 (0.18916) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.13739 (3.93119) + | > current_lr: 0.00002 + | > step_time: 5.10010 (3.71334) + | > loader_time: 0.00470 (0.52013) + + + --> STEP: 36/234 -- GLOBAL_STEP: 15480 + | > loss: 0.06681 (0.06991) + | > log_mle: -0.13176 (-0.11942) + | > loss_dur: 0.19857 (0.18933) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.75079 (4.09822) + | > current_lr: 0.00002 + | > step_time: 1.70260 (3.58869) + | > loader_time: 0.00270 (0.45091) + + + --> STEP: 41/234 -- GLOBAL_STEP: 15485 + | > loss: 0.03854 (0.06835) + | > log_mle: -0.12332 (-0.11988) + | > loss_dur: 0.16187 (0.18823) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 1.94354 (4.04021) + | > current_lr: 0.00002 + | > step_time: 1.40210 (3.38982) + | > loader_time: 0.08610 (0.40478) + + + --> STEP: 46/234 -- GLOBAL_STEP: 15490 + | > loss: 0.05999 (0.06664) + | > log_mle: -0.12701 (-0.12061) + | > loss_dur: 0.18700 (0.18725) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.79358 (4.09416) + | > current_lr: 0.00002 + | > step_time: 2.19650 (3.30394) + | > loader_time: 0.10680 (0.36493) + + + --> STEP: 51/234 -- GLOBAL_STEP: 15495 + | > loss: 0.05754 (0.06500) + | > log_mle: -0.11161 (-0.12043) + | > loss_dur: 0.16915 (0.18543) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.12260 (4.01818) + | > current_lr: 0.00002 + | > step_time: 1.29950 (3.10873) + | > loader_time: 0.00240 (0.32934) + + + --> STEP: 56/234 -- GLOBAL_STEP: 15500 + | > loss: 0.08699 (0.06444) + | > log_mle: -0.13082 (-0.12131) + | > loss_dur: 0.21781 (0.18575) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.41089 (3.93803) + | > current_lr: 0.00002 + | > step_time: 1.70440 (2.94876) + | > loader_time: 0.08640 (0.30310) + + + --> STEP: 61/234 -- GLOBAL_STEP: 15505 + | > loss: 0.03133 (0.06206) + | > log_mle: -0.12628 (-0.12226) + | > loss_dur: 0.15760 (0.18432) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.52629 (3.94397) + | > current_lr: 0.00002 + | > step_time: 2.04860 (2.84000) + | > loader_time: 0.00210 (0.27990) + + + --> STEP: 66/234 -- GLOBAL_STEP: 15510 + | > loss: 0.06634 (0.06173) + | > log_mle: -0.11659 (-0.12302) + | > loss_dur: 0.18293 (0.18474) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.74518 (4.11725) + | > current_lr: 0.00002 + | > step_time: 1.99650 (2.76458) + | > loader_time: 0.01120 (0.26030) + + + --> STEP: 71/234 -- GLOBAL_STEP: 15515 + | > loss: 0.04478 (0.06107) + | > log_mle: -0.16412 (-0.12397) + | > loss_dur: 0.20889 (0.18504) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.32947 (4.39598) + | > current_lr: 0.00002 + | > step_time: 2.19840 (2.67786) + | > loader_time: 0.00230 (0.24215) + + + --> STEP: 76/234 -- GLOBAL_STEP: 15520 + | > loss: 0.04056 (0.06006) + | > log_mle: -0.14313 (-0.12501) + | > loss_dur: 0.18369 (0.18506) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.20693 (4.49047) + | > current_lr: 0.00002 + | > step_time: 1.90010 (2.63964) + | > loader_time: 0.00270 (0.22754) + + + --> STEP: 81/234 -- GLOBAL_STEP: 15525 + | > loss: 0.02780 (0.05851) + | > log_mle: -0.15427 (-0.12576) + | > loss_dur: 0.18207 (0.18427) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.59410 (4.54578) + | > current_lr: 0.00002 + | > step_time: 1.78850 (2.59302) + | > loader_time: 0.00250 (0.21465) + + + --> STEP: 86/234 -- GLOBAL_STEP: 15530 + | > loss: 0.02522 (0.05740) + | > log_mle: -0.15454 (-0.12685) + | > loss_dur: 0.17976 (0.18424) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.40337 (4.64943) + | > current_lr: 0.00002 + | > step_time: 2.21010 (2.57034) + | > loader_time: 0.07870 (0.20321) + + + --> STEP: 91/234 -- GLOBAL_STEP: 15535 + | > loss: 0.03754 (0.05571) + | > log_mle: -0.16310 (-0.12912) + | > loss_dur: 0.20065 (0.18483) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.59900 (4.89797) + | > current_lr: 0.00002 + | > step_time: 4.12100 (2.54225) + | > loader_time: 0.09740 (0.19417) + + + --> STEP: 96/234 -- GLOBAL_STEP: 15540 + | > loss: 0.03878 (0.05287) + | > log_mle: -0.14988 (-0.13271) + | > loss_dur: 0.18866 (0.18558) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.53224 (5.64156) + | > current_lr: 0.00002 + | > step_time: 1.50660 (2.55020) + | > loader_time: 0.00290 (0.18519) + + + --> STEP: 101/234 -- GLOBAL_STEP: 15545 + | > loss: 0.00203 (0.05111) + | > log_mle: -0.20605 (-0.13533) + | > loss_dur: 0.20808 (0.18644) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 17.27082 (6.04065) + | > current_lr: 0.00002 + | > step_time: 1.77610 (2.53918) + | > loader_time: 0.00150 (0.17703) + + + --> STEP: 106/234 -- GLOBAL_STEP: 15550 + | > loss: 0.02398 (0.04916) + | > log_mle: -0.20803 (-0.13828) + | > loss_dur: 0.23201 (0.18744) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.09267 (6.48144) + | > current_lr: 0.00002 + | > step_time: 2.90200 (2.51247) + | > loader_time: 0.00270 (0.17038) + + + --> STEP: 111/234 -- GLOBAL_STEP: 15555 + | > loss: 0.00709 (0.04748) + | > log_mle: -0.24521 (-0.14126) + | > loss_dur: 0.25230 (0.18874) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 29.84356 (7.03709) + | > current_lr: 0.00002 + | > step_time: 5.21590 (2.50465) + | > loader_time: 0.08230 (0.16435) + + + --> STEP: 116/234 -- GLOBAL_STEP: 15560 + | > loss: 0.04248 (0.04624) + | > log_mle: -0.21622 (-0.14418) + | > loss_dur: 0.25870 (0.19042) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.32841 (7.38001) + | > current_lr: 0.00002 + | > step_time: 1.84790 (2.48124) + | > loader_time: 0.00240 (0.15895) + + + --> STEP: 121/234 -- GLOBAL_STEP: 15565 + | > loss: 0.06700 (0.04516) + | > log_mle: -0.13474 (-0.14624) + | > loss_dur: 0.20174 (0.19140) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.30700 (7.66926) + | > current_lr: 0.00002 + | > step_time: 3.11830 (2.46571) + | > loader_time: 0.09150 (0.15471) + + + --> STEP: 126/234 -- GLOBAL_STEP: 15570 + | > loss: -0.03110 (0.04309) + | > log_mle: -0.25870 (-0.14886) + | > loss_dur: 0.22759 (0.19196) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.42511 (8.14926) + | > current_lr: 0.00002 + | > step_time: 3.01040 (2.46227) + | > loader_time: 0.00330 (0.14951) + + + --> STEP: 131/234 -- GLOBAL_STEP: 15575 + | > loss: -0.04796 (0.04078) + | > log_mle: -0.29505 (-0.15234) + | > loss_dur: 0.24708 (0.19311) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.82482 (8.71927) + | > current_lr: 0.00002 + | > step_time: 4.09930 (2.46456) + | > loader_time: 0.00340 (0.14395) + + + --> STEP: 136/234 -- GLOBAL_STEP: 15580 + | > loss: -0.07240 (0.03857) + | > log_mle: -0.33884 (-0.15578) + | > loss_dur: 0.26644 (0.19435) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 39.33851 (9.33362) + | > current_lr: 0.00002 + | > step_time: 4.29940 (2.46899) + | > loader_time: 0.09550 (0.14205) + + + --> STEP: 141/234 -- GLOBAL_STEP: 15585 + | > loss: -0.01723 (0.03672) + | > log_mle: -0.25664 (-0.15879) + | > loss_dur: 0.23941 (0.19551) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.58536 (9.76732) + | > current_lr: 0.00002 + | > step_time: 2.90460 (2.50583) + | > loader_time: 0.00330 (0.13792) + + + --> STEP: 146/234 -- GLOBAL_STEP: 15590 + | > loss: -0.06863 (0.03371) + | > log_mle: -0.30473 (-0.16335) + | > loss_dur: 0.23610 (0.19706) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.76066 (10.42965) + | > current_lr: 0.00002 + | > step_time: 2.89130 (2.48649) + | > loader_time: 0.00280 (0.13329) + + + --> STEP: 151/234 -- GLOBAL_STEP: 15595 + | > loss: -0.06009 (0.03105) + | > log_mle: -0.27349 (-0.16709) + | > loss_dur: 0.21340 (0.19813) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.10902 (11.02974) + | > current_lr: 0.00002 + | > step_time: 2.49840 (2.47744) + | > loader_time: 0.00420 (0.13023) + + + --> STEP: 156/234 -- GLOBAL_STEP: 15600 + | > loss: -0.07195 (0.02755) + | > log_mle: -0.30998 (-0.17215) + | > loss_dur: 0.23803 (0.19970) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.60195 (11.84410) + | > current_lr: 0.00002 + | > step_time: 8.09040 (2.53895) + | > loader_time: 0.00440 (0.12733) + + + --> STEP: 161/234 -- GLOBAL_STEP: 15605 + | > loss: -0.09103 (0.02465) + | > log_mle: -0.32945 (-0.17646) + | > loss_dur: 0.23842 (0.20111) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.64713 (12.48723) + | > current_lr: 0.00002 + | > step_time: 2.51060 (2.55027) + | > loader_time: 0.00360 (0.12529) + + + --> STEP: 166/234 -- GLOBAL_STEP: 15610 + | > loss: -0.06185 (0.02205) + | > log_mle: -0.27602 (-0.18019) + | > loss_dur: 0.21417 (0.20224) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.27966 (13.02973) + | > current_lr: 0.00002 + | > step_time: 4.90420 (2.55594) + | > loader_time: 0.08440 (0.12265) + + + --> STEP: 171/234 -- GLOBAL_STEP: 15615 + | > loss: -0.13483 (0.01869) + | > log_mle: -0.37431 (-0.18517) + | > loss_dur: 0.23948 (0.20386) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.30220 (13.61988) + | > current_lr: 0.00002 + | > step_time: 5.51100 (2.55260) + | > loader_time: 0.00490 (0.11964) + + + --> STEP: 176/234 -- GLOBAL_STEP: 15620 + | > loss: -0.08976 (0.01544) + | > log_mle: -0.34246 (-0.19003) + | > loss_dur: 0.25270 (0.20547) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.19932 (14.14754) + | > current_lr: 0.00002 + | > step_time: 2.40420 (2.56656) + | > loader_time: 0.08900 (0.11736) + + + --> STEP: 181/234 -- GLOBAL_STEP: 15625 + | > loss: -0.03833 (0.01277) + | > log_mle: -0.28717 (-0.19434) + | > loss_dur: 0.24884 (0.20711) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.68528 (14.85167) + | > current_lr: 0.00002 + | > step_time: 4.80080 (2.73266) + | > loader_time: 0.19310 (0.11633) + + + --> STEP: 186/234 -- GLOBAL_STEP: 15630 + | > loss: -0.04524 (0.01005) + | > log_mle: -0.32438 (-0.19884) + | > loss_dur: 0.27914 (0.20889) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.54132 (15.51945) + | > current_lr: 0.00002 + | > step_time: 2.89890 (2.78892) + | > loader_time: 0.00260 (0.11418) + + + --> STEP: 191/234 -- GLOBAL_STEP: 15635 + | > loss: -0.09860 (0.00699) + | > log_mle: -0.33779 (-0.20313) + | > loss_dur: 0.23919 (0.21011) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.56134 (16.15076) + | > current_lr: 0.00002 + | > step_time: 3.00910 (2.80797) + | > loader_time: 0.00350 (0.11236) + + + --> STEP: 196/234 -- GLOBAL_STEP: 15640 + | > loss: -0.06754 (0.00407) + | > log_mle: -0.33507 (-0.20747) + | > loss_dur: 0.26754 (0.21154) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.38811 (16.97260) + | > current_lr: 0.00002 + | > step_time: 4.90830 (2.87364) + | > loader_time: 0.39120 (0.11246) + + + --> STEP: 201/234 -- GLOBAL_STEP: 15645 + | > loss: -0.03080 (0.00162) + | > log_mle: -0.30826 (-0.21137) + | > loss_dur: 0.27747 (0.21299) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 26.96030 (17.60795) + | > current_lr: 0.00002 + | > step_time: 3.10080 (2.89037) + | > loader_time: 0.09170 (0.11071) + + + --> STEP: 206/234 -- GLOBAL_STEP: 15650 + | > loss: -0.12726 (-0.00132) + | > log_mle: -0.39889 (-0.21575) + | > loss_dur: 0.27162 (0.21442) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.27254 (18.20020) + | > current_lr: 0.00002 + | > step_time: 3.99870 (2.90379) + | > loader_time: 0.00460 (0.11130) + + + --> STEP: 211/234 -- GLOBAL_STEP: 15655 + | > loss: -0.16666 (-0.00462) + | > log_mle: -0.46300 (-0.22065) + | > loss_dur: 0.29633 (0.21603) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 56.34980 (18.92960) + | > current_lr: 0.00002 + | > step_time: 8.90310 (2.97950) + | > loader_time: 0.08930 (0.11000) + + + --> STEP: 216/234 -- GLOBAL_STEP: 15660 + | > loss: -0.16088 (-0.00786) + | > log_mle: -0.45300 (-0.22534) + | > loss_dur: 0.29211 (0.21747) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 49.22607 (19.60011) + | > current_lr: 0.00002 + | > step_time: 3.59250 (3.02912) + | > loader_time: 0.20440 (0.11154) + + + --> STEP: 221/234 -- GLOBAL_STEP: 15665 + | > loss: -0.10860 (-0.01101) + | > log_mle: -0.37862 (-0.23000) + | > loss_dur: 0.27002 (0.21899) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.29893 (20.20244) + | > current_lr: 0.00002 + | > step_time: 2.58670 (3.03698) + | > loader_time: 0.00300 (0.10949) + + + --> STEP: 226/234 -- GLOBAL_STEP: 15670 + | > loss: -0.17181 (-0.01436) + | > log_mle: -0.47058 (-0.23516) + | > loss_dur: 0.29877 (0.22081) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 50.32710 (20.91617) + | > current_lr: 0.00002 + | > step_time: 1.11200 (3.01900) + | > loader_time: 0.00460 (0.10795) + + + --> STEP: 231/234 -- GLOBAL_STEP: 15675 + | > loss: -0.09435 (-0.01677) + | > log_mle: -0.52921 (-0.24056) + | > loss_dur: 0.43487 (0.22379) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 66.41341 (21.73810) + | > current_lr: 0.00002 + | > step_time: 0.27850 (2.95917) + | > loader_time: 0.00500 (0.10571) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.35979 (+0.03732) + | > avg_loss: -0.02956 (+0.03058) + | > avg_log_mle: -0.28604 (+0.02818) + | > avg_loss_dur: 0.25648 (+0.00240) + + + > EPOCH: 67/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 23:21:32)  + + --> STEP: 2/234 -- GLOBAL_STEP: 15680 + | > loss: 0.16978 (0.13046) + | > log_mle: -0.09295 (-0.10432) + | > loss_dur: 0.26274 (0.23479) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.58311 (3.47673) + | > current_lr: 0.00002 + | > step_time: 2.01120 (2.55497) + | > loader_time: 0.00140 (0.45786) + + + --> STEP: 7/234 -- GLOBAL_STEP: 15685 + | > loss: 0.05791 (0.10176) + | > log_mle: -0.13527 (-0.11811) + | > loss_dur: 0.19318 (0.21987) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.86729 (4.44250) + | > current_lr: 0.00002 + | > step_time: 2.19780 (3.50277) + | > loader_time: 0.10030 (0.33019) + + + --> STEP: 12/234 -- GLOBAL_STEP: 15690 + | > loss: 0.07268 (0.08841) + | > log_mle: -0.12267 (-0.12173) + | > loss_dur: 0.19535 (0.21014) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.47899 (4.75689) + | > current_lr: 0.00002 + | > step_time: 7.19340 (4.82709) + | > loader_time: 0.00190 (0.21738) + + + --> STEP: 17/234 -- GLOBAL_STEP: 15695 + | > loss: 0.07999 (0.08053) + | > log_mle: -0.10568 (-0.12086) + | > loss_dur: 0.18567 (0.20139) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.73336 (4.53670) + | > current_lr: 0.00002 + | > step_time: 3.30270 (4.91901) + | > loader_time: 0.00130 (0.15984) + + + --> STEP: 22/234 -- GLOBAL_STEP: 15700 + | > loss: 0.02437 (0.07402) + | > log_mle: -0.13072 (-0.12004) + | > loss_dur: 0.15509 (0.19406) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.79007 (4.39016) + | > current_lr: 0.00002 + | > step_time: 3.69390 (4.63677) + | > loader_time: 0.00400 (0.12776) + + + --> STEP: 27/234 -- GLOBAL_STEP: 15705 + | > loss: 0.04049 (0.06972) + | > log_mle: -0.13152 (-0.12015) + | > loss_dur: 0.17202 (0.18987) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.56953 (4.22614) + | > current_lr: 0.00002 + | > step_time: 3.31330 (4.34156) + | > loader_time: 0.18780 (0.12136) + + + --> STEP: 32/234 -- GLOBAL_STEP: 15710 + | > loss: 0.02330 (0.06526) + | > log_mle: -0.14388 (-0.12143) + | > loss_dur: 0.16718 (0.18669) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.64864 (4.07790) + | > current_lr: 0.00002 + | > step_time: 0.80420 (4.22605) + | > loader_time: 0.09580 (0.12730) + + + --> STEP: 37/234 -- GLOBAL_STEP: 15715 + | > loss: 0.03784 (0.06410) + | > log_mle: -0.12386 (-0.12206) + | > loss_dur: 0.16170 (0.18616) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.27436 (4.29159) + | > current_lr: 0.00002 + | > step_time: 2.50760 (3.84454) + | > loader_time: 0.08070 (0.11481) + + + --> STEP: 42/234 -- GLOBAL_STEP: 15720 + | > loss: 0.06788 (0.06436) + | > log_mle: -0.11091 (-0.12213) + | > loss_dur: 0.17879 (0.18649) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.46196 (4.27145) + | > current_lr: 0.00002 + | > step_time: 1.29100 (3.61093) + | > loader_time: 0.00220 (0.10330) + + + --> STEP: 47/234 -- GLOBAL_STEP: 15725 + | > loss: 0.04918 (0.06304) + | > log_mle: -0.12524 (-0.12313) + | > loss_dur: 0.17441 (0.18617) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.71068 (4.37356) + | > current_lr: 0.00002 + | > step_time: 1.20460 (3.38912) + | > loader_time: 0.00290 (0.09259) + + + --> STEP: 52/234 -- GLOBAL_STEP: 15730 + | > loss: 0.05523 (0.06153) + | > log_mle: -0.11813 (-0.12277) + | > loss_dur: 0.17336 (0.18430) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.53326 (4.25726) + | > current_lr: 0.00002 + | > step_time: 1.90650 (3.21481) + | > loader_time: 0.00200 (0.08391) + + + --> STEP: 57/234 -- GLOBAL_STEP: 15735 + | > loss: 0.06596 (0.06100) + | > log_mle: -0.11576 (-0.12360) + | > loss_dur: 0.18172 (0.18459) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.56765 (4.14409) + | > current_lr: 0.00002 + | > step_time: 5.10340 (3.11622) + | > loader_time: 0.07930 (0.07810) + + + --> STEP: 62/234 -- GLOBAL_STEP: 15740 + | > loss: 0.05045 (0.05830) + | > log_mle: -0.16887 (-0.12541) + | > loss_dur: 0.21932 (0.18371) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.20380 (4.31289) + | > current_lr: 0.00002 + | > step_time: 2.07040 (3.03921) + | > loader_time: 0.00220 (0.07347) + + + --> STEP: 67/234 -- GLOBAL_STEP: 15745 + | > loss: 0.03092 (0.05750) + | > log_mle: -0.15301 (-0.12587) + | > loss_dur: 0.18393 (0.18337) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.52521 (4.28948) + | > current_lr: 0.00002 + | > step_time: 1.30770 (2.90774) + | > loader_time: 0.00200 (0.07053) + + + --> STEP: 72/234 -- GLOBAL_STEP: 15750 + | > loss: 0.05551 (0.05729) + | > log_mle: -0.13223 (-0.12640) + | > loss_dur: 0.18774 (0.18369) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.09995 (4.62702) + | > current_lr: 0.00002 + | > step_time: 0.99940 (2.80402) + | > loader_time: 0.08260 (0.06690) + + + --> STEP: 77/234 -- GLOBAL_STEP: 15755 + | > loss: 0.01778 (0.05563) + | > log_mle: -0.14655 (-0.12761) + | > loss_dur: 0.16433 (0.18324) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.59203 (4.73683) + | > current_lr: 0.00002 + | > step_time: 1.48450 (2.75934) + | > loader_time: 0.00230 (0.06270) + + + --> STEP: 82/234 -- GLOBAL_STEP: 15760 + | > loss: 0.02934 (0.05426) + | > log_mle: -0.13547 (-0.12817) + | > loss_dur: 0.16481 (0.18243) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.72810 (4.78546) + | > current_lr: 0.00002 + | > step_time: 1.89390 (2.69002) + | > loader_time: 0.00370 (0.06112) + + + --> STEP: 87/234 -- GLOBAL_STEP: 15765 + | > loss: 0.04363 (0.05357) + | > log_mle: -0.14826 (-0.12941) + | > loss_dur: 0.19189 (0.18298) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.26862 (4.91009) + | > current_lr: 0.00002 + | > step_time: 2.06360 (2.66599) + | > loader_time: 0.00190 (0.05867) + + + --> STEP: 92/234 -- GLOBAL_STEP: 15770 + | > loss: -0.00854 (0.05144) + | > log_mle: -0.19032 (-0.13213) + | > loss_dur: 0.18177 (0.18357) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 9.66235 (5.13104) + | > current_lr: 0.00002 + | > step_time: 1.80450 (2.61370) + | > loader_time: 0.00630 (0.05656) + + + --> STEP: 97/234 -- GLOBAL_STEP: 15775 + | > loss: 0.00150 (0.04876) + | > log_mle: -0.18163 (-0.13566) + | > loss_dur: 0.18312 (0.18442) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.27849 (5.65012) + | > current_lr: 0.00002 + | > step_time: 1.49850 (2.58407) + | > loader_time: 0.00300 (0.05564) + + + --> STEP: 102/234 -- GLOBAL_STEP: 15780 + | > loss: 0.03067 (0.04730) + | > log_mle: -0.16079 (-0.13807) + | > loss_dur: 0.19146 (0.18536) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 8.37722 (5.99839) + | > current_lr: 0.00002 + | > step_time: 1.50030 (2.58017) + | > loader_time: 0.08660 (0.05388) + + + --> STEP: 107/234 -- GLOBAL_STEP: 15785 + | > loss: -0.00562 (0.04519) + | > log_mle: -0.20724 (-0.14143) + | > loss_dur: 0.20162 (0.18662) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.15720 (6.43937) + | > current_lr: 0.00002 + | > step_time: 1.70110 (2.54194) + | > loader_time: 0.08590 (0.05554) + + + --> STEP: 112/234 -- GLOBAL_STEP: 15790 + | > loss: 0.00340 (0.04375) + | > log_mle: -0.21376 (-0.14441) + | > loss_dur: 0.21716 (0.18816) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.10611 (6.82459) + | > current_lr: 0.00002 + | > step_time: 2.69780 (2.53645) + | > loader_time: 0.00280 (0.05393) + + + --> STEP: 117/234 -- GLOBAL_STEP: 15795 + | > loss: 0.00088 (0.04216) + | > log_mle: -0.20850 (-0.14727) + | > loss_dur: 0.20937 (0.18943) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.98316 (7.19550) + | > current_lr: 0.00002 + | > step_time: 3.50910 (2.54445) + | > loader_time: 0.08530 (0.05404) + + + --> STEP: 122/234 -- GLOBAL_STEP: 15800 + | > loss: 0.00336 (0.04092) + | > log_mle: -0.18971 (-0.14915) + | > loss_dur: 0.19307 (0.19007) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.72668 (7.38611) + | > current_lr: 0.00002 + | > step_time: 1.30220 (2.51157) + | > loader_time: 0.00340 (0.05196) + + + --> STEP: 127/234 -- GLOBAL_STEP: 15805 + | > loss: -0.01444 (0.03887) + | > log_mle: -0.23756 (-0.15214) + | > loss_dur: 0.22312 (0.19101) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.80922 (7.91370) + | > current_lr: 0.00002 + | > step_time: 1.79750 (2.49388) + | > loader_time: 0.00740 (0.05135) + + + --> STEP: 132/234 -- GLOBAL_STEP: 15810 + | > loss: -0.03273 (0.03644) + | > log_mle: -0.22082 (-0.15545) + | > loss_dur: 0.18809 (0.19189) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 15.33421 (8.36482) + | > current_lr: 0.00002 + | > step_time: 1.87050 (2.48344) + | > loader_time: 0.00280 (0.05222) + + + --> STEP: 137/234 -- GLOBAL_STEP: 15815 + | > loss: 0.00889 (0.03452) + | > log_mle: -0.23207 (-0.15901) + | > loss_dur: 0.24096 (0.19352) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.65255 (8.82224) + | > current_lr: 0.00002 + | > step_time: 2.41290 (2.46832) + | > loader_time: 0.07850 (0.05096) + + + --> STEP: 142/234 -- GLOBAL_STEP: 15820 + | > loss: -0.02385 (0.03231) + | > log_mle: -0.24637 (-0.16209) + | > loss_dur: 0.22252 (0.19440) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.80373 (9.56264) + | > current_lr: 0.00002 + | > step_time: 2.40900 (2.44783) + | > loader_time: 0.00220 (0.04991) + + + --> STEP: 147/234 -- GLOBAL_STEP: 15825 + | > loss: -0.02008 (0.02959) + | > log_mle: -0.25123 (-0.16660) + | > loss_dur: 0.23115 (0.19619) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.02411 (10.37734) + | > current_lr: 0.00002 + | > step_time: 3.09930 (2.44361) + | > loader_time: 0.09070 (0.04949) + + + --> STEP: 152/234 -- GLOBAL_STEP: 15830 + | > loss: -0.06173 (0.02649) + | > log_mle: -0.31980 (-0.17080) + | > loss_dur: 0.25808 (0.19729) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.25025 (11.05389) + | > current_lr: 0.00002 + | > step_time: 2.60110 (2.47524) + | > loader_time: 0.09720 (0.05109) + + + --> STEP: 157/234 -- GLOBAL_STEP: 15835 + | > loss: -0.03991 (0.02309) + | > log_mle: -0.27418 (-0.17554) + | > loss_dur: 0.23427 (0.19863) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 20.94915 (11.77778) + | > current_lr: 0.00002 + | > step_time: 4.89800 (2.50402) + | > loader_time: 0.19890 (0.05196) + + + --> STEP: 162/234 -- GLOBAL_STEP: 15840 + | > loss: -0.07908 (0.01994) + | > log_mle: -0.30743 (-0.18004) + | > loss_dur: 0.22835 (0.19997) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.76956 (12.49051) + | > current_lr: 0.00002 + | > step_time: 1.20120 (2.53423) + | > loader_time: 0.00300 (0.05344) + + + --> STEP: 167/234 -- GLOBAL_STEP: 15845 + | > loss: -0.13638 (0.01700) + | > log_mle: -0.37614 (-0.18419) + | > loss_dur: 0.23976 (0.20118) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 44.85905 (13.06932) + | > current_lr: 0.00002 + | > step_time: 2.38950 (2.53621) + | > loader_time: 0.00770 (0.05257) + + + --> STEP: 172/234 -- GLOBAL_STEP: 15850 + | > loss: -0.10518 (0.01398) + | > log_mle: -0.36860 (-0.18899) + | > loss_dur: 0.26342 (0.20298) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 46.31187 (13.89214) + | > current_lr: 0.00002 + | > step_time: 4.70790 (2.56481) + | > loader_time: 0.00690 (0.05115) + + + --> STEP: 177/234 -- GLOBAL_STEP: 15855 + | > loss: -0.06653 (0.01083) + | > log_mle: -0.33191 (-0.19358) + | > loss_dur: 0.26538 (0.20441) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 35.77032 (14.58614) + | > current_lr: 0.00002 + | > step_time: 2.09900 (2.58332) + | > loader_time: 0.00370 (0.05086) + + + --> STEP: 182/234 -- GLOBAL_STEP: 15860 + | > loss: -0.09771 (0.00809) + | > log_mle: -0.37603 (-0.19812) + | > loss_dur: 0.27832 (0.20622) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 33.02338 (15.16202) + | > current_lr: 0.00002 + | > step_time: 5.10360 (2.58060) + | > loader_time: 0.00730 (0.04959) + + + --> STEP: 187/234 -- GLOBAL_STEP: 15865 + | > loss: -0.12131 (0.00519) + | > log_mle: -0.37588 (-0.20263) + | > loss_dur: 0.25457 (0.20782) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 32.67078 (15.63504) + | > current_lr: 0.00002 + | > step_time: 3.09170 (2.62661) + | > loader_time: 0.00180 (0.05024) + + + --> STEP: 192/234 -- GLOBAL_STEP: 15870 + | > loss: -0.15534 (0.00201) + | > log_mle: -0.40077 (-0.20704) + | > loss_dur: 0.24544 (0.20904) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 30.00299 (16.13733) + | > current_lr: 0.00002 + | > step_time: 8.49860 (2.65449) + | > loader_time: 0.10240 (0.05103) + + + --> STEP: 197/234 -- GLOBAL_STEP: 15875 + | > loss: -0.12686 (-0.00079) + | > log_mle: -0.37339 (-0.21123) + | > loss_dur: 0.24653 (0.21045) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.26548 (16.87541) + | > current_lr: 0.00002 + | > step_time: 4.70500 (2.70692) + | > loader_time: 0.00410 (0.05125) + + + --> STEP: 202/234 -- GLOBAL_STEP: 15880 + | > loss: -0.18273 (-0.00350) + | > log_mle: -0.45322 (-0.21553) + | > loss_dur: 0.27048 (0.21203) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 51.69207 (17.54427) + | > current_lr: 0.00002 + | > step_time: 2.81540 (2.72277) + | > loader_time: 0.00300 (0.05180) + + + --> STEP: 207/234 -- GLOBAL_STEP: 15885 + | > loss: -0.16040 (-0.00628) + | > log_mle: -0.44202 (-0.21980) + | > loss_dur: 0.28162 (0.21352) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.13280 (18.26597) + | > current_lr: 0.00002 + | > step_time: 2.80880 (2.70520) + | > loader_time: 0.08050 (0.05146) + + + --> STEP: 212/234 -- GLOBAL_STEP: 15890 + | > loss: -0.14799 (-0.00942) + | > log_mle: -0.42567 (-0.22458) + | > loss_dur: 0.27767 (0.21517) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 52.67966 (19.17357) + | > current_lr: 0.00002 + | > step_time: 6.38640 (2.71862) + | > loader_time: 0.10090 (0.05167) + + + --> STEP: 217/234 -- GLOBAL_STEP: 15895 + | > loss: -0.16175 (-0.01273) + | > log_mle: -0.44202 (-0.22936) + | > loss_dur: 0.28027 (0.21663) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 63.63208 (19.80810) + | > current_lr: 0.00002 + | > step_time: 5.60320 (2.78507) + | > loader_time: 0.09590 (0.05271) + + + --> STEP: 222/234 -- GLOBAL_STEP: 15900 + | > loss: -0.14419 (-0.01568) + | > log_mle: -0.45098 (-0.23399) + | > loss_dur: 0.30678 (0.21830) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 54.61483 (20.59050) + | > current_lr: 0.00002 + | > step_time: 3.29420 (2.79667) + | > loader_time: 0.00300 (0.05288) + + + --> STEP: 227/234 -- GLOBAL_STEP: 15905 + | > loss: -0.12708 (-0.01891) + | > log_mle: -0.43226 (-0.23899) + | > loss_dur: 0.30518 (0.22008) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 45.14764 (21.31993) + | > current_lr: 0.00002 + | > step_time: 0.26120 (2.76575) + | > loader_time: 0.00320 (0.05311) + + + --> STEP: 232/234 -- GLOBAL_STEP: 15910 + | > loss: -0.04018 (-0.02094) + | > log_mle: -0.60482 (-0.24511) + | > loss_dur: 0.56464 (0.22417) + | > amp_scaler: 4096.00000 (8174.34483) + | > grad_norm: 0.00000 (21.81808) + | > current_lr: 0.00002 + | > step_time: 0.29390 (2.71189) + | > loader_time: 0.00460 (0.05205) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40158 (+0.04179) + | > avg_loss: -0.05462 (-0.02505) + | > avg_log_mle: -0.30796 (-0.02192) + | > avg_loss_dur: 0.25335 (-0.00313) + + + > EPOCH: 68/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 23:33:16)  + + --> STEP: 3/234 -- GLOBAL_STEP: 15915 + | > loss: 0.13736 (0.12389) + | > log_mle: -0.12030 (-0.11090) + | > loss_dur: 0.25766 (0.23479) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.27140 (3.39913) + | > current_lr: 0.00002 + | > step_time: 1.40390 (6.59686) + | > loader_time: 0.00290 (0.06411) + + + --> STEP: 8/234 -- GLOBAL_STEP: 15920 + | > loss: 0.04684 (0.09277) + | > log_mle: -0.13465 (-0.12207) + | > loss_dur: 0.18149 (0.21483) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.61475 (4.46666) + | > current_lr: 0.00002 + | > step_time: 2.09250 (5.73803) + | > loader_time: 0.00240 (0.07008) + + + --> STEP: 13/234 -- GLOBAL_STEP: 15925 + | > loss: 0.07770 (0.08161) + | > log_mle: -0.11842 (-0.12389) + | > loss_dur: 0.19611 (0.20550) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.48124 (4.58903) + | > current_lr: 0.00002 + | > step_time: 10.91620 (6.02516) + | > loader_time: 0.19020 (0.10354) + + + --> STEP: 18/234 -- GLOBAL_STEP: 15930 + | > loss: 0.05557 (0.07452) + | > log_mle: -0.12308 (-0.12340) + | > loss_dur: 0.17864 (0.19793) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.93598 (4.37240) + | > current_lr: 0.00002 + | > step_time: 0.94080 (5.23375) + | > loader_time: 0.00130 (0.08654) + + + --> STEP: 23/234 -- GLOBAL_STEP: 15935 + | > loss: 0.02723 (0.06967) + | > log_mle: -0.12728 (-0.12268) + | > loss_dur: 0.15451 (0.19235) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.14389 (4.21573) + | > current_lr: 0.00002 + | > step_time: 1.69780 (4.62180) + | > loader_time: 0.00100 (0.07140) + + + --> STEP: 28/234 -- GLOBAL_STEP: 15940 + | > loss: 0.03440 (0.06619) + | > log_mle: -0.12109 (-0.12244) + | > loss_dur: 0.15548 (0.18863) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.25151 (3.99011) + | > current_lr: 0.00002 + | > step_time: 2.90930 (4.77557) + | > loader_time: 0.09260 (0.07621) + + + --> STEP: 33/234 -- GLOBAL_STEP: 15945 + | > loss: 0.06001 (0.06310) + | > log_mle: -0.11433 (-0.12352) + | > loss_dur: 0.17434 (0.18663) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.48271 (3.87549) + | > current_lr: 0.00002 + | > step_time: 4.20920 (4.59627) + | > loader_time: 0.08770 (0.07852) + + + --> STEP: 38/234 -- GLOBAL_STEP: 15950 + | > loss: 0.07148 (0.06193) + | > log_mle: -0.13482 (-0.12465) + | > loss_dur: 0.20630 (0.18658) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.27548 (4.15645) + | > current_lr: 0.00002 + | > step_time: 5.89440 (4.65230) + | > loader_time: 0.00130 (0.08367) + + + --> STEP: 43/234 -- GLOBAL_STEP: 15955 + | > loss: 0.05497 (0.06205) + | > log_mle: -0.13491 (-0.12473) + | > loss_dur: 0.18988 (0.18678) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.55270 (4.20959) + | > current_lr: 0.00002 + | > step_time: 0.83430 (4.28908) + | > loader_time: 0.00230 (0.07434) + + + --> STEP: 48/234 -- GLOBAL_STEP: 15960 + | > loss: 0.03850 (0.06026) + | > log_mle: -0.12169 (-0.12547) + | > loss_dur: 0.16019 (0.18573) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.95604 (4.22873) + | > current_lr: 0.00002 + | > step_time: 1.82870 (3.98601) + | > loader_time: 0.07510 (0.06993) + + + --> STEP: 53/234 -- GLOBAL_STEP: 15965 + | > loss: 0.04987 (0.05899) + | > log_mle: -0.14187 (-0.12554) + | > loss_dur: 0.19174 (0.18453) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.45202 (4.15682) + | > current_lr: 0.00002 + | > step_time: 4.71390 (3.80569) + | > loader_time: 0.00200 (0.06359) + + + --> STEP: 58/234 -- GLOBAL_STEP: 15970 + | > loss: 0.03168 (0.05793) + | > log_mle: -0.12739 (-0.12611) + | > loss_dur: 0.15907 (0.18404) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.13003 (4.06676) + | > current_lr: 0.00002 + | > step_time: 1.35340 (3.63931) + | > loader_time: 0.00160 (0.05970) + + + --> STEP: 63/234 -- GLOBAL_STEP: 15975 + | > loss: 0.06173 (0.05582) + | > log_mle: -0.14020 (-0.12817) + | > loss_dur: 0.20192 (0.18399) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.36858 (4.25919) + | > current_lr: 0.00002 + | > step_time: 2.30420 (3.48077) + | > loader_time: 0.08700 (0.05651) + + + --> STEP: 68/234 -- GLOBAL_STEP: 15980 + | > loss: 0.05269 (0.05501) + | > log_mle: -0.13188 (-0.12850) + | > loss_dur: 0.18457 (0.18351) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.20031 (4.26604) + | > current_lr: 0.00002 + | > step_time: 1.37270 (3.35443) + | > loader_time: 0.00220 (0.05482) + + + --> STEP: 73/234 -- GLOBAL_STEP: 15985 + | > loss: 0.04300 (0.05491) + | > log_mle: -0.15674 (-0.12931) + | > loss_dur: 0.19973 (0.18422) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.92229 (4.77503) + | > current_lr: 0.00002 + | > step_time: 1.80180 (3.23281) + | > loader_time: 0.07860 (0.05227) + + + --> STEP: 78/234 -- GLOBAL_STEP: 15990 + | > loss: 0.05880 (0.05374) + | > log_mle: -0.13018 (-0.13015) + | > loss_dur: 0.18898 (0.18389) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.36028 (4.87277) + | > current_lr: 0.00002 + | > step_time: 1.40970 (3.15498) + | > loader_time: 0.08510 (0.05122) + + + --> STEP: 83/234 -- GLOBAL_STEP: 15995 + | > loss: 0.04380 (0.05243) + | > log_mle: -0.16012 (-0.13106) + | > loss_dur: 0.20393 (0.18349) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.18897 (5.01552) + | > current_lr: 0.00002 + | > step_time: 1.11050 (3.05537) + | > loader_time: 0.00340 (0.05031) + + + --> STEP: 88/234 -- GLOBAL_STEP: 16000 + | > loss: -0.00486 (0.05111) + | > log_mle: -0.19461 (-0.13261) + | > loss_dur: 0.18975 (0.18371) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.53227 (5.24911) + | > current_lr: 0.00002 + | > step_time: 2.40190 (2.99764) + | > loader_time: 0.00370 (0.04765) + + + --> STEP: 93/234 -- GLOBAL_STEP: 16005 + | > loss: 0.00360 (0.04914) + | > log_mle: -0.20665 (-0.13529) + | > loss_dur: 0.21025 (0.18443) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.39660 (5.66836) + | > current_lr: 0.00002 + | > step_time: 0.77830 (2.90722) + | > loader_time: 0.00270 (0.04725) + + + --> STEP: 98/234 -- GLOBAL_STEP: 16010 + | > loss: 0.05588 (0.04721) + | > log_mle: -0.13549 (-0.13796) + | > loss_dur: 0.19138 (0.18516) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.86241 (6.04153) + | > current_lr: 0.00002 + | > step_time: 1.59790 (2.85982) + | > loader_time: 0.08390 (0.04581) + + + --> STEP: 103/234 -- GLOBAL_STEP: 16015 + | > loss: -0.00504 (0.04496) + | > log_mle: -0.22962 (-0.14126) + | > loss_dur: 0.22459 (0.18623) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.67582 (6.52046) + | > current_lr: 0.00002 + | > step_time: 3.69140 (2.82973) + | > loader_time: 0.00320 (0.04546) + + + --> STEP: 108/234 -- GLOBAL_STEP: 16020 + | > loss: 0.02000 (0.04307) + | > log_mle: -0.17842 (-0.14407) + | > loss_dur: 0.19842 (0.18714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.60414 (6.91853) + | > current_lr: 0.00002 + | > step_time: 1.08190 (2.79195) + | > loader_time: 0.00240 (0.04349) + + + --> STEP: 113/234 -- GLOBAL_STEP: 16025 + | > loss: -0.01424 (0.04122) + | > log_mle: -0.22594 (-0.14749) + | > loss_dur: 0.21170 (0.18871) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.06237 (7.37818) + | > current_lr: 0.00002 + | > step_time: 2.43690 (2.76351) + | > loader_time: 0.00400 (0.04171) + + + --> STEP: 118/234 -- GLOBAL_STEP: 16030 + | > loss: 0.00574 (0.03987) + | > log_mle: -0.19347 (-0.14998) + | > loss_dur: 0.19921 (0.18986) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.56254 (7.70677) + | > current_lr: 0.00002 + | > step_time: 1.20040 (2.70875) + | > loader_time: 0.00210 (0.04152) + + + --> STEP: 123/234 -- GLOBAL_STEP: 16035 + | > loss: 0.02790 (0.03876) + | > log_mle: -0.16517 (-0.15159) + | > loss_dur: 0.19307 (0.19035) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.22329 (7.89947) + | > current_lr: 0.00002 + | > step_time: 2.59460 (2.67992) + | > loader_time: 0.00410 (0.04056) + + + --> STEP: 128/234 -- GLOBAL_STEP: 16040 + | > loss: -0.02637 (0.03648) + | > log_mle: -0.22318 (-0.15496) + | > loss_dur: 0.19681 (0.19144) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.71209 (8.49326) + | > current_lr: 0.00002 + | > step_time: 1.59510 (2.66456) + | > loader_time: 0.00250 (0.04050) + + + --> STEP: 133/234 -- GLOBAL_STEP: 16045 + | > loss: -0.00176 (0.03437) + | > log_mle: -0.24460 (-0.15837) + | > loss_dur: 0.24284 (0.19273) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.32315 (8.96664) + | > current_lr: 0.00002 + | > step_time: 2.00680 (2.64931) + | > loader_time: 0.09360 (0.04037) + + + --> STEP: 138/234 -- GLOBAL_STEP: 16050 + | > loss: 0.00368 (0.03255) + | > log_mle: -0.20366 (-0.16154) + | > loss_dur: 0.20734 (0.19410) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.62270 (9.43699) + | > current_lr: 0.00002 + | > step_time: 4.09530 (2.64982) + | > loader_time: 0.10680 (0.04098) + + + --> STEP: 143/234 -- GLOBAL_STEP: 16055 + | > loss: -0.05414 (0.02999) + | > log_mle: -0.32767 (-0.16549) + | > loss_dur: 0.27353 (0.19548) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.16550 (10.07730) + | > current_lr: 0.00002 + | > step_time: 1.70510 (2.66477) + | > loader_time: 0.00350 (0.04032) + + + --> STEP: 148/234 -- GLOBAL_STEP: 16060 + | > loss: -0.04368 (0.02719) + | > log_mle: -0.25414 (-0.16949) + | > loss_dur: 0.21046 (0.19668) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.31676 (10.57152) + | > current_lr: 0.00002 + | > step_time: 1.30810 (2.65797) + | > loader_time: 0.08810 (0.04130) + + + --> STEP: 153/234 -- GLOBAL_STEP: 16065 + | > loss: -0.11592 (0.02386) + | > log_mle: -0.36490 (-0.17434) + | > loss_dur: 0.24898 (0.19820) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.43075 (11.36386) + | > current_lr: 0.00002 + | > step_time: 2.60130 (2.64762) + | > loader_time: 0.19630 (0.04138) + + + --> STEP: 158/234 -- GLOBAL_STEP: 16070 + | > loss: -0.06133 (0.02097) + | > log_mle: -0.31078 (-0.17863) + | > loss_dur: 0.24945 (0.19960) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.44063 (12.13432) + | > current_lr: 0.00002 + | > step_time: 3.30830 (2.65118) + | > loader_time: 0.08730 (0.04127) + + + --> STEP: 163/234 -- GLOBAL_STEP: 16075 + | > loss: -0.05453 (0.01781) + | > log_mle: -0.28512 (-0.18292) + | > loss_dur: 0.23059 (0.20073) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.16792 (12.83342) + | > current_lr: 0.00002 + | > step_time: 2.61140 (2.64209) + | > loader_time: 0.00270 (0.04117) + + + --> STEP: 168/234 -- GLOBAL_STEP: 16080 + | > loss: -0.06812 (0.01483) + | > log_mle: -0.33078 (-0.18729) + | > loss_dur: 0.26266 (0.20213) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.03143 (13.45941) + | > current_lr: 0.00002 + | > step_time: 1.49980 (2.65222) + | > loader_time: 0.08020 (0.04149) + + + --> STEP: 173/234 -- GLOBAL_STEP: 16085 + | > loss: -0.08921 (0.01169) + | > log_mle: -0.33590 (-0.19213) + | > loss_dur: 0.24669 (0.20381) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.56954 (14.05642) + | > current_lr: 0.00002 + | > step_time: 2.89700 (2.66977) + | > loader_time: 0.00650 (0.04190) + + + --> STEP: 178/234 -- GLOBAL_STEP: 16090 + | > loss: -0.11029 (0.00858) + | > log_mle: -0.38784 (-0.19690) + | > loss_dur: 0.27755 (0.20548) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.72805 (15.02839) + | > current_lr: 0.00002 + | > step_time: 8.19480 (2.73471) + | > loader_time: 0.10370 (0.04252) + + + --> STEP: 183/234 -- GLOBAL_STEP: 16095 + | > loss: -0.12167 (0.00580) + | > log_mle: -0.38732 (-0.20134) + | > loss_dur: 0.26565 (0.20714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.77486 (15.68423) + | > current_lr: 0.00002 + | > step_time: 3.31220 (2.80108) + | > loader_time: 0.09460 (0.04356) + + + --> STEP: 188/234 -- GLOBAL_STEP: 16100 + | > loss: -0.13354 (0.00303) + | > log_mle: -0.40108 (-0.20581) + | > loss_dur: 0.26755 (0.20884) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.89814 (16.25093) + | > current_lr: 0.00002 + | > step_time: 2.59930 (2.85051) + | > loader_time: 0.00660 (0.04508) + + + --> STEP: 193/234 -- GLOBAL_STEP: 16105 + | > loss: -0.12804 (-0.00001) + | > log_mle: -0.40113 (-0.21013) + | > loss_dur: 0.27310 (0.21012) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.36081 (16.84879) + | > current_lr: 0.00002 + | > step_time: 7.09600 (2.89796) + | > loader_time: 0.01020 (0.04545) + + + --> STEP: 198/234 -- GLOBAL_STEP: 16110 + | > loss: -0.12070 (-0.00279) + | > log_mle: -0.39130 (-0.21429) + | > loss_dur: 0.27059 (0.21150) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.21587 (17.35701) + | > current_lr: 0.00002 + | > step_time: 2.08710 (2.91067) + | > loader_time: 0.00360 (0.04829) + + + --> STEP: 203/234 -- GLOBAL_STEP: 16115 + | > loss: -0.07805 (-0.00533) + | > log_mle: -0.33328 (-0.21827) + | > loss_dur: 0.25523 (0.21294) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.34298 (18.11472) + | > current_lr: 0.00002 + | > step_time: 7.70720 (3.04840) + | > loader_time: 0.29470 (0.05010) + + + --> STEP: 208/234 -- GLOBAL_STEP: 16120 + | > loss: -0.12311 (-0.00822) + | > log_mle: -0.40112 (-0.22274) + | > loss_dur: 0.27801 (0.21453) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.90372 (19.29761) + | > current_lr: 0.00002 + | > step_time: 6.20640 (3.09237) + | > loader_time: 0.59630 (0.05366) + + + --> STEP: 213/234 -- GLOBAL_STEP: 16125 + | > loss: -0.15217 (-0.01148) + | > log_mle: -0.44494 (-0.22768) + | > loss_dur: 0.29277 (0.21620) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.06279 (20.33972) + | > current_lr: 0.00002 + | > step_time: 3.29420 (3.16103) + | > loader_time: 0.09370 (0.05428) + + + --> STEP: 218/234 -- GLOBAL_STEP: 16130 + | > loss: -0.12701 (-0.01456) + | > log_mle: -0.41104 (-0.23215) + | > loss_dur: 0.28403 (0.21760) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.87529 (21.32107) + | > current_lr: 0.00002 + | > step_time: 3.89160 (3.22067) + | > loader_time: 0.00400 (0.05495) + + + --> STEP: 223/234 -- GLOBAL_STEP: 16135 + | > loss: -0.16443 (-0.01770) + | > log_mle: -0.44978 (-0.23694) + | > loss_dur: 0.28535 (0.21924) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.18194 (22.12440) + | > current_lr: 0.00002 + | > step_time: 0.82190 (3.18858) + | > loader_time: 0.07930 (0.05454) + + + --> STEP: 228/234 -- GLOBAL_STEP: 16140 + | > loss: -0.14592 (-0.02088) + | > log_mle: -0.45229 (-0.24193) + | > loss_dur: 0.30637 (0.22105) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.87196 (22.88220) + | > current_lr: 0.00002 + | > step_time: 0.24840 (3.12395) + | > loader_time: 0.00310 (0.05342) + + + --> STEP: 233/234 -- GLOBAL_STEP: 16145 + | > loss: 0.51874 (-0.02010) + | > log_mle: -0.41335 (-0.24787) + | > loss_dur: 0.93208 (0.22777) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.26116 (23.71977) + | > current_lr: 0.00002 + | > step_time: 0.18290 (3.06258) + | > loader_time: 0.00310 (0.05237) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02531 (-0.37626) + | > avg_loss: -0.02769 (+0.02692) + | > avg_log_mle: -0.28223 (+0.02574) + | > avg_loss_dur: 0.25453 (+0.00119) + + + > EPOCH: 69/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 23:46:31)  + + --> STEP: 4/234 -- GLOBAL_STEP: 16150 + | > loss: 0.09521 (0.11605) + | > log_mle: -0.13469 (-0.11940) + | > loss_dur: 0.22990 (0.23545) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.65789 (4.88075) + | > current_lr: 0.00002 + | > step_time: 7.00720 (4.80590) + | > loader_time: 0.00180 (0.02483) + + + --> STEP: 9/234 -- GLOBAL_STEP: 16155 + | > loss: 0.04794 (0.09135) + | > log_mle: -0.14128 (-0.12689) + | > loss_dur: 0.18922 (0.21823) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.35583 (4.62288) + | > current_lr: 0.00002 + | > step_time: 1.79050 (4.50579) + | > loader_time: 0.00700 (0.05344) + + + --> STEP: 14/234 -- GLOBAL_STEP: 16160 + | > loss: 0.05353 (0.07942) + | > log_mle: -0.13255 (-0.12719) + | > loss_dur: 0.18608 (0.20660) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.25255 (4.58190) + | > current_lr: 0.00002 + | > step_time: 1.59670 (4.36859) + | > loader_time: 0.00310 (0.11119) + + + --> STEP: 19/234 -- GLOBAL_STEP: 16165 + | > loss: 0.04700 (0.07190) + | > log_mle: -0.11956 (-0.12575) + | > loss_dur: 0.16656 (0.19765) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.33172 (4.51115) + | > current_lr: 0.00002 + | > step_time: 0.70720 (4.22534) + | > loader_time: 0.00310 (0.09208) + + + --> STEP: 24/234 -- GLOBAL_STEP: 16170 + | > loss: 0.03783 (0.06470) + | > log_mle: -0.11922 (-0.12515) + | > loss_dur: 0.15705 (0.18985) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.27426 (4.33875) + | > current_lr: 0.00002 + | > step_time: 1.99920 (4.09857) + | > loader_time: 0.00430 (0.08114) + + + --> STEP: 29/234 -- GLOBAL_STEP: 16175 + | > loss: 0.05616 (0.06281) + | > log_mle: -0.11897 (-0.12490) + | > loss_dur: 0.17513 (0.18772) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.18219 (4.12833) + | > current_lr: 0.00002 + | > step_time: 1.40410 (4.04033) + | > loader_time: 0.07610 (0.07723) + + + --> STEP: 34/234 -- GLOBAL_STEP: 16180 + | > loss: 0.06355 (0.06065) + | > log_mle: -0.12942 (-0.12622) + | > loss_dur: 0.19297 (0.18687) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.76267 (4.04918) + | > current_lr: 0.00002 + | > step_time: 1.40620 (3.66014) + | > loader_time: 0.00210 (0.06620) + + + --> STEP: 39/234 -- GLOBAL_STEP: 16185 + | > loss: 0.05116 (0.05870) + | > log_mle: -0.13591 (-0.12748) + | > loss_dur: 0.18707 (0.18618) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.68215 (4.33558) + | > current_lr: 0.00002 + | > step_time: 2.59540 (3.40968) + | > loader_time: 0.00200 (0.05797) + + + --> STEP: 44/234 -- GLOBAL_STEP: 16190 + | > loss: 0.03775 (0.05826) + | > log_mle: -0.12838 (-0.12738) + | > loss_dur: 0.16612 (0.18564) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.74698 (4.19874) + | > current_lr: 0.00002 + | > step_time: 1.14060 (3.14868) + | > loader_time: 0.00190 (0.05166) + + + --> STEP: 49/234 -- GLOBAL_STEP: 16195 + | > loss: 0.01705 (0.05578) + | > log_mle: -0.13549 (-0.12824) + | > loss_dur: 0.15253 (0.18402) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.61458 (4.17341) + | > current_lr: 0.00002 + | > step_time: 1.29180 (2.97346) + | > loader_time: 0.00500 (0.04666) + + + --> STEP: 54/234 -- GLOBAL_STEP: 16200 + | > loss: 0.02806 (0.05543) + | > log_mle: -0.14055 (-0.12840) + | > loss_dur: 0.16861 (0.18383) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.50790 (4.06275) + | > current_lr: 0.00002 + | > step_time: 1.51000 (2.83263) + | > loader_time: 0.00270 (0.04255) + + + --> STEP: 59/234 -- GLOBAL_STEP: 16205 + | > loss: -0.00395 (0.05365) + | > log_mle: -0.15796 (-0.12918) + | > loss_dur: 0.15401 (0.18282) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.41012 (3.98439) + | > current_lr: 0.00002 + | > step_time: 1.70190 (2.73603) + | > loader_time: 0.00190 (0.04227) + + + --> STEP: 64/234 -- GLOBAL_STEP: 16210 + | > loss: 0.02533 (0.05219) + | > log_mle: -0.12494 (-0.13061) + | > loss_dur: 0.15027 (0.18280) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.10406 (4.15065) + | > current_lr: 0.00002 + | > step_time: 2.79770 (2.66402) + | > loader_time: 0.08930 (0.04050) + + + --> STEP: 69/234 -- GLOBAL_STEP: 16215 + | > loss: 0.06916 (0.05164) + | > log_mle: -0.11268 (-0.13070) + | > loss_dur: 0.18184 (0.18235) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.11182 (4.35759) + | > current_lr: 0.00002 + | > step_time: 1.58060 (2.60374) + | > loader_time: 0.00220 (0.04009) + + + --> STEP: 74/234 -- GLOBAL_STEP: 16220 + | > loss: 0.03604 (0.05100) + | > log_mle: -0.13499 (-0.13182) + | > loss_dur: 0.17103 (0.18281) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.76108 (4.84802) + | > current_lr: 0.00002 + | > step_time: 1.36240 (2.56654) + | > loader_time: 0.00220 (0.03872) + + + --> STEP: 79/234 -- GLOBAL_STEP: 16225 + | > loss: 0.01871 (0.04990) + | > log_mle: -0.14575 (-0.13259) + | > loss_dur: 0.16445 (0.18250) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.55745 (4.99086) + | > current_lr: 0.00002 + | > step_time: 1.70740 (2.50683) + | > loader_time: 0.08450 (0.03749) + + + --> STEP: 84/234 -- GLOBAL_STEP: 16230 + | > loss: 0.04241 (0.04886) + | > log_mle: -0.14637 (-0.13344) + | > loss_dur: 0.18878 (0.18230) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.22782 (5.08509) + | > current_lr: 0.00002 + | > step_time: 1.89740 (2.47536) + | > loader_time: 0.00210 (0.03539) + + + --> STEP: 89/234 -- GLOBAL_STEP: 16235 + | > loss: 0.00397 (0.04719) + | > log_mle: -0.17526 (-0.13526) + | > loss_dur: 0.17923 (0.18245) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.76527 (5.37846) + | > current_lr: 0.00002 + | > step_time: 2.10410 (2.43920) + | > loader_time: 0.00240 (0.03450) + + + --> STEP: 94/234 -- GLOBAL_STEP: 16240 + | > loss: -0.00776 (0.04501) + | > log_mle: -0.21046 (-0.13828) + | > loss_dur: 0.20270 (0.18329) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.64685 (5.76580) + | > current_lr: 0.00002 + | > step_time: 1.46540 (2.41778) + | > loader_time: 0.00240 (0.03281) + + + --> STEP: 99/234 -- GLOBAL_STEP: 16245 + | > loss: -0.02995 (0.04291) + | > log_mle: -0.23780 (-0.14111) + | > loss_dur: 0.20785 (0.18402) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.68855 (6.35496) + | > current_lr: 0.00002 + | > step_time: 1.91130 (2.40005) + | > loader_time: 0.07590 (0.03298) + + + --> STEP: 104/234 -- GLOBAL_STEP: 16250 + | > loss: -0.02544 (0.04093) + | > log_mle: -0.24840 (-0.14427) + | > loss_dur: 0.22296 (0.18520) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.29602 (6.79460) + | > current_lr: 0.00002 + | > step_time: 2.41020 (2.38838) + | > loader_time: 0.00240 (0.03155) + + + --> STEP: 109/234 -- GLOBAL_STEP: 16255 + | > loss: 0.01825 (0.03990) + | > log_mle: -0.22531 (-0.14649) + | > loss_dur: 0.24356 (0.18639) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.48019 (7.19336) + | > current_lr: 0.00002 + | > step_time: 1.10430 (2.37582) + | > loader_time: 0.09280 (0.03108) + + + --> STEP: 114/234 -- GLOBAL_STEP: 16260 + | > loss: -0.00645 (0.03822) + | > log_mle: -0.20461 (-0.14952) + | > loss_dur: 0.19816 (0.18773) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.11757 (7.59245) + | > current_lr: 0.00002 + | > step_time: 1.50980 (2.36910) + | > loader_time: 0.00260 (0.02990) + + + --> STEP: 119/234 -- GLOBAL_STEP: 16265 + | > loss: 0.01318 (0.03710) + | > log_mle: -0.20206 (-0.15188) + | > loss_dur: 0.21524 (0.18899) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.94652 (8.04795) + | > current_lr: 0.00002 + | > step_time: 2.51790 (2.36739) + | > loader_time: 0.08730 (0.03189) + + + --> STEP: 124/234 -- GLOBAL_STEP: 16270 + | > loss: -0.03743 (0.03559) + | > log_mle: -0.23069 (-0.15368) + | > loss_dur: 0.19326 (0.18927) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.94207 (8.23365) + | > current_lr: 0.00002 + | > step_time: 2.21680 (2.37545) + | > loader_time: 0.18610 (0.03371) + + + --> STEP: 129/234 -- GLOBAL_STEP: 16275 + | > loss: 0.00502 (0.03345) + | > log_mle: -0.21637 (-0.15699) + | > loss_dur: 0.22139 (0.19044) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.98603 (8.69438) + | > current_lr: 0.00002 + | > step_time: 1.98640 (2.34841) + | > loader_time: 0.00330 (0.03311) + + + --> STEP: 134/234 -- GLOBAL_STEP: 16280 + | > loss: -0.02292 (0.03097) + | > log_mle: -0.26858 (-0.16080) + | > loss_dur: 0.24567 (0.19177) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.15548 (9.15810) + | > current_lr: 0.00002 + | > step_time: 1.29370 (2.38034) + | > loader_time: 0.00240 (0.03335) + + + --> STEP: 139/234 -- GLOBAL_STEP: 16285 + | > loss: -0.09234 (0.02859) + | > log_mle: -0.32000 (-0.16437) + | > loss_dur: 0.22766 (0.19296) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.31730 (9.83278) + | > current_lr: 0.00002 + | > step_time: 1.11160 (2.36380) + | > loader_time: 0.00310 (0.03287) + + + --> STEP: 144/234 -- GLOBAL_STEP: 16290 + | > loss: -0.05380 (0.02654) + | > log_mle: -0.29929 (-0.16806) + | > loss_dur: 0.24549 (0.19460) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.02588 (10.43257) + | > current_lr: 0.00002 + | > step_time: 3.70180 (2.37108) + | > loader_time: 0.00770 (0.03194) + + + --> STEP: 149/234 -- GLOBAL_STEP: 16295 + | > loss: -0.09442 (0.02351) + | > log_mle: -0.34022 (-0.17228) + | > loss_dur: 0.24580 (0.19578) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.96047 (10.98514) + | > current_lr: 0.00002 + | > step_time: 6.90460 (2.40572) + | > loader_time: 0.00400 (0.03219) + + + --> STEP: 154/234 -- GLOBAL_STEP: 16300 + | > loss: -0.07451 (0.02024) + | > log_mle: -0.30533 (-0.17690) + | > loss_dur: 0.23083 (0.19714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.19182 (11.54173) + | > current_lr: 0.00002 + | > step_time: 3.38920 (2.43737) + | > loader_time: 0.00390 (0.03255) + + + --> STEP: 159/234 -- GLOBAL_STEP: 16305 + | > loss: -0.08587 (0.01728) + | > log_mle: -0.32492 (-0.18133) + | > loss_dur: 0.23906 (0.19862) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.63598 (12.16237) + | > current_lr: 0.00002 + | > step_time: 5.30370 (2.45819) + | > loader_time: 0.19740 (0.03518) + + + --> STEP: 164/234 -- GLOBAL_STEP: 16310 + | > loss: -0.07306 (0.01432) + | > log_mle: -0.31630 (-0.18554) + | > loss_dur: 0.24324 (0.19987) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.01880 (12.86893) + | > current_lr: 0.00002 + | > step_time: 6.29590 (2.54434) + | > loader_time: 0.00500 (0.03597) + + + --> STEP: 169/234 -- GLOBAL_STEP: 16315 + | > loss: -0.04418 (0.01154) + | > log_mle: -0.30852 (-0.18983) + | > loss_dur: 0.26434 (0.20138) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.52236 (13.50449) + | > current_lr: 0.00002 + | > step_time: 3.19010 (2.54018) + | > loader_time: 0.00360 (0.03662) + + + --> STEP: 174/234 -- GLOBAL_STEP: 16320 + | > loss: -0.14638 (0.00786) + | > log_mle: -0.39616 (-0.19513) + | > loss_dur: 0.24979 (0.20299) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.43906 (14.34533) + | > current_lr: 0.00002 + | > step_time: 4.09680 (2.55911) + | > loader_time: 0.10510 (0.03690) + + + --> STEP: 179/234 -- GLOBAL_STEP: 16325 + | > loss: -0.11995 (0.00490) + | > log_mle: -0.38696 (-0.19988) + | > loss_dur: 0.26701 (0.20477) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.95824 (15.00987) + | > current_lr: 0.00002 + | > step_time: 3.59650 (2.55912) + | > loader_time: 0.00660 (0.03695) + + + --> STEP: 184/234 -- GLOBAL_STEP: 16330 + | > loss: -0.09293 (0.00216) + | > log_mle: -0.36009 (-0.20418) + | > loss_dur: 0.26717 (0.20633) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.35768 (15.59647) + | > current_lr: 0.00002 + | > step_time: 6.81250 (2.63211) + | > loader_time: 0.18330 (0.03842) + + + --> STEP: 189/234 -- GLOBAL_STEP: 16335 + | > loss: -0.09390 (-0.00068) + | > log_mle: -0.35805 (-0.20864) + | > loss_dur: 0.26415 (0.20796) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.72275 (16.19293) + | > current_lr: 0.00002 + | > step_time: 3.10030 (2.66151) + | > loader_time: 0.10930 (0.03940) + + + --> STEP: 194/234 -- GLOBAL_STEP: 16340 + | > loss: -0.12628 (-0.00390) + | > log_mle: -0.38723 (-0.21309) + | > loss_dur: 0.26095 (0.20919) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.77475 (16.90761) + | > current_lr: 0.00002 + | > step_time: 2.11310 (2.74404) + | > loader_time: 0.08530 (0.04182) + + + --> STEP: 199/234 -- GLOBAL_STEP: 16345 + | > loss: -0.13049 (-0.00659) + | > log_mle: -0.39284 (-0.21717) + | > loss_dur: 0.26235 (0.21058) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.56662 (17.86669) + | > current_lr: 0.00002 + | > step_time: 3.80030 (2.86413) + | > loader_time: 0.00400 (0.04322) + + + --> STEP: 204/234 -- GLOBAL_STEP: 16350 + | > loss: -0.12801 (-0.00895) + | > log_mle: -0.42110 (-0.22118) + | > loss_dur: 0.29309 (0.21223) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.28747 (18.74851) + | > current_lr: 0.00002 + | > step_time: 3.20140 (2.88809) + | > loader_time: 0.08970 (0.04408) + + + --> STEP: 209/234 -- GLOBAL_STEP: 16355 + | > loss: -0.11140 (-0.01186) + | > log_mle: -0.38478 (-0.22554) + | > loss_dur: 0.27339 (0.21368) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.76639 (19.40791) + | > current_lr: 0.00002 + | > step_time: 3.49820 (2.89897) + | > loader_time: 0.10160 (0.04448) + + + --> STEP: 214/234 -- GLOBAL_STEP: 16360 + | > loss: -0.16097 (-0.01541) + | > log_mle: -0.41788 (-0.23066) + | > loss_dur: 0.25692 (0.21525) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.68884 (20.19471) + | > current_lr: 0.00002 + | > step_time: 2.40030 (2.96812) + | > loader_time: 0.00430 (0.04668) + + + --> STEP: 219/234 -- GLOBAL_STEP: 16365 + | > loss: -0.20822 (-0.01872) + | > log_mle: -0.49706 (-0.23557) + | > loss_dur: 0.28883 (0.21685) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 82.64416 (20.94916) + | > current_lr: 0.00002 + | > step_time: 3.50060 (3.03003) + | > loader_time: 0.08500 (0.04820) + + + --> STEP: 224/234 -- GLOBAL_STEP: 16370 + | > loss: -0.16672 (-0.02168) + | > log_mle: -0.45842 (-0.24013) + | > loss_dur: 0.29170 (0.21845) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.23486 (21.68471) + | > current_lr: 0.00002 + | > step_time: 1.37970 (3.03428) + | > loader_time: 0.00290 (0.04752) + + + --> STEP: 229/234 -- GLOBAL_STEP: 16375 + | > loss: -0.13060 (-0.02467) + | > log_mle: -0.48629 (-0.24521) + | > loss_dur: 0.35569 (0.22053) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.41486 (22.44550) + | > current_lr: 0.00002 + | > step_time: 0.25610 (2.97550) + | > loader_time: 0.00410 (0.04657) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40713 (+0.38182) + | > avg_loss: -0.06916 (-0.04147) + | > avg_log_mle: -0.32200 (-0.03977) + | > avg_loss_dur: 0.25284 (-0.00169) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_16380.pth + + > EPOCH: 70/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-27 23:58:58)  + + --> STEP: 0/234 -- GLOBAL_STEP: 16380 + | > loss: 0.08130 (0.08130) + | > log_mle: -0.16720 (-0.16720) + | > loss_dur: 0.24850 (0.24850) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.76246 (4.76246) + | > current_lr: 0.00002 + | > step_time: 8.70210 (8.70207) + | > loader_time: 8.52240 (8.52237) + + + --> STEP: 5/234 -- GLOBAL_STEP: 16385 + | > loss: 0.05007 (0.09749) + | > log_mle: -0.13153 (-0.12353) + | > loss_dur: 0.18159 (0.22102) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.96041 (5.49667) + | > current_lr: 0.00002 + | > step_time: 4.90890 (4.21906) + | > loader_time: 0.08460 (0.67579) + + + --> STEP: 10/234 -- GLOBAL_STEP: 16390 + | > loss: 0.03709 (0.07831) + | > log_mle: -0.13770 (-0.12964) + | > loss_dur: 0.17479 (0.20795) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.56131 (5.54070) + | > current_lr: 0.00002 + | > step_time: 1.00540 (6.19004) + | > loader_time: 0.08950 (0.36728) + + + --> STEP: 15/234 -- GLOBAL_STEP: 16395 + | > loss: 0.04394 (0.07095) + | > log_mle: -0.13005 (-0.12909) + | > loss_dur: 0.17399 (0.20004) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.48744 (5.24639) + | > current_lr: 0.00002 + | > step_time: 0.91410 (4.54574) + | > loader_time: 0.00120 (0.25080) + + + --> STEP: 20/234 -- GLOBAL_STEP: 16400 + | > loss: 0.05479 (0.06465) + | > log_mle: -0.11993 (-0.12741) + | > loss_dur: 0.17472 (0.19206) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.60959 (4.90584) + | > current_lr: 0.00002 + | > step_time: 1.90560 (3.83838) + | > loader_time: 0.00120 (0.18849) + + + --> STEP: 25/234 -- GLOBAL_STEP: 16405 + | > loss: 0.06912 (0.06091) + | > log_mle: -0.11339 (-0.12664) + | > loss_dur: 0.18251 (0.18755) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.33140 (4.79182) + | > current_lr: 0.00002 + | > step_time: 2.79380 (3.40507) + | > loader_time: 0.00450 (0.15133) + + + --> STEP: 30/234 -- GLOBAL_STEP: 16410 + | > loss: 0.00884 (0.05558) + | > log_mle: -0.14465 (-0.12758) + | > loss_dur: 0.15348 (0.18315) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.63070 (4.57808) + | > current_lr: 0.00002 + | > step_time: 1.67340 (3.10667) + | > loader_time: 0.00200 (0.12932) + + + --> STEP: 35/234 -- GLOBAL_STEP: 16415 + | > loss: 0.01951 (0.05436) + | > log_mle: -0.14329 (-0.12881) + | > loss_dur: 0.16280 (0.18317) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.65903 (4.62570) + | > current_lr: 0.00002 + | > step_time: 2.09070 (2.91323) + | > loader_time: 0.00200 (0.11588) + + + --> STEP: 40/234 -- GLOBAL_STEP: 16420 + | > loss: 0.06713 (0.05406) + | > log_mle: -0.12143 (-0.12956) + | > loss_dur: 0.18856 (0.18362) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.80082 (4.55600) + | > current_lr: 0.00002 + | > step_time: 1.80080 (2.72602) + | > loader_time: 0.00290 (0.10167) + + + --> STEP: 45/234 -- GLOBAL_STEP: 16425 + | > loss: 0.01743 (0.05229) + | > log_mle: -0.15666 (-0.13020) + | > loss_dur: 0.17409 (0.18249) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.12463 (4.49858) + | > current_lr: 0.00002 + | > step_time: 1.60510 (2.60057) + | > loader_time: 0.00260 (0.09064) + + + --> STEP: 50/234 -- GLOBAL_STEP: 16430 + | > loss: 0.05238 (0.05072) + | > log_mle: -0.12332 (-0.13028) + | > loss_dur: 0.17570 (0.18100) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.24572 (4.37262) + | > current_lr: 0.00002 + | > step_time: 1.42410 (2.52663) + | > loader_time: 0.00210 (0.08547) + + + --> STEP: 55/234 -- GLOBAL_STEP: 16435 + | > loss: 0.03013 (0.04971) + | > log_mle: -0.14487 (-0.13080) + | > loss_dur: 0.17500 (0.18051) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.26181 (4.28676) + | > current_lr: 0.00002 + | > step_time: 1.48980 (2.41206) + | > loader_time: 0.00220 (0.07899) + + + --> STEP: 60/234 -- GLOBAL_STEP: 16440 + | > loss: 0.00188 (0.04782) + | > log_mle: -0.16366 (-0.13183) + | > loss_dur: 0.16554 (0.17965) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.93786 (4.26691) + | > current_lr: 0.00002 + | > step_time: 3.90540 (2.42242) + | > loader_time: 0.08460 (0.07404) + + + --> STEP: 65/234 -- GLOBAL_STEP: 16445 + | > loss: 0.03349 (0.04716) + | > log_mle: -0.13412 (-0.13269) + | > loss_dur: 0.16761 (0.17985) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.44032 (4.45897) + | > current_lr: 0.00002 + | > step_time: 1.38950 (2.38099) + | > loader_time: 0.00320 (0.06986) + + + --> STEP: 70/234 -- GLOBAL_STEP: 16450 + | > loss: 0.03502 (0.04689) + | > log_mle: -0.14152 (-0.13283) + | > loss_dur: 0.17654 (0.17972) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.36102 (4.61532) + | > current_lr: 0.00002 + | > step_time: 2.25900 (2.33894) + | > loader_time: 0.08380 (0.06859) + + + --> STEP: 75/234 -- GLOBAL_STEP: 16455 + | > loss: 0.03612 (0.04620) + | > log_mle: -0.15195 (-0.13409) + | > loss_dur: 0.18807 (0.18029) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.49255 (4.88149) + | > current_lr: 0.00002 + | > step_time: 2.37890 (2.32839) + | > loader_time: 0.00230 (0.06661) + + + --> STEP: 80/234 -- GLOBAL_STEP: 16460 + | > loss: 0.03374 (0.04546) + | > log_mle: -0.13117 (-0.13466) + | > loss_dur: 0.16490 (0.18012) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.02505 (4.88760) + | > current_lr: 0.00002 + | > step_time: 2.84050 (2.31180) + | > loader_time: 0.08090 (0.06362) + + + --> STEP: 85/234 -- GLOBAL_STEP: 16465 + | > loss: 0.03123 (0.04460) + | > log_mle: -0.14746 (-0.13559) + | > loss_dur: 0.17869 (0.18019) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.92392 (5.23065) + | > current_lr: 0.00002 + | > step_time: 3.69800 (2.30586) + | > loader_time: 0.10170 (0.06314) + + + --> STEP: 90/234 -- GLOBAL_STEP: 16470 + | > loss: 0.02489 (0.04328) + | > log_mle: -0.17319 (-0.13748) + | > loss_dur: 0.19809 (0.18075) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.03024 (5.64187) + | > current_lr: 0.00002 + | > step_time: 2.40670 (2.29884) + | > loader_time: 0.00280 (0.06176) + + + --> STEP: 95/234 -- GLOBAL_STEP: 16475 + | > loss: -0.03670 (0.04083) + | > log_mle: -0.25825 (-0.14120) + | > loss_dur: 0.22155 (0.18202) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.59835 (6.08921) + | > current_lr: 0.00002 + | > step_time: 2.20300 (2.31372) + | > loader_time: 0.00300 (0.06063) + + + --> STEP: 100/234 -- GLOBAL_STEP: 16480 + | > loss: 0.00713 (0.03943) + | > log_mle: -0.18593 (-0.14325) + | > loss_dur: 0.19306 (0.18268) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.06654 (6.33881) + | > current_lr: 0.00002 + | > step_time: 1.80290 (2.27429) + | > loader_time: 0.00260 (0.05867) + + + --> STEP: 105/234 -- GLOBAL_STEP: 16485 + | > loss: 0.02234 (0.03750) + | > log_mle: -0.15828 (-0.14620) + | > loss_dur: 0.18061 (0.18370) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.87099 (6.78018) + | > current_lr: 0.00002 + | > step_time: 2.09800 (2.26326) + | > loader_time: 0.00420 (0.06239) + + + --> STEP: 110/234 -- GLOBAL_STEP: 16490 + | > loss: -0.01002 (0.03632) + | > log_mle: -0.18949 (-0.14874) + | > loss_dur: 0.17947 (0.18506) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.40794 (7.38920) + | > current_lr: 0.00002 + | > step_time: 1.80390 (2.28684) + | > loader_time: 0.08740 (0.06125) + + + --> STEP: 115/234 -- GLOBAL_STEP: 16495 + | > loss: 0.00702 (0.03449) + | > log_mle: -0.20886 (-0.15191) + | > loss_dur: 0.21588 (0.18640) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.06296 (7.82795) + | > current_lr: 0.00002 + | > step_time: 2.69450 (2.29613) + | > loader_time: 0.00210 (0.05870) + + + --> STEP: 120/234 -- GLOBAL_STEP: 16500 + | > loss: -0.02746 (0.03303) + | > log_mle: -0.25361 (-0.15470) + | > loss_dur: 0.22615 (0.18773) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.22221 (8.07639) + | > current_lr: 0.00002 + | > step_time: 2.91140 (2.28659) + | > loader_time: 0.08630 (0.05844) + + + --> STEP: 125/234 -- GLOBAL_STEP: 16505 + | > loss: -0.01179 (0.03194) + | > log_mle: -0.24192 (-0.15638) + | > loss_dur: 0.23012 (0.18832) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.02681 (8.21387) + | > current_lr: 0.00002 + | > step_time: 2.20310 (2.28966) + | > loader_time: 0.00260 (0.05624) + + + --> STEP: 130/234 -- GLOBAL_STEP: 16510 + | > loss: -0.03137 (0.02977) + | > log_mle: -0.25366 (-0.15972) + | > loss_dur: 0.22229 (0.18948) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.58993 (8.59738) + | > current_lr: 0.00002 + | > step_time: 1.08910 (2.27620) + | > loader_time: 0.00240 (0.05555) + + + --> STEP: 135/234 -- GLOBAL_STEP: 16515 + | > loss: 0.00578 (0.02771) + | > log_mle: -0.18701 (-0.16297) + | > loss_dur: 0.19279 (0.19069) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.89927 (9.05481) + | > current_lr: 0.00002 + | > step_time: 2.59350 (2.28450) + | > loader_time: 0.00300 (0.05483) + + + --> STEP: 140/234 -- GLOBAL_STEP: 16520 + | > loss: -0.00104 (0.02528) + | > log_mle: -0.21928 (-0.16668) + | > loss_dur: 0.21824 (0.19195) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.01343 (9.66273) + | > current_lr: 0.00002 + | > step_time: 3.60280 (2.29525) + | > loader_time: 0.09350 (0.05484) + + + --> STEP: 145/234 -- GLOBAL_STEP: 16525 + | > loss: -0.07933 (0.02267) + | > log_mle: -0.30773 (-0.17092) + | > loss_dur: 0.22841 (0.19359) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.53231 (10.59596) + | > current_lr: 0.00002 + | > step_time: 1.68280 (2.28422) + | > loader_time: 0.00750 (0.05315) + + + --> STEP: 150/234 -- GLOBAL_STEP: 16530 + | > loss: -0.05307 (0.01985) + | > log_mle: -0.29384 (-0.17500) + | > loss_dur: 0.24078 (0.19486) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.08021 (11.14108) + | > current_lr: 0.00002 + | > step_time: 3.10670 (2.32004) + | > loader_time: 0.00310 (0.05265) + + + --> STEP: 155/234 -- GLOBAL_STEP: 16535 + | > loss: -0.10317 (0.01625) + | > log_mle: -0.35750 (-0.18001) + | > loss_dur: 0.25433 (0.19626) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.89153 (11.81859) + | > current_lr: 0.00002 + | > step_time: 2.29200 (2.33186) + | > loader_time: 0.00360 (0.05223) + + + --> STEP: 160/234 -- GLOBAL_STEP: 16540 + | > loss: -0.10871 (0.01312) + | > log_mle: -0.35249 (-0.18440) + | > loss_dur: 0.24378 (0.19751) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.73102 (12.27522) + | > current_lr: 0.00002 + | > step_time: 1.80920 (2.37910) + | > loader_time: 0.19520 (0.05362) + + + --> STEP: 165/234 -- GLOBAL_STEP: 16545 + | > loss: -0.08259 (0.01030) + | > log_mle: -0.35041 (-0.18861) + | > loss_dur: 0.26782 (0.19890) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.57204 (12.80112) + | > current_lr: 0.00002 + | > step_time: 2.49190 (2.37542) + | > loader_time: 0.09230 (0.05316) + + + --> STEP: 170/234 -- GLOBAL_STEP: 16550 + | > loss: -0.10765 (0.00736) + | > log_mle: -0.38607 (-0.19315) + | > loss_dur: 0.27842 (0.20051) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.26223 (13.41558) + | > current_lr: 0.00002 + | > step_time: 1.08650 (2.37277) + | > loader_time: 0.00430 (0.05174) + + + --> STEP: 175/234 -- GLOBAL_STEP: 16555 + | > loss: -0.08859 (0.00386) + | > log_mle: -0.35883 (-0.19825) + | > loss_dur: 0.27024 (0.20210) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.25149 (14.40260) + | > current_lr: 0.00002 + | > step_time: 3.19300 (2.38710) + | > loader_time: 0.10450 (0.05141) + + + --> STEP: 180/234 -- GLOBAL_STEP: 16560 + | > loss: -0.11238 (0.00081) + | > log_mle: -0.36710 (-0.20298) + | > loss_dur: 0.25472 (0.20380) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.64855 (15.18785) + | > current_lr: 0.00002 + | > step_time: 7.00330 (2.43867) + | > loader_time: 0.19480 (0.05426) + + + --> STEP: 185/234 -- GLOBAL_STEP: 16565 + | > loss: -0.11571 (-0.00190) + | > log_mle: -0.39080 (-0.20737) + | > loss_dur: 0.27509 (0.20546) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.53345 (15.94584) + | > current_lr: 0.00002 + | > step_time: 4.09750 (2.48956) + | > loader_time: 0.19550 (0.05595) + + + --> STEP: 190/234 -- GLOBAL_STEP: 16570 + | > loss: -0.12133 (-0.00479) + | > log_mle: -0.37006 (-0.21169) + | > loss_dur: 0.24873 (0.20689) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.70350 (16.55299) + | > current_lr: 0.00002 + | > step_time: 4.70750 (2.55205) + | > loader_time: 0.00490 (0.05504) + + + --> STEP: 195/234 -- GLOBAL_STEP: 16575 + | > loss: -0.11017 (-0.00790) + | > log_mle: -0.38024 (-0.21610) + | > loss_dur: 0.27007 (0.20820) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.26001 (17.48033) + | > current_lr: 0.00002 + | > step_time: 1.99740 (2.56204) + | > loader_time: 0.00640 (0.05716) + + + --> STEP: 200/234 -- GLOBAL_STEP: 16580 + | > loss: -0.10247 (-0.01057) + | > log_mle: -0.38926 (-0.22023) + | > loss_dur: 0.28680 (0.20966) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.89305 (18.19494) + | > current_lr: 0.00002 + | > step_time: 4.30240 (2.67495) + | > loader_time: 0.00470 (0.05935) + + + --> STEP: 205/234 -- GLOBAL_STEP: 16585 + | > loss: -0.12138 (-0.01313) + | > log_mle: -0.37894 (-0.22424) + | > loss_dur: 0.25756 (0.21111) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.32641 (18.83850) + | > current_lr: 0.00002 + | > step_time: 5.58930 (2.69651) + | > loader_time: 0.10400 (0.05946) + + + --> STEP: 210/234 -- GLOBAL_STEP: 16590 + | > loss: -0.17042 (-0.01632) + | > log_mle: -0.45269 (-0.22896) + | > loss_dur: 0.28226 (0.21264) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.18557 (19.56789) + | > current_lr: 0.00002 + | > step_time: 2.01420 (2.71425) + | > loader_time: 0.08390 (0.05892) + + + --> STEP: 215/234 -- GLOBAL_STEP: 16595 + | > loss: -0.14346 (-0.01967) + | > log_mle: -0.40420 (-0.23381) + | > loss_dur: 0.26073 (0.21414) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.33777 (20.31088) + | > current_lr: 0.00002 + | > step_time: 16.30900 (2.82547) + | > loader_time: 0.49130 (0.05993) + + + --> STEP: 220/234 -- GLOBAL_STEP: 16600 + | > loss: -0.16755 (-0.02313) + | > log_mle: -0.44981 (-0.23891) + | > loss_dur: 0.28227 (0.21578) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 58.22461 (21.08769) + | > current_lr: 0.00002 + | > step_time: 2.49710 (2.92670) + | > loader_time: 0.00380 (0.06350) + + + --> STEP: 225/234 -- GLOBAL_STEP: 16605 + | > loss: -0.20557 (-0.02620) + | > log_mle: -0.50596 (-0.24374) + | > loss_dur: 0.30039 (0.21754) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.75432 (21.83467) + | > current_lr: 0.00002 + | > step_time: 0.24010 (2.89183) + | > loader_time: 0.00440 (0.06294) + + + --> STEP: 230/234 -- GLOBAL_STEP: 16610 + | > loss: -0.15646 (-0.02881) + | > log_mle: -0.54444 (-0.24882) + | > loss_dur: 0.38798 (0.22001) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 94.98944 (23.22608) + | > current_lr: 0.00002 + | > step_time: 0.25890 (2.83446) + | > loader_time: 0.00350 (0.06165) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.08267 (-0.32445) + | > avg_loss: -0.06075 (+0.00841) + | > avg_log_mle: -0.31344 (+0.00856) + | > avg_loss_dur: 0.25269 (-0.00015) + + + > EPOCH: 71/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 00:11:05)  + + --> STEP: 1/234 -- GLOBAL_STEP: 16615 + | > loss: 0.07726 (0.07726) + | > log_mle: -0.12637 (-0.12637) + | > loss_dur: 0.20364 (0.20364) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.14765 (4.14765) + | > current_lr: 0.00002 + | > step_time: 2.49870 (2.49873) + | > loader_time: 0.99710 (0.99706) + + + --> STEP: 6/234 -- GLOBAL_STEP: 16620 + | > loss: 0.08079 (0.08983) + | > log_mle: -0.11915 (-0.12583) + | > loss_dur: 0.19994 (0.21565) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.69997 (5.32483) + | > current_lr: 0.00002 + | > step_time: 6.91040 (7.06987) + | > loader_time: 0.08290 (0.18197) + + + --> STEP: 11/234 -- GLOBAL_STEP: 16625 + | > loss: 0.03924 (0.07128) + | > log_mle: -0.12426 (-0.13185) + | > loss_dur: 0.16349 (0.20312) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.28499 (5.64815) + | > current_lr: 0.00002 + | > step_time: 5.38990 (5.49384) + | > loader_time: 0.19420 (0.13255) + + + --> STEP: 16/234 -- GLOBAL_STEP: 16630 + | > loss: 0.02757 (0.06239) + | > log_mle: -0.13084 (-0.13163) + | > loss_dur: 0.15841 (0.19401) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.32120 (5.44078) + | > current_lr: 0.00002 + | > step_time: 6.10510 (5.03922) + | > loader_time: 0.10920 (0.10504) + + + --> STEP: 21/234 -- GLOBAL_STEP: 16635 + | > loss: 0.05910 (0.06045) + | > log_mle: -0.11386 (-0.12908) + | > loss_dur: 0.17296 (0.18953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.70428 (5.16535) + | > current_lr: 0.00002 + | > step_time: 1.24610 (4.45094) + | > loader_time: 0.00360 (0.08567) + + + --> STEP: 26/234 -- GLOBAL_STEP: 16640 + | > loss: 0.03968 (0.05455) + | > log_mle: -0.13181 (-0.12913) + | > loss_dur: 0.17150 (0.18368) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.30323 (4.88442) + | > current_lr: 0.00002 + | > step_time: 1.68990 (3.83047) + | > loader_time: 0.00110 (0.06960) + + + --> STEP: 31/234 -- GLOBAL_STEP: 16645 + | > loss: 0.06849 (0.05164) + | > log_mle: -0.13791 (-0.13015) + | > loss_dur: 0.20639 (0.18179) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.02575 (4.68078) + | > current_lr: 0.00002 + | > step_time: 3.09320 (3.54106) + | > loader_time: 0.00380 (0.05884) + + + --> STEP: 36/234 -- GLOBAL_STEP: 16650 + | > loss: 0.05311 (0.05026) + | > log_mle: -0.14092 (-0.13127) + | > loss_dur: 0.19403 (0.18152) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.70901 (5.19092) + | > current_lr: 0.00002 + | > step_time: 2.90590 (3.68544) + | > loader_time: 0.00320 (0.05643) + + + --> STEP: 41/234 -- GLOBAL_STEP: 16655 + | > loss: 0.02840 (0.04966) + | > log_mle: -0.13371 (-0.13161) + | > loss_dur: 0.16211 (0.18127) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.40664 (5.09389) + | > current_lr: 0.00002 + | > step_time: 3.48700 (3.44317) + | > loader_time: 0.00770 (0.04996) + + + --> STEP: 46/234 -- GLOBAL_STEP: 16660 + | > loss: 0.05079 (0.04887) + | > log_mle: -0.13744 (-0.13224) + | > loss_dur: 0.18822 (0.18111) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.95836 (5.11775) + | > current_lr: 0.00002 + | > step_time: 1.70440 (3.24914) + | > loader_time: 0.08980 (0.04851) + + + --> STEP: 51/234 -- GLOBAL_STEP: 16665 + | > loss: 0.04039 (0.04753) + | > log_mle: -0.12261 (-0.13200) + | > loss_dur: 0.16300 (0.17953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.35928 (4.94618) + | > current_lr: 0.00002 + | > step_time: 2.10050 (3.13835) + | > loader_time: 0.00430 (0.04401) + + + --> STEP: 56/234 -- GLOBAL_STEP: 16670 + | > loss: 0.05884 (0.04721) + | > log_mle: -0.14191 (-0.13287) + | > loss_dur: 0.20075 (0.18008) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.05935 (4.79331) + | > current_lr: 0.00002 + | > step_time: 1.29170 (3.03219) + | > loader_time: 0.00200 (0.04642) + + + --> STEP: 61/234 -- GLOBAL_STEP: 16675 + | > loss: 0.01919 (0.04515) + | > log_mle: -0.13853 (-0.13383) + | > loss_dur: 0.15772 (0.17898) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.66110 (4.72036) + | > current_lr: 0.00002 + | > step_time: 2.08880 (2.98666) + | > loader_time: 0.00150 (0.04608) + + + --> STEP: 66/234 -- GLOBAL_STEP: 16680 + | > loss: 0.05150 (0.04489) + | > log_mle: -0.12758 (-0.13461) + | > loss_dur: 0.17908 (0.17950) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.83182 (4.75353) + | > current_lr: 0.00002 + | > step_time: 1.38460 (2.87970) + | > loader_time: 0.00230 (0.04387) + + + --> STEP: 71/234 -- GLOBAL_STEP: 16685 + | > loss: 0.03226 (0.04459) + | > log_mle: -0.17443 (-0.13552) + | > loss_dur: 0.20669 (0.18011) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.54040 (5.05241) + | > current_lr: 0.00002 + | > step_time: 1.40810 (2.82075) + | > loader_time: 0.00290 (0.04243) + + + --> STEP: 76/234 -- GLOBAL_STEP: 16690 + | > loss: 0.03060 (0.04377) + | > log_mle: -0.15428 (-0.13653) + | > loss_dur: 0.18488 (0.18029) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.85587 (5.08917) + | > current_lr: 0.00002 + | > step_time: 1.22640 (2.77945) + | > loader_time: 0.08760 (0.04451) + + + --> STEP: 81/234 -- GLOBAL_STEP: 16695 + | > loss: 0.00355 (0.04210) + | > log_mle: -0.16456 (-0.13724) + | > loss_dur: 0.16810 (0.17934) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.59992 (5.10890) + | > current_lr: 0.00002 + | > step_time: 2.19840 (2.76720) + | > loader_time: 0.08560 (0.04491) + + + --> STEP: 86/234 -- GLOBAL_STEP: 16700 + | > loss: 0.02587 (0.04115) + | > log_mle: -0.16426 (-0.13827) + | > loss_dur: 0.19013 (0.17942) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.56637 (5.19714) + | > current_lr: 0.00002 + | > step_time: 1.60640 (2.74793) + | > loader_time: 0.00570 (0.04268) + + + --> STEP: 91/234 -- GLOBAL_STEP: 16705 + | > loss: 0.03545 (0.03967) + | > log_mle: -0.17274 (-0.14043) + | > loss_dur: 0.20819 (0.18010) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.49964 (5.49877) + | > current_lr: 0.00002 + | > step_time: 1.21010 (2.68826) + | > loader_time: 0.00410 (0.04147) + + + --> STEP: 96/234 -- GLOBAL_STEP: 16710 + | > loss: 0.03059 (0.03701) + | > log_mle: -0.16180 (-0.14399) + | > loss_dur: 0.19239 (0.18100) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.20415 (6.13213) + | > current_lr: 0.00002 + | > step_time: 2.10730 (2.66311) + | > loader_time: 0.07790 (0.04283) + + + --> STEP: 101/234 -- GLOBAL_STEP: 16715 + | > loss: -0.01407 (0.03523) + | > log_mle: -0.21805 (-0.14662) + | > loss_dur: 0.20398 (0.18185) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.54690 (6.51088) + | > current_lr: 0.00002 + | > step_time: 1.70110 (2.62729) + | > loader_time: 0.00380 (0.04166) + + + --> STEP: 106/234 -- GLOBAL_STEP: 16720 + | > loss: 0.01038 (0.03346) + | > log_mle: -0.21798 (-0.14955) + | > loss_dur: 0.22835 (0.18301) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.54502 (6.94269) + | > current_lr: 0.00002 + | > step_time: 2.79400 (2.60036) + | > loader_time: 0.10670 (0.04316) + + + --> STEP: 111/234 -- GLOBAL_STEP: 16725 + | > loss: -0.01377 (0.03166) + | > log_mle: -0.25723 (-0.15253) + | > loss_dur: 0.24347 (0.18419) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.73255 (7.40259) + | > current_lr: 0.00002 + | > step_time: 1.69410 (2.58414) + | > loader_time: 0.10220 (0.04383) + + + --> STEP: 116/234 -- GLOBAL_STEP: 16730 + | > loss: 0.01354 (0.03019) + | > log_mle: -0.22701 (-0.15544) + | > loss_dur: 0.24055 (0.18564) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.44297 (7.82974) + | > current_lr: 0.00002 + | > step_time: 3.51870 (2.56544) + | > loader_time: 0.00270 (0.04341) + + + --> STEP: 121/234 -- GLOBAL_STEP: 16735 + | > loss: 0.04541 (0.02884) + | > log_mle: -0.14641 (-0.15752) + | > loss_dur: 0.19182 (0.18636) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.04298 (8.03510) + | > current_lr: 0.00002 + | > step_time: 2.30470 (2.55469) + | > loader_time: 0.08930 (0.04395) + + + --> STEP: 126/234 -- GLOBAL_STEP: 16740 + | > loss: -0.05619 (0.02677) + | > log_mle: -0.27107 (-0.16019) + | > loss_dur: 0.21488 (0.18696) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.22532 (8.41580) + | > current_lr: 0.00002 + | > step_time: 1.99730 (2.54524) + | > loader_time: 0.08390 (0.04441) + + + --> STEP: 131/234 -- GLOBAL_STEP: 16745 + | > loss: -0.06263 (0.02453) + | > log_mle: -0.30906 (-0.16372) + | > loss_dur: 0.24644 (0.18825) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.76859 (9.00539) + | > current_lr: 0.00002 + | > step_time: 4.42380 (2.56888) + | > loader_time: 0.00360 (0.04498) + + + --> STEP: 136/234 -- GLOBAL_STEP: 16750 + | > loss: -0.08645 (0.02238) + | > log_mle: -0.34915 (-0.16713) + | > loss_dur: 0.26271 (0.18951) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.71227 (9.72761) + | > current_lr: 0.00002 + | > step_time: 3.21340 (2.54573) + | > loader_time: 0.08790 (0.04478) + + + --> STEP: 141/234 -- GLOBAL_STEP: 16755 + | > loss: -0.03022 (0.02050) + | > log_mle: -0.26593 (-0.17016) + | > loss_dur: 0.23572 (0.19066) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.30169 (10.18475) + | > current_lr: 0.00002 + | > step_time: 3.99870 (2.55888) + | > loader_time: 0.19900 (0.04538) + + + --> STEP: 146/234 -- GLOBAL_STEP: 16760 + | > loss: -0.07723 (0.01755) + | > log_mle: -0.31829 (-0.17477) + | > loss_dur: 0.24106 (0.19231) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.60150 (10.84650) + | > current_lr: 0.00002 + | > step_time: 3.60770 (2.57214) + | > loader_time: 0.19730 (0.04648) + + + --> STEP: 151/234 -- GLOBAL_STEP: 16765 + | > loss: -0.07232 (0.01486) + | > log_mle: -0.28523 (-0.17856) + | > loss_dur: 0.21292 (0.19342) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.38445 (11.31595) + | > current_lr: 0.00002 + | > step_time: 5.71130 (2.62353) + | > loader_time: 0.19900 (0.04837) + + + --> STEP: 156/234 -- GLOBAL_STEP: 16770 + | > loss: -0.09481 (0.01124) + | > log_mle: -0.32242 (-0.18369) + | > loss_dur: 0.22762 (0.19492) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.32677 (12.04493) + | > current_lr: 0.00002 + | > step_time: 2.72060 (2.61914) + | > loader_time: 0.00430 (0.04853) + + + --> STEP: 161/234 -- GLOBAL_STEP: 16775 + | > loss: -0.10238 (0.00822) + | > log_mle: -0.34042 (-0.18800) + | > loss_dur: 0.23803 (0.19622) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.54599 (12.82084) + | > current_lr: 0.00002 + | > step_time: 15.10610 (2.69486) + | > loader_time: 0.09190 (0.04771) + + + --> STEP: 166/234 -- GLOBAL_STEP: 16780 + | > loss: -0.06958 (0.00562) + | > log_mle: -0.28752 (-0.19174) + | > loss_dur: 0.21794 (0.19735) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.30315 (13.47776) + | > current_lr: 0.00002 + | > step_time: 1.29680 (2.74940) + | > loader_time: 0.00370 (0.04857) + + + --> STEP: 171/234 -- GLOBAL_STEP: 16785 + | > loss: -0.14240 (0.00241) + | > log_mle: -0.38641 (-0.19670) + | > loss_dur: 0.24401 (0.19912) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.72992 (14.30530) + | > current_lr: 0.00002 + | > step_time: 3.49850 (2.75725) + | > loader_time: 0.00420 (0.04829) + + + --> STEP: 176/234 -- GLOBAL_STEP: 16790 + | > loss: -0.11318 (-0.00080) + | > log_mle: -0.35757 (-0.20157) + | > loss_dur: 0.24439 (0.20077) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.98544 (15.03026) + | > current_lr: 0.00002 + | > step_time: 4.39290 (2.81928) + | > loader_time: 0.00400 (0.04955) + + + --> STEP: 181/234 -- GLOBAL_STEP: 16795 + | > loss: -0.04867 (-0.00347) + | > log_mle: -0.29899 (-0.20596) + | > loss_dur: 0.25032 (0.20250) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.63849 (15.64719) + | > current_lr: 0.00002 + | > step_time: 4.01180 (2.86911) + | > loader_time: 0.19080 (0.05314) + + + --> STEP: 186/234 -- GLOBAL_STEP: 16800 + | > loss: -0.06518 (-0.00621) + | > log_mle: -0.33751 (-0.21051) + | > loss_dur: 0.27234 (0.20430) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.33781 (16.32607) + | > current_lr: 0.00002 + | > step_time: 3.32230 (2.87705) + | > loader_time: 0.08490 (0.05226) + + + --> STEP: 191/234 -- GLOBAL_STEP: 16805 + | > loss: -0.11040 (-0.00923) + | > log_mle: -0.34954 (-0.21483) + | > loss_dur: 0.23914 (0.20560) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.95701 (17.01307) + | > current_lr: 0.00002 + | > step_time: 1.69950 (2.84739) + | > loader_time: 0.00880 (0.05152) + + + --> STEP: 196/234 -- GLOBAL_STEP: 16810 + | > loss: -0.08455 (-0.01226) + | > log_mle: -0.34894 (-0.21926) + | > loss_dur: 0.26439 (0.20699) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.85218 (17.68869) + | > current_lr: 0.00002 + | > step_time: 3.40450 (2.87165) + | > loader_time: 0.19320 (0.05223) + + + --> STEP: 201/234 -- GLOBAL_STEP: 16815 + | > loss: -0.04706 (-0.01468) + | > log_mle: -0.31612 (-0.22317) + | > loss_dur: 0.26906 (0.20849) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.41827 (18.67678) + | > current_lr: 0.00002 + | > step_time: 2.68730 (2.88374) + | > loader_time: 0.00840 (0.05234) + + + --> STEP: 206/234 -- GLOBAL_STEP: 16820 + | > loss: -0.14809 (-0.01757) + | > log_mle: -0.41028 (-0.22750) + | > loss_dur: 0.26219 (0.20993) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.48640 (19.80844) + | > current_lr: 0.00002 + | > step_time: 3.40000 (2.89239) + | > loader_time: 0.00480 (0.05295) + + + --> STEP: 211/234 -- GLOBAL_STEP: 16825 + | > loss: -0.18641 (-0.02085) + | > log_mle: -0.47814 (-0.23242) + | > loss_dur: 0.29174 (0.21157) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 63.44854 (20.84282) + | > current_lr: 0.00002 + | > step_time: 8.69530 (2.99488) + | > loader_time: 0.00810 (0.05220) + + + --> STEP: 216/234 -- GLOBAL_STEP: 16830 + | > loss: -0.18422 (-0.02414) + | > log_mle: -0.46458 (-0.23713) + | > loss_dur: 0.28036 (0.21299) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.98602 (21.70007) + | > current_lr: 0.00002 + | > step_time: 8.89040 (3.03290) + | > loader_time: 0.00430 (0.05198) + + + --> STEP: 221/234 -- GLOBAL_STEP: 16835 + | > loss: -0.12153 (-0.02717) + | > log_mle: -0.39112 (-0.24181) + | > loss_dur: 0.26960 (0.21464) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.15044 (22.47198) + | > current_lr: 0.00002 + | > step_time: 2.09850 (3.04209) + | > loader_time: 0.00530 (0.05133) + + + --> STEP: 226/234 -- GLOBAL_STEP: 16840 + | > loss: -0.18606 (-0.03054) + | > log_mle: -0.48397 (-0.24700) + | > loss_dur: 0.29791 (0.21646) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.63693 (23.18722) + | > current_lr: 0.00002 + | > step_time: 1.90400 (3.03152) + | > loader_time: 0.09390 (0.05144) + + + --> STEP: 231/234 -- GLOBAL_STEP: 16845 + | > loss: -0.10164 (-0.03272) + | > log_mle: -0.53609 (-0.25228) + | > loss_dur: 0.43445 (0.21956) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.71497 (24.34492) + | > current_lr: 0.00002 + | > step_time: 0.27480 (2.97150) + | > loader_time: 0.00470 (0.05042) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.24952 (+0.16685) + | > avg_loss: -0.05145 (+0.00931) + | > avg_log_mle: -0.30201 (+0.01144) + | > avg_loss_dur: 0.25056 (-0.00213) + + + > EPOCH: 72/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 00:23:37)  + + --> STEP: 2/234 -- GLOBAL_STEP: 16850 + | > loss: 0.13317 (0.10212) + | > log_mle: -0.10729 (-0.11727) + | > loss_dur: 0.24046 (0.21939) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.02727 (3.73498) + | > current_lr: 0.00002 + | > step_time: 4.39380 (4.14760) + | > loader_time: 0.00320 (5.09132) + + + --> STEP: 7/234 -- GLOBAL_STEP: 16855 + | > loss: 0.02676 (0.07696) + | > log_mle: -0.14630 (-0.13003) + | > loss_dur: 0.17306 (0.20698) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.67708 (5.60683) + | > current_lr: 0.00002 + | > step_time: 7.61490 (4.84416) + | > loader_time: 0.19390 (1.51011) + + + --> STEP: 12/234 -- GLOBAL_STEP: 16860 + | > loss: 0.03393 (0.06388) + | > log_mle: -0.13379 (-0.13369) + | > loss_dur: 0.16772 (0.19757) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.52026 (5.45366) + | > current_lr: 0.00002 + | > step_time: 2.70140 (5.00939) + | > loader_time: 0.09530 (0.95274) + + + --> STEP: 17/234 -- GLOBAL_STEP: 16865 + | > loss: 0.07013 (0.05621) + | > log_mle: -0.11754 (-0.13265) + | > loss_dur: 0.18767 (0.18886) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.14771 (4.98126) + | > current_lr: 0.00002 + | > step_time: 4.99210 (4.88806) + | > loader_time: 0.00160 (0.69061) + + + --> STEP: 22/234 -- GLOBAL_STEP: 16870 + | > loss: 0.01148 (0.05298) + | > log_mle: -0.14003 (-0.13141) + | > loss_dur: 0.15151 (0.18439) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.26900 (4.80215) + | > current_lr: 0.00002 + | > step_time: 1.19850 (4.23956) + | > loader_time: 0.00150 (0.53810) + + + --> STEP: 27/234 -- GLOBAL_STEP: 16875 + | > loss: 0.02849 (0.05047) + | > log_mle: -0.14315 (-0.13152) + | > loss_dur: 0.17163 (0.18198) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.92735 (4.57971) + | > current_lr: 0.00002 + | > step_time: 1.90840 (3.77972) + | > loader_time: 0.00200 (0.44213) + + + --> STEP: 32/234 -- GLOBAL_STEP: 16880 + | > loss: -0.00119 (0.04645) + | > log_mle: -0.15483 (-0.13273) + | > loss_dur: 0.15364 (0.17918) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.83264 (4.43859) + | > current_lr: 0.00002 + | > step_time: 1.30810 (3.55460) + | > loader_time: 0.00220 (0.37650) + + + --> STEP: 37/234 -- GLOBAL_STEP: 16885 + | > loss: 0.02606 (0.04591) + | > log_mle: -0.13435 (-0.13327) + | > loss_dur: 0.16042 (0.17918) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.20646 (4.78611) + | > current_lr: 0.00002 + | > step_time: 1.79670 (3.25766) + | > loader_time: 0.00190 (0.33001) + + + --> STEP: 42/234 -- GLOBAL_STEP: 16890 + | > loss: 0.05690 (0.04624) + | > log_mle: -0.12237 (-0.13330) + | > loss_dur: 0.17926 (0.17954) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.31986 (4.77598) + | > current_lr: 0.00002 + | > step_time: 1.41650 (3.10978) + | > loader_time: 0.08390 (0.29294) + + + --> STEP: 47/234 -- GLOBAL_STEP: 16895 + | > loss: 0.03270 (0.04474) + | > log_mle: -0.13654 (-0.13424) + | > loss_dur: 0.16924 (0.17898) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.22623 (4.89438) + | > current_lr: 0.00002 + | > step_time: 1.05490 (2.96320) + | > loader_time: 0.00200 (0.26407) + + + --> STEP: 52/234 -- GLOBAL_STEP: 16900 + | > loss: 0.05353 (0.04384) + | > log_mle: -0.12967 (-0.13387) + | > loss_dur: 0.18319 (0.17771) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.86359 (4.75968) + | > current_lr: 0.00002 + | > step_time: 2.40760 (2.88786) + | > loader_time: 0.00290 (0.23891) + + + --> STEP: 57/234 -- GLOBAL_STEP: 16905 + | > loss: 0.06072 (0.04366) + | > log_mle: -0.12686 (-0.13469) + | > loss_dur: 0.18758 (0.17835) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.65636 (4.61611) + | > current_lr: 0.00002 + | > step_time: 1.00800 (2.76721) + | > loader_time: 0.00260 (0.21815) + + + --> STEP: 62/234 -- GLOBAL_STEP: 16910 + | > loss: 0.03919 (0.04118) + | > log_mle: -0.18050 (-0.13651) + | > loss_dur: 0.21969 (0.17769) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.59924 (4.78202) + | > current_lr: 0.00002 + | > step_time: 2.70940 (2.69458) + | > loader_time: 0.08660 (0.20344) + + + --> STEP: 67/234 -- GLOBAL_STEP: 16915 + | > loss: 0.01488 (0.04071) + | > log_mle: -0.16249 (-0.13689) + | > loss_dur: 0.17737 (0.17760) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.73128 (4.74677) + | > current_lr: 0.00002 + | > step_time: 2.60070 (2.64325) + | > loader_time: 0.03680 (0.18893) + + + --> STEP: 72/234 -- GLOBAL_STEP: 16920 + | > loss: 0.05545 (0.04105) + | > log_mle: -0.14212 (-0.13739) + | > loss_dur: 0.19757 (0.17845) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.75188 (5.02792) + | > current_lr: 0.00002 + | > step_time: 3.95030 (2.62118) + | > loader_time: 0.20150 (0.17875) + + + --> STEP: 77/234 -- GLOBAL_STEP: 16925 + | > loss: 0.01246 (0.03962) + | > log_mle: -0.15660 (-0.13854) + | > loss_dur: 0.16906 (0.17816) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.52442 (5.14759) + | > current_lr: 0.00002 + | > step_time: 1.98800 (2.56148) + | > loader_time: 0.00160 (0.16729) + + + --> STEP: 82/234 -- GLOBAL_STEP: 16930 + | > loss: 0.01812 (0.03848) + | > log_mle: -0.14529 (-0.13909) + | > loss_dur: 0.16341 (0.17756) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.43681 (5.20124) + | > current_lr: 0.00002 + | > step_time: 2.05490 (2.52035) + | > loader_time: 0.09720 (0.15838) + + + --> STEP: 87/234 -- GLOBAL_STEP: 16935 + | > loss: 0.03221 (0.03792) + | > log_mle: -0.15745 (-0.14022) + | > loss_dur: 0.18966 (0.17814) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.96372 (5.49020) + | > current_lr: 0.00002 + | > step_time: 1.31300 (2.48700) + | > loader_time: 0.00180 (0.15037) + + + --> STEP: 92/234 -- GLOBAL_STEP: 16940 + | > loss: -0.03204 (0.03585) + | > log_mle: -0.19929 (-0.14282) + | > loss_dur: 0.16724 (0.17867) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.02888 (5.83077) + | > current_lr: 0.00002 + | > step_time: 2.11420 (2.46442) + | > loader_time: 0.08840 (0.14427) + + + --> STEP: 97/234 -- GLOBAL_STEP: 16945 + | > loss: -0.00664 (0.03344) + | > log_mle: -0.19048 (-0.14631) + | > loss_dur: 0.18384 (0.17975) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.04526 (6.27756) + | > current_lr: 0.00002 + | > step_time: 2.35450 (2.47006) + | > loader_time: 0.00250 (0.13868) + + + --> STEP: 102/234 -- GLOBAL_STEP: 16950 + | > loss: 0.02433 (0.03189) + | > log_mle: -0.17056 (-0.14870) + | > loss_dur: 0.19489 (0.18060) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.06605 (6.58257) + | > current_lr: 0.00002 + | > step_time: 1.02160 (2.43995) + | > loader_time: 0.00360 (0.13291) + + + --> STEP: 107/234 -- GLOBAL_STEP: 16955 + | > loss: -0.01011 (0.02975) + | > log_mle: -0.21938 (-0.15208) + | > loss_dur: 0.20927 (0.18183) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.06011 (6.97721) + | > current_lr: 0.00002 + | > step_time: 2.40170 (2.46711) + | > loader_time: 0.00290 (0.12769) + + + --> STEP: 112/234 -- GLOBAL_STEP: 16960 + | > loss: -0.01249 (0.02817) + | > log_mle: -0.22603 (-0.15511) + | > loss_dur: 0.21354 (0.18328) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.55330 (7.32741) + | > current_lr: 0.00002 + | > step_time: 1.60540 (2.44825) + | > loader_time: 0.08350 (0.12332) + + + --> STEP: 117/234 -- GLOBAL_STEP: 16965 + | > loss: -0.02309 (0.02660) + | > log_mle: -0.21867 (-0.15796) + | > loss_dur: 0.19557 (0.18456) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.09937 (7.73240) + | > current_lr: 0.00002 + | > step_time: 1.21050 (2.44080) + | > loader_time: 0.00240 (0.11972) + + + --> STEP: 122/234 -- GLOBAL_STEP: 16970 + | > loss: -0.01059 (0.02556) + | > log_mle: -0.20013 (-0.15980) + | > loss_dur: 0.18954 (0.18536) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.42846 (7.93044) + | > current_lr: 0.00002 + | > step_time: 3.59350 (2.46708) + | > loader_time: 0.09360 (0.11722) + + + --> STEP: 127/234 -- GLOBAL_STEP: 16975 + | > loss: -0.03392 (0.02335) + | > log_mle: -0.25051 (-0.16284) + | > loss_dur: 0.21659 (0.18619) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.33120 (8.34441) + | > current_lr: 0.00002 + | > step_time: 2.59900 (2.43690) + | > loader_time: 0.00330 (0.11419) + + + --> STEP: 132/234 -- GLOBAL_STEP: 16980 + | > loss: -0.04405 (0.02095) + | > log_mle: -0.23309 (-0.16619) + | > loss_dur: 0.18904 (0.18714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.34012 (8.77651) + | > current_lr: 0.00002 + | > step_time: 2.29710 (2.40676) + | > loader_time: 0.08830 (0.11189) + + + --> STEP: 137/234 -- GLOBAL_STEP: 16985 + | > loss: -0.01121 (0.01903) + | > log_mle: -0.24449 (-0.16978) + | > loss_dur: 0.23328 (0.18881) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.44818 (9.12275) + | > current_lr: 0.00002 + | > step_time: 4.29680 (2.42381) + | > loader_time: 0.08920 (0.10918) + + + --> STEP: 142/234 -- GLOBAL_STEP: 16990 + | > loss: -0.03702 (0.01696) + | > log_mle: -0.25938 (-0.17295) + | > loss_dur: 0.22236 (0.18991) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.38865 (9.53777) + | > current_lr: 0.00002 + | > step_time: 1.80450 (2.51038) + | > loader_time: 0.08620 (0.11071) + + + --> STEP: 147/234 -- GLOBAL_STEP: 16995 + | > loss: -0.04187 (0.01400) + | > log_mle: -0.26206 (-0.17756) + | > loss_dur: 0.22019 (0.19156) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.09223 (10.21532) + | > current_lr: 0.00002 + | > step_time: 2.19370 (2.50813) + | > loader_time: 0.00550 (0.10818) + + + --> STEP: 152/234 -- GLOBAL_STEP: 17000 + | > loss: -0.08138 (0.01102) + | > log_mle: -0.33393 (-0.18178) + | > loss_dur: 0.25256 (0.19280) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.48103 (10.81992) + | > current_lr: 0.00002 + | > step_time: 2.00570 (2.49057) + | > loader_time: 0.00260 (0.10590) + + + --> STEP: 157/234 -- GLOBAL_STEP: 17005 + | > loss: -0.05713 (0.00772) + | > log_mle: -0.28534 (-0.18651) + | > loss_dur: 0.22820 (0.19423) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.64603 (11.71830) + | > current_lr: 0.00002 + | > step_time: 1.50450 (2.54054) + | > loader_time: 0.08060 (0.10380) + + + --> STEP: 162/234 -- GLOBAL_STEP: 17010 + | > loss: -0.09671 (0.00444) + | > log_mle: -0.31836 (-0.19102) + | > loss_dur: 0.22165 (0.19547) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.96545 (12.43220) + | > current_lr: 0.00002 + | > step_time: 5.72050 (2.57784) + | > loader_time: 0.08830 (0.10175) + + + --> STEP: 167/234 -- GLOBAL_STEP: 17015 + | > loss: -0.14825 (0.00150) + | > log_mle: -0.39024 (-0.19520) + | > loss_dur: 0.24199 (0.19670) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.46419 (13.00956) + | > current_lr: 0.00002 + | > step_time: 1.29580 (2.56211) + | > loader_time: 0.00400 (0.09992) + + + --> STEP: 172/234 -- GLOBAL_STEP: 17020 + | > loss: -0.11147 (-0.00152) + | > log_mle: -0.38361 (-0.20012) + | > loss_dur: 0.27214 (0.19860) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.03147 (13.68369) + | > current_lr: 0.00002 + | > step_time: 1.19650 (2.53359) + | > loader_time: 0.00300 (0.09762) + + + --> STEP: 177/234 -- GLOBAL_STEP: 17025 + | > loss: -0.08453 (-0.00446) + | > log_mle: -0.34532 (-0.20472) + | > loss_dur: 0.26079 (0.20026) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.74808 (14.54693) + | > current_lr: 0.00002 + | > step_time: 4.01360 (2.54802) + | > loader_time: 0.09540 (0.09713) + + + --> STEP: 182/234 -- GLOBAL_STEP: 17030 + | > loss: -0.11166 (-0.00723) + | > log_mle: -0.38669 (-0.20928) + | > loss_dur: 0.27502 (0.20206) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.00484 (15.35741) + | > current_lr: 0.00002 + | > step_time: 4.91550 (2.61876) + | > loader_time: 0.18940 (0.09881) + + + --> STEP: 187/234 -- GLOBAL_STEP: 17035 + | > loss: -0.13779 (-0.01006) + | > log_mle: -0.38724 (-0.21376) + | > loss_dur: 0.24944 (0.20371) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.98263 (16.09560) + | > current_lr: 0.00002 + | > step_time: 8.70400 (2.68670) + | > loader_time: 0.19610 (0.09875) + + + --> STEP: 192/234 -- GLOBAL_STEP: 17040 + | > loss: -0.15967 (-0.01313) + | > log_mle: -0.40683 (-0.21815) + | > loss_dur: 0.24716 (0.20502) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.10362 (16.85375) + | > current_lr: 0.00002 + | > step_time: 4.60050 (2.76724) + | > loader_time: 0.09150 (0.09870) + + + --> STEP: 197/234 -- GLOBAL_STEP: 17045 + | > loss: -0.13764 (-0.01593) + | > log_mle: -0.38505 (-0.22238) + | > loss_dur: 0.24740 (0.20644) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.93993 (17.66829) + | > current_lr: 0.00002 + | > step_time: 5.59740 (2.77411) + | > loader_time: 0.11670 (0.09738) + + + --> STEP: 202/234 -- GLOBAL_STEP: 17050 + | > loss: -0.19962 (-0.01868) + | > log_mle: -0.46502 (-0.22666) + | > loss_dur: 0.26539 (0.20798) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 67.46283 (18.49897) + | > current_lr: 0.00002 + | > step_time: 5.50610 (2.82572) + | > loader_time: 0.00440 (0.09649) + + + --> STEP: 207/234 -- GLOBAL_STEP: 17055 + | > loss: -0.18061 (-0.02143) + | > log_mle: -0.45276 (-0.23096) + | > loss_dur: 0.27215 (0.20953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.52081 (19.13274) + | > current_lr: 0.00002 + | > step_time: 8.61080 (2.87390) + | > loader_time: 0.08560 (0.09601) + + + --> STEP: 212/234 -- GLOBAL_STEP: 17060 + | > loss: -0.15911 (-0.02461) + | > log_mle: -0.43888 (-0.23580) + | > loss_dur: 0.27976 (0.21119) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.73891 (19.86616) + | > current_lr: 0.00002 + | > step_time: 5.80680 (2.98302) + | > loader_time: 0.09020 (0.09649) + + + --> STEP: 217/234 -- GLOBAL_STEP: 17065 + | > loss: -0.16969 (-0.02788) + | > log_mle: -0.45484 (-0.24056) + | > loss_dur: 0.28515 (0.21268) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.09674 (20.75343) + | > current_lr: 0.00002 + | > step_time: 2.39710 (3.04285) + | > loader_time: 0.08770 (0.09833) + + + --> STEP: 222/234 -- GLOBAL_STEP: 17070 + | > loss: -0.15742 (-0.03087) + | > log_mle: -0.46622 (-0.24521) + | > loss_dur: 0.30880 (0.21435) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.18007 (21.66721) + | > current_lr: 0.00002 + | > step_time: 0.37890 (3.04860) + | > loader_time: 0.00350 (0.09796) + + + --> STEP: 227/234 -- GLOBAL_STEP: 17075 + | > loss: -0.13850 (-0.03410) + | > log_mle: -0.43820 (-0.25021) + | > loss_dur: 0.29969 (0.21611) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.19360 (22.62555) + | > current_lr: 0.00002 + | > step_time: 0.24130 (2.98685) + | > loader_time: 0.00310 (0.09589) + + + --> STEP: 232/234 -- GLOBAL_STEP: 17080 + | > loss: -0.05685 (-0.03602) + | > log_mle: -0.62177 (-0.25633) + | > loss_dur: 0.56492 (0.22031) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 96.70032 (23.79493) + | > current_lr: 0.00002 + | > step_time: 0.40870 (2.92875) + | > loader_time: 0.02670 (0.09401) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.76960 (+0.52008) + | > avg_loss: -0.06407 (-0.01262) + | > avg_log_mle: -0.31497 (-0.01296) + | > avg_loss_dur: 0.25090 (+0.00033) + + + > EPOCH: 73/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 00:36:08)  + + --> STEP: 3/234 -- GLOBAL_STEP: 17085 + | > loss: 0.11552 (0.10561) + | > log_mle: -0.13376 (-0.12470) + | > loss_dur: 0.24927 (0.23032) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.30745 (4.57738) + | > current_lr: 0.00002 + | > step_time: 1.48230 (1.52662) + | > loader_time: 0.00080 (0.00284) + + + --> STEP: 8/234 -- GLOBAL_STEP: 17090 + | > loss: 0.05776 (0.07075) + | > log_mle: -0.14541 (-0.13356) + | > loss_dur: 0.20317 (0.20431) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.20401 (6.34184) + | > current_lr: 0.00002 + | > step_time: 8.70190 (4.72168) + | > loader_time: 0.09390 (0.16141) + + + --> STEP: 13/234 -- GLOBAL_STEP: 17095 + | > loss: 0.06043 (0.06254) + | > log_mle: -0.12781 (-0.13462) + | > loss_dur: 0.18824 (0.19716) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.98328 (6.20966) + | > current_lr: 0.00002 + | > step_time: 9.58940 (5.42755) + | > loader_time: 0.00560 (0.12264) + + + --> STEP: 18/234 -- GLOBAL_STEP: 17100 + | > loss: 0.05935 (0.05645) + | > log_mle: -0.13460 (-0.13410) + | > loss_dur: 0.19395 (0.19054) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.38585 (5.64545) + | > current_lr: 0.00002 + | > step_time: 1.90540 (4.50811) + | > loader_time: 0.08710 (0.09860) + + + --> STEP: 23/234 -- GLOBAL_STEP: 17105 + | > loss: 0.03425 (0.05263) + | > log_mle: -0.13839 (-0.13335) + | > loss_dur: 0.17265 (0.18599) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.87284 (5.23808) + | > current_lr: 0.00002 + | > step_time: 0.73890 (4.85932) + | > loader_time: 0.00120 (0.09012) + + + --> STEP: 28/234 -- GLOBAL_STEP: 17110 + | > loss: 0.01140 (0.04995) + | > log_mle: -0.13052 (-0.13318) + | > loss_dur: 0.14192 (0.18313) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.65122 (4.89685) + | > current_lr: 0.00002 + | > step_time: 1.60090 (4.18255) + | > loader_time: 0.00250 (0.07438) + + + --> STEP: 33/234 -- GLOBAL_STEP: 17115 + | > loss: 0.05223 (0.04778) + | > log_mle: -0.12613 (-0.13429) + | > loss_dur: 0.17837 (0.18207) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.55447 (4.68739) + | > current_lr: 0.00002 + | > step_time: 3.59800 (3.91504) + | > loader_time: 0.11110 (0.06925) + + + --> STEP: 38/234 -- GLOBAL_STEP: 17120 + | > loss: 0.05542 (0.04609) + | > log_mle: -0.14541 (-0.13542) + | > loss_dur: 0.20083 (0.18151) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.54372 (4.81883) + | > current_lr: 0.00002 + | > step_time: 3.33590 (4.02033) + | > loader_time: 0.10440 (0.07045) + + + --> STEP: 43/234 -- GLOBAL_STEP: 17125 + | > loss: 0.03031 (0.04593) + | > log_mle: -0.14573 (-0.13548) + | > loss_dur: 0.17604 (0.18141) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.75234 (4.87284) + | > current_lr: 0.00002 + | > step_time: 1.98960 (3.70478) + | > loader_time: 0.00250 (0.06254) + + + --> STEP: 48/234 -- GLOBAL_STEP: 17130 + | > loss: 0.01817 (0.04399) + | > log_mle: -0.13064 (-0.13607) + | > loss_dur: 0.14881 (0.18006) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.82217 (4.84786) + | > current_lr: 0.00002 + | > step_time: 2.24680 (3.46585) + | > loader_time: 0.00160 (0.05626) + + + --> STEP: 53/234 -- GLOBAL_STEP: 17135 + | > loss: 0.03921 (0.04320) + | > log_mle: -0.15200 (-0.13614) + | > loss_dur: 0.19121 (0.17934) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.61673 (4.70681) + | > current_lr: 0.00002 + | > step_time: 3.38690 (3.33577) + | > loader_time: 0.00170 (0.05254) + + + --> STEP: 58/234 -- GLOBAL_STEP: 17140 + | > loss: 0.01569 (0.04227) + | > log_mle: -0.13676 (-0.13666) + | > loss_dur: 0.15245 (0.17893) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.70705 (4.56712) + | > current_lr: 0.00002 + | > step_time: 2.68850 (3.19236) + | > loader_time: 0.07390 (0.04946) + + + --> STEP: 63/234 -- GLOBAL_STEP: 17145 + | > loss: 0.04496 (0.04018) + | > log_mle: -0.14998 (-0.13870) + | > loss_dur: 0.19493 (0.17888) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.49615 (4.71427) + | > current_lr: 0.00002 + | > step_time: 2.22800 (3.09813) + | > loader_time: 0.00200 (0.04736) + + + --> STEP: 68/234 -- GLOBAL_STEP: 17150 + | > loss: 0.04620 (0.03946) + | > log_mle: -0.14156 (-0.13898) + | > loss_dur: 0.18776 (0.17844) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.53092 (4.64932) + | > current_lr: 0.00002 + | > step_time: 2.68510 (3.01951) + | > loader_time: 0.00200 (0.04408) + + + --> STEP: 73/234 -- GLOBAL_STEP: 17155 + | > loss: 0.02331 (0.03942) + | > log_mle: -0.16478 (-0.13964) + | > loss_dur: 0.18809 (0.17907) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.14583 (5.01815) + | > current_lr: 0.00002 + | > step_time: 3.20730 (2.94006) + | > loader_time: 0.00370 (0.04235) + + + --> STEP: 78/234 -- GLOBAL_STEP: 17160 + | > loss: 0.04163 (0.03823) + | > log_mle: -0.14079 (-0.14042) + | > loss_dur: 0.18242 (0.17864) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.47202 (5.11544) + | > current_lr: 0.00002 + | > step_time: 1.41740 (2.86861) + | > loader_time: 0.08560 (0.04323) + + + --> STEP: 83/234 -- GLOBAL_STEP: 17165 + | > loss: 0.03279 (0.03691) + | > log_mle: -0.16833 (-0.14129) + | > loss_dur: 0.20112 (0.17819) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.46304 (5.27782) + | > current_lr: 0.00002 + | > step_time: 2.09420 (2.80724) + | > loader_time: 0.00310 (0.04079) + + + --> STEP: 88/234 -- GLOBAL_STEP: 17170 + | > loss: -0.01070 (0.03584) + | > log_mle: -0.20431 (-0.14277) + | > loss_dur: 0.19361 (0.17861) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.68798 (5.49759) + | > current_lr: 0.00002 + | > step_time: 2.02590 (2.75717) + | > loader_time: 0.00290 (0.04070) + + + --> STEP: 93/234 -- GLOBAL_STEP: 17175 + | > loss: -0.01176 (0.03395) + | > log_mle: -0.21574 (-0.14542) + | > loss_dur: 0.20398 (0.17937) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.84659 (5.89171) + | > current_lr: 0.00002 + | > step_time: 2.00670 (2.70454) + | > loader_time: 0.00290 (0.03968) + + + --> STEP: 98/234 -- GLOBAL_STEP: 17180 + | > loss: 0.04779 (0.03215) + | > log_mle: -0.14496 (-0.14807) + | > loss_dur: 0.19275 (0.18022) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.08559 (6.29166) + | > current_lr: 0.00002 + | > step_time: 1.27650 (2.66333) + | > loader_time: 0.00230 (0.03784) + + + --> STEP: 103/234 -- GLOBAL_STEP: 17185 + | > loss: -0.02532 (0.02984) + | > log_mle: -0.23997 (-0.15136) + | > loss_dur: 0.21464 (0.18121) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.95333 (6.75910) + | > current_lr: 0.00002 + | > step_time: 3.71790 (2.65178) + | > loader_time: 0.09030 (0.03892) + + + --> STEP: 108/234 -- GLOBAL_STEP: 17190 + | > loss: -0.00096 (0.02810) + | > log_mle: -0.18933 (-0.15419) + | > loss_dur: 0.18837 (0.18229) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.87671 (7.13364) + | > current_lr: 0.00002 + | > step_time: 3.50240 (2.64840) + | > loader_time: 0.00670 (0.03892) + + + --> STEP: 113/234 -- GLOBAL_STEP: 17195 + | > loss: -0.03356 (0.02623) + | > log_mle: -0.23556 (-0.15756) + | > loss_dur: 0.20200 (0.18379) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.51698 (7.64461) + | > current_lr: 0.00002 + | > step_time: 1.70120 (2.61882) + | > loader_time: 0.00510 (0.03828) + + + --> STEP: 118/234 -- GLOBAL_STEP: 17200 + | > loss: -0.00199 (0.02498) + | > log_mle: -0.20465 (-0.16009) + | > loss_dur: 0.20265 (0.18507) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.56044 (7.98264) + | > current_lr: 0.00002 + | > step_time: 2.10180 (2.57901) + | > loader_time: 0.08470 (0.03887) + + + --> STEP: 123/234 -- GLOBAL_STEP: 17205 + | > loss: 0.00446 (0.02390) + | > log_mle: -0.17557 (-0.16170) + | > loss_dur: 0.18003 (0.18560) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.43013 (8.23406) + | > current_lr: 0.00002 + | > step_time: 2.60460 (2.58013) + | > loader_time: 0.00390 (0.03949) + + + --> STEP: 128/234 -- GLOBAL_STEP: 17210 + | > loss: -0.04586 (0.02127) + | > log_mle: -0.23500 (-0.16519) + | > loss_dur: 0.18914 (0.18646) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.02391 (8.61267) + | > current_lr: 0.00002 + | > step_time: 3.70010 (2.58018) + | > loader_time: 0.09670 (0.03948) + + + --> STEP: 133/234 -- GLOBAL_STEP: 17215 + | > loss: -0.03728 (0.01896) + | > log_mle: -0.25633 (-0.16864) + | > loss_dur: 0.21905 (0.18759) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.94586 (9.14152) + | > current_lr: 0.00002 + | > step_time: 3.30550 (2.55483) + | > loader_time: 0.00300 (0.03943) + + + --> STEP: 138/234 -- GLOBAL_STEP: 17220 + | > loss: -0.01287 (0.01709) + | > log_mle: -0.21430 (-0.17184) + | > loss_dur: 0.20143 (0.18893) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.56487 (9.64515) + | > current_lr: 0.00002 + | > step_time: 1.19020 (2.56923) + | > loader_time: 0.00270 (0.03963) + + + --> STEP: 143/234 -- GLOBAL_STEP: 17225 + | > loss: -0.06141 (0.01467) + | > log_mle: -0.33553 (-0.17578) + | > loss_dur: 0.27412 (0.19045) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.50129 (10.32929) + | > current_lr: 0.00002 + | > step_time: 2.40340 (2.54237) + | > loader_time: 0.08430 (0.03893) + + + --> STEP: 148/234 -- GLOBAL_STEP: 17230 + | > loss: -0.06052 (0.01193) + | > log_mle: -0.26393 (-0.17978) + | > loss_dur: 0.20341 (0.19171) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.82890 (10.93145) + | > current_lr: 0.00002 + | > step_time: 1.60920 (2.51519) + | > loader_time: 0.09720 (0.03837) + + + --> STEP: 153/234 -- GLOBAL_STEP: 17235 + | > loss: -0.13253 (0.00864) + | > log_mle: -0.37423 (-0.18466) + | > loss_dur: 0.24170 (0.19330) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.34854 (11.78126) + | > current_lr: 0.00002 + | > step_time: 1.88920 (2.51015) + | > loader_time: 0.00420 (0.03723) + + + --> STEP: 158/234 -- GLOBAL_STEP: 17240 + | > loss: -0.07018 (0.00570) + | > log_mle: -0.31797 (-0.18895) + | > loss_dur: 0.24779 (0.19465) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.01419 (12.51976) + | > current_lr: 0.00002 + | > step_time: 4.91450 (2.56960) + | > loader_time: 0.19910 (0.04150) + + + --> STEP: 163/234 -- GLOBAL_STEP: 17245 + | > loss: -0.06852 (0.00252) + | > log_mle: -0.29336 (-0.19324) + | > loss_dur: 0.22484 (0.19576) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.10420 (13.17023) + | > current_lr: 0.00002 + | > step_time: 2.80020 (2.58347) + | > loader_time: 0.08000 (0.04203) + + + --> STEP: 168/234 -- GLOBAL_STEP: 17250 + | > loss: -0.08329 (-0.00042) + | > log_mle: -0.34177 (-0.19758) + | > loss_dur: 0.25848 (0.19716) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.49389 (13.95333) + | > current_lr: 0.00002 + | > step_time: 4.28920 (2.61433) + | > loader_time: 0.10580 (0.04300) + + + --> STEP: 173/234 -- GLOBAL_STEP: 17255 + | > loss: -0.10370 (-0.00364) + | > log_mle: -0.35029 (-0.20249) + | > loss_dur: 0.24659 (0.19885) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.75332 (14.55822) + | > current_lr: 0.00002 + | > step_time: 2.90660 (2.65621) + | > loader_time: 0.09780 (0.04294) + + + --> STEP: 178/234 -- GLOBAL_STEP: 17260 + | > loss: -0.12473 (-0.00681) + | > log_mle: -0.40301 (-0.20734) + | > loss_dur: 0.27828 (0.20053) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.79483 (15.43842) + | > current_lr: 0.00002 + | > step_time: 4.50720 (2.65593) + | > loader_time: 0.08620 (0.04326) + + + --> STEP: 183/234 -- GLOBAL_STEP: 17265 + | > loss: -0.13754 (-0.00970) + | > log_mle: -0.39897 (-0.21183) + | > loss_dur: 0.26144 (0.20213) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.85765 (16.15057) + | > current_lr: 0.00002 + | > step_time: 6.10470 (2.70589) + | > loader_time: 0.09120 (0.04354) + + + --> STEP: 188/234 -- GLOBAL_STEP: 17270 + | > loss: -0.14630 (-0.01253) + | > log_mle: -0.41354 (-0.21636) + | > loss_dur: 0.26724 (0.20384) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.16798 (16.75150) + | > current_lr: 0.00002 + | > step_time: 2.40020 (2.72011) + | > loader_time: 0.00440 (0.04353) + + + --> STEP: 193/234 -- GLOBAL_STEP: 17275 + | > loss: -0.14562 (-0.01563) + | > log_mle: -0.41319 (-0.22074) + | > loss_dur: 0.26758 (0.20511) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.40134 (17.36377) + | > current_lr: 0.00002 + | > step_time: 2.89630 (2.78758) + | > loader_time: 0.00330 (0.04548) + + + --> STEP: 198/234 -- GLOBAL_STEP: 17280 + | > loss: -0.13683 (-0.01841) + | > log_mle: -0.40266 (-0.22494) + | > loss_dur: 0.26583 (0.20653) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.57386 (17.93538) + | > current_lr: 0.00002 + | > step_time: 5.28480 (2.81719) + | > loader_time: 0.11690 (0.04648) + + + --> STEP: 203/234 -- GLOBAL_STEP: 17285 + | > loss: -0.08535 (-0.02090) + | > log_mle: -0.34438 (-0.22894) + | > loss_dur: 0.25903 (0.20804) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.39381 (18.61516) + | > current_lr: 0.00002 + | > step_time: 5.30310 (2.85320) + | > loader_time: 0.08440 (0.04678) + + + --> STEP: 208/234 -- GLOBAL_STEP: 17290 + | > loss: -0.14322 (-0.02400) + | > log_mle: -0.41697 (-0.23361) + | > loss_dur: 0.27375 (0.20960) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.65948 (19.38102) + | > current_lr: 0.00002 + | > step_time: 4.70060 (2.94992) + | > loader_time: 0.19560 (0.04898) + + + --> STEP: 213/234 -- GLOBAL_STEP: 17295 + | > loss: -0.17669 (-0.02734) + | > log_mle: -0.45940 (-0.23861) + | > loss_dur: 0.28271 (0.21126) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 68.65417 (20.34737) + | > current_lr: 0.00002 + | > step_time: 3.39650 (3.02296) + | > loader_time: 0.00520 (0.04967) + + + --> STEP: 218/234 -- GLOBAL_STEP: 17300 + | > loss: -0.13795 (-0.03049) + | > log_mle: -0.42589 (-0.24318) + | > loss_dur: 0.28794 (0.21269) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.47712 (21.06873) + | > current_lr: 0.00002 + | > step_time: 9.19700 (3.08525) + | > loader_time: 0.00370 (0.04990) + + + --> STEP: 223/234 -- GLOBAL_STEP: 17305 + | > loss: -0.17559 (-0.03366) + | > log_mle: -0.46479 (-0.24804) + | > loss_dur: 0.28919 (0.21438) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.66079 (21.71692) + | > current_lr: 0.00002 + | > step_time: 0.23740 (3.05321) + | > loader_time: 0.00370 (0.04967) + + + --> STEP: 228/234 -- GLOBAL_STEP: 17310 + | > loss: -0.15295 (-0.03683) + | > log_mle: -0.46201 (-0.25303) + | > loss_dur: 0.30906 (0.21620) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.59285 (22.58119) + | > current_lr: 0.00002 + | > step_time: 0.25140 (2.99153) + | > loader_time: 0.00350 (0.04866) + + + --> STEP: 233/234 -- GLOBAL_STEP: 17315 + | > loss: 0.48273 (-0.03608) + | > log_mle: -0.42546 (-0.25898) + | > loss_dur: 0.90818 (0.22290) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.86013 (23.63933) + | > current_lr: 0.00002 + | > step_time: 0.19630 (2.93331) + | > loader_time: 0.00300 (0.04780) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.98351 (+0.21391) + | > avg_loss: -0.06832 (-0.00425) + | > avg_log_mle: -0.31915 (-0.00418) + | > avg_loss_dur: 0.25083 (-0.00006) + + + > EPOCH: 74/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 00:48:48)  + + --> STEP: 4/234 -- GLOBAL_STEP: 17320 + | > loss: 0.07691 (0.08037) + | > log_mle: -0.14150 (-0.13049) + | > loss_dur: 0.21841 (0.21086) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.29106 (5.97094) + | > current_lr: 0.00002 + | > step_time: 9.81250 (9.35591) + | > loader_time: 0.00310 (0.04898) + + + --> STEP: 9/234 -- GLOBAL_STEP: 17325 + | > loss: 0.05333 (0.06373) + | > log_mle: -0.15123 (-0.13734) + | > loss_dur: 0.20456 (0.20107) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.51015 (5.14582) + | > current_lr: 0.00002 + | > step_time: 2.70540 (5.32281) + | > loader_time: 0.19180 (0.04369) + + + --> STEP: 14/234 -- GLOBAL_STEP: 17330 + | > loss: 0.01377 (0.05400) + | > log_mle: -0.14303 (-0.13730) + | > loss_dur: 0.15681 (0.19131) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.60873 (5.02862) + | > current_lr: 0.00002 + | > step_time: 1.51110 (4.20792) + | > loader_time: 0.08010 (0.03472) + + + --> STEP: 19/234 -- GLOBAL_STEP: 17335 + | > loss: 0.03244 (0.04788) + | > log_mle: -0.12846 (-0.13590) + | > loss_dur: 0.16089 (0.18378) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.89075 (4.73686) + | > current_lr: 0.00002 + | > step_time: 0.79130 (3.43489) + | > loader_time: 0.00170 (0.02609) + + + --> STEP: 24/234 -- GLOBAL_STEP: 17340 + | > loss: 0.01060 (0.04366) + | > log_mle: -0.13157 (-0.13533) + | > loss_dur: 0.14217 (0.17899) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.84794 (4.56874) + | > current_lr: 0.00002 + | > step_time: 3.91200 (3.57434) + | > loader_time: 0.18490 (0.04018) + + + --> STEP: 29/234 -- GLOBAL_STEP: 17345 + | > loss: 0.03415 (0.04240) + | > log_mle: -0.12919 (-0.13508) + | > loss_dur: 0.16334 (0.17748) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.63750 (4.37351) + | > current_lr: 0.00002 + | > step_time: 3.41050 (3.66174) + | > loader_time: 0.10510 (0.04031) + + + --> STEP: 34/234 -- GLOBAL_STEP: 17350 + | > loss: 0.05190 (0.04115) + | > log_mle: -0.13784 (-0.13633) + | > loss_dur: 0.18975 (0.17748) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.31737 (4.26372) + | > current_lr: 0.00002 + | > step_time: 2.61030 (3.49049) + | > loader_time: 0.10320 (0.04325) + + + --> STEP: 39/234 -- GLOBAL_STEP: 17355 + | > loss: 0.04337 (0.04065) + | > log_mle: -0.14602 (-0.13755) + | > loss_dur: 0.18939 (0.17819) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.27789 (4.58314) + | > current_lr: 0.00002 + | > step_time: 1.70610 (3.24030) + | > loader_time: 0.00190 (0.03798) + + + --> STEP: 44/234 -- GLOBAL_STEP: 17360 + | > loss: 0.02567 (0.04011) + | > log_mle: -0.13527 (-0.13722) + | > loss_dur: 0.16094 (0.17732) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.52941 (4.64279) + | > current_lr: 0.00002 + | > step_time: 1.17900 (3.00055) + | > loader_time: 0.08890 (0.03586) + + + --> STEP: 49/234 -- GLOBAL_STEP: 17365 + | > loss: 0.00687 (0.03817) + | > log_mle: -0.14479 (-0.13798) + | > loss_dur: 0.15166 (0.17615) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.19253 (4.62081) + | > current_lr: 0.00002 + | > step_time: 1.06450 (2.82291) + | > loader_time: 0.00230 (0.03415) + + + --> STEP: 54/234 -- GLOBAL_STEP: 17370 + | > loss: 0.01104 (0.03795) + | > log_mle: -0.15000 (-0.13807) + | > loss_dur: 0.16104 (0.17602) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.79944 (4.51492) + | > current_lr: 0.00002 + | > step_time: 1.00630 (2.68942) + | > loader_time: 0.00200 (0.03126) + + + --> STEP: 59/234 -- GLOBAL_STEP: 17375 + | > loss: -0.01808 (0.03679) + | > log_mle: -0.16673 (-0.13877) + | > loss_dur: 0.14864 (0.17556) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.86930 (4.46322) + | > current_lr: 0.00002 + | > step_time: 2.20130 (2.63772) + | > loader_time: 0.00240 (0.03166) + + + --> STEP: 64/234 -- GLOBAL_STEP: 17380 + | > loss: 0.02517 (0.03609) + | > log_mle: -0.13400 (-0.14012) + | > loss_dur: 0.15916 (0.17622) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.27186 (4.61889) + | > current_lr: 0.00002 + | > step_time: 2.30390 (2.56265) + | > loader_time: 0.00260 (0.02947) + + + --> STEP: 69/234 -- GLOBAL_STEP: 17385 + | > loss: 0.06091 (0.03587) + | > log_mle: -0.12332 (-0.14022) + | > loss_dur: 0.18423 (0.17608) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.50572 (4.63683) + | > current_lr: 0.00002 + | > step_time: 2.00580 (2.53226) + | > loader_time: 0.08320 (0.03005) + + + --> STEP: 74/234 -- GLOBAL_STEP: 17390 + | > loss: 0.00215 (0.03539) + | > log_mle: -0.14702 (-0.14124) + | > loss_dur: 0.14916 (0.17663) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.66651 (5.22663) + | > current_lr: 0.00002 + | > step_time: 2.30460 (2.52639) + | > loader_time: 0.00330 (0.03049) + + + --> STEP: 79/234 -- GLOBAL_STEP: 17395 + | > loss: 0.00855 (0.03415) + | > log_mle: -0.15615 (-0.14211) + | > loss_dur: 0.16470 (0.17626) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.24247 (5.29627) + | > current_lr: 0.00002 + | > step_time: 1.08560 (2.47466) + | > loader_time: 0.00220 (0.03081) + + + --> STEP: 84/234 -- GLOBAL_STEP: 17400 + | > loss: 0.02794 (0.03357) + | > log_mle: -0.15216 (-0.14280) + | > loss_dur: 0.18010 (0.17637) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.90374 (5.75333) + | > current_lr: 0.00002 + | > step_time: 4.21000 (2.48919) + | > loader_time: 0.00370 (0.02919) + + + --> STEP: 89/234 -- GLOBAL_STEP: 17405 + | > loss: -0.01142 (0.03204) + | > log_mle: -0.18313 (-0.14453) + | > loss_dur: 0.17171 (0.17657) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.61084 (5.98839) + | > current_lr: 0.00002 + | > step_time: 1.50300 (2.43463) + | > loader_time: 0.08760 (0.03074) + + + --> STEP: 94/234 -- GLOBAL_STEP: 17410 + | > loss: -0.02240 (0.03014) + | > log_mle: -0.21830 (-0.14750) + | > loss_dur: 0.19589 (0.17763) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.75972 (6.39386) + | > current_lr: 0.00002 + | > step_time: 7.80040 (2.49791) + | > loader_time: 0.10590 (0.03415) + + + --> STEP: 99/234 -- GLOBAL_STEP: 17415 + | > loss: -0.04067 (0.02830) + | > log_mle: -0.25102 (-0.15036) + | > loss_dur: 0.21035 (0.17866) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.67595 (6.83258) + | > current_lr: 0.00002 + | > step_time: 1.31430 (2.44984) + | > loader_time: 0.00190 (0.03337) + + + --> STEP: 104/234 -- GLOBAL_STEP: 17420 + | > loss: -0.05570 (0.02616) + | > log_mle: -0.26347 (-0.15369) + | > loss_dur: 0.20777 (0.17986) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.18555 (7.20509) + | > current_lr: 0.00002 + | > step_time: 1.20760 (2.41097) + | > loader_time: 0.08290 (0.03349) + + + --> STEP: 109/234 -- GLOBAL_STEP: 17425 + | > loss: 0.01764 (0.02508) + | > log_mle: -0.23328 (-0.15613) + | > loss_dur: 0.25091 (0.18121) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.49842 (7.59283) + | > current_lr: 0.00002 + | > step_time: 2.69780 (2.38297) + | > loader_time: 0.10170 (0.03381) + + + --> STEP: 114/234 -- GLOBAL_STEP: 17430 + | > loss: -0.02444 (0.02313) + | > log_mle: -0.21454 (-0.15924) + | > loss_dur: 0.19010 (0.18236) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.77700 (8.07682) + | > current_lr: 0.00002 + | > step_time: 0.92350 (2.36630) + | > loader_time: 0.00250 (0.03455) + + + --> STEP: 119/234 -- GLOBAL_STEP: 17435 + | > loss: -0.00284 (0.02216) + | > log_mle: -0.21277 (-0.16167) + | > loss_dur: 0.20993 (0.18383) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.52861 (8.33830) + | > current_lr: 0.00002 + | > step_time: 2.09770 (2.36410) + | > loader_time: 0.08540 (0.03391) + + + --> STEP: 124/234 -- GLOBAL_STEP: 17440 + | > loss: -0.04054 (0.02090) + | > log_mle: -0.24067 (-0.16349) + | > loss_dur: 0.20013 (0.18439) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.61927 (8.49905) + | > current_lr: 0.00002 + | > step_time: 2.31060 (2.36324) + | > loader_time: 0.08600 (0.03467) + + + --> STEP: 129/234 -- GLOBAL_STEP: 17445 + | > loss: -0.01773 (0.01879) + | > log_mle: -0.22752 (-0.16682) + | > loss_dur: 0.20979 (0.18561) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.74260 (8.91894) + | > current_lr: 0.00002 + | > step_time: 2.08350 (2.35890) + | > loader_time: 0.00250 (0.03343) + + + --> STEP: 134/234 -- GLOBAL_STEP: 17450 + | > loss: -0.03482 (0.01645) + | > log_mle: -0.27514 (-0.17055) + | > loss_dur: 0.24032 (0.18700) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.58823 (9.82420) + | > current_lr: 0.00002 + | > step_time: 2.32090 (2.35478) + | > loader_time: 0.07220 (0.03561) + + + --> STEP: 139/234 -- GLOBAL_STEP: 17455 + | > loss: -0.11155 (0.01406) + | > log_mle: -0.33282 (-0.17412) + | > loss_dur: 0.22127 (0.18818) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.09478 (10.36645) + | > current_lr: 0.00002 + | > step_time: 1.89530 (2.34853) + | > loader_time: 0.09520 (0.03572) + + + --> STEP: 144/234 -- GLOBAL_STEP: 17460 + | > loss: -0.07674 (0.01201) + | > log_mle: -0.30953 (-0.17788) + | > loss_dur: 0.23279 (0.18989) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.47473 (10.94686) + | > current_lr: 0.00002 + | > step_time: 7.40230 (2.39555) + | > loader_time: 0.19690 (0.03711) + + + --> STEP: 149/234 -- GLOBAL_STEP: 17465 + | > loss: -0.10890 (0.00894) + | > log_mle: -0.35116 (-0.18213) + | > loss_dur: 0.24226 (0.19108) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.25280 (11.52642) + | > current_lr: 0.00002 + | > step_time: 1.40840 (2.44122) + | > loader_time: 0.08640 (0.03785) + + + --> STEP: 154/234 -- GLOBAL_STEP: 17470 + | > loss: -0.09170 (0.00562) + | > log_mle: -0.31600 (-0.18677) + | > loss_dur: 0.22430 (0.19239) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.65393 (12.18410) + | > current_lr: 0.00002 + | > step_time: 4.51060 (2.51266) + | > loader_time: 0.09280 (0.04043) + + + --> STEP: 159/234 -- GLOBAL_STEP: 17475 + | > loss: -0.09697 (0.00270) + | > log_mle: -0.33172 (-0.19121) + | > loss_dur: 0.23475 (0.19390) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.36404 (12.91100) + | > current_lr: 0.00002 + | > step_time: 2.50070 (2.51416) + | > loader_time: 0.09950 (0.04270) + + + --> STEP: 164/234 -- GLOBAL_STEP: 17480 + | > loss: -0.09506 (-0.00038) + | > log_mle: -0.32895 (-0.19546) + | > loss_dur: 0.23388 (0.19508) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.92980 (13.45230) + | > current_lr: 0.00002 + | > step_time: 4.79680 (2.53308) + | > loader_time: 0.09730 (0.04389) + + + --> STEP: 169/234 -- GLOBAL_STEP: 17485 + | > loss: -0.06692 (-0.00320) + | > log_mle: -0.32021 (-0.19977) + | > loss_dur: 0.25329 (0.19657) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.01786 (14.22736) + | > current_lr: 0.00002 + | > step_time: 1.70750 (2.52108) + | > loader_time: 0.07800 (0.04638) + + + --> STEP: 174/234 -- GLOBAL_STEP: 17490 + | > loss: -0.16041 (-0.00687) + | > log_mle: -0.40875 (-0.20515) + | > loss_dur: 0.24834 (0.19828) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.58390 (14.92637) + | > current_lr: 0.00002 + | > step_time: 2.48650 (2.54566) + | > loader_time: 0.00250 (0.04571) + + + --> STEP: 179/234 -- GLOBAL_STEP: 17495 + | > loss: -0.12423 (-0.00985) + | > log_mle: -0.39647 (-0.20996) + | > loss_dur: 0.27224 (0.20011) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.76236 (15.60041) + | > current_lr: 0.00002 + | > step_time: 3.91050 (2.56562) + | > loader_time: 0.08540 (0.04561) + + + --> STEP: 184/234 -- GLOBAL_STEP: 17500 + | > loss: -0.11070 (-0.01261) + | > log_mle: -0.36868 (-0.21428) + | > loss_dur: 0.25798 (0.20167) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.72872 (16.42779) + | > current_lr: 0.00002 + | > step_time: 1.59780 (2.59808) + | > loader_time: 0.08680 (0.04742) + + + --> STEP: 189/234 -- GLOBAL_STEP: 17505 + | > loss: -0.09831 (-0.01538) + | > log_mle: -0.36864 (-0.21874) + | > loss_dur: 0.27033 (0.20336) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.89591 (17.19601) + | > current_lr: 0.00002 + | > step_time: 3.89190 (2.66005) + | > loader_time: 0.00380 (0.04926) + + + --> STEP: 194/234 -- GLOBAL_STEP: 17510 + | > loss: -0.14381 (-0.01862) + | > log_mle: -0.40160 (-0.22321) + | > loss_dur: 0.25779 (0.20459) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.86961 (17.97715) + | > current_lr: 0.00002 + | > step_time: 2.88990 (2.75447) + | > loader_time: 0.10850 (0.05251) + + + --> STEP: 199/234 -- GLOBAL_STEP: 17515 + | > loss: -0.14320 (-0.02133) + | > log_mle: -0.40333 (-0.22734) + | > loss_dur: 0.26012 (0.20601) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.26716 (18.89470) + | > current_lr: 0.00002 + | > step_time: 10.70300 (2.80491) + | > loader_time: 0.08820 (0.05263) + + + --> STEP: 204/234 -- GLOBAL_STEP: 17520 + | > loss: -0.14934 (-0.02376) + | > log_mle: -0.43528 (-0.23142) + | > loss_dur: 0.28593 (0.20765) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.44735 (19.63081) + | > current_lr: 0.00002 + | > step_time: 2.00080 (2.85133) + | > loader_time: 1.50040 (0.06114) + + + --> STEP: 209/234 -- GLOBAL_STEP: 17525 + | > loss: -0.13002 (-0.02674) + | > log_mle: -0.39760 (-0.23584) + | > loss_dur: 0.26759 (0.20910) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.51448 (20.30263) + | > current_lr: 0.00002 + | > step_time: 6.10690 (2.90363) + | > loader_time: 0.08330 (0.06017) + + + --> STEP: 214/234 -- GLOBAL_STEP: 17530 + | > loss: -0.17044 (-0.03025) + | > log_mle: -0.42743 (-0.24098) + | > loss_dur: 0.25698 (0.21073) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.70986 (21.19176) + | > current_lr: 0.00002 + | > step_time: 2.11460 (2.90460) + | > loader_time: 0.18530 (0.06008) + + + --> STEP: 219/234 -- GLOBAL_STEP: 17535 + | > loss: -0.23364 (-0.03364) + | > log_mle: -0.51727 (-0.24593) + | > loss_dur: 0.28362 (0.21229) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.76213 (22.02666) + | > current_lr: 0.00002 + | > step_time: 3.61670 (3.01562) + | > loader_time: 0.09290 (0.06227) + + + --> STEP: 224/234 -- GLOBAL_STEP: 17540 + | > loss: -0.17628 (-0.03656) + | > log_mle: -0.46971 (-0.25054) + | > loss_dur: 0.29343 (0.21398) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.53922 (22.85807) + | > current_lr: 0.00002 + | > step_time: 1.39480 (2.97972) + | > loader_time: 0.00530 (0.06134) + + + --> STEP: 229/234 -- GLOBAL_STEP: 17545 + | > loss: -0.13114 (-0.03954) + | > log_mle: -0.48908 (-0.25561) + | > loss_dur: 0.35793 (0.21608) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 173.32022 (24.16167) + | > current_lr: 0.00002 + | > step_time: 0.24490 (2.92070) + | > loader_time: 0.00390 (0.06047) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.14717 (-0.83634) + | > avg_loss: -0.07778 (-0.00946) + | > avg_log_mle: -0.32790 (-0.00875) + | > avg_loss_dur: 0.25012 (-0.00072) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_17550.pth + + > EPOCH: 75/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 01:01:09)  + + --> STEP: 0/234 -- GLOBAL_STEP: 17550 + | > loss: 0.04741 (0.04741) + | > log_mle: -0.18208 (-0.18208) + | > loss_dur: 0.22949 (0.22949) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.36208 (5.36208) + | > current_lr: 0.00002 + | > step_time: 4.48880 (4.48884) + | > loader_time: 4.75470 (4.75471) + + + --> STEP: 5/234 -- GLOBAL_STEP: 17555 + | > loss: 0.03068 (0.06863) + | > log_mle: -0.14343 (-0.13577) + | > loss_dur: 0.17411 (0.20440) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.14007 (5.89387) + | > current_lr: 0.00002 + | > step_time: 1.61030 (7.84828) + | > loader_time: 0.08690 (0.39910) + + + --> STEP: 10/234 -- GLOBAL_STEP: 17560 + | > loss: 0.03091 (0.05718) + | > log_mle: -0.14763 (-0.14083) + | > loss_dur: 0.17854 (0.19801) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.46160 (5.45583) + | > current_lr: 0.00002 + | > step_time: 8.49800 (6.52597) + | > loader_time: 0.10170 (0.21816) + + + --> STEP: 15/234 -- GLOBAL_STEP: 17565 + | > loss: 0.02354 (0.05171) + | > log_mle: -0.14114 (-0.14021) + | > loss_dur: 0.16468 (0.19192) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.46775 (4.91150) + | > current_lr: 0.00002 + | > step_time: 2.00900 (4.94292) + | > loader_time: 0.18940 (0.16359) + + + --> STEP: 20/234 -- GLOBAL_STEP: 17570 + | > loss: 0.03659 (0.04763) + | > log_mle: -0.13042 (-0.13830) + | > loss_dur: 0.16701 (0.18594) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.12352 (4.62822) + | > current_lr: 0.00002 + | > step_time: 1.61310 (4.61234) + | > loader_time: 0.08540 (0.13197) + + + --> STEP: 25/234 -- GLOBAL_STEP: 17575 + | > loss: 0.04927 (0.04377) + | > log_mle: -0.12499 (-0.13742) + | > loss_dur: 0.17425 (0.18119) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.77404 (4.48970) + | > current_lr: 0.00002 + | > step_time: 4.29940 (4.74216) + | > loader_time: 0.08460 (0.11297) + + + --> STEP: 30/234 -- GLOBAL_STEP: 17580 + | > loss: 0.00668 (0.04052) + | > log_mle: -0.15258 (-0.13812) + | > loss_dur: 0.15925 (0.17864) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.43123 (4.31664) + | > current_lr: 0.00002 + | > step_time: 6.00170 (4.76224) + | > loader_time: 0.00250 (0.11365) + + + --> STEP: 35/234 -- GLOBAL_STEP: 17585 + | > loss: 0.02022 (0.03964) + | > log_mle: -0.15192 (-0.13919) + | > loss_dur: 0.17214 (0.17883) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.94613 (4.47441) + | > current_lr: 0.00002 + | > step_time: 1.27910 (4.27279) + | > loader_time: 0.00400 (0.09785) + + + --> STEP: 40/234 -- GLOBAL_STEP: 17590 + | > loss: 0.05940 (0.03974) + | > log_mle: -0.13148 (-0.13984) + | > loss_dur: 0.19088 (0.17957) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.43187 (4.47099) + | > current_lr: 0.00002 + | > step_time: 1.08250 (3.91467) + | > loader_time: 0.00220 (0.08962) + + + --> STEP: 45/234 -- GLOBAL_STEP: 17595 + | > loss: 0.00206 (0.03788) + | > log_mle: -0.16557 (-0.14037) + | > loss_dur: 0.16763 (0.17825) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.53607 (4.54118) + | > current_lr: 0.00002 + | > step_time: 2.54830 (3.68193) + | > loader_time: 0.00980 (0.08007) + + + --> STEP: 50/234 -- GLOBAL_STEP: 17600 + | > loss: 0.04638 (0.03663) + | > log_mle: -0.13300 (-0.14039) + | > loss_dur: 0.17938 (0.17701) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.81246 (4.45948) + | > current_lr: 0.00002 + | > step_time: 1.60690 (3.47604) + | > loader_time: 0.00260 (0.07237) + + + --> STEP: 55/234 -- GLOBAL_STEP: 17605 + | > loss: 0.01907 (0.03600) + | > log_mle: -0.15441 (-0.14091) + | > loss_dur: 0.17348 (0.17690) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.64543 (4.38858) + | > current_lr: 0.00002 + | > step_time: 1.31310 (3.31674) + | > loader_time: 0.00210 (0.06787) + + + --> STEP: 60/234 -- GLOBAL_STEP: 17610 + | > loss: -0.01045 (0.03419) + | > log_mle: -0.17308 (-0.14188) + | > loss_dur: 0.16264 (0.17607) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.86909 (4.38268) + | > current_lr: 0.00002 + | > step_time: 1.17600 (3.16946) + | > loader_time: 0.00260 (0.06413) + + + --> STEP: 65/234 -- GLOBAL_STEP: 17615 + | > loss: 0.02300 (0.03360) + | > log_mle: -0.14481 (-0.14273) + | > loss_dur: 0.16781 (0.17633) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.87119 (4.55385) + | > current_lr: 0.00002 + | > step_time: 1.10190 (3.08891) + | > loader_time: 0.00200 (0.06336) + + + --> STEP: 70/234 -- GLOBAL_STEP: 17620 + | > loss: 0.03092 (0.03360) + | > log_mle: -0.14838 (-0.14279) + | > loss_dur: 0.17930 (0.17639) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.55659 (4.79877) + | > current_lr: 0.00002 + | > step_time: 1.84370 (3.00323) + | > loader_time: 0.00200 (0.05899) + + + --> STEP: 75/234 -- GLOBAL_STEP: 17625 + | > loss: 0.01408 (0.03299) + | > log_mle: -0.16123 (-0.14401) + | > loss_dur: 0.17531 (0.17700) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.30838 (5.06336) + | > current_lr: 0.00002 + | > step_time: 1.79070 (2.93220) + | > loader_time: 0.00220 (0.05642) + + + --> STEP: 80/234 -- GLOBAL_STEP: 17630 + | > loss: 0.01274 (0.03188) + | > log_mle: -0.14024 (-0.14458) + | > loss_dur: 0.15298 (0.17645) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.93178 (5.05484) + | > current_lr: 0.00002 + | > step_time: 1.58650 (2.84833) + | > loader_time: 0.00210 (0.05303) + + + --> STEP: 85/234 -- GLOBAL_STEP: 17635 + | > loss: 0.00813 (0.03093) + | > log_mle: -0.15746 (-0.14564) + | > loss_dur: 0.16559 (0.17657) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.52266 (5.15549) + | > current_lr: 0.00002 + | > step_time: 2.60270 (2.82061) + | > loader_time: 0.00230 (0.05118) + + + --> STEP: 90/234 -- GLOBAL_STEP: 17640 + | > loss: 0.01260 (0.02934) + | > log_mle: -0.18737 (-0.14767) + | > loss_dur: 0.19998 (0.17701) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.00492 (5.69420) + | > current_lr: 0.00002 + | > step_time: 3.21810 (2.79561) + | > loader_time: 0.00300 (0.04949) + + + --> STEP: 95/234 -- GLOBAL_STEP: 17645 + | > loss: -0.06779 (0.02656) + | > log_mle: -0.26901 (-0.15142) + | > loss_dur: 0.20122 (0.17798) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.43849 (6.31305) + | > current_lr: 0.00002 + | > step_time: 1.93130 (2.79176) + | > loader_time: 0.00300 (0.04705) + + + --> STEP: 100/234 -- GLOBAL_STEP: 17650 + | > loss: -0.00687 (0.02544) + | > log_mle: -0.19984 (-0.15349) + | > loss_dur: 0.19297 (0.17894) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.52271 (6.57906) + | > current_lr: 0.00002 + | > step_time: 1.50110 (2.74160) + | > loader_time: 0.00270 (0.04668) + + + --> STEP: 105/234 -- GLOBAL_STEP: 17655 + | > loss: -0.00535 (0.02321) + | > log_mle: -0.17341 (-0.15654) + | > loss_dur: 0.16806 (0.17976) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.96051 (6.98871) + | > current_lr: 0.00002 + | > step_time: 1.25400 (2.69613) + | > loader_time: 0.00450 (0.04460) + + + --> STEP: 110/234 -- GLOBAL_STEP: 17660 + | > loss: -0.01608 (0.02170) + | > log_mle: -0.20102 (-0.15928) + | > loss_dur: 0.18493 (0.18098) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.86531 (7.49598) + | > current_lr: 0.00002 + | > step_time: 2.39840 (2.71291) + | > loader_time: 0.10720 (0.04525) + + + --> STEP: 115/234 -- GLOBAL_STEP: 17665 + | > loss: -0.00055 (0.01989) + | > log_mle: -0.21764 (-0.16255) + | > loss_dur: 0.21710 (0.18244) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.28237 (7.98885) + | > current_lr: 0.00002 + | > step_time: 1.41640 (2.70197) + | > loader_time: 0.00270 (0.04472) + + + --> STEP: 120/234 -- GLOBAL_STEP: 17670 + | > loss: -0.04050 (0.01842) + | > log_mle: -0.26511 (-0.16538) + | > loss_dur: 0.22461 (0.18380) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.38873 (8.30864) + | > current_lr: 0.00002 + | > step_time: 1.58290 (2.66843) + | > loader_time: 0.00260 (0.04385) + + + --> STEP: 125/234 -- GLOBAL_STEP: 17675 + | > loss: -0.02307 (0.01723) + | > log_mle: -0.25252 (-0.16705) + | > loss_dur: 0.22944 (0.18427) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.31437 (8.55743) + | > current_lr: 0.00002 + | > step_time: 1.00880 (2.61824) + | > loader_time: 0.00270 (0.04366) + + + --> STEP: 130/234 -- GLOBAL_STEP: 17680 + | > loss: -0.03587 (0.01510) + | > log_mle: -0.26115 (-0.17036) + | > loss_dur: 0.22528 (0.18546) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.66244 (9.07582) + | > current_lr: 0.00002 + | > step_time: 1.29450 (2.58427) + | > loader_time: 0.00200 (0.04277) + + + --> STEP: 135/234 -- GLOBAL_STEP: 17685 + | > loss: -0.00546 (0.01294) + | > log_mle: -0.19814 (-0.17362) + | > loss_dur: 0.19267 (0.18656) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.55177 (9.62045) + | > current_lr: 0.00002 + | > step_time: 1.19600 (2.56920) + | > loader_time: 0.00280 (0.04192) + + + --> STEP: 140/234 -- GLOBAL_STEP: 17690 + | > loss: -0.01573 (0.01063) + | > log_mle: -0.22738 (-0.17735) + | > loss_dur: 0.21165 (0.18798) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.88235 (10.28096) + | > current_lr: 0.00002 + | > step_time: 3.60180 (2.61528) + | > loader_time: 0.09390 (0.04387) + + + --> STEP: 145/234 -- GLOBAL_STEP: 17695 + | > loss: -0.08770 (0.00807) + | > log_mle: -0.31851 (-0.18170) + | > loss_dur: 0.23081 (0.18977) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.55454 (10.88107) + | > current_lr: 0.00002 + | > step_time: 2.80310 (2.62435) + | > loader_time: 0.00860 (0.04252) + + + --> STEP: 150/234 -- GLOBAL_STEP: 17700 + | > loss: -0.07150 (0.00516) + | > log_mle: -0.30542 (-0.18580) + | > loss_dur: 0.23392 (0.19096) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.05563 (11.46527) + | > current_lr: 0.00002 + | > step_time: 2.50870 (2.62949) + | > loader_time: 0.08140 (0.04226) + + + --> STEP: 155/234 -- GLOBAL_STEP: 17705 + | > loss: -0.11559 (0.00157) + | > log_mle: -0.36611 (-0.19081) + | > loss_dur: 0.25052 (0.19238) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.19872 (12.20874) + | > current_lr: 0.00002 + | > step_time: 5.11030 (2.63905) + | > loader_time: 0.00400 (0.04281) + + + --> STEP: 160/234 -- GLOBAL_STEP: 17710 + | > loss: -0.11484 (-0.00135) + | > log_mle: -0.35739 (-0.19512) + | > loss_dur: 0.24255 (0.19377) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.68069 (12.92068) + | > current_lr: 0.00002 + | > step_time: 3.20270 (2.67543) + | > loader_time: 0.00350 (0.04764) + + + --> STEP: 165/234 -- GLOBAL_STEP: 17715 + | > loss: -0.09653 (-0.00431) + | > log_mle: -0.35715 (-0.19925) + | > loss_dur: 0.26062 (0.19494) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.91634 (13.72222) + | > current_lr: 0.00002 + | > step_time: 2.90950 (2.68652) + | > loader_time: 0.00370 (0.04633) + + + --> STEP: 170/234 -- GLOBAL_STEP: 17720 + | > loss: -0.11832 (-0.00715) + | > log_mle: -0.39576 (-0.20372) + | > loss_dur: 0.27744 (0.19657) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.14256 (14.54935) + | > current_lr: 0.00002 + | > step_time: 1.39420 (2.72348) + | > loader_time: 0.00580 (0.05367) + + + --> STEP: 175/234 -- GLOBAL_STEP: 17725 + | > loss: -0.10377 (-0.01065) + | > log_mle: -0.36970 (-0.20880) + | > loss_dur: 0.26592 (0.19814) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.44574 (15.53474) + | > current_lr: 0.00002 + | > step_time: 4.19820 (2.72128) + | > loader_time: 0.00330 (0.05270) + + + --> STEP: 180/234 -- GLOBAL_STEP: 17730 + | > loss: -0.12211 (-0.01377) + | > log_mle: -0.37859 (-0.21359) + | > loss_dur: 0.25648 (0.19982) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.68403 (16.30947) + | > current_lr: 0.00002 + | > step_time: 3.90810 (2.71342) + | > loader_time: 0.08600 (0.05182) + + + --> STEP: 185/234 -- GLOBAL_STEP: 17735 + | > loss: -0.12078 (-0.01645) + | > log_mle: -0.39957 (-0.21793) + | > loss_dur: 0.27880 (0.20147) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.85293 (17.39862) + | > current_lr: 0.00002 + | > step_time: 5.50600 (2.74234) + | > loader_time: 0.00460 (0.05206) + + + --> STEP: 190/234 -- GLOBAL_STEP: 17740 + | > loss: -0.13183 (-0.01920) + | > log_mle: -0.38095 (-0.22220) + | > loss_dur: 0.24911 (0.20300) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.15595 (18.20837) + | > current_lr: 0.00002 + | > step_time: 2.27760 (2.75597) + | > loader_time: 0.00250 (0.05180) + + + --> STEP: 195/234 -- GLOBAL_STEP: 17745 + | > loss: -0.12236 (-0.02229) + | > log_mle: -0.39254 (-0.22667) + | > loss_dur: 0.27018 (0.20438) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.18713 (18.94515) + | > current_lr: 0.00002 + | > step_time: 1.98700 (2.75548) + | > loader_time: 0.00300 (0.05108) + + + --> STEP: 200/234 -- GLOBAL_STEP: 17750 + | > loss: -0.10799 (-0.02493) + | > log_mle: -0.40396 (-0.23088) + | > loss_dur: 0.29597 (0.20595) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.99896 (19.50915) + | > current_lr: 0.00002 + | > step_time: 3.11250 (2.78661) + | > loader_time: 0.00640 (0.05035) + + + --> STEP: 205/234 -- GLOBAL_STEP: 17755 + | > loss: -0.12633 (-0.02744) + | > log_mle: -0.38607 (-0.23482) + | > loss_dur: 0.25975 (0.20738) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.20973 (20.39237) + | > current_lr: 0.00002 + | > step_time: 6.09760 (2.82055) + | > loader_time: 0.08550 (0.05059) + + + --> STEP: 210/234 -- GLOBAL_STEP: 17760 + | > loss: -0.18177 (-0.03051) + | > log_mle: -0.46217 (-0.23946) + | > loss_dur: 0.28040 (0.20895) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.05287 (21.33831) + | > current_lr: 0.00002 + | > step_time: 4.60910 (2.88213) + | > loader_time: 0.08480 (0.05438) + + + --> STEP: 215/234 -- GLOBAL_STEP: 17765 + | > loss: -0.14930 (-0.03382) + | > log_mle: -0.41409 (-0.24429) + | > loss_dur: 0.26479 (0.21047) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.53815 (22.22208) + | > current_lr: 0.00002 + | > step_time: 3.40710 (2.90389) + | > loader_time: 0.09640 (0.05407) + + + --> STEP: 220/234 -- GLOBAL_STEP: 17770 + | > loss: -0.18656 (-0.03728) + | > log_mle: -0.46259 (-0.24943) + | > loss_dur: 0.27602 (0.21215) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.92617 (23.07452) + | > current_lr: 0.00002 + | > step_time: 3.80930 (2.96574) + | > loader_time: 0.10070 (0.06509) + + + --> STEP: 225/234 -- GLOBAL_STEP: 17775 + | > loss: -0.22279 (-0.04031) + | > log_mle: -0.52299 (-0.25428) + | > loss_dur: 0.30020 (0.21397) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.04162 (23.84817) + | > current_lr: 0.00002 + | > step_time: 0.24370 (2.93772) + | > loader_time: 0.00270 (0.06451) + + + --> STEP: 230/234 -- GLOBAL_STEP: 17780 + | > loss: -0.18383 (-0.04312) + | > log_mle: -0.56124 (-0.25952) + | > loss_dur: 0.37741 (0.21640) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 87.74027 (24.79128) + | > current_lr: 0.00002 + | > step_time: 0.26410 (2.87932) + | > loader_time: 0.00350 (0.06319) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17286 (+0.02569) + | > avg_loss: -0.08152 (-0.00374) + | > avg_log_mle: -0.32851 (-0.00061) + | > avg_loss_dur: 0.24699 (-0.00313) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_17784.pth + + > EPOCH: 76/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 01:13:25)  + + --> STEP: 1/234 -- GLOBAL_STEP: 17785 + | > loss: 0.06083 (0.06083) + | > log_mle: -0.13940 (-0.13940) + | > loss_dur: 0.20023 (0.20023) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.65977 (5.65977) + | > current_lr: 0.00002 + | > step_time: 9.38450 (9.38454) + | > loader_time: 0.00100 (0.00097) + + + --> STEP: 6/234 -- GLOBAL_STEP: 17790 + | > loss: 0.07630 (0.07287) + | > log_mle: -0.13043 (-0.13758) + | > loss_dur: 0.20674 (0.21046) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.86949 (5.73984) + | > current_lr: 0.00002 + | > step_time: 2.21280 (5.91665) + | > loader_time: 0.09300 (0.04810) + + + --> STEP: 11/234 -- GLOBAL_STEP: 17795 + | > loss: 0.03975 (0.05519) + | > log_mle: -0.13604 (-0.14330) + | > loss_dur: 0.17579 (0.19849) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.49484 (5.33090) + | > current_lr: 0.00002 + | > step_time: 2.10530 (4.80946) + | > loader_time: 0.00180 (0.04315) + + + --> STEP: 16/234 -- GLOBAL_STEP: 17800 + | > loss: 0.01220 (0.04914) + | > log_mle: -0.14030 (-0.14268) + | > loss_dur: 0.15250 (0.19182) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.58186 (5.21063) + | > current_lr: 0.00002 + | > step_time: 1.75530 (4.23640) + | > loader_time: 0.00290 (0.03670) + + + --> STEP: 21/234 -- GLOBAL_STEP: 17805 + | > loss: 0.06005 (0.04631) + | > log_mle: -0.12431 (-0.13985) + | > loss_dur: 0.18436 (0.18616) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.13383 (5.08728) + | > current_lr: 0.00002 + | > step_time: 2.08760 (3.59579) + | > loader_time: 0.09830 (0.03677) + + + --> STEP: 26/234 -- GLOBAL_STEP: 17810 + | > loss: 0.03220 (0.04150) + | > log_mle: -0.14243 (-0.13970) + | > loss_dur: 0.17463 (0.18120) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.54461 (4.84250) + | > current_lr: 0.00002 + | > step_time: 0.93840 (3.26304) + | > loader_time: 0.00240 (0.03409) + + + --> STEP: 31/234 -- GLOBAL_STEP: 17815 + | > loss: 0.06126 (0.03840) + | > log_mle: -0.14712 (-0.14048) + | > loss_dur: 0.20838 (0.17888) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.45339 (4.69893) + | > current_lr: 0.00002 + | > step_time: 1.38750 (2.89832) + | > loader_time: 0.00140 (0.02889) + + + --> STEP: 36/234 -- GLOBAL_STEP: 17820 + | > loss: 0.03832 (0.03632) + | > log_mle: -0.15258 (-0.14156) + | > loss_dur: 0.19089 (0.17788) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.10311 (4.99194) + | > current_lr: 0.00002 + | > step_time: 2.19190 (2.77832) + | > loader_time: 0.00230 (0.02760) + + + --> STEP: 41/234 -- GLOBAL_STEP: 17825 + | > loss: 0.02528 (0.03623) + | > log_mle: -0.14127 (-0.14179) + | > loss_dur: 0.16655 (0.17801) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.59324 (5.05555) + | > current_lr: 0.00002 + | > step_time: 1.30580 (2.64838) + | > loader_time: 0.00270 (0.02646) + + + --> STEP: 46/234 -- GLOBAL_STEP: 17830 + | > loss: 0.03243 (0.03539) + | > log_mle: -0.14624 (-0.14237) + | > loss_dur: 0.17867 (0.17776) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.07641 (5.16795) + | > current_lr: 0.00002 + | > step_time: 1.08920 (2.53858) + | > loader_time: 0.00210 (0.02562) + + + --> STEP: 51/234 -- GLOBAL_STEP: 17835 + | > loss: 0.02214 (0.03406) + | > log_mle: -0.13162 (-0.14203) + | > loss_dur: 0.15376 (0.17609) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.69273 (5.05009) + | > current_lr: 0.00002 + | > step_time: 1.09520 (2.40854) + | > loader_time: 0.00220 (0.02691) + + + --> STEP: 56/234 -- GLOBAL_STEP: 17840 + | > loss: 0.05633 (0.03390) + | > log_mle: -0.15162 (-0.14288) + | > loss_dur: 0.20795 (0.17677) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.23648 (4.91756) + | > current_lr: 0.00002 + | > step_time: 1.01490 (2.33777) + | > loader_time: 0.08840 (0.02772) + + + --> STEP: 61/234 -- GLOBAL_STEP: 17845 + | > loss: 0.01961 (0.03158) + | > log_mle: -0.14721 (-0.14377) + | > loss_dur: 0.16682 (0.17535) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.58145 (4.87622) + | > current_lr: 0.00002 + | > step_time: 1.67680 (2.33783) + | > loader_time: 0.00210 (0.02721) + + + --> STEP: 66/234 -- GLOBAL_STEP: 17850 + | > loss: 0.03794 (0.03132) + | > log_mle: -0.13701 (-0.14448) + | > loss_dur: 0.17495 (0.17580) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.05986 (4.93627) + | > current_lr: 0.00002 + | > step_time: 1.41650 (2.27933) + | > loader_time: 0.08720 (0.02905) + + + --> STEP: 71/234 -- GLOBAL_STEP: 17855 + | > loss: 0.03639 (0.03105) + | > log_mle: -0.17913 (-0.14520) + | > loss_dur: 0.21552 (0.17625) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.01446 (5.30983) + | > current_lr: 0.00002 + | > step_time: 1.29400 (2.22028) + | > loader_time: 0.00280 (0.02838) + + + --> STEP: 76/234 -- GLOBAL_STEP: 17860 + | > loss: 0.01623 (0.03047) + | > log_mle: -0.16387 (-0.14613) + | > loss_dur: 0.18010 (0.17660) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.66405 (5.34768) + | > current_lr: 0.00002 + | > step_time: 1.37760 (2.20281) + | > loader_time: 0.00230 (0.02898) + + + --> STEP: 81/234 -- GLOBAL_STEP: 17865 + | > loss: 0.00616 (0.02933) + | > log_mle: -0.17386 (-0.14681) + | > loss_dur: 0.18003 (0.17614) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.71933 (5.35884) + | > current_lr: 0.00002 + | > step_time: 1.40650 (2.17288) + | > loader_time: 0.09220 (0.02956) + + + --> STEP: 86/234 -- GLOBAL_STEP: 17870 + | > loss: 0.01063 (0.02826) + | > log_mle: -0.17477 (-0.14785) + | > loss_dur: 0.18540 (0.17610) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.35015 (5.45530) + | > current_lr: 0.00002 + | > step_time: 2.30200 (2.18485) + | > loader_time: 0.08180 (0.03197) + + + --> STEP: 91/234 -- GLOBAL_STEP: 17875 + | > loss: 0.01906 (0.02666) + | > log_mle: -0.18339 (-0.15006) + | > loss_dur: 0.20245 (0.17673) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.60262 (5.68575) + | > current_lr: 0.00002 + | > step_time: 2.22460 (2.18144) + | > loader_time: 0.00230 (0.03035) + + + --> STEP: 96/234 -- GLOBAL_STEP: 17880 + | > loss: 0.01826 (0.02373) + | > log_mle: -0.17250 (-0.15374) + | > loss_dur: 0.19076 (0.17747) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.48923 (6.16320) + | > current_lr: 0.00002 + | > step_time: 1.60250 (2.17622) + | > loader_time: 0.08370 (0.03078) + + + --> STEP: 101/234 -- GLOBAL_STEP: 17885 + | > loss: -0.01733 (0.02211) + | > log_mle: -0.22909 (-0.15640) + | > loss_dur: 0.21177 (0.17851) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.78254 (6.45868) + | > current_lr: 0.00002 + | > step_time: 2.58850 (2.16826) + | > loader_time: 0.00610 (0.03117) + + + --> STEP: 106/234 -- GLOBAL_STEP: 17890 + | > loss: -0.00359 (0.02032) + | > log_mle: -0.22972 (-0.15938) + | > loss_dur: 0.22613 (0.17970) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.12927 (6.85756) + | > current_lr: 0.00002 + | > step_time: 2.41850 (2.15288) + | > loader_time: 0.00330 (0.03078) + + + --> STEP: 111/234 -- GLOBAL_STEP: 17895 + | > loss: -0.02324 (0.01867) + | > log_mle: -0.26889 (-0.16243) + | > loss_dur: 0.24565 (0.18110) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.33698 (7.26725) + | > current_lr: 0.00002 + | > step_time: 4.60180 (2.16148) + | > loader_time: 0.09410 (0.03033) + + + --> STEP: 116/234 -- GLOBAL_STEP: 17900 + | > loss: -0.00106 (0.01714) + | > log_mle: -0.23849 (-0.16540) + | > loss_dur: 0.23742 (0.18254) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.60231 (7.69783) + | > current_lr: 0.00002 + | > step_time: 2.40710 (2.17083) + | > loader_time: 0.00280 (0.03004) + + + --> STEP: 121/234 -- GLOBAL_STEP: 17905 + | > loss: 0.03038 (0.01573) + | > log_mle: -0.15601 (-0.16749) + | > loss_dur: 0.18639 (0.18322) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.40009 (8.00497) + | > current_lr: 0.00002 + | > step_time: 2.33550 (2.16564) + | > loader_time: 0.08810 (0.03117) + + + --> STEP: 126/234 -- GLOBAL_STEP: 17910 + | > loss: -0.06994 (0.01369) + | > log_mle: -0.28312 (-0.17017) + | > loss_dur: 0.21318 (0.18386) + | > amp_scaler: 8192.00000 (4128.50794) + | > grad_norm: 25.75681 (8.38441) + | > current_lr: 0.00002 + | > step_time: 1.51610 (2.14850) + | > loader_time: 0.08780 (0.03207) + + + --> STEP: 131/234 -- GLOBAL_STEP: 17915 + | > loss: -0.07645 (0.01154) + | > log_mle: -0.32108 (-0.17373) + | > loss_dur: 0.24463 (0.18527) + | > amp_scaler: 8192.00000 (4283.60305) + | > grad_norm: 53.26233 (9.09882) + | > current_lr: 0.00002 + | > step_time: 2.20930 (2.12997) + | > loader_time: 0.00430 (0.03101) + + + --> STEP: 136/234 -- GLOBAL_STEP: 17920 + | > loss: -0.11198 (0.00919) + | > log_mle: -0.36237 (-0.17722) + | > loss_dur: 0.25039 (0.18641) + | > amp_scaler: 8192.00000 (4427.29412) + | > grad_norm: 43.44053 (9.63101) + | > current_lr: 0.00002 + | > step_time: 2.39620 (2.13885) + | > loader_time: 0.00280 (0.03000) + + + --> STEP: 141/234 -- GLOBAL_STEP: 17925 + | > loss: -0.04551 (0.00732) + | > log_mle: -0.27817 (-0.18026) + | > loss_dur: 0.23267 (0.18758) + | > amp_scaler: 8192.00000 (4560.79433) + | > grad_norm: 20.90631 (10.13837) + | > current_lr: 0.00002 + | > step_time: 4.99500 (2.17368) + | > loader_time: 0.01180 (0.03091) + + + --> STEP: 146/234 -- GLOBAL_STEP: 17930 + | > loss: -0.09605 (0.00430) + | > log_mle: -0.32776 (-0.18490) + | > loss_dur: 0.23172 (0.18920) + | > amp_scaler: 8192.00000 (4685.15068) + | > grad_norm: 38.05954 (10.93358) + | > current_lr: 0.00002 + | > step_time: 2.08560 (2.17322) + | > loader_time: 0.00320 (0.03182) + + + --> STEP: 151/234 -- GLOBAL_STEP: 17935 + | > loss: -0.08868 (0.00165) + | > log_mle: -0.29526 (-0.18869) + | > loss_dur: 0.20658 (0.19034) + | > amp_scaler: 8192.00000 (4801.27152) + | > grad_norm: 36.03571 (11.59767) + | > current_lr: 0.00002 + | > step_time: 3.09240 (2.17923) + | > loader_time: 0.00920 (0.03215) + + + --> STEP: 156/234 -- GLOBAL_STEP: 17940 + | > loss: -0.10417 (-0.00196) + | > log_mle: -0.33543 (-0.19385) + | > loss_dur: 0.23125 (0.19189) + | > amp_scaler: 8192.00000 (4909.94872) + | > grad_norm: 32.66019 (12.52967) + | > current_lr: 0.00002 + | > step_time: 2.40090 (2.20723) + | > loader_time: 0.10220 (0.03247) + + + --> STEP: 161/234 -- GLOBAL_STEP: 17945 + | > loss: -0.11891 (-0.00505) + | > log_mle: -0.35303 (-0.19827) + | > loss_dur: 0.23412 (0.19323) + | > amp_scaler: 8192.00000 (5011.87578) + | > grad_norm: 40.65154 (13.17844) + | > current_lr: 0.00002 + | > step_time: 4.40610 (2.24679) + | > loader_time: 0.01080 (0.03393) + + + --> STEP: 166/234 -- GLOBAL_STEP: 17950 + | > loss: -0.08341 (-0.00764) + | > log_mle: -0.29969 (-0.20209) + | > loss_dur: 0.21629 (0.19445) + | > amp_scaler: 8192.00000 (5107.66265) + | > grad_norm: 22.86930 (13.64122) + | > current_lr: 0.00002 + | > step_time: 2.39900 (2.33571) + | > loader_time: 0.08810 (0.03477) + + + --> STEP: 171/234 -- GLOBAL_STEP: 17955 + | > loss: -0.16089 (-0.01099) + | > log_mle: -0.39873 (-0.20716) + | > loss_dur: 0.23784 (0.19617) + | > amp_scaler: 8192.00000 (5197.84795) + | > grad_norm: 32.72158 (14.28460) + | > current_lr: 0.00002 + | > step_time: 3.99860 (2.34813) + | > loader_time: 0.00230 (0.03436) + + + --> STEP: 176/234 -- GLOBAL_STEP: 17960 + | > loss: -0.12130 (-0.01423) + | > log_mle: -0.37034 (-0.21212) + | > loss_dur: 0.24904 (0.19790) + | > amp_scaler: 8192.00000 (5282.90909) + | > grad_norm: 42.06954 (15.11649) + | > current_lr: 0.00002 + | > step_time: 0.89530 (2.45077) + | > loader_time: 0.00220 (0.03568) + + + --> STEP: 181/234 -- GLOBAL_STEP: 17965 + | > loss: -0.06398 (-0.01694) + | > log_mle: -0.31156 (-0.21655) + | > loss_dur: 0.24758 (0.19961) + | > amp_scaler: 8192.00000 (5363.27072) + | > grad_norm: 27.39883 (15.74185) + | > current_lr: 0.00002 + | > step_time: 4.28960 (2.46268) + | > loader_time: 0.10300 (0.03624) + + + --> STEP: 186/234 -- GLOBAL_STEP: 17970 + | > loss: -0.09066 (-0.01983) + | > log_mle: -0.34952 (-0.22116) + | > loss_dur: 0.25886 (0.20133) + | > amp_scaler: 8192.00000 (5439.31183) + | > grad_norm: 35.76627 (16.47981) + | > current_lr: 0.00002 + | > step_time: 1.48440 (2.50395) + | > loader_time: 0.00280 (0.03680) + + + --> STEP: 191/234 -- GLOBAL_STEP: 17975 + | > loss: -0.12371 (-0.02290) + | > log_mle: -0.36378 (-0.22556) + | > loss_dur: 0.24008 (0.20265) + | > amp_scaler: 8192.00000 (5511.37173) + | > grad_norm: 32.32683 (17.12197) + | > current_lr: 0.00002 + | > step_time: 1.70960 (2.48338) + | > loader_time: 0.08510 (0.03679) + + + --> STEP: 196/234 -- GLOBAL_STEP: 17980 + | > loss: -0.10537 (-0.02595) + | > log_mle: -0.36287 (-0.23001) + | > loss_dur: 0.25750 (0.20406) + | > amp_scaler: 8192.00000 (5579.75510) + | > grad_norm: 42.16776 (17.98301) + | > current_lr: 0.00002 + | > step_time: 2.69820 (2.57975) + | > loader_time: 0.19380 (0.03839) + + + --> STEP: 201/234 -- GLOBAL_STEP: 17985 + | > loss: -0.06128 (-0.02846) + | > log_mle: -0.33150 (-0.23402) + | > loss_dur: 0.27022 (0.20556) + | > amp_scaler: 8192.00000 (5644.73632) + | > grad_norm: 32.12816 (18.71770) + | > current_lr: 0.00002 + | > step_time: 4.09630 (2.65334) + | > loader_time: 0.08870 (0.04191) + + + --> STEP: 206/234 -- GLOBAL_STEP: 17990 + | > loss: -0.16025 (-0.03145) + | > log_mle: -0.42527 (-0.23849) + | > loss_dur: 0.26502 (0.20704) + | > amp_scaler: 8192.00000 (5706.56311) + | > grad_norm: 56.80917 (19.56195) + | > current_lr: 0.00002 + | > step_time: 7.79390 (2.73698) + | > loader_time: 0.00450 (0.04284) + + + --> STEP: 211/234 -- GLOBAL_STEP: 17995 + | > loss: -0.20331 (-0.03487) + | > log_mle: -0.49492 (-0.24355) + | > loss_dur: 0.29160 (0.20868) + | > amp_scaler: 8192.00000 (5765.45972) + | > grad_norm: 62.98969 (20.42087) + | > current_lr: 0.00002 + | > step_time: 2.00270 (2.75603) + | > loader_time: 0.00420 (0.04193) + + + --> STEP: 216/234 -- GLOBAL_STEP: 18000 + | > loss: -0.18935 (-0.03819) + | > log_mle: -0.47564 (-0.24831) + | > loss_dur: 0.28629 (0.21012) + | > amp_scaler: 8192.00000 (5821.62963) + | > grad_norm: 62.77859 (21.19220) + | > current_lr: 0.00002 + | > step_time: 5.19540 (2.80703) + | > loader_time: 0.00710 (0.04278) + + + --> STEP: 221/234 -- GLOBAL_STEP: 18005 + | > loss: -0.13101 (-0.04133) + | > log_mle: -0.40232 (-0.25302) + | > loss_dur: 0.27130 (0.21170) + | > amp_scaler: 8192.00000 (5875.25792) + | > grad_norm: 43.13974 (22.04892) + | > current_lr: 0.00002 + | > step_time: 2.09160 (2.84655) + | > loader_time: 0.00320 (0.04275) + + + --> STEP: 226/234 -- GLOBAL_STEP: 18010 + | > loss: -0.20364 (-0.04469) + | > log_mle: -0.49049 (-0.25816) + | > loss_dur: 0.28685 (0.21346) + | > amp_scaler: 8192.00000 (5926.51327) + | > grad_norm: 73.43642 (23.10817) + | > current_lr: 0.00002 + | > step_time: 0.25620 (2.80284) + | > loader_time: 0.00500 (0.04189) + + + --> STEP: 231/234 -- GLOBAL_STEP: 18015 + | > loss: -0.12101 (-0.04706) + | > log_mle: -0.55322 (-0.26355) + | > loss_dur: 0.43221 (0.21650) + | > amp_scaler: 8192.00000 (5975.54978) + | > grad_norm: 64.37179 (24.14612) + | > current_lr: 0.00002 + | > step_time: 0.27150 (2.74767) + | > loader_time: 0.00380 (0.04107) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.75071 (+0.57784) + | > avg_loss: -0.07596 (+0.00557) + | > avg_log_mle: -0.32281 (+0.00570) + | > avg_loss_dur: 0.24685 (-0.00014) + + + > EPOCH: 77/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 01:25:15)  + + --> STEP: 2/234 -- GLOBAL_STEP: 18020 + | > loss: 0.12485 (0.08111) + | > log_mle: -0.11997 (-0.13101) + | > loss_dur: 0.24482 (0.21212) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.84946 (4.19689) + | > current_lr: 0.00002 + | > step_time: 6.30820 (4.60993) + | > loader_time: 0.00210 (0.00231) + + + --> STEP: 7/234 -- GLOBAL_STEP: 18025 + | > loss: -0.00173 (0.06002) + | > log_mle: -0.15725 (-0.14184) + | > loss_dur: 0.15552 (0.20187) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.98038 (6.00864) + | > current_lr: 0.00002 + | > step_time: 9.49940 (4.64486) + | > loader_time: 0.21230 (0.04692) + + + --> STEP: 12/234 -- GLOBAL_STEP: 18030 + | > loss: 0.03748 (0.05081) + | > log_mle: -0.14518 (-0.14487) + | > loss_dur: 0.18266 (0.19568) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.04705 (5.89820) + | > current_lr: 0.00002 + | > step_time: 0.80390 (4.52446) + | > loader_time: 0.00540 (0.02848) + + + --> STEP: 17/234 -- GLOBAL_STEP: 18035 + | > loss: 0.03899 (0.04169) + | > log_mle: -0.12602 (-0.14357) + | > loss_dur: 0.16501 (0.18526) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.50186 (5.50377) + | > current_lr: 0.00002 + | > step_time: 2.20370 (3.77718) + | > loader_time: 0.00190 (0.02508) + + + --> STEP: 22/234 -- GLOBAL_STEP: 18040 + | > loss: -0.00507 (0.03802) + | > log_mle: -0.15166 (-0.14236) + | > loss_dur: 0.14660 (0.18038) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.67125 (5.22519) + | > current_lr: 0.00002 + | > step_time: 3.79930 (4.04964) + | > loader_time: 0.00240 (0.02473) + + + --> STEP: 27/234 -- GLOBAL_STEP: 18045 + | > loss: -0.00125 (0.03417) + | > log_mle: -0.15375 (-0.14238) + | > loss_dur: 0.15251 (0.17654) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.26222 (4.93312) + | > current_lr: 0.00002 + | > step_time: 3.19820 (4.12227) + | > loader_time: 0.09680 (0.03505) + + + --> STEP: 32/234 -- GLOBAL_STEP: 18050 + | > loss: -0.00807 (0.03178) + | > log_mle: -0.16427 (-0.14337) + | > loss_dur: 0.15621 (0.17515) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.54166 (4.71941) + | > current_lr: 0.00002 + | > step_time: 1.72000 (3.96319) + | > loader_time: 0.06840 (0.04080) + + + --> STEP: 37/234 -- GLOBAL_STEP: 18055 + | > loss: 0.00543 (0.03108) + | > log_mle: -0.14346 (-0.14369) + | > loss_dur: 0.14889 (0.17477) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.29040 (4.91935) + | > current_lr: 0.00002 + | > step_time: 1.60120 (3.64868) + | > loader_time: 0.08460 (0.03995) + + + --> STEP: 42/234 -- GLOBAL_STEP: 18060 + | > loss: 0.04463 (0.03141) + | > log_mle: -0.13229 (-0.14365) + | > loss_dur: 0.17692 (0.17507) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.70523 (4.91266) + | > current_lr: 0.00002 + | > step_time: 0.98340 (3.45704) + | > loader_time: 0.00120 (0.03949) + + + --> STEP: 47/234 -- GLOBAL_STEP: 18065 + | > loss: 0.01919 (0.03004) + | > log_mle: -0.14589 (-0.14449) + | > loss_dur: 0.16509 (0.17453) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 5.06350 (5.08710) + | > current_lr: 0.00002 + | > step_time: 1.04530 (3.30805) + | > loader_time: 0.00380 (0.03961) + + + --> STEP: 52/234 -- GLOBAL_STEP: 18070 + | > loss: 0.02786 (0.02899) + | > log_mle: -0.14002 (-0.14407) + | > loss_dur: 0.16788 (0.17306) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.39035 (4.93222) + | > current_lr: 0.00002 + | > step_time: 1.38930 (3.12851) + | > loader_time: 0.00260 (0.03600) + + + --> STEP: 57/234 -- GLOBAL_STEP: 18075 + | > loss: 0.04136 (0.02883) + | > log_mle: -0.13606 (-0.14483) + | > loss_dur: 0.17743 (0.17366) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 2.84391 (4.78875) + | > current_lr: 0.00002 + | > step_time: 1.40000 (2.99960) + | > loader_time: 0.00250 (0.03643) + + + --> STEP: 62/234 -- GLOBAL_STEP: 18080 + | > loss: 0.03195 (0.02673) + | > log_mle: -0.19163 (-0.14666) + | > loss_dur: 0.22358 (0.17339) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 13.93801 (4.90213) + | > current_lr: 0.00002 + | > step_time: 1.46780 (2.89608) + | > loader_time: 0.00160 (0.03648) + + + --> STEP: 67/234 -- GLOBAL_STEP: 18085 + | > loss: -0.00013 (0.02619) + | > log_mle: -0.17119 (-0.14701) + | > loss_dur: 0.17106 (0.17320) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.05113 (4.88061) + | > current_lr: 0.00002 + | > step_time: 1.42040 (2.80106) + | > loader_time: 0.09810 (0.03807) + + + --> STEP: 72/234 -- GLOBAL_STEP: 18090 + | > loss: 0.04163 (0.02664) + | > log_mle: -0.14977 (-0.14733) + | > loss_dur: 0.19140 (0.17397) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 3.37144 (5.18465) + | > current_lr: 0.00002 + | > step_time: 1.00480 (2.69648) + | > loader_time: 0.00290 (0.03680) + + + --> STEP: 77/234 -- GLOBAL_STEP: 18095 + | > loss: -0.00880 (0.02535) + | > log_mle: -0.16665 (-0.14843) + | > loss_dur: 0.15785 (0.17378) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 6.55989 (5.30160) + | > current_lr: 0.00002 + | > step_time: 2.11570 (2.65154) + | > loader_time: 0.07550 (0.03551) + + + --> STEP: 82/234 -- GLOBAL_STEP: 18100 + | > loss: 0.00167 (0.02430) + | > log_mle: -0.15484 (-0.14895) + | > loss_dur: 0.15651 (0.17325) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 4.82565 (5.38154) + | > current_lr: 0.00002 + | > step_time: 1.51920 (2.58871) + | > loader_time: 0.08590 (0.03681) + + + --> STEP: 87/234 -- GLOBAL_STEP: 18105 + | > loss: 0.01891 (0.02388) + | > log_mle: -0.16555 (-0.15006) + | > loss_dur: 0.18445 (0.17394) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 7.99307 (5.58201) + | > current_lr: 0.00002 + | > step_time: 1.30100 (2.52807) + | > loader_time: 0.00500 (0.03491) + + + --> STEP: 92/234 -- GLOBAL_STEP: 18110 + | > loss: -0.04041 (0.02209) + | > log_mle: -0.20927 (-0.15267) + | > loss_dur: 0.16886 (0.17476) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.13595 (5.91082) + | > current_lr: 0.00002 + | > step_time: 1.38770 (2.51465) + | > loader_time: 0.00280 (0.03603) + + + --> STEP: 97/234 -- GLOBAL_STEP: 18115 + | > loss: -0.02408 (0.01947) + | > log_mle: -0.19956 (-0.15611) + | > loss_dur: 0.17548 (0.17558) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 11.57603 (6.46253) + | > current_lr: 0.00002 + | > step_time: 1.69310 (2.47496) + | > loader_time: 0.00260 (0.03609) + + + --> STEP: 102/234 -- GLOBAL_STEP: 18120 + | > loss: 0.01250 (0.01795) + | > log_mle: -0.18068 (-0.15848) + | > loss_dur: 0.19317 (0.17643) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 10.13041 (6.92050) + | > current_lr: 0.00002 + | > step_time: 2.41180 (2.47444) + | > loader_time: 0.08400 (0.03605) + + + --> STEP: 107/234 -- GLOBAL_STEP: 18125 + | > loss: -0.03307 (0.01547) + | > log_mle: -0.22877 (-0.16187) + | > loss_dur: 0.19569 (0.17734) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 18.43087 (7.43104) + | > current_lr: 0.00002 + | > step_time: 2.39290 (2.45704) + | > loader_time: 0.20060 (0.03713) + + + --> STEP: 112/234 -- GLOBAL_STEP: 18130 + | > loss: -0.02273 (0.01399) + | > log_mle: -0.23392 (-0.16487) + | > loss_dur: 0.21119 (0.17886) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 19.47165 (8.01280) + | > current_lr: 0.00002 + | > step_time: 0.80360 (2.48409) + | > loader_time: 0.00310 (0.03867) + + + --> STEP: 117/234 -- GLOBAL_STEP: 18135 + | > loss: -0.03342 (0.01251) + | > log_mle: -0.23016 (-0.16773) + | > loss_dur: 0.19675 (0.18024) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 16.26340 (8.45425) + | > current_lr: 0.00002 + | > step_time: 1.49240 (2.45919) + | > loader_time: 0.09760 (0.03858) + + + --> STEP: 122/234 -- GLOBAL_STEP: 18140 + | > loss: -0.01878 (0.01142) + | > log_mle: -0.20953 (-0.16959) + | > loss_dur: 0.19075 (0.18101) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 14.62848 (8.70819) + | > current_lr: 0.00002 + | > step_time: 2.60720 (2.43798) + | > loader_time: 0.00280 (0.03850) + + + --> STEP: 127/234 -- GLOBAL_STEP: 18145 + | > loss: -0.03310 (0.00957) + | > log_mle: -0.25830 (-0.17255) + | > loss_dur: 0.22521 (0.18212) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 24.55336 (9.25932) + | > current_lr: 0.00002 + | > step_time: 1.40750 (2.42006) + | > loader_time: 0.00300 (0.03922) + + + --> STEP: 132/234 -- GLOBAL_STEP: 18150 + | > loss: -0.06192 (0.00707) + | > log_mle: -0.24190 (-0.17592) + | > loss_dur: 0.17999 (0.18298) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 21.14957 (9.79713) + | > current_lr: 0.00002 + | > step_time: 1.69720 (2.40626) + | > loader_time: 0.09550 (0.03855) + + + --> STEP: 137/234 -- GLOBAL_STEP: 18155 + | > loss: -0.01874 (0.00508) + | > log_mle: -0.25180 (-0.17947) + | > loss_dur: 0.23306 (0.18455) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.53217 (10.50794) + | > current_lr: 0.00002 + | > step_time: 1.48960 (2.38114) + | > loader_time: 0.00410 (0.03923) + + + --> STEP: 142/234 -- GLOBAL_STEP: 18160 + | > loss: -0.04347 (0.00320) + | > log_mle: -0.26876 (-0.18257) + | > loss_dur: 0.22529 (0.18577) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.40044 (11.13244) + | > current_lr: 0.00002 + | > step_time: 7.50910 (2.42216) + | > loader_time: 0.10460 (0.03985) + + + --> STEP: 147/234 -- GLOBAL_STEP: 18165 + | > loss: -0.05307 (0.00041) + | > log_mle: -0.27065 (-0.18713) + | > loss_dur: 0.21758 (0.18754) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 25.09221 (12.03627) + | > current_lr: 0.00002 + | > step_time: 3.69130 (2.43513) + | > loader_time: 0.00260 (0.04058) + + + --> STEP: 152/234 -- GLOBAL_STEP: 18170 + | > loss: -0.08625 (-0.00251) + | > log_mle: -0.34550 (-0.19138) + | > loss_dur: 0.25925 (0.18887) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 28.64335 (12.63143) + | > current_lr: 0.00002 + | > step_time: 4.90050 (2.49924) + | > loader_time: 0.09510 (0.04376) + + + --> STEP: 157/234 -- GLOBAL_STEP: 18175 + | > loss: -0.06229 (-0.00585) + | > log_mle: -0.29676 (-0.19618) + | > loss_dur: 0.23447 (0.19033) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 27.26365 (13.41062) + | > current_lr: 0.00002 + | > step_time: 1.50420 (2.51462) + | > loader_time: 0.07500 (0.04537) + + + --> STEP: 162/234 -- GLOBAL_STEP: 18180 + | > loss: -0.10985 (-0.00909) + | > log_mle: -0.32985 (-0.20078) + | > loss_dur: 0.22000 (0.19169) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 40.16785 (14.06389) + | > current_lr: 0.00002 + | > step_time: 1.02500 (2.52427) + | > loader_time: 0.06710 (0.04562) + + + --> STEP: 167/234 -- GLOBAL_STEP: 18185 + | > loss: -0.16281 (-0.01197) + | > log_mle: -0.40158 (-0.20498) + | > loss_dur: 0.23877 (0.19301) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 53.36178 (14.72462) + | > current_lr: 0.00002 + | > step_time: 4.63290 (2.53493) + | > loader_time: 0.19460 (0.04651) + + + --> STEP: 172/234 -- GLOBAL_STEP: 18190 + | > loss: -0.13058 (-0.01505) + | > log_mle: -0.39467 (-0.20993) + | > loss_dur: 0.26409 (0.19488) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.20802 (15.34642) + | > current_lr: 0.00002 + | > step_time: 2.11530 (2.49846) + | > loader_time: 0.08980 (0.04619) + + + --> STEP: 177/234 -- GLOBAL_STEP: 18195 + | > loss: -0.09273 (-0.01808) + | > log_mle: -0.35652 (-0.21463) + | > loss_dur: 0.26379 (0.19655) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 41.92723 (16.04079) + | > current_lr: 0.00002 + | > step_time: 1.78270 (2.53907) + | > loader_time: 0.00280 (0.04551) + + + --> STEP: 182/234 -- GLOBAL_STEP: 18200 + | > loss: -0.12412 (-0.02088) + | > log_mle: -0.39741 (-0.21927) + | > loss_dur: 0.27329 (0.19839) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 48.52320 (16.75842) + | > current_lr: 0.00002 + | > step_time: 3.29170 (2.63026) + | > loader_time: 0.10370 (0.04592) + + + --> STEP: 187/234 -- GLOBAL_STEP: 18205 + | > loss: -0.14948 (-0.02381) + | > log_mle: -0.39871 (-0.22384) + | > loss_dur: 0.24923 (0.20003) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 43.90120 (17.47381) + | > current_lr: 0.00002 + | > step_time: 1.21040 (2.68558) + | > loader_time: 0.09260 (0.04742) + + + --> STEP: 192/234 -- GLOBAL_STEP: 18210 + | > loss: -0.16857 (-0.02692) + | > log_mle: -0.41442 (-0.22824) + | > loss_dur: 0.24585 (0.20132) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 125.43929 (18.69929) + | > current_lr: 0.00002 + | > step_time: 6.00810 (2.72826) + | > loader_time: 0.00680 (0.04726) + + + --> STEP: 197/234 -- GLOBAL_STEP: 18215 + | > loss: -0.14520 (-0.02966) + | > log_mle: -0.38948 (-0.23238) + | > loss_dur: 0.24428 (0.20272) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 68.61813 (19.98799) + | > current_lr: 0.00002 + | > step_time: 1.30840 (2.79859) + | > loader_time: 0.00370 (0.04814) + + + --> STEP: 202/234 -- GLOBAL_STEP: 18220 + | > loss: -0.21532 (-0.03233) + | > log_mle: -0.47692 (-0.23668) + | > loss_dur: 0.26160 (0.20435) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 60.76336 (20.83846) + | > current_lr: 0.00002 + | > step_time: 2.90180 (2.79921) + | > loader_time: 0.09810 (0.04751) + + + --> STEP: 207/234 -- GLOBAL_STEP: 18225 + | > loss: -0.19585 (-0.03510) + | > log_mle: -0.46505 (-0.24101) + | > loss_dur: 0.26920 (0.20591) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 49.32179 (21.62323) + | > current_lr: 0.00002 + | > step_time: 4.40780 (2.86838) + | > loader_time: 0.29050 (0.04972) + + + --> STEP: 212/234 -- GLOBAL_STEP: 18230 + | > loss: -0.17058 (-0.03818) + | > log_mle: -0.44742 (-0.24581) + | > loss_dur: 0.27683 (0.20762) + | > amp_scaler: 8192.00000 (8192.00000) + | > grad_norm: 67.35101 (22.76160) + | > current_lr: 0.00002 + | > step_time: 6.00960 (2.96025) + | > loader_time: 0.29700 (0.05176) + + + --> STEP: 217/234 -- GLOBAL_STEP: 18235 + | > loss: -0.18076 (-0.04138) + | > log_mle: -0.46003 (-0.25051) + | > loss_dur: 0.27927 (0.20913) + | > amp_scaler: 4096.00000 (8154.24885) + | > grad_norm: 101.97343 (23.62302) + | > current_lr: 0.00002 + | > step_time: 4.10910 (2.99718) + | > loader_time: 0.00540 (0.05283) + + + --> STEP: 222/234 -- GLOBAL_STEP: 18240 + | > loss: -0.17060 (-0.04427) + | > log_mle: -0.47617 (-0.25517) + | > loss_dur: 0.30557 (0.21089) + | > amp_scaler: 4096.00000 (8062.84685) + | > grad_norm: 64.46159 (24.67936) + | > current_lr: 0.00002 + | > step_time: 0.78170 (3.00803) + | > loader_time: 0.06880 (0.05244) + + + --> STEP: 227/234 -- GLOBAL_STEP: 18245 + | > loss: -0.16208 (-0.04762) + | > log_mle: -0.45770 (-0.26024) + | > loss_dur: 0.29562 (0.21262) + | > amp_scaler: 4096.00000 (7975.47137) + | > grad_norm: 52.20390 (25.56992) + | > current_lr: 0.00002 + | > step_time: 0.23860 (2.94695) + | > loader_time: 0.00310 (0.05136) + + + --> STEP: 232/234 -- GLOBAL_STEP: 18250 + | > loss: -0.08475 (-0.04971) + | > log_mle: -0.64222 (-0.26647) + | > loss_dur: 0.55747 (0.21677) + | > amp_scaler: 4096.00000 (7891.86207) + | > grad_norm: 86.07883 (26.61542) + | > current_lr: 0.00002 + | > step_time: 0.32700 (2.88932) + | > loader_time: 0.00830 (0.05041) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.70466 (-0.04604) + | > avg_loss: -0.09056 (-0.01460) + | > avg_log_mle: -0.33790 (-0.01510) + | > avg_loss_dur: 0.24735 (+0.00050) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_18252.pth + + > EPOCH: 78/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 01:37:37)  + + --> STEP: 3/234 -- GLOBAL_STEP: 18255 + | > loss: 0.08707 (0.07880) + | > log_mle: -0.14737 (-0.13746) + | > loss_dur: 0.23444 (0.21626) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.71299 (5.15450) + | > current_lr: 0.00002 + | > step_time: 3.71300 (6.73925) + | > loader_time: 0.00400 (1.26421) + + + --> STEP: 8/234 -- GLOBAL_STEP: 18260 + | > loss: 0.04976 (0.05278) + | > log_mle: -0.15601 (-0.14555) + | > loss_dur: 0.20577 (0.19832) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.44837 (6.10028) + | > current_lr: 0.00002 + | > step_time: 4.80800 (5.41908) + | > loader_time: 0.00400 (0.50156) + + + --> STEP: 13/234 -- GLOBAL_STEP: 18265 + | > loss: 0.05136 (0.04554) + | > log_mle: -0.13888 (-0.14622) + | > loss_dur: 0.19024 (0.19176) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.83931 (6.17000) + | > current_lr: 0.00002 + | > step_time: 1.50210 (5.46514) + | > loader_time: 0.08780 (0.34017) + + + --> STEP: 18/234 -- GLOBAL_STEP: 18270 + | > loss: 0.02177 (0.03679) + | > log_mle: -0.14394 (-0.14539) + | > loss_dur: 0.16571 (0.18218) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.60385 (5.75409) + | > current_lr: 0.00002 + | > step_time: 2.19560 (4.42303) + | > loader_time: 0.01000 (0.24669) + + + --> STEP: 23/234 -- GLOBAL_STEP: 18275 + | > loss: 0.01337 (0.03337) + | > log_mle: -0.14708 (-0.14430) + | > loss_dur: 0.16045 (0.17766) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.39595 (5.45410) + | > current_lr: 0.00002 + | > step_time: 1.39250 (3.92575) + | > loader_time: 0.00170 (0.19780) + + + --> STEP: 28/234 -- GLOBAL_STEP: 18280 + | > loss: 0.00077 (0.03127) + | > log_mle: -0.13994 (-0.14392) + | > loss_dur: 0.14071 (0.17519) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.84986 (5.10145) + | > current_lr: 0.00002 + | > step_time: 7.10310 (4.04727) + | > loader_time: 0.00170 (0.18232) + + + --> STEP: 33/234 -- GLOBAL_STEP: 18285 + | > loss: 0.04096 (0.02895) + | > log_mle: -0.13624 (-0.14491) + | > loss_dur: 0.17720 (0.17387) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.60928 (4.82731) + | > current_lr: 0.00002 + | > step_time: 1.49690 (3.94626) + | > loader_time: 0.00230 (0.16106) + + + --> STEP: 38/234 -- GLOBAL_STEP: 18290 + | > loss: 0.03951 (0.02884) + | > log_mle: -0.15447 (-0.14590) + | > loss_dur: 0.19398 (0.17474) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.90970 (4.94292) + | > current_lr: 0.00002 + | > step_time: 2.31040 (3.72282) + | > loader_time: 0.00250 (0.14447) + + + --> STEP: 43/234 -- GLOBAL_STEP: 18295 + | > loss: 0.01826 (0.02927) + | > log_mle: -0.15625 (-0.14585) + | > loss_dur: 0.17451 (0.17512) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.55152 (4.95904) + | > current_lr: 0.00002 + | > step_time: 1.65520 (3.44977) + | > loader_time: 0.00200 (0.13341) + + + --> STEP: 48/234 -- GLOBAL_STEP: 18300 + | > loss: 0.00768 (0.02776) + | > log_mle: -0.14031 (-0.14637) + | > loss_dur: 0.14799 (0.17413) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.93809 (5.00003) + | > current_lr: 0.00002 + | > step_time: 1.68560 (3.30498) + | > loader_time: 0.00120 (0.12375) + + + --> STEP: 53/234 -- GLOBAL_STEP: 18305 + | > loss: 0.03193 (0.02725) + | > log_mle: -0.16202 (-0.14638) + | > loss_dur: 0.19395 (0.17363) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.48910 (4.85717) + | > current_lr: 0.00002 + | > step_time: 2.09760 (3.20237) + | > loader_time: 0.00160 (0.11415) + + + --> STEP: 58/234 -- GLOBAL_STEP: 18310 + | > loss: -0.00039 (0.02670) + | > log_mle: -0.14526 (-0.14678) + | > loss_dur: 0.14487 (0.17348) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.34782 (4.78315) + | > current_lr: 0.00002 + | > step_time: 1.87830 (3.06661) + | > loader_time: 0.00150 (0.10710) + + + --> STEP: 63/234 -- GLOBAL_STEP: 18315 + | > loss: 0.04332 (0.02498) + | > log_mle: -0.15816 (-0.14875) + | > loss_dur: 0.20149 (0.17372) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.89806 (4.94476) + | > current_lr: 0.00002 + | > step_time: 1.24460 (2.99110) + | > loader_time: 0.00220 (0.10026) + + + --> STEP: 68/234 -- GLOBAL_STEP: 18320 + | > loss: 0.03546 (0.02460) + | > log_mle: -0.15184 (-0.14896) + | > loss_dur: 0.18729 (0.17356) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.92957 (4.92705) + | > current_lr: 0.00002 + | > step_time: 1.88740 (2.89172) + | > loader_time: 0.00260 (0.09428) + + + --> STEP: 73/234 -- GLOBAL_STEP: 18325 + | > loss: 0.01612 (0.02493) + | > log_mle: -0.17577 (-0.14973) + | > loss_dur: 0.19189 (0.17466) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.77046 (5.18994) + | > current_lr: 0.00002 + | > step_time: 2.29130 (2.82942) + | > loader_time: 0.00250 (0.09040) + + + --> STEP: 78/234 -- GLOBAL_STEP: 18330 + | > loss: 0.01875 (0.02358) + | > log_mle: -0.14936 (-0.15045) + | > loss_dur: 0.16810 (0.17403) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.65005 (5.31446) + | > current_lr: 0.00002 + | > step_time: 1.70520 (2.73923) + | > loader_time: 0.00210 (0.08781) + + + --> STEP: 83/234 -- GLOBAL_STEP: 18335 + | > loss: 0.01385 (0.02234) + | > log_mle: -0.17824 (-0.15131) + | > loss_dur: 0.19209 (0.17364) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.86484 (5.42868) + | > current_lr: 0.00002 + | > step_time: 2.71270 (2.67668) + | > loader_time: 0.00220 (0.08368) + + + --> STEP: 88/234 -- GLOBAL_STEP: 18340 + | > loss: -0.01759 (0.02143) + | > log_mle: -0.21361 (-0.15276) + | > loss_dur: 0.19602 (0.17419) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.07756 (5.63551) + | > current_lr: 0.00002 + | > step_time: 2.19690 (2.61625) + | > loader_time: 0.00280 (0.08004) + + + --> STEP: 93/234 -- GLOBAL_STEP: 18345 + | > loss: -0.00625 (0.01981) + | > log_mle: -0.22644 (-0.15540) + | > loss_dur: 0.22019 (0.17521) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.44953 (5.99711) + | > current_lr: 0.00002 + | > step_time: 1.90390 (2.55846) + | > loader_time: 0.00510 (0.07802) + + + --> STEP: 98/234 -- GLOBAL_STEP: 18350 + | > loss: 0.02908 (0.01790) + | > log_mle: -0.15292 (-0.15802) + | > loss_dur: 0.18200 (0.17593) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.84647 (6.36530) + | > current_lr: 0.00002 + | > step_time: 1.40200 (2.53098) + | > loader_time: 0.08900 (0.07602) + + + --> STEP: 103/234 -- GLOBAL_STEP: 18355 + | > loss: -0.01771 (0.01611) + | > log_mle: -0.24734 (-0.16124) + | > loss_dur: 0.22963 (0.17735) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.59907 (6.98766) + | > current_lr: 0.00002 + | > step_time: 1.50440 (2.51209) + | > loader_time: 0.09480 (0.07495) + + + --> STEP: 108/234 -- GLOBAL_STEP: 18360 + | > loss: -0.00356 (0.01443) + | > log_mle: -0.19642 (-0.16399) + | > loss_dur: 0.19286 (0.17842) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.56382 (7.44563) + | > current_lr: 0.00002 + | > step_time: 2.19960 (2.50690) + | > loader_time: 0.08900 (0.07317) + + + --> STEP: 113/234 -- GLOBAL_STEP: 18365 + | > loss: -0.03834 (0.01266) + | > log_mle: -0.24475 (-0.16729) + | > loss_dur: 0.20642 (0.17995) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.68246 (8.04694) + | > current_lr: 0.00002 + | > step_time: 2.21000 (2.47285) + | > loader_time: 0.10910 (0.07100) + + + --> STEP: 118/234 -- GLOBAL_STEP: 18370 + | > loss: -0.00420 (0.01155) + | > log_mle: -0.21088 (-0.16977) + | > loss_dur: 0.20668 (0.18132) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.68949 (8.35830) + | > current_lr: 0.00002 + | > step_time: 2.29370 (2.44513) + | > loader_time: 0.00260 (0.06958) + + + --> STEP: 123/234 -- GLOBAL_STEP: 18375 + | > loss: 0.00189 (0.01066) + | > log_mle: -0.18561 (-0.17137) + | > loss_dur: 0.18749 (0.18203) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.66969 (8.52232) + | > current_lr: 0.00002 + | > step_time: 3.02630 (2.44600) + | > loader_time: 0.08990 (0.06826) + + + --> STEP: 128/234 -- GLOBAL_STEP: 18380 + | > loss: -0.04753 (0.00836) + | > log_mle: -0.24460 (-0.17479) + | > loss_dur: 0.19707 (0.18315) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.58637 (9.10085) + | > current_lr: 0.00002 + | > step_time: 1.39560 (2.43628) + | > loader_time: 0.00400 (0.06638) + + + --> STEP: 133/234 -- GLOBAL_STEP: 18385 + | > loss: -0.03504 (0.00609) + | > log_mle: -0.26609 (-0.17827) + | > loss_dur: 0.23105 (0.18436) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.53947 (9.62099) + | > current_lr: 0.00002 + | > step_time: 1.79080 (2.41835) + | > loader_time: 0.00440 (0.06533) + + + --> STEP: 138/234 -- GLOBAL_STEP: 18390 + | > loss: -0.02800 (0.00422) + | > log_mle: -0.22383 (-0.18152) + | > loss_dur: 0.19583 (0.18574) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.24034 (10.08045) + | > current_lr: 0.00002 + | > step_time: 1.51520 (2.42573) + | > loader_time: 0.08860 (0.06505) + + + --> STEP: 143/234 -- GLOBAL_STEP: 18395 + | > loss: -0.07532 (0.00184) + | > log_mle: -0.34503 (-0.18549) + | > loss_dur: 0.26970 (0.18732) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.97758 (10.85851) + | > current_lr: 0.00002 + | > step_time: 1.80300 (2.39768) + | > loader_time: 0.00270 (0.06292) + + + --> STEP: 148/234 -- GLOBAL_STEP: 18400 + | > loss: -0.05924 (-0.00081) + | > log_mle: -0.27149 (-0.18940) + | > loss_dur: 0.21225 (0.18859) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.57766 (11.76253) + | > current_lr: 0.00002 + | > step_time: 2.19260 (2.40988) + | > loader_time: 0.00370 (0.06203) + + + --> STEP: 153/234 -- GLOBAL_STEP: 18405 + | > loss: -0.14608 (-0.00418) + | > log_mle: -0.38280 (-0.19427) + | > loss_dur: 0.23673 (0.19009) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.20824 (12.64646) + | > current_lr: 0.00002 + | > step_time: 2.11540 (2.39550) + | > loader_time: 0.08300 (0.06301) + + + --> STEP: 158/234 -- GLOBAL_STEP: 18410 + | > loss: -0.07413 (-0.00705) + | > log_mle: -0.32861 (-0.19858) + | > loss_dur: 0.25449 (0.19153) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.05558 (13.50635) + | > current_lr: 0.00002 + | > step_time: 0.96100 (2.37960) + | > loader_time: 0.00260 (0.06158) + + + --> STEP: 163/234 -- GLOBAL_STEP: 18415 + | > loss: -0.08867 (-0.01031) + | > log_mle: -0.30517 (-0.20292) + | > loss_dur: 0.21649 (0.19262) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.94405 (14.22532) + | > current_lr: 0.00002 + | > step_time: 4.18610 (2.39617) + | > loader_time: 0.00300 (0.06079) + + + --> STEP: 168/234 -- GLOBAL_STEP: 18420 + | > loss: -0.09358 (-0.01321) + | > log_mle: -0.35419 (-0.20737) + | > loss_dur: 0.26061 (0.19417) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.11428 (14.79522) + | > current_lr: 0.00002 + | > step_time: 5.40250 (2.43338) + | > loader_time: 0.10010 (0.06183) + + + --> STEP: 173/234 -- GLOBAL_STEP: 18425 + | > loss: -0.11002 (-0.01617) + | > log_mle: -0.35722 (-0.21215) + | > loss_dur: 0.24720 (0.19599) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.82716 (15.82522) + | > current_lr: 0.00002 + | > step_time: 2.31030 (2.45033) + | > loader_time: 0.00360 (0.06181) + + + --> STEP: 178/234 -- GLOBAL_STEP: 18430 + | > loss: -0.14360 (-0.01932) + | > log_mle: -0.41494 (-0.21706) + | > loss_dur: 0.27134 (0.19774) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.20248 (16.55524) + | > current_lr: 0.00002 + | > step_time: 5.10780 (2.45862) + | > loader_time: 0.00390 (0.06122) + + + --> STEP: 183/234 -- GLOBAL_STEP: 18435 + | > loss: -0.15509 (-0.02227) + | > log_mle: -0.41356 (-0.22165) + | > loss_dur: 0.25847 (0.19938) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.11400 (17.14048) + | > current_lr: 0.00002 + | > step_time: 4.20390 (2.46970) + | > loader_time: 0.07540 (0.06052) + + + --> STEP: 188/234 -- GLOBAL_STEP: 18440 + | > loss: -0.16362 (-0.02521) + | > log_mle: -0.42540 (-0.22615) + | > loss_dur: 0.26178 (0.20094) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.46748 (17.98744) + | > current_lr: 0.00002 + | > step_time: 2.00770 (2.46793) + | > loader_time: 0.00420 (0.06040) + + + --> STEP: 193/234 -- GLOBAL_STEP: 18445 + | > loss: -0.16105 (-0.02831) + | > log_mle: -0.41966 (-0.23055) + | > loss_dur: 0.25861 (0.20223) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.44759 (18.65791) + | > current_lr: 0.00002 + | > step_time: 4.40430 (2.46729) + | > loader_time: 0.08660 (0.06108) + + + --> STEP: 198/234 -- GLOBAL_STEP: 18450 + | > loss: -0.15478 (-0.03112) + | > log_mle: -0.41318 (-0.23472) + | > loss_dur: 0.25840 (0.20361) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.80185 (19.34097) + | > current_lr: 0.00002 + | > step_time: 3.30080 (2.53839) + | > loader_time: 0.08810 (0.06334) + + + --> STEP: 203/234 -- GLOBAL_STEP: 18455 + | > loss: -0.09810 (-0.03363) + | > log_mle: -0.35487 (-0.23874) + | > loss_dur: 0.25677 (0.20511) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.89335 (19.95078) + | > current_lr: 0.00002 + | > step_time: 3.81130 (2.60402) + | > loader_time: 0.18460 (0.06449) + + + --> STEP: 208/234 -- GLOBAL_STEP: 18460 + | > loss: -0.15069 (-0.03674) + | > log_mle: -0.42746 (-0.24343) + | > loss_dur: 0.27677 (0.20669) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.12310 (20.71802) + | > current_lr: 0.00002 + | > step_time: 6.20270 (2.64671) + | > loader_time: 0.70440 (0.06863) + + + --> STEP: 213/234 -- GLOBAL_STEP: 18465 + | > loss: -0.19157 (-0.04009) + | > log_mle: -0.47098 (-0.24842) + | > loss_dur: 0.27941 (0.20833) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.86279 (21.92361) + | > current_lr: 0.00002 + | > step_time: 3.40330 (2.69910) + | > loader_time: 1.98360 (0.07719) + + + --> STEP: 218/234 -- GLOBAL_STEP: 18470 + | > loss: -0.15967 (-0.04329) + | > log_mle: -0.43679 (-0.25305) + | > loss_dur: 0.27712 (0.20975) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.95059 (22.61863) + | > current_lr: 0.00002 + | > step_time: 2.00220 (2.75245) + | > loader_time: 0.08780 (0.07752) + + + --> STEP: 223/234 -- GLOBAL_STEP: 18475 + | > loss: -0.19894 (-0.04659) + | > log_mle: -0.47868 (-0.25798) + | > loss_dur: 0.27973 (0.21139) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.84713 (23.33955) + | > current_lr: 0.00002 + | > step_time: 3.41110 (2.74769) + | > loader_time: 0.00330 (0.07627) + + + --> STEP: 228/234 -- GLOBAL_STEP: 18480 + | > loss: -0.16949 (-0.04985) + | > log_mle: -0.47618 (-0.26303) + | > loss_dur: 0.30669 (0.21318) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 68.40100 (24.32817) + | > current_lr: 0.00002 + | > step_time: 0.24050 (2.70171) + | > loader_time: 0.00270 (0.07470) + + + --> STEP: 233/234 -- GLOBAL_STEP: 18485 + | > loss: 0.48391 (-0.04905) + | > log_mle: -0.42526 (-0.26893) + | > loss_dur: 0.90917 (0.21988) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.66840 (25.88251) + | > current_lr: 0.00002 + | > step_time: 0.19980 (2.64931) + | > loader_time: 0.00350 (0.07319) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55475 (-0.14992) + | > avg_loss: -0.10393 (-0.01337) + | > avg_log_mle: -0.34947 (-0.01156) + | > avg_loss_dur: 0.24554 (-0.00181) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_18486.pth + + > EPOCH: 79/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 01:49:03)  + + --> STEP: 4/234 -- GLOBAL_STEP: 18490 + | > loss: 0.06106 (0.07506) + | > log_mle: -0.15154 (-0.14322) + | > loss_dur: 0.21260 (0.21828) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.75048 (7.14895) + | > current_lr: 0.00002 + | > step_time: 6.09780 (5.50472) + | > loader_time: 0.00150 (0.19839) + + + --> STEP: 9/234 -- GLOBAL_STEP: 18495 + | > loss: 0.03525 (0.05450) + | > log_mle: -0.16209 (-0.14919) + | > loss_dur: 0.19734 (0.20368) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.02996 (6.00639) + | > current_lr: 0.00002 + | > step_time: 1.69700 (5.50314) + | > loader_time: 0.00300 (0.12881) + + + --> STEP: 14/234 -- GLOBAL_STEP: 18500 + | > loss: 0.00427 (0.04219) + | > log_mle: -0.15566 (-0.14911) + | > loss_dur: 0.15993 (0.19130) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.99666 (5.60223) + | > current_lr: 0.00002 + | > step_time: 0.80620 (4.23797) + | > loader_time: 0.00130 (0.09720) + + + --> STEP: 19/234 -- GLOBAL_STEP: 18505 + | > loss: 0.01835 (0.03621) + | > log_mle: -0.13901 (-0.14743) + | > loss_dur: 0.15737 (0.18364) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.27235 (5.30132) + | > current_lr: 0.00002 + | > step_time: 2.59710 (3.97679) + | > loader_time: 0.00490 (0.09173) + + + --> STEP: 24/234 -- GLOBAL_STEP: 18510 + | > loss: -0.00345 (0.03122) + | > log_mle: -0.14040 (-0.14644) + | > loss_dur: 0.13696 (0.17766) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.74833 (5.13286) + | > current_lr: 0.00002 + | > step_time: 1.40920 (3.60854) + | > loader_time: 0.00170 (0.10074) + + + --> STEP: 29/234 -- GLOBAL_STEP: 18515 + | > loss: 0.00482 (0.02850) + | > log_mle: -0.13982 (-0.14605) + | > loss_dur: 0.14464 (0.17455) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.28537 (4.99116) + | > current_lr: 0.00002 + | > step_time: 2.89970 (3.76563) + | > loader_time: 0.00450 (0.08682) + + + --> STEP: 34/234 -- GLOBAL_STEP: 18520 + | > loss: 0.02710 (0.02696) + | > log_mle: -0.14820 (-0.14719) + | > loss_dur: 0.17529 (0.17415) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.81934 (4.89020) + | > current_lr: 0.00002 + | > step_time: 6.49560 (3.86750) + | > loader_time: 0.00510 (0.08640) + + + --> STEP: 39/234 -- GLOBAL_STEP: 18525 + | > loss: 0.02550 (0.02594) + | > log_mle: -0.15699 (-0.14828) + | > loss_dur: 0.18249 (0.17423) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.27072 (5.19056) + | > current_lr: 0.00002 + | > step_time: 2.09850 (3.60698) + | > loader_time: 0.00440 (0.07579) + + + --> STEP: 44/234 -- GLOBAL_STEP: 18530 + | > loss: 0.01656 (0.02647) + | > log_mle: -0.14730 (-0.14804) + | > loss_dur: 0.16386 (0.17451) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.56109 (5.00349) + | > current_lr: 0.00002 + | > step_time: 1.48920 (3.36712) + | > loader_time: 0.09680 (0.07137) + + + --> STEP: 49/234 -- GLOBAL_STEP: 18535 + | > loss: -0.01348 (0.02396) + | > log_mle: -0.15428 (-0.14875) + | > loss_dur: 0.14080 (0.17272) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.37878 (4.97770) + | > current_lr: 0.00002 + | > step_time: 1.28940 (3.16795) + | > loader_time: 0.00240 (0.06434) + + + --> STEP: 54/234 -- GLOBAL_STEP: 18540 + | > loss: -0.00189 (0.02395) + | > log_mle: -0.16021 (-0.14882) + | > loss_dur: 0.15832 (0.17278) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.59766 (4.86652) + | > current_lr: 0.00002 + | > step_time: 2.00970 (3.02824) + | > loader_time: 0.00250 (0.06005) + + + --> STEP: 59/234 -- GLOBAL_STEP: 18545 + | > loss: -0.02246 (0.02272) + | > log_mle: -0.17614 (-0.14949) + | > loss_dur: 0.15368 (0.17221) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.98245 (4.80728) + | > current_lr: 0.00002 + | > step_time: 1.35180 (2.93192) + | > loader_time: 0.00200 (0.05516) + + + --> STEP: 64/234 -- GLOBAL_STEP: 18550 + | > loss: 0.01180 (0.02211) + | > log_mle: -0.14293 (-0.15075) + | > loss_dur: 0.15473 (0.17285) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.44448 (5.01068) + | > current_lr: 0.00002 + | > step_time: 1.19640 (2.84137) + | > loader_time: 0.00230 (0.05427) + + + --> STEP: 69/234 -- GLOBAL_STEP: 18555 + | > loss: 0.04277 (0.02208) + | > log_mle: -0.13332 (-0.15077) + | > loss_dur: 0.17609 (0.17285) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.56254 (4.98924) + | > current_lr: 0.00002 + | > step_time: 1.50250 (2.76994) + | > loader_time: 0.00300 (0.05177) + + + --> STEP: 74/234 -- GLOBAL_STEP: 18560 + | > loss: 0.00442 (0.02165) + | > log_mle: -0.15023 (-0.15174) + | > loss_dur: 0.15464 (0.17339) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.43217 (5.50249) + | > current_lr: 0.00002 + | > step_time: 1.15560 (2.67400) + | > loader_time: 0.00270 (0.04845) + + + --> STEP: 79/234 -- GLOBAL_STEP: 18565 + | > loss: -0.00386 (0.02095) + | > log_mle: -0.16391 (-0.15246) + | > loss_dur: 0.16006 (0.17341) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.60184 (5.59083) + | > current_lr: 0.00002 + | > step_time: 1.50080 (2.63130) + | > loader_time: 0.08280 (0.04900) + + + --> STEP: 84/234 -- GLOBAL_STEP: 18570 + | > loss: 0.01904 (0.01999) + | > log_mle: -0.15986 (-0.15315) + | > loss_dur: 0.17890 (0.17314) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.44079 (5.72702) + | > current_lr: 0.00002 + | > step_time: 5.60350 (2.61982) + | > loader_time: 0.10320 (0.04943) + + + --> STEP: 89/234 -- GLOBAL_STEP: 18575 + | > loss: -0.02571 (0.01883) + | > log_mle: -0.19164 (-0.15482) + | > loss_dur: 0.16593 (0.17365) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.27869 (6.00820) + | > current_lr: 0.00002 + | > step_time: 1.22270 (2.63631) + | > loader_time: 0.00220 (0.04799) + + + --> STEP: 94/234 -- GLOBAL_STEP: 18580 + | > loss: -0.04356 (0.01685) + | > log_mle: -0.22572 (-0.15766) + | > loss_dur: 0.18216 (0.17451) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.70996 (6.51428) + | > current_lr: 0.00002 + | > step_time: 1.60430 (2.57172) + | > loader_time: 0.00230 (0.04654) + + + --> STEP: 99/234 -- GLOBAL_STEP: 18585 + | > loss: -0.06110 (0.01503) + | > log_mle: -0.25982 (-0.16049) + | > loss_dur: 0.19873 (0.17552) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.19008 (6.96067) + | > current_lr: 0.00002 + | > step_time: 1.37220 (2.54502) + | > loader_time: 0.00260 (0.04717) + + + --> STEP: 104/234 -- GLOBAL_STEP: 18590 + | > loss: -0.06833 (0.01319) + | > log_mle: -0.27250 (-0.16379) + | > loss_dur: 0.20418 (0.17698) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.41222 (7.42192) + | > current_lr: 0.00002 + | > step_time: 2.40190 (2.55325) + | > loader_time: 0.00330 (0.04603) + + + --> STEP: 109/234 -- GLOBAL_STEP: 18595 + | > loss: -0.00854 (0.01174) + | > log_mle: -0.24291 (-0.16627) + | > loss_dur: 0.23437 (0.17801) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.53809 (7.78080) + | > current_lr: 0.00002 + | > step_time: 1.29860 (2.53548) + | > loader_time: 0.00320 (0.04578) + + + --> STEP: 114/234 -- GLOBAL_STEP: 18600 + | > loss: -0.03873 (0.00973) + | > log_mle: -0.22571 (-0.16945) + | > loss_dur: 0.18698 (0.17918) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.54087 (8.26730) + | > current_lr: 0.00002 + | > step_time: 1.17690 (2.52133) + | > loader_time: 0.00560 (0.04484) + + + --> STEP: 119/234 -- GLOBAL_STEP: 18605 + | > loss: -0.01401 (0.00869) + | > log_mle: -0.22191 (-0.17185) + | > loss_dur: 0.20790 (0.18054) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.75254 (8.62978) + | > current_lr: 0.00002 + | > step_time: 1.46570 (2.51031) + | > loader_time: 0.00250 (0.04457) + + + --> STEP: 124/234 -- GLOBAL_STEP: 18610 + | > loss: -0.05675 (0.00735) + | > log_mle: -0.25094 (-0.17363) + | > loss_dur: 0.19419 (0.18098) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.29161 (8.86342) + | > current_lr: 0.00002 + | > step_time: 1.08890 (2.48886) + | > loader_time: 0.00270 (0.04519) + + + --> STEP: 129/234 -- GLOBAL_STEP: 18615 + | > loss: -0.03573 (0.00500) + | > log_mle: -0.23599 (-0.17696) + | > loss_dur: 0.20026 (0.18196) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.03301 (9.38174) + | > current_lr: 0.00002 + | > step_time: 2.49510 (2.48535) + | > loader_time: 0.00320 (0.04420) + + + --> STEP: 134/234 -- GLOBAL_STEP: 18620 + | > loss: -0.04569 (0.00269) + | > log_mle: -0.28691 (-0.18075) + | > loss_dur: 0.24121 (0.18344) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.85668 (10.04651) + | > current_lr: 0.00002 + | > step_time: 8.39110 (2.50673) + | > loader_time: 0.10090 (0.04477) + + + --> STEP: 139/234 -- GLOBAL_STEP: 18625 + | > loss: -0.12580 (0.00028) + | > log_mle: -0.34401 (-0.18436) + | > loss_dur: 0.21820 (0.18464) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.78933 (10.54502) + | > current_lr: 0.00002 + | > step_time: 4.90430 (2.55826) + | > loader_time: 0.08650 (0.04520) + + + --> STEP: 144/234 -- GLOBAL_STEP: 18630 + | > loss: -0.09020 (-0.00182) + | > log_mle: -0.32166 (-0.18814) + | > loss_dur: 0.23146 (0.18632) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.07289 (11.17516) + | > current_lr: 0.00002 + | > step_time: 1.40720 (2.53059) + | > loader_time: 0.09900 (0.04441) + + + --> STEP: 149/234 -- GLOBAL_STEP: 18635 + | > loss: -0.11640 (-0.00486) + | > log_mle: -0.36245 (-0.19239) + | > loss_dur: 0.24604 (0.18753) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.27206 (11.83162) + | > current_lr: 0.00002 + | > step_time: 1.70580 (2.52508) + | > loader_time: 0.00230 (0.04722) + + + --> STEP: 154/234 -- GLOBAL_STEP: 18640 + | > loss: -0.11139 (-0.00833) + | > log_mle: -0.32870 (-0.19711) + | > loss_dur: 0.21731 (0.18878) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.23758 (12.48128) + | > current_lr: 0.00002 + | > step_time: 1.90560 (2.50527) + | > loader_time: 0.00540 (0.04582) + + + --> STEP: 159/234 -- GLOBAL_STEP: 18645 + | > loss: -0.12232 (-0.01141) + | > log_mle: -0.34532 (-0.20160) + | > loss_dur: 0.22300 (0.19019) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.63181 (13.23057) + | > current_lr: 0.00002 + | > step_time: 2.40690 (2.51776) + | > loader_time: 0.18250 (0.04730) + + + --> STEP: 164/234 -- GLOBAL_STEP: 18650 + | > loss: -0.09803 (-0.01446) + | > log_mle: -0.33718 (-0.20587) + | > loss_dur: 0.23916 (0.19141) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.14800 (14.04921) + | > current_lr: 0.00002 + | > step_time: 2.39940 (2.52160) + | > loader_time: 0.00480 (0.04655) + + + --> STEP: 169/234 -- GLOBAL_STEP: 18655 + | > loss: -0.07082 (-0.01725) + | > log_mle: -0.32939 (-0.21022) + | > loss_dur: 0.25856 (0.19297) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.90726 (14.78272) + | > current_lr: 0.00002 + | > step_time: 4.48430 (2.55642) + | > loader_time: 0.10680 (0.04747) + + + --> STEP: 174/234 -- GLOBAL_STEP: 18660 + | > loss: -0.17075 (-0.02088) + | > log_mle: -0.42196 (-0.21563) + | > loss_dur: 0.25121 (0.19475) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.85915 (15.79465) + | > current_lr: 0.00002 + | > step_time: 1.80340 (2.60087) + | > loader_time: 0.09780 (0.04946) + + + --> STEP: 179/234 -- GLOBAL_STEP: 18665 + | > loss: -0.13941 (-0.02383) + | > log_mle: -0.41336 (-0.22051) + | > loss_dur: 0.27395 (0.19669) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.57778 (16.51390) + | > current_lr: 0.00002 + | > step_time: 1.18890 (2.57504) + | > loader_time: 0.00320 (0.04874) + + + --> STEP: 184/234 -- GLOBAL_STEP: 18670 + | > loss: -0.12654 (-0.02666) + | > log_mle: -0.38271 (-0.22491) + | > loss_dur: 0.25618 (0.19825) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.92928 (17.31061) + | > current_lr: 0.00002 + | > step_time: 2.08400 (2.57078) + | > loader_time: 0.00470 (0.04752) + + + --> STEP: 189/234 -- GLOBAL_STEP: 18675 + | > loss: -0.11585 (-0.02947) + | > log_mle: -0.37886 (-0.22941) + | > loss_dur: 0.26301 (0.19994) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.30608 (18.17835) + | > current_lr: 0.00002 + | > step_time: 2.01890 (2.56094) + | > loader_time: 0.08580 (0.05041) + + + --> STEP: 194/234 -- GLOBAL_STEP: 18680 + | > loss: -0.15521 (-0.03272) + | > log_mle: -0.40966 (-0.23388) + | > loss_dur: 0.25445 (0.20117) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.52559 (19.03567) + | > current_lr: 0.00002 + | > step_time: 5.38730 (2.58980) + | > loader_time: 0.10240 (0.05061) + + + --> STEP: 199/234 -- GLOBAL_STEP: 18685 + | > loss: -0.15949 (-0.03550) + | > log_mle: -0.41750 (-0.23808) + | > loss_dur: 0.25801 (0.20258) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.86275 (19.71589) + | > current_lr: 0.00002 + | > step_time: 1.90520 (2.61663) + | > loader_time: 0.19870 (0.05182) + + + --> STEP: 204/234 -- GLOBAL_STEP: 18690 + | > loss: -0.16068 (-0.03801) + | > log_mle: -0.45040 (-0.24224) + | > loss_dur: 0.28972 (0.20423) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.19390 (20.31127) + | > current_lr: 0.00002 + | > step_time: 5.10730 (2.67017) + | > loader_time: 0.09440 (0.05157) + + + --> STEP: 209/234 -- GLOBAL_STEP: 18695 + | > loss: -0.14691 (-0.04095) + | > log_mle: -0.41070 (-0.24666) + | > loss_dur: 0.26379 (0.20571) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.49630 (21.06672) + | > current_lr: 0.00002 + | > step_time: 6.19180 (2.78628) + | > loader_time: 0.10880 (0.05794) + + + --> STEP: 214/234 -- GLOBAL_STEP: 18700 + | > loss: -0.18645 (-0.04453) + | > log_mle: -0.44261 (-0.25187) + | > loss_dur: 0.25615 (0.20734) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.18141 (22.00029) + | > current_lr: 0.00002 + | > step_time: 4.29870 (2.86548) + | > loader_time: 0.19030 (0.05990) + + + --> STEP: 219/234 -- GLOBAL_STEP: 18705 + | > loss: -0.24157 (-0.04790) + | > log_mle: -0.53012 (-0.25683) + | > loss_dur: 0.28855 (0.20893) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 81.38696 (23.11366) + | > current_lr: 0.00002 + | > step_time: 3.39860 (2.87043) + | > loader_time: 0.09120 (0.05986) + + + --> STEP: 224/234 -- GLOBAL_STEP: 18710 + | > loss: -0.18975 (-0.05086) + | > log_mle: -0.48385 (-0.26150) + | > loss_dur: 0.29410 (0.21064) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.56388 (23.93653) + | > current_lr: 0.00002 + | > step_time: 2.41160 (2.86187) + | > loader_time: 0.09710 (0.06017) + + + --> STEP: 229/234 -- GLOBAL_STEP: 18715 + | > loss: -0.16203 (-0.05388) + | > log_mle: -0.50947 (-0.26660) + | > loss_dur: 0.34744 (0.21272) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 73.81214 (24.94684) + | > current_lr: 0.00002 + | > step_time: 0.24910 (2.80682) + | > loader_time: 0.00360 (0.05895) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.18447 (-0.37028) + | > avg_loss: -0.09148 (+0.01245) + | > avg_log_mle: -0.33613 (+0.01334) + | > avg_loss_dur: 0.24465 (-0.00089) + + + > EPOCH: 80/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 02:01:01)  + + --> STEP: 0/234 -- GLOBAL_STEP: 18720 + | > loss: -0.01747 (-0.01747) + | > log_mle: -0.19731 (-0.19731) + | > loss_dur: 0.17984 (0.17984) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.39385 (6.39385) + | > current_lr: 0.00002 + | > step_time: 6.50720 (6.50717) + | > loader_time: 5.64310 (5.64314) + + + --> STEP: 5/234 -- GLOBAL_STEP: 18725 + | > loss: 0.01942 (0.05994) + | > log_mle: -0.15388 (-0.14689) + | > loss_dur: 0.17330 (0.20683) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.61777 (6.01517) + | > current_lr: 0.00002 + | > step_time: 5.80270 (4.88318) + | > loader_time: 0.20190 (0.18157) + + + --> STEP: 10/234 -- GLOBAL_STEP: 18730 + | > loss: 0.01897 (0.04831) + | > log_mle: -0.15637 (-0.15148) + | > loss_dur: 0.17534 (0.19979) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.56257 (6.03012) + | > current_lr: 0.00002 + | > step_time: 7.49670 (5.61100) + | > loader_time: 0.09920 (0.11026) + + + --> STEP: 15/234 -- GLOBAL_STEP: 18735 + | > loss: -0.00388 (0.03954) + | > log_mle: -0.15133 (-0.15053) + | > loss_dur: 0.14745 (0.19007) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.83038 (5.81537) + | > current_lr: 0.00002 + | > step_time: 1.75960 (4.77143) + | > loader_time: 0.00110 (0.09340) + + + --> STEP: 20/234 -- GLOBAL_STEP: 18740 + | > loss: 0.03059 (0.03569) + | > log_mle: -0.14014 (-0.14819) + | > loss_dur: 0.17073 (0.18388) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.79357 (5.53724) + | > current_lr: 0.00002 + | > step_time: 2.50500 (4.11364) + | > loader_time: 0.00730 (0.07923) + + + --> STEP: 25/234 -- GLOBAL_STEP: 18745 + | > loss: 0.03275 (0.03179) + | > log_mle: -0.13431 (-0.14719) + | > loss_dur: 0.16706 (0.17898) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.96018 (5.27556) + | > current_lr: 0.00002 + | > step_time: 2.69830 (4.13050) + | > loader_time: 0.00250 (0.07160) + + + --> STEP: 30/234 -- GLOBAL_STEP: 18750 + | > loss: -0.01348 (0.02706) + | > log_mle: -0.16384 (-0.14793) + | > loss_dur: 0.15036 (0.17499) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.74493 (4.99948) + | > current_lr: 0.00002 + | > step_time: 0.91010 (3.87281) + | > loader_time: 0.00210 (0.06619) + + + --> STEP: 35/234 -- GLOBAL_STEP: 18755 + | > loss: 0.01456 (0.02657) + | > log_mle: -0.16062 (-0.14890) + | > loss_dur: 0.17519 (0.17547) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.81704 (5.09142) + | > current_lr: 0.00002 + | > step_time: 4.29070 (3.75024) + | > loader_time: 0.00150 (0.05709) + + + --> STEP: 40/234 -- GLOBAL_STEP: 18760 + | > loss: 0.05114 (0.02656) + | > log_mle: -0.14010 (-0.14940) + | > loss_dur: 0.19125 (0.17596) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.62536 (5.13996) + | > current_lr: 0.00002 + | > step_time: 1.81620 (3.80756) + | > loader_time: 0.07660 (0.05403) + + + --> STEP: 45/234 -- GLOBAL_STEP: 18765 + | > loss: -0.00025 (0.02491) + | > log_mle: -0.17417 (-0.14990) + | > loss_dur: 0.17392 (0.17481) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.37661 (5.18185) + | > current_lr: 0.00002 + | > step_time: 1.04060 (3.53615) + | > loader_time: 0.00170 (0.05011) + + + --> STEP: 50/234 -- GLOBAL_STEP: 18770 + | > loss: 0.03292 (0.02368) + | > log_mle: -0.14161 (-0.14985) + | > loss_dur: 0.17452 (0.17353) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.14452 (5.08126) + | > current_lr: 0.00002 + | > step_time: 1.65650 (3.33836) + | > loader_time: 0.00230 (0.04712) + + + --> STEP: 55/234 -- GLOBAL_STEP: 18775 + | > loss: -0.00755 (0.02256) + | > log_mle: -0.16352 (-0.15033) + | > loss_dur: 0.15597 (0.17290) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.66988 (4.98934) + | > current_lr: 0.00002 + | > step_time: 1.27880 (3.19781) + | > loader_time: 0.00210 (0.04646) + + + --> STEP: 60/234 -- GLOBAL_STEP: 18780 + | > loss: -0.02524 (0.02125) + | > log_mle: -0.18090 (-0.15125) + | > loss_dur: 0.15567 (0.17250) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.17968 (4.97241) + | > current_lr: 0.00002 + | > step_time: 1.38900 (3.09452) + | > loader_time: 0.00180 (0.04601) + + + --> STEP: 65/234 -- GLOBAL_STEP: 18785 + | > loss: 0.00370 (0.02083) + | > log_mle: -0.15450 (-0.15208) + | > loss_dur: 0.15820 (0.17291) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.45763 (5.13579) + | > current_lr: 0.00002 + | > step_time: 1.40680 (2.96881) + | > loader_time: 0.08960 (0.04530) + + + --> STEP: 70/234 -- GLOBAL_STEP: 18790 + | > loss: 0.01678 (0.02084) + | > log_mle: -0.15457 (-0.15212) + | > loss_dur: 0.17135 (0.17296) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.49751 (5.27607) + | > current_lr: 0.00002 + | > step_time: 3.21170 (2.88081) + | > loader_time: 0.08340 (0.04337) + + + --> STEP: 75/234 -- GLOBAL_STEP: 18795 + | > loss: 0.01094 (0.02013) + | > log_mle: -0.17052 (-0.15329) + | > loss_dur: 0.18146 (0.17342) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.59528 (5.67575) + | > current_lr: 0.00002 + | > step_time: 2.11770 (2.81537) + | > loader_time: 0.00290 (0.04182) + + + --> STEP: 80/234 -- GLOBAL_STEP: 18800 + | > loss: 0.00261 (0.01909) + | > log_mle: -0.14769 (-0.15381) + | > loss_dur: 0.15030 (0.17290) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.50051 (5.64932) + | > current_lr: 0.00002 + | > step_time: 2.48130 (2.78571) + | > loader_time: 0.00860 (0.04057) + + + --> STEP: 85/234 -- GLOBAL_STEP: 18805 + | > loss: 0.00267 (0.01812) + | > log_mle: -0.16532 (-0.15475) + | > loss_dur: 0.16799 (0.17287) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.85881 (5.96889) + | > current_lr: 0.00002 + | > step_time: 3.00450 (2.73973) + | > loader_time: 0.00240 (0.04021) + + + --> STEP: 90/234 -- GLOBAL_STEP: 18810 + | > loss: -0.00856 (0.01654) + | > log_mle: -0.19669 (-0.15683) + | > loss_dur: 0.18813 (0.17338) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.30910 (6.27592) + | > current_lr: 0.00002 + | > step_time: 1.10540 (2.68045) + | > loader_time: 0.00260 (0.03824) + + + --> STEP: 95/234 -- GLOBAL_STEP: 18815 + | > loss: -0.07467 (0.01353) + | > log_mle: -0.27965 (-0.16062) + | > loss_dur: 0.20497 (0.17416) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.78375 (6.73717) + | > current_lr: 0.00002 + | > step_time: 1.71750 (2.64311) + | > loader_time: 0.00240 (0.03812) + + + --> STEP: 100/234 -- GLOBAL_STEP: 18820 + | > loss: -0.02269 (0.01200) + | > log_mle: -0.20584 (-0.16268) + | > loss_dur: 0.18315 (0.17468) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.12064 (6.97561) + | > current_lr: 0.00002 + | > step_time: 0.98720 (2.57927) + | > loader_time: 0.00230 (0.03803) + + + --> STEP: 105/234 -- GLOBAL_STEP: 18825 + | > loss: -0.01889 (0.01000) + | > log_mle: -0.18073 (-0.16569) + | > loss_dur: 0.16184 (0.17569) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.41014 (7.39164) + | > current_lr: 0.00002 + | > step_time: 2.10120 (2.54763) + | > loader_time: 0.00340 (0.03834) + + + --> STEP: 110/234 -- GLOBAL_STEP: 18830 + | > loss: -0.02589 (0.00865) + | > log_mle: -0.20945 (-0.16842) + | > loss_dur: 0.18356 (0.17707) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.23162 (7.80623) + | > current_lr: 0.00002 + | > step_time: 2.05300 (2.53710) + | > loader_time: 0.00400 (0.03675) + + + --> STEP: 115/234 -- GLOBAL_STEP: 18835 + | > loss: -0.01751 (0.00692) + | > log_mle: -0.22740 (-0.17171) + | > loss_dur: 0.20988 (0.17863) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.35165 (8.19197) + | > current_lr: 0.00002 + | > step_time: 1.40780 (2.50162) + | > loader_time: 0.00240 (0.03605) + + + --> STEP: 120/234 -- GLOBAL_STEP: 18840 + | > loss: -0.05783 (0.00542) + | > log_mle: -0.27181 (-0.17453) + | > loss_dur: 0.21398 (0.17995) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.81911 (8.59936) + | > current_lr: 0.00002 + | > step_time: 2.88840 (2.47593) + | > loader_time: 0.00320 (0.03466) + + + --> STEP: 125/234 -- GLOBAL_STEP: 18845 + | > loss: -0.05383 (0.00415) + | > log_mle: -0.26070 (-0.17616) + | > loss_dur: 0.20687 (0.18031) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.64501 (8.92618) + | > current_lr: 0.00002 + | > step_time: 1.99380 (2.46793) + | > loader_time: 0.00260 (0.03473) + + + --> STEP: 130/234 -- GLOBAL_STEP: 18850 + | > loss: -0.04338 (0.00202) + | > log_mle: -0.27194 (-0.17947) + | > loss_dur: 0.22856 (0.18149) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.06309 (9.53887) + | > current_lr: 0.00002 + | > step_time: 1.00870 (2.44945) + | > loader_time: 0.00270 (0.03414) + + + --> STEP: 135/234 -- GLOBAL_STEP: 18855 + | > loss: -0.02341 (-0.00010) + | > log_mle: -0.20649 (-0.18274) + | > loss_dur: 0.18308 (0.18264) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.01100 (10.03973) + | > current_lr: 0.00002 + | > step_time: 3.19730 (2.43514) + | > loader_time: 0.00270 (0.03301) + + + --> STEP: 140/234 -- GLOBAL_STEP: 18860 + | > loss: -0.01876 (-0.00236) + | > log_mle: -0.23646 (-0.18650) + | > loss_dur: 0.21770 (0.18414) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.93250 (10.88296) + | > current_lr: 0.00002 + | > step_time: 3.20350 (2.44596) + | > loader_time: 0.00490 (0.03259) + + + --> STEP: 145/234 -- GLOBAL_STEP: 18865 + | > loss: -0.09844 (-0.00487) + | > log_mle: -0.32353 (-0.19078) + | > loss_dur: 0.22509 (0.18592) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.79082 (11.99278) + | > current_lr: 0.00002 + | > step_time: 2.39890 (2.41895) + | > loader_time: 0.08540 (0.03325) + + + --> STEP: 150/234 -- GLOBAL_STEP: 18870 + | > loss: -0.08208 (-0.00776) + | > log_mle: -0.31505 (-0.19485) + | > loss_dur: 0.23297 (0.18709) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.59768 (12.75387) + | > current_lr: 0.00002 + | > step_time: 1.89040 (2.44374) + | > loader_time: 0.00830 (0.03231) + + + --> STEP: 155/234 -- GLOBAL_STEP: 18875 + | > loss: -0.13266 (-0.01137) + | > log_mle: -0.37775 (-0.19987) + | > loss_dur: 0.24508 (0.18850) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.14219 (13.54512) + | > current_lr: 0.00002 + | > step_time: 1.78700 (2.43729) + | > loader_time: 0.00380 (0.03240) + + + --> STEP: 160/234 -- GLOBAL_STEP: 18880 + | > loss: -0.12963 (-0.01429) + | > log_mle: -0.37397 (-0.20425) + | > loss_dur: 0.24435 (0.18997) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.28336 (14.19132) + | > current_lr: 0.00002 + | > step_time: 1.79660 (2.43102) + | > loader_time: 0.00400 (0.03375) + + + --> STEP: 165/234 -- GLOBAL_STEP: 18885 + | > loss: -0.11113 (-0.01709) + | > log_mle: -0.36817 (-0.20846) + | > loss_dur: 0.25704 (0.19137) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.65200 (14.83340) + | > current_lr: 0.00002 + | > step_time: 1.89410 (2.49315) + | > loader_time: 0.01270 (0.03582) + + + --> STEP: 170/234 -- GLOBAL_STEP: 18890 + | > loss: -0.13160 (-0.02007) + | > log_mle: -0.40451 (-0.21295) + | > loss_dur: 0.27290 (0.19288) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.57360 (15.56827) + | > current_lr: 0.00002 + | > step_time: 8.50740 (2.60049) + | > loader_time: 0.08420 (0.03765) + + + --> STEP: 175/234 -- GLOBAL_STEP: 18895 + | > loss: -0.12040 (-0.02358) + | > log_mle: -0.38135 (-0.21804) + | > loss_dur: 0.26096 (0.19446) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.94788 (16.39073) + | > current_lr: 0.00002 + | > step_time: 1.49900 (2.60099) + | > loader_time: 0.08160 (0.03772) + + + --> STEP: 180/234 -- GLOBAL_STEP: 18900 + | > loss: -0.14248 (-0.02668) + | > log_mle: -0.38830 (-0.22288) + | > loss_dur: 0.24582 (0.19619) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.65204 (17.01750) + | > current_lr: 0.00002 + | > step_time: 3.99360 (2.62424) + | > loader_time: 0.00590 (0.03736) + + + --> STEP: 185/234 -- GLOBAL_STEP: 18905 + | > loss: -0.14487 (-0.02939) + | > log_mle: -0.41459 (-0.22732) + | > loss_dur: 0.26972 (0.19793) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.79947 (17.75024) + | > current_lr: 0.00002 + | > step_time: 2.30940 (2.63067) + | > loader_time: 0.00280 (0.03745) + + + --> STEP: 190/234 -- GLOBAL_STEP: 18910 + | > loss: -0.14775 (-0.03223) + | > log_mle: -0.39338 (-0.23171) + | > loss_dur: 0.24563 (0.19947) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.62407 (18.43666) + | > current_lr: 0.00002 + | > step_time: 12.20190 (2.70895) + | > loader_time: 0.39780 (0.03964) + + + --> STEP: 195/234 -- GLOBAL_STEP: 18915 + | > loss: -0.13069 (-0.03536) + | > log_mle: -0.40120 (-0.23619) + | > loss_dur: 0.27050 (0.20083) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.22530 (19.47068) + | > current_lr: 0.00002 + | > step_time: 2.30130 (2.76204) + | > loader_time: 0.00330 (0.04272) + + + --> STEP: 200/234 -- GLOBAL_STEP: 18920 + | > loss: -0.13130 (-0.03805) + | > log_mle: -0.40765 (-0.24037) + | > loss_dur: 0.27635 (0.20233) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.49156 (20.34000) + | > current_lr: 0.00002 + | > step_time: 4.19930 (2.78352) + | > loader_time: 0.30450 (0.04556) + + + --> STEP: 205/234 -- GLOBAL_STEP: 18925 + | > loss: -0.14117 (-0.04049) + | > log_mle: -0.39808 (-0.24428) + | > loss_dur: 0.25691 (0.20380) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.73368 (21.42036) + | > current_lr: 0.00002 + | > step_time: 6.29860 (2.86006) + | > loader_time: 0.00560 (0.04603) + + + --> STEP: 210/234 -- GLOBAL_STEP: 18930 + | > loss: -0.19486 (-0.04362) + | > log_mle: -0.47268 (-0.24897) + | > loss_dur: 0.27782 (0.20535) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.72880 (22.33439) + | > current_lr: 0.00002 + | > step_time: 4.70750 (2.91241) + | > loader_time: 0.08410 (0.04915) + + + --> STEP: 215/234 -- GLOBAL_STEP: 18935 + | > loss: -0.17208 (-0.04700) + | > log_mle: -0.42860 (-0.25389) + | > loss_dur: 0.25652 (0.20690) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.05985 (23.12480) + | > current_lr: 0.00002 + | > step_time: 2.50310 (2.97207) + | > loader_time: 0.08620 (0.05037) + + + --> STEP: 220/234 -- GLOBAL_STEP: 18940 + | > loss: -0.19897 (-0.05053) + | > log_mle: -0.47787 (-0.25911) + | > loss_dur: 0.27889 (0.20858) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.37105 (23.88103) + | > current_lr: 0.00002 + | > step_time: 5.09230 (2.99479) + | > loader_time: 0.00820 (0.04981) + + + --> STEP: 225/234 -- GLOBAL_STEP: 18945 + | > loss: -0.23251 (-0.05359) + | > log_mle: -0.53093 (-0.26392) + | > loss_dur: 0.29843 (0.21033) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 74.27705 (25.00089) + | > current_lr: 0.00002 + | > step_time: 0.23720 (2.94142) + | > loader_time: 0.00430 (0.04912) + + + --> STEP: 230/234 -- GLOBAL_STEP: 18950 + | > loss: -0.19734 (-0.05638) + | > log_mle: -0.57695 (-0.26920) + | > loss_dur: 0.37962 (0.21283) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.84272 (25.94199) + | > current_lr: 0.00002 + | > step_time: 0.26240 (2.88305) + | > loader_time: 0.00470 (0.04816) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.09986 (-0.08461) + | > avg_loss: -0.10528 (-0.01380) + | > avg_log_mle: -0.34971 (-0.01359) + | > avg_loss_dur: 0.24443 (-0.00022) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_18954.pth + + > EPOCH: 81/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 02:13:12)  + + --> STEP: 1/234 -- GLOBAL_STEP: 18955 + | > loss: 0.01762 (0.01762) + | > log_mle: -0.15116 (-0.15116) + | > loss_dur: 0.16878 (0.16878) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.77995 (4.77995) + | > current_lr: 0.00002 + | > step_time: 2.69330 (2.69334) + | > loader_time: 0.09340 (0.09339) + + + --> STEP: 6/234 -- GLOBAL_STEP: 18960 + | > loss: 0.07024 (0.05277) + | > log_mle: -0.14280 (-0.14880) + | > loss_dur: 0.21304 (0.20157) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.12726 (5.39636) + | > current_lr: 0.00002 + | > step_time: 1.70790 (2.06563) + | > loader_time: 0.00310 (0.01807) + + + --> STEP: 11/234 -- GLOBAL_STEP: 18965 + | > loss: 0.02602 (0.03937) + | > log_mle: -0.14635 (-0.15384) + | > loss_dur: 0.17237 (0.19320) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.31272 (5.16538) + | > current_lr: 0.00002 + | > step_time: 10.01160 (4.15489) + | > loader_time: 0.09550 (0.76242) + + + --> STEP: 16/234 -- GLOBAL_STEP: 18970 + | > loss: 0.00465 (0.03134) + | > log_mle: -0.15049 (-0.15305) + | > loss_dur: 0.15514 (0.18439) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.63542 (5.18100) + | > current_lr: 0.00002 + | > step_time: 1.29070 (3.94575) + | > loader_time: 0.00260 (0.54191) + + + --> STEP: 21/234 -- GLOBAL_STEP: 18975 + | > loss: 0.04320 (0.03058) + | > log_mle: -0.13299 (-0.14996) + | > loss_dur: 0.17620 (0.18054) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.49899 (5.22904) + | > current_lr: 0.00002 + | > step_time: 1.14880 (3.22203) + | > loader_time: 0.00130 (0.41325) + + + --> STEP: 26/234 -- GLOBAL_STEP: 18980 + | > loss: 0.00487 (0.02595) + | > log_mle: -0.15317 (-0.14982) + | > loss_dur: 0.15804 (0.17577) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.13157 (4.91436) + | > current_lr: 0.00002 + | > step_time: 1.98650 (2.88314) + | > loader_time: 0.00170 (0.33413) + + + --> STEP: 31/234 -- GLOBAL_STEP: 18985 + | > loss: 0.04646 (0.02241) + | > log_mle: -0.15743 (-0.15050) + | > loss_dur: 0.20388 (0.17291) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.44928 (4.80546) + | > current_lr: 0.00002 + | > step_time: 1.69740 (2.65409) + | > loader_time: 0.00240 (0.28904) + + + --> STEP: 36/234 -- GLOBAL_STEP: 18990 + | > loss: 0.01480 (0.02084) + | > log_mle: -0.16133 (-0.15153) + | > loss_dur: 0.17613 (0.17237) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.45808 (5.02408) + | > current_lr: 0.00002 + | > step_time: 2.61080 (2.56770) + | > loader_time: 0.00220 (0.25625) + + + --> STEP: 41/234 -- GLOBAL_STEP: 18995 + | > loss: 0.00576 (0.02006) + | > log_mle: -0.15225 (-0.15173) + | > loss_dur: 0.15801 (0.17179) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.81997 (4.97553) + | > current_lr: 0.00002 + | > step_time: 1.66290 (2.48185) + | > loader_time: 0.00210 (0.22708) + + + --> STEP: 46/234 -- GLOBAL_STEP: 19000 + | > loss: 0.01390 (0.01946) + | > log_mle: -0.15636 (-0.15232) + | > loss_dur: 0.17027 (0.17179) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.85236 (5.04265) + | > current_lr: 0.00002 + | > step_time: 2.09370 (2.41629) + | > loader_time: 0.10050 (0.20877) + + + --> STEP: 51/234 -- GLOBAL_STEP: 19005 + | > loss: 0.01925 (0.01845) + | > log_mle: -0.14097 (-0.15193) + | > loss_dur: 0.16022 (0.17037) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.07486 (4.91915) + | > current_lr: 0.00002 + | > step_time: 2.29220 (2.36912) + | > loader_time: 0.00150 (0.19031) + + + --> STEP: 56/234 -- GLOBAL_STEP: 19010 + | > loss: 0.03988 (0.01865) + | > log_mle: -0.16018 (-0.15269) + | > loss_dur: 0.20006 (0.17134) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.13309 (4.82577) + | > current_lr: 0.00002 + | > step_time: 1.20420 (2.29521) + | > loader_time: 0.00460 (0.17359) + + + --> STEP: 61/234 -- GLOBAL_STEP: 19015 + | > loss: -0.00187 (0.01634) + | > log_mle: -0.15794 (-0.15349) + | > loss_dur: 0.15607 (0.16983) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.15729 (4.85881) + | > current_lr: 0.00002 + | > step_time: 2.39530 (2.23311) + | > loader_time: 0.00360 (0.15957) + + + --> STEP: 66/234 -- GLOBAL_STEP: 19020 + | > loss: 0.02491 (0.01642) + | > log_mle: -0.14514 (-0.15404) + | > loss_dur: 0.17004 (0.17047) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.37980 (5.01795) + | > current_lr: 0.00002 + | > step_time: 1.58640 (2.19395) + | > loader_time: 0.00220 (0.14917) + + + --> STEP: 71/234 -- GLOBAL_STEP: 19025 + | > loss: 0.00951 (0.01634) + | > log_mle: -0.18949 (-0.15473) + | > loss_dur: 0.19900 (0.17107) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.24439 (5.35124) + | > current_lr: 0.00002 + | > step_time: 2.20520 (2.18556) + | > loader_time: 0.00460 (0.14016) + + + --> STEP: 76/234 -- GLOBAL_STEP: 19030 + | > loss: 0.01072 (0.01570) + | > log_mle: -0.17094 (-0.15555) + | > loss_dur: 0.18167 (0.17125) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.88654 (5.46431) + | > current_lr: 0.00002 + | > step_time: 2.26460 (2.18384) + | > loader_time: 0.00240 (0.13110) + + + --> STEP: 81/234 -- GLOBAL_STEP: 19035 + | > loss: -0.01401 (0.01475) + | > log_mle: -0.17962 (-0.15609) + | > loss_dur: 0.16561 (0.17084) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.73892 (5.53297) + | > current_lr: 0.00002 + | > step_time: 1.51170 (2.17405) + | > loader_time: 0.18520 (0.12654) + + + --> STEP: 86/234 -- GLOBAL_STEP: 19040 + | > loss: 0.00515 (0.01407) + | > log_mle: -0.17860 (-0.15698) + | > loss_dur: 0.18375 (0.17105) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.55729 (5.77988) + | > current_lr: 0.00002 + | > step_time: 1.49230 (2.15375) + | > loader_time: 0.00340 (0.12038) + + + --> STEP: 91/234 -- GLOBAL_STEP: 19045 + | > loss: 0.00842 (0.01304) + | > log_mle: -0.19078 (-0.15903) + | > loss_dur: 0.19920 (0.17207) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.99489 (6.09551) + | > current_lr: 0.00002 + | > step_time: 1.30380 (2.14085) + | > loader_time: 0.00240 (0.11601) + + + --> STEP: 96/234 -- GLOBAL_STEP: 19050 + | > loss: 0.00336 (0.01027) + | > log_mle: -0.18037 (-0.16261) + | > loss_dur: 0.18373 (0.17288) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.05597 (6.63401) + | > current_lr: 0.00002 + | > step_time: 4.50460 (2.19214) + | > loader_time: 0.48770 (0.11708) + + + --> STEP: 101/234 -- GLOBAL_STEP: 19055 + | > loss: -0.03466 (0.00861) + | > log_mle: -0.23707 (-0.16519) + | > loss_dur: 0.20241 (0.17379) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.65538 (7.01145) + | > current_lr: 0.00002 + | > step_time: 2.40510 (2.18483) + | > loader_time: 0.08520 (0.11297) + + + --> STEP: 106/234 -- GLOBAL_STEP: 19060 + | > loss: -0.00412 (0.00703) + | > log_mle: -0.23655 (-0.16812) + | > loss_dur: 0.23243 (0.17515) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.01083 (7.49769) + | > current_lr: 0.00002 + | > step_time: 1.70250 (2.21424) + | > loader_time: 0.00280 (0.11200) + + + --> STEP: 111/234 -- GLOBAL_STEP: 19065 + | > loss: -0.03356 (0.00552) + | > log_mle: -0.27562 (-0.17107) + | > loss_dur: 0.24206 (0.17660) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.71445 (7.97750) + | > current_lr: 0.00002 + | > step_time: 1.46660 (2.23155) + | > loader_time: 0.00180 (0.10883) + + + --> STEP: 116/234 -- GLOBAL_STEP: 19070 + | > loss: -0.00944 (0.00403) + | > log_mle: -0.24754 (-0.17400) + | > loss_dur: 0.23810 (0.17804) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.01497 (8.41746) + | > current_lr: 0.00002 + | > step_time: 2.90160 (2.22533) + | > loader_time: 0.08670 (0.10636) + + + --> STEP: 121/234 -- GLOBAL_STEP: 19075 + | > loss: 0.02051 (0.00289) + | > log_mle: -0.16477 (-0.17608) + | > loss_dur: 0.18528 (0.17898) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.85519 (8.62810) + | > current_lr: 0.00002 + | > step_time: 3.80470 (2.24916) + | > loader_time: 0.00330 (0.10343) + + + --> STEP: 126/234 -- GLOBAL_STEP: 19080 + | > loss: -0.07672 (0.00113) + | > log_mle: -0.29033 (-0.17873) + | > loss_dur: 0.21361 (0.17985) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.93102 (9.08286) + | > current_lr: 0.00002 + | > step_time: 1.29260 (2.25359) + | > loader_time: 0.00220 (0.09943) + + + --> STEP: 131/234 -- GLOBAL_STEP: 19085 + | > loss: -0.09719 (-0.00116) + | > log_mle: -0.32970 (-0.18231) + | > loss_dur: 0.23250 (0.18115) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.93036 (9.62000) + | > current_lr: 0.00002 + | > step_time: 2.00050 (2.27003) + | > loader_time: 0.00380 (0.09784) + + + --> STEP: 136/234 -- GLOBAL_STEP: 19090 + | > loss: -0.12136 (-0.00342) + | > log_mle: -0.37776 (-0.18588) + | > loss_dur: 0.25640 (0.18246) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.52998 (9.99051) + | > current_lr: 0.00002 + | > step_time: 1.70620 (2.35218) + | > loader_time: 0.00360 (0.09652) + + + --> STEP: 141/234 -- GLOBAL_STEP: 19095 + | > loss: -0.05981 (-0.00531) + | > log_mle: -0.28887 (-0.18899) + | > loss_dur: 0.22906 (0.18368) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.91843 (10.52957) + | > current_lr: 0.00002 + | > step_time: 2.33650 (2.35846) + | > loader_time: 0.00280 (0.09524) + + + --> STEP: 146/234 -- GLOBAL_STEP: 19100 + | > loss: -0.11387 (-0.00823) + | > log_mle: -0.34095 (-0.19372) + | > loss_dur: 0.22708 (0.18549) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.36360 (11.18771) + | > current_lr: 0.00002 + | > step_time: 2.11150 (2.35297) + | > loader_time: 0.00270 (0.09269) + + + --> STEP: 151/234 -- GLOBAL_STEP: 19105 + | > loss: -0.09570 (-0.01105) + | > log_mle: -0.30447 (-0.19759) + | > loss_dur: 0.20877 (0.18655) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.43581 (11.76075) + | > current_lr: 0.00002 + | > step_time: 1.79190 (2.36914) + | > loader_time: 0.00530 (0.09029) + + + --> STEP: 156/234 -- GLOBAL_STEP: 19110 + | > loss: -0.11933 (-0.01466) + | > log_mle: -0.34325 (-0.20282) + | > loss_dur: 0.22391 (0.18817) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.22541 (12.82523) + | > current_lr: 0.00002 + | > step_time: 1.19850 (2.35867) + | > loader_time: 0.00310 (0.08853) + + + --> STEP: 161/234 -- GLOBAL_STEP: 19115 + | > loss: -0.12649 (-0.01754) + | > log_mle: -0.36380 (-0.20727) + | > loss_dur: 0.23730 (0.18974) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.35345 (13.61361) + | > current_lr: 0.00002 + | > step_time: 3.50340 (2.36386) + | > loader_time: 0.09220 (0.08643) + + + --> STEP: 166/234 -- GLOBAL_STEP: 19120 + | > loss: -0.09526 (-0.02015) + | > log_mle: -0.30742 (-0.21109) + | > loss_dur: 0.21216 (0.19095) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.00556 (14.34350) + | > current_lr: 0.00002 + | > step_time: 3.30720 (2.38303) + | > loader_time: 0.08620 (0.08493) + + + --> STEP: 171/234 -- GLOBAL_STEP: 19125 + | > loss: -0.17507 (-0.02353) + | > log_mle: -0.40871 (-0.21617) + | > loss_dur: 0.23364 (0.19264) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.31865 (15.24966) + | > current_lr: 0.00002 + | > step_time: 2.09870 (2.39057) + | > loader_time: 0.19670 (0.08687) + + + --> STEP: 176/234 -- GLOBAL_STEP: 19130 + | > loss: -0.14004 (-0.02688) + | > log_mle: -0.38207 (-0.22116) + | > loss_dur: 0.24202 (0.19428) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.00442 (16.29451) + | > current_lr: 0.00002 + | > step_time: 2.28890 (2.40833) + | > loader_time: 0.00490 (0.08452) + + + --> STEP: 181/234 -- GLOBAL_STEP: 19135 + | > loss: -0.08538 (-0.02962) + | > log_mle: -0.32345 (-0.22562) + | > loss_dur: 0.23806 (0.19600) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.09782 (17.05383) + | > current_lr: 0.00002 + | > step_time: 0.78720 (2.43804) + | > loader_time: 0.00260 (0.08387) + + + --> STEP: 186/234 -- GLOBAL_STEP: 19140 + | > loss: -0.09508 (-0.03228) + | > log_mle: -0.35611 (-0.23008) + | > loss_dur: 0.26103 (0.19780) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.78411 (18.22836) + | > current_lr: 0.00002 + | > step_time: 4.30120 (2.46668) + | > loader_time: 0.00410 (0.08271) + + + --> STEP: 191/234 -- GLOBAL_STEP: 19145 + | > loss: -0.13837 (-0.03514) + | > log_mle: -0.37222 (-0.23437) + | > loss_dur: 0.23385 (0.19923) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.76740 (19.21395) + | > current_lr: 0.00002 + | > step_time: 5.10270 (2.52508) + | > loader_time: 0.29720 (0.08319) + + + --> STEP: 196/234 -- GLOBAL_STEP: 19150 + | > loss: -0.10797 (-0.03804) + | > log_mle: -0.36789 (-0.23866) + | > loss_dur: 0.25992 (0.20062) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.89862 (20.23194) + | > current_lr: 0.00002 + | > step_time: 1.36970 (2.55147) + | > loader_time: 0.00330 (0.08212) + + + --> STEP: 201/234 -- GLOBAL_STEP: 19155 + | > loss: -0.06792 (-0.04038) + | > log_mle: -0.33663 (-0.24252) + | > loss_dur: 0.26871 (0.20214) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.72944 (21.09845) + | > current_lr: 0.00002 + | > step_time: 4.89790 (2.55912) + | > loader_time: 0.00770 (0.08024) + + + --> STEP: 206/234 -- GLOBAL_STEP: 19160 + | > loss: -0.16007 (-0.04331) + | > log_mle: -0.43495 (-0.24697) + | > loss_dur: 0.27488 (0.20367) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.38076 (21.90875) + | > current_lr: 0.00002 + | > step_time: 3.09470 (2.57378) + | > loader_time: 0.00500 (0.08021) + + + --> STEP: 211/234 -- GLOBAL_STEP: 19165 + | > loss: -0.21424 (-0.04669) + | > log_mle: -0.50246 (-0.25202) + | > loss_dur: 0.28822 (0.20532) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.10255 (22.76234) + | > current_lr: 0.00002 + | > step_time: 3.80930 (2.60577) + | > loader_time: 0.10020 (0.08015) + + + --> STEP: 216/234 -- GLOBAL_STEP: 19170 + | > loss: -0.20733 (-0.05006) + | > log_mle: -0.48881 (-0.25683) + | > loss_dur: 0.28148 (0.20677) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 68.37408 (23.58764) + | > current_lr: 0.00002 + | > step_time: 3.30360 (2.67654) + | > loader_time: 0.00240 (0.07922) + + + --> STEP: 221/234 -- GLOBAL_STEP: 19175 + | > loss: -0.14313 (-0.05331) + | > log_mle: -0.40833 (-0.26160) + | > loss_dur: 0.26520 (0.20829) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 85.23534 (24.65558) + | > current_lr: 0.00002 + | > step_time: 1.99740 (2.73269) + | > loader_time: 0.08730 (0.08054) + + + --> STEP: 226/234 -- GLOBAL_STEP: 19180 + | > loss: -0.22552 (-0.05670) + | > log_mle: -0.50583 (-0.26681) + | > loss_dur: 0.28032 (0.21011) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 68.65215 (25.83348) + | > current_lr: 0.00002 + | > step_time: 0.81980 (2.70382) + | > loader_time: 0.07840 (0.07988) + + + --> STEP: 231/234 -- GLOBAL_STEP: 19185 + | > loss: -0.13618 (-0.05922) + | > log_mle: -0.56891 (-0.27236) + | > loss_dur: 0.43273 (0.21314) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 78.91635 (26.84596) + | > current_lr: 0.00002 + | > step_time: 0.33980 (2.65105) + | > loader_time: 0.00470 (0.07824) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00164 (-0.09822) + | > avg_loss: -0.10402 (+0.00126) + | > avg_log_mle: -0.34816 (+0.00156) + | > avg_loss_dur: 0.24413 (-0.00030) + + + > EPOCH: 82/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 02:24:39)  + + --> STEP: 2/234 -- GLOBAL_STEP: 19190 + | > loss: 0.10974 (0.05897) + | > log_mle: -0.13196 (-0.14230) + | > loss_dur: 0.24171 (0.20127) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.08320 (5.03991) + | > current_lr: 0.00002 + | > step_time: 2.20260 (4.70483) + | > loader_time: 0.00270 (0.04753) + + + --> STEP: 7/234 -- GLOBAL_STEP: 19195 + | > loss: -0.00369 (0.04037) + | > log_mle: -0.16910 (-0.15319) + | > loss_dur: 0.16541 (0.19356) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.57797 (6.19841) + | > current_lr: 0.00002 + | > step_time: 1.42170 (4.18799) + | > loader_time: 0.08180 (0.05303) + + + --> STEP: 12/234 -- GLOBAL_STEP: 19200 + | > loss: 0.02139 (0.03401) + | > log_mle: -0.15371 (-0.15550) + | > loss_dur: 0.17511 (0.18951) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.04225 (6.14851) + | > current_lr: 0.00002 + | > step_time: 5.18490 (3.34355) + | > loader_time: 0.00240 (0.03218) + + + --> STEP: 17/234 -- GLOBAL_STEP: 19205 + | > loss: 0.03310 (0.02789) + | > log_mle: -0.13521 (-0.15393) + | > loss_dur: 0.16831 (0.18182) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.60626 (5.69133) + | > current_lr: 0.00002 + | > step_time: 7.41400 (3.57324) + | > loader_time: 0.19290 (0.03965) + + + --> STEP: 22/234 -- GLOBAL_STEP: 19210 + | > loss: -0.01271 (0.02529) + | > log_mle: -0.16075 (-0.15247) + | > loss_dur: 0.14805 (0.17777) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.21105 (5.51728) + | > current_lr: 0.00002 + | > step_time: 2.60100 (3.18482) + | > loader_time: 0.00260 (0.03121) + + + --> STEP: 27/234 -- GLOBAL_STEP: 19215 + | > loss: -0.01009 (0.02152) + | > log_mle: -0.16198 (-0.15217) + | > loss_dur: 0.15190 (0.17369) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.23876 (5.22331) + | > current_lr: 0.00002 + | > step_time: 8.89990 (3.78858) + | > loader_time: 0.00270 (0.03658) + + + --> STEP: 32/234 -- GLOBAL_STEP: 19220 + | > loss: -0.03695 (0.01821) + | > log_mle: -0.17217 (-0.15297) + | > loss_dur: 0.13522 (0.17118) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.37107 (5.04179) + | > current_lr: 0.00002 + | > step_time: 0.85370 (3.48286) + | > loader_time: 0.00370 (0.03696) + + + --> STEP: 37/234 -- GLOBAL_STEP: 19225 + | > loss: 0.00072 (0.01781) + | > log_mle: -0.15348 (-0.15323) + | > loss_dur: 0.15421 (0.17103) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.04709 (5.17185) + | > current_lr: 0.00002 + | > step_time: 1.15910 (3.30070) + | > loader_time: 0.00140 (0.03237) + + + --> STEP: 42/234 -- GLOBAL_STEP: 19230 + | > loss: 0.02505 (0.01816) + | > log_mle: -0.14232 (-0.15314) + | > loss_dur: 0.16737 (0.17130) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.87249 (5.17882) + | > current_lr: 0.00002 + | > step_time: 2.80020 (3.11455) + | > loader_time: 0.00170 (0.03080) + + + --> STEP: 47/234 -- GLOBAL_STEP: 19235 + | > loss: 0.00571 (0.01718) + | > log_mle: -0.15519 (-0.15406) + | > loss_dur: 0.16090 (0.17123) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.15602 (5.26119) + | > current_lr: 0.00002 + | > step_time: 0.99140 (2.92587) + | > loader_time: 0.00270 (0.02787) + + + --> STEP: 52/234 -- GLOBAL_STEP: 19240 + | > loss: 0.02794 (0.01661) + | > log_mle: -0.14933 (-0.15353) + | > loss_dur: 0.17727 (0.17014) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.48932 (5.12193) + | > current_lr: 0.00002 + | > step_time: 1.19340 (2.75257) + | > loader_time: 0.00230 (0.02540) + + + --> STEP: 57/234 -- GLOBAL_STEP: 19245 + | > loss: 0.03133 (0.01665) + | > log_mle: -0.14422 (-0.15419) + | > loss_dur: 0.17555 (0.17084) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.44848 (5.01113) + | > current_lr: 0.00002 + | > step_time: 1.30010 (2.66012) + | > loader_time: 0.08540 (0.02489) + + + --> STEP: 62/234 -- GLOBAL_STEP: 19250 + | > loss: 0.03127 (0.01480) + | > log_mle: -0.19899 (-0.15583) + | > loss_dur: 0.23026 (0.17063) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.47930 (5.24202) + | > current_lr: 0.00002 + | > step_time: 1.50050 (2.57434) + | > loader_time: 0.00270 (0.02453) + + + --> STEP: 67/234 -- GLOBAL_STEP: 19255 + | > loss: -0.01817 (0.01441) + | > log_mle: -0.17932 (-0.15610) + | > loss_dur: 0.16115 (0.17051) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.28493 (5.19864) + | > current_lr: 0.00002 + | > step_time: 1.59050 (2.49082) + | > loader_time: 0.00240 (0.02413) + + + --> STEP: 72/234 -- GLOBAL_STEP: 19260 + | > loss: 0.02317 (0.01499) + | > log_mle: -0.15699 (-0.15636) + | > loss_dur: 0.18016 (0.17135) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.87146 (5.55518) + | > current_lr: 0.00002 + | > step_time: 3.64920 (2.46842) + | > loader_time: 0.09580 (0.02395) + + + --> STEP: 77/234 -- GLOBAL_STEP: 19265 + | > loss: -0.00745 (0.01409) + | > log_mle: -0.17403 (-0.15742) + | > loss_dur: 0.16658 (0.17151) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.49084 (5.68153) + | > current_lr: 0.00002 + | > step_time: 3.36000 (2.44847) + | > loader_time: 0.00450 (0.02259) + + + --> STEP: 82/234 -- GLOBAL_STEP: 19270 + | > loss: -0.00588 (0.01334) + | > log_mle: -0.16273 (-0.15783) + | > loss_dur: 0.15685 (0.17118) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.32705 (5.79704) + | > current_lr: 0.00002 + | > step_time: 2.60620 (2.42568) + | > loader_time: 0.00310 (0.02331) + + + --> STEP: 87/234 -- GLOBAL_STEP: 19275 + | > loss: 0.01916 (0.01287) + | > log_mle: -0.17231 (-0.15883) + | > loss_dur: 0.19147 (0.17170) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.22996 (5.94486) + | > current_lr: 0.00002 + | > step_time: 1.89840 (2.40699) + | > loader_time: 0.08810 (0.02310) + + + --> STEP: 92/234 -- GLOBAL_STEP: 19280 + | > loss: -0.04670 (0.01088) + | > log_mle: -0.21790 (-0.16134) + | > loss_dur: 0.17120 (0.17222) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.81250 (6.28279) + | > current_lr: 0.00002 + | > step_time: 1.30850 (2.36118) + | > loader_time: 0.00290 (0.02296) + + + --> STEP: 97/234 -- GLOBAL_STEP: 19285 + | > loss: -0.02574 (0.00850) + | > log_mle: -0.20891 (-0.16475) + | > loss_dur: 0.18317 (0.17326) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.15205 (6.79661) + | > current_lr: 0.00002 + | > step_time: 1.14600 (2.32143) + | > loader_time: 0.00250 (0.02191) + + + --> STEP: 102/234 -- GLOBAL_STEP: 19290 + | > loss: 0.01173 (0.00720) + | > log_mle: -0.18796 (-0.16710) + | > loss_dur: 0.19968 (0.17430) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.82167 (7.10377) + | > current_lr: 0.00002 + | > step_time: 2.09650 (2.30855) + | > loader_time: 0.00280 (0.02099) + + + --> STEP: 107/234 -- GLOBAL_STEP: 19295 + | > loss: -0.05163 (0.00493) + | > log_mle: -0.23483 (-0.17045) + | > loss_dur: 0.18320 (0.17539) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.96349 (7.57224) + | > current_lr: 0.00002 + | > step_time: 1.03360 (2.28502) + | > loader_time: 0.00230 (0.02100) + + + --> STEP: 112/234 -- GLOBAL_STEP: 19300 + | > loss: -0.03642 (0.00330) + | > log_mle: -0.24205 (-0.17345) + | > loss_dur: 0.20562 (0.17675) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.22158 (7.98395) + | > current_lr: 0.00002 + | > step_time: 1.88750 (2.28480) + | > loader_time: 0.10480 (0.02261) + + + --> STEP: 117/234 -- GLOBAL_STEP: 19305 + | > loss: -0.04973 (0.00165) + | > log_mle: -0.23797 (-0.17632) + | > loss_dur: 0.18824 (0.17796) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.01524 (8.40187) + | > current_lr: 0.00002 + | > step_time: 1.90320 (2.27957) + | > loader_time: 0.08040 (0.02315) + + + --> STEP: 122/234 -- GLOBAL_STEP: 19310 + | > loss: -0.03619 (0.00051) + | > log_mle: -0.21733 (-0.17816) + | > loss_dur: 0.18114 (0.17867) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.80203 (8.67292) + | > current_lr: 0.00002 + | > step_time: 1.31140 (2.24863) + | > loader_time: 0.00280 (0.02303) + + + --> STEP: 127/234 -- GLOBAL_STEP: 19315 + | > loss: -0.04823 (-0.00144) + | > log_mle: -0.26742 (-0.18111) + | > loss_dur: 0.21919 (0.17967) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.47063 (9.33666) + | > current_lr: 0.00002 + | > step_time: 1.48360 (2.25517) + | > loader_time: 0.00250 (0.02292) + + + --> STEP: 132/234 -- GLOBAL_STEP: 19320 + | > loss: -0.07086 (-0.00387) + | > log_mle: -0.25160 (-0.18448) + | > loss_dur: 0.18074 (0.18061) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.34537 (9.91455) + | > current_lr: 0.00002 + | > step_time: 1.38660 (2.23911) + | > loader_time: 0.00280 (0.02220) + + + --> STEP: 137/234 -- GLOBAL_STEP: 19325 + | > loss: -0.03545 (-0.00582) + | > log_mle: -0.26400 (-0.18809) + | > loss_dur: 0.22855 (0.18227) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.27870 (10.43463) + | > current_lr: 0.00002 + | > step_time: 2.51130 (2.26773) + | > loader_time: 0.00280 (0.02355) + + + --> STEP: 142/234 -- GLOBAL_STEP: 19330 + | > loss: -0.05108 (-0.00773) + | > log_mle: -0.27746 (-0.19124) + | > loss_dur: 0.22638 (0.18352) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.57364 (11.01967) + | > current_lr: 0.00002 + | > step_time: 1.88570 (2.26453) + | > loader_time: 0.00290 (0.02343) + + + --> STEP: 147/234 -- GLOBAL_STEP: 19335 + | > loss: -0.06700 (-0.01069) + | > log_mle: -0.27929 (-0.19589) + | > loss_dur: 0.21229 (0.18520) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.61723 (11.79071) + | > current_lr: 0.00002 + | > step_time: 2.50950 (2.26160) + | > loader_time: 0.08770 (0.02468) + + + --> STEP: 152/234 -- GLOBAL_STEP: 19340 + | > loss: -0.09729 (-0.01363) + | > log_mle: -0.35355 (-0.20014) + | > loss_dur: 0.25626 (0.18651) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.33482 (12.47571) + | > current_lr: 0.00002 + | > step_time: 4.41420 (2.26626) + | > loader_time: 0.09930 (0.02574) + + + --> STEP: 157/234 -- GLOBAL_STEP: 19345 + | > loss: -0.07022 (-0.01696) + | > log_mle: -0.30470 (-0.20496) + | > loss_dur: 0.23448 (0.18801) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.08444 (13.28061) + | > current_lr: 0.00002 + | > step_time: 2.90390 (2.29138) + | > loader_time: 0.00360 (0.02621) + + + --> STEP: 162/234 -- GLOBAL_STEP: 19350 + | > loss: -0.12503 (-0.02023) + | > log_mle: -0.34032 (-0.20957) + | > loss_dur: 0.21529 (0.18934) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.45724 (14.04619) + | > current_lr: 0.00002 + | > step_time: 1.50380 (2.28989) + | > loader_time: 0.00370 (0.02667) + + + --> STEP: 167/234 -- GLOBAL_STEP: 19355 + | > loss: -0.17692 (-0.02312) + | > log_mle: -0.41294 (-0.21380) + | > loss_dur: 0.23602 (0.19068) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.99271 (14.72214) + | > current_lr: 0.00002 + | > step_time: 1.11280 (2.30954) + | > loader_time: 0.08470 (0.02824) + + + --> STEP: 172/234 -- GLOBAL_STEP: 19360 + | > loss: -0.14211 (-0.02610) + | > log_mle: -0.40233 (-0.21873) + | > loss_dur: 0.26022 (0.19262) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.83464 (15.81461) + | > current_lr: 0.00002 + | > step_time: 6.29380 (2.40762) + | > loader_time: 0.10470 (0.02971) + + + --> STEP: 177/234 -- GLOBAL_STEP: 19365 + | > loss: -0.09746 (-0.02906) + | > log_mle: -0.35471 (-0.22332) + | > loss_dur: 0.25725 (0.19426) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.42463 (17.13318) + | > current_lr: 0.00002 + | > step_time: 1.69650 (2.42741) + | > loader_time: 0.00320 (0.03199) + + + --> STEP: 182/234 -- GLOBAL_STEP: 19370 + | > loss: -0.14068 (-0.03181) + | > log_mle: -0.39996 (-0.22782) + | > loss_dur: 0.25927 (0.19601) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.72587 (18.36548) + | > current_lr: 0.00002 + | > step_time: 5.09230 (2.53756) + | > loader_time: 0.00570 (0.03265) + + + --> STEP: 187/234 -- GLOBAL_STEP: 19375 + | > loss: -0.15601 (-0.03467) + | > log_mle: -0.40498 (-0.23231) + | > loss_dur: 0.24896 (0.19764) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.57894 (19.30656) + | > current_lr: 0.00002 + | > step_time: 4.49110 (2.58084) + | > loader_time: 0.10550 (0.03704) + + + --> STEP: 192/234 -- GLOBAL_STEP: 19380 + | > loss: -0.18213 (-0.03765) + | > log_mle: -0.42736 (-0.23673) + | > loss_dur: 0.24524 (0.19907) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 90.25473 (20.30507) + | > current_lr: 0.00002 + | > step_time: 1.51290 (2.66489) + | > loader_time: 0.08640 (0.03903) + + + --> STEP: 197/234 -- GLOBAL_STEP: 19385 + | > loss: -0.16110 (-0.04048) + | > log_mle: -0.40837 (-0.24102) + | > loss_dur: 0.24727 (0.20054) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.10733 (21.06651) + | > current_lr: 0.00002 + | > step_time: 3.50220 (2.75416) + | > loader_time: 0.00270 (0.03905) + + + --> STEP: 202/234 -- GLOBAL_STEP: 19390 + | > loss: -0.22892 (-0.04331) + | > log_mle: -0.48810 (-0.24540) + | > loss_dur: 0.25918 (0.20209) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 58.64682 (21.72928) + | > current_lr: 0.00002 + | > step_time: 3.19440 (2.79345) + | > loader_time: 0.00470 (0.04090) + + + --> STEP: 207/234 -- GLOBAL_STEP: 19395 + | > loss: -0.19830 (-0.04621) + | > log_mle: -0.47148 (-0.24977) + | > loss_dur: 0.27318 (0.20356) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 83.49442 (22.54613) + | > current_lr: 0.00002 + | > step_time: 4.30010 (2.87519) + | > loader_time: 0.00450 (0.04094) + + + --> STEP: 212/234 -- GLOBAL_STEP: 19400 + | > loss: -0.19320 (-0.04949) + | > log_mle: -0.45983 (-0.25469) + | > loss_dur: 0.26662 (0.20520) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.75479 (23.46244) + | > current_lr: 0.00002 + | > step_time: 7.00390 (2.95847) + | > loader_time: 0.09280 (0.04217) + + + --> STEP: 217/234 -- GLOBAL_STEP: 19405 + | > loss: -0.21342 (-0.05290) + | > log_mle: -0.48203 (-0.25958) + | > loss_dur: 0.26861 (0.20668) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.48307 (24.25213) + | > current_lr: 0.00002 + | > step_time: 6.80240 (2.98772) + | > loader_time: 0.08870 (0.04365) + + + --> STEP: 222/234 -- GLOBAL_STEP: 19410 + | > loss: -0.18051 (-0.05595) + | > log_mle: -0.48393 (-0.26432) + | > loss_dur: 0.30342 (0.20837) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 106.26269 (25.50320) + | > current_lr: 0.00002 + | > step_time: 0.23490 (3.00944) + | > loader_time: 0.00640 (0.04446) + + + --> STEP: 227/234 -- GLOBAL_STEP: 19415 + | > loss: -0.17724 (-0.05929) + | > log_mle: -0.46765 (-0.26940) + | > loss_dur: 0.29041 (0.21011) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.91371 (26.54071) + | > current_lr: 0.00002 + | > step_time: 0.24210 (2.94829) + | > loader_time: 0.00320 (0.04356) + + + --> STEP: 232/234 -- GLOBAL_STEP: 19420 + | > loss: -0.08017 (-0.06128) + | > log_mle: -0.64770 (-0.27561) + | > loss_dur: 0.56753 (0.21432) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 101.98113 (27.78626) + | > current_lr: 0.00002 + | > step_time: 0.34730 (2.89071) + | > loader_time: 0.00480 (0.04272) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.37994 (+0.37830) + | > avg_loss: -0.08310 (+0.02092) + | > avg_log_mle: -0.32753 (+0.02062) + | > avg_loss_dur: 0.24443 (+0.00030) + + + > EPOCH: 83/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 02:37:20)  + + --> STEP: 3/234 -- GLOBAL_STEP: 19425 + | > loss: 0.06222 (0.05180) + | > log_mle: -0.15807 (-0.14970) + | > loss_dur: 0.22029 (0.20150) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.87780 (4.72731) + | > current_lr: 0.00002 + | > step_time: 8.10120 (6.96957) + | > loader_time: 0.29520 (0.13107) + + + --> STEP: 8/234 -- GLOBAL_STEP: 19430 + | > loss: 0.04158 (0.03677) + | > log_mle: -0.16603 (-0.15583) + | > loss_dur: 0.20761 (0.19260) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.16091 (6.08943) + | > current_lr: 0.00002 + | > step_time: 2.40900 (3.41311) + | > loader_time: 0.08740 (0.07296) + + + --> STEP: 13/234 -- GLOBAL_STEP: 19435 + | > loss: 0.03348 (0.03092) + | > log_mle: -0.14759 (-0.15585) + | > loss_dur: 0.18107 (0.18677) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.89280 (6.26208) + | > current_lr: 0.00002 + | > step_time: 2.39560 (3.35399) + | > loader_time: 0.00120 (0.08266) + + + --> STEP: 18/234 -- GLOBAL_STEP: 19440 + | > loss: 0.01505 (0.02424) + | > log_mle: -0.15221 (-0.15477) + | > loss_dur: 0.16726 (0.17901) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.92088 (5.87759) + | > current_lr: 0.00002 + | > step_time: 1.02580 (2.96512) + | > loader_time: 0.09360 (0.06528) + + + --> STEP: 23/234 -- GLOBAL_STEP: 19445 + | > loss: -0.00102 (0.02114) + | > log_mle: -0.15658 (-0.15351) + | > loss_dur: 0.15556 (0.17465) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.85701 (5.51119) + | > current_lr: 0.00002 + | > step_time: 1.79390 (2.63116) + | > loader_time: 0.00240 (0.05151) + + + --> STEP: 28/234 -- GLOBAL_STEP: 19450 + | > loss: -0.01216 (0.01758) + | > log_mle: -0.14853 (-0.15305) + | > loss_dur: 0.13637 (0.17064) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.85025 (5.19527) + | > current_lr: 0.00002 + | > step_time: 3.10300 (2.79018) + | > loader_time: 0.00200 (0.04892) + + + --> STEP: 33/234 -- GLOBAL_STEP: 19455 + | > loss: 0.02603 (0.01564) + | > log_mle: -0.14513 (-0.15391) + | > loss_dur: 0.17117 (0.16955) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.21633 (5.07140) + | > current_lr: 0.00002 + | > step_time: 1.61090 (2.67939) + | > loader_time: 0.00270 (0.05294) + + + --> STEP: 38/234 -- GLOBAL_STEP: 19460 + | > loss: 0.02508 (0.01481) + | > log_mle: -0.16324 (-0.15474) + | > loss_dur: 0.18832 (0.16955) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.65308 (5.25251) + | > current_lr: 0.00002 + | > step_time: 1.36280 (2.53385) + | > loader_time: 0.07480 (0.05019) + + + --> STEP: 43/234 -- GLOBAL_STEP: 19465 + | > loss: 0.00971 (0.01554) + | > log_mle: -0.16660 (-0.15473) + | > loss_dur: 0.17631 (0.17028) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.76897 (5.25674) + | > current_lr: 0.00002 + | > step_time: 1.06950 (2.43756) + | > loader_time: 0.00160 (0.04694) + + + --> STEP: 48/234 -- GLOBAL_STEP: 19470 + | > loss: -0.00483 (0.01431) + | > log_mle: -0.14861 (-0.15526) + | > loss_dur: 0.14378 (0.16958) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.28314 (5.30969) + | > current_lr: 0.00002 + | > step_time: 0.77810 (2.34914) + | > loader_time: 0.00190 (0.04229) + + + --> STEP: 53/234 -- GLOBAL_STEP: 19475 + | > loss: 0.01650 (0.01405) + | > log_mle: -0.17013 (-0.15518) + | > loss_dur: 0.18663 (0.16923) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.49312 (5.21467) + | > current_lr: 0.00002 + | > step_time: 1.51770 (2.33703) + | > loader_time: 0.00690 (0.04021) + + + --> STEP: 58/234 -- GLOBAL_STEP: 19480 + | > loss: -0.00375 (0.01387) + | > log_mle: -0.15517 (-0.15558) + | > loss_dur: 0.15142 (0.16945) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.63717 (5.08069) + | > current_lr: 0.00002 + | > step_time: 2.18750 (2.31237) + | > loader_time: 0.00210 (0.03701) + + + --> STEP: 63/234 -- GLOBAL_STEP: 19485 + | > loss: 0.03019 (0.01284) + | > log_mle: -0.16654 (-0.15743) + | > loss_dur: 0.19673 (0.17027) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.89101 (5.26015) + | > current_lr: 0.00002 + | > step_time: 1.33150 (2.26638) + | > loader_time: 0.00210 (0.03428) + + + --> STEP: 68/234 -- GLOBAL_STEP: 19490 + | > loss: 0.02149 (0.01242) + | > log_mle: -0.15913 (-0.15758) + | > loss_dur: 0.18061 (0.17000) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.62460 (5.23916) + | > current_lr: 0.00002 + | > step_time: 1.42890 (2.21234) + | > loader_time: 0.00280 (0.03303) + + + --> STEP: 73/234 -- GLOBAL_STEP: 19495 + | > loss: 0.00174 (0.01304) + | > log_mle: -0.18275 (-0.15799) + | > loss_dur: 0.18449 (0.17103) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.81550 (6.01785) + | > current_lr: 0.00002 + | > step_time: 3.61980 (2.19331) + | > loader_time: 0.08190 (0.03203) + + + --> STEP: 78/234 -- GLOBAL_STEP: 19500 + | > loss: 0.02442 (0.01235) + | > log_mle: -0.15642 (-0.15865) + | > loss_dur: 0.18085 (0.17100) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.31206 (6.07785) + | > current_lr: 0.00002 + | > step_time: 1.80820 (2.16933) + | > loader_time: 0.00270 (0.03138) + + + --> STEP: 83/234 -- GLOBAL_STEP: 19505 + | > loss: 0.00078 (0.01105) + | > log_mle: -0.18588 (-0.15946) + | > loss_dur: 0.18666 (0.17051) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.94422 (6.17263) + | > current_lr: 0.00002 + | > step_time: 1.31480 (2.16154) + | > loader_time: 0.08850 (0.03174) + + + --> STEP: 88/234 -- GLOBAL_STEP: 19510 + | > loss: -0.02722 (0.01004) + | > log_mle: -0.22275 (-0.16091) + | > loss_dur: 0.19554 (0.17095) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.47031 (6.34691) + | > current_lr: 0.00002 + | > step_time: 1.09490 (2.13745) + | > loader_time: 0.00210 (0.03100) + + + --> STEP: 93/234 -- GLOBAL_STEP: 19515 + | > loss: -0.03608 (0.00814) + | > log_mle: -0.23513 (-0.16358) + | > loss_dur: 0.19906 (0.17171) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.80123 (6.71431) + | > current_lr: 0.00002 + | > step_time: 2.49760 (2.13206) + | > loader_time: 0.00270 (0.03039) + + + --> STEP: 98/234 -- GLOBAL_STEP: 19520 + | > loss: 0.02066 (0.00629) + | > log_mle: -0.16100 (-0.16624) + | > loss_dur: 0.18165 (0.17252) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.86795 (7.03280) + | > current_lr: 0.00002 + | > step_time: 2.99070 (2.13779) + | > loader_time: 0.00500 (0.03163) + + + --> STEP: 103/234 -- GLOBAL_STEP: 19525 + | > loss: -0.03508 (0.00437) + | > log_mle: -0.25843 (-0.16952) + | > loss_dur: 0.22335 (0.17390) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.20204 (7.60692) + | > current_lr: 0.00002 + | > step_time: 1.50870 (2.11671) + | > loader_time: 0.00290 (0.03187) + + + --> STEP: 108/234 -- GLOBAL_STEP: 19530 + | > loss: -0.01729 (0.00250) + | > log_mle: -0.20616 (-0.17234) + | > loss_dur: 0.18888 (0.17485) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.63101 (8.12253) + | > current_lr: 0.00002 + | > step_time: 1.60060 (2.10672) + | > loader_time: 0.00200 (0.03131) + + + --> STEP: 113/234 -- GLOBAL_STEP: 19535 + | > loss: -0.05192 (0.00077) + | > log_mle: -0.25386 (-0.17574) + | > loss_dur: 0.20194 (0.17651) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.03689 (8.80275) + | > current_lr: 0.00002 + | > step_time: 2.69780 (2.11157) + | > loader_time: 0.00180 (0.03004) + + + --> STEP: 118/234 -- GLOBAL_STEP: 19540 + | > loss: -0.01763 (-0.00045) + | > log_mle: -0.22411 (-0.17830) + | > loss_dur: 0.20648 (0.17785) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.30636 (9.09855) + | > current_lr: 0.00002 + | > step_time: 1.59240 (2.08959) + | > loader_time: 0.00240 (0.02888) + + + --> STEP: 123/234 -- GLOBAL_STEP: 19545 + | > loss: -0.00991 (-0.00160) + | > log_mle: -0.19446 (-0.17993) + | > loss_dur: 0.18456 (0.17833) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.02134 (9.22075) + | > current_lr: 0.00002 + | > step_time: 1.09410 (2.07142) + | > loader_time: 0.00260 (0.02846) + + + --> STEP: 128/234 -- GLOBAL_STEP: 19550 + | > loss: -0.06665 (-0.00396) + | > log_mle: -0.25546 (-0.18348) + | > loss_dur: 0.18882 (0.17952) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.48437 (9.64455) + | > current_lr: 0.00002 + | > step_time: 1.91500 (2.07346) + | > loader_time: 0.07720 (0.02869) + + + --> STEP: 133/234 -- GLOBAL_STEP: 19555 + | > loss: -0.05619 (-0.00620) + | > log_mle: -0.27471 (-0.18700) + | > loss_dur: 0.21852 (0.18080) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.97609 (10.09900) + | > current_lr: 0.00002 + | > step_time: 1.30320 (2.04917) + | > loader_time: 0.00350 (0.02900) + + + --> STEP: 138/234 -- GLOBAL_STEP: 19560 + | > loss: -0.03675 (-0.00803) + | > log_mle: -0.23324 (-0.19029) + | > loss_dur: 0.19649 (0.18226) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.66528 (10.52532) + | > current_lr: 0.00002 + | > step_time: 1.20430 (2.06481) + | > loader_time: 0.08930 (0.02932) + + + --> STEP: 143/234 -- GLOBAL_STEP: 19565 + | > loss: -0.09319 (-0.01039) + | > log_mle: -0.35852 (-0.19428) + | > loss_dur: 0.26533 (0.18389) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.66371 (11.39396) + | > current_lr: 0.00002 + | > step_time: 1.45400 (2.08323) + | > loader_time: 0.00280 (0.02963) + + + --> STEP: 148/234 -- GLOBAL_STEP: 19570 + | > loss: -0.08626 (-0.01308) + | > log_mle: -0.28329 (-0.19834) + | > loss_dur: 0.19703 (0.18527) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.34586 (12.02228) + | > current_lr: 0.00002 + | > step_time: 1.90050 (2.09543) + | > loader_time: 0.00620 (0.03049) + + + --> STEP: 153/234 -- GLOBAL_STEP: 19575 + | > loss: -0.16620 (-0.01659) + | > log_mle: -0.40235 (-0.20338) + | > loss_dur: 0.23615 (0.18680) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.34356 (12.63809) + | > current_lr: 0.00002 + | > step_time: 3.30020 (2.10870) + | > loader_time: 0.07560 (0.03227) + + + --> STEP: 158/234 -- GLOBAL_STEP: 19580 + | > loss: -0.09936 (-0.01958) + | > log_mle: -0.34206 (-0.20780) + | > loss_dur: 0.24270 (0.18822) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.95184 (13.41305) + | > current_lr: 0.00002 + | > step_time: 1.59460 (2.09886) + | > loader_time: 0.00210 (0.03204) + + + --> STEP: 163/234 -- GLOBAL_STEP: 19585 + | > loss: -0.09202 (-0.02273) + | > log_mle: -0.31617 (-0.21222) + | > loss_dur: 0.22415 (0.18948) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.95504 (14.08874) + | > current_lr: 0.00002 + | > step_time: 3.70760 (2.13269) + | > loader_time: 0.00260 (0.03349) + + + --> STEP: 168/234 -- GLOBAL_STEP: 19590 + | > loss: -0.10636 (-0.02580) + | > log_mle: -0.36458 (-0.21668) + | > loss_dur: 0.25822 (0.19088) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.99583 (14.92889) + | > current_lr: 0.00002 + | > step_time: 3.29460 (2.23108) + | > loader_time: 0.00280 (0.03652) + + + --> STEP: 173/234 -- GLOBAL_STEP: 19595 + | > loss: -0.12621 (-0.02912) + | > log_mle: -0.37093 (-0.22170) + | > loss_dur: 0.24471 (0.19258) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.52398 (15.62813) + | > current_lr: 0.00002 + | > step_time: 8.59890 (2.37181) + | > loader_time: 0.10120 (0.04046) + + + --> STEP: 178/234 -- GLOBAL_STEP: 19600 + | > loss: -0.15885 (-0.03245) + | > log_mle: -0.42025 (-0.22666) + | > loss_dur: 0.26140 (0.19420) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.98356 (16.64293) + | > current_lr: 0.00002 + | > step_time: 1.90950 (2.35767) + | > loader_time: 0.00320 (0.04147) + + + --> STEP: 183/234 -- GLOBAL_STEP: 19605 + | > loss: -0.16932 (-0.03531) + | > log_mle: -0.42073 (-0.23118) + | > loss_dur: 0.25141 (0.19587) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.48513 (17.55550) + | > current_lr: 0.00002 + | > step_time: 2.10240 (2.39812) + | > loader_time: 0.08240 (0.04090) + + + --> STEP: 188/234 -- GLOBAL_STEP: 19610 + | > loss: -0.17644 (-0.03815) + | > log_mle: -0.43490 (-0.23574) + | > loss_dur: 0.25846 (0.19759) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.64915 (18.42173) + | > current_lr: 0.00002 + | > step_time: 9.00210 (2.47313) + | > loader_time: 0.10280 (0.04096) + + + --> STEP: 193/234 -- GLOBAL_STEP: 19615 + | > loss: -0.17189 (-0.04122) + | > log_mle: -0.43063 (-0.24012) + | > loss_dur: 0.25875 (0.19890) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 73.52656 (19.53487) + | > current_lr: 0.00002 + | > step_time: 5.40100 (2.51226) + | > loader_time: 0.09990 (0.04133) + + + --> STEP: 198/234 -- GLOBAL_STEP: 19620 + | > loss: -0.16320 (-0.04406) + | > log_mle: -0.42581 (-0.24437) + | > loss_dur: 0.26261 (0.20031) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 58.43461 (20.29712) + | > current_lr: 0.00002 + | > step_time: 5.90140 (2.58573) + | > loader_time: 0.08730 (0.04312) + + + --> STEP: 203/234 -- GLOBAL_STEP: 19625 + | > loss: -0.12385 (-0.04671) + | > log_mle: -0.36715 (-0.24847) + | > loss_dur: 0.24330 (0.20176) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.61499 (21.05893) + | > current_lr: 0.00002 + | > step_time: 5.39730 (2.62350) + | > loader_time: 0.18340 (0.04496) + + + --> STEP: 208/234 -- GLOBAL_STEP: 19630 + | > loss: -0.15542 (-0.04980) + | > log_mle: -0.42623 (-0.25317) + | > loss_dur: 0.27081 (0.20337) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 161.03865 (22.36756) + | > current_lr: 0.00002 + | > step_time: 11.69530 (2.74298) + | > loader_time: 0.00570 (0.04452) + + + --> STEP: 213/234 -- GLOBAL_STEP: 19635 + | > loss: -0.19656 (-0.05296) + | > log_mle: -0.47485 (-0.25802) + | > loss_dur: 0.27829 (0.20507) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.42904 (23.66940) + | > current_lr: 0.00002 + | > step_time: 6.79700 (2.81134) + | > loader_time: 0.19970 (0.04633) + + + --> STEP: 218/234 -- GLOBAL_STEP: 19640 + | > loss: -0.16368 (-0.05605) + | > log_mle: -0.44466 (-0.26258) + | > loss_dur: 0.28097 (0.20653) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.04628 (24.70485) + | > current_lr: 0.00002 + | > step_time: 3.79530 (2.85454) + | > loader_time: 0.00450 (0.04627) + + + --> STEP: 223/234 -- GLOBAL_STEP: 19645 + | > loss: -0.21312 (-0.05935) + | > log_mle: -0.48756 (-0.26752) + | > loss_dur: 0.27444 (0.20817) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.33607 (25.57015) + | > current_lr: 0.00002 + | > step_time: 0.24120 (2.81107) + | > loader_time: 0.00360 (0.04568) + + + --> STEP: 228/234 -- GLOBAL_STEP: 19650 + | > loss: -0.18502 (-0.06257) + | > log_mle: -0.48613 (-0.27258) + | > loss_dur: 0.30111 (0.21001) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.30045 (26.64110) + | > current_lr: 0.00002 + | > step_time: 0.24820 (2.75470) + | > loader_time: 0.00480 (0.04477) + + + --> STEP: 233/234 -- GLOBAL_STEP: 19655 + | > loss: 0.47209 (-0.06184) + | > log_mle: -0.43598 (-0.27857) + | > loss_dur: 0.90808 (0.21673) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 127.74635 (28.13191) + | > current_lr: 0.00002 + | > step_time: 0.19770 (2.70135) + | > loader_time: 0.00310 (0.04391) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07592 (-0.30402) + | > avg_loss: -0.10626 (-0.02316) + | > avg_log_mle: -0.35192 (-0.02438) + | > avg_loss_dur: 0.24566 (+0.00123) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_19656.pth + + > EPOCH: 84/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 02:49:10)  + + --> STEP: 4/234 -- GLOBAL_STEP: 19660 + | > loss: 0.04170 (0.04422) + | > log_mle: -0.16281 (-0.15441) + | > loss_dur: 0.20451 (0.19863) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.73172 (6.33738) + | > current_lr: 0.00002 + | > step_time: 4.10790 (4.74683) + | > loader_time: 0.00180 (0.27809) + + + --> STEP: 9/234 -- GLOBAL_STEP: 19665 + | > loss: 0.02081 (0.03066) + | > log_mle: -0.17063 (-0.15998) + | > loss_dur: 0.19144 (0.19064) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.19935 (5.75442) + | > current_lr: 0.00002 + | > step_time: 2.59210 (4.89869) + | > loader_time: 0.00350 (0.15625) + + + --> STEP: 14/234 -- GLOBAL_STEP: 19670 + | > loss: -0.00548 (0.02279) + | > log_mle: -0.16350 (-0.15908) + | > loss_dur: 0.15802 (0.18186) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.88455 (5.60527) + | > current_lr: 0.00002 + | > step_time: 5.19820 (4.64994) + | > loader_time: 0.08700 (0.11463) + + + --> STEP: 19/234 -- GLOBAL_STEP: 19675 + | > loss: 0.00774 (0.01943) + | > log_mle: -0.14726 (-0.15706) + | > loss_dur: 0.15500 (0.17648) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.83060 (5.32521) + | > current_lr: 0.00002 + | > step_time: 4.69580 (4.56309) + | > loader_time: 0.01020 (0.10444) + + + --> STEP: 24/234 -- GLOBAL_STEP: 19680 + | > loss: -0.00749 (0.01527) + | > log_mle: -0.14928 (-0.15590) + | > loss_dur: 0.14179 (0.17118) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.69990 (5.05667) + | > current_lr: 0.00002 + | > step_time: 1.81140 (4.39195) + | > loader_time: 0.00250 (0.10658) + + + --> STEP: 29/234 -- GLOBAL_STEP: 19685 + | > loss: -0.00552 (0.01330) + | > log_mle: -0.14809 (-0.15547) + | > loss_dur: 0.14257 (0.16877) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.28721 (4.82692) + | > current_lr: 0.00002 + | > step_time: 8.20400 (4.53511) + | > loader_time: 0.10700 (0.11610) + + + --> STEP: 34/234 -- GLOBAL_STEP: 19690 + | > loss: 0.02433 (0.01211) + | > log_mle: -0.15705 (-0.15648) + | > loss_dur: 0.18138 (0.16860) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.56037 (4.70866) + | > current_lr: 0.00002 + | > step_time: 3.49420 (4.24193) + | > loader_time: 0.10300 (0.10472) + + + --> STEP: 39/234 -- GLOBAL_STEP: 19695 + | > loss: 0.01994 (0.01237) + | > log_mle: -0.16560 (-0.15744) + | > loss_dur: 0.18554 (0.16981) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.13482 (5.20341) + | > current_lr: 0.00002 + | > step_time: 1.18330 (3.89251) + | > loader_time: 0.00230 (0.09165) + + + --> STEP: 44/234 -- GLOBAL_STEP: 19700 + | > loss: -0.00981 (0.01266) + | > log_mle: -0.15240 (-0.15697) + | > loss_dur: 0.14258 (0.16962) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.97968 (5.15970) + | > current_lr: 0.00002 + | > step_time: 1.78960 (3.58301) + | > loader_time: 0.00120 (0.08148) + + + --> STEP: 49/234 -- GLOBAL_STEP: 19705 + | > loss: -0.01548 (0.01100) + | > log_mle: -0.16316 (-0.15755) + | > loss_dur: 0.14767 (0.16856) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.26994 (5.18108) + | > current_lr: 0.00002 + | > step_time: 2.03100 (3.39697) + | > loader_time: 0.00180 (0.07358) + + + --> STEP: 54/234 -- GLOBAL_STEP: 19710 + | > loss: -0.00708 (0.01106) + | > log_mle: -0.16833 (-0.15755) + | > loss_dur: 0.16125 (0.16862) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.48822 (5.04164) + | > current_lr: 0.00002 + | > step_time: 1.63300 (3.23111) + | > loader_time: 0.00180 (0.07006) + + + --> STEP: 59/234 -- GLOBAL_STEP: 19715 + | > loss: -0.03881 (0.01024) + | > log_mle: -0.18308 (-0.15812) + | > loss_dur: 0.14427 (0.16836) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.86245 (5.00056) + | > current_lr: 0.00002 + | > step_time: 1.91070 (3.07619) + | > loader_time: 0.00240 (0.06568) + + + --> STEP: 64/234 -- GLOBAL_STEP: 19720 + | > loss: 0.00126 (0.00958) + | > log_mle: -0.15168 (-0.15939) + | > loss_dur: 0.15294 (0.16898) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.37404 (5.20633) + | > current_lr: 0.00002 + | > step_time: 1.00440 (2.97819) + | > loader_time: 0.08920 (0.06484) + + + --> STEP: 69/234 -- GLOBAL_STEP: 19725 + | > loss: 0.03844 (0.00968) + | > log_mle: -0.14116 (-0.15936) + | > loss_dur: 0.17961 (0.16904) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.66949 (5.25058) + | > current_lr: 0.00002 + | > step_time: 1.21430 (2.89158) + | > loader_time: 0.08800 (0.06383) + + + --> STEP: 74/234 -- GLOBAL_STEP: 19730 + | > loss: -0.01835 (0.00950) + | > log_mle: -0.16286 (-0.16026) + | > loss_dur: 0.14451 (0.16976) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.58344 (5.62477) + | > current_lr: 0.00002 + | > step_time: 2.71770 (2.82345) + | > loader_time: 0.08940 (0.06086) + + + --> STEP: 79/234 -- GLOBAL_STEP: 19735 + | > loss: -0.01353 (0.00858) + | > log_mle: -0.17430 (-0.16102) + | > loss_dur: 0.16077 (0.16959) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.26967 (5.72344) + | > current_lr: 0.00002 + | > step_time: 3.92200 (2.82079) + | > loader_time: 0.09940 (0.06069) + + + --> STEP: 84/234 -- GLOBAL_STEP: 19740 + | > loss: 0.00312 (0.00759) + | > log_mle: -0.16918 (-0.16170) + | > loss_dur: 0.17230 (0.16929) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.06324 (5.90267) + | > current_lr: 0.00002 + | > step_time: 1.99350 (2.75158) + | > loader_time: 0.00270 (0.05725) + + + --> STEP: 89/234 -- GLOBAL_STEP: 19745 + | > loss: -0.03334 (0.00623) + | > log_mle: -0.20187 (-0.16331) + | > loss_dur: 0.16853 (0.16954) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.17217 (6.25752) + | > current_lr: 0.00002 + | > step_time: 2.90220 (2.71604) + | > loader_time: 0.00260 (0.05422) + + + --> STEP: 94/234 -- GLOBAL_STEP: 19750 + | > loss: -0.05663 (0.00407) + | > log_mle: -0.23583 (-0.16618) + | > loss_dur: 0.17921 (0.17024) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.11841 (6.66895) + | > current_lr: 0.00002 + | > step_time: 2.28450 (2.68135) + | > loader_time: 0.00170 (0.05148) + + + --> STEP: 99/234 -- GLOBAL_STEP: 19755 + | > loss: -0.08025 (0.00213) + | > log_mle: -0.26849 (-0.16897) + | > loss_dur: 0.18824 (0.17110) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.00747 (7.14678) + | > current_lr: 0.00002 + | > step_time: 2.09260 (2.66769) + | > loader_time: 0.00230 (0.04999) + + + --> STEP: 104/234 -- GLOBAL_STEP: 19760 + | > loss: -0.08725 (-0.00006) + | > log_mle: -0.28013 (-0.17226) + | > loss_dur: 0.19288 (0.17220) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.18087 (7.65954) + | > current_lr: 0.00002 + | > step_time: 2.21220 (2.66174) + | > loader_time: 0.09160 (0.04942) + + + --> STEP: 109/234 -- GLOBAL_STEP: 19765 + | > loss: -0.02125 (-0.00113) + | > log_mle: -0.24852 (-0.17464) + | > loss_dur: 0.22727 (0.17351) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.42598 (8.18495) + | > current_lr: 0.00002 + | > step_time: 1.56260 (2.64158) + | > loader_time: 0.00230 (0.04962) + + + --> STEP: 114/234 -- GLOBAL_STEP: 19770 + | > loss: -0.03786 (-0.00296) + | > log_mle: -0.23261 (-0.17775) + | > loss_dur: 0.19475 (0.17480) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.03327 (8.74036) + | > current_lr: 0.00002 + | > step_time: 2.30690 (2.59663) + | > loader_time: 0.10320 (0.04844) + + + --> STEP: 119/234 -- GLOBAL_STEP: 19775 + | > loss: -0.02715 (-0.00388) + | > log_mle: -0.23158 (-0.18019) + | > loss_dur: 0.20442 (0.17632) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.51198 (8.99631) + | > current_lr: 0.00002 + | > step_time: 3.12390 (2.57665) + | > loader_time: 0.00280 (0.04725) + + + --> STEP: 124/234 -- GLOBAL_STEP: 19780 + | > loss: -0.06561 (-0.00515) + | > log_mle: -0.26015 (-0.18200) + | > loss_dur: 0.19455 (0.17685) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.62644 (9.20555) + | > current_lr: 0.00002 + | > step_time: 2.92630 (2.55380) + | > loader_time: 0.00470 (0.04618) + + + --> STEP: 129/234 -- GLOBAL_STEP: 19785 + | > loss: -0.03725 (-0.00714) + | > log_mle: -0.24499 (-0.18532) + | > loss_dur: 0.20774 (0.17817) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.95705 (9.89905) + | > current_lr: 0.00002 + | > step_time: 3.09350 (2.52961) + | > loader_time: 0.00370 (0.04591) + + + --> STEP: 134/234 -- GLOBAL_STEP: 19790 + | > loss: -0.06387 (-0.00942) + | > log_mle: -0.29827 (-0.18921) + | > loss_dur: 0.23440 (0.17979) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.91646 (10.49570) + | > current_lr: 0.00002 + | > step_time: 3.49140 (2.52019) + | > loader_time: 0.09620 (0.04627) + + + --> STEP: 139/234 -- GLOBAL_STEP: 19795 + | > loss: -0.14073 (-0.01185) + | > log_mle: -0.35071 (-0.19281) + | > loss_dur: 0.20999 (0.18096) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.69006 (11.13645) + | > current_lr: 0.00002 + | > step_time: 1.40070 (2.50960) + | > loader_time: 0.00380 (0.04535) + + + --> STEP: 144/234 -- GLOBAL_STEP: 19800 + | > loss: -0.09403 (-0.01380) + | > log_mle: -0.32963 (-0.19658) + | > loss_dur: 0.23560 (0.18278) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.60334 (12.01580) + | > current_lr: 0.00002 + | > step_time: 1.50970 (2.49421) + | > loader_time: 0.00290 (0.04514) + + + --> STEP: 149/234 -- GLOBAL_STEP: 19805 + | > loss: -0.13739 (-0.01688) + | > log_mle: -0.37472 (-0.20090) + | > loss_dur: 0.23733 (0.18402) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.25832 (12.67766) + | > current_lr: 0.00002 + | > step_time: 1.40280 (2.48762) + | > loader_time: 0.08750 (0.04628) + + + --> STEP: 154/234 -- GLOBAL_STEP: 19810 + | > loss: -0.12421 (-0.02031) + | > log_mle: -0.33695 (-0.20564) + | > loss_dur: 0.21274 (0.18532) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.65859 (13.40270) + | > current_lr: 0.00002 + | > step_time: 3.08530 (2.51916) + | > loader_time: 0.09760 (0.04687) + + + --> STEP: 159/234 -- GLOBAL_STEP: 19815 + | > loss: -0.12514 (-0.02327) + | > log_mle: -0.35505 (-0.21015) + | > loss_dur: 0.22990 (0.18687) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.33320 (14.23338) + | > current_lr: 0.00002 + | > step_time: 1.39050 (2.53488) + | > loader_time: 0.09760 (0.04730) + + + --> STEP: 164/234 -- GLOBAL_STEP: 19820 + | > loss: -0.09890 (-0.02615) + | > log_mle: -0.33689 (-0.21434) + | > loss_dur: 0.23798 (0.18819) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.24896 (15.36741) + | > current_lr: 0.00002 + | > step_time: 1.90250 (2.54663) + | > loader_time: 0.00510 (0.04658) + + + --> STEP: 169/234 -- GLOBAL_STEP: 19825 + | > loss: -0.09168 (-0.02888) + | > log_mle: -0.33768 (-0.21856) + | > loss_dur: 0.24600 (0.18968) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.49280 (16.36927) + | > current_lr: 0.00002 + | > step_time: 2.39310 (2.53223) + | > loader_time: 0.00330 (0.04626) + + + --> STEP: 174/234 -- GLOBAL_STEP: 19830 + | > loss: -0.18696 (-0.03256) + | > log_mle: -0.42637 (-0.22391) + | > loss_dur: 0.23941 (0.19135) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.94910 (17.39335) + | > current_lr: 0.00002 + | > step_time: 2.99800 (2.54907) + | > loader_time: 0.10190 (0.04770) + + + --> STEP: 179/234 -- GLOBAL_STEP: 19835 + | > loss: -0.15046 (-0.03549) + | > log_mle: -0.41537 (-0.22870) + | > loss_dur: 0.26491 (0.19321) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.97388 (18.49209) + | > current_lr: 0.00002 + | > step_time: 4.38710 (2.64369) + | > loader_time: 0.01060 (0.05086) + + + --> STEP: 184/234 -- GLOBAL_STEP: 19840 + | > loss: -0.14040 (-0.03825) + | > log_mle: -0.38891 (-0.23302) + | > loss_dur: 0.24851 (0.19477) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.87035 (19.35618) + | > current_lr: 0.00002 + | > step_time: 5.70370 (2.68816) + | > loader_time: 0.18870 (0.05216) + + + --> STEP: 189/234 -- GLOBAL_STEP: 19845 + | > loss: -0.12563 (-0.04101) + | > log_mle: -0.38904 (-0.23753) + | > loss_dur: 0.26342 (0.19652) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.83010 (20.10266) + | > current_lr: 0.00002 + | > step_time: 6.61060 (2.72343) + | > loader_time: 0.19550 (0.05338) + + + --> STEP: 194/234 -- GLOBAL_STEP: 19850 + | > loss: -0.16603 (-0.04426) + | > log_mle: -0.41862 (-0.24203) + | > loss_dur: 0.25259 (0.19777) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.70980 (20.84714) + | > current_lr: 0.00002 + | > step_time: 5.88820 (2.77050) + | > loader_time: 0.00210 (0.05363) + + + --> STEP: 199/234 -- GLOBAL_STEP: 19855 + | > loss: -0.17881 (-0.04714) + | > log_mle: -0.42977 (-0.24630) + | > loss_dur: 0.25096 (0.19916) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.92684 (21.41331) + | > current_lr: 0.00002 + | > step_time: 3.71560 (2.85586) + | > loader_time: 0.09280 (0.05520) + + + --> STEP: 204/234 -- GLOBAL_STEP: 19860 + | > loss: -0.17863 (-0.04966) + | > log_mle: -0.45669 (-0.25045) + | > loss_dur: 0.27806 (0.20080) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.27021 (22.01241) + | > current_lr: 0.00002 + | > step_time: 2.80430 (2.89570) + | > loader_time: 0.18810 (0.05621) + + + --> STEP: 209/234 -- GLOBAL_STEP: 19865 + | > loss: -0.15861 (-0.05263) + | > log_mle: -0.41920 (-0.25491) + | > loss_dur: 0.26059 (0.20227) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.00805 (22.75456) + | > current_lr: 0.00002 + | > step_time: 8.49320 (2.96363) + | > loader_time: 0.09640 (0.05594) + + + --> STEP: 214/234 -- GLOBAL_STEP: 19870 + | > loss: -0.20315 (-0.05624) + | > log_mle: -0.45040 (-0.26015) + | > loss_dur: 0.24725 (0.20391) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.23862 (23.51603) + | > current_lr: 0.00002 + | > step_time: 8.08260 (2.99907) + | > loader_time: 0.11030 (0.05708) + + + --> STEP: 219/234 -- GLOBAL_STEP: 19875 + | > loss: -0.25694 (-0.05957) + | > log_mle: -0.53680 (-0.26516) + | > loss_dur: 0.27986 (0.20558) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 79.69873 (24.50741) + | > current_lr: 0.00002 + | > step_time: 1.41290 (2.99719) + | > loader_time: 0.08590 (0.05629) + + + --> STEP: 224/234 -- GLOBAL_STEP: 19880 + | > loss: -0.20945 (-0.06265) + | > log_mle: -0.49443 (-0.26988) + | > loss_dur: 0.28498 (0.20723) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 78.57977 (25.35480) + | > current_lr: 0.00002 + | > step_time: 0.98440 (2.96703) + | > loader_time: 0.00510 (0.05550) + + + --> STEP: 229/234 -- GLOBAL_STEP: 19885 + | > loss: -0.16949 (-0.06566) + | > log_mle: -0.51307 (-0.27497) + | > loss_dur: 0.34357 (0.20932) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 148.65446 (26.80034) + | > current_lr: 0.00002 + | > step_time: 0.25370 (2.90775) + | > loader_time: 0.00420 (0.05436) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55090 (+0.47498) + | > avg_loss: -0.10296 (+0.00330) + | > avg_log_mle: -0.34669 (+0.00523) + | > avg_loss_dur: 0.24373 (-0.00193) + + + > EPOCH: 85/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 03:01:37)  + + --> STEP: 0/234 -- GLOBAL_STEP: 19890 + | > loss: -0.01822 (-0.01822) + | > log_mle: -0.20895 (-0.20895) + | > loss_dur: 0.19072 (0.19072) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.99468 (5.99468) + | > current_lr: 0.00002 + | > step_time: 4.09690 (4.09686) + | > loader_time: 10.95160 (10.95156) + + + --> STEP: 5/234 -- GLOBAL_STEP: 19895 + | > loss: 0.02448 (0.04487) + | > log_mle: -0.16366 (-0.15715) + | > loss_dur: 0.18814 (0.20202) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.28098 (9.29645) + | > current_lr: 0.00002 + | > step_time: 6.39320 (3.03967) + | > loader_time: 0.29420 (1.31778) + + + --> STEP: 10/234 -- GLOBAL_STEP: 19900 + | > loss: 0.01380 (0.03182) + | > log_mle: -0.16437 (-0.16127) + | > loss_dur: 0.17818 (0.19308) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.19136 (7.87999) + | > current_lr: 0.00002 + | > step_time: 14.19460 (4.30044) + | > loader_time: 0.09290 (0.69777) + + + --> STEP: 15/234 -- GLOBAL_STEP: 19905 + | > loss: -0.02003 (0.02320) + | > log_mle: -0.16040 (-0.16008) + | > loss_dur: 0.14037 (0.18328) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.41057 (7.12837) + | > current_lr: 0.00002 + | > step_time: 1.70200 (4.30778) + | > loader_time: 0.00160 (0.47916) + + + --> STEP: 20/234 -- GLOBAL_STEP: 19910 + | > loss: 0.01320 (0.02121) + | > log_mle: -0.14830 (-0.15744) + | > loss_dur: 0.16150 (0.17865) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.04925 (6.56756) + | > current_lr: 0.00002 + | > step_time: 1.19400 (3.62069) + | > loader_time: 0.00150 (0.39961) + + + --> STEP: 25/234 -- GLOBAL_STEP: 19915 + | > loss: 0.02477 (0.01733) + | > log_mle: -0.14327 (-0.15634) + | > loss_dur: 0.16804 (0.17366) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.81884 (6.24017) + | > current_lr: 0.00002 + | > step_time: 1.70210 (3.13715) + | > loader_time: 0.00400 (0.32020) + + + --> STEP: 30/234 -- GLOBAL_STEP: 19920 + | > loss: -0.02594 (0.01228) + | > log_mle: -0.17195 (-0.15691) + | > loss_dur: 0.14601 (0.16919) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.72481 (5.98369) + | > current_lr: 0.00002 + | > step_time: 1.90210 (3.06437) + | > loader_time: 0.00130 (0.26726) + + + --> STEP: 35/234 -- GLOBAL_STEP: 19925 + | > loss: -0.00908 (0.01183) + | > log_mle: -0.16796 (-0.15770) + | > loss_dur: 0.15888 (0.16953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.17186 (6.14413) + | > current_lr: 0.00002 + | > step_time: 2.10180 (3.08367) + | > loader_time: 0.09500 (0.23745) + + + --> STEP: 40/234 -- GLOBAL_STEP: 19930 + | > loss: 0.03485 (0.01242) + | > log_mle: -0.14709 (-0.15813) + | > loss_dur: 0.18194 (0.17055) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.66970 (6.18334) + | > current_lr: 0.00002 + | > step_time: 1.11800 (2.89388) + | > loader_time: 0.00240 (0.21201) + + + --> STEP: 45/234 -- GLOBAL_STEP: 19935 + | > loss: -0.01788 (0.01137) + | > log_mle: -0.18201 (-0.15859) + | > loss_dur: 0.16413 (0.16995) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.35574 (6.14114) + | > current_lr: 0.00002 + | > step_time: 1.17460 (2.72931) + | > loader_time: 0.00150 (0.19058) + + + --> STEP: 50/234 -- GLOBAL_STEP: 19940 + | > loss: 0.01588 (0.00989) + | > log_mle: -0.15089 (-0.15848) + | > loss_dur: 0.16677 (0.16837) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.59599 (5.95160) + | > current_lr: 0.00002 + | > step_time: 1.11410 (2.63423) + | > loader_time: 0.08440 (0.17541) + + + --> STEP: 55/234 -- GLOBAL_STEP: 19945 + | > loss: -0.01069 (0.00899) + | > log_mle: -0.17137 (-0.15893) + | > loss_dur: 0.16067 (0.16792) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.69520 (5.80888) + | > current_lr: 0.00002 + | > step_time: 2.31580 (2.59308) + | > loader_time: 0.10050 (0.16299) + + + --> STEP: 60/234 -- GLOBAL_STEP: 19950 + | > loss: -0.03232 (0.00774) + | > log_mle: -0.18872 (-0.15983) + | > loss_dur: 0.15640 (0.16757) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.90654 (5.82752) + | > current_lr: 0.00002 + | > step_time: 1.90290 (2.55748) + | > loader_time: 0.09760 (0.15261) + + + --> STEP: 65/234 -- GLOBAL_STEP: 19955 + | > loss: -0.01040 (0.00700) + | > log_mle: -0.16273 (-0.16070) + | > loss_dur: 0.15233 (0.16770) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.67829 (5.84198) + | > current_lr: 0.00002 + | > step_time: 1.98650 (2.48266) + | > loader_time: 0.00270 (0.14258) + + + --> STEP: 70/234 -- GLOBAL_STEP: 19960 + | > loss: 0.00856 (0.00697) + | > log_mle: -0.16512 (-0.16076) + | > loss_dur: 0.17368 (0.16774) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.35187 (6.13350) + | > current_lr: 0.00002 + | > step_time: 4.00220 (2.47088) + | > loader_time: 0.00650 (0.13526) + + + --> STEP: 75/234 -- GLOBAL_STEP: 19965 + | > loss: -0.00504 (0.00666) + | > log_mle: -0.17762 (-0.16182) + | > loss_dur: 0.17259 (0.16848) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.06280 (6.40813) + | > current_lr: 0.00002 + | > step_time: 3.30160 (2.43451) + | > loader_time: 0.00280 (0.12652) + + + --> STEP: 80/234 -- GLOBAL_STEP: 19970 + | > loss: -0.01517 (0.00578) + | > log_mle: -0.15661 (-0.16228) + | > loss_dur: 0.14143 (0.16806) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 2.69882 (6.39614) + | > current_lr: 0.00002 + | > step_time: 1.91830 (2.40113) + | > loader_time: 0.08450 (0.12183) + + + --> STEP: 85/234 -- GLOBAL_STEP: 19975 + | > loss: -0.00343 (0.00509) + | > log_mle: -0.17326 (-0.16321) + | > loss_dur: 0.16983 (0.16830) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.65333 (6.50038) + | > current_lr: 0.00002 + | > step_time: 1.90980 (2.34818) + | > loader_time: 0.00260 (0.11482) + + + --> STEP: 90/234 -- GLOBAL_STEP: 19980 + | > loss: -0.00693 (0.00402) + | > log_mle: -0.20384 (-0.16518) + | > loss_dur: 0.19691 (0.16920) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.82009 (6.93010) + | > current_lr: 0.00002 + | > step_time: 9.49450 (2.41343) + | > loader_time: 0.19860 (0.11170) + + + --> STEP: 95/234 -- GLOBAL_STEP: 19985 + | > loss: -0.07881 (0.00172) + | > log_mle: -0.28608 (-0.16887) + | > loss_dur: 0.20727 (0.17059) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.96680 (7.67067) + | > current_lr: 0.00002 + | > step_time: 1.20090 (2.36310) + | > loader_time: 0.00250 (0.10773) + + + --> STEP: 100/234 -- GLOBAL_STEP: 19990 + | > loss: -0.03764 (0.00039) + | > log_mle: -0.21602 (-0.17093) + | > loss_dur: 0.17838 (0.17132) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.59546 (7.95179) + | > current_lr: 0.00002 + | > step_time: 2.69000 (2.32415) + | > loader_time: 0.00310 (0.10330) + + + --> STEP: 105/234 -- GLOBAL_STEP: 19995 + | > loss: -0.02827 (-0.00158) + | > log_mle: -0.19035 (-0.17396) + | > loss_dur: 0.16208 (0.17238) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.04828 (8.36410) + | > current_lr: 0.00002 + | > step_time: 1.06470 (2.27913) + | > loader_time: 0.00200 (0.09853) + + + --> STEP: 110/234 -- GLOBAL_STEP: 20000 + | > loss: -0.04582 (-0.00285) + | > log_mle: -0.21650 (-0.17661) + | > loss_dur: 0.17068 (0.17376) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.44931 (8.90010) + | > current_lr: 0.00002 + | > step_time: 0.87930 (2.25899) + | > loader_time: 0.00250 (0.09487) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_20000.pth + + --> STEP: 115/234 -- GLOBAL_STEP: 20005 + | > loss: -0.02141 (-0.00458) + | > log_mle: -0.23548 (-0.17987) + | > loss_dur: 0.21407 (0.17529) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.03288 (9.42352) + | > current_lr: 0.00002 + | > step_time: 1.61490 (2.23825) + | > loader_time: 0.07590 (0.09256) + + + --> STEP: 120/234 -- GLOBAL_STEP: 20010 + | > loss: -0.06592 (-0.00595) + | > log_mle: -0.28311 (-0.18269) + | > loss_dur: 0.21719 (0.17674) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.70919 (9.70863) + | > current_lr: 0.00002 + | > step_time: 1.68840 (2.23055) + | > loader_time: 0.00410 (0.08957) + + + --> STEP: 125/234 -- GLOBAL_STEP: 20015 + | > loss: -0.05748 (-0.00710) + | > log_mle: -0.26782 (-0.18433) + | > loss_dur: 0.21035 (0.17723) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.20350 (9.99510) + | > current_lr: 0.00002 + | > step_time: 3.61590 (2.26311) + | > loader_time: 0.00410 (0.08684) + + + --> STEP: 130/234 -- GLOBAL_STEP: 20020 + | > loss: -0.05846 (-0.00916) + | > log_mle: -0.27785 (-0.18761) + | > loss_dur: 0.21938 (0.17845) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.73376 (10.61358) + | > current_lr: 0.00002 + | > step_time: 3.29990 (2.26788) + | > loader_time: 0.19780 (0.08577) + + + --> STEP: 135/234 -- GLOBAL_STEP: 20025 + | > loss: -0.02363 (-0.01109) + | > log_mle: -0.21407 (-0.19088) + | > loss_dur: 0.19045 (0.17980) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.75531 (11.02329) + | > current_lr: 0.00002 + | > step_time: 1.59140 (2.25786) + | > loader_time: 0.00310 (0.08407) + + + --> STEP: 140/234 -- GLOBAL_STEP: 20030 + | > loss: -0.01892 (-0.01311) + | > log_mle: -0.23374 (-0.19444) + | > loss_dur: 0.21482 (0.18133) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.39476 (12.40069) + | > current_lr: 0.00002 + | > step_time: 5.60260 (2.31591) + | > loader_time: 0.29210 (0.08526) + + + --> STEP: 145/234 -- GLOBAL_STEP: 20035 + | > loss: -0.11886 (-0.01540) + | > log_mle: -0.33428 (-0.19853) + | > loss_dur: 0.21542 (0.18313) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.40601 (13.41267) + | > current_lr: 0.00002 + | > step_time: 2.39160 (2.28731) + | > loader_time: 0.00280 (0.08464) + + + --> STEP: 150/234 -- GLOBAL_STEP: 20040 + | > loss: -0.09502 (-0.01823) + | > log_mle: -0.31831 (-0.20252) + | > loss_dur: 0.22329 (0.18429) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.60493 (14.16350) + | > current_lr: 0.00002 + | > step_time: 3.61220 (2.32464) + | > loader_time: 0.09220 (0.08425) + + + --> STEP: 155/234 -- GLOBAL_STEP: 20045 + | > loss: -0.13325 (-0.02154) + | > log_mle: -0.37910 (-0.20738) + | > loss_dur: 0.24585 (0.18584) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.93538 (15.06297) + | > current_lr: 0.00002 + | > step_time: 2.49150 (2.32620) + | > loader_time: 0.00200 (0.08226) + + + --> STEP: 160/234 -- GLOBAL_STEP: 20050 + | > loss: -0.13462 (-0.02418) + | > log_mle: -0.36900 (-0.21144) + | > loss_dur: 0.23438 (0.18726) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.65385 (16.19739) + | > current_lr: 0.00002 + | > step_time: 2.21610 (2.33193) + | > loader_time: 0.08810 (0.08197) + + + --> STEP: 165/234 -- GLOBAL_STEP: 20055 + | > loss: -0.11659 (-0.02680) + | > log_mle: -0.36940 (-0.21542) + | > loss_dur: 0.25282 (0.18862) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.38697 (17.03374) + | > current_lr: 0.00002 + | > step_time: 5.40630 (2.34208) + | > loader_time: 0.10300 (0.08227) + + + --> STEP: 170/234 -- GLOBAL_STEP: 20060 + | > loss: -0.14377 (-0.02968) + | > log_mle: -0.41106 (-0.21985) + | > loss_dur: 0.26729 (0.19017) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.75649 (17.69227) + | > current_lr: 0.00002 + | > step_time: 3.49280 (2.37623) + | > loader_time: 0.20600 (0.08210) + + + --> STEP: 175/234 -- GLOBAL_STEP: 20065 + | > loss: -0.12565 (-0.03331) + | > log_mle: -0.38681 (-0.22503) + | > loss_dur: 0.26116 (0.19172) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.75835 (18.33154) + | > current_lr: 0.00002 + | > step_time: 5.98950 (2.42610) + | > loader_time: 0.19380 (0.08300) + + + --> STEP: 180/234 -- GLOBAL_STEP: 20070 + | > loss: -0.14642 (-0.03644) + | > log_mle: -0.39667 (-0.22988) + | > loss_dur: 0.25024 (0.19344) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 41.90359 (19.02978) + | > current_lr: 0.00002 + | > step_time: 1.00390 (2.52277) + | > loader_time: 0.07640 (0.08715) + + + --> STEP: 185/234 -- GLOBAL_STEP: 20075 + | > loss: -0.14225 (-0.03924) + | > log_mle: -0.42121 (-0.23436) + | > loss_dur: 0.27896 (0.19512) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.14877 (19.73809) + | > current_lr: 0.00002 + | > step_time: 2.39680 (2.52929) + | > loader_time: 0.10910 (0.08690) + + + --> STEP: 190/234 -- GLOBAL_STEP: 20080 + | > loss: -0.15210 (-0.04195) + | > log_mle: -0.39711 (-0.23868) + | > loss_dur: 0.24501 (0.19673) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.60185 (20.68222) + | > current_lr: 0.00002 + | > step_time: 10.59880 (2.69224) + | > loader_time: 0.20520 (0.08832) + + + --> STEP: 195/234 -- GLOBAL_STEP: 20085 + | > loss: -0.14754 (-0.04515) + | > log_mle: -0.41150 (-0.24319) + | > loss_dur: 0.26396 (0.19804) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.35738 (21.62901) + | > current_lr: 0.00002 + | > step_time: 2.50270 (2.68787) + | > loader_time: 0.00310 (0.08662) + + + --> STEP: 200/234 -- GLOBAL_STEP: 20090 + | > loss: -0.14080 (-0.04784) + | > log_mle: -0.41981 (-0.24743) + | > loss_dur: 0.27900 (0.19959) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.49507 (22.33572) + | > current_lr: 0.00002 + | > step_time: 3.39680 (2.71852) + | > loader_time: 0.10300 (0.08703) + + + --> STEP: 205/234 -- GLOBAL_STEP: 20095 + | > loss: -0.15724 (-0.05050) + | > log_mle: -0.40980 (-0.25154) + | > loss_dur: 0.25256 (0.20104) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.49537 (23.12085) + | > current_lr: 0.00002 + | > step_time: 6.19120 (2.75467) + | > loader_time: 0.00210 (0.08632) + + + --> STEP: 210/234 -- GLOBAL_STEP: 20100 + | > loss: -0.20380 (-0.05371) + | > log_mle: -0.48377 (-0.25636) + | > loss_dur: 0.27996 (0.20265) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.64669 (23.97619) + | > current_lr: 0.00002 + | > step_time: 5.00070 (2.82095) + | > loader_time: 0.08540 (0.08571) + + + --> STEP: 215/234 -- GLOBAL_STEP: 20105 + | > loss: -0.18715 (-0.05717) + | > log_mle: -0.43930 (-0.26136) + | > loss_dur: 0.25215 (0.20419) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.98488 (24.72993) + | > current_lr: 0.00002 + | > step_time: 10.20960 (2.86092) + | > loader_time: 0.00460 (0.08463) + + + --> STEP: 220/234 -- GLOBAL_STEP: 20110 + | > loss: -0.20716 (-0.06072) + | > log_mle: -0.48248 (-0.26657) + | > loss_dur: 0.27532 (0.20585) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 67.00211 (25.64433) + | > current_lr: 0.00002 + | > step_time: 3.19440 (2.87277) + | > loader_time: 0.00660 (0.08319) + + + --> STEP: 225/234 -- GLOBAL_STEP: 20115 + | > loss: -0.25485 (-0.06401) + | > log_mle: -0.54551 (-0.27156) + | > loss_dur: 0.29066 (0.20755) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.45954 (26.34777) + | > current_lr: 0.00002 + | > step_time: 2.91190 (2.87563) + | > loader_time: 0.00880 (0.08186) + + + --> STEP: 230/234 -- GLOBAL_STEP: 20120 + | > loss: -0.22279 (-0.06704) + | > log_mle: -0.59138 (-0.27697) + | > loss_dur: 0.36859 (0.20993) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 85.11322 (27.29598) + | > current_lr: 0.00002 + | > step_time: 0.26560 (2.82378) + | > loader_time: 0.00360 (0.08048) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27391 (-0.27698) + | > avg_loss: -0.10186 (+0.00110) + | > avg_log_mle: -0.34835 (-0.00166) + | > avg_loss_dur: 0.24648 (+0.00276) + + + > EPOCH: 86/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 03:13:48)  + + --> STEP: 1/234 -- GLOBAL_STEP: 20125 + | > loss: -0.01015 (-0.01015) + | > log_mle: -0.16320 (-0.16320) + | > loss_dur: 0.15305 (0.15305) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.06632 (5.06632) + | > current_lr: 0.00002 + | > step_time: 4.60710 (4.60707) + | > loader_time: 0.08230 (0.08232) + + + --> STEP: 6/234 -- GLOBAL_STEP: 20130 + | > loss: 0.02572 (0.03706) + | > log_mle: -0.15158 (-0.15851) + | > loss_dur: 0.17730 (0.19557) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.93287 (6.91241) + | > current_lr: 0.00002 + | > step_time: 9.29010 (5.41893) + | > loader_time: 0.50780 (0.14583) + + + --> STEP: 11/234 -- GLOBAL_STEP: 20135 + | > loss: 0.00424 (0.02133) + | > log_mle: -0.15674 (-0.16307) + | > loss_dur: 0.16098 (0.18441) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.66258 (6.72475) + | > current_lr: 0.00002 + | > step_time: 2.68880 (3.87187) + | > loader_time: 0.00120 (0.08094) + + + --> STEP: 16/234 -- GLOBAL_STEP: 20140 + | > loss: -0.00912 (0.01456) + | > log_mle: -0.15983 (-0.16250) + | > loss_dur: 0.15072 (0.17706) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.90959 (6.19772) + | > current_lr: 0.00002 + | > step_time: 2.30490 (3.98808) + | > loader_time: 0.08840 (0.07328) + + + --> STEP: 21/234 -- GLOBAL_STEP: 20145 + | > loss: 0.01928 (0.01401) + | > log_mle: -0.14367 (-0.15924) + | > loss_dur: 0.16295 (0.17326) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.58564 (5.90819) + | > current_lr: 0.00002 + | > step_time: 0.99870 (4.29085) + | > loader_time: 0.00150 (0.07956) + + + --> STEP: 26/234 -- GLOBAL_STEP: 20150 + | > loss: -0.00053 (0.01013) + | > log_mle: -0.16187 (-0.15893) + | > loss_dur: 0.16134 (0.16906) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.58287 (5.56599) + | > current_lr: 0.00002 + | > step_time: 2.70060 (3.98432) + | > loader_time: 0.08290 (0.07168) + + + --> STEP: 31/234 -- GLOBAL_STEP: 20155 + | > loss: 0.03571 (0.00766) + | > log_mle: -0.16409 (-0.15943) + | > loss_dur: 0.19980 (0.16709) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.20848 (5.35574) + | > current_lr: 0.00002 + | > step_time: 6.30130 (3.79023) + | > loader_time: 0.19110 (0.06678) + + + --> STEP: 36/234 -- GLOBAL_STEP: 20160 + | > loss: 0.01651 (0.00717) + | > log_mle: -0.16877 (-0.16023) + | > loss_dur: 0.18528 (0.16740) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.97293 (5.50393) + | > current_lr: 0.00002 + | > step_time: 4.39560 (3.75803) + | > loader_time: 0.19960 (0.06571) + + + --> STEP: 41/234 -- GLOBAL_STEP: 20165 + | > loss: -0.00315 (0.00694) + | > log_mle: -0.15968 (-0.16019) + | > loss_dur: 0.15653 (0.16713) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.29525 (5.65194) + | > current_lr: 0.00002 + | > step_time: 0.81240 (3.65875) + | > loader_time: 0.00280 (0.06242) + + + --> STEP: 46/234 -- GLOBAL_STEP: 20170 + | > loss: 0.00252 (0.00606) + | > log_mle: -0.16421 (-0.16068) + | > loss_dur: 0.16673 (0.16674) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.38098 (5.69109) + | > current_lr: 0.00002 + | > step_time: 1.57300 (3.45211) + | > loader_time: 0.00160 (0.05598) + + + --> STEP: 51/234 -- GLOBAL_STEP: 20175 + | > loss: 0.00054 (0.00515) + | > log_mle: -0.14980 (-0.16027) + | > loss_dur: 0.15034 (0.16542) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.86589 (5.52326) + | > current_lr: 0.00002 + | > step_time: 1.29420 (3.24917) + | > loader_time: 0.00230 (0.05229) + + + --> STEP: 56/234 -- GLOBAL_STEP: 20180 + | > loss: 0.01830 (0.00490) + | > log_mle: -0.16959 (-0.16106) + | > loss_dur: 0.18789 (0.16596) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.57322 (5.39252) + | > current_lr: 0.00002 + | > step_time: 3.10320 (3.11507) + | > loader_time: 0.00580 (0.04789) + + + --> STEP: 61/234 -- GLOBAL_STEP: 20185 + | > loss: -0.01817 (0.00313) + | > log_mle: -0.16472 (-0.16183) + | > loss_dur: 0.14654 (0.16495) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.61849 (5.44370) + | > current_lr: 0.00002 + | > step_time: 1.41240 (2.99907) + | > loader_time: 0.00220 (0.04678) + + + --> STEP: 66/234 -- GLOBAL_STEP: 20190 + | > loss: 0.00343 (0.00315) + | > log_mle: -0.15321 (-0.16239) + | > loss_dur: 0.15664 (0.16554) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.64037 (5.59577) + | > current_lr: 0.00002 + | > step_time: 2.81050 (2.90492) + | > loader_time: 0.07690 (0.04457) + + + --> STEP: 71/234 -- GLOBAL_STEP: 20195 + | > loss: 0.00828 (0.00359) + | > log_mle: -0.19535 (-0.16297) + | > loss_dur: 0.20364 (0.16656) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.10628 (6.03233) + | > current_lr: 0.00002 + | > step_time: 2.17240 (2.81156) + | > loader_time: 0.00150 (0.04274) + + + --> STEP: 76/234 -- GLOBAL_STEP: 20200 + | > loss: -0.00548 (0.00325) + | > log_mle: -0.17847 (-0.16373) + | > loss_dur: 0.17300 (0.16698) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.64150 (6.11085) + | > current_lr: 0.00002 + | > step_time: 2.02740 (2.79117) + | > loader_time: 0.00360 (0.04015) + + + --> STEP: 81/234 -- GLOBAL_STEP: 20205 + | > loss: -0.02278 (0.00228) + | > log_mle: -0.18936 (-0.16425) + | > loss_dur: 0.16658 (0.16653) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.98848 (6.12325) + | > current_lr: 0.00002 + | > step_time: 0.80530 (2.70189) + | > loader_time: 0.00220 (0.03897) + + + --> STEP: 86/234 -- GLOBAL_STEP: 20210 + | > loss: -0.01689 (0.00185) + | > log_mle: -0.19128 (-0.16521) + | > loss_dur: 0.17440 (0.16706) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.62990 (6.20765) + | > current_lr: 0.00002 + | > step_time: 3.39320 (2.65975) + | > loader_time: 0.00280 (0.03690) + + + --> STEP: 91/234 -- GLOBAL_STEP: 20215 + | > loss: -0.00252 (0.00066) + | > log_mle: -0.20051 (-0.16739) + | > loss_dur: 0.19799 (0.16804) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.67353 (6.39771) + | > current_lr: 0.00002 + | > step_time: 3.00570 (2.67864) + | > loader_time: 0.10010 (0.03877) + + + --> STEP: 96/234 -- GLOBAL_STEP: 20220 + | > loss: -0.01040 (-0.00214) + | > log_mle: -0.18772 (-0.17104) + | > loss_dur: 0.17733 (0.16890) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.04402 (6.90419) + | > current_lr: 0.00002 + | > step_time: 3.99650 (2.64637) + | > loader_time: 0.00320 (0.03965) + + + --> STEP: 101/234 -- GLOBAL_STEP: 20225 + | > loss: -0.04790 (-0.00380) + | > log_mle: -0.24502 (-0.17363) + | > loss_dur: 0.19711 (0.16983) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.87339 (7.31540) + | > current_lr: 0.00002 + | > step_time: 3.39780 (2.61801) + | > loader_time: 0.10270 (0.03880) + + + --> STEP: 106/234 -- GLOBAL_STEP: 20230 + | > loss: -0.02103 (-0.00542) + | > log_mle: -0.24651 (-0.17659) + | > loss_dur: 0.22548 (0.17117) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.24481 (7.73804) + | > current_lr: 0.00002 + | > step_time: 4.81540 (2.62394) + | > loader_time: 0.00260 (0.03878) + + + --> STEP: 111/234 -- GLOBAL_STEP: 20235 + | > loss: -0.03767 (-0.00672) + | > log_mle: -0.28620 (-0.17959) + | > loss_dur: 0.24853 (0.17287) + | > amp_scaler: 8192.00000 (4169.80180) + | > grad_norm: 32.56540 (8.32182) + | > current_lr: 0.00002 + | > step_time: 1.99400 (2.64264) + | > loader_time: 0.00230 (0.03957) + + + --> STEP: 116/234 -- GLOBAL_STEP: 20240 + | > loss: -0.02188 (-0.00825) + | > log_mle: -0.25462 (-0.18250) + | > loss_dur: 0.23274 (0.17424) + | > amp_scaler: 8192.00000 (4343.17241) + | > grad_norm: 26.78500 (8.87059) + | > current_lr: 0.00002 + | > step_time: 4.29240 (2.65806) + | > loader_time: 0.00310 (0.03900) + + + --> STEP: 121/234 -- GLOBAL_STEP: 20245 + | > loss: 0.01166 (-0.00932) + | > log_mle: -0.17118 (-0.18452) + | > loss_dur: 0.18283 (0.17520) + | > amp_scaler: 8192.00000 (4502.21488) + | > grad_norm: 6.28422 (9.15272) + | > current_lr: 0.00002 + | > step_time: 6.59920 (2.69312) + | > loader_time: 0.29210 (0.03990) + + + --> STEP: 126/234 -- GLOBAL_STEP: 20250 + | > loss: -0.07998 (-0.01117) + | > log_mle: -0.30013 (-0.18715) + | > loss_dur: 0.22015 (0.17598) + | > amp_scaler: 8192.00000 (4648.63492) + | > grad_norm: 35.51289 (9.65871) + | > current_lr: 0.00002 + | > step_time: 1.73130 (2.67773) + | > loader_time: 0.07860 (0.03978) + + + --> STEP: 131/234 -- GLOBAL_STEP: 20255 + | > loss: -0.10689 (-0.01339) + | > log_mle: -0.33907 (-0.19071) + | > loss_dur: 0.23218 (0.17732) + | > amp_scaler: 8192.00000 (4783.87786) + | > grad_norm: 42.80559 (10.33296) + | > current_lr: 0.00002 + | > step_time: 1.69970 (2.65663) + | > loader_time: 0.08860 (0.03939) + + + --> STEP: 136/234 -- GLOBAL_STEP: 20260 + | > loss: -0.12900 (-0.01551) + | > log_mle: -0.38408 (-0.19428) + | > loss_dur: 0.25507 (0.17878) + | > amp_scaler: 8192.00000 (4909.17647) + | > grad_norm: 40.90364 (10.84637) + | > current_lr: 0.00002 + | > step_time: 2.71020 (2.64875) + | > loader_time: 0.00570 (0.04006) + + + --> STEP: 141/234 -- GLOBAL_STEP: 20265 + | > loss: -0.06811 (-0.01732) + | > log_mle: -0.29766 (-0.19739) + | > loss_dur: 0.22955 (0.18007) + | > amp_scaler: 8192.00000 (5025.58865) + | > grad_norm: 22.88451 (11.34404) + | > current_lr: 0.00002 + | > step_time: 0.68740 (2.65699) + | > loader_time: 0.00220 (0.04005) + + + --> STEP: 146/234 -- GLOBAL_STEP: 20270 + | > loss: -0.12214 (-0.02021) + | > log_mle: -0.34476 (-0.20206) + | > loss_dur: 0.22262 (0.18185) + | > amp_scaler: 8192.00000 (5134.02740) + | > grad_norm: 35.15216 (12.16801) + | > current_lr: 0.00002 + | > step_time: 1.68030 (2.63035) + | > loader_time: 0.00320 (0.04036) + + + --> STEP: 151/234 -- GLOBAL_STEP: 20275 + | > loss: -0.11036 (-0.02288) + | > log_mle: -0.31173 (-0.20588) + | > loss_dur: 0.20136 (0.18300) + | > amp_scaler: 8192.00000 (5235.28477) + | > grad_norm: 31.25377 (12.78777) + | > current_lr: 0.00002 + | > step_time: 1.06350 (2.61670) + | > loader_time: 0.00220 (0.04077) + + + --> STEP: 156/234 -- GLOBAL_STEP: 20280 + | > loss: -0.13178 (-0.02651) + | > log_mle: -0.35123 (-0.21109) + | > loss_dur: 0.21946 (0.18458) + | > amp_scaler: 8192.00000 (5330.05128) + | > grad_norm: 37.81307 (13.73281) + | > current_lr: 0.00002 + | > step_time: 2.40720 (2.65602) + | > loader_time: 0.00270 (0.04142) + + + --> STEP: 161/234 -- GLOBAL_STEP: 20285 + | > loss: -0.13883 (-0.02943) + | > log_mle: -0.37480 (-0.21557) + | > loss_dur: 0.23597 (0.18614) + | > amp_scaler: 8192.00000 (5418.93168) + | > grad_norm: 32.30191 (14.39026) + | > current_lr: 0.00002 + | > step_time: 1.15540 (2.63862) + | > loader_time: 0.00290 (0.04132) + + + --> STEP: 166/234 -- GLOBAL_STEP: 20290 + | > loss: -0.10359 (-0.03204) + | > log_mle: -0.31708 (-0.21936) + | > loss_dur: 0.21349 (0.18732) + | > amp_scaler: 8192.00000 (5502.45783) + | > grad_norm: 26.04210 (15.23382) + | > current_lr: 0.00002 + | > step_time: 0.92400 (2.62402) + | > loader_time: 0.00280 (0.04069) + + + --> STEP: 171/234 -- GLOBAL_STEP: 20295 + | > loss: -0.19014 (-0.03536) + | > log_mle: -0.41294 (-0.22440) + | > loss_dur: 0.22280 (0.18903) + | > amp_scaler: 8192.00000 (5581.09942) + | > grad_norm: 79.74494 (16.35131) + | > current_lr: 0.00002 + | > step_time: 2.51280 (2.61281) + | > loader_time: 0.08740 (0.04064) + + + --> STEP: 176/234 -- GLOBAL_STEP: 20300 + | > loss: -0.15135 (-0.03860) + | > log_mle: -0.38516 (-0.22929) + | > loss_dur: 0.23381 (0.19069) + | > amp_scaler: 8192.00000 (5655.27273) + | > grad_norm: 57.83403 (17.57969) + | > current_lr: 0.00002 + | > step_time: 1.59260 (2.61581) + | > loader_time: 0.00220 (0.04175) + + + --> STEP: 181/234 -- GLOBAL_STEP: 20305 + | > loss: -0.07864 (-0.04114) + | > log_mle: -0.32688 (-0.23365) + | > loss_dur: 0.24824 (0.19251) + | > amp_scaler: 8192.00000 (5725.34807) + | > grad_norm: 40.38202 (18.69149) + | > current_lr: 0.00002 + | > step_time: 4.49850 (2.63654) + | > loader_time: 0.10390 (0.04279) + + + --> STEP: 186/234 -- GLOBAL_STEP: 20310 + | > loss: -0.10919 (-0.04387) + | > log_mle: -0.36552 (-0.23820) + | > loss_dur: 0.25633 (0.19432) + | > amp_scaler: 8192.00000 (5791.65591) + | > grad_norm: 48.18471 (19.79885) + | > current_lr: 0.00002 + | > step_time: 8.08800 (2.74950) + | > loader_time: 0.20370 (0.04386) + + + --> STEP: 191/234 -- GLOBAL_STEP: 20315 + | > loss: -0.15341 (-0.04689) + | > log_mle: -0.38475 (-0.24262) + | > loss_dur: 0.23134 (0.19573) + | > amp_scaler: 8192.00000 (5854.49215) + | > grad_norm: 34.20162 (20.50212) + | > current_lr: 0.00002 + | > step_time: 7.30290 (2.79164) + | > loader_time: 0.18780 (0.04565) + + + --> STEP: 196/234 -- GLOBAL_STEP: 20320 + | > loss: -0.12268 (-0.04996) + | > log_mle: -0.38135 (-0.24710) + | > loss_dur: 0.25867 (0.19714) + | > amp_scaler: 8192.00000 (5914.12245) + | > grad_norm: 40.67521 (21.39000) + | > current_lr: 0.00002 + | > step_time: 3.89090 (2.80674) + | > loader_time: 0.00590 (0.04580) + + + --> STEP: 201/234 -- GLOBAL_STEP: 20325 + | > loss: -0.08748 (-0.05254) + | > log_mle: -0.34890 (-0.25117) + | > loss_dur: 0.26142 (0.19863) + | > amp_scaler: 8192.00000 (5970.78607) + | > grad_norm: 33.47158 (21.99078) + | > current_lr: 0.00002 + | > step_time: 2.70300 (2.83262) + | > loader_time: 0.00290 (0.04656) + + + --> STEP: 206/234 -- GLOBAL_STEP: 20330 + | > loss: -0.18320 (-0.05551) + | > log_mle: -0.44723 (-0.25570) + | > loss_dur: 0.26403 (0.20020) + | > amp_scaler: 8192.00000 (6024.69903) + | > grad_norm: 47.94243 (22.70631) + | > current_lr: 0.00002 + | > step_time: 5.71130 (2.90043) + | > loader_time: 0.09350 (0.04777) + + + --> STEP: 211/234 -- GLOBAL_STEP: 20335 + | > loss: -0.21822 (-0.05871) + | > log_mle: -0.50332 (-0.26062) + | > loss_dur: 0.28510 (0.20191) + | > amp_scaler: 4096.00000 (5978.99526) + | > grad_norm: 77.56522 (23.64368) + | > current_lr: 0.00002 + | > step_time: 4.39330 (2.91835) + | > loader_time: 0.00270 (0.04813) + + + --> STEP: 216/234 -- GLOBAL_STEP: 20340 + | > loss: -0.20702 (-0.06192) + | > log_mle: -0.49532 (-0.26535) + | > loss_dur: 0.28829 (0.20343) + | > amp_scaler: 4096.00000 (5935.40741) + | > grad_norm: 74.29557 (24.79343) + | > current_lr: 0.00002 + | > step_time: 2.50120 (2.99150) + | > loader_time: 0.00520 (0.04934) + + + --> STEP: 221/234 -- GLOBAL_STEP: 20345 + | > loss: -0.15504 (-0.06511) + | > log_mle: -0.41957 (-0.27011) + | > loss_dur: 0.26452 (0.20500) + | > amp_scaler: 4096.00000 (5893.79186) + | > grad_norm: 53.90870 (25.76019) + | > current_lr: 0.00002 + | > step_time: 1.32180 (2.97296) + | > loader_time: 0.07600 (0.04981) + + + --> STEP: 226/234 -- GLOBAL_STEP: 20350 + | > loss: -0.23704 (-0.06864) + | > log_mle: -0.51450 (-0.27541) + | > loss_dur: 0.27746 (0.20676) + | > amp_scaler: 4096.00000 (5854.01770) + | > grad_norm: 73.19580 (26.70905) + | > current_lr: 0.00002 + | > step_time: 0.23550 (2.93773) + | > loader_time: 0.00360 (0.04958) + + + --> STEP: 231/234 -- GLOBAL_STEP: 20355 + | > loss: -0.15926 (-0.07120) + | > log_mle: -0.57970 (-0.28100) + | > loss_dur: 0.42044 (0.20980) + | > amp_scaler: 4096.00000 (5815.96537) + | > grad_norm: 59.14633 (27.49928) + | > current_lr: 0.00002 + | > step_time: 0.27470 (2.87971) + | > loader_time: 0.00470 (0.04860) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15173 (-0.12219) + | > avg_loss: -0.11003 (-0.00816) + | > avg_log_mle: -0.35372 (-0.00537) + | > avg_loss_dur: 0.24370 (-0.00279) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_20358.pth + + > EPOCH: 87/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 03:26:18)  + + --> STEP: 2/234 -- GLOBAL_STEP: 20360 + | > loss: 0.08207 (0.03827) + | > log_mle: -0.14288 (-0.15311) + | > loss_dur: 0.22495 (0.19138) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.66175 (6.16210) + | > current_lr: 0.00002 + | > step_time: 2.81580 (2.16220) + | > loader_time: 0.09080 (0.09187) + + + --> STEP: 7/234 -- GLOBAL_STEP: 20365 + | > loss: -0.03002 (0.02591) + | > log_mle: -0.17646 (-0.16240) + | > loss_dur: 0.14644 (0.18831) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.77376 (6.66519) + | > current_lr: 0.00002 + | > step_time: 2.19880 (6.17597) + | > loader_time: 0.09590 (0.11049) + + + --> STEP: 12/234 -- GLOBAL_STEP: 20370 + | > loss: 0.00303 (0.01661) + | > log_mle: -0.16359 (-0.16450) + | > loss_dur: 0.16661 (0.18111) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.38656 (6.70529) + | > current_lr: 0.00002 + | > step_time: 1.11950 (4.20468) + | > loader_time: 0.00130 (0.07132) + + + --> STEP: 17/234 -- GLOBAL_STEP: 20375 + | > loss: 0.02501 (0.01184) + | > log_mle: -0.14413 (-0.16292) + | > loss_dur: 0.16914 (0.17476) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.98329 (6.10058) + | > current_lr: 0.00002 + | > step_time: 0.59150 (3.24728) + | > loader_time: 0.00200 (0.05472) + + + --> STEP: 22/234 -- GLOBAL_STEP: 20380 + | > loss: -0.00724 (0.01107) + | > log_mle: -0.17049 (-0.16123) + | > loss_dur: 0.16325 (0.17231) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.36312 (5.91460) + | > current_lr: 0.00002 + | > step_time: 2.00210 (2.87525) + | > loader_time: 0.00150 (0.04684) + + + --> STEP: 27/234 -- GLOBAL_STEP: 20385 + | > loss: -0.02118 (0.00787) + | > log_mle: -0.17195 (-0.16104) + | > loss_dur: 0.15077 (0.16891) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.20563 (5.59582) + | > current_lr: 0.00002 + | > step_time: 1.27660 (2.64477) + | > loader_time: 0.00260 (0.03860) + + + --> STEP: 32/234 -- GLOBAL_STEP: 20390 + | > loss: -0.03648 (0.00542) + | > log_mle: -0.17995 (-0.16174) + | > loss_dur: 0.14348 (0.16716) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.15789 (5.45963) + | > current_lr: 0.00002 + | > step_time: 1.99260 (2.53209) + | > loader_time: 0.00270 (0.03551) + + + --> STEP: 37/234 -- GLOBAL_STEP: 20395 + | > loss: -0.01514 (0.00581) + | > log_mle: -0.16072 (-0.16182) + | > loss_dur: 0.14558 (0.16762) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.58563 (5.72141) + | > current_lr: 0.00002 + | > step_time: 0.93940 (2.37351) + | > loader_time: 0.00240 (0.03770) + + + --> STEP: 42/234 -- GLOBAL_STEP: 20400 + | > loss: 0.02339 (0.00651) + | > log_mle: -0.14939 (-0.16161) + | > loss_dur: 0.17278 (0.16812) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.58068 (5.66514) + | > current_lr: 0.00002 + | > step_time: 1.70640 (2.26172) + | > loader_time: 0.08520 (0.03775) + + + --> STEP: 47/234 -- GLOBAL_STEP: 20405 + | > loss: -0.00558 (0.00565) + | > log_mle: -0.16265 (-0.16242) + | > loss_dur: 0.15707 (0.16807) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.66203 (5.75259) + | > current_lr: 0.00002 + | > step_time: 1.35860 (2.19106) + | > loader_time: 0.00200 (0.03807) + + + --> STEP: 52/234 -- GLOBAL_STEP: 20410 + | > loss: 0.01663 (0.00517) + | > log_mle: -0.15866 (-0.16189) + | > loss_dur: 0.17529 (0.16706) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.03899 (5.59418) + | > current_lr: 0.00002 + | > step_time: 1.52030 (2.15674) + | > loader_time: 0.00240 (0.03620) + + + --> STEP: 57/234 -- GLOBAL_STEP: 20415 + | > loss: 0.03344 (0.00546) + | > log_mle: -0.15320 (-0.16260) + | > loss_dur: 0.18664 (0.16806) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.76051 (5.47365) + | > current_lr: 0.00002 + | > step_time: 2.44190 (2.11835) + | > loader_time: 0.00270 (0.03324) + + + --> STEP: 62/234 -- GLOBAL_STEP: 20420 + | > loss: 0.01535 (0.00335) + | > log_mle: -0.20691 (-0.16425) + | > loss_dur: 0.22226 (0.16760) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.07563 (5.72264) + | > current_lr: 0.00002 + | > step_time: 1.11960 (2.07802) + | > loader_time: 0.00210 (0.03208) + + + --> STEP: 67/234 -- GLOBAL_STEP: 20425 + | > loss: -0.02358 (0.00283) + | > log_mle: -0.18790 (-0.16450) + | > loss_dur: 0.16432 (0.16733) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.86882 (5.67371) + | > current_lr: 0.00002 + | > step_time: 1.29580 (2.05913) + | > loader_time: 0.00240 (0.03117) + + + --> STEP: 72/234 -- GLOBAL_STEP: 20430 + | > loss: 0.01856 (0.00352) + | > log_mle: -0.16643 (-0.16482) + | > loss_dur: 0.18498 (0.16834) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.24653 (5.85762) + | > current_lr: 0.00002 + | > step_time: 1.80700 (2.04670) + | > loader_time: 0.08380 (0.03402) + + + --> STEP: 77/234 -- GLOBAL_STEP: 20435 + | > loss: -0.03235 (0.00224) + | > log_mle: -0.18312 (-0.16584) + | > loss_dur: 0.15077 (0.16808) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.04256 (5.98282) + | > current_lr: 0.00002 + | > step_time: 1.56990 (2.04450) + | > loader_time: 0.00210 (0.03319) + + + --> STEP: 82/234 -- GLOBAL_STEP: 20440 + | > loss: -0.01354 (0.00155) + | > log_mle: -0.17129 (-0.16622) + | > loss_dur: 0.15775 (0.16777) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.88225 (6.04963) + | > current_lr: 0.00002 + | > step_time: 2.59860 (2.07191) + | > loader_time: 0.18990 (0.03362) + + + --> STEP: 87/234 -- GLOBAL_STEP: 20445 + | > loss: -0.00233 (0.00113) + | > log_mle: -0.18056 (-0.16719) + | > loss_dur: 0.17823 (0.16833) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.81229 (6.23410) + | > current_lr: 0.00002 + | > step_time: 1.38750 (2.05243) + | > loader_time: 0.00180 (0.03183) + + + --> STEP: 92/234 -- GLOBAL_STEP: 20450 + | > loss: -0.06092 (-0.00053) + | > log_mle: -0.22487 (-0.16969) + | > loss_dur: 0.16396 (0.16916) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.58416 (6.54477) + | > current_lr: 0.00002 + | > step_time: 1.79080 (2.05387) + | > loader_time: 0.00340 (0.03129) + + + --> STEP: 97/234 -- GLOBAL_STEP: 20455 + | > loss: -0.03854 (-0.00283) + | > log_mle: -0.21526 (-0.17305) + | > loss_dur: 0.17672 (0.17023) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.62028 (7.09482) + | > current_lr: 0.00002 + | > step_time: 2.19830 (2.05819) + | > loader_time: 0.08370 (0.03151) + + + --> STEP: 102/234 -- GLOBAL_STEP: 20460 + | > loss: -0.00752 (-0.00424) + | > log_mle: -0.19623 (-0.17536) + | > loss_dur: 0.18871 (0.17111) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.35065 (7.48731) + | > current_lr: 0.00002 + | > step_time: 1.51070 (2.06136) + | > loader_time: 0.00230 (0.03255) + + + --> STEP: 107/234 -- GLOBAL_STEP: 20465 + | > loss: -0.05452 (-0.00632) + | > log_mle: -0.24272 (-0.17867) + | > loss_dur: 0.18820 (0.17235) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.66395 (8.06741) + | > current_lr: 0.00002 + | > step_time: 3.09980 (2.10165) + | > loader_time: 0.09960 (0.03549) + + + --> STEP: 112/234 -- GLOBAL_STEP: 20470 + | > loss: -0.03596 (-0.00774) + | > log_mle: -0.25062 (-0.18167) + | > loss_dur: 0.21466 (0.17393) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.79280 (8.59213) + | > current_lr: 0.00002 + | > step_time: 3.80150 (2.12316) + | > loader_time: 0.10170 (0.03639) + + + --> STEP: 117/234 -- GLOBAL_STEP: 20475 + | > loss: -0.05967 (-0.00925) + | > log_mle: -0.24562 (-0.18451) + | > loss_dur: 0.18595 (0.17526) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.57388 (9.12372) + | > current_lr: 0.00002 + | > step_time: 2.30160 (2.09661) + | > loader_time: 0.00330 (0.03640) + + + --> STEP: 122/234 -- GLOBAL_STEP: 20480 + | > loss: -0.05067 (-0.01038) + | > log_mle: -0.22540 (-0.18634) + | > loss_dur: 0.17473 (0.17596) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.65300 (9.30449) + | > current_lr: 0.00002 + | > step_time: 1.86950 (2.08678) + | > loader_time: 0.00190 (0.03644) + + + --> STEP: 127/234 -- GLOBAL_STEP: 20485 + | > loss: -0.06361 (-0.01248) + | > log_mle: -0.27865 (-0.18937) + | > loss_dur: 0.21504 (0.17690) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.69274 (9.75797) + | > current_lr: 0.00002 + | > step_time: 2.99500 (2.08786) + | > loader_time: 0.10100 (0.03653) + + + --> STEP: 132/234 -- GLOBAL_STEP: 20490 + | > loss: -0.07682 (-0.01480) + | > log_mle: -0.25754 (-0.19270) + | > loss_dur: 0.18072 (0.17790) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.76606 (10.40003) + | > current_lr: 0.00002 + | > step_time: 4.51420 (2.15741) + | > loader_time: 0.09260 (0.03874) + + + --> STEP: 137/234 -- GLOBAL_STEP: 20495 + | > loss: -0.04693 (-0.01676) + | > log_mle: -0.26890 (-0.19623) + | > loss_dur: 0.22197 (0.17947) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.83178 (11.11372) + | > current_lr: 0.00002 + | > step_time: 1.90760 (2.14434) + | > loader_time: 0.00340 (0.03864) + + + --> STEP: 142/234 -- GLOBAL_STEP: 20500 + | > loss: -0.06698 (-0.01865) + | > log_mle: -0.28614 (-0.19934) + | > loss_dur: 0.21916 (0.18069) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.23728 (11.72479) + | > current_lr: 0.00002 + | > step_time: 1.01300 (2.15757) + | > loader_time: 0.00210 (0.03926) + + + --> STEP: 147/234 -- GLOBAL_STEP: 20505 + | > loss: -0.07368 (-0.02158) + | > log_mle: -0.28850 (-0.20400) + | > loss_dur: 0.21482 (0.18242) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.52185 (12.48776) + | > current_lr: 0.00002 + | > step_time: 1.40210 (2.17747) + | > loader_time: 0.00310 (0.03978) + + + --> STEP: 152/234 -- GLOBAL_STEP: 20510 + | > loss: -0.11794 (-0.02461) + | > log_mle: -0.36178 (-0.20829) + | > loss_dur: 0.24384 (0.18369) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.04055 (13.11367) + | > current_lr: 0.00002 + | > step_time: 2.09660 (2.18806) + | > loader_time: 0.00280 (0.04046) + + + --> STEP: 157/234 -- GLOBAL_STEP: 20515 + | > loss: -0.07907 (-0.02807) + | > log_mle: -0.31139 (-0.21314) + | > loss_dur: 0.23232 (0.18508) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.24787 (14.01919) + | > current_lr: 0.00002 + | > step_time: 1.97040 (2.19985) + | > loader_time: 0.00270 (0.03987) + + + --> STEP: 162/234 -- GLOBAL_STEP: 20520 + | > loss: -0.13092 (-0.03124) + | > log_mle: -0.34611 (-0.21773) + | > loss_dur: 0.21519 (0.18648) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.22584 (14.93644) + | > current_lr: 0.00002 + | > step_time: 1.41640 (2.20990) + | > loader_time: 0.07720 (0.04145) + + + --> STEP: 167/234 -- GLOBAL_STEP: 20525 + | > loss: -0.18446 (-0.03411) + | > log_mle: -0.41605 (-0.22192) + | > loss_dur: 0.23158 (0.18780) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.47940 (15.78870) + | > current_lr: 0.00002 + | > step_time: 1.39160 (2.20178) + | > loader_time: 0.00290 (0.04031) + + + --> STEP: 172/234 -- GLOBAL_STEP: 20530 + | > loss: -0.14943 (-0.03715) + | > log_mle: -0.41147 (-0.22680) + | > loss_dur: 0.26204 (0.18965) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.46113 (16.89668) + | > current_lr: 0.00002 + | > step_time: 5.10500 (2.20479) + | > loader_time: 0.18340 (0.04077) + + + --> STEP: 177/234 -- GLOBAL_STEP: 20535 + | > loss: -0.12588 (-0.04025) + | > log_mle: -0.37300 (-0.23153) + | > loss_dur: 0.24712 (0.19128) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.50360 (17.80801) + | > current_lr: 0.00002 + | > step_time: 5.20030 (2.27022) + | > loader_time: 0.09570 (0.04131) + + + --> STEP: 182/234 -- GLOBAL_STEP: 20540 + | > loss: -0.15660 (-0.04312) + | > log_mle: -0.41913 (-0.23621) + | > loss_dur: 0.26254 (0.19309) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.09541 (18.60285) + | > current_lr: 0.00002 + | > step_time: 2.49480 (2.30947) + | > loader_time: 0.00340 (0.04086) + + + --> STEP: 187/234 -- GLOBAL_STEP: 20545 + | > loss: -0.17100 (-0.04605) + | > log_mle: -0.42029 (-0.24084) + | > loss_dur: 0.24928 (0.19479) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.97470 (19.39265) + | > current_lr: 0.00002 + | > step_time: 1.91490 (2.32119) + | > loader_time: 0.00290 (0.04185) + + + --> STEP: 192/234 -- GLOBAL_STEP: 20550 + | > loss: -0.20292 (-0.04920) + | > log_mle: -0.44292 (-0.24537) + | > loss_dur: 0.24001 (0.19617) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.04059 (20.22552) + | > current_lr: 0.00002 + | > step_time: 3.59760 (2.32680) + | > loader_time: 0.00690 (0.04088) + + + --> STEP: 197/234 -- GLOBAL_STEP: 20555 + | > loss: -0.18464 (-0.05218) + | > log_mle: -0.41924 (-0.24978) + | > loss_dur: 0.23460 (0.19760) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.69806 (20.91424) + | > current_lr: 0.00002 + | > step_time: 3.20670 (2.33587) + | > loader_time: 0.00700 (0.04134) + + + --> STEP: 202/234 -- GLOBAL_STEP: 20560 + | > loss: -0.22927 (-0.05495) + | > log_mle: -0.48625 (-0.25419) + | > loss_dur: 0.25698 (0.19924) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 146.67577 (22.01539) + | > current_lr: 0.00002 + | > step_time: 1.59350 (2.41362) + | > loader_time: 0.09450 (0.04325) + + + --> STEP: 207/234 -- GLOBAL_STEP: 20565 + | > loss: -0.21339 (-0.05767) + | > log_mle: -0.48313 (-0.25844) + | > loss_dur: 0.26974 (0.20076) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.43790 (23.07792) + | > current_lr: 0.00002 + | > step_time: 6.79620 (2.43747) + | > loader_time: 0.00570 (0.04406) + + + --> STEP: 212/234 -- GLOBAL_STEP: 20570 + | > loss: -0.19711 (-0.06089) + | > log_mle: -0.46882 (-0.26333) + | > loss_dur: 0.27171 (0.20245) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.04029 (23.99763) + | > current_lr: 0.00002 + | > step_time: 5.09700 (2.47260) + | > loader_time: 0.10470 (0.04488) + + + --> STEP: 217/234 -- GLOBAL_STEP: 20575 + | > loss: -0.21822 (-0.06433) + | > log_mle: -0.49020 (-0.26825) + | > loss_dur: 0.27197 (0.20392) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 58.75222 (24.77864) + | > current_lr: 0.00002 + | > step_time: 2.09290 (2.55485) + | > loader_time: 0.00590 (0.04928) + + + --> STEP: 222/234 -- GLOBAL_STEP: 20580 + | > loss: -0.20122 (-0.06753) + | > log_mle: -0.50234 (-0.27309) + | > loss_dur: 0.30113 (0.20556) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 67.95287 (25.79695) + | > current_lr: 0.00002 + | > step_time: 1.51370 (2.55235) + | > loader_time: 0.08430 (0.04978) + + + --> STEP: 227/234 -- GLOBAL_STEP: 20585 + | > loss: -0.18873 (-0.07093) + | > log_mle: -0.47951 (-0.27827) + | > loss_dur: 0.29079 (0.20734) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 63.04688 (26.87152) + | > current_lr: 0.00002 + | > step_time: 0.25080 (2.51985) + | > loader_time: 0.00390 (0.04913) + + + --> STEP: 232/234 -- GLOBAL_STEP: 20590 + | > loss: -0.08277 (-0.07285) + | > log_mle: -0.63526 (-0.28430) + | > loss_dur: 0.55249 (0.21145) + | > amp_scaler: 2048.00000 (4069.51724) + | > grad_norm: 155.06625 (28.59575) + | > current_lr: 0.00002 + | > step_time: 0.34920 (2.47135) + | > loader_time: 0.00560 (0.04817) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07975 (-0.07198) + | > avg_loss: -0.11810 (-0.00807) + | > avg_log_mle: -0.35765 (-0.00393) + | > avg_loss_dur: 0.23956 (-0.00414) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_20592.pth + + > EPOCH: 88/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 03:37:06)  + + --> STEP: 3/234 -- GLOBAL_STEP: 20595 + | > loss: 0.05327 (0.03500) + | > log_mle: -0.16855 (-0.16030) + | > loss_dur: 0.22182 (0.19530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.48305 (6.01435) + | > current_lr: 0.00002 + | > step_time: 2.29020 (6.72736) + | > loader_time: 0.09820 (0.29645) + + + --> STEP: 8/234 -- GLOBAL_STEP: 20600 + | > loss: 0.00044 (0.02227) + | > log_mle: -0.17549 (-0.16603) + | > loss_dur: 0.17593 (0.18830) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.92860 (6.52755) + | > current_lr: 0.00002 + | > step_time: 1.41780 (3.37719) + | > loader_time: 0.00210 (0.11227) + + + --> STEP: 13/234 -- GLOBAL_STEP: 20605 + | > loss: 0.01802 (0.01457) + | > log_mle: -0.15778 (-0.16594) + | > loss_dur: 0.17580 (0.18051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.05960 (6.28626) + | > current_lr: 0.00002 + | > step_time: 1.10830 (2.74006) + | > loader_time: 0.09530 (0.09730) + + + --> STEP: 18/234 -- GLOBAL_STEP: 20610 + | > loss: 0.01019 (0.00932) + | > log_mle: -0.16085 (-0.16457) + | > loss_dur: 0.17104 (0.17389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.15669 (6.00765) + | > current_lr: 0.00002 + | > step_time: 2.50880 (2.65233) + | > loader_time: 0.08750 (0.07587) + + + --> STEP: 23/234 -- GLOBAL_STEP: 20615 + | > loss: -0.01933 (0.00606) + | > log_mle: -0.16529 (-0.16306) + | > loss_dur: 0.14597 (0.16912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.40510 (5.75892) + | > current_lr: 0.00002 + | > step_time: 1.79740 (2.93667) + | > loader_time: 0.09740 (0.06870) + + + --> STEP: 28/234 -- GLOBAL_STEP: 20620 + | > loss: -0.01717 (0.00312) + | > log_mle: -0.15498 (-0.16245) + | > loss_dur: 0.13781 (0.16557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.67883 (5.49556) + | > current_lr: 0.00002 + | > step_time: 4.69580 (3.30181) + | > loader_time: 0.00170 (0.06948) + + + --> STEP: 33/234 -- GLOBAL_STEP: 20625 + | > loss: 0.01738 (0.00152) + | > log_mle: -0.15421 (-0.16319) + | > loss_dur: 0.17160 (0.16470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.34496 (5.38267) + | > current_lr: 0.00002 + | > step_time: 2.09270 (3.32048) + | > loader_time: 0.00380 (0.06499) + + + --> STEP: 38/234 -- GLOBAL_STEP: 20630 + | > loss: -0.00421 (0.00149) + | > log_mle: -0.17277 (-0.16382) + | > loss_dur: 0.16856 (0.16531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.27752 (5.69971) + | > current_lr: 0.00002 + | > step_time: 4.62020 (3.17034) + | > loader_time: 0.08740 (0.05897) + + + --> STEP: 43/234 -- GLOBAL_STEP: 20635 + | > loss: 0.01368 (0.00209) + | > log_mle: -0.17302 (-0.16364) + | > loss_dur: 0.18669 (0.16573) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.42591 (5.77743) + | > current_lr: 0.00002 + | > step_time: 6.50050 (3.15385) + | > loader_time: 0.08310 (0.05650) + + + --> STEP: 48/234 -- GLOBAL_STEP: 20640 + | > loss: -0.00999 (0.00085) + | > log_mle: -0.15543 (-0.16399) + | > loss_dur: 0.14544 (0.16484) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.38634 (5.85358) + | > current_lr: 0.00002 + | > step_time: 1.77160 (2.98343) + | > loader_time: 0.00230 (0.05102) + + + --> STEP: 53/234 -- GLOBAL_STEP: 20645 + | > loss: -0.00514 (0.00047) + | > log_mle: -0.17770 (-0.16388) + | > loss_dur: 0.17256 (0.16435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.44311 (5.70403) + | > current_lr: 0.00002 + | > step_time: 1.20300 (2.83221) + | > loader_time: 0.00280 (0.04812) + + + --> STEP: 58/234 -- GLOBAL_STEP: 20650 + | > loss: -0.01274 (0.00027) + | > log_mle: -0.16116 (-0.16417) + | > loss_dur: 0.14842 (0.16444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.01579 (5.56582) + | > current_lr: 0.00002 + | > step_time: 0.81510 (2.76962) + | > loader_time: 0.00280 (0.04584) + + + --> STEP: 63/234 -- GLOBAL_STEP: 20655 + | > loss: 0.01363 (-0.00079) + | > log_mle: -0.17515 (-0.16598) + | > loss_dur: 0.18878 (0.16519) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.46726 (5.68428) + | > current_lr: 0.00002 + | > step_time: 1.62470 (2.71565) + | > loader_time: 0.00360 (0.04527) + + + --> STEP: 68/234 -- GLOBAL_STEP: 20660 + | > loss: 0.01091 (-0.00089) + | > log_mle: -0.16697 (-0.16607) + | > loss_dur: 0.17787 (0.16518) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.60970 (5.63675) + | > current_lr: 0.00002 + | > step_time: 1.80040 (2.69841) + | > loader_time: 0.00350 (0.04219) + + + --> STEP: 73/234 -- GLOBAL_STEP: 20665 + | > loss: -0.00276 (-0.00035) + | > log_mle: -0.19030 (-0.16669) + | > loss_dur: 0.18755 (0.16634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.28908 (5.85144) + | > current_lr: 0.00002 + | > step_time: 4.39800 (2.66958) + | > loader_time: 0.00290 (0.03956) + + + --> STEP: 78/234 -- GLOBAL_STEP: 20670 + | > loss: 0.01250 (-0.00096) + | > log_mle: -0.16316 (-0.16723) + | > loss_dur: 0.17567 (0.16628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.20321 (5.96607) + | > current_lr: 0.00002 + | > step_time: 1.39600 (2.69074) + | > loader_time: 0.00210 (0.03960) + + + --> STEP: 83/234 -- GLOBAL_STEP: 20675 + | > loss: -0.00873 (-0.00194) + | > log_mle: -0.19501 (-0.16799) + | > loss_dur: 0.18628 (0.16605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99165 (6.04865) + | > current_lr: 0.00002 + | > step_time: 3.10000 (2.69973) + | > loader_time: 0.00250 (0.03980) + + + --> STEP: 88/234 -- GLOBAL_STEP: 20680 + | > loss: -0.04101 (-0.00264) + | > log_mle: -0.23008 (-0.16932) + | > loss_dur: 0.18907 (0.16668) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.29086 (6.25462) + | > current_lr: 0.00002 + | > step_time: 2.40830 (2.66394) + | > loader_time: 0.00520 (0.03956) + + + --> STEP: 93/234 -- GLOBAL_STEP: 20685 + | > loss: -0.03722 (-0.00449) + | > log_mle: -0.23997 (-0.17191) + | > loss_dur: 0.20275 (0.16743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.03759 (6.65987) + | > current_lr: 0.00002 + | > step_time: 1.95470 (2.65503) + | > loader_time: 0.00340 (0.03838) + + + --> STEP: 98/234 -- GLOBAL_STEP: 20690 + | > loss: 0.00672 (-0.00624) + | > log_mle: -0.16926 (-0.17451) + | > loss_dur: 0.17598 (0.16827) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.63072 (7.01795) + | > current_lr: 0.00002 + | > step_time: 2.30830 (2.65331) + | > loader_time: 0.00240 (0.03741) + + + --> STEP: 103/234 -- GLOBAL_STEP: 20695 + | > loss: -0.05142 (-0.00811) + | > log_mle: -0.26706 (-0.17771) + | > loss_dur: 0.21564 (0.16960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.72288 (7.53067) + | > current_lr: 0.00002 + | > step_time: 1.69450 (2.65437) + | > loader_time: 0.02180 (0.03713) + + + --> STEP: 108/234 -- GLOBAL_STEP: 20700 + | > loss: -0.03180 (-0.00982) + | > log_mle: -0.20663 (-0.18035) + | > loss_dur: 0.17484 (0.17054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.27645 (8.00389) + | > current_lr: 0.00002 + | > step_time: 3.41810 (2.68347) + | > loader_time: 0.09910 (0.03812) + + + --> STEP: 113/234 -- GLOBAL_STEP: 20705 + | > loss: -0.06899 (-0.01150) + | > log_mle: -0.25995 (-0.18364) + | > loss_dur: 0.19096 (0.17214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.22850 (8.64413) + | > current_lr: 0.00002 + | > step_time: 0.97330 (2.66527) + | > loader_time: 0.00290 (0.03833) + + + --> STEP: 118/234 -- GLOBAL_STEP: 20710 + | > loss: -0.02012 (-0.01241) + | > log_mle: -0.22803 (-0.18613) + | > loss_dur: 0.20791 (0.17372) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.08382 (8.95042) + | > current_lr: 0.00002 + | > step_time: 2.80980 (2.64733) + | > loader_time: 0.00230 (0.03683) + + + --> STEP: 123/234 -- GLOBAL_STEP: 20715 + | > loss: -0.02912 (-0.01334) + | > log_mle: -0.19944 (-0.18769) + | > loss_dur: 0.17032 (0.17435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.31193 (9.17304) + | > current_lr: 0.00002 + | > step_time: 1.10130 (2.64420) + | > loader_time: 0.00340 (0.03748) + + + --> STEP: 128/234 -- GLOBAL_STEP: 20720 + | > loss: -0.08383 (-0.01569) + | > log_mle: -0.26064 (-0.19116) + | > loss_dur: 0.17680 (0.17547) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.07555 (9.66204) + | > current_lr: 0.00002 + | > step_time: 1.50970 (2.66278) + | > loader_time: 0.10320 (0.03834) + + + --> STEP: 133/234 -- GLOBAL_STEP: 20725 + | > loss: -0.05944 (-0.01787) + | > log_mle: -0.28362 (-0.19467) + | > loss_dur: 0.22418 (0.17679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.73247 (10.24938) + | > current_lr: 0.00002 + | > step_time: 2.80970 (2.65304) + | > loader_time: 0.08100 (0.03830) + + + --> STEP: 138/234 -- GLOBAL_STEP: 20730 + | > loss: -0.04251 (-0.01975) + | > log_mle: -0.23972 (-0.19793) + | > loss_dur: 0.19720 (0.17819) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.29237 (10.82083) + | > current_lr: 0.00002 + | > step_time: 2.69880 (2.67219) + | > loader_time: 0.00440 (0.03895) + + + --> STEP: 143/234 -- GLOBAL_STEP: 20735 + | > loss: -0.10390 (-0.02206) + | > log_mle: -0.37205 (-0.20202) + | > loss_dur: 0.26814 (0.17997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.62201 (11.62034) + | > current_lr: 0.00002 + | > step_time: 1.21080 (2.66146) + | > loader_time: 0.00630 (0.03962) + + + --> STEP: 148/234 -- GLOBAL_STEP: 20740 + | > loss: -0.09290 (-0.02478) + | > log_mle: -0.28747 (-0.20600) + | > loss_dur: 0.19457 (0.18122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.28778 (12.85586) + | > current_lr: 0.00002 + | > step_time: 2.60660 (2.67776) + | > loader_time: 0.08970 (0.04021) + + + --> STEP: 153/234 -- GLOBAL_STEP: 20745 + | > loss: -0.17948 (-0.02814) + | > log_mle: -0.40550 (-0.21098) + | > loss_dur: 0.22603 (0.18284) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.64364 (13.88789) + | > current_lr: 0.00002 + | > step_time: 6.01050 (2.76220) + | > loader_time: 0.20270 (0.04403) + + + --> STEP: 158/234 -- GLOBAL_STEP: 20750 + | > loss: -0.10803 (-0.03113) + | > log_mle: -0.34994 (-0.21544) + | > loss_dur: 0.24191 (0.18431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.76040 (14.69331) + | > current_lr: 0.00002 + | > step_time: 3.40850 (2.73270) + | > loader_time: 0.10180 (0.04337) + + + --> STEP: 163/234 -- GLOBAL_STEP: 20755 + | > loss: -0.09511 (-0.03424) + | > log_mle: -0.31994 (-0.21981) + | > loss_dur: 0.22483 (0.18557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.92521 (15.47070) + | > current_lr: 0.00002 + | > step_time: 1.49130 (2.73758) + | > loader_time: 0.00320 (0.04324) + + + --> STEP: 168/234 -- GLOBAL_STEP: 20760 + | > loss: -0.12869 (-0.03740) + | > log_mle: -0.37411 (-0.22434) + | > loss_dur: 0.24542 (0.18694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.44016 (16.09436) + | > current_lr: 0.00002 + | > step_time: 3.50800 (2.72283) + | > loader_time: 0.00400 (0.04256) + + + --> STEP: 173/234 -- GLOBAL_STEP: 20765 + | > loss: -0.13445 (-0.04059) + | > log_mle: -0.37813 (-0.22934) + | > loss_dur: 0.24367 (0.18875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.52340 (16.89153) + | > current_lr: 0.00002 + | > step_time: 2.80560 (2.73434) + | > loader_time: 0.08270 (0.04285) + + + --> STEP: 178/234 -- GLOBAL_STEP: 20770 + | > loss: -0.16466 (-0.04373) + | > log_mle: -0.43138 (-0.23424) + | > loss_dur: 0.26672 (0.19052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.02571 (18.06151) + | > current_lr: 0.00002 + | > step_time: 3.78990 (2.80510) + | > loader_time: 0.00530 (0.04180) + + + --> STEP: 183/234 -- GLOBAL_STEP: 20775 + | > loss: -0.18564 (-0.04668) + | > log_mle: -0.43079 (-0.23883) + | > loss_dur: 0.24515 (0.19215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.80757 (18.88052) + | > current_lr: 0.00002 + | > step_time: 10.00230 (2.85707) + | > loader_time: 0.19060 (0.04316) + + + --> STEP: 188/234 -- GLOBAL_STEP: 20780 + | > loss: -0.18560 (-0.04957) + | > log_mle: -0.44176 (-0.24344) + | > loss_dur: 0.25616 (0.19387) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.61757 (19.63700) + | > current_lr: 0.00002 + | > step_time: 1.48400 (2.93733) + | > loader_time: 0.02340 (0.04909) + + + --> STEP: 193/234 -- GLOBAL_STEP: 20785 + | > loss: -0.17382 (-0.05253) + | > log_mle: -0.42744 (-0.24774) + | > loss_dur: 0.25362 (0.19521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.34093 (20.88221) + | > current_lr: 0.00002 + | > step_time: 7.90450 (3.00270) + | > loader_time: 0.09510 (0.04887) + + + --> STEP: 198/234 -- GLOBAL_STEP: 20790 + | > loss: -0.17672 (-0.05519) + | > log_mle: -0.43175 (-0.25190) + | > loss_dur: 0.25503 (0.19671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.73641 (21.75437) + | > current_lr: 0.00002 + | > step_time: 5.61230 (3.03008) + | > loader_time: 0.09830 (0.04866) + + + --> STEP: 203/234 -- GLOBAL_STEP: 20795 + | > loss: -0.13383 (-0.05767) + | > log_mle: -0.37172 (-0.25595) + | > loss_dur: 0.23789 (0.19828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.08583 (22.54268) + | > current_lr: 0.00002 + | > step_time: 5.89290 (3.08848) + | > loader_time: 0.11110 (0.05198) + + + --> STEP: 208/234 -- GLOBAL_STEP: 20800 + | > loss: -0.17234 (-0.06076) + | > log_mle: -0.44611 (-0.26068) + | > loss_dur: 0.27377 (0.19992) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.22451 (23.46440) + | > current_lr: 0.00002 + | > step_time: 6.19790 (3.16952) + | > loader_time: 0.10900 (0.05457) + + + --> STEP: 213/234 -- GLOBAL_STEP: 20805 + | > loss: -0.21863 (-0.06419) + | > log_mle: -0.49713 (-0.26587) + | > loss_dur: 0.27850 (0.20168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.26229 (24.31984) + | > current_lr: 0.00002 + | > step_time: 6.89380 (3.22791) + | > loader_time: 0.19460 (0.05555) + + + --> STEP: 218/234 -- GLOBAL_STEP: 20810 + | > loss: -0.18885 (-0.06747) + | > log_mle: -0.45528 (-0.27060) + | > loss_dur: 0.26643 (0.20313) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.37137 (25.22310) + | > current_lr: 0.00002 + | > step_time: 4.81670 (3.27275) + | > loader_time: 0.08550 (0.05691) + + + --> STEP: 223/234 -- GLOBAL_STEP: 20815 + | > loss: -0.22372 (-0.07074) + | > log_mle: -0.49905 (-0.27553) + | > loss_dur: 0.27533 (0.20479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.72900 (26.32699) + | > current_lr: 0.00002 + | > step_time: 1.41640 (3.25960) + | > loader_time: 0.09810 (0.05701) + + + --> STEP: 228/234 -- GLOBAL_STEP: 20820 + | > loss: -0.19173 (-0.07390) + | > log_mle: -0.49431 (-0.28058) + | > loss_dur: 0.30258 (0.20668) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.06158 (27.66403) + | > current_lr: 0.00002 + | > step_time: 0.24690 (3.19356) + | > loader_time: 0.00330 (0.05585) + + + --> STEP: 233/234 -- GLOBAL_STEP: 20825 + | > loss: 0.40728 (-0.07349) + | > log_mle: -0.45568 (-0.28670) + | > loss_dur: 0.86295 (0.21322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.50061 (28.86814) + | > current_lr: 0.00002 + | > step_time: 0.19260 (3.13061) + | > loader_time: 0.00270 (0.05478) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00481 (-0.07494) + | > avg_loss: -0.11142 (+0.00668) + | > avg_log_mle: -0.35361 (+0.00405) + | > avg_loss_dur: 0.24219 (+0.00263) + + + > EPOCH: 89/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 03:50:47)  + + --> STEP: 4/234 -- GLOBAL_STEP: 20830 + | > loss: 0.04360 (0.03648) + | > log_mle: -0.17102 (-0.16440) + | > loss_dur: 0.21462 (0.20088) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.42063 (6.85961) + | > current_lr: 0.00002 + | > step_time: 3.09780 (2.44966) + | > loader_time: 0.00280 (0.32573) + + + --> STEP: 9/234 -- GLOBAL_STEP: 20835 + | > loss: 0.00610 (0.02107) + | > log_mle: -0.17842 (-0.16942) + | > loss_dur: 0.18452 (0.19048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58561 (6.60214) + | > current_lr: 0.00002 + | > step_time: 6.79240 (5.85214) + | > loader_time: 0.10930 (0.43594) + + + --> STEP: 14/234 -- GLOBAL_STEP: 20840 + | > loss: -0.00845 (0.01264) + | > log_mle: -0.17146 (-0.16841) + | > loss_dur: 0.16300 (0.18105) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.48272 (6.33650) + | > current_lr: 0.00002 + | > step_time: 2.99960 (5.45649) + | > loader_time: 0.09410 (0.30931) + + + --> STEP: 19/234 -- GLOBAL_STEP: 20845 + | > loss: -0.01935 (0.00701) + | > log_mle: -0.15551 (-0.16610) + | > loss_dur: 0.13616 (0.17312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.97004 (6.04688) + | > current_lr: 0.00002 + | > step_time: 0.71580 (5.31146) + | > loader_time: 0.00420 (0.24867) + + + --> STEP: 24/234 -- GLOBAL_STEP: 20850 + | > loss: -0.01354 (0.00417) + | > log_mle: -0.15783 (-0.16478) + | > loss_dur: 0.14429 (0.16895) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.84192 (5.71831) + | > current_lr: 0.00002 + | > step_time: 4.40510 (5.04232) + | > loader_time: 0.00260 (0.20998) + + + --> STEP: 29/234 -- GLOBAL_STEP: 20855 + | > loss: -0.00101 (0.00197) + | > log_mle: -0.15581 (-0.16418) + | > loss_dur: 0.15480 (0.16615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.63154 (5.48622) + | > current_lr: 0.00002 + | > step_time: 3.40930 (4.65658) + | > loader_time: 0.00250 (0.17452) + + + --> STEP: 34/234 -- GLOBAL_STEP: 20860 + | > loss: 0.01392 (0.00098) + | > log_mle: -0.16389 (-0.16503) + | > loss_dur: 0.17780 (0.16601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.53478 (5.37421) + | > current_lr: 0.00002 + | > step_time: 1.28550 (4.68615) + | > loader_time: 0.00140 (0.15482) + + + --> STEP: 39/234 -- GLOBAL_STEP: 20865 + | > loss: 0.00066 (0.00053) + | > log_mle: -0.17104 (-0.16580) + | > loss_dur: 0.17170 (0.16634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.35727 (5.73357) + | > current_lr: 0.00002 + | > step_time: 1.20690 (4.36375) + | > loader_time: 0.07990 (0.13730) + + + --> STEP: 44/234 -- GLOBAL_STEP: 20870 + | > loss: -0.01309 (0.00103) + | > log_mle: -0.16227 (-0.16535) + | > loss_dur: 0.14918 (0.16638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.20402 (5.57837) + | > current_lr: 0.00002 + | > step_time: 0.95960 (4.11016) + | > loader_time: 0.00130 (0.12368) + + + --> STEP: 49/234 -- GLOBAL_STEP: 20875 + | > loss: -0.03470 (-0.00073) + | > log_mle: -0.17030 (-0.16588) + | > loss_dur: 0.13560 (0.16515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.37151 (5.54456) + | > current_lr: 0.00002 + | > step_time: 7.40540 (4.02171) + | > loader_time: 0.10840 (0.11720) + + + --> STEP: 54/234 -- GLOBAL_STEP: 20880 + | > loss: -0.01427 (-0.00052) + | > log_mle: -0.17629 (-0.16585) + | > loss_dur: 0.16202 (0.16533) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.38894 (5.46688) + | > current_lr: 0.00002 + | > step_time: 1.29720 (3.83969) + | > loader_time: 0.00230 (0.11130) + + + --> STEP: 59/234 -- GLOBAL_STEP: 20885 + | > loss: -0.05099 (-0.00129) + | > log_mle: -0.19216 (-0.16641) + | > loss_dur: 0.14117 (0.16513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.13835 (5.43773) + | > current_lr: 0.00002 + | > step_time: 2.89040 (3.70492) + | > loader_time: 0.20290 (0.10734) + + + --> STEP: 64/234 -- GLOBAL_STEP: 20890 + | > loss: 0.00051 (-0.00188) + | > log_mle: -0.15950 (-0.16765) + | > loss_dur: 0.16001 (0.16577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 2.63607 (5.69586) + | > current_lr: 0.00002 + | > step_time: 1.38310 (3.55338) + | > loader_time: 0.00270 (0.09965) + + + --> STEP: 69/234 -- GLOBAL_STEP: 20895 + | > loss: 0.02694 (-0.00156) + | > log_mle: -0.14926 (-0.16762) + | > loss_dur: 0.17620 (0.16607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.16197 (5.65881) + | > current_lr: 0.00002 + | > step_time: 0.99590 (3.47467) + | > loader_time: 0.00330 (0.09679) + + + --> STEP: 74/234 -- GLOBAL_STEP: 20900 + | > loss: -0.01726 (-0.00153) + | > log_mle: -0.16799 (-0.16837) + | > loss_dur: 0.15074 (0.16684) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06793 (6.55985) + | > current_lr: 0.00002 + | > step_time: 5.50250 (3.45068) + | > loader_time: 0.00400 (0.09400) + + + --> STEP: 79/234 -- GLOBAL_STEP: 20905 + | > loss: -0.02509 (-0.00248) + | > log_mle: -0.18044 (-0.16908) + | > loss_dur: 0.15535 (0.16661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.37394 (6.57449) + | > current_lr: 0.00002 + | > step_time: 1.91690 (3.35405) + | > loader_time: 0.00280 (0.08824) + + + --> STEP: 84/234 -- GLOBAL_STEP: 20910 + | > loss: 0.00135 (-0.00319) + | > log_mle: -0.17623 (-0.16974) + | > loss_dur: 0.17757 (0.16655) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.26901 (6.68031) + | > current_lr: 0.00002 + | > step_time: 2.00410 (3.29029) + | > loader_time: 0.08220 (0.08704) + + + --> STEP: 89/234 -- GLOBAL_STEP: 20915 + | > loss: -0.04357 (-0.00442) + | > log_mle: -0.20908 (-0.17138) + | > loss_dur: 0.16552 (0.16696) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.62159 (6.88120) + | > current_lr: 0.00002 + | > step_time: 2.30060 (3.25111) + | > loader_time: 0.00420 (0.08236) + + + --> STEP: 94/234 -- GLOBAL_STEP: 20920 + | > loss: -0.05702 (-0.00617) + | > log_mle: -0.24376 (-0.17417) + | > loss_dur: 0.18674 (0.16799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.28141 (7.40859) + | > current_lr: 0.00002 + | > step_time: 5.29960 (3.28035) + | > loader_time: 0.08750 (0.08225) + + + --> STEP: 99/234 -- GLOBAL_STEP: 20925 + | > loss: -0.08054 (-0.00802) + | > log_mle: -0.27560 (-0.17696) + | > loss_dur: 0.19506 (0.16894) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.39866 (7.86153) + | > current_lr: 0.00002 + | > step_time: 0.85880 (3.26113) + | > loader_time: 0.00240 (0.08310) + + + --> STEP: 104/234 -- GLOBAL_STEP: 20930 + | > loss: -0.10048 (-0.01023) + | > log_mle: -0.28899 (-0.18028) + | > loss_dur: 0.18851 (0.17005) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.21812 (8.30141) + | > current_lr: 0.00002 + | > step_time: 2.58570 (3.25389) + | > loader_time: 0.09780 (0.08394) + + + --> STEP: 109/234 -- GLOBAL_STEP: 20935 + | > loss: -0.02029 (-0.01141) + | > log_mle: -0.26099 (-0.18273) + | > loss_dur: 0.24070 (0.17132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.39486 (8.59579) + | > current_lr: 0.00002 + | > step_time: 1.48910 (3.20841) + | > loader_time: 0.00190 (0.08206) + + + --> STEP: 114/234 -- GLOBAL_STEP: 20940 + | > loss: -0.04867 (-0.01342) + | > log_mle: -0.24102 (-0.18588) + | > loss_dur: 0.19235 (0.17246) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.19255 (9.08268) + | > current_lr: 0.00002 + | > step_time: 2.59920 (3.17645) + | > loader_time: 0.68920 (0.08545) + + + --> STEP: 119/234 -- GLOBAL_STEP: 20945 + | > loss: -0.04835 (-0.01447) + | > log_mle: -0.23860 (-0.18830) + | > loss_dur: 0.19025 (0.17383) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.01234 (9.40908) + | > current_lr: 0.00002 + | > step_time: 4.00490 (3.15666) + | > loader_time: 0.09530 (0.08432) + + + --> STEP: 124/234 -- GLOBAL_STEP: 20950 + | > loss: -0.07353 (-0.01582) + | > log_mle: -0.26616 (-0.19007) + | > loss_dur: 0.19264 (0.17425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.77657 (9.63987) + | > current_lr: 0.00002 + | > step_time: 3.08460 (3.13727) + | > loader_time: 0.00300 (0.08256) + + + --> STEP: 129/234 -- GLOBAL_STEP: 20955 + | > loss: -0.04841 (-0.01790) + | > log_mle: -0.25448 (-0.19334) + | > loss_dur: 0.20607 (0.17544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.30123 (10.37082) + | > current_lr: 0.00002 + | > step_time: 2.50250 (3.14202) + | > loader_time: 0.08420 (0.08230) + + + --> STEP: 134/234 -- GLOBAL_STEP: 20960 + | > loss: -0.06072 (-0.02003) + | > log_mle: -0.29830 (-0.19703) + | > loss_dur: 0.23758 (0.17700) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.59140 (11.25376) + | > current_lr: 0.00002 + | > step_time: 1.57420 (3.12476) + | > loader_time: 0.00190 (0.08062) + + + --> STEP: 139/234 -- GLOBAL_STEP: 20965 + | > loss: -0.13650 (-0.02211) + | > log_mle: -0.35704 (-0.20049) + | > loss_dur: 0.22053 (0.17838) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.81152 (12.05262) + | > current_lr: 0.00002 + | > step_time: 3.00490 (3.09862) + | > loader_time: 0.09470 (0.07967) + + + --> STEP: 144/234 -- GLOBAL_STEP: 20970 + | > loss: -0.10863 (-0.02395) + | > log_mle: -0.33650 (-0.20417) + | > loss_dur: 0.22787 (0.18022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.89656 (12.90904) + | > current_lr: 0.00002 + | > step_time: 2.19880 (3.08810) + | > loader_time: 0.00400 (0.07708) + + + --> STEP: 149/234 -- GLOBAL_STEP: 20975 + | > loss: -0.14926 (-0.02690) + | > log_mle: -0.38077 (-0.20845) + | > loss_dur: 0.23150 (0.18155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.27634 (13.50226) + | > current_lr: 0.00002 + | > step_time: 3.71060 (3.13696) + | > loader_time: 0.10150 (0.07785) + + + --> STEP: 154/234 -- GLOBAL_STEP: 20980 + | > loss: -0.13670 (-0.03025) + | > log_mle: -0.34554 (-0.21321) + | > loss_dur: 0.20884 (0.18295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.93779 (14.12594) + | > current_lr: 0.00002 + | > step_time: 0.98750 (3.09833) + | > loader_time: 0.00390 (0.07657) + + + --> STEP: 159/234 -- GLOBAL_STEP: 20985 + | > loss: -0.13574 (-0.03319) + | > log_mle: -0.36089 (-0.21769) + | > loss_dur: 0.22515 (0.18450) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.29971 (15.01167) + | > current_lr: 0.00002 + | > step_time: 2.80550 (3.08902) + | > loader_time: 0.00260 (0.07543) + + + --> STEP: 164/234 -- GLOBAL_STEP: 20990 + | > loss: -0.11200 (-0.03617) + | > log_mle: -0.35453 (-0.22194) + | > loss_dur: 0.24253 (0.18578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.04823 (15.85359) + | > current_lr: 0.00002 + | > step_time: 4.69750 (3.07662) + | > loader_time: 0.00920 (0.07429) + + + --> STEP: 169/234 -- GLOBAL_STEP: 20995 + | > loss: -0.10692 (-0.03907) + | > log_mle: -0.35146 (-0.22637) + | > loss_dur: 0.24453 (0.18731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.38943 (16.67111) + | > current_lr: 0.00002 + | > step_time: 4.99320 (3.14706) + | > loader_time: 0.80270 (0.07925) + + + --> STEP: 174/234 -- GLOBAL_STEP: 21000 + | > loss: -0.20224 (-0.04281) + | > log_mle: -0.43905 (-0.23187) + | > loss_dur: 0.23681 (0.18906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.14947 (17.62967) + | > current_lr: 0.00002 + | > step_time: 2.98360 (3.17848) + | > loader_time: 0.00400 (0.07750) + + + --> STEP: 179/234 -- GLOBAL_STEP: 21005 + | > loss: -0.16314 (-0.04577) + | > log_mle: -0.42605 (-0.23672) + | > loss_dur: 0.26292 (0.19095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.04103 (18.65789) + | > current_lr: 0.00002 + | > step_time: 3.78970 (3.18519) + | > loader_time: 0.00270 (0.07593) + + + --> STEP: 184/234 -- GLOBAL_STEP: 21010 + | > loss: -0.14718 (-0.04854) + | > log_mle: -0.40100 (-0.24114) + | > loss_dur: 0.25382 (0.19260) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.34365 (19.47119) + | > current_lr: 0.00002 + | > step_time: 4.68330 (3.22261) + | > loader_time: 0.11670 (0.07764) + + + --> STEP: 189/234 -- GLOBAL_STEP: 21015 + | > loss: -0.13678 (-0.05138) + | > log_mle: -0.39927 (-0.24575) + | > loss_dur: 0.26249 (0.19437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.18650 (20.28151) + | > current_lr: 0.00002 + | > step_time: 8.39770 (3.33570) + | > loader_time: 0.10650 (0.07779) + + + --> STEP: 194/234 -- GLOBAL_STEP: 21020 + | > loss: -0.17715 (-0.05475) + | > log_mle: -0.42782 (-0.25037) + | > loss_dur: 0.25067 (0.19562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.73497 (20.99284) + | > current_lr: 0.00002 + | > step_time: 8.58860 (3.40990) + | > loader_time: 0.01110 (0.07689) + + + --> STEP: 199/234 -- GLOBAL_STEP: 21025 + | > loss: -0.17986 (-0.05755) + | > log_mle: -0.43787 (-0.25465) + | > loss_dur: 0.25802 (0.19710) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.07723 (21.70786) + | > current_lr: 0.00002 + | > step_time: 12.51180 (3.49171) + | > loader_time: 0.09490 (0.07830) + + + --> STEP: 204/234 -- GLOBAL_STEP: 21030 + | > loss: -0.19340 (-0.06016) + | > log_mle: -0.47169 (-0.25888) + | > loss_dur: 0.27829 (0.19871) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.22231 (22.47550) + | > current_lr: 0.00002 + | > step_time: 7.50090 (3.50461) + | > loader_time: 0.19310 (0.07869) + + + --> STEP: 209/234 -- GLOBAL_STEP: 21035 + | > loss: -0.16389 (-0.06317) + | > log_mle: -0.42501 (-0.26340) + | > loss_dur: 0.26112 (0.20023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.14915 (23.31309) + | > current_lr: 0.00002 + | > step_time: 3.20700 (3.53322) + | > loader_time: 0.00470 (0.07821) + + + --> STEP: 214/234 -- GLOBAL_STEP: 21040 + | > loss: -0.21533 (-0.06676) + | > log_mle: -0.45933 (-0.26859) + | > loss_dur: 0.24400 (0.20184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.80755 (24.52781) + | > current_lr: 0.00002 + | > step_time: 1.30060 (3.57543) + | > loader_time: 0.00310 (0.07740) + + + --> STEP: 219/234 -- GLOBAL_STEP: 21045 + | > loss: -0.26598 (-0.07016) + | > log_mle: -0.54569 (-0.27363) + | > loss_dur: 0.27971 (0.20347) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.46235 (25.46488) + | > current_lr: 0.00002 + | > step_time: 4.69080 (3.63031) + | > loader_time: 0.78640 (0.08104) + + + --> STEP: 224/234 -- GLOBAL_STEP: 21050 + | > loss: -0.22049 (-0.07323) + | > log_mle: -0.50385 (-0.27835) + | > loss_dur: 0.28337 (0.20513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.81320 (26.35513) + | > current_lr: 0.00002 + | > step_time: 0.22650 (3.57807) + | > loader_time: 0.00300 (0.07934) + + + --> STEP: 229/234 -- GLOBAL_STEP: 21055 + | > loss: -0.18609 (-0.07637) + | > log_mle: -0.52873 (-0.28359) + | > loss_dur: 0.34264 (0.20722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.21410 (27.38002) + | > current_lr: 0.00002 + | > step_time: 0.24040 (3.50509) + | > loader_time: 0.00400 (0.07768) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.33511 (+0.33031) + | > avg_loss: -0.12302 (-0.01160) + | > avg_log_mle: -0.36349 (-0.00988) + | > avg_loss_dur: 0.24047 (-0.00171) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_21060.pth + + > EPOCH: 90/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 04:05:33)  + + --> STEP: 0/234 -- GLOBAL_STEP: 21060 + | > loss: -0.02196 (-0.02196) + | > log_mle: -0.22074 (-0.22074) + | > loss_dur: 0.19878 (0.19878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.51797 (6.51797) + | > current_lr: 0.00002 + | > step_time: 5.00820 (5.00821) + | > loader_time: 8.65720 (8.65720) + + + --> STEP: 5/234 -- GLOBAL_STEP: 21065 + | > loss: 0.01773 (0.02418) + | > log_mle: -0.17066 (-0.16629) + | > loss_dur: 0.18838 (0.19048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.90911 (14.03625) + | > current_lr: 0.00002 + | > step_time: 4.59450 (6.04274) + | > loader_time: 0.00470 (0.13983) + + + --> STEP: 10/234 -- GLOBAL_STEP: 21070 + | > loss: -0.00944 (0.00997) + | > log_mle: -0.17273 (-0.17041) + | > loss_dur: 0.16329 (0.18038) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.83626 (10.43866) + | > current_lr: 0.00002 + | > step_time: 1.39140 (4.92144) + | > loader_time: 0.00230 (0.07099) + + + --> STEP: 15/234 -- GLOBAL_STEP: 21075 + | > loss: -0.02785 (0.00586) + | > log_mle: -0.16979 (-0.16914) + | > loss_dur: 0.14193 (0.17500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.16982 (8.89225) + | > current_lr: 0.00002 + | > step_time: 3.39710 (4.27330) + | > loader_time: 0.00310 (0.05404) + + + --> STEP: 20/234 -- GLOBAL_STEP: 21080 + | > loss: 0.00289 (0.00200) + | > log_mle: -0.15603 (-0.16643) + | > loss_dur: 0.15893 (0.16843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.51140 (8.06870) + | > current_lr: 0.00002 + | > step_time: 2.11070 (3.54526) + | > loader_time: 0.00220 (0.04522) + + + --> STEP: 25/234 -- GLOBAL_STEP: 21085 + | > loss: 0.01676 (0.00063) + | > log_mle: -0.15036 (-0.16505) + | > loss_dur: 0.16712 (0.16568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.16229 (7.51274) + | > current_lr: 0.00002 + | > step_time: 1.37940 (3.67088) + | > loader_time: 0.08360 (0.04707) + + + --> STEP: 30/234 -- GLOBAL_STEP: 21090 + | > loss: -0.03241 (-0.00264) + | > log_mle: -0.17944 (-0.16539) + | > loss_dur: 0.14704 (0.16275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.21293 (7.08242) + | > current_lr: 0.00002 + | > step_time: 0.89570 (3.62630) + | > loader_time: 0.00210 (0.03998) + + + --> STEP: 35/234 -- GLOBAL_STEP: 21095 + | > loss: -0.02108 (-0.00235) + | > log_mle: -0.17147 (-0.16600) + | > loss_dur: 0.15039 (0.16365) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.77176 (7.18601) + | > current_lr: 0.00002 + | > step_time: 3.11930 (3.66580) + | > loader_time: 0.00340 (0.04339) + + + --> STEP: 40/234 -- GLOBAL_STEP: 21100 + | > loss: 0.02025 (-0.00141) + | > log_mle: -0.15474 (-0.16624) + | > loss_dur: 0.17498 (0.16483) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.55788 (7.15879) + | > current_lr: 0.00002 + | > step_time: 1.01420 (3.38487) + | > loader_time: 0.08500 (0.04496) + + + --> STEP: 45/234 -- GLOBAL_STEP: 21105 + | > loss: -0.02915 (-0.00192) + | > log_mle: -0.19088 (-0.16667) + | > loss_dur: 0.16173 (0.16474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.37271 (7.02030) + | > current_lr: 0.00002 + | > step_time: 1.89700 (3.25745) + | > loader_time: 0.00620 (0.04259) + + + --> STEP: 50/234 -- GLOBAL_STEP: 21110 + | > loss: 0.00168 (-0.00307) + | > log_mle: -0.15796 (-0.16650) + | > loss_dur: 0.15964 (0.16343) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.36195 (6.79774) + | > current_lr: 0.00002 + | > step_time: 1.02730 (3.12577) + | > loader_time: 0.00350 (0.04189) + + + --> STEP: 55/234 -- GLOBAL_STEP: 21115 + | > loss: -0.02465 (-0.00344) + | > log_mle: -0.17791 (-0.16689) + | > loss_dur: 0.15327 (0.16345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.95235 (6.57577) + | > current_lr: 0.00002 + | > step_time: 2.02170 (3.00254) + | > loader_time: 0.07710 (0.03966) + + + --> STEP: 60/234 -- GLOBAL_STEP: 21120 + | > loss: -0.03358 (-0.00445) + | > log_mle: -0.19658 (-0.16778) + | > loss_dur: 0.16300 (0.16333) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76451 (6.50056) + | > current_lr: 0.00002 + | > step_time: 1.30100 (2.88503) + | > loader_time: 0.00270 (0.03658) + + + --> STEP: 65/234 -- GLOBAL_STEP: 21125 + | > loss: -0.01228 (-0.00477) + | > log_mle: -0.16961 (-0.16863) + | > loss_dur: 0.15733 (0.16386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.69985 (6.49186) + | > current_lr: 0.00002 + | > step_time: 2.49410 (2.86587) + | > loader_time: 0.00730 (0.03410) + + + --> STEP: 70/234 -- GLOBAL_STEP: 21130 + | > loss: 0.00943 (-0.00385) + | > log_mle: -0.17025 (-0.16858) + | > loss_dur: 0.17968 (0.16473) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.86204 (6.58598) + | > current_lr: 0.00002 + | > step_time: 2.70030 (2.80713) + | > loader_time: 0.00360 (0.03298) + + + --> STEP: 75/234 -- GLOBAL_STEP: 21135 + | > loss: -0.00226 (-0.00376) + | > log_mle: -0.18356 (-0.16962) + | > loss_dur: 0.18130 (0.16586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.55181 (6.84573) + | > current_lr: 0.00002 + | > step_time: 1.01210 (2.75484) + | > loader_time: 0.07770 (0.03210) + + + --> STEP: 80/234 -- GLOBAL_STEP: 21140 + | > loss: -0.01895 (-0.00451) + | > log_mle: -0.16426 (-0.17010) + | > loss_dur: 0.14531 (0.16559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 2.83759 (6.82479) + | > current_lr: 0.00002 + | > step_time: 3.44100 (2.76562) + | > loader_time: 0.00640 (0.03453) + + + --> STEP: 85/234 -- GLOBAL_STEP: 21145 + | > loss: -0.02399 (-0.00528) + | > log_mle: -0.18267 (-0.17105) + | > loss_dur: 0.15868 (0.16577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.37769 (6.99742) + | > current_lr: 0.00002 + | > step_time: 1.88660 (2.69519) + | > loader_time: 0.00280 (0.03585) + + + --> STEP: 90/234 -- GLOBAL_STEP: 21150 + | > loss: -0.01029 (-0.00619) + | > log_mle: -0.21168 (-0.17308) + | > loss_dur: 0.20139 (0.16689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.67888 (7.28623) + | > current_lr: 0.00002 + | > step_time: 2.69900 (2.66345) + | > loader_time: 0.00350 (0.03581) + + + --> STEP: 95/234 -- GLOBAL_STEP: 21155 + | > loss: -0.09323 (-0.00886) + | > log_mle: -0.29479 (-0.17684) + | > loss_dur: 0.20155 (0.16798) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.05451 (7.85373) + | > current_lr: 0.00002 + | > step_time: 3.40370 (2.62878) + | > loader_time: 0.07530 (0.03572) + + + --> STEP: 100/234 -- GLOBAL_STEP: 21160 + | > loss: -0.03371 (-0.01022) + | > log_mle: -0.22274 (-0.17887) + | > loss_dur: 0.18903 (0.16865) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.29621 (8.12699) + | > current_lr: 0.00002 + | > step_time: 1.39710 (2.59773) + | > loader_time: 0.07910 (0.03727) + + + --> STEP: 105/234 -- GLOBAL_STEP: 21165 + | > loss: -0.04237 (-0.01221) + | > log_mle: -0.19691 (-0.18189) + | > loss_dur: 0.15454 (0.16967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.44918 (8.55473) + | > current_lr: 0.00002 + | > step_time: 3.48860 (2.59577) + | > loader_time: 0.00280 (0.03643) + + + --> STEP: 110/234 -- GLOBAL_STEP: 21170 + | > loss: -0.05435 (-0.01353) + | > log_mle: -0.22496 (-0.18460) + | > loss_dur: 0.17061 (0.17107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.20576 (8.94366) + | > current_lr: 0.00002 + | > step_time: 5.01180 (2.60500) + | > loader_time: 0.18550 (0.03740) + + + --> STEP: 115/234 -- GLOBAL_STEP: 21175 + | > loss: -0.03732 (-0.01537) + | > log_mle: -0.24310 (-0.18787) + | > loss_dur: 0.20577 (0.17250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.82123 (9.51163) + | > current_lr: 0.00002 + | > step_time: 1.06980 (2.59132) + | > loader_time: 0.00240 (0.03667) + + + --> STEP: 120/234 -- GLOBAL_STEP: 21180 + | > loss: -0.07906 (-0.01687) + | > log_mle: -0.28889 (-0.19068) + | > loss_dur: 0.20983 (0.17381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.48713 (9.91608) + | > current_lr: 0.00002 + | > step_time: 2.09950 (2.58252) + | > loader_time: 0.07850 (0.03893) + + + --> STEP: 125/234 -- GLOBAL_STEP: 21185 + | > loss: -0.07227 (-0.01803) + | > log_mle: -0.27603 (-0.19228) + | > loss_dur: 0.20377 (0.17424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.87883 (10.20764) + | > current_lr: 0.00002 + | > step_time: 4.31830 (2.61557) + | > loader_time: 0.19800 (0.04184) + + + --> STEP: 130/234 -- GLOBAL_STEP: 21190 + | > loss: -0.06741 (-0.01993) + | > log_mle: -0.28610 (-0.19555) + | > loss_dur: 0.21869 (0.17562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.24876 (10.92916) + | > current_lr: 0.00002 + | > step_time: 3.29850 (2.60271) + | > loader_time: 0.10440 (0.04306) + + + --> STEP: 135/234 -- GLOBAL_STEP: 21195 + | > loss: -0.03667 (-0.02197) + | > log_mle: -0.22218 (-0.19885) + | > loss_dur: 0.18551 (0.17688) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.76677 (11.54439) + | > current_lr: 0.00002 + | > step_time: 1.60720 (2.59078) + | > loader_time: 0.00950 (0.04232) + + + --> STEP: 140/234 -- GLOBAL_STEP: 21200 + | > loss: -0.04032 (-0.02430) + | > log_mle: -0.25600 (-0.20270) + | > loss_dur: 0.21567 (0.17839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.50805 (12.21673) + | > current_lr: 0.00002 + | > step_time: 2.50550 (2.58104) + | > loader_time: 0.10050 (0.04288) + + + --> STEP: 145/234 -- GLOBAL_STEP: 21205 + | > loss: -0.12805 (-0.02688) + | > log_mle: -0.35064 (-0.20717) + | > loss_dur: 0.22259 (0.18029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.53185 (12.85520) + | > current_lr: 0.00002 + | > step_time: 3.99790 (2.60922) + | > loader_time: 0.00880 (0.04426) + + + --> STEP: 150/234 -- GLOBAL_STEP: 21210 + | > loss: -0.10181 (-0.02972) + | > log_mle: -0.33318 (-0.21132) + | > loss_dur: 0.23137 (0.18160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.93002 (13.51365) + | > current_lr: 0.00002 + | > step_time: 5.30510 (2.63243) + | > loader_time: 0.08360 (0.04413) + + + --> STEP: 155/234 -- GLOBAL_STEP: 21215 + | > loss: -0.16117 (-0.03329) + | > log_mle: -0.39844 (-0.21640) + | > loss_dur: 0.23727 (0.18311) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.67541 (14.35138) + | > current_lr: 0.00002 + | > step_time: 1.49300 (2.63189) + | > loader_time: 0.00230 (0.04385) + + + --> STEP: 160/234 -- GLOBAL_STEP: 21220 + | > loss: -0.16654 (-0.03635) + | > log_mle: -0.39214 (-0.22081) + | > loss_dur: 0.22560 (0.18446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.94152 (15.14916) + | > current_lr: 0.00002 + | > step_time: 2.48970 (2.69015) + | > loader_time: 0.00290 (0.04452) + + + --> STEP: 165/234 -- GLOBAL_STEP: 21225 + | > loss: -0.13666 (-0.03930) + | > log_mle: -0.38958 (-0.22508) + | > loss_dur: 0.25292 (0.18578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.13013 (15.90039) + | > current_lr: 0.00002 + | > step_time: 1.91200 (2.72933) + | > loader_time: 0.08240 (0.04425) + + + --> STEP: 170/234 -- GLOBAL_STEP: 21230 + | > loss: -0.14680 (-0.04212) + | > log_mle: -0.41925 (-0.22960) + | > loss_dur: 0.27245 (0.18748) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.67010 (17.06458) + | > current_lr: 0.00002 + | > step_time: 4.70420 (2.74016) + | > loader_time: 0.08730 (0.04467) + + + --> STEP: 175/234 -- GLOBAL_STEP: 21235 + | > loss: -0.14289 (-0.04569) + | > log_mle: -0.40113 (-0.23483) + | > loss_dur: 0.25824 (0.18914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.54556 (18.14891) + | > current_lr: 0.00002 + | > step_time: 1.51660 (2.75685) + | > loader_time: 0.08250 (0.04408) + + + --> STEP: 180/234 -- GLOBAL_STEP: 21240 + | > loss: -0.15947 (-0.04877) + | > log_mle: -0.40993 (-0.23978) + | > loss_dur: 0.25046 (0.19101) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.64975 (18.85223) + | > current_lr: 0.00002 + | > step_time: 5.89830 (2.76255) + | > loader_time: 0.00600 (0.04383) + + + --> STEP: 185/234 -- GLOBAL_STEP: 21245 + | > loss: -0.16268 (-0.05152) + | > log_mle: -0.43138 (-0.24428) + | > loss_dur: 0.26870 (0.19276) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.35007 (19.76311) + | > current_lr: 0.00002 + | > step_time: 2.20360 (2.82114) + | > loader_time: 0.08200 (0.04572) + + + --> STEP: 190/234 -- GLOBAL_STEP: 21250 + | > loss: -0.17046 (-0.05436) + | > log_mle: -0.40812 (-0.24860) + | > loss_dur: 0.23766 (0.19424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.20551 (20.80220) + | > current_lr: 0.00002 + | > step_time: 1.61800 (2.82082) + | > loader_time: 0.00400 (0.04603) + + + --> STEP: 195/234 -- GLOBAL_STEP: 21255 + | > loss: -0.16053 (-0.05767) + | > log_mle: -0.42309 (-0.25320) + | > loss_dur: 0.26256 (0.19553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.86376 (21.65775) + | > current_lr: 0.00002 + | > step_time: 5.39860 (2.83377) + | > loader_time: 0.07680 (0.04617) + + + --> STEP: 200/234 -- GLOBAL_STEP: 21260 + | > loss: -0.16099 (-0.06036) + | > log_mle: -0.43542 (-0.25750) + | > loss_dur: 0.27443 (0.19714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.33161 (22.45174) + | > current_lr: 0.00002 + | > step_time: 4.30070 (2.85684) + | > loader_time: 0.00720 (0.04604) + + + --> STEP: 205/234 -- GLOBAL_STEP: 21265 + | > loss: -0.16963 (-0.06304) + | > log_mle: -0.41932 (-0.26167) + | > loss_dur: 0.24970 (0.19864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.77679 (23.16321) + | > current_lr: 0.00002 + | > step_time: 5.19970 (2.89779) + | > loader_time: 0.18800 (0.04648) + + + --> STEP: 210/234 -- GLOBAL_STEP: 21270 + | > loss: -0.22100 (-0.06610) + | > log_mle: -0.48775 (-0.26634) + | > loss_dur: 0.26675 (0.20025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.34228 (24.32758) + | > current_lr: 0.00002 + | > step_time: 2.12340 (2.99843) + | > loader_time: 0.07990 (0.04863) + + + --> STEP: 215/234 -- GLOBAL_STEP: 21275 + | > loss: -0.18940 (-0.06936) + | > log_mle: -0.44127 (-0.27116) + | > loss_dur: 0.25187 (0.20180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.44005 (25.58564) + | > current_lr: 0.00002 + | > step_time: 2.18690 (3.08092) + | > loader_time: 0.00490 (0.05027) + + + --> STEP: 220/234 -- GLOBAL_STEP: 21280 + | > loss: -0.22208 (-0.07292) + | > log_mle: -0.49257 (-0.27639) + | > loss_dur: 0.27049 (0.20346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.62291 (26.71599) + | > current_lr: 0.00002 + | > step_time: 5.99100 (3.11950) + | > loader_time: 0.40230 (0.05227) + + + --> STEP: 225/234 -- GLOBAL_STEP: 21285 + | > loss: -0.25699 (-0.07612) + | > log_mle: -0.54881 (-0.28127) + | > loss_dur: 0.29182 (0.20515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.91417 (27.91891) + | > current_lr: 0.00002 + | > step_time: 0.24580 (3.06644) + | > loader_time: 0.00350 (0.05154) + + + --> STEP: 230/234 -- GLOBAL_STEP: 21290 + | > loss: -0.23529 (-0.07901) + | > log_mle: -0.59914 (-0.28659) + | > loss_dur: 0.36385 (0.20757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.36129 (29.01101) + | > current_lr: 0.00002 + | > step_time: 0.30300 (3.00529) + | > loader_time: 0.00400 (0.05050) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27759 (-0.05752) + | > avg_loss: -0.13130 (-0.00828) + | > avg_log_mle: -0.37351 (-0.01002) + | > avg_loss_dur: 0.24221 (+0.00174) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_21294.pth + + > EPOCH: 91/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 04:18:14)  + + --> STEP: 1/234 -- GLOBAL_STEP: 21295 + | > loss: -0.01359 (-0.01359) + | > log_mle: -0.17070 (-0.17070) + | > loss_dur: 0.15711 (0.15711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.83734 (7.83734) + | > current_lr: 0.00002 + | > step_time: 2.60110 (2.60108) + | > loader_time: 0.00230 (0.00235) + + + --> STEP: 6/234 -- GLOBAL_STEP: 21300 + | > loss: 0.02362 (0.02405) + | > log_mle: -0.15975 (-0.16741) + | > loss_dur: 0.18337 (0.19146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.03901 (7.38756) + | > current_lr: 0.00002 + | > step_time: 7.79160 (3.73634) + | > loader_time: 0.00290 (0.03053) + + + --> STEP: 11/234 -- GLOBAL_STEP: 21305 + | > loss: 0.00516 (0.00895) + | > log_mle: -0.16589 (-0.17166) + | > loss_dur: 0.17105 (0.18062) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.95347 (7.08869) + | > current_lr: 0.00002 + | > step_time: 2.20140 (4.48496) + | > loader_time: 0.00230 (0.05157) + + + --> STEP: 16/234 -- GLOBAL_STEP: 21310 + | > loss: -0.04288 (-0.00003) + | > log_mle: -0.16795 (-0.17097) + | > loss_dur: 0.12507 (0.17094) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.45970 (6.64791) + | > current_lr: 0.00002 + | > step_time: 5.21170 (3.85345) + | > loader_time: 0.20140 (0.05317) + + + --> STEP: 21/234 -- GLOBAL_STEP: 21315 + | > loss: 0.01567 (0.00115) + | > log_mle: -0.15187 (-0.16762) + | > loss_dur: 0.16754 (0.16877) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.82663 (6.35857) + | > current_lr: 0.00002 + | > step_time: 3.80120 (3.40270) + | > loader_time: 0.18710 (0.04983) + + + --> STEP: 26/234 -- GLOBAL_STEP: 21320 + | > loss: -0.00194 (-0.00166) + | > log_mle: -0.17029 (-0.16734) + | > loss_dur: 0.16835 (0.16569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.16003 (5.97673) + | > current_lr: 0.00002 + | > step_time: 4.00500 (3.42229) + | > loader_time: 0.00540 (0.04110) + + + --> STEP: 31/234 -- GLOBAL_STEP: 21325 + | > loss: 0.02709 (-0.00393) + | > log_mle: -0.17134 (-0.16773) + | > loss_dur: 0.19843 (0.16380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42803 (5.96813) + | > current_lr: 0.00002 + | > step_time: 3.09620 (3.33170) + | > loader_time: 0.00220 (0.04138) + + + --> STEP: 36/234 -- GLOBAL_STEP: 21330 + | > loss: 0.01198 (-0.00352) + | > log_mle: -0.17238 (-0.16807) + | > loss_dur: 0.18436 (0.16455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.72274 (6.23435) + | > current_lr: 0.00002 + | > step_time: 0.80530 (3.16869) + | > loader_time: 0.00410 (0.04443) + + + --> STEP: 41/234 -- GLOBAL_STEP: 21335 + | > loss: -0.02073 (-0.00270) + | > log_mle: -0.16588 (-0.16796) + | > loss_dur: 0.14515 (0.16526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.72910 (6.34673) + | > current_lr: 0.00002 + | > step_time: 5.88350 (3.29136) + | > loader_time: 0.00900 (0.04439) + + + --> STEP: 46/234 -- GLOBAL_STEP: 21340 + | > loss: -0.00511 (-0.00350) + | > log_mle: -0.17128 (-0.16842) + | > loss_dur: 0.16617 (0.16492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.79061 (6.39305) + | > current_lr: 0.00002 + | > step_time: 2.18390 (3.15266) + | > loader_time: 0.00890 (0.04207) + + + --> STEP: 51/234 -- GLOBAL_STEP: 21345 + | > loss: -0.00690 (-0.00446) + | > log_mle: -0.15593 (-0.16789) + | > loss_dur: 0.14903 (0.16343) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.48763 (6.24212) + | > current_lr: 0.00002 + | > step_time: 2.19480 (3.08044) + | > loader_time: 0.10490 (0.04381) + + + --> STEP: 56/234 -- GLOBAL_STEP: 21350 + | > loss: 0.01871 (-0.00482) + | > log_mle: -0.17570 (-0.16867) + | > loss_dur: 0.19441 (0.16385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.94524 (6.09540) + | > current_lr: 0.00002 + | > step_time: 1.99490 (2.93704) + | > loader_time: 0.00320 (0.04019) + + + --> STEP: 61/234 -- GLOBAL_STEP: 21355 + | > loss: -0.02400 (-0.00637) + | > log_mle: -0.17379 (-0.16944) + | > loss_dur: 0.14979 (0.16307) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.60127 (6.13279) + | > current_lr: 0.00002 + | > step_time: 1.69410 (2.87516) + | > loader_time: 0.00880 (0.03855) + + + --> STEP: 66/234 -- GLOBAL_STEP: 21360 + | > loss: 0.00212 (-0.00616) + | > log_mle: -0.16137 (-0.17002) + | > loss_dur: 0.16349 (0.16386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.97208 (6.24232) + | > current_lr: 0.00002 + | > step_time: 1.31550 (2.76300) + | > loader_time: 0.08630 (0.03817) + + + --> STEP: 71/234 -- GLOBAL_STEP: 21365 + | > loss: -0.00944 (-0.00594) + | > log_mle: -0.20522 (-0.17064) + | > loss_dur: 0.19578 (0.16470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.29216 (6.46924) + | > current_lr: 0.00002 + | > step_time: 1.20980 (2.69719) + | > loader_time: 0.00270 (0.03571) + + + --> STEP: 76/234 -- GLOBAL_STEP: 21370 + | > loss: -0.02159 (-0.00649) + | > log_mle: -0.18700 (-0.17137) + | > loss_dur: 0.16541 (0.16489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.75022 (6.52837) + | > current_lr: 0.00002 + | > step_time: 1.79790 (2.68051) + | > loader_time: 0.00300 (0.03706) + + + --> STEP: 81/234 -- GLOBAL_STEP: 21375 + | > loss: -0.03935 (-0.00748) + | > log_mle: -0.19714 (-0.17191) + | > loss_dur: 0.15779 (0.16443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06057 (6.54538) + | > current_lr: 0.00002 + | > step_time: 2.91540 (2.67071) + | > loader_time: 0.00340 (0.03494) + + + --> STEP: 86/234 -- GLOBAL_STEP: 21380 + | > loss: -0.01462 (-0.00803) + | > log_mle: -0.19579 (-0.17276) + | > loss_dur: 0.18116 (0.16474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.11327 (6.70445) + | > current_lr: 0.00002 + | > step_time: 1.70510 (2.63966) + | > loader_time: 0.00220 (0.03416) + + + --> STEP: 91/234 -- GLOBAL_STEP: 21385 + | > loss: -0.01512 (-0.00915) + | > log_mle: -0.20613 (-0.17484) + | > loss_dur: 0.19101 (0.16568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.43427 (6.99484) + | > current_lr: 0.00002 + | > step_time: 4.00320 (2.64316) + | > loader_time: 0.00360 (0.03425) + + + --> STEP: 96/234 -- GLOBAL_STEP: 21390 + | > loss: -0.01691 (-0.01178) + | > log_mle: -0.19428 (-0.17842) + | > loss_dur: 0.17736 (0.16663) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.08697 (7.48276) + | > current_lr: 0.00002 + | > step_time: 1.99540 (2.60954) + | > loader_time: 0.00300 (0.03436) + + + --> STEP: 101/234 -- GLOBAL_STEP: 21395 + | > loss: -0.05293 (-0.01348) + | > log_mle: -0.25382 (-0.18099) + | > loss_dur: 0.20089 (0.16752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.18844 (7.91172) + | > current_lr: 0.00002 + | > step_time: 2.31620 (2.58866) + | > loader_time: 0.08740 (0.03367) + + + --> STEP: 106/234 -- GLOBAL_STEP: 21400 + | > loss: -0.02979 (-0.01505) + | > log_mle: -0.25090 (-0.18394) + | > loss_dur: 0.22111 (0.16890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.39970 (8.40847) + | > current_lr: 0.00002 + | > step_time: 1.80030 (2.61744) + | > loader_time: 0.07530 (0.03555) + + + --> STEP: 111/234 -- GLOBAL_STEP: 21405 + | > loss: -0.05574 (-0.01653) + | > log_mle: -0.29460 (-0.18695) + | > loss_dur: 0.23886 (0.17042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.94369 (8.91483) + | > current_lr: 0.00002 + | > step_time: 1.99460 (2.64297) + | > loader_time: 0.00290 (0.03496) + + + --> STEP: 116/234 -- GLOBAL_STEP: 21410 + | > loss: -0.02251 (-0.01786) + | > log_mle: -0.26306 (-0.18990) + | > loss_dur: 0.24056 (0.17204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.96463 (9.35316) + | > current_lr: 0.00002 + | > step_time: 1.18680 (2.66710) + | > loader_time: 0.00650 (0.03366) + + + --> STEP: 121/234 -- GLOBAL_STEP: 21415 + | > loss: -0.00103 (-0.01911) + | > log_mle: -0.17902 (-0.19199) + | > loss_dur: 0.17799 (0.17288) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.37151 (9.57993) + | > current_lr: 0.00002 + | > step_time: 1.38720 (2.64233) + | > loader_time: 0.00320 (0.03247) + + + --> STEP: 126/234 -- GLOBAL_STEP: 21420 + | > loss: -0.09440 (-0.02103) + | > log_mle: -0.31100 (-0.19469) + | > loss_dur: 0.21659 (0.17366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.30771 (9.98735) + | > current_lr: 0.00002 + | > step_time: 4.39840 (2.67009) + | > loader_time: 0.10200 (0.03574) + + + --> STEP: 131/234 -- GLOBAL_STEP: 21425 + | > loss: -0.10992 (-0.02308) + | > log_mle: -0.34810 (-0.19821) + | > loss_dur: 0.23818 (0.17513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.54892 (10.76932) + | > current_lr: 0.00002 + | > step_time: 2.41180 (2.71445) + | > loader_time: 0.07450 (0.03721) + + + --> STEP: 136/234 -- GLOBAL_STEP: 21430 + | > loss: -0.14042 (-0.02538) + | > log_mle: -0.39345 (-0.20177) + | > loss_dur: 0.25303 (0.17639) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.07654 (11.38488) + | > current_lr: 0.00002 + | > step_time: 3.10130 (2.73500) + | > loader_time: 0.00400 (0.03744) + + + --> STEP: 141/234 -- GLOBAL_STEP: 21435 + | > loss: -0.08590 (-0.02730) + | > log_mle: -0.30438 (-0.20489) + | > loss_dur: 0.21848 (0.17758) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.61869 (11.95321) + | > current_lr: 0.00002 + | > step_time: 2.39220 (2.72352) + | > loader_time: 0.00250 (0.03701) + + + --> STEP: 146/234 -- GLOBAL_STEP: 21440 + | > loss: -0.12590 (-0.03023) + | > log_mle: -0.35337 (-0.20960) + | > loss_dur: 0.22748 (0.17937) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.80972 (12.82095) + | > current_lr: 0.00002 + | > step_time: 3.20610 (2.77292) + | > loader_time: 0.10050 (0.03813) + + + --> STEP: 151/234 -- GLOBAL_STEP: 21445 + | > loss: -0.11631 (-0.03296) + | > log_mle: -0.32290 (-0.21347) + | > loss_dur: 0.20658 (0.18051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.17199 (13.40328) + | > current_lr: 0.00002 + | > step_time: 6.65350 (2.77744) + | > loader_time: 0.10380 (0.03867) + + + --> STEP: 156/234 -- GLOBAL_STEP: 21450 + | > loss: -0.14315 (-0.03676) + | > log_mle: -0.35949 (-0.21878) + | > loss_dur: 0.21634 (0.18202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.81758 (14.32469) + | > current_lr: 0.00002 + | > step_time: 4.01780 (2.77485) + | > loader_time: 0.00310 (0.03852) + + + --> STEP: 161/234 -- GLOBAL_STEP: 21455 + | > loss: -0.15358 (-0.03986) + | > log_mle: -0.37800 (-0.22325) + | > loss_dur: 0.22442 (0.18339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.14611 (15.18840) + | > current_lr: 0.00002 + | > step_time: 3.09300 (2.77015) + | > loader_time: 0.08930 (0.03852) + + + --> STEP: 166/234 -- GLOBAL_STEP: 21460 + | > loss: -0.12493 (-0.04245) + | > log_mle: -0.32635 (-0.22706) + | > loss_dur: 0.20142 (0.18461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.61626 (16.00660) + | > current_lr: 0.00002 + | > step_time: 3.41520 (2.77309) + | > loader_time: 0.00290 (0.03954) + + + --> STEP: 171/234 -- GLOBAL_STEP: 21465 + | > loss: -0.19110 (-0.04589) + | > log_mle: -0.42589 (-0.23220) + | > loss_dur: 0.23479 (0.18631) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.16939 (16.91766) + | > current_lr: 0.00002 + | > step_time: 3.79550 (2.81068) + | > loader_time: 0.00260 (0.04134) + + + --> STEP: 176/234 -- GLOBAL_STEP: 21470 + | > loss: -0.15906 (-0.04930) + | > log_mle: -0.39889 (-0.23727) + | > loss_dur: 0.23982 (0.18797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.97504 (17.72229) + | > current_lr: 0.00002 + | > step_time: 7.79280 (2.83777) + | > loader_time: 0.00840 (0.04236) + + + --> STEP: 181/234 -- GLOBAL_STEP: 21475 + | > loss: -0.10451 (-0.05207) + | > log_mle: -0.33960 (-0.24184) + | > loss_dur: 0.23508 (0.18977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.86901 (18.44620) + | > current_lr: 0.00002 + | > step_time: 3.60590 (2.87937) + | > loader_time: 0.10860 (0.04291) + + + --> STEP: 186/234 -- GLOBAL_STEP: 21480 + | > loss: -0.12466 (-0.05492) + | > log_mle: -0.37727 (-0.24650) + | > loss_dur: 0.25261 (0.19158) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.88688 (19.46695) + | > current_lr: 0.00002 + | > step_time: 3.00460 (2.89351) + | > loader_time: 0.00640 (0.04279) + + + --> STEP: 191/234 -- GLOBAL_STEP: 21485 + | > loss: -0.16198 (-0.05781) + | > log_mle: -0.38887 (-0.25086) + | > loss_dur: 0.22690 (0.19305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.46151 (20.54579) + | > current_lr: 0.00002 + | > step_time: 7.69570 (2.97641) + | > loader_time: 0.20310 (0.04688) + + + --> STEP: 196/234 -- GLOBAL_STEP: 21490 + | > loss: -0.13048 (-0.06080) + | > log_mle: -0.39034 (-0.25533) + | > loss_dur: 0.25986 (0.19453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.36124 (21.63366) + | > current_lr: 0.00002 + | > step_time: 0.88490 (2.97585) + | > loader_time: 0.00220 (0.04673) + + + --> STEP: 201/234 -- GLOBAL_STEP: 21495 + | > loss: -0.09361 (-0.06327) + | > log_mle: -0.35629 (-0.25938) + | > loss_dur: 0.26267 (0.19612) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.03180 (22.54718) + | > current_lr: 0.00002 + | > step_time: 11.70340 (3.04361) + | > loader_time: 0.08530 (0.04608) + + + --> STEP: 206/234 -- GLOBAL_STEP: 21500 + | > loss: -0.19198 (-0.06628) + | > log_mle: -0.45357 (-0.26391) + | > loss_dur: 0.26159 (0.19763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.11796 (23.45016) + | > current_lr: 0.00002 + | > step_time: 6.20770 (3.07999) + | > loader_time: 0.29210 (0.04776) + + + --> STEP: 211/234 -- GLOBAL_STEP: 21505 + | > loss: -0.23155 (-0.06971) + | > log_mle: -0.52431 (-0.26906) + | > loss_dur: 0.29276 (0.19935) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.17761 (24.35384) + | > current_lr: 0.00002 + | > step_time: 5.19770 (3.19186) + | > loader_time: 0.00440 (0.05693) + + + --> STEP: 216/234 -- GLOBAL_STEP: 21510 + | > loss: -0.22519 (-0.07308) + | > log_mle: -0.51116 (-0.27397) + | > loss_dur: 0.28597 (0.20088) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.58549 (25.29416) + | > current_lr: 0.00002 + | > step_time: 16.91440 (3.27964) + | > loader_time: 0.18590 (0.05829) + + + --> STEP: 221/234 -- GLOBAL_STEP: 21515 + | > loss: -0.17127 (-0.07644) + | > log_mle: -0.43192 (-0.27887) + | > loss_dur: 0.26065 (0.20244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.39757 (26.17816) + | > current_lr: 0.00002 + | > step_time: 3.20010 (3.28779) + | > loader_time: 0.09010 (0.05783) + + + --> STEP: 226/234 -- GLOBAL_STEP: 21520 + | > loss: -0.25063 (-0.07995) + | > log_mle: -0.52490 (-0.28421) + | > loss_dur: 0.27427 (0.20426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.32399 (27.20537) + | > current_lr: 0.00002 + | > step_time: 0.24740 (3.22744) + | > loader_time: 0.00400 (0.05664) + + + --> STEP: 231/234 -- GLOBAL_STEP: 21525 + | > loss: -0.15710 (-0.08241) + | > log_mle: -0.57851 (-0.28973) + | > loss_dur: 0.42141 (0.20732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.27875 (28.71734) + | > current_lr: 0.00002 + | > step_time: 0.27250 (3.16318) + | > loader_time: 0.00550 (0.05551) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00198 (-0.27561) + | > avg_loss: -0.11406 (+0.01724) + | > avg_log_mle: -0.35593 (+0.01759) + | > avg_loss_dur: 0.24186 (-0.00035) + + + > EPOCH: 92/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 04:31:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 21530 + | > loss: 0.04229 (0.00374) + | > log_mle: -0.15284 (-0.16295) + | > loss_dur: 0.19514 (0.16669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.88562 (5.77683) + | > current_lr: 0.00002 + | > step_time: 1.61190 (3.00109) + | > loader_time: 0.09320 (0.04723) + + + --> STEP: 7/234 -- GLOBAL_STEP: 21535 + | > loss: -0.03293 (0.01020) + | > log_mle: -0.18452 (-0.17109) + | > loss_dur: 0.15159 (0.18129) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.74201 (6.71638) + | > current_lr: 0.00002 + | > step_time: 3.00460 (4.09812) + | > loader_time: 0.09350 (0.07018) + + + --> STEP: 12/234 -- GLOBAL_STEP: 21540 + | > loss: -0.00866 (0.00336) + | > log_mle: -0.17101 (-0.17320) + | > loss_dur: 0.16236 (0.17656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.26743 (6.99552) + | > current_lr: 0.00002 + | > step_time: 2.30940 (3.83345) + | > loader_time: 0.08810 (0.06957) + + + --> STEP: 17/234 -- GLOBAL_STEP: 21545 + | > loss: 0.00418 (-0.00121) + | > log_mle: -0.15010 (-0.17136) + | > loss_dur: 0.15428 (0.17015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.05858 (6.28892) + | > current_lr: 0.00002 + | > step_time: 3.60030 (4.02370) + | > loader_time: 0.08870 (0.06056) + + + --> STEP: 22/234 -- GLOBAL_STEP: 21550 + | > loss: -0.02822 (-0.00268) + | > log_mle: -0.17583 (-0.16948) + | > loss_dur: 0.14762 (0.16680) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.72205 (6.05419) + | > current_lr: 0.00002 + | > step_time: 5.59640 (3.97262) + | > loader_time: 0.10360 (0.39053) + + + --> STEP: 27/234 -- GLOBAL_STEP: 21555 + | > loss: -0.02433 (-0.00552) + | > log_mle: -0.17844 (-0.16906) + | > loss_dur: 0.15412 (0.16354) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.90609 (5.79417) + | > current_lr: 0.00002 + | > step_time: 2.88890 (4.09153) + | > loader_time: 0.00140 (0.44060) + + + --> STEP: 32/234 -- GLOBAL_STEP: 21560 + | > loss: -0.04586 (-0.00763) + | > log_mle: -0.18787 (-0.16962) + | > loss_dur: 0.14201 (0.16199) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.26185 (5.72213) + | > current_lr: 0.00002 + | > step_time: 4.19610 (4.05614) + | > loader_time: 0.00400 (0.37462) + + + --> STEP: 37/234 -- GLOBAL_STEP: 21565 + | > loss: -0.02019 (-0.00758) + | > log_mle: -0.16806 (-0.16966) + | > loss_dur: 0.14786 (0.16208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.47560 (5.92262) + | > current_lr: 0.00002 + | > step_time: 0.69650 (4.12129) + | > loader_time: 0.00150 (0.34795) + + + --> STEP: 42/234 -- GLOBAL_STEP: 21570 + | > loss: -0.00086 (-0.00699) + | > log_mle: -0.15807 (-0.16938) + | > loss_dur: 0.15720 (0.16238) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.39281 (6.00494) + | > current_lr: 0.00002 + | > step_time: 1.59150 (3.82307) + | > loader_time: 0.00480 (0.30727) + + + --> STEP: 47/234 -- GLOBAL_STEP: 21575 + | > loss: -0.01753 (-0.00789) + | > log_mle: -0.17055 (-0.17011) + | > loss_dur: 0.15302 (0.16222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.94142 (6.03796) + | > current_lr: 0.00002 + | > step_time: 1.39930 (3.56974) + | > loader_time: 0.00180 (0.27679) + + + --> STEP: 52/234 -- GLOBAL_STEP: 21580 + | > loss: 0.00025 (-0.00824) + | > log_mle: -0.16484 (-0.16954) + | > loss_dur: 0.16509 (0.16130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.83674 (5.89060) + | > current_lr: 0.00002 + | > step_time: 1.51860 (3.45713) + | > loader_time: 0.08210 (0.25563) + + + --> STEP: 57/234 -- GLOBAL_STEP: 21585 + | > loss: -0.00785 (-0.00854) + | > log_mle: -0.16083 (-0.17023) + | > loss_dur: 0.15298 (0.16170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.29926 (5.76743) + | > current_lr: 0.00002 + | > step_time: 0.87150 (3.25428) + | > loader_time: 0.00180 (0.23340) + + + --> STEP: 62/234 -- GLOBAL_STEP: 21590 + | > loss: 0.01316 (-0.01002) + | > log_mle: -0.21497 (-0.17186) + | > loss_dur: 0.22813 (0.16184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.52287 (6.05783) + | > current_lr: 0.00002 + | > step_time: 1.66180 (3.20698) + | > loader_time: 0.00290 (0.21643) + + + --> STEP: 67/234 -- GLOBAL_STEP: 21595 + | > loss: -0.02995 (-0.01016) + | > log_mle: -0.19565 (-0.17211) + | > loss_dur: 0.16570 (0.16194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.47193 (5.94216) + | > current_lr: 0.00002 + | > step_time: 2.81000 (3.13374) + | > loader_time: 0.07550 (0.20249) + + + --> STEP: 72/234 -- GLOBAL_STEP: 21600 + | > loss: 0.00582 (-0.00936) + | > log_mle: -0.17396 (-0.17237) + | > loss_dur: 0.17978 (0.16301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.57304 (6.25470) + | > current_lr: 0.00002 + | > step_time: 2.41120 (3.08640) + | > loader_time: 0.08550 (0.19114) + + + --> STEP: 77/234 -- GLOBAL_STEP: 21605 + | > loss: -0.03056 (-0.01021) + | > log_mle: -0.18885 (-0.17338) + | > loss_dur: 0.15829 (0.16317) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.28988 (6.35665) + | > current_lr: 0.00002 + | > step_time: 1.86640 (3.09993) + | > loader_time: 0.00260 (0.18155) + + + --> STEP: 82/234 -- GLOBAL_STEP: 21610 + | > loss: -0.02566 (-0.01083) + | > log_mle: -0.18026 (-0.17383) + | > loss_dur: 0.15460 (0.16300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.59078 (6.35708) + | > current_lr: 0.00002 + | > step_time: 2.18790 (3.05113) + | > loader_time: 0.00200 (0.17263) + + + --> STEP: 87/234 -- GLOBAL_STEP: 21615 + | > loss: -0.00553 (-0.01115) + | > log_mle: -0.18848 (-0.17481) + | > loss_dur: 0.18296 (0.16366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.68613 (6.60473) + | > current_lr: 0.00002 + | > step_time: 1.47750 (2.97536) + | > loader_time: 0.00480 (0.16405) + + + --> STEP: 92/234 -- GLOBAL_STEP: 21620 + | > loss: -0.07108 (-0.01265) + | > log_mle: -0.23271 (-0.17733) + | > loss_dur: 0.16163 (0.16467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.11620 (6.90865) + | > current_lr: 0.00002 + | > step_time: 1.89800 (2.90401) + | > loader_time: 0.00300 (0.15613) + + + --> STEP: 97/234 -- GLOBAL_STEP: 21625 + | > loss: -0.04632 (-0.01483) + | > log_mle: -0.22365 (-0.18072) + | > loss_dur: 0.17733 (0.16589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18978 (7.42379) + | > current_lr: 0.00002 + | > step_time: 2.41850 (2.87982) + | > loader_time: 0.00280 (0.15055) + + + --> STEP: 102/234 -- GLOBAL_STEP: 21630 + | > loss: -0.02715 (-0.01627) + | > log_mle: -0.20432 (-0.18305) + | > loss_dur: 0.17717 (0.16679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.52068 (7.79262) + | > current_lr: 0.00002 + | > step_time: 1.53370 (2.84828) + | > loader_time: 0.00220 (0.14402) + + + --> STEP: 107/234 -- GLOBAL_STEP: 21635 + | > loss: -0.06103 (-0.01810) + | > log_mle: -0.25092 (-0.18638) + | > loss_dur: 0.18989 (0.16829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.09058 (8.31549) + | > current_lr: 0.00002 + | > step_time: 1.71530 (2.81692) + | > loader_time: 0.08190 (0.13893) + + + --> STEP: 112/234 -- GLOBAL_STEP: 21640 + | > loss: -0.05653 (-0.01949) + | > log_mle: -0.25891 (-0.18939) + | > loss_dur: 0.20238 (0.16991) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.28861 (8.84351) + | > current_lr: 0.00002 + | > step_time: 1.40110 (2.78054) + | > loader_time: 0.00340 (0.13493) + + + --> STEP: 117/234 -- GLOBAL_STEP: 21645 + | > loss: -0.06369 (-0.02094) + | > log_mle: -0.25550 (-0.19227) + | > loss_dur: 0.19181 (0.17133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.57887 (9.25347) + | > current_lr: 0.00002 + | > step_time: 1.61100 (2.76166) + | > loader_time: 0.00250 (0.12933) + + + --> STEP: 122/234 -- GLOBAL_STEP: 21650 + | > loss: -0.04997 (-0.02199) + | > log_mle: -0.23211 (-0.19411) + | > loss_dur: 0.18214 (0.17212) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.48225 (9.53551) + | > current_lr: 0.00002 + | > step_time: 2.60630 (2.73447) + | > loader_time: 0.08000 (0.12554) + + + --> STEP: 127/234 -- GLOBAL_STEP: 21655 + | > loss: -0.06321 (-0.02382) + | > log_mle: -0.28699 (-0.19713) + | > loss_dur: 0.22378 (0.17330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.77957 (10.16643) + | > current_lr: 0.00002 + | > step_time: 1.62010 (2.69313) + | > loader_time: 0.08270 (0.12329) + + + --> STEP: 132/234 -- GLOBAL_STEP: 21660 + | > loss: -0.08944 (-0.02606) + | > log_mle: -0.26814 (-0.20050) + | > loss_dur: 0.17870 (0.17444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.81107 (10.88273) + | > current_lr: 0.00002 + | > step_time: 2.29910 (2.68963) + | > loader_time: 0.00360 (0.11941) + + + --> STEP: 137/234 -- GLOBAL_STEP: 21665 + | > loss: -0.05651 (-0.02808) + | > log_mle: -0.28091 (-0.20415) + | > loss_dur: 0.22439 (0.17607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.98687 (11.44503) + | > current_lr: 0.00002 + | > step_time: 2.79910 (2.66250) + | > loader_time: 0.00550 (0.11703) + + + --> STEP: 142/234 -- GLOBAL_STEP: 21670 + | > loss: -0.07524 (-0.02998) + | > log_mle: -0.29405 (-0.20735) + | > loss_dur: 0.21881 (0.17737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.97421 (12.01404) + | > current_lr: 0.00002 + | > step_time: 2.00730 (2.67497) + | > loader_time: 0.00250 (0.11431) + + + --> STEP: 147/234 -- GLOBAL_STEP: 21675 + | > loss: -0.07570 (-0.03279) + | > log_mle: -0.29710 (-0.21206) + | > loss_dur: 0.22140 (0.17927) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.23490 (12.78584) + | > current_lr: 0.00002 + | > step_time: 2.39910 (2.67251) + | > loader_time: 0.08790 (0.11164) + + + --> STEP: 152/234 -- GLOBAL_STEP: 21680 + | > loss: -0.12899 (-0.03582) + | > log_mle: -0.37350 (-0.21640) + | > loss_dur: 0.24452 (0.18058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.36509 (13.53771) + | > current_lr: 0.00002 + | > step_time: 2.51580 (2.69454) + | > loader_time: 0.08340 (0.10926) + + + --> STEP: 157/234 -- GLOBAL_STEP: 21685 + | > loss: -0.09778 (-0.03938) + | > log_mle: -0.32537 (-0.22138) + | > loss_dur: 0.22759 (0.18200) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.42527 (14.44893) + | > current_lr: 0.00002 + | > step_time: 1.58750 (2.71449) + | > loader_time: 0.00380 (0.10653) + + + --> STEP: 162/234 -- GLOBAL_STEP: 21690 + | > loss: -0.15271 (-0.04270) + | > log_mle: -0.35857 (-0.22609) + | > loss_dur: 0.20586 (0.18339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.32019 (15.29475) + | > current_lr: 0.00002 + | > step_time: 3.80570 (2.73035) + | > loader_time: 0.00390 (0.10481) + + + --> STEP: 167/234 -- GLOBAL_STEP: 21695 + | > loss: -0.20382 (-0.04574) + | > log_mle: -0.43589 (-0.23041) + | > loss_dur: 0.23207 (0.18467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.63797 (16.06523) + | > current_lr: 0.00002 + | > step_time: 3.41460 (2.75837) + | > loader_time: 0.00370 (0.10571) + + + --> STEP: 172/234 -- GLOBAL_STEP: 21700 + | > loss: -0.16749 (-0.04887) + | > log_mle: -0.42872 (-0.23553) + | > loss_dur: 0.26123 (0.18666) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.86351 (16.87932) + | > current_lr: 0.00002 + | > step_time: 4.92130 (2.80498) + | > loader_time: 0.09450 (0.10555) + + + --> STEP: 177/234 -- GLOBAL_STEP: 21705 + | > loss: -0.13735 (-0.05207) + | > log_mle: -0.38537 (-0.24032) + | > loss_dur: 0.24802 (0.18825) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.97163 (17.81370) + | > current_lr: 0.00002 + | > step_time: 3.59460 (2.79765) + | > loader_time: 0.10300 (0.10375) + + + --> STEP: 182/234 -- GLOBAL_STEP: 21710 + | > loss: -0.16087 (-0.05484) + | > log_mle: -0.42672 (-0.24501) + | > loss_dur: 0.26585 (0.19017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.01719 (18.83127) + | > current_lr: 0.00002 + | > step_time: 5.60830 (2.87514) + | > loader_time: 0.10080 (0.10314) + + + --> STEP: 187/234 -- GLOBAL_STEP: 21715 + | > loss: -0.18000 (-0.05771) + | > log_mle: -0.42466 (-0.24957) + | > loss_dur: 0.24466 (0.19186) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.14119 (19.77239) + | > current_lr: 0.00002 + | > step_time: 5.70280 (2.94010) + | > loader_time: 0.10350 (0.10197) + + + --> STEP: 192/234 -- GLOBAL_STEP: 21720 + | > loss: -0.20248 (-0.06077) + | > log_mle: -0.44762 (-0.25406) + | > loss_dur: 0.24514 (0.19329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.81966 (20.84806) + | > current_lr: 0.00002 + | > step_time: 1.68170 (3.01623) + | > loader_time: 0.00620 (0.10285) + + + --> STEP: 197/234 -- GLOBAL_STEP: 21725 + | > loss: -0.19064 (-0.06372) + | > log_mle: -0.42810 (-0.25844) + | > loss_dur: 0.23746 (0.19472) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.69637 (21.79351) + | > current_lr: 0.00002 + | > step_time: 7.19140 (3.03759) + | > loader_time: 0.00490 (0.10076) + + + --> STEP: 202/234 -- GLOBAL_STEP: 21730 + | > loss: -0.24279 (-0.06651) + | > log_mle: -0.50496 (-0.26288) + | > loss_dur: 0.26217 (0.19637) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.71411 (22.71305) + | > current_lr: 0.00002 + | > step_time: 5.50050 (3.11742) + | > loader_time: 0.10120 (0.10262) + + + --> STEP: 207/234 -- GLOBAL_STEP: 21735 + | > loss: -0.23311 (-0.06944) + | > log_mle: -0.49981 (-0.26736) + | > loss_dur: 0.26670 (0.19792) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.95977 (23.46242) + | > current_lr: 0.00002 + | > step_time: 13.08480 (3.22459) + | > loader_time: 0.31450 (0.10369) + + + --> STEP: 212/234 -- GLOBAL_STEP: 21740 + | > loss: -0.21004 (-0.07267) + | > log_mle: -0.47784 (-0.27239) + | > loss_dur: 0.26780 (0.19972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.78783 (24.43473) + | > current_lr: 0.00002 + | > step_time: 9.20560 (3.30451) + | > loader_time: 0.09780 (0.10308) + + + --> STEP: 217/234 -- GLOBAL_STEP: 21745 + | > loss: -0.22779 (-0.07607) + | > log_mle: -0.49711 (-0.27731) + | > loss_dur: 0.26933 (0.20124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.80832 (25.46092) + | > current_lr: 0.00002 + | > step_time: 5.60940 (3.32621) + | > loader_time: 0.19570 (0.10286) + + + --> STEP: 222/234 -- GLOBAL_STEP: 21750 + | > loss: -0.21582 (-0.07925) + | > log_mle: -0.51637 (-0.28221) + | > loss_dur: 0.30056 (0.20297) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.08422 (26.29062) + | > current_lr: 0.00002 + | > step_time: 1.70060 (3.33094) + | > loader_time: 0.00340 (0.10138) + + + --> STEP: 227/234 -- GLOBAL_STEP: 21755 + | > loss: -0.19089 (-0.08260) + | > log_mle: -0.48070 (-0.28730) + | > loss_dur: 0.28982 (0.20470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.26937 (27.67822) + | > current_lr: 0.00002 + | > step_time: 0.23840 (3.27501) + | > loader_time: 0.00300 (0.09925) + + + --> STEP: 232/234 -- GLOBAL_STEP: 21760 + | > loss: -0.13473 (-0.08487) + | > log_mle: -0.68068 (-0.29368) + | > loss_dur: 0.54594 (0.20881) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.64716 (29.15401) + | > current_lr: 0.00002 + | > step_time: 0.35630 (3.21058) + | > loader_time: 0.01010 (0.09723) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.14985 (+0.14787) + | > avg_loss: -0.10484 (+0.00923) + | > avg_log_mle: -0.34844 (+0.00749) + | > avg_loss_dur: 0.24360 (+0.00174) + + + > EPOCH: 93/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 04:45:37)  + + --> STEP: 3/234 -- GLOBAL_STEP: 21765 + | > loss: 0.01064 (0.01511) + | > log_mle: -0.17815 (-0.17075) + | > loss_dur: 0.18879 (0.18586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.33348 (6.79158) + | > current_lr: 0.00002 + | > step_time: 8.11850 (6.10905) + | > loader_time: 0.00230 (0.23318) + + + --> STEP: 8/234 -- GLOBAL_STEP: 21770 + | > loss: 0.00273 (0.00820) + | > log_mle: -0.18399 (-0.17500) + | > loss_dur: 0.18672 (0.18320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.57220 (8.02153) + | > current_lr: 0.00002 + | > step_time: 5.99220 (5.27806) + | > loader_time: 0.09550 (0.13525) + + + --> STEP: 13/234 -- GLOBAL_STEP: 21775 + | > loss: 0.01701 (0.00495) + | > log_mle: -0.16545 (-0.17464) + | > loss_dur: 0.18246 (0.17959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.38269 (7.76074) + | > current_lr: 0.00002 + | > step_time: 3.60210 (4.51708) + | > loader_time: 0.00230 (0.12791) + + + --> STEP: 18/234 -- GLOBAL_STEP: 21780 + | > loss: 0.00812 (-0.00198) + | > log_mle: -0.16897 (-0.17318) + | > loss_dur: 0.17709 (0.17121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.97569 (7.20495) + | > current_lr: 0.00002 + | > step_time: 2.40290 (4.09900) + | > loader_time: 0.08180 (0.09795) + + + --> STEP: 23/234 -- GLOBAL_STEP: 21785 + | > loss: -0.03021 (-0.00532) + | > log_mle: -0.17370 (-0.17166) + | > loss_dur: 0.14349 (0.16633) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.05683 (6.71174) + | > current_lr: 0.00002 + | > step_time: 1.10410 (4.29019) + | > loader_time: 0.00260 (0.08635) + + + --> STEP: 28/234 -- GLOBAL_STEP: 21790 + | > loss: -0.03400 (-0.00827) + | > log_mle: -0.16289 (-0.17099) + | > loss_dur: 0.12888 (0.16272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.59749 (6.34229) + | > current_lr: 0.00002 + | > step_time: 1.89820 (4.09515) + | > loader_time: 0.09780 (0.08142) + + + --> STEP: 33/234 -- GLOBAL_STEP: 21795 + | > loss: 0.00085 (-0.00965) + | > log_mle: -0.16139 (-0.17146) + | > loss_dur: 0.16224 (0.16181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.04057 (6.21473) + | > current_lr: 0.00002 + | > step_time: 2.80000 (4.09863) + | > loader_time: 0.29490 (0.08429) + + + --> STEP: 38/234 -- GLOBAL_STEP: 21800 + | > loss: -0.00364 (-0.00898) + | > log_mle: -0.18055 (-0.17199) + | > loss_dur: 0.17691 (0.16301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.48955 (6.43352) + | > current_lr: 0.00002 + | > step_time: 1.80110 (3.97331) + | > loader_time: 0.00270 (0.08091) + + + --> STEP: 43/234 -- GLOBAL_STEP: 21805 + | > loss: -0.00858 (-0.00856) + | > log_mle: -0.18167 (-0.17169) + | > loss_dur: 0.17309 (0.16312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.93073 (6.38003) + | > current_lr: 0.00002 + | > step_time: 1.10260 (3.71535) + | > loader_time: 0.00230 (0.07374) + + + --> STEP: 48/234 -- GLOBAL_STEP: 21810 + | > loss: -0.02438 (-0.00992) + | > log_mle: -0.16427 (-0.17202) + | > loss_dur: 0.13988 (0.16210) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.46946 (6.40792) + | > current_lr: 0.00002 + | > step_time: 1.38790 (3.58901) + | > loader_time: 0.02070 (0.07389) + + + --> STEP: 53/234 -- GLOBAL_STEP: 21815 + | > loss: -0.01169 (-0.00994) + | > log_mle: -0.18640 (-0.17192) + | > loss_dur: 0.17470 (0.16197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.69365 (6.18495) + | > current_lr: 0.00002 + | > step_time: 2.60730 (3.56192) + | > loader_time: 0.00280 (0.06717) + + + --> STEP: 58/234 -- GLOBAL_STEP: 21820 + | > loss: -0.02469 (-0.01025) + | > log_mle: -0.17015 (-0.17228) + | > loss_dur: 0.14547 (0.16203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.57678 (6.00315) + | > current_lr: 0.00002 + | > step_time: 1.21810 (3.43985) + | > loader_time: 0.07960 (0.06478) + + + --> STEP: 63/234 -- GLOBAL_STEP: 21825 + | > loss: 0.00656 (-0.01117) + | > log_mle: -0.18190 (-0.17409) + | > loss_dur: 0.18846 (0.16292) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.79596 (6.19414) + | > current_lr: 0.00002 + | > step_time: 1.98510 (3.36497) + | > loader_time: 0.00290 (0.06252) + + + --> STEP: 68/234 -- GLOBAL_STEP: 21830 + | > loss: 0.00412 (-0.01113) + | > log_mle: -0.17377 (-0.17419) + | > loss_dur: 0.17789 (0.16306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.85056 (6.11479) + | > current_lr: 0.00002 + | > step_time: 3.50730 (3.29084) + | > loader_time: 0.09890 (0.06082) + + + --> STEP: 73/234 -- GLOBAL_STEP: 21835 + | > loss: -0.01442 (-0.01066) + | > log_mle: -0.19684 (-0.17473) + | > loss_dur: 0.18242 (0.16407) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.76011 (6.40968) + | > current_lr: 0.00002 + | > step_time: 1.42500 (3.20565) + | > loader_time: 0.00360 (0.05824) + + + --> STEP: 78/234 -- GLOBAL_STEP: 21840 + | > loss: 0.00444 (-0.01140) + | > log_mle: -0.17116 (-0.17524) + | > loss_dur: 0.17560 (0.16384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.37188 (6.53869) + | > current_lr: 0.00002 + | > step_time: 1.89620 (3.11254) + | > loader_time: 0.00290 (0.05581) + + + --> STEP: 83/234 -- GLOBAL_STEP: 21845 + | > loss: -0.02500 (-0.01247) + | > log_mle: -0.20224 (-0.17599) + | > loss_dur: 0.17724 (0.16352) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.64101 (6.57742) + | > current_lr: 0.00002 + | > step_time: 1.39850 (3.03627) + | > loader_time: 0.00260 (0.05451) + + + --> STEP: 88/234 -- GLOBAL_STEP: 21850 + | > loss: -0.04819 (-0.01303) + | > log_mle: -0.23068 (-0.17717) + | > loss_dur: 0.18249 (0.16414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.14529 (6.91669) + | > current_lr: 0.00002 + | > step_time: 0.91340 (3.00177) + | > loader_time: 0.09660 (0.05473) + + + --> STEP: 93/234 -- GLOBAL_STEP: 21855 + | > loss: -0.03926 (-0.01448) + | > log_mle: -0.24661 (-0.17963) + | > loss_dur: 0.20735 (0.16515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.26081 (7.35018) + | > current_lr: 0.00002 + | > step_time: 2.70200 (2.99820) + | > loader_time: 0.10690 (0.05416) + + + --> STEP: 98/234 -- GLOBAL_STEP: 21860 + | > loss: -0.00536 (-0.01632) + | > log_mle: -0.17548 (-0.18214) + | > loss_dur: 0.17012 (0.16582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.16698 (7.75283) + | > current_lr: 0.00002 + | > step_time: 1.11250 (2.99763) + | > loader_time: 0.00520 (0.05512) + + + --> STEP: 103/234 -- GLOBAL_STEP: 21865 + | > loss: -0.06323 (-0.01812) + | > log_mle: -0.27449 (-0.18534) + | > loss_dur: 0.21126 (0.16722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.06342 (8.32474) + | > current_lr: 0.00002 + | > step_time: 1.70950 (2.96575) + | > loader_time: 0.00650 (0.05524) + + + --> STEP: 108/234 -- GLOBAL_STEP: 21870 + | > loss: -0.03710 (-0.01974) + | > log_mle: -0.21813 (-0.18800) + | > loss_dur: 0.18104 (0.16826) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.81678 (8.76465) + | > current_lr: 0.00002 + | > step_time: 0.88870 (2.94805) + | > loader_time: 0.00380 (0.05463) + + + --> STEP: 113/234 -- GLOBAL_STEP: 21875 + | > loss: -0.07084 (-0.02142) + | > log_mle: -0.26922 (-0.19132) + | > loss_dur: 0.19838 (0.16990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.04141 (9.41521) + | > current_lr: 0.00002 + | > step_time: 3.40790 (2.92167) + | > loader_time: 0.08010 (0.05323) + + + --> STEP: 118/234 -- GLOBAL_STEP: 21880 + | > loss: -0.03344 (-0.02255) + | > log_mle: -0.23908 (-0.19385) + | > loss_dur: 0.20564 (0.17130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.53971 (9.83275) + | > current_lr: 0.00002 + | > step_time: 3.39490 (2.92225) + | > loader_time: 0.00720 (0.05344) + + + --> STEP: 123/234 -- GLOBAL_STEP: 21885 + | > loss: -0.03247 (-0.02352) + | > log_mle: -0.20682 (-0.19544) + | > loss_dur: 0.17435 (0.17192) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.25613 (9.93548) + | > current_lr: 0.00002 + | > step_time: 2.50900 (2.92062) + | > loader_time: 0.09910 (0.05222) + + + --> STEP: 128/234 -- GLOBAL_STEP: 21890 + | > loss: -0.08392 (-0.02591) + | > log_mle: -0.26992 (-0.19898) + | > loss_dur: 0.18600 (0.17306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.27344 (10.48215) + | > current_lr: 0.00002 + | > step_time: 2.70010 (2.91159) + | > loader_time: 0.00420 (0.05090) + + + --> STEP: 133/234 -- GLOBAL_STEP: 21895 + | > loss: -0.07146 (-0.02804) + | > log_mle: -0.28909 (-0.20245) + | > loss_dur: 0.21763 (0.17441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.29445 (10.96327) + | > current_lr: 0.00002 + | > step_time: 1.65040 (2.90532) + | > loader_time: 0.00360 (0.05111) + + + --> STEP: 138/234 -- GLOBAL_STEP: 21900 + | > loss: -0.05325 (-0.02970) + | > log_mle: -0.24561 (-0.20562) + | > loss_dur: 0.19235 (0.17591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.64444 (11.60884) + | > current_lr: 0.00002 + | > step_time: 2.70880 (2.89438) + | > loader_time: 0.00600 (0.05136) + + + --> STEP: 143/234 -- GLOBAL_STEP: 21905 + | > loss: -0.11075 (-0.03201) + | > log_mle: -0.37762 (-0.20961) + | > loss_dur: 0.26687 (0.17760) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.83636 (12.62183) + | > current_lr: 0.00002 + | > step_time: 1.38290 (2.91203) + | > loader_time: 0.09340 (0.05368) + + + --> STEP: 148/234 -- GLOBAL_STEP: 21910 + | > loss: -0.10881 (-0.03474) + | > log_mle: -0.30005 (-0.21371) + | > loss_dur: 0.19124 (0.17898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.86739 (13.31636) + | > current_lr: 0.00002 + | > step_time: 2.19630 (2.89237) + | > loader_time: 0.07690 (0.05410) + + + --> STEP: 153/234 -- GLOBAL_STEP: 21915 + | > loss: -0.17982 (-0.03818) + | > log_mle: -0.41725 (-0.21879) + | > loss_dur: 0.23743 (0.18060) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.43304 (14.15256) + | > current_lr: 0.00002 + | > step_time: 4.30280 (2.91426) + | > loader_time: 0.09200 (0.05365) + + + --> STEP: 158/234 -- GLOBAL_STEP: 21920 + | > loss: -0.11936 (-0.04116) + | > log_mle: -0.35845 (-0.22323) + | > loss_dur: 0.23909 (0.18208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.17419 (15.08436) + | > current_lr: 0.00002 + | > step_time: 1.78920 (2.96666) + | > loader_time: 0.00540 (0.05391) + + + --> STEP: 163/234 -- GLOBAL_STEP: 21925 + | > loss: -0.11327 (-0.04428) + | > log_mle: -0.33082 (-0.22764) + | > loss_dur: 0.21754 (0.18336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.00609 (16.02208) + | > current_lr: 0.00002 + | > step_time: 1.81970 (2.97686) + | > loader_time: 0.08650 (0.05579) + + + --> STEP: 168/234 -- GLOBAL_STEP: 21930 + | > loss: -0.13292 (-0.04750) + | > log_mle: -0.38355 (-0.23223) + | > loss_dur: 0.25063 (0.18474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.98249 (16.79003) + | > current_lr: 0.00002 + | > step_time: 5.91170 (3.01578) + | > loader_time: 0.09130 (0.05582) + + + --> STEP: 173/234 -- GLOBAL_STEP: 21935 + | > loss: -0.15888 (-0.05083) + | > log_mle: -0.39085 (-0.23733) + | > loss_dur: 0.23197 (0.18650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.57771 (17.73867) + | > current_lr: 0.00002 + | > step_time: 4.30240 (3.04534) + | > loader_time: 0.08580 (0.05592) + + + --> STEP: 178/234 -- GLOBAL_STEP: 21940 + | > loss: -0.18425 (-0.05421) + | > log_mle: -0.44578 (-0.24247) + | > loss_dur: 0.26152 (0.18826) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.02795 (18.63454) + | > current_lr: 0.00002 + | > step_time: 4.20050 (3.06887) + | > loader_time: 0.08720 (0.05649) + + + --> STEP: 183/234 -- GLOBAL_STEP: 21945 + | > loss: -0.18770 (-0.05707) + | > log_mle: -0.43592 (-0.24714) + | > loss_dur: 0.24822 (0.19007) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.98104 (19.53231) + | > current_lr: 0.00002 + | > step_time: 3.22120 (3.06932) + | > loader_time: 0.00310 (0.05654) + + + --> STEP: 188/234 -- GLOBAL_STEP: 21950 + | > loss: -0.20310 (-0.06007) + | > log_mle: -0.45521 (-0.25179) + | > loss_dur: 0.25212 (0.19172) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.44937 (20.44049) + | > current_lr: 0.00002 + | > step_time: 1.99760 (3.05428) + | > loader_time: 0.09600 (0.05667) + + + --> STEP: 193/234 -- GLOBAL_STEP: 21955 + | > loss: -0.20517 (-0.06320) + | > log_mle: -0.45344 (-0.25625) + | > loss_dur: 0.24827 (0.19305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.73352 (21.36334) + | > current_lr: 0.00002 + | > step_time: 2.19490 (3.09131) + | > loader_time: 0.00770 (0.05626) + + + --> STEP: 198/234 -- GLOBAL_STEP: 21960 + | > loss: -0.18086 (-0.06586) + | > log_mle: -0.44021 (-0.26043) + | > loss_dur: 0.25935 (0.19457) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.07652 (22.63662) + | > current_lr: 0.00002 + | > step_time: 7.90510 (3.15455) + | > loader_time: 0.09060 (0.05690) + + + --> STEP: 203/234 -- GLOBAL_STEP: 21965 + | > loss: -0.13334 (-0.06848) + | > log_mle: -0.38096 (-0.26454) + | > loss_dur: 0.24762 (0.19606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.23980 (23.44086) + | > current_lr: 0.00002 + | > step_time: 9.40000 (3.20294) + | > loader_time: 0.49270 (0.05847) + + + --> STEP: 208/234 -- GLOBAL_STEP: 21970 + | > loss: -0.18175 (-0.07160) + | > log_mle: -0.45298 (-0.26928) + | > loss_dur: 0.27123 (0.19768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.37963 (24.44336) + | > current_lr: 0.00002 + | > step_time: 8.20940 (3.24096) + | > loader_time: 0.09410 (0.05906) + + + --> STEP: 213/234 -- GLOBAL_STEP: 21975 + | > loss: -0.22493 (-0.07510) + | > log_mle: -0.50351 (-0.27447) + | > loss_dur: 0.27858 (0.19937) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.25501 (25.42345) + | > current_lr: 0.00002 + | > step_time: 5.30140 (3.31463) + | > loader_time: 0.00820 (0.06108) + + + --> STEP: 218/234 -- GLOBAL_STEP: 21980 + | > loss: -0.19554 (-0.07839) + | > log_mle: -0.46916 (-0.27922) + | > loss_dur: 0.27361 (0.20083) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.50570 (26.35775) + | > current_lr: 0.00002 + | > step_time: 8.21550 (3.40209) + | > loader_time: 0.19540 (0.06424) + + + --> STEP: 223/234 -- GLOBAL_STEP: 21985 + | > loss: -0.22868 (-0.08161) + | > log_mle: -0.49901 (-0.28413) + | > loss_dur: 0.27033 (0.20252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.00617 (27.62778) + | > current_lr: 0.00002 + | > step_time: 0.23330 (3.35459) + | > loader_time: 0.00310 (0.06327) + + + --> STEP: 228/234 -- GLOBAL_STEP: 21990 + | > loss: -0.20441 (-0.08482) + | > log_mle: -0.50085 (-0.28917) + | > loss_dur: 0.29644 (0.20434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.34962 (29.00325) + | > current_lr: 0.00002 + | > step_time: 0.24840 (3.28624) + | > loader_time: 0.00660 (0.06198) + + + --> STEP: 233/234 -- GLOBAL_STEP: 21995 + | > loss: 0.41441 (-0.08434) + | > log_mle: -0.46549 (-0.29530) + | > loss_dur: 0.87990 (0.21096) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.78020 (30.31181) + | > current_lr: 0.00002 + | > step_time: 0.21320 (3.22161) + | > loader_time: 0.00300 (0.06077) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32671 (+0.17686) + | > avg_loss: -0.12720 (-0.02236) + | > avg_log_mle: -0.36822 (-0.01978) + | > avg_loss_dur: 0.24102 (-0.00258) + + + > EPOCH: 94/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 04:59:24)  + + --> STEP: 4/234 -- GLOBAL_STEP: 22000 + | > loss: 0.05149 (0.02281) + | > log_mle: -0.17865 (-0.17358) + | > loss_dur: 0.23014 (0.19639) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.29072 (7.61995) + | > current_lr: 0.00002 + | > step_time: 1.18580 (9.09581) + | > loader_time: 0.00230 (0.00158) + + + --> STEP: 9/234 -- GLOBAL_STEP: 22005 + | > loss: -0.00007 (0.00400) + | > log_mle: -0.18646 (-0.17757) + | > loss_dur: 0.18638 (0.18157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.67351 (7.77421) + | > current_lr: 0.00002 + | > step_time: 2.98570 (4.92833) + | > loader_time: 0.10680 (0.01345) + + + --> STEP: 14/234 -- GLOBAL_STEP: 22010 + | > loss: -0.01170 (-0.00040) + | > log_mle: -0.17997 (-0.17644) + | > loss_dur: 0.16827 (0.17604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.66164 (7.68463) + | > current_lr: 0.00002 + | > step_time: 2.79380 (3.91582) + | > loader_time: 0.00340 (0.03641) + + + --> STEP: 19/234 -- GLOBAL_STEP: 22015 + | > loss: -0.01023 (-0.00589) + | > log_mle: -0.16452 (-0.17429) + | > loss_dur: 0.15430 (0.16840) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.30023 (7.10404) + | > current_lr: 0.00002 + | > step_time: 7.80660 (4.46977) + | > loader_time: 0.20250 (0.04331) + + + --> STEP: 24/234 -- GLOBAL_STEP: 22020 + | > loss: -0.03058 (-0.00845) + | > log_mle: -0.16660 (-0.17291) + | > loss_dur: 0.13602 (0.16446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.19017 (6.74549) + | > current_lr: 0.00002 + | > step_time: 3.00800 (4.14062) + | > loader_time: 0.08990 (0.04597) + + + --> STEP: 29/234 -- GLOBAL_STEP: 22025 + | > loss: -0.01562 (-0.01022) + | > log_mle: -0.16239 (-0.17210) + | > loss_dur: 0.14677 (0.16188) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.18877 (6.46435) + | > current_lr: 0.00002 + | > step_time: 8.61650 (4.43035) + | > loader_time: 0.00290 (0.05187) + + + --> STEP: 34/234 -- GLOBAL_STEP: 22030 + | > loss: 0.00469 (-0.01113) + | > log_mle: -0.16994 (-0.17283) + | > loss_dur: 0.17463 (0.16171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.76288 (6.29186) + | > current_lr: 0.00002 + | > step_time: 0.81510 (4.17035) + | > loader_time: 0.00300 (0.05073) + + + --> STEP: 39/234 -- GLOBAL_STEP: 22035 + | > loss: -0.01075 (-0.01120) + | > log_mle: -0.18033 (-0.17366) + | > loss_dur: 0.16957 (0.16245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.98844 (6.62674) + | > current_lr: 0.00002 + | > step_time: 2.61100 (4.04333) + | > loader_time: 0.08260 (0.04882) + + + --> STEP: 44/234 -- GLOBAL_STEP: 22040 + | > loss: -0.02780 (-0.01086) + | > log_mle: -0.16993 (-0.17317) + | > loss_dur: 0.14214 (0.16231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.12088 (6.47895) + | > current_lr: 0.00002 + | > step_time: 1.90880 (3.83346) + | > loader_time: 0.00400 (0.04601) + + + --> STEP: 49/234 -- GLOBAL_STEP: 22045 + | > loss: -0.03376 (-0.01192) + | > log_mle: -0.17889 (-0.17373) + | > loss_dur: 0.14513 (0.16182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.60585 (6.42983) + | > current_lr: 0.00002 + | > step_time: 0.98150 (3.60666) + | > loader_time: 0.00150 (0.04151) + + + --> STEP: 54/234 -- GLOBAL_STEP: 22050 + | > loss: -0.02823 (-0.01163) + | > log_mle: -0.18373 (-0.17367) + | > loss_dur: 0.15550 (0.16204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.55324 (6.23633) + | > current_lr: 0.00002 + | > step_time: 1.27460 (3.39297) + | > loader_time: 0.00150 (0.04069) + + + --> STEP: 59/234 -- GLOBAL_STEP: 22055 + | > loss: -0.05765 (-0.01217) + | > log_mle: -0.19871 (-0.17418) + | > loss_dur: 0.14105 (0.16201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.20588 (6.15731) + | > current_lr: 0.00002 + | > step_time: 1.02100 (3.28598) + | > loader_time: 0.00190 (0.04198) + + + --> STEP: 64/234 -- GLOBAL_STEP: 22060 + | > loss: -0.02295 (-0.01279) + | > log_mle: -0.16655 (-0.17537) + | > loss_dur: 0.14360 (0.16258) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 2.99712 (6.30352) + | > current_lr: 0.00002 + | > step_time: 3.20930 (3.21879) + | > loader_time: 0.09250 (0.04056) + + + --> STEP: 69/234 -- GLOBAL_STEP: 22065 + | > loss: 0.00371 (-0.01281) + | > log_mle: -0.15727 (-0.17531) + | > loss_dur: 0.16098 (0.16250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.60479 (6.21561) + | > current_lr: 0.00002 + | > step_time: 2.28580 (3.16190) + | > loader_time: 0.00230 (0.04178) + + + --> STEP: 74/234 -- GLOBAL_STEP: 22070 + | > loss: -0.03264 (-0.01261) + | > log_mle: -0.17475 (-0.17603) + | > loss_dur: 0.14211 (0.16343) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.06781 (6.62264) + | > current_lr: 0.00002 + | > step_time: 1.99600 (3.06134) + | > loader_time: 0.00270 (0.04014) + + + --> STEP: 79/234 -- GLOBAL_STEP: 22075 + | > loss: -0.03651 (-0.01333) + | > log_mle: -0.18903 (-0.17672) + | > loss_dur: 0.15253 (0.16339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.46338 (6.63647) + | > current_lr: 0.00002 + | > step_time: 2.60410 (3.01421) + | > loader_time: 0.00230 (0.03992) + + + --> STEP: 84/234 -- GLOBAL_STEP: 22080 + | > loss: -0.01381 (-0.01411) + | > log_mle: -0.18420 (-0.17732) + | > loss_dur: 0.17039 (0.16321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.23482 (6.75077) + | > current_lr: 0.00002 + | > step_time: 2.37810 (2.95660) + | > loader_time: 0.01200 (0.03785) + + + --> STEP: 89/234 -- GLOBAL_STEP: 22085 + | > loss: -0.05400 (-0.01518) + | > log_mle: -0.21709 (-0.17888) + | > loss_dur: 0.16308 (0.16370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.20237 (7.12063) + | > current_lr: 0.00002 + | > step_time: 1.81450 (2.95039) + | > loader_time: 0.08000 (0.03792) + + + --> STEP: 94/234 -- GLOBAL_STEP: 22090 + | > loss: -0.06954 (-0.01691) + | > log_mle: -0.24962 (-0.18167) + | > loss_dur: 0.18008 (0.16477) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.70230 (7.58888) + | > current_lr: 0.00002 + | > step_time: 1.50150 (2.98043) + | > loader_time: 0.00330 (0.03713) + + + --> STEP: 99/234 -- GLOBAL_STEP: 22095 + | > loss: -0.08325 (-0.01871) + | > log_mle: -0.28393 (-0.18449) + | > loss_dur: 0.20067 (0.16578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.58488 (8.03093) + | > current_lr: 0.00002 + | > step_time: 1.51340 (2.94542) + | > loader_time: 0.00420 (0.04025) + + + --> STEP: 104/234 -- GLOBAL_STEP: 22100 + | > loss: -0.10513 (-0.02063) + | > log_mle: -0.29456 (-0.18777) + | > loss_dur: 0.18943 (0.16714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.78873 (8.54997) + | > current_lr: 0.00002 + | > step_time: 2.51000 (3.02209) + | > loader_time: 0.00510 (0.04033) + + + --> STEP: 109/234 -- GLOBAL_STEP: 22105 + | > loss: -0.02268 (-0.02155) + | > log_mle: -0.26623 (-0.19013) + | > loss_dur: 0.24355 (0.16858) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.63894 (8.96767) + | > current_lr: 0.00002 + | > step_time: 3.49200 (3.00986) + | > loader_time: 0.00290 (0.03867) + + + --> STEP: 114/234 -- GLOBAL_STEP: 22110 + | > loss: -0.07007 (-0.02357) + | > log_mle: -0.25074 (-0.19327) + | > loss_dur: 0.18067 (0.16970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.55215 (9.46539) + | > current_lr: 0.00002 + | > step_time: 2.79980 (2.97980) + | > loader_time: 0.00220 (0.03881) + + + --> STEP: 119/234 -- GLOBAL_STEP: 22115 + | > loss: -0.06122 (-0.02469) + | > log_mle: -0.24654 (-0.19573) + | > loss_dur: 0.18532 (0.17105) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.35029 (9.73763) + | > current_lr: 0.00002 + | > step_time: 1.11360 (2.94557) + | > loader_time: 0.07900 (0.04010) + + + --> STEP: 124/234 -- GLOBAL_STEP: 22120 + | > loss: -0.09315 (-0.02578) + | > log_mle: -0.27634 (-0.19755) + | > loss_dur: 0.18319 (0.17176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.67248 (9.94553) + | > current_lr: 0.00002 + | > step_time: 1.81300 (2.91376) + | > loader_time: 0.08720 (0.04005) + + + --> STEP: 129/234 -- GLOBAL_STEP: 22125 + | > loss: -0.05445 (-0.02773) + | > log_mle: -0.25892 (-0.20078) + | > loss_dur: 0.20447 (0.17305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.63876 (10.80585) + | > current_lr: 0.00002 + | > step_time: 1.19900 (2.88216) + | > loader_time: 0.00320 (0.03919) + + + --> STEP: 134/234 -- GLOBAL_STEP: 22130 + | > loss: -0.07906 (-0.03001) + | > log_mle: -0.31328 (-0.20459) + | > loss_dur: 0.23422 (0.17458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.06800 (11.55096) + | > current_lr: 0.00002 + | > step_time: 6.42630 (2.85844) + | > loader_time: 0.08980 (0.03921) + + + --> STEP: 139/234 -- GLOBAL_STEP: 22135 + | > loss: -0.15393 (-0.03227) + | > log_mle: -0.37059 (-0.20819) + | > loss_dur: 0.21665 (0.17592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.75676 (12.30496) + | > current_lr: 0.00002 + | > step_time: 1.60330 (2.85191) + | > loader_time: 0.08650 (0.03977) + + + --> STEP: 144/234 -- GLOBAL_STEP: 22140 + | > loss: -0.12138 (-0.03432) + | > log_mle: -0.35002 (-0.21205) + | > loss_dur: 0.22863 (0.17772) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.70094 (13.00886) + | > current_lr: 0.00002 + | > step_time: 2.99440 (2.83885) + | > loader_time: 0.19980 (0.04105) + + + --> STEP: 149/234 -- GLOBAL_STEP: 22145 + | > loss: -0.15702 (-0.03742) + | > log_mle: -0.38986 (-0.21640) + | > loss_dur: 0.23285 (0.17898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.76573 (13.77308) + | > current_lr: 0.00002 + | > step_time: 4.39740 (2.82398) + | > loader_time: 0.10760 (0.04234) + + + --> STEP: 154/234 -- GLOBAL_STEP: 22150 + | > loss: -0.13554 (-0.04071) + | > log_mle: -0.35472 (-0.22120) + | > loss_dur: 0.21919 (0.18050) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.09031 (14.57946) + | > current_lr: 0.00002 + | > step_time: 2.70770 (2.86790) + | > loader_time: 0.08710 (0.04167) + + + --> STEP: 159/234 -- GLOBAL_STEP: 22155 + | > loss: -0.15113 (-0.04382) + | > log_mle: -0.37467 (-0.22580) + | > loss_dur: 0.22354 (0.18198) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.89864 (15.49478) + | > current_lr: 0.00002 + | > step_time: 1.00510 (2.86136) + | > loader_time: 0.07860 (0.04206) + + + --> STEP: 164/234 -- GLOBAL_STEP: 22160 + | > loss: -0.13353 (-0.04693) + | > log_mle: -0.36414 (-0.23015) + | > loss_dur: 0.23061 (0.18322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.00215 (16.38547) + | > current_lr: 0.00002 + | > step_time: 2.70920 (2.84729) + | > loader_time: 0.00810 (0.04144) + + + --> STEP: 169/234 -- GLOBAL_STEP: 22165 + | > loss: -0.12022 (-0.04983) + | > log_mle: -0.35926 (-0.23461) + | > loss_dur: 0.23904 (0.18479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.37594 (17.25529) + | > current_lr: 0.00002 + | > step_time: 4.80670 (2.84838) + | > loader_time: 0.09390 (0.04196) + + + --> STEP: 174/234 -- GLOBAL_STEP: 22170 + | > loss: -0.20824 (-0.05351) + | > log_mle: -0.44480 (-0.24005) + | > loss_dur: 0.23655 (0.18653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.65258 (18.53701) + | > current_lr: 0.00002 + | > step_time: 4.69580 (2.87514) + | > loader_time: 0.19400 (0.04246) + + + --> STEP: 179/234 -- GLOBAL_STEP: 22175 + | > loss: -0.17482 (-0.05654) + | > log_mle: -0.43587 (-0.24493) + | > loss_dur: 0.26105 (0.18839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.86545 (19.57802) + | > current_lr: 0.00002 + | > step_time: 5.70570 (2.89158) + | > loader_time: 0.09280 (0.04249) + + + --> STEP: 184/234 -- GLOBAL_STEP: 22180 + | > loss: -0.16520 (-0.05943) + | > log_mle: -0.41203 (-0.24941) + | > loss_dur: 0.24683 (0.18998) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.79627 (20.35640) + | > current_lr: 0.00002 + | > step_time: 3.21430 (2.87302) + | > loader_time: 0.00280 (0.04722) + + + --> STEP: 189/234 -- GLOBAL_STEP: 22185 + | > loss: -0.15483 (-0.06229) + | > log_mle: -0.41297 (-0.25409) + | > loss_dur: 0.25813 (0.19180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.32916 (21.09959) + | > current_lr: 0.00002 + | > step_time: 6.19510 (2.94999) + | > loader_time: 0.09590 (0.04851) + + + --> STEP: 194/234 -- GLOBAL_STEP: 22190 + | > loss: -0.19682 (-0.06576) + | > log_mle: -0.44001 (-0.25869) + | > loss_dur: 0.24320 (0.19293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.14735 (22.14972) + | > current_lr: 0.00002 + | > step_time: 6.39850 (2.98225) + | > loader_time: 0.10990 (0.04889) + + + --> STEP: 199/234 -- GLOBAL_STEP: 22195 + | > loss: -0.20094 (-0.06865) + | > log_mle: -0.44941 (-0.26301) + | > loss_dur: 0.24847 (0.19436) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.68646 (22.91801) + | > current_lr: 0.00002 + | > step_time: 3.20360 (3.09263) + | > loader_time: 0.00670 (0.04977) + + + --> STEP: 204/234 -- GLOBAL_STEP: 22200 + | > loss: -0.20549 (-0.07122) + | > log_mle: -0.48003 (-0.26724) + | > loss_dur: 0.27454 (0.19602) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.93794 (23.85299) + | > current_lr: 0.00002 + | > step_time: 5.00310 (3.13942) + | > loader_time: 0.19410 (0.05134) + + + --> STEP: 209/234 -- GLOBAL_STEP: 22205 + | > loss: -0.17768 (-0.07435) + | > log_mle: -0.44010 (-0.27185) + | > loss_dur: 0.26242 (0.19750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.21580 (24.58970) + | > current_lr: 0.00002 + | > step_time: 6.89760 (3.21779) + | > loader_time: 0.11290 (0.05162) + + + --> STEP: 214/234 -- GLOBAL_STEP: 22210 + | > loss: -0.22043 (-0.07804) + | > log_mle: -0.47090 (-0.27718) + | > loss_dur: 0.25048 (0.19914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.85986 (25.76887) + | > current_lr: 0.00002 + | > step_time: 6.80070 (3.25908) + | > loader_time: 0.00680 (0.05274) + + + --> STEP: 219/234 -- GLOBAL_STEP: 22215 + | > loss: -0.28178 (-0.08153) + | > log_mle: -0.55406 (-0.28224) + | > loss_dur: 0.27228 (0.20071) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 118.04766 (27.15644) + | > current_lr: 0.00002 + | > step_time: 5.19930 (3.29211) + | > loader_time: 0.00550 (0.05471) + + + --> STEP: 224/234 -- GLOBAL_STEP: 22220 + | > loss: -0.22659 (-0.08454) + | > log_mle: -0.50945 (-0.28696) + | > loss_dur: 0.28285 (0.20242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.46079 (28.22955) + | > current_lr: 0.00002 + | > step_time: 2.00280 (3.27274) + | > loader_time: 0.00510 (0.05397) + + + --> STEP: 229/234 -- GLOBAL_STEP: 22225 + | > loss: -0.20185 (-0.08753) + | > log_mle: -0.54463 (-0.29211) + | > loss_dur: 0.34278 (0.20458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.97468 (29.62873) + | > current_lr: 0.00002 + | > step_time: 0.25330 (3.21145) + | > loader_time: 0.00820 (0.05295) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00198 (-0.32473) + | > avg_loss: -0.10427 (+0.02293) + | > avg_log_mle: -0.34609 (+0.02213) + | > avg_loss_dur: 0.24182 (+0.00080) + + + > EPOCH: 95/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 05:12:56)  + + --> STEP: 0/234 -- GLOBAL_STEP: 22230 + | > loss: -0.05688 (-0.05688) + | > log_mle: -0.22965 (-0.22965) + | > loss_dur: 0.17278 (0.17278) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.83544 (9.83544) + | > current_lr: 0.00002 + | > step_time: 4.11630 (4.11630) + | > loader_time: 8.43150 (8.43149) + + + --> STEP: 5/234 -- GLOBAL_STEP: 22235 + | > loss: 0.00085 (0.00888) + | > log_mle: -0.18244 (-0.17646) + | > loss_dur: 0.18329 (0.18534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.66954 (7.69113) + | > current_lr: 0.00002 + | > step_time: 2.60610 (5.62568) + | > loader_time: 0.00180 (1.19807) + + + --> STEP: 10/234 -- GLOBAL_STEP: 22240 + | > loss: -0.00244 (-0.00135) + | > log_mle: -0.18070 (-0.17972) + | > loss_dur: 0.17826 (0.17838) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.54659 (7.03506) + | > current_lr: 0.00002 + | > step_time: 6.90480 (4.48108) + | > loader_time: 0.01090 (0.60354) + + + --> STEP: 15/234 -- GLOBAL_STEP: 22245 + | > loss: -0.04628 (-0.00904) + | > log_mle: -0.18040 (-0.17860) + | > loss_dur: 0.13411 (0.16956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.37822 (6.76587) + | > current_lr: 0.00002 + | > step_time: 3.00160 (3.92865) + | > loader_time: 0.00180 (0.40926) + + + --> STEP: 20/234 -- GLOBAL_STEP: 22250 + | > loss: -0.01354 (-0.00883) + | > log_mle: -0.16478 (-0.17569) + | > loss_dur: 0.15124 (0.16687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.86025 (6.53754) + | > current_lr: 0.00002 + | > step_time: 4.51140 (4.14282) + | > loader_time: 0.09780 (0.33389) + + + --> STEP: 25/234 -- GLOBAL_STEP: 22255 + | > loss: -0.00172 (-0.01147) + | > log_mle: -0.15924 (-0.17427) + | > loss_dur: 0.15752 (0.16279) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.14234 (6.38934) + | > current_lr: 0.00002 + | > step_time: 1.56370 (3.97546) + | > loader_time: 0.00240 (0.27679) + + + --> STEP: 30/234 -- GLOBAL_STEP: 22260 + | > loss: -0.04673 (-0.01517) + | > log_mle: -0.18855 (-0.17456) + | > loss_dur: 0.14182 (0.15939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.45021 (6.19714) + | > current_lr: 0.00002 + | > step_time: 3.90820 (3.63277) + | > loader_time: 0.00290 (0.23173) + + + --> STEP: 35/234 -- GLOBAL_STEP: 22265 + | > loss: -0.01926 (-0.01493) + | > log_mle: -0.18140 (-0.17497) + | > loss_dur: 0.16214 (0.16004) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.30813 (6.38856) + | > current_lr: 0.00002 + | > step_time: 1.59210 (3.32430) + | > loader_time: 0.00250 (0.19895) + + + --> STEP: 40/234 -- GLOBAL_STEP: 22270 + | > loss: 0.00014 (-0.01472) + | > log_mle: -0.16244 (-0.17514) + | > loss_dur: 0.16258 (0.16042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.39174 (6.35000) + | > current_lr: 0.00002 + | > step_time: 0.87890 (3.11387) + | > loader_time: 0.00190 (0.17626) + + + --> STEP: 45/234 -- GLOBAL_STEP: 22275 + | > loss: -0.03823 (-0.01532) + | > log_mle: -0.19969 (-0.17552) + | > loss_dur: 0.16146 (0.16019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.50258 (6.25474) + | > current_lr: 0.00002 + | > step_time: 0.84850 (2.89994) + | > loader_time: 0.00310 (0.15863) + + + --> STEP: 50/234 -- GLOBAL_STEP: 22280 + | > loss: -0.00377 (-0.01572) + | > log_mle: -0.16533 (-0.17525) + | > loss_dur: 0.16157 (0.15953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.24569 (6.04376) + | > current_lr: 0.00002 + | > step_time: 3.20590 (2.86491) + | > loader_time: 0.08340 (0.14465) + + + --> STEP: 55/234 -- GLOBAL_STEP: 22285 + | > loss: -0.03838 (-0.01584) + | > log_mle: -0.18505 (-0.17553) + | > loss_dur: 0.14667 (0.15969) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.89421 (5.99992) + | > current_lr: 0.00002 + | > step_time: 1.02360 (2.72688) + | > loader_time: 0.08670 (0.13337) + + + --> STEP: 60/234 -- GLOBAL_STEP: 22290 + | > loss: -0.03991 (-0.01659) + | > log_mle: -0.20207 (-0.17619) + | > loss_dur: 0.16216 (0.15959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.49754 (6.08422) + | > current_lr: 0.00002 + | > step_time: 3.00730 (2.66945) + | > loader_time: 0.00210 (0.12524) + + + --> STEP: 65/234 -- GLOBAL_STEP: 22295 + | > loss: -0.02199 (-0.01640) + | > log_mle: -0.17729 (-0.17687) + | > loss_dur: 0.15530 (0.16047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.29382 (6.22788) + | > current_lr: 0.00002 + | > step_time: 3.12870 (2.61690) + | > loader_time: 0.09450 (0.11727) + + + --> STEP: 70/234 -- GLOBAL_STEP: 22300 + | > loss: -0.00858 (-0.01590) + | > log_mle: -0.17692 (-0.17675) + | > loss_dur: 0.16834 (0.16084) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.12190 (6.44870) + | > current_lr: 0.00002 + | > step_time: 2.19980 (2.59483) + | > loader_time: 0.10430 (0.11199) + + + --> STEP: 75/234 -- GLOBAL_STEP: 22305 + | > loss: -0.02465 (-0.01613) + | > log_mle: -0.18959 (-0.17761) + | > loss_dur: 0.16494 (0.16148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.15239 (6.82184) + | > current_lr: 0.00002 + | > step_time: 2.80280 (2.56159) + | > loader_time: 0.08960 (0.10702) + + + --> STEP: 80/234 -- GLOBAL_STEP: 22310 + | > loss: -0.02982 (-0.01666) + | > log_mle: -0.17098 (-0.17789) + | > loss_dur: 0.14116 (0.16124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.05307 (6.95375) + | > current_lr: 0.00002 + | > step_time: 1.20610 (2.52508) + | > loader_time: 0.00560 (0.10054) + + + --> STEP: 85/234 -- GLOBAL_STEP: 22315 + | > loss: -0.02762 (-0.01701) + | > log_mle: -0.18672 (-0.17854) + | > loss_dur: 0.15910 (0.16153) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.70546 (7.15162) + | > current_lr: 0.00002 + | > step_time: 7.80950 (2.59682) + | > loader_time: 0.29000 (0.09912) + + + --> STEP: 90/234 -- GLOBAL_STEP: 22320 + | > loss: -0.03848 (-0.01799) + | > log_mle: -0.21800 (-0.18038) + | > loss_dur: 0.17952 (0.16240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.69860 (7.56700) + | > current_lr: 0.00002 + | > step_time: 2.21120 (2.57836) + | > loader_time: 0.09640 (0.09481) + + + --> STEP: 95/234 -- GLOBAL_STEP: 22325 + | > loss: -0.10305 (-0.02068) + | > log_mle: -0.30367 (-0.18412) + | > loss_dur: 0.20063 (0.16345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.90635 (8.07199) + | > current_lr: 0.00002 + | > step_time: 3.59060 (2.58686) + | > loader_time: 0.00310 (0.09185) + + + --> STEP: 100/234 -- GLOBAL_STEP: 22330 + | > loss: -0.04818 (-0.02171) + | > log_mle: -0.22996 (-0.18613) + | > loss_dur: 0.18178 (0.16443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.58698 (8.31128) + | > current_lr: 0.00002 + | > step_time: 4.31500 (2.59769) + | > loader_time: 0.39610 (0.09293) + + + --> STEP: 105/234 -- GLOBAL_STEP: 22335 + | > loss: -0.04510 (-0.02357) + | > log_mle: -0.20500 (-0.18918) + | > loss_dur: 0.15990 (0.16561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.44343 (8.68174) + | > current_lr: 0.00002 + | > step_time: 4.01110 (2.66559) + | > loader_time: 0.00310 (0.09039) + + + --> STEP: 110/234 -- GLOBAL_STEP: 22340 + | > loss: -0.06332 (-0.02485) + | > log_mle: -0.23321 (-0.19185) + | > loss_dur: 0.16989 (0.16701) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.71812 (9.18003) + | > current_lr: 0.00002 + | > step_time: 5.90800 (2.66367) + | > loader_time: 0.29290 (0.08973) + + + --> STEP: 115/234 -- GLOBAL_STEP: 22345 + | > loss: -0.04529 (-0.02658) + | > log_mle: -0.25090 (-0.19515) + | > loss_dur: 0.20560 (0.16857) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.60942 (9.67089) + | > current_lr: 0.00002 + | > step_time: 2.21080 (2.65026) + | > loader_time: 0.09160 (0.08769) + + + --> STEP: 120/234 -- GLOBAL_STEP: 22350 + | > loss: -0.07884 (-0.02793) + | > log_mle: -0.29856 (-0.19801) + | > loss_dur: 0.21972 (0.17008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.56752 (9.97535) + | > current_lr: 0.00002 + | > step_time: 1.79230 (2.61155) + | > loader_time: 0.00380 (0.08486) + + + --> STEP: 125/234 -- GLOBAL_STEP: 22355 + | > loss: -0.08461 (-0.02900) + | > log_mle: -0.28512 (-0.19967) + | > loss_dur: 0.20052 (0.17067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.91965 (10.26636) + | > current_lr: 0.00002 + | > step_time: 2.00880 (2.58577) + | > loader_time: 0.09340 (0.08376) + + + --> STEP: 130/234 -- GLOBAL_STEP: 22360 + | > loss: -0.08863 (-0.03112) + | > log_mle: -0.29670 (-0.20302) + | > loss_dur: 0.20807 (0.17189) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.44660 (10.92703) + | > current_lr: 0.00002 + | > step_time: 3.58920 (2.58528) + | > loader_time: 0.00370 (0.08130) + + + --> STEP: 135/234 -- GLOBAL_STEP: 22365 + | > loss: -0.04605 (-0.03331) + | > log_mle: -0.23076 (-0.20640) + | > loss_dur: 0.18471 (0.17309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.86585 (11.46322) + | > current_lr: 0.00002 + | > step_time: 2.49950 (2.58886) + | > loader_time: 0.00450 (0.07957) + + + --> STEP: 140/234 -- GLOBAL_STEP: 22370 + | > loss: -0.05048 (-0.03561) + | > log_mle: -0.25879 (-0.21028) + | > loss_dur: 0.20832 (0.17468) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.67365 (12.15232) + | > current_lr: 0.00002 + | > step_time: 1.99430 (2.60091) + | > loader_time: 0.00480 (0.07862) + + + --> STEP: 145/234 -- GLOBAL_STEP: 22375 + | > loss: -0.13351 (-0.03820) + | > log_mle: -0.35601 (-0.21470) + | > loss_dur: 0.22250 (0.17650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.42386 (13.18652) + | > current_lr: 0.00002 + | > step_time: 1.59460 (2.61546) + | > loader_time: 0.10290 (0.07932) + + + --> STEP: 150/234 -- GLOBAL_STEP: 22380 + | > loss: -0.11670 (-0.04097) + | > log_mle: -0.34249 (-0.21886) + | > loss_dur: 0.22579 (0.17789) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.03577 (13.92127) + | > current_lr: 0.00002 + | > step_time: 3.89960 (2.67704) + | > loader_time: 0.10280 (0.07993) + + + --> STEP: 155/234 -- GLOBAL_STEP: 22385 + | > loss: -0.16810 (-0.04463) + | > log_mle: -0.40652 (-0.22404) + | > loss_dur: 0.23842 (0.17941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.05561 (14.76495) + | > current_lr: 0.00002 + | > step_time: 2.90610 (2.71008) + | > loader_time: 0.08620 (0.07870) + + + --> STEP: 160/234 -- GLOBAL_STEP: 22390 + | > loss: -0.17492 (-0.04765) + | > log_mle: -0.40033 (-0.22851) + | > loss_dur: 0.22541 (0.18086) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.59655 (15.56636) + | > current_lr: 0.00002 + | > step_time: 6.75380 (2.75145) + | > loader_time: 1.00030 (0.08415) + + + --> STEP: 165/234 -- GLOBAL_STEP: 22395 + | > loss: -0.15018 (-0.05052) + | > log_mle: -0.39801 (-0.23270) + | > loss_dur: 0.24783 (0.18218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.14499 (16.53246) + | > current_lr: 0.00002 + | > step_time: 1.29210 (2.72329) + | > loader_time: 0.09400 (0.08226) + + + --> STEP: 170/234 -- GLOBAL_STEP: 22400 + | > loss: -0.16469 (-0.05330) + | > log_mle: -0.43320 (-0.23724) + | > loss_dur: 0.26851 (0.18394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.71074 (17.46463) + | > current_lr: 0.00002 + | > step_time: 1.40080 (2.70904) + | > loader_time: 0.00490 (0.08171) + + + --> STEP: 175/234 -- GLOBAL_STEP: 22405 + | > loss: -0.15059 (-0.05695) + | > log_mle: -0.40636 (-0.24252) + | > loss_dur: 0.25577 (0.18557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.09962 (18.56217) + | > current_lr: 0.00002 + | > step_time: 2.51360 (2.75691) + | > loader_time: 0.00310 (0.08114) + + + --> STEP: 180/234 -- GLOBAL_STEP: 22410 + | > loss: -0.17341 (-0.05998) + | > log_mle: -0.41289 (-0.24741) + | > loss_dur: 0.23947 (0.18743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.31433 (19.64365) + | > current_lr: 0.00002 + | > step_time: 2.09790 (2.74516) + | > loader_time: 0.09700 (0.08046) + + + --> STEP: 185/234 -- GLOBAL_STEP: 22415 + | > loss: -0.17717 (-0.06278) + | > log_mle: -0.44233 (-0.25195) + | > loss_dur: 0.26516 (0.18917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.45757 (20.40513) + | > current_lr: 0.00002 + | > step_time: 3.29950 (2.75364) + | > loader_time: 0.00570 (0.07996) + + + --> STEP: 190/234 -- GLOBAL_STEP: 22420 + | > loss: -0.18158 (-0.06559) + | > log_mle: -0.41356 (-0.25632) + | > loss_dur: 0.23198 (0.19073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.71684 (21.57197) + | > current_lr: 0.00002 + | > step_time: 2.48970 (2.82232) + | > loader_time: 0.00630 (0.07895) + + + --> STEP: 195/234 -- GLOBAL_STEP: 22425 + | > loss: -0.17643 (-0.06881) + | > log_mle: -0.42922 (-0.26085) + | > loss_dur: 0.25279 (0.19204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.72185 (22.64720) + | > current_lr: 0.00002 + | > step_time: 2.19210 (2.84032) + | > loader_time: 0.00450 (0.07934) + + + --> STEP: 200/234 -- GLOBAL_STEP: 22430 + | > loss: -0.16304 (-0.07150) + | > log_mle: -0.43651 (-0.26502) + | > loss_dur: 0.27346 (0.19352) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.35664 (23.55232) + | > current_lr: 0.00002 + | > step_time: 4.51310 (2.91339) + | > loader_time: 0.09570 (0.07982) + + + --> STEP: 205/234 -- GLOBAL_STEP: 22435 + | > loss: -0.17581 (-0.07398) + | > log_mle: -0.42693 (-0.26907) + | > loss_dur: 0.25112 (0.19509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.39294 (24.36945) + | > current_lr: 0.00002 + | > step_time: 9.72060 (3.02824) + | > loader_time: 0.08390 (0.08013) + + + --> STEP: 210/234 -- GLOBAL_STEP: 22440 + | > loss: -0.23855 (-0.07716) + | > log_mle: -0.50529 (-0.27387) + | > loss_dur: 0.26674 (0.19671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.23404 (25.31925) + | > current_lr: 0.00002 + | > step_time: 3.88850 (3.04987) + | > loader_time: 0.00310 (0.08070) + + + --> STEP: 215/234 -- GLOBAL_STEP: 22445 + | > loss: -0.21249 (-0.08066) + | > log_mle: -0.45760 (-0.27890) + | > loss_dur: 0.24511 (0.19824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.23777 (26.26605) + | > current_lr: 0.00002 + | > step_time: 2.62110 (3.08231) + | > loader_time: 0.09240 (0.08201) + + + --> STEP: 220/234 -- GLOBAL_STEP: 22450 + | > loss: -0.23520 (-0.08437) + | > log_mle: -0.50622 (-0.28427) + | > loss_dur: 0.27102 (0.19990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.24744 (27.11326) + | > current_lr: 0.00002 + | > step_time: 4.79030 (3.16809) + | > loader_time: 0.00390 (0.08066) + + + --> STEP: 225/234 -- GLOBAL_STEP: 22455 + | > loss: -0.28383 (-0.08766) + | > log_mle: -0.56807 (-0.28934) + | > loss_dur: 0.28424 (0.20168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.50671 (28.09123) + | > current_lr: 0.00002 + | > step_time: 0.26050 (3.12956) + | > loader_time: 0.00610 (0.07897) + + + --> STEP: 230/234 -- GLOBAL_STEP: 22460 + | > loss: -0.21507 (-0.09024) + | > log_mle: -0.58461 (-0.29442) + | > loss_dur: 0.36954 (0.20419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 338.71490 (30.98316) + | > current_lr: 0.00002 + | > step_time: 0.24880 (3.06686) + | > loader_time: 0.00350 (0.07734) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04417 (+0.04219) + | > avg_loss: -0.11095 (-0.00668) + | > avg_log_mle: -0.35650 (-0.01042) + | > avg_loss_dur: 0.24555 (+0.00373) + + + > EPOCH: 96/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 05:25:57)  + + --> STEP: 1/234 -- GLOBAL_STEP: 22465 + | > loss: -0.05408 (-0.05408) + | > log_mle: -0.18031 (-0.18031) + | > loss_dur: 0.12623 (0.12623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.10183 (10.10183) + | > current_lr: 0.00002 + | > step_time: 8.21010 (8.21006) + | > loader_time: 0.00160 (0.00163) + + + --> STEP: 6/234 -- GLOBAL_STEP: 22470 + | > loss: 0.00419 (0.00893) + | > log_mle: -0.16433 (-0.17153) + | > loss_dur: 0.16851 (0.18047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.73443 (26.10402) + | > current_lr: 0.00002 + | > step_time: 2.89590 (4.80291) + | > loader_time: 0.20390 (0.10019) + + + --> STEP: 11/234 -- GLOBAL_STEP: 22475 + | > loss: -0.03147 (-0.00156) + | > log_mle: -0.17282 (-0.17649) + | > loss_dur: 0.14135 (0.17493) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.08031 (17.65136) + | > current_lr: 0.00002 + | > step_time: 0.99850 (3.89012) + | > loader_time: 0.00180 (0.10855) + + + --> STEP: 16/234 -- GLOBAL_STEP: 22480 + | > loss: -0.05456 (-0.00820) + | > log_mle: -0.17601 (-0.17687) + | > loss_dur: 0.12145 (0.16866) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.94374 (13.94715) + | > current_lr: 0.00002 + | > step_time: 1.89040 (3.59410) + | > loader_time: 0.00330 (0.08163) + + + --> STEP: 21/234 -- GLOBAL_STEP: 22485 + | > loss: -0.01745 (-0.00907) + | > log_mle: -0.15822 (-0.17387) + | > loss_dur: 0.14077 (0.16480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.40112 (11.96162) + | > current_lr: 0.00002 + | > step_time: 5.99830 (3.37663) + | > loader_time: 0.00220 (0.06365) + + + --> STEP: 26/234 -- GLOBAL_STEP: 22490 + | > loss: -0.02706 (-0.01396) + | > log_mle: -0.17734 (-0.17379) + | > loss_dur: 0.15028 (0.15983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.33698 (10.64991) + | > current_lr: 0.00002 + | > step_time: 3.89380 (3.42411) + | > loader_time: 0.00190 (0.06245) + + + --> STEP: 31/234 -- GLOBAL_STEP: 22495 + | > loss: 0.01245 (-0.01600) + | > log_mle: -0.17893 (-0.17434) + | > loss_dur: 0.19138 (0.15834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.11516 (9.76893) + | > current_lr: 0.00002 + | > step_time: 4.60140 (3.62673) + | > loader_time: 0.08310 (0.06186) + + + --> STEP: 36/234 -- GLOBAL_STEP: 22500 + | > loss: -0.01033 (-0.01621) + | > log_mle: -0.18229 (-0.17512) + | > loss_dur: 0.17196 (0.15890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.59826 (9.40045) + | > current_lr: 0.00002 + | > step_time: 0.80320 (3.74557) + | > loader_time: 0.06910 (0.05840) + + + --> STEP: 41/234 -- GLOBAL_STEP: 22505 + | > loss: -0.02992 (-0.01610) + | > log_mle: -0.17643 (-0.17530) + | > loss_dur: 0.14651 (0.15921) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.91226 (8.99591) + | > current_lr: 0.00002 + | > step_time: 1.15720 (3.50730) + | > loader_time: 0.00180 (0.05573) + + + --> STEP: 46/234 -- GLOBAL_STEP: 22510 + | > loss: -0.01912 (-0.01666) + | > log_mle: -0.18119 (-0.17600) + | > loss_dur: 0.16207 (0.15934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.75879 (8.72546) + | > current_lr: 0.00002 + | > step_time: 2.60890 (3.31720) + | > loader_time: 0.00320 (0.04993) + + + --> STEP: 51/234 -- GLOBAL_STEP: 22515 + | > loss: -0.01917 (-0.01744) + | > log_mle: -0.16475 (-0.17562) + | > loss_dur: 0.14558 (0.15818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.25965 (8.33573) + | > current_lr: 0.00002 + | > step_time: 1.80620 (3.19193) + | > loader_time: 0.00230 (0.04533) + + + --> STEP: 56/234 -- GLOBAL_STEP: 22520 + | > loss: -0.00269 (-0.01801) + | > log_mle: -0.18464 (-0.17647) + | > loss_dur: 0.18195 (0.15846) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.23690 (8.02516) + | > current_lr: 0.00002 + | > step_time: 1.21100 (3.08223) + | > loader_time: 0.00250 (0.04443) + + + --> STEP: 61/234 -- GLOBAL_STEP: 22525 + | > loss: -0.04060 (-0.01958) + | > log_mle: -0.18213 (-0.17729) + | > loss_dur: 0.14153 (0.15770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.47598 (7.91840) + | > current_lr: 0.00002 + | > step_time: 4.81670 (3.02470) + | > loader_time: 0.10760 (0.04283) + + + --> STEP: 66/234 -- GLOBAL_STEP: 22530 + | > loss: -0.01454 (-0.01910) + | > log_mle: -0.16976 (-0.17794) + | > loss_dur: 0.15522 (0.15884) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.51285 (7.81846) + | > current_lr: 0.00002 + | > step_time: 1.80430 (2.95528) + | > loader_time: 0.00600 (0.04115) + + + --> STEP: 71/234 -- GLOBAL_STEP: 22535 + | > loss: -0.00884 (-0.01832) + | > log_mle: -0.21171 (-0.17858) + | > loss_dur: 0.20287 (0.16025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.79676 (8.01285) + | > current_lr: 0.00002 + | > step_time: 2.95620 (2.93499) + | > loader_time: 0.20430 (0.04248) + + + --> STEP: 76/234 -- GLOBAL_STEP: 22540 + | > loss: -0.03189 (-0.01865) + | > log_mle: -0.19615 (-0.17941) + | > loss_dur: 0.16426 (0.16076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.86647 (7.98836) + | > current_lr: 0.00002 + | > step_time: 4.90710 (2.90583) + | > loader_time: 0.10450 (0.04448) + + + --> STEP: 81/234 -- GLOBAL_STEP: 22545 + | > loss: -0.04700 (-0.01943) + | > log_mle: -0.20377 (-0.17994) + | > loss_dur: 0.15677 (0.16051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.12857 (8.00075) + | > current_lr: 0.00002 + | > step_time: 4.19230 (2.90423) + | > loader_time: 0.19900 (0.04626) + + + --> STEP: 86/234 -- GLOBAL_STEP: 22550 + | > loss: -0.02605 (-0.02011) + | > log_mle: -0.20288 (-0.18076) + | > loss_dur: 0.17683 (0.16065) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.80750 (8.14609) + | > current_lr: 0.00002 + | > step_time: 2.29140 (2.87846) + | > loader_time: 0.10760 (0.04502) + + + --> STEP: 91/234 -- GLOBAL_STEP: 22555 + | > loss: -0.02379 (-0.02069) + | > log_mle: -0.21136 (-0.18273) + | > loss_dur: 0.18757 (0.16203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57129 (8.44247) + | > current_lr: 0.00002 + | > step_time: 2.80740 (2.85230) + | > loader_time: 0.08430 (0.04528) + + + --> STEP: 96/234 -- GLOBAL_STEP: 22560 + | > loss: -0.02326 (-0.02310) + | > log_mle: -0.20191 (-0.18626) + | > loss_dur: 0.17865 (0.16316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79684 (8.94089) + | > current_lr: 0.00002 + | > step_time: 1.20490 (2.82016) + | > loader_time: 0.00450 (0.04406) + + + --> STEP: 101/234 -- GLOBAL_STEP: 22565 + | > loss: -0.07144 (-0.02485) + | > log_mle: -0.26175 (-0.18884) + | > loss_dur: 0.19031 (0.16399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.26936 (9.24164) + | > current_lr: 0.00002 + | > step_time: 4.11030 (2.79897) + | > loader_time: 0.08620 (0.04438) + + + --> STEP: 106/234 -- GLOBAL_STEP: 22570 + | > loss: -0.04749 (-0.02662) + | > log_mle: -0.25929 (-0.19178) + | > loss_dur: 0.21181 (0.16516) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.13422 (9.68595) + | > current_lr: 0.00002 + | > step_time: 2.80800 (2.78228) + | > loader_time: 0.07610 (0.04562) + + + --> STEP: 111/234 -- GLOBAL_STEP: 22575 + | > loss: -0.07315 (-0.02795) + | > log_mle: -0.30330 (-0.19478) + | > loss_dur: 0.23015 (0.16682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.00308 (10.11944) + | > current_lr: 0.00002 + | > step_time: 1.58990 (2.75590) + | > loader_time: 0.00290 (0.04705) + + + --> STEP: 116/234 -- GLOBAL_STEP: 22580 + | > loss: -0.03059 (-0.02917) + | > log_mle: -0.26940 (-0.19771) + | > loss_dur: 0.23881 (0.16854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.00657 (10.53181) + | > current_lr: 0.00002 + | > step_time: 4.92540 (2.76591) + | > loader_time: 0.00450 (0.04658) + + + --> STEP: 121/234 -- GLOBAL_STEP: 22585 + | > loss: -0.00776 (-0.03025) + | > log_mle: -0.18641 (-0.19975) + | > loss_dur: 0.17865 (0.16950) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.78836 (10.78008) + | > current_lr: 0.00002 + | > step_time: 1.99080 (2.73518) + | > loader_time: 0.08880 (0.04610) + + + --> STEP: 126/234 -- GLOBAL_STEP: 22590 + | > loss: -0.10443 (-0.03203) + | > log_mle: -0.31664 (-0.20243) + | > loss_dur: 0.21221 (0.17039) + | > amp_scaler: 4096.00000 (2096.76190) + | > grad_norm: 27.75268 (11.10928) + | > current_lr: 0.00002 + | > step_time: 2.30660 (2.71006) + | > loader_time: 0.10090 (0.04592) + + + --> STEP: 131/234 -- GLOBAL_STEP: 22595 + | > loss: -0.12336 (-0.03413) + | > log_mle: -0.35453 (-0.20601) + | > loss_dur: 0.23116 (0.17188) + | > amp_scaler: 4096.00000 (2173.06870) + | > grad_norm: 36.08327 (11.65685) + | > current_lr: 0.00002 + | > step_time: 6.29120 (2.74402) + | > loader_time: 0.00210 (0.04628) + + + --> STEP: 136/234 -- GLOBAL_STEP: 22600 + | > loss: -0.16037 (-0.03629) + | > log_mle: -0.40014 (-0.20954) + | > loss_dur: 0.23977 (0.17325) + | > amp_scaler: 4096.00000 (2243.76471) + | > grad_norm: 51.50101 (12.35892) + | > current_lr: 0.00002 + | > step_time: 4.11350 (2.74847) + | > loader_time: 0.00310 (0.04605) + + + --> STEP: 141/234 -- GLOBAL_STEP: 22605 + | > loss: -0.09560 (-0.03805) + | > log_mle: -0.31249 (-0.21264) + | > loss_dur: 0.21689 (0.17459) + | > amp_scaler: 4096.00000 (2309.44681) + | > grad_norm: 28.45630 (12.96126) + | > current_lr: 0.00002 + | > step_time: 1.64150 (2.83818) + | > loader_time: 0.00270 (0.04931) + + + --> STEP: 146/234 -- GLOBAL_STEP: 22610 + | > loss: -0.13663 (-0.04092) + | > log_mle: -0.35662 (-0.21733) + | > loss_dur: 0.21999 (0.17641) + | > amp_scaler: 4096.00000 (2370.63014) + | > grad_norm: 44.45480 (14.09638) + | > current_lr: 0.00002 + | > step_time: 1.20180 (2.82801) + | > loader_time: 0.08270 (0.04934) + + + --> STEP: 151/234 -- GLOBAL_STEP: 22615 + | > loss: -0.12965 (-0.04352) + | > log_mle: -0.32923 (-0.22116) + | > loss_dur: 0.19958 (0.17764) + | > amp_scaler: 4096.00000 (2427.76159) + | > grad_norm: 27.25220 (14.78078) + | > current_lr: 0.00002 + | > step_time: 1.39320 (2.80202) + | > loader_time: 0.00330 (0.04830) + + + --> STEP: 156/234 -- GLOBAL_STEP: 22620 + | > loss: -0.15161 (-0.04714) + | > log_mle: -0.36722 (-0.22634) + | > loss_dur: 0.21561 (0.17920) + | > amp_scaler: 4096.00000 (2481.23077) + | > grad_norm: 44.80963 (16.00954) + | > current_lr: 0.00002 + | > step_time: 8.19850 (2.89182) + | > loader_time: 0.08690 (0.04852) + + + --> STEP: 161/234 -- GLOBAL_STEP: 22625 + | > loss: -0.16421 (-0.05014) + | > log_mle: -0.39089 (-0.23087) + | > loss_dur: 0.22668 (0.18073) + | > amp_scaler: 4096.00000 (2531.37888) + | > grad_norm: 42.14550 (16.76400) + | > current_lr: 0.00002 + | > step_time: 1.48800 (2.90320) + | > loader_time: 0.00890 (0.04830) + + + --> STEP: 166/234 -- GLOBAL_STEP: 22630 + | > loss: -0.13289 (-0.05283) + | > log_mle: -0.33346 (-0.23477) + | > loss_dur: 0.20057 (0.18194) + | > amp_scaler: 4096.00000 (2578.50602) + | > grad_norm: 31.06101 (17.45971) + | > current_lr: 0.00002 + | > step_time: 6.39590 (2.90700) + | > loader_time: 0.00690 (0.04813) + + + --> STEP: 171/234 -- GLOBAL_STEP: 22635 + | > loss: -0.21210 (-0.05630) + | > log_mle: -0.43891 (-0.24000) + | > loss_dur: 0.22681 (0.18369) + | > amp_scaler: 4096.00000 (2622.87719) + | > grad_norm: 43.85900 (18.37381) + | > current_lr: 0.00002 + | > step_time: 3.60790 (2.89428) + | > loader_time: 0.19230 (0.04796) + + + --> STEP: 176/234 -- GLOBAL_STEP: 22640 + | > loss: -0.17104 (-0.05968) + | > log_mle: -0.41026 (-0.24516) + | > loss_dur: 0.23923 (0.18547) + | > amp_scaler: 4096.00000 (2664.72727) + | > grad_norm: 43.31188 (19.20828) + | > current_lr: 0.00002 + | > step_time: 2.09930 (2.88921) + | > loader_time: 0.00300 (0.04853) + + + --> STEP: 181/234 -- GLOBAL_STEP: 22645 + | > loss: -0.11377 (-0.06256) + | > log_mle: -0.35100 (-0.24979) + | > loss_dur: 0.23722 (0.18723) + | > amp_scaler: 4096.00000 (2704.26519) + | > grad_norm: 32.26495 (19.99718) + | > current_lr: 0.00002 + | > step_time: 2.89130 (2.92212) + | > loader_time: 0.00450 (0.04951) + + + --> STEP: 186/234 -- GLOBAL_STEP: 22650 + | > loss: -0.12554 (-0.06547) + | > log_mle: -0.38565 (-0.25458) + | > loss_dur: 0.26011 (0.18911) + | > amp_scaler: 4096.00000 (2741.67742) + | > grad_norm: 47.25529 (20.83023) + | > current_lr: 0.00002 + | > step_time: 3.08040 (2.91397) + | > loader_time: 0.00380 (0.04930) + + + --> STEP: 191/234 -- GLOBAL_STEP: 22655 + | > loss: -0.17933 (-0.06849) + | > log_mle: -0.40033 (-0.25904) + | > loss_dur: 0.22101 (0.19055) + | > amp_scaler: 4096.00000 (2777.13089) + | > grad_norm: 46.62757 (21.80879) + | > current_lr: 0.00002 + | > step_time: 2.49510 (2.94599) + | > loader_time: 0.00780 (0.04955) + + + --> STEP: 196/234 -- GLOBAL_STEP: 22660 + | > loss: -0.15064 (-0.07169) + | > log_mle: -0.40126 (-0.26365) + | > loss_dur: 0.25062 (0.19195) + | > amp_scaler: 4096.00000 (2810.77551) + | > grad_norm: 47.82127 (22.77850) + | > current_lr: 0.00002 + | > step_time: 3.00780 (3.01366) + | > loader_time: 0.00400 (0.05043) + + + --> STEP: 201/234 -- GLOBAL_STEP: 22665 + | > loss: -0.11178 (-0.07426) + | > log_mle: -0.36797 (-0.26780) + | > loss_dur: 0.25620 (0.19354) + | > amp_scaler: 4096.00000 (2842.74627) + | > grad_norm: 35.26276 (23.47399) + | > current_lr: 0.00002 + | > step_time: 15.28540 (3.09346) + | > loader_time: 0.21780 (0.05305) + + + --> STEP: 206/234 -- GLOBAL_STEP: 22670 + | > loss: -0.20307 (-0.07724) + | > log_mle: -0.46619 (-0.27235) + | > loss_dur: 0.26312 (0.19511) + | > amp_scaler: 4096.00000 (2873.16505) + | > grad_norm: 53.43842 (24.59767) + | > current_lr: 0.00002 + | > step_time: 4.32140 (3.17722) + | > loader_time: 0.08560 (0.05369) + + + --> STEP: 211/234 -- GLOBAL_STEP: 22675 + | > loss: -0.25110 (-0.08064) + | > log_mle: -0.53393 (-0.27753) + | > loss_dur: 0.28284 (0.19688) + | > amp_scaler: 4096.00000 (2902.14218) + | > grad_norm: 85.58341 (25.64680) + | > current_lr: 0.00002 + | > step_time: 4.31320 (3.23150) + | > loader_time: 0.07980 (0.05330) + + + --> STEP: 216/234 -- GLOBAL_STEP: 22680 + | > loss: -0.23658 (-0.08404) + | > log_mle: -0.51853 (-0.28243) + | > loss_dur: 0.28195 (0.19840) + | > amp_scaler: 4096.00000 (2929.77778) + | > grad_norm: 73.67214 (26.60646) + | > current_lr: 0.00002 + | > step_time: 4.30370 (3.30994) + | > loader_time: 0.00370 (0.05266) + + + --> STEP: 221/234 -- GLOBAL_STEP: 22685 + | > loss: -0.18426 (-0.08729) + | > log_mle: -0.44222 (-0.28730) + | > loss_dur: 0.25797 (0.20001) + | > amp_scaler: 4096.00000 (2956.16290) + | > grad_norm: 57.08987 (27.67921) + | > current_lr: 0.00002 + | > step_time: 1.40510 (3.35041) + | > loader_time: 0.00320 (0.05288) + + + --> STEP: 226/234 -- GLOBAL_STEP: 22690 + | > loss: -0.26708 (-0.09090) + | > log_mle: -0.54070 (-0.29269) + | > loss_dur: 0.27362 (0.20179) + | > amp_scaler: 4096.00000 (2981.38053) + | > grad_norm: 71.31867 (28.85232) + | > current_lr: 0.00002 + | > step_time: 0.23390 (3.28661) + | > loader_time: 0.00380 (0.05213) + + + --> STEP: 231/234 -- GLOBAL_STEP: 22695 + | > loss: -0.18372 (-0.09355) + | > log_mle: -0.60317 (-0.29838) + | > loss_dur: 0.41945 (0.20484) + | > amp_scaler: 4096.00000 (3005.50649) + | > grad_norm: 82.78725 (30.11871) + | > current_lr: 0.00002 + | > step_time: 0.27800 (3.22099) + | > loader_time: 0.00480 (0.05108) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.84842 (+0.80426) + | > avg_loss: -0.14191 (-0.03096) + | > avg_log_mle: -0.38146 (-0.02496) + | > avg_loss_dur: 0.23955 (-0.00600) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_22698.pth + + > EPOCH: 97/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 05:39:50)  + + --> STEP: 2/234 -- GLOBAL_STEP: 22700 + | > loss: 0.03091 (-0.00279) + | > log_mle: -0.16486 (-0.17396) + | > loss_dur: 0.19577 (0.17117) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.61757 (6.86036) + | > current_lr: 0.00002 + | > step_time: 5.61170 (5.91240) + | > loader_time: 0.09240 (0.14402) + + + --> STEP: 7/234 -- GLOBAL_STEP: 22705 + | > loss: -0.04487 (-0.00043) + | > log_mle: -0.19284 (-0.17895) + | > loss_dur: 0.14797 (0.17853) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.57436 (10.51230) + | > current_lr: 0.00002 + | > step_time: 9.51410 (5.68003) + | > loader_time: 0.19960 (0.09743) + + + --> STEP: 12/234 -- GLOBAL_STEP: 22710 + | > loss: -0.01814 (-0.00516) + | > log_mle: -0.17917 (-0.18089) + | > loss_dur: 0.16103 (0.17573) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.88062 (9.69921) + | > current_lr: 0.00002 + | > step_time: 1.30190 (4.76256) + | > loader_time: 0.00300 (0.08178) + + + --> STEP: 17/234 -- GLOBAL_STEP: 22715 + | > loss: 0.00534 (-0.00967) + | > log_mle: -0.15757 (-0.17907) + | > loss_dur: 0.16291 (0.16940) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.75304 (8.61265) + | > current_lr: 0.00002 + | > step_time: 5.59410 (4.31923) + | > loader_time: 0.00100 (0.06854) + + + --> STEP: 22/234 -- GLOBAL_STEP: 22720 + | > loss: -0.04282 (-0.01112) + | > log_mle: -0.18417 (-0.17725) + | > loss_dur: 0.14136 (0.16613) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.17530 (8.12340) + | > current_lr: 0.00002 + | > step_time: 4.98800 (3.96077) + | > loader_time: 0.20320 (0.06691) + + + --> STEP: 27/234 -- GLOBAL_STEP: 22725 + | > loss: -0.04806 (-0.01486) + | > log_mle: -0.18604 (-0.17681) + | > loss_dur: 0.13798 (0.16195) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.75420 (7.65224) + | > current_lr: 0.00002 + | > step_time: 12.39610 (4.26058) + | > loader_time: 0.00340 (0.06226) + + + --> STEP: 32/234 -- GLOBAL_STEP: 22730 + | > loss: -0.04860 (-0.01689) + | > log_mle: -0.19452 (-0.17733) + | > loss_dur: 0.14592 (0.16043) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.21884 (7.39484) + | > current_lr: 0.00002 + | > step_time: 0.59400 (3.94918) + | > loader_time: 0.00270 (0.05523) + + + --> STEP: 37/234 -- GLOBAL_STEP: 22735 + | > loss: -0.02222 (-0.01621) + | > log_mle: -0.17544 (-0.17728) + | > loss_dur: 0.15322 (0.16106) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.32096 (7.57520) + | > current_lr: 0.00002 + | > step_time: 2.83070 (3.83555) + | > loader_time: 0.00270 (0.05286) + + + --> STEP: 42/234 -- GLOBAL_STEP: 22740 + | > loss: -0.00479 (-0.01541) + | > log_mle: -0.16567 (-0.17701) + | > loss_dur: 0.16088 (0.16160) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.00146 (7.43004) + | > current_lr: 0.00002 + | > step_time: 1.55910 (3.58139) + | > loader_time: 0.01170 (0.05331) + + + --> STEP: 47/234 -- GLOBAL_STEP: 22745 + | > loss: -0.01845 (-0.01653) + | > log_mle: -0.17687 (-0.17772) + | > loss_dur: 0.15842 (0.16120) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.27097 (7.40430) + | > current_lr: 0.00002 + | > step_time: 4.11300 (3.40990) + | > loader_time: 0.09990 (0.05155) + + + --> STEP: 52/234 -- GLOBAL_STEP: 22750 + | > loss: -0.00007 (-0.01658) + | > log_mle: -0.17448 (-0.17719) + | > loss_dur: 0.17441 (0.16061) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.12363 (7.10160) + | > current_lr: 0.00002 + | > step_time: 2.61040 (3.31636) + | > loader_time: 0.00500 (0.05043) + + + --> STEP: 57/234 -- GLOBAL_STEP: 22755 + | > loss: 0.00073 (-0.01646) + | > log_mle: -0.16854 (-0.17788) + | > loss_dur: 0.16926 (0.16142) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.68672 (6.84467) + | > current_lr: 0.00002 + | > step_time: 2.53470 (3.26337) + | > loader_time: 0.07600 (0.05236) + + + --> STEP: 62/234 -- GLOBAL_STEP: 22760 + | > loss: -0.00886 (-0.01838) + | > log_mle: -0.22228 (-0.17952) + | > loss_dur: 0.21342 (0.16114) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.79956 (7.01334) + | > current_lr: 0.00002 + | > step_time: 8.60780 (3.28102) + | > loader_time: 0.09180 (0.05286) + + + --> STEP: 67/234 -- GLOBAL_STEP: 22765 + | > loss: -0.03148 (-0.01871) + | > log_mle: -0.20177 (-0.17972) + | > loss_dur: 0.17028 (0.16101) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.15148 (6.89816) + | > current_lr: 0.00002 + | > step_time: 5.00500 (3.20714) + | > loader_time: 0.00800 (0.05212) + + + --> STEP: 72/234 -- GLOBAL_STEP: 22770 + | > loss: -0.00835 (-0.01771) + | > log_mle: -0.17907 (-0.17991) + | > loss_dur: 0.17072 (0.16220) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.80589 (7.14176) + | > current_lr: 0.00002 + | > step_time: 2.51930 (3.14710) + | > loader_time: 0.09390 (0.04998) + + + --> STEP: 77/234 -- GLOBAL_STEP: 22775 + | > loss: -0.03813 (-0.01832) + | > log_mle: -0.19435 (-0.18078) + | > loss_dur: 0.15622 (0.16245) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.18332 (7.29807) + | > current_lr: 0.00002 + | > step_time: 2.59030 (3.12310) + | > loader_time: 0.00350 (0.04819) + + + --> STEP: 82/234 -- GLOBAL_STEP: 22780 + | > loss: -0.03820 (-0.01917) + | > log_mle: -0.18545 (-0.18110) + | > loss_dur: 0.14725 (0.16193) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.51015 (7.31878) + | > current_lr: 0.00002 + | > step_time: 1.10750 (3.07953) + | > loader_time: 0.00280 (0.04859) + + + --> STEP: 87/234 -- GLOBAL_STEP: 22785 + | > loss: -0.02092 (-0.01970) + | > log_mle: -0.19656 (-0.18209) + | > loss_dur: 0.17564 (0.16239) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.24327 (7.37344) + | > current_lr: 0.00002 + | > step_time: 1.69200 (3.02560) + | > loader_time: 0.00290 (0.04806) + + + --> STEP: 92/234 -- GLOBAL_STEP: 22790 + | > loss: -0.08167 (-0.02154) + | > log_mle: -0.24025 (-0.18459) + | > loss_dur: 0.15858 (0.16304) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.43509 (7.69913) + | > current_lr: 0.00002 + | > step_time: 2.80240 (2.98627) + | > loader_time: 0.01810 (0.04665) + + + --> STEP: 97/234 -- GLOBAL_STEP: 22795 + | > loss: -0.04780 (-0.02366) + | > log_mle: -0.23062 (-0.18798) + | > loss_dur: 0.18282 (0.16432) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.04564 (8.20285) + | > current_lr: 0.00002 + | > step_time: 6.11710 (3.03877) + | > loader_time: 0.59450 (0.05222) + + + --> STEP: 102/234 -- GLOBAL_STEP: 22800 + | > loss: -0.03378 (-0.02521) + | > log_mle: -0.21195 (-0.19033) + | > loss_dur: 0.17817 (0.16511) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.15354 (8.53126) + | > current_lr: 0.00002 + | > step_time: 1.80070 (2.99874) + | > loader_time: 0.00260 (0.05052) + + + --> STEP: 107/234 -- GLOBAL_STEP: 22805 + | > loss: -0.07137 (-0.02731) + | > log_mle: -0.25854 (-0.19370) + | > loss_dur: 0.18717 (0.16639) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.33588 (9.11986) + | > current_lr: 0.00002 + | > step_time: 1.19050 (3.00218) + | > loader_time: 0.00740 (0.05017) + + + --> STEP: 112/234 -- GLOBAL_STEP: 22810 + | > loss: -0.05643 (-0.02865) + | > log_mle: -0.26511 (-0.19667) + | > loss_dur: 0.20868 (0.16802) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.78353 (9.78818) + | > current_lr: 0.00002 + | > step_time: 2.19680 (2.97964) + | > loader_time: 0.00300 (0.05048) + + + --> STEP: 117/234 -- GLOBAL_STEP: 22815 + | > loss: -0.07431 (-0.03016) + | > log_mle: -0.25793 (-0.19944) + | > loss_dur: 0.18361 (0.16928) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.52158 (10.54141) + | > current_lr: 0.00002 + | > step_time: 2.76110 (2.98616) + | > loader_time: 0.00550 (0.05008) + + + --> STEP: 122/234 -- GLOBAL_STEP: 22820 + | > loss: -0.05709 (-0.03089) + | > log_mle: -0.23794 (-0.20116) + | > loss_dur: 0.18085 (0.17027) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.32032 (10.84429) + | > current_lr: 0.00002 + | > step_time: 2.59720 (2.97237) + | > loader_time: 0.00620 (0.04964) + + + --> STEP: 127/234 -- GLOBAL_STEP: 22825 + | > loss: -0.08033 (-0.03299) + | > log_mle: -0.29338 (-0.20417) + | > loss_dur: 0.21305 (0.17119) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.17617 (11.28505) + | > current_lr: 0.00002 + | > step_time: 1.19280 (2.92634) + | > loader_time: 0.00300 (0.04976) + + + --> STEP: 132/234 -- GLOBAL_STEP: 22830 + | > loss: -0.09361 (-0.03541) + | > log_mle: -0.27250 (-0.20757) + | > loss_dur: 0.17889 (0.17216) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.55231 (11.98398) + | > current_lr: 0.00002 + | > step_time: 7.20220 (2.96297) + | > loader_time: 0.00390 (0.04946) + + + --> STEP: 137/234 -- GLOBAL_STEP: 22835 + | > loss: -0.06688 (-0.03741) + | > log_mle: -0.28456 (-0.21116) + | > loss_dur: 0.21769 (0.17375) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.78208 (12.58280) + | > current_lr: 0.00002 + | > step_time: 0.99640 (2.97831) + | > loader_time: 0.00330 (0.05042) + + + --> STEP: 142/234 -- GLOBAL_STEP: 22840 + | > loss: -0.08451 (-0.03927) + | > log_mle: -0.30102 (-0.21429) + | > loss_dur: 0.21652 (0.17503) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.79373 (13.15153) + | > current_lr: 0.00002 + | > step_time: 2.99680 (2.94103) + | > loader_time: 0.00430 (0.04934) + + + --> STEP: 147/234 -- GLOBAL_STEP: 22845 + | > loss: -0.09359 (-0.04219) + | > log_mle: -0.30407 (-0.21900) + | > loss_dur: 0.21048 (0.17680) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.01254 (14.08822) + | > current_lr: 0.00002 + | > step_time: 1.50990 (2.93297) + | > loader_time: 0.07420 (0.04933) + + + --> STEP: 152/234 -- GLOBAL_STEP: 22850 + | > loss: -0.14367 (-0.04536) + | > log_mle: -0.38218 (-0.22340) + | > loss_dur: 0.23850 (0.17805) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.71976 (14.64598) + | > current_lr: 0.00002 + | > step_time: 3.09680 (2.91602) + | > loader_time: 0.00220 (0.04826) + + + --> STEP: 157/234 -- GLOBAL_STEP: 22855 + | > loss: -0.10805 (-0.04888) + | > log_mle: -0.33038 (-0.22838) + | > loss_dur: 0.22233 (0.17951) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.06509 (15.56615) + | > current_lr: 0.00002 + | > step_time: 2.18330 (2.92120) + | > loader_time: 0.01130 (0.05043) + + + --> STEP: 162/234 -- GLOBAL_STEP: 22860 + | > loss: -0.16007 (-0.05219) + | > log_mle: -0.36702 (-0.23310) + | > loss_dur: 0.20696 (0.18091) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.90851 (16.45566) + | > current_lr: 0.00002 + | > step_time: 3.50050 (2.93338) + | > loader_time: 0.00290 (0.04963) + + + --> STEP: 167/234 -- GLOBAL_STEP: 22865 + | > loss: -0.21503 (-0.05518) + | > log_mle: -0.43777 (-0.23731) + | > loss_dur: 0.22274 (0.18213) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.75460 (17.51210) + | > current_lr: 0.00002 + | > step_time: 6.31430 (2.98951) + | > loader_time: 0.09110 (0.05038) + + + --> STEP: 172/234 -- GLOBAL_STEP: 22870 + | > loss: -0.17265 (-0.05824) + | > log_mle: -0.43015 (-0.24227) + | > loss_dur: 0.25749 (0.18402) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.62972 (18.70884) + | > current_lr: 0.00002 + | > step_time: 4.00460 (3.04860) + | > loader_time: 0.09150 (0.05049) + + + --> STEP: 177/234 -- GLOBAL_STEP: 22875 + | > loss: -0.15551 (-0.06159) + | > log_mle: -0.39341 (-0.24711) + | > loss_dur: 0.23790 (0.18552) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.61599 (19.58968) + | > current_lr: 0.00002 + | > step_time: 9.41070 (3.13722) + | > loader_time: 0.10100 (0.05238) + + + --> STEP: 182/234 -- GLOBAL_STEP: 22880 + | > loss: -0.18477 (-0.06449) + | > log_mle: -0.43965 (-0.25193) + | > loss_dur: 0.25488 (0.18744) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.52234 (20.43147) + | > current_lr: 0.00002 + | > step_time: 1.49910 (3.12031) + | > loader_time: 0.00340 (0.05247) + + + --> STEP: 187/234 -- GLOBAL_STEP: 22885 + | > loss: -0.19964 (-0.06750) + | > log_mle: -0.43908 (-0.25669) + | > loss_dur: 0.23944 (0.18918) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.30549 (21.27498) + | > current_lr: 0.00002 + | > step_time: 2.20000 (3.14554) + | > loader_time: 0.00330 (0.05171) + + + --> STEP: 192/234 -- GLOBAL_STEP: 22890 + | > loss: -0.22039 (-0.07062) + | > log_mle: -0.45642 (-0.26120) + | > loss_dur: 0.23603 (0.19058) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 83.06316 (22.54115) + | > current_lr: 0.00002 + | > step_time: 4.50020 (3.19958) + | > loader_time: 0.10360 (0.05336) + + + --> STEP: 197/234 -- GLOBAL_STEP: 22895 + | > loss: -0.19729 (-0.07359) + | > log_mle: -0.43332 (-0.26558) + | > loss_dur: 0.23603 (0.19200) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 59.89769 (23.48424) + | > current_lr: 0.00002 + | > step_time: 4.09290 (3.26112) + | > loader_time: 0.00650 (0.05548) + + + --> STEP: 202/234 -- GLOBAL_STEP: 22900 + | > loss: -0.26160 (-0.07647) + | > log_mle: -0.52011 (-0.27011) + | > loss_dur: 0.25852 (0.19364) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.28075 (24.28419) + | > current_lr: 0.00002 + | > step_time: 8.91660 (3.36415) + | > loader_time: 0.17980 (0.06486) + + + --> STEP: 207/234 -- GLOBAL_STEP: 22905 + | > loss: -0.25332 (-0.07951) + | > log_mle: -0.51145 (-0.27465) + | > loss_dur: 0.25814 (0.19515) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.60346 (25.10909) + | > current_lr: 0.00002 + | > step_time: 5.21580 (3.39436) + | > loader_time: 0.08810 (0.06430) + + + --> STEP: 212/234 -- GLOBAL_STEP: 22910 + | > loss: -0.22807 (-0.08289) + | > log_mle: -0.48629 (-0.27973) + | > loss_dur: 0.25822 (0.19683) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 79.93543 (26.22251) + | > current_lr: 0.00002 + | > step_time: 7.59980 (3.40852) + | > loader_time: 0.10340 (0.06465) + + + --> STEP: 217/234 -- GLOBAL_STEP: 22915 + | > loss: -0.23801 (-0.08620) + | > log_mle: -0.50565 (-0.28463) + | > loss_dur: 0.26764 (0.19842) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.53376 (27.55283) + | > current_lr: 0.00002 + | > step_time: 9.50370 (3.44624) + | > loader_time: 0.00710 (0.06497) + + + --> STEP: 222/234 -- GLOBAL_STEP: 22920 + | > loss: -0.22302 (-0.08938) + | > log_mle: -0.51562 (-0.28944) + | > loss_dur: 0.29260 (0.20005) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 102.63349 (28.69176) + | > current_lr: 0.00002 + | > step_time: 1.49080 (3.44963) + | > loader_time: 0.00460 (0.06360) + + + --> STEP: 227/234 -- GLOBAL_STEP: 22925 + | > loss: -0.21315 (-0.09278) + | > log_mle: -0.49269 (-0.29455) + | > loss_dur: 0.27954 (0.20177) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 77.07787 (29.68573) + | > current_lr: 0.00002 + | > step_time: 0.26380 (3.39138) + | > loader_time: 0.00240 (0.06266) + + + --> STEP: 232/234 -- GLOBAL_STEP: 22930 + | > loss: -0.12808 (-0.09507) + | > log_mle: -0.68324 (-0.30095) + | > loss_dur: 0.55516 (0.20588) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 121.61678 (31.04181) + | > current_lr: 0.00002 + | > step_time: 0.35090 (3.32456) + | > loader_time: 0.00430 (0.06144) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.25779 (-0.59063) + | > avg_loss: -0.13902 (+0.00288) + | > avg_log_mle: -0.38000 (+0.00146) + | > avg_loss_dur: 0.24098 (+0.00143) + + + > EPOCH: 98/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 05:53:58)  + + --> STEP: 3/234 -- GLOBAL_STEP: 22935 + | > loss: 0.02796 (0.00407) + | > log_mle: -0.18308 (-0.17704) + | > loss_dur: 0.21104 (0.18112) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.22841 (8.56650) + | > current_lr: 0.00002 + | > step_time: 6.50180 (4.40343) + | > loader_time: 0.09480 (0.03404) + + + --> STEP: 8/234 -- GLOBAL_STEP: 22940 + | > loss: -0.00761 (-0.00243) + | > log_mle: -0.19284 (-0.18188) + | > loss_dur: 0.18523 (0.17946) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.33054 (8.46313) + | > current_lr: 0.00002 + | > step_time: 9.91130 (5.59113) + | > loader_time: 0.18350 (0.05975) + + + --> STEP: 13/234 -- GLOBAL_STEP: 22945 + | > loss: 0.00621 (-0.00715) + | > log_mle: -0.17341 (-0.18195) + | > loss_dur: 0.17962 (0.17480) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.80176 (8.45083) + | > current_lr: 0.00002 + | > step_time: 5.59890 (5.73998) + | > loader_time: 0.09870 (0.04513) + + + --> STEP: 18/234 -- GLOBAL_STEP: 22950 + | > loss: -0.01551 (-0.01317) + | > log_mle: -0.17806 (-0.18055) + | > loss_dur: 0.16255 (0.16738) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.12862 (7.75543) + | > current_lr: 0.00002 + | > step_time: 2.71520 (5.10066) + | > loader_time: 0.18700 (0.05483) + + + --> STEP: 23/234 -- GLOBAL_STEP: 22955 + | > loss: -0.04268 (-0.01624) + | > log_mle: -0.18136 (-0.17895) + | > loss_dur: 0.13867 (0.16271) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.60239 (7.15616) + | > current_lr: 0.00002 + | > step_time: 2.81290 (4.81447) + | > loader_time: 0.00230 (0.04344) + + + --> STEP: 28/234 -- GLOBAL_STEP: 22960 + | > loss: -0.04646 (-0.01902) + | > log_mle: -0.17041 (-0.17823) + | > loss_dur: 0.12395 (0.15920) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.30349 (6.90353) + | > current_lr: 0.00002 + | > step_time: 1.59030 (4.61632) + | > loader_time: 0.00210 (0.05291) + + + --> STEP: 33/234 -- GLOBAL_STEP: 22965 + | > loss: -0.00294 (-0.01993) + | > log_mle: -0.16817 (-0.17862) + | > loss_dur: 0.16523 (0.15869) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.37078 (6.85438) + | > current_lr: 0.00002 + | > step_time: 4.60220 (4.68315) + | > loader_time: 0.00260 (0.05096) + + + --> STEP: 38/234 -- GLOBAL_STEP: 22970 + | > loss: -0.01142 (-0.01931) + | > log_mle: -0.18758 (-0.17912) + | > loss_dur: 0.17617 (0.15981) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.27579 (7.01488) + | > current_lr: 0.00002 + | > step_time: 2.90030 (4.29138) + | > loader_time: 0.10350 (0.05210) + + + --> STEP: 43/234 -- GLOBAL_STEP: 22975 + | > loss: -0.01404 (-0.01906) + | > log_mle: -0.18653 (-0.17875) + | > loss_dur: 0.17249 (0.15969) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.84506 (6.95428) + | > current_lr: 0.00002 + | > step_time: 1.11510 (4.02225) + | > loader_time: 0.00180 (0.04647) + + + --> STEP: 48/234 -- GLOBAL_STEP: 22980 + | > loss: -0.03781 (-0.02026) + | > log_mle: -0.17096 (-0.17906) + | > loss_dur: 0.13315 (0.15880) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.07855 (6.92044) + | > current_lr: 0.00002 + | > step_time: 1.38440 (3.85703) + | > loader_time: 0.00140 (0.04382) + + + --> STEP: 53/234 -- GLOBAL_STEP: 22985 + | > loss: -0.02293 (-0.02032) + | > log_mle: -0.19267 (-0.17895) + | > loss_dur: 0.16974 (0.15863) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.59697 (6.68949) + | > current_lr: 0.00002 + | > step_time: 1.29530 (3.71807) + | > loader_time: 0.00290 (0.04500) + + + --> STEP: 58/234 -- GLOBAL_STEP: 22990 + | > loss: -0.03439 (-0.02048) + | > log_mle: -0.17568 (-0.17925) + | > loss_dur: 0.14129 (0.15878) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.96955 (6.57603) + | > current_lr: 0.00002 + | > step_time: 1.07190 (3.53777) + | > loader_time: 0.00200 (0.04263) + + + --> STEP: 63/234 -- GLOBAL_STEP: 22995 + | > loss: -0.00318 (-0.02146) + | > log_mle: -0.18745 (-0.18098) + | > loss_dur: 0.18427 (0.15953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.70635 (6.93020) + | > current_lr: 0.00002 + | > step_time: 3.94980 (3.41685) + | > loader_time: 0.00290 (0.03951) + + + --> STEP: 68/234 -- GLOBAL_STEP: 23000 + | > loss: -0.00296 (-0.02166) + | > log_mle: -0.18107 (-0.18106) + | > loss_dur: 0.17811 (0.15940) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.56029 (6.87052) + | > current_lr: 0.00002 + | > step_time: 1.78490 (3.28966) + | > loader_time: 0.00230 (0.03828) + + + --> STEP: 73/234 -- GLOBAL_STEP: 23005 + | > loss: -0.02846 (-0.02094) + | > log_mle: -0.20526 (-0.18164) + | > loss_dur: 0.17679 (0.16070) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.80871 (7.07122) + | > current_lr: 0.00002 + | > step_time: 2.31200 (3.23855) + | > loader_time: 0.08230 (0.03812) + + + --> STEP: 78/234 -- GLOBAL_STEP: 23010 + | > loss: -0.00857 (-0.02146) + | > log_mle: -0.17706 (-0.18215) + | > loss_dur: 0.16849 (0.16070) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.29890 (7.24447) + | > current_lr: 0.00002 + | > step_time: 2.39180 (3.17608) + | > loader_time: 0.00270 (0.03810) + + + --> STEP: 83/234 -- GLOBAL_STEP: 23015 + | > loss: -0.02412 (-0.02229) + | > log_mle: -0.20720 (-0.18282) + | > loss_dur: 0.18308 (0.16053) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.08738 (7.34780) + | > current_lr: 0.00002 + | > step_time: 3.21680 (3.21746) + | > loader_time: 0.08540 (0.03935) + + + --> STEP: 88/234 -- GLOBAL_STEP: 23020 + | > loss: -0.05956 (-0.02314) + | > log_mle: -0.24258 (-0.18413) + | > loss_dur: 0.18302 (0.16099) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.88997 (7.57353) + | > current_lr: 0.00002 + | > step_time: 3.50870 (3.15926) + | > loader_time: 0.07120 (0.03912) + + + --> STEP: 93/234 -- GLOBAL_STEP: 23025 + | > loss: -0.06432 (-0.02481) + | > log_mle: -0.25670 (-0.18670) + | > loss_dur: 0.19238 (0.16188) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.88775 (7.94315) + | > current_lr: 0.00002 + | > step_time: 1.71060 (3.10683) + | > loader_time: 0.08310 (0.04065) + + + --> STEP: 98/234 -- GLOBAL_STEP: 23030 + | > loss: -0.01790 (-0.02659) + | > log_mle: -0.18410 (-0.18925) + | > loss_dur: 0.16620 (0.16266) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.42658 (8.28351) + | > current_lr: 0.00002 + | > step_time: 1.70640 (3.07393) + | > loader_time: 0.00390 (0.04152) + + + --> STEP: 103/234 -- GLOBAL_STEP: 23035 + | > loss: -0.07833 (-0.02849) + | > log_mle: -0.28080 (-0.19244) + | > loss_dur: 0.20248 (0.16395) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.08274 (8.95827) + | > current_lr: 0.00002 + | > step_time: 0.87770 (3.05751) + | > loader_time: 0.00270 (0.04143) + + + --> STEP: 108/234 -- GLOBAL_STEP: 23040 + | > loss: -0.04646 (-0.03021) + | > log_mle: -0.22385 (-0.19514) + | > loss_dur: 0.17739 (0.16493) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.75515 (9.29260) + | > current_lr: 0.00002 + | > step_time: 2.93440 (3.02953) + | > loader_time: 0.00350 (0.04196) + + + --> STEP: 113/234 -- GLOBAL_STEP: 23045 + | > loss: -0.08537 (-0.03166) + | > log_mle: -0.27416 (-0.19841) + | > loss_dur: 0.18879 (0.16675) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.64681 (9.97890) + | > current_lr: 0.00002 + | > step_time: 1.88840 (3.01008) + | > loader_time: 0.00340 (0.04187) + + + --> STEP: 118/234 -- GLOBAL_STEP: 23050 + | > loss: -0.04812 (-0.03264) + | > log_mle: -0.24246 (-0.20088) + | > loss_dur: 0.19434 (0.16824) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.77798 (10.33728) + | > current_lr: 0.00002 + | > step_time: 1.54650 (2.97708) + | > loader_time: 0.00740 (0.04272) + + + --> STEP: 123/234 -- GLOBAL_STEP: 23055 + | > loss: -0.05028 (-0.03365) + | > log_mle: -0.21478 (-0.20243) + | > loss_dur: 0.16450 (0.16878) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.03026 (10.50374) + | > current_lr: 0.00002 + | > step_time: 5.70030 (2.98474) + | > loader_time: 0.00320 (0.04118) + + + --> STEP: 128/234 -- GLOBAL_STEP: 23060 + | > loss: -0.09249 (-0.03602) + | > log_mle: -0.27374 (-0.20591) + | > loss_dur: 0.18125 (0.16989) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.34266 (11.13664) + | > current_lr: 0.00002 + | > step_time: 2.49060 (3.01550) + | > loader_time: 0.00300 (0.04057) + + + --> STEP: 133/234 -- GLOBAL_STEP: 23065 + | > loss: -0.08557 (-0.03815) + | > log_mle: -0.29929 (-0.20942) + | > loss_dur: 0.21372 (0.17128) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.11667 (11.65731) + | > current_lr: 0.00002 + | > step_time: 2.41300 (3.01505) + | > loader_time: 0.09030 (0.04108) + + + --> STEP: 138/234 -- GLOBAL_STEP: 23070 + | > loss: -0.06412 (-0.03991) + | > log_mle: -0.25556 (-0.21275) + | > loss_dur: 0.19144 (0.17284) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.60473 (12.13770) + | > current_lr: 0.00002 + | > step_time: 5.50820 (3.01256) + | > loader_time: 0.07900 (0.04149) + + + --> STEP: 143/234 -- GLOBAL_STEP: 23075 + | > loss: -0.13525 (-0.04229) + | > log_mle: -0.38949 (-0.21685) + | > loss_dur: 0.25424 (0.17456) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.27105 (12.89574) + | > current_lr: 0.00002 + | > step_time: 4.89570 (3.04866) + | > loader_time: 0.00260 (0.04135) + + + --> STEP: 148/234 -- GLOBAL_STEP: 23080 + | > loss: -0.12093 (-0.04517) + | > log_mle: -0.30895 (-0.22103) + | > loss_dur: 0.18803 (0.17586) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.60667 (13.52883) + | > current_lr: 0.00002 + | > step_time: 3.39770 (3.03417) + | > loader_time: 0.00610 (0.04076) + + + --> STEP: 153/234 -- GLOBAL_STEP: 23085 + | > loss: -0.19286 (-0.04856) + | > log_mle: -0.42412 (-0.22606) + | > loss_dur: 0.23125 (0.17750) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.71326 (14.76870) + | > current_lr: 0.00002 + | > step_time: 3.70600 (3.09854) + | > loader_time: 0.00260 (0.04275) + + + --> STEP: 158/234 -- GLOBAL_STEP: 23090 + | > loss: -0.12982 (-0.05153) + | > log_mle: -0.36608 (-0.23054) + | > loss_dur: 0.23626 (0.17901) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.83303 (15.83129) + | > current_lr: 0.00002 + | > step_time: 9.63710 (3.16455) + | > loader_time: 0.28790 (0.04334) + + + --> STEP: 163/234 -- GLOBAL_STEP: 23095 + | > loss: -0.11789 (-0.05468) + | > log_mle: -0.33579 (-0.23497) + | > loss_dur: 0.21790 (0.18029) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.81118 (16.72495) + | > current_lr: 0.00002 + | > step_time: 3.58800 (3.16543) + | > loader_time: 0.00310 (0.04456) + + + --> STEP: 168/234 -- GLOBAL_STEP: 23100 + | > loss: -0.13908 (-0.05769) + | > log_mle: -0.39332 (-0.23957) + | > loss_dur: 0.25424 (0.18187) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.73981 (17.51089) + | > current_lr: 0.00002 + | > step_time: 5.08880 (3.20863) + | > loader_time: 0.00450 (0.04452) + + + --> STEP: 173/234 -- GLOBAL_STEP: 23105 + | > loss: -0.16838 (-0.06091) + | > log_mle: -0.39813 (-0.24457) + | > loss_dur: 0.22976 (0.18366) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.77899 (18.78982) + | > current_lr: 0.00002 + | > step_time: 2.18330 (3.19111) + | > loader_time: 0.00500 (0.04498) + + + --> STEP: 178/234 -- GLOBAL_STEP: 23110 + | > loss: -0.17843 (-0.06414) + | > log_mle: -0.44209 (-0.24952) + | > loss_dur: 0.26366 (0.18538) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 91.21174 (20.13769) + | > current_lr: 0.00002 + | > step_time: 6.30850 (3.20826) + | > loader_time: 0.08360 (0.04627) + + + --> STEP: 183/234 -- GLOBAL_STEP: 23115 + | > loss: -0.20540 (-0.06680) + | > log_mle: -0.44805 (-0.25400) + | > loss_dur: 0.24265 (0.18719) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.34886 (21.27619) + | > current_lr: 0.00002 + | > step_time: 4.70240 (3.23472) + | > loader_time: 0.00900 (0.04633) + + + --> STEP: 188/234 -- GLOBAL_STEP: 23120 + | > loss: -0.20660 (-0.06970) + | > log_mle: -0.46134 (-0.25862) + | > loss_dur: 0.25474 (0.18892) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.49120 (22.20321) + | > current_lr: 0.00002 + | > step_time: 3.39630 (3.26097) + | > loader_time: 0.09090 (0.04672) + + + --> STEP: 193/234 -- GLOBAL_STEP: 23125 + | > loss: -0.22238 (-0.07291) + | > log_mle: -0.46531 (-0.26314) + | > loss_dur: 0.24293 (0.19023) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.81594 (23.17628) + | > current_lr: 0.00002 + | > step_time: 6.39190 (3.27847) + | > loader_time: 0.10250 (0.04769) + + + --> STEP: 198/234 -- GLOBAL_STEP: 23130 + | > loss: -0.20628 (-0.07582) + | > log_mle: -0.45682 (-0.26751) + | > loss_dur: 0.25054 (0.19169) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.63795 (23.98614) + | > current_lr: 0.00002 + | > step_time: 6.70040 (3.39462) + | > loader_time: 0.19910 (0.05000) + + + --> STEP: 203/234 -- GLOBAL_STEP: 23135 + | > loss: -0.15755 (-0.07839) + | > log_mle: -0.39440 (-0.27167) + | > loss_dur: 0.23685 (0.19328) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.70743 (24.78930) + | > current_lr: 0.00002 + | > step_time: 6.50650 (3.40655) + | > loader_time: 0.09660 (0.04935) + + + --> STEP: 208/234 -- GLOBAL_STEP: 23140 + | > loss: -0.19694 (-0.08152) + | > log_mle: -0.46276 (-0.27642) + | > loss_dur: 0.26582 (0.19490) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.85969 (25.84987) + | > current_lr: 0.00002 + | > step_time: 7.00470 (3.46803) + | > loader_time: 0.09650 (0.05194) + + + --> STEP: 213/234 -- GLOBAL_STEP: 23145 + | > loss: -0.23204 (-0.08498) + | > log_mle: -0.51026 (-0.28161) + | > loss_dur: 0.27822 (0.19663) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.21690 (26.83838) + | > current_lr: 0.00002 + | > step_time: 5.51280 (3.51871) + | > loader_time: 0.10030 (0.05294) + + + --> STEP: 218/234 -- GLOBAL_STEP: 23150 + | > loss: -0.20556 (-0.08811) + | > log_mle: -0.47494 (-0.28626) + | > loss_dur: 0.26937 (0.19815) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.38489 (28.03131) + | > current_lr: 0.00002 + | > step_time: 5.49640 (3.52596) + | > loader_time: 0.00460 (0.05253) + + + --> STEP: 223/234 -- GLOBAL_STEP: 23155 + | > loss: -0.24685 (-0.09155) + | > log_mle: -0.51946 (-0.29133) + | > loss_dur: 0.27261 (0.19978) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.30038 (28.94995) + | > current_lr: 0.00002 + | > step_time: 0.54100 (3.56369) + | > loader_time: 0.00340 (0.05146) + + + --> STEP: 228/234 -- GLOBAL_STEP: 23160 + | > loss: -0.22198 (-0.09495) + | > log_mle: -0.51838 (-0.29660) + | > loss_dur: 0.29640 (0.20165) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 91.18686 (30.04946) + | > current_lr: 0.00002 + | > step_time: 0.25480 (3.49093) + | > loader_time: 0.00690 (0.05044) + + + --> STEP: 233/234 -- GLOBAL_STEP: 23165 + | > loss: 0.37215 (-0.09477) + | > log_mle: -0.47457 (-0.30290) + | > loss_dur: 0.84673 (0.20813) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 100.07848 (31.46278) + | > current_lr: 0.00002 + | > step_time: 0.19170 (3.42167) + | > loader_time: 0.00370 (0.04946) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.82649 (+0.56870) + | > avg_loss: -0.12956 (+0.00946) + | > avg_log_mle: -0.36964 (+0.01036) + | > avg_loss_dur: 0.24008 (-0.00090) + + + > EPOCH: 99/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 06:08:43)  + + --> STEP: 4/234 -- GLOBAL_STEP: 23170 + | > loss: 0.01307 (0.00305) + | > log_mle: -0.18655 (-0.18181) + | > loss_dur: 0.19963 (0.18485) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.96455 (10.10852) + | > current_lr: 0.00002 + | > step_time: 11.00090 (7.88041) + | > loader_time: 0.10200 (0.04827) + + + --> STEP: 9/234 -- GLOBAL_STEP: 23175 + | > loss: -0.01419 (-0.00873) + | > log_mle: -0.19512 (-0.18587) + | > loss_dur: 0.18094 (0.17714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.75578 (8.06890) + | > current_lr: 0.00002 + | > step_time: 1.00480 (5.95715) + | > loader_time: 0.07930 (0.14104) + + + --> STEP: 14/234 -- GLOBAL_STEP: 23180 + | > loss: -0.02451 (-0.01319) + | > log_mle: -0.18754 (-0.18462) + | > loss_dur: 0.16302 (0.17143) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.75015 (7.54312) + | > current_lr: 0.00002 + | > step_time: 1.79220 (4.39031) + | > loader_time: 0.00620 (0.09315) + + + --> STEP: 19/234 -- GLOBAL_STEP: 23185 + | > loss: -0.03098 (-0.01714) + | > log_mle: -0.17170 (-0.18225) + | > loss_dur: 0.14073 (0.16511) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.41356 (7.09663) + | > current_lr: 0.00002 + | > step_time: 1.19520 (3.95699) + | > loader_time: 0.00150 (0.07866) + + + --> STEP: 24/234 -- GLOBAL_STEP: 23190 + | > loss: -0.04460 (-0.02058) + | > log_mle: -0.17412 (-0.18075) + | > loss_dur: 0.12951 (0.16017) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.88251 (6.82235) + | > current_lr: 0.00002 + | > step_time: 0.86510 (3.51731) + | > loader_time: 0.00130 (0.06299) + + + --> STEP: 29/234 -- GLOBAL_STEP: 23195 + | > loss: -0.01291 (-0.02177) + | > log_mle: -0.16541 (-0.17957) + | > loss_dur: 0.15250 (0.15780) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.57196 (6.88847) + | > current_lr: 0.00002 + | > step_time: 1.60170 (3.36269) + | > loader_time: 0.00210 (0.06218) + + + --> STEP: 34/234 -- GLOBAL_STEP: 23200 + | > loss: -0.01308 (-0.02183) + | > log_mle: -0.17819 (-0.18002) + | > loss_dur: 0.16511 (0.15820) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.26605 (6.86117) + | > current_lr: 0.00002 + | > step_time: 2.92270 (3.41888) + | > loader_time: 0.09920 (0.06756) + + + --> STEP: 39/234 -- GLOBAL_STEP: 23205 + | > loss: -0.02310 (-0.02231) + | > log_mle: -0.18703 (-0.18061) + | > loss_dur: 0.16393 (0.15830) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.03584 (7.17024) + | > current_lr: 0.00002 + | > step_time: 1.88620 (3.23861) + | > loader_time: 0.00270 (0.06135) + + + --> STEP: 44/234 -- GLOBAL_STEP: 23210 + | > loss: -0.03658 (-0.02189) + | > log_mle: -0.17640 (-0.18017) + | > loss_dur: 0.13982 (0.15827) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.53823 (7.02370) + | > current_lr: 0.00002 + | > step_time: 2.20860 (3.11907) + | > loader_time: 0.06790 (0.05807) + + + --> STEP: 49/234 -- GLOBAL_STEP: 23215 + | > loss: -0.04191 (-0.02325) + | > log_mle: -0.18503 (-0.18073) + | > loss_dur: 0.14312 (0.15747) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.71384 (6.94152) + | > current_lr: 0.00002 + | > step_time: 2.69870 (3.01990) + | > loader_time: 0.00160 (0.05576) + + + --> STEP: 54/234 -- GLOBAL_STEP: 23220 + | > loss: -0.04251 (-0.02349) + | > log_mle: -0.19219 (-0.18076) + | > loss_dur: 0.14968 (0.15727) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.05239 (6.74108) + | > current_lr: 0.00002 + | > step_time: 5.60710 (2.97341) + | > loader_time: 0.18410 (0.05730) + + + --> STEP: 59/234 -- GLOBAL_STEP: 23225 + | > loss: -0.05629 (-0.02356) + | > log_mle: -0.20439 (-0.18130) + | > loss_dur: 0.14810 (0.15774) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.12587 (6.66685) + | > current_lr: 0.00002 + | > step_time: 1.20390 (2.87592) + | > loader_time: 0.00480 (0.05271) + + + --> STEP: 64/234 -- GLOBAL_STEP: 23230 + | > loss: -0.03654 (-0.02407) + | > log_mle: -0.17252 (-0.18241) + | > loss_dur: 0.13598 (0.15834) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.80871 (6.92982) + | > current_lr: 0.00002 + | > step_time: 1.40090 (2.84321) + | > loader_time: 0.01960 (0.05174) + + + --> STEP: 69/234 -- GLOBAL_STEP: 23235 + | > loss: 0.00134 (-0.02379) + | > log_mle: -0.16259 (-0.18237) + | > loss_dur: 0.16393 (0.15858) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.82060 (6.86204) + | > current_lr: 0.00002 + | > step_time: 2.49700 (2.81266) + | > loader_time: 0.00300 (0.04830) + + + --> STEP: 74/234 -- GLOBAL_STEP: 23240 + | > loss: -0.04359 (-0.02381) + | > log_mle: -0.18239 (-0.18313) + | > loss_dur: 0.13880 (0.15933) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.87160 (7.18140) + | > current_lr: 0.00002 + | > step_time: 1.99380 (2.73872) + | > loader_time: 0.00360 (0.04757) + + + --> STEP: 79/234 -- GLOBAL_STEP: 23245 + | > loss: -0.04117 (-0.02451) + | > log_mle: -0.19519 (-0.18378) + | > loss_dur: 0.15402 (0.15928) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.29321 (7.26243) + | > current_lr: 0.00002 + | > step_time: 2.90060 (2.66956) + | > loader_time: 0.08680 (0.04673) + + + --> STEP: 84/234 -- GLOBAL_STEP: 23250 + | > loss: -0.02842 (-0.02520) + | > log_mle: -0.19298 (-0.18442) + | > loss_dur: 0.16455 (0.15922) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.90147 (7.34506) + | > current_lr: 0.00002 + | > step_time: 3.39240 (2.66639) + | > loader_time: 0.09410 (0.04610) + + + --> STEP: 89/234 -- GLOBAL_STEP: 23255 + | > loss: -0.05320 (-0.02607) + | > log_mle: -0.22337 (-0.18605) + | > loss_dur: 0.17016 (0.15998) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.39553 (7.55073) + | > current_lr: 0.00002 + | > step_time: 3.89010 (2.61291) + | > loader_time: 0.10230 (0.04479) + + + --> STEP: 94/234 -- GLOBAL_STEP: 23260 + | > loss: -0.08246 (-0.02807) + | > log_mle: -0.25916 (-0.18892) + | > loss_dur: 0.17670 (0.16086) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.91921 (7.98063) + | > current_lr: 0.00002 + | > step_time: 5.80680 (2.61593) + | > loader_time: 0.10560 (0.04543) + + + --> STEP: 99/234 -- GLOBAL_STEP: 23265 + | > loss: -0.09650 (-0.02986) + | > log_mle: -0.29067 (-0.19172) + | > loss_dur: 0.19417 (0.16186) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.41985 (8.57666) + | > current_lr: 0.00002 + | > step_time: 2.10250 (2.57748) + | > loader_time: 0.00350 (0.04327) + + + --> STEP: 104/234 -- GLOBAL_STEP: 23270 + | > loss: -0.11244 (-0.03176) + | > log_mle: -0.30285 (-0.19500) + | > loss_dur: 0.19041 (0.16324) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.88578 (9.06440) + | > current_lr: 0.00002 + | > step_time: 1.50450 (2.55857) + | > loader_time: 0.00350 (0.04314) + + + --> STEP: 109/234 -- GLOBAL_STEP: 23275 + | > loss: -0.03673 (-0.03263) + | > log_mle: -0.27174 (-0.19738) + | > loss_dur: 0.23502 (0.16475) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.26024 (9.58875) + | > current_lr: 0.00002 + | > step_time: 3.29340 (2.56250) + | > loader_time: 0.00680 (0.04216) + + + --> STEP: 114/234 -- GLOBAL_STEP: 23280 + | > loss: -0.06603 (-0.03423) + | > log_mle: -0.25488 (-0.20048) + | > loss_dur: 0.18884 (0.16625) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.26046 (10.21855) + | > current_lr: 0.00002 + | > step_time: 2.31630 (2.56089) + | > loader_time: 0.00530 (0.04279) + + + --> STEP: 119/234 -- GLOBAL_STEP: 23285 + | > loss: -0.06476 (-0.03509) + | > log_mle: -0.25295 (-0.20295) + | > loss_dur: 0.18819 (0.16786) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.12136 (10.48750) + | > current_lr: 0.00002 + | > step_time: 3.48530 (2.55231) + | > loader_time: 0.00430 (0.04334) + + + --> STEP: 124/234 -- GLOBAL_STEP: 23290 + | > loss: -0.09764 (-0.03618) + | > log_mle: -0.28181 (-0.20474) + | > loss_dur: 0.18417 (0.16855) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.06897 (10.74955) + | > current_lr: 0.00002 + | > step_time: 0.89260 (2.52593) + | > loader_time: 0.00210 (0.04318) + + + --> STEP: 129/234 -- GLOBAL_STEP: 23295 + | > loss: -0.06808 (-0.03824) + | > log_mle: -0.26984 (-0.20807) + | > loss_dur: 0.20175 (0.16983) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.46525 (11.41758) + | > current_lr: 0.00002 + | > step_time: 2.69860 (2.51030) + | > loader_time: 0.00340 (0.04230) + + + --> STEP: 134/234 -- GLOBAL_STEP: 23300 + | > loss: -0.10039 (-0.04066) + | > log_mle: -0.32096 (-0.21198) + | > loss_dur: 0.22057 (0.17132) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.41947 (12.00670) + | > current_lr: 0.00002 + | > step_time: 1.43660 (2.49130) + | > loader_time: 0.00260 (0.04085) + + + --> STEP: 139/234 -- GLOBAL_STEP: 23305 + | > loss: -0.17512 (-0.04290) + | > log_mle: -0.38283 (-0.21564) + | > loss_dur: 0.20771 (0.17273) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.89268 (12.75803) + | > current_lr: 0.00002 + | > step_time: 1.70230 (2.50887) + | > loader_time: 0.00350 (0.04146) + + + --> STEP: 144/234 -- GLOBAL_STEP: 23310 + | > loss: -0.12404 (-0.04482) + | > log_mle: -0.35561 (-0.21946) + | > loss_dur: 0.23157 (0.17464) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.95141 (13.66843) + | > current_lr: 0.00002 + | > step_time: 1.48830 (2.48642) + | > loader_time: 0.00410 (0.04073) + + + --> STEP: 149/234 -- GLOBAL_STEP: 23315 + | > loss: -0.15630 (-0.04773) + | > log_mle: -0.39205 (-0.22374) + | > loss_dur: 0.23575 (0.17601) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.58353 (14.62725) + | > current_lr: 0.00002 + | > step_time: 2.59480 (2.50616) + | > loader_time: 0.00470 (0.04140) + + + --> STEP: 154/234 -- GLOBAL_STEP: 23320 + | > loss: -0.14577 (-0.05101) + | > log_mle: -0.35883 (-0.22846) + | > loss_dur: 0.21307 (0.17746) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.16501 (15.54677) + | > current_lr: 0.00002 + | > step_time: 1.59100 (2.57537) + | > loader_time: 0.00840 (0.04260) + + + --> STEP: 159/234 -- GLOBAL_STEP: 23325 + | > loss: -0.15436 (-0.05398) + | > log_mle: -0.37135 (-0.23293) + | > loss_dur: 0.21699 (0.17894) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.16196 (16.61619) + | > current_lr: 0.00002 + | > step_time: 2.57800 (2.56932) + | > loader_time: 0.32000 (0.04430) + + + --> STEP: 164/234 -- GLOBAL_STEP: 23330 + | > loss: -0.14321 (-0.05690) + | > log_mle: -0.37011 (-0.23712) + | > loss_dur: 0.22690 (0.18022) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.72908 (17.60143) + | > current_lr: 0.00002 + | > step_time: 2.61480 (2.56006) + | > loader_time: 0.08660 (0.04511) + + + --> STEP: 169/234 -- GLOBAL_STEP: 23335 + | > loss: -0.12660 (-0.05973) + | > log_mle: -0.36601 (-0.24152) + | > loss_dur: 0.23941 (0.18179) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.73689 (18.37648) + | > current_lr: 0.00002 + | > step_time: 2.00750 (2.58151) + | > loader_time: 0.38560 (0.04774) + + + --> STEP: 174/234 -- GLOBAL_STEP: 23340 + | > loss: -0.21336 (-0.06341) + | > log_mle: -0.44994 (-0.24688) + | > loss_dur: 0.23658 (0.18347) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 67.28226 (19.84347) + | > current_lr: 0.00002 + | > step_time: 3.60890 (2.59048) + | > loader_time: 0.09380 (0.04819) + + + --> STEP: 179/234 -- GLOBAL_STEP: 23345 + | > loss: -0.18078 (-0.06648) + | > log_mle: -0.44714 (-0.25178) + | > loss_dur: 0.26636 (0.18530) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.70915 (20.80871) + | > current_lr: 0.00002 + | > step_time: 5.20570 (2.62619) + | > loader_time: 0.08660 (0.05007) + + + --> STEP: 184/234 -- GLOBAL_STEP: 23350 + | > loss: -0.17707 (-0.06934) + | > log_mle: -0.41542 (-0.25624) + | > loss_dur: 0.23835 (0.18690) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.68491 (21.64102) + | > current_lr: 0.00002 + | > step_time: 5.51700 (2.65926) + | > loader_time: 0.19930 (0.05091) + + + --> STEP: 189/234 -- GLOBAL_STEP: 23355 + | > loss: -0.15689 (-0.07220) + | > log_mle: -0.41950 (-0.26093) + | > loss_dur: 0.26261 (0.18873) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.25710 (22.51229) + | > current_lr: 0.00002 + | > step_time: 1.49520 (2.66720) + | > loader_time: 0.00660 (0.05218) + + + --> STEP: 194/234 -- GLOBAL_STEP: 23360 + | > loss: -0.20088 (-0.07556) + | > log_mle: -0.44441 (-0.26559) + | > loss_dur: 0.24353 (0.19003) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 77.98501 (23.58522) + | > current_lr: 0.00002 + | > step_time: 9.88270 (2.76122) + | > loader_time: 0.00830 (0.05141) + + + --> STEP: 199/234 -- GLOBAL_STEP: 23365 + | > loss: -0.19837 (-0.07829) + | > log_mle: -0.44859 (-0.26980) + | > loss_dur: 0.25022 (0.19151) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.15151 (24.60895) + | > current_lr: 0.00002 + | > step_time: 3.60400 (2.82061) + | > loader_time: 0.00690 (0.05154) + + + --> STEP: 204/234 -- GLOBAL_STEP: 23370 + | > loss: -0.22289 (-0.08097) + | > log_mle: -0.48810 (-0.27404) + | > loss_dur: 0.26521 (0.19307) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.59418 (25.39319) + | > current_lr: 0.00002 + | > step_time: 3.61760 (2.91759) + | > loader_time: 0.09830 (0.05143) + + + --> STEP: 209/234 -- GLOBAL_STEP: 23375 + | > loss: -0.18832 (-0.08408) + | > log_mle: -0.44647 (-0.27865) + | > loss_dur: 0.25815 (0.19457) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.51421 (26.09501) + | > current_lr: 0.00002 + | > step_time: 4.08290 (2.96863) + | > loader_time: 0.00590 (0.05266) + + + --> STEP: 214/234 -- GLOBAL_STEP: 23380 + | > loss: -0.24014 (-0.08771) + | > log_mle: -0.48044 (-0.28390) + | > loss_dur: 0.24030 (0.19620) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.72785 (27.10881) + | > current_lr: 0.00002 + | > step_time: 9.41380 (3.09846) + | > loader_time: 0.20330 (0.05463) + + + --> STEP: 219/234 -- GLOBAL_STEP: 23385 + | > loss: -0.29738 (-0.09122) + | > log_mle: -0.56919 (-0.28902) + | > loss_dur: 0.27181 (0.19780) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 81.35728 (28.00674) + | > current_lr: 0.00002 + | > step_time: 2.50650 (3.14502) + | > loader_time: 0.09690 (0.05555) + + + --> STEP: 224/234 -- GLOBAL_STEP: 23390 + | > loss: -0.24992 (-0.09429) + | > log_mle: -0.52502 (-0.29379) + | > loss_dur: 0.27510 (0.19950) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 81.59161 (29.00978) + | > current_lr: 0.00002 + | > step_time: 0.23720 (3.11214) + | > loader_time: 0.00550 (0.05602) + + + --> STEP: 229/234 -- GLOBAL_STEP: 23395 + | > loss: -0.21591 (-0.09739) + | > log_mle: -0.55252 (-0.29903) + | > loss_dur: 0.33661 (0.20163) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 95.65428 (30.35419) + | > current_lr: 0.00002 + | > step_time: 0.24590 (3.04952) + | > loader_time: 0.00380 (0.05487) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05436 (-0.77214) + | > avg_loss: -0.11974 (+0.00982) + | > avg_log_mle: -0.36111 (+0.00853) + | > avg_loss_dur: 0.24137 (+0.00129) + + + > EPOCH: 100/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 06:21:42)  + + --> STEP: 0/234 -- GLOBAL_STEP: 23400 + | > loss: -0.05842 (-0.05842) + | > log_mle: -0.24025 (-0.24025) + | > loss_dur: 0.18183 (0.18183) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.90860 (9.90860) + | > current_lr: 0.00002 + | > step_time: 10.40400 (10.40397) + | > loader_time: 10.54050 (10.54053) + + + --> STEP: 5/234 -- GLOBAL_STEP: 23405 + | > loss: -0.01692 (-0.01182) + | > log_mle: -0.18678 (-0.18261) + | > loss_dur: 0.16986 (0.17079) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.91302 (10.56566) + | > current_lr: 0.00002 + | > step_time: 3.91670 (4.12253) + | > loader_time: 0.08370 (0.03795) + + + --> STEP: 10/234 -- GLOBAL_STEP: 23410 + | > loss: -0.01819 (-0.01752) + | > log_mle: -0.18734 (-0.18663) + | > loss_dur: 0.16914 (0.16911) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.81066 (8.74592) + | > current_lr: 0.00002 + | > step_time: 9.10730 (4.24227) + | > loader_time: 0.20450 (0.05940) + + + --> STEP: 15/234 -- GLOBAL_STEP: 23415 + | > loss: -0.04277 (-0.01986) + | > log_mle: -0.18673 (-0.18569) + | > loss_dur: 0.14397 (0.16583) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.56064 (7.86944) + | > current_lr: 0.00002 + | > step_time: 6.30550 (4.59658) + | > loader_time: 0.00160 (0.04027) + + + --> STEP: 20/234 -- GLOBAL_STEP: 23420 + | > loss: -0.03262 (-0.02311) + | > log_mle: -0.17425 (-0.18325) + | > loss_dur: 0.14163 (0.16013) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.18459 (7.18560) + | > current_lr: 0.00002 + | > step_time: 2.51500 (4.44823) + | > loader_time: 0.07610 (0.04342) + + + --> STEP: 25/234 -- GLOBAL_STEP: 23425 + | > loss: -0.01377 (-0.02551) + | > log_mle: -0.16828 (-0.18197) + | > loss_dur: 0.15451 (0.15646) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.73896 (6.87255) + | > current_lr: 0.00002 + | > step_time: 1.20030 (4.46743) + | > loader_time: 0.18740 (0.05371) + + + --> STEP: 30/234 -- GLOBAL_STEP: 23430 + | > loss: -0.05561 (-0.02843) + | > log_mle: -0.19437 (-0.18201) + | > loss_dur: 0.13876 (0.15359) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.80695 (6.65773) + | > current_lr: 0.00002 + | > step_time: 4.11050 (4.34643) + | > loader_time: 0.07840 (0.05389) + + + --> STEP: 35/234 -- GLOBAL_STEP: 23435 + | > loss: -0.02116 (-0.02716) + | > log_mle: -0.18898 (-0.18247) + | > loss_dur: 0.16782 (0.15532) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.33547 (6.71600) + | > current_lr: 0.00002 + | > step_time: 3.39680 (4.29230) + | > loader_time: 0.00340 (0.05145) + + + --> STEP: 40/234 -- GLOBAL_STEP: 23440 + | > loss: -0.00113 (-0.02642) + | > log_mle: -0.16919 (-0.18266) + | > loss_dur: 0.16806 (0.15624) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.01487 (6.74205) + | > current_lr: 0.00002 + | > step_time: 2.00110 (4.02536) + | > loader_time: 0.00380 (0.04530) + + + --> STEP: 45/234 -- GLOBAL_STEP: 23445 + | > loss: -0.05128 (-0.02724) + | > log_mle: -0.20751 (-0.18308) + | > loss_dur: 0.15623 (0.15585) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.47680 (6.68752) + | > current_lr: 0.00002 + | > step_time: 5.09280 (3.88452) + | > loader_time: 0.10720 (0.04891) + + + --> STEP: 50/234 -- GLOBAL_STEP: 23450 + | > loss: -0.01784 (-0.02742) + | > log_mle: -0.17352 (-0.18289) + | > loss_dur: 0.15568 (0.15546) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.99061 (6.50745) + | > current_lr: 0.00002 + | > step_time: 1.39950 (3.65177) + | > loader_time: 0.08170 (0.04588) + + + --> STEP: 55/234 -- GLOBAL_STEP: 23455 + | > loss: -0.05055 (-0.02805) + | > log_mle: -0.19374 (-0.18323) + | > loss_dur: 0.14319 (0.15518) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.47904 (6.40026) + | > current_lr: 0.00002 + | > step_time: 3.28710 (3.68529) + | > loader_time: 0.00380 (0.04691) + + + --> STEP: 60/234 -- GLOBAL_STEP: 23460 + | > loss: -0.05708 (-0.02849) + | > log_mle: -0.20956 (-0.18392) + | > loss_dur: 0.15248 (0.15543) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.11250 (6.49531) + | > current_lr: 0.00002 + | > step_time: 1.80840 (3.56726) + | > loader_time: 0.00470 (0.04639) + + + --> STEP: 65/234 -- GLOBAL_STEP: 23465 + | > loss: -0.04344 (-0.02838) + | > log_mle: -0.18410 (-0.18455) + | > loss_dur: 0.14067 (0.15617) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.82395 (6.64291) + | > current_lr: 0.00002 + | > step_time: 2.99700 (3.48967) + | > loader_time: 0.00250 (0.04623) + + + --> STEP: 70/234 -- GLOBAL_STEP: 23470 + | > loss: -0.01791 (-0.02770) + | > log_mle: -0.18432 (-0.18435) + | > loss_dur: 0.16641 (0.15666) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.09242 (6.76578) + | > current_lr: 0.00002 + | > step_time: 1.38530 (3.35774) + | > loader_time: 0.00250 (0.04596) + + + --> STEP: 75/234 -- GLOBAL_STEP: 23475 + | > loss: -0.02429 (-0.02721) + | > log_mle: -0.19281 (-0.18503) + | > loss_dur: 0.16852 (0.15781) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.77181 (7.03399) + | > current_lr: 0.00002 + | > step_time: 4.11220 (3.29016) + | > loader_time: 0.18500 (0.04730) + + + --> STEP: 80/234 -- GLOBAL_STEP: 23480 + | > loss: -0.04364 (-0.02764) + | > log_mle: -0.17846 (-0.18532) + | > loss_dur: 0.13482 (0.15767) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.61524 (7.02830) + | > current_lr: 0.00002 + | > step_time: 1.12060 (3.22235) + | > loader_time: 0.07720 (0.04661) + + + --> STEP: 85/234 -- GLOBAL_STEP: 23485 + | > loss: -0.03841 (-0.02831) + | > log_mle: -0.19524 (-0.18607) + | > loss_dur: 0.15684 (0.15776) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.23951 (7.16193) + | > current_lr: 0.00002 + | > step_time: 2.90580 (3.15611) + | > loader_time: 0.00550 (0.04415) + + + --> STEP: 90/234 -- GLOBAL_STEP: 23490 + | > loss: -0.04678 (-0.02936) + | > log_mle: -0.22497 (-0.18798) + | > loss_dur: 0.17819 (0.15862) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.27567 (7.48074) + | > current_lr: 0.00002 + | > step_time: 2.60440 (3.10969) + | > loader_time: 0.07690 (0.04549) + + + --> STEP: 95/234 -- GLOBAL_STEP: 23495 + | > loss: -0.10588 (-0.03161) + | > log_mle: -0.30775 (-0.19161) + | > loss_dur: 0.20188 (0.16001) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.04358 (8.14834) + | > current_lr: 0.00002 + | > step_time: 3.79550 (3.05064) + | > loader_time: 0.00220 (0.04606) + + + --> STEP: 100/234 -- GLOBAL_STEP: 23500 + | > loss: -0.06369 (-0.03264) + | > log_mle: -0.23548 (-0.19351) + | > loss_dur: 0.17179 (0.16087) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.47493 (8.47298) + | > current_lr: 0.00002 + | > step_time: 2.92840 (3.06456) + | > loader_time: 0.00250 (0.04849) + + + --> STEP: 105/234 -- GLOBAL_STEP: 23505 + | > loss: -0.05463 (-0.03439) + | > log_mle: -0.21052 (-0.19646) + | > loss_dur: 0.15589 (0.16207) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.56944 (8.96636) + | > current_lr: 0.00002 + | > step_time: 1.90810 (3.04025) + | > loader_time: 0.09120 (0.04866) + + + --> STEP: 110/234 -- GLOBAL_STEP: 23510 + | > loss: -0.06672 (-0.03529) + | > log_mle: -0.23706 (-0.19895) + | > loss_dur: 0.17035 (0.16366) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.81438 (9.54948) + | > current_lr: 0.00002 + | > step_time: 1.39230 (3.00752) + | > loader_time: 0.00410 (0.04721) + + + --> STEP: 115/234 -- GLOBAL_STEP: 23515 + | > loss: -0.04813 (-0.03691) + | > log_mle: -0.25555 (-0.20216) + | > loss_dur: 0.20742 (0.16524) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.24996 (10.03834) + | > current_lr: 0.00002 + | > step_time: 4.81740 (3.00052) + | > loader_time: 0.00330 (0.04769) + + + --> STEP: 120/234 -- GLOBAL_STEP: 23520 + | > loss: -0.10059 (-0.03827) + | > log_mle: -0.30623 (-0.20500) + | > loss_dur: 0.20565 (0.16673) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.19613 (10.35801) + | > current_lr: 0.00002 + | > step_time: 2.28030 (2.97750) + | > loader_time: 0.00300 (0.04816) + + + --> STEP: 125/234 -- GLOBAL_STEP: 23525 + | > loss: -0.08580 (-0.03927) + | > log_mle: -0.29123 (-0.20665) + | > loss_dur: 0.20543 (0.16738) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.90813 (10.69684) + | > current_lr: 0.00002 + | > step_time: 1.19310 (2.92916) + | > loader_time: 0.00240 (0.04768) + + + --> STEP: 130/234 -- GLOBAL_STEP: 23530 + | > loss: -0.09214 (-0.04127) + | > log_mle: -0.30365 (-0.20996) + | > loss_dur: 0.21151 (0.16869) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.53283 (11.53514) + | > current_lr: 0.00002 + | > step_time: 2.69610 (2.90488) + | > loader_time: 0.00830 (0.04666) + + + --> STEP: 135/234 -- GLOBAL_STEP: 23535 + | > loss: -0.06246 (-0.04333) + | > log_mle: -0.23673 (-0.21329) + | > loss_dur: 0.17427 (0.16996) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.30661 (12.09300) + | > current_lr: 0.00002 + | > step_time: 4.10730 (2.93584) + | > loader_time: 0.08420 (0.04628) + + + --> STEP: 140/234 -- GLOBAL_STEP: 23540 + | > loss: -0.06199 (-0.04560) + | > log_mle: -0.26922 (-0.21713) + | > loss_dur: 0.20723 (0.17153) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.50349 (12.87517) + | > current_lr: 0.00002 + | > step_time: 0.67800 (2.89284) + | > loader_time: 0.00270 (0.04541) + + + --> STEP: 145/234 -- GLOBAL_STEP: 23545 + | > loss: -0.14686 (-0.04810) + | > log_mle: -0.36780 (-0.22162) + | > loss_dur: 0.22093 (0.17353) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.31749 (13.73150) + | > current_lr: 0.00002 + | > step_time: 3.59290 (2.86751) + | > loader_time: 0.00670 (0.04464) + + + --> STEP: 150/234 -- GLOBAL_STEP: 23550 + | > loss: -0.11772 (-0.05070) + | > log_mle: -0.34423 (-0.22571) + | > loss_dur: 0.22652 (0.17501) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 53.16320 (14.76507) + | > current_lr: 0.00002 + | > step_time: 1.30280 (2.84323) + | > loader_time: 0.00310 (0.04379) + + + --> STEP: 155/234 -- GLOBAL_STEP: 23555 + | > loss: -0.17461 (-0.05421) + | > log_mle: -0.40918 (-0.23073) + | > loss_dur: 0.23457 (0.17651) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.23873 (15.87839) + | > current_lr: 0.00002 + | > step_time: 1.90010 (2.81906) + | > loader_time: 0.00440 (0.04254) + + + --> STEP: 160/234 -- GLOBAL_STEP: 23560 + | > loss: -0.16737 (-0.05708) + | > log_mle: -0.39715 (-0.23512) + | > loss_dur: 0.22978 (0.17804) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.99306 (16.81294) + | > current_lr: 0.00002 + | > step_time: 1.70670 (2.81121) + | > loader_time: 0.09180 (0.04237) + + + --> STEP: 165/234 -- GLOBAL_STEP: 23565 + | > loss: -0.15629 (-0.05993) + | > log_mle: -0.40515 (-0.23939) + | > loss_dur: 0.24886 (0.17945) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.08434 (17.62659) + | > current_lr: 0.00002 + | > step_time: 1.69380 (2.78427) + | > loader_time: 0.00360 (0.04213) + + + --> STEP: 170/234 -- GLOBAL_STEP: 23570 + | > loss: -0.17235 (-0.06297) + | > log_mle: -0.44684 (-0.24410) + | > loss_dur: 0.27450 (0.18113) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.61818 (18.32059) + | > current_lr: 0.00002 + | > step_time: 2.20210 (2.80026) + | > loader_time: 0.00280 (0.04478) + + + --> STEP: 175/234 -- GLOBAL_STEP: 23575 + | > loss: -0.15855 (-0.06649) + | > log_mle: -0.41684 (-0.24942) + | > loss_dur: 0.25829 (0.18294) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.88780 (19.40694) + | > current_lr: 0.00002 + | > step_time: 0.77990 (2.81976) + | > loader_time: 0.00250 (0.04449) + + + --> STEP: 180/234 -- GLOBAL_STEP: 23580 + | > loss: -0.18563 (-0.06977) + | > log_mle: -0.42847 (-0.25445) + | > loss_dur: 0.24284 (0.18468) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.37202 (20.23911) + | > current_lr: 0.00002 + | > step_time: 2.78690 (2.79655) + | > loader_time: 0.00450 (0.04379) + + + --> STEP: 185/234 -- GLOBAL_STEP: 23585 + | > loss: -0.18251 (-0.07252) + | > log_mle: -0.45167 (-0.25906) + | > loss_dur: 0.26917 (0.18654) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 67.02598 (21.15259) + | > current_lr: 0.00002 + | > step_time: 1.30070 (2.78207) + | > loader_time: 0.07560 (0.04368) + + + --> STEP: 190/234 -- GLOBAL_STEP: 23590 + | > loss: -0.18935 (-0.07545) + | > log_mle: -0.42504 (-0.26359) + | > loss_dur: 0.23569 (0.18814) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.74654 (22.06330) + | > current_lr: 0.00002 + | > step_time: 3.01270 (2.78319) + | > loader_time: 0.00310 (0.04440) + + + --> STEP: 195/234 -- GLOBAL_STEP: 23595 + | > loss: -0.18032 (-0.07870) + | > log_mle: -0.43853 (-0.26823) + | > loss_dur: 0.25822 (0.18953) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.66966 (23.19616) + | > current_lr: 0.00002 + | > step_time: 3.21140 (2.81647) + | > loader_time: 0.10950 (0.04496) + + + --> STEP: 200/234 -- GLOBAL_STEP: 23600 + | > loss: -0.18642 (-0.08144) + | > log_mle: -0.44857 (-0.27250) + | > loss_dur: 0.26215 (0.19106) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.01870 (24.21971) + | > current_lr: 0.00002 + | > step_time: 7.41490 (2.90109) + | > loader_time: 0.39360 (0.05087) + + + --> STEP: 205/234 -- GLOBAL_STEP: 23605 + | > loss: -0.18952 (-0.08406) + | > log_mle: -0.43986 (-0.27668) + | > loss_dur: 0.25034 (0.19262) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.83998 (24.99760) + | > current_lr: 0.00002 + | > step_time: 3.60010 (3.00207) + | > loader_time: 0.00520 (0.05165) + + + --> STEP: 210/234 -- GLOBAL_STEP: 23610 + | > loss: -0.25210 (-0.08733) + | > log_mle: -0.51853 (-0.28162) + | > loss_dur: 0.26643 (0.19429) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.45493 (25.90706) + | > current_lr: 0.00002 + | > step_time: 4.30110 (3.04240) + | > loader_time: 0.09820 (0.05234) + + + --> STEP: 215/234 -- GLOBAL_STEP: 23615 + | > loss: -0.21516 (-0.09085) + | > log_mle: -0.46570 (-0.28671) + | > loss_dur: 0.25053 (0.19586) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.83356 (26.81310) + | > current_lr: 0.00002 + | > step_time: 5.09720 (3.12567) + | > loader_time: 0.00230 (0.05299) + + + --> STEP: 220/234 -- GLOBAL_STEP: 23620 + | > loss: -0.24295 (-0.09448) + | > log_mle: -0.51173 (-0.29202) + | > loss_dur: 0.26878 (0.19754) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 92.48330 (28.21601) + | > current_lr: 0.00002 + | > step_time: 1.00360 (3.13379) + | > loader_time: 0.00420 (0.05310) + + + --> STEP: 225/234 -- GLOBAL_STEP: 23625 + | > loss: -0.27919 (-0.09774) + | > log_mle: -0.56533 (-0.29701) + | > loss_dur: 0.28614 (0.19927) + | > amp_scaler: 2048.00000 (4086.89778) + | > grad_norm: 0.00000 (29.11236) + | > current_lr: 0.00002 + | > step_time: 0.20930 (3.07147) + | > loader_time: 0.00310 (0.05235) + + + --> STEP: 230/234 -- GLOBAL_STEP: 23630 + | > loss: -0.26787 (-0.10066) + | > log_mle: -0.62295 (-0.30233) + | > loss_dur: 0.35509 (0.20168) + | > amp_scaler: 2048.00000 (4042.57391) + | > grad_norm: 87.72652 (30.88789) + | > current_lr: 0.00002 + | > step_time: 0.25320 (3.01008) + | > loader_time: 0.00520 (0.05130) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.77647 (+0.72211) + | > avg_loss: -0.13738 (-0.01764) + | > avg_log_mle: -0.38265 (-0.02154) + | > avg_loss_dur: 0.24527 (+0.00390) + + + > EPOCH: 101/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 06:34:32)  + + --> STEP: 1/234 -- GLOBAL_STEP: 23635 + | > loss: -0.06559 (-0.06559) + | > log_mle: -0.18809 (-0.18809) + | > loss_dur: 0.12249 (0.12249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.08425 (9.08425) + | > current_lr: 0.00003 + | > step_time: 2.69770 (2.69769) + | > loader_time: 0.00360 (0.00359) + + + --> STEP: 6/234 -- GLOBAL_STEP: 23640 + | > loss: -0.02886 (-0.01105) + | > log_mle: -0.17588 (-0.18216) + | > loss_dur: 0.14702 (0.17111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.81459 (11.19695) + | > current_lr: 0.00003 + | > step_time: 2.20300 (5.33722) + | > loader_time: 0.08420 (0.47849) + + + --> STEP: 11/234 -- GLOBAL_STEP: 23645 + | > loss: -0.04651 (-0.02403) + | > log_mle: -0.18073 (-0.18665) + | > loss_dur: 0.13422 (0.16262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.18369 (9.61233) + | > current_lr: 0.00003 + | > step_time: 5.19690 (4.85625) + | > loader_time: 0.20200 (0.28900) + + + --> STEP: 16/234 -- GLOBAL_STEP: 23650 + | > loss: -0.06555 (-0.03008) + | > log_mle: -0.18394 (-0.18627) + | > loss_dur: 0.11838 (0.15619) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.36441 (8.61606) + | > current_lr: 0.00003 + | > step_time: 3.09350 (5.16922) + | > loader_time: 0.00220 (0.22353) + + + --> STEP: 21/234 -- GLOBAL_STEP: 23655 + | > loss: -0.01089 (-0.02835) + | > log_mle: -0.16499 (-0.18282) + | > loss_dur: 0.15411 (0.15447) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.38154 (7.94752) + | > current_lr: 0.00003 + | > step_time: 3.71360 (4.80546) + | > loader_time: 0.00300 (0.17534) + + + --> STEP: 26/234 -- GLOBAL_STEP: 23660 + | > loss: -0.03120 (-0.02998) + | > log_mle: -0.18359 (-0.18242) + | > loss_dur: 0.15239 (0.15244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.71256 (7.42405) + | > current_lr: 0.00003 + | > step_time: 2.30630 (4.25540) + | > loader_time: 0.00200 (0.14827) + + + --> STEP: 31/234 -- GLOBAL_STEP: 23665 + | > loss: 0.01318 (-0.03046) + | > log_mle: -0.18652 (-0.18260) + | > loss_dur: 0.19970 (0.15214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.00736 (7.16958) + | > current_lr: 0.00003 + | > step_time: 6.48690 (4.38154) + | > loader_time: 0.19440 (0.14404) + + + --> STEP: 36/234 -- GLOBAL_STEP: 23670 + | > loss: -0.02624 (-0.03004) + | > log_mle: -0.18873 (-0.18305) + | > loss_dur: 0.16248 (0.15301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.22794 (7.25126) + | > current_lr: 0.00003 + | > step_time: 5.19530 (4.20591) + | > loader_time: 0.10060 (0.12959) + + + --> STEP: 41/234 -- GLOBAL_STEP: 23675 + | > loss: -0.04141 (-0.02977) + | > log_mle: -0.18267 (-0.18304) + | > loss_dur: 0.14126 (0.15327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.07026 (7.03066) + | > current_lr: 0.00003 + | > step_time: 1.42460 (3.97950) + | > loader_time: 0.08250 (0.11870) + + + --> STEP: 46/234 -- GLOBAL_STEP: 23680 + | > loss: -0.02493 (-0.02965) + | > log_mle: -0.18595 (-0.18357) + | > loss_dur: 0.16101 (0.15393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.54697 (7.02766) + | > current_lr: 0.00003 + | > step_time: 2.00520 (3.70545) + | > loader_time: 0.00390 (0.10609) + + + --> STEP: 51/234 -- GLOBAL_STEP: 23685 + | > loss: -0.01946 (-0.02958) + | > log_mle: -0.17221 (-0.18310) + | > loss_dur: 0.15275 (0.15351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.25661 (6.78420) + | > current_lr: 0.00003 + | > step_time: 2.79210 (3.54308) + | > loader_time: 0.00150 (0.09666) + + + --> STEP: 56/234 -- GLOBAL_STEP: 23690 + | > loss: -0.00502 (-0.02943) + | > log_mle: -0.19136 (-0.18392) + | > loss_dur: 0.18634 (0.15449) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.43542 (6.65582) + | > current_lr: 0.00003 + | > step_time: 1.39190 (3.36229) + | > loader_time: 0.00280 (0.08824) + + + --> STEP: 61/234 -- GLOBAL_STEP: 23695 + | > loss: -0.04722 (-0.03037) + | > log_mle: -0.18873 (-0.18461) + | > loss_dur: 0.14150 (0.15424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.20113 (6.63199) + | > current_lr: 0.00003 + | > step_time: 1.30990 (3.21803) + | > loader_time: 0.00240 (0.08259) + + + --> STEP: 66/234 -- GLOBAL_STEP: 23700 + | > loss: -0.02679 (-0.03000) + | > log_mle: -0.17603 (-0.18515) + | > loss_dur: 0.14924 (0.15515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.97584 (6.70299) + | > current_lr: 0.00003 + | > step_time: 1.70670 (3.13466) + | > loader_time: 0.00450 (0.07816) + + + --> STEP: 71/234 -- GLOBAL_STEP: 23705 + | > loss: -0.01489 (-0.02923) + | > log_mle: -0.21768 (-0.18570) + | > loss_dur: 0.20279 (0.15647) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.20604 (6.96007) + | > current_lr: 0.00003 + | > step_time: 2.19540 (3.01786) + | > loader_time: 0.00330 (0.07517) + + + --> STEP: 76/234 -- GLOBAL_STEP: 23710 + | > loss: -0.03704 (-0.02940) + | > log_mle: -0.20323 (-0.18650) + | > loss_dur: 0.16620 (0.15710) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.37850 (7.00313) + | > current_lr: 0.00003 + | > step_time: 1.21060 (2.95369) + | > loader_time: 0.08550 (0.07261) + + + --> STEP: 81/234 -- GLOBAL_STEP: 23715 + | > loss: -0.05063 (-0.03013) + | > log_mle: -0.21083 (-0.18701) + | > loss_dur: 0.16020 (0.15688) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03926 (7.05502) + | > current_lr: 0.00003 + | > step_time: 1.30620 (2.87168) + | > loader_time: 0.08420 (0.07252) + + + --> STEP: 86/234 -- GLOBAL_STEP: 23720 + | > loss: -0.03883 (-0.03049) + | > log_mle: -0.21068 (-0.18784) + | > loss_dur: 0.17185 (0.15735) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.40671 (7.21439) + | > current_lr: 0.00003 + | > step_time: 3.61290 (2.83025) + | > loader_time: 0.00310 (0.06941) + + + --> STEP: 91/234 -- GLOBAL_STEP: 23725 + | > loss: -0.04366 (-0.03145) + | > log_mle: -0.22115 (-0.18988) + | > loss_dur: 0.17749 (0.15842) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.75580 (7.49436) + | > current_lr: 0.00003 + | > step_time: 1.60560 (2.76483) + | > loader_time: 0.00320 (0.06577) + + + --> STEP: 96/234 -- GLOBAL_STEP: 23730 + | > loss: -0.03363 (-0.03377) + | > log_mle: -0.20910 (-0.19342) + | > loss_dur: 0.17547 (0.15965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.73924 (8.04659) + | > current_lr: 0.00003 + | > step_time: 2.19760 (2.74402) + | > loader_time: 0.00280 (0.06339) + + + --> STEP: 101/234 -- GLOBAL_STEP: 23735 + | > loss: -0.08106 (-0.03537) + | > log_mle: -0.26752 (-0.19594) + | > loss_dur: 0.18646 (0.16057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.05522 (8.51901) + | > current_lr: 0.00003 + | > step_time: 1.48650 (2.74549) + | > loader_time: 0.00240 (0.06201) + + + --> STEP: 106/234 -- GLOBAL_STEP: 23740 + | > loss: -0.05241 (-0.03683) + | > log_mle: -0.26774 (-0.19886) + | > loss_dur: 0.21533 (0.16204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.11084 (9.05894) + | > current_lr: 0.00003 + | > step_time: 2.21010 (2.70566) + | > loader_time: 0.00240 (0.06005) + + + --> STEP: 111/234 -- GLOBAL_STEP: 23745 + | > loss: -0.09658 (-0.03822) + | > log_mle: -0.31251 (-0.20186) + | > loss_dur: 0.21593 (0.16364) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.46894 (9.54729) + | > current_lr: 0.00003 + | > step_time: 1.49940 (2.65677) + | > loader_time: 0.00320 (0.05823) + + + --> STEP: 116/234 -- GLOBAL_STEP: 23750 + | > loss: -0.05159 (-0.03952) + | > log_mle: -0.27785 (-0.20474) + | > loss_dur: 0.22626 (0.16522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.70885 (10.17411) + | > current_lr: 0.00003 + | > step_time: 1.48020 (2.61188) + | > loader_time: 0.00270 (0.05664) + + + --> STEP: 121/234 -- GLOBAL_STEP: 23755 + | > loss: -0.01490 (-0.04043) + | > log_mle: -0.19277 (-0.20678) + | > loss_dur: 0.17787 (0.16634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.41213 (10.42360) + | > current_lr: 0.00003 + | > step_time: 2.39150 (2.59594) + | > loader_time: 0.00290 (0.05584) + + + --> STEP: 126/234 -- GLOBAL_STEP: 23760 + | > loss: -0.11491 (-0.04234) + | > log_mle: -0.32513 (-0.20948) + | > loss_dur: 0.21022 (0.16714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.37865 (10.87619) + | > current_lr: 0.00003 + | > step_time: 3.70380 (2.60898) + | > loader_time: 0.00280 (0.05454) + + + --> STEP: 131/234 -- GLOBAL_STEP: 23765 + | > loss: -0.13504 (-0.04453) + | > log_mle: -0.36459 (-0.21312) + | > loss_dur: 0.22955 (0.16859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.31142 (11.52241) + | > current_lr: 0.00003 + | > step_time: 1.11390 (2.59979) + | > loader_time: 0.00250 (0.05585) + + + --> STEP: 136/234 -- GLOBAL_STEP: 23770 + | > loss: -0.17370 (-0.04671) + | > log_mle: -0.41174 (-0.21672) + | > loss_dur: 0.23804 (0.17001) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.35789 (12.08040) + | > current_lr: 0.00003 + | > step_time: 1.20570 (2.57848) + | > loader_time: 0.00240 (0.05540) + + + --> STEP: 141/234 -- GLOBAL_STEP: 23775 + | > loss: -0.10067 (-0.04843) + | > log_mle: -0.32051 (-0.21984) + | > loss_dur: 0.21984 (0.17141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.56440 (12.72107) + | > current_lr: 0.00003 + | > step_time: 2.90340 (2.58552) + | > loader_time: 0.00450 (0.05471) + + + --> STEP: 146/234 -- GLOBAL_STEP: 23780 + | > loss: -0.14995 (-0.05134) + | > log_mle: -0.36963 (-0.22463) + | > loss_dur: 0.21968 (0.17329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.22793 (13.84956) + | > current_lr: 0.00003 + | > step_time: 4.19980 (2.60317) + | > loader_time: 0.08630 (0.05593) + + + --> STEP: 151/234 -- GLOBAL_STEP: 23785 + | > loss: -0.13245 (-0.05398) + | > log_mle: -0.33850 (-0.22852) + | > loss_dur: 0.20605 (0.17454) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.11395 (14.56333) + | > current_lr: 0.00003 + | > step_time: 2.79660 (2.60705) + | > loader_time: 0.00470 (0.05727) + + + --> STEP: 156/234 -- GLOBAL_STEP: 23790 + | > loss: -0.16198 (-0.05771) + | > log_mle: -0.37861 (-0.23386) + | > loss_dur: 0.21663 (0.17615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.15391 (15.67298) + | > current_lr: 0.00003 + | > step_time: 4.30870 (2.62441) + | > loader_time: 0.00260 (0.05655) + + + --> STEP: 161/234 -- GLOBAL_STEP: 23795 + | > loss: -0.17654 (-0.06070) + | > log_mle: -0.39970 (-0.23838) + | > loss_dur: 0.22316 (0.17768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.30671 (16.60033) + | > current_lr: 0.00003 + | > step_time: 3.39890 (2.64049) + | > loader_time: 0.00470 (0.05646) + + + --> STEP: 166/234 -- GLOBAL_STEP: 23800 + | > loss: -0.14227 (-0.06334) + | > log_mle: -0.34315 (-0.24233) + | > loss_dur: 0.20088 (0.17900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.95605 (17.27261) + | > current_lr: 0.00003 + | > step_time: 1.18010 (2.68977) + | > loader_time: 0.00290 (0.05602) + + + --> STEP: 171/234 -- GLOBAL_STEP: 23805 + | > loss: -0.21866 (-0.06683) + | > log_mle: -0.44443 (-0.24757) + | > loss_dur: 0.22577 (0.18074) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.81453 (18.22230) + | > current_lr: 0.00003 + | > step_time: 4.29810 (2.68724) + | > loader_time: 0.08760 (0.05687) + + + --> STEP: 176/234 -- GLOBAL_STEP: 23810 + | > loss: -0.17661 (-0.07020) + | > log_mle: -0.41765 (-0.25272) + | > loss_dur: 0.24104 (0.18252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.83588 (19.22331) + | > current_lr: 0.00003 + | > step_time: 7.90230 (2.74326) + | > loader_time: 0.09870 (0.05753) + + + --> STEP: 181/234 -- GLOBAL_STEP: 23815 + | > loss: -0.12156 (-0.07299) + | > log_mle: -0.35434 (-0.25724) + | > loss_dur: 0.23279 (0.18424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.64037 (20.19383) + | > current_lr: 0.00003 + | > step_time: 3.91170 (2.78091) + | > loader_time: 0.10810 (0.05706) + + + --> STEP: 186/234 -- GLOBAL_STEP: 23820 + | > loss: -0.14353 (-0.07586) + | > log_mle: -0.39411 (-0.26196) + | > loss_dur: 0.25058 (0.18610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.57437 (21.14205) + | > current_lr: 0.00003 + | > step_time: 2.39900 (2.78726) + | > loader_time: 0.00400 (0.05571) + + + --> STEP: 191/234 -- GLOBAL_STEP: 23825 + | > loss: -0.19034 (-0.07888) + | > log_mle: -0.41360 (-0.26649) + | > loss_dur: 0.22326 (0.18761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.29253 (22.03913) + | > current_lr: 0.00003 + | > step_time: 2.40350 (2.78809) + | > loader_time: 0.00330 (0.05489) + + + --> STEP: 196/234 -- GLOBAL_STEP: 23830 + | > loss: -0.15977 (-0.08209) + | > log_mle: -0.40659 (-0.27111) + | > loss_dur: 0.24682 (0.18902) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.96595 (23.05097) + | > current_lr: 0.00003 + | > step_time: 10.00430 (2.83184) + | > loader_time: 0.11500 (0.05601) + + + --> STEP: 201/234 -- GLOBAL_STEP: 23835 + | > loss: -0.11917 (-0.08469) + | > log_mle: -0.37516 (-0.27526) + | > loss_dur: 0.25599 (0.19057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.72505 (23.84157) + | > current_lr: 0.00003 + | > step_time: 4.01340 (2.83518) + | > loader_time: 0.08810 (0.05642) + + + --> STEP: 206/234 -- GLOBAL_STEP: 23840 + | > loss: -0.21821 (-0.08782) + | > log_mle: -0.47410 (-0.27989) + | > loss_dur: 0.25589 (0.19207) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.69889 (24.80617) + | > current_lr: 0.00003 + | > step_time: 9.09690 (2.90914) + | > loader_time: 0.19510 (0.05837) + + + --> STEP: 211/234 -- GLOBAL_STEP: 23845 + | > loss: -0.25385 (-0.09131) + | > log_mle: -0.54282 (-0.28512) + | > loss_dur: 0.28896 (0.19381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.18022 (25.88935) + | > current_lr: 0.00003 + | > step_time: 6.51070 (2.95362) + | > loader_time: 0.00270 (0.05791) + + + --> STEP: 216/234 -- GLOBAL_STEP: 23850 + | > loss: -0.25620 (-0.09477) + | > log_mle: -0.53053 (-0.29008) + | > loss_dur: 0.27434 (0.19532) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.31756 (26.90158) + | > current_lr: 0.00003 + | > step_time: 5.39340 (3.01763) + | > loader_time: 0.20500 (0.05982) + + + --> STEP: 221/234 -- GLOBAL_STEP: 23855 + | > loss: -0.20076 (-0.09814) + | > log_mle: -0.45483 (-0.29503) + | > loss_dur: 0.25407 (0.19689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.56612 (27.89579) + | > current_lr: 0.00003 + | > step_time: 3.30070 (3.02413) + | > loader_time: 0.00740 (0.05934) + + + --> STEP: 226/234 -- GLOBAL_STEP: 23860 + | > loss: -0.27655 (-0.10174) + | > log_mle: -0.54986 (-0.30046) + | > loss_dur: 0.27330 (0.19872) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.08566 (29.04072) + | > current_lr: 0.00003 + | > step_time: 1.79840 (3.01557) + | > loader_time: 0.08870 (0.05925) + + + --> STEP: 231/234 -- GLOBAL_STEP: 23865 + | > loss: -0.19538 (-0.10444) + | > log_mle: -0.61117 (-0.30622) + | > loss_dur: 0.41579 (0.20178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.68439 (30.17138) + | > current_lr: 0.00003 + | > step_time: 0.27390 (2.95787) + | > loader_time: 0.00500 (0.05807) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05840 (-0.71807) + | > avg_loss: -0.13079 (+0.00659) + | > avg_log_mle: -0.36880 (+0.01386) + | > avg_loss_dur: 0.23800 (-0.00727) + + + > EPOCH: 102/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 06:47:09)  + + --> STEP: 2/234 -- GLOBAL_STEP: 23870 + | > loss: 0.01597 (-0.01635) + | > log_mle: -0.17296 (-0.18043) + | > loss_dur: 0.18892 (0.16408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.73035 (9.89744) + | > current_lr: 0.00003 + | > step_time: 2.11600 (4.60789) + | > loader_time: 0.08530 (0.09070) + + + --> STEP: 7/234 -- GLOBAL_STEP: 23875 + | > loss: -0.05443 (-0.01124) + | > log_mle: -0.19969 (-0.18654) + | > loss_dur: 0.14526 (0.17529) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.30974 (11.55034) + | > current_lr: 0.00003 + | > step_time: 4.41330 (6.34609) + | > loader_time: 0.00370 (1.18386) + + + --> STEP: 12/234 -- GLOBAL_STEP: 23880 + | > loss: -0.03422 (-0.01863) + | > log_mle: -0.18773 (-0.18849) + | > loss_dur: 0.15351 (0.16986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.83145 (10.35971) + | > current_lr: 0.00003 + | > step_time: 0.83130 (4.21405) + | > loader_time: 0.00120 (0.69970) + + + --> STEP: 17/234 -- GLOBAL_STEP: 23885 + | > loss: -0.00545 (-0.02367) + | > log_mle: -0.16419 (-0.18679) + | > loss_dur: 0.15873 (0.16312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.31020 (9.19431) + | > current_lr: 0.00003 + | > step_time: 0.95440 (3.32700) + | > loader_time: 0.00130 (0.49467) + + + --> STEP: 22/234 -- GLOBAL_STEP: 23890 + | > loss: -0.03924 (-0.02468) + | > log_mle: -0.19216 (-0.18512) + | > loss_dur: 0.15292 (0.16043) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.32772 (8.54036) + | > current_lr: 0.00003 + | > step_time: 0.84350 (2.80199) + | > loader_time: 0.00210 (0.38264) + + + --> STEP: 27/234 -- GLOBAL_STEP: 23895 + | > loss: -0.05205 (-0.02660) + | > log_mle: -0.19314 (-0.18471) + | > loss_dur: 0.14109 (0.15810) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.41871 (7.95496) + | > current_lr: 0.00003 + | > step_time: 2.80670 (2.67959) + | > loader_time: 0.00500 (0.31235) + + + --> STEP: 32/234 -- GLOBAL_STEP: 23900 + | > loss: -0.06684 (-0.02892) + | > log_mle: -0.20057 (-0.18501) + | > loss_dur: 0.13373 (0.15609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.04465 (7.56480) + | > current_lr: 0.00003 + | > step_time: 7.29520 (3.17028) + | > loader_time: 0.10770 (0.27294) + + + --> STEP: 37/234 -- GLOBAL_STEP: 23905 + | > loss: -0.04814 (-0.02819) + | > log_mle: -0.18398 (-0.18497) + | > loss_dur: 0.13584 (0.15679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.61134 (7.58924) + | > current_lr: 0.00003 + | > step_time: 3.19860 (3.23610) + | > loader_time: 0.08520 (0.25116) + + + --> STEP: 42/234 -- GLOBAL_STEP: 23910 + | > loss: -0.01819 (-0.02816) + | > log_mle: -0.17271 (-0.18476) + | > loss_dur: 0.15452 (0.15660) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.85276 (7.42217) + | > current_lr: 0.00003 + | > step_time: 1.10660 (3.02371) + | > loader_time: 0.00170 (0.22167) + + + --> STEP: 47/234 -- GLOBAL_STEP: 23915 + | > loss: -0.02596 (-0.02875) + | > log_mle: -0.18393 (-0.18540) + | > loss_dur: 0.15797 (0.15665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.70309 (7.43643) + | > current_lr: 0.00003 + | > step_time: 2.29510 (2.88014) + | > loader_time: 0.00210 (0.20007) + + + --> STEP: 52/234 -- GLOBAL_STEP: 23920 + | > loss: -0.01801 (-0.02933) + | > log_mle: -0.18148 (-0.18483) + | > loss_dur: 0.16346 (0.15551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.67828 (7.16980) + | > current_lr: 0.00003 + | > step_time: 1.22840 (2.78408) + | > loader_time: 0.00220 (0.18109) + + + --> STEP: 57/234 -- GLOBAL_STEP: 23925 + | > loss: -0.01415 (-0.02927) + | > log_mle: -0.17520 (-0.18546) + | > loss_dur: 0.16106 (0.15619) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.43084 (6.96650) + | > current_lr: 0.00003 + | > step_time: 1.08230 (2.66067) + | > loader_time: 0.00200 (0.16716) + + + --> STEP: 62/234 -- GLOBAL_STEP: 23930 + | > loss: -0.01854 (-0.03065) + | > log_mle: -0.23034 (-0.18704) + | > loss_dur: 0.21180 (0.15640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.44904 (7.12546) + | > current_lr: 0.00003 + | > step_time: 2.59940 (2.59705) + | > loader_time: 0.09310 (0.15669) + + + --> STEP: 67/234 -- GLOBAL_STEP: 23935 + | > loss: -0.04782 (-0.03116) + | > log_mle: -0.21085 (-0.18726) + | > loss_dur: 0.16303 (0.15609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.34669 (6.97896) + | > current_lr: 0.00003 + | > step_time: 1.47200 (2.55187) + | > loader_time: 0.00210 (0.14794) + + + --> STEP: 72/234 -- GLOBAL_STEP: 23940 + | > loss: -0.01346 (-0.02996) + | > log_mle: -0.18777 (-0.18746) + | > loss_dur: 0.17431 (0.15750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.17320 (7.18169) + | > current_lr: 0.00003 + | > step_time: 2.48820 (2.48288) + | > loader_time: 0.00220 (0.14105) + + + --> STEP: 77/234 -- GLOBAL_STEP: 23945 + | > loss: -0.05831 (-0.03056) + | > log_mle: -0.20213 (-0.18834) + | > loss_dur: 0.14382 (0.15778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.49363 (7.36132) + | > current_lr: 0.00003 + | > step_time: 1.15130 (2.42783) + | > loader_time: 0.00230 (0.13324) + + + --> STEP: 82/234 -- GLOBAL_STEP: 23950 + | > loss: -0.04457 (-0.03126) + | > log_mle: -0.19232 (-0.18866) + | > loss_dur: 0.14775 (0.15741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.27357 (7.40171) + | > current_lr: 0.00003 + | > step_time: 2.20070 (2.43227) + | > loader_time: 0.00440 (0.12865) + + + --> STEP: 87/234 -- GLOBAL_STEP: 23955 + | > loss: -0.02347 (-0.03143) + | > log_mle: -0.20177 (-0.18956) + | > loss_dur: 0.17831 (0.15813) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.11928 (7.57069) + | > current_lr: 0.00003 + | > step_time: 4.00410 (2.42953) + | > loader_time: 0.09860 (0.12352) + + + --> STEP: 92/234 -- GLOBAL_STEP: 23960 + | > loss: -0.09637 (-0.03328) + | > log_mle: -0.24862 (-0.19204) + | > loss_dur: 0.15225 (0.15876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.81161 (7.94345) + | > current_lr: 0.00003 + | > step_time: 1.60420 (2.39758) + | > loader_time: 0.00320 (0.11872) + + + --> STEP: 97/234 -- GLOBAL_STEP: 23965 + | > loss: -0.06568 (-0.03548) + | > log_mle: -0.23735 (-0.19542) + | > loss_dur: 0.17168 (0.15995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.70156 (8.49882) + | > current_lr: 0.00003 + | > step_time: 2.90640 (2.49367) + | > loader_time: 0.08890 (0.11452) + + + --> STEP: 102/234 -- GLOBAL_STEP: 23970 + | > loss: -0.04320 (-0.03681) + | > log_mle: -0.21618 (-0.19771) + | > loss_dur: 0.17298 (0.16091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.31243 (8.89375) + | > current_lr: 0.00003 + | > step_time: 1.40270 (2.43918) + | > loader_time: 0.00250 (0.10968) + + + --> STEP: 107/234 -- GLOBAL_STEP: 23975 + | > loss: -0.08349 (-0.03875) + | > log_mle: -0.26308 (-0.20097) + | > loss_dur: 0.17959 (0.16222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.72638 (9.45957) + | > current_lr: 0.00003 + | > step_time: 2.54430 (2.41625) + | > loader_time: 0.00370 (0.10468) + + + --> STEP: 112/234 -- GLOBAL_STEP: 23980 + | > loss: -0.06872 (-0.04005) + | > log_mle: -0.27315 (-0.20392) + | > loss_dur: 0.20443 (0.16387) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.97207 (10.07451) + | > current_lr: 0.00003 + | > step_time: 3.31070 (2.39376) + | > loader_time: 0.08740 (0.10162) + + + --> STEP: 117/234 -- GLOBAL_STEP: 23985 + | > loss: -0.07371 (-0.04135) + | > log_mle: -0.26742 (-0.20671) + | > loss_dur: 0.19370 (0.16536) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.13872 (10.59516) + | > current_lr: 0.00003 + | > step_time: 1.67070 (2.39034) + | > loader_time: 0.00310 (0.09903) + + + --> STEP: 122/234 -- GLOBAL_STEP: 23990 + | > loss: -0.07427 (-0.04230) + | > log_mle: -0.24713 (-0.20853) + | > loss_dur: 0.17286 (0.16623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.73582 (10.82451) + | > current_lr: 0.00003 + | > step_time: 1.50140 (2.38166) + | > loader_time: 0.09020 (0.09721) + + + --> STEP: 127/234 -- GLOBAL_STEP: 23995 + | > loss: -0.09056 (-0.04440) + | > log_mle: -0.30113 (-0.21164) + | > loss_dur: 0.21057 (0.16723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.56075 (11.29871) + | > current_lr: 0.00003 + | > step_time: 2.33890 (2.38084) + | > loader_time: 0.00270 (0.09607) + + + --> STEP: 132/234 -- GLOBAL_STEP: 24000 + | > loss: -0.10750 (-0.04675) + | > log_mle: -0.28177 (-0.21510) + | > loss_dur: 0.17427 (0.16834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.54194 (11.91739) + | > current_lr: 0.00003 + | > step_time: 2.61780 (2.39554) + | > loader_time: 0.00340 (0.09326) + + + --> STEP: 137/234 -- GLOBAL_STEP: 24005 + | > loss: -0.06626 (-0.04884) + | > log_mle: -0.29411 (-0.21874) + | > loss_dur: 0.22785 (0.16990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.04014 (12.53971) + | > current_lr: 0.00003 + | > step_time: 0.99730 (2.37682) + | > loader_time: 0.00340 (0.09179) + + + --> STEP: 142/234 -- GLOBAL_STEP: 24010 + | > loss: -0.09341 (-0.05065) + | > log_mle: -0.30933 (-0.22192) + | > loss_dur: 0.21592 (0.17127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.33736 (13.11377) + | > current_lr: 0.00003 + | > step_time: 3.03740 (2.39125) + | > loader_time: 0.01550 (0.08991) + + + --> STEP: 147/234 -- GLOBAL_STEP: 24015 + | > loss: -0.09896 (-0.05356) + | > log_mle: -0.31038 (-0.22667) + | > loss_dur: 0.21142 (0.17312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.79685 (14.10134) + | > current_lr: 0.00003 + | > step_time: 1.91620 (2.36618) + | > loader_time: 0.08560 (0.08862) + + + --> STEP: 152/234 -- GLOBAL_STEP: 24020 + | > loss: -0.15098 (-0.05658) + | > log_mle: -0.38939 (-0.23106) + | > loss_dur: 0.23842 (0.17448) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.19767 (14.89654) + | > current_lr: 0.00003 + | > step_time: 1.80770 (2.35344) + | > loader_time: 0.08810 (0.08748) + + + --> STEP: 157/234 -- GLOBAL_STEP: 24025 + | > loss: -0.11569 (-0.06007) + | > log_mle: -0.33959 (-0.23609) + | > loss_dur: 0.22390 (0.17602) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.05505 (15.99848) + | > current_lr: 0.00003 + | > step_time: 2.39750 (2.38106) + | > loader_time: 0.10470 (0.08857) + + + --> STEP: 162/234 -- GLOBAL_STEP: 24030 + | > loss: -0.15604 (-0.06320) + | > log_mle: -0.36965 (-0.24073) + | > loss_dur: 0.21362 (0.17753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.24854 (17.15360) + | > current_lr: 0.00003 + | > step_time: 1.29550 (2.37904) + | > loader_time: 0.00400 (0.08595) + + + --> STEP: 167/234 -- GLOBAL_STEP: 24035 + | > loss: -0.22570 (-0.06608) + | > log_mle: -0.45269 (-0.24504) + | > loss_dur: 0.22699 (0.17896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.09892 (18.04863) + | > current_lr: 0.00003 + | > step_time: 2.61140 (2.40551) + | > loader_time: 0.07530 (0.08498) + + + --> STEP: 172/234 -- GLOBAL_STEP: 24040 + | > loss: -0.19324 (-0.06932) + | > log_mle: -0.44382 (-0.25021) + | > loss_dur: 0.25058 (0.18089) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.75859 (19.00626) + | > current_lr: 0.00003 + | > step_time: 5.30810 (2.42281) + | > loader_time: 0.00380 (0.08377) + + + --> STEP: 177/234 -- GLOBAL_STEP: 24045 + | > loss: -0.16108 (-0.07251) + | > log_mle: -0.40520 (-0.25513) + | > loss_dur: 0.24413 (0.18263) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.19838 (19.90615) + | > current_lr: 0.00003 + | > step_time: 1.71350 (2.40598) + | > loader_time: 0.00300 (0.08299) + + + --> STEP: 182/234 -- GLOBAL_STEP: 24050 + | > loss: -0.19581 (-0.07561) + | > log_mle: -0.44904 (-0.26000) + | > loss_dur: 0.25323 (0.18439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.77912 (20.80409) + | > current_lr: 0.00003 + | > step_time: 3.41920 (2.41356) + | > loader_time: 0.09220 (0.08234) + + + --> STEP: 187/234 -- GLOBAL_STEP: 24055 + | > loss: -0.20794 (-0.07862) + | > log_mle: -0.44969 (-0.26476) + | > loss_dur: 0.24176 (0.18614) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.38546 (21.85784) + | > current_lr: 0.00003 + | > step_time: 2.30530 (2.40728) + | > loader_time: 0.19870 (0.08131) + + + --> STEP: 192/234 -- GLOBAL_STEP: 24060 + | > loss: -0.22217 (-0.08176) + | > log_mle: -0.45627 (-0.26926) + | > loss_dur: 0.23410 (0.18750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 133.44492 (23.44517) + | > current_lr: 0.00003 + | > step_time: 2.00090 (2.43472) + | > loader_time: 0.20050 (0.08236) + + + --> STEP: 197/234 -- GLOBAL_STEP: 24065 + | > loss: -0.19829 (-0.08459) + | > log_mle: -0.43476 (-0.27356) + | > loss_dur: 0.23647 (0.18896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.87692 (24.76483) + | > current_lr: 0.00003 + | > step_time: 6.39460 (2.49989) + | > loader_time: 0.00490 (0.08168) + + + --> STEP: 202/234 -- GLOBAL_STEP: 24070 + | > loss: -0.27312 (-0.08738) + | > log_mle: -0.52881 (-0.27801) + | > loss_dur: 0.25569 (0.19063) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.78381 (25.73250) + | > current_lr: 0.00003 + | > step_time: 2.70170 (2.51235) + | > loader_time: 0.09410 (0.08159) + + + --> STEP: 207/234 -- GLOBAL_STEP: 24075 + | > loss: -0.26018 (-0.09039) + | > log_mle: -0.51854 (-0.28257) + | > loss_dur: 0.25835 (0.19218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.42975 (26.54790) + | > current_lr: 0.00003 + | > step_time: 2.30460 (2.57866) + | > loader_time: 0.08950 (0.08058) + + + --> STEP: 212/234 -- GLOBAL_STEP: 24080 + | > loss: -0.22898 (-0.09374) + | > log_mle: -0.49052 (-0.28766) + | > loss_dur: 0.26154 (0.19392) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.13509 (27.67656) + | > current_lr: 0.00003 + | > step_time: 3.41470 (2.64342) + | > loader_time: 0.08320 (0.08086) + + + --> STEP: 217/234 -- GLOBAL_STEP: 24085 + | > loss: -0.24372 (-0.09699) + | > log_mle: -0.51414 (-0.29254) + | > loss_dur: 0.27042 (0.19555) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.82169 (28.91954) + | > current_lr: 0.00003 + | > step_time: 2.79660 (2.64928) + | > loader_time: 0.01030 (0.08001) + + + --> STEP: 222/234 -- GLOBAL_STEP: 24090 + | > loss: -0.24020 (-0.10026) + | > log_mle: -0.53347 (-0.29748) + | > loss_dur: 0.29328 (0.19722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.53341 (29.82529) + | > current_lr: 0.00003 + | > step_time: 1.49400 (2.64769) + | > loader_time: 0.00490 (0.07951) + + + --> STEP: 227/234 -- GLOBAL_STEP: 24095 + | > loss: -0.21860 (-0.10381) + | > log_mle: -0.50139 (-0.30279) + | > loss_dur: 0.28279 (0.19898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.22000 (30.98531) + | > current_lr: 0.00003 + | > step_time: 1.78160 (2.62319) + | > loader_time: 0.00490 (0.07818) + + + --> STEP: 232/234 -- GLOBAL_STEP: 24100 + | > loss: -0.15125 (-0.10612) + | > log_mle: -0.69879 (-0.30927) + | > loss_dur: 0.54754 (0.20315) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 123.33535 (32.35553) + | > current_lr: 0.00003 + | > step_time: 0.34030 (2.59380) + | > loader_time: 0.11080 (0.10798) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.20392 (+1.14552) + | > avg_loss: -0.14932 (-0.01852) + | > avg_log_mle: -0.38933 (-0.02053) + | > avg_loss_dur: 0.24001 (+0.00201) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_24102.pth + + > EPOCH: 103/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 06:58:29)  + + --> STEP: 3/234 -- GLOBAL_STEP: 24105 + | > loss: 0.02093 (-0.00754) + | > log_mle: -0.19087 (-0.18593) + | > loss_dur: 0.21179 (0.17840) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.10404 (8.13271) + | > current_lr: 0.00003 + | > step_time: 1.31350 (2.24131) + | > loader_time: 0.00250 (0.00237) + + + --> STEP: 8/234 -- GLOBAL_STEP: 24110 + | > loss: -0.02641 (-0.01327) + | > log_mle: -0.20148 (-0.18950) + | > loss_dur: 0.17507 (0.17623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.40166 (8.56272) + | > current_lr: 0.00003 + | > step_time: 1.51640 (2.30428) + | > loader_time: 0.08940 (0.04807) + + + --> STEP: 13/234 -- GLOBAL_STEP: 24115 + | > loss: -0.01560 (-0.01946) + | > log_mle: -0.18440 (-0.18973) + | > loss_dur: 0.16880 (0.17027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.38361 (8.43339) + | > current_lr: 0.00003 + | > step_time: 8.10000 (3.70952) + | > loader_time: 0.19460 (0.06014) + + + --> STEP: 18/234 -- GLOBAL_STEP: 24120 + | > loss: -0.01746 (-0.02494) + | > log_mle: -0.18568 (-0.18840) + | > loss_dur: 0.16822 (0.16346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.01725 (8.03757) + | > current_lr: 0.00003 + | > step_time: 1.79040 (3.30627) + | > loader_time: 0.00230 (0.04908) + + + --> STEP: 23/234 -- GLOBAL_STEP: 24125 + | > loss: -0.04601 (-0.02745) + | > log_mle: -0.18861 (-0.18690) + | > loss_dur: 0.14260 (0.15946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.38810 (7.61489) + | > current_lr: 0.00003 + | > step_time: 5.30570 (3.21919) + | > loader_time: 0.00160 (0.04217) + + + --> STEP: 28/234 -- GLOBAL_STEP: 24130 + | > loss: -0.05563 (-0.03118) + | > log_mle: -0.17934 (-0.18630) + | > loss_dur: 0.12371 (0.15512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.03602 (7.12934) + | > current_lr: 0.00003 + | > step_time: 5.39900 (3.24205) + | > loader_time: 0.00550 (0.04146) + + + --> STEP: 33/234 -- GLOBAL_STEP: 24135 + | > loss: -0.01103 (-0.03196) + | > log_mle: -0.17671 (-0.18667) + | > loss_dur: 0.16568 (0.15471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.22773 (6.96484) + | > current_lr: 0.00003 + | > step_time: 4.41220 (3.32624) + | > loader_time: 0.08970 (0.04451) + + + --> STEP: 38/234 -- GLOBAL_STEP: 24140 + | > loss: -0.03345 (-0.03185) + | > log_mle: -0.19705 (-0.18727) + | > loss_dur: 0.16361 (0.15542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.24809 (7.03525) + | > current_lr: 0.00003 + | > step_time: 1.55620 (3.32477) + | > loader_time: 0.09010 (0.05629) + + + --> STEP: 43/234 -- GLOBAL_STEP: 24145 + | > loss: -0.03180 (-0.03122) + | > log_mle: -0.19554 (-0.18700) + | > loss_dur: 0.16374 (0.15579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.06187 (7.04340) + | > current_lr: 0.00003 + | > step_time: 1.71130 (3.11853) + | > loader_time: 0.00290 (0.05230) + + + --> STEP: 48/234 -- GLOBAL_STEP: 24150 + | > loss: -0.04221 (-0.03176) + | > log_mle: -0.17777 (-0.18724) + | > loss_dur: 0.13556 (0.15548) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.36253 (7.10430) + | > current_lr: 0.00003 + | > step_time: 1.37740 (2.95052) + | > loader_time: 0.00210 (0.05067) + + + --> STEP: 53/234 -- GLOBAL_STEP: 24155 + | > loss: -0.03072 (-0.03174) + | > log_mle: -0.20116 (-0.18715) + | > loss_dur: 0.17045 (0.15541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.75040 (6.84686) + | > current_lr: 0.00003 + | > step_time: 1.17550 (2.79260) + | > loader_time: 0.00200 (0.04764) + + + --> STEP: 58/234 -- GLOBAL_STEP: 24160 + | > loss: -0.03954 (-0.03238) + | > log_mle: -0.18311 (-0.18741) + | > loss_dur: 0.14357 (0.15503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.04832 (6.71278) + | > current_lr: 0.00003 + | > step_time: 1.21440 (2.68659) + | > loader_time: 0.08380 (0.04665) + + + --> STEP: 63/234 -- GLOBAL_STEP: 24165 + | > loss: -0.01840 (-0.03349) + | > log_mle: -0.19450 (-0.18907) + | > loss_dur: 0.17610 (0.15557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76408 (6.99327) + | > current_lr: 0.00003 + | > step_time: 1.54380 (2.64757) + | > loader_time: 0.00250 (0.04880) + + + --> STEP: 68/234 -- GLOBAL_STEP: 24170 + | > loss: -0.02127 (-0.03371) + | > log_mle: -0.18745 (-0.18910) + | > loss_dur: 0.16618 (0.15539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.79128 (6.96451) + | > current_lr: 0.00003 + | > step_time: 3.09270 (2.57844) + | > loader_time: 0.00450 (0.04545) + + + --> STEP: 73/234 -- GLOBAL_STEP: 24175 + | > loss: -0.02757 (-0.03274) + | > log_mle: -0.21254 (-0.18957) + | > loss_dur: 0.18497 (0.15683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.40139 (7.20491) + | > current_lr: 0.00003 + | > step_time: 2.40460 (2.62024) + | > loader_time: 0.00350 (0.04621) + + + --> STEP: 78/234 -- GLOBAL_STEP: 24180 + | > loss: -0.01899 (-0.03319) + | > log_mle: -0.18587 (-0.19010) + | > loss_dur: 0.16688 (0.15691) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.15218 (7.30872) + | > current_lr: 0.00003 + | > step_time: 3.68790 (2.57415) + | > loader_time: 0.00380 (0.04550) + + + --> STEP: 83/234 -- GLOBAL_STEP: 24185 + | > loss: -0.03817 (-0.03398) + | > log_mle: -0.21524 (-0.19081) + | > loss_dur: 0.17707 (0.15683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.91492 (7.35494) + | > current_lr: 0.00003 + | > step_time: 3.01310 (2.57805) + | > loader_time: 0.00340 (0.04519) + + + --> STEP: 88/234 -- GLOBAL_STEP: 24190 + | > loss: -0.05900 (-0.03447) + | > log_mle: -0.24777 (-0.19198) + | > loss_dur: 0.18877 (0.15751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.27279 (7.61626) + | > current_lr: 0.00003 + | > step_time: 1.11700 (2.52629) + | > loader_time: 0.00230 (0.04395) + + + --> STEP: 93/234 -- GLOBAL_STEP: 24195 + | > loss: -0.06999 (-0.03602) + | > log_mle: -0.26293 (-0.19445) + | > loss_dur: 0.19294 (0.15843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.48366 (8.02722) + | > current_lr: 0.00003 + | > step_time: 1.31120 (2.48843) + | > loader_time: 0.00330 (0.04269) + + + --> STEP: 98/234 -- GLOBAL_STEP: 24200 + | > loss: -0.02721 (-0.03774) + | > log_mle: -0.19080 (-0.19697) + | > loss_dur: 0.16359 (0.15924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.38414 (8.36885) + | > current_lr: 0.00003 + | > step_time: 2.00950 (2.46137) + | > loader_time: 0.00280 (0.04237) + + + --> STEP: 103/234 -- GLOBAL_STEP: 24205 + | > loss: -0.08289 (-0.03951) + | > log_mle: -0.29015 (-0.20021) + | > loss_dur: 0.20726 (0.16070) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.63319 (8.93184) + | > current_lr: 0.00003 + | > step_time: 2.59740 (2.45620) + | > loader_time: 0.00300 (0.04128) + + + --> STEP: 108/234 -- GLOBAL_STEP: 24210 + | > loss: -0.06238 (-0.04122) + | > log_mle: -0.23422 (-0.20293) + | > loss_dur: 0.17184 (0.16171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.54227 (9.40989) + | > current_lr: 0.00003 + | > step_time: 1.20110 (2.41480) + | > loader_time: 0.00280 (0.04103) + + + --> STEP: 113/234 -- GLOBAL_STEP: 24215 + | > loss: -0.09747 (-0.04271) + | > log_mle: -0.28286 (-0.20624) + | > loss_dur: 0.18539 (0.16353) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.02531 (10.27570) + | > current_lr: 0.00003 + | > step_time: 1.09450 (2.38134) + | > loader_time: 0.00140 (0.04002) + + + --> STEP: 118/234 -- GLOBAL_STEP: 24220 + | > loss: -0.05213 (-0.04377) + | > log_mle: -0.25168 (-0.20873) + | > loss_dur: 0.19956 (0.16496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.10565 (10.66526) + | > current_lr: 0.00003 + | > step_time: 1.29850 (2.34833) + | > loader_time: 0.09000 (0.04140) + + + --> STEP: 123/234 -- GLOBAL_STEP: 24225 + | > loss: -0.05272 (-0.04449) + | > log_mle: -0.22289 (-0.21027) + | > loss_dur: 0.17017 (0.16578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.49459 (10.87365) + | > current_lr: 0.00003 + | > step_time: 1.60240 (2.34490) + | > loader_time: 0.00850 (0.04123) + + + --> STEP: 128/234 -- GLOBAL_STEP: 24230 + | > loss: -0.10477 (-0.04684) + | > log_mle: -0.28313 (-0.21378) + | > loss_dur: 0.17836 (0.16694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.36342 (11.53379) + | > current_lr: 0.00003 + | > step_time: 3.29900 (2.33995) + | > loader_time: 0.00310 (0.04104) + + + --> STEP: 133/234 -- GLOBAL_STEP: 24235 + | > loss: -0.09686 (-0.04900) + | > log_mle: -0.30197 (-0.21728) + | > loss_dur: 0.20510 (0.16828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.51436 (12.28056) + | > current_lr: 0.00003 + | > step_time: 1.70390 (2.37012) + | > loader_time: 0.08870 (0.04310) + + + --> STEP: 138/234 -- GLOBAL_STEP: 24240 + | > loss: -0.07170 (-0.05064) + | > log_mle: -0.26030 (-0.22047) + | > loss_dur: 0.18860 (0.16983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.69368 (12.95284) + | > current_lr: 0.00003 + | > step_time: 1.80930 (2.34750) + | > loader_time: 0.00310 (0.04281) + + + --> STEP: 143/234 -- GLOBAL_STEP: 24245 + | > loss: -0.15040 (-0.05302) + | > log_mle: -0.39731 (-0.22454) + | > loss_dur: 0.24691 (0.17152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.59595 (13.74544) + | > current_lr: 0.00003 + | > step_time: 6.99360 (2.47457) + | > loader_time: 0.00260 (0.04341) + + + --> STEP: 148/234 -- GLOBAL_STEP: 24250 + | > loss: -0.12591 (-0.05584) + | > log_mle: -0.31683 (-0.22870) + | > loss_dur: 0.19092 (0.17286) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.75616 (14.44241) + | > current_lr: 0.00003 + | > step_time: 2.30150 (2.48427) + | > loader_time: 0.00470 (0.04258) + + + --> STEP: 153/234 -- GLOBAL_STEP: 24255 + | > loss: -0.21989 (-0.05933) + | > log_mle: -0.43572 (-0.23382) + | > loss_dur: 0.21583 (0.17449) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.03411 (15.28539) + | > current_lr: 0.00003 + | > step_time: 3.99800 (2.51666) + | > loader_time: 0.01050 (0.04193) + + + --> STEP: 158/234 -- GLOBAL_STEP: 24260 + | > loss: -0.13718 (-0.06235) + | > log_mle: -0.37440 (-0.23835) + | > loss_dur: 0.23722 (0.17600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.44999 (16.20467) + | > current_lr: 0.00003 + | > step_time: 2.31490 (2.51979) + | > loader_time: 0.00290 (0.04286) + + + --> STEP: 163/234 -- GLOBAL_STEP: 24265 + | > loss: -0.12669 (-0.06537) + | > log_mle: -0.34359 (-0.24277) + | > loss_dur: 0.21690 (0.17739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.11440 (17.29850) + | > current_lr: 0.00003 + | > step_time: 2.10920 (2.50688) + | > loader_time: 0.18470 (0.04277) + + + --> STEP: 168/234 -- GLOBAL_STEP: 24270 + | > loss: -0.15553 (-0.06834) + | > log_mle: -0.39767 (-0.24728) + | > loss_dur: 0.24214 (0.17894) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.78839 (18.27360) + | > current_lr: 0.00003 + | > step_time: 3.70180 (2.56265) + | > loader_time: 0.00340 (0.04273) + + + --> STEP: 173/234 -- GLOBAL_STEP: 24275 + | > loss: -0.17700 (-0.07160) + | > log_mle: -0.40839 (-0.25234) + | > loss_dur: 0.23138 (0.18075) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.50030 (19.41514) + | > current_lr: 0.00003 + | > step_time: 4.70800 (2.63678) + | > loader_time: 0.00370 (0.04375) + + + --> STEP: 178/234 -- GLOBAL_STEP: 24280 + | > loss: -0.20338 (-0.07497) + | > log_mle: -0.46441 (-0.25750) + | > loss_dur: 0.26103 (0.18253) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.92902 (20.46682) + | > current_lr: 0.00003 + | > step_time: 2.50000 (2.63796) + | > loader_time: 0.08440 (0.04416) + + + --> STEP: 183/234 -- GLOBAL_STEP: 24285 + | > loss: -0.21356 (-0.07800) + | > log_mle: -0.46499 (-0.26228) + | > loss_dur: 0.25143 (0.18428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.54324 (21.35216) + | > current_lr: 0.00003 + | > step_time: 3.80110 (2.65675) + | > loader_time: 0.10400 (0.04872) + + + --> STEP: 188/234 -- GLOBAL_STEP: 24290 + | > loss: -0.22330 (-0.08101) + | > log_mle: -0.46917 (-0.26705) + | > loss_dur: 0.24586 (0.18604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.72662 (22.37886) + | > current_lr: 0.00003 + | > step_time: 2.80280 (2.69615) + | > loader_time: 0.09060 (0.04957) + + + --> STEP: 193/234 -- GLOBAL_STEP: 24295 + | > loss: -0.21443 (-0.08397) + | > log_mle: -0.46412 (-0.27145) + | > loss_dur: 0.24968 (0.18748) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.35558 (23.75688) + | > current_lr: 0.00003 + | > step_time: 2.49890 (2.71185) + | > loader_time: 0.07650 (0.05311) + + + --> STEP: 198/234 -- GLOBAL_STEP: 24300 + | > loss: -0.20257 (-0.08662) + | > log_mle: -0.45971 (-0.27567) + | > loss_dur: 0.25714 (0.18904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.74062 (24.90119) + | > current_lr: 0.00003 + | > step_time: 1.60840 (2.70247) + | > loader_time: 0.00470 (0.05278) + + + --> STEP: 203/234 -- GLOBAL_STEP: 24305 + | > loss: -0.15680 (-0.08919) + | > log_mle: -0.39721 (-0.27982) + | > loss_dur: 0.24041 (0.19063) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.74519 (25.72565) + | > current_lr: 0.00003 + | > step_time: 6.69970 (2.77644) + | > loader_time: 0.08550 (0.05323) + + + --> STEP: 208/234 -- GLOBAL_STEP: 24310 + | > loss: -0.20289 (-0.09232) + | > log_mle: -0.46914 (-0.28462) + | > loss_dur: 0.26625 (0.19231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.17058 (26.86544) + | > current_lr: 0.00003 + | > step_time: 2.59100 (2.86008) + | > loader_time: 0.00560 (0.05626) + + + --> STEP: 213/234 -- GLOBAL_STEP: 24315 + | > loss: -0.24490 (-0.09578) + | > log_mle: -0.51897 (-0.28982) + | > loss_dur: 0.27407 (0.19403) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.27641 (28.05834) + | > current_lr: 0.00003 + | > step_time: 2.48540 (2.98570) + | > loader_time: 0.00270 (0.05829) + + + --> STEP: 218/234 -- GLOBAL_STEP: 24320 + | > loss: -0.21762 (-0.09907) + | > log_mle: -0.48438 (-0.29460) + | > loss_dur: 0.26676 (0.19553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.03423 (29.04987) + | > current_lr: 0.00003 + | > step_time: 1.99510 (2.97182) + | > loader_time: 0.00590 (0.05746) + + + --> STEP: 223/234 -- GLOBAL_STEP: 24325 + | > loss: -0.25836 (-0.10253) + | > log_mle: -0.52554 (-0.29975) + | > loss_dur: 0.26719 (0.19722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.49249 (29.99773) + | > current_lr: 0.00003 + | > step_time: 5.90530 (2.99264) + | > loader_time: 0.00470 (0.05663) + + + --> STEP: 228/234 -- GLOBAL_STEP: 24330 + | > loss: -0.22982 (-0.10588) + | > log_mle: -0.52517 (-0.30496) + | > loss_dur: 0.29535 (0.19908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.29201 (31.31466) + | > current_lr: 0.00003 + | > step_time: 0.25040 (2.95239) + | > loader_time: 0.00380 (0.05547) + + + --> STEP: 233/234 -- GLOBAL_STEP: 24335 + | > loss: 0.36024 (-0.10559) + | > log_mle: -0.47281 (-0.31110) + | > loss_dur: 0.83305 (0.20551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.48429 (33.00009) + | > current_lr: 0.00003 + | > step_time: 0.18350 (2.89472) + | > loader_time: 0.00280 (0.05437) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00364 (-1.20028) + | > avg_loss: -0.15509 (-0.00578) + | > avg_log_mle: -0.39213 (-0.00281) + | > avg_loss_dur: 0.23704 (-0.00297) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_24336.pth + + > EPOCH: 104/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 07:11:27)  + + --> STEP: 4/234 -- GLOBAL_STEP: 24340 + | > loss: -0.01211 (-0.01747) + | > log_mle: -0.18822 (-0.18735) + | > loss_dur: 0.17610 (0.16988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.55910 (12.06869) + | > current_lr: 0.00003 + | > step_time: 1.29890 (3.24553) + | > loader_time: 0.00110 (0.09894) + + + --> STEP: 9/234 -- GLOBAL_STEP: 24345 + | > loss: -0.03188 (-0.02615) + | > log_mle: -0.20334 (-0.19253) + | > loss_dur: 0.17146 (0.16638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.78543 (9.46922) + | > current_lr: 0.00003 + | > step_time: 6.69720 (3.43197) + | > loader_time: 0.09540 (0.08833) + + + --> STEP: 14/234 -- GLOBAL_STEP: 24350 + | > loss: -0.03155 (-0.02753) + | > log_mle: -0.19608 (-0.19180) + | > loss_dur: 0.16453 (0.16428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.07050 (8.83627) + | > current_lr: 0.00003 + | > step_time: 8.60450 (3.97795) + | > loader_time: 0.10170 (0.06461) + + + --> STEP: 19/234 -- GLOBAL_STEP: 24355 + | > loss: -0.04846 (-0.03148) + | > log_mle: -0.18064 (-0.18963) + | > loss_dur: 0.13217 (0.15815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.14366 (8.04443) + | > current_lr: 0.00003 + | > step_time: 0.89920 (3.56277) + | > loader_time: 0.00540 (0.04858) + + + --> STEP: 24/234 -- GLOBAL_STEP: 24360 + | > loss: -0.05602 (-0.03445) + | > log_mle: -0.18263 (-0.18828) + | > loss_dur: 0.12660 (0.15383) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.79076 (7.52462) + | > current_lr: 0.00003 + | > step_time: 1.19240 (3.34074) + | > loader_time: 0.00230 (0.03892) + + + --> STEP: 29/234 -- GLOBAL_STEP: 24365 + | > loss: -0.02469 (-0.03565) + | > log_mle: -0.17474 (-0.18739) + | > loss_dur: 0.15004 (0.15174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.63690 (7.18517) + | > current_lr: 0.00003 + | > step_time: 1.68840 (3.00249) + | > loader_time: 0.20350 (0.03951) + + + --> STEP: 34/234 -- GLOBAL_STEP: 24370 + | > loss: -0.02658 (-0.03603) + | > log_mle: -0.18578 (-0.18800) + | > loss_dur: 0.15920 (0.15197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.71858 (6.97275) + | > current_lr: 0.00003 + | > step_time: 2.49970 (3.01761) + | > loader_time: 0.00210 (0.05326) + + + --> STEP: 39/234 -- GLOBAL_STEP: 24375 + | > loss: -0.04159 (-0.03687) + | > log_mle: -0.19568 (-0.18884) + | > loss_dur: 0.15409 (0.15196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10913 (7.18017) + | > current_lr: 0.00003 + | > step_time: 2.43810 (3.08807) + | > loader_time: 0.19280 (0.05167) + + + --> STEP: 44/234 -- GLOBAL_STEP: 24380 + | > loss: -0.04772 (-0.03593) + | > log_mle: -0.18378 (-0.18824) + | > loss_dur: 0.13606 (0.15232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.85073 (6.96675) + | > current_lr: 0.00003 + | > step_time: 1.38640 (2.92115) + | > loader_time: 0.00230 (0.04782) + + + --> STEP: 49/234 -- GLOBAL_STEP: 24385 + | > loss: -0.05688 (-0.03666) + | > log_mle: -0.19296 (-0.18868) + | > loss_dur: 0.13608 (0.15202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.28103 (6.93498) + | > current_lr: 0.00003 + | > step_time: 1.24740 (2.75796) + | > loader_time: 0.00310 (0.04319) + + + --> STEP: 54/234 -- GLOBAL_STEP: 24390 + | > loss: -0.05113 (-0.03654) + | > log_mle: -0.19941 (-0.18868) + | > loss_dur: 0.14827 (0.15214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.76380 (6.72585) + | > current_lr: 0.00003 + | > step_time: 1.38340 (2.63853) + | > loader_time: 0.00310 (0.04437) + + + --> STEP: 59/234 -- GLOBAL_STEP: 24395 + | > loss: -0.06522 (-0.03686) + | > log_mle: -0.21171 (-0.18914) + | > loss_dur: 0.14649 (0.15229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.30773 (6.65471) + | > current_lr: 0.00003 + | > step_time: 1.29560 (2.59654) + | > loader_time: 0.00240 (0.04536) + + + --> STEP: 64/234 -- GLOBAL_STEP: 24400 + | > loss: -0.03900 (-0.03709) + | > log_mle: -0.18060 (-0.19030) + | > loss_dur: 0.14160 (0.15321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.60980 (6.86430) + | > current_lr: 0.00003 + | > step_time: 1.15150 (2.50118) + | > loader_time: 0.00190 (0.04199) + + + --> STEP: 69/234 -- GLOBAL_STEP: 24405 + | > loss: -0.01309 (-0.03668) + | > log_mle: -0.17138 (-0.19022) + | > loss_dur: 0.15829 (0.15354) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.11135 (6.78660) + | > current_lr: 0.00003 + | > step_time: 1.21660 (2.42965) + | > loader_time: 0.00280 (0.04147) + + + --> STEP: 74/234 -- GLOBAL_STEP: 24410 + | > loss: -0.04166 (-0.03585) + | > log_mle: -0.18631 (-0.19085) + | > loss_dur: 0.14465 (0.15499) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.92289 (7.21681) + | > current_lr: 0.00003 + | > step_time: 1.07390 (2.36322) + | > loader_time: 0.00220 (0.03881) + + + --> STEP: 79/234 -- GLOBAL_STEP: 24415 + | > loss: -0.05364 (-0.03598) + | > log_mle: -0.20006 (-0.19134) + | > loss_dur: 0.14643 (0.15536) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.01227 (7.32162) + | > current_lr: 0.00003 + | > step_time: 2.51320 (2.32265) + | > loader_time: 0.00290 (0.03865) + + + --> STEP: 84/234 -- GLOBAL_STEP: 24420 + | > loss: -0.03525 (-0.03644) + | > log_mle: -0.19847 (-0.19188) + | > loss_dur: 0.16322 (0.15545) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.28576 (7.45522) + | > current_lr: 0.00003 + | > step_time: 2.80500 (2.32251) + | > loader_time: 0.08490 (0.03969) + + + --> STEP: 89/234 -- GLOBAL_STEP: 24425 + | > loss: -0.07186 (-0.03733) + | > log_mle: -0.23041 (-0.19347) + | > loss_dur: 0.15855 (0.15614) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.41476 (7.65686) + | > current_lr: 0.00003 + | > step_time: 1.37460 (2.29290) + | > loader_time: 0.00210 (0.03872) + + + --> STEP: 94/234 -- GLOBAL_STEP: 24430 + | > loss: -0.09398 (-0.03896) + | > log_mle: -0.26423 (-0.19626) + | > loss_dur: 0.17025 (0.15730) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.43191 (8.22972) + | > current_lr: 0.00003 + | > step_time: 0.91090 (2.26501) + | > loader_time: 0.06630 (0.03923) + + + --> STEP: 99/234 -- GLOBAL_STEP: 24435 + | > loss: -0.10497 (-0.04060) + | > log_mle: -0.29742 (-0.19905) + | > loss_dur: 0.19245 (0.15845) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.26282 (8.74594) + | > current_lr: 0.00003 + | > step_time: 2.70220 (2.23964) + | > loader_time: 0.00360 (0.03825) + + + --> STEP: 104/234 -- GLOBAL_STEP: 24440 + | > loss: -0.12002 (-0.04254) + | > log_mle: -0.30970 (-0.20232) + | > loss_dur: 0.18967 (0.15978) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.84110 (9.27389) + | > current_lr: 0.00003 + | > step_time: 1.42620 (2.22692) + | > loader_time: 0.00230 (0.03654) + + + --> STEP: 109/234 -- GLOBAL_STEP: 24445 + | > loss: -0.05426 (-0.04339) + | > log_mle: -0.28092 (-0.20468) + | > loss_dur: 0.22666 (0.16129) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.39663 (9.85759) + | > current_lr: 0.00003 + | > step_time: 2.52470 (2.20573) + | > loader_time: 0.08500 (0.03651) + + + --> STEP: 114/234 -- GLOBAL_STEP: 24450 + | > loss: -0.08360 (-0.04524) + | > log_mle: -0.26224 (-0.20779) + | > loss_dur: 0.17864 (0.16255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.91828 (10.51616) + | > current_lr: 0.00003 + | > step_time: 2.60700 (2.19856) + | > loader_time: 0.08530 (0.03648) + + + --> STEP: 119/234 -- GLOBAL_STEP: 24455 + | > loss: -0.07110 (-0.04608) + | > log_mle: -0.26105 (-0.21027) + | > loss_dur: 0.18995 (0.16420) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.04899 (10.81016) + | > current_lr: 0.00003 + | > step_time: 1.61700 (2.19790) + | > loader_time: 0.00300 (0.03765) + + + --> STEP: 124/234 -- GLOBAL_STEP: 24460 + | > loss: -0.10928 (-0.04728) + | > log_mle: -0.28989 (-0.21206) + | > loss_dur: 0.18060 (0.16478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.15674 (11.03722) + | > current_lr: 0.00003 + | > step_time: 2.30550 (2.17551) + | > loader_time: 0.00310 (0.03693) + + + --> STEP: 129/234 -- GLOBAL_STEP: 24465 + | > loss: -0.08145 (-0.04928) + | > log_mle: -0.27637 (-0.21534) + | > loss_dur: 0.19492 (0.16606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.09013 (11.78648) + | > current_lr: 0.00003 + | > step_time: 4.00530 (2.16923) + | > loader_time: 0.18570 (0.03768) + + + --> STEP: 134/234 -- GLOBAL_STEP: 24470 + | > loss: -0.10613 (-0.05154) + | > log_mle: -0.32997 (-0.21922) + | > loss_dur: 0.22383 (0.16768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.67799 (12.37799) + | > current_lr: 0.00003 + | > step_time: 1.40080 (2.16057) + | > loader_time: 0.08340 (0.03700) + + + --> STEP: 139/234 -- GLOBAL_STEP: 24475 + | > loss: -0.17182 (-0.05372) + | > log_mle: -0.38519 (-0.22288) + | > loss_dur: 0.21337 (0.16917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.71352 (13.13924) + | > current_lr: 0.00003 + | > step_time: 2.09550 (2.16628) + | > loader_time: 0.09180 (0.03827) + + + --> STEP: 144/234 -- GLOBAL_STEP: 24480 + | > loss: -0.13697 (-0.05560) + | > log_mle: -0.36038 (-0.22670) + | > loss_dur: 0.22341 (0.17110) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.89470 (14.10943) + | > current_lr: 0.00003 + | > step_time: 1.90660 (2.18268) + | > loader_time: 0.08620 (0.03825) + + + --> STEP: 149/234 -- GLOBAL_STEP: 24485 + | > loss: -0.17707 (-0.05861) + | > log_mle: -0.40981 (-0.23108) + | > loss_dur: 0.23274 (0.17247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.06762 (14.84945) + | > current_lr: 0.00003 + | > step_time: 5.09510 (2.20092) + | > loader_time: 0.00330 (0.03814) + + + --> STEP: 154/234 -- GLOBAL_STEP: 24490 + | > loss: -0.16052 (-0.06184) + | > log_mle: -0.36770 (-0.23586) + | > loss_dur: 0.20718 (0.17402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.24824 (15.72481) + | > current_lr: 0.00003 + | > step_time: 2.60410 (2.22631) + | > loader_time: 0.00320 (0.03824) + + + --> STEP: 159/234 -- GLOBAL_STEP: 24495 + | > loss: -0.16763 (-0.06474) + | > log_mle: -0.38706 (-0.24042) + | > loss_dur: 0.21943 (0.17568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.74141 (16.69770) + | > current_lr: 0.00003 + | > step_time: 1.81250 (2.21183) + | > loader_time: 0.08690 (0.03928) + + + --> STEP: 164/234 -- GLOBAL_STEP: 24500 + | > loss: -0.15845 (-0.06775) + | > log_mle: -0.38116 (-0.24482) + | > loss_dur: 0.22270 (0.17707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.12090 (17.60948) + | > current_lr: 0.00003 + | > step_time: 2.29960 (2.25420) + | > loader_time: 0.07810 (0.03978) + + + --> STEP: 169/234 -- GLOBAL_STEP: 24505 + | > loss: -0.14295 (-0.07078) + | > log_mle: -0.37929 (-0.24941) + | > loss_dur: 0.23634 (0.17863) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.35971 (18.40913) + | > current_lr: 0.00003 + | > step_time: 1.90460 (2.23809) + | > loader_time: 0.00230 (0.03967) + + + --> STEP: 174/234 -- GLOBAL_STEP: 24510 + | > loss: -0.23205 (-0.07462) + | > log_mle: -0.46860 (-0.25503) + | > loss_dur: 0.23655 (0.18042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.23394 (19.69445) + | > current_lr: 0.00003 + | > step_time: 2.60660 (2.29189) + | > loader_time: 0.09270 (0.04063) + + + --> STEP: 179/234 -- GLOBAL_STEP: 24515 + | > loss: -0.18920 (-0.07759) + | > log_mle: -0.45609 (-0.26004) + | > loss_dur: 0.26689 (0.18245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.26721 (20.82342) + | > current_lr: 0.00003 + | > step_time: 6.29520 (2.39650) + | > loader_time: 0.00370 (0.04068) + + + --> STEP: 184/234 -- GLOBAL_STEP: 24520 + | > loss: -0.18047 (-0.08042) + | > log_mle: -0.42698 (-0.26456) + | > loss_dur: 0.24650 (0.18414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.68151 (21.68956) + | > current_lr: 0.00003 + | > step_time: 6.80250 (2.47433) + | > loader_time: 0.18980 (0.04219) + + + --> STEP: 189/234 -- GLOBAL_STEP: 24525 + | > loss: -0.16819 (-0.08337) + | > log_mle: -0.42654 (-0.26926) + | > loss_dur: 0.25835 (0.18589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.73139 (22.66116) + | > current_lr: 0.00003 + | > step_time: 7.50270 (2.53154) + | > loader_time: 0.09820 (0.04312) + + + --> STEP: 194/234 -- GLOBAL_STEP: 24530 + | > loss: -0.22504 (-0.08684) + | > log_mle: -0.46380 (-0.27402) + | > loss_dur: 0.23876 (0.18718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.69333 (23.53044) + | > current_lr: 0.00003 + | > step_time: 8.50270 (2.62989) + | > loader_time: 0.10610 (0.04557) + + + --> STEP: 199/234 -- GLOBAL_STEP: 24535 + | > loss: -0.21151 (-0.08969) + | > log_mle: -0.46811 (-0.27842) + | > loss_dur: 0.25660 (0.18873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.44296 (24.45279) + | > current_lr: 0.00003 + | > step_time: 1.59890 (2.66578) + | > loader_time: 0.00420 (0.04537) + + + --> STEP: 204/234 -- GLOBAL_STEP: 24540 + | > loss: -0.23050 (-0.09234) + | > log_mle: -0.49780 (-0.28273) + | > loss_dur: 0.26730 (0.19039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.95208 (25.38225) + | > current_lr: 0.00003 + | > step_time: 5.10250 (2.71716) + | > loader_time: 0.10050 (0.04609) + + + --> STEP: 209/234 -- GLOBAL_STEP: 24545 + | > loss: -0.19691 (-0.09532) + | > log_mle: -0.45284 (-0.28729) + | > loss_dur: 0.25593 (0.19197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.31349 (26.46563) + | > current_lr: 0.00003 + | > step_time: 5.70640 (2.76468) + | > loader_time: 0.00810 (0.04558) + + + --> STEP: 214/234 -- GLOBAL_STEP: 24550 + | > loss: -0.25126 (-0.09908) + | > log_mle: -0.49106 (-0.29267) + | > loss_dur: 0.23980 (0.19359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.73826 (27.63653) + | > current_lr: 0.00003 + | > step_time: 4.49150 (2.88745) + | > loader_time: 0.00450 (0.04644) + + + --> STEP: 219/234 -- GLOBAL_STEP: 24555 + | > loss: -0.31551 (-0.10270) + | > log_mle: -0.58177 (-0.29789) + | > loss_dur: 0.26626 (0.19518) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.15105 (28.71439) + | > current_lr: 0.00003 + | > step_time: 5.49820 (2.96862) + | > loader_time: 0.00280 (0.04765) + + + --> STEP: 224/234 -- GLOBAL_STEP: 24560 + | > loss: -0.24246 (-0.10581) + | > log_mle: -0.51984 (-0.30268) + | > loss_dur: 0.27739 (0.19687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 129.66519 (29.93649) + | > current_lr: 0.00003 + | > step_time: 0.23190 (2.95516) + | > loader_time: 0.00450 (0.04705) + + + --> STEP: 229/234 -- GLOBAL_STEP: 24565 + | > loss: -0.22581 (-0.10887) + | > log_mle: -0.56071 (-0.30787) + | > loss_dur: 0.33491 (0.19901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.01393 (31.30864) + | > current_lr: 0.00003 + | > step_time: 0.25110 (2.89605) + | > loader_time: 0.00580 (0.04610) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.69992 (+0.69627) + | > avg_loss: -0.15968 (-0.00459) + | > avg_log_mle: -0.40223 (-0.01009) + | > avg_loss_dur: 0.24254 (+0.00550) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_24570.pth + + > EPOCH: 105/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 07:24:04)  + + --> STEP: 0/234 -- GLOBAL_STEP: 24570 + | > loss: -0.07942 (-0.07942) + | > log_mle: -0.25124 (-0.25124) + | > loss_dur: 0.17182 (0.17182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.16580 (11.16580) + | > current_lr: 0.00003 + | > step_time: 2.70410 (2.70408) + | > loader_time: 12.43660 (12.43660) + + + --> STEP: 5/234 -- GLOBAL_STEP: 24575 + | > loss: -0.03269 (-0.01471) + | > log_mle: -0.19658 (-0.19194) + | > loss_dur: 0.16389 (0.17723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.95763 (8.73234) + | > current_lr: 0.00003 + | > step_time: 5.18750 (5.29373) + | > loader_time: 0.00850 (3.14414) + + + --> STEP: 10/234 -- GLOBAL_STEP: 24580 + | > loss: -0.00677 (-0.02352) + | > log_mle: -0.19100 (-0.19470) + | > loss_dur: 0.18423 (0.17118) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.96280 (8.51222) + | > current_lr: 0.00003 + | > step_time: 2.63560 (3.61247) + | > loader_time: 0.00200 (1.58209) + + + --> STEP: 15/234 -- GLOBAL_STEP: 24585 + | > loss: -0.06104 (-0.02898) + | > log_mle: -0.19400 (-0.19352) + | > loss_dur: 0.13296 (0.16454) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.05322 (8.22864) + | > current_lr: 0.00003 + | > step_time: 4.39060 (2.98405) + | > loader_time: 0.10240 (1.06194) + + + --> STEP: 20/234 -- GLOBAL_STEP: 24590 + | > loss: -0.04107 (-0.03083) + | > log_mle: -0.17793 (-0.19029) + | > loss_dur: 0.13686 (0.15945) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.32422 (7.75001) + | > current_lr: 0.00003 + | > step_time: 3.79260 (3.13222) + | > loader_time: 0.79980 (0.84093) + + + --> STEP: 25/234 -- GLOBAL_STEP: 24595 + | > loss: -0.02807 (-0.03343) + | > log_mle: -0.17481 (-0.18873) + | > loss_dur: 0.14674 (0.15530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.17814 (7.44886) + | > current_lr: 0.00003 + | > step_time: 3.19260 (3.35773) + | > loader_time: 0.10220 (0.68507) + + + --> STEP: 30/234 -- GLOBAL_STEP: 24600 + | > loss: -0.06502 (-0.03629) + | > log_mle: -0.19772 (-0.18863) + | > loss_dur: 0.13270 (0.15234) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.48540 (7.42167) + | > current_lr: 0.00003 + | > step_time: 1.08770 (3.36463) + | > loader_time: 0.00180 (0.57408) + + + --> STEP: 35/234 -- GLOBAL_STEP: 24605 + | > loss: -0.03111 (-0.03573) + | > log_mle: -0.19491 (-0.18898) + | > loss_dur: 0.16380 (0.15325) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.54058 (7.48485) + | > current_lr: 0.00003 + | > step_time: 0.64810 (3.10701) + | > loader_time: 0.00140 (0.49510) + + + --> STEP: 40/234 -- GLOBAL_STEP: 24610 + | > loss: 0.00036 (-0.03547) + | > log_mle: -0.17541 (-0.18914) + | > loss_dur: 0.17577 (0.15367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.04828 (7.36268) + | > current_lr: 0.00003 + | > step_time: 1.58530 (2.92780) + | > loader_time: 0.02390 (0.43407) + + + --> STEP: 45/234 -- GLOBAL_STEP: 24615 + | > loss: -0.05481 (-0.03626) + | > log_mle: -0.21153 (-0.18940) + | > loss_dur: 0.15672 (0.15314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99597 (7.34732) + | > current_lr: 0.00003 + | > step_time: 1.27930 (2.76560) + | > loader_time: 0.00270 (0.38611) + + + --> STEP: 50/234 -- GLOBAL_STEP: 24620 + | > loss: -0.02226 (-0.03634) + | > log_mle: -0.17982 (-0.18913) + | > loss_dur: 0.15756 (0.15280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.56724 (7.17819) + | > current_lr: 0.00003 + | > step_time: 1.43860 (2.73624) + | > loader_time: 0.00120 (0.35300) + + + --> STEP: 55/234 -- GLOBAL_STEP: 24625 + | > loss: -0.06417 (-0.03720) + | > log_mle: -0.20106 (-0.18956) + | > loss_dur: 0.13690 (0.15236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.69858 (7.05704) + | > current_lr: 0.00003 + | > step_time: 0.76860 (2.61864) + | > loader_time: 0.00190 (0.32268) + + + --> STEP: 60/234 -- GLOBAL_STEP: 24630 + | > loss: -0.05787 (-0.03775) + | > log_mle: -0.21721 (-0.19032) + | > loss_dur: 0.15934 (0.15257) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.92512 (7.08924) + | > current_lr: 0.00003 + | > step_time: 1.87210 (2.52807) + | > loader_time: 0.00280 (0.29740) + + + --> STEP: 65/234 -- GLOBAL_STEP: 24635 + | > loss: -0.04760 (-0.03763) + | > log_mle: -0.19158 (-0.19100) + | > loss_dur: 0.14398 (0.15336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.52144 (7.33826) + | > current_lr: 0.00003 + | > step_time: 2.13670 (2.45024) + | > loader_time: 0.00200 (0.27598) + + + --> STEP: 70/234 -- GLOBAL_STEP: 24640 + | > loss: -0.02197 (-0.03696) + | > log_mle: -0.19211 (-0.19097) + | > loss_dur: 0.17014 (0.15401) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.53044 (7.39457) + | > current_lr: 0.00003 + | > step_time: 1.20100 (2.38868) + | > loader_time: 0.00220 (0.25645) + + + --> STEP: 75/234 -- GLOBAL_STEP: 24645 + | > loss: -0.03333 (-0.03656) + | > log_mle: -0.20507 (-0.19185) + | > loss_dur: 0.17173 (0.15529) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.21502 (7.58888) + | > current_lr: 0.00003 + | > step_time: 1.19990 (2.35701) + | > loader_time: 0.00210 (0.23952) + + + --> STEP: 80/234 -- GLOBAL_STEP: 24650 + | > loss: -0.04637 (-0.03698) + | > log_mle: -0.18572 (-0.19224) + | > loss_dur: 0.13934 (0.15527) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.71680 (7.52863) + | > current_lr: 0.00003 + | > step_time: 1.99840 (2.29926) + | > loader_time: 0.00530 (0.22474) + + + --> STEP: 85/234 -- GLOBAL_STEP: 24655 + | > loss: -0.05142 (-0.03772) + | > log_mle: -0.20255 (-0.19304) + | > loss_dur: 0.15113 (0.15532) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.04778 (7.63711) + | > current_lr: 0.00003 + | > step_time: 3.80110 (2.28844) + | > loader_time: 0.00280 (0.21252) + + + --> STEP: 90/234 -- GLOBAL_STEP: 24660 + | > loss: -0.04716 (-0.03858) + | > log_mle: -0.23088 (-0.19495) + | > loss_dur: 0.18373 (0.15638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.81010 (7.93988) + | > current_lr: 0.00003 + | > step_time: 2.12030 (2.29862) + | > loader_time: 0.00340 (0.20392) + + + --> STEP: 95/234 -- GLOBAL_STEP: 24665 + | > loss: -0.11824 (-0.04090) + | > log_mle: -0.31634 (-0.19866) + | > loss_dur: 0.19810 (0.15775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.26230 (8.62106) + | > current_lr: 0.00003 + | > step_time: 1.19230 (2.27611) + | > loader_time: 0.00220 (0.19422) + + + --> STEP: 100/234 -- GLOBAL_STEP: 24670 + | > loss: -0.06301 (-0.04180) + | > log_mle: -0.24399 (-0.20059) + | > loss_dur: 0.18099 (0.15879) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.44146 (8.98950) + | > current_lr: 0.00003 + | > step_time: 1.00040 (2.25153) + | > loader_time: 0.00220 (0.18642) + + + --> STEP: 105/234 -- GLOBAL_STEP: 24675 + | > loss: -0.06057 (-0.04363) + | > log_mle: -0.21906 (-0.20360) + | > loss_dur: 0.15850 (0.15997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.97542 (9.45723) + | > current_lr: 0.00003 + | > step_time: 2.60730 (2.33573) + | > loader_time: 0.00330 (0.18319) + + + --> STEP: 110/234 -- GLOBAL_STEP: 24680 + | > loss: -0.08057 (-0.04465) + | > log_mle: -0.24649 (-0.20626) + | > loss_dur: 0.16592 (0.16161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.02031 (10.01489) + | > current_lr: 0.00003 + | > step_time: 1.80510 (2.32409) + | > loader_time: 0.07810 (0.17650) + + + --> STEP: 115/234 -- GLOBAL_STEP: 24685 + | > loss: -0.06722 (-0.04646) + | > log_mle: -0.26576 (-0.20960) + | > loss_dur: 0.19854 (0.16314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.45700 (10.48627) + | > current_lr: 0.00003 + | > step_time: 2.31180 (2.30142) + | > loader_time: 0.00250 (0.16981) + + + --> STEP: 120/234 -- GLOBAL_STEP: 24690 + | > loss: -0.11029 (-0.04794) + | > log_mle: -0.31503 (-0.21248) + | > loss_dur: 0.20474 (0.16454) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.46395 (10.81304) + | > current_lr: 0.00003 + | > step_time: 3.00190 (2.32216) + | > loader_time: 0.08760 (0.16426) + + + --> STEP: 125/234 -- GLOBAL_STEP: 24695 + | > loss: -0.10974 (-0.04907) + | > log_mle: -0.30347 (-0.21416) + | > loss_dur: 0.19374 (0.16509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.13913 (11.08772) + | > current_lr: 0.00003 + | > step_time: 1.00570 (2.28459) + | > loader_time: 0.00320 (0.15909) + + + --> STEP: 130/234 -- GLOBAL_STEP: 24700 + | > loss: -0.10572 (-0.05101) + | > log_mle: -0.31325 (-0.21758) + | > loss_dur: 0.20753 (0.16657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.99968 (11.74230) + | > current_lr: 0.00003 + | > step_time: 0.90220 (2.26449) + | > loader_time: 0.00350 (0.15377) + + + --> STEP: 135/234 -- GLOBAL_STEP: 24705 + | > loss: -0.06739 (-0.05308) + | > log_mle: -0.24408 (-0.22092) + | > loss_dur: 0.17669 (0.16783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.34264 (12.39949) + | > current_lr: 0.00003 + | > step_time: 2.09050 (2.23686) + | > loader_time: 0.00270 (0.14873) + + + --> STEP: 140/234 -- GLOBAL_STEP: 24710 + | > loss: -0.07215 (-0.05529) + | > log_mle: -0.27795 (-0.22478) + | > loss_dur: 0.20580 (0.16949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.38017 (13.31004) + | > current_lr: 0.00003 + | > step_time: 3.21980 (2.23348) + | > loader_time: 0.08410 (0.14539) + + + --> STEP: 145/234 -- GLOBAL_STEP: 24715 + | > loss: -0.15451 (-0.05800) + | > log_mle: -0.37389 (-0.22934) + | > loss_dur: 0.21939 (0.17134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.66460 (14.17470) + | > current_lr: 0.00003 + | > step_time: 0.81610 (2.21091) + | > loader_time: 0.08560 (0.14230) + + + --> STEP: 150/234 -- GLOBAL_STEP: 24720 + | > loss: -0.13875 (-0.06086) + | > log_mle: -0.35972 (-0.23361) + | > loss_dur: 0.22097 (0.17275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.04834 (14.83583) + | > current_lr: 0.00003 + | > step_time: 7.01100 (2.26989) + | > loader_time: 0.19780 (0.13958) + + + --> STEP: 155/234 -- GLOBAL_STEP: 24725 + | > loss: -0.19203 (-0.06451) + | > log_mle: -0.42211 (-0.23882) + | > loss_dur: 0.23008 (0.17431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.28603 (15.81623) + | > current_lr: 0.00003 + | > step_time: 2.60420 (2.34310) + | > loader_time: 0.08420 (0.13889) + + + --> STEP: 160/234 -- GLOBAL_STEP: 24730 + | > loss: -0.17166 (-0.06725) + | > log_mle: -0.40499 (-0.24310) + | > loss_dur: 0.23332 (0.17585) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.97957 (17.31337) + | > current_lr: 0.00003 + | > step_time: 7.91060 (2.41310) + | > loader_time: 0.08700 (0.13520) + + + --> STEP: 165/234 -- GLOBAL_STEP: 24735 + | > loss: -0.16901 (-0.06998) + | > log_mle: -0.41334 (-0.24730) + | > loss_dur: 0.24432 (0.17732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.53069 (18.30285) + | > current_lr: 0.00003 + | > step_time: 7.90510 (2.57881) + | > loader_time: 0.19980 (0.13648) + + + --> STEP: 170/234 -- GLOBAL_STEP: 24740 + | > loss: -0.19155 (-0.07298) + | > log_mle: -0.45554 (-0.25200) + | > loss_dur: 0.26398 (0.17903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.90822 (19.06775) + | > current_lr: 0.00003 + | > step_time: 1.50940 (2.57593) + | > loader_time: 0.08630 (0.13349) + + + --> STEP: 175/234 -- GLOBAL_STEP: 24745 + | > loss: -0.17665 (-0.07677) + | > log_mle: -0.42757 (-0.25741) + | > loss_dur: 0.25093 (0.18064) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.75093 (20.20926) + | > current_lr: 0.00003 + | > step_time: 1.50610 (2.59036) + | > loader_time: 0.08090 (0.13077) + + + --> STEP: 180/234 -- GLOBAL_STEP: 24750 + | > loss: -0.19116 (-0.07992) + | > log_mle: -0.43363 (-0.26244) + | > loss_dur: 0.24248 (0.18252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.57327 (21.30719) + | > current_lr: 0.00003 + | > step_time: 1.11410 (2.62022) + | > loader_time: 0.18380 (0.13196) + + + --> STEP: 185/234 -- GLOBAL_STEP: 24755 + | > loss: -0.19460 (-0.08274) + | > log_mle: -0.45521 (-0.26701) + | > loss_dur: 0.26061 (0.18427) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.40374 (22.33054) + | > current_lr: 0.00003 + | > step_time: 4.90050 (2.64526) + | > loader_time: 0.00480 (0.12950) + + + --> STEP: 190/234 -- GLOBAL_STEP: 24760 + | > loss: -0.20651 (-0.08570) + | > log_mle: -0.43826 (-0.27153) + | > loss_dur: 0.23174 (0.18582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.11512 (23.37302) + | > current_lr: 0.00003 + | > step_time: 1.70640 (2.64046) + | > loader_time: 0.08280 (0.12842) + + + --> STEP: 195/234 -- GLOBAL_STEP: 24765 + | > loss: -0.19670 (-0.08907) + | > log_mle: -0.45406 (-0.27631) + | > loss_dur: 0.25737 (0.18725) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.87072 (24.27829) + | > current_lr: 0.00003 + | > step_time: 3.09910 (2.68037) + | > loader_time: 0.09060 (0.12674) + + + --> STEP: 200/234 -- GLOBAL_STEP: 24770 + | > loss: -0.19148 (-0.09181) + | > log_mle: -0.45877 (-0.28066) + | > loss_dur: 0.26730 (0.18885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.41081 (25.29367) + | > current_lr: 0.00003 + | > step_time: 11.80900 (2.79596) + | > loader_time: 0.09800 (0.12692) + + + --> STEP: 205/234 -- GLOBAL_STEP: 24775 + | > loss: -0.18900 (-0.09440) + | > log_mle: -0.44314 (-0.28481) + | > loss_dur: 0.25415 (0.19040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.75604 (26.25884) + | > current_lr: 0.00003 + | > step_time: 2.89220 (2.88287) + | > loader_time: 0.09120 (0.12668) + + + --> STEP: 210/234 -- GLOBAL_STEP: 24780 + | > loss: -0.26051 (-0.09765) + | > log_mle: -0.52424 (-0.28971) + | > loss_dur: 0.26373 (0.19206) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.03170 (27.38187) + | > current_lr: 0.00003 + | > step_time: 6.10040 (2.94413) + | > loader_time: 0.19440 (0.12609) + + + --> STEP: 215/234 -- GLOBAL_STEP: 24785 + | > loss: -0.21985 (-0.10116) + | > log_mle: -0.47359 (-0.29481) + | > loss_dur: 0.25375 (0.19365) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.59253 (28.55485) + | > current_lr: 0.00003 + | > step_time: 6.70880 (3.01205) + | > loader_time: 0.09320 (0.12728) + + + --> STEP: 220/234 -- GLOBAL_STEP: 24790 + | > loss: -0.25521 (-0.10481) + | > log_mle: -0.52120 (-0.30015) + | > loss_dur: 0.26598 (0.19534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.17740 (29.77243) + | > current_lr: 0.00003 + | > step_time: 2.60840 (3.04275) + | > loader_time: 0.00420 (0.12618) + + + --> STEP: 225/234 -- GLOBAL_STEP: 24795 + | > loss: -0.30360 (-0.10821) + | > log_mle: -0.58660 (-0.30528) + | > loss_dur: 0.28300 (0.19708) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.04037 (30.77050) + | > current_lr: 0.00003 + | > step_time: 0.23800 (3.00878) + | > loader_time: 0.00300 (0.12449) + + + --> STEP: 230/234 -- GLOBAL_STEP: 24800 + | > loss: -0.27577 (-0.11133) + | > log_mle: -0.63846 (-0.31087) + | > loss_dur: 0.36269 (0.19954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.74963 (31.95620) + | > current_lr: 0.00003 + | > step_time: 0.24890 (2.94906) + | > loader_time: 0.00400 (0.12187) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.75545 (+0.05554) + | > avg_loss: -0.14568 (+0.01400) + | > avg_log_mle: -0.38655 (+0.01568) + | > avg_loss_dur: 0.24087 (-0.00167) + + + > EPOCH: 106/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 07:36:52)  + + --> STEP: 1/234 -- GLOBAL_STEP: 24805 + | > loss: -0.05381 (-0.05381) + | > log_mle: -0.19319 (-0.19319) + | > loss_dur: 0.13937 (0.13937) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.44531 (18.44531) + | > current_lr: 0.00003 + | > step_time: 2.80070 (2.80066) + | > loader_time: 0.29600 (0.29600) + + + --> STEP: 6/234 -- GLOBAL_STEP: 24810 + | > loss: -0.03673 (-0.02308) + | > log_mle: -0.18543 (-0.19214) + | > loss_dur: 0.14870 (0.16905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.17391 (11.21589) + | > current_lr: 0.00003 + | > step_time: 4.60040 (3.26526) + | > loader_time: 0.09490 (0.08298) + + + --> STEP: 11/234 -- GLOBAL_STEP: 24815 + | > loss: -0.05413 (-0.03074) + | > log_mle: -0.19180 (-0.19655) + | > loss_dur: 0.13766 (0.16581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.06464 (9.76719) + | > current_lr: 0.00003 + | > step_time: 3.20940 (4.64595) + | > loader_time: 0.00120 (0.12702) + + + --> STEP: 16/234 -- GLOBAL_STEP: 24820 + | > loss: -0.08975 (-0.03752) + | > log_mle: -0.19288 (-0.19584) + | > loss_dur: 0.10313 (0.15832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.22575 (8.63022) + | > current_lr: 0.00003 + | > step_time: 8.00070 (4.75654) + | > loader_time: 0.19460 (0.11188) + + + --> STEP: 21/234 -- GLOBAL_STEP: 24825 + | > loss: -0.02050 (-0.03707) + | > log_mle: -0.17394 (-0.19214) + | > loss_dur: 0.15344 (0.15507) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.93810 (8.20745) + | > current_lr: 0.00003 + | > step_time: 5.40760 (4.81371) + | > loader_time: 0.00270 (0.08592) + + + --> STEP: 26/234 -- GLOBAL_STEP: 24830 + | > loss: -0.04045 (-0.03918) + | > log_mle: -0.19074 (-0.19146) + | > loss_dur: 0.15029 (0.15228) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.36374 (7.93182) + | > current_lr: 0.00003 + | > step_time: 5.30550 (4.61840) + | > loader_time: 0.09440 (0.08417) + + + --> STEP: 31/234 -- GLOBAL_STEP: 24835 + | > loss: -0.00609 (-0.04033) + | > log_mle: -0.19395 (-0.19148) + | > loss_dur: 0.18785 (0.15115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.16745 (7.62168) + | > current_lr: 0.00003 + | > step_time: 4.00530 (4.45757) + | > loader_time: 0.00400 (0.08037) + + + --> STEP: 36/234 -- GLOBAL_STEP: 24840 + | > loss: -0.02508 (-0.03947) + | > log_mle: -0.19597 (-0.19179) + | > loss_dur: 0.17089 (0.15232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.11919 (7.80306) + | > current_lr: 0.00003 + | > step_time: 1.51400 (4.36167) + | > loader_time: 0.08210 (0.08127) + + + --> STEP: 41/234 -- GLOBAL_STEP: 24845 + | > loss: -0.05496 (-0.03948) + | > log_mle: -0.19050 (-0.19165) + | > loss_dur: 0.13554 (0.15217) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.55950 (7.63923) + | > current_lr: 0.00003 + | > step_time: 1.07580 (4.04141) + | > loader_time: 0.00240 (0.07371) + + + --> STEP: 46/234 -- GLOBAL_STEP: 24850 + | > loss: -0.03808 (-0.03973) + | > log_mle: -0.19329 (-0.19205) + | > loss_dur: 0.15521 (0.15233) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.87233 (7.53726) + | > current_lr: 0.00003 + | > step_time: 1.16610 (3.78572) + | > loader_time: 0.00140 (0.06784) + + + --> STEP: 51/234 -- GLOBAL_STEP: 24855 + | > loss: -0.02579 (-0.03956) + | > log_mle: -0.17883 (-0.19141) + | > loss_dur: 0.15304 (0.15185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.14475 (7.28597) + | > current_lr: 0.00003 + | > step_time: 1.15890 (3.59430) + | > loader_time: 0.00280 (0.06144) + + + --> STEP: 56/234 -- GLOBAL_STEP: 24860 + | > loss: -0.01419 (-0.03984) + | > log_mle: -0.19891 (-0.19215) + | > loss_dur: 0.18471 (0.15231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.33928 (7.13622) + | > current_lr: 0.00003 + | > step_time: 1.20850 (3.41629) + | > loader_time: 0.00140 (0.05615) + + + --> STEP: 61/234 -- GLOBAL_STEP: 24865 + | > loss: -0.05410 (-0.04107) + | > log_mle: -0.19543 (-0.19276) + | > loss_dur: 0.14132 (0.15169) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.57098 (7.08737) + | > current_lr: 0.00003 + | > step_time: 2.05320 (3.30658) + | > loader_time: 0.00160 (0.05475) + + + --> STEP: 66/234 -- GLOBAL_STEP: 24870 + | > loss: -0.03682 (-0.04070) + | > log_mle: -0.18395 (-0.19315) + | > loss_dur: 0.14713 (0.15245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.46291 (7.23282) + | > current_lr: 0.00003 + | > step_time: 1.98080 (3.20285) + | > loader_time: 0.00150 (0.05078) + + + --> STEP: 71/234 -- GLOBAL_STEP: 24875 + | > loss: -0.03081 (-0.03990) + | > log_mle: -0.22853 (-0.19366) + | > loss_dur: 0.19772 (0.15376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.61209 (7.39679) + | > current_lr: 0.00003 + | > step_time: 2.08270 (3.11052) + | > loader_time: 0.00190 (0.04874) + + + --> STEP: 76/234 -- GLOBAL_STEP: 24880 + | > loss: -0.04499 (-0.03964) + | > log_mle: -0.20883 (-0.19415) + | > loss_dur: 0.16384 (0.15451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.09290 (7.53912) + | > current_lr: 0.00003 + | > step_time: 2.19630 (3.01924) + | > loader_time: 0.00230 (0.04573) + + + --> STEP: 81/234 -- GLOBAL_STEP: 24885 + | > loss: -0.05272 (-0.04007) + | > log_mle: -0.21474 (-0.19450) + | > loss_dur: 0.16202 (0.15442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.07019 (7.56960) + | > current_lr: 0.00003 + | > step_time: 2.43730 (2.98271) + | > loader_time: 0.00380 (0.04524) + + + --> STEP: 86/234 -- GLOBAL_STEP: 24890 + | > loss: -0.04406 (-0.04025) + | > log_mle: -0.21312 (-0.19515) + | > loss_dur: 0.16906 (0.15490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.48754 (7.72325) + | > current_lr: 0.00003 + | > step_time: 5.12150 (2.94220) + | > loader_time: 0.39940 (0.04781) + + + --> STEP: 91/234 -- GLOBAL_STEP: 24895 + | > loss: -0.03589 (-0.04086) + | > log_mle: -0.22312 (-0.19689) + | > loss_dur: 0.18723 (0.15603) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.07097 (8.08628) + | > current_lr: 0.00003 + | > step_time: 3.60820 (2.89011) + | > loader_time: 0.00410 (0.04628) + + + --> STEP: 96/234 -- GLOBAL_STEP: 24900 + | > loss: -0.04587 (-0.04308) + | > log_mle: -0.21423 (-0.20026) + | > loss_dur: 0.16836 (0.15718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.45516 (8.71636) + | > current_lr: 0.00003 + | > step_time: 1.79740 (2.83142) + | > loader_time: 0.10900 (0.04670) + + + --> STEP: 101/234 -- GLOBAL_STEP: 24905 + | > loss: -0.06896 (-0.04426) + | > log_mle: -0.26428 (-0.20259) + | > loss_dur: 0.19532 (0.15833) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.08104 (9.18937) + | > current_lr: 0.00003 + | > step_time: 2.50660 (2.79469) + | > loader_time: 0.09460 (0.04542) + + + --> STEP: 106/234 -- GLOBAL_STEP: 24910 + | > loss: -0.05926 (-0.04562) + | > log_mle: -0.26983 (-0.20530) + | > loss_dur: 0.21057 (0.15968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.81258 (9.73980) + | > current_lr: 0.00003 + | > step_time: 1.59360 (2.74654) + | > loader_time: 0.00370 (0.04505) + + + --> STEP: 111/234 -- GLOBAL_STEP: 24915 + | > loss: -0.09013 (-0.04672) + | > log_mle: -0.31325 (-0.20804) + | > loss_dur: 0.22311 (0.16132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.42638 (10.31203) + | > current_lr: 0.00003 + | > step_time: 1.78960 (2.71514) + | > loader_time: 0.00270 (0.04477) + + + --> STEP: 116/234 -- GLOBAL_STEP: 24920 + | > loss: -0.06033 (-0.04809) + | > log_mle: -0.28471 (-0.21090) + | > loss_dur: 0.22438 (0.16281) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.08645 (10.79696) + | > current_lr: 0.00003 + | > step_time: 1.70180 (2.70589) + | > loader_time: 0.08690 (0.04515) + + + --> STEP: 121/234 -- GLOBAL_STEP: 24925 + | > loss: -0.02487 (-0.04909) + | > log_mle: -0.19701 (-0.21294) + | > loss_dur: 0.17214 (0.16385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.61923 (11.02693) + | > current_lr: 0.00003 + | > step_time: 4.51200 (2.69359) + | > loader_time: 0.19040 (0.04634) + + + --> STEP: 126/234 -- GLOBAL_STEP: 24930 + | > loss: -0.11337 (-0.05077) + | > log_mle: -0.32884 (-0.21557) + | > loss_dur: 0.21547 (0.16479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.96774 (11.47506) + | > current_lr: 0.00003 + | > step_time: 7.00830 (2.73606) + | > loader_time: 0.08850 (0.04877) + + + --> STEP: 131/234 -- GLOBAL_STEP: 24935 + | > loss: -0.14649 (-0.05296) + | > log_mle: -0.37063 (-0.21918) + | > loss_dur: 0.22414 (0.16622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.68407 (12.29120) + | > current_lr: 0.00003 + | > step_time: 1.98940 (2.76682) + | > loader_time: 0.00320 (0.05198) + + + --> STEP: 136/234 -- GLOBAL_STEP: 24940 + | > loss: -0.17866 (-0.05521) + | > log_mle: -0.41534 (-0.22282) + | > loss_dur: 0.23669 (0.16761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.53704 (12.87898) + | > current_lr: 0.00003 + | > step_time: 3.40370 (2.74735) + | > loader_time: 0.08620 (0.05141) + + + --> STEP: 141/234 -- GLOBAL_STEP: 24945 + | > loss: -0.10474 (-0.05689) + | > log_mle: -0.32810 (-0.22602) + | > loss_dur: 0.22335 (0.16914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.02179 (13.46496) + | > current_lr: 0.00003 + | > step_time: 2.60510 (2.71883) + | > loader_time: 0.00380 (0.05174) + + + --> STEP: 146/234 -- GLOBAL_STEP: 24950 + | > loss: -0.16493 (-0.05988) + | > log_mle: -0.38079 (-0.23091) + | > loss_dur: 0.21586 (0.17103) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.94241 (14.30836) + | > current_lr: 0.00003 + | > step_time: 3.31300 (2.71537) + | > loader_time: 0.00730 (0.05078) + + + --> STEP: 151/234 -- GLOBAL_STEP: 24955 + | > loss: -0.14883 (-0.06255) + | > log_mle: -0.34377 (-0.23481) + | > loss_dur: 0.19494 (0.17226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.47989 (15.19532) + | > current_lr: 0.00003 + | > step_time: 3.10170 (2.70781) + | > loader_time: 0.09710 (0.05095) + + + --> STEP: 156/234 -- GLOBAL_STEP: 24960 + | > loss: -0.17460 (-0.06619) + | > log_mle: -0.38389 (-0.24013) + | > loss_dur: 0.20929 (0.17394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.82619 (16.39440) + | > current_lr: 0.00003 + | > step_time: 4.29160 (2.75809) + | > loader_time: 0.10910 (0.05075) + + + --> STEP: 161/234 -- GLOBAL_STEP: 24965 + | > loss: -0.18794 (-0.06931) + | > log_mle: -0.40828 (-0.24469) + | > loss_dur: 0.22034 (0.17538) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.46230 (17.23240) + | > current_lr: 0.00003 + | > step_time: 1.89600 (2.75251) + | > loader_time: 0.09610 (0.05037) + + + --> STEP: 166/234 -- GLOBAL_STEP: 24970 + | > loss: -0.14796 (-0.07194) + | > log_mle: -0.35340 (-0.24865) + | > loss_dur: 0.20545 (0.17671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.07512 (17.94581) + | > current_lr: 0.00003 + | > step_time: 2.21040 (2.73041) + | > loader_time: 0.08290 (0.04945) + + + --> STEP: 171/234 -- GLOBAL_STEP: 24975 + | > loss: -0.22354 (-0.07540) + | > log_mle: -0.44803 (-0.25383) + | > loss_dur: 0.22449 (0.17843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.63581 (19.35736) + | > current_lr: 0.00003 + | > step_time: 1.60520 (2.71861) + | > loader_time: 0.08030 (0.04958) + + + --> STEP: 176/234 -- GLOBAL_STEP: 24980 + | > loss: -0.18605 (-0.07882) + | > log_mle: -0.42437 (-0.25899) + | > loss_dur: 0.23832 (0.18017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.81842 (20.46625) + | > current_lr: 0.00003 + | > step_time: 0.81330 (2.71040) + | > loader_time: 0.08540 (0.05020) + + + --> STEP: 181/234 -- GLOBAL_STEP: 24985 + | > loss: -0.13208 (-0.08155) + | > log_mle: -0.36509 (-0.26360) + | > loss_dur: 0.23300 (0.18205) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.04557 (21.41289) + | > current_lr: 0.00003 + | > step_time: 10.08690 (2.76858) + | > loader_time: 0.00600 (0.04936) + + + --> STEP: 186/234 -- GLOBAL_STEP: 24990 + | > loss: -0.15344 (-0.08449) + | > log_mle: -0.40448 (-0.26845) + | > loss_dur: 0.25104 (0.18396) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.03101 (22.26538) + | > current_lr: 0.00003 + | > step_time: 1.58590 (2.83283) + | > loader_time: 0.00360 (0.04928) + + + --> STEP: 191/234 -- GLOBAL_STEP: 24995 + | > loss: -0.19838 (-0.08755) + | > log_mle: -0.42072 (-0.27302) + | > loss_dur: 0.22233 (0.18547) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.21012 (23.34332) + | > current_lr: 0.00003 + | > step_time: 3.58970 (2.83667) + | > loader_time: 0.00350 (0.04982) + + + --> STEP: 196/234 -- GLOBAL_STEP: 25000 + | > loss: -0.16703 (-0.09063) + | > log_mle: -0.41425 (-0.27763) + | > loss_dur: 0.24722 (0.18699) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.04043 (24.42897) + | > current_lr: 0.00003 + | > step_time: 5.79780 (2.85208) + | > loader_time: 0.20310 (0.05007) + + + --> STEP: 201/234 -- GLOBAL_STEP: 25005 + | > loss: -0.12435 (-0.09322) + | > log_mle: -0.38160 (-0.28183) + | > loss_dur: 0.25726 (0.18860) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.63031 (25.35673) + | > current_lr: 0.00003 + | > step_time: 1.81110 (2.90645) + | > loader_time: 0.08540 (0.05323) + + + --> STEP: 206/234 -- GLOBAL_STEP: 25010 + | > loss: -0.22752 (-0.09650) + | > log_mle: -0.48288 (-0.28658) + | > loss_dur: 0.25536 (0.19008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.60504 (26.30582) + | > current_lr: 0.00003 + | > step_time: 3.19340 (2.95681) + | > loader_time: 0.00840 (0.05626) + + + --> STEP: 211/234 -- GLOBAL_STEP: 25015 + | > loss: -0.26743 (-0.10007) + | > log_mle: -0.54850 (-0.29187) + | > loss_dur: 0.28108 (0.19180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.03865 (27.47353) + | > current_lr: 0.00003 + | > step_time: 8.90380 (3.03743) + | > loader_time: 0.29340 (0.05806) + + + --> STEP: 216/234 -- GLOBAL_STEP: 25020 + | > loss: -0.25812 (-0.10349) + | > log_mle: -0.53736 (-0.29682) + | > loss_dur: 0.27924 (0.19334) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.96567 (28.70632) + | > current_lr: 0.00003 + | > step_time: 5.49730 (3.10730) + | > loader_time: 0.00670 (0.05859) + + + --> STEP: 221/234 -- GLOBAL_STEP: 25025 + | > loss: -0.20520 (-0.10686) + | > log_mle: -0.45892 (-0.30179) + | > loss_dur: 0.25372 (0.19493) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.10767 (29.88746) + | > current_lr: 0.00003 + | > step_time: 2.69640 (3.11074) + | > loader_time: 0.00510 (0.05776) + + + --> STEP: 226/234 -- GLOBAL_STEP: 25030 + | > loss: -0.28107 (-0.11055) + | > log_mle: -0.55466 (-0.30725) + | > loss_dur: 0.27359 (0.19670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.87429 (30.99594) + | > current_lr: 0.00003 + | > step_time: 0.24100 (3.06611) + | > loader_time: 0.00390 (0.05662) + + + --> STEP: 231/234 -- GLOBAL_STEP: 25035 + | > loss: -0.20594 (-0.11331) + | > log_mle: -0.62377 (-0.31304) + | > loss_dur: 0.41783 (0.19973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.91169 (32.16451) + | > current_lr: 0.00003 + | > step_time: 0.27340 (3.00518) + | > loader_time: 0.00340 (0.05548) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00271 (-0.75275) + | > avg_loss: -0.16066 (-0.01498) + | > avg_log_mle: -0.39590 (-0.00935) + | > avg_loss_dur: 0.23524 (-0.00563) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_25038.pth + + > EPOCH: 107/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 07:49:50)  + + --> STEP: 2/234 -- GLOBAL_STEP: 25040 + | > loss: -0.01264 (-0.03072) + | > log_mle: -0.18439 (-0.19031) + | > loss_dur: 0.17174 (0.15959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.72450 (7.36992) + | > current_lr: 0.00003 + | > step_time: 2.50960 (5.65509) + | > loader_time: 0.00710 (3.10774) + + + --> STEP: 7/234 -- GLOBAL_STEP: 25045 + | > loss: -0.05666 (-0.02811) + | > log_mle: -0.20834 (-0.19564) + | > loss_dur: 0.15168 (0.16754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.55784 (9.26730) + | > current_lr: 0.00003 + | > step_time: 2.50490 (3.70373) + | > loader_time: 0.00280 (0.92738) + + + --> STEP: 12/234 -- GLOBAL_STEP: 25050 + | > loss: -0.03896 (-0.03539) + | > log_mle: -0.19555 (-0.19707) + | > loss_dur: 0.15660 (0.16168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.91019 (9.14332) + | > current_lr: 0.00003 + | > step_time: 3.99590 (4.00260) + | > loader_time: 0.00440 (0.56572) + + + --> STEP: 17/234 -- GLOBAL_STEP: 25055 + | > loss: -0.02230 (-0.03969) + | > log_mle: -0.17247 (-0.19516) + | > loss_dur: 0.15018 (0.15548) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.00020 (8.50169) + | > current_lr: 0.00003 + | > step_time: 2.91410 (4.47329) + | > loader_time: 0.08600 (0.42607) + + + --> STEP: 22/234 -- GLOBAL_STEP: 25060 + | > loss: -0.06842 (-0.04153) + | > log_mle: -0.20041 (-0.19334) + | > loss_dur: 0.13199 (0.15182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.47656 (8.12322) + | > current_lr: 0.00003 + | > step_time: 3.00750 (5.08539) + | > loader_time: 0.00160 (0.33383) + + + --> STEP: 27/234 -- GLOBAL_STEP: 25065 + | > loss: -0.07271 (-0.04422) + | > log_mle: -0.20143 (-0.19291) + | > loss_dur: 0.12872 (0.14870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.21830 (7.62992) + | > current_lr: 0.00003 + | > step_time: 9.99850 (5.06898) + | > loader_time: 0.00220 (0.27651) + + + --> STEP: 32/234 -- GLOBAL_STEP: 25070 + | > loss: -0.08306 (-0.04570) + | > log_mle: -0.20736 (-0.19310) + | > loss_dur: 0.12430 (0.14739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.96855 (7.54235) + | > current_lr: 0.00003 + | > step_time: 1.60980 (4.78965) + | > loader_time: 0.10090 (0.25172) + + + --> STEP: 37/234 -- GLOBAL_STEP: 25075 + | > loss: -0.05487 (-0.04390) + | > log_mle: -0.18948 (-0.19281) + | > loss_dur: 0.13461 (0.14891) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.41873 (7.74277) + | > current_lr: 0.00003 + | > step_time: 3.62570 (4.56207) + | > loader_time: 0.10330 (0.22326) + + + --> STEP: 42/234 -- GLOBAL_STEP: 25080 + | > loss: -0.03681 (-0.04288) + | > log_mle: -0.18106 (-0.19248) + | > loss_dur: 0.14425 (0.14960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.77147 (7.63074) + | > current_lr: 0.00003 + | > step_time: 0.78480 (4.16084) + | > loader_time: 0.00230 (0.19866) + + + --> STEP: 47/234 -- GLOBAL_STEP: 25085 + | > loss: -0.03855 (-0.04302) + | > log_mle: -0.19213 (-0.19317) + | > loss_dur: 0.15358 (0.15015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.25370 (7.56416) + | > current_lr: 0.00003 + | > step_time: 0.71960 (3.87321) + | > loader_time: 0.09260 (0.17963) + + + --> STEP: 52/234 -- GLOBAL_STEP: 25090 + | > loss: -0.02483 (-0.04251) + | > log_mle: -0.18872 (-0.19256) + | > loss_dur: 0.16389 (0.15005) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.78456 (7.37664) + | > current_lr: 0.00003 + | > step_time: 4.40790 (3.71256) + | > loader_time: 0.00380 (0.16413) + + + --> STEP: 57/234 -- GLOBAL_STEP: 25095 + | > loss: -0.02349 (-0.04261) + | > log_mle: -0.18288 (-0.19318) + | > loss_dur: 0.15939 (0.15056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.28257 (7.18272) + | > current_lr: 0.00003 + | > step_time: 1.22350 (3.53113) + | > loader_time: 0.08720 (0.15403) + + + --> STEP: 62/234 -- GLOBAL_STEP: 25100 + | > loss: -0.03501 (-0.04359) + | > log_mle: -0.23346 (-0.19467) + | > loss_dur: 0.19845 (0.15108) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.16206 (7.47738) + | > current_lr: 0.00003 + | > step_time: 1.32610 (3.38994) + | > loader_time: 0.08540 (0.14594) + + + --> STEP: 67/234 -- GLOBAL_STEP: 25105 + | > loss: -0.05438 (-0.04362) + | > log_mle: -0.21774 (-0.19484) + | > loss_dur: 0.16336 (0.15121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.64285 (7.34724) + | > current_lr: 0.00003 + | > step_time: 1.61770 (3.27220) + | > loader_time: 0.00200 (0.13525) + + + --> STEP: 72/234 -- GLOBAL_STEP: 25110 + | > loss: -0.01892 (-0.04216) + | > log_mle: -0.19285 (-0.19489) + | > loss_dur: 0.17393 (0.15273) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.50778 (7.86834) + | > current_lr: 0.00003 + | > step_time: 1.76220 (3.19214) + | > loader_time: 0.00200 (0.12727) + + + --> STEP: 77/234 -- GLOBAL_STEP: 25115 + | > loss: -0.05900 (-0.04254) + | > log_mle: -0.20819 (-0.19569) + | > loss_dur: 0.14919 (0.15315) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.71674 (7.96557) + | > current_lr: 0.00003 + | > step_time: 2.14090 (3.12647) + | > loader_time: 0.08750 (0.12268) + + + --> STEP: 82/234 -- GLOBAL_STEP: 25120 + | > loss: -0.04967 (-0.04284) + | > log_mle: -0.19623 (-0.19593) + | > loss_dur: 0.14656 (0.15309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.35344 (8.04433) + | > current_lr: 0.00003 + | > step_time: 2.01070 (3.03065) + | > loader_time: 0.08700 (0.11760) + + + --> STEP: 87/234 -- GLOBAL_STEP: 25125 + | > loss: -0.03814 (-0.04294) + | > log_mle: -0.20659 (-0.19672) + | > loss_dur: 0.16845 (0.15377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.35145 (8.21390) + | > current_lr: 0.00003 + | > step_time: 1.49760 (2.97265) + | > loader_time: 0.00280 (0.11404) + + + --> STEP: 92/234 -- GLOBAL_STEP: 25130 + | > loss: -0.09303 (-0.04429) + | > log_mle: -0.25495 (-0.19916) + | > loss_dur: 0.16192 (0.15488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.80789 (8.53758) + | > current_lr: 0.00003 + | > step_time: 1.17970 (2.90685) + | > loader_time: 0.00220 (0.11161) + + + --> STEP: 97/234 -- GLOBAL_STEP: 25135 + | > loss: -0.07271 (-0.04636) + | > log_mle: -0.24587 (-0.20259) + | > loss_dur: 0.17317 (0.15623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.15742 (8.96818) + | > current_lr: 0.00003 + | > step_time: 1.69800 (2.83555) + | > loader_time: 0.00320 (0.10859) + + + --> STEP: 102/234 -- GLOBAL_STEP: 25140 + | > loss: -0.04222 (-0.04734) + | > log_mle: -0.22411 (-0.20479) + | > loss_dur: 0.18189 (0.15745) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.89884 (9.45095) + | > current_lr: 0.00003 + | > step_time: 1.20000 (2.78862) + | > loader_time: 0.08830 (0.10521) + + + --> STEP: 107/234 -- GLOBAL_STEP: 25145 + | > loss: -0.08345 (-0.04920) + | > log_mle: -0.27026 (-0.20805) + | > loss_dur: 0.18680 (0.15885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.83294 (10.05145) + | > current_lr: 0.00003 + | > step_time: 1.82580 (2.74567) + | > loader_time: 0.00570 (0.10124) + + + --> STEP: 112/234 -- GLOBAL_STEP: 25150 + | > loss: -0.07850 (-0.05049) + | > log_mle: -0.28066 (-0.21100) + | > loss_dur: 0.20215 (0.16051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.16348 (10.64863) + | > current_lr: 0.00003 + | > step_time: 1.39700 (2.70295) + | > loader_time: 0.00410 (0.09686) + + + --> STEP: 117/234 -- GLOBAL_STEP: 25155 + | > loss: -0.10230 (-0.05186) + | > log_mle: -0.27624 (-0.21383) + | > loss_dur: 0.17394 (0.16197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.34405 (11.18858) + | > current_lr: 0.00003 + | > step_time: 1.19800 (2.67125) + | > loader_time: 0.00250 (0.09509) + + + --> STEP: 122/234 -- GLOBAL_STEP: 25160 + | > loss: -0.07806 (-0.05268) + | > log_mle: -0.25518 (-0.21567) + | > loss_dur: 0.17712 (0.16299) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.87344 (11.34940) + | > current_lr: 0.00003 + | > step_time: 4.51850 (2.66841) + | > loader_time: 0.00330 (0.09218) + + + --> STEP: 127/234 -- GLOBAL_STEP: 25165 + | > loss: -0.10049 (-0.05462) + | > log_mle: -0.30804 (-0.21873) + | > loss_dur: 0.20755 (0.16411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.20979 (11.94556) + | > current_lr: 0.00003 + | > step_time: 1.29150 (2.62937) + | > loader_time: 0.00320 (0.09000) + + + --> STEP: 132/234 -- GLOBAL_STEP: 25170 + | > loss: -0.10956 (-0.05675) + | > log_mle: -0.28775 (-0.22213) + | > loss_dur: 0.17819 (0.16539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.72229 (12.64383) + | > current_lr: 0.00003 + | > step_time: 1.99530 (2.63459) + | > loader_time: 0.00860 (0.08856) + + + --> STEP: 137/234 -- GLOBAL_STEP: 25175 + | > loss: -0.08399 (-0.05873) + | > log_mle: -0.30276 (-0.22583) + | > loss_dur: 0.21877 (0.16711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.73179 (13.21184) + | > current_lr: 0.00003 + | > step_time: 4.26290 (2.62938) + | > loader_time: 0.10650 (0.08750) + + + --> STEP: 142/234 -- GLOBAL_STEP: 25180 + | > loss: -0.10949 (-0.06053) + | > log_mle: -0.31791 (-0.22903) + | > loss_dur: 0.20842 (0.16850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.52722 (13.89090) + | > current_lr: 0.00003 + | > step_time: 1.49880 (2.59230) + | > loader_time: 0.08760 (0.08574) + + + --> STEP: 147/234 -- GLOBAL_STEP: 25185 + | > loss: -0.11572 (-0.06342) + | > log_mle: -0.32099 (-0.23384) + | > loss_dur: 0.20527 (0.17043) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.44860 (14.76949) + | > current_lr: 0.00003 + | > step_time: 4.21250 (2.61086) + | > loader_time: 0.18570 (0.08547) + + + --> STEP: 152/234 -- GLOBAL_STEP: 25190 + | > loss: -0.16536 (-0.06648) + | > log_mle: -0.40145 (-0.23832) + | > loss_dur: 0.23609 (0.17184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.30227 (15.49536) + | > current_lr: 0.00003 + | > step_time: 1.50950 (2.60737) + | > loader_time: 0.00450 (0.08509) + + + --> STEP: 157/234 -- GLOBAL_STEP: 25195 + | > loss: -0.12659 (-0.06996) + | > log_mle: -0.34435 (-0.24331) + | > loss_dur: 0.21776 (0.17335) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.17905 (16.80777) + | > current_lr: 0.00003 + | > step_time: 2.70540 (2.61287) + | > loader_time: 0.00350 (0.08307) + + + --> STEP: 162/234 -- GLOBAL_STEP: 25200 + | > loss: -0.16882 (-0.07310) + | > log_mle: -0.37950 (-0.24800) + | > loss_dur: 0.21068 (0.17490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.57104 (17.95894) + | > current_lr: 0.00003 + | > step_time: 1.27740 (2.60443) + | > loader_time: 0.00300 (0.08222) + + + --> STEP: 167/234 -- GLOBAL_STEP: 25205 + | > loss: -0.24740 (-0.07619) + | > log_mle: -0.46243 (-0.25241) + | > loss_dur: 0.21504 (0.17622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.88742 (18.71061) + | > current_lr: 0.00003 + | > step_time: 2.90420 (2.63001) + | > loader_time: 0.00230 (0.08166) + + + --> STEP: 172/234 -- GLOBAL_STEP: 25210 + | > loss: -0.19652 (-0.07939) + | > log_mle: -0.44321 (-0.25751) + | > loss_dur: 0.24669 (0.17812) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.99426 (19.90986) + | > current_lr: 0.00003 + | > step_time: 5.60850 (2.72299) + | > loader_time: 0.20840 (0.08317) + + + --> STEP: 177/234 -- GLOBAL_STEP: 25215 + | > loss: -0.15615 (-0.08243) + | > log_mle: -0.40342 (-0.26229) + | > loss_dur: 0.24727 (0.17986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.74964 (21.17447) + | > current_lr: 0.00003 + | > step_time: 2.50360 (2.71158) + | > loader_time: 0.00390 (0.08137) + + + --> STEP: 182/234 -- GLOBAL_STEP: 25220 + | > loss: -0.18790 (-0.08521) + | > log_mle: -0.45189 (-0.26700) + | > loss_dur: 0.26400 (0.18179) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.50600 (22.38394) + | > current_lr: 0.00003 + | > step_time: 1.17550 (2.70853) + | > loader_time: 0.00270 (0.07977) + + + --> STEP: 187/234 -- GLOBAL_STEP: 25225 + | > loss: -0.20251 (-0.08808) + | > log_mle: -0.44647 (-0.27156) + | > loss_dur: 0.24396 (0.18348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.98676 (23.65870) + | > current_lr: 0.00003 + | > step_time: 2.00340 (2.68047) + | > loader_time: 0.00390 (0.07775) + + + --> STEP: 192/234 -- GLOBAL_STEP: 25230 + | > loss: -0.24035 (-0.09117) + | > log_mle: -0.47298 (-0.27612) + | > loss_dur: 0.23263 (0.18495) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.16321 (24.69744) + | > current_lr: 0.00003 + | > step_time: 2.71270 (2.70550) + | > loader_time: 0.08730 (0.07768) + + + --> STEP: 197/234 -- GLOBAL_STEP: 25235 + | > loss: -0.22089 (-0.09406) + | > log_mle: -0.45219 (-0.28055) + | > loss_dur: 0.23130 (0.18649) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.66869 (25.67782) + | > current_lr: 0.00003 + | > step_time: 2.20520 (2.76897) + | > loader_time: 0.09800 (0.07872) + + + --> STEP: 202/234 -- GLOBAL_STEP: 25240 + | > loss: -0.29084 (-0.09704) + | > log_mle: -0.53413 (-0.28509) + | > loss_dur: 0.24329 (0.18805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.61167 (26.59095) + | > current_lr: 0.00003 + | > step_time: 4.50550 (2.80303) + | > loader_time: 0.10850 (0.07865) + + + --> STEP: 207/234 -- GLOBAL_STEP: 25245 + | > loss: -0.26533 (-0.09998) + | > log_mle: -0.52619 (-0.28965) + | > loss_dur: 0.26086 (0.18967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.29761 (27.39233) + | > current_lr: 0.00003 + | > step_time: 5.10020 (2.81111) + | > loader_time: 0.08870 (0.07814) + + + --> STEP: 212/234 -- GLOBAL_STEP: 25250 + | > loss: -0.25824 (-0.10339) + | > log_mle: -0.51164 (-0.29481) + | > loss_dur: 0.25340 (0.19142) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.42519 (28.37141) + | > current_lr: 0.00003 + | > step_time: 2.90150 (2.85431) + | > loader_time: 0.00430 (0.07765) + + + --> STEP: 217/234 -- GLOBAL_STEP: 25255 + | > loss: -0.25450 (-0.10686) + | > log_mle: -0.52007 (-0.29985) + | > loss_dur: 0.26557 (0.19299) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.92870 (29.59907) + | > current_lr: 0.00003 + | > step_time: 3.70280 (2.92870) + | > loader_time: 0.09160 (0.07859) + + + --> STEP: 222/234 -- GLOBAL_STEP: 25260 + | > loss: -0.24858 (-0.11015) + | > log_mle: -0.53868 (-0.30476) + | > loss_dur: 0.29009 (0.19461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.89858 (30.79035) + | > current_lr: 0.00003 + | > step_time: 2.98520 (2.96731) + | > loader_time: 0.00420 (0.07776) + + + --> STEP: 227/234 -- GLOBAL_STEP: 25265 + | > loss: -0.23814 (-0.11372) + | > log_mle: -0.51973 (-0.31010) + | > loss_dur: 0.28160 (0.19638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.42975 (31.84574) + | > current_lr: 0.00003 + | > step_time: 0.24880 (2.91581) + | > loader_time: 0.00300 (0.07646) + + + --> STEP: 232/234 -- GLOBAL_STEP: 25270 + | > loss: -0.14288 (-0.11595) + | > log_mle: -0.68609 (-0.31648) + | > loss_dur: 0.54321 (0.20053) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 189.47691 (33.66660) + | > current_lr: 0.00003 + | > step_time: 0.38600 (2.85921) + | > loader_time: 0.05680 (0.07512) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.10176 (+0.09905) + | > avg_loss: -0.15370 (+0.00696) + | > avg_log_mle: -0.39021 (+0.00569) + | > avg_loss_dur: 0.23651 (+0.00127) + + + > EPOCH: 108/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 08:02:02)  + + --> STEP: 3/234 -- GLOBAL_STEP: 25275 + | > loss: 0.02455 (-0.01897) + | > log_mle: -0.19992 (-0.19425) + | > loss_dur: 0.22447 (0.17528) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.13490 (7.58082) + | > current_lr: 0.00003 + | > step_time: 6.20490 (5.50417) + | > loader_time: 0.00260 (0.03056) + + + --> STEP: 8/234 -- GLOBAL_STEP: 25280 + | > loss: -0.04292 (-0.02203) + | > log_mle: -0.20720 (-0.19579) + | > loss_dur: 0.16428 (0.17376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.09978 (16.73824) + | > current_lr: 0.00003 + | > step_time: 9.69720 (4.12692) + | > loader_time: 0.00300 (0.01272) + + + --> STEP: 13/234 -- GLOBAL_STEP: 25285 + | > loss: -0.03355 (-0.02671) + | > log_mle: -0.19385 (-0.19617) + | > loss_dur: 0.16030 (0.16946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.83735 (13.39622) + | > current_lr: 0.00003 + | > step_time: 2.68600 (3.58605) + | > loader_time: 0.00110 (0.02307) + + + --> STEP: 18/234 -- GLOBAL_STEP: 25290 + | > loss: -0.04333 (-0.03305) + | > log_mle: -0.19073 (-0.19489) + | > loss_dur: 0.14740 (0.16184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.53598 (11.80687) + | > current_lr: 0.00003 + | > step_time: 1.77340 (3.78886) + | > loader_time: 0.00400 (0.02272) + + + --> STEP: 23/234 -- GLOBAL_STEP: 25295 + | > loss: -0.04930 (-0.03506) + | > log_mle: -0.19606 (-0.19338) + | > loss_dur: 0.14675 (0.15832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.97459 (10.56454) + | > current_lr: 0.00003 + | > step_time: 4.19830 (3.40083) + | > loader_time: 0.08480 (0.02492) + + + --> STEP: 28/234 -- GLOBAL_STEP: 25300 + | > loss: -0.07494 (-0.03834) + | > log_mle: -0.18545 (-0.19269) + | > loss_dur: 0.11051 (0.15435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.43760 (9.61617) + | > current_lr: 0.00003 + | > step_time: 1.20730 (3.64716) + | > loader_time: 0.07620 (0.03372) + + + --> STEP: 33/234 -- GLOBAL_STEP: 25305 + | > loss: -0.03273 (-0.03973) + | > log_mle: -0.18355 (-0.19299) + | > loss_dur: 0.15082 (0.15326) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.37988 (9.22299) + | > current_lr: 0.00003 + | > step_time: 6.71010 (3.63776) + | > loader_time: 0.18280 (0.04255) + + + --> STEP: 38/234 -- GLOBAL_STEP: 25310 + | > loss: -0.04606 (-0.03985) + | > log_mle: -0.20446 (-0.19363) + | > loss_dur: 0.15840 (0.15378) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.42821 (8.97069) + | > current_lr: 0.00003 + | > step_time: 5.69590 (3.64092) + | > loader_time: 0.10800 (0.04968) + + + --> STEP: 43/234 -- GLOBAL_STEP: 25315 + | > loss: -0.03606 (-0.03962) + | > log_mle: -0.20511 (-0.19356) + | > loss_dur: 0.16905 (0.15394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.76816 (8.70370) + | > current_lr: 0.00003 + | > step_time: 1.60630 (3.39425) + | > loader_time: 0.00210 (0.04809) + + + --> STEP: 48/234 -- GLOBAL_STEP: 25320 + | > loss: -0.05693 (-0.04067) + | > log_mle: -0.18552 (-0.19391) + | > loss_dur: 0.12860 (0.15324) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.79167 (8.50684) + | > current_lr: 0.00003 + | > step_time: 1.00110 (3.15737) + | > loader_time: 0.00200 (0.04339) + + + --> STEP: 53/234 -- GLOBAL_STEP: 25325 + | > loss: -0.04169 (-0.04057) + | > log_mle: -0.20759 (-0.19379) + | > loss_dur: 0.16590 (0.15322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.62748 (8.24198) + | > current_lr: 0.00003 + | > step_time: 2.00760 (3.07844) + | > loader_time: 0.07510 (0.04091) + + + --> STEP: 58/234 -- GLOBAL_STEP: 25330 + | > loss: -0.05334 (-0.04057) + | > log_mle: -0.18987 (-0.19408) + | > loss_dur: 0.13653 (0.15351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.03744 (8.10541) + | > current_lr: 0.00003 + | > step_time: 1.49120 (2.92163) + | > loader_time: 0.00240 (0.03768) + + + --> STEP: 63/234 -- GLOBAL_STEP: 25335 + | > loss: -0.03037 (-0.04154) + | > log_mle: -0.19969 (-0.19575) + | > loss_dur: 0.16932 (0.15421) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.26785 (8.36178) + | > current_lr: 0.00003 + | > step_time: 3.89920 (2.86246) + | > loader_time: 0.00250 (0.03625) + + + --> STEP: 68/234 -- GLOBAL_STEP: 25340 + | > loss: -0.02191 (-0.04154) + | > log_mle: -0.19314 (-0.19573) + | > loss_dur: 0.17123 (0.15420) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.30334 (8.24041) + | > current_lr: 0.00003 + | > step_time: 2.46930 (2.85769) + | > loader_time: 0.00270 (0.03644) + + + --> STEP: 73/234 -- GLOBAL_STEP: 25345 + | > loss: -0.03640 (-0.04024) + | > log_mle: -0.21668 (-0.19592) + | > loss_dur: 0.18028 (0.15568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.55036 (8.70430) + | > current_lr: 0.00003 + | > step_time: 1.98880 (2.81169) + | > loader_time: 0.00370 (0.03798) + + + --> STEP: 78/234 -- GLOBAL_STEP: 25350 + | > loss: -0.02245 (-0.04054) + | > log_mle: -0.18960 (-0.19623) + | > loss_dur: 0.16715 (0.15569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.59759 (8.77636) + | > current_lr: 0.00003 + | > step_time: 2.70610 (2.80972) + | > loader_time: 0.00290 (0.03815) + + + --> STEP: 83/234 -- GLOBAL_STEP: 25355 + | > loss: -0.03991 (-0.04117) + | > log_mle: -0.22060 (-0.19683) + | > loss_dur: 0.18070 (0.15565) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.67591 (8.82218) + | > current_lr: 0.00003 + | > step_time: 4.01260 (2.83222) + | > loader_time: 0.18910 (0.04145) + + + --> STEP: 88/234 -- GLOBAL_STEP: 25360 + | > loss: -0.06825 (-0.04166) + | > log_mle: -0.25537 (-0.19810) + | > loss_dur: 0.18712 (0.15644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.84799 (8.88035) + | > current_lr: 0.00003 + | > step_time: 2.00800 (2.75696) + | > loader_time: 0.00480 (0.04028) + + + --> STEP: 93/234 -- GLOBAL_STEP: 25365 + | > loss: -0.07325 (-0.04312) + | > log_mle: -0.27033 (-0.20064) + | > loss_dur: 0.19708 (0.15753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.16077 (9.17331) + | > current_lr: 0.00003 + | > step_time: 2.80310 (2.72580) + | > loader_time: 0.00230 (0.03830) + + + --> STEP: 98/234 -- GLOBAL_STEP: 25370 + | > loss: -0.03434 (-0.04487) + | > log_mle: -0.19651 (-0.20319) + | > loss_dur: 0.16217 (0.15832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.95153 (9.48258) + | > current_lr: 0.00003 + | > step_time: 1.98810 (2.70326) + | > loader_time: 0.01110 (0.03933) + + + --> STEP: 103/234 -- GLOBAL_STEP: 25375 + | > loss: -0.10669 (-0.04705) + | > log_mle: -0.29878 (-0.20648) + | > loss_dur: 0.19209 (0.15943) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.26198 (10.05742) + | > current_lr: 0.00003 + | > step_time: 2.08910 (2.74568) + | > loader_time: 0.00290 (0.03837) + + + --> STEP: 108/234 -- GLOBAL_STEP: 25380 + | > loss: -0.07143 (-0.04874) + | > log_mle: -0.24185 (-0.20928) + | > loss_dur: 0.17041 (0.16053) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.93112 (10.43878) + | > current_lr: 0.00003 + | > step_time: 1.21050 (2.71430) + | > loader_time: 0.00270 (0.03818) + + + --> STEP: 113/234 -- GLOBAL_STEP: 25385 + | > loss: -0.11094 (-0.05052) + | > log_mle: -0.29255 (-0.21270) + | > loss_dur: 0.18161 (0.16218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.85668 (11.10057) + | > current_lr: 0.00003 + | > step_time: 1.79890 (2.67200) + | > loader_time: 0.00680 (0.03737) + + + --> STEP: 118/234 -- GLOBAL_STEP: 25390 + | > loss: -0.06686 (-0.05166) + | > log_mle: -0.25634 (-0.21518) + | > loss_dur: 0.18947 (0.16353) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.06250 (11.58316) + | > current_lr: 0.00003 + | > step_time: 4.10480 (2.72834) + | > loader_time: 0.08820 (0.03908) + + + --> STEP: 123/234 -- GLOBAL_STEP: 25395 + | > loss: -0.06367 (-0.05269) + | > log_mle: -0.22938 (-0.21672) + | > loss_dur: 0.16571 (0.16403) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.77253 (11.94777) + | > current_lr: 0.00003 + | > step_time: 1.28900 (2.67852) + | > loader_time: 0.00370 (0.03896) + + + --> STEP: 128/234 -- GLOBAL_STEP: 25400 + | > loss: -0.10571 (-0.05489) + | > log_mle: -0.28981 (-0.22020) + | > loss_dur: 0.18410 (0.16531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.09995 (12.72709) + | > current_lr: 0.00003 + | > step_time: 3.20450 (2.70749) + | > loader_time: 0.00520 (0.03829) + + + --> STEP: 133/234 -- GLOBAL_STEP: 25405 + | > loss: -0.10900 (-0.05716) + | > log_mle: -0.31173 (-0.22377) + | > loss_dur: 0.20274 (0.16661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.88605 (13.44731) + | > current_lr: 0.00003 + | > step_time: 2.68860 (2.69959) + | > loader_time: 0.00240 (0.03829) + + + --> STEP: 138/234 -- GLOBAL_STEP: 25410 + | > loss: -0.08216 (-0.05883) + | > log_mle: -0.26919 (-0.22705) + | > loss_dur: 0.18703 (0.16822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.79985 (14.15946) + | > current_lr: 0.00003 + | > step_time: 1.89740 (2.67365) + | > loader_time: 0.06730 (0.03811) + + + --> STEP: 143/234 -- GLOBAL_STEP: 25415 + | > loss: -0.14314 (-0.06108) + | > log_mle: -0.40574 (-0.23116) + | > loss_dur: 0.26259 (0.17009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.87115 (15.02622) + | > current_lr: 0.00003 + | > step_time: 2.29730 (2.65377) + | > loader_time: 0.08600 (0.03853) + + + --> STEP: 148/234 -- GLOBAL_STEP: 25420 + | > loss: -0.13391 (-0.06366) + | > log_mle: -0.31604 (-0.23521) + | > loss_dur: 0.18214 (0.17154) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.95875 (15.96232) + | > current_lr: 0.00003 + | > step_time: 3.29860 (2.65777) + | > loader_time: 0.08460 (0.03858) + + + --> STEP: 153/234 -- GLOBAL_STEP: 25425 + | > loss: -0.21755 (-0.06701) + | > log_mle: -0.44095 (-0.24021) + | > loss_dur: 0.22340 (0.17320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.54502 (16.96654) + | > current_lr: 0.00003 + | > step_time: 3.41750 (2.65274) + | > loader_time: 0.09260 (0.03859) + + + --> STEP: 158/234 -- GLOBAL_STEP: 25430 + | > loss: -0.15384 (-0.07002) + | > log_mle: -0.38332 (-0.24471) + | > loss_dur: 0.22948 (0.17469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.78653 (17.87309) + | > current_lr: 0.00003 + | > step_time: 3.50690 (2.76325) + | > loader_time: 0.08350 (0.03977) + + + --> STEP: 163/234 -- GLOBAL_STEP: 25435 + | > loss: -0.13235 (-0.07311) + | > log_mle: -0.34477 (-0.24918) + | > loss_dur: 0.21242 (0.17607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.58823 (18.73550) + | > current_lr: 0.00003 + | > step_time: 3.09020 (2.78845) + | > loader_time: 0.00420 (0.04105) + + + --> STEP: 168/234 -- GLOBAL_STEP: 25440 + | > loss: -0.16003 (-0.07623) + | > log_mle: -0.40165 (-0.25374) + | > loss_dur: 0.24163 (0.17751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.34528 (19.66538) + | > current_lr: 0.00003 + | > step_time: 3.09170 (2.79066) + | > loader_time: 0.00710 (0.04091) + + + --> STEP: 173/234 -- GLOBAL_STEP: 25445 + | > loss: -0.17892 (-0.07963) + | > log_mle: -0.40792 (-0.25882) + | > loss_dur: 0.22901 (0.17919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.93117 (20.77338) + | > current_lr: 0.00003 + | > step_time: 3.89640 (2.80186) + | > loader_time: 0.19990 (0.04197) + + + --> STEP: 178/234 -- GLOBAL_STEP: 25450 + | > loss: -0.21199 (-0.08297) + | > log_mle: -0.47190 (-0.26397) + | > loss_dur: 0.25992 (0.18100) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.53421 (21.76671) + | > current_lr: 0.00003 + | > step_time: 1.92540 (2.82783) + | > loader_time: 0.08380 (0.04227) + + + --> STEP: 183/234 -- GLOBAL_STEP: 25455 + | > loss: -0.20279 (-0.08559) + | > log_mle: -0.45442 (-0.26848) + | > loss_dur: 0.25163 (0.18289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.00552 (23.13093) + | > current_lr: 0.00003 + | > step_time: 2.99600 (2.82966) + | > loader_time: 0.00300 (0.04122) + + + --> STEP: 188/234 -- GLOBAL_STEP: 25460 + | > loss: -0.21677 (-0.08836) + | > log_mle: -0.47133 (-0.27305) + | > loss_dur: 0.25456 (0.18469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.10857 (24.27595) + | > current_lr: 0.00003 + | > step_time: 2.81120 (2.89119) + | > loader_time: 0.08900 (0.04736) + + + --> STEP: 193/234 -- GLOBAL_STEP: 25465 + | > loss: -0.23667 (-0.09161) + | > log_mle: -0.47922 (-0.27761) + | > loss_dur: 0.24255 (0.18601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.52012 (25.16784) + | > current_lr: 0.00003 + | > step_time: 6.49740 (2.96082) + | > loader_time: 0.00800 (0.04720) + + + --> STEP: 198/234 -- GLOBAL_STEP: 25470 + | > loss: -0.22422 (-0.09452) + | > log_mle: -0.46988 (-0.28201) + | > loss_dur: 0.24566 (0.18750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.85075 (26.19649) + | > current_lr: 0.00003 + | > step_time: 3.18770 (3.01979) + | > loader_time: 0.10660 (0.04806) + + + --> STEP: 203/234 -- GLOBAL_STEP: 25475 + | > loss: -0.15547 (-0.09704) + | > log_mle: -0.39836 (-0.28613) + | > loss_dur: 0.24289 (0.18909) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.70070 (27.27887) + | > current_lr: 0.00003 + | > step_time: 1.59260 (3.06897) + | > loader_time: 0.01040 (0.04840) + + + --> STEP: 208/234 -- GLOBAL_STEP: 25480 + | > loss: -0.22495 (-0.10035) + | > log_mle: -0.48276 (-0.29101) + | > loss_dur: 0.25781 (0.19065) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.72462 (28.12652) + | > current_lr: 0.00003 + | > step_time: 6.00470 (3.12793) + | > loader_time: 0.00280 (0.05059) + + + --> STEP: 213/234 -- GLOBAL_STEP: 25485 + | > loss: -0.25562 (-0.10387) + | > log_mle: -0.52341 (-0.29628) + | > loss_dur: 0.26779 (0.19241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.95461 (29.25123) + | > current_lr: 0.00003 + | > step_time: 4.40170 (3.13534) + | > loader_time: 0.08470 (0.05114) + + + --> STEP: 218/234 -- GLOBAL_STEP: 25490 + | > loss: -0.23304 (-0.10720) + | > log_mle: -0.49592 (-0.30116) + | > loss_dur: 0.26288 (0.19396) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.16852 (30.20601) + | > current_lr: 0.00003 + | > step_time: 6.09170 (3.26252) + | > loader_time: 0.00280 (0.05302) + + + --> STEP: 223/234 -- GLOBAL_STEP: 25495 + | > loss: -0.27494 (-0.11075) + | > log_mle: -0.53858 (-0.30636) + | > loss_dur: 0.26365 (0.19562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.60232 (31.15851) + | > current_lr: 0.00003 + | > step_time: 0.23840 (3.24875) + | > loader_time: 0.00440 (0.05276) + + + --> STEP: 228/234 -- GLOBAL_STEP: 25500 + | > loss: -0.23691 (-0.11406) + | > log_mle: -0.52895 (-0.31158) + | > loss_dur: 0.29204 (0.19752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.26155 (32.48423) + | > current_lr: 0.00003 + | > step_time: 0.24350 (3.18274) + | > loader_time: 0.00320 (0.05167) + + + --> STEP: 233/234 -- GLOBAL_STEP: 25505 + | > loss: 0.38110 (-0.11379) + | > log_mle: -0.48177 (-0.31783) + | > loss_dur: 0.86286 (0.20404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.42386 (33.99042) + | > current_lr: 0.00003 + | > step_time: 0.18960 (3.12001) + | > loader_time: 0.00290 (0.05095) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.10669 (+0.00493) + | > avg_loss: -0.10601 (+0.04769) + | > avg_log_mle: -0.34506 (+0.04516) + | > avg_loss_dur: 0.23905 (+0.00254) + + + > EPOCH: 109/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 08:15:21)  + + --> STEP: 4/234 -- GLOBAL_STEP: 25510 + | > loss: -0.00125 (-0.01409) + | > log_mle: -0.19533 (-0.19612) + | > loss_dur: 0.19408 (0.18203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.20155 (17.17634) + | > current_lr: 0.00003 + | > step_time: 6.20230 (5.52307) + | > loader_time: 0.00500 (0.00235) + + + --> STEP: 9/234 -- GLOBAL_STEP: 25515 + | > loss: -0.04791 (-0.03047) + | > log_mle: -0.20827 (-0.20004) + | > loss_dur: 0.16035 (0.16957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.69642 (13.25421) + | > current_lr: 0.00003 + | > step_time: 0.89690 (3.49871) + | > loader_time: 0.00120 (0.02218) + + + --> STEP: 14/234 -- GLOBAL_STEP: 25520 + | > loss: -0.04346 (-0.03302) + | > log_mle: -0.20228 (-0.19893) + | > loss_dur: 0.15882 (0.16591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.19668 (11.59516) + | > current_lr: 0.00003 + | > step_time: 1.41420 (3.23646) + | > loader_time: 0.00290 (0.02590) + + + --> STEP: 19/234 -- GLOBAL_STEP: 25525 + | > loss: -0.06406 (-0.03856) + | > log_mle: -0.18890 (-0.19673) + | > loss_dur: 0.12484 (0.15818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.46514 (10.39419) + | > current_lr: 0.00003 + | > step_time: 3.09820 (3.04339) + | > loader_time: 0.00300 (0.02886) + + + --> STEP: 24/234 -- GLOBAL_STEP: 25530 + | > loss: -0.06351 (-0.04132) + | > log_mle: -0.19040 (-0.19550) + | > loss_dur: 0.12690 (0.15419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.36968 (9.60648) + | > current_lr: 0.00003 + | > step_time: 5.90230 (3.29363) + | > loader_time: 0.00430 (0.03077) + + + --> STEP: 29/234 -- GLOBAL_STEP: 25535 + | > loss: -0.03936 (-0.04333) + | > log_mle: -0.18192 (-0.19469) + | > loss_dur: 0.14256 (0.15136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79595 (8.99355) + | > current_lr: 0.00003 + | > step_time: 9.51900 (3.62742) + | > loader_time: 0.18580 (0.04077) + + + --> STEP: 34/234 -- GLOBAL_STEP: 25540 + | > loss: -0.03040 (-0.04372) + | > log_mle: -0.19323 (-0.19536) + | > loss_dur: 0.16283 (0.15165) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.69067 (8.52350) + | > current_lr: 0.00003 + | > step_time: 7.49740 (3.74990) + | > loader_time: 0.00320 (0.03797) + + + --> STEP: 39/234 -- GLOBAL_STEP: 25545 + | > loss: -0.04657 (-0.04454) + | > log_mle: -0.20237 (-0.19610) + | > loss_dur: 0.15579 (0.15156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10868 (8.62998) + | > current_lr: 0.00003 + | > step_time: 1.10000 (3.72837) + | > loader_time: 0.00110 (0.03814) + + + --> STEP: 44/234 -- GLOBAL_STEP: 25550 + | > loss: -0.05154 (-0.04424) + | > log_mle: -0.19177 (-0.19561) + | > loss_dur: 0.14023 (0.15137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.07736 (8.27206) + | > current_lr: 0.00003 + | > step_time: 0.94990 (3.47412) + | > loader_time: 0.00160 (0.03402) + + + --> STEP: 49/234 -- GLOBAL_STEP: 25555 + | > loss: -0.06738 (-0.04496) + | > log_mle: -0.19982 (-0.19606) + | > loss_dur: 0.13245 (0.15111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.56981 (8.08542) + | > current_lr: 0.00003 + | > step_time: 1.98710 (3.28647) + | > loader_time: 0.00190 (0.03390) + + + --> STEP: 54/234 -- GLOBAL_STEP: 25560 + | > loss: -0.06160 (-0.04483) + | > log_mle: -0.20749 (-0.19602) + | > loss_dur: 0.14589 (0.15119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.67809 (7.81136) + | > current_lr: 0.00003 + | > step_time: 1.59310 (3.15947) + | > loader_time: 0.00120 (0.03096) + + + --> STEP: 59/234 -- GLOBAL_STEP: 25565 + | > loss: -0.07662 (-0.04518) + | > log_mle: -0.21720 (-0.19641) + | > loss_dur: 0.14058 (0.15123) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.39793 (7.71152) + | > current_lr: 0.00003 + | > step_time: 2.22830 (3.13956) + | > loader_time: 0.00160 (0.03310) + + + --> STEP: 64/234 -- GLOBAL_STEP: 25570 + | > loss: -0.04909 (-0.04545) + | > log_mle: -0.18733 (-0.19745) + | > loss_dur: 0.13824 (0.15200) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.50801 (8.02153) + | > current_lr: 0.00003 + | > step_time: 2.98790 (3.03021) + | > loader_time: 0.10890 (0.03382) + + + --> STEP: 69/234 -- GLOBAL_STEP: 25575 + | > loss: -0.01586 (-0.04502) + | > log_mle: -0.17753 (-0.19730) + | > loss_dur: 0.16167 (0.15229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.97856 (7.96434) + | > current_lr: 0.00003 + | > step_time: 1.50130 (2.97440) + | > loader_time: 0.00340 (0.03437) + + + --> STEP: 74/234 -- GLOBAL_STEP: 25580 + | > loss: -0.04636 (-0.04414) + | > log_mle: -0.19309 (-0.19792) + | > loss_dur: 0.14673 (0.15378) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.37754 (8.21070) + | > current_lr: 0.00003 + | > step_time: 1.57210 (2.90851) + | > loader_time: 0.00310 (0.03338) + + + --> STEP: 79/234 -- GLOBAL_STEP: 25585 + | > loss: -0.05484 (-0.04425) + | > log_mle: -0.20755 (-0.19847) + | > loss_dur: 0.15271 (0.15422) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.47189 (8.19291) + | > current_lr: 0.00003 + | > step_time: 1.10570 (2.82740) + | > loader_time: 0.00270 (0.03342) + + + --> STEP: 84/234 -- GLOBAL_STEP: 25590 + | > loss: -0.05137 (-0.04486) + | > log_mle: -0.20489 (-0.19895) + | > loss_dur: 0.15352 (0.15409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.23654 (8.27987) + | > current_lr: 0.00003 + | > step_time: 3.68570 (2.83407) + | > loader_time: 0.00170 (0.03252) + + + --> STEP: 89/234 -- GLOBAL_STEP: 25595 + | > loss: -0.07216 (-0.04570) + | > log_mle: -0.23726 (-0.20047) + | > loss_dur: 0.16510 (0.15477) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.40890 (8.46177) + | > current_lr: 0.00003 + | > step_time: 2.49930 (2.79651) + | > loader_time: 0.00260 (0.03178) + + + --> STEP: 94/234 -- GLOBAL_STEP: 25600 + | > loss: -0.09821 (-0.04744) + | > log_mle: -0.27305 (-0.20328) + | > loss_dur: 0.17484 (0.15584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.45849 (8.92678) + | > current_lr: 0.00003 + | > step_time: 3.21390 (2.83218) + | > loader_time: 0.00350 (0.03231) + + + --> STEP: 99/234 -- GLOBAL_STEP: 25605 + | > loss: -0.11116 (-0.04922) + | > log_mle: -0.29995 (-0.20599) + | > loss_dur: 0.18879 (0.15678) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.23680 (9.55133) + | > current_lr: 0.00003 + | > step_time: 1.11250 (2.78717) + | > loader_time: 0.08740 (0.03243) + + + --> STEP: 104/234 -- GLOBAL_STEP: 25610 + | > loss: -0.12998 (-0.05104) + | > log_mle: -0.31584 (-0.20920) + | > loss_dur: 0.18586 (0.15816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.54119 (10.10227) + | > current_lr: 0.00003 + | > step_time: 2.16510 (2.80814) + | > loader_time: 0.00240 (0.03358) + + + --> STEP: 109/234 -- GLOBAL_STEP: 25615 + | > loss: -0.05016 (-0.05179) + | > log_mle: -0.28483 (-0.21152) + | > loss_dur: 0.23467 (0.15973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.84240 (10.58693) + | > current_lr: 0.00003 + | > step_time: 4.31080 (2.77915) + | > loader_time: 0.08490 (0.03367) + + + --> STEP: 114/234 -- GLOBAL_STEP: 25620 + | > loss: -0.09035 (-0.05341) + | > log_mle: -0.26988 (-0.21466) + | > loss_dur: 0.17953 (0.16126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.07154 (11.12070) + | > current_lr: 0.00003 + | > step_time: 1.48980 (2.83697) + | > loader_time: 0.00270 (0.03408) + + + --> STEP: 119/234 -- GLOBAL_STEP: 25625 + | > loss: -0.08134 (-0.05438) + | > log_mle: -0.26643 (-0.21709) + | > loss_dur: 0.18509 (0.16271) + | > amp_scaler: 4096.00000 (2065.21008) + | > grad_norm: 18.15642 (11.41939) + | > current_lr: 0.00003 + | > step_time: 1.00130 (2.79563) + | > loader_time: 0.00730 (0.03282) + + + --> STEP: 124/234 -- GLOBAL_STEP: 25630 + | > loss: -0.12626 (-0.05553) + | > log_mle: -0.29783 (-0.21887) + | > loss_dur: 0.17157 (0.16335) + | > amp_scaler: 4096.00000 (2147.09677) + | > grad_norm: 21.52521 (11.62996) + | > current_lr: 0.00003 + | > step_time: 2.20860 (2.76126) + | > loader_time: 0.19810 (0.03446) + + + --> STEP: 129/234 -- GLOBAL_STEP: 25635 + | > loss: -0.07285 (-0.05742) + | > log_mle: -0.27715 (-0.22211) + | > loss_dur: 0.20430 (0.16469) + | > amp_scaler: 4096.00000 (2222.63566) + | > grad_norm: 47.25532 (12.51146) + | > current_lr: 0.00003 + | > step_time: 5.60170 (2.78392) + | > loader_time: 0.19900 (0.03478) + + + --> STEP: 134/234 -- GLOBAL_STEP: 25640 + | > loss: -0.11836 (-0.05988) + | > log_mle: -0.33542 (-0.22591) + | > loss_dur: 0.21706 (0.16602) + | > amp_scaler: 4096.00000 (2292.53731) + | > grad_norm: 35.31830 (13.29114) + | > current_lr: 0.00003 + | > step_time: 2.98690 (2.77433) + | > loader_time: 0.00300 (0.03619) + + + --> STEP: 139/234 -- GLOBAL_STEP: 25645 + | > loss: -0.18407 (-0.06206) + | > log_mle: -0.39529 (-0.22954) + | > loss_dur: 0.21123 (0.16748) + | > amp_scaler: 4096.00000 (2357.41007) + | > grad_norm: 40.31950 (13.97873) + | > current_lr: 0.00003 + | > step_time: 2.89190 (2.74067) + | > loader_time: 0.00220 (0.03619) + + + --> STEP: 144/234 -- GLOBAL_STEP: 25650 + | > loss: -0.14455 (-0.06396) + | > log_mle: -0.36535 (-0.23336) + | > loss_dur: 0.22080 (0.16940) + | > amp_scaler: 4096.00000 (2417.77778) + | > grad_norm: 58.19630 (14.91008) + | > current_lr: 0.00003 + | > step_time: 3.38600 (2.74055) + | > loader_time: 0.00270 (0.03629) + + + --> STEP: 149/234 -- GLOBAL_STEP: 25655 + | > loss: -0.18912 (-0.06690) + | > log_mle: -0.41349 (-0.23771) + | > loss_dur: 0.22436 (0.17081) + | > amp_scaler: 4096.00000 (2474.09396) + | > grad_norm: 42.99543 (15.69064) + | > current_lr: 0.00003 + | > step_time: 3.99450 (2.73982) + | > loader_time: 0.09770 (0.03697) + + + --> STEP: 154/234 -- GLOBAL_STEP: 25660 + | > loss: -0.16729 (-0.07037) + | > log_mle: -0.37430 (-0.24253) + | > loss_dur: 0.20701 (0.17215) + | > amp_scaler: 4096.00000 (2526.75325) + | > grad_norm: 55.30813 (16.68724) + | > current_lr: 0.00003 + | > step_time: 7.89700 (2.78534) + | > loader_time: 0.11070 (0.03903) + + + --> STEP: 159/234 -- GLOBAL_STEP: 25665 + | > loss: -0.17516 (-0.07352) + | > log_mle: -0.38984 (-0.24703) + | > loss_dur: 0.21467 (0.17351) + | > amp_scaler: 4096.00000 (2576.10063) + | > grad_norm: 59.58195 (17.80770) + | > current_lr: 0.00003 + | > step_time: 2.21170 (2.81563) + | > loader_time: 0.07890 (0.04134) + + + --> STEP: 164/234 -- GLOBAL_STEP: 25670 + | > loss: -0.16773 (-0.07642) + | > log_mle: -0.38625 (-0.25137) + | > loss_dur: 0.21852 (0.17495) + | > amp_scaler: 4096.00000 (2622.43902) + | > grad_norm: 52.64521 (18.73751) + | > current_lr: 0.00003 + | > step_time: 5.58710 (2.80838) + | > loader_time: 0.10530 (0.04080) + + + --> STEP: 169/234 -- GLOBAL_STEP: 25675 + | > loss: -0.14389 (-0.07943) + | > log_mle: -0.38660 (-0.25594) + | > loss_dur: 0.24271 (0.17651) + | > amp_scaler: 4096.00000 (2666.03550) + | > grad_norm: 36.12404 (19.52000) + | > current_lr: 0.00003 + | > step_time: 4.79470 (2.81046) + | > loader_time: 0.00250 (0.04013) + + + --> STEP: 174/234 -- GLOBAL_STEP: 25680 + | > loss: -0.24896 (-0.08333) + | > log_mle: -0.47620 (-0.26160) + | > loss_dur: 0.22724 (0.17826) + | > amp_scaler: 4096.00000 (2707.12644) + | > grad_norm: 51.71962 (20.56504) + | > current_lr: 0.00003 + | > step_time: 11.79960 (2.90052) + | > loader_time: 0.09730 (0.04178) + + + --> STEP: 179/234 -- GLOBAL_STEP: 25685 + | > loss: -0.20011 (-0.08644) + | > log_mle: -0.46525 (-0.26667) + | > loss_dur: 0.26514 (0.18023) + | > amp_scaler: 4096.00000 (2745.92179) + | > grad_norm: 59.91502 (21.48474) + | > current_lr: 0.00003 + | > step_time: 4.48980 (2.95302) + | > loader_time: 0.01130 (0.04220) + + + --> STEP: 184/234 -- GLOBAL_STEP: 25690 + | > loss: -0.19059 (-0.08929) + | > log_mle: -0.43281 (-0.27123) + | > loss_dur: 0.24222 (0.18194) + | > amp_scaler: 4096.00000 (2782.60870) + | > grad_norm: 62.94547 (22.43519) + | > current_lr: 0.00003 + | > step_time: 4.19220 (2.97932) + | > loader_time: 0.00360 (0.04271) + + + --> STEP: 189/234 -- GLOBAL_STEP: 25695 + | > loss: -0.18586 (-0.09220) + | > log_mle: -0.43658 (-0.27592) + | > loss_dur: 0.25073 (0.18372) + | > amp_scaler: 4096.00000 (2817.35450) + | > grad_norm: 50.18093 (23.50253) + | > current_lr: 0.00003 + | > step_time: 9.80060 (3.11590) + | > loader_time: 0.09560 (0.04528) + + + --> STEP: 194/234 -- GLOBAL_STEP: 25700 + | > loss: -0.22948 (-0.09573) + | > log_mle: -0.46679 (-0.28064) + | > loss_dur: 0.23731 (0.18491) + | > amp_scaler: 4096.00000 (2850.30928) + | > grad_norm: 62.98124 (24.66239) + | > current_lr: 0.00003 + | > step_time: 4.61080 (3.11770) + | > loader_time: 0.07550 (0.04559) + + + --> STEP: 199/234 -- GLOBAL_STEP: 25705 + | > loss: -0.22554 (-0.09858) + | > log_mle: -0.47363 (-0.28503) + | > loss_dur: 0.24810 (0.18645) + | > amp_scaler: 4096.00000 (2881.60804) + | > grad_norm: 59.71913 (25.68203) + | > current_lr: 0.00003 + | > step_time: 6.40030 (3.22435) + | > loader_time: 0.00470 (0.04687) + + + --> STEP: 204/234 -- GLOBAL_STEP: 25710 + | > loss: -0.24692 (-0.10130) + | > log_mle: -0.51126 (-0.28933) + | > loss_dur: 0.26434 (0.18803) + | > amp_scaler: 4096.00000 (2911.37255) + | > grad_norm: 68.09470 (26.83526) + | > current_lr: 0.00003 + | > step_time: 4.19980 (3.24223) + | > loader_time: 0.18900 (0.04715) + + + --> STEP: 209/234 -- GLOBAL_STEP: 25715 + | > loss: -0.20667 (-0.10424) + | > log_mle: -0.45721 (-0.29387) + | > loss_dur: 0.25054 (0.18963) + | > amp_scaler: 4096.00000 (2939.71292) + | > grad_norm: 83.49506 (28.06331) + | > current_lr: 0.00003 + | > step_time: 1.80750 (3.28711) + | > loader_time: 0.00410 (0.05266) + + + --> STEP: 214/234 -- GLOBAL_STEP: 25720 + | > loss: -0.25902 (-0.10790) + | > log_mle: -0.49765 (-0.29917) + | > loss_dur: 0.23863 (0.19126) + | > amp_scaler: 4096.00000 (2966.72897) + | > grad_norm: 64.71108 (29.34369) + | > current_lr: 0.00003 + | > step_time: 9.79440 (3.36264) + | > loader_time: 0.20190 (0.05246) + + + --> STEP: 219/234 -- GLOBAL_STEP: 25725 + | > loss: -0.32328 (-0.11146) + | > log_mle: -0.58858 (-0.30435) + | > loss_dur: 0.26530 (0.19289) + | > amp_scaler: 4096.00000 (2992.51142) + | > grad_norm: 108.92085 (30.58725) + | > current_lr: 0.00003 + | > step_time: 3.70280 (3.38499) + | > loader_time: 0.08790 (0.05296) + + + --> STEP: 224/234 -- GLOBAL_STEP: 25730 + | > loss: -0.25396 (-0.11452) + | > log_mle: -0.52647 (-0.30908) + | > loss_dur: 0.27250 (0.19455) + | > amp_scaler: 4096.00000 (3017.14286) + | > grad_norm: 94.65899 (31.87075) + | > current_lr: 0.00003 + | > step_time: 0.23350 (3.32202) + | > loader_time: 0.00440 (0.05188) + + + --> STEP: 229/234 -- GLOBAL_STEP: 25735 + | > loss: -0.24488 (-0.11768) + | > log_mle: -0.57438 (-0.31438) + | > loss_dur: 0.32951 (0.19670) + | > amp_scaler: 4096.00000 (3040.69869) + | > grad_norm: 76.51702 (33.06189) + | > current_lr: 0.00003 + | > step_time: 0.25600 (3.25491) + | > loader_time: 0.00400 (0.05081) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.52811 (+0.42142) + | > avg_loss: -0.16437 (-0.05837) + | > avg_log_mle: -0.40039 (-0.05533) + | > avg_loss_dur: 0.23602 (-0.00303) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_25740.pth + + > EPOCH: 110/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 08:29:20)  + + --> STEP: 0/234 -- GLOBAL_STEP: 25740 + | > loss: -0.10540 (-0.10540) + | > log_mle: -0.26043 (-0.26043) + | > loss_dur: 0.15503 (0.15503) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.80491 (13.80491) + | > current_lr: 0.00003 + | > step_time: 2.10400 (2.10403) + | > loader_time: 3.82030 (3.82032) + + + --> STEP: 5/234 -- GLOBAL_STEP: 25745 + | > loss: -0.03625 (-0.02450) + | > log_mle: -0.20154 (-0.19833) + | > loss_dur: 0.16529 (0.17384) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.12151 (12.25334) + | > current_lr: 0.00003 + | > step_time: 6.60600 (3.78275) + | > loader_time: 0.29020 (0.07663) + + + --> STEP: 10/234 -- GLOBAL_STEP: 25750 + | > loss: -0.03789 (-0.03742) + | > log_mle: -0.20266 (-0.20209) + | > loss_dur: 0.16477 (0.16467) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.40853 (10.93716) + | > current_lr: 0.00003 + | > step_time: 10.20260 (5.48191) + | > loader_time: 0.00180 (0.05919) + + + --> STEP: 15/234 -- GLOBAL_STEP: 25755 + | > loss: -0.06244 (-0.04110) + | > log_mle: -0.20239 (-0.20157) + | > loss_dur: 0.13995 (0.16046) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.36234 (9.64943) + | > current_lr: 0.00003 + | > step_time: 5.48510 (5.28728) + | > loader_time: 0.00520 (0.05968) + + + --> STEP: 20/234 -- GLOBAL_STEP: 25760 + | > loss: -0.04678 (-0.04379) + | > log_mle: -0.18716 (-0.19854) + | > loss_dur: 0.14038 (0.15475) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.43235 (8.73770) + | > current_lr: 0.00003 + | > step_time: 3.18510 (5.05299) + | > loader_time: 0.00120 (0.05512) + + + --> STEP: 25/234 -- GLOBAL_STEP: 25765 + | > loss: -0.04163 (-0.04625) + | > log_mle: -0.18300 (-0.19694) + | > loss_dur: 0.14137 (0.15069) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.83487 (8.44798) + | > current_lr: 0.00003 + | > step_time: 4.79850 (4.75793) + | > loader_time: 0.00140 (0.05640) + + + --> STEP: 30/234 -- GLOBAL_STEP: 25770 + | > loss: -0.09020 (-0.04881) + | > log_mle: -0.20782 (-0.19686) + | > loss_dur: 0.11762 (0.14805) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.47711 (8.20948) + | > current_lr: 0.00003 + | > step_time: 3.21810 (4.89939) + | > loader_time: 0.08630 (0.05917) + + + --> STEP: 35/234 -- GLOBAL_STEP: 25775 + | > loss: -0.04266 (-0.04784) + | > log_mle: -0.20150 (-0.19708) + | > loss_dur: 0.15885 (0.14924) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.64662 (8.21534) + | > current_lr: 0.00003 + | > step_time: 4.00920 (4.70527) + | > loader_time: 0.08690 (0.06193) + + + --> STEP: 40/234 -- GLOBAL_STEP: 25780 + | > loss: -0.02284 (-0.04718) + | > log_mle: -0.18199 (-0.19718) + | > loss_dur: 0.15915 (0.15000) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.53122 (8.18304) + | > current_lr: 0.00003 + | > step_time: 3.11660 (4.33870) + | > loader_time: 0.00150 (0.05607) + + + --> STEP: 45/234 -- GLOBAL_STEP: 25785 + | > loss: -0.06078 (-0.04746) + | > log_mle: -0.21995 (-0.19750) + | > loss_dur: 0.15917 (0.15003) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.03394 (8.04507) + | > current_lr: 0.00003 + | > step_time: 1.46970 (4.03662) + | > loader_time: 0.00200 (0.05431) + + + --> STEP: 50/234 -- GLOBAL_STEP: 25790 + | > loss: -0.02333 (-0.04746) + | > log_mle: -0.18674 (-0.19711) + | > loss_dur: 0.16340 (0.14965) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.92442 (7.75928) + | > current_lr: 0.00003 + | > step_time: 1.48630 (3.78986) + | > loader_time: 0.00200 (0.05067) + + + --> STEP: 55/234 -- GLOBAL_STEP: 25795 + | > loss: -0.05697 (-0.04750) + | > log_mle: -0.20718 (-0.19738) + | > loss_dur: 0.15021 (0.14988) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.08027 (7.56212) + | > current_lr: 0.00003 + | > step_time: 1.54640 (3.60688) + | > loader_time: 0.00230 (0.04941) + + + --> STEP: 60/234 -- GLOBAL_STEP: 25800 + | > loss: -0.06471 (-0.04787) + | > log_mle: -0.22363 (-0.19800) + | > loss_dur: 0.15892 (0.15013) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.20932 (7.57084) + | > current_lr: 0.00003 + | > step_time: 1.19090 (3.45595) + | > loader_time: 0.00890 (0.04700) + + + --> STEP: 65/234 -- GLOBAL_STEP: 25805 + | > loss: -0.05399 (-0.04756) + | > log_mle: -0.19872 (-0.19865) + | > loss_dur: 0.14473 (0.15109) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.87382 (7.63186) + | > current_lr: 0.00003 + | > step_time: 2.40680 (3.36890) + | > loader_time: 0.08080 (0.04736) + + + --> STEP: 70/234 -- GLOBAL_STEP: 25810 + | > loss: -0.03256 (-0.04653) + | > log_mle: -0.19749 (-0.19850) + | > loss_dur: 0.16494 (0.15198) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.58795 (7.66348) + | > current_lr: 0.00003 + | > step_time: 2.90730 (3.28395) + | > loader_time: 0.09700 (0.04683) + + + --> STEP: 75/234 -- GLOBAL_STEP: 25815 + | > loss: -0.04954 (-0.04604) + | > log_mle: -0.21117 (-0.19929) + | > loss_dur: 0.16163 (0.15326) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.05062 (7.86019) + | > current_lr: 0.00003 + | > step_time: 2.19480 (3.20118) + | > loader_time: 0.00300 (0.04509) + + + --> STEP: 80/234 -- GLOBAL_STEP: 25820 + | > loss: -0.06219 (-0.04664) + | > log_mle: -0.19198 (-0.19962) + | > loss_dur: 0.12979 (0.15298) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.93230 (7.77917) + | > current_lr: 0.00003 + | > step_time: 1.92520 (3.12035) + | > loader_time: 0.08170 (0.04568) + + + --> STEP: 85/234 -- GLOBAL_STEP: 25825 + | > loss: -0.06460 (-0.04716) + | > log_mle: -0.21026 (-0.20037) + | > loss_dur: 0.14566 (0.15321) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.91394 (7.88710) + | > current_lr: 0.00003 + | > step_time: 2.79340 (3.04100) + | > loader_time: 0.00260 (0.04312) + + + --> STEP: 90/234 -- GLOBAL_STEP: 25830 + | > loss: -0.04909 (-0.04796) + | > log_mle: -0.23828 (-0.20226) + | > loss_dur: 0.18919 (0.15430) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.92141 (8.26473) + | > current_lr: 0.00003 + | > step_time: 1.50760 (2.97668) + | > loader_time: 0.09130 (0.04282) + + + --> STEP: 95/234 -- GLOBAL_STEP: 25835 + | > loss: -0.12901 (-0.05048) + | > log_mle: -0.32583 (-0.20599) + | > loss_dur: 0.19682 (0.15551) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.37996 (8.85902) + | > current_lr: 0.00003 + | > step_time: 1.98440 (2.95001) + | > loader_time: 0.00420 (0.04073) + + + --> STEP: 100/234 -- GLOBAL_STEP: 25840 + | > loss: -0.07553 (-0.05161) + | > log_mle: -0.25051 (-0.20798) + | > loss_dur: 0.17498 (0.15636) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.86792 (9.15332) + | > current_lr: 0.00003 + | > step_time: 1.11470 (2.88543) + | > loader_time: 0.08830 (0.04052) + + + --> STEP: 105/234 -- GLOBAL_STEP: 25845 + | > loss: -0.07249 (-0.05358) + | > log_mle: -0.22563 (-0.21099) + | > loss_dur: 0.15314 (0.15741) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.47430 (9.59530) + | > current_lr: 0.00003 + | > step_time: 2.09630 (2.86916) + | > loader_time: 0.00500 (0.03961) + + + --> STEP: 110/234 -- GLOBAL_STEP: 25850 + | > loss: -0.08212 (-0.05445) + | > log_mle: -0.25410 (-0.21365) + | > loss_dur: 0.17198 (0.15920) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.92869 (10.06632) + | > current_lr: 0.00003 + | > step_time: 1.05120 (2.83219) + | > loader_time: 0.00240 (0.03794) + + + --> STEP: 115/234 -- GLOBAL_STEP: 25855 + | > loss: -0.07016 (-0.05610) + | > log_mle: -0.27129 (-0.21688) + | > loss_dur: 0.20113 (0.16078) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.18189 (10.82560) + | > current_lr: 0.00003 + | > step_time: 2.09780 (2.80049) + | > loader_time: 0.00260 (0.03794) + + + --> STEP: 120/234 -- GLOBAL_STEP: 25860 + | > loss: -0.12763 (-0.05748) + | > log_mle: -0.32165 (-0.21970) + | > loss_dur: 0.19401 (0.16222) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.81913 (11.25917) + | > current_lr: 0.00003 + | > step_time: 1.59950 (2.77714) + | > loader_time: 0.00380 (0.03795) + + + --> STEP: 125/234 -- GLOBAL_STEP: 25865 + | > loss: -0.11361 (-0.05851) + | > log_mle: -0.30670 (-0.22132) + | > loss_dur: 0.19308 (0.16282) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.44281 (11.51015) + | > current_lr: 0.00003 + | > step_time: 1.50310 (2.76692) + | > loader_time: 0.00230 (0.03657) + + + --> STEP: 130/234 -- GLOBAL_STEP: 25870 + | > loss: -0.10629 (-0.06038) + | > log_mle: -0.31906 (-0.22471) + | > loss_dur: 0.21277 (0.16434) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.59824 (12.23996) + | > current_lr: 0.00003 + | > step_time: 3.09780 (2.75659) + | > loader_time: 0.08540 (0.03736) + + + --> STEP: 135/234 -- GLOBAL_STEP: 25875 + | > loss: -0.07634 (-0.06247) + | > log_mle: -0.25083 (-0.22808) + | > loss_dur: 0.17449 (0.16562) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.49589 (12.87792) + | > current_lr: 0.00003 + | > step_time: 3.19850 (2.75359) + | > loader_time: 0.10160 (0.03812) + + + --> STEP: 140/234 -- GLOBAL_STEP: 25880 + | > loss: -0.08108 (-0.06471) + | > log_mle: -0.28422 (-0.23197) + | > loss_dur: 0.20314 (0.16726) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.51342 (13.66381) + | > current_lr: 0.00003 + | > step_time: 4.00260 (2.74018) + | > loader_time: 0.18970 (0.04026) + + + --> STEP: 145/234 -- GLOBAL_STEP: 25885 + | > loss: -0.16283 (-0.06732) + | > log_mle: -0.38328 (-0.23656) + | > loss_dur: 0.22045 (0.16924) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.58075 (14.48995) + | > current_lr: 0.00003 + | > step_time: 4.59560 (2.72212) + | > loader_time: 0.10450 (0.04080) + + + --> STEP: 150/234 -- GLOBAL_STEP: 25890 + | > loss: -0.15162 (-0.07014) + | > log_mle: -0.36401 (-0.24080) + | > loss_dur: 0.21239 (0.17066) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.11247 (15.47809) + | > current_lr: 0.00003 + | > step_time: 4.00530 (2.70401) + | > loader_time: 0.00800 (0.04012) + + + --> STEP: 155/234 -- GLOBAL_STEP: 25895 + | > loss: -0.20005 (-0.07390) + | > log_mle: -0.43317 (-0.24606) + | > loss_dur: 0.23312 (0.17216) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 50.33126 (16.60398) + | > current_lr: 0.00003 + | > step_time: 2.20930 (2.74452) + | > loader_time: 0.00560 (0.04012) + + + --> STEP: 160/234 -- GLOBAL_STEP: 25900 + | > loss: -0.20103 (-0.07697) + | > log_mle: -0.42788 (-0.25064) + | > loss_dur: 0.22686 (0.17367) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.92799 (17.49948) + | > current_lr: 0.00003 + | > step_time: 3.38620 (2.77433) + | > loader_time: 0.00300 (0.04064) + + + --> STEP: 165/234 -- GLOBAL_STEP: 25905 + | > loss: -0.17111 (-0.07977) + | > log_mle: -0.41389 (-0.25490) + | > loss_dur: 0.24278 (0.17513) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 82.94543 (18.68170) + | > current_lr: 0.00003 + | > step_time: 2.89050 (2.78958) + | > loader_time: 0.00450 (0.04074) + + + --> STEP: 170/234 -- GLOBAL_STEP: 25910 + | > loss: -0.19087 (-0.08262) + | > log_mle: -0.45116 (-0.25947) + | > loss_dur: 0.26028 (0.17685) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 80.56162 (19.92915) + | > current_lr: 0.00003 + | > step_time: 1.90120 (2.75568) + | > loader_time: 0.08710 (0.04105) + + + --> STEP: 175/234 -- GLOBAL_STEP: 25915 + | > loss: -0.18945 (-0.08636) + | > log_mle: -0.43179 (-0.26477) + | > loss_dur: 0.24234 (0.17841) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.32449 (21.03579) + | > current_lr: 0.00003 + | > step_time: 1.32180 (2.75197) + | > loader_time: 0.08900 (0.04261) + + + --> STEP: 180/234 -- GLOBAL_STEP: 25920 + | > loss: -0.20206 (-0.08948) + | > log_mle: -0.43910 (-0.26971) + | > loss_dur: 0.23704 (0.18023) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.89623 (22.23259) + | > current_lr: 0.00003 + | > step_time: 11.91220 (2.81673) + | > loader_time: 0.09630 (0.04354) + + + --> STEP: 185/234 -- GLOBAL_STEP: 25925 + | > loss: -0.20238 (-0.09229) + | > log_mle: -0.46147 (-0.27423) + | > loss_dur: 0.25909 (0.18194) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 83.29821 (23.33809) + | > current_lr: 0.00003 + | > step_time: 7.69490 (2.87081) + | > loader_time: 0.10490 (0.04460) + + + --> STEP: 190/234 -- GLOBAL_STEP: 25930 + | > loss: -0.21405 (-0.09515) + | > log_mle: -0.44426 (-0.27870) + | > loss_dur: 0.23021 (0.18354) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.91171 (24.24278) + | > current_lr: 0.00003 + | > step_time: 7.79330 (2.91730) + | > loader_time: 0.00440 (0.04662) + + + --> STEP: 195/234 -- GLOBAL_STEP: 25935 + | > loss: -0.20946 (-0.09846) + | > log_mle: -0.45988 (-0.28341) + | > loss_dur: 0.25042 (0.18495) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.27532 (25.17732) + | > current_lr: 0.00003 + | > step_time: 4.30720 (2.98522) + | > loader_time: 0.19580 (0.05046) + + + --> STEP: 200/234 -- GLOBAL_STEP: 25940 + | > loss: -0.19713 (-0.10129) + | > log_mle: -0.47108 (-0.28785) + | > loss_dur: 0.27395 (0.18656) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.04822 (26.02033) + | > current_lr: 0.00003 + | > step_time: 5.11350 (3.03774) + | > loader_time: 0.19480 (0.05161) + + + --> STEP: 205/234 -- GLOBAL_STEP: 25945 + | > loss: -0.21185 (-0.10395) + | > log_mle: -0.45943 (-0.29206) + | > loss_dur: 0.24757 (0.18810) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 46.82362 (26.91321) + | > current_lr: 0.00003 + | > step_time: 3.50250 (3.16953) + | > loader_time: 0.19890 (0.05363) + + + --> STEP: 210/234 -- GLOBAL_STEP: 25950 + | > loss: -0.27419 (-0.10721) + | > log_mle: -0.53241 (-0.29695) + | > loss_dur: 0.25821 (0.18974) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.27561 (27.97647) + | > current_lr: 0.00003 + | > step_time: 4.49640 (3.18642) + | > loader_time: 0.00420 (0.05474) + + + --> STEP: 215/234 -- GLOBAL_STEP: 25955 + | > loss: -0.23409 (-0.11070) + | > log_mle: -0.47999 (-0.30200) + | > loss_dur: 0.24589 (0.19130) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 55.01707 (29.12513) + | > current_lr: 0.00003 + | > step_time: 2.70820 (3.24591) + | > loader_time: 0.19460 (0.05615) + + + --> STEP: 220/234 -- GLOBAL_STEP: 25960 + | > loss: -0.26658 (-0.11425) + | > log_mle: -0.52710 (-0.30724) + | > loss_dur: 0.26052 (0.19299) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 85.49673 (30.55624) + | > current_lr: 0.00003 + | > step_time: 4.60850 (3.27500) + | > loader_time: 0.19860 (0.05667) + + + --> STEP: 225/234 -- GLOBAL_STEP: 25965 + | > loss: -0.30913 (-0.11750) + | > log_mle: -0.59160 (-0.31230) + | > loss_dur: 0.28247 (0.19479) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 91.92752 (31.69507) + | > current_lr: 0.00003 + | > step_time: 0.23820 (3.21924) + | > loader_time: 0.00380 (0.05586) + + + --> STEP: 230/234 -- GLOBAL_STEP: 25970 + | > loss: -0.27968 (-0.12056) + | > log_mle: -0.63982 (-0.31778) + | > loss_dur: 0.36013 (0.19723) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 97.73514 (33.08428) + | > current_lr: 0.00003 + | > step_time: 0.25020 (3.15450) + | > loader_time: 0.00330 (0.05473) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07890 (-0.44921) + | > avg_loss: -0.17195 (-0.00758) + | > avg_log_mle: -0.40629 (-0.00590) + | > avg_loss_dur: 0.23434 (-0.00168) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_25974.pth + + > EPOCH: 111/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 08:42:34)  + + --> STEP: 1/234 -- GLOBAL_STEP: 25975 + | > loss: -0.04531 (-0.04531) + | > log_mle: -0.20265 (-0.20265) + | > loss_dur: 0.15734 (0.15734) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.49418 (12.49418) + | > current_lr: 0.00003 + | > step_time: 1.48920 (1.48921) + | > loader_time: 0.19910 (0.19907) + + + --> STEP: 6/234 -- GLOBAL_STEP: 25980 + | > loss: -0.04801 (-0.02051) + | > log_mle: -0.19252 (-0.19808) + | > loss_dur: 0.14451 (0.17757) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.82863 (11.62977) + | > current_lr: 0.00003 + | > step_time: 11.19950 (4.43853) + | > loader_time: 0.00160 (0.79462) + + + --> STEP: 11/234 -- GLOBAL_STEP: 25985 + | > loss: -0.06922 (-0.03551) + | > log_mle: -0.19909 (-0.20254) + | > loss_dur: 0.12987 (0.16702) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.36406 (10.29266) + | > current_lr: 0.00003 + | > step_time: 7.99780 (4.71008) + | > loader_time: 0.29160 (0.48637) + + + --> STEP: 16/234 -- GLOBAL_STEP: 25990 + | > loss: -0.08512 (-0.04302) + | > log_mle: -0.19950 (-0.20217) + | > loss_dur: 0.11438 (0.15915) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.61797 (9.18408) + | > current_lr: 0.00003 + | > step_time: 5.30870 (4.65802) + | > loader_time: 0.09400 (0.34624) + + + --> STEP: 21/234 -- GLOBAL_STEP: 25995 + | > loss: -0.03136 (-0.04099) + | > log_mle: -0.18075 (-0.19849) + | > loss_dur: 0.14939 (0.15750) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.57364 (8.68200) + | > current_lr: 0.00003 + | > step_time: 1.93180 (4.13742) + | > loader_time: 0.00340 (0.26445) + + + --> STEP: 26/234 -- GLOBAL_STEP: 26000 + | > loss: -0.05238 (-0.04467) + | > log_mle: -0.19794 (-0.19800) + | > loss_dur: 0.14556 (0.15333) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.17485 (8.29528) + | > current_lr: 0.00003 + | > step_time: 1.49010 (3.59732) + | > loader_time: 0.00140 (0.21393) + + + --> STEP: 31/234 -- GLOBAL_STEP: 26005 + | > loss: -0.01051 (-0.04653) + | > log_mle: -0.20031 (-0.19817) + | > loss_dur: 0.18980 (0.15164) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.61660 (7.96568) + | > current_lr: 0.00003 + | > step_time: 3.00320 (3.51073) + | > loader_time: 0.00110 (0.18620) + + + --> STEP: 36/234 -- GLOBAL_STEP: 26010 + | > loss: -0.04070 (-0.04749) + | > log_mle: -0.20329 (-0.19871) + | > loss_dur: 0.16259 (0.15122) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.36141 (7.93905) + | > current_lr: 0.00003 + | > step_time: 0.91560 (3.20916) + | > loader_time: 0.00180 (0.16269) + + + --> STEP: 41/234 -- GLOBAL_STEP: 26015 + | > loss: -0.05332 (-0.04778) + | > log_mle: -0.19858 (-0.19877) + | > loss_dur: 0.14526 (0.15099) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.29812 (7.76535) + | > current_lr: 0.00003 + | > step_time: 1.40310 (2.98330) + | > loader_time: 0.08690 (0.14521) + + + --> STEP: 46/234 -- GLOBAL_STEP: 26020 + | > loss: -0.04337 (-0.04792) + | > log_mle: -0.20029 (-0.19923) + | > loss_dur: 0.15693 (0.15132) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.32469 (7.79777) + | > current_lr: 0.00003 + | > step_time: 1.29820 (2.80289) + | > loader_time: 0.00210 (0.12967) + + + --> STEP: 51/234 -- GLOBAL_STEP: 26025 + | > loss: -0.05113 (-0.04836) + | > log_mle: -0.18654 (-0.19866) + | > loss_dur: 0.13541 (0.15030) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.09025 (7.58871) + | > current_lr: 0.00003 + | > step_time: 1.69290 (2.67265) + | > loader_time: 0.00250 (0.11720) + + + --> STEP: 56/234 -- GLOBAL_STEP: 26030 + | > loss: -0.03418 (-0.04904) + | > log_mle: -0.20583 (-0.19931) + | > loss_dur: 0.17166 (0.15028) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.49683 (7.55820) + | > current_lr: 0.00003 + | > step_time: 1.18500 (2.56654) + | > loader_time: 0.00160 (0.10854) + + + --> STEP: 61/234 -- GLOBAL_STEP: 26035 + | > loss: -0.06236 (-0.05000) + | > log_mle: -0.20353 (-0.19991) + | > loss_dur: 0.14117 (0.14992) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.56218 (7.50852) + | > current_lr: 0.00003 + | > step_time: 1.80620 (2.46970) + | > loader_time: 0.08590 (0.10256) + + + --> STEP: 66/234 -- GLOBAL_STEP: 26040 + | > loss: -0.05306 (-0.04951) + | > log_mle: -0.19176 (-0.20036) + | > loss_dur: 0.13870 (0.15085) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.78071 (7.51885) + | > current_lr: 0.00003 + | > step_time: 1.20710 (2.40567) + | > loader_time: 0.00200 (0.09501) + + + --> STEP: 71/234 -- GLOBAL_STEP: 26045 + | > loss: -0.03631 (-0.04857) + | > log_mle: -0.22839 (-0.20070) + | > loss_dur: 0.19208 (0.15213) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.82888 (7.87141) + | > current_lr: 0.00003 + | > step_time: 1.28340 (2.37246) + | > loader_time: 0.00220 (0.09081) + + + --> STEP: 76/234 -- GLOBAL_STEP: 26050 + | > loss: -0.06277 (-0.04851) + | > log_mle: -0.21697 (-0.20127) + | > loss_dur: 0.15420 (0.15275) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.29500 (7.88326) + | > current_lr: 0.00003 + | > step_time: 2.08930 (2.36584) + | > loader_time: 0.00250 (0.08613) + + + --> STEP: 81/234 -- GLOBAL_STEP: 26055 + | > loss: -0.05830 (-0.04907) + | > log_mle: -0.22291 (-0.20164) + | > loss_dur: 0.16461 (0.15257) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.94996 (7.87490) + | > current_lr: 0.00003 + | > step_time: 2.59770 (2.34963) + | > loader_time: 0.00290 (0.08202) + + + --> STEP: 86/234 -- GLOBAL_STEP: 26060 + | > loss: -0.05482 (-0.04947) + | > log_mle: -0.22261 (-0.20235) + | > loss_dur: 0.16779 (0.15288) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.20535 (8.00686) + | > current_lr: 0.00003 + | > step_time: 1.28070 (2.36185) + | > loader_time: 0.00170 (0.07835) + + + --> STEP: 91/234 -- GLOBAL_STEP: 26065 + | > loss: -0.05668 (-0.05021) + | > log_mle: -0.23468 (-0.20430) + | > loss_dur: 0.17800 (0.15409) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.25211 (8.30024) + | > current_lr: 0.00003 + | > step_time: 1.21290 (2.30519) + | > loader_time: 0.08820 (0.07728) + + + --> STEP: 96/234 -- GLOBAL_STEP: 26070 + | > loss: -0.04177 (-0.05249) + | > log_mle: -0.22022 (-0.20777) + | > loss_dur: 0.17845 (0.15528) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.65208 (9.02985) + | > current_lr: 0.00003 + | > step_time: 1.80580 (2.29267) + | > loader_time: 0.08750 (0.07596) + + + --> STEP: 101/234 -- GLOBAL_STEP: 26075 + | > loss: -0.08224 (-0.05377) + | > log_mle: -0.27520 (-0.21014) + | > loss_dur: 0.19296 (0.15638) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.83322 (9.45329) + | > current_lr: 0.00003 + | > step_time: 6.38590 (2.34337) + | > loader_time: 0.00130 (0.07623) + + + --> STEP: 106/234 -- GLOBAL_STEP: 26080 + | > loss: -0.07200 (-0.05527) + | > log_mle: -0.27802 (-0.21295) + | > loss_dur: 0.20601 (0.15768) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.82711 (9.95912) + | > current_lr: 0.00003 + | > step_time: 3.09290 (2.38782) + | > loader_time: 0.01340 (0.07548) + + + --> STEP: 111/234 -- GLOBAL_STEP: 26085 + | > loss: -0.09304 (-0.05633) + | > log_mle: -0.32159 (-0.21581) + | > loss_dur: 0.22855 (0.15948) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.18178 (10.50100) + | > current_lr: 0.00003 + | > step_time: 1.10580 (2.34747) + | > loader_time: 0.00330 (0.07296) + + + --> STEP: 116/234 -- GLOBAL_STEP: 26090 + | > loss: -0.06844 (-0.05749) + | > log_mle: -0.28941 (-0.21865) + | > loss_dur: 0.22098 (0.16116) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.15768 (11.00885) + | > current_lr: 0.00003 + | > step_time: 2.60900 (2.38617) + | > loader_time: 0.00340 (0.07085) + + + --> STEP: 121/234 -- GLOBAL_STEP: 26095 + | > loss: -0.03542 (-0.05844) + | > log_mle: -0.20533 (-0.22065) + | > loss_dur: 0.16991 (0.16221) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.59349 (11.24956) + | > current_lr: 0.00003 + | > step_time: 0.95070 (2.35268) + | > loader_time: 0.00330 (0.06871) + + + --> STEP: 126/234 -- GLOBAL_STEP: 26100 + | > loss: -0.12855 (-0.06015) + | > log_mle: -0.33861 (-0.22329) + | > loss_dur: 0.21006 (0.16314) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.84062 (11.72186) + | > current_lr: 0.00003 + | > step_time: 2.29970 (2.33804) + | > loader_time: 0.00330 (0.06820) + + + --> STEP: 131/234 -- GLOBAL_STEP: 26105 + | > loss: -0.16514 (-0.06235) + | > log_mle: -0.37824 (-0.22691) + | > loss_dur: 0.21310 (0.16456) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.04876 (12.40685) + | > current_lr: 0.00003 + | > step_time: 1.07010 (2.33339) + | > loader_time: 0.00290 (0.06715) + + + --> STEP: 136/234 -- GLOBAL_STEP: 26110 + | > loss: -0.17727 (-0.06461) + | > log_mle: -0.41696 (-0.23048) + | > loss_dur: 0.23969 (0.16587) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.01819 (13.20614) + | > current_lr: 0.00003 + | > step_time: 1.60450 (2.31660) + | > loader_time: 0.08920 (0.06603) + + + --> STEP: 141/234 -- GLOBAL_STEP: 26115 + | > loss: -0.11441 (-0.06612) + | > log_mle: -0.33273 (-0.23355) + | > loss_dur: 0.21832 (0.16743) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.53382 (13.96222) + | > current_lr: 0.00003 + | > step_time: 1.48670 (2.33658) + | > loader_time: 0.00280 (0.06504) + + + --> STEP: 146/234 -- GLOBAL_STEP: 26120 + | > loss: -0.16613 (-0.06896) + | > log_mle: -0.38397 (-0.23834) + | > loss_dur: 0.21784 (0.16938) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.19610 (14.99831) + | > current_lr: 0.00003 + | > step_time: 3.99900 (2.32863) + | > loader_time: 0.19140 (0.06534) + + + --> STEP: 151/234 -- GLOBAL_STEP: 26125 + | > loss: -0.16346 (-0.07178) + | > log_mle: -0.35446 (-0.24230) + | > loss_dur: 0.19100 (0.17052) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 38.57022 (15.72447) + | > current_lr: 0.00003 + | > step_time: 3.01990 (2.33556) + | > loader_time: 0.10260 (0.06464) + + + --> STEP: 156/234 -- GLOBAL_STEP: 26130 + | > loss: -0.17844 (-0.07553) + | > log_mle: -0.38825 (-0.24762) + | > loss_dur: 0.20981 (0.17210) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.64987 (17.00699) + | > current_lr: 0.00003 + | > step_time: 1.89950 (2.35412) + | > loader_time: 0.09570 (0.06431) + + + --> STEP: 161/234 -- GLOBAL_STEP: 26135 + | > loss: -0.19944 (-0.07854) + | > log_mle: -0.41647 (-0.25223) + | > loss_dur: 0.21703 (0.17368) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.49798 (17.92212) + | > current_lr: 0.00003 + | > step_time: 5.60280 (2.43950) + | > loader_time: 0.09700 (0.06783) + + + --> STEP: 166/234 -- GLOBAL_STEP: 26140 + | > loss: -0.15379 (-0.08114) + | > log_mle: -0.35582 (-0.25616) + | > loss_dur: 0.20203 (0.17502) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 37.69774 (18.72232) + | > current_lr: 0.00003 + | > step_time: 7.88980 (2.47552) + | > loader_time: 0.00780 (0.06657) + + + --> STEP: 171/234 -- GLOBAL_STEP: 26145 + | > loss: -0.23042 (-0.08466) + | > log_mle: -0.45517 (-0.26139) + | > loss_dur: 0.22475 (0.17673) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.36463 (19.87863) + | > current_lr: 0.00003 + | > step_time: 4.91060 (2.55891) + | > loader_time: 0.08900 (0.06671) + + + --> STEP: 176/234 -- GLOBAL_STEP: 26150 + | > loss: -0.20471 (-0.08810) + | > log_mle: -0.43280 (-0.26655) + | > loss_dur: 0.22809 (0.17845) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.58589 (21.02042) + | > current_lr: 0.00003 + | > step_time: 1.69920 (2.58633) + | > loader_time: 0.08020 (0.06728) + + + --> STEP: 181/234 -- GLOBAL_STEP: 26155 + | > loss: -0.13735 (-0.09091) + | > log_mle: -0.36938 (-0.27117) + | > loss_dur: 0.23203 (0.18026) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.45321 (21.90349) + | > current_lr: 0.00003 + | > step_time: 1.49820 (2.56315) + | > loader_time: 0.00250 (0.06636) + + + --> STEP: 186/234 -- GLOBAL_STEP: 26160 + | > loss: -0.15797 (-0.09363) + | > log_mle: -0.40849 (-0.27584) + | > loss_dur: 0.25051 (0.18221) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.64787 (23.10917) + | > current_lr: 0.00003 + | > step_time: 3.61070 (2.64875) + | > loader_time: 0.08990 (0.07632) + + + --> STEP: 191/234 -- GLOBAL_STEP: 26165 + | > loss: -0.19848 (-0.09662) + | > log_mle: -0.42114 (-0.28032) + | > loss_dur: 0.22267 (0.18369) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.17298 (24.24814) + | > current_lr: 0.00003 + | > step_time: 2.29610 (2.63291) + | > loader_time: 0.00420 (0.07533) + + + --> STEP: 196/234 -- GLOBAL_STEP: 26170 + | > loss: -0.16815 (-0.09973) + | > log_mle: -0.41849 (-0.28490) + | > loss_dur: 0.25034 (0.18517) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 74.25200 (25.42656) + | > current_lr: 0.00003 + | > step_time: 6.60160 (2.68608) + | > loader_time: 0.00450 (0.07446) + + + --> STEP: 201/234 -- GLOBAL_STEP: 26175 + | > loss: -0.13642 (-0.10219) + | > log_mle: -0.38710 (-0.28900) + | > loss_dur: 0.25068 (0.18681) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.78633 (26.30964) + | > current_lr: 0.00003 + | > step_time: 12.90530 (2.78058) + | > loader_time: 0.50030 (0.07697) + + + --> STEP: 206/234 -- GLOBAL_STEP: 26180 + | > loss: -0.24044 (-0.10540) + | > log_mle: -0.49059 (-0.29369) + | > loss_dur: 0.25015 (0.18829) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.11926 (27.23368) + | > current_lr: 0.00003 + | > step_time: 3.80590 (2.85251) + | > loader_time: 0.09630 (0.07838) + + + --> STEP: 211/234 -- GLOBAL_STEP: 26185 + | > loss: -0.27372 (-0.10892) + | > log_mle: -0.55426 (-0.29894) + | > loss_dur: 0.28054 (0.19002) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 100.26490 (28.32831) + | > current_lr: 0.00003 + | > step_time: 2.99770 (2.86924) + | > loader_time: 0.00300 (0.07887) + + + --> STEP: 216/234 -- GLOBAL_STEP: 26190 + | > loss: -0.26516 (-0.11238) + | > log_mle: -0.54490 (-0.30395) + | > loss_dur: 0.27974 (0.19157) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 83.55995 (29.28304) + | > current_lr: 0.00003 + | > step_time: 6.10230 (2.91902) + | > loader_time: 0.09390 (0.08107) + + + --> STEP: 221/234 -- GLOBAL_STEP: 26195 + | > loss: -0.21607 (-0.11591) + | > log_mle: -0.46931 (-0.30904) + | > loss_dur: 0.25324 (0.19313) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.59658 (30.29061) + | > current_lr: 0.00003 + | > step_time: 3.20160 (2.95747) + | > loader_time: 0.09100 (0.08290) + + + --> STEP: 226/234 -- GLOBAL_STEP: 26200 + | > loss: -0.30063 (-0.11971) + | > log_mle: -0.56683 (-0.31460) + | > loss_dur: 0.26620 (0.19489) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 79.11407 (31.52545) + | > current_lr: 0.00003 + | > step_time: 0.65950 (2.97284) + | > loader_time: 0.07510 (0.08230) + + + --> STEP: 231/234 -- GLOBAL_STEP: 26205 + | > loss: -0.20595 (-0.12234) + | > log_mle: -0.62236 (-0.32028) + | > loss_dur: 0.41641 (0.19794) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 113.36635 (33.17539) + | > current_lr: 0.00003 + | > step_time: 0.26220 (2.91387) + | > loader_time: 0.00380 (0.08060) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.81801 (+0.73911) + | > avg_loss: -0.16985 (+0.00210) + | > avg_log_mle: -0.40591 (+0.00038) + | > avg_loss_dur: 0.23606 (+0.00172) + + + > EPOCH: 112/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 08:55:03)  + + --> STEP: 2/234 -- GLOBAL_STEP: 26210 + | > loss: -0.02332 (-0.05013) + | > log_mle: -0.19299 (-0.19865) + | > loss_dur: 0.16967 (0.14851) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.41185 (10.18519) + | > current_lr: 0.00003 + | > step_time: 6.19950 (5.75252) + | > loader_time: 0.50160 (0.29555) + + + --> STEP: 7/234 -- GLOBAL_STEP: 26215 + | > loss: -0.04960 (-0.03277) + | > log_mle: -0.21404 (-0.20235) + | > loss_dur: 0.16444 (0.16959) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.30831 (11.08482) + | > current_lr: 0.00003 + | > step_time: 8.29720 (6.77170) + | > loader_time: 0.09820 (0.11305) + + + --> STEP: 12/234 -- GLOBAL_STEP: 26220 + | > loss: -0.04791 (-0.03928) + | > log_mle: -0.20195 (-0.20444) + | > loss_dur: 0.15404 (0.16516) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.82406 (10.82551) + | > current_lr: 0.00003 + | > step_time: 7.30740 (6.19373) + | > loader_time: 0.00330 (0.09931) + + + --> STEP: 17/234 -- GLOBAL_STEP: 26225 + | > loss: -0.02061 (-0.04373) + | > log_mle: -0.17924 (-0.20264) + | > loss_dur: 0.15863 (0.15891) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.47975 (10.05450) + | > current_lr: 0.00003 + | > step_time: 1.90620 (5.53761) + | > loader_time: 0.09960 (0.08797) + + + --> STEP: 22/234 -- GLOBAL_STEP: 26230 + | > loss: -0.07412 (-0.04611) + | > log_mle: -0.20594 (-0.20063) + | > loss_dur: 0.13182 (0.15452) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.22656 (9.52639) + | > current_lr: 0.00003 + | > step_time: 1.90430 (4.69259) + | > loader_time: 0.00510 (0.07214) + + + --> STEP: 27/234 -- GLOBAL_STEP: 26235 + | > loss: -0.06842 (-0.04919) + | > log_mle: -0.20651 (-0.20001) + | > loss_dur: 0.13809 (0.15082) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.09201 (9.05822) + | > current_lr: 0.00003 + | > step_time: 2.68480 (4.20030) + | > loader_time: 0.00440 (0.05924) + + + --> STEP: 32/234 -- GLOBAL_STEP: 26240 + | > loss: -0.08864 (-0.05081) + | > log_mle: -0.21638 (-0.20027) + | > loss_dur: 0.12774 (0.14946) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.19089 (8.59318) + | > current_lr: 0.00003 + | > step_time: 2.19010 (3.86843) + | > loader_time: 0.00450 (0.05293) + + + --> STEP: 37/234 -- GLOBAL_STEP: 26245 + | > loss: -0.05805 (-0.04955) + | > log_mle: -0.19543 (-0.19996) + | > loss_dur: 0.13738 (0.15042) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.33913 (8.64181) + | > current_lr: 0.00003 + | > step_time: 3.41160 (3.67519) + | > loader_time: 0.08080 (0.04817) + + + --> STEP: 42/234 -- GLOBAL_STEP: 26250 + | > loss: -0.05783 (-0.04948) + | > log_mle: -0.18824 (-0.19960) + | > loss_dur: 0.13042 (0.15012) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.03634 (8.40693) + | > current_lr: 0.00003 + | > step_time: 1.67200 (3.53746) + | > loader_time: 0.00190 (0.04721) + + + --> STEP: 47/234 -- GLOBAL_STEP: 26255 + | > loss: -0.03662 (-0.05002) + | > log_mle: -0.19809 (-0.20019) + | > loss_dur: 0.16147 (0.15017) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.41845 (8.32334) + | > current_lr: 0.00003 + | > step_time: 2.71170 (3.44213) + | > loader_time: 0.09650 (0.04446) + + + --> STEP: 52/234 -- GLOBAL_STEP: 26260 + | > loss: -0.03752 (-0.05017) + | > log_mle: -0.19462 (-0.19960) + | > loss_dur: 0.15710 (0.14942) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.51337 (8.01025) + | > current_lr: 0.00003 + | > step_time: 1.86830 (3.28936) + | > loader_time: 0.00190 (0.04182) + + + --> STEP: 57/234 -- GLOBAL_STEP: 26265 + | > loss: -0.04256 (-0.05048) + | > log_mle: -0.19002 (-0.20019) + | > loss_dur: 0.14745 (0.14972) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 4.93738 (7.79102) + | > current_lr: 0.00003 + | > step_time: 1.91090 (3.14975) + | > loader_time: 0.08260 (0.04125) + + + --> STEP: 62/234 -- GLOBAL_STEP: 26270 + | > loss: -0.04208 (-0.05169) + | > log_mle: -0.24255 (-0.20172) + | > loss_dur: 0.20048 (0.15003) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.70304 (7.95933) + | > current_lr: 0.00003 + | > step_time: 3.32670 (3.10265) + | > loader_time: 0.00710 (0.03822) + + + --> STEP: 67/234 -- GLOBAL_STEP: 26275 + | > loss: -0.05920 (-0.05162) + | > log_mle: -0.22409 (-0.20183) + | > loss_dur: 0.16489 (0.15021) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.80120 (7.80584) + | > current_lr: 0.00003 + | > step_time: 1.60520 (2.98793) + | > loader_time: 0.08400 (0.03676) + + + --> STEP: 72/234 -- GLOBAL_STEP: 26280 + | > loss: -0.03832 (-0.05006) + | > log_mle: -0.19923 (-0.20189) + | > loss_dur: 0.16091 (0.15183) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.33252 (7.94867) + | > current_lr: 0.00003 + | > step_time: 1.49150 (2.91107) + | > loader_time: 0.00320 (0.03560) + + + --> STEP: 77/234 -- GLOBAL_STEP: 26285 + | > loss: -0.06220 (-0.05015) + | > log_mle: -0.21527 (-0.20261) + | > loss_dur: 0.15307 (0.15246) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.69172 (8.06834) + | > current_lr: 0.00003 + | > step_time: 1.39970 (2.81570) + | > loader_time: 0.00290 (0.03818) + + + --> STEP: 82/234 -- GLOBAL_STEP: 26290 + | > loss: -0.05825 (-0.05040) + | > log_mle: -0.20263 (-0.20285) + | > loss_dur: 0.14438 (0.15245) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.48339 (8.10052) + | > current_lr: 0.00003 + | > step_time: 1.60510 (2.75975) + | > loader_time: 0.00290 (0.03820) + + + --> STEP: 87/234 -- GLOBAL_STEP: 26295 + | > loss: -0.04544 (-0.05048) + | > log_mle: -0.21376 (-0.20362) + | > loss_dur: 0.16832 (0.15313) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.73966 (8.27886) + | > current_lr: 0.00003 + | > step_time: 1.47590 (2.71238) + | > loader_time: 0.00220 (0.03615) + + + --> STEP: 92/234 -- GLOBAL_STEP: 26300 + | > loss: -0.10535 (-0.05185) + | > log_mle: -0.26124 (-0.20603) + | > loss_dur: 0.15589 (0.15417) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.45058 (8.66916) + | > current_lr: 0.00003 + | > step_time: 3.81100 (2.68451) + | > loader_time: 0.20110 (0.03649) + + + --> STEP: 97/234 -- GLOBAL_STEP: 26305 + | > loss: -0.09077 (-0.05388) + | > log_mle: -0.25037 (-0.20938) + | > loss_dur: 0.15960 (0.15550) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.36075 (9.20883) + | > current_lr: 0.00003 + | > step_time: 0.91950 (2.64387) + | > loader_time: 0.08630 (0.03645) + + + --> STEP: 102/234 -- GLOBAL_STEP: 26310 + | > loss: -0.05394 (-0.05497) + | > log_mle: -0.23243 (-0.21166) + | > loss_dur: 0.17849 (0.15669) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.62728 (9.67001) + | > current_lr: 0.00003 + | > step_time: 1.11670 (2.59741) + | > loader_time: 0.00270 (0.03552) + + + --> STEP: 107/234 -- GLOBAL_STEP: 26315 + | > loss: -0.09218 (-0.05693) + | > log_mle: -0.27909 (-0.21498) + | > loss_dur: 0.18691 (0.15805) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.11245 (10.24283) + | > current_lr: 0.00003 + | > step_time: 2.42810 (2.61906) + | > loader_time: 0.00380 (0.03654) + + + --> STEP: 112/234 -- GLOBAL_STEP: 26320 + | > loss: -0.08633 (-0.05813) + | > log_mle: -0.28613 (-0.21792) + | > loss_dur: 0.19980 (0.15979) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.74779 (10.96412) + | > current_lr: 0.00003 + | > step_time: 3.70260 (2.62501) + | > loader_time: 0.19710 (0.03839) + + + --> STEP: 117/234 -- GLOBAL_STEP: 26325 + | > loss: -0.10055 (-0.05948) + | > log_mle: -0.28517 (-0.22083) + | > loss_dur: 0.18462 (0.16135) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.24467 (11.32617) + | > current_lr: 0.00003 + | > step_time: 1.26130 (2.59921) + | > loader_time: 0.00230 (0.03990) + + + --> STEP: 122/234 -- GLOBAL_STEP: 26330 + | > loss: -0.08375 (-0.06037) + | > log_mle: -0.25876 (-0.22263) + | > loss_dur: 0.17501 (0.16227) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.41985 (11.62967) + | > current_lr: 0.00003 + | > step_time: 1.59640 (2.56509) + | > loader_time: 0.00340 (0.03898) + + + --> STEP: 127/234 -- GLOBAL_STEP: 26335 + | > loss: -0.10723 (-0.06229) + | > log_mle: -0.31376 (-0.22565) + | > loss_dur: 0.20653 (0.16335) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.47290 (12.37740) + | > current_lr: 0.00003 + | > step_time: 3.79370 (2.60425) + | > loader_time: 0.00590 (0.03898) + + + --> STEP: 132/234 -- GLOBAL_STEP: 26340 + | > loss: -0.11892 (-0.06459) + | > log_mle: -0.29573 (-0.22912) + | > loss_dur: 0.17681 (0.16453) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.34811 (12.97943) + | > current_lr: 0.00003 + | > step_time: 1.11380 (2.61602) + | > loader_time: 0.00250 (0.03900) + + + --> STEP: 137/234 -- GLOBAL_STEP: 26345 + | > loss: -0.08276 (-0.06654) + | > log_mle: -0.31028 (-0.23280) + | > loss_dur: 0.22752 (0.16626) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.95324 (13.65971) + | > current_lr: 0.00003 + | > step_time: 1.98950 (2.61029) + | > loader_time: 0.00260 (0.03976) + + + --> STEP: 142/234 -- GLOBAL_STEP: 26350 + | > loss: -0.11403 (-0.06830) + | > log_mle: -0.31785 (-0.23590) + | > loss_dur: 0.20382 (0.16760) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.38276 (14.60631) + | > current_lr: 0.00003 + | > step_time: 1.60000 (2.61638) + | > loader_time: 0.00980 (0.04028) + + + --> STEP: 147/234 -- GLOBAL_STEP: 26355 + | > loss: -0.10995 (-0.07102) + | > log_mle: -0.32577 (-0.24058) + | > loss_dur: 0.21581 (0.16956) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.76860 (15.69839) + | > current_lr: 0.00003 + | > step_time: 3.00300 (2.61662) + | > loader_time: 0.00260 (0.04063) + + + --> STEP: 152/234 -- GLOBAL_STEP: 26360 + | > loss: -0.17169 (-0.07405) + | > log_mle: -0.40537 (-0.24496) + | > loss_dur: 0.23368 (0.17091) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.14968 (16.50539) + | > current_lr: 0.00003 + | > step_time: 2.59860 (2.61621) + | > loader_time: 0.00410 (0.04065) + + + --> STEP: 157/234 -- GLOBAL_STEP: 26365 + | > loss: -0.13948 (-0.07761) + | > log_mle: -0.35650 (-0.24996) + | > loss_dur: 0.21701 (0.17235) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.06698 (17.58107) + | > current_lr: 0.00003 + | > step_time: 3.19260 (2.63936) + | > loader_time: 0.00380 (0.04196) + + + --> STEP: 162/234 -- GLOBAL_STEP: 26370 + | > loss: -0.18837 (-0.08103) + | > log_mle: -0.39003 (-0.25477) + | > loss_dur: 0.20166 (0.17374) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.62527 (18.49402) + | > current_lr: 0.00003 + | > step_time: 2.50080 (2.63396) + | > loader_time: 0.19010 (0.04297) + + + --> STEP: 167/234 -- GLOBAL_STEP: 26375 + | > loss: -0.25257 (-0.08405) + | > log_mle: -0.46766 (-0.25913) + | > loss_dur: 0.21509 (0.17507) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.26006 (19.39834) + | > current_lr: 0.00003 + | > step_time: 3.10120 (2.73674) + | > loader_time: 0.00270 (0.04525) + + + --> STEP: 172/234 -- GLOBAL_STEP: 26380 + | > loss: -0.20602 (-0.08739) + | > log_mle: -0.46061 (-0.26434) + | > loss_dur: 0.25459 (0.17695) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.20719 (20.48108) + | > current_lr: 0.00003 + | > step_time: 4.51370 (2.79050) + | > loader_time: 0.00360 (0.04615) + + + --> STEP: 177/234 -- GLOBAL_STEP: 26385 + | > loss: -0.18279 (-0.09074) + | > log_mle: -0.41954 (-0.26930) + | > loss_dur: 0.23676 (0.17855) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 47.50586 (21.45436) + | > current_lr: 0.00003 + | > step_time: 1.99970 (2.76694) + | > loader_time: 0.00680 (0.04591) + + + --> STEP: 182/234 -- GLOBAL_STEP: 26390 + | > loss: -0.20026 (-0.09365) + | > log_mle: -0.46193 (-0.27412) + | > loss_dur: 0.26168 (0.18047) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.01247 (22.59219) + | > current_lr: 0.00003 + | > step_time: 2.90330 (2.77019) + | > loader_time: 0.00640 (0.04565) + + + --> STEP: 187/234 -- GLOBAL_STEP: 26395 + | > loss: -0.22215 (-0.09665) + | > log_mle: -0.46101 (-0.27887) + | > loss_dur: 0.23886 (0.18222) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.04310 (23.54429) + | > current_lr: 0.00003 + | > step_time: 2.00350 (2.78550) + | > loader_time: 0.00330 (0.04597) + + + --> STEP: 192/234 -- GLOBAL_STEP: 26400 + | > loss: -0.25128 (-0.09982) + | > log_mle: -0.48450 (-0.28352) + | > loss_dur: 0.23322 (0.18370) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 71.49885 (24.53418) + | > current_lr: 0.00003 + | > step_time: 2.99670 (2.77807) + | > loader_time: 0.09810 (0.04629) + + + --> STEP: 197/234 -- GLOBAL_STEP: 26405 + | > loss: -0.23131 (-0.10285) + | > log_mle: -0.46334 (-0.28802) + | > loss_dur: 0.23202 (0.18517) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 63.07544 (25.70511) + | > current_lr: 0.00003 + | > step_time: 5.18660 (2.78272) + | > loader_time: 0.02060 (0.04669) + + + --> STEP: 202/234 -- GLOBAL_STEP: 26410 + | > loss: -0.30056 (-0.10584) + | > log_mle: -0.54386 (-0.29261) + | > loss_dur: 0.24330 (0.18677) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 106.96479 (26.73806) + | > current_lr: 0.00003 + | > step_time: 3.19650 (2.78546) + | > loader_time: 0.00430 (0.04615) + + + --> STEP: 207/234 -- GLOBAL_STEP: 26415 + | > loss: -0.27341 (-0.10883) + | > log_mle: -0.52866 (-0.29712) + | > loss_dur: 0.25526 (0.18829) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.65041 (27.82006) + | > current_lr: 0.00003 + | > step_time: 2.99770 (2.92534) + | > loader_time: 0.00430 (0.04705) + + + --> STEP: 212/234 -- GLOBAL_STEP: 26420 + | > loss: -0.26131 (-0.11226) + | > log_mle: -0.51444 (-0.30224) + | > loss_dur: 0.25313 (0.18998) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.08809 (28.87822) + | > current_lr: 0.00003 + | > step_time: 4.49750 (2.94886) + | > loader_time: 0.00410 (0.04686) + + + --> STEP: 217/234 -- GLOBAL_STEP: 26425 + | > loss: -0.26794 (-0.11573) + | > log_mle: -0.53576 (-0.30733) + | > loss_dur: 0.26782 (0.19159) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.66053 (29.97511) + | > current_lr: 0.00003 + | > step_time: 5.69460 (3.05518) + | > loader_time: 0.00480 (0.04635) + + + --> STEP: 222/234 -- GLOBAL_STEP: 26430 + | > loss: -0.26062 (-0.11915) + | > log_mle: -0.55005 (-0.31240) + | > loss_dur: 0.28943 (0.19325) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.12760 (31.16955) + | > current_lr: 0.00003 + | > step_time: 2.10060 (3.05704) + | > loader_time: 0.00390 (0.04716) + + + --> STEP: 227/234 -- GLOBAL_STEP: 26435 + | > loss: -0.24539 (-0.12284) + | > log_mle: -0.52630 (-0.31781) + | > loss_dur: 0.28091 (0.19497) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 78.72329 (32.32405) + | > current_lr: 0.00003 + | > step_time: 0.24190 (3.01625) + | > loader_time: 0.00300 (0.04657) + + + --> STEP: 232/234 -- GLOBAL_STEP: 26440 + | > loss: -0.17489 (-0.12540) + | > log_mle: -0.72022 (-0.32449) + | > loss_dur: 0.54534 (0.19908) + | > amp_scaler: 2048.00000 (4087.17241) + | > grad_norm: 0.00000 (33.21896) + | > current_lr: 0.00003 + | > step_time: 0.31380 (2.95712) + | > loader_time: 0.02690 (0.04575) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02310 (-0.79490) + | > avg_loss: -0.17022 (-0.00038) + | > avg_log_mle: -0.40778 (-0.00187) + | > avg_loss_dur: 0.23756 (+0.00150) + + + > EPOCH: 113/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 09:07:37)  + + --> STEP: 3/234 -- GLOBAL_STEP: 26445 + | > loss: -0.00251 (-0.02001) + | > log_mle: -0.20553 (-0.20192) + | > loss_dur: 0.20301 (0.18191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.47588 (10.18256) + | > current_lr: 0.00003 + | > step_time: 5.60730 (4.79712) + | > loader_time: 0.00210 (0.10908) + + + --> STEP: 8/234 -- GLOBAL_STEP: 26450 + | > loss: -0.06807 (-0.03364) + | > log_mle: -0.21611 (-0.20481) + | > loss_dur: 0.14803 (0.17117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.63858 (10.33749) + | > current_lr: 0.00003 + | > step_time: 5.11560 (5.36294) + | > loader_time: 0.18910 (0.06714) + + + --> STEP: 13/234 -- GLOBAL_STEP: 26455 + | > loss: -0.05178 (-0.04048) + | > log_mle: -0.20068 (-0.20429) + | > loss_dur: 0.14891 (0.16382) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.61674 (10.30369) + | > current_lr: 0.00003 + | > step_time: 1.29960 (4.12998) + | > loader_time: 0.00160 (0.06611) + + + --> STEP: 18/234 -- GLOBAL_STEP: 26460 + | > loss: -0.04669 (-0.04550) + | > log_mle: -0.19842 (-0.20271) + | > loss_dur: 0.15173 (0.15720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.30523 (9.53027) + | > current_lr: 0.00003 + | > step_time: 3.88910 (3.62157) + | > loader_time: 0.10570 (0.06794) + + + --> STEP: 23/234 -- GLOBAL_STEP: 26465 + | > loss: -0.06988 (-0.04765) + | > log_mle: -0.20306 (-0.20097) + | > loss_dur: 0.13318 (0.15332) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.88465 (8.96230) + | > current_lr: 0.00003 + | > step_time: 1.16170 (3.35116) + | > loader_time: 0.00130 (0.05825) + + + --> STEP: 28/234 -- GLOBAL_STEP: 26470 + | > loss: -0.07917 (-0.05061) + | > log_mle: -0.19295 (-0.20014) + | > loss_dur: 0.11378 (0.14952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.93580 (8.49673) + | > current_lr: 0.00003 + | > step_time: 1.18150 (3.00421) + | > loader_time: 0.00220 (0.04820) + + + --> STEP: 33/234 -- GLOBAL_STEP: 26475 + | > loss: -0.03934 (-0.05141) + | > log_mle: -0.19110 (-0.20047) + | > loss_dur: 0.15176 (0.14906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.42269 (8.20600) + | > current_lr: 0.00003 + | > step_time: 2.19440 (2.77847) + | > loader_time: 0.00210 (0.04359) + + + --> STEP: 38/234 -- GLOBAL_STEP: 26480 + | > loss: -0.04254 (-0.05124) + | > log_mle: -0.20976 (-0.20087) + | > loss_dur: 0.16722 (0.14963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.06558 (8.45940) + | > current_lr: 0.00003 + | > step_time: 1.02330 (2.60690) + | > loader_time: 0.00230 (0.04486) + + + --> STEP: 43/234 -- GLOBAL_STEP: 26485 + | > loss: -0.03324 (-0.05053) + | > log_mle: -0.20985 (-0.20061) + | > loss_dur: 0.17661 (0.15008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.24255 (8.36495) + | > current_lr: 0.00003 + | > step_time: 2.09740 (2.52294) + | > loader_time: 0.00280 (0.04342) + + + --> STEP: 48/234 -- GLOBAL_STEP: 26490 + | > loss: -0.06319 (-0.05120) + | > log_mle: -0.19227 (-0.20094) + | > loss_dur: 0.12908 (0.14974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.67044 (8.23710) + | > current_lr: 0.00003 + | > step_time: 1.28630 (2.39666) + | > loader_time: 0.00270 (0.04083) + + + --> STEP: 53/234 -- GLOBAL_STEP: 26495 + | > loss: -0.05733 (-0.05160) + | > log_mle: -0.21356 (-0.20081) + | > loss_dur: 0.15623 (0.14921) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.06645 (8.02192) + | > current_lr: 0.00003 + | > step_time: 1.10730 (2.30683) + | > loader_time: 0.00190 (0.03717) + + + --> STEP: 58/234 -- GLOBAL_STEP: 26500 + | > loss: -0.05941 (-0.05179) + | > log_mle: -0.19750 (-0.20110) + | > loss_dur: 0.13808 (0.14932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.27866 (7.88260) + | > current_lr: 0.00003 + | > step_time: 1.58290 (2.22215) + | > loader_time: 0.08530 (0.03560) + + + --> STEP: 63/234 -- GLOBAL_STEP: 26505 + | > loss: -0.04280 (-0.05235) + | > log_mle: -0.20808 (-0.20275) + | > loss_dur: 0.16527 (0.15040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.41532 (8.07941) + | > current_lr: 0.00003 + | > step_time: 1.48720 (2.15927) + | > loader_time: 0.00220 (0.03298) + + + --> STEP: 68/234 -- GLOBAL_STEP: 26510 + | > loss: -0.03045 (-0.05211) + | > log_mle: -0.20145 (-0.20282) + | > loss_dur: 0.17100 (0.15071) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.73726 (7.99868) + | > current_lr: 0.00003 + | > step_time: 1.00020 (2.12260) + | > loader_time: 0.09230 (0.03472) + + + --> STEP: 73/234 -- GLOBAL_STEP: 26515 + | > loss: -0.03986 (-0.05111) + | > log_mle: -0.22470 (-0.20315) + | > loss_dur: 0.18485 (0.15203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.33684 (8.34209) + | > current_lr: 0.00003 + | > step_time: 1.51340 (2.10354) + | > loader_time: 0.08640 (0.03598) + + + --> STEP: 78/234 -- GLOBAL_STEP: 26520 + | > loss: -0.03657 (-0.05132) + | > log_mle: -0.19799 (-0.20353) + | > loss_dur: 0.16143 (0.15222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.17875 (8.45220) + | > current_lr: 0.00003 + | > step_time: 1.29430 (2.05267) + | > loader_time: 0.09500 (0.03502) + + + --> STEP: 83/234 -- GLOBAL_STEP: 26525 + | > loss: -0.04807 (-0.05187) + | > log_mle: -0.22453 (-0.20410) + | > loss_dur: 0.17647 (0.15223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.40285 (8.55361) + | > current_lr: 0.00003 + | > step_time: 1.29750 (2.02273) + | > loader_time: 0.00280 (0.03414) + + + --> STEP: 88/234 -- GLOBAL_STEP: 26530 + | > loss: -0.08804 (-0.05265) + | > log_mle: -0.26322 (-0.20528) + | > loss_dur: 0.17517 (0.15263) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.17220 (8.71866) + | > current_lr: 0.00003 + | > step_time: 1.99010 (2.00538) + | > loader_time: 0.00230 (0.03327) + + + --> STEP: 93/234 -- GLOBAL_STEP: 26535 + | > loss: -0.08126 (-0.05400) + | > log_mle: -0.27562 (-0.20774) + | > loss_dur: 0.19436 (0.15374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.76324 (9.12360) + | > current_lr: 0.00003 + | > step_time: 2.98520 (2.02509) + | > loader_time: 0.00320 (0.03368) + + + --> STEP: 98/234 -- GLOBAL_STEP: 26540 + | > loss: -0.03846 (-0.05552) + | > log_mle: -0.20269 (-0.21020) + | > loss_dur: 0.16423 (0.15467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.54175 (9.55718) + | > current_lr: 0.00003 + | > step_time: 1.11530 (2.02075) + | > loader_time: 0.08560 (0.03294) + + + --> STEP: 103/234 -- GLOBAL_STEP: 26545 + | > loss: -0.09886 (-0.05725) + | > log_mle: -0.30304 (-0.21338) + | > loss_dur: 0.20418 (0.15613) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.32176 (10.11820) + | > current_lr: 0.00003 + | > step_time: 1.18010 (1.99256) + | > loader_time: 0.00250 (0.03228) + + + --> STEP: 108/234 -- GLOBAL_STEP: 26550 + | > loss: -0.08194 (-0.05887) + | > log_mle: -0.24840 (-0.21613) + | > loss_dur: 0.16647 (0.15727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.93545 (10.54357) + | > current_lr: 0.00003 + | > step_time: 1.41290 (2.01622) + | > loader_time: 0.08330 (0.03171) + + + --> STEP: 113/234 -- GLOBAL_STEP: 26555 + | > loss: -0.11373 (-0.06045) + | > log_mle: -0.29987 (-0.21956) + | > loss_dur: 0.18614 (0.15911) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.45610 (11.14044) + | > current_lr: 0.00003 + | > step_time: 2.53650 (2.00790) + | > loader_time: 0.00290 (0.03044) + + + --> STEP: 118/234 -- GLOBAL_STEP: 26560 + | > loss: -0.07079 (-0.06138) + | > log_mle: -0.26541 (-0.22203) + | > loss_dur: 0.19462 (0.16065) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.63983 (11.60797) + | > current_lr: 0.00003 + | > step_time: 1.30860 (1.99885) + | > loader_time: 0.00410 (0.03076) + + + --> STEP: 123/234 -- GLOBAL_STEP: 26565 + | > loss: -0.06798 (-0.06205) + | > log_mle: -0.23537 (-0.22354) + | > loss_dur: 0.16739 (0.16149) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.03027 (11.81503) + | > current_lr: 0.00003 + | > step_time: 4.08860 (2.02949) + | > loader_time: 0.00180 (0.03033) + + + --> STEP: 128/234 -- GLOBAL_STEP: 26570 + | > loss: -0.11877 (-0.06428) + | > log_mle: -0.29314 (-0.22691) + | > loss_dur: 0.17438 (0.16262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.48445 (12.72170) + | > current_lr: 0.00003 + | > step_time: 1.58890 (2.06092) + | > loader_time: 0.00290 (0.03137) + + + --> STEP: 133/234 -- GLOBAL_STEP: 26575 + | > loss: -0.11418 (-0.06626) + | > log_mle: -0.31686 (-0.23039) + | > loss_dur: 0.20268 (0.16414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.12261 (13.49024) + | > current_lr: 0.00003 + | > step_time: 1.90100 (2.07168) + | > loader_time: 0.09810 (0.03434) + + + --> STEP: 138/234 -- GLOBAL_STEP: 26580 + | > loss: -0.08975 (-0.06793) + | > log_mle: -0.27677 (-0.23369) + | > loss_dur: 0.18702 (0.16576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.07165 (14.02835) + | > current_lr: 0.00003 + | > step_time: 2.58850 (2.14228) + | > loader_time: 0.10570 (0.03588) + + + --> STEP: 143/234 -- GLOBAL_STEP: 26585 + | > loss: -0.15553 (-0.07012) + | > log_mle: -0.40768 (-0.23774) + | > loss_dur: 0.25215 (0.16762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.77503 (15.05696) + | > current_lr: 0.00003 + | > step_time: 1.81390 (2.12843) + | > loader_time: 0.09520 (0.03594) + + + --> STEP: 148/234 -- GLOBAL_STEP: 26590 + | > loss: -0.14567 (-0.07292) + | > log_mle: -0.32987 (-0.24190) + | > loss_dur: 0.18420 (0.16898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.88382 (15.76146) + | > current_lr: 0.00003 + | > step_time: 2.80530 (2.16474) + | > loader_time: 0.00240 (0.03539) + + + --> STEP: 153/234 -- GLOBAL_STEP: 26595 + | > loss: -0.23164 (-0.07646) + | > log_mle: -0.45416 (-0.24713) + | > loss_dur: 0.22253 (0.17067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.24050 (16.62514) + | > current_lr: 0.00003 + | > step_time: 2.00630 (2.16944) + | > loader_time: 0.00270 (0.03557) + + + --> STEP: 158/234 -- GLOBAL_STEP: 26600 + | > loss: -0.16453 (-0.07957) + | > log_mle: -0.39397 (-0.25179) + | > loss_dur: 0.22944 (0.17222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.56118 (17.52314) + | > current_lr: 0.00003 + | > step_time: 1.31070 (2.17429) + | > loader_time: 0.28550 (0.03858) + + + --> STEP: 163/234 -- GLOBAL_STEP: 26605 + | > loss: -0.15120 (-0.08290) + | > log_mle: -0.36424 (-0.25640) + | > loss_dur: 0.21304 (0.17350) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.00478 (18.50981) + | > current_lr: 0.00003 + | > step_time: 6.61390 (2.21636) + | > loader_time: 0.18330 (0.03964) + + + --> STEP: 168/234 -- GLOBAL_STEP: 26610 + | > loss: -0.16880 (-0.08601) + | > log_mle: -0.41656 (-0.26109) + | > loss_dur: 0.24776 (0.17507) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.93573 (19.69727) + | > current_lr: 0.00003 + | > step_time: 2.70210 (2.22085) + | > loader_time: 0.00380 (0.04006) + + + --> STEP: 173/234 -- GLOBAL_STEP: 26615 + | > loss: -0.19240 (-0.08937) + | > log_mle: -0.42033 (-0.26623) + | > loss_dur: 0.22793 (0.17686) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.37938 (20.97198) + | > current_lr: 0.00003 + | > step_time: 3.90450 (2.24179) + | > loader_time: 0.09610 (0.04008) + + + --> STEP: 178/234 -- GLOBAL_STEP: 26620 + | > loss: -0.22220 (-0.09263) + | > log_mle: -0.47670 (-0.27133) + | > loss_dur: 0.25450 (0.17870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.32861 (22.26475) + | > current_lr: 0.00003 + | > step_time: 1.89840 (2.28928) + | > loader_time: 0.09930 (0.03964) + + + --> STEP: 183/234 -- GLOBAL_STEP: 26625 + | > loss: -0.22810 (-0.09546) + | > log_mle: -0.47142 (-0.27599) + | > loss_dur: 0.24331 (0.18053) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.75890 (23.44152) + | > current_lr: 0.00003 + | > step_time: 4.00860 (2.30724) + | > loader_time: 0.08480 (0.04053) + + + --> STEP: 188/234 -- GLOBAL_STEP: 26630 + | > loss: -0.24015 (-0.09845) + | > log_mle: -0.48904 (-0.28076) + | > loss_dur: 0.24889 (0.18231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.02649 (24.40327) + | > current_lr: 0.00003 + | > step_time: 3.78970 (2.38469) + | > loader_time: 0.00850 (0.04055) + + + --> STEP: 193/234 -- GLOBAL_STEP: 26635 + | > loss: -0.22779 (-0.10155) + | > log_mle: -0.47604 (-0.28527) + | > loss_dur: 0.24825 (0.18372) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.03125 (25.68066) + | > current_lr: 0.00003 + | > step_time: 2.79830 (2.45607) + | > loader_time: 0.00480 (0.04375) + + + --> STEP: 198/234 -- GLOBAL_STEP: 26640 + | > loss: -0.23298 (-0.10440) + | > log_mle: -0.47634 (-0.28953) + | > loss_dur: 0.24336 (0.18514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.56476 (26.69965) + | > current_lr: 0.00003 + | > step_time: 15.19820 (2.56681) + | > loader_time: 0.19580 (0.04715) + + + --> STEP: 203/234 -- GLOBAL_STEP: 26645 + | > loss: -0.17970 (-0.10709) + | > log_mle: -0.41153 (-0.29371) + | > loss_dur: 0.23183 (0.18662) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.64948 (27.58122) + | > current_lr: 0.00003 + | > step_time: 2.99160 (2.58142) + | > loader_time: 0.00560 (0.04752) + + + --> STEP: 208/234 -- GLOBAL_STEP: 26650 + | > loss: -0.22697 (-0.11033) + | > log_mle: -0.49368 (-0.29866) + | > loss_dur: 0.26671 (0.18833) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.79584 (28.57473) + | > current_lr: 0.00003 + | > step_time: 6.19720 (2.62994) + | > loader_time: 0.09950 (0.04787) + + + --> STEP: 213/234 -- GLOBAL_STEP: 26655 + | > loss: -0.28281 (-0.11402) + | > log_mle: -0.54603 (-0.30405) + | > loss_dur: 0.26323 (0.19002) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.06300 (29.76053) + | > current_lr: 0.00003 + | > step_time: 4.39190 (2.66496) + | > loader_time: 0.00340 (0.04807) + + + --> STEP: 218/234 -- GLOBAL_STEP: 26660 + | > loss: -0.25151 (-0.11753) + | > log_mle: -0.50897 (-0.30903) + | > loss_dur: 0.25746 (0.19150) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.42958 (30.79843) + | > current_lr: 0.00003 + | > step_time: 2.30320 (2.70389) + | > loader_time: 0.00560 (0.10109) + + + --> STEP: 223/234 -- GLOBAL_STEP: 26665 + | > loss: -0.28233 (-0.12112) + | > log_mle: -0.54490 (-0.31428) + | > loss_dur: 0.26257 (0.19316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.86069 (32.07209) + | > current_lr: 0.00003 + | > step_time: 2.10680 (2.69305) + | > loader_time: 0.09090 (0.09933) + + + --> STEP: 228/234 -- GLOBAL_STEP: 26670 + | > loss: -0.24207 (-0.12455) + | > log_mle: -0.53878 (-0.31958) + | > loss_dur: 0.29672 (0.19503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.37467 (33.57874) + | > current_lr: 0.00003 + | > step_time: 0.25220 (2.64931) + | > loader_time: 0.00340 (0.21125) + + + --> STEP: 233/234 -- GLOBAL_STEP: 26675 + | > loss: 0.36757 (-0.12426) + | > log_mle: -0.49959 (-0.32586) + | > loss_dur: 0.86716 (0.20160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.78867 (35.30854) + | > current_lr: 0.00003 + | > step_time: 0.21130 (2.59847) + | > loader_time: 0.00360 (0.20680) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.90377 (+0.88066) + | > avg_loss: -0.13995 (+0.03027) + | > avg_log_mle: -0.38370 (+0.02408) + | > avg_loss_dur: 0.24375 (+0.00619) + + + > EPOCH: 114/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 09:19:30)  + + --> STEP: 4/234 -- GLOBAL_STEP: 26680 + | > loss: -0.01305 (-0.03066) + | > log_mle: -0.20139 (-0.20363) + | > loss_dur: 0.18834 (0.17298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.92800 (12.01580) + | > current_lr: 0.00003 + | > step_time: 5.98120 (6.27104) + | > loader_time: 0.00130 (0.69703) + + + --> STEP: 9/234 -- GLOBAL_STEP: 26685 + | > loss: -0.04939 (-0.04557) + | > log_mle: -0.21607 (-0.20838) + | > loss_dur: 0.16668 (0.16282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.42798 (10.42813) + | > current_lr: 0.00003 + | > step_time: 2.28680 (6.06437) + | > loader_time: 0.00220 (0.37595) + + + --> STEP: 14/234 -- GLOBAL_STEP: 26690 + | > loss: -0.05944 (-0.04863) + | > log_mle: -0.21087 (-0.20770) + | > loss_dur: 0.15143 (0.15908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.18813 (9.62860) + | > current_lr: 0.00003 + | > step_time: 0.88670 (4.37146) + | > loader_time: 0.00130 (0.24827) + + + --> STEP: 19/234 -- GLOBAL_STEP: 26695 + | > loss: -0.06946 (-0.05242) + | > log_mle: -0.19732 (-0.20548) + | > loss_dur: 0.12785 (0.15306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.77372 (8.87183) + | > current_lr: 0.00003 + | > step_time: 0.89850 (3.55591) + | > loader_time: 0.00230 (0.18338) + + + --> STEP: 24/234 -- GLOBAL_STEP: 26700 + | > loss: -0.08420 (-0.05496) + | > log_mle: -0.19895 (-0.20401) + | > loss_dur: 0.11475 (0.14905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.94981 (8.42884) + | > current_lr: 0.00003 + | > step_time: 1.19500 (3.12415) + | > loader_time: 0.00130 (0.14556) + + + --> STEP: 29/234 -- GLOBAL_STEP: 26705 + | > loss: -0.04445 (-0.05657) + | > log_mle: -0.18964 (-0.20299) + | > loss_dur: 0.14519 (0.14642) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.92678 (8.05232) + | > current_lr: 0.00003 + | > step_time: 2.90180 (3.04873) + | > loader_time: 0.01440 (0.12745) + + + --> STEP: 34/234 -- GLOBAL_STEP: 26710 + | > loss: -0.04037 (-0.05750) + | > log_mle: -0.20017 (-0.20355) + | > loss_dur: 0.15980 (0.14605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.47670 (7.83610) + | > current_lr: 0.00003 + | > step_time: 1.30900 (2.83806) + | > loader_time: 0.00240 (0.11151) + + + --> STEP: 39/234 -- GLOBAL_STEP: 26715 + | > loss: -0.06279 (-0.05755) + | > log_mle: -0.20980 (-0.20413) + | > loss_dur: 0.14701 (0.14659) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.91779 (7.98717) + | > current_lr: 0.00003 + | > step_time: 1.17180 (2.66096) + | > loader_time: 0.00200 (0.09962) + + + --> STEP: 44/234 -- GLOBAL_STEP: 26720 + | > loss: -0.07123 (-0.05735) + | > log_mle: -0.19992 (-0.20357) + | > loss_dur: 0.12870 (0.14622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.01087 (7.75948) + | > current_lr: 0.00003 + | > step_time: 1.80220 (2.58548) + | > loader_time: 0.00230 (0.08858) + + + --> STEP: 49/234 -- GLOBAL_STEP: 26725 + | > loss: -0.07676 (-0.05753) + | > log_mle: -0.20719 (-0.20391) + | > loss_dur: 0.13043 (0.14638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.16253 (7.76090) + | > current_lr: 0.00003 + | > step_time: 1.64850 (2.50181) + | > loader_time: 0.00460 (0.07982) + + + --> STEP: 54/234 -- GLOBAL_STEP: 26730 + | > loss: -0.07770 (-0.05704) + | > log_mle: -0.21456 (-0.20377) + | > loss_dur: 0.13686 (0.14672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.53605 (7.60667) + | > current_lr: 0.00003 + | > step_time: 1.90050 (2.43632) + | > loader_time: 0.00250 (0.07416) + + + --> STEP: 59/234 -- GLOBAL_STEP: 26735 + | > loss: -0.08582 (-0.05715) + | > log_mle: -0.22429 (-0.20409) + | > loss_dur: 0.13847 (0.14694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.39048 (7.55078) + | > current_lr: 0.00003 + | > step_time: 1.93240 (2.45873) + | > loader_time: 0.00200 (0.07249) + + + --> STEP: 64/234 -- GLOBAL_STEP: 26740 + | > loss: -0.05417 (-0.05677) + | > log_mle: -0.19559 (-0.20512) + | > loss_dur: 0.14142 (0.14835) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.68542 (7.74370) + | > current_lr: 0.00003 + | > step_time: 2.60380 (2.42618) + | > loader_time: 0.09340 (0.06852) + + + --> STEP: 69/234 -- GLOBAL_STEP: 26745 + | > loss: -0.02916 (-0.05630) + | > log_mle: -0.18552 (-0.20495) + | > loss_dur: 0.15636 (0.14865) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.98422 (7.64804) + | > current_lr: 0.00003 + | > step_time: 3.30210 (2.45721) + | > loader_time: 0.00280 (0.06507) + + + --> STEP: 74/234 -- GLOBAL_STEP: 26750 + | > loss: -0.05833 (-0.05527) + | > log_mle: -0.20150 (-0.20545) + | > loss_dur: 0.14317 (0.15017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.36095 (8.18248) + | > current_lr: 0.00003 + | > step_time: 1.49250 (2.44195) + | > loader_time: 0.00170 (0.06350) + + + --> STEP: 79/234 -- GLOBAL_STEP: 26755 + | > loss: -0.06859 (-0.05552) + | > log_mle: -0.21591 (-0.20600) + | > loss_dur: 0.14731 (0.15048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.34947 (8.20551) + | > current_lr: 0.00003 + | > step_time: 1.86230 (2.44190) + | > loader_time: 0.00220 (0.06179) + + + --> STEP: 84/234 -- GLOBAL_STEP: 26760 + | > loss: -0.05963 (-0.05626) + | > log_mle: -0.21132 (-0.20653) + | > loss_dur: 0.15169 (0.15027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.98909 (8.28868) + | > current_lr: 0.00003 + | > step_time: 1.97880 (2.43206) + | > loader_time: 0.01060 (0.06162) + + + --> STEP: 89/234 -- GLOBAL_STEP: 26765 + | > loss: -0.07386 (-0.05680) + | > log_mle: -0.24354 (-0.20805) + | > loss_dur: 0.16969 (0.15125) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.75924 (8.48769) + | > current_lr: 0.00003 + | > step_time: 1.21200 (2.42710) + | > loader_time: 0.07790 (0.06222) + + + --> STEP: 94/234 -- GLOBAL_STEP: 26770 + | > loss: -0.11264 (-0.05836) + | > log_mle: -0.28058 (-0.21089) + | > loss_dur: 0.16794 (0.15253) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.70191 (8.95912) + | > current_lr: 0.00003 + | > step_time: 1.20110 (2.43952) + | > loader_time: 0.00300 (0.06008) + + + --> STEP: 99/234 -- GLOBAL_STEP: 26775 + | > loss: -0.12012 (-0.05987) + | > log_mle: -0.31197 (-0.21365) + | > loss_dur: 0.19185 (0.15378) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.06503 (9.47105) + | > current_lr: 0.00003 + | > step_time: 3.50660 (2.42074) + | > loader_time: 0.08780 (0.05810) + + + --> STEP: 104/234 -- GLOBAL_STEP: 26780 + | > loss: -0.13745 (-0.06163) + | > log_mle: -0.32145 (-0.21678) + | > loss_dur: 0.18400 (0.15515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.11176 (10.13928) + | > current_lr: 0.00003 + | > step_time: 1.60210 (2.40800) + | > loader_time: 0.00210 (0.05626) + + + --> STEP: 109/234 -- GLOBAL_STEP: 26785 + | > loss: -0.06753 (-0.06232) + | > log_mle: -0.29516 (-0.21908) + | > loss_dur: 0.22764 (0.15676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.80688 (10.65555) + | > current_lr: 0.00003 + | > step_time: 2.30330 (2.42067) + | > loader_time: 0.09250 (0.05544) + + + --> STEP: 114/234 -- GLOBAL_STEP: 26790 + | > loss: -0.10845 (-0.06403) + | > log_mle: -0.27738 (-0.22221) + | > loss_dur: 0.16893 (0.15818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.76772 (11.20902) + | > current_lr: 0.00003 + | > step_time: 1.48910 (2.41471) + | > loader_time: 0.00200 (0.05446) + + + --> STEP: 119/234 -- GLOBAL_STEP: 26795 + | > loss: -0.08586 (-0.06485) + | > log_mle: -0.27403 (-0.22463) + | > loss_dur: 0.18817 (0.15978) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.26088 (11.61129) + | > current_lr: 0.00003 + | > step_time: 1.96810 (2.43579) + | > loader_time: 0.08380 (0.05434) + + + --> STEP: 124/234 -- GLOBAL_STEP: 26800 + | > loss: -0.12168 (-0.06594) + | > log_mle: -0.30355 (-0.22638) + | > loss_dur: 0.18187 (0.16044) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.45273 (11.87308) + | > current_lr: 0.00003 + | > step_time: 3.40090 (2.42648) + | > loader_time: 0.09710 (0.05453) + + + --> STEP: 129/234 -- GLOBAL_STEP: 26805 + | > loss: -0.08838 (-0.06778) + | > log_mle: -0.29199 (-0.22972) + | > loss_dur: 0.20361 (0.16194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.28719 (12.62994) + | > current_lr: 0.00003 + | > step_time: 2.09530 (2.45716) + | > loader_time: 0.00370 (0.05399) + + + --> STEP: 134/234 -- GLOBAL_STEP: 26810 + | > loss: -0.11437 (-0.06997) + | > log_mle: -0.34104 (-0.23358) + | > loss_dur: 0.22666 (0.16361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.67440 (13.37103) + | > current_lr: 0.00003 + | > step_time: 1.18450 (2.43438) + | > loader_time: 0.00250 (0.05207) + + + --> STEP: 139/234 -- GLOBAL_STEP: 26815 + | > loss: -0.18474 (-0.07214) + | > log_mle: -0.40021 (-0.23720) + | > loss_dur: 0.21547 (0.16506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.15713 (14.26772) + | > current_lr: 0.00003 + | > step_time: 2.31580 (2.44300) + | > loader_time: 0.08640 (0.05233) + + + --> STEP: 144/234 -- GLOBAL_STEP: 26820 + | > loss: -0.15438 (-0.07418) + | > log_mle: -0.37935 (-0.24113) + | > loss_dur: 0.22497 (0.16695) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.88608 (15.18519) + | > current_lr: 0.00003 + | > step_time: 6.00080 (2.45685) + | > loader_time: 0.00260 (0.05121) + + + --> STEP: 149/234 -- GLOBAL_STEP: 26825 + | > loss: -0.19925 (-0.07717) + | > log_mle: -0.42555 (-0.24558) + | > loss_dur: 0.22630 (0.16841) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.38203 (15.95351) + | > current_lr: 0.00003 + | > step_time: 3.41310 (2.54494) + | > loader_time: 0.08530 (0.05140) + + + --> STEP: 154/234 -- GLOBAL_STEP: 26830 + | > loss: -0.17696 (-0.08058) + | > log_mle: -0.38540 (-0.25044) + | > loss_dur: 0.20844 (0.16986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.26947 (16.99593) + | > current_lr: 0.00003 + | > step_time: 2.16760 (2.53961) + | > loader_time: 0.00380 (0.05158) + + + --> STEP: 159/234 -- GLOBAL_STEP: 26835 + | > loss: -0.18441 (-0.08352) + | > log_mle: -0.40559 (-0.25504) + | > loss_dur: 0.22118 (0.17152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.98624 (18.00049) + | > current_lr: 0.00003 + | > step_time: 5.51430 (2.58260) + | > loader_time: 0.11170 (0.05192) + + + --> STEP: 164/234 -- GLOBAL_STEP: 26840 + | > loss: -0.17377 (-0.08656) + | > log_mle: -0.39779 (-0.25943) + | > loss_dur: 0.22402 (0.17287) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.58855 (18.98843) + | > current_lr: 0.00003 + | > step_time: 0.79960 (2.57061) + | > loader_time: 0.08180 (0.05095) + + + --> STEP: 169/234 -- GLOBAL_STEP: 26845 + | > loss: -0.16197 (-0.08948) + | > log_mle: -0.39344 (-0.26396) + | > loss_dur: 0.23147 (0.17448) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.09922 (20.17740) + | > current_lr: 0.00003 + | > step_time: 1.51110 (2.62540) + | > loader_time: 0.18180 (0.05118) + + + --> STEP: 174/234 -- GLOBAL_STEP: 26850 + | > loss: -0.24345 (-0.09322) + | > log_mle: -0.47689 (-0.26956) + | > loss_dur: 0.23343 (0.17634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.19507 (21.58812) + | > current_lr: 0.00003 + | > step_time: 4.20260 (2.63287) + | > loader_time: 0.00280 (0.05087) + + + --> STEP: 179/234 -- GLOBAL_STEP: 26855 + | > loss: -0.21328 (-0.09631) + | > log_mle: -0.47602 (-0.27464) + | > loss_dur: 0.26274 (0.17834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.33969 (22.65427) + | > current_lr: 0.00003 + | > step_time: 2.49490 (2.70575) + | > loader_time: 0.10840 (0.05177) + + + --> STEP: 184/234 -- GLOBAL_STEP: 26860 + | > loss: -0.19584 (-0.09918) + | > log_mle: -0.43676 (-0.27918) + | > loss_dur: 0.24092 (0.18000) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.57450 (23.62557) + | > current_lr: 0.00003 + | > step_time: 2.80990 (2.70999) + | > loader_time: 0.09360 (0.05146) + + + --> STEP: 189/234 -- GLOBAL_STEP: 26865 + | > loss: -0.19503 (-0.10197) + | > log_mle: -0.44332 (-0.28382) + | > loss_dur: 0.24829 (0.18186) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.94992 (24.86229) + | > current_lr: 0.00003 + | > step_time: 1.50310 (2.71918) + | > loader_time: 0.00280 (0.05121) + + + --> STEP: 194/234 -- GLOBAL_STEP: 26870 + | > loss: -0.24352 (-0.10542) + | > log_mle: -0.47492 (-0.28851) + | > loss_dur: 0.23140 (0.18309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.73058 (25.85539) + | > current_lr: 0.00003 + | > step_time: 1.89900 (2.72810) + | > loader_time: 0.00520 (0.05103) + + + --> STEP: 199/234 -- GLOBAL_STEP: 26875 + | > loss: -0.23958 (-0.10840) + | > log_mle: -0.47538 (-0.29287) + | > loss_dur: 0.23579 (0.18447) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 110.43787 (27.02777) + | > current_lr: 0.00003 + | > step_time: 3.21430 (2.73052) + | > loader_time: 0.08720 (0.05085) + + + --> STEP: 204/234 -- GLOBAL_STEP: 26880 + | > loss: -0.25327 (-0.11105) + | > log_mle: -0.51897 (-0.29722) + | > loss_dur: 0.26570 (0.18616) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.96835 (28.01468) + | > current_lr: 0.00003 + | > step_time: 2.79330 (2.78848) + | > loader_time: 0.00360 (0.05113) + + + --> STEP: 209/234 -- GLOBAL_STEP: 26885 + | > loss: -0.22582 (-0.11428) + | > log_mle: -0.47441 (-0.30195) + | > loss_dur: 0.24859 (0.18767) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.10551 (28.84719) + | > current_lr: 0.00003 + | > step_time: 6.10670 (2.90405) + | > loader_time: 0.08330 (0.05262) + + + --> STEP: 214/234 -- GLOBAL_STEP: 26890 + | > loss: -0.26788 (-0.11810) + | > log_mle: -0.50848 (-0.30741) + | > loss_dur: 0.24060 (0.18932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.07909 (30.08983) + | > current_lr: 0.00003 + | > step_time: 3.09860 (2.91862) + | > loader_time: 0.00930 (0.05419) + + + --> STEP: 219/234 -- GLOBAL_STEP: 26895 + | > loss: -0.34972 (-0.12184) + | > log_mle: -0.60849 (-0.31276) + | > loss_dur: 0.25877 (0.19092) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.20329 (31.16688) + | > current_lr: 0.00003 + | > step_time: 2.80250 (2.96070) + | > loader_time: 0.08790 (0.05515) + + + --> STEP: 224/234 -- GLOBAL_STEP: 26900 + | > loss: -0.28035 (-0.12514) + | > log_mle: -0.54802 (-0.31766) + | > loss_dur: 0.26767 (0.19252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.55767 (32.45457) + | > current_lr: 0.00003 + | > step_time: 0.22430 (2.95534) + | > loader_time: 0.00300 (0.05402) + + + --> STEP: 229/234 -- GLOBAL_STEP: 26905 + | > loss: -0.25234 (-0.12836) + | > log_mle: -0.57997 (-0.32303) + | > loss_dur: 0.32763 (0.19467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.23208 (33.84054) + | > current_lr: 0.00003 + | > step_time: 0.25210 (2.89603) + | > loader_time: 0.00430 (0.05292) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22580 (-0.67796) + | > avg_loss: -0.15493 (-0.01498) + | > avg_log_mle: -0.39501 (-0.01131) + | > avg_loss_dur: 0.24008 (-0.00367) + + + > EPOCH: 115/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 09:31:40)  + + --> STEP: 0/234 -- GLOBAL_STEP: 26910 + | > loss: -0.13285 (-0.13285) + | > log_mle: -0.27353 (-0.27353) + | > loss_dur: 0.14068 (0.14068) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.66772 (10.66772) + | > current_lr: 0.00003 + | > step_time: 9.40270 (9.40272) + | > loader_time: 30.54680 (30.54678) + + + --> STEP: 5/234 -- GLOBAL_STEP: 26915 + | > loss: -0.05025 (-0.03684) + | > log_mle: -0.20897 (-0.20506) + | > loss_dur: 0.15872 (0.16822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.92847 (15.84412) + | > current_lr: 0.00003 + | > step_time: 4.74660 (5.02803) + | > loader_time: 0.00780 (0.08051) + + + --> STEP: 10/234 -- GLOBAL_STEP: 26920 + | > loss: -0.05321 (-0.04911) + | > log_mle: -0.20661 (-0.20849) + | > loss_dur: 0.15340 (0.15939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.46605 (13.15224) + | > current_lr: 0.00003 + | > step_time: 0.79270 (3.03617) + | > loader_time: 0.00230 (0.04117) + + + --> STEP: 15/234 -- GLOBAL_STEP: 26925 + | > loss: -0.08449 (-0.05369) + | > log_mle: -0.20928 (-0.20797) + | > loss_dur: 0.12479 (0.15428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.94139 (11.55182) + | > current_lr: 0.00003 + | > step_time: 1.56860 (2.39336) + | > loader_time: 0.00220 (0.02797) + + + --> STEP: 20/234 -- GLOBAL_STEP: 26930 + | > loss: -0.05010 (-0.05493) + | > log_mle: -0.19554 (-0.20546) + | > loss_dur: 0.14543 (0.15053) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.88416 (10.45448) + | > current_lr: 0.00003 + | > step_time: 0.95510 (2.07322) + | > loader_time: 0.00130 (0.02145) + + + --> STEP: 25/234 -- GLOBAL_STEP: 26935 + | > loss: -0.05576 (-0.05782) + | > log_mle: -0.19050 (-0.20423) + | > loss_dur: 0.13474 (0.14641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.68947 (9.71821) + | > current_lr: 0.00003 + | > step_time: 4.99420 (2.16050) + | > loader_time: 0.09840 (0.02473) + + + --> STEP: 30/234 -- GLOBAL_STEP: 26940 + | > loss: -0.10057 (-0.06027) + | > log_mle: -0.21280 (-0.20396) + | > loss_dur: 0.11223 (0.14369) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.60114 (9.51059) + | > current_lr: 0.00003 + | > step_time: 1.15720 (2.22890) + | > loader_time: 0.00200 (0.02132) + + + --> STEP: 35/234 -- GLOBAL_STEP: 26945 + | > loss: -0.05423 (-0.05838) + | > log_mle: -0.20704 (-0.20395) + | > loss_dur: 0.15280 (0.14557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.25632 (9.56934) + | > current_lr: 0.00003 + | > step_time: 1.79630 (2.11482) + | > loader_time: 0.00190 (0.01857) + + + --> STEP: 40/234 -- GLOBAL_STEP: 26950 + | > loss: -0.02255 (-0.05776) + | > log_mle: -0.18928 (-0.20396) + | > loss_dur: 0.16673 (0.14620) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.94621 (9.50286) + | > current_lr: 0.00003 + | > step_time: 1.67420 (2.05971) + | > loader_time: 0.00180 (0.01861) + + + --> STEP: 45/234 -- GLOBAL_STEP: 26955 + | > loss: -0.07102 (-0.05843) + | > log_mle: -0.22783 (-0.20435) + | > loss_dur: 0.15681 (0.14591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.53332 (9.28619) + | > current_lr: 0.00003 + | > step_time: 1.27190 (2.08973) + | > loader_time: 0.00210 (0.01712) + + + --> STEP: 50/234 -- GLOBAL_STEP: 26960 + | > loss: -0.04382 (-0.05853) + | > log_mle: -0.19514 (-0.20404) + | > loss_dur: 0.15133 (0.14551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.21990 (8.98242) + | > current_lr: 0.00003 + | > step_time: 4.81450 (2.14752) + | > loader_time: 0.18830 (0.02271) + + + --> STEP: 55/234 -- GLOBAL_STEP: 26965 + | > loss: -0.08199 (-0.05877) + | > log_mle: -0.21422 (-0.20435) + | > loss_dur: 0.13223 (0.14558) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.94801 (8.66358) + | > current_lr: 0.00003 + | > step_time: 2.02800 (2.15944) + | > loader_time: 0.00550 (0.02268) + + + --> STEP: 60/234 -- GLOBAL_STEP: 26970 + | > loss: -0.07069 (-0.05888) + | > log_mle: -0.23115 (-0.20497) + | > loss_dur: 0.16046 (0.14608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.19142 (8.61823) + | > current_lr: 0.00003 + | > step_time: 4.20800 (2.15274) + | > loader_time: 0.01040 (0.02247) + + + --> STEP: 65/234 -- GLOBAL_STEP: 26975 + | > loss: -0.07209 (-0.05818) + | > log_mle: -0.20610 (-0.20562) + | > loss_dur: 0.13400 (0.14744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.04947 (8.60447) + | > current_lr: 0.00003 + | > step_time: 1.82300 (2.12915) + | > loader_time: 0.00190 (0.02220) + + + --> STEP: 70/234 -- GLOBAL_STEP: 26980 + | > loss: -0.03912 (-0.05713) + | > log_mle: -0.20484 (-0.20549) + | > loss_dur: 0.16572 (0.14837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.21115 (8.59279) + | > current_lr: 0.00003 + | > step_time: 2.70640 (2.10245) + | > loader_time: 0.08280 (0.02192) + + + --> STEP: 75/234 -- GLOBAL_STEP: 26985 + | > loss: -0.05339 (-0.05650) + | > log_mle: -0.21687 (-0.20633) + | > loss_dur: 0.16348 (0.14983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.42168 (8.82876) + | > current_lr: 0.00003 + | > step_time: 3.21410 (2.14127) + | > loader_time: 0.08390 (0.03471) + + + --> STEP: 80/234 -- GLOBAL_STEP: 26990 + | > loss: -0.06902 (-0.05707) + | > log_mle: -0.19885 (-0.20665) + | > loss_dur: 0.12983 (0.14958) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.49023 (8.76890) + | > current_lr: 0.00003 + | > step_time: 1.89680 (2.15621) + | > loader_time: 0.00630 (0.03388) + + + --> STEP: 85/234 -- GLOBAL_STEP: 26995 + | > loss: -0.06836 (-0.05741) + | > log_mle: -0.21592 (-0.20736) + | > loss_dur: 0.14756 (0.14995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.23693 (8.86515) + | > current_lr: 0.00003 + | > step_time: 1.69600 (2.14262) + | > loader_time: 0.00250 (0.03390) + + + --> STEP: 90/234 -- GLOBAL_STEP: 27000 + | > loss: -0.05869 (-0.05785) + | > log_mle: -0.23996 (-0.20906) + | > loss_dur: 0.18127 (0.15121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.60317 (9.17894) + | > current_lr: 0.00003 + | > step_time: 2.40870 (2.18164) + | > loader_time: 0.07530 (0.03394) + + + --> STEP: 95/234 -- GLOBAL_STEP: 27005 + | > loss: -0.13036 (-0.05985) + | > log_mle: -0.33124 (-0.21270) + | > loss_dur: 0.20088 (0.15286) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.39470 (9.74736) + | > current_lr: 0.00003 + | > step_time: 2.54320 (2.16832) + | > loader_time: 0.08580 (0.03325) + + + --> STEP: 100/234 -- GLOBAL_STEP: 27010 + | > loss: -0.08301 (-0.06070) + | > log_mle: -0.25573 (-0.21456) + | > loss_dur: 0.17272 (0.15386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.41058 (10.17887) + | > current_lr: 0.00003 + | > step_time: 2.09320 (2.17422) + | > loader_time: 0.00220 (0.03181) + + + --> STEP: 105/234 -- GLOBAL_STEP: 27015 + | > loss: -0.08124 (-0.06249) + | > log_mle: -0.23112 (-0.21752) + | > loss_dur: 0.14987 (0.15503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.54836 (10.70367) + | > current_lr: 0.00003 + | > step_time: 2.16350 (2.19859) + | > loader_time: 0.00180 (0.03048) + + + --> STEP: 110/234 -- GLOBAL_STEP: 27020 + | > loss: -0.08643 (-0.06325) + | > log_mle: -0.25746 (-0.22007) + | > loss_dur: 0.17103 (0.15682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.08511 (11.21497) + | > current_lr: 0.00003 + | > step_time: 1.76490 (2.20184) + | > loader_time: 0.00280 (0.03093) + + + --> STEP: 115/234 -- GLOBAL_STEP: 27025 + | > loss: -0.08065 (-0.06485) + | > log_mle: -0.27814 (-0.22334) + | > loss_dur: 0.19748 (0.15849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.76020 (11.77394) + | > current_lr: 0.00003 + | > step_time: 1.60790 (2.20887) + | > loader_time: 0.00270 (0.03114) + + + --> STEP: 120/234 -- GLOBAL_STEP: 27030 + | > loss: -0.12336 (-0.06613) + | > log_mle: -0.32698 (-0.22616) + | > loss_dur: 0.20362 (0.16003) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.40593 (12.21218) + | > current_lr: 0.00003 + | > step_time: 1.30460 (2.22804) + | > loader_time: 0.00380 (0.03140) + + + --> STEP: 125/234 -- GLOBAL_STEP: 27035 + | > loss: -0.11984 (-0.06720) + | > log_mle: -0.31396 (-0.22776) + | > loss_dur: 0.19412 (0.16056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.46705 (12.56852) + | > current_lr: 0.00003 + | > step_time: 1.17310 (2.23704) + | > loader_time: 0.00250 (0.03234) + + + --> STEP: 130/234 -- GLOBAL_STEP: 27040 + | > loss: -0.12263 (-0.06911) + | > log_mle: -0.32338 (-0.23113) + | > loss_dur: 0.20075 (0.16202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.45716 (13.31488) + | > current_lr: 0.00003 + | > step_time: 3.09660 (2.24030) + | > loader_time: 0.00250 (0.03333) + + + --> STEP: 135/234 -- GLOBAL_STEP: 27045 + | > loss: -0.08370 (-0.07118) + | > log_mle: -0.25707 (-0.23447) + | > loss_dur: 0.17337 (0.16329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.22072 (14.02965) + | > current_lr: 0.00003 + | > step_time: 2.30220 (2.23816) + | > loader_time: 0.00240 (0.03407) + + + --> STEP: 140/234 -- GLOBAL_STEP: 27050 + | > loss: -0.08432 (-0.07330) + | > log_mle: -0.29025 (-0.23833) + | > loss_dur: 0.20593 (0.16503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.67085 (14.97401) + | > current_lr: 0.00003 + | > step_time: 3.69110 (2.25667) + | > loader_time: 0.00580 (0.03444) + + + --> STEP: 145/234 -- GLOBAL_STEP: 27055 + | > loss: -0.17525 (-0.07583) + | > log_mle: -0.38995 (-0.24287) + | > loss_dur: 0.21470 (0.16705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.76812 (15.81613) + | > current_lr: 0.00003 + | > step_time: 1.11660 (2.25011) + | > loader_time: 0.08040 (0.03445) + + + --> STEP: 150/234 -- GLOBAL_STEP: 27060 + | > loss: -0.15012 (-0.07880) + | > log_mle: -0.37118 (-0.24717) + | > loss_dur: 0.22106 (0.16837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.69183 (16.54827) + | > current_lr: 0.00003 + | > step_time: 4.80720 (2.29773) + | > loader_time: 0.09220 (0.03534) + + + --> STEP: 155/234 -- GLOBAL_STEP: 27065 + | > loss: -0.20020 (-0.08245) + | > log_mle: -0.43029 (-0.25235) + | > loss_dur: 0.23009 (0.16990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.69299 (17.76677) + | > current_lr: 0.00003 + | > step_time: 0.91710 (2.29601) + | > loader_time: 0.00440 (0.03602) + + + --> STEP: 160/234 -- GLOBAL_STEP: 27070 + | > loss: -0.19777 (-0.08520) + | > log_mle: -0.42266 (-0.25670) + | > loss_dur: 0.22489 (0.17151) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.85126 (18.98778) + | > current_lr: 0.00003 + | > step_time: 4.00800 (2.30553) + | > loader_time: 0.00380 (0.03501) + + + --> STEP: 165/234 -- GLOBAL_STEP: 27075 + | > loss: -0.18554 (-0.08799) + | > log_mle: -0.42894 (-0.26094) + | > loss_dur: 0.24340 (0.17295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.25966 (19.84273) + | > current_lr: 0.00003 + | > step_time: 2.49580 (2.42602) + | > loader_time: 0.10080 (0.03817) + + + --> STEP: 170/234 -- GLOBAL_STEP: 27080 + | > loss: -0.21888 (-0.09095) + | > log_mle: -0.46921 (-0.26565) + | > loss_dur: 0.25033 (0.17470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.60622 (20.85920) + | > current_lr: 0.00003 + | > step_time: 4.79160 (2.48042) + | > loader_time: 0.30040 (0.03994) + + + --> STEP: 175/234 -- GLOBAL_STEP: 27085 + | > loss: -0.19682 (-0.09472) + | > log_mle: -0.44250 (-0.27106) + | > loss_dur: 0.24569 (0.17634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.95113 (22.05439) + | > current_lr: 0.00003 + | > step_time: 1.30420 (2.51942) + | > loader_time: 0.00240 (0.04107) + + + --> STEP: 180/234 -- GLOBAL_STEP: 27090 + | > loss: -0.21221 (-0.09781) + | > log_mle: -0.44800 (-0.27604) + | > loss_dur: 0.23579 (0.17823) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.54068 (23.11193) + | > current_lr: 0.00003 + | > step_time: 5.70160 (2.56898) + | > loader_time: 0.00300 (0.04268) + + + --> STEP: 185/234 -- GLOBAL_STEP: 27095 + | > loss: -0.22065 (-0.10072) + | > log_mle: -0.47562 (-0.28068) + | > loss_dur: 0.25497 (0.17996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.54875 (24.07893) + | > current_lr: 0.00003 + | > step_time: 11.61270 (2.60494) + | > loader_time: 0.00530 (0.04250) + + + --> STEP: 190/234 -- GLOBAL_STEP: 27100 + | > loss: -0.22450 (-0.10359) + | > log_mle: -0.45139 (-0.28521) + | > loss_dur: 0.22690 (0.18162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.03666 (25.03440) + | > current_lr: 0.00003 + | > step_time: 4.70040 (2.63961) + | > loader_time: 0.09870 (0.04382) + + + --> STEP: 195/234 -- GLOBAL_STEP: 27105 + | > loss: -0.21251 (-0.10695) + | > log_mle: -0.46655 (-0.28999) + | > loss_dur: 0.25404 (0.18304) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.66805 (25.97929) + | > current_lr: 0.00003 + | > step_time: 6.00360 (2.67087) + | > loader_time: 0.09230 (0.04379) + + + --> STEP: 200/234 -- GLOBAL_STEP: 27110 + | > loss: -0.20516 (-0.10978) + | > log_mle: -0.47329 (-0.29440) + | > loss_dur: 0.26812 (0.18462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.12860 (27.00117) + | > current_lr: 0.00003 + | > step_time: 5.30480 (2.78511) + | > loader_time: 0.00510 (0.04373) + + + --> STEP: 205/234 -- GLOBAL_STEP: 27115 + | > loss: -0.21502 (-0.11239) + | > log_mle: -0.45939 (-0.29858) + | > loss_dur: 0.24436 (0.18620) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.25463 (28.19156) + | > current_lr: 0.00003 + | > step_time: 6.69550 (2.87822) + | > loader_time: 0.10490 (0.04504) + + + --> STEP: 210/234 -- GLOBAL_STEP: 27120 + | > loss: -0.27896 (-0.11577) + | > log_mle: -0.53854 (-0.30362) + | > loss_dur: 0.25958 (0.18785) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.48899 (29.20835) + | > current_lr: 0.00003 + | > step_time: 15.09540 (3.02062) + | > loader_time: 0.11070 (0.04559) + + + --> STEP: 215/234 -- GLOBAL_STEP: 27125 + | > loss: -0.24047 (-0.11935) + | > log_mle: -0.49045 (-0.30877) + | > loss_dur: 0.24998 (0.18942) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.52354 (30.55146) + | > current_lr: 0.00003 + | > step_time: 5.39930 (3.04655) + | > loader_time: 0.09430 (0.04506) + + + --> STEP: 220/234 -- GLOBAL_STEP: 27130 + | > loss: -0.27102 (-0.12300) + | > log_mle: -0.53288 (-0.31410) + | > loss_dur: 0.26186 (0.19110) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.76151 (32.00797) + | > current_lr: 0.00003 + | > step_time: 2.79750 (3.06916) + | > loader_time: 0.00970 (0.04500) + + + --> STEP: 225/234 -- GLOBAL_STEP: 27135 + | > loss: -0.32551 (-0.12640) + | > log_mle: -0.60803 (-0.31926) + | > loss_dur: 0.28253 (0.19285) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.61166 (33.10480) + | > current_lr: 0.00003 + | > step_time: 0.96560 (3.04708) + | > loader_time: 0.00360 (0.04449) + + + --> STEP: 230/234 -- GLOBAL_STEP: 27140 + | > loss: -0.28660 (-0.12956) + | > log_mle: -0.64242 (-0.32483) + | > loss_dur: 0.35581 (0.19527) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 143.92305 (34.58324) + | > current_lr: 0.00003 + | > step_time: 0.26730 (2.99036) + | > loader_time: 0.00350 (0.06399) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32718 (+0.10138) + | > avg_loss: -0.18259 (-0.02766) + | > avg_log_mle: -0.41358 (-0.01857) + | > avg_loss_dur: 0.23098 (-0.00909) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_27144.pth + + > EPOCH: 116/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 09:44:51)  + + --> STEP: 1/234 -- GLOBAL_STEP: 27145 + | > loss: -0.07629 (-0.07629) + | > log_mle: -0.21064 (-0.21064) + | > loss_dur: 0.13436 (0.13436) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.29881 (11.29881) + | > current_lr: 0.00003 + | > step_time: 11.80750 (11.80747) + | > loader_time: 0.00220 (0.00224) + + + --> STEP: 6/234 -- GLOBAL_STEP: 27150 + | > loss: -0.04510 (-0.04022) + | > log_mle: -0.20090 (-0.20578) + | > loss_dur: 0.15579 (0.16556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.71944 (12.40365) + | > current_lr: 0.00003 + | > step_time: 3.09230 (6.78547) + | > loader_time: 0.01390 (0.06570) + + + --> STEP: 11/234 -- GLOBAL_STEP: 27155 + | > loss: -0.06748 (-0.04663) + | > log_mle: -0.20702 (-0.21021) + | > loss_dur: 0.13954 (0.16358) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.35567 (11.03608) + | > current_lr: 0.00003 + | > step_time: 2.89300 (4.71796) + | > loader_time: 0.00240 (0.05290) + + + --> STEP: 16/234 -- GLOBAL_STEP: 27160 + | > loss: -0.09815 (-0.05611) + | > log_mle: -0.20764 (-0.20988) + | > loss_dur: 0.10949 (0.15377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.57256 (10.09069) + | > current_lr: 0.00003 + | > step_time: 1.01830 (3.71608) + | > loader_time: 0.00130 (0.04288) + + + --> STEP: 21/234 -- GLOBAL_STEP: 27165 + | > loss: -0.05061 (-0.05568) + | > log_mle: -0.18736 (-0.20636) + | > loss_dur: 0.13675 (0.15067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.93210 (9.39279) + | > current_lr: 0.00003 + | > step_time: 1.41080 (3.41826) + | > loader_time: 0.00200 (0.03768) + + + --> STEP: 26/234 -- GLOBAL_STEP: 27170 + | > loss: -0.06001 (-0.05863) + | > log_mle: -0.20586 (-0.20594) + | > loss_dur: 0.14586 (0.14731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.54734 (8.77228) + | > current_lr: 0.00003 + | > step_time: 3.39690 (3.31048) + | > loader_time: 0.18690 (0.04469) + + + --> STEP: 31/234 -- GLOBAL_STEP: 27175 + | > loss: -0.02790 (-0.06014) + | > log_mle: -0.20690 (-0.20591) + | > loss_dur: 0.17900 (0.14577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.50489 (8.37932) + | > current_lr: 0.00003 + | > step_time: 1.21580 (3.29544) + | > loader_time: 0.00200 (0.04073) + + + --> STEP: 36/234 -- GLOBAL_STEP: 27180 + | > loss: -0.05455 (-0.06024) + | > log_mle: -0.20851 (-0.20631) + | > loss_dur: 0.15396 (0.14607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.00818 (8.29513) + | > current_lr: 0.00003 + | > step_time: 2.10840 (3.08633) + | > loader_time: 0.00310 (0.03789) + + + --> STEP: 41/234 -- GLOBAL_STEP: 27185 + | > loss: -0.07736 (-0.06032) + | > log_mle: -0.20363 (-0.20604) + | > loss_dur: 0.12627 (0.14572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.65263 (8.19314) + | > current_lr: 0.00003 + | > step_time: 1.32540 (2.92422) + | > loader_time: 0.00230 (0.03534) + + + --> STEP: 46/234 -- GLOBAL_STEP: 27190 + | > loss: -0.05884 (-0.06041) + | > log_mle: -0.20417 (-0.20630) + | > loss_dur: 0.14533 (0.14589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.26610 (8.19556) + | > current_lr: 0.00003 + | > step_time: 2.51200 (2.83599) + | > loader_time: 0.00200 (0.03172) + + + --> STEP: 51/234 -- GLOBAL_STEP: 27195 + | > loss: -0.04609 (-0.06011) + | > log_mle: -0.19302 (-0.20564) + | > loss_dur: 0.14693 (0.14553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.54368 (7.90676) + | > current_lr: 0.00003 + | > step_time: 1.70510 (2.74567) + | > loader_time: 0.00370 (0.03055) + + + --> STEP: 56/234 -- GLOBAL_STEP: 27200 + | > loss: -0.03192 (-0.05987) + | > log_mle: -0.21295 (-0.20628) + | > loss_dur: 0.18104 (0.14641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.83883 (7.78557) + | > current_lr: 0.00003 + | > step_time: 2.98960 (2.68368) + | > loader_time: 0.18340 (0.03130) + + + --> STEP: 61/234 -- GLOBAL_STEP: 27205 + | > loss: -0.07632 (-0.06075) + | > log_mle: -0.21085 (-0.20686) + | > loss_dur: 0.13453 (0.14611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.64708 (7.77512) + | > current_lr: 0.00003 + | > step_time: 3.61070 (2.69330) + | > loader_time: 0.09500 (0.03180) + + + --> STEP: 66/234 -- GLOBAL_STEP: 27210 + | > loss: -0.06570 (-0.06021) + | > log_mle: -0.19745 (-0.20716) + | > loss_dur: 0.13175 (0.14694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.78903 (7.98585) + | > current_lr: 0.00003 + | > step_time: 7.60280 (2.69227) + | > loader_time: 0.10570 (0.03243) + + + --> STEP: 71/234 -- GLOBAL_STEP: 27215 + | > loss: -0.03284 (-0.05865) + | > log_mle: -0.23064 (-0.20735) + | > loss_dur: 0.19780 (0.14870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.01484 (8.32526) + | > current_lr: 0.00003 + | > step_time: 1.71580 (2.62063) + | > loader_time: 0.00490 (0.03046) + + + --> STEP: 76/234 -- GLOBAL_STEP: 27220 + | > loss: -0.06749 (-0.05833) + | > log_mle: -0.22019 (-0.20766) + | > loss_dur: 0.15271 (0.14933) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.59860 (8.40894) + | > current_lr: 0.00003 + | > step_time: 2.18460 (2.62271) + | > loader_time: 0.00240 (0.02991) + + + --> STEP: 81/234 -- GLOBAL_STEP: 27225 + | > loss: -0.07155 (-0.05860) + | > log_mle: -0.22651 (-0.20793) + | > loss_dur: 0.15496 (0.14932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.36081 (8.40172) + | > current_lr: 0.00003 + | > step_time: 2.50610 (2.58573) + | > loader_time: 0.07560 (0.03124) + + + --> STEP: 86/234 -- GLOBAL_STEP: 27230 + | > loss: -0.06325 (-0.05856) + | > log_mle: -0.22639 (-0.20853) + | > loss_dur: 0.16314 (0.14997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.97087 (8.51775) + | > current_lr: 0.00003 + | > step_time: 1.10170 (2.55638) + | > loader_time: 0.00230 (0.03044) + + + --> STEP: 91/234 -- GLOBAL_STEP: 27235 + | > loss: -0.05478 (-0.05911) + | > log_mle: -0.24050 (-0.21039) + | > loss_dur: 0.18572 (0.15128) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.79673 (8.73562) + | > current_lr: 0.00003 + | > step_time: 1.79540 (2.53226) + | > loader_time: 0.00280 (0.02995) + + + --> STEP: 96/234 -- GLOBAL_STEP: 27240 + | > loss: -0.06950 (-0.06159) + | > log_mle: -0.22801 (-0.21383) + | > loss_dur: 0.15851 (0.15224) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.98172 (9.31684) + | > current_lr: 0.00003 + | > step_time: 2.98760 (2.56307) + | > loader_time: 0.00660 (0.03027) + + + --> STEP: 101/234 -- GLOBAL_STEP: 27245 + | > loss: -0.10110 (-0.06303) + | > log_mle: -0.28597 (-0.21628) + | > loss_dur: 0.18487 (0.15325) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.11889 (9.86174) + | > current_lr: 0.00003 + | > step_time: 2.30570 (2.54334) + | > loader_time: 0.08720 (0.03058) + + + --> STEP: 106/234 -- GLOBAL_STEP: 27250 + | > loss: -0.08440 (-0.06459) + | > log_mle: -0.28810 (-0.21924) + | > loss_dur: 0.20370 (0.15466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.33046 (10.40186) + | > current_lr: 0.00003 + | > step_time: 1.88860 (2.54901) + | > loader_time: 0.00500 (0.03088) + + + --> STEP: 111/234 -- GLOBAL_STEP: 27255 + | > loss: -0.10936 (-0.06583) + | > log_mle: -0.33019 (-0.22219) + | > loss_dur: 0.22083 (0.15636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.47704 (10.98090) + | > current_lr: 0.00003 + | > step_time: 1.60760 (2.54327) + | > loader_time: 0.08850 (0.03124) + + + --> STEP: 116/234 -- GLOBAL_STEP: 27260 + | > loss: -0.07785 (-0.06701) + | > log_mle: -0.29822 (-0.22510) + | > loss_dur: 0.22037 (0.15809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.19088 (11.47875) + | > current_lr: 0.00003 + | > step_time: 3.80370 (2.54594) + | > loader_time: 0.00270 (0.03164) + + + --> STEP: 121/234 -- GLOBAL_STEP: 27265 + | > loss: -0.04138 (-0.06812) + | > log_mle: -0.21109 (-0.22717) + | > loss_dur: 0.16971 (0.15905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.91871 (11.75154) + | > current_lr: 0.00003 + | > step_time: 1.29790 (2.50112) + | > loader_time: 0.00270 (0.03048) + + + --> STEP: 126/234 -- GLOBAL_STEP: 27270 + | > loss: -0.14238 (-0.06988) + | > log_mle: -0.34220 (-0.22981) + | > loss_dur: 0.19982 (0.15993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.93858 (12.39357) + | > current_lr: 0.00003 + | > step_time: 1.10080 (2.50020) + | > loader_time: 0.09020 (0.03234) + + + --> STEP: 131/234 -- GLOBAL_STEP: 27275 + | > loss: -0.17190 (-0.07195) + | > log_mle: -0.38735 (-0.23339) + | > loss_dur: 0.21544 (0.16144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.96536 (13.15711) + | > current_lr: 0.00003 + | > step_time: 1.87440 (2.50463) + | > loader_time: 0.08400 (0.03400) + + + --> STEP: 136/234 -- GLOBAL_STEP: 27280 + | > loss: -0.20653 (-0.07424) + | > log_mle: -0.43614 (-0.23707) + | > loss_dur: 0.22961 (0.16283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.51950 (13.71603) + | > current_lr: 0.00003 + | > step_time: 3.30560 (2.48038) + | > loader_time: 0.09530 (0.03418) + + + --> STEP: 141/234 -- GLOBAL_STEP: 27285 + | > loss: -0.12616 (-0.07582) + | > log_mle: -0.33922 (-0.24023) + | > loss_dur: 0.21305 (0.16441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.29871 (14.47205) + | > current_lr: 0.00003 + | > step_time: 1.60880 (2.44438) + | > loader_time: 0.00360 (0.03308) + + + --> STEP: 146/234 -- GLOBAL_STEP: 27290 + | > loss: -0.17705 (-0.07882) + | > log_mle: -0.39525 (-0.24513) + | > loss_dur: 0.21819 (0.16631) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.18176 (15.35826) + | > current_lr: 0.00003 + | > step_time: 2.19650 (2.44489) + | > loader_time: 0.01020 (0.03272) + + + --> STEP: 151/234 -- GLOBAL_STEP: 27295 + | > loss: -0.16908 (-0.08153) + | > log_mle: -0.36276 (-0.24915) + | > loss_dur: 0.19368 (0.16762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.23658 (16.19351) + | > current_lr: 0.00003 + | > step_time: 1.31490 (2.41094) + | > loader_time: 0.08790 (0.03342) + + + --> STEP: 156/234 -- GLOBAL_STEP: 27300 + | > loss: -0.17706 (-0.08525) + | > log_mle: -0.39239 (-0.25455) + | > loss_dur: 0.21532 (0.16930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.33017 (17.57159) + | > current_lr: 0.00003 + | > step_time: 3.60640 (2.41661) + | > loader_time: 0.00310 (0.03359) + + + --> STEP: 161/234 -- GLOBAL_STEP: 27305 + | > loss: -0.20060 (-0.08810) + | > log_mle: -0.42182 (-0.25908) + | > loss_dur: 0.22122 (0.17098) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.56542 (18.53902) + | > current_lr: 0.00003 + | > step_time: 5.10160 (2.43458) + | > loader_time: 0.08830 (0.03319) + + + --> STEP: 166/234 -- GLOBAL_STEP: 27310 + | > loss: -0.16302 (-0.09078) + | > log_mle: -0.36688 (-0.26307) + | > loss_dur: 0.20385 (0.17229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.93688 (19.20650) + | > current_lr: 0.00003 + | > step_time: 1.20380 (2.43365) + | > loader_time: 0.00370 (0.03383) + + + --> STEP: 171/234 -- GLOBAL_STEP: 27315 + | > loss: -0.25415 (-0.09430) + | > log_mle: -0.47110 (-0.26838) + | > loss_dur: 0.21695 (0.17408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.91222 (20.48931) + | > current_lr: 0.00003 + | > step_time: 1.70450 (2.41650) + | > loader_time: 0.08760 (0.03501) + + + --> STEP: 176/234 -- GLOBAL_STEP: 27320 + | > loss: -0.21491 (-0.09773) + | > log_mle: -0.44373 (-0.27362) + | > loss_dur: 0.22882 (0.17589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.59937 (21.57197) + | > current_lr: 0.00003 + | > step_time: 5.39730 (2.46834) + | > loader_time: 0.09830 (0.03623) + + + --> STEP: 181/234 -- GLOBAL_STEP: 27325 + | > loss: -0.15367 (-0.10058) + | > log_mle: -0.38003 (-0.27835) + | > loss_dur: 0.22635 (0.17777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.79813 (22.52732) + | > current_lr: 0.00003 + | > step_time: 4.10410 (2.50290) + | > loader_time: 0.00350 (0.03735) + + + --> STEP: 186/234 -- GLOBAL_STEP: 27330 + | > loss: -0.17488 (-0.10356) + | > log_mle: -0.42020 (-0.28321) + | > loss_dur: 0.24531 (0.17965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.44690 (23.81517) + | > current_lr: 0.00003 + | > step_time: 2.51200 (2.49266) + | > loader_time: 0.00270 (0.03797) + + + --> STEP: 191/234 -- GLOBAL_STEP: 27335 + | > loss: -0.21103 (-0.10667) + | > log_mle: -0.43294 (-0.28782) + | > loss_dur: 0.22191 (0.18115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.76302 (25.00183) + | > current_lr: 0.00003 + | > step_time: 0.67900 (2.60031) + | > loader_time: 0.00450 (0.03959) + + + --> STEP: 196/234 -- GLOBAL_STEP: 27340 + | > loss: -0.19193 (-0.10989) + | > log_mle: -0.42967 (-0.29245) + | > loss_dur: 0.23774 (0.18256) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.96174 (26.31102) + | > current_lr: 0.00003 + | > step_time: 3.51290 (2.61205) + | > loader_time: 0.09290 (0.04003) + + + --> STEP: 201/234 -- GLOBAL_STEP: 27345 + | > loss: -0.15556 (-0.11247) + | > log_mle: -0.39781 (-0.29660) + | > loss_dur: 0.24225 (0.18413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.35586 (27.51847) + | > current_lr: 0.00003 + | > step_time: 5.51080 (2.65310) + | > loader_time: 0.19350 (0.04203) + + + --> STEP: 206/234 -- GLOBAL_STEP: 27350 + | > loss: -0.25322 (-0.11553) + | > log_mle: -0.49902 (-0.30120) + | > loss_dur: 0.24580 (0.18567) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.80372 (28.95089) + | > current_lr: 0.00003 + | > step_time: 4.10150 (2.70867) + | > loader_time: 0.09670 (0.04295) + + + --> STEP: 211/234 -- GLOBAL_STEP: 27355 + | > loss: -0.29075 (-0.11898) + | > log_mle: -0.57086 (-0.30642) + | > loss_dur: 0.28011 (0.18745) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.90643 (30.15079) + | > current_lr: 0.00003 + | > step_time: 3.20340 (2.78381) + | > loader_time: 0.08850 (0.04376) + + + --> STEP: 216/234 -- GLOBAL_STEP: 27360 + | > loss: -0.28138 (-0.12235) + | > log_mle: -0.55579 (-0.31143) + | > loss_dur: 0.27441 (0.18908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.01385 (31.48984) + | > current_lr: 0.00003 + | > step_time: 3.71110 (2.85736) + | > loader_time: 0.08560 (0.04649) + + + --> STEP: 221/234 -- GLOBAL_STEP: 27365 + | > loss: -0.23960 (-0.12587) + | > log_mle: -0.48209 (-0.31654) + | > loss_dur: 0.24249 (0.19067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.32444 (32.50188) + | > current_lr: 0.00003 + | > step_time: 1.90010 (2.87556) + | > loader_time: 0.00420 (0.04706) + + + --> STEP: 226/234 -- GLOBAL_STEP: 27370 + | > loss: -0.30833 (-0.12968) + | > log_mle: -0.57925 (-0.32219) + | > loss_dur: 0.27092 (0.19251) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.78391 (33.71464) + | > current_lr: 0.00003 + | > step_time: 0.25080 (2.82306) + | > loader_time: 0.00440 (0.04646) + + + --> STEP: 231/234 -- GLOBAL_STEP: 27375 + | > loss: -0.23196 (-0.13246) + | > log_mle: -0.63917 (-0.32803) + | > loss_dur: 0.40721 (0.19557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.75845 (35.10278) + | > current_lr: 0.00003 + | > step_time: 0.27630 (2.76757) + | > loader_time: 0.00400 (0.04555) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.92732 (+0.60014) + | > avg_loss: -0.15039 (+0.03220) + | > avg_log_mle: -0.38544 (+0.02813) + | > avg_loss_dur: 0.23505 (+0.00407) + + + > EPOCH: 117/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 09:56:41)  + + --> STEP: 2/234 -- GLOBAL_STEP: 27380 + | > loss: -0.04400 (-0.04980) + | > log_mle: -0.20221 (-0.20682) + | > loss_dur: 0.15821 (0.15702) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.56769 (10.39957) + | > current_lr: 0.00003 + | > step_time: 1.99240 (3.19611) + | > loader_time: 0.10020 (0.05183) + + + --> STEP: 7/234 -- GLOBAL_STEP: 27385 + | > loss: -0.08982 (-0.04734) + | > log_mle: -0.21944 (-0.20711) + | > loss_dur: 0.12961 (0.15977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.60885 (17.33621) + | > current_lr: 0.00003 + | > step_time: 1.01380 (2.57053) + | > loader_time: 0.08130 (0.22551) + + + --> STEP: 12/234 -- GLOBAL_STEP: 27390 + | > loss: -0.04632 (-0.05090) + | > log_mle: -0.20831 (-0.20955) + | > loss_dur: 0.16199 (0.15865) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.61308 (14.34790) + | > current_lr: 0.00003 + | > step_time: 5.69370 (3.42601) + | > loader_time: 0.00520 (0.70624) + + + --> STEP: 17/234 -- GLOBAL_STEP: 27395 + | > loss: -0.05284 (-0.05743) + | > log_mle: -0.18726 (-0.20852) + | > loss_dur: 0.13442 (0.15109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.45578 (12.03889) + | > current_lr: 0.00003 + | > step_time: 2.31150 (4.41933) + | > loader_time: 0.38940 (0.55670) + + + --> STEP: 22/234 -- GLOBAL_STEP: 27400 + | > loss: -0.07960 (-0.05827) + | > log_mle: -0.21357 (-0.20706) + | > loss_dur: 0.13397 (0.14878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.24250 (10.87269) + | > current_lr: 0.00003 + | > step_time: 4.59890 (3.89163) + | > loader_time: 0.00170 (0.43421) + + + --> STEP: 27/234 -- GLOBAL_STEP: 27405 + | > loss: -0.08132 (-0.06059) + | > log_mle: -0.21427 (-0.20676) + | > loss_dur: 0.13295 (0.14617) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.80987 (10.06543) + | > current_lr: 0.00003 + | > step_time: 0.83020 (3.62724) + | > loader_time: 0.08600 (0.36470) + + + --> STEP: 32/234 -- GLOBAL_STEP: 27410 + | > loss: -0.10027 (-0.06278) + | > log_mle: -0.22322 (-0.20717) + | > loss_dur: 0.12295 (0.14439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.86028 (9.54442) + | > current_lr: 0.00003 + | > step_time: 2.47900 (3.48976) + | > loader_time: 0.00190 (0.31373) + + + --> STEP: 37/234 -- GLOBAL_STEP: 27415 + | > loss: -0.08548 (-0.06235) + | > log_mle: -0.20454 (-0.20701) + | > loss_dur: 0.11907 (0.14466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.48096 (9.54716) + | > current_lr: 0.00003 + | > step_time: 5.49300 (3.30736) + | > loader_time: 0.10700 (0.27879) + + + --> STEP: 42/234 -- GLOBAL_STEP: 27420 + | > loss: -0.05954 (-0.06181) + | > log_mle: -0.19622 (-0.20686) + | > loss_dur: 0.13668 (0.14505) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.29278 (9.17876) + | > current_lr: 0.00003 + | > step_time: 1.38940 (3.08508) + | > loader_time: 0.00560 (0.24994) + + + --> STEP: 47/234 -- GLOBAL_STEP: 27425 + | > loss: -0.05584 (-0.06207) + | > log_mle: -0.20602 (-0.20750) + | > loss_dur: 0.15017 (0.14542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.26061 (9.08677) + | > current_lr: 0.00003 + | > step_time: 2.11210 (2.90671) + | > loader_time: 0.00240 (0.22539) + + + --> STEP: 52/234 -- GLOBAL_STEP: 27430 + | > loss: -0.05018 (-0.06217) + | > log_mle: -0.20217 (-0.20685) + | > loss_dur: 0.15198 (0.14468) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.39437 (8.79350) + | > current_lr: 0.00003 + | > step_time: 1.10690 (2.74090) + | > loader_time: 0.00290 (0.20399) + + + --> STEP: 57/234 -- GLOBAL_STEP: 27435 + | > loss: -0.04887 (-0.06240) + | > log_mle: -0.19667 (-0.20740) + | > loss_dur: 0.14781 (0.14500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.83484 (8.62565) + | > current_lr: 0.00003 + | > step_time: 2.01500 (2.60955) + | > loader_time: 0.08760 (0.19053) + + + --> STEP: 62/234 -- GLOBAL_STEP: 27440 + | > loss: -0.03977 (-0.06330) + | > log_mle: -0.24961 (-0.20894) + | > loss_dur: 0.20984 (0.14564) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.45804 (8.74931) + | > current_lr: 0.00003 + | > step_time: 0.99660 (2.51338) + | > loader_time: 0.00460 (0.17676) + + + --> STEP: 67/234 -- GLOBAL_STEP: 27445 + | > loss: -0.06923 (-0.06331) + | > log_mle: -0.22991 (-0.20906) + | > loss_dur: 0.16068 (0.14575) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.84124 (8.60727) + | > current_lr: 0.00003 + | > step_time: 0.80810 (2.42276) + | > loader_time: 0.08580 (0.16504) + + + --> STEP: 72/234 -- GLOBAL_STEP: 27450 + | > loss: -0.04603 (-0.06184) + | > log_mle: -0.20929 (-0.20921) + | > loss_dur: 0.16327 (0.14737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.02253 (8.73556) + | > current_lr: 0.00003 + | > step_time: 0.79140 (2.39057) + | > loader_time: 0.00350 (0.15593) + + + --> STEP: 77/234 -- GLOBAL_STEP: 27455 + | > loss: -0.08320 (-0.06200) + | > log_mle: -0.22201 (-0.20999) + | > loss_dur: 0.13882 (0.14799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.32701 (8.82779) + | > current_lr: 0.00003 + | > step_time: 2.30510 (2.38637) + | > loader_time: 0.00250 (0.14688) + + + --> STEP: 82/234 -- GLOBAL_STEP: 27460 + | > loss: -0.06739 (-0.06230) + | > log_mle: -0.21206 (-0.21020) + | > loss_dur: 0.14467 (0.14790) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.73993 (8.91076) + | > current_lr: 0.00003 + | > step_time: 2.68970 (2.34959) + | > loader_time: 0.00280 (0.14211) + + + --> STEP: 87/234 -- GLOBAL_STEP: 27465 + | > loss: -0.04594 (-0.06212) + | > log_mle: -0.21909 (-0.21090) + | > loss_dur: 0.17315 (0.14878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.95360 (9.17648) + | > current_lr: 0.00003 + | > step_time: 1.18370 (2.34800) + | > loader_time: 0.00220 (0.13626) + + + --> STEP: 92/234 -- GLOBAL_STEP: 27470 + | > loss: -0.11231 (-0.06333) + | > log_mle: -0.26641 (-0.21321) + | > loss_dur: 0.15410 (0.14988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.42360 (9.65683) + | > current_lr: 0.00003 + | > step_time: 1.19920 (2.32904) + | > loader_time: 0.00230 (0.13198) + + + --> STEP: 97/234 -- GLOBAL_STEP: 27475 + | > loss: -0.09952 (-0.06526) + | > log_mle: -0.25522 (-0.21640) + | > loss_dur: 0.15570 (0.15114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.36774 (10.33849) + | > current_lr: 0.00003 + | > step_time: 3.10960 (2.31421) + | > loader_time: 0.08890 (0.12621) + + + --> STEP: 102/234 -- GLOBAL_STEP: 27480 + | > loss: -0.06568 (-0.06638) + | > log_mle: -0.23658 (-0.21857) + | > loss_dur: 0.17090 (0.15218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.82882 (10.72522) + | > current_lr: 0.00003 + | > step_time: 2.40850 (2.30328) + | > loader_time: 0.00240 (0.12015) + + + --> STEP: 107/234 -- GLOBAL_STEP: 27485 + | > loss: -0.09598 (-0.06808) + | > log_mle: -0.28071 (-0.22175) + | > loss_dur: 0.18473 (0.15367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.36314 (11.28908) + | > current_lr: 0.00003 + | > step_time: 2.70410 (2.28212) + | > loader_time: 0.08320 (0.11616) + + + --> STEP: 112/234 -- GLOBAL_STEP: 27490 + | > loss: -0.08396 (-0.06914) + | > log_mle: -0.28927 (-0.22466) + | > loss_dur: 0.20530 (0.15552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.02442 (11.87392) + | > current_lr: 0.00003 + | > step_time: 2.31480 (2.28281) + | > loader_time: 0.08430 (0.11258) + + + --> STEP: 117/234 -- GLOBAL_STEP: 27495 + | > loss: -0.10367 (-0.07017) + | > log_mle: -0.28724 (-0.22738) + | > loss_dur: 0.18356 (0.15721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.06102 (12.35632) + | > current_lr: 0.00003 + | > step_time: 1.08670 (2.26449) + | > loader_time: 0.00240 (0.10790) + + + --> STEP: 122/234 -- GLOBAL_STEP: 27500 + | > loss: -0.09166 (-0.07097) + | > log_mle: -0.26615 (-0.22916) + | > loss_dur: 0.17449 (0.15819) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.81738 (12.57766) + | > current_lr: 0.00003 + | > step_time: 1.10700 (2.24693) + | > loader_time: 0.00330 (0.10505) + + + --> STEP: 127/234 -- GLOBAL_STEP: 27505 + | > loss: -0.11651 (-0.07268) + | > log_mle: -0.31921 (-0.23215) + | > loss_dur: 0.20271 (0.15947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.10810 (13.22917) + | > current_lr: 0.00003 + | > step_time: 1.70380 (2.24436) + | > loader_time: 0.08940 (0.10371) + + + --> STEP: 132/234 -- GLOBAL_STEP: 27510 + | > loss: -0.13594 (-0.07489) + | > log_mle: -0.29912 (-0.23554) + | > loss_dur: 0.16318 (0.16066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.04700 (13.96010) + | > current_lr: 0.00003 + | > step_time: 3.60230 (2.24735) + | > loader_time: 0.08250 (0.10318) + + + --> STEP: 137/234 -- GLOBAL_STEP: 27515 + | > loss: -0.09903 (-0.07679) + | > log_mle: -0.31279 (-0.23915) + | > loss_dur: 0.21376 (0.16236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.46611 (14.72343) + | > current_lr: 0.00003 + | > step_time: 1.35130 (2.25066) + | > loader_time: 0.00210 (0.10209) + + + --> STEP: 142/234 -- GLOBAL_STEP: 27520 + | > loss: -0.11706 (-0.07843) + | > log_mle: -0.32985 (-0.24232) + | > loss_dur: 0.21279 (0.16390) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.53993 (15.36340) + | > current_lr: 0.00003 + | > step_time: 4.21020 (2.26650) + | > loader_time: 0.00350 (0.09925) + + + --> STEP: 147/234 -- GLOBAL_STEP: 27525 + | > loss: -0.12250 (-0.08122) + | > log_mle: -0.33358 (-0.24717) + | > loss_dur: 0.21109 (0.16595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.74666 (16.16654) + | > current_lr: 0.00003 + | > step_time: 2.42510 (2.26444) + | > loader_time: 0.00300 (0.09656) + + + --> STEP: 152/234 -- GLOBAL_STEP: 27530 + | > loss: -0.17533 (-0.08422) + | > log_mle: -0.40873 (-0.25159) + | > loss_dur: 0.23340 (0.16737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.95481 (17.06472) + | > current_lr: 0.00003 + | > step_time: 5.50250 (2.29305) + | > loader_time: 0.19680 (0.09600) + + + --> STEP: 157/234 -- GLOBAL_STEP: 27535 + | > loss: -0.14196 (-0.08767) + | > log_mle: -0.35631 (-0.25658) + | > loss_dur: 0.21435 (0.16891) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.94965 (18.10731) + | > current_lr: 0.00003 + | > step_time: 2.67520 (2.33205) + | > loader_time: 0.00350 (0.09535) + + + --> STEP: 162/234 -- GLOBAL_STEP: 27540 + | > loss: -0.19067 (-0.09080) + | > log_mle: -0.39007 (-0.26119) + | > loss_dur: 0.19940 (0.17040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.14663 (19.14378) + | > current_lr: 0.00003 + | > step_time: 1.60500 (2.32069) + | > loader_time: 0.07860 (0.09346) + + + --> STEP: 167/234 -- GLOBAL_STEP: 27545 + | > loss: -0.22761 (-0.09322) + | > log_mle: -0.44899 (-0.26504) + | > loss_dur: 0.22138 (0.17181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.58683 (21.08550) + | > current_lr: 0.00003 + | > step_time: 1.60550 (2.30579) + | > loader_time: 0.00250 (0.09127) + + + --> STEP: 172/234 -- GLOBAL_STEP: 27550 + | > loss: -0.21502 (-0.09629) + | > log_mle: -0.45996 (-0.26995) + | > loss_dur: 0.24494 (0.17366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.24545 (22.03867) + | > current_lr: 0.00003 + | > step_time: 3.49030 (2.31136) + | > loader_time: 0.00570 (0.08927) + + + --> STEP: 177/234 -- GLOBAL_STEP: 27555 + | > loss: -0.19134 (-0.09948) + | > log_mle: -0.42263 (-0.27478) + | > loss_dur: 0.23129 (0.17530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.34608 (22.99353) + | > current_lr: 0.00003 + | > step_time: 1.61410 (2.30586) + | > loader_time: 0.08750 (0.08853) + + + --> STEP: 182/234 -- GLOBAL_STEP: 27560 + | > loss: -0.21879 (-0.10234) + | > log_mle: -0.47005 (-0.27964) + | > loss_dur: 0.25125 (0.17730) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.64627 (23.95789) + | > current_lr: 0.00003 + | > step_time: 3.10270 (2.30300) + | > loader_time: 0.00840 (0.08746) + + + --> STEP: 187/234 -- GLOBAL_STEP: 27565 + | > loss: -0.22664 (-0.10523) + | > log_mle: -0.47106 (-0.28435) + | > loss_dur: 0.24442 (0.17912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.39596 (25.03410) + | > current_lr: 0.00003 + | > step_time: 7.79870 (2.38792) + | > loader_time: 0.10540 (0.08680) + + + --> STEP: 192/234 -- GLOBAL_STEP: 27570 + | > loss: -0.26881 (-0.10856) + | > log_mle: -0.49522 (-0.28908) + | > loss_dur: 0.22641 (0.18052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.35616 (26.02868) + | > current_lr: 0.00003 + | > step_time: 3.99910 (2.49345) + | > loader_time: 0.00390 (0.08613) + + + --> STEP: 197/234 -- GLOBAL_STEP: 27575 + | > loss: -0.24218 (-0.11161) + | > log_mle: -0.46898 (-0.29365) + | > loss_dur: 0.22680 (0.18204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.06231 (27.09721) + | > current_lr: 0.00003 + | > step_time: 2.91180 (2.50849) + | > loader_time: 0.19190 (0.08583) + + + --> STEP: 202/234 -- GLOBAL_STEP: 27580 + | > loss: -0.31092 (-0.11448) + | > log_mle: -0.55944 (-0.29830) + | > loss_dur: 0.24853 (0.18382) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.96925 (28.23981) + | > current_lr: 0.00003 + | > step_time: 5.80090 (2.57714) + | > loader_time: 0.28210 (0.08806) + + + --> STEP: 207/234 -- GLOBAL_STEP: 27585 + | > loss: -0.27683 (-0.11748) + | > log_mle: -0.52822 (-0.30290) + | > loss_dur: 0.25139 (0.18542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 133.74759 (29.49680) + | > current_lr: 0.00003 + | > step_time: 4.89340 (2.62781) + | > loader_time: 0.00790 (0.08651) + + + --> STEP: 212/234 -- GLOBAL_STEP: 27590 + | > loss: -0.26157 (-0.12073) + | > log_mle: -0.51522 (-0.30786) + | > loss_dur: 0.25364 (0.18713) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.20180 (30.85056) + | > current_lr: 0.00003 + | > step_time: 1.99790 (2.71312) + | > loader_time: 0.00440 (0.08720) + + + --> STEP: 217/234 -- GLOBAL_STEP: 27595 + | > loss: -0.27791 (-0.12418) + | > log_mle: -0.54013 (-0.31287) + | > loss_dur: 0.26222 (0.18869) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.85689 (31.94660) + | > current_lr: 0.00003 + | > step_time: 8.20530 (2.79898) + | > loader_time: 0.00540 (0.08622) + + + --> STEP: 222/234 -- GLOBAL_STEP: 27600 + | > loss: -0.26755 (-0.12762) + | > log_mle: -0.55057 (-0.31792) + | > loss_dur: 0.28301 (0.19031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.06264 (33.00837) + | > current_lr: 0.00003 + | > step_time: 0.63630 (2.79867) + | > loader_time: 0.00290 (0.08514) + + + --> STEP: 227/234 -- GLOBAL_STEP: 27605 + | > loss: -0.24774 (-0.13120) + | > log_mle: -0.52957 (-0.32329) + | > loss_dur: 0.28183 (0.19209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.75919 (34.37902) + | > current_lr: 0.00003 + | > step_time: 0.24120 (2.74224) + | > loader_time: 0.00320 (0.08335) + + + --> STEP: 232/234 -- GLOBAL_STEP: 27610 + | > loss: -0.17681 (-0.13358) + | > log_mle: -0.71216 (-0.32978) + | > loss_dur: 0.53535 (0.19621) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 151.84889 (36.03479) + | > current_lr: 0.00003 + | > step_time: 0.32510 (2.68887) + | > loader_time: 0.00550 (0.08165) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67903 (-0.24829) + | > avg_loss: -0.16518 (-0.01478) + | > avg_log_mle: -0.40185 (-0.01640) + | > avg_loss_dur: 0.23667 (+0.00162) + + + > EPOCH: 118/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 10:08:24)  + + --> STEP: 3/234 -- GLOBAL_STEP: 27615 + | > loss: -0.03303 (-0.05271) + | > log_mle: -0.21135 (-0.20849) + | > loss_dur: 0.17832 (0.15578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.36050 (13.28918) + | > current_lr: 0.00003 + | > step_time: 4.19350 (3.83180) + | > loader_time: 0.00510 (0.73286) + + + --> STEP: 8/234 -- GLOBAL_STEP: 27620 + | > loss: -0.07137 (-0.05441) + | > log_mle: -0.22408 (-0.21176) + | > loss_dur: 0.15271 (0.15736) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.59213 (12.76468) + | > current_lr: 0.00003 + | > step_time: 3.71880 (6.01196) + | > loader_time: 0.19370 (0.31401) + + + --> STEP: 13/234 -- GLOBAL_STEP: 27625 + | > loss: -0.05645 (-0.05625) + | > log_mle: -0.20983 (-0.21220) + | > loss_dur: 0.15338 (0.15595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.78767 (11.60782) + | > current_lr: 0.00003 + | > step_time: 1.08250 (4.21541) + | > loader_time: 0.00130 (0.19390) + + + --> STEP: 18/234 -- GLOBAL_STEP: 27630 + | > loss: -0.04988 (-0.05917) + | > log_mle: -0.20710 (-0.21083) + | > loss_dur: 0.15722 (0.15167) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.75038 (10.78290) + | > current_lr: 0.00003 + | > step_time: 1.28100 (3.38422) + | > loader_time: 0.00360 (0.14067) + + + --> STEP: 23/234 -- GLOBAL_STEP: 27635 + | > loss: -0.07179 (-0.06125) + | > log_mle: -0.21281 (-0.20940) + | > loss_dur: 0.14102 (0.14814) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.93489 (9.98318) + | > current_lr: 0.00003 + | > step_time: 1.50540 (2.91904) + | > loader_time: 0.00270 (0.11048) + + + --> STEP: 28/234 -- GLOBAL_STEP: 27640 + | > loss: -0.09243 (-0.06408) + | > log_mle: -0.20181 (-0.20852) + | > loss_dur: 0.10938 (0.14444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 3.49267 (9.51048) + | > current_lr: 0.00003 + | > step_time: 2.31160 (3.08842) + | > loader_time: 0.00210 (0.09424) + + + --> STEP: 33/234 -- GLOBAL_STEP: 27645 + | > loss: -0.05488 (-0.06427) + | > log_mle: -0.19912 (-0.20869) + | > loss_dur: 0.14424 (0.14442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.11546 (9.09248) + | > current_lr: 0.00003 + | > step_time: 1.48140 (2.80422) + | > loader_time: 0.00170 (0.08277) + + + --> STEP: 38/234 -- GLOBAL_STEP: 27650 + | > loss: -0.07617 (-0.06415) + | > log_mle: -0.21758 (-0.20887) + | > loss_dur: 0.14141 (0.14472) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.92520 (9.15181) + | > current_lr: 0.00003 + | > step_time: 1.28820 (2.61320) + | > loader_time: 0.00180 (0.07216) + + + --> STEP: 43/234 -- GLOBAL_STEP: 27655 + | > loss: -0.04438 (-0.06323) + | > log_mle: -0.21569 (-0.20847) + | > loss_dur: 0.17131 (0.14524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.37594 (9.19935) + | > current_lr: 0.00003 + | > step_time: 1.52770 (2.47515) + | > loader_time: 0.00220 (0.06408) + + + --> STEP: 48/234 -- GLOBAL_STEP: 27660 + | > loss: -0.06640 (-0.06358) + | > log_mle: -0.19724 (-0.20852) + | > loss_dur: 0.13084 (0.14495) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.21942 (9.19028) + | > current_lr: 0.00003 + | > step_time: 0.99770 (2.34680) + | > loader_time: 0.00220 (0.05761) + + + --> STEP: 53/234 -- GLOBAL_STEP: 27665 + | > loss: -0.06617 (-0.06336) + | > log_mle: -0.22055 (-0.20834) + | > loss_dur: 0.15438 (0.14497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.93719 (8.89286) + | > current_lr: 0.00003 + | > step_time: 2.30220 (2.29930) + | > loader_time: 0.08300 (0.05576) + + + --> STEP: 58/234 -- GLOBAL_STEP: 27670 + | > loss: -0.07876 (-0.06326) + | > log_mle: -0.20444 (-0.20857) + | > loss_dur: 0.12568 (0.14532) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.89367 (8.71685) + | > current_lr: 0.00003 + | > step_time: 1.09110 (2.24945) + | > loader_time: 0.00270 (0.05404) + + + --> STEP: 63/234 -- GLOBAL_STEP: 27675 + | > loss: -0.04473 (-0.06376) + | > log_mle: -0.21427 (-0.21018) + | > loss_dur: 0.16954 (0.14643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.60016 (8.88357) + | > current_lr: 0.00003 + | > step_time: 1.67520 (2.19291) + | > loader_time: 0.09700 (0.05416) + + + --> STEP: 68/234 -- GLOBAL_STEP: 27680 + | > loss: -0.04227 (-0.06372) + | > log_mle: -0.20821 (-0.21024) + | > loss_dur: 0.16594 (0.14653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.05663 (8.69714) + | > current_lr: 0.00003 + | > step_time: 2.29160 (2.16337) + | > loader_time: 0.00240 (0.05189) + + + --> STEP: 73/234 -- GLOBAL_STEP: 27685 + | > loss: -0.04524 (-0.06247) + | > log_mle: -0.23033 (-0.21075) + | > loss_dur: 0.18509 (0.14828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.44021 (8.89772) + | > current_lr: 0.00003 + | > step_time: 1.50920 (2.14275) + | > loader_time: 0.08830 (0.05310) + + + --> STEP: 78/234 -- GLOBAL_STEP: 27690 + | > loss: -0.05221 (-0.06245) + | > log_mle: -0.20361 (-0.21094) + | > loss_dur: 0.15141 (0.14849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.13152 (8.98015) + | > current_lr: 0.00003 + | > step_time: 1.27240 (2.13979) + | > loader_time: 0.00230 (0.04990) + + + --> STEP: 83/234 -- GLOBAL_STEP: 27695 + | > loss: -0.05462 (-0.06264) + | > log_mle: -0.23050 (-0.21144) + | > loss_dur: 0.17589 (0.14880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.48339 (9.00701) + | > current_lr: 0.00003 + | > step_time: 1.39640 (2.10114) + | > loader_time: 0.00300 (0.04909) + + + --> STEP: 88/234 -- GLOBAL_STEP: 27700 + | > loss: -0.08780 (-0.06294) + | > log_mle: -0.26670 (-0.21254) + | > loss_dur: 0.17890 (0.14960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.71501 (9.09514) + | > current_lr: 0.00003 + | > step_time: 3.50950 (2.11940) + | > loader_time: 0.18270 (0.05038) + + + --> STEP: 93/234 -- GLOBAL_STEP: 27705 + | > loss: -0.08236 (-0.06411) + | > log_mle: -0.27847 (-0.21488) + | > loss_dur: 0.19611 (0.15078) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.61037 (9.60244) + | > current_lr: 0.00003 + | > step_time: 1.27820 (2.09235) + | > loader_time: 0.00790 (0.04993) + + + --> STEP: 98/234 -- GLOBAL_STEP: 27710 + | > loss: -0.04639 (-0.06548) + | > log_mle: -0.20920 (-0.21718) + | > loss_dur: 0.16281 (0.15170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.72122 (10.03989) + | > current_lr: 0.00003 + | > step_time: 1.80790 (2.11244) + | > loader_time: 0.00290 (0.04933) + + + --> STEP: 103/234 -- GLOBAL_STEP: 27715 + | > loss: -0.11076 (-0.06714) + | > log_mle: -0.30839 (-0.22026) + | > loss_dur: 0.19763 (0.15311) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.42822 (10.57671) + | > current_lr: 0.00003 + | > step_time: 2.60430 (2.11263) + | > loader_time: 0.08750 (0.04874) + + + --> STEP: 108/234 -- GLOBAL_STEP: 27720 + | > loss: -0.08990 (-0.06850) + | > log_mle: -0.25455 (-0.22293) + | > loss_dur: 0.16465 (0.15443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58004 (10.97649) + | > current_lr: 0.00003 + | > step_time: 2.30200 (2.11770) + | > loader_time: 0.00230 (0.04828) + + + --> STEP: 113/234 -- GLOBAL_STEP: 27725 + | > loss: -0.12033 (-0.06979) + | > log_mle: -0.30066 (-0.22620) + | > loss_dur: 0.18033 (0.15640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.53493 (11.60793) + | > current_lr: 0.00003 + | > step_time: 2.39530 (2.15950) + | > loader_time: 0.00480 (0.04629) + + + --> STEP: 118/234 -- GLOBAL_STEP: 27730 + | > loss: -0.07385 (-0.07062) + | > log_mle: -0.26832 (-0.22863) + | > loss_dur: 0.19446 (0.15800) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.10542 (12.07149) + | > current_lr: 0.00003 + | > step_time: 2.81170 (2.15720) + | > loader_time: 0.08680 (0.04757) + + + --> STEP: 123/234 -- GLOBAL_STEP: 27735 + | > loss: -0.07103 (-0.07126) + | > log_mle: -0.24179 (-0.23011) + | > loss_dur: 0.17077 (0.15885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.00130 (12.37904) + | > current_lr: 0.00003 + | > step_time: 1.15690 (2.18812) + | > loader_time: 0.00200 (0.04721) + + + --> STEP: 128/234 -- GLOBAL_STEP: 27740 + | > loss: -0.12737 (-0.07352) + | > log_mle: -0.30301 (-0.23361) + | > loss_dur: 0.17564 (0.16009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.95694 (13.01711) + | > current_lr: 0.00003 + | > step_time: 1.59120 (2.16071) + | > loader_time: 0.00250 (0.04679) + + + --> STEP: 133/234 -- GLOBAL_STEP: 27745 + | > loss: -0.13255 (-0.07573) + | > log_mle: -0.32875 (-0.23724) + | > loss_dur: 0.19620 (0.16151) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.45008 (13.64327) + | > current_lr: 0.00003 + | > step_time: 1.81230 (2.15763) + | > loader_time: 0.00320 (0.04577) + + + --> STEP: 138/234 -- GLOBAL_STEP: 27750 + | > loss: -0.09673 (-0.07753) + | > log_mle: -0.28383 (-0.24063) + | > loss_dur: 0.18709 (0.16310) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.92869 (14.17047) + | > current_lr: 0.00003 + | > step_time: 2.10560 (2.14940) + | > loader_time: 0.09010 (0.04722) + + + --> STEP: 143/234 -- GLOBAL_STEP: 27755 + | > loss: -0.16872 (-0.07980) + | > log_mle: -0.42117 (-0.24482) + | > loss_dur: 0.25245 (0.16502) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.78822 (15.17576) + | > current_lr: 0.00003 + | > step_time: 2.39790 (2.14482) + | > loader_time: 0.00400 (0.04626) + + + --> STEP: 148/234 -- GLOBAL_STEP: 27760 + | > loss: -0.15398 (-0.08264) + | > log_mle: -0.33514 (-0.24897) + | > loss_dur: 0.18117 (0.16633) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.87823 (16.06972) + | > current_lr: 0.00003 + | > step_time: 1.40950 (2.16296) + | > loader_time: 0.00580 (0.04611) + + + --> STEP: 153/234 -- GLOBAL_STEP: 27765 + | > loss: -0.25421 (-0.08622) + | > log_mle: -0.46009 (-0.25417) + | > loss_dur: 0.20588 (0.16795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.45866 (16.98721) + | > current_lr: 0.00003 + | > step_time: 2.10720 (2.16039) + | > loader_time: 0.00280 (0.04471) + + + --> STEP: 158/234 -- GLOBAL_STEP: 27770 + | > loss: -0.16857 (-0.08923) + | > log_mle: -0.39838 (-0.25877) + | > loss_dur: 0.22981 (0.16954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.09644 (18.08312) + | > current_lr: 0.00003 + | > step_time: 3.49670 (2.18008) + | > loader_time: 0.10660 (0.04458) + + + --> STEP: 163/234 -- GLOBAL_STEP: 27775 + | > loss: -0.14859 (-0.09234) + | > log_mle: -0.36333 (-0.26334) + | > loss_dur: 0.21474 (0.17100) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.94600 (19.14711) + | > current_lr: 0.00003 + | > step_time: 3.71240 (2.22929) + | > loader_time: 0.08860 (0.04733) + + + --> STEP: 168/234 -- GLOBAL_STEP: 27780 + | > loss: -0.17214 (-0.09536) + | > log_mle: -0.41803 (-0.26794) + | > loss_dur: 0.24589 (0.17258) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.60488 (20.16693) + | > current_lr: 0.00003 + | > step_time: 1.48290 (2.22470) + | > loader_time: 0.00360 (0.04719) + + + --> STEP: 173/234 -- GLOBAL_STEP: 27785 + | > loss: -0.20560 (-0.09866) + | > log_mle: -0.42733 (-0.27312) + | > loss_dur: 0.22173 (0.17445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.42673 (21.36642) + | > current_lr: 0.00003 + | > step_time: 2.48360 (2.27235) + | > loader_time: 0.00200 (0.04595) + + + --> STEP: 178/234 -- GLOBAL_STEP: 27790 + | > loss: -0.23948 (-0.10217) + | > log_mle: -0.48783 (-0.27838) + | > loss_dur: 0.24835 (0.17621) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.38662 (22.41739) + | > current_lr: 0.00003 + | > step_time: 2.11240 (2.30585) + | > loader_time: 0.07690 (0.05004) + + + --> STEP: 183/234 -- GLOBAL_STEP: 27795 + | > loss: -0.25085 (-0.10507) + | > log_mle: -0.48246 (-0.28307) + | > loss_dur: 0.23161 (0.17799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.33908 (23.71358) + | > current_lr: 0.00003 + | > step_time: 6.29680 (2.36134) + | > loader_time: 0.10900 (0.04984) + + + --> STEP: 188/234 -- GLOBAL_STEP: 27800 + | > loss: -0.23710 (-0.10782) + | > log_mle: -0.48415 (-0.28766) + | > loss_dur: 0.24705 (0.17985) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.16542 (25.03312) + | > current_lr: 0.00003 + | > step_time: 1.00830 (2.38053) + | > loader_time: 0.00300 (0.04960) + + + --> STEP: 193/234 -- GLOBAL_STEP: 27805 + | > loss: -0.25959 (-0.11096) + | > log_mle: -0.49304 (-0.29215) + | > loss_dur: 0.23345 (0.18119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.38943 (26.23089) + | > current_lr: 0.00003 + | > step_time: 2.34410 (2.37477) + | > loader_time: 0.00250 (0.04841) + + + --> STEP: 198/234 -- GLOBAL_STEP: 27810 + | > loss: -0.23625 (-0.11379) + | > log_mle: -0.48370 (-0.29650) + | > loss_dur: 0.24745 (0.18271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.54203 (27.33419) + | > current_lr: 0.00003 + | > step_time: 0.96810 (2.37557) + | > loader_time: 0.00220 (0.04863) + + + --> STEP: 203/234 -- GLOBAL_STEP: 27815 + | > loss: -0.18669 (-0.11646) + | > log_mle: -0.42201 (-0.30074) + | > loss_dur: 0.23531 (0.18428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.54313 (28.21447) + | > current_lr: 0.00003 + | > step_time: 9.60760 (2.41670) + | > loader_time: 0.10480 (0.04891) + + + --> STEP: 208/234 -- GLOBAL_STEP: 27820 + | > loss: -0.24368 (-0.11963) + | > log_mle: -0.49722 (-0.30556) + | > loss_dur: 0.25354 (0.18593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.45319 (29.37856) + | > current_lr: 0.00003 + | > step_time: 3.11220 (2.48234) + | > loader_time: 0.19200 (0.05293) + + + --> STEP: 213/234 -- GLOBAL_STEP: 27825 + | > loss: -0.28252 (-0.12326) + | > log_mle: -0.54883 (-0.31089) + | > loss_dur: 0.26631 (0.18763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.31135 (30.48514) + | > current_lr: 0.00003 + | > step_time: 3.41570 (2.49745) + | > loader_time: 0.19310 (0.05590) + + + --> STEP: 218/234 -- GLOBAL_STEP: 27830 + | > loss: -0.25362 (-0.12662) + | > log_mle: -0.51385 (-0.31582) + | > loss_dur: 0.26023 (0.18920) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.64636 (31.49463) + | > current_lr: 0.00003 + | > step_time: 8.39260 (2.57273) + | > loader_time: 0.00340 (0.05511) + + + --> STEP: 223/234 -- GLOBAL_STEP: 27835 + | > loss: -0.29844 (-0.13027) + | > log_mle: -0.55791 (-0.32111) + | > loss_dur: 0.25947 (0.19085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.92353 (32.61135) + | > current_lr: 0.00003 + | > step_time: 5.49210 (2.66178) + | > loader_time: 0.09760 (0.05658) + + + --> STEP: 228/234 -- GLOBAL_STEP: 27840 + | > loss: -0.26859 (-0.13384) + | > log_mle: -0.55770 (-0.32656) + | > loss_dur: 0.28911 (0.19271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.35553 (33.78671) + | > current_lr: 0.00003 + | > step_time: 0.26110 (2.62970) + | > loader_time: 0.00500 (0.05615) + + + --> STEP: 233/234 -- GLOBAL_STEP: 27845 + | > loss: 0.28643 (-0.13407) + | > log_mle: -0.51557 (-0.33303) + | > loss_dur: 0.80200 (0.19896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.86015 (35.50043) + | > current_lr: 0.00003 + | > step_time: 0.20870 (2.57896) + | > loader_time: 0.00330 (0.05509) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02337 (-0.65566) + | > avg_loss: -0.16494 (+0.00023) + | > avg_log_mle: -0.40163 (+0.00021) + | > avg_loss_dur: 0.23669 (+0.00002) + + + > EPOCH: 119/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 10:19:33)  + + --> STEP: 4/234 -- GLOBAL_STEP: 27850 + | > loss: -0.00635 (-0.03042) + | > log_mle: -0.20497 (-0.20771) + | > loss_dur: 0.19862 (0.17728) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.99195 (17.74960) + | > current_lr: 0.00003 + | > step_time: 7.91120 (7.75482) + | > loader_time: 0.10180 (0.05172) + + + --> STEP: 9/234 -- GLOBAL_STEP: 27855 + | > loss: -0.06222 (-0.04696) + | > log_mle: -0.22330 (-0.21344) + | > loss_dur: 0.16108 (0.16648) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.98703 (13.20966) + | > current_lr: 0.00003 + | > step_time: 6.10420 (5.57340) + | > loader_time: 0.00670 (0.07592) + + + --> STEP: 14/234 -- GLOBAL_STEP: 27860 + | > loss: -0.05458 (-0.05181) + | > log_mle: -0.21565 (-0.21298) + | > loss_dur: 0.16107 (0.16117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.15305 (11.58620) + | > current_lr: 0.00003 + | > step_time: 2.09630 (5.04051) + | > loader_time: 0.00430 (0.06368) + + + --> STEP: 19/234 -- GLOBAL_STEP: 27865 + | > loss: -0.09049 (-0.05818) + | > log_mle: -0.20332 (-0.21065) + | > loss_dur: 0.11283 (0.15248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.42694 (10.60452) + | > current_lr: 0.00003 + | > step_time: 7.08980 (5.09244) + | > loader_time: 0.10700 (0.06862) + + + --> STEP: 24/234 -- GLOBAL_STEP: 27870 + | > loss: -0.09238 (-0.06115) + | > log_mle: -0.20518 (-0.20943) + | > loss_dur: 0.11280 (0.14828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.68491 (9.94406) + | > current_lr: 0.00003 + | > step_time: 1.03100 (4.39965) + | > loader_time: 0.00320 (0.05892) + + + --> STEP: 29/234 -- GLOBAL_STEP: 27875 + | > loss: -0.05871 (-0.06258) + | > log_mle: -0.19725 (-0.20847) + | > loss_dur: 0.13854 (0.14589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.13813 (9.44687) + | > current_lr: 0.00003 + | > step_time: 1.66920 (3.94527) + | > loader_time: 0.00270 (0.04914) + + + --> STEP: 34/234 -- GLOBAL_STEP: 27880 + | > loss: -0.04634 (-0.06350) + | > log_mle: -0.20554 (-0.20902) + | > loss_dur: 0.15920 (0.14552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.61168 (8.96407) + | > current_lr: 0.00003 + | > step_time: 1.10420 (3.56346) + | > loader_time: 0.07720 (0.04685) + + + --> STEP: 39/234 -- GLOBAL_STEP: 27885 + | > loss: -0.06684 (-0.06387) + | > log_mle: -0.21532 (-0.20959) + | > loss_dur: 0.14848 (0.14572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.02133 (9.10640) + | > current_lr: 0.00003 + | > step_time: 3.80000 (3.59649) + | > loader_time: 0.08570 (0.05000) + + + --> STEP: 44/234 -- GLOBAL_STEP: 27890 + | > loss: -0.08912 (-0.06399) + | > log_mle: -0.20519 (-0.20909) + | > loss_dur: 0.11606 (0.14510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.64140 (8.81678) + | > current_lr: 0.00003 + | > step_time: 0.86690 (3.33992) + | > loader_time: 0.00180 (0.04459) + + + --> STEP: 49/234 -- GLOBAL_STEP: 27895 + | > loss: -0.08119 (-0.06438) + | > log_mle: -0.21429 (-0.20958) + | > loss_dur: 0.13310 (0.14521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.02402 (8.63897) + | > current_lr: 0.00003 + | > step_time: 1.21490 (3.16719) + | > loader_time: 0.00210 (0.04364) + + + --> STEP: 54/234 -- GLOBAL_STEP: 27900 + | > loss: -0.08241 (-0.06423) + | > log_mle: -0.22069 (-0.20956) + | > loss_dur: 0.13828 (0.14533) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.14298 (8.50031) + | > current_lr: 0.00003 + | > step_time: 1.30960 (3.01674) + | > loader_time: 0.00270 (0.03980) + + + --> STEP: 59/234 -- GLOBAL_STEP: 27905 + | > loss: -0.10020 (-0.06449) + | > log_mle: -0.23051 (-0.20992) + | > loss_dur: 0.13031 (0.14544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.88439 (8.51419) + | > current_lr: 0.00003 + | > step_time: 2.20230 (2.92589) + | > loader_time: 0.10140 (0.03966) + + + --> STEP: 64/234 -- GLOBAL_STEP: 27910 + | > loss: -0.06721 (-0.06444) + | > log_mle: -0.20267 (-0.21105) + | > loss_dur: 0.13546 (0.14661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.02581 (8.68674) + | > current_lr: 0.00003 + | > step_time: 1.40020 (2.82583) + | > loader_time: 0.00200 (0.03791) + + + --> STEP: 69/234 -- GLOBAL_STEP: 27915 + | > loss: -0.03253 (-0.06393) + | > log_mle: -0.19329 (-0.21099) + | > loss_dur: 0.16076 (0.14706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.39520 (8.56577) + | > current_lr: 0.00003 + | > step_time: 2.11200 (2.74000) + | > loader_time: 0.00180 (0.03538) + + + --> STEP: 74/234 -- GLOBAL_STEP: 27920 + | > loss: -0.06727 (-0.06336) + | > log_mle: -0.20841 (-0.21172) + | > loss_dur: 0.14114 (0.14836) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42493 (8.74571) + | > current_lr: 0.00003 + | > step_time: 2.50650 (2.68567) + | > loader_time: 0.00290 (0.03319) + + + --> STEP: 79/234 -- GLOBAL_STEP: 27925 + | > loss: -0.07226 (-0.06370) + | > log_mle: -0.22258 (-0.21237) + | > loss_dur: 0.15032 (0.14867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.81341 (8.76133) + | > current_lr: 0.00003 + | > step_time: 1.48710 (2.66105) + | > loader_time: 0.00230 (0.03131) + + + --> STEP: 84/234 -- GLOBAL_STEP: 27930 + | > loss: -0.07621 (-0.06445) + | > log_mle: -0.21850 (-0.21293) + | > loss_dur: 0.14230 (0.14849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.08385 (8.80070) + | > current_lr: 0.00003 + | > step_time: 1.64470 (2.62619) + | > loader_time: 0.00230 (0.03080) + + + --> STEP: 89/234 -- GLOBAL_STEP: 27935 + | > loss: -0.09057 (-0.06530) + | > log_mle: -0.25006 (-0.21440) + | > loss_dur: 0.15949 (0.14910) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.30581 (8.94861) + | > current_lr: 0.00003 + | > step_time: 2.61600 (2.58090) + | > loader_time: 0.00400 (0.02922) + + + --> STEP: 94/234 -- GLOBAL_STEP: 27940 + | > loss: -0.11902 (-0.06658) + | > log_mle: -0.28454 (-0.21714) + | > loss_dur: 0.16551 (0.15056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.48072 (9.47240) + | > current_lr: 0.00003 + | > step_time: 5.39720 (2.59554) + | > loader_time: 0.00320 (0.02880) + + + --> STEP: 99/234 -- GLOBAL_STEP: 27945 + | > loss: -0.12443 (-0.06797) + | > log_mle: -0.31922 (-0.21987) + | > loss_dur: 0.19479 (0.15190) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.89285 (10.07640) + | > current_lr: 0.00003 + | > step_time: 1.91170 (2.58173) + | > loader_time: 0.08340 (0.03192) + + + --> STEP: 104/234 -- GLOBAL_STEP: 27950 + | > loss: -0.15039 (-0.07000) + | > log_mle: -0.33311 (-0.22316) + | > loss_dur: 0.18272 (0.15315) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.91456 (10.54859) + | > current_lr: 0.00003 + | > step_time: 2.31030 (2.54161) + | > loader_time: 0.09110 (0.03208) + + + --> STEP: 109/234 -- GLOBAL_STEP: 27955 + | > loss: -0.07654 (-0.07078) + | > log_mle: -0.30288 (-0.22557) + | > loss_dur: 0.22634 (0.15479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.87305 (10.94967) + | > current_lr: 0.00003 + | > step_time: 3.60540 (2.54429) + | > loader_time: 0.00270 (0.03238) + + + --> STEP: 114/234 -- GLOBAL_STEP: 27960 + | > loss: -0.10847 (-0.07250) + | > log_mle: -0.28299 (-0.22878) + | > loss_dur: 0.17452 (0.15628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.54815 (11.50242) + | > current_lr: 0.00003 + | > step_time: 0.89920 (2.51272) + | > loader_time: 0.00290 (0.03265) + + + --> STEP: 119/234 -- GLOBAL_STEP: 27965 + | > loss: -0.09160 (-0.07329) + | > log_mle: -0.27974 (-0.23119) + | > loss_dur: 0.18814 (0.15790) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.61063 (11.97378) + | > current_lr: 0.00003 + | > step_time: 2.48910 (2.50786) + | > loader_time: 0.00400 (0.03303) + + + --> STEP: 124/234 -- GLOBAL_STEP: 27970 + | > loss: -0.13621 (-0.07441) + | > log_mle: -0.31171 (-0.23293) + | > loss_dur: 0.17550 (0.15851) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.96899 (12.23968) + | > current_lr: 0.00003 + | > step_time: 1.39410 (2.50837) + | > loader_time: 0.00880 (0.03270) + + + --> STEP: 129/234 -- GLOBAL_STEP: 27975 + | > loss: -0.10050 (-0.07638) + | > log_mle: -0.29796 (-0.23628) + | > loss_dur: 0.19746 (0.15990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.30396 (12.92924) + | > current_lr: 0.00003 + | > step_time: 1.29890 (2.48974) + | > loader_time: 0.00240 (0.03295) + + + --> STEP: 134/234 -- GLOBAL_STEP: 27980 + | > loss: -0.12255 (-0.07874) + | > log_mle: -0.35098 (-0.24022) + | > loss_dur: 0.22844 (0.16147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.74482 (13.72399) + | > current_lr: 0.00003 + | > step_time: 1.90510 (2.46179) + | > loader_time: 0.08740 (0.03385) + + + --> STEP: 139/234 -- GLOBAL_STEP: 27985 + | > loss: -0.19671 (-0.08076) + | > log_mle: -0.40478 (-0.24382) + | > loss_dur: 0.20807 (0.16306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.26680 (14.60218) + | > current_lr: 0.00003 + | > step_time: 0.96660 (2.43297) + | > loader_time: 0.00200 (0.03464) + + + --> STEP: 144/234 -- GLOBAL_STEP: 27990 + | > loss: -0.15166 (-0.08249) + | > log_mle: -0.37552 (-0.24755) + | > loss_dur: 0.22387 (0.16506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.99883 (15.79176) + | > current_lr: 0.00003 + | > step_time: 1.88610 (2.45006) + | > loader_time: 0.00270 (0.03479) + + + --> STEP: 149/234 -- GLOBAL_STEP: 27995 + | > loss: -0.20356 (-0.08530) + | > log_mle: -0.42763 (-0.25186) + | > loss_dur: 0.22408 (0.16656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.32205 (16.73999) + | > current_lr: 0.00003 + | > step_time: 1.80290 (2.43766) + | > loader_time: 0.00240 (0.03494) + + + --> STEP: 154/234 -- GLOBAL_STEP: 28000 + | > loss: -0.19097 (-0.08867) + | > log_mle: -0.39413 (-0.25670) + | > loss_dur: 0.20316 (0.16803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.88326 (17.75184) + | > current_lr: 0.00003 + | > step_time: 3.20610 (2.44035) + | > loader_time: 0.00300 (0.03496) + + + --> STEP: 159/234 -- GLOBAL_STEP: 28005 + | > loss: -0.19438 (-0.09170) + | > log_mle: -0.41062 (-0.26138) + | > loss_dur: 0.21625 (0.16968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.65238 (18.71984) + | > current_lr: 0.00003 + | > step_time: 1.80050 (2.48762) + | > loader_time: 0.07820 (0.03913) + + + --> STEP: 164/234 -- GLOBAL_STEP: 28010 + | > loss: -0.18051 (-0.09475) + | > log_mle: -0.40742 (-0.26587) + | > loss_dur: 0.22691 (0.17112) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.14156 (19.52896) + | > current_lr: 0.00003 + | > step_time: 3.31830 (2.51568) + | > loader_time: 0.18690 (0.04137) + + + --> STEP: 169/234 -- GLOBAL_STEP: 28015 + | > loss: -0.17216 (-0.09773) + | > log_mle: -0.40238 (-0.27043) + | > loss_dur: 0.23022 (0.17270) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.29240 (20.55597) + | > current_lr: 0.00003 + | > step_time: 1.90140 (2.50172) + | > loader_time: 0.07590 (0.04123) + + + --> STEP: 174/234 -- GLOBAL_STEP: 28020 + | > loss: -0.27296 (-0.10173) + | > log_mle: -0.49492 (-0.27613) + | > loss_dur: 0.22196 (0.17441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.75680 (21.81178) + | > current_lr: 0.00003 + | > step_time: 2.00570 (2.49202) + | > loader_time: 0.00470 (0.04171) + + + --> STEP: 179/234 -- GLOBAL_STEP: 28025 + | > loss: -0.22882 (-0.10489) + | > log_mle: -0.48375 (-0.28127) + | > loss_dur: 0.25493 (0.17638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.09314 (22.78535) + | > current_lr: 0.00003 + | > step_time: 3.90090 (2.50897) + | > loader_time: 0.18970 (0.04225) + + + --> STEP: 184/234 -- GLOBAL_STEP: 28030 + | > loss: -0.21279 (-0.10779) + | > log_mle: -0.45090 (-0.28585) + | > loss_dur: 0.23811 (0.17807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.52728 (23.80264) + | > current_lr: 0.00003 + | > step_time: 5.08670 (2.56801) + | > loader_time: 0.00380 (0.04376) + + + --> STEP: 189/234 -- GLOBAL_STEP: 28035 + | > loss: -0.20213 (-0.11071) + | > log_mle: -0.44329 (-0.29056) + | > loss_dur: 0.24117 (0.17986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.25390 (24.86398) + | > current_lr: 0.00003 + | > step_time: 4.09660 (2.63399) + | > loader_time: 0.10050 (0.04569) + + + --> STEP: 194/234 -- GLOBAL_STEP: 28040 + | > loss: -0.24562 (-0.11422) + | > log_mle: -0.47627 (-0.29527) + | > loss_dur: 0.23065 (0.18106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.49841 (25.99116) + | > current_lr: 0.00003 + | > step_time: 3.18070 (2.68356) + | > loader_time: 0.00510 (0.04904) + + + --> STEP: 199/234 -- GLOBAL_STEP: 28045 + | > loss: -0.25156 (-0.11716) + | > log_mle: -0.48925 (-0.29969) + | > loss_dur: 0.23768 (0.18252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.48907 (26.94124) + | > current_lr: 0.00003 + | > step_time: 4.69200 (2.76079) + | > loader_time: 0.20040 (0.05125) + + + --> STEP: 204/234 -- GLOBAL_STEP: 28050 + | > loss: -0.24958 (-0.11978) + | > log_mle: -0.51892 (-0.30400) + | > loss_dur: 0.26935 (0.18422) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.19273 (27.96379) + | > current_lr: 0.00003 + | > step_time: 3.61190 (2.82954) + | > loader_time: 0.00490 (0.05297) + + + --> STEP: 209/234 -- GLOBAL_STEP: 28055 + | > loss: -0.22165 (-0.12280) + | > log_mle: -0.46951 (-0.30862) + | > loss_dur: 0.24786 (0.18581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 130.53415 (29.32448) + | > current_lr: 0.00003 + | > step_time: 3.69230 (2.88717) + | > loader_time: 0.00710 (0.05312) + + + --> STEP: 214/234 -- GLOBAL_STEP: 28060 + | > loss: -0.27024 (-0.12627) + | > log_mle: -0.50751 (-0.31379) + | > loss_dur: 0.23728 (0.18752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.95853 (30.81879) + | > current_lr: 0.00003 + | > step_time: 5.00520 (2.96749) + | > loader_time: 0.00480 (0.05364) + + + --> STEP: 219/234 -- GLOBAL_STEP: 28065 + | > loss: -0.34740 (-0.12981) + | > log_mle: -0.60753 (-0.31901) + | > loss_dur: 0.26013 (0.18920) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.18266 (31.99906) + | > current_lr: 0.00003 + | > step_time: 4.51370 (3.02577) + | > loader_time: 0.08920 (0.05748) + + + --> STEP: 224/234 -- GLOBAL_STEP: 28070 + | > loss: -0.28936 (-0.13304) + | > log_mle: -0.55308 (-0.32388) + | > loss_dur: 0.26371 (0.19084) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.67066 (33.23676) + | > current_lr: 0.00003 + | > step_time: 0.22940 (2.97705) + | > loader_time: 0.00440 (0.05672) + + + --> STEP: 229/234 -- GLOBAL_STEP: 28075 + | > loss: -0.26675 (-0.13629) + | > log_mle: -0.58700 (-0.32919) + | > loss_dur: 0.32025 (0.19290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.50874 (34.63193) + | > current_lr: 0.00003 + | > step_time: 0.26230 (2.91743) + | > loader_time: 0.00530 (0.05556) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02905 (+0.00568) + | > avg_loss: -0.18822 (-0.02328) + | > avg_log_mle: -0.42243 (-0.02080) + | > avg_loss_dur: 0.23421 (-0.00248) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_28080.pth + + > EPOCH: 120/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 10:32:09)  + + --> STEP: 0/234 -- GLOBAL_STEP: 28080 + | > loss: -0.13154 (-0.13154) + | > log_mle: -0.28217 (-0.28217) + | > loss_dur: 0.15064 (0.15064) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.10529 (9.10529) + | > current_lr: 0.00003 + | > step_time: 18.20890 (18.20889) + | > loader_time: 24.76530 (24.76532) + + + --> STEP: 5/234 -- GLOBAL_STEP: 28085 + | > loss: -0.05820 (-0.03234) + | > log_mle: -0.21184 (-0.20953) + | > loss_dur: 0.15363 (0.17720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.67617 (22.32203) + | > current_lr: 0.00003 + | > step_time: 0.80360 (2.70481) + | > loader_time: 0.00180 (0.03920) + + + --> STEP: 10/234 -- GLOBAL_STEP: 28090 + | > loss: -0.05472 (-0.04864) + | > log_mle: -0.21002 (-0.21316) + | > loss_dur: 0.15530 (0.16452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.59471 (16.86925) + | > current_lr: 0.00003 + | > step_time: 1.09360 (1.75517) + | > loader_time: 0.00120 (0.02690) + + + --> STEP: 15/234 -- GLOBAL_STEP: 28095 + | > loss: -0.08788 (-0.05672) + | > log_mle: -0.21430 (-0.21320) + | > loss_dur: 0.12642 (0.15648) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.65182 (13.82321) + | > current_lr: 0.00003 + | > step_time: 0.87700 (1.91506) + | > loader_time: 0.00120 (0.03187) + + + --> STEP: 20/234 -- GLOBAL_STEP: 28100 + | > loss: -0.07121 (-0.05938) + | > log_mle: -0.20327 (-0.21094) + | > loss_dur: 0.13206 (0.15156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.82030 (11.91552) + | > current_lr: 0.00003 + | > step_time: 1.29220 (1.74788) + | > loader_time: 0.00170 (0.02845) + + + --> STEP: 25/234 -- GLOBAL_STEP: 28105 + | > loss: -0.04800 (-0.06233) + | > log_mle: -0.19519 (-0.20985) + | > loss_dur: 0.14719 (0.14752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.46449 (10.88983) + | > current_lr: 0.00003 + | > step_time: 1.25650 (1.61914) + | > loader_time: 0.00180 (0.02316) + + + --> STEP: 30/234 -- GLOBAL_STEP: 28110 + | > loss: -0.09730 (-0.06524) + | > log_mle: -0.22109 (-0.21001) + | > loss_dur: 0.12380 (0.14477) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.83009 (10.23646) + | > current_lr: 0.00003 + | > step_time: 1.30180 (1.95235) + | > loader_time: 0.00360 (0.02276) + + + --> STEP: 35/234 -- GLOBAL_STEP: 28115 + | > loss: -0.05077 (-0.06462) + | > log_mle: -0.21492 (-0.21034) + | > loss_dur: 0.16415 (0.14572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.73534 (10.09117) + | > current_lr: 0.00003 + | > step_time: 2.40240 (1.97349) + | > loader_time: 0.00220 (0.02477) + + + --> STEP: 40/234 -- GLOBAL_STEP: 28120 + | > loss: -0.03632 (-0.06486) + | > log_mle: -0.19620 (-0.21042) + | > loss_dur: 0.15988 (0.14556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.90772 (9.86164) + | > current_lr: 0.00003 + | > step_time: 1.28210 (1.94363) + | > loader_time: 0.00350 (0.02386) + + + --> STEP: 45/234 -- GLOBAL_STEP: 28125 + | > loss: -0.08619 (-0.06571) + | > log_mle: -0.23507 (-0.21084) + | > loss_dur: 0.14888 (0.14512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.95171 (9.73322) + | > current_lr: 0.00003 + | > step_time: 1.39440 (1.86479) + | > loader_time: 0.00280 (0.02148) + + + --> STEP: 50/234 -- GLOBAL_STEP: 28130 + | > loss: -0.05962 (-0.06588) + | > log_mle: -0.20197 (-0.21048) + | > loss_dur: 0.14235 (0.14461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.19181 (9.52220) + | > current_lr: 0.00003 + | > step_time: 1.69050 (1.88464) + | > loader_time: 0.00650 (0.02292) + + + --> STEP: 55/234 -- GLOBAL_STEP: 28135 + | > loss: -0.08503 (-0.06618) + | > log_mle: -0.22250 (-0.21090) + | > loss_dur: 0.13747 (0.14472) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.47784 (9.26074) + | > current_lr: 0.00003 + | > step_time: 1.38880 (1.86026) + | > loader_time: 0.00220 (0.02250) + + + --> STEP: 60/234 -- GLOBAL_STEP: 28140 + | > loss: -0.08795 (-0.06665) + | > log_mle: -0.23872 (-0.21163) + | > loss_dur: 0.15077 (0.14498) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.66258 (9.16649) + | > current_lr: 0.00003 + | > step_time: 3.64610 (1.87638) + | > loader_time: 0.00280 (0.02085) + + + --> STEP: 65/234 -- GLOBAL_STEP: 28145 + | > loss: -0.07885 (-0.06656) + | > log_mle: -0.21421 (-0.21234) + | > loss_dur: 0.13536 (0.14578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.98320 (9.16959) + | > current_lr: 0.00003 + | > step_time: 1.48450 (1.85579) + | > loader_time: 0.00230 (0.02070) + + + --> STEP: 70/234 -- GLOBAL_STEP: 28150 + | > loss: -0.04519 (-0.06555) + | > log_mle: -0.21162 (-0.21229) + | > loss_dur: 0.16642 (0.14674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.06926 (9.12252) + | > current_lr: 0.00003 + | > step_time: 2.33320 (1.86815) + | > loader_time: 0.00250 (0.02183) + + + --> STEP: 75/234 -- GLOBAL_STEP: 28155 + | > loss: -0.05747 (-0.06513) + | > log_mle: -0.22561 (-0.21318) + | > loss_dur: 0.16814 (0.14805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.72935 (9.25227) + | > current_lr: 0.00003 + | > step_time: 1.41250 (1.85687) + | > loader_time: 0.00260 (0.02055) + + + --> STEP: 80/234 -- GLOBAL_STEP: 28160 + | > loss: -0.07761 (-0.06534) + | > log_mle: -0.20583 (-0.21351) + | > loss_dur: 0.12821 (0.14817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.93304 (9.14330) + | > current_lr: 0.00003 + | > step_time: 1.68170 (1.87102) + | > loader_time: 0.00230 (0.02064) + + + --> STEP: 85/234 -- GLOBAL_STEP: 28165 + | > loss: -0.08508 (-0.06564) + | > log_mle: -0.22441 (-0.21428) + | > loss_dur: 0.13933 (0.14864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.80389 (9.20517) + | > current_lr: 0.00003 + | > step_time: 3.70090 (1.86980) + | > loader_time: 0.09750 (0.02068) + + + --> STEP: 90/234 -- GLOBAL_STEP: 28170 + | > loss: -0.07812 (-0.06653) + | > log_mle: -0.25148 (-0.21614) + | > loss_dur: 0.17336 (0.14961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.57569 (9.53618) + | > current_lr: 0.00003 + | > step_time: 1.31460 (1.84818) + | > loader_time: 0.08770 (0.02158) + + + --> STEP: 95/234 -- GLOBAL_STEP: 28175 + | > loss: -0.15028 (-0.06883) + | > log_mle: -0.34003 (-0.21981) + | > loss_dur: 0.18974 (0.15098) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.69231 (10.20110) + | > current_lr: 0.00003 + | > step_time: 2.30660 (1.88885) + | > loader_time: 0.08720 (0.02340) + + + --> STEP: 100/234 -- GLOBAL_STEP: 28180 + | > loss: -0.10259 (-0.06989) + | > log_mle: -0.26343 (-0.22174) + | > loss_dur: 0.16084 (0.15185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.82511 (10.52614) + | > current_lr: 0.00003 + | > step_time: 1.17340 (1.90108) + | > loader_time: 0.00220 (0.02242) + + + --> STEP: 105/234 -- GLOBAL_STEP: 28185 + | > loss: -0.08115 (-0.07155) + | > log_mle: -0.23926 (-0.22474) + | > loss_dur: 0.15811 (0.15319) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.22968 (10.95142) + | > current_lr: 0.00003 + | > step_time: 2.31420 (1.91079) + | > loader_time: 0.06510 (0.02293) + + + --> STEP: 110/234 -- GLOBAL_STEP: 28190 + | > loss: -0.10063 (-0.07242) + | > log_mle: -0.26718 (-0.22737) + | > loss_dur: 0.16654 (0.15495) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.47811 (11.43625) + | > current_lr: 0.00003 + | > step_time: 1.59270 (1.91830) + | > loader_time: 0.00250 (0.02283) + + + --> STEP: 115/234 -- GLOBAL_STEP: 28195 + | > loss: -0.07875 (-0.07395) + | > log_mle: -0.28639 (-0.23070) + | > loss_dur: 0.20764 (0.15675) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.76235 (12.05621) + | > current_lr: 0.00003 + | > step_time: 2.31300 (1.93146) + | > loader_time: 0.09480 (0.02349) + + + --> STEP: 120/234 -- GLOBAL_STEP: 28200 + | > loss: -0.14918 (-0.07521) + | > log_mle: -0.33429 (-0.23348) + | > loss_dur: 0.18511 (0.15827) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.85593 (12.61334) + | > current_lr: 0.00003 + | > step_time: 1.91210 (1.93608) + | > loader_time: 0.00380 (0.02478) + + + --> STEP: 125/234 -- GLOBAL_STEP: 28205 + | > loss: -0.12671 (-0.07612) + | > log_mle: -0.31947 (-0.23504) + | > loss_dur: 0.19276 (0.15893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.06781 (12.98972) + | > current_lr: 0.00003 + | > step_time: 2.00350 (1.95724) + | > loader_time: 0.00310 (0.02608) + + + --> STEP: 130/234 -- GLOBAL_STEP: 28210 + | > loss: -0.13375 (-0.07808) + | > log_mle: -0.33147 (-0.23827) + | > loss_dur: 0.19772 (0.16019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.82261 (13.86406) + | > current_lr: 0.00003 + | > step_time: 1.66190 (1.95662) + | > loader_time: 0.00230 (0.02590) + + + --> STEP: 135/234 -- GLOBAL_STEP: 28215 + | > loss: -0.09039 (-0.08010) + | > log_mle: -0.26354 (-0.24158) + | > loss_dur: 0.17315 (0.16148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.38509 (14.51875) + | > current_lr: 0.00003 + | > step_time: 3.40890 (1.95915) + | > loader_time: 0.09000 (0.02632) + + + --> STEP: 140/234 -- GLOBAL_STEP: 28220 + | > loss: -0.08162 (-0.08226) + | > log_mle: -0.29443 (-0.24545) + | > loss_dur: 0.21281 (0.16319) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.16454 (15.29782) + | > current_lr: 0.00003 + | > step_time: 3.21300 (2.00487) + | > loader_time: 0.08610 (0.02743) + + + --> STEP: 145/234 -- GLOBAL_STEP: 28225 + | > loss: -0.18087 (-0.08483) + | > log_mle: -0.39409 (-0.24995) + | > loss_dur: 0.21322 (0.16513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.47779 (16.32805) + | > current_lr: 0.00003 + | > step_time: 1.00350 (1.98890) + | > loader_time: 0.08530 (0.02845) + + + --> STEP: 150/234 -- GLOBAL_STEP: 28230 + | > loss: -0.16464 (-0.08767) + | > log_mle: -0.38049 (-0.25420) + | > loss_dur: 0.21585 (0.16653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.71079 (17.12835) + | > current_lr: 0.00003 + | > step_time: 2.59540 (2.01604) + | > loader_time: 0.00270 (0.02816) + + + --> STEP: 155/234 -- GLOBAL_STEP: 28235 + | > loss: -0.22219 (-0.09132) + | > log_mle: -0.44606 (-0.25944) + | > loss_dur: 0.22387 (0.16811) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.35945 (18.38585) + | > current_lr: 0.00003 + | > step_time: 2.98540 (2.01026) + | > loader_time: 0.00350 (0.02851) + + + --> STEP: 160/234 -- GLOBAL_STEP: 28240 + | > loss: -0.20920 (-0.09420) + | > log_mle: -0.43794 (-0.26394) + | > loss_dur: 0.22874 (0.16974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.92773 (19.50804) + | > current_lr: 0.00003 + | > step_time: 1.39880 (2.01169) + | > loader_time: 0.08720 (0.02944) + + + --> STEP: 165/234 -- GLOBAL_STEP: 28245 + | > loss: -0.20127 (-0.09714) + | > log_mle: -0.43340 (-0.26825) + | > loss_dur: 0.23213 (0.17111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.80221 (20.47670) + | > current_lr: 0.00003 + | > step_time: 4.51130 (2.04860) + | > loader_time: 0.10240 (0.03025) + + + --> STEP: 170/234 -- GLOBAL_STEP: 28250 + | > loss: -0.22193 (-0.10025) + | > log_mle: -0.47523 (-0.27301) + | > loss_dur: 0.25329 (0.17276) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.67088 (21.49533) + | > current_lr: 0.00003 + | > step_time: 2.89190 (2.05293) + | > loader_time: 0.00670 (0.02947) + + + --> STEP: 175/234 -- GLOBAL_STEP: 28255 + | > loss: -0.19777 (-0.10393) + | > log_mle: -0.45113 (-0.27848) + | > loss_dur: 0.25337 (0.17455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.45781 (22.80706) + | > current_lr: 0.00003 + | > step_time: 2.49230 (2.06060) + | > loader_time: 0.00350 (0.02922) + + + --> STEP: 180/234 -- GLOBAL_STEP: 28260 + | > loss: -0.22048 (-0.10716) + | > log_mle: -0.45847 (-0.28359) + | > loss_dur: 0.23799 (0.17643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.81577 (23.93568) + | > current_lr: 0.00003 + | > step_time: 4.61190 (2.07127) + | > loader_time: 0.08510 (0.02939) + + + --> STEP: 185/234 -- GLOBAL_STEP: 28265 + | > loss: -0.23479 (-0.11017) + | > log_mle: -0.49230 (-0.28837) + | > loss_dur: 0.25751 (0.17820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.43496 (25.00561) + | > current_lr: 0.00003 + | > step_time: 3.29910 (2.10566) + | > loader_time: 0.19510 (0.03075) + + + --> STEP: 190/234 -- GLOBAL_STEP: 28270 + | > loss: -0.23158 (-0.11308) + | > log_mle: -0.45688 (-0.29291) + | > loss_dur: 0.22530 (0.17983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.41813 (26.07481) + | > current_lr: 0.00003 + | > step_time: 6.81170 (2.15714) + | > loader_time: 0.20190 (0.03261) + + + --> STEP: 195/234 -- GLOBAL_STEP: 28275 + | > loss: -0.22884 (-0.11645) + | > log_mle: -0.47758 (-0.29768) + | > loss_dur: 0.24874 (0.18123) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.40095 (27.06608) + | > current_lr: 0.00003 + | > step_time: 6.09700 (2.22999) + | > loader_time: 0.10130 (0.03337) + + + --> STEP: 200/234 -- GLOBAL_STEP: 28280 + | > loss: -0.20283 (-0.11916) + | > log_mle: -0.46777 (-0.30200) + | > loss_dur: 0.26495 (0.18284) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.91792 (28.39617) + | > current_lr: 0.00003 + | > step_time: 2.78780 (2.22413) + | > loader_time: 0.00380 (0.03343) + + + --> STEP: 205/234 -- GLOBAL_STEP: 28285 + | > loss: -0.22731 (-0.12165) + | > log_mle: -0.46765 (-0.30609) + | > loss_dur: 0.24034 (0.18443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.12862 (29.64207) + | > current_lr: 0.00003 + | > step_time: 7.10140 (2.27331) + | > loader_time: 0.00420 (0.03366) + + + --> STEP: 210/234 -- GLOBAL_STEP: 28290 + | > loss: -0.27987 (-0.12482) + | > log_mle: -0.53833 (-0.31095) + | > loss_dur: 0.25846 (0.18612) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.75159 (30.83132) + | > current_lr: 0.00003 + | > step_time: 5.32080 (2.32887) + | > loader_time: 0.18760 (0.03552) + + + --> STEP: 215/234 -- GLOBAL_STEP: 28295 + | > loss: -0.25160 (-0.12834) + | > log_mle: -0.49213 (-0.31602) + | > loss_dur: 0.24053 (0.18768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.59998 (31.96147) + | > current_lr: 0.00003 + | > step_time: 2.70380 (2.40799) + | > loader_time: 0.09580 (0.03658) + + + --> STEP: 220/234 -- GLOBAL_STEP: 28300 + | > loss: -0.29932 (-0.13215) + | > log_mle: -0.55178 (-0.32148) + | > loss_dur: 0.25246 (0.18934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.77718 (33.12167) + | > current_lr: 0.00003 + | > step_time: 3.90290 (2.42457) + | > loader_time: 0.09560 (0.03629) + + + --> STEP: 225/234 -- GLOBAL_STEP: 28305 + | > loss: -0.33626 (-0.13552) + | > log_mle: -0.61096 (-0.32654) + | > loss_dur: 0.27470 (0.19102) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.87153 (34.44579) + | > current_lr: 0.00003 + | > step_time: 0.25340 (2.38007) + | > loader_time: 0.00600 (0.03598) + + + --> STEP: 230/234 -- GLOBAL_STEP: 28310 + | > loss: -0.30873 (-0.13868) + | > log_mle: -0.65666 (-0.33210) + | > loss_dur: 0.34793 (0.19342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.38749 (35.58768) + | > current_lr: 0.00003 + | > step_time: 0.27970 (2.33405) + | > loader_time: 0.00470 (0.03527) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.57760 (+0.54855) + | > avg_loss: -0.18255 (+0.00567) + | > avg_log_mle: -0.41536 (+0.00707) + | > avg_loss_dur: 0.23282 (-0.00140) + + + > EPOCH: 121/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 10:42:38)  + + --> STEP: 1/234 -- GLOBAL_STEP: 28315 + | > loss: -0.08130 (-0.08130) + | > log_mle: -0.21628 (-0.21628) + | > loss_dur: 0.13498 (0.13498) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.93088 (13.93088) + | > current_lr: 0.00003 + | > step_time: 7.70570 (7.70569) + | > loader_time: 0.10040 (0.10045) + + + --> STEP: 6/234 -- GLOBAL_STEP: 28320 + | > loss: -0.06941 (-0.05050) + | > log_mle: -0.20611 (-0.21167) + | > loss_dur: 0.13670 (0.16116) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.04778 (14.14253) + | > current_lr: 0.00003 + | > step_time: 3.19170 (6.71884) + | > loader_time: 0.00490 (0.04939) + + + --> STEP: 11/234 -- GLOBAL_STEP: 28325 + | > loss: -0.08951 (-0.06101) + | > log_mle: -0.21615 (-0.21671) + | > loss_dur: 0.12664 (0.15569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.52019 (12.08361) + | > current_lr: 0.00003 + | > step_time: 6.20130 (5.08313) + | > loader_time: 0.00430 (0.04459) + + + --> STEP: 16/234 -- GLOBAL_STEP: 28330 + | > loss: -0.10536 (-0.06662) + | > log_mle: -0.21590 (-0.21689) + | > loss_dur: 0.11054 (0.15027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.78306 (10.97768) + | > current_lr: 0.00003 + | > step_time: 2.29880 (5.11292) + | > loader_time: 0.10100 (0.09778) + + + --> STEP: 21/234 -- GLOBAL_STEP: 28335 + | > loss: -0.04769 (-0.06470) + | > log_mle: -0.19578 (-0.21356) + | > loss_dur: 0.14809 (0.14887) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.15092 (10.21396) + | > current_lr: 0.00003 + | > step_time: 1.28620 (4.13601) + | > loader_time: 0.00180 (0.07489) + + + --> STEP: 26/234 -- GLOBAL_STEP: 28340 + | > loss: -0.06743 (-0.06812) + | > log_mle: -0.21463 (-0.21326) + | > loss_dur: 0.14720 (0.14514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.83930 (9.53216) + | > current_lr: 0.00003 + | > step_time: 2.32620 (3.75364) + | > loader_time: 0.00220 (0.06696) + + + --> STEP: 31/234 -- GLOBAL_STEP: 28345 + | > loss: -0.03907 (-0.06995) + | > log_mle: -0.21419 (-0.21336) + | > loss_dur: 0.17512 (0.14341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.27704 (9.15663) + | > current_lr: 0.00003 + | > step_time: 1.90620 (3.37286) + | > loader_time: 0.07790 (0.06215) + + + --> STEP: 36/234 -- GLOBAL_STEP: 28350 + | > loss: -0.06570 (-0.07082) + | > log_mle: -0.21744 (-0.21368) + | > loss_dur: 0.15174 (0.14286) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81345 (9.26891) + | > current_lr: 0.00003 + | > step_time: 1.30610 (3.15132) + | > loader_time: 0.00250 (0.05588) + + + --> STEP: 41/234 -- GLOBAL_STEP: 28355 + | > loss: -0.07611 (-0.07084) + | > log_mle: -0.21365 (-0.21363) + | > loss_dur: 0.13755 (0.14279) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.36538 (9.00932) + | > current_lr: 0.00003 + | > step_time: 1.21500 (2.90873) + | > loader_time: 0.00210 (0.04936) + + + --> STEP: 46/234 -- GLOBAL_STEP: 28360 + | > loss: -0.06704 (-0.07096) + | > log_mle: -0.21435 (-0.21394) + | > loss_dur: 0.14730 (0.14298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.33898 (8.88278) + | > current_lr: 0.00003 + | > step_time: 1.33290 (2.75011) + | > loader_time: 0.08310 (0.04778) + + + --> STEP: 51/234 -- GLOBAL_STEP: 28365 + | > loss: -0.06142 (-0.07038) + | > log_mle: -0.20127 (-0.21332) + | > loss_dur: 0.13985 (0.14294) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.29067 (8.53662) + | > current_lr: 0.00003 + | > step_time: 2.00790 (2.64732) + | > loader_time: 0.00230 (0.04524) + + + --> STEP: 56/234 -- GLOBAL_STEP: 28370 + | > loss: -0.03603 (-0.07036) + | > log_mle: -0.21936 (-0.21391) + | > loss_dur: 0.18333 (0.14355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.25874 (8.42398) + | > current_lr: 0.00003 + | > step_time: 1.30300 (2.56794) + | > loader_time: 0.00260 (0.04293) + + + --> STEP: 61/234 -- GLOBAL_STEP: 28375 + | > loss: -0.07689 (-0.07087) + | > log_mle: -0.21669 (-0.21444) + | > loss_dur: 0.13980 (0.14357) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.44417 (8.40802) + | > current_lr: 0.00003 + | > step_time: 1.56140 (2.50145) + | > loader_time: 0.00170 (0.04236) + + + --> STEP: 66/234 -- GLOBAL_STEP: 28380 + | > loss: -0.06587 (-0.07048) + | > log_mle: -0.20388 (-0.21475) + | > loss_dur: 0.13801 (0.14427) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.26952 (8.49502) + | > current_lr: 0.00003 + | > step_time: 1.48690 (2.42368) + | > loader_time: 0.00280 (0.03936) + + + --> STEP: 71/234 -- GLOBAL_STEP: 28385 + | > loss: -0.05456 (-0.06941) + | > log_mle: -0.24315 (-0.21504) + | > loss_dur: 0.18859 (0.14563) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.41587 (8.75449) + | > current_lr: 0.00003 + | > step_time: 2.31400 (2.40646) + | > loader_time: 0.08600 (0.04031) + + + --> STEP: 76/234 -- GLOBAL_STEP: 28390 + | > loss: -0.07681 (-0.06900) + | > log_mle: -0.22974 (-0.21550) + | > loss_dur: 0.15293 (0.14650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.05378 (8.77792) + | > current_lr: 0.00003 + | > step_time: 4.78930 (2.39601) + | > loader_time: 0.00280 (0.03781) + + + --> STEP: 81/234 -- GLOBAL_STEP: 28395 + | > loss: -0.08804 (-0.06932) + | > log_mle: -0.23661 (-0.21581) + | > loss_dur: 0.14857 (0.14649) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.66139 (8.73639) + | > current_lr: 0.00003 + | > step_time: 2.19940 (2.35054) + | > loader_time: 0.00260 (0.03670) + + + --> STEP: 86/234 -- GLOBAL_STEP: 28400 + | > loss: -0.07184 (-0.06942) + | > log_mle: -0.23257 (-0.21641) + | > loss_dur: 0.16073 (0.14699) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.99336 (8.93028) + | > current_lr: 0.00003 + | > step_time: 1.00410 (2.30213) + | > loader_time: 0.09190 (0.03665) + + + --> STEP: 91/234 -- GLOBAL_STEP: 28405 + | > loss: -0.07553 (-0.06995) + | > log_mle: -0.24623 (-0.21818) + | > loss_dur: 0.17070 (0.14822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.83844 (9.27938) + | > current_lr: 0.00003 + | > step_time: 2.00170 (2.31287) + | > loader_time: 0.00350 (0.03487) + + + --> STEP: 96/234 -- GLOBAL_STEP: 28410 + | > loss: -0.06536 (-0.07173) + | > log_mle: -0.23323 (-0.22143) + | > loss_dur: 0.16788 (0.14970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.86293 (10.14027) + | > current_lr: 0.00003 + | > step_time: 2.39770 (2.34056) + | > loader_time: 0.09540 (0.03589) + + + --> STEP: 101/234 -- GLOBAL_STEP: 28415 + | > loss: -0.10780 (-0.07286) + | > log_mle: -0.29380 (-0.22379) + | > loss_dur: 0.18600 (0.15093) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.21337 (10.60193) + | > current_lr: 0.00003 + | > step_time: 1.21350 (2.36044) + | > loader_time: 0.00280 (0.03707) + + + --> STEP: 106/234 -- GLOBAL_STEP: 28420 + | > loss: -0.08289 (-0.07418) + | > log_mle: -0.29385 (-0.22664) + | > loss_dur: 0.21096 (0.15246) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.96394 (11.09197) + | > current_lr: 0.00003 + | > step_time: 2.00040 (2.39755) + | > loader_time: 0.08670 (0.03879) + + + --> STEP: 111/234 -- GLOBAL_STEP: 28425 + | > loss: -0.11746 (-0.07529) + | > log_mle: -0.33825 (-0.22952) + | > loss_dur: 0.22079 (0.15423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.49213 (11.66259) + | > current_lr: 0.00003 + | > step_time: 4.21810 (2.51320) + | > loader_time: 0.08250 (0.04210) + + + --> STEP: 116/234 -- GLOBAL_STEP: 28430 + | > loss: -0.08698 (-0.07658) + | > log_mle: -0.30660 (-0.23240) + | > loss_dur: 0.21962 (0.15582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.39972 (12.09453) + | > current_lr: 0.00003 + | > step_time: 2.89700 (2.55232) + | > loader_time: 0.00320 (0.04361) + + + --> STEP: 121/234 -- GLOBAL_STEP: 28435 + | > loss: -0.04761 (-0.07744) + | > log_mle: -0.21786 (-0.23434) + | > loss_dur: 0.17026 (0.15690) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.75821 (12.45043) + | > current_lr: 0.00003 + | > step_time: 1.60250 (2.54608) + | > loader_time: 0.09320 (0.04418) + + + --> STEP: 126/234 -- GLOBAL_STEP: 28440 + | > loss: -0.13802 (-0.07900) + | > log_mle: -0.35123 (-0.23695) + | > loss_dur: 0.21321 (0.15795) + | > amp_scaler: 4096.00000 (2064.25397) + | > grad_norm: 39.27713 (12.95245) + | > current_lr: 0.00003 + | > step_time: 1.30770 (2.52207) + | > loader_time: 0.00290 (0.04379) + + + --> STEP: 131/234 -- GLOBAL_STEP: 28445 + | > loss: -0.19217 (-0.08130) + | > log_mle: -0.39698 (-0.24062) + | > loss_dur: 0.20480 (0.15932) + | > amp_scaler: 4096.00000 (2141.80153) + | > grad_norm: 39.36988 (13.66743) + | > current_lr: 0.00003 + | > step_time: 2.01310 (2.50600) + | > loader_time: 0.08620 (0.04351) + + + --> STEP: 136/234 -- GLOBAL_STEP: 28450 + | > loss: -0.21429 (-0.08353) + | > log_mle: -0.44335 (-0.24430) + | > loss_dur: 0.22906 (0.16077) + | > amp_scaler: 4096.00000 (2213.64706) + | > grad_norm: 62.52479 (14.48296) + | > current_lr: 0.00003 + | > step_time: 1.20720 (2.49016) + | > loader_time: 0.00200 (0.04206) + + + --> STEP: 141/234 -- GLOBAL_STEP: 28455 + | > loss: -0.13718 (-0.08520) + | > log_mle: -0.35043 (-0.24751) + | > loss_dur: 0.21325 (0.16231) + | > amp_scaler: 4096.00000 (2280.39716) + | > grad_norm: 33.18471 (15.14823) + | > current_lr: 0.00003 + | > step_time: 1.80120 (2.48857) + | > loader_time: 0.00250 (0.04281) + + + --> STEP: 146/234 -- GLOBAL_STEP: 28460 + | > loss: -0.17923 (-0.08811) + | > log_mle: -0.39560 (-0.25236) + | > loss_dur: 0.21637 (0.16424) + | > amp_scaler: 4096.00000 (2342.57534) + | > grad_norm: 69.58750 (16.37309) + | > current_lr: 0.00003 + | > step_time: 2.00580 (2.49239) + | > loader_time: 0.08580 (0.04208) + + + --> STEP: 151/234 -- GLOBAL_STEP: 28465 + | > loss: -0.17031 (-0.09086) + | > log_mle: -0.36551 (-0.25632) + | > loss_dur: 0.19520 (0.16546) + | > amp_scaler: 4096.00000 (2400.63576) + | > grad_norm: 48.54665 (17.20428) + | > current_lr: 0.00003 + | > step_time: 1.89910 (2.53170) + | > loader_time: 0.10480 (0.04335) + + + --> STEP: 156/234 -- GLOBAL_STEP: 28470 + | > loss: -0.19637 (-0.09459) + | > log_mle: -0.40757 (-0.26176) + | > loss_dur: 0.21120 (0.16717) + | > amp_scaler: 4096.00000 (2454.97436) + | > grad_norm: 61.07438 (18.47819) + | > current_lr: 0.00003 + | > step_time: 2.11430 (2.51413) + | > loader_time: 0.00540 (0.04263) + + + --> STEP: 161/234 -- GLOBAL_STEP: 28475 + | > loss: -0.21931 (-0.09769) + | > log_mle: -0.43279 (-0.26641) + | > loss_dur: 0.21348 (0.16872) + | > amp_scaler: 4096.00000 (2505.93789) + | > grad_norm: 42.95507 (19.36175) + | > current_lr: 0.00003 + | > step_time: 4.90100 (2.52914) + | > loader_time: 0.09530 (0.04426) + + + --> STEP: 166/234 -- GLOBAL_STEP: 28480 + | > loss: -0.16818 (-0.10004) + | > log_mle: -0.36435 (-0.27010) + | > loss_dur: 0.19617 (0.17007) + | > amp_scaler: 4096.00000 (2553.83133) + | > grad_norm: 39.97453 (20.93358) + | > current_lr: 0.00003 + | > step_time: 2.49210 (2.56627) + | > loader_time: 0.09820 (0.04706) + + + --> STEP: 171/234 -- GLOBAL_STEP: 28485 + | > loss: -0.24314 (-0.10344) + | > log_mle: -0.47091 (-0.27531) + | > loss_dur: 0.22777 (0.17186) + | > amp_scaler: 4096.00000 (2598.92398) + | > grad_norm: 75.01415 (22.35593) + | > current_lr: 0.00003 + | > step_time: 1.69710 (2.57361) + | > loader_time: 0.09400 (0.04738) + + + --> STEP: 176/234 -- GLOBAL_STEP: 28490 + | > loss: -0.22506 (-0.10693) + | > log_mle: -0.45134 (-0.28056) + | > loss_dur: 0.22628 (0.17363) + | > amp_scaler: 4096.00000 (2641.45455) + | > grad_norm: 41.90060 (23.32663) + | > current_lr: 0.00003 + | > step_time: 3.29380 (2.57665) + | > loader_time: 0.01360 (0.04728) + + + --> STEP: 181/234 -- GLOBAL_STEP: 28495 + | > loss: -0.16175 (-0.10965) + | > log_mle: -0.38678 (-0.28512) + | > loss_dur: 0.22503 (0.17547) + | > amp_scaler: 4096.00000 (2681.63536) + | > grad_norm: 38.49274 (24.49895) + | > current_lr: 0.00003 + | > step_time: 6.19310 (2.66841) + | > loader_time: 0.00440 (0.04772) + + + --> STEP: 186/234 -- GLOBAL_STEP: 28500 + | > loss: -0.18306 (-0.11262) + | > log_mle: -0.42449 (-0.28995) + | > loss_dur: 0.24142 (0.17733) + | > amp_scaler: 4096.00000 (2719.65591) + | > grad_norm: 49.97387 (25.49496) + | > current_lr: 0.00003 + | > step_time: 3.70280 (2.69031) + | > loader_time: 0.09460 (0.04842) + + + --> STEP: 191/234 -- GLOBAL_STEP: 28505 + | > loss: -0.21918 (-0.11560) + | > log_mle: -0.43976 (-0.29449) + | > loss_dur: 0.22059 (0.17889) + | > amp_scaler: 4096.00000 (2755.68586) + | > grad_norm: 54.52176 (26.72134) + | > current_lr: 0.00003 + | > step_time: 5.21010 (2.78065) + | > loader_time: 0.09860 (0.04986) + + + --> STEP: 196/234 -- GLOBAL_STEP: 28510 + | > loss: -0.19600 (-0.11869) + | > log_mle: -0.43750 (-0.29908) + | > loss_dur: 0.24150 (0.18039) + | > amp_scaler: 4096.00000 (2789.87755) + | > grad_norm: 49.66652 (27.89448) + | > current_lr: 0.00003 + | > step_time: 4.59040 (2.80038) + | > loader_time: 0.10250 (0.05018) + + + --> STEP: 201/234 -- GLOBAL_STEP: 28515 + | > loss: -0.16196 (-0.12133) + | > log_mle: -0.40178 (-0.30330) + | > loss_dur: 0.23982 (0.18197) + | > amp_scaler: 4096.00000 (2822.36816) + | > grad_norm: 49.64595 (28.70461) + | > current_lr: 0.00003 + | > step_time: 7.59050 (2.85012) + | > loader_time: 0.00650 (0.04989) + + + --> STEP: 206/234 -- GLOBAL_STEP: 28520 + | > loss: -0.26220 (-0.12449) + | > log_mle: -0.50480 (-0.30803) + | > loss_dur: 0.24260 (0.18354) + | > amp_scaler: 4096.00000 (2853.28155) + | > grad_norm: 68.85336 (29.65329) + | > current_lr: 0.00003 + | > step_time: 5.40200 (2.90624) + | > loader_time: 0.09300 (0.05003) + + + --> STEP: 211/234 -- GLOBAL_STEP: 28525 + | > loss: -0.29967 (-0.12806) + | > log_mle: -0.57684 (-0.31334) + | > loss_dur: 0.27717 (0.18529) + | > amp_scaler: 4096.00000 (2882.72986) + | > grad_norm: 86.03510 (30.76239) + | > current_lr: 0.00003 + | > step_time: 3.59620 (2.92086) + | > loader_time: 0.00480 (0.06066) + + + --> STEP: 216/234 -- GLOBAL_STEP: 28530 + | > loss: -0.29014 (-0.13153) + | > log_mle: -0.56132 (-0.31838) + | > loss_dur: 0.27118 (0.18685) + | > amp_scaler: 4096.00000 (2910.81481) + | > grad_norm: 91.96183 (31.96048) + | > current_lr: 0.00003 + | > step_time: 5.00910 (2.97867) + | > loader_time: 2.29250 (0.16394) + + + --> STEP: 221/234 -- GLOBAL_STEP: 28535 + | > loss: -0.24463 (-0.13496) + | > log_mle: -0.48716 (-0.32342) + | > loss_dur: 0.24252 (0.18847) + | > amp_scaler: 4096.00000 (2937.62896) + | > grad_norm: 65.53299 (33.29441) + | > current_lr: 0.00003 + | > step_time: 5.21000 (3.03344) + | > loader_time: 0.09650 (0.16709) + + + --> STEP: 226/234 -- GLOBAL_STEP: 28540 + | > loss: -0.32182 (-0.13887) + | > log_mle: -0.58782 (-0.32911) + | > loss_dur: 0.26600 (0.19024) + | > amp_scaler: 4096.00000 (2963.25664) + | > grad_norm: 86.29856 (34.49102) + | > current_lr: 0.00003 + | > step_time: 1.29670 (3.07345) + | > loader_time: 0.00300 (0.16351) + + + --> STEP: 231/234 -- GLOBAL_STEP: 28545 + | > loss: -0.24009 (-0.14177) + | > log_mle: -0.64767 (-0.33503) + | > loss_dur: 0.40758 (0.19327) + | > amp_scaler: 4096.00000 (2987.77489) + | > grad_norm: 92.61974 (35.70507) + | > current_lr: 0.00003 + | > step_time: 0.26750 (3.01682) + | > loader_time: 0.00370 (0.16040) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00180 (-0.57580) + | > avg_loss: -0.17359 (+0.00896) + | > avg_log_mle: -0.40821 (+0.00715) + | > avg_loss_dur: 0.23462 (+0.00181) + + + > EPOCH: 122/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 10:55:58)  + + --> STEP: 2/234 -- GLOBAL_STEP: 28550 + | > loss: -0.04959 (-0.06420) + | > log_mle: -0.21244 (-0.21426) + | > loss_dur: 0.16284 (0.15006) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.88372 (10.24246) + | > current_lr: 0.00003 + | > step_time: 1.70570 (1.75444) + | > loader_time: 0.08250 (0.04268) + + + --> STEP: 7/234 -- GLOBAL_STEP: 28555 + | > loss: -0.08079 (-0.04934) + | > log_mle: -0.22686 (-0.21613) + | > loss_dur: 0.14608 (0.16679) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.60620 (11.01220) + | > current_lr: 0.00003 + | > step_time: 6.01600 (4.57568) + | > loader_time: 0.18770 (1.17719) + + + --> STEP: 12/234 -- GLOBAL_STEP: 28560 + | > loss: -0.07418 (-0.05824) + | > log_mle: -0.21815 (-0.21817) + | > loss_dur: 0.14398 (0.15993) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.30909 (10.82353) + | > current_lr: 0.00003 + | > step_time: 0.98820 (5.10199) + | > loader_time: 0.00250 (0.71918) + + + --> STEP: 17/234 -- GLOBAL_STEP: 28565 + | > loss: -0.05003 (-0.06390) + | > log_mle: -0.19453 (-0.21687) + | > loss_dur: 0.14449 (0.15297) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.35458 (9.97784) + | > current_lr: 0.00003 + | > step_time: 4.48800 (4.77660) + | > loader_time: 0.00140 (0.50855) + + + --> STEP: 22/234 -- GLOBAL_STEP: 28570 + | > loss: -0.09299 (-0.06613) + | > log_mle: -0.21962 (-0.21501) + | > loss_dur: 0.12663 (0.14888) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.76123 (9.48435) + | > current_lr: 0.00003 + | > step_time: 1.80240 (4.23237) + | > loader_time: 0.00220 (0.40221) + + + --> STEP: 27/234 -- GLOBAL_STEP: 28575 + | > loss: -0.09820 (-0.06926) + | > log_mle: -0.22006 (-0.21453) + | > loss_dur: 0.12186 (0.14527) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.89734 (9.00399) + | > current_lr: 0.00003 + | > step_time: 2.71250 (4.38606) + | > loader_time: 0.08430 (0.35243) + + + --> STEP: 32/234 -- GLOBAL_STEP: 28580 + | > loss: -0.10701 (-0.07098) + | > log_mle: -0.22830 (-0.21469) + | > loss_dur: 0.12128 (0.14371) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.97058 (8.72568) + | > current_lr: 0.00003 + | > step_time: 0.82510 (3.93750) + | > loader_time: 0.00230 (0.30258) + + + --> STEP: 37/234 -- GLOBAL_STEP: 28585 + | > loss: -0.08844 (-0.07062) + | > log_mle: -0.21196 (-0.21449) + | > loss_dur: 0.12351 (0.14387) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.96862 (8.81369) + | > current_lr: 0.00003 + | > step_time: 1.20810 (3.57786) + | > loader_time: 0.00150 (0.26399) + + + --> STEP: 42/234 -- GLOBAL_STEP: 28590 + | > loss: -0.06766 (-0.07041) + | > log_mle: -0.20194 (-0.21414) + | > loss_dur: 0.13428 (0.14373) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.95355 (8.83744) + | > current_lr: 0.00003 + | > step_time: 1.61360 (3.33039) + | > loader_time: 0.08860 (0.23487) + + + --> STEP: 47/234 -- GLOBAL_STEP: 28595 + | > loss: -0.06285 (-0.07049) + | > log_mle: -0.21258 (-0.21467) + | > loss_dur: 0.14973 (0.14419) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.08385 (8.84033) + | > current_lr: 0.00003 + | > step_time: 2.29990 (3.21256) + | > loader_time: 0.08860 (0.21370) + + + --> STEP: 52/234 -- GLOBAL_STEP: 28600 + | > loss: -0.05785 (-0.07046) + | > log_mle: -0.21048 (-0.21408) + | > loss_dur: 0.15263 (0.14362) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.30361 (8.51768) + | > current_lr: 0.00003 + | > step_time: 1.08850 (3.03856) + | > loader_time: 0.00350 (0.19341) + + + --> STEP: 57/234 -- GLOBAL_STEP: 28605 + | > loss: -0.04648 (-0.07041) + | > log_mle: -0.20232 (-0.21452) + | > loss_dur: 0.15584 (0.14410) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.47098 (8.40656) + | > current_lr: 0.00003 + | > step_time: 2.81140 (2.92774) + | > loader_time: 0.08120 (0.17808) + + + --> STEP: 62/234 -- GLOBAL_STEP: 28610 + | > loss: -0.04674 (-0.07128) + | > log_mle: -0.25284 (-0.21586) + | > loss_dur: 0.20609 (0.14458) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.30942 (8.65628) + | > current_lr: 0.00003 + | > step_time: 1.37590 (2.84623) + | > loader_time: 0.00260 (0.16641) + + + --> STEP: 67/234 -- GLOBAL_STEP: 28615 + | > loss: -0.06875 (-0.07123) + | > log_mle: -0.23408 (-0.21588) + | > loss_dur: 0.16533 (0.14465) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.36357 (8.56833) + | > current_lr: 0.00003 + | > step_time: 2.01440 (2.75456) + | > loader_time: 0.08020 (0.15534) + + + --> STEP: 72/234 -- GLOBAL_STEP: 28620 + | > loss: -0.05677 (-0.06963) + | > log_mle: -0.21149 (-0.21574) + | > loss_dur: 0.15472 (0.14611) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.14474 (8.74327) + | > current_lr: 0.00003 + | > step_time: 2.60230 (2.71753) + | > loader_time: 0.00370 (0.14705) + + + --> STEP: 77/234 -- GLOBAL_STEP: 28625 + | > loss: -0.08478 (-0.06908) + | > log_mle: -0.22442 (-0.21607) + | > loss_dur: 0.13964 (0.14699) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.61936 (9.01387) + | > current_lr: 0.00003 + | > step_time: 1.80660 (2.65942) + | > loader_time: 0.00230 (0.13774) + + + --> STEP: 82/234 -- GLOBAL_STEP: 28630 + | > loss: -0.07717 (-0.06943) + | > log_mle: -0.21460 (-0.21612) + | > loss_dur: 0.13743 (0.14669) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.69098 (9.02952) + | > current_lr: 0.00003 + | > step_time: 2.70870 (2.62171) + | > loader_time: 0.00300 (0.13050) + + + --> STEP: 87/234 -- GLOBAL_STEP: 28635 + | > loss: -0.06228 (-0.06946) + | > log_mle: -0.22635 (-0.21679) + | > loss_dur: 0.16407 (0.14733) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.99035 (9.20194) + | > current_lr: 0.00003 + | > step_time: 2.59990 (2.59506) + | > loader_time: 0.00280 (0.12402) + + + --> STEP: 92/234 -- GLOBAL_STEP: 28640 + | > loss: -0.12053 (-0.07060) + | > log_mle: -0.27438 (-0.21910) + | > loss_dur: 0.15385 (0.14850) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.19964 (9.57532) + | > current_lr: 0.00003 + | > step_time: 1.51510 (2.54995) + | > loader_time: 0.00320 (0.12046) + + + --> STEP: 97/234 -- GLOBAL_STEP: 28645 + | > loss: -0.10172 (-0.07271) + | > log_mle: -0.26215 (-0.22238) + | > loss_dur: 0.16044 (0.14967) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.73608 (10.14817) + | > current_lr: 0.00003 + | > step_time: 1.27580 (2.54367) + | > loader_time: 0.00200 (0.11627) + + + --> STEP: 102/234 -- GLOBAL_STEP: 28650 + | > loss: -0.07096 (-0.07374) + | > log_mle: -0.24251 (-0.22460) + | > loss_dur: 0.17156 (0.15086) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.50988 (10.49075) + | > current_lr: 0.00003 + | > step_time: 2.69510 (2.51031) + | > loader_time: 0.00340 (0.11261) + + + --> STEP: 107/234 -- GLOBAL_STEP: 28655 + | > loss: -0.10082 (-0.07537) + | > log_mle: -0.28909 (-0.22787) + | > loss_dur: 0.18827 (0.15250) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.02225 (11.18828) + | > current_lr: 0.00003 + | > step_time: 3.70040 (2.51245) + | > loader_time: 0.08800 (0.10902) + + + --> STEP: 112/234 -- GLOBAL_STEP: 28660 + | > loss: -0.10474 (-0.07649) + | > log_mle: -0.29829 (-0.23084) + | > loss_dur: 0.19355 (0.15435) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.34059 (11.80260) + | > current_lr: 0.00003 + | > step_time: 1.20260 (2.47873) + | > loader_time: 0.00230 (0.10427) + + + --> STEP: 117/234 -- GLOBAL_STEP: 28665 + | > loss: -0.12058 (-0.07777) + | > log_mle: -0.29614 (-0.23369) + | > loss_dur: 0.17556 (0.15592) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.97870 (12.23175) + | > current_lr: 0.00003 + | > step_time: 1.89300 (2.43952) + | > loader_time: 0.00330 (0.10077) + + + --> STEP: 122/234 -- GLOBAL_STEP: 28670 + | > loss: -0.09225 (-0.07852) + | > log_mle: -0.27164 (-0.23549) + | > loss_dur: 0.17939 (0.15696) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.20049 (12.49059) + | > current_lr: 0.00003 + | > step_time: 2.69920 (2.41512) + | > loader_time: 0.09150 (0.09888) + + + --> STEP: 127/234 -- GLOBAL_STEP: 28675 + | > loss: -0.12352 (-0.08044) + | > log_mle: -0.32874 (-0.23854) + | > loss_dur: 0.20522 (0.15810) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.67955 (13.13272) + | > current_lr: 0.00003 + | > step_time: 1.91030 (2.38601) + | > loader_time: 0.00350 (0.09577) + + + --> STEP: 132/234 -- GLOBAL_STEP: 28680 + | > loss: -0.13313 (-0.08260) + | > log_mle: -0.30934 (-0.24201) + | > loss_dur: 0.17621 (0.15941) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 24.16084 (13.80073) + | > current_lr: 0.00003 + | > step_time: 1.50790 (2.35778) + | > loader_time: 0.09010 (0.09353) + + + --> STEP: 137/234 -- GLOBAL_STEP: 28685 + | > loss: -0.11328 (-0.08465) + | > log_mle: -0.32322 (-0.24575) + | > loss_dur: 0.20994 (0.16109) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.99938 (14.53322) + | > current_lr: 0.00003 + | > step_time: 1.90820 (2.34791) + | > loader_time: 0.00250 (0.09149) + + + --> STEP: 142/234 -- GLOBAL_STEP: 28690 + | > loss: -0.13830 (-0.08655) + | > log_mle: -0.33859 (-0.24902) + | > loss_dur: 0.20029 (0.16247) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.78590 (15.16446) + | > current_lr: 0.00003 + | > step_time: 5.50120 (2.41457) + | > loader_time: 0.09020 (0.09143) + + + --> STEP: 147/234 -- GLOBAL_STEP: 28695 + | > loss: -0.13164 (-0.08925) + | > log_mle: -0.33682 (-0.25368) + | > loss_dur: 0.20518 (0.16443) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.18848 (16.52851) + | > current_lr: 0.00003 + | > step_time: 1.61000 (2.46721) + | > loader_time: 0.00390 (0.09039) + + + --> STEP: 152/234 -- GLOBAL_STEP: 28700 + | > loss: -0.18350 (-0.09210) + | > log_mle: -0.41417 (-0.25796) + | > loss_dur: 0.23067 (0.16586) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.26867 (17.56036) + | > current_lr: 0.00003 + | > step_time: 1.69070 (2.52078) + | > loader_time: 0.00300 (0.09142) + + + --> STEP: 157/234 -- GLOBAL_STEP: 28705 + | > loss: -0.13853 (-0.09530) + | > log_mle: -0.35375 (-0.26275) + | > loss_dur: 0.21522 (0.16745) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 45.08860 (19.24396) + | > current_lr: 0.00003 + | > step_time: 5.10530 (2.52790) + | > loader_time: 0.09210 (0.09023) + + + --> STEP: 162/234 -- GLOBAL_STEP: 28710 + | > loss: -0.20302 (-0.09826) + | > log_mle: -0.39966 (-0.26735) + | > loss_dur: 0.19663 (0.16909) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.95362 (20.28497) + | > current_lr: 0.00003 + | > step_time: 2.30630 (2.51544) + | > loader_time: 0.00450 (0.08808) + + + --> STEP: 167/234 -- GLOBAL_STEP: 28715 + | > loss: -0.26537 (-0.10117) + | > log_mle: -0.47604 (-0.27159) + | > loss_dur: 0.21067 (0.17042) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.00052 (21.26231) + | > current_lr: 0.00003 + | > step_time: 3.20080 (2.53717) + | > loader_time: 0.09920 (0.08612) + + + --> STEP: 172/234 -- GLOBAL_STEP: 28720 + | > loss: -0.22471 (-0.10441) + | > log_mle: -0.46569 (-0.27677) + | > loss_dur: 0.24097 (0.17236) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 77.50733 (22.33491) + | > current_lr: 0.00003 + | > step_time: 2.11380 (2.54673) + | > loader_time: 0.00550 (0.08514) + + + --> STEP: 177/234 -- GLOBAL_STEP: 28725 + | > loss: -0.20034 (-0.10763) + | > log_mle: -0.43324 (-0.28172) + | > loss_dur: 0.23290 (0.17409) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.19456 (23.37227) + | > current_lr: 0.00003 + | > step_time: 2.80340 (2.55731) + | > loader_time: 0.00440 (0.08337) + + + --> STEP: 182/234 -- GLOBAL_STEP: 28730 + | > loss: -0.21530 (-0.11059) + | > log_mle: -0.47522 (-0.28658) + | > loss_dur: 0.25992 (0.17599) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.00965 (24.48333) + | > current_lr: 0.00003 + | > step_time: 5.00300 (2.57595) + | > loader_time: 0.00510 (0.08224) + + + --> STEP: 187/234 -- GLOBAL_STEP: 28735 + | > loss: -0.23734 (-0.11360) + | > log_mle: -0.47469 (-0.29135) + | > loss_dur: 0.23735 (0.17774) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.07615 (25.61723) + | > current_lr: 0.00003 + | > step_time: 5.68970 (2.61972) + | > loader_time: 0.10640 (0.08169) + + + --> STEP: 192/234 -- GLOBAL_STEP: 28740 + | > loss: -0.27547 (-0.11683) + | > log_mle: -0.50256 (-0.29605) + | > loss_dur: 0.22709 (0.17922) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.49365 (26.57330) + | > current_lr: 0.00003 + | > step_time: 9.19880 (2.70048) + | > loader_time: 0.10820 (0.08159) + + + --> STEP: 197/234 -- GLOBAL_STEP: 28745 + | > loss: -0.25531 (-0.11992) + | > log_mle: -0.47274 (-0.30053) + | > loss_dur: 0.21742 (0.18061) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.32190 (27.69600) + | > current_lr: 0.00003 + | > step_time: 2.00100 (2.71268) + | > loader_time: 0.00470 (0.08021) + + + --> STEP: 202/234 -- GLOBAL_STEP: 28750 + | > loss: -0.31957 (-0.12290) + | > log_mle: -0.56216 (-0.30511) + | > loss_dur: 0.24259 (0.18221) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 79.08597 (28.70237) + | > current_lr: 0.00003 + | > step_time: 4.30070 (2.72434) + | > loader_time: 0.19900 (0.08218) + + + --> STEP: 207/234 -- GLOBAL_STEP: 28755 + | > loss: -0.29779 (-0.12596) + | > log_mle: -0.55032 (-0.30974) + | > loss_dur: 0.25253 (0.18378) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 77.12218 (29.70976) + | > current_lr: 0.00003 + | > step_time: 3.29370 (2.74515) + | > loader_time: 0.00300 (0.08076) + + + --> STEP: 212/234 -- GLOBAL_STEP: 28760 + | > loss: -0.28722 (-0.12947) + | > log_mle: -0.53734 (-0.31498) + | > loss_dur: 0.25012 (0.18551) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.17244 (30.95410) + | > current_lr: 0.00003 + | > step_time: 7.19900 (2.80180) + | > loader_time: 0.29310 (0.08158) + + + --> STEP: 217/234 -- GLOBAL_STEP: 28765 + | > loss: -0.28588 (-0.13294) + | > log_mle: -0.54618 (-0.32006) + | > loss_dur: 0.26030 (0.18712) + | > amp_scaler: 2048.00000 (4086.56221) + | > grad_norm: 0.00000 (31.89992) + | > current_lr: 0.00003 + | > step_time: 7.90440 (2.84461) + | > loader_time: 0.10400 (0.08070) + + + --> STEP: 222/234 -- GLOBAL_STEP: 28770 + | > loss: -0.27111 (-0.13630) + | > log_mle: -0.56167 (-0.32509) + | > loss_dur: 0.29055 (0.18879) + | > amp_scaler: 2048.00000 (4040.64865) + | > grad_norm: 92.43415 (33.19035) + | > current_lr: 0.00003 + | > step_time: 1.79650 (2.85040) + | > loader_time: 0.00330 (0.08158) + + + --> STEP: 227/234 -- GLOBAL_STEP: 28775 + | > loss: -0.22220 (-0.13947) + | > log_mle: -0.50658 (-0.32997) + | > loss_dur: 0.28437 (0.19050) + | > amp_scaler: 2048.00000 (3996.75771) + | > grad_norm: 112.66394 (35.38443) + | > current_lr: 0.00003 + | > step_time: 0.24210 (2.81459) + | > loader_time: 0.00350 (0.07988) + + + --> STEP: 232/234 -- GLOBAL_STEP: 28780 + | > loss: -0.17859 (-0.14146) + | > log_mle: -0.70923 (-0.33609) + | > loss_dur: 0.53063 (0.19463) + | > amp_scaler: 2048.00000 (3954.75862) + | > grad_norm: 121.01516 (36.89401) + | > current_lr: 0.00003 + | > step_time: 0.33910 (2.75979) + | > loader_time: 0.00580 (0.07826) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.19837 (+0.19657) + | > avg_loss: -0.17038 (+0.00321) + | > avg_log_mle: -0.40515 (+0.00306) + | > avg_loss_dur: 0.23477 (+0.00015) + + + > EPOCH: 123/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 11:08:01)  + + --> STEP: 3/234 -- GLOBAL_STEP: 28785 + | > loss: -0.01777 (-0.05481) + | > log_mle: -0.21930 (-0.21698) + | > loss_dur: 0.20152 (0.16217) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01289 (8.75534) + | > current_lr: 0.00003 + | > step_time: 7.61810 (7.91134) + | > loader_time: 0.00180 (1.23624) + + + --> STEP: 8/234 -- GLOBAL_STEP: 28790 + | > loss: -0.08834 (-0.06555) + | > log_mle: -0.23269 (-0.21956) + | > loss_dur: 0.14435 (0.15402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.09775 (9.99295) + | > current_lr: 0.00003 + | > step_time: 8.41160 (5.94280) + | > loader_time: 0.18880 (0.49788) + + + --> STEP: 13/234 -- GLOBAL_STEP: 28795 + | > loss: -0.07899 (-0.06785) + | > log_mle: -0.21828 (-0.21955) + | > loss_dur: 0.13930 (0.15170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.93515 (10.12704) + | > current_lr: 0.00003 + | > step_time: 3.49590 (4.52615) + | > loader_time: 0.09990 (0.32132) + + + --> STEP: 18/234 -- GLOBAL_STEP: 28800 + | > loss: -0.05967 (-0.06959) + | > log_mle: -0.21333 (-0.21791) + | > loss_dur: 0.15366 (0.14832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.31776 (9.57138) + | > current_lr: 0.00003 + | > step_time: 1.44470 (3.87835) + | > loader_time: 0.00120 (0.23263) + + + --> STEP: 23/234 -- GLOBAL_STEP: 28805 + | > loss: -0.10329 (-0.07221) + | > log_mle: -0.22167 (-0.21653) + | > loss_dur: 0.11838 (0.14432) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.85136 (9.19384) + | > current_lr: 0.00003 + | > step_time: 1.29560 (3.33159) + | > loader_time: 0.00270 (0.18253) + + + --> STEP: 28/234 -- GLOBAL_STEP: 28810 + | > loss: -0.09923 (-0.07413) + | > log_mle: -0.21103 (-0.21585) + | > loss_dur: 0.11180 (0.14172) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.45768 (8.79952) + | > current_lr: 0.00003 + | > step_time: 1.15880 (2.96747) + | > loader_time: 0.00230 (0.15311) + + + --> STEP: 33/234 -- GLOBAL_STEP: 28815 + | > loss: -0.06584 (-0.07434) + | > log_mle: -0.20658 (-0.21580) + | > loss_dur: 0.14074 (0.14146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.80314 (8.63958) + | > current_lr: 0.00003 + | > step_time: 2.39830 (2.71314) + | > loader_time: 0.00230 (0.13021) + + + --> STEP: 38/234 -- GLOBAL_STEP: 28820 + | > loss: -0.07942 (-0.07366) + | > log_mle: -0.22523 (-0.21606) + | > loss_dur: 0.14581 (0.14240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.62374 (8.59796) + | > current_lr: 0.00003 + | > step_time: 5.23710 (2.78844) + | > loader_time: 0.09370 (0.12631) + + + --> STEP: 43/234 -- GLOBAL_STEP: 28825 + | > loss: -0.05961 (-0.07303) + | > log_mle: -0.22396 (-0.21578) + | > loss_dur: 0.16435 (0.14275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.94598 (8.44900) + | > current_lr: 0.00003 + | > step_time: 1.18700 (2.64668) + | > loader_time: 0.00190 (0.11383) + + + --> STEP: 48/234 -- GLOBAL_STEP: 28830 + | > loss: -0.06968 (-0.07350) + | > log_mle: -0.20695 (-0.21594) + | > loss_dur: 0.13726 (0.14245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.24569 (8.44058) + | > current_lr: 0.00003 + | > step_time: 1.15870 (2.51865) + | > loader_time: 0.00170 (0.10572) + + + --> STEP: 53/234 -- GLOBAL_STEP: 28835 + | > loss: -0.07277 (-0.07319) + | > log_mle: -0.22637 (-0.21572) + | > loss_dur: 0.15360 (0.14253) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.75289 (8.34897) + | > current_lr: 0.00003 + | > step_time: 1.37840 (2.41277) + | > loader_time: 0.00150 (0.09759) + + + --> STEP: 58/234 -- GLOBAL_STEP: 28840 + | > loss: -0.07431 (-0.07350) + | > log_mle: -0.20934 (-0.21578) + | > loss_dur: 0.13504 (0.14228) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.19932 (8.41481) + | > current_lr: 0.00003 + | > step_time: 2.21460 (2.35002) + | > loader_time: 0.08520 (0.09230) + + + --> STEP: 63/234 -- GLOBAL_STEP: 28845 + | > loss: -0.04881 (-0.07378) + | > log_mle: -0.21777 (-0.21715) + | > loss_dur: 0.16896 (0.14337) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.59660 (8.93587) + | > current_lr: 0.00003 + | > step_time: 1.50850 (2.30147) + | > loader_time: 0.00290 (0.08518) + + + --> STEP: 68/234 -- GLOBAL_STEP: 28850 + | > loss: -0.04505 (-0.07354) + | > log_mle: -0.21286 (-0.21698) + | > loss_dur: 0.16781 (0.14344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.44536 (8.88957) + | > current_lr: 0.00003 + | > step_time: 1.07980 (2.24840) + | > loader_time: 0.00200 (0.08255) + + + --> STEP: 73/234 -- GLOBAL_STEP: 28855 + | > loss: -0.04830 (-0.07179) + | > log_mle: -0.23655 (-0.21716) + | > loss_dur: 0.18825 (0.14536) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.12821 (9.18110) + | > current_lr: 0.00003 + | > step_time: 3.71080 (2.22728) + | > loader_time: 0.00280 (0.07825) + + + --> STEP: 78/234 -- GLOBAL_STEP: 28860 + | > loss: -0.05252 (-0.07159) + | > log_mle: -0.21210 (-0.21745) + | > loss_dur: 0.15958 (0.14586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.48013 (9.23957) + | > current_lr: 0.00003 + | > step_time: 1.61090 (2.19603) + | > loader_time: 0.09920 (0.07600) + + + --> STEP: 83/234 -- GLOBAL_STEP: 28865 + | > loss: -0.05789 (-0.07162) + | > log_mle: -0.23723 (-0.21798) + | > loss_dur: 0.17934 (0.14636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.90290 (9.28045) + | > current_lr: 0.00003 + | > step_time: 2.78740 (2.16730) + | > loader_time: 0.00240 (0.07247) + + + --> STEP: 88/234 -- GLOBAL_STEP: 28870 + | > loss: -0.09886 (-0.07200) + | > log_mle: -0.27216 (-0.21907) + | > loss_dur: 0.17330 (0.14707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.76293 (9.39365) + | > current_lr: 0.00003 + | > step_time: 1.08880 (2.13607) + | > loader_time: 0.00190 (0.06961) + + + --> STEP: 93/234 -- GLOBAL_STEP: 28875 + | > loss: -0.09705 (-0.07299) + | > log_mle: -0.28751 (-0.22147) + | > loss_dur: 0.19046 (0.14849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.06877 (9.71824) + | > current_lr: 0.00003 + | > step_time: 1.78090 (2.14648) + | > loader_time: 0.00260 (0.06796) + + + --> STEP: 98/234 -- GLOBAL_STEP: 28880 + | > loss: -0.06064 (-0.07445) + | > log_mle: -0.21590 (-0.22389) + | > loss_dur: 0.15526 (0.14944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.53685 (10.04800) + | > current_lr: 0.00003 + | > step_time: 4.70620 (2.18110) + | > loader_time: 0.08940 (0.06650) + + + --> STEP: 103/234 -- GLOBAL_STEP: 28885 + | > loss: -0.12550 (-0.07623) + | > log_mle: -0.31681 (-0.22708) + | > loss_dur: 0.19131 (0.15085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.01621 (10.55807) + | > current_lr: 0.00003 + | > step_time: 4.49180 (2.18719) + | > loader_time: 0.10600 (0.06611) + + + --> STEP: 108/234 -- GLOBAL_STEP: 28890 + | > loss: -0.09189 (-0.07755) + | > log_mle: -0.26325 (-0.22974) + | > loss_dur: 0.17136 (0.15219) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.09627 (11.08673) + | > current_lr: 0.00003 + | > step_time: 1.18200 (2.23974) + | > loader_time: 0.00260 (0.06470) + + + --> STEP: 113/234 -- GLOBAL_STEP: 28895 + | > loss: -0.12638 (-0.07885) + | > log_mle: -0.30933 (-0.23299) + | > loss_dur: 0.18295 (0.15414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.29504 (12.01020) + | > current_lr: 0.00003 + | > step_time: 1.90840 (2.21068) + | > loader_time: 0.08500 (0.06424) + + + --> STEP: 118/234 -- GLOBAL_STEP: 28900 + | > loss: -0.09261 (-0.07975) + | > log_mle: -0.27831 (-0.23549) + | > loss_dur: 0.18570 (0.15574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.59318 (12.37464) + | > current_lr: 0.00003 + | > step_time: 1.10130 (2.19331) + | > loader_time: 0.07530 (0.06229) + + + --> STEP: 123/234 -- GLOBAL_STEP: 28905 + | > loss: -0.08316 (-0.08049) + | > log_mle: -0.24804 (-0.23702) + | > loss_dur: 0.16488 (0.15653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.93651 (12.60281) + | > current_lr: 0.00003 + | > step_time: 1.31650 (2.17686) + | > loader_time: 0.07590 (0.06178) + + + --> STEP: 128/234 -- GLOBAL_STEP: 28910 + | > loss: -0.13928 (-0.08274) + | > log_mle: -0.31042 (-0.24057) + | > loss_dur: 0.17114 (0.15784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.85770 (13.22737) + | > current_lr: 0.00003 + | > step_time: 2.00420 (2.18469) + | > loader_time: 0.08780 (0.06091) + + + --> STEP: 133/234 -- GLOBAL_STEP: 28915 + | > loss: -0.13553 (-0.08476) + | > log_mle: -0.33210 (-0.24416) + | > loss_dur: 0.19657 (0.15940) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.21533 (14.00887) + | > current_lr: 0.00003 + | > step_time: 1.20990 (2.17842) + | > loader_time: 0.00260 (0.05872) + + + --> STEP: 138/234 -- GLOBAL_STEP: 28920 + | > loss: -0.10681 (-0.08644) + | > log_mle: -0.29021 (-0.24752) + | > loss_dur: 0.18341 (0.16109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.97002 (14.66422) + | > current_lr: 0.00003 + | > step_time: 4.69230 (2.19130) + | > loader_time: 0.01070 (0.05675) + + + --> STEP: 143/234 -- GLOBAL_STEP: 28925 + | > loss: -0.17957 (-0.08870) + | > log_mle: -0.42656 (-0.25170) + | > loss_dur: 0.24699 (0.16300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.15521 (15.70966) + | > current_lr: 0.00003 + | > step_time: 2.20790 (2.20275) + | > loader_time: 0.08430 (0.05545) + + + --> STEP: 148/234 -- GLOBAL_STEP: 28930 + | > loss: -0.16292 (-0.09139) + | > log_mle: -0.34325 (-0.25584) + | > loss_dur: 0.18032 (0.16445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.00962 (16.60809) + | > current_lr: 0.00003 + | > step_time: 2.49930 (2.21417) + | > loader_time: 0.07880 (0.05658) + + + --> STEP: 153/234 -- GLOBAL_STEP: 28935 + | > loss: -0.25985 (-0.09504) + | > log_mle: -0.46959 (-0.26111) + | > loss_dur: 0.20974 (0.16606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.80997 (17.63489) + | > current_lr: 0.00003 + | > step_time: 4.39480 (2.22810) + | > loader_time: 0.09640 (0.05715) + + + --> STEP: 158/234 -- GLOBAL_STEP: 28940 + | > loss: -0.17814 (-0.09816) + | > log_mle: -0.40057 (-0.26571) + | > loss_dur: 0.22244 (0.16755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.77605 (18.75406) + | > current_lr: 0.00003 + | > step_time: 2.70910 (2.27605) + | > loader_time: 0.08590 (0.05776) + + + --> STEP: 163/234 -- GLOBAL_STEP: 28945 + | > loss: -0.15754 (-0.10117) + | > log_mle: -0.37202 (-0.27017) + | > loss_dur: 0.21448 (0.16900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.81962 (19.85552) + | > current_lr: 0.00003 + | > step_time: 2.11180 (2.26224) + | > loader_time: 0.09340 (0.05805) + + + --> STEP: 168/234 -- GLOBAL_STEP: 28950 + | > loss: -0.17327 (-0.10416) + | > log_mle: -0.41925 (-0.27468) + | > loss_dur: 0.24598 (0.17052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.21831 (21.17726) + | > current_lr: 0.00003 + | > step_time: 2.01360 (2.29255) + | > loader_time: 0.08790 (0.05806) + + + --> STEP: 173/234 -- GLOBAL_STEP: 28955 + | > loss: -0.20913 (-0.10741) + | > log_mle: -0.43588 (-0.27977) + | > loss_dur: 0.22675 (0.17236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.98743 (22.49857) + | > current_lr: 0.00003 + | > step_time: 2.71260 (2.29510) + | > loader_time: 0.08280 (0.05790) + + + --> STEP: 178/234 -- GLOBAL_STEP: 28960 + | > loss: -0.22828 (-0.11041) + | > log_mle: -0.47876 (-0.28462) + | > loss_dur: 0.25048 (0.17422) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.79742 (24.16867) + | > current_lr: 0.00003 + | > step_time: 2.81080 (2.29689) + | > loader_time: 0.00450 (0.05788) + + + --> STEP: 183/234 -- GLOBAL_STEP: 28965 + | > loss: -0.25402 (-0.11315) + | > log_mle: -0.48948 (-0.28923) + | > loss_dur: 0.23545 (0.17609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.05689 (25.10736) + | > current_lr: 0.00003 + | > step_time: 6.18710 (2.39048) + | > loader_time: 0.00850 (0.05953) + + + --> STEP: 188/234 -- GLOBAL_STEP: 28970 + | > loss: -0.25081 (-0.11607) + | > log_mle: -0.49539 (-0.29391) + | > loss_dur: 0.24458 (0.17784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.25404 (26.25268) + | > current_lr: 0.00003 + | > step_time: 3.40080 (2.44713) + | > loader_time: 0.08390 (0.06096) + + + --> STEP: 193/234 -- GLOBAL_STEP: 28975 + | > loss: -0.26623 (-0.11937) + | > log_mle: -0.50455 (-0.29858) + | > loss_dur: 0.23831 (0.17922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.67288 (27.07061) + | > current_lr: 0.00003 + | > step_time: 2.30150 (2.50878) + | > loader_time: 0.09130 (0.06344) + + + --> STEP: 198/234 -- GLOBAL_STEP: 28980 + | > loss: -0.25429 (-0.12232) + | > log_mle: -0.49782 (-0.30307) + | > loss_dur: 0.24353 (0.18075) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.78507 (27.87455) + | > current_lr: 0.00003 + | > step_time: 3.19840 (2.53177) + | > loader_time: 0.10130 (0.06337) + + + --> STEP: 203/234 -- GLOBAL_STEP: 28985 + | > loss: -0.19027 (-0.12497) + | > log_mle: -0.42666 (-0.30732) + | > loss_dur: 0.23639 (0.18234) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.57377 (28.95968) + | > current_lr: 0.00003 + | > step_time: 2.10240 (2.56207) + | > loader_time: 0.00310 (0.06190) + + + --> STEP: 208/234 -- GLOBAL_STEP: 28990 + | > loss: -0.24915 (-0.12832) + | > log_mle: -0.50688 (-0.31232) + | > loss_dur: 0.25773 (0.18400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.58669 (30.05553) + | > current_lr: 0.00003 + | > step_time: 8.50120 (2.61262) + | > loader_time: 0.20630 (0.06321) + + + --> STEP: 213/234 -- GLOBAL_STEP: 28995 + | > loss: -0.28506 (-0.13192) + | > log_mle: -0.55252 (-0.31771) + | > loss_dur: 0.26746 (0.18579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.52045 (31.28497) + | > current_lr: 0.00003 + | > step_time: 5.29780 (2.69868) + | > loader_time: 0.08430 (0.06269) + + + --> STEP: 218/234 -- GLOBAL_STEP: 29000 + | > loss: -0.26133 (-0.13525) + | > log_mle: -0.51678 (-0.32259) + | > loss_dur: 0.25546 (0.18734) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.29108 (32.60359) + | > current_lr: 0.00003 + | > step_time: 2.59810 (2.80145) + | > loader_time: 0.00350 (0.06362) + + + --> STEP: 223/234 -- GLOBAL_STEP: 29005 + | > loss: -0.30179 (-0.13879) + | > log_mle: -0.55800 (-0.32777) + | > loss_dur: 0.25621 (0.18898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.63231 (33.94262) + | > current_lr: 0.00003 + | > step_time: 1.82400 (2.79715) + | > loader_time: 0.07610 (0.06262) + + + --> STEP: 228/234 -- GLOBAL_STEP: 29010 + | > loss: -0.26828 (-0.14226) + | > log_mle: -0.55857 (-0.33313) + | > loss_dur: 0.29029 (0.19087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.54559 (35.37762) + | > current_lr: 0.00003 + | > step_time: 0.25400 (2.74119) + | > loader_time: 0.00370 (0.06131) + + + --> STEP: 233/234 -- GLOBAL_STEP: 29015 + | > loss: 0.31798 (-0.14245) + | > log_mle: -0.51665 (-0.33967) + | > loss_dur: 0.83463 (0.19722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.47206 (36.81160) + | > current_lr: 0.00003 + | > step_time: 0.18610 (2.68788) + | > loader_time: 0.00260 (0.06035) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.33146 (+0.13309) + | > avg_loss: -0.19448 (-0.02409) + | > avg_log_mle: -0.42266 (-0.01751) + | > avg_loss_dur: 0.22818 (-0.00658) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_29016.pth + + > EPOCH: 124/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 11:19:39)  + + --> STEP: 4/234 -- GLOBAL_STEP: 29020 + | > loss: -0.01743 (-0.05035) + | > log_mle: -0.21863 (-0.21950) + | > loss_dur: 0.20121 (0.16915) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.17644 (15.84699) + | > current_lr: 0.00003 + | > step_time: 3.00140 (8.77145) + | > loader_time: 0.00300 (0.04715) + + + --> STEP: 9/234 -- GLOBAL_STEP: 29025 + | > loss: -0.06030 (-0.06563) + | > log_mle: -0.22973 (-0.22357) + | > loss_dur: 0.16943 (0.15794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.58815 (12.92183) + | > current_lr: 0.00003 + | > step_time: 4.10750 (6.21960) + | > loader_time: 0.19440 (0.07433) + + + --> STEP: 14/234 -- GLOBAL_STEP: 29030 + | > loss: -0.07007 (-0.07047) + | > log_mle: -0.22480 (-0.22287) + | > loss_dur: 0.15473 (0.15240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.83521 (11.76262) + | > current_lr: 0.00003 + | > step_time: 3.69270 (5.02495) + | > loader_time: 0.10130 (0.06294) + + + --> STEP: 19/234 -- GLOBAL_STEP: 29035 + | > loss: -0.09491 (-0.07363) + | > log_mle: -0.21207 (-0.22065) + | > loss_dur: 0.11716 (0.14702) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.09197 (10.79930) + | > current_lr: 0.00003 + | > step_time: 2.41550 (4.94172) + | > loader_time: 0.07840 (0.09197) + + + --> STEP: 24/234 -- GLOBAL_STEP: 29040 + | > loss: -0.10521 (-0.07621) + | > log_mle: -0.21300 (-0.21889) + | > loss_dur: 0.10779 (0.14268) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.93331 (10.53864) + | > current_lr: 0.00003 + | > step_time: 1.41520 (4.19958) + | > loader_time: 0.08290 (0.08015) + + + --> STEP: 29/234 -- GLOBAL_STEP: 29045 + | > loss: -0.07183 (-0.07726) + | > log_mle: -0.20302 (-0.21756) + | > loss_dur: 0.13120 (0.14030) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.46983 (10.06024) + | > current_lr: 0.00003 + | > step_time: 1.69520 (3.85038) + | > loader_time: 0.00160 (0.07013) + + + --> STEP: 34/234 -- GLOBAL_STEP: 29050 + | > loss: -0.05511 (-0.07768) + | > log_mle: -0.21301 (-0.21786) + | > loss_dur: 0.15790 (0.14019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.40302 (9.61977) + | > current_lr: 0.00003 + | > step_time: 1.86390 (3.50280) + | > loader_time: 0.00310 (0.06237) + + + --> STEP: 39/234 -- GLOBAL_STEP: 29055 + | > loss: -0.07635 (-0.07763) + | > log_mle: -0.22119 (-0.21815) + | > loss_dur: 0.14484 (0.14052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.60474 (9.87872) + | > current_lr: 0.00003 + | > step_time: 2.64640 (3.25209) + | > loader_time: 0.00290 (0.05462) + + + --> STEP: 44/234 -- GLOBAL_STEP: 29060 + | > loss: -0.08413 (-0.07632) + | > log_mle: -0.21243 (-0.21746) + | > loss_dur: 0.12830 (0.14113) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.36495 (9.59066) + | > current_lr: 0.00003 + | > step_time: 1.58500 (3.09226) + | > loader_time: 0.00170 (0.04871) + + + --> STEP: 49/234 -- GLOBAL_STEP: 29065 + | > loss: -0.08680 (-0.07666) + | > log_mle: -0.22199 (-0.21780) + | > loss_dur: 0.13519 (0.14114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.49047 (9.45298) + | > current_lr: 0.00003 + | > step_time: 1.69580 (2.95579) + | > loader_time: 0.00250 (0.04582) + + + --> STEP: 54/234 -- GLOBAL_STEP: 29070 + | > loss: -0.08742 (-0.07609) + | > log_mle: -0.22694 (-0.21766) + | > loss_dur: 0.13952 (0.14157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.61023 (9.17079) + | > current_lr: 0.00003 + | > step_time: 1.48560 (3.03179) + | > loader_time: 0.00250 (0.05288) + + + --> STEP: 59/234 -- GLOBAL_STEP: 29075 + | > loss: -0.10636 (-0.07609) + | > log_mle: -0.23729 (-0.21796) + | > loss_dur: 0.13093 (0.14187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.73903 (8.98254) + | > current_lr: 0.00003 + | > step_time: 2.88970 (2.96659) + | > loader_time: 0.09780 (0.05168) + + + --> STEP: 64/234 -- GLOBAL_STEP: 29080 + | > loss: -0.06755 (-0.07561) + | > log_mle: -0.20872 (-0.21899) + | > loss_dur: 0.14117 (0.14338) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.30397 (9.04798) + | > current_lr: 0.00003 + | > step_time: 2.39640 (2.85702) + | > loader_time: 0.00410 (0.05040) + + + --> STEP: 69/234 -- GLOBAL_STEP: 29085 + | > loss: -0.04767 (-0.07501) + | > log_mle: -0.19851 (-0.21876) + | > loss_dur: 0.15084 (0.14375) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.16373 (8.97343) + | > current_lr: 0.00003 + | > step_time: 1.79480 (2.77528) + | > loader_time: 0.00370 (0.04696) + + + --> STEP: 74/234 -- GLOBAL_STEP: 29090 + | > loss: -0.07327 (-0.07372) + | > log_mle: -0.20863 (-0.21914) + | > loss_dur: 0.13536 (0.14542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.06852 (9.33062) + | > current_lr: 0.00003 + | > step_time: 1.69780 (2.68367) + | > loader_time: 0.00220 (0.04393) + + + --> STEP: 79/234 -- GLOBAL_STEP: 29095 + | > loss: -0.07886 (-0.07363) + | > log_mle: -0.22710 (-0.21948) + | > loss_dur: 0.14825 (0.14585) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.68084 (9.38133) + | > current_lr: 0.00003 + | > step_time: 1.38870 (2.65439) + | > loader_time: 0.00250 (0.04137) + + + --> STEP: 84/234 -- GLOBAL_STEP: 29100 + | > loss: -0.06830 (-0.07378) + | > log_mle: -0.22264 (-0.21983) + | > loss_dur: 0.15434 (0.14605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.80779 (9.51204) + | > current_lr: 0.00003 + | > step_time: 1.69640 (2.59690) + | > loader_time: 0.00190 (0.04025) + + + --> STEP: 89/234 -- GLOBAL_STEP: 29105 + | > loss: -0.09273 (-0.07408) + | > log_mle: -0.25441 (-0.22118) + | > loss_dur: 0.16168 (0.14710) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.56998 (9.74095) + | > current_lr: 0.00003 + | > step_time: 2.00870 (2.55651) + | > loader_time: 0.00300 (0.03926) + + + --> STEP: 94/234 -- GLOBAL_STEP: 29110 + | > loss: -0.12954 (-0.07547) + | > log_mle: -0.29167 (-0.22384) + | > loss_dur: 0.16212 (0.14837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.44340 (10.21277) + | > current_lr: 0.00003 + | > step_time: 1.70100 (2.54603) + | > loader_time: 0.09190 (0.04014) + + + --> STEP: 99/234 -- GLOBAL_STEP: 29115 + | > loss: -0.13220 (-0.07682) + | > log_mle: -0.32405 (-0.22654) + | > loss_dur: 0.19185 (0.14972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.30418 (10.79701) + | > current_lr: 0.00003 + | > step_time: 1.40420 (2.50847) + | > loader_time: 0.08420 (0.03993) + + + --> STEP: 104/234 -- GLOBAL_STEP: 29120 + | > loss: -0.15503 (-0.07881) + | > log_mle: -0.33537 (-0.22969) + | > loss_dur: 0.18034 (0.15087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.78978 (11.34043) + | > current_lr: 0.00003 + | > step_time: 1.85400 (2.52583) + | > loader_time: 0.09340 (0.04084) + + + --> STEP: 109/234 -- GLOBAL_STEP: 29125 + | > loss: -0.08044 (-0.07955) + | > log_mle: -0.30390 (-0.23195) + | > loss_dur: 0.22347 (0.15240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.51973 (11.81670) + | > current_lr: 0.00003 + | > step_time: 2.10390 (2.50024) + | > loader_time: 0.08490 (0.04053) + + + --> STEP: 114/234 -- GLOBAL_STEP: 29130 + | > loss: -0.11112 (-0.08102) + | > log_mle: -0.28805 (-0.23506) + | > loss_dur: 0.17693 (0.15404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.34959 (12.33804) + | > current_lr: 0.00003 + | > step_time: 1.01180 (2.49173) + | > loader_time: 0.08830 (0.04200) + + + --> STEP: 119/234 -- GLOBAL_STEP: 29135 + | > loss: -0.10572 (-0.08179) + | > log_mle: -0.28833 (-0.23750) + | > loss_dur: 0.18262 (0.15571) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.62391 (12.66820) + | > current_lr: 0.00003 + | > step_time: 1.36150 (2.46573) + | > loader_time: 0.00210 (0.04034) + + + --> STEP: 124/234 -- GLOBAL_STEP: 29140 + | > loss: -0.14419 (-0.08268) + | > log_mle: -0.31545 (-0.23919) + | > loss_dur: 0.17125 (0.15651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.28955 (13.06333) + | > current_lr: 0.00003 + | > step_time: 1.81240 (2.50269) + | > loader_time: 0.08480 (0.04194) + + + --> STEP: 129/234 -- GLOBAL_STEP: 29145 + | > loss: -0.09778 (-0.08448) + | > log_mle: -0.30157 (-0.24246) + | > loss_dur: 0.20379 (0.15798) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.02555 (13.97093) + | > current_lr: 0.00003 + | > step_time: 2.50330 (2.51796) + | > loader_time: 0.00240 (0.04120) + + + --> STEP: 134/234 -- GLOBAL_STEP: 29150 + | > loss: -0.13409 (-0.08668) + | > log_mle: -0.35511 (-0.24632) + | > loss_dur: 0.22102 (0.15964) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.90532 (14.88738) + | > current_lr: 0.00003 + | > step_time: 1.72310 (2.50631) + | > loader_time: 0.07690 (0.04102) + + + --> STEP: 139/234 -- GLOBAL_STEP: 29155 + | > loss: -0.19710 (-0.08872) + | > log_mle: -0.41000 (-0.24986) + | > loss_dur: 0.21290 (0.16115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.44192 (15.76341) + | > current_lr: 0.00003 + | > step_time: 2.70530 (2.53707) + | > loader_time: 0.00340 (0.04115) + + + --> STEP: 144/234 -- GLOBAL_STEP: 29160 + | > loss: -0.17976 (-0.09068) + | > log_mle: -0.39198 (-0.25376) + | > loss_dur: 0.21223 (0.16308) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.71711 (16.51186) + | > current_lr: 0.00003 + | > step_time: 1.80900 (2.52831) + | > loader_time: 0.08510 (0.04041) + + + --> STEP: 149/234 -- GLOBAL_STEP: 29165 + | > loss: -0.22287 (-0.09367) + | > log_mle: -0.43851 (-0.25819) + | > loss_dur: 0.21564 (0.16452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.15414 (17.33103) + | > current_lr: 0.00003 + | > step_time: 1.60260 (2.50465) + | > loader_time: 0.00260 (0.04022) + + + --> STEP: 154/234 -- GLOBAL_STEP: 29170 + | > loss: -0.19253 (-0.09722) + | > log_mle: -0.39806 (-0.26315) + | > loss_dur: 0.20553 (0.16593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.98953 (18.21319) + | > current_lr: 0.00003 + | > step_time: 2.72570 (2.49899) + | > loader_time: 0.19180 (0.04139) + + + --> STEP: 159/234 -- GLOBAL_STEP: 29175 + | > loss: -0.19571 (-0.10017) + | > log_mle: -0.41530 (-0.26776) + | > loss_dur: 0.21959 (0.16759) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.24651 (19.23113) + | > current_lr: 0.00003 + | > step_time: 3.49840 (2.50702) + | > loader_time: 0.00600 (0.04084) + + + --> STEP: 164/234 -- GLOBAL_STEP: 29180 + | > loss: -0.18884 (-0.10323) + | > log_mle: -0.41198 (-0.27216) + | > loss_dur: 0.22314 (0.16892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.34614 (20.21884) + | > current_lr: 0.00003 + | > step_time: 2.40460 (2.52636) + | > loader_time: 0.08760 (0.04141) + + + --> STEP: 169/234 -- GLOBAL_STEP: 29185 + | > loss: -0.18396 (-0.10630) + | > log_mle: -0.40741 (-0.27678) + | > loss_dur: 0.22345 (0.17048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.28372 (21.36949) + | > current_lr: 0.00003 + | > step_time: 14.09740 (2.66533) + | > loader_time: 0.29650 (0.04366) + + + --> STEP: 174/234 -- GLOBAL_STEP: 29190 + | > loss: -0.27000 (-0.11027) + | > log_mle: -0.49355 (-0.28241) + | > loss_dur: 0.22355 (0.17214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.01315 (22.99983) + | > current_lr: 0.00003 + | > step_time: 3.12110 (2.69798) + | > loader_time: 0.00170 (0.04352) + + + --> STEP: 179/234 -- GLOBAL_STEP: 29195 + | > loss: -0.22876 (-0.11337) + | > log_mle: -0.48899 (-0.28749) + | > loss_dur: 0.26023 (0.17412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.18287 (24.08082) + | > current_lr: 0.00003 + | > step_time: 2.49690 (2.68557) + | > loader_time: 0.08000 (0.04284) + + + --> STEP: 184/234 -- GLOBAL_STEP: 29200 + | > loss: -0.22269 (-0.11619) + | > log_mle: -0.45955 (-0.29204) + | > loss_dur: 0.23686 (0.17584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.49195 (25.08196) + | > current_lr: 0.00003 + | > step_time: 6.70800 (2.72799) + | > loader_time: 0.20150 (0.04537) + + + --> STEP: 189/234 -- GLOBAL_STEP: 29205 + | > loss: -0.20879 (-0.11911) + | > log_mle: -0.45821 (-0.29681) + | > loss_dur: 0.24942 (0.17770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.04775 (26.14274) + | > current_lr: 0.00003 + | > step_time: 5.30870 (2.80201) + | > loader_time: 0.00480 (0.04525) + + + --> STEP: 194/234 -- GLOBAL_STEP: 29210 + | > loss: -0.26150 (-0.12265) + | > log_mle: -0.49226 (-0.30166) + | > loss_dur: 0.23076 (0.17901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.84577 (27.03353) + | > current_lr: 0.00003 + | > step_time: 4.18520 (2.82989) + | > loader_time: 0.00370 (0.04458) + + + --> STEP: 199/234 -- GLOBAL_STEP: 29215 + | > loss: -0.25231 (-0.12555) + | > log_mle: -0.49877 (-0.30615) + | > loss_dur: 0.24646 (0.18060) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.92363 (28.08990) + | > current_lr: 0.00003 + | > step_time: 11.30300 (2.90307) + | > loader_time: 0.28950 (0.04597) + + + --> STEP: 204/234 -- GLOBAL_STEP: 29220 + | > loss: -0.25070 (-0.12795) + | > log_mle: -0.51242 (-0.31024) + | > loss_dur: 0.26172 (0.18230) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.65113 (29.54815) + | > current_lr: 0.00003 + | > step_time: 6.20860 (2.94583) + | > loader_time: 0.18450 (0.04753) + + + --> STEP: 209/234 -- GLOBAL_STEP: 29225 + | > loss: -0.23972 (-0.13088) + | > log_mle: -0.48414 (-0.31475) + | > loss_dur: 0.24442 (0.18387) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.66674 (30.52722) + | > current_lr: 0.00003 + | > step_time: 5.09240 (2.97673) + | > loader_time: 0.10730 (0.04793) + + + --> STEP: 214/234 -- GLOBAL_STEP: 29230 + | > loss: -0.27968 (-0.13443) + | > log_mle: -0.51607 (-0.31998) + | > loss_dur: 0.23639 (0.18554) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.23589 (31.85638) + | > current_lr: 0.00003 + | > step_time: 6.09160 (2.98755) + | > loader_time: 0.00290 (0.04781) + + + --> STEP: 219/234 -- GLOBAL_STEP: 29235 + | > loss: -0.35051 (-0.13800) + | > log_mle: -0.60596 (-0.32519) + | > loss_dur: 0.25544 (0.18719) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.45020 (33.00301) + | > current_lr: 0.00003 + | > step_time: 3.58860 (3.09601) + | > loader_time: 0.00250 (0.04824) + + + --> STEP: 224/234 -- GLOBAL_STEP: 29240 + | > loss: -0.30236 (-0.14126) + | > log_mle: -0.56886 (-0.33013) + | > loss_dur: 0.26650 (0.18887) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.85004 (33.95353) + | > current_lr: 0.00003 + | > step_time: 0.22920 (3.05978) + | > loader_time: 0.00300 (0.04764) + + + --> STEP: 229/234 -- GLOBAL_STEP: 29245 + | > loss: -0.28051 (-0.14467) + | > log_mle: -0.59705 (-0.33560) + | > loss_dur: 0.31654 (0.19093) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 118.57953 (35.27971) + | > current_lr: 0.00003 + | > step_time: 0.25690 (2.99832) + | > loader_time: 0.00310 (0.04668) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05073 (-0.28072) + | > avg_loss: -0.18355 (+0.01093) + | > avg_log_mle: -0.41271 (+0.00995) + | > avg_loss_dur: 0.22916 (+0.00097) + + + > EPOCH: 125/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 11:32:12)  + + --> STEP: 0/234 -- GLOBAL_STEP: 29250 + | > loss: -0.13207 (-0.13207) + | > log_mle: -0.29022 (-0.29022) + | > loss_dur: 0.15815 (0.15815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.17177 (12.17177) + | > current_lr: 0.00003 + | > step_time: 13.71350 (13.71348) + | > loader_time: 15.30610 (15.30615) + + + --> STEP: 5/234 -- GLOBAL_STEP: 29255 + | > loss: -0.07048 (-0.05828) + | > log_mle: -0.22420 (-0.21793) + | > loss_dur: 0.15372 (0.15965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.09132 (16.20243) + | > current_lr: 0.00003 + | > step_time: 1.99380 (6.33455) + | > loader_time: 0.00120 (0.09806) + + + --> STEP: 10/234 -- GLOBAL_STEP: 29260 + | > loss: -0.06653 (-0.06619) + | > log_mle: -0.21964 (-0.22212) + | > loss_dur: 0.15310 (0.15593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.24132 (14.02007) + | > current_lr: 0.00003 + | > step_time: 4.00200 (4.46924) + | > loader_time: 0.18160 (0.08674) + + + --> STEP: 15/234 -- GLOBAL_STEP: 29265 + | > loss: -0.09333 (-0.07208) + | > log_mle: -0.22343 (-0.22269) + | > loss_dur: 0.13010 (0.15061) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.55543 (12.18552) + | > current_lr: 0.00003 + | > step_time: 1.17780 (4.12452) + | > loader_time: 0.00150 (0.07130) + + + --> STEP: 20/234 -- GLOBAL_STEP: 29270 + | > loss: -0.08056 (-0.07445) + | > log_mle: -0.21167 (-0.22037) + | > loss_dur: 0.13112 (0.14592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.30872 (10.77376) + | > current_lr: 0.00003 + | > step_time: 3.60710 (3.51202) + | > loader_time: 0.08300 (0.05790) + + + --> STEP: 25/234 -- GLOBAL_STEP: 29275 + | > loss: -0.05971 (-0.07701) + | > log_mle: -0.20712 (-0.21910) + | > loss_dur: 0.14741 (0.14209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.35482 (10.14483) + | > current_lr: 0.00003 + | > step_time: 3.21330 (3.25051) + | > loader_time: 0.38260 (0.06567) + + + --> STEP: 30/234 -- GLOBAL_STEP: 29280 + | > loss: -0.11436 (-0.07948) + | > log_mle: -0.22943 (-0.21915) + | > loss_dur: 0.11506 (0.13967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.12675 (9.69750) + | > current_lr: 0.00003 + | > step_time: 10.89810 (3.64217) + | > loader_time: 0.20620 (0.06511) + + + --> STEP: 35/234 -- GLOBAL_STEP: 29285 + | > loss: -0.06830 (-0.07913) + | > log_mle: -0.22120 (-0.21931) + | > loss_dur: 0.15290 (0.14018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76491 (9.66835) + | > current_lr: 0.00003 + | > step_time: 5.30140 (3.66173) + | > loader_time: 0.09540 (0.05926) + + + --> STEP: 40/234 -- GLOBAL_STEP: 29290 + | > loss: -0.04537 (-0.07874) + | > log_mle: -0.20345 (-0.21925) + | > loss_dur: 0.15808 (0.14051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.89105 (9.64926) + | > current_lr: 0.00003 + | > step_time: 1.86420 (3.57280) + | > loader_time: 0.09300 (0.06148) + + + --> STEP: 45/234 -- GLOBAL_STEP: 29295 + | > loss: -0.07827 (-0.07905) + | > log_mle: -0.24175 (-0.21952) + | > loss_dur: 0.16348 (0.14048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.80967 (9.48414) + | > current_lr: 0.00003 + | > step_time: 2.71140 (3.32999) + | > loader_time: 0.00350 (0.05493) + + + --> STEP: 50/234 -- GLOBAL_STEP: 29300 + | > loss: -0.06587 (-0.07881) + | > log_mle: -0.20867 (-0.21914) + | > loss_dur: 0.14280 (0.14033) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.66458 (9.13821) + | > current_lr: 0.00003 + | > step_time: 1.00280 (3.14659) + | > loader_time: 0.00170 (0.05280) + + + --> STEP: 55/234 -- GLOBAL_STEP: 29305 + | > loss: -0.09111 (-0.07900) + | > log_mle: -0.22849 (-0.21936) + | > loss_dur: 0.13738 (0.14035) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.99052 (8.91945) + | > current_lr: 0.00003 + | > step_time: 1.07120 (2.97675) + | > loader_time: 0.00210 (0.04953) + + + --> STEP: 60/234 -- GLOBAL_STEP: 29310 + | > loss: -0.09347 (-0.07886) + | > log_mle: -0.24563 (-0.21993) + | > loss_dur: 0.15215 (0.14107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.45046 (8.89130) + | > current_lr: 0.00003 + | > step_time: 1.70240 (2.85670) + | > loader_time: 0.00780 (0.04928) + + + --> STEP: 65/234 -- GLOBAL_STEP: 29315 + | > loss: -0.07992 (-0.07808) + | > log_mle: -0.21924 (-0.22049) + | > loss_dur: 0.13933 (0.14242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.64546 (8.87686) + | > current_lr: 0.00003 + | > step_time: 2.09940 (2.76588) + | > loader_time: 0.00290 (0.04708) + + + --> STEP: 70/234 -- GLOBAL_STEP: 29320 + | > loss: -0.05196 (-0.07699) + | > log_mle: -0.21718 (-0.22026) + | > loss_dur: 0.16523 (0.14327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.30243 (8.87835) + | > current_lr: 0.00003 + | > step_time: 2.79850 (2.71511) + | > loader_time: 0.09500 (0.04651) + + + --> STEP: 75/234 -- GLOBAL_STEP: 29325 + | > loss: -0.06502 (-0.07622) + | > log_mle: -0.23198 (-0.22100) + | > loss_dur: 0.16696 (0.14478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.02050 (9.30361) + | > current_lr: 0.00003 + | > step_time: 3.09110 (2.68334) + | > loader_time: 0.00300 (0.04359) + + + --> STEP: 80/234 -- GLOBAL_STEP: 29330 + | > loss: -0.08859 (-0.07633) + | > log_mle: -0.21287 (-0.22122) + | > loss_dur: 0.12428 (0.14488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.40129 (9.37939) + | > current_lr: 0.00003 + | > step_time: 1.49070 (2.67082) + | > loader_time: 0.00240 (0.04423) + + + --> STEP: 85/234 -- GLOBAL_STEP: 29335 + | > loss: -0.09650 (-0.07662) + | > log_mle: -0.23000 (-0.22172) + | > loss_dur: 0.13350 (0.14511) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01035 (9.64067) + | > current_lr: 0.00003 + | > step_time: 2.10190 (2.61252) + | > loader_time: 0.00360 (0.04289) + + + --> STEP: 90/234 -- GLOBAL_STEP: 29340 + | > loss: -0.08136 (-0.07683) + | > log_mle: -0.25668 (-0.22338) + | > loss_dur: 0.17532 (0.14654) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.04511 (9.93412) + | > current_lr: 0.00003 + | > step_time: 1.49860 (2.60395) + | > loader_time: 0.00280 (0.04159) + + + --> STEP: 95/234 -- GLOBAL_STEP: 29345 + | > loss: -0.14971 (-0.07866) + | > log_mle: -0.33838 (-0.22682) + | > loss_dur: 0.18867 (0.14817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.82753 (10.65702) + | > current_lr: 0.00003 + | > step_time: 4.81970 (2.60493) + | > loader_time: 0.10030 (0.04226) + + + --> STEP: 100/234 -- GLOBAL_STEP: 29350 + | > loss: -0.10309 (-0.07943) + | > log_mle: -0.26757 (-0.22856) + | > loss_dur: 0.16448 (0.14913) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.44977 (10.93223) + | > current_lr: 0.00003 + | > step_time: 1.00680 (2.57171) + | > loader_time: 0.00340 (0.04195) + + + --> STEP: 105/234 -- GLOBAL_STEP: 29355 + | > loss: -0.09577 (-0.08125) + | > log_mle: -0.24409 (-0.23143) + | > loss_dur: 0.14832 (0.15018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.75751 (11.29747) + | > current_lr: 0.00003 + | > step_time: 2.60000 (2.56344) + | > loader_time: 0.08720 (0.04267) + + + --> STEP: 110/234 -- GLOBAL_STEP: 29360 + | > loss: -0.09970 (-0.08182) + | > log_mle: -0.26940 (-0.23389) + | > loss_dur: 0.16970 (0.15208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.90023 (11.81633) + | > current_lr: 0.00003 + | > step_time: 0.99250 (2.53329) + | > loader_time: 0.00190 (0.04320) + + + --> STEP: 115/234 -- GLOBAL_STEP: 29365 + | > loss: -0.09876 (-0.08331) + | > log_mle: -0.29017 (-0.23711) + | > loss_dur: 0.19140 (0.15380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.31165 (12.36302) + | > current_lr: 0.00003 + | > step_time: 1.61410 (2.51874) + | > loader_time: 0.08530 (0.04217) + + + --> STEP: 120/234 -- GLOBAL_STEP: 29370 + | > loss: -0.13775 (-0.08441) + | > log_mle: -0.33766 (-0.23983) + | > loss_dur: 0.19992 (0.15543) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.67748 (12.84143) + | > current_lr: 0.00003 + | > step_time: 2.08900 (2.49855) + | > loader_time: 0.00360 (0.04054) + + + --> STEP: 125/234 -- GLOBAL_STEP: 29375 + | > loss: -0.13847 (-0.08525) + | > log_mle: -0.32600 (-0.24136) + | > loss_dur: 0.18753 (0.15611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.62226 (13.16131) + | > current_lr: 0.00003 + | > step_time: 2.71510 (2.51461) + | > loader_time: 0.08570 (0.04044) + + + --> STEP: 130/234 -- GLOBAL_STEP: 29380 + | > loss: -0.12708 (-0.08692) + | > log_mle: -0.33447 (-0.24472) + | > loss_dur: 0.20739 (0.15780) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.73751 (13.87066) + | > current_lr: 0.00003 + | > step_time: 1.18630 (2.52841) + | > loader_time: 0.00490 (0.04036) + + + --> STEP: 135/234 -- GLOBAL_STEP: 29385 + | > loss: -0.09747 (-0.08872) + | > log_mle: -0.26895 (-0.24801) + | > loss_dur: 0.17148 (0.15928) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.81801 (14.44881) + | > current_lr: 0.00003 + | > step_time: 2.21360 (2.51285) + | > loader_time: 0.08680 (0.04155) + + + --> STEP: 140/234 -- GLOBAL_STEP: 29390 + | > loss: -0.10943 (-0.09088) + | > log_mle: -0.30053 (-0.25183) + | > loss_dur: 0.19110 (0.16095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.47363 (15.35618) + | > current_lr: 0.00003 + | > step_time: 2.19220 (2.54682) + | > loader_time: 0.00210 (0.04146) + + + --> STEP: 145/234 -- GLOBAL_STEP: 29395 + | > loss: -0.19431 (-0.09346) + | > log_mle: -0.40279 (-0.25632) + | > loss_dur: 0.20848 (0.16286) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.13481 (16.31248) + | > current_lr: 0.00003 + | > step_time: 2.19610 (2.54793) + | > loader_time: 0.00280 (0.04078) + + + --> STEP: 150/234 -- GLOBAL_STEP: 29400 + | > loss: -0.16915 (-0.09618) + | > log_mle: -0.38512 (-0.26053) + | > loss_dur: 0.21597 (0.16435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.00240 (17.10349) + | > current_lr: 0.00003 + | > step_time: 1.42390 (2.52369) + | > loader_time: 0.08730 (0.04013) + + + --> STEP: 155/234 -- GLOBAL_STEP: 29405 + | > loss: -0.22826 (-0.09988) + | > log_mle: -0.45182 (-0.26576) + | > loss_dur: 0.22356 (0.16589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.15056 (18.38026) + | > current_lr: 0.00003 + | > step_time: 8.90590 (2.57484) + | > loader_time: 0.19730 (0.04184) + + + --> STEP: 160/234 -- GLOBAL_STEP: 29410 + | > loss: -0.22102 (-0.10285) + | > log_mle: -0.44629 (-0.27031) + | > loss_dur: 0.22527 (0.16746) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.93554 (19.42875) + | > current_lr: 0.00003 + | > step_time: 2.29540 (2.56612) + | > loader_time: 0.00340 (0.04071) + + + --> STEP: 165/234 -- GLOBAL_STEP: 29415 + | > loss: -0.20062 (-0.10569) + | > log_mle: -0.43203 (-0.27459) + | > loss_dur: 0.23141 (0.16890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.16582 (20.68285) + | > current_lr: 0.00003 + | > step_time: 3.89990 (2.63382) + | > loader_time: 0.00340 (0.04139) + + + --> STEP: 170/234 -- GLOBAL_STEP: 29420 + | > loss: -0.22023 (-0.10848) + | > log_mle: -0.47157 (-0.27915) + | > loss_dur: 0.25134 (0.17067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.76363 (22.01141) + | > current_lr: 0.00003 + | > step_time: 1.98640 (2.62155) + | > loader_time: 0.00600 (0.04078) + + + --> STEP: 175/234 -- GLOBAL_STEP: 29425 + | > loss: -0.21118 (-0.11216) + | > log_mle: -0.45412 (-0.28455) + | > loss_dur: 0.24294 (0.17238) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.49449 (23.18479) + | > current_lr: 0.00003 + | > step_time: 3.80520 (2.61766) + | > loader_time: 0.29040 (0.04339) + + + --> STEP: 180/234 -- GLOBAL_STEP: 29430 + | > loss: -0.22764 (-0.11538) + | > log_mle: -0.46359 (-0.28966) + | > loss_dur: 0.23595 (0.17428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.09798 (24.24035) + | > current_lr: 0.00003 + | > step_time: 7.91220 (2.67956) + | > loader_time: 0.19380 (0.04534) + + + --> STEP: 185/234 -- GLOBAL_STEP: 29435 + | > loss: -0.23839 (-0.11820) + | > log_mle: -0.48847 (-0.29429) + | > loss_dur: 0.25008 (0.17608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.54472 (25.64962) + | > current_lr: 0.00003 + | > step_time: 5.78890 (2.78128) + | > loader_time: 0.00360 (0.04523) + + + --> STEP: 190/234 -- GLOBAL_STEP: 29440 + | > loss: -0.24882 (-0.12118) + | > log_mle: -0.47056 (-0.29891) + | > loss_dur: 0.22174 (0.17773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.34833 (26.52905) + | > current_lr: 0.00003 + | > step_time: 5.19020 (2.78911) + | > loader_time: 0.49590 (0.04763) + + + --> STEP: 195/234 -- GLOBAL_STEP: 29445 + | > loss: -0.23513 (-0.12461) + | > log_mle: -0.48412 (-0.30378) + | > loss_dur: 0.24899 (0.17917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.68203 (27.55741) + | > current_lr: 0.00003 + | > step_time: 1.89590 (2.88837) + | > loader_time: 0.00770 (0.04796) + + + --> STEP: 200/234 -- GLOBAL_STEP: 29450 + | > loss: -0.22439 (-0.12726) + | > log_mle: -0.48528 (-0.30802) + | > loss_dur: 0.26089 (0.18076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.85554 (28.84728) + | > current_lr: 0.00003 + | > step_time: 3.48870 (2.94395) + | > loader_time: 0.00670 (0.04781) + + + --> STEP: 205/234 -- GLOBAL_STEP: 29455 + | > loss: -0.23124 (-0.12991) + | > log_mle: -0.47832 (-0.31226) + | > loss_dur: 0.24708 (0.18235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.60533 (29.78985) + | > current_lr: 0.00003 + | > step_time: 5.09280 (2.98394) + | > loader_time: 0.00710 (0.04817) + + + --> STEP: 210/234 -- GLOBAL_STEP: 29460 + | > loss: -0.29710 (-0.13328) + | > log_mle: -0.55244 (-0.31728) + | > loss_dur: 0.25535 (0.18400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.44418 (31.06659) + | > current_lr: 0.00003 + | > step_time: 2.70310 (2.98904) + | > loader_time: 0.00340 (0.04856) + + + --> STEP: 215/234 -- GLOBAL_STEP: 29465 + | > loss: -0.26016 (-0.13683) + | > log_mle: -0.50614 (-0.32245) + | > loss_dur: 0.24598 (0.18562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.01702 (32.37859) + | > current_lr: 0.00003 + | > step_time: 5.29970 (3.04401) + | > loader_time: 0.19550 (0.05116) + + + --> STEP: 220/234 -- GLOBAL_STEP: 29470 + | > loss: -0.29809 (-0.14064) + | > log_mle: -0.55345 (-0.32791) + | > loss_dur: 0.25536 (0.18727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.21957 (33.52512) + | > current_lr: 0.00003 + | > step_time: 2.49930 (3.11705) + | > loader_time: 0.08790 (0.05315) + + + --> STEP: 225/234 -- GLOBAL_STEP: 29475 + | > loss: -0.34083 (-0.14415) + | > log_mle: -0.61752 (-0.33309) + | > loss_dur: 0.27669 (0.18894) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.22177 (34.84743) + | > current_lr: 0.00003 + | > step_time: 0.35920 (3.08861) + | > loader_time: 0.07650 (0.05358) + + + --> STEP: 230/234 -- GLOBAL_STEP: 29480 + | > loss: -0.31620 (-0.14744) + | > log_mle: -0.66857 (-0.33873) + | > loss_dur: 0.35237 (0.19129) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.85542 (36.31502) + | > current_lr: 0.00003 + | > step_time: 0.24250 (3.02666) + | > loader_time: 0.00260 (0.05251) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04545 (-0.00528) + | > avg_loss: -0.13470 (+0.04885) + | > avg_log_mle: -0.36393 (+0.04878) + | > avg_loss_dur: 0.22923 (+0.00007) + + + > EPOCH: 126/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 11:45:15)  + + --> STEP: 1/234 -- GLOBAL_STEP: 29485 + | > loss: -0.07873 (-0.07873) + | > log_mle: -0.22530 (-0.22530) + | > loss_dur: 0.14657 (0.14657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.97851 (9.97851) + | > current_lr: 0.00003 + | > step_time: 3.50490 (3.50495) + | > loader_time: 0.00530 (0.00528) + + + --> STEP: 6/234 -- GLOBAL_STEP: 29490 + | > loss: -0.07220 (-0.05942) + | > log_mle: -0.21352 (-0.22044) + | > loss_dur: 0.14132 (0.16102) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.28751 (12.14132) + | > current_lr: 0.00003 + | > step_time: 3.78790 (4.98369) + | > loader_time: 0.00110 (0.06626) + + + --> STEP: 11/234 -- GLOBAL_STEP: 29495 + | > loss: -0.09905 (-0.06965) + | > log_mle: -0.22448 (-0.22459) + | > loss_dur: 0.12543 (0.15494) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.58577 (11.91958) + | > current_lr: 0.00003 + | > step_time: 3.41080 (4.30034) + | > loader_time: 0.00420 (0.03723) + + + --> STEP: 16/234 -- GLOBAL_STEP: 29500 + | > loss: -0.11863 (-0.07619) + | > log_mle: -0.22415 (-0.22478) + | > loss_dur: 0.10552 (0.14859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.02620 (10.99232) + | > current_lr: 0.00003 + | > step_time: 6.41650 (4.72718) + | > loader_time: 0.09400 (0.05055) + + + --> STEP: 21/234 -- GLOBAL_STEP: 29505 + | > loss: -0.06473 (-0.07531) + | > log_mle: -0.20295 (-0.22146) + | > loss_dur: 0.13822 (0.14615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.26298 (9.97761) + | > current_lr: 0.00003 + | > step_time: 2.10530 (4.66537) + | > loader_time: 0.00270 (0.05643) + + + --> STEP: 26/234 -- GLOBAL_STEP: 29510 + | > loss: -0.07939 (-0.07789) + | > log_mle: -0.22068 (-0.22095) + | > loss_dur: 0.14129 (0.14305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.74312 (9.45680) + | > current_lr: 0.00003 + | > step_time: 3.40700 (4.60734) + | > loader_time: 0.00370 (0.05309) + + + --> STEP: 31/234 -- GLOBAL_STEP: 29515 + | > loss: -0.04497 (-0.07955) + | > log_mle: -0.22124 (-0.22092) + | > loss_dur: 0.17627 (0.14137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.56537 (9.15082) + | > current_lr: 0.00003 + | > step_time: 2.29260 (4.38712) + | > loader_time: 0.00190 (0.05695) + + + --> STEP: 36/234 -- GLOBAL_STEP: 29520 + | > loss: -0.06662 (-0.08008) + | > log_mle: -0.22295 (-0.22119) + | > loss_dur: 0.15632 (0.14111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.95328 (9.08992) + | > current_lr: 0.00003 + | > step_time: 3.39770 (4.06942) + | > loader_time: 0.00300 (0.05426) + + + --> STEP: 41/234 -- GLOBAL_STEP: 29525 + | > loss: -0.10155 (-0.07999) + | > log_mle: -0.21899 (-0.22088) + | > loss_dur: 0.11744 (0.14090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.61026 (9.00696) + | > current_lr: 0.00003 + | > step_time: 2.10910 (3.84195) + | > loader_time: 0.00310 (0.04822) + + + --> STEP: 46/234 -- GLOBAL_STEP: 29530 + | > loss: -0.07048 (-0.07992) + | > log_mle: -0.21993 (-0.22112) + | > loss_dur: 0.14946 (0.14120) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.05747 (8.98375) + | > current_lr: 0.00003 + | > step_time: 1.82290 (3.58340) + | > loader_time: 0.00290 (0.04688) + + + --> STEP: 51/234 -- GLOBAL_STEP: 29535 + | > loss: -0.06378 (-0.07960) + | > log_mle: -0.20868 (-0.22043) + | > loss_dur: 0.14490 (0.14083) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.38450 (8.75725) + | > current_lr: 0.00003 + | > step_time: 1.67460 (3.40904) + | > loader_time: 0.00200 (0.04413) + + + --> STEP: 56/234 -- GLOBAL_STEP: 29540 + | > loss: -0.05230 (-0.07942) + | > log_mle: -0.22498 (-0.22089) + | > loss_dur: 0.17268 (0.14147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.14372 (8.66261) + | > current_lr: 0.00003 + | > step_time: 1.48430 (3.26751) + | > loader_time: 0.00250 (0.04044) + + + --> STEP: 61/234 -- GLOBAL_STEP: 29545 + | > loss: -0.09011 (-0.08016) + | > log_mle: -0.22467 (-0.22138) + | > loss_dur: 0.13456 (0.14122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.35696 (8.65806) + | > current_lr: 0.00003 + | > step_time: 1.49840 (3.14061) + | > loader_time: 0.00310 (0.03868) + + + --> STEP: 66/234 -- GLOBAL_STEP: 29550 + | > loss: -0.07403 (-0.07950) + | > log_mle: -0.21088 (-0.22173) + | > loss_dur: 0.13685 (0.14223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.72783 (8.76058) + | > current_lr: 0.00003 + | > step_time: 2.09230 (3.03942) + | > loader_time: 0.00360 (0.03599) + + + --> STEP: 71/234 -- GLOBAL_STEP: 29555 + | > loss: -0.06351 (-0.07817) + | > log_mle: -0.25202 (-0.22206) + | > loss_dur: 0.18850 (0.14390) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.66233 (8.99561) + | > current_lr: 0.00003 + | > step_time: 1.81290 (2.94789) + | > loader_time: 0.08670 (0.03602) + + + --> STEP: 76/234 -- GLOBAL_STEP: 29560 + | > loss: -0.07862 (-0.07747) + | > log_mle: -0.23441 (-0.22237) + | > loss_dur: 0.15579 (0.14490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.66427 (9.04785) + | > current_lr: 0.00003 + | > step_time: 2.80150 (2.89503) + | > loader_time: 0.10200 (0.03624) + + + --> STEP: 81/234 -- GLOBAL_STEP: 29565 + | > loss: -0.09233 (-0.07786) + | > log_mle: -0.24098 (-0.22255) + | > loss_dur: 0.14866 (0.14469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.51965 (9.06194) + | > current_lr: 0.00003 + | > step_time: 2.21140 (2.85454) + | > loader_time: 0.00200 (0.03729) + + + --> STEP: 86/234 -- GLOBAL_STEP: 29570 + | > loss: -0.07063 (-0.07785) + | > log_mle: -0.23777 (-0.22302) + | > loss_dur: 0.16714 (0.14517) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.65768 (9.25359) + | > current_lr: 0.00003 + | > step_time: 2.00070 (2.78864) + | > loader_time: 0.00960 (0.03645) + + + --> STEP: 91/234 -- GLOBAL_STEP: 29575 + | > loss: -0.08268 (-0.07821) + | > log_mle: -0.25044 (-0.22460) + | > loss_dur: 0.16776 (0.14639) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.91882 (9.56413) + | > current_lr: 0.00003 + | > step_time: 4.40160 (2.82096) + | > loader_time: 0.08670 (0.03654) + + + --> STEP: 96/234 -- GLOBAL_STEP: 29580 + | > loss: -0.07123 (-0.07995) + | > log_mle: -0.23758 (-0.22781) + | > loss_dur: 0.16635 (0.14786) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.98383 (10.21457) + | > current_lr: 0.00003 + | > step_time: 1.59270 (2.77944) + | > loader_time: 0.00120 (0.03557) + + + --> STEP: 101/234 -- GLOBAL_STEP: 29585 + | > loss: -0.11609 (-0.08104) + | > log_mle: -0.29564 (-0.23010) + | > loss_dur: 0.17955 (0.14906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.80230 (10.57113) + | > current_lr: 0.00003 + | > step_time: 1.99270 (2.73858) + | > loader_time: 0.00590 (0.03494) + + + --> STEP: 106/234 -- GLOBAL_STEP: 29590 + | > loss: -0.09032 (-0.08233) + | > log_mle: -0.29525 (-0.23285) + | > loss_dur: 0.20493 (0.15052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.25965 (11.10066) + | > current_lr: 0.00003 + | > step_time: 1.57630 (2.70349) + | > loader_time: 0.00220 (0.03420) + + + --> STEP: 111/234 -- GLOBAL_STEP: 29595 + | > loss: -0.13564 (-0.08343) + | > log_mle: -0.34260 (-0.23567) + | > loss_dur: 0.20696 (0.15224) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.48269 (11.75787) + | > current_lr: 0.00003 + | > step_time: 4.71330 (2.75761) + | > loader_time: 0.09410 (0.03532) + + + --> STEP: 116/234 -- GLOBAL_STEP: 29600 + | > loss: -0.08794 (-0.08443) + | > log_mle: -0.30857 (-0.23847) + | > loss_dur: 0.22063 (0.15404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.05867 (12.35351) + | > current_lr: 0.00003 + | > step_time: 0.79530 (2.70598) + | > loader_time: 0.00320 (0.03532) + + + --> STEP: 121/234 -- GLOBAL_STEP: 29605 + | > loss: -0.05454 (-0.08484) + | > log_mle: -0.22068 (-0.24009) + | > loss_dur: 0.16614 (0.15525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.76097 (13.50495) + | > current_lr: 0.00003 + | > step_time: 3.40160 (2.68916) + | > loader_time: 0.18620 (0.03694) + + + --> STEP: 126/234 -- GLOBAL_STEP: 29610 + | > loss: -0.15100 (-0.08621) + | > log_mle: -0.35312 (-0.24250) + | > loss_dur: 0.20213 (0.15628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.95703 (14.01663) + | > current_lr: 0.00003 + | > step_time: 1.20090 (2.64692) + | > loader_time: 0.00490 (0.03699) + + + --> STEP: 131/234 -- GLOBAL_STEP: 29615 + | > loss: -0.18579 (-0.08822) + | > log_mle: -0.39700 (-0.24600) + | > loss_dur: 0.21121 (0.15778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.97635 (14.73543) + | > current_lr: 0.00003 + | > step_time: 1.60000 (2.61677) + | > loader_time: 0.09920 (0.03643) + + + --> STEP: 136/234 -- GLOBAL_STEP: 29620 + | > loss: -0.20592 (-0.09038) + | > log_mle: -0.44233 (-0.24957) + | > loss_dur: 0.23642 (0.15919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.53569 (15.31778) + | > current_lr: 0.00003 + | > step_time: 3.19030 (2.61415) + | > loader_time: 0.00350 (0.03647) + + + --> STEP: 141/234 -- GLOBAL_STEP: 29625 + | > loss: -0.14232 (-0.09191) + | > log_mle: -0.35399 (-0.25268) + | > loss_dur: 0.21167 (0.16077) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.42094 (15.89771) + | > current_lr: 0.00003 + | > step_time: 7.31280 (2.62880) + | > loader_time: 0.09630 (0.03722) + + + --> STEP: 146/234 -- GLOBAL_STEP: 29630 + | > loss: -0.18767 (-0.09455) + | > log_mle: -0.39508 (-0.25725) + | > loss_dur: 0.20741 (0.16271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.20052 (17.17511) + | > current_lr: 0.00003 + | > step_time: 6.81530 (2.64567) + | > loader_time: 0.08490 (0.03739) + + + --> STEP: 151/234 -- GLOBAL_STEP: 29635 + | > loss: -0.15550 (-0.09667) + | > log_mle: -0.35555 (-0.26075) + | > loss_dur: 0.20005 (0.16408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.05091 (19.17150) + | > current_lr: 0.00003 + | > step_time: 3.30270 (2.61760) + | > loader_time: 0.00370 (0.03742) + + + --> STEP: 156/234 -- GLOBAL_STEP: 29640 + | > loss: -0.19166 (-0.10001) + | > log_mle: -0.40675 (-0.26585) + | > loss_dur: 0.21510 (0.16584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.11544 (20.04692) + | > current_lr: 0.00003 + | > step_time: 12.50300 (2.68759) + | > loader_time: 0.11070 (0.03877) + + + --> STEP: 161/234 -- GLOBAL_STEP: 29645 + | > loss: -0.21327 (-0.10298) + | > log_mle: -0.42968 (-0.27039) + | > loss_dur: 0.21641 (0.16741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.01287 (20.87924) + | > current_lr: 0.00003 + | > step_time: 3.01040 (2.67180) + | > loader_time: 0.29700 (0.03950) + + + --> STEP: 166/234 -- GLOBAL_STEP: 29650 + | > loss: -0.17764 (-0.10556) + | > log_mle: -0.37647 (-0.27432) + | > loss_dur: 0.19883 (0.16876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.49151 (21.53869) + | > current_lr: 0.00003 + | > step_time: 2.51040 (2.73651) + | > loader_time: 0.19380 (0.04246) + + + --> STEP: 171/234 -- GLOBAL_STEP: 29655 + | > loss: -0.26184 (-0.10913) + | > log_mle: -0.48017 (-0.27962) + | > loss_dur: 0.21834 (0.17049) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.02202 (22.65050) + | > current_lr: 0.00003 + | > step_time: 4.00630 (2.77704) + | > loader_time: 0.00430 (0.04240) + + + --> STEP: 176/234 -- GLOBAL_STEP: 29660 + | > loss: -0.21979 (-0.11265) + | > log_mle: -0.45668 (-0.28494) + | > loss_dur: 0.23689 (0.17229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.42704 (23.60958) + | > current_lr: 0.00003 + | > step_time: 1.70980 (2.82550) + | > loader_time: 0.00490 (0.04307) + + + --> STEP: 181/234 -- GLOBAL_STEP: 29665 + | > loss: -0.16688 (-0.11550) + | > log_mle: -0.39109 (-0.28965) + | > loss_dur: 0.22421 (0.17415) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.01129 (24.56224) + | > current_lr: 0.00003 + | > step_time: 4.72330 (2.85093) + | > loader_time: 0.00360 (0.04386) + + + --> STEP: 186/234 -- GLOBAL_STEP: 29670 + | > loss: -0.18593 (-0.11841) + | > log_mle: -0.42930 (-0.29446) + | > loss_dur: 0.24337 (0.17605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.36194 (25.60983) + | > current_lr: 0.00003 + | > step_time: 3.89550 (2.87205) + | > loader_time: 0.00690 (0.04372) + + + --> STEP: 191/234 -- GLOBAL_STEP: 29675 + | > loss: -0.22981 (-0.12143) + | > log_mle: -0.44762 (-0.29905) + | > loss_dur: 0.21782 (0.17762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.22946 (26.91951) + | > current_lr: 0.00003 + | > step_time: 5.19800 (2.89278) + | > loader_time: 0.09270 (0.04455) + + + --> STEP: 196/234 -- GLOBAL_STEP: 29680 + | > loss: -0.20195 (-0.12464) + | > log_mle: -0.44572 (-0.30382) + | > loss_dur: 0.24378 (0.17918) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.54814 (27.99990) + | > current_lr: 0.00003 + | > step_time: 4.30660 (2.98228) + | > loader_time: 0.00410 (0.04654) + + + --> STEP: 201/234 -- GLOBAL_STEP: 29685 + | > loss: -0.16296 (-0.12734) + | > log_mle: -0.41087 (-0.30815) + | > loss_dur: 0.24791 (0.18081) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.65662 (28.90743) + | > current_lr: 0.00003 + | > step_time: 3.30150 (3.03854) + | > loader_time: 0.00440 (0.04944) + + + --> STEP: 206/234 -- GLOBAL_STEP: 29690 + | > loss: -0.25711 (-0.13051) + | > log_mle: -0.50486 (-0.31292) + | > loss_dur: 0.24775 (0.18241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.59651 (30.06703) + | > current_lr: 0.00003 + | > step_time: 8.29860 (3.06820) + | > loader_time: 0.09130 (0.04929) + + + --> STEP: 211/234 -- GLOBAL_STEP: 29695 + | > loss: -0.30910 (-0.13401) + | > log_mle: -0.58428 (-0.31819) + | > loss_dur: 0.27519 (0.18418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.87386 (31.47814) + | > current_lr: 0.00003 + | > step_time: 3.29610 (3.08702) + | > loader_time: 0.00290 (0.05024) + + + --> STEP: 216/234 -- GLOBAL_STEP: 29700 + | > loss: -0.30792 (-0.13761) + | > log_mle: -0.57594 (-0.32333) + | > loss_dur: 0.26802 (0.18572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.29662 (32.52849) + | > current_lr: 0.00003 + | > step_time: 6.09910 (3.13600) + | > loader_time: 0.10360 (0.05083) + + + --> STEP: 221/234 -- GLOBAL_STEP: 29705 + | > loss: -0.25696 (-0.14129) + | > log_mle: -0.49733 (-0.32850) + | > loss_dur: 0.24037 (0.18721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.98302 (33.67547) + | > current_lr: 0.00003 + | > step_time: 1.69620 (3.19896) + | > loader_time: 0.00760 (0.05025) + + + --> STEP: 226/234 -- GLOBAL_STEP: 29710 + | > loss: -0.33014 (-0.14515) + | > log_mle: -0.59274 (-0.33414) + | > loss_dur: 0.26260 (0.18899) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.52478 (34.81226) + | > current_lr: 0.00003 + | > step_time: 1.49220 (3.17178) + | > loader_time: 0.00300 (0.04958) + + + --> STEP: 231/234 -- GLOBAL_STEP: 29715 + | > loss: -0.25758 (-0.14812) + | > log_mle: -0.66496 (-0.34013) + | > loss_dur: 0.40738 (0.19201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.88654 (36.01461) + | > current_lr: 0.00003 + | > step_time: 0.27770 (3.11004) + | > loader_time: 0.00390 (0.04861) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.60201 (+0.55656) + | > avg_loss: -0.12850 (+0.00620) + | > avg_log_mle: -0.36829 (-0.00436) + | > avg_loss_dur: 0.23979 (+0.01056) + + + > EPOCH: 127/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 11:58:29)  + + --> STEP: 2/234 -- GLOBAL_STEP: 29720 + | > loss: -0.06308 (-0.07219) + | > log_mle: -0.22029 (-0.22406) + | > loss_dur: 0.15721 (0.15187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.97021 (10.12182) + | > current_lr: 0.00003 + | > step_time: 8.50190 (8.89439) + | > loader_time: 0.10290 (0.15126) + + + --> STEP: 7/234 -- GLOBAL_STEP: 29725 + | > loss: -0.08926 (-0.05938) + | > log_mle: -0.23128 (-0.22146) + | > loss_dur: 0.14202 (0.16208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.37881 (17.78881) + | > current_lr: 0.00003 + | > step_time: 6.39920 (4.32339) + | > loader_time: 0.10730 (0.05981) + + + --> STEP: 12/234 -- GLOBAL_STEP: 29730 + | > loss: -0.09035 (-0.07031) + | > log_mle: -0.22499 (-0.22419) + | > loss_dur: 0.13464 (0.15388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.88260 (14.84551) + | > current_lr: 0.00003 + | > step_time: 1.20000 (3.19943) + | > loader_time: 0.00170 (0.04292) + + + --> STEP: 17/234 -- GLOBAL_STEP: 29735 + | > loss: -0.06222 (-0.07674) + | > log_mle: -0.20244 (-0.22335) + | > loss_dur: 0.14021 (0.14661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.86254 (12.70426) + | > current_lr: 0.00003 + | > step_time: 4.59930 (3.07159) + | > loader_time: 0.00170 (0.03104) + + + --> STEP: 22/234 -- GLOBAL_STEP: 29740 + | > loss: -0.09329 (-0.07811) + | > log_mle: -0.22432 (-0.22127) + | > loss_dur: 0.13103 (0.14316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.59678 (11.94271) + | > current_lr: 0.00003 + | > step_time: 1.56810 (3.09537) + | > loader_time: 0.01060 (0.03252) + + + --> STEP: 27/234 -- GLOBAL_STEP: 29745 + | > loss: -0.08910 (-0.08039) + | > log_mle: -0.22499 (-0.22071) + | > loss_dur: 0.13588 (0.14031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.05271 (11.23127) + | > current_lr: 0.00003 + | > step_time: 4.01010 (2.86575) + | > loader_time: 0.00230 (0.02683) + + + --> STEP: 32/234 -- GLOBAL_STEP: 29750 + | > loss: -0.11325 (-0.08137) + | > log_mle: -0.23363 (-0.22078) + | > loss_dur: 0.12039 (0.13941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.08550 (10.64054) + | > current_lr: 0.00003 + | > step_time: 4.39690 (2.98697) + | > loader_time: 0.09950 (0.02891) + + + --> STEP: 37/234 -- GLOBAL_STEP: 29755 + | > loss: -0.10200 (-0.08100) + | > log_mle: -0.21716 (-0.22036) + | > loss_dur: 0.11516 (0.13935) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.38579 (10.57532) + | > current_lr: 0.00003 + | > step_time: 2.79950 (3.10766) + | > loader_time: 0.08570 (0.03809) + + + --> STEP: 42/234 -- GLOBAL_STEP: 29760 + | > loss: -0.06472 (-0.07999) + | > log_mle: -0.20894 (-0.21998) + | > loss_dur: 0.14422 (0.13999) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.02109 (10.31137) + | > current_lr: 0.00003 + | > step_time: 1.28750 (2.92143) + | > loader_time: 0.00260 (0.03758) + + + --> STEP: 47/234 -- GLOBAL_STEP: 29765 + | > loss: -0.06540 (-0.07997) + | > log_mle: -0.21831 (-0.22054) + | > loss_dur: 0.15291 (0.14057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.57288 (10.12776) + | > current_lr: 0.00003 + | > step_time: 1.10330 (2.79124) + | > loader_time: 0.08540 (0.03736) + + + --> STEP: 52/234 -- GLOBAL_STEP: 29770 + | > loss: -0.06272 (-0.07965) + | > log_mle: -0.21630 (-0.22001) + | > loss_dur: 0.15358 (0.14036) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.91774 (9.75321) + | > current_lr: 0.00003 + | > step_time: 1.80660 (2.67153) + | > loader_time: 0.08740 (0.03862) + + + --> STEP: 57/234 -- GLOBAL_STEP: 29775 + | > loss: -0.06206 (-0.07971) + | > log_mle: -0.20820 (-0.22051) + | > loss_dur: 0.14615 (0.14080) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.75679 (9.54258) + | > current_lr: 0.00003 + | > step_time: 1.80050 (2.58064) + | > loader_time: 0.09770 (0.03884) + + + --> STEP: 62/234 -- GLOBAL_STEP: 29780 + | > loss: -0.06625 (-0.08056) + | > log_mle: -0.26173 (-0.22193) + | > loss_dur: 0.19549 (0.14137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.36690 (9.67572) + | > current_lr: 0.00003 + | > step_time: 1.97500 (2.50155) + | > loader_time: 0.00230 (0.03878) + + + --> STEP: 67/234 -- GLOBAL_STEP: 29785 + | > loss: -0.07286 (-0.08027) + | > log_mle: -0.24287 (-0.22207) + | > loss_dur: 0.17002 (0.14180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.90273 (9.47305) + | > current_lr: 0.00003 + | > step_time: 1.60520 (2.43301) + | > loader_time: 0.00330 (0.03860) + + + --> STEP: 72/234 -- GLOBAL_STEP: 29790 + | > loss: -0.06683 (-0.07886) + | > log_mle: -0.22133 (-0.22225) + | > loss_dur: 0.15449 (0.14339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.61154 (9.49093) + | > current_lr: 0.00003 + | > step_time: 2.18980 (2.40965) + | > loader_time: 0.00220 (0.03845) + + + --> STEP: 77/234 -- GLOBAL_STEP: 29795 + | > loss: -0.08968 (-0.07892) + | > log_mle: -0.23392 (-0.22292) + | > loss_dur: 0.14424 (0.14400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.57915 (9.53257) + | > current_lr: 0.00003 + | > step_time: 4.24120 (2.37695) + | > loader_time: 0.00390 (0.03613) + + + --> STEP: 82/234 -- GLOBAL_STEP: 29800 + | > loss: -0.08619 (-0.07907) + | > log_mle: -0.22375 (-0.22311) + | > loss_dur: 0.13755 (0.14404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.84957 (9.51581) + | > current_lr: 0.00003 + | > step_time: 3.50810 (2.37481) + | > loader_time: 0.09970 (0.03644) + + + --> STEP: 87/234 -- GLOBAL_STEP: 29805 + | > loss: -0.06844 (-0.07883) + | > log_mle: -0.23157 (-0.22379) + | > loss_dur: 0.16313 (0.14497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.99498 (9.64252) + | > current_lr: 0.00003 + | > step_time: 3.30410 (2.35686) + | > loader_time: 0.07690 (0.03538) + + + --> STEP: 92/234 -- GLOBAL_STEP: 29810 + | > loss: -0.12502 (-0.07996) + | > log_mle: -0.28027 (-0.22613) + | > loss_dur: 0.15525 (0.14617) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.91664 (9.93454) + | > current_lr: 0.00003 + | > step_time: 1.70340 (2.34499) + | > loader_time: 0.00250 (0.03553) + + + --> STEP: 97/234 -- GLOBAL_STEP: 29815 + | > loss: -0.09863 (-0.08185) + | > log_mle: -0.26839 (-0.22941) + | > loss_dur: 0.16977 (0.14756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.19306 (10.47894) + | > current_lr: 0.00003 + | > step_time: 2.30260 (2.30467) + | > loader_time: 0.00320 (0.03468) + + + --> STEP: 102/234 -- GLOBAL_STEP: 29820 + | > loss: -0.07622 (-0.08290) + | > log_mle: -0.24873 (-0.23159) + | > loss_dur: 0.17251 (0.14868) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.60848 (10.96754) + | > current_lr: 0.00003 + | > step_time: 2.29350 (2.29069) + | > loader_time: 0.00440 (0.03566) + + + --> STEP: 107/234 -- GLOBAL_STEP: 29825 + | > loss: -0.11657 (-0.08458) + | > log_mle: -0.29760 (-0.23486) + | > loss_dur: 0.18103 (0.15028) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.34908 (11.67488) + | > current_lr: 0.00003 + | > step_time: 1.80020 (2.31090) + | > loader_time: 0.07590 (0.03725) + + + --> STEP: 112/234 -- GLOBAL_STEP: 29830 + | > loss: -0.11056 (-0.08579) + | > log_mle: -0.30502 (-0.23783) + | > loss_dur: 0.19446 (0.15204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.90796 (12.42289) + | > current_lr: 0.00003 + | > step_time: 1.50030 (2.27929) + | > loader_time: 0.08360 (0.03717) + + + --> STEP: 117/234 -- GLOBAL_STEP: 29835 + | > loss: -0.12561 (-0.08701) + | > log_mle: -0.30380 (-0.24062) + | > loss_dur: 0.17820 (0.15361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.81471 (12.99330) + | > current_lr: 0.00003 + | > step_time: 1.39590 (2.30161) + | > loader_time: 0.00210 (0.03806) + + + --> STEP: 122/234 -- GLOBAL_STEP: 29840 + | > loss: -0.10881 (-0.08769) + | > log_mle: -0.28043 (-0.24245) + | > loss_dur: 0.17162 (0.15475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.40171 (13.16449) + | > current_lr: 0.00003 + | > step_time: 2.39050 (2.29821) + | > loader_time: 0.00260 (0.03799) + + + --> STEP: 127/234 -- GLOBAL_STEP: 29845 + | > loss: -0.13453 (-0.08945) + | > log_mle: -0.33368 (-0.24543) + | > loss_dur: 0.19915 (0.15598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.54109 (13.88806) + | > current_lr: 0.00003 + | > step_time: 2.20980 (2.29606) + | > loader_time: 0.00320 (0.03733) + + + --> STEP: 132/234 -- GLOBAL_STEP: 29850 + | > loss: -0.14169 (-0.09160) + | > log_mle: -0.31661 (-0.24889) + | > loss_dur: 0.17493 (0.15730) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.97343 (14.65939) + | > current_lr: 0.00003 + | > step_time: 1.58820 (2.28612) + | > loader_time: 0.00330 (0.03817) + + + --> STEP: 137/234 -- GLOBAL_STEP: 29855 + | > loss: -0.11180 (-0.09353) + | > log_mle: -0.32735 (-0.25259) + | > loss_dur: 0.21555 (0.15907) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.10666 (15.37245) + | > current_lr: 0.00003 + | > step_time: 1.19570 (2.29098) + | > loader_time: 0.00340 (0.03803) + + + --> STEP: 142/234 -- GLOBAL_STEP: 29860 + | > loss: -0.14112 (-0.09518) + | > log_mle: -0.34297 (-0.25576) + | > loss_dur: 0.20185 (0.16057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.55704 (16.10636) + | > current_lr: 0.00003 + | > step_time: 1.29800 (2.30827) + | > loader_time: 0.00360 (0.03802) + + + --> STEP: 147/234 -- GLOBAL_STEP: 29865 + | > loss: -0.15299 (-0.09811) + | > log_mle: -0.34769 (-0.26056) + | > loss_dur: 0.19470 (0.16245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.60499 (17.13881) + | > current_lr: 0.00003 + | > step_time: 1.09320 (2.30279) + | > loader_time: 0.00320 (0.03860) + + + --> STEP: 152/234 -- GLOBAL_STEP: 29870 + | > loss: -0.18930 (-0.10105) + | > log_mle: -0.42586 (-0.26495) + | > loss_dur: 0.23656 (0.16390) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.93167 (18.13194) + | > current_lr: 0.00003 + | > step_time: 5.99460 (2.31314) + | > loader_time: 0.10550 (0.03813) + + + --> STEP: 157/234 -- GLOBAL_STEP: 29875 + | > loss: -0.15529 (-0.10450) + | > log_mle: -0.37659 (-0.27001) + | > loss_dur: 0.22131 (0.16552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.36496 (19.17534) + | > current_lr: 0.00003 + | > step_time: 1.40870 (2.32617) + | > loader_time: 0.00270 (0.03812) + + + --> STEP: 162/234 -- GLOBAL_STEP: 29880 + | > loss: -0.20419 (-0.10773) + | > log_mle: -0.40925 (-0.27488) + | > loss_dur: 0.20506 (0.16714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.64058 (20.17672) + | > current_lr: 0.00003 + | > step_time: 2.30940 (2.33581) + | > loader_time: 0.09640 (0.03877) + + + --> STEP: 167/234 -- GLOBAL_STEP: 29885 + | > loss: -0.28106 (-0.11089) + | > log_mle: -0.49396 (-0.27935) + | > loss_dur: 0.21290 (0.16846) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.02940 (21.08967) + | > current_lr: 0.00003 + | > step_time: 5.69670 (2.34615) + | > loader_time: 0.09850 (0.03928) + + + --> STEP: 172/234 -- GLOBAL_STEP: 29890 + | > loss: -0.24070 (-0.11438) + | > log_mle: -0.48757 (-0.28473) + | > loss_dur: 0.24687 (0.17035) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.91945 (22.10202) + | > current_lr: 0.00003 + | > step_time: 1.51350 (2.35348) + | > loader_time: 0.00470 (0.03952) + + + --> STEP: 177/234 -- GLOBAL_STEP: 29895 + | > loss: -0.20051 (-0.11758) + | > log_mle: -0.43438 (-0.28965) + | > loss_dur: 0.23388 (0.17208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.45650 (23.56257) + | > current_lr: 0.00003 + | > step_time: 2.30240 (2.35158) + | > loader_time: 0.09840 (0.03905) + + + --> STEP: 182/234 -- GLOBAL_STEP: 29900 + | > loss: -0.22391 (-0.12024) + | > log_mle: -0.47754 (-0.29437) + | > loss_dur: 0.25364 (0.17413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.59903 (24.94636) + | > current_lr: 0.00003 + | > step_time: 8.39980 (2.44132) + | > loader_time: 0.18900 (0.04284) + + + --> STEP: 187/234 -- GLOBAL_STEP: 29905 + | > loss: -0.25312 (-0.12329) + | > log_mle: -0.48766 (-0.29919) + | > loss_dur: 0.23454 (0.17590) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.40324 (25.84725) + | > current_lr: 0.00003 + | > step_time: 5.09540 (2.54392) + | > loader_time: 0.10000 (0.04384) + + + --> STEP: 192/234 -- GLOBAL_STEP: 29910 + | > loss: -0.29037 (-0.12663) + | > log_mle: -0.51435 (-0.30398) + | > loss_dur: 0.22398 (0.17735) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.83681 (26.82866) + | > current_lr: 0.00003 + | > step_time: 6.29830 (2.65000) + | > loader_time: 0.10580 (0.04489) + + + --> STEP: 197/234 -- GLOBAL_STEP: 29915 + | > loss: -0.26076 (-0.12975) + | > log_mle: -0.48655 (-0.30861) + | > loss_dur: 0.22578 (0.17886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.56063 (27.78157) + | > current_lr: 0.00003 + | > step_time: 9.98950 (2.73949) + | > loader_time: 0.00380 (0.04580) + + + --> STEP: 202/234 -- GLOBAL_STEP: 29920 + | > loss: -0.33307 (-0.13275) + | > log_mle: -0.56900 (-0.31324) + | > loss_dur: 0.23593 (0.18049) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.27215 (28.99713) + | > current_lr: 0.00003 + | > step_time: 10.81150 (2.83909) + | > loader_time: 0.09570 (0.04859) + + + --> STEP: 207/234 -- GLOBAL_STEP: 29925 + | > loss: -0.30577 (-0.13570) + | > log_mle: -0.55057 (-0.31779) + | > loss_dur: 0.24481 (0.18209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.40218 (30.14169) + | > current_lr: 0.00003 + | > step_time: 8.00150 (2.89713) + | > loader_time: 0.29510 (0.05036) + + + --> STEP: 212/234 -- GLOBAL_STEP: 29930 + | > loss: -0.28780 (-0.13917) + | > log_mle: -0.53886 (-0.32298) + | > loss_dur: 0.25106 (0.18381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.65304 (31.32078) + | > current_lr: 0.00003 + | > step_time: 5.30380 (2.95000) + | > loader_time: 0.08520 (0.05184) + + + --> STEP: 217/234 -- GLOBAL_STEP: 29935 + | > loss: -0.30249 (-0.14275) + | > log_mle: -0.56171 (-0.32818) + | > loss_dur: 0.25922 (0.18542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.55019 (32.53834) + | > current_lr: 0.00003 + | > step_time: 7.90310 (3.00472) + | > loader_time: 0.09210 (0.05204) + + + --> STEP: 222/234 -- GLOBAL_STEP: 29940 + | > loss: -0.29518 (-0.14628) + | > log_mle: -0.57871 (-0.33333) + | > loss_dur: 0.28353 (0.18705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.78472 (33.61717) + | > current_lr: 0.00003 + | > step_time: 3.60390 (3.02312) + | > loader_time: 0.08840 (0.05216) + + + --> STEP: 227/234 -- GLOBAL_STEP: 29945 + | > loss: -0.26623 (-0.14978) + | > log_mle: -0.53840 (-0.33859) + | > loss_dur: 0.27217 (0.18881) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.59618 (35.08158) + | > current_lr: 0.00003 + | > step_time: 0.24460 (2.98518) + | > loader_time: 0.00380 (0.05186) + + + --> STEP: 232/234 -- GLOBAL_STEP: 29950 + | > loss: -0.20504 (-0.15222) + | > log_mle: -0.73697 (-0.34512) + | > loss_dur: 0.53193 (0.19290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 142.45947 (36.88049) + | > current_lr: 0.00003 + | > step_time: 0.37090 (2.92696) + | > loader_time: 0.07230 (0.05112) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40039 (-0.20161) + | > avg_loss: -0.18497 (-0.05646) + | > avg_log_mle: -0.41275 (-0.04446) + | > avg_loss_dur: 0.22779 (-0.01200) + + + > EPOCH: 128/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 12:11:11)  + + --> STEP: 3/234 -- GLOBAL_STEP: 29955 + | > loss: -0.02816 (-0.06021) + | > log_mle: -0.22781 (-0.22620) + | > loss_dur: 0.19964 (0.16600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.63740 (12.59739) + | > current_lr: 0.00003 + | > step_time: 3.50120 (3.16896) + | > loader_time: 0.09530 (1.73369) + + + --> STEP: 8/234 -- GLOBAL_STEP: 29960 + | > loss: -0.08567 (-0.06832) + | > log_mle: -0.23974 (-0.22791) + | > loss_dur: 0.15408 (0.15959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.30409 (11.51882) + | > current_lr: 0.00003 + | > step_time: 8.49800 (4.42872) + | > loader_time: 0.00210 (0.67653) + + + --> STEP: 13/234 -- GLOBAL_STEP: 29965 + | > loss: -0.08061 (-0.07398) + | > log_mle: -0.22653 (-0.22841) + | > loss_dur: 0.14593 (0.15444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.67989 (11.16758) + | > current_lr: 0.00003 + | > step_time: 1.70190 (5.35031) + | > loader_time: 0.00130 (0.45469) + + + --> STEP: 18/234 -- GLOBAL_STEP: 29970 + | > loss: -0.07548 (-0.07845) + | > log_mle: -0.22221 (-0.22668) + | > loss_dur: 0.14673 (0.14824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.74445 (10.85826) + | > current_lr: 0.00003 + | > step_time: 1.09930 (4.33264) + | > loader_time: 0.10400 (0.33909) + + + --> STEP: 23/234 -- GLOBAL_STEP: 29975 + | > loss: -0.10960 (-0.08211) + | > log_mle: -0.22807 (-0.22493) + | > loss_dur: 0.11847 (0.14283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.74355 (10.32974) + | > current_lr: 0.00003 + | > step_time: 1.09700 (3.66268) + | > loader_time: 0.00640 (0.27012) + + + --> STEP: 28/234 -- GLOBAL_STEP: 29980 + | > loss: -0.10760 (-0.08485) + | > log_mle: -0.21976 (-0.22409) + | > loss_dur: 0.11216 (0.13923) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.18486 (9.66430) + | > current_lr: 0.00003 + | > step_time: 1.30680 (3.26532) + | > loader_time: 0.00260 (0.22225) + + + --> STEP: 33/234 -- GLOBAL_STEP: 29985 + | > loss: -0.07444 (-0.08506) + | > log_mle: -0.21322 (-0.22384) + | > loss_dur: 0.13877 (0.13878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.09701 (9.32048) + | > current_lr: 0.00003 + | > step_time: 3.59220 (3.11903) + | > loader_time: 0.00570 (0.18914) + + + --> STEP: 38/234 -- GLOBAL_STEP: 29990 + | > loss: -0.08926 (-0.08471) + | > log_mle: -0.23060 (-0.22372) + | > loss_dur: 0.14134 (0.13901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.68264 (9.42801) + | > current_lr: 0.00003 + | > step_time: 2.80520 (3.13796) + | > loader_time: 0.00220 (0.17155) + + + --> STEP: 43/234 -- GLOBAL_STEP: 29995 + | > loss: -0.06610 (-0.08390) + | > log_mle: -0.23084 (-0.22318) + | > loss_dur: 0.16474 (0.13928) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.14083 (9.34548) + | > current_lr: 0.00003 + | > step_time: 1.48810 (3.06352) + | > loader_time: 0.00200 (0.15206) + + + --> STEP: 48/234 -- GLOBAL_STEP: 30000 + | > loss: -0.08207 (-0.08411) + | > log_mle: -0.21267 (-0.22319) + | > loss_dur: 0.13060 (0.13908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.57886 (9.39794) + | > current_lr: 0.00003 + | > step_time: 1.20530 (2.93071) + | > loader_time: 0.07380 (0.13948) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_30000.pth + + --> STEP: 53/234 -- GLOBAL_STEP: 30005 + | > loss: -0.09350 (-0.08416) + | > log_mle: -0.23419 (-0.22297) + | > loss_dur: 0.14069 (0.13881) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.15729 (9.15331) + | > current_lr: 0.00003 + | > step_time: 1.08950 (2.80132) + | > loader_time: 0.00300 (0.12686) + + + --> STEP: 58/234 -- GLOBAL_STEP: 30010 + | > loss: -0.08283 (-0.08391) + | > log_mle: -0.21781 (-0.22313) + | > loss_dur: 0.13497 (0.13922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.42916 (8.93097) + | > current_lr: 0.00003 + | > step_time: 2.00610 (2.68582) + | > loader_time: 0.08630 (0.11909) + + + --> STEP: 63/234 -- GLOBAL_STEP: 30015 + | > loss: -0.05461 (-0.08363) + | > log_mle: -0.22550 (-0.22450) + | > loss_dur: 0.17090 (0.14087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.38996 (9.24643) + | > current_lr: 0.00003 + | > step_time: 0.89460 (2.62519) + | > loader_time: 0.00390 (0.11250) + + + --> STEP: 68/234 -- GLOBAL_STEP: 30020 + | > loss: -0.06177 (-0.08330) + | > log_mle: -0.21941 (-0.22439) + | > loss_dur: 0.15764 (0.14109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.88731 (9.18528) + | > current_lr: 0.00003 + | > step_time: 1.89060 (2.64265) + | > loader_time: 0.10030 (0.10836) + + + --> STEP: 73/234 -- GLOBAL_STEP: 30025 + | > loss: -0.06698 (-0.08168) + | > log_mle: -0.24301 (-0.22476) + | > loss_dur: 0.17602 (0.14307) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.36091 (9.33015) + | > current_lr: 0.00003 + | > step_time: 3.63630 (2.64275) + | > loader_time: 0.20000 (0.10499) + + + --> STEP: 78/234 -- GLOBAL_STEP: 30030 + | > loss: -0.06811 (-0.08132) + | > log_mle: -0.21469 (-0.22487) + | > loss_dur: 0.14658 (0.14355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05295 (9.58454) + | > current_lr: 0.00003 + | > step_time: 1.50990 (2.58555) + | > loader_time: 0.09810 (0.09964) + + + --> STEP: 83/234 -- GLOBAL_STEP: 30035 + | > loss: -0.06045 (-0.08127) + | > log_mle: -0.24143 (-0.22525) + | > loss_dur: 0.18098 (0.14398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.48982 (9.74013) + | > current_lr: 0.00003 + | > step_time: 0.99940 (2.51437) + | > loader_time: 0.00300 (0.09596) + + + --> STEP: 88/234 -- GLOBAL_STEP: 30040 + | > loss: -0.10491 (-0.08154) + | > log_mle: -0.28012 (-0.22628) + | > loss_dur: 0.17521 (0.14474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.66418 (9.84342) + | > current_lr: 0.00003 + | > step_time: 1.98950 (2.47821) + | > loader_time: 0.00210 (0.09165) + + + --> STEP: 93/234 -- GLOBAL_STEP: 30045 + | > loss: -0.09928 (-0.08249) + | > log_mle: -0.29164 (-0.22860) + | > loss_dur: 0.19236 (0.14611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.35839 (10.23847) + | > current_lr: 0.00003 + | > step_time: 1.50270 (2.43772) + | > loader_time: 0.00260 (0.08856) + + + --> STEP: 98/234 -- GLOBAL_STEP: 30050 + | > loss: -0.06498 (-0.08357) + | > log_mle: -0.22195 (-0.23090) + | > loss_dur: 0.15697 (0.14733) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.04508 (10.65047) + | > current_lr: 0.00003 + | > step_time: 1.30090 (2.46240) + | > loader_time: 0.00270 (0.08516) + + + --> STEP: 103/234 -- GLOBAL_STEP: 30055 + | > loss: -0.13765 (-0.08510) + | > log_mle: -0.32547 (-0.23402) + | > loss_dur: 0.18782 (0.14893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.24788 (11.16308) + | > current_lr: 0.00003 + | > step_time: 1.18720 (2.44760) + | > loader_time: 0.00230 (0.08201) + + + --> STEP: 108/234 -- GLOBAL_STEP: 30060 + | > loss: -0.11039 (-0.08628) + | > log_mle: -0.26754 (-0.23664) + | > loss_dur: 0.15715 (0.15035) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.05658 (11.78333) + | > current_lr: 0.00003 + | > step_time: 2.42500 (2.45848) + | > loader_time: 0.00320 (0.07928) + + + --> STEP: 113/234 -- GLOBAL_STEP: 30065 + | > loss: -0.13441 (-0.08740) + | > log_mle: -0.31643 (-0.23991) + | > loss_dur: 0.18202 (0.15251) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.48714 (12.60256) + | > current_lr: 0.00003 + | > step_time: 2.68710 (2.42884) + | > loader_time: 0.00250 (0.07669) + + + --> STEP: 118/234 -- GLOBAL_STEP: 30070 + | > loss: -0.09491 (-0.08835) + | > log_mle: -0.28447 (-0.24237) + | > loss_dur: 0.18956 (0.15402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.66901 (13.02483) + | > current_lr: 0.00003 + | > step_time: 1.55990 (2.39534) + | > loader_time: 0.00360 (0.07362) + + + --> STEP: 123/234 -- GLOBAL_STEP: 30075 + | > loss: -0.09700 (-0.08910) + | > log_mle: -0.25360 (-0.24385) + | > loss_dur: 0.15659 (0.15475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.53817 (13.19632) + | > current_lr: 0.00003 + | > step_time: 1.09810 (2.38500) + | > loader_time: 0.00290 (0.07232) + + + --> STEP: 128/234 -- GLOBAL_STEP: 30080 + | > loss: -0.13966 (-0.09120) + | > log_mle: -0.31340 (-0.24727) + | > loss_dur: 0.17374 (0.15607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.74276 (13.92707) + | > current_lr: 0.00003 + | > step_time: 3.21970 (2.36068) + | > loader_time: 0.00270 (0.06961) + + + --> STEP: 133/234 -- GLOBAL_STEP: 30085 + | > loss: -0.14600 (-0.09341) + | > log_mle: -0.33990 (-0.25082) + | > loss_dur: 0.19390 (0.15742) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.87435 (14.71087) + | > current_lr: 0.00003 + | > step_time: 3.70970 (2.36065) + | > loader_time: 0.08540 (0.06828) + + + --> STEP: 138/234 -- GLOBAL_STEP: 30090 + | > loss: -0.11119 (-0.09492) + | > log_mle: -0.29361 (-0.25409) + | > loss_dur: 0.18242 (0.15917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.95999 (15.44576) + | > current_lr: 0.00003 + | > step_time: 1.60530 (2.34826) + | > loader_time: 0.00460 (0.06596) + + + --> STEP: 143/234 -- GLOBAL_STEP: 30095 + | > loss: -0.19149 (-0.09727) + | > log_mle: -0.43535 (-0.25821) + | > loss_dur: 0.24385 (0.16095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.16975 (16.26046) + | > current_lr: 0.00003 + | > step_time: 2.41770 (2.32067) + | > loader_time: 0.08630 (0.06508) + + + --> STEP: 148/234 -- GLOBAL_STEP: 30100 + | > loss: -0.16019 (-0.09972) + | > log_mle: -0.34673 (-0.26220) + | > loss_dur: 0.18654 (0.16248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.21713 (17.28154) + | > current_lr: 0.00003 + | > step_time: 3.29400 (2.35628) + | > loader_time: 0.00280 (0.06477) + + + --> STEP: 153/234 -- GLOBAL_STEP: 30105 + | > loss: -0.25579 (-0.10315) + | > log_mle: -0.46655 (-0.26729) + | > loss_dur: 0.21076 (0.16414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.31335 (18.50120) + | > current_lr: 0.00003 + | > step_time: 1.99960 (2.35420) + | > loader_time: 0.00290 (0.06345) + + + --> STEP: 158/234 -- GLOBAL_STEP: 30110 + | > loss: -0.19305 (-0.10607) + | > log_mle: -0.40935 (-0.27180) + | > loss_dur: 0.21630 (0.16573) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.71945 (19.47534) + | > current_lr: 0.00003 + | > step_time: 2.60690 (2.34616) + | > loader_time: 0.00410 (0.06317) + + + --> STEP: 163/234 -- GLOBAL_STEP: 30115 + | > loss: -0.17567 (-0.10915) + | > log_mle: -0.38137 (-0.27635) + | > loss_dur: 0.20571 (0.16720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.57007 (20.33167) + | > current_lr: 0.00003 + | > step_time: 2.51070 (2.38705) + | > loader_time: 0.29470 (0.06423) + + + --> STEP: 168/234 -- GLOBAL_STEP: 30120 + | > loss: -0.19088 (-0.11238) + | > log_mle: -0.43695 (-0.28108) + | > loss_dur: 0.24607 (0.16871) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.01902 (21.20608) + | > current_lr: 0.00003 + | > step_time: 3.00230 (2.37921) + | > loader_time: 0.10450 (0.06397) + + + --> STEP: 173/234 -- GLOBAL_STEP: 30125 + | > loss: -0.21618 (-0.11579) + | > log_mle: -0.44765 (-0.28635) + | > loss_dur: 0.23147 (0.17056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.36428 (22.44286) + | > current_lr: 0.00003 + | > step_time: 2.20970 (2.39596) + | > loader_time: 0.08530 (0.06444) + + + --> STEP: 178/234 -- GLOBAL_STEP: 30130 + | > loss: -0.25109 (-0.11927) + | > log_mle: -0.50317 (-0.29164) + | > loss_dur: 0.25209 (0.17236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.82310 (23.56043) + | > current_lr: 0.00003 + | > step_time: 6.58590 (2.46233) + | > loader_time: 0.40090 (0.06710) + + + --> STEP: 183/234 -- GLOBAL_STEP: 30135 + | > loss: -0.26472 (-0.12227) + | > log_mle: -0.50340 (-0.29650) + | > loss_dur: 0.23868 (0.17423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.47600 (24.66162) + | > current_lr: 0.00003 + | > step_time: 6.39170 (2.54860) + | > loader_time: 0.19890 (0.06849) + + + --> STEP: 188/234 -- GLOBAL_STEP: 30140 + | > loss: -0.26917 (-0.12526) + | > log_mle: -0.51443 (-0.30128) + | > loss_dur: 0.24526 (0.17602) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.44857 (25.85923) + | > current_lr: 0.00003 + | > step_time: 1.88950 (2.57977) + | > loader_time: 0.00460 (0.06781) + + + --> STEP: 193/234 -- GLOBAL_STEP: 30145 + | > loss: -0.25578 (-0.12783) + | > log_mle: -0.48611 (-0.30520) + | > loss_dur: 0.23032 (0.17737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.93837 (28.42715) + | > current_lr: 0.00003 + | > step_time: 2.10270 (2.59425) + | > loader_time: 0.00480 (0.06812) + + + --> STEP: 198/234 -- GLOBAL_STEP: 30150 + | > loss: -0.23992 (-0.13035) + | > log_mle: -0.48678 (-0.30922) + | > loss_dur: 0.24686 (0.17887) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.17337 (29.51498) + | > current_lr: 0.00003 + | > step_time: 8.79950 (2.62572) + | > loader_time: 0.00640 (0.06744) + + + --> STEP: 203/234 -- GLOBAL_STEP: 30155 + | > loss: -0.20050 (-0.13284) + | > log_mle: -0.42846 (-0.31328) + | > loss_dur: 0.22796 (0.18043) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.78352 (30.57184) + | > current_lr: 0.00003 + | > step_time: 8.30820 (2.74334) + | > loader_time: 0.00460 (0.06877) + + + --> STEP: 208/234 -- GLOBAL_STEP: 30160 + | > loss: -0.25548 (-0.13605) + | > log_mle: -0.51227 (-0.31818) + | > loss_dur: 0.25678 (0.18213) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.34846 (31.45862) + | > current_lr: 0.00003 + | > step_time: 2.19680 (2.73746) + | > loader_time: 0.08640 (0.06858) + + + --> STEP: 213/234 -- GLOBAL_STEP: 30165 + | > loss: -0.29147 (-0.13966) + | > log_mle: -0.55899 (-0.32353) + | > loss_dur: 0.26752 (0.18388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.86668 (32.36434) + | > current_lr: 0.00003 + | > step_time: 3.50330 (2.76804) + | > loader_time: 0.00350 (0.06833) + + + --> STEP: 218/234 -- GLOBAL_STEP: 30170 + | > loss: -0.27034 (-0.14303) + | > log_mle: -0.52315 (-0.32844) + | > loss_dur: 0.25281 (0.18541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.13971 (33.44053) + | > current_lr: 0.00003 + | > step_time: 4.19770 (2.83666) + | > loader_time: 0.09050 (0.07501) + + + --> STEP: 223/234 -- GLOBAL_STEP: 30175 + | > loss: -0.30266 (-0.14653) + | > log_mle: -0.55713 (-0.33359) + | > loss_dur: 0.25447 (0.18707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.60478 (34.64383) + | > current_lr: 0.00003 + | > step_time: 3.70360 (2.87800) + | > loader_time: 0.09680 (0.07424) + + + --> STEP: 228/234 -- GLOBAL_STEP: 30180 + | > loss: -0.27904 (-0.14992) + | > log_mle: -0.56381 (-0.33882) + | > loss_dur: 0.28477 (0.18890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.80563 (36.09408) + | > current_lr: 0.00003 + | > step_time: 0.25060 (2.83954) + | > loader_time: 0.00510 (0.07307) + + + --> STEP: 233/234 -- GLOBAL_STEP: 30185 + | > loss: 0.27032 (-0.15006) + | > log_mle: -0.52083 (-0.34510) + | > loss_dur: 0.79115 (0.19503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.32261 (38.14883) + | > current_lr: 0.00003 + | > step_time: 0.18850 (2.78423) + | > loader_time: 0.00290 (0.07194) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.41942 (+0.01902) + | > avg_loss: -0.19102 (-0.00605) + | > avg_log_mle: -0.41684 (-0.00408) + | > avg_loss_dur: 0.22582 (-0.00197) + + + > EPOCH: 129/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 12:23:14)  + + --> STEP: 4/234 -- GLOBAL_STEP: 30190 + | > loss: -0.05165 (-0.06809) + | > log_mle: -0.22048 (-0.22610) + | > loss_dur: 0.16884 (0.15801) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.36971 (12.05852) + | > current_lr: 0.00003 + | > step_time: 7.29260 (6.82739) + | > loader_time: 0.39740 (1.49985) + + + --> STEP: 9/234 -- GLOBAL_STEP: 30195 + | > loss: -0.09721 (-0.07996) + | > log_mle: -0.23770 (-0.22996) + | > loss_dur: 0.14049 (0.15001) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.86909 (11.66381) + | > current_lr: 0.00003 + | > step_time: 5.09850 (5.58837) + | > loader_time: 0.00150 (0.69930) + + + --> STEP: 14/234 -- GLOBAL_STEP: 30200 + | > loss: -0.07901 (-0.08196) + | > log_mle: -0.23202 (-0.22968) + | > loss_dur: 0.15301 (0.14772) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.86481 (10.64921) + | > current_lr: 0.00003 + | > step_time: 1.18680 (4.60311) + | > loader_time: 0.00160 (0.45667) + + + --> STEP: 19/234 -- GLOBAL_STEP: 30205 + | > loss: -0.09888 (-0.08448) + | > log_mle: -0.21675 (-0.22711) + | > loss_dur: 0.11787 (0.14263) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.89356 (9.99912) + | > current_lr: 0.00003 + | > step_time: 1.21090 (3.73260) + | > loader_time: 0.09200 (0.34165) + + + --> STEP: 24/234 -- GLOBAL_STEP: 30210 + | > loss: -0.10567 (-0.08616) + | > log_mle: -0.21925 (-0.22567) + | > loss_dur: 0.11358 (0.13951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.56105 (9.68219) + | > current_lr: 0.00003 + | > step_time: 1.14850 (3.20941) + | > loader_time: 0.00140 (0.27077) + + + --> STEP: 29/234 -- GLOBAL_STEP: 30215 + | > loss: -0.06957 (-0.08684) + | > log_mle: -0.20889 (-0.22454) + | > loss_dur: 0.13933 (0.13770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.22773 (9.26720) + | > current_lr: 0.00003 + | > step_time: 0.87780 (2.85568) + | > loader_time: 0.00200 (0.22439) + + + --> STEP: 34/234 -- GLOBAL_STEP: 30220 + | > loss: -0.06499 (-0.08692) + | > log_mle: -0.22200 (-0.22506) + | > loss_dur: 0.15701 (0.13815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.62504 (8.97481) + | > current_lr: 0.00003 + | > step_time: 2.10590 (2.90953) + | > loader_time: 0.00280 (0.19431) + + + --> STEP: 39/234 -- GLOBAL_STEP: 30225 + | > loss: -0.09901 (-0.08812) + | > log_mle: -0.23046 (-0.22549) + | > loss_dur: 0.13145 (0.13737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.34002 (9.00477) + | > current_lr: 0.00003 + | > step_time: 0.97370 (2.69145) + | > loader_time: 0.00160 (0.16969) + + + --> STEP: 44/234 -- GLOBAL_STEP: 30230 + | > loss: -0.10081 (-0.08718) + | > log_mle: -0.22047 (-0.22482) + | > loss_dur: 0.11966 (0.13764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.06044 (8.71902) + | > current_lr: 0.00003 + | > step_time: 1.40880 (2.53303) + | > loader_time: 0.08410 (0.15397) + + + --> STEP: 49/234 -- GLOBAL_STEP: 30235 + | > loss: -0.10772 (-0.08747) + | > log_mle: -0.22960 (-0.22511) + | > loss_dur: 0.12188 (0.13764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.09773 (8.64633) + | > current_lr: 0.00003 + | > step_time: 3.10110 (2.50272) + | > loader_time: 0.00340 (0.14233) + + + --> STEP: 54/234 -- GLOBAL_STEP: 30240 + | > loss: -0.10081 (-0.08662) + | > log_mle: -0.23416 (-0.22489) + | > loss_dur: 0.13335 (0.13827) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81673 (8.62994) + | > current_lr: 0.00003 + | > step_time: 1.04590 (2.41311) + | > loader_time: 0.00190 (0.13069) + + + --> STEP: 59/234 -- GLOBAL_STEP: 30245 + | > loss: -0.12081 (-0.08678) + | > log_mle: -0.24347 (-0.22512) + | > loss_dur: 0.12267 (0.13834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.79544 (8.62364) + | > current_lr: 0.00003 + | > step_time: 1.98320 (2.38618) + | > loader_time: 0.00370 (0.12127) + + + --> STEP: 64/234 -- GLOBAL_STEP: 30250 + | > loss: -0.09622 (-0.08652) + | > log_mle: -0.21708 (-0.22612) + | > loss_dur: 0.12086 (0.13961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.57838 (8.89564) + | > current_lr: 0.00003 + | > step_time: 1.71160 (2.33614) + | > loader_time: 0.08280 (0.11743) + + + --> STEP: 69/234 -- GLOBAL_STEP: 30255 + | > loss: -0.06334 (-0.08555) + | > log_mle: -0.20665 (-0.22583) + | > loss_dur: 0.14331 (0.14028) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.26311 (8.88883) + | > current_lr: 0.00003 + | > step_time: 1.39040 (2.26813) + | > loader_time: 0.00220 (0.11026) + + + --> STEP: 74/234 -- GLOBAL_STEP: 30260 + | > loss: -0.08638 (-0.08439) + | > log_mle: -0.21872 (-0.22604) + | > loss_dur: 0.13234 (0.14165) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.71758 (9.45134) + | > current_lr: 0.00003 + | > step_time: 1.70900 (2.24448) + | > loader_time: 0.00240 (0.10523) + + + --> STEP: 79/234 -- GLOBAL_STEP: 30265 + | > loss: -0.08900 (-0.08399) + | > log_mle: -0.23482 (-0.22644) + | > loss_dur: 0.14582 (0.14245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.49493 (9.49792) + | > current_lr: 0.00003 + | > step_time: 3.79850 (2.25178) + | > loader_time: 0.09360 (0.10095) + + + --> STEP: 84/234 -- GLOBAL_STEP: 30270 + | > loss: -0.08061 (-0.08388) + | > log_mle: -0.23093 (-0.22685) + | > loss_dur: 0.15032 (0.14297) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.38295 (9.53385) + | > current_lr: 0.00003 + | > step_time: 1.30420 (2.20677) + | > loader_time: 0.08800 (0.09613) + + + --> STEP: 89/234 -- GLOBAL_STEP: 30275 + | > loss: -0.10918 (-0.08452) + | > log_mle: -0.26100 (-0.22823) + | > loss_dur: 0.15182 (0.14371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.24183 (9.72239) + | > current_lr: 0.00003 + | > step_time: 1.20120 (2.16484) + | > loader_time: 0.00230 (0.09276) + + + --> STEP: 94/234 -- GLOBAL_STEP: 30280 + | > loss: -0.13678 (-0.08583) + | > log_mle: -0.29719 (-0.23089) + | > loss_dur: 0.16041 (0.14506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.56999 (10.11813) + | > current_lr: 0.00003 + | > step_time: 1.19370 (2.12782) + | > loader_time: 0.00330 (0.08795) + + + --> STEP: 99/234 -- GLOBAL_STEP: 30285 + | > loss: -0.13009 (-0.08704) + | > log_mle: -0.32636 (-0.23347) + | > loss_dur: 0.19627 (0.14644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.68301 (10.54608) + | > current_lr: 0.00003 + | > step_time: 2.09910 (2.10685) + | > loader_time: 0.18370 (0.08546) + + + --> STEP: 104/234 -- GLOBAL_STEP: 30290 + | > loss: -0.16422 (-0.08871) + | > log_mle: -0.34190 (-0.23662) + | > loss_dur: 0.17768 (0.14791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.98606 (11.12124) + | > current_lr: 0.00003 + | > step_time: 1.61150 (2.10216) + | > loader_time: 0.00230 (0.08312) + + + --> STEP: 109/234 -- GLOBAL_STEP: 30295 + | > loss: -0.08896 (-0.08931) + | > log_mle: -0.30677 (-0.23885) + | > loss_dur: 0.21782 (0.14953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.42444 (11.57137) + | > current_lr: 0.00003 + | > step_time: 3.99780 (2.15361) + | > loader_time: 0.00850 (0.08199) + + + --> STEP: 114/234 -- GLOBAL_STEP: 30300 + | > loss: -0.11794 (-0.09073) + | > log_mle: -0.29404 (-0.24187) + | > loss_dur: 0.17610 (0.15114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.12773 (12.21313) + | > current_lr: 0.00003 + | > step_time: 1.20440 (2.16358) + | > loader_time: 0.00230 (0.08085) + + + --> STEP: 119/234 -- GLOBAL_STEP: 30305 + | > loss: -0.10827 (-0.09146) + | > log_mle: -0.29387 (-0.24427) + | > loss_dur: 0.18560 (0.15280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.39320 (12.53801) + | > current_lr: 0.00003 + | > step_time: 1.70100 (2.15521) + | > loader_time: 0.00940 (0.07971) + + + --> STEP: 124/234 -- GLOBAL_STEP: 30310 + | > loss: -0.15129 (-0.09236) + | > log_mle: -0.32169 (-0.24595) + | > loss_dur: 0.17040 (0.15359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.17010 (12.84126) + | > current_lr: 0.00003 + | > step_time: 4.80040 (2.20555) + | > loader_time: 0.10230 (0.07823) + + + --> STEP: 129/234 -- GLOBAL_STEP: 30315 + | > loss: -0.11318 (-0.09425) + | > log_mle: -0.31194 (-0.24928) + | > loss_dur: 0.19876 (0.15503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.03804 (13.56818) + | > current_lr: 0.00003 + | > step_time: 2.19840 (2.18116) + | > loader_time: 0.08800 (0.07661) + + + --> STEP: 134/234 -- GLOBAL_STEP: 30320 + | > loss: -0.14122 (-0.09646) + | > log_mle: -0.36253 (-0.25321) + | > loss_dur: 0.22132 (0.15674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.81464 (14.34624) + | > current_lr: 0.00003 + | > step_time: 3.58350 (2.20569) + | > loader_time: 0.00170 (0.07737) + + + --> STEP: 139/234 -- GLOBAL_STEP: 30325 + | > loss: -0.21656 (-0.09859) + | > log_mle: -0.42590 (-0.25689) + | > loss_dur: 0.20935 (0.15830) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.33153 (15.26585) + | > current_lr: 0.00003 + | > step_time: 1.09390 (2.17954) + | > loader_time: 0.00340 (0.07591) + + + --> STEP: 144/234 -- GLOBAL_STEP: 30330 + | > loss: -0.18302 (-0.10050) + | > log_mle: -0.40056 (-0.26081) + | > loss_dur: 0.21754 (0.16030) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.57488 (16.20002) + | > current_lr: 0.00003 + | > step_time: 1.17920 (2.18418) + | > loader_time: 0.00250 (0.07410) + + + --> STEP: 149/234 -- GLOBAL_STEP: 30335 + | > loss: -0.23125 (-0.10344) + | > log_mle: -0.44451 (-0.26524) + | > loss_dur: 0.21326 (0.16180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.17784 (17.03108) + | > current_lr: 0.00003 + | > step_time: 1.89710 (2.19196) + | > loader_time: 0.08980 (0.07359) + + + --> STEP: 154/234 -- GLOBAL_STEP: 30340 + | > loss: -0.20396 (-0.10674) + | > log_mle: -0.40494 (-0.27011) + | > loss_dur: 0.20098 (0.16336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.12404 (18.03054) + | > current_lr: 0.00003 + | > step_time: 1.70790 (2.22543) + | > loader_time: 0.00220 (0.07251) + + + --> STEP: 159/234 -- GLOBAL_STEP: 30345 + | > loss: -0.21393 (-0.10975) + | > log_mle: -0.42297 (-0.27471) + | > loss_dur: 0.20904 (0.16497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.42283 (19.12073) + | > current_lr: 0.00003 + | > step_time: 6.70220 (2.31150) + | > loader_time: 0.10320 (0.07276) + + + --> STEP: 164/234 -- GLOBAL_STEP: 30350 + | > loss: -0.19499 (-0.11278) + | > log_mle: -0.41430 (-0.27910) + | > loss_dur: 0.21931 (0.16632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.98242 (20.14592) + | > current_lr: 0.00003 + | > step_time: 3.19960 (2.32137) + | > loader_time: 0.00380 (0.07411) + + + --> STEP: 169/234 -- GLOBAL_STEP: 30355 + | > loss: -0.18745 (-0.11575) + | > log_mle: -0.41495 (-0.28369) + | > loss_dur: 0.22750 (0.16794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.12386 (21.30903) + | > current_lr: 0.00003 + | > step_time: 5.39660 (2.33908) + | > loader_time: 0.00620 (0.07254) + + + --> STEP: 174/234 -- GLOBAL_STEP: 30360 + | > loss: -0.27208 (-0.11951) + | > log_mle: -0.50053 (-0.28924) + | > loss_dur: 0.22845 (0.16973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.58910 (22.93503) + | > current_lr: 0.00003 + | > step_time: 4.40040 (2.40179) + | > loader_time: 0.00270 (0.07218) + + + --> STEP: 179/234 -- GLOBAL_STEP: 30365 + | > loss: -0.23772 (-0.12247) + | > log_mle: -0.49833 (-0.29434) + | > loss_dur: 0.26060 (0.17187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.94088 (24.00015) + | > current_lr: 0.00003 + | > step_time: 4.38980 (2.41345) + | > loader_time: 0.10010 (0.07181) + + + --> STEP: 184/234 -- GLOBAL_STEP: 30370 + | > loss: -0.22375 (-0.12517) + | > log_mle: -0.45968 (-0.29885) + | > loss_dur: 0.23593 (0.17368) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.87737 (25.16465) + | > current_lr: 0.00003 + | > step_time: 4.50150 (2.48707) + | > loader_time: 0.08520 (0.07204) + + + --> STEP: 189/234 -- GLOBAL_STEP: 30375 + | > loss: -0.21574 (-0.12796) + | > log_mle: -0.46090 (-0.30355) + | > loss_dur: 0.24516 (0.17559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.41217 (26.19860) + | > current_lr: 0.00003 + | > step_time: 5.09480 (2.53553) + | > loader_time: 0.00290 (0.07120) + + + --> STEP: 194/234 -- GLOBAL_STEP: 30380 + | > loss: -0.26476 (-0.13135) + | > log_mle: -0.49575 (-0.30828) + | > loss_dur: 0.23098 (0.17692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.16278 (27.35027) + | > current_lr: 0.00003 + | > step_time: 8.20450 (2.64339) + | > loader_time: 0.08350 (0.07077) + + + --> STEP: 199/234 -- GLOBAL_STEP: 30385 + | > loss: -0.25661 (-0.13420) + | > log_mle: -0.49995 (-0.31264) + | > loss_dur: 0.24334 (0.17844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.24648 (28.49483) + | > current_lr: 0.00003 + | > step_time: 8.00910 (2.71926) + | > loader_time: 0.19760 (0.07202) + + + --> STEP: 204/234 -- GLOBAL_STEP: 30390 + | > loss: -0.27556 (-0.13670) + | > log_mle: -0.53067 (-0.31682) + | > loss_dur: 0.25511 (0.18012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.48643 (29.73669) + | > current_lr: 0.00003 + | > step_time: 5.49260 (2.80853) + | > loader_time: 0.11090 (0.07369) + + + --> STEP: 209/234 -- GLOBAL_STEP: 30395 + | > loss: -0.24604 (-0.13969) + | > log_mle: -0.48987 (-0.32142) + | > loss_dur: 0.24383 (0.18173) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.44842 (30.85052) + | > current_lr: 0.00003 + | > step_time: 9.10300 (2.89638) + | > loader_time: 0.19930 (0.07439) + + + --> STEP: 214/234 -- GLOBAL_STEP: 30400 + | > loss: -0.29867 (-0.14351) + | > log_mle: -0.52210 (-0.32688) + | > loss_dur: 0.22344 (0.18336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.79214 (32.17528) + | > current_lr: 0.00003 + | > step_time: 2.00910 (2.87312) + | > loader_time: 0.07630 (0.07437) + + + --> STEP: 219/234 -- GLOBAL_STEP: 30405 + | > loss: -0.36825 (-0.14724) + | > log_mle: -0.62657 (-0.33222) + | > loss_dur: 0.25832 (0.18498) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.62588 (33.38800) + | > current_lr: 0.00003 + | > step_time: 1.19230 (2.94535) + | > loader_time: 0.00290 (0.07457) + + + --> STEP: 224/234 -- GLOBAL_STEP: 30410 + | > loss: -0.31996 (-0.15068) + | > log_mle: -0.57839 (-0.33729) + | > loss_dur: 0.25843 (0.18661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.55437 (34.44717) + | > current_lr: 0.00003 + | > step_time: 0.24000 (2.88811) + | > loader_time: 0.00430 (0.07333) + + + --> STEP: 229/234 -- GLOBAL_STEP: 30415 + | > loss: -0.29082 (-0.15421) + | > log_mle: -0.61320 (-0.34293) + | > loss_dur: 0.32238 (0.18872) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.26577 (35.82719) + | > current_lr: 0.00003 + | > step_time: 0.26730 (2.83047) + | > loader_time: 0.00370 (0.07183) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.38379 (-0.03562) + | > avg_loss: -0.19740 (-0.00638) + | > avg_log_mle: -0.42554 (-0.00870) + | > avg_loss_dur: 0.22814 (+0.00232) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_30420.pth + + > EPOCH: 130/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 12:35:19)  + + --> STEP: 0/234 -- GLOBAL_STEP: 30420 + | > loss: -0.12471 (-0.12471) + | > log_mle: -0.29827 (-0.29827) + | > loss_dur: 0.17356 (0.17356) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.80569 (12.80569) + | > current_lr: 0.00003 + | > step_time: 21.81080 (21.81077) + | > loader_time: 12.94720 (12.94723) + + + --> STEP: 5/234 -- GLOBAL_STEP: 30425 + | > loss: -0.06623 (-0.06807) + | > log_mle: -0.23004 (-0.22821) + | > loss_dur: 0.16381 (0.16014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.63589 (15.39970) + | > current_lr: 0.00003 + | > step_time: 0.61940 (3.39892) + | > loader_time: 0.00240 (0.05328) + + + --> STEP: 10/234 -- GLOBAL_STEP: 30430 + | > loss: -0.06864 (-0.08117) + | > log_mle: -0.22961 (-0.23107) + | > loss_dur: 0.16098 (0.14989) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.14745 (13.71836) + | > current_lr: 0.00003 + | > step_time: 0.83700 (2.31375) + | > loader_time: 0.08420 (0.04402) + + + --> STEP: 15/234 -- GLOBAL_STEP: 30435 + | > loss: -0.11201 (-0.08599) + | > log_mle: -0.23096 (-0.23141) + | > loss_dur: 0.11895 (0.14542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.48411 (12.22343) + | > current_lr: 0.00003 + | > step_time: 4.70590 (2.16817) + | > loader_time: 0.19030 (0.05358) + + + --> STEP: 20/234 -- GLOBAL_STEP: 30440 + | > loss: -0.09188 (-0.08782) + | > log_mle: -0.21742 (-0.22867) + | > loss_dur: 0.12554 (0.14086) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.30040 (10.93776) + | > current_lr: 0.00003 + | > step_time: 6.40890 (2.59742) + | > loader_time: 0.08820 (0.05401) + + + --> STEP: 25/234 -- GLOBAL_STEP: 30445 + | > loss: -0.08404 (-0.08970) + | > log_mle: -0.21287 (-0.22709) + | > loss_dur: 0.12883 (0.13739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.68306 (10.22759) + | > current_lr: 0.00003 + | > step_time: 7.19580 (2.96153) + | > loader_time: 0.09860 (0.05467) + + + --> STEP: 30/234 -- GLOBAL_STEP: 30450 + | > loss: -0.12584 (-0.09126) + | > log_mle: -0.23588 (-0.22689) + | > loss_dur: 0.11004 (0.13563) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.01584 (9.75933) + | > current_lr: 0.00003 + | > step_time: 4.41230 (3.03566) + | > loader_time: 0.10370 (0.06494) + + + --> STEP: 35/234 -- GLOBAL_STEP: 30455 + | > loss: -0.08416 (-0.09014) + | > log_mle: -0.22897 (-0.22687) + | > loss_dur: 0.14480 (0.13673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.33962 (9.66063) + | > current_lr: 0.00003 + | > step_time: 1.09750 (2.99976) + | > loader_time: 0.00150 (0.06915) + + + --> STEP: 40/234 -- GLOBAL_STEP: 30460 + | > loss: -0.05148 (-0.08958) + | > log_mle: -0.21020 (-0.22672) + | > loss_dur: 0.15872 (0.13714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.12839 (9.66399) + | > current_lr: 0.00003 + | > step_time: 0.95010 (2.78246) + | > loader_time: 0.00180 (0.06076) + + + --> STEP: 45/234 -- GLOBAL_STEP: 30465 + | > loss: -0.09350 (-0.08999) + | > log_mle: -0.25013 (-0.22696) + | > loss_dur: 0.15663 (0.13698) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.90510 (9.51341) + | > current_lr: 0.00003 + | > step_time: 1.32140 (2.60982) + | > loader_time: 0.00170 (0.05422) + + + --> STEP: 50/234 -- GLOBAL_STEP: 30470 + | > loss: -0.07744 (-0.08977) + | > log_mle: -0.21591 (-0.22650) + | > loss_dur: 0.13847 (0.13673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.73931 (9.24259) + | > current_lr: 0.00003 + | > step_time: 1.48900 (2.52397) + | > loader_time: 0.00180 (0.04904) + + + --> STEP: 55/234 -- GLOBAL_STEP: 30475 + | > loss: -0.10904 (-0.08995) + | > log_mle: -0.23470 (-0.22664) + | > loss_dur: 0.12566 (0.13669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.86770 (9.01629) + | > current_lr: 0.00003 + | > step_time: 0.98280 (2.43261) + | > loader_time: 0.00210 (0.04479) + + + --> STEP: 60/234 -- GLOBAL_STEP: 30480 + | > loss: -0.09856 (-0.08969) + | > log_mle: -0.25207 (-0.22709) + | > loss_dur: 0.15351 (0.13740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.77422 (8.98540) + | > current_lr: 0.00003 + | > step_time: 1.40580 (2.40174) + | > loader_time: 0.08450 (0.04407) + + + --> STEP: 65/234 -- GLOBAL_STEP: 30485 + | > loss: -0.10153 (-0.08906) + | > log_mle: -0.22609 (-0.22754) + | > loss_dur: 0.12456 (0.13848) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.27773 (9.10632) + | > current_lr: 0.00003 + | > step_time: 1.10270 (2.35186) + | > loader_time: 0.08630 (0.04350) + + + --> STEP: 70/234 -- GLOBAL_STEP: 30490 + | > loss: -0.06886 (-0.08767) + | > log_mle: -0.22154 (-0.22714) + | > loss_dur: 0.15268 (0.13947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.46567 (9.19471) + | > current_lr: 0.00003 + | > step_time: 0.90420 (2.37262) + | > loader_time: 0.00270 (0.04459) + + + --> STEP: 75/234 -- GLOBAL_STEP: 30495 + | > loss: -0.06758 (-0.08651) + | > log_mle: -0.23590 (-0.22765) + | > loss_dur: 0.16832 (0.14114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.33291 (9.47746) + | > current_lr: 0.00003 + | > step_time: 1.71300 (2.33671) + | > loader_time: 0.00230 (0.04177) + + + --> STEP: 80/234 -- GLOBAL_STEP: 30500 + | > loss: -0.09643 (-0.08646) + | > log_mle: -0.21849 (-0.22774) + | > loss_dur: 0.12206 (0.14128) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.19353 (9.49259) + | > current_lr: 0.00003 + | > step_time: 1.56200 (2.30668) + | > loader_time: 0.00230 (0.04035) + + + --> STEP: 85/234 -- GLOBAL_STEP: 30505 + | > loss: -0.09581 (-0.08642) + | > log_mle: -0.23534 (-0.22825) + | > loss_dur: 0.13953 (0.14182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.79144 (9.59028) + | > current_lr: 0.00003 + | > step_time: 2.39770 (2.29192) + | > loader_time: 0.00160 (0.03814) + + + --> STEP: 90/234 -- GLOBAL_STEP: 30510 + | > loss: -0.08605 (-0.08667) + | > log_mle: -0.26334 (-0.22988) + | > loss_dur: 0.17729 (0.14321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.82455 (9.88780) + | > current_lr: 0.00003 + | > step_time: 1.47450 (2.27316) + | > loader_time: 0.00230 (0.03794) + + + --> STEP: 95/234 -- GLOBAL_STEP: 30515 + | > loss: -0.15587 (-0.08852) + | > log_mle: -0.34653 (-0.23341) + | > loss_dur: 0.19066 (0.14490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.86397 (10.58404) + | > current_lr: 0.00003 + | > step_time: 2.90560 (2.25663) + | > loader_time: 0.00320 (0.03610) + + + --> STEP: 100/234 -- GLOBAL_STEP: 30520 + | > loss: -0.11117 (-0.08929) + | > log_mle: -0.27447 (-0.23521) + | > loss_dur: 0.16331 (0.14592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.75533 (10.90310) + | > current_lr: 0.00003 + | > step_time: 2.71600 (2.24186) + | > loader_time: 0.08790 (0.03616) + + + --> STEP: 105/234 -- GLOBAL_STEP: 30525 + | > loss: -0.11165 (-0.09079) + | > log_mle: -0.25116 (-0.23810) + | > loss_dur: 0.13951 (0.14731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.03102 (11.43409) + | > current_lr: 0.00003 + | > step_time: 2.01920 (2.21937) + | > loader_time: 0.09090 (0.03540) + + + --> STEP: 110/234 -- GLOBAL_STEP: 30530 + | > loss: -0.10786 (-0.09150) + | > log_mle: -0.27486 (-0.24051) + | > loss_dur: 0.16699 (0.14901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.83039 (12.07482) + | > current_lr: 0.00003 + | > step_time: 1.30660 (2.21308) + | > loader_time: 0.00310 (0.03396) + + + --> STEP: 115/234 -- GLOBAL_STEP: 30535 + | > loss: -0.10396 (-0.09282) + | > log_mle: -0.29806 (-0.24372) + | > loss_dur: 0.19410 (0.15090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.29759 (12.65677) + | > current_lr: 0.00003 + | > step_time: 2.51000 (2.21432) + | > loader_time: 0.00230 (0.03259) + + + --> STEP: 120/234 -- GLOBAL_STEP: 30540 + | > loss: -0.15886 (-0.09381) + | > log_mle: -0.34676 (-0.24645) + | > loss_dur: 0.18790 (0.15264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.33111 (13.17644) + | > current_lr: 0.00003 + | > step_time: 2.50830 (2.26130) + | > loader_time: 0.08540 (0.03355) + + + --> STEP: 125/234 -- GLOBAL_STEP: 30545 + | > loss: -0.14261 (-0.09450) + | > log_mle: -0.33158 (-0.24797) + | > loss_dur: 0.18897 (0.15347) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.36449 (13.52273) + | > current_lr: 0.00003 + | > step_time: 2.90400 (2.27000) + | > loader_time: 0.00770 (0.03370) + + + --> STEP: 130/234 -- GLOBAL_STEP: 30550 + | > loss: -0.15410 (-0.09641) + | > log_mle: -0.34795 (-0.25139) + | > loss_dur: 0.19385 (0.15497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.90464 (14.10567) + | > current_lr: 0.00003 + | > step_time: 2.38500 (2.27657) + | > loader_time: 0.00240 (0.03382) + + + --> STEP: 135/234 -- GLOBAL_STEP: 30555 + | > loss: -0.10277 (-0.09843) + | > log_mle: -0.27350 (-0.25474) + | > loss_dur: 0.17073 (0.15631) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.00672 (14.88768) + | > current_lr: 0.00003 + | > step_time: 1.99940 (2.28895) + | > loader_time: 0.00260 (0.03345) + + + --> STEP: 140/234 -- GLOBAL_STEP: 30560 + | > loss: -0.10369 (-0.10036) + | > log_mle: -0.30639 (-0.25854) + | > loss_dur: 0.20270 (0.15818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.12475 (15.81875) + | > current_lr: 0.00003 + | > step_time: 2.11700 (2.27852) + | > loader_time: 0.00580 (0.03240) + + + --> STEP: 145/234 -- GLOBAL_STEP: 30565 + | > loss: -0.19234 (-0.10268) + | > log_mle: -0.40333 (-0.26292) + | > loss_dur: 0.21100 (0.16024) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.53461 (16.81380) + | > current_lr: 0.00003 + | > step_time: 1.99750 (2.27142) + | > loader_time: 0.00300 (0.03197) + + + --> STEP: 150/234 -- GLOBAL_STEP: 30570 + | > loss: -0.17733 (-0.10534) + | > log_mle: -0.39270 (-0.26714) + | > loss_dur: 0.21537 (0.16180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.49467 (17.71642) + | > current_lr: 0.00003 + | > step_time: 2.59780 (2.26636) + | > loader_time: 0.09150 (0.03410) + + + --> STEP: 155/234 -- GLOBAL_STEP: 30575 + | > loss: -0.22771 (-0.10888) + | > log_mle: -0.45610 (-0.27233) + | > loss_dur: 0.22839 (0.16345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.76289 (18.85180) + | > current_lr: 0.00003 + | > step_time: 1.80410 (2.25266) + | > loader_time: 0.08810 (0.03480) + + + --> STEP: 160/234 -- GLOBAL_STEP: 30580 + | > loss: -0.23579 (-0.11182) + | > log_mle: -0.45121 (-0.27688) + | > loss_dur: 0.21542 (0.16507) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.76487 (19.82961) + | > current_lr: 0.00003 + | > step_time: 1.41370 (2.23418) + | > loader_time: 0.08770 (0.03547) + + + --> STEP: 165/234 -- GLOBAL_STEP: 30585 + | > loss: -0.22168 (-0.11479) + | > log_mle: -0.45577 (-0.28133) + | > loss_dur: 0.23409 (0.16654) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.26085 (20.73373) + | > current_lr: 0.00003 + | > step_time: 3.90780 (2.31158) + | > loader_time: 0.09600 (0.03722) + + + --> STEP: 170/234 -- GLOBAL_STEP: 30590 + | > loss: -0.24263 (-0.11790) + | > log_mle: -0.49441 (-0.28618) + | > loss_dur: 0.25178 (0.16828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.09399 (21.70388) + | > current_lr: 0.00003 + | > step_time: 1.20120 (2.36954) + | > loader_time: 0.00290 (0.03837) + + + --> STEP: 175/234 -- GLOBAL_STEP: 30595 + | > loss: -0.21894 (-0.12165) + | > log_mle: -0.46629 (-0.29176) + | > loss_dur: 0.24735 (0.17011) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.66488 (22.90469) + | > current_lr: 0.00003 + | > step_time: 2.98890 (2.36275) + | > loader_time: 0.00850 (0.03858) + + + --> STEP: 180/234 -- GLOBAL_STEP: 30600 + | > loss: -0.24399 (-0.12491) + | > log_mle: -0.47828 (-0.29699) + | > loss_dur: 0.23429 (0.17209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.75386 (24.05953) + | > current_lr: 0.00003 + | > step_time: 1.40590 (2.37373) + | > loader_time: 0.08410 (0.03860) + + + --> STEP: 185/234 -- GLOBAL_STEP: 30605 + | > loss: -0.23430 (-0.12775) + | > log_mle: -0.49013 (-0.30164) + | > loss_dur: 0.25582 (0.17389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.25235 (25.52290) + | > current_lr: 0.00003 + | > step_time: 1.80140 (2.45284) + | > loader_time: 0.00430 (0.03975) + + + --> STEP: 190/234 -- GLOBAL_STEP: 30610 + | > loss: -0.24180 (-0.13061) + | > log_mle: -0.47322 (-0.30623) + | > loss_dur: 0.23143 (0.17562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.58464 (26.68884) + | > current_lr: 0.00003 + | > step_time: 6.31730 (2.54263) + | > loader_time: 0.19710 (0.04184) + + + --> STEP: 195/234 -- GLOBAL_STEP: 30615 + | > loss: -0.25003 (-0.13410) + | > log_mle: -0.49594 (-0.31116) + | > loss_dur: 0.24592 (0.17706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.98045 (27.74623) + | > current_lr: 0.00003 + | > step_time: 6.20080 (2.59993) + | > loader_time: 0.00240 (0.04335) + + + --> STEP: 200/234 -- GLOBAL_STEP: 30620 + | > loss: -0.24197 (-0.13700) + | > log_mle: -0.50068 (-0.31564) + | > loss_dur: 0.25871 (0.17864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.16325 (28.95502) + | > current_lr: 0.00003 + | > step_time: 6.50740 (2.70594) + | > loader_time: 0.08360 (0.04415) + + + --> STEP: 205/234 -- GLOBAL_STEP: 30625 + | > loss: -0.24734 (-0.13972) + | > log_mle: -0.48853 (-0.31995) + | > loss_dur: 0.24119 (0.18023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.94056 (29.91961) + | > current_lr: 0.00003 + | > step_time: 2.40760 (2.72488) + | > loader_time: 0.08970 (0.04405) + + + --> STEP: 210/234 -- GLOBAL_STEP: 30630 + | > loss: -0.30691 (-0.14313) + | > log_mle: -0.56353 (-0.32500) + | > loss_dur: 0.25662 (0.18187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.85595 (31.26418) + | > current_lr: 0.00003 + | > step_time: 4.90420 (2.82009) + | > loader_time: 0.11260 (0.04584) + + + --> STEP: 215/234 -- GLOBAL_STEP: 30635 + | > loss: -0.27450 (-0.14678) + | > log_mle: -0.51636 (-0.33020) + | > loss_dur: 0.24186 (0.18342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.50854 (32.65434) + | > current_lr: 0.00003 + | > step_time: 4.60730 (2.86986) + | > loader_time: 0.00430 (0.04537) + + + --> STEP: 220/234 -- GLOBAL_STEP: 30640 + | > loss: -0.31013 (-0.15062) + | > log_mle: -0.56446 (-0.33577) + | > loss_dur: 0.25432 (0.18515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.24043 (33.79370) + | > current_lr: 0.00003 + | > step_time: 2.70270 (2.92970) + | > loader_time: 0.00460 (0.04619) + + + --> STEP: 225/234 -- GLOBAL_STEP: 30645 + | > loss: -0.35397 (-0.15408) + | > log_mle: -0.62343 (-0.34094) + | > loss_dur: 0.26946 (0.18686) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 130.80882 (35.24858) + | > current_lr: 0.00003 + | > step_time: 0.22490 (2.89960) + | > loader_time: 0.00260 (0.04677) + + + --> STEP: 230/234 -- GLOBAL_STEP: 30650 + | > loss: -0.32728 (-0.15723) + | > log_mle: -0.67551 (-0.34656) + | > loss_dur: 0.34823 (0.18934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 119.85196 (36.74424) + | > current_lr: 0.00003 + | > step_time: 0.24950 (2.84175) + | > loader_time: 0.00370 (0.04584) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00228 (-0.38152) + | > avg_loss: -0.19951 (-0.00211) + | > avg_log_mle: -0.42291 (+0.00263) + | > avg_loss_dur: 0.22340 (-0.00474) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_30654.pth + + > EPOCH: 131/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 12:47:44)  + + --> STEP: 1/234 -- GLOBAL_STEP: 30655 + | > loss: -0.10594 (-0.10594) + | > log_mle: -0.23119 (-0.23119) + | > loss_dur: 0.12525 (0.12525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.83973 (17.83973) + | > current_lr: 0.00003 + | > step_time: 6.40390 (6.40395) + | > loader_time: 0.00330 (0.00328) + + + --> STEP: 6/234 -- GLOBAL_STEP: 30660 + | > loss: -0.07845 (-0.06855) + | > log_mle: -0.22211 (-0.22767) + | > loss_dur: 0.14365 (0.15912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99610 (15.37057) + | > current_lr: 0.00003 + | > step_time: 0.90740 (5.13437) + | > loader_time: 0.00220 (0.03579) + + + --> STEP: 11/234 -- GLOBAL_STEP: 30665 + | > loss: -0.11058 (-0.08033) + | > log_mle: -0.23278 (-0.23207) + | > loss_dur: 0.12220 (0.15174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01116 (13.45979) + | > current_lr: 0.00003 + | > step_time: 0.91010 (4.67194) + | > loader_time: 0.00230 (0.06420) + + + --> STEP: 16/234 -- GLOBAL_STEP: 30670 + | > loss: -0.13149 (-0.08645) + | > log_mle: -0.23187 (-0.23226) + | > loss_dur: 0.10038 (0.14581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.62710 (12.26276) + | > current_lr: 0.00003 + | > step_time: 5.69400 (4.19876) + | > loader_time: 0.11350 (0.05153) + + + --> STEP: 21/234 -- GLOBAL_STEP: 30675 + | > loss: -0.08204 (-0.08632) + | > log_mle: -0.20822 (-0.22872) + | > loss_dur: 0.12618 (0.14241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.81470 (11.40514) + | > current_lr: 0.00003 + | > step_time: 1.94530 (3.90015) + | > loader_time: 0.09520 (0.05255) + + + --> STEP: 26/234 -- GLOBAL_STEP: 30680 + | > loss: -0.08311 (-0.08928) + | > log_mle: -0.22758 (-0.22827) + | > loss_dur: 0.14447 (0.13900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.75531 (10.87366) + | > current_lr: 0.00003 + | > step_time: 1.27140 (3.41112) + | > loader_time: 0.00190 (0.04293) + + + --> STEP: 31/234 -- GLOBAL_STEP: 30685 + | > loss: -0.05492 (-0.09029) + | > log_mle: -0.22673 (-0.22807) + | > loss_dur: 0.17181 (0.13778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01184 (10.36675) + | > current_lr: 0.00003 + | > step_time: 4.19510 (3.35509) + | > loader_time: 0.10430 (0.05425) + + + --> STEP: 36/234 -- GLOBAL_STEP: 30690 + | > loss: -0.07579 (-0.09026) + | > log_mle: -0.22719 (-0.22793) + | > loss_dur: 0.15140 (0.13768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.70902 (10.29681) + | > current_lr: 0.00003 + | > step_time: 4.49240 (3.56929) + | > loader_time: 0.31640 (0.06085) + + + --> STEP: 41/234 -- GLOBAL_STEP: 30695 + | > loss: -0.09981 (-0.08968) + | > log_mle: -0.22644 (-0.22755) + | > loss_dur: 0.12662 (0.13787) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.27532 (10.08780) + | > current_lr: 0.00003 + | > step_time: 4.09470 (3.44839) + | > loader_time: 0.00290 (0.05814) + + + --> STEP: 46/234 -- GLOBAL_STEP: 30700 + | > loss: -0.07952 (-0.08945) + | > log_mle: -0.22441 (-0.22768) + | > loss_dur: 0.14489 (0.13823) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.09596 (10.10717) + | > current_lr: 0.00003 + | > step_time: 1.19580 (3.22827) + | > loader_time: 0.00270 (0.05210) + + + --> STEP: 51/234 -- GLOBAL_STEP: 30705 + | > loss: -0.07950 (-0.08894) + | > log_mle: -0.21442 (-0.22695) + | > loss_dur: 0.13493 (0.13800) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.22692 (9.71064) + | > current_lr: 0.00003 + | > step_time: 2.04440 (3.06588) + | > loader_time: 0.00250 (0.04908) + + + --> STEP: 56/234 -- GLOBAL_STEP: 30710 + | > loss: -0.05799 (-0.08850) + | > log_mle: -0.23173 (-0.22744) + | > loss_dur: 0.17374 (0.13895) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.97476 (9.58414) + | > current_lr: 0.00003 + | > step_time: 1.29270 (2.91450) + | > loader_time: 0.00210 (0.04662) + + + --> STEP: 61/234 -- GLOBAL_STEP: 30715 + | > loss: -0.10439 (-0.08881) + | > log_mle: -0.22921 (-0.22781) + | > loss_dur: 0.12482 (0.13900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.57351 (9.55458) + | > current_lr: 0.00003 + | > step_time: 1.89980 (2.80367) + | > loader_time: 0.00370 (0.04441) + + + --> STEP: 66/234 -- GLOBAL_STEP: 30720 + | > loss: -0.09078 (-0.08846) + | > log_mle: -0.21792 (-0.22812) + | > loss_dur: 0.12714 (0.13966) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.71258 (9.62715) + | > current_lr: 0.00003 + | > step_time: 1.60140 (2.73671) + | > loader_time: 0.00190 (0.04249) + + + --> STEP: 71/234 -- GLOBAL_STEP: 30725 + | > loss: -0.06789 (-0.08713) + | > log_mle: -0.26072 (-0.22845) + | > loss_dur: 0.19283 (0.14132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.58498 (9.86166) + | > current_lr: 0.00003 + | > step_time: 1.60110 (2.66465) + | > loader_time: 0.00290 (0.04091) + + + --> STEP: 76/234 -- GLOBAL_STEP: 30730 + | > loss: -0.08791 (-0.08667) + | > log_mle: -0.24309 (-0.22896) + | > loss_dur: 0.15518 (0.14229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.66048 (9.90673) + | > current_lr: 0.00003 + | > step_time: 1.71430 (2.59563) + | > loader_time: 0.08560 (0.04078) + + + --> STEP: 81/234 -- GLOBAL_STEP: 30735 + | > loss: -0.10149 (-0.08713) + | > log_mle: -0.24981 (-0.22926) + | > loss_dur: 0.14832 (0.14212) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.58567 (9.91771) + | > current_lr: 0.00003 + | > step_time: 1.41070 (2.52629) + | > loader_time: 0.08370 (0.04066) + + + --> STEP: 86/234 -- GLOBAL_STEP: 30740 + | > loss: -0.08908 (-0.08742) + | > log_mle: -0.24825 (-0.22986) + | > loss_dur: 0.15917 (0.14244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.22748 (9.98306) + | > current_lr: 0.00003 + | > step_time: 2.10800 (2.47363) + | > loader_time: 0.08480 (0.03940) + + + --> STEP: 91/234 -- GLOBAL_STEP: 30745 + | > loss: -0.08931 (-0.08792) + | > log_mle: -0.26026 (-0.23171) + | > loss_dur: 0.17095 (0.14378) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.42200 (10.22362) + | > current_lr: 0.00003 + | > step_time: 1.10720 (2.44123) + | > loader_time: 0.07590 (0.03910) + + + --> STEP: 96/234 -- GLOBAL_STEP: 30750 + | > loss: -0.08667 (-0.09001) + | > log_mle: -0.24696 (-0.23511) + | > loss_dur: 0.16029 (0.14510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.08911 (10.91610) + | > current_lr: 0.00003 + | > step_time: 2.71050 (2.43307) + | > loader_time: 0.00280 (0.03814) + + + --> STEP: 101/234 -- GLOBAL_STEP: 30755 + | > loss: -0.12487 (-0.09124) + | > log_mle: -0.30230 (-0.23746) + | > loss_dur: 0.17744 (0.14622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.12601 (11.38253) + | > current_lr: 0.00003 + | > step_time: 2.79860 (2.44931) + | > loader_time: 0.08690 (0.03994) + + + --> STEP: 106/234 -- GLOBAL_STEP: 30760 + | > loss: -0.09549 (-0.09257) + | > log_mle: -0.30369 (-0.24025) + | > loss_dur: 0.20820 (0.14768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.09074 (11.92064) + | > current_lr: 0.00003 + | > step_time: 2.10660 (2.42449) + | > loader_time: 0.08540 (0.04051) + + + --> STEP: 111/234 -- GLOBAL_STEP: 30765 + | > loss: -0.13442 (-0.09355) + | > log_mle: -0.34573 (-0.24299) + | > loss_dur: 0.21131 (0.14944) + | > amp_scaler: 4096.00000 (2066.45045) + | > grad_norm: 42.92066 (12.77494) + | > current_lr: 0.00003 + | > step_time: 4.00350 (2.48918) + | > loader_time: 0.00340 (0.04118) + + + --> STEP: 116/234 -- GLOBAL_STEP: 30770 + | > loss: -0.10255 (-0.09464) + | > log_mle: -0.31753 (-0.24574) + | > loss_dur: 0.21498 (0.15110) + | > amp_scaler: 4096.00000 (2153.93103) + | > grad_norm: 27.60841 (13.37010) + | > current_lr: 0.00003 + | > step_time: 1.42170 (2.47680) + | > loader_time: 0.09290 (0.04183) + + + --> STEP: 121/234 -- GLOBAL_STEP: 30775 + | > loss: -0.06776 (-0.09533) + | > log_mle: -0.22886 (-0.24771) + | > loss_dur: 0.16110 (0.15238) + | > amp_scaler: 4096.00000 (2234.18182) + | > grad_norm: 13.36109 (13.68026) + | > current_lr: 0.00003 + | > step_time: 0.89820 (2.43991) + | > loader_time: 0.00440 (0.04095) + + + --> STEP: 126/234 -- GLOBAL_STEP: 30780 + | > loss: -0.15805 (-0.09674) + | > log_mle: -0.36560 (-0.25027) + | > loss_dur: 0.20756 (0.15353) + | > amp_scaler: 4096.00000 (2308.06349) + | > grad_norm: 39.33709 (14.19765) + | > current_lr: 0.00003 + | > step_time: 2.80230 (2.41908) + | > loader_time: 0.08770 (0.04076) + + + --> STEP: 131/234 -- GLOBAL_STEP: 30785 + | > loss: -0.19361 (-0.09850) + | > log_mle: -0.40518 (-0.25376) + | > loss_dur: 0.21157 (0.15527) + | > amp_scaler: 4096.00000 (2376.30534) + | > grad_norm: 44.23506 (14.93541) + | > current_lr: 0.00003 + | > step_time: 1.40630 (2.38711) + | > loader_time: 0.00290 (0.04065) + + + --> STEP: 136/234 -- GLOBAL_STEP: 30790 + | > loss: -0.22011 (-0.10055) + | > log_mle: -0.45401 (-0.25735) + | > loss_dur: 0.23390 (0.15680) + | > amp_scaler: 4096.00000 (2439.52941) + | > grad_norm: 50.02772 (15.60561) + | > current_lr: 0.00003 + | > step_time: 2.09820 (2.36154) + | > loader_time: 0.07900 (0.03983) + + + --> STEP: 141/234 -- GLOBAL_STEP: 30795 + | > loss: -0.15398 (-0.10201) + | > log_mle: -0.35509 (-0.26039) + | > loss_dur: 0.20111 (0.15838) + | > amp_scaler: 4096.00000 (2498.26950) + | > grad_norm: 68.86453 (16.68827) + | > current_lr: 0.00003 + | > step_time: 3.31470 (2.36306) + | > loader_time: 0.00310 (0.03977) + + + --> STEP: 146/234 -- GLOBAL_STEP: 30800 + | > loss: -0.19618 (-0.10457) + | > log_mle: -0.40862 (-0.26506) + | > loss_dur: 0.21244 (0.16049) + | > amp_scaler: 4096.00000 (2552.98630) + | > grad_norm: 42.54970 (17.69979) + | > current_lr: 0.00003 + | > step_time: 2.90180 (2.42204) + | > loader_time: 0.00300 (0.04056) + + + --> STEP: 151/234 -- GLOBAL_STEP: 30805 + | > loss: -0.18798 (-0.10721) + | > log_mle: -0.37756 (-0.26899) + | > loss_dur: 0.18957 (0.16178) + | > amp_scaler: 4096.00000 (2604.07947) + | > grad_norm: 41.73781 (18.35665) + | > current_lr: 0.00003 + | > step_time: 2.51020 (2.41806) + | > loader_time: 0.07950 (0.04038) + + + --> STEP: 156/234 -- GLOBAL_STEP: 30810 + | > loss: -0.21644 (-0.11101) + | > log_mle: -0.42373 (-0.27447) + | > loss_dur: 0.20729 (0.16346) + | > amp_scaler: 4096.00000 (2651.89744) + | > grad_norm: 37.79096 (19.43413) + | > current_lr: 0.00003 + | > step_time: 1.19770 (2.41689) + | > loader_time: 0.00390 (0.03919) + + + --> STEP: 161/234 -- GLOBAL_STEP: 30815 + | > loss: -0.23390 (-0.11403) + | > log_mle: -0.44627 (-0.27909) + | > loss_dur: 0.21237 (0.16506) + | > amp_scaler: 4096.00000 (2696.74534) + | > grad_norm: 47.60944 (20.44260) + | > current_lr: 0.00003 + | > step_time: 2.41230 (2.43019) + | > loader_time: 0.08890 (0.03979) + + + --> STEP: 166/234 -- GLOBAL_STEP: 30820 + | > loss: -0.19271 (-0.11665) + | > log_mle: -0.38814 (-0.28307) + | > loss_dur: 0.19543 (0.16642) + | > amp_scaler: 4096.00000 (2738.89157) + | > grad_norm: 31.07464 (21.27568) + | > current_lr: 0.00003 + | > step_time: 1.90340 (2.41990) + | > loader_time: 0.00420 (0.03934) + + + --> STEP: 171/234 -- GLOBAL_STEP: 30825 + | > loss: -0.27646 (-0.12018) + | > log_mle: -0.49124 (-0.28836) + | > loss_dur: 0.21479 (0.16818) + | > amp_scaler: 4096.00000 (2778.57310) + | > grad_norm: 66.65770 (22.69506) + | > current_lr: 0.00003 + | > step_time: 1.10540 (2.41290) + | > loader_time: 0.00370 (0.03886) + + + --> STEP: 176/234 -- GLOBAL_STEP: 30830 + | > loss: -0.24103 (-0.12370) + | > log_mle: -0.46573 (-0.29366) + | > loss_dur: 0.22470 (0.16997) + | > amp_scaler: 4096.00000 (2816.00000) + | > grad_norm: 55.21889 (23.80180) + | > current_lr: 0.00003 + | > step_time: 2.60070 (2.43527) + | > loader_time: 0.00340 (0.03896) + + + --> STEP: 181/234 -- GLOBAL_STEP: 30835 + | > loss: -0.17907 (-0.12653) + | > log_mle: -0.40372 (-0.29840) + | > loss_dur: 0.22464 (0.17187) + | > amp_scaler: 4096.00000 (2851.35912) + | > grad_norm: 32.54604 (24.95597) + | > current_lr: 0.00003 + | > step_time: 4.50420 (2.44878) + | > loader_time: 0.00330 (0.03905) + + + --> STEP: 186/234 -- GLOBAL_STEP: 30840 + | > loss: -0.19506 (-0.12944) + | > log_mle: -0.44240 (-0.30323) + | > loss_dur: 0.24734 (0.17378) + | > amp_scaler: 4096.00000 (2884.81720) + | > grad_norm: 52.76904 (26.18126) + | > current_lr: 0.00003 + | > step_time: 4.45750 (2.50752) + | > loader_time: 0.00920 (0.03963) + + + --> STEP: 191/234 -- GLOBAL_STEP: 30845 + | > loss: -0.22986 (-0.13264) + | > log_mle: -0.45213 (-0.30785) + | > loss_dur: 0.22227 (0.17521) + | > amp_scaler: 4096.00000 (2916.52356) + | > grad_norm: 72.12377 (27.32553) + | > current_lr: 0.00003 + | > step_time: 2.50380 (2.51021) + | > loader_time: 0.08470 (0.04009) + + + --> STEP: 196/234 -- GLOBAL_STEP: 30850 + | > loss: -0.20973 (-0.13583) + | > log_mle: -0.45296 (-0.31257) + | > loss_dur: 0.24324 (0.17674) + | > amp_scaler: 4096.00000 (2946.61224) + | > grad_norm: 58.75001 (28.30806) + | > current_lr: 0.00003 + | > step_time: 10.99930 (2.59049) + | > loader_time: 0.10580 (0.04120) + + + --> STEP: 201/234 -- GLOBAL_STEP: 30855 + | > loss: -0.16731 (-0.13850) + | > log_mle: -0.41641 (-0.31686) + | > loss_dur: 0.24910 (0.17836) + | > amp_scaler: 4096.00000 (2975.20398) + | > grad_norm: 53.85706 (29.32439) + | > current_lr: 0.00003 + | > step_time: 2.98610 (2.62101) + | > loader_time: 0.10810 (0.04219) + + + --> STEP: 206/234 -- GLOBAL_STEP: 30860 + | > loss: -0.28041 (-0.14171) + | > log_mle: -0.52597 (-0.32168) + | > loss_dur: 0.24556 (0.17997) + | > amp_scaler: 4096.00000 (3002.40777) + | > grad_norm: 66.14774 (30.39025) + | > current_lr: 0.00003 + | > step_time: 4.20370 (2.66292) + | > loader_time: 0.09820 (0.04357) + + + --> STEP: 211/234 -- GLOBAL_STEP: 30865 + | > loss: -0.32819 (-0.14544) + | > log_mle: -0.59976 (-0.32717) + | > loss_dur: 0.27158 (0.18173) + | > amp_scaler: 4096.00000 (3028.32227) + | > grad_norm: 87.18015 (31.45338) + | > current_lr: 0.00003 + | > step_time: 6.09810 (2.74490) + | > loader_time: 0.00480 (0.04396) + + + --> STEP: 216/234 -- GLOBAL_STEP: 30870 + | > loss: -0.31731 (-0.14900) + | > log_mle: -0.57735 (-0.33231) + | > loss_dur: 0.26003 (0.18331) + | > amp_scaler: 4096.00000 (3053.03704) + | > grad_norm: 127.47085 (32.95556) + | > current_lr: 0.00003 + | > step_time: 3.40670 (2.81473) + | > loader_time: 0.58960 (0.04881) + + + --> STEP: 221/234 -- GLOBAL_STEP: 30875 + | > loss: -0.26076 (-0.15261) + | > log_mle: -0.49707 (-0.33742) + | > loss_dur: 0.23631 (0.18480) + | > amp_scaler: 4096.00000 (3076.63348) + | > grad_norm: 69.94815 (34.24335) + | > current_lr: 0.00003 + | > step_time: 2.40650 (2.83022) + | > loader_time: 0.00410 (0.04963) + + + --> STEP: 226/234 -- GLOBAL_STEP: 30880 + | > loss: -0.33719 (-0.15643) + | > log_mle: -0.60129 (-0.34302) + | > loss_dur: 0.26410 (0.18659) + | > amp_scaler: 4096.00000 (3099.18584) + | > grad_norm: 78.08322 (35.64807) + | > current_lr: 0.00003 + | > step_time: 0.25130 (2.77864) + | > loader_time: 0.00440 (0.04986) + + + --> STEP: 231/234 -- GLOBAL_STEP: 30885 + | > loss: -0.26301 (-0.15922) + | > log_mle: -0.66659 (-0.34883) + | > loss_dur: 0.40357 (0.18961) + | > amp_scaler: 4096.00000 (3120.76190) + | > grad_norm: 99.32404 (37.18863) + | > current_lr: 0.00003 + | > step_time: 0.28560 (2.72426) + | > loader_time: 0.00390 (0.04887) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22032 (+0.21804) + | > avg_loss: -0.19800 (+0.00150) + | > avg_log_mle: -0.42526 (-0.00235) + | > avg_loss_dur: 0.22725 (+0.00385) + + + > EPOCH: 132/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 12:59:37)  + + --> STEP: 2/234 -- GLOBAL_STEP: 30890 + | > loss: -0.07793 (-0.08504) + | > log_mle: -0.22639 (-0.22956) + | > loss_dur: 0.14846 (0.14451) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.77744 (12.36910) + | > current_lr: 0.00003 + | > step_time: 5.69480 (3.84559) + | > loader_time: 4.69550 (6.09707) + + + --> STEP: 7/234 -- GLOBAL_STEP: 30895 + | > loss: -0.10944 (-0.07958) + | > log_mle: -0.24208 (-0.23136) + | > loss_dur: 0.13264 (0.15179) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.73625 (12.66904) + | > current_lr: 0.00003 + | > step_time: 4.81380 (4.30173) + | > loader_time: 0.08150 (1.85502) + + + --> STEP: 12/234 -- GLOBAL_STEP: 30900 + | > loss: -0.10023 (-0.08451) + | > log_mle: -0.23404 (-0.23368) + | > loss_dur: 0.13381 (0.14918) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.56904 (12.28990) + | > current_lr: 0.00003 + | > step_time: 5.41930 (5.02098) + | > loader_time: 0.00410 (1.10624) + + + --> STEP: 17/234 -- GLOBAL_STEP: 30905 + | > loss: -0.07532 (-0.08918) + | > log_mle: -0.21165 (-0.23252) + | > loss_dur: 0.13633 (0.14333) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.99563 (11.23820) + | > current_lr: 0.00003 + | > step_time: 4.58530 (4.20263) + | > loader_time: 0.10040 (0.78725) + + + --> STEP: 22/234 -- GLOBAL_STEP: 30910 + | > loss: -0.11457 (-0.09058) + | > log_mle: -0.23244 (-0.23024) + | > loss_dur: 0.11787 (0.13965) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.22164 (10.97860) + | > current_lr: 0.00003 + | > step_time: 1.99290 (4.13819) + | > loader_time: 0.00310 (0.61296) + + + --> STEP: 27/234 -- GLOBAL_STEP: 30915 + | > loss: -0.11300 (-0.09355) + | > log_mle: -0.23736 (-0.22987) + | > loss_dur: 0.12435 (0.13631) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.31257 (10.28396) + | > current_lr: 0.00003 + | > step_time: 4.48930 (4.02616) + | > loader_time: 0.11030 (0.51876) + + + --> STEP: 32/234 -- GLOBAL_STEP: 30920 + | > loss: -0.13132 (-0.09504) + | > log_mle: -0.24371 (-0.22997) + | > loss_dur: 0.11238 (0.13494) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.83942 (9.75585) + | > current_lr: 0.00003 + | > step_time: 6.49750 (4.20016) + | > loader_time: 0.09860 (0.44989) + + + --> STEP: 37/234 -- GLOBAL_STEP: 30925 + | > loss: -0.10548 (-0.09420) + | > log_mle: -0.22484 (-0.22952) + | > loss_dur: 0.11936 (0.13532) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.88528 (9.65823) + | > current_lr: 0.00003 + | > step_time: 3.71030 (4.04652) + | > loader_time: 0.10630 (0.39701) + + + --> STEP: 42/234 -- GLOBAL_STEP: 30930 + | > loss: -0.08139 (-0.09306) + | > log_mle: -0.21610 (-0.22903) + | > loss_dur: 0.13471 (0.13597) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.02903 (9.57907) + | > current_lr: 0.00003 + | > step_time: 0.90730 (3.96355) + | > loader_time: 0.00240 (0.35004) + + + --> STEP: 47/234 -- GLOBAL_STEP: 30935 + | > loss: -0.07316 (-0.09285) + | > log_mle: -0.22457 (-0.22946) + | > loss_dur: 0.15140 (0.13661) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.71986 (9.67264) + | > current_lr: 0.00003 + | > step_time: 1.01520 (3.64868) + | > loader_time: 0.00240 (0.31466) + + + --> STEP: 52/234 -- GLOBAL_STEP: 30940 + | > loss: -0.08624 (-0.09264) + | > log_mle: -0.22390 (-0.22881) + | > loss_dur: 0.13766 (0.13617) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.81385 (9.32911) + | > current_lr: 0.00003 + | > step_time: 2.00830 (3.45699) + | > loader_time: 0.00300 (0.28466) + + + --> STEP: 57/234 -- GLOBAL_STEP: 30945 + | > loss: -0.07587 (-0.09259) + | > log_mle: -0.21543 (-0.22922) + | > loss_dur: 0.13957 (0.13664) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.88657 (9.10564) + | > current_lr: 0.00003 + | > step_time: 1.51460 (3.32023) + | > loader_time: 0.08370 (0.26132) + + + --> STEP: 62/234 -- GLOBAL_STEP: 30950 + | > loss: -0.06241 (-0.09295) + | > log_mle: -0.26480 (-0.23040) + | > loss_dur: 0.20239 (0.13745) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.38622 (9.41467) + | > current_lr: 0.00003 + | > step_time: 2.38950 (3.21641) + | > loader_time: 0.00200 (0.24178) + + + --> STEP: 67/234 -- GLOBAL_STEP: 30955 + | > loss: -0.08470 (-0.09257) + | > log_mle: -0.24757 (-0.23032) + | > loss_dur: 0.16288 (0.13775) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.92722 (9.29247) + | > current_lr: 0.00003 + | > step_time: 1.39250 (3.11807) + | > loader_time: 0.00240 (0.22774) + + + --> STEP: 72/234 -- GLOBAL_STEP: 30960 + | > loss: -0.06787 (-0.09065) + | > log_mle: -0.22849 (-0.23029) + | > loss_dur: 0.16062 (0.13965) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.27974 (9.42783) + | > current_lr: 0.00003 + | > step_time: 1.80680 (3.03218) + | > loader_time: 0.00460 (0.21330) + + + --> STEP: 77/234 -- GLOBAL_STEP: 30965 + | > loss: -0.09681 (-0.09026) + | > log_mle: -0.23882 (-0.23081) + | > loss_dur: 0.14201 (0.14055) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.98088 (9.62456) + | > current_lr: 0.00003 + | > step_time: 1.23670 (2.93431) + | > loader_time: 0.00200 (0.20216) + + + --> STEP: 82/234 -- GLOBAL_STEP: 30970 + | > loss: -0.09654 (-0.09033) + | > log_mle: -0.23157 (-0.23092) + | > loss_dur: 0.13503 (0.14058) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.14159 (9.60956) + | > current_lr: 0.00003 + | > step_time: 1.61520 (2.88876) + | > loader_time: 0.00230 (0.19222) + + + --> STEP: 87/234 -- GLOBAL_STEP: 30975 + | > loss: -0.08603 (-0.09002) + | > log_mle: -0.23932 (-0.23151) + | > loss_dur: 0.15330 (0.14148) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.85082 (9.77792) + | > current_lr: 0.00003 + | > step_time: 4.49850 (2.85952) + | > loader_time: 0.00340 (0.18345) + + + --> STEP: 92/234 -- GLOBAL_STEP: 30980 + | > loss: -0.13309 (-0.09082) + | > log_mle: -0.28536 (-0.23373) + | > loss_dur: 0.15226 (0.14291) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.04990 (10.08937) + | > current_lr: 0.00003 + | > step_time: 2.31780 (2.84778) + | > loader_time: 0.00300 (0.17558) + + + --> STEP: 97/234 -- GLOBAL_STEP: 30985 + | > loss: -0.11259 (-0.09246) + | > log_mle: -0.27132 (-0.23668) + | > loss_dur: 0.15874 (0.14422) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.87416 (10.87654) + | > current_lr: 0.00003 + | > step_time: 1.19750 (2.77332) + | > loader_time: 0.00510 (0.16670) + + + --> STEP: 102/234 -- GLOBAL_STEP: 30990 + | > loss: -0.08084 (-0.09326) + | > log_mle: -0.25383 (-0.23872) + | > loss_dur: 0.17298 (0.14546) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.13106 (11.29986) + | > current_lr: 0.00003 + | > step_time: 1.88480 (2.71593) + | > loader_time: 0.10470 (0.16129) + + + --> STEP: 107/234 -- GLOBAL_STEP: 30995 + | > loss: -0.11792 (-0.09473) + | > log_mle: -0.30068 (-0.24186) + | > loss_dur: 0.18275 (0.14714) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.85408 (11.93869) + | > current_lr: 0.00003 + | > step_time: 1.11040 (2.64515) + | > loader_time: 0.08780 (0.15542) + + + --> STEP: 112/234 -- GLOBAL_STEP: 31000 + | > loss: -0.12178 (-0.09556) + | > log_mle: -0.31004 (-0.24462) + | > loss_dur: 0.18826 (0.14906) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.38477 (12.63731) + | > current_lr: 0.00003 + | > step_time: 1.29860 (2.61518) + | > loader_time: 0.08700 (0.15014) + + + --> STEP: 117/234 -- GLOBAL_STEP: 31005 + | > loss: -0.13410 (-0.09672) + | > log_mle: -0.30544 (-0.24727) + | > loss_dur: 0.17134 (0.15055) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.48666 (13.19559) + | > current_lr: 0.00003 + | > step_time: 1.18680 (2.57687) + | > loader_time: 0.00250 (0.14462) + + + --> STEP: 122/234 -- GLOBAL_STEP: 31010 + | > loss: -0.11648 (-0.09744) + | > log_mle: -0.28466 (-0.24898) + | > loss_dur: 0.16818 (0.15154) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.49495 (13.42094) + | > current_lr: 0.00003 + | > step_time: 2.99260 (2.55966) + | > loader_time: 0.00330 (0.13958) + + + --> STEP: 127/234 -- GLOBAL_STEP: 31015 + | > loss: -0.14333 (-0.09904) + | > log_mle: -0.34017 (-0.25195) + | > loss_dur: 0.19684 (0.15291) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.92174 (14.00072) + | > current_lr: 0.00003 + | > step_time: 1.35820 (2.53756) + | > loader_time: 0.00290 (0.13551) + + + --> STEP: 132/234 -- GLOBAL_STEP: 31020 + | > loss: -0.14467 (-0.10111) + | > log_mle: -0.31876 (-0.25540) + | > loss_dur: 0.17409 (0.15429) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.69392 (14.64445) + | > current_lr: 0.00003 + | > step_time: 4.31350 (2.55666) + | > loader_time: 0.09830 (0.13260) + + + --> STEP: 137/234 -- GLOBAL_STEP: 31025 + | > loss: -0.11627 (-0.10293) + | > log_mle: -0.33492 (-0.25905) + | > loss_dur: 0.21865 (0.15613) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 39.55973 (15.41023) + | > current_lr: 0.00003 + | > step_time: 3.61190 (2.63060) + | > loader_time: 0.00410 (0.13061) + + + --> STEP: 142/234 -- GLOBAL_STEP: 31030 + | > loss: -0.14349 (-0.10446) + | > log_mle: -0.35008 (-0.26219) + | > loss_dur: 0.20659 (0.15773) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.62785 (16.10731) + | > current_lr: 0.00003 + | > step_time: 2.25560 (2.60848) + | > loader_time: 0.00280 (0.12794) + + + --> STEP: 147/234 -- GLOBAL_STEP: 31035 + | > loss: -0.15236 (-0.10731) + | > log_mle: -0.34993 (-0.26691) + | > loss_dur: 0.19757 (0.15961) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.97669 (17.13851) + | > current_lr: 0.00003 + | > step_time: 2.80670 (2.58313) + | > loader_time: 0.08480 (0.12543) + + + --> STEP: 152/234 -- GLOBAL_STEP: 31040 + | > loss: -0.20447 (-0.11014) + | > log_mle: -0.43161 (-0.27133) + | > loss_dur: 0.22714 (0.16119) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 54.07493 (18.06318) + | > current_lr: 0.00003 + | > step_time: 2.69310 (2.57520) + | > loader_time: 0.00250 (0.12255) + + + --> STEP: 157/234 -- GLOBAL_STEP: 31045 + | > loss: -0.16530 (-0.11355) + | > log_mle: -0.38184 (-0.27636) + | > loss_dur: 0.21653 (0.16281) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.73727 (19.21890) + | > current_lr: 0.00003 + | > step_time: 2.30920 (2.58682) + | > loader_time: 0.00260 (0.11998) + + + --> STEP: 162/234 -- GLOBAL_STEP: 31050 + | > loss: -0.20442 (-0.11664) + | > log_mle: -0.41331 (-0.28114) + | > loss_dur: 0.20890 (0.16449) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.17110 (20.37012) + | > current_lr: 0.00003 + | > step_time: 1.80790 (2.57181) + | > loader_time: 0.00370 (0.11643) + + + --> STEP: 167/234 -- GLOBAL_STEP: 31055 + | > loss: -0.28782 (-0.11970) + | > log_mle: -0.49834 (-0.28559) + | > loss_dur: 0.21052 (0.16589) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.87548 (21.27964) + | > current_lr: 0.00003 + | > step_time: 4.11660 (2.57798) + | > loader_time: 0.08860 (0.11408) + + + --> STEP: 172/234 -- GLOBAL_STEP: 31060 + | > loss: -0.25399 (-0.12298) + | > log_mle: -0.49258 (-0.29092) + | > loss_dur: 0.23858 (0.16794) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 70.00948 (22.42560) + | > current_lr: 0.00003 + | > step_time: 3.79660 (2.58616) + | > loader_time: 0.09440 (0.11144) + + + --> STEP: 177/234 -- GLOBAL_STEP: 31065 + | > loss: -0.20890 (-0.12617) + | > log_mle: -0.43935 (-0.29588) + | > loss_dur: 0.23045 (0.16971) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 74.50146 (23.66524) + | > current_lr: 0.00003 + | > step_time: 2.69860 (2.59794) + | > loader_time: 0.00340 (0.11098) + + + --> STEP: 182/234 -- GLOBAL_STEP: 31070 + | > loss: -0.22735 (-0.12882) + | > log_mle: -0.48647 (-0.30067) + | > loss_dur: 0.25912 (0.17185) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.54305 (24.76684) + | > current_lr: 0.00003 + | > step_time: 4.49180 (2.60848) + | > loader_time: 0.10510 (0.11011) + + + --> STEP: 187/234 -- GLOBAL_STEP: 31075 + | > loss: -0.24781 (-0.13175) + | > log_mle: -0.48677 (-0.30546) + | > loss_dur: 0.23896 (0.17372) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 65.92863 (25.96390) + | > current_lr: 0.00003 + | > step_time: 5.98640 (2.69336) + | > loader_time: 0.19940 (0.10983) + + + --> STEP: 192/234 -- GLOBAL_STEP: 31080 + | > loss: -0.29675 (-0.13505) + | > log_mle: -0.52183 (-0.31021) + | > loss_dur: 0.22509 (0.17516) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 63.52129 (27.11849) + | > current_lr: 0.00003 + | > step_time: 3.40590 (2.78632) + | > loader_time: 0.09820 (0.11118) + + + --> STEP: 197/234 -- GLOBAL_STEP: 31085 + | > loss: -0.26600 (-0.13814) + | > log_mle: -0.48818 (-0.31481) + | > loss_dur: 0.22218 (0.17667) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.79343 (28.16939) + | > current_lr: 0.00003 + | > step_time: 3.11360 (2.79986) + | > loader_time: 0.18450 (0.11030) + + + --> STEP: 202/234 -- GLOBAL_STEP: 31090 + | > loss: -0.32994 (-0.14104) + | > log_mle: -0.57613 (-0.31946) + | > loss_dur: 0.24619 (0.17842) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 93.76728 (29.36011) + | > current_lr: 0.00003 + | > step_time: 6.10500 (2.86175) + | > loader_time: 0.09340 (0.10949) + + + --> STEP: 207/234 -- GLOBAL_STEP: 31095 + | > loss: -0.30277 (-0.14395) + | > log_mle: -0.56237 (-0.32409) + | > loss_dur: 0.25960 (0.18014) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 82.57469 (30.35830) + | > current_lr: 0.00003 + | > step_time: 3.31090 (2.92415) + | > loader_time: 0.08520 (0.11012) + + + --> STEP: 212/234 -- GLOBAL_STEP: 31100 + | > loss: -0.30018 (-0.14748) + | > log_mle: -0.54613 (-0.32932) + | > loss_dur: 0.24595 (0.18184) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 89.80972 (31.60687) + | > current_lr: 0.00003 + | > step_time: 6.30370 (3.01883) + | > loader_time: 0.00580 (0.10902) + + + --> STEP: 217/234 -- GLOBAL_STEP: 31105 + | > loss: -0.32437 (-0.15112) + | > log_mle: -0.57501 (-0.33457) + | > loss_dur: 0.25064 (0.18344) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 78.49551 (32.97356) + | > current_lr: 0.00003 + | > step_time: 2.80000 (3.04248) + | > loader_time: 0.19670 (0.10829) + + + --> STEP: 222/234 -- GLOBAL_STEP: 31110 + | > loss: -0.30046 (-0.15460) + | > log_mle: -0.58437 (-0.33969) + | > loss_dur: 0.28390 (0.18509) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 86.52996 (34.22225) + | > current_lr: 0.00003 + | > step_time: 1.98770 (3.04515) + | > loader_time: 0.00300 (0.10713) + + + --> STEP: 227/234 -- GLOBAL_STEP: 31115 + | > loss: -0.28398 (-0.15834) + | > log_mle: -0.55592 (-0.34517) + | > loss_dur: 0.27195 (0.18682) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 91.98989 (35.54644) + | > current_lr: 0.00003 + | > step_time: 0.31700 (3.00543) + | > loader_time: 0.00430 (0.10487) + + + --> STEP: 232/234 -- GLOBAL_STEP: 31120 + | > loss: -0.22936 (-0.16097) + | > log_mle: -0.75242 (-0.35187) + | > loss_dur: 0.52306 (0.19090) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 172.37709 (37.52590) + | > current_lr: 0.00003 + | > step_time: 0.32320 (2.94661) + | > loader_time: 0.11790 (0.10318) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.77106 (+0.55074) + | > avg_loss: -0.19211 (+0.00590) + | > avg_log_mle: -0.42457 (+0.00068) + | > avg_loss_dur: 0.23247 (+0.00521) + + + > EPOCH: 133/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 13:12:17)  + + --> STEP: 3/234 -- GLOBAL_STEP: 31125 + | > loss: -0.03414 (-0.07877) + | > log_mle: -0.22946 (-0.23107) + | > loss_dur: 0.19532 (0.15230) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.52960 (16.92556) + | > current_lr: 0.00003 + | > step_time: 4.89270 (4.26046) + | > loader_time: 0.00210 (0.03460) + + + --> STEP: 8/234 -- GLOBAL_STEP: 31130 + | > loss: -0.10850 (-0.08856) + | > log_mle: -0.24490 (-0.23250) + | > loss_dur: 0.13640 (0.14394) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.85673 (15.57580) + | > current_lr: 0.00003 + | > step_time: 6.09680 (4.87255) + | > loader_time: 0.00380 (0.05110) + + + --> STEP: 13/234 -- GLOBAL_STEP: 31135 + | > loss: -0.09783 (-0.09040) + | > log_mle: -0.23228 (-0.23314) + | > loss_dur: 0.13445 (0.14274) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.93741 (14.10048) + | > current_lr: 0.00003 + | > step_time: 1.31160 (4.32260) + | > loader_time: 0.00230 (0.05376) + + + --> STEP: 18/234 -- GLOBAL_STEP: 31140 + | > loss: -0.08248 (-0.09237) + | > log_mle: -0.22957 (-0.23184) + | > loss_dur: 0.14708 (0.13947) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.97010 (12.79440) + | > current_lr: 0.00003 + | > step_time: 1.12890 (3.52841) + | > loader_time: 0.00120 (0.03934) + + + --> STEP: 23/234 -- GLOBAL_STEP: 31145 + | > loss: -0.12230 (-0.09514) + | > log_mle: -0.23440 (-0.23031) + | > loss_dur: 0.11210 (0.13517) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.84337 (11.79506) + | > current_lr: 0.00003 + | > step_time: 1.77380 (3.06955) + | > loader_time: 0.00120 (0.03117) + + + --> STEP: 28/234 -- GLOBAL_STEP: 31150 + | > loss: -0.11836 (-0.09757) + | > log_mle: -0.22781 (-0.22992) + | > loss_dur: 0.10945 (0.13235) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 3.97842 (10.84069) + | > current_lr: 0.00003 + | > step_time: 0.86130 (2.70812) + | > loader_time: 0.00210 (0.02604) + + + --> STEP: 33/234 -- GLOBAL_STEP: 31155 + | > loss: -0.09811 (-0.09778) + | > log_mle: -0.22367 (-0.23010) + | > loss_dur: 0.12556 (0.13232) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.23707 (10.24282) + | > current_lr: 0.00003 + | > step_time: 2.17900 (2.57212) + | > loader_time: 0.00160 (0.02241) + + + --> STEP: 38/234 -- GLOBAL_STEP: 31160 + | > loss: -0.09923 (-0.09702) + | > log_mle: -0.23962 (-0.23024) + | > loss_dur: 0.14040 (0.13321) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.85940 (10.15331) + | > current_lr: 0.00003 + | > step_time: 1.75290 (2.42404) + | > loader_time: 0.00230 (0.01977) + + + --> STEP: 43/234 -- GLOBAL_STEP: 31165 + | > loss: -0.08626 (-0.09620) + | > log_mle: -0.23524 (-0.22982) + | > loss_dur: 0.14898 (0.13362) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.20054 (9.96670) + | > current_lr: 0.00003 + | > step_time: 1.36480 (2.27448) + | > loader_time: 0.00200 (0.01772) + + + --> STEP: 48/234 -- GLOBAL_STEP: 31170 + | > loss: -0.09614 (-0.09563) + | > log_mle: -0.21936 (-0.22977) + | > loss_dur: 0.12322 (0.13414) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.84832 (9.90172) + | > current_lr: 0.00003 + | > step_time: 2.10610 (2.23770) + | > loader_time: 0.00440 (0.01800) + + + --> STEP: 53/234 -- GLOBAL_STEP: 31175 + | > loss: -0.08988 (-0.09517) + | > log_mle: -0.23871 (-0.22954) + | > loss_dur: 0.14883 (0.13437) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.35571 (9.63260) + | > current_lr: 0.00003 + | > step_time: 1.20170 (2.20261) + | > loader_time: 0.09250 (0.02278) + + + --> STEP: 58/234 -- GLOBAL_STEP: 31180 + | > loss: -0.09890 (-0.09480) + | > log_mle: -0.22375 (-0.22960) + | > loss_dur: 0.12485 (0.13479) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.62596 (9.46802) + | > current_lr: 0.00003 + | > step_time: 1.67130 (2.16693) + | > loader_time: 0.09450 (0.02260) + + + --> STEP: 63/234 -- GLOBAL_STEP: 31185 + | > loss: -0.06962 (-0.09456) + | > log_mle: -0.23073 (-0.23092) + | > loss_dur: 0.16111 (0.13636) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.89496 (9.79344) + | > current_lr: 0.00003 + | > step_time: 2.59470 (2.16161) + | > loader_time: 0.00290 (0.02235) + + + --> STEP: 68/234 -- GLOBAL_STEP: 31190 + | > loss: -0.06324 (-0.09413) + | > log_mle: -0.22527 (-0.23082) + | > loss_dur: 0.16203 (0.13669) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.04807 (9.69054) + | > current_lr: 0.00003 + | > step_time: 1.18520 (2.12448) + | > loader_time: 0.00170 (0.02092) + + + --> STEP: 73/234 -- GLOBAL_STEP: 31195 + | > loss: -0.07961 (-0.09266) + | > log_mle: -0.25054 (-0.23115) + | > loss_dur: 0.17092 (0.13848) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.35601 (9.80175) + | > current_lr: 0.00003 + | > step_time: 2.70160 (2.11005) + | > loader_time: 0.00680 (0.02209) + + + --> STEP: 78/234 -- GLOBAL_STEP: 31200 + | > loss: -0.07578 (-0.09219) + | > log_mle: -0.22430 (-0.23148) + | > loss_dur: 0.14852 (0.13929) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.85754 (9.81499) + | > current_lr: 0.00003 + | > step_time: 1.28800 (2.14107) + | > loader_time: 0.00210 (0.02320) + + + --> STEP: 83/234 -- GLOBAL_STEP: 31205 + | > loss: -0.07071 (-0.09216) + | > log_mle: -0.25097 (-0.23201) + | > loss_dur: 0.18026 (0.13985) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.98958 (9.82649) + | > current_lr: 0.00003 + | > step_time: 1.62100 (2.09792) + | > loader_time: 0.10740 (0.02424) + + + --> STEP: 88/234 -- GLOBAL_STEP: 31210 + | > loss: -0.10847 (-0.09241) + | > log_mle: -0.28524 (-0.23308) + | > loss_dur: 0.17677 (0.14067) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.80124 (9.93711) + | > current_lr: 0.00003 + | > step_time: 1.00440 (2.06913) + | > loader_time: 0.00190 (0.02300) + + + --> STEP: 93/234 -- GLOBAL_STEP: 31215 + | > loss: -0.10900 (-0.09314) + | > log_mle: -0.29330 (-0.23533) + | > loss_dur: 0.18430 (0.14219) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.14880 (10.47584) + | > current_lr: 0.00003 + | > step_time: 1.41450 (2.04829) + | > loader_time: 0.08460 (0.02383) + + + --> STEP: 98/234 -- GLOBAL_STEP: 31220 + | > loss: -0.07708 (-0.09418) + | > log_mle: -0.22741 (-0.23757) + | > loss_dur: 0.15033 (0.14339) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.38791 (10.80986) + | > current_lr: 0.00003 + | > step_time: 1.18220 (2.04985) + | > loader_time: 0.00210 (0.02448) + + + --> STEP: 103/234 -- GLOBAL_STEP: 31225 + | > loss: -0.12801 (-0.09547) + | > log_mle: -0.32890 (-0.24063) + | > loss_dur: 0.20090 (0.14516) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.94037 (11.35197) + | > current_lr: 0.00003 + | > step_time: 2.50090 (2.03866) + | > loader_time: 0.00320 (0.02423) + + + --> STEP: 108/234 -- GLOBAL_STEP: 31230 + | > loss: -0.11718 (-0.09680) + | > log_mle: -0.27434 (-0.24322) + | > loss_dur: 0.15716 (0.14642) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.50006 (11.83101) + | > current_lr: 0.00003 + | > step_time: 2.84140 (2.04738) + | > loader_time: 0.00430 (0.02327) + + + --> STEP: 113/234 -- GLOBAL_STEP: 31235 + | > loss: -0.14279 (-0.09806) + | > log_mle: -0.32327 (-0.24651) + | > loss_dur: 0.18048 (0.14844) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.54979 (12.52646) + | > current_lr: 0.00003 + | > step_time: 1.59710 (2.03243) + | > loader_time: 0.00310 (0.02238) + + + --> STEP: 118/234 -- GLOBAL_STEP: 31240 + | > loss: -0.10295 (-0.09893) + | > log_mle: -0.29134 (-0.24894) + | > loss_dur: 0.18840 (0.15002) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.35146 (12.98727) + | > current_lr: 0.00003 + | > step_time: 1.08910 (2.05222) + | > loader_time: 0.00340 (0.02304) + + + --> STEP: 123/234 -- GLOBAL_STEP: 31245 + | > loss: -0.08806 (-0.09934) + | > log_mle: -0.26161 (-0.25045) + | > loss_dur: 0.17356 (0.15111) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.78699 (13.25199) + | > current_lr: 0.00003 + | > step_time: 2.20610 (2.07056) + | > loader_time: 0.10180 (0.02441) + + + --> STEP: 128/234 -- GLOBAL_STEP: 31250 + | > loss: -0.15060 (-0.10140) + | > log_mle: -0.31946 (-0.25388) + | > loss_dur: 0.16886 (0.15248) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.01491 (14.20467) + | > current_lr: 0.00003 + | > step_time: 1.99760 (2.07709) + | > loader_time: 0.08730 (0.02697) + + + --> STEP: 133/234 -- GLOBAL_STEP: 31255 + | > loss: -0.16059 (-0.10348) + | > log_mle: -0.35033 (-0.25755) + | > loss_dur: 0.18974 (0.15407) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.14465 (14.83131) + | > current_lr: 0.00003 + | > step_time: 2.60400 (2.10149) + | > loader_time: 0.00310 (0.02749) + + + --> STEP: 138/234 -- GLOBAL_STEP: 31260 + | > loss: -0.11757 (-0.10505) + | > log_mle: -0.30109 (-0.26088) + | > loss_dur: 0.18352 (0.15583) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 25.98398 (15.60467) + | > current_lr: 0.00003 + | > step_time: 1.69270 (2.10167) + | > loader_time: 0.00350 (0.02834) + + + --> STEP: 143/234 -- GLOBAL_STEP: 31265 + | > loss: -0.19894 (-0.10725) + | > log_mle: -0.44156 (-0.26500) + | > loss_dur: 0.24262 (0.15775) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.39968 (16.62281) + | > current_lr: 0.00003 + | > step_time: 1.17430 (2.11211) + | > loader_time: 0.00260 (0.02854) + + + --> STEP: 148/234 -- GLOBAL_STEP: 31270 + | > loss: -0.18361 (-0.10978) + | > log_mle: -0.35629 (-0.26903) + | > loss_dur: 0.17268 (0.15925) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.82599 (17.56633) + | > current_lr: 0.00003 + | > step_time: 3.10200 (2.13318) + | > loader_time: 0.08520 (0.02825) + + + --> STEP: 153/234 -- GLOBAL_STEP: 31275 + | > loss: -0.27691 (-0.11328) + | > log_mle: -0.47618 (-0.27415) + | > loss_dur: 0.19928 (0.16087) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 75.65317 (18.96872) + | > current_lr: 0.00003 + | > step_time: 4.80500 (2.16615) + | > loader_time: 0.88760 (0.03437) + + + --> STEP: 158/234 -- GLOBAL_STEP: 31280 + | > loss: -0.18813 (-0.11600) + | > log_mle: -0.41206 (-0.27860) + | > loss_dur: 0.22392 (0.16260) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.85419 (20.26029) + | > current_lr: 0.00003 + | > step_time: 1.60740 (2.17604) + | > loader_time: 0.08510 (0.03512) + + + --> STEP: 163/234 -- GLOBAL_STEP: 31285 + | > loss: -0.17809 (-0.11902) + | > log_mle: -0.38670 (-0.28310) + | > loss_dur: 0.20861 (0.16408) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 40.01339 (21.24640) + | > current_lr: 0.00003 + | > step_time: 2.21880 (2.17703) + | > loader_time: 0.19170 (0.03530) + + + --> STEP: 168/234 -- GLOBAL_STEP: 31290 + | > loss: -0.20542 (-0.12200) + | > log_mle: -0.44023 (-0.28774) + | > loss_dur: 0.23481 (0.16574) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 69.03558 (22.54282) + | > current_lr: 0.00003 + | > step_time: 1.70150 (2.18470) + | > loader_time: 0.08710 (0.03547) + + + --> STEP: 173/234 -- GLOBAL_STEP: 31295 + | > loss: -0.22815 (-0.12538) + | > log_mle: -0.44618 (-0.29294) + | > loss_dur: 0.21803 (0.16756) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.94852 (23.92475) + | > current_lr: 0.00003 + | > step_time: 4.41600 (2.21757) + | > loader_time: 0.00330 (0.03554) + + + --> STEP: 178/234 -- GLOBAL_STEP: 31300 + | > loss: -0.26108 (-0.12870) + | > log_mle: -0.50101 (-0.29813) + | > loss_dur: 0.23993 (0.16943) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 84.28586 (25.19483) + | > current_lr: 0.00003 + | > step_time: 2.31850 (2.21966) + | > loader_time: 0.08550 (0.03611) + + + --> STEP: 183/234 -- GLOBAL_STEP: 31305 + | > loss: -0.27297 (-0.13162) + | > log_mle: -0.50586 (-0.30289) + | > loss_dur: 0.23289 (0.17127) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.12151 (26.37877) + | > current_lr: 0.00003 + | > step_time: 6.19530 (2.28585) + | > loader_time: 0.10300 (0.03671) + + + --> STEP: 188/234 -- GLOBAL_STEP: 31310 + | > loss: -0.27652 (-0.13450) + | > log_mle: -0.51458 (-0.30759) + | > loss_dur: 0.23806 (0.17310) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.92297 (27.52353) + | > current_lr: 0.00003 + | > step_time: 7.68630 (2.35468) + | > loader_time: 0.11070 (0.03795) + + + --> STEP: 193/234 -- GLOBAL_STEP: 31315 + | > loss: -0.28516 (-0.13775) + | > log_mle: -0.51624 (-0.31219) + | > loss_dur: 0.23108 (0.17444) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.73063 (28.63444) + | > current_lr: 0.00003 + | > step_time: 2.28270 (2.34537) + | > loader_time: 0.00510 (0.03752) + + + --> STEP: 198/234 -- GLOBAL_STEP: 31320 + | > loss: -0.24996 (-0.14055) + | > log_mle: -0.49550 (-0.31659) + | > loss_dur: 0.24554 (0.17604) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 103.65142 (29.95132) + | > current_lr: 0.00003 + | > step_time: 5.50460 (2.38255) + | > loader_time: 0.19980 (0.03814) + + + --> STEP: 203/234 -- GLOBAL_STEP: 31325 + | > loss: -0.20462 (-0.14288) + | > log_mle: -0.43505 (-0.32057) + | > loss_dur: 0.23043 (0.17769) + | > amp_scaler: 2048.00000 (4055.64532) + | > grad_norm: 76.21696 (30.84150) + | > current_lr: 0.00003 + | > step_time: 6.80170 (2.45250) + | > loader_time: 0.19810 (0.03965) + + + --> STEP: 208/234 -- GLOBAL_STEP: 31330 + | > loss: -0.26817 (-0.14600) + | > log_mle: -0.52184 (-0.32541) + | > loss_dur: 0.25367 (0.17940) + | > amp_scaler: 2048.00000 (4007.38462) + | > grad_norm: 58.66539 (31.87271) + | > current_lr: 0.00003 + | > step_time: 10.40120 (2.55276) + | > loader_time: 0.20020 (0.04112) + + + --> STEP: 213/234 -- GLOBAL_STEP: 31335 + | > loss: -0.29945 (-0.14951) + | > log_mle: -0.56403 (-0.33069) + | > loss_dur: 0.26458 (0.18118) + | > amp_scaler: 2048.00000 (3961.38967) + | > grad_norm: 80.65541 (32.95600) + | > current_lr: 0.00003 + | > step_time: 6.88720 (2.63884) + | > loader_time: 0.10680 (0.04248) + + + --> STEP: 218/234 -- GLOBAL_STEP: 31340 + | > loss: -0.28019 (-0.15266) + | > log_mle: -0.53031 (-0.33545) + | > loss_dur: 0.25011 (0.18279) + | > amp_scaler: 2048.00000 (3917.50459) + | > grad_norm: 53.38924 (34.03627) + | > current_lr: 0.00003 + | > step_time: 2.40560 (2.68661) + | > loader_time: 0.00450 (0.04249) + + + --> STEP: 223/234 -- GLOBAL_STEP: 31345 + | > loss: -0.31089 (-0.15603) + | > log_mle: -0.56336 (-0.34050) + | > loss_dur: 0.25246 (0.18448) + | > amp_scaler: 2048.00000 (3875.58744) + | > grad_norm: 82.99760 (35.02422) + | > current_lr: 0.00003 + | > step_time: 0.24000 (2.65821) + | > loader_time: 0.00290 (0.04380) + + + --> STEP: 228/234 -- GLOBAL_STEP: 31350 + | > loss: -0.27926 (-0.15938) + | > log_mle: -0.55730 (-0.34571) + | > loss_dur: 0.27804 (0.18633) + | > amp_scaler: 2048.00000 (3835.50877) + | > grad_norm: 120.02704 (36.48542) + | > current_lr: 0.00003 + | > step_time: 0.24480 (2.60528) + | > loader_time: 0.00390 (0.04292) + + + --> STEP: 233/234 -- GLOBAL_STEP: 31355 + | > loss: 0.26736 (-0.15947) + | > log_mle: -0.51967 (-0.35201) + | > loss_dur: 0.78704 (0.19254) + | > amp_scaler: 2048.00000 (3797.15021) + | > grad_norm: 107.45329 (38.04821) + | > current_lr: 0.00003 + | > step_time: 0.19200 (2.55501) + | > loader_time: 0.00320 (0.04226) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.24949 (-0.52157) + | > avg_loss: -0.19128 (+0.00082) + | > avg_log_mle: -0.41601 (+0.00856) + | > avg_loss_dur: 0.22473 (-0.00774) + + + > EPOCH: 134/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 13:23:20)  + + --> STEP: 4/234 -- GLOBAL_STEP: 31360 + | > loss: -0.04652 (-0.05939) + | > log_mle: -0.22977 (-0.23188) + | > loss_dur: 0.18325 (0.17249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.29879 (17.71552) + | > current_lr: 0.00003 + | > step_time: 2.60050 (8.59545) + | > loader_time: 0.29700 (0.15260) + + + --> STEP: 9/234 -- GLOBAL_STEP: 31365 + | > loss: -0.06971 (-0.07885) + | > log_mle: -0.24362 (-0.23577) + | > loss_dur: 0.17391 (0.15692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.48824 (14.54311) + | > current_lr: 0.00003 + | > step_time: 10.39980 (5.75109) + | > loader_time: 0.00140 (0.07893) + + + --> STEP: 14/234 -- GLOBAL_STEP: 31370 + | > loss: -0.09476 (-0.08607) + | > log_mle: -0.23870 (-0.23585) + | > loss_dur: 0.14395 (0.14978) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.41827 (12.84646) + | > current_lr: 0.00003 + | > step_time: 1.99330 (4.86811) + | > loader_time: 0.00260 (0.06514) + + + --> STEP: 19/234 -- GLOBAL_STEP: 31375 + | > loss: -0.11876 (-0.09208) + | > log_mle: -0.22595 (-0.23374) + | > loss_dur: 0.10720 (0.14166) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.72776 (11.76192) + | > current_lr: 0.00003 + | > step_time: 1.24190 (4.31066) + | > loader_time: 0.06660 (0.05712) + + + --> STEP: 24/234 -- GLOBAL_STEP: 31380 + | > loss: -0.12288 (-0.09526) + | > log_mle: -0.22773 (-0.23239) + | > loss_dur: 0.10486 (0.13713) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.72622 (11.13831) + | > current_lr: 0.00003 + | > step_time: 0.90150 (3.68086) + | > loader_time: 0.00120 (0.04880) + + + --> STEP: 29/234 -- GLOBAL_STEP: 31385 + | > loss: -0.08752 (-0.09694) + | > log_mle: -0.21884 (-0.23158) + | > loss_dur: 0.13132 (0.13464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.27325 (10.55744) + | > current_lr: 0.00003 + | > step_time: 5.60070 (3.54965) + | > loader_time: 0.00250 (0.04083) + + + --> STEP: 34/234 -- GLOBAL_STEP: 31390 + | > loss: -0.07852 (-0.09772) + | > log_mle: -0.22674 (-0.23207) + | > loss_dur: 0.14822 (0.13435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.12291 (10.32369) + | > current_lr: 0.00003 + | > step_time: 5.60050 (3.78974) + | > loader_time: 0.11010 (0.04696) + + + --> STEP: 39/234 -- GLOBAL_STEP: 31395 + | > loss: -0.09560 (-0.09708) + | > log_mle: -0.23640 (-0.23220) + | > loss_dur: 0.14080 (0.13511) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.24287 (10.58585) + | > current_lr: 0.00003 + | > step_time: 3.42700 (3.92262) + | > loader_time: 0.08380 (0.05045) + + + --> STEP: 44/234 -- GLOBAL_STEP: 31400 + | > loss: -0.11859 (-0.09618) + | > log_mle: -0.22853 (-0.23163) + | > loss_dur: 0.10995 (0.13545) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.37215 (10.24034) + | > current_lr: 0.00003 + | > step_time: 1.71040 (3.65014) + | > loader_time: 0.00230 (0.04683) + + + --> STEP: 49/234 -- GLOBAL_STEP: 31405 + | > loss: -0.11622 (-0.09628) + | > log_mle: -0.23762 (-0.23185) + | > loss_dur: 0.12140 (0.13557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.48438 (10.22594) + | > current_lr: 0.00003 + | > step_time: 1.11150 (3.45984) + | > loader_time: 0.00160 (0.04618) + + + --> STEP: 54/234 -- GLOBAL_STEP: 31410 + | > loss: -0.11431 (-0.09577) + | > log_mle: -0.24282 (-0.23176) + | > loss_dur: 0.12850 (0.13599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.06466 (10.00279) + | > current_lr: 0.00003 + | > step_time: 0.73190 (3.25649) + | > loader_time: 0.00120 (0.04376) + + + --> STEP: 59/234 -- GLOBAL_STEP: 31415 + | > loss: -0.12675 (-0.09606) + | > log_mle: -0.24906 (-0.23195) + | > loss_dur: 0.12231 (0.13589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.47168 (9.85775) + | > current_lr: 0.00003 + | > step_time: 1.25920 (3.12419) + | > loader_time: 0.00200 (0.04150) + + + --> STEP: 64/234 -- GLOBAL_STEP: 31420 + | > loss: -0.10030 (-0.09558) + | > log_mle: -0.22576 (-0.23292) + | > loss_dur: 0.12546 (0.13734) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.00072 (9.85497) + | > current_lr: 0.00003 + | > step_time: 2.49570 (3.07376) + | > loader_time: 0.00280 (0.04264) + + + --> STEP: 69/234 -- GLOBAL_STEP: 31425 + | > loss: -0.06696 (-0.09474) + | > log_mle: -0.21209 (-0.23267) + | > loss_dur: 0.14513 (0.13794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.21750 (9.79718) + | > current_lr: 0.00003 + | > step_time: 3.92810 (2.98613) + | > loader_time: 0.09120 (0.04345) + + + --> STEP: 74/234 -- GLOBAL_STEP: 31430 + | > loss: -0.08992 (-0.09326) + | > log_mle: -0.22239 (-0.23309) + | > loss_dur: 0.13247 (0.13983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.67209 (9.90527) + | > current_lr: 0.00003 + | > step_time: 1.25290 (2.86875) + | > loader_time: 0.00280 (0.04185) + + + --> STEP: 79/234 -- GLOBAL_STEP: 31435 + | > loss: -0.08674 (-0.09270) + | > log_mle: -0.24200 (-0.23346) + | > loss_dur: 0.15526 (0.14076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.42250 (9.94644) + | > current_lr: 0.00003 + | > step_time: 2.10370 (2.80485) + | > loader_time: 0.00300 (0.04142) + + + --> STEP: 84/234 -- GLOBAL_STEP: 31440 + | > loss: -0.08901 (-0.09274) + | > log_mle: -0.23686 (-0.23385) + | > loss_dur: 0.14785 (0.14110) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.66694 (9.95526) + | > current_lr: 0.00003 + | > step_time: 1.19450 (2.78537) + | > loader_time: 0.00180 (0.04235) + + + --> STEP: 89/234 -- GLOBAL_STEP: 31445 + | > loss: -0.11863 (-0.09314) + | > log_mle: -0.26853 (-0.23522) + | > loss_dur: 0.14990 (0.14208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.29141 (10.13756) + | > current_lr: 0.00003 + | > step_time: 2.21380 (2.73835) + | > loader_time: 0.00220 (0.04220) + + + --> STEP: 94/234 -- GLOBAL_STEP: 31450 + | > loss: -0.14603 (-0.09430) + | > log_mle: -0.30480 (-0.23790) + | > loss_dur: 0.15877 (0.14360) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.19760 (10.61415) + | > current_lr: 0.00003 + | > step_time: 1.69770 (2.69800) + | > loader_time: 0.00260 (0.04010) + + + --> STEP: 99/234 -- GLOBAL_STEP: 31455 + | > loss: -0.13569 (-0.09549) + | > log_mle: -0.33377 (-0.24049) + | > loss_dur: 0.19809 (0.14499) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.76943 (11.20149) + | > current_lr: 0.00003 + | > step_time: 2.10220 (2.64272) + | > loader_time: 0.07890 (0.03997) + + + --> STEP: 104/234 -- GLOBAL_STEP: 31460 + | > loss: -0.16850 (-0.09716) + | > log_mle: -0.35084 (-0.24365) + | > loss_dur: 0.18234 (0.14649) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.98911 (11.70440) + | > current_lr: 0.00003 + | > step_time: 1.31250 (2.59187) + | > loader_time: 0.08780 (0.04159) + + + --> STEP: 109/234 -- GLOBAL_STEP: 31465 + | > loss: -0.10010 (-0.09791) + | > log_mle: -0.32156 (-0.24594) + | > loss_dur: 0.22146 (0.14803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.02132 (12.07895) + | > current_lr: 0.00003 + | > step_time: 1.50530 (2.59408) + | > loader_time: 0.08480 (0.04059) + + + --> STEP: 114/234 -- GLOBAL_STEP: 31470 + | > loss: -0.13978 (-0.09945) + | > log_mle: -0.30102 (-0.24897) + | > loss_dur: 0.16124 (0.14952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.90310 (12.81432) + | > current_lr: 0.00003 + | > step_time: 0.89810 (2.57088) + | > loader_time: 0.00340 (0.04061) + + + --> STEP: 119/234 -- GLOBAL_STEP: 31475 + | > loss: -0.10770 (-0.10016) + | > log_mle: -0.29706 (-0.25124) + | > loss_dur: 0.18936 (0.15108) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.41155 (13.41083) + | > current_lr: 0.00003 + | > step_time: 1.20720 (2.53043) + | > loader_time: 0.00870 (0.03909) + + + --> STEP: 124/234 -- GLOBAL_STEP: 31480 + | > loss: -0.15536 (-0.10095) + | > log_mle: -0.32957 (-0.25288) + | > loss_dur: 0.17421 (0.15193) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.43025 (13.69653) + | > current_lr: 0.00003 + | > step_time: 1.90080 (2.50190) + | > loader_time: 0.00660 (0.03835) + + + --> STEP: 129/234 -- GLOBAL_STEP: 31485 + | > loss: -0.12306 (-0.10275) + | > log_mle: -0.31879 (-0.25621) + | > loss_dur: 0.19573 (0.15346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.55289 (14.47340) + | > current_lr: 0.00003 + | > step_time: 2.50680 (2.48393) + | > loader_time: 0.00270 (0.03697) + + + --> STEP: 134/234 -- GLOBAL_STEP: 31490 + | > loss: -0.15089 (-0.10516) + | > log_mle: -0.36631 (-0.26007) + | > loss_dur: 0.21542 (0.15491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.54037 (15.28569) + | > current_lr: 0.00003 + | > step_time: 2.41520 (2.48230) + | > loader_time: 0.18370 (0.03903) + + + --> STEP: 139/234 -- GLOBAL_STEP: 31495 + | > loss: -0.21859 (-0.10696) + | > log_mle: -0.42529 (-0.26357) + | > loss_dur: 0.20669 (0.15661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.28673 (16.15383) + | > current_lr: 0.00003 + | > step_time: 1.79810 (2.45307) + | > loader_time: 0.00350 (0.03773) + + + --> STEP: 144/234 -- GLOBAL_STEP: 31500 + | > loss: -0.19332 (-0.10879) + | > log_mle: -0.40447 (-0.26738) + | > loss_dur: 0.21115 (0.15859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.70028 (16.97819) + | > current_lr: 0.00003 + | > step_time: 2.01950 (2.45309) + | > loader_time: 0.00250 (0.03772) + + + --> STEP: 149/234 -- GLOBAL_STEP: 31505 + | > loss: -0.22811 (-0.11170) + | > log_mle: -0.45069 (-0.27177) + | > loss_dur: 0.22257 (0.16007) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.95919 (17.87729) + | > current_lr: 0.00003 + | > step_time: 3.10350 (2.44062) + | > loader_time: 0.00330 (0.03656) + + + --> STEP: 154/234 -- GLOBAL_STEP: 31510 + | > loss: -0.21665 (-0.11515) + | > log_mle: -0.41634 (-0.27670) + | > loss_dur: 0.19969 (0.16155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.87513 (18.84585) + | > current_lr: 0.00003 + | > step_time: 1.59100 (2.42122) + | > loader_time: 0.00340 (0.03651) + + + --> STEP: 159/234 -- GLOBAL_STEP: 31515 + | > loss: -0.22262 (-0.11818) + | > log_mle: -0.43420 (-0.28136) + | > loss_dur: 0.21158 (0.16318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.54422 (20.00141) + | > current_lr: 0.00003 + | > step_time: 5.00530 (2.44638) + | > loader_time: 0.19360 (0.03769) + + + --> STEP: 164/234 -- GLOBAL_STEP: 31520 + | > loss: -0.19299 (-0.12102) + | > log_mle: -0.41517 (-0.28565) + | > loss_dur: 0.22218 (0.16464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.82494 (21.22525) + | > current_lr: 0.00003 + | > step_time: 1.80060 (2.47796) + | > loader_time: 0.00590 (0.03844) + + + --> STEP: 169/234 -- GLOBAL_STEP: 31525 + | > loss: -0.19005 (-0.12388) + | > log_mle: -0.41527 (-0.29008) + | > loss_dur: 0.22522 (0.16620) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.09207 (22.36350) + | > current_lr: 0.00003 + | > step_time: 1.41030 (2.49640) + | > loader_time: 0.08920 (0.04024) + + + --> STEP: 174/234 -- GLOBAL_STEP: 31530 + | > loss: -0.27519 (-0.12763) + | > log_mle: -0.50894 (-0.29571) + | > loss_dur: 0.23375 (0.16808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.93375 (23.71524) + | > current_lr: 0.00003 + | > step_time: 2.68950 (2.50731) + | > loader_time: 0.00650 (0.03920) + + + --> STEP: 179/234 -- GLOBAL_STEP: 31535 + | > loss: -0.24322 (-0.13074) + | > log_mle: -0.50223 (-0.30083) + | > loss_dur: 0.25901 (0.17009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.09610 (24.91773) + | > current_lr: 0.00003 + | > step_time: 3.99520 (2.52955) + | > loader_time: 0.00290 (0.03872) + + + --> STEP: 184/234 -- GLOBAL_STEP: 31540 + | > loss: -0.23521 (-0.13362) + | > log_mle: -0.47297 (-0.30548) + | > loss_dur: 0.23776 (0.17186) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.07627 (25.73973) + | > current_lr: 0.00003 + | > step_time: 1.50820 (2.54590) + | > loader_time: 0.00500 (0.04084) + + + --> STEP: 189/234 -- GLOBAL_STEP: 31545 + | > loss: -0.21887 (-0.13647) + | > log_mle: -0.46454 (-0.31021) + | > loss_dur: 0.24567 (0.17374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.26669 (26.94179) + | > current_lr: 0.00003 + | > step_time: 5.30350 (2.57281) + | > loader_time: 0.09380 (0.04260) + + + --> STEP: 194/234 -- GLOBAL_STEP: 31550 + | > loss: -0.27441 (-0.14000) + | > log_mle: -0.50407 (-0.31507) + | > loss_dur: 0.22966 (0.17506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.06641 (27.94008) + | > current_lr: 0.00003 + | > step_time: 1.90040 (2.61059) + | > loader_time: 0.00630 (0.04358) + + + --> STEP: 199/234 -- GLOBAL_STEP: 31555 + | > loss: -0.26872 (-0.14291) + | > log_mle: -0.51165 (-0.31962) + | > loss_dur: 0.24293 (0.17671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.82536 (29.13362) + | > current_lr: 0.00003 + | > step_time: 2.79330 (2.64741) + | > loader_time: 0.00300 (0.04355) + + + --> STEP: 204/234 -- GLOBAL_STEP: 31560 + | > loss: -0.28473 (-0.14558) + | > log_mle: -0.54667 (-0.32401) + | > loss_dur: 0.26194 (0.17843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.92036 (30.35330) + | > current_lr: 0.00003 + | > step_time: 3.88330 (2.68822) + | > loader_time: 0.01570 (0.04357) + + + --> STEP: 209/234 -- GLOBAL_STEP: 31565 + | > loss: -0.26354 (-0.14875) + | > log_mle: -0.50411 (-0.32878) + | > loss_dur: 0.24057 (0.18004) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.55768 (31.41424) + | > current_lr: 0.00003 + | > step_time: 4.70770 (2.76976) + | > loader_time: 0.00550 (0.04451) + + + --> STEP: 214/234 -- GLOBAL_STEP: 31570 + | > loss: -0.31098 (-0.15257) + | > log_mle: -0.53466 (-0.33431) + | > loss_dur: 0.22368 (0.18174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.41537 (32.86375) + | > current_lr: 0.00003 + | > step_time: 3.69000 (2.82130) + | > loader_time: 0.30750 (0.04672) + + + --> STEP: 219/234 -- GLOBAL_STEP: 31575 + | > loss: -0.38233 (-0.15627) + | > log_mle: -0.63635 (-0.33966) + | > loss_dur: 0.25402 (0.18340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.13741 (34.15745) + | > current_lr: 0.00003 + | > step_time: 2.00510 (2.86050) + | > loader_time: 0.00490 (0.04842) + + + --> STEP: 224/234 -- GLOBAL_STEP: 31580 + | > loss: -0.31967 (-0.15961) + | > log_mle: -0.57910 (-0.34463) + | > loss_dur: 0.25942 (0.18502) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.60811 (35.32985) + | > current_lr: 0.00003 + | > step_time: 0.23450 (2.81243) + | > loader_time: 0.00310 (0.04743) + + + --> STEP: 229/234 -- GLOBAL_STEP: 31585 + | > loss: -0.29826 (-0.16306) + | > log_mle: -0.61610 (-0.35016) + | > loss_dur: 0.31784 (0.18710) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 128.51395 (36.82811) + | > current_lr: 0.00003 + | > step_time: 0.24390 (2.75623) + | > loader_time: 0.00330 (0.04648) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22860 (-0.02089) + | > avg_loss: -0.16961 (+0.02167) + | > avg_log_mle: -0.39898 (+0.01703) + | > avg_loss_dur: 0.22937 (+0.00464) + + + > EPOCH: 135/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 13:35:09)  + + --> STEP: 0/234 -- GLOBAL_STEP: 31590 + | > loss: -0.18354 (-0.18354) + | > log_mle: -0.30844 (-0.30844) + | > loss_dur: 0.12490 (0.12490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.41835 (18.41835) + | > current_lr: 0.00003 + | > step_time: 8.89060 (8.89058) + | > loader_time: 17.43500 (17.43501) + + + --> STEP: 5/234 -- GLOBAL_STEP: 31595 + | > loss: -0.10052 (-0.07579) + | > log_mle: -0.23539 (-0.23359) + | > loss_dur: 0.13487 (0.15781) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.85230 (16.90395) + | > current_lr: 0.00003 + | > step_time: 1.48880 (6.04219) + | > loader_time: 0.00130 (0.05638) + + + --> STEP: 10/234 -- GLOBAL_STEP: 31600 + | > loss: -0.06343 (-0.08452) + | > log_mle: -0.23478 (-0.23701) + | > loss_dur: 0.17135 (0.15249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.65014 (14.41704) + | > current_lr: 0.00003 + | > step_time: 2.00000 (3.70902) + | > loader_time: 0.00150 (0.02950) + + + --> STEP: 15/234 -- GLOBAL_STEP: 31605 + | > loss: -0.11470 (-0.09202) + | > log_mle: -0.23649 (-0.23754) + | > loss_dur: 0.12179 (0.14552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.30828 (12.85834) + | > current_lr: 0.00003 + | > step_time: 1.29880 (3.07787) + | > loader_time: 0.00570 (0.02087) + + + --> STEP: 20/234 -- GLOBAL_STEP: 31610 + | > loss: -0.09885 (-0.09531) + | > log_mle: -0.22627 (-0.23499) + | > loss_dur: 0.12743 (0.13967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.02996 (11.50331) + | > current_lr: 0.00003 + | > step_time: 0.98900 (2.60007) + | > loader_time: 0.00140 (0.01939) + + + --> STEP: 25/234 -- GLOBAL_STEP: 31615 + | > loss: -0.10110 (-0.09844) + | > log_mle: -0.22290 (-0.23360) + | > loss_dur: 0.12180 (0.13516) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.88489 (10.85992) + | > current_lr: 0.00003 + | > step_time: 0.90370 (2.30871) + | > loader_time: 0.00190 (0.02590) + + + --> STEP: 30/234 -- GLOBAL_STEP: 31620 + | > loss: -0.13680 (-0.10141) + | > log_mle: -0.24476 (-0.23367) + | > loss_dur: 0.10796 (0.13226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.50535 (10.24891) + | > current_lr: 0.00003 + | > step_time: 1.30450 (2.12414) + | > loader_time: 0.00220 (0.02189) + + + --> STEP: 35/234 -- GLOBAL_STEP: 31625 + | > loss: -0.08966 (-0.10018) + | > log_mle: -0.23552 (-0.23359) + | > loss_dur: 0.14586 (0.13340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.51121 (10.29868) + | > current_lr: 0.00003 + | > step_time: 1.34990 (2.00100) + | > loader_time: 0.00300 (0.02147) + + + --> STEP: 40/234 -- GLOBAL_STEP: 31630 + | > loss: -0.06266 (-0.09923) + | > log_mle: -0.21558 (-0.23329) + | > loss_dur: 0.15292 (0.13406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.00583 (10.19593) + | > current_lr: 0.00003 + | > step_time: 1.33350 (1.90205) + | > loader_time: 0.08640 (0.02117) + + + --> STEP: 45/234 -- GLOBAL_STEP: 31635 + | > loss: -0.10293 (-0.09929) + | > log_mle: -0.25454 (-0.23342) + | > loss_dur: 0.15161 (0.13413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.74313 (10.02933) + | > current_lr: 0.00003 + | > step_time: 1.19800 (1.85841) + | > loader_time: 0.00190 (0.02294) + + + --> STEP: 50/234 -- GLOBAL_STEP: 31640 + | > loss: -0.07834 (-0.09905) + | > log_mle: -0.22286 (-0.23294) + | > loss_dur: 0.14452 (0.13388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.15358 (9.70577) + | > current_lr: 0.00003 + | > step_time: 1.70910 (1.84606) + | > loader_time: 0.00420 (0.02310) + + + --> STEP: 55/234 -- GLOBAL_STEP: 31645 + | > loss: -0.11497 (-0.09904) + | > log_mle: -0.24244 (-0.23315) + | > loss_dur: 0.12747 (0.13411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76258 (9.45514) + | > current_lr: 0.00003 + | > step_time: 1.24530 (1.79008) + | > loader_time: 0.00250 (0.02120) + + + --> STEP: 60/234 -- GLOBAL_STEP: 31650 + | > loss: -0.11040 (-0.09896) + | > log_mle: -0.25771 (-0.23358) + | > loss_dur: 0.14731 (0.13462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.53913 (9.37872) + | > current_lr: 0.00003 + | > step_time: 1.09710 (1.77444) + | > loader_time: 0.00210 (0.01966) + + + --> STEP: 65/234 -- GLOBAL_STEP: 31655 + | > loss: -0.10065 (-0.09829) + | > log_mle: -0.23211 (-0.23398) + | > loss_dur: 0.13146 (0.13569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.93212 (9.57059) + | > current_lr: 0.00003 + | > step_time: 1.60360 (1.81535) + | > loader_time: 0.08270 (0.02206) + + + --> STEP: 70/234 -- GLOBAL_STEP: 31660 + | > loss: -0.07395 (-0.09680) + | > log_mle: -0.22962 (-0.23355) + | > loss_dur: 0.15567 (0.13676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.55686 (9.59324) + | > current_lr: 0.00003 + | > step_time: 1.59210 (1.86145) + | > loader_time: 0.00300 (0.02352) + + + --> STEP: 75/234 -- GLOBAL_STEP: 31665 + | > loss: -0.05604 (-0.09503) + | > log_mle: -0.23162 (-0.23376) + | > loss_dur: 0.17558 (0.13873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.61505 (9.83287) + | > current_lr: 0.00003 + | > step_time: 1.61500 (1.83633) + | > loader_time: 0.08960 (0.02444) + + + --> STEP: 80/234 -- GLOBAL_STEP: 31670 + | > loss: -0.09698 (-0.09511) + | > log_mle: -0.22154 (-0.23368) + | > loss_dur: 0.12456 (0.13857) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.91669 (9.75425) + | > current_lr: 0.00003 + | > step_time: 1.26970 (1.87379) + | > loader_time: 0.00220 (0.02555) + + + --> STEP: 85/234 -- GLOBAL_STEP: 31675 + | > loss: -0.10518 (-0.09477) + | > log_mle: -0.24139 (-0.23403) + | > loss_dur: 0.13620 (0.13926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.65106 (9.88502) + | > current_lr: 0.00003 + | > step_time: 5.39790 (1.89142) + | > loader_time: 0.09330 (0.02527) + + + --> STEP: 90/234 -- GLOBAL_STEP: 31680 + | > loss: -0.08758 (-0.09483) + | > log_mle: -0.26508 (-0.23549) + | > loss_dur: 0.17750 (0.14066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.75076 (10.18887) + | > current_lr: 0.00003 + | > step_time: 5.98800 (1.95947) + | > loader_time: 0.20020 (0.02809) + + + --> STEP: 95/234 -- GLOBAL_STEP: 31685 + | > loss: -0.16128 (-0.09664) + | > log_mle: -0.35347 (-0.23894) + | > loss_dur: 0.19219 (0.14230) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.89965 (10.81230) + | > current_lr: 0.00003 + | > step_time: 1.49840 (1.98511) + | > loader_time: 0.00500 (0.02679) + + + --> STEP: 100/234 -- GLOBAL_STEP: 31690 + | > loss: -0.12639 (-0.09749) + | > log_mle: -0.27898 (-0.24069) + | > loss_dur: 0.15259 (0.14320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.15535 (11.20031) + | > current_lr: 0.00003 + | > step_time: 3.90590 (1.98290) + | > loader_time: 0.00350 (0.02642) + + + --> STEP: 105/234 -- GLOBAL_STEP: 31695 + | > loss: -0.11267 (-0.09889) + | > log_mle: -0.25659 (-0.24353) + | > loss_dur: 0.14392 (0.14463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.65636 (11.68701) + | > current_lr: 0.00003 + | > step_time: 2.60200 (1.99802) + | > loader_time: 0.00300 (0.02703) + + + --> STEP: 110/234 -- GLOBAL_STEP: 31700 + | > loss: -0.11942 (-0.09951) + | > log_mle: -0.28098 (-0.24596) + | > loss_dur: 0.16156 (0.14645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.20748 (12.25029) + | > current_lr: 0.00003 + | > step_time: 1.89530 (1.98374) + | > loader_time: 0.00210 (0.02738) + + + --> STEP: 115/234 -- GLOBAL_STEP: 31705 + | > loss: -0.11697 (-0.10087) + | > log_mle: -0.30326 (-0.24918) + | > loss_dur: 0.18628 (0.14831) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.36660 (12.79504) + | > current_lr: 0.00003 + | > step_time: 1.40400 (1.97667) + | > loader_time: 0.00330 (0.02779) + + + --> STEP: 120/234 -- GLOBAL_STEP: 31710 + | > loss: -0.16158 (-0.10197) + | > log_mle: -0.35371 (-0.25194) + | > loss_dur: 0.19214 (0.14997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.87396 (13.27302) + | > current_lr: 0.00003 + | > step_time: 1.68010 (1.98443) + | > loader_time: 0.00260 (0.02906) + + + --> STEP: 125/234 -- GLOBAL_STEP: 31715 + | > loss: -0.15044 (-0.10270) + | > log_mle: -0.34073 (-0.25348) + | > loss_dur: 0.19029 (0.15078) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.35941 (13.62305) + | > current_lr: 0.00003 + | > step_time: 1.20950 (1.98840) + | > loader_time: 0.00290 (0.03007) + + + --> STEP: 130/234 -- GLOBAL_STEP: 31720 + | > loss: -0.14298 (-0.10433) + | > log_mle: -0.34832 (-0.25678) + | > loss_dur: 0.20534 (0.15245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.16017 (14.64942) + | > current_lr: 0.00003 + | > step_time: 2.70420 (2.01245) + | > loader_time: 0.00500 (0.02971) + + + --> STEP: 135/234 -- GLOBAL_STEP: 31725 + | > loss: -0.11528 (-0.10622) + | > log_mle: -0.27994 (-0.26009) + | > loss_dur: 0.16465 (0.15388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.82331 (15.54851) + | > current_lr: 0.00003 + | > step_time: 4.40150 (2.02143) + | > loader_time: 0.00320 (0.02937) + + + --> STEP: 140/234 -- GLOBAL_STEP: 31730 + | > loss: -0.12711 (-0.10827) + | > log_mle: -0.31698 (-0.26394) + | > loss_dur: 0.18987 (0.15566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.26722 (16.52924) + | > current_lr: 0.00003 + | > step_time: 4.00000 (2.02855) + | > loader_time: 0.00330 (0.03025) + + + --> STEP: 145/234 -- GLOBAL_STEP: 31735 + | > loss: -0.21413 (-0.11075) + | > log_mle: -0.41771 (-0.26850) + | > loss_dur: 0.20358 (0.15775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.37714 (17.43497) + | > current_lr: 0.00003 + | > step_time: 2.19880 (2.07376) + | > loader_time: 0.20640 (0.03188) + + + --> STEP: 150/234 -- GLOBAL_STEP: 31740 + | > loss: -0.18076 (-0.11346) + | > log_mle: -0.39898 (-0.27275) + | > loss_dur: 0.21823 (0.15929) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.92593 (18.22460) + | > current_lr: 0.00003 + | > step_time: 4.28910 (2.09806) + | > loader_time: 0.00380 (0.03142) + + + --> STEP: 155/234 -- GLOBAL_STEP: 31745 + | > loss: -0.24325 (-0.11706) + | > log_mle: -0.46480 (-0.27804) + | > loss_dur: 0.22155 (0.16099) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.55014 (19.32414) + | > current_lr: 0.00003 + | > step_time: 4.41100 (2.16718) + | > loader_time: 0.00350 (0.03176) + + + --> STEP: 160/234 -- GLOBAL_STEP: 31750 + | > loss: -0.23730 (-0.11992) + | > log_mle: -0.46178 (-0.28266) + | > loss_dur: 0.22448 (0.16274) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.15569 (20.30774) + | > current_lr: 0.00003 + | > step_time: 2.70860 (2.19508) + | > loader_time: 0.09480 (0.03315) + + + --> STEP: 165/234 -- GLOBAL_STEP: 31755 + | > loss: -0.22281 (-0.12272) + | > log_mle: -0.45174 (-0.28698) + | > loss_dur: 0.22893 (0.16425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.84725 (21.42338) + | > current_lr: 0.00003 + | > step_time: 2.49860 (2.20127) + | > loader_time: 0.10460 (0.03343) + + + --> STEP: 170/234 -- GLOBAL_STEP: 31760 + | > loss: -0.25428 (-0.12566) + | > log_mle: -0.49569 (-0.29165) + | > loss_dur: 0.24141 (0.16599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.79823 (22.59036) + | > current_lr: 0.00003 + | > step_time: 5.41430 (2.26311) + | > loader_time: 0.00400 (0.03542) + + + --> STEP: 175/234 -- GLOBAL_STEP: 31765 + | > loss: -0.22702 (-0.12927) + | > log_mle: -0.47206 (-0.29714) + | > loss_dur: 0.24504 (0.16788) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.87188 (23.83731) + | > current_lr: 0.00003 + | > step_time: 1.79070 (2.27197) + | > loader_time: 0.00320 (0.03513) + + + --> STEP: 180/234 -- GLOBAL_STEP: 31770 + | > loss: -0.25007 (-0.13243) + | > log_mle: -0.47965 (-0.30228) + | > loss_dur: 0.22958 (0.16985) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.21065 (25.04673) + | > current_lr: 0.00003 + | > step_time: 4.11050 (2.33457) + | > loader_time: 0.18970 (0.03799) + + + --> STEP: 185/234 -- GLOBAL_STEP: 31775 + | > loss: -0.25867 (-0.13535) + | > log_mle: -0.49975 (-0.30697) + | > loss_dur: 0.24108 (0.17162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.67443 (26.14952) + | > current_lr: 0.00003 + | > step_time: 1.65170 (2.35952) + | > loader_time: 0.00290 (0.03858) + + + --> STEP: 190/234 -- GLOBAL_STEP: 31780 + | > loss: -0.25512 (-0.13818) + | > log_mle: -0.47880 (-0.31151) + | > loss_dur: 0.22368 (0.17333) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.36560 (27.25784) + | > current_lr: 0.00003 + | > step_time: 5.19730 (2.37321) + | > loader_time: 0.10580 (0.03916) + + + --> STEP: 195/234 -- GLOBAL_STEP: 31785 + | > loss: -0.25891 (-0.14160) + | > log_mle: -0.49613 (-0.31638) + | > loss_dur: 0.23722 (0.17479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.56842 (28.41295) + | > current_lr: 0.00003 + | > step_time: 7.10480 (2.50218) + | > loader_time: 0.00450 (0.04176) + + + --> STEP: 200/234 -- GLOBAL_STEP: 31790 + | > loss: -0.25079 (-0.14450) + | > log_mle: -0.50621 (-0.32094) + | > loss_dur: 0.25542 (0.17644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.38428 (29.46297) + | > current_lr: 0.00003 + | > step_time: 8.99370 (2.53363) + | > loader_time: 0.89990 (0.04617) + + + --> STEP: 205/234 -- GLOBAL_STEP: 31795 + | > loss: -0.25745 (-0.14720) + | > log_mle: -0.49143 (-0.32525) + | > loss_dur: 0.23397 (0.17805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.15699 (30.44835) + | > current_lr: 0.00003 + | > step_time: 3.49960 (2.59629) + | > loader_time: 0.10920 (0.04651) + + + --> STEP: 210/234 -- GLOBAL_STEP: 31800 + | > loss: -0.31607 (-0.15055) + | > log_mle: -0.56735 (-0.33028) + | > loss_dur: 0.25129 (0.17973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.57541 (31.72375) + | > current_lr: 0.00003 + | > step_time: 2.80450 (2.62169) + | > loader_time: 0.18970 (0.04824) + + + --> STEP: 215/234 -- GLOBAL_STEP: 31805 + | > loss: -0.27464 (-0.15418) + | > log_mle: -0.51806 (-0.33545) + | > loss_dur: 0.24342 (0.18127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.83307 (33.26899) + | > current_lr: 0.00003 + | > step_time: 7.51090 (2.71238) + | > loader_time: 0.09620 (0.04853) + + + --> STEP: 220/234 -- GLOBAL_STEP: 31810 + | > loss: -0.31177 (-0.15804) + | > log_mle: -0.56764 (-0.34098) + | > loss_dur: 0.25587 (0.18294) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.29673 (34.62753) + | > current_lr: 0.00003 + | > step_time: 3.80290 (2.76837) + | > loader_time: 0.08870 (0.04925) + + + --> STEP: 225/234 -- GLOBAL_STEP: 31815 + | > loss: -0.35836 (-0.16156) + | > log_mle: -0.62643 (-0.34620) + | > loss_dur: 0.26806 (0.18464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.04196 (35.83745) + | > current_lr: 0.00003 + | > step_time: 0.24450 (2.74071) + | > loader_time: 0.00350 (0.04900) + + + --> STEP: 230/234 -- GLOBAL_STEP: 31820 + | > loss: -0.33816 (-0.16485) + | > log_mle: -0.67953 (-0.35190) + | > loss_dur: 0.34137 (0.18705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 142.74429 (37.39650) + | > current_lr: 0.00003 + | > step_time: 0.25360 (2.68650) + | > loader_time: 0.00480 (0.04802) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00990 (-0.21870) + | > avg_loss: -0.20885 (-0.03923) + | > avg_log_mle: -0.43309 (-0.03411) + | > avg_loss_dur: 0.22424 (-0.00513) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_31824.pth + + > EPOCH: 136/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 13:46:49)  + + --> STEP: 1/234 -- GLOBAL_STEP: 31825 + | > loss: -0.12215 (-0.12215) + | > log_mle: -0.24044 (-0.24044) + | > loss_dur: 0.11829 (0.11829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.01580 (15.01580) + | > current_lr: 0.00003 + | > step_time: 10.60600 (10.60604) + | > loader_time: 0.09080 (0.09076) + + + --> STEP: 6/234 -- GLOBAL_STEP: 31830 + | > loss: -0.10325 (-0.08388) + | > log_mle: -0.22700 (-0.23381) + | > loss_dur: 0.12375 (0.14993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01060 (15.62066) + | > current_lr: 0.00003 + | > step_time: 7.89870 (7.10035) + | > loader_time: 2.79710 (0.49967) + + + --> STEP: 11/234 -- GLOBAL_STEP: 31835 + | > loss: -0.12368 (-0.09051) + | > log_mle: -0.23906 (-0.23836) + | > loss_dur: 0.11538 (0.14785) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76822 (13.27424) + | > current_lr: 0.00003 + | > step_time: 1.39960 (5.70723) + | > loader_time: 0.00190 (0.31015) + + + --> STEP: 16/234 -- GLOBAL_STEP: 31840 + | > loss: -0.13081 (-0.09730) + | > log_mle: -0.23796 (-0.23887) + | > loss_dur: 0.10714 (0.14157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42087 (12.01417) + | > current_lr: 0.00003 + | > step_time: 6.81530 (5.53712) + | > loader_time: 0.08180 (0.24928) + + + --> STEP: 21/234 -- GLOBAL_STEP: 31845 + | > loss: -0.09675 (-0.09707) + | > log_mle: -0.21708 (-0.23551) + | > loss_dur: 0.12033 (0.13844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.09853 (11.08862) + | > current_lr: 0.00003 + | > step_time: 0.87390 (4.97414) + | > loader_time: 0.00200 (0.19525) + + + --> STEP: 26/234 -- GLOBAL_STEP: 31850 + | > loss: -0.09088 (-0.09996) + | > log_mle: -0.23219 (-0.23506) + | > loss_dur: 0.14132 (0.13510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.92819 (10.54276) + | > current_lr: 0.00003 + | > step_time: 3.70340 (4.60263) + | > loader_time: 0.09650 (0.16908) + + + --> STEP: 31/234 -- GLOBAL_STEP: 31855 + | > loss: -0.05299 (-0.10100) + | > log_mle: -0.23165 (-0.23478) + | > loss_dur: 0.17867 (0.13377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.12936 (10.20053) + | > current_lr: 0.00003 + | > step_time: 3.18620 (4.56953) + | > loader_time: 0.00380 (0.14533) + + + --> STEP: 36/234 -- GLOBAL_STEP: 31860 + | > loss: -0.08140 (-0.10030) + | > log_mle: -0.23615 (-0.23483) + | > loss_dur: 0.15475 (0.13453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.70943 (10.13771) + | > current_lr: 0.00003 + | > step_time: 5.38860 (4.41250) + | > loader_time: 0.00190 (0.12799) + + + --> STEP: 41/234 -- GLOBAL_STEP: 31865 + | > loss: -0.10660 (-0.10020) + | > log_mle: -0.23581 (-0.23460) + | > loss_dur: 0.12921 (0.13440) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.50296 (9.91223) + | > current_lr: 0.00003 + | > step_time: 1.29270 (4.32605) + | > loader_time: 0.00200 (0.12214) + + + --> STEP: 46/234 -- GLOBAL_STEP: 31870 + | > loss: -0.09032 (-0.09982) + | > log_mle: -0.23057 (-0.23468) + | > loss_dur: 0.14025 (0.13487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.60883 (9.99208) + | > current_lr: 0.00003 + | > step_time: 1.09400 (4.00847) + | > loader_time: 0.01210 (0.11315) + + + --> STEP: 51/234 -- GLOBAL_STEP: 31875 + | > loss: -0.08637 (-0.09956) + | > log_mle: -0.22125 (-0.23394) + | > loss_dur: 0.13487 (0.13439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.42435 (9.77695) + | > current_lr: 0.00003 + | > step_time: 1.96040 (3.78356) + | > loader_time: 0.00280 (0.10232) + + + --> STEP: 56/234 -- GLOBAL_STEP: 31880 + | > loss: -0.07822 (-0.09934) + | > log_mle: -0.23882 (-0.23443) + | > loss_dur: 0.16060 (0.13509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.22777 (9.69886) + | > current_lr: 0.00003 + | > step_time: 0.90920 (3.56785) + | > loader_time: 0.00140 (0.09499) + + + --> STEP: 61/234 -- GLOBAL_STEP: 31885 + | > loss: -0.11162 (-0.09931) + | > log_mle: -0.23538 (-0.23469) + | > loss_dur: 0.12375 (0.13539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.91322 (9.72480) + | > current_lr: 0.00003 + | > step_time: 1.50230 (3.45745) + | > loader_time: 0.00250 (0.09051) + + + --> STEP: 66/234 -- GLOBAL_STEP: 31890 + | > loss: -0.09395 (-0.09836) + | > log_mle: -0.22250 (-0.23486) + | > loss_dur: 0.12855 (0.13651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.51143 (9.72455) + | > current_lr: 0.00003 + | > step_time: 2.60370 (3.33210) + | > loader_time: 0.08330 (0.08652) + + + --> STEP: 71/234 -- GLOBAL_STEP: 31895 + | > loss: -0.07623 (-0.09676) + | > log_mle: -0.25478 (-0.23484) + | > loss_dur: 0.17855 (0.13808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.81832 (10.02385) + | > current_lr: 0.00003 + | > step_time: 1.20320 (3.18384) + | > loader_time: 0.00350 (0.08189) + + + --> STEP: 76/234 -- GLOBAL_STEP: 31900 + | > loss: -0.09573 (-0.09561) + | > log_mle: -0.24459 (-0.23495) + | > loss_dur: 0.14887 (0.13934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.98359 (10.13628) + | > current_lr: 0.00003 + | > step_time: 2.00100 (3.09025) + | > loader_time: 0.00230 (0.07777) + + + --> STEP: 81/234 -- GLOBAL_STEP: 31905 + | > loss: -0.10087 (-0.09557) + | > log_mle: -0.25438 (-0.23509) + | > loss_dur: 0.15351 (0.13952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.10308 (10.10058) + | > current_lr: 0.00003 + | > step_time: 1.49460 (3.02047) + | > loader_time: 0.00680 (0.07526) + + + --> STEP: 86/234 -- GLOBAL_STEP: 31910 + | > loss: -0.08864 (-0.09525) + | > log_mle: -0.25085 (-0.23550) + | > loss_dur: 0.16221 (0.14025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.99468 (10.26964) + | > current_lr: 0.00003 + | > step_time: 1.71710 (2.95674) + | > loader_time: 0.07620 (0.07283) + + + --> STEP: 91/234 -- GLOBAL_STEP: 31915 + | > loss: -0.09499 (-0.09548) + | > log_mle: -0.26510 (-0.23713) + | > loss_dur: 0.17011 (0.14165) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.91034 (10.52703) + | > current_lr: 0.00003 + | > step_time: 2.71360 (2.88759) + | > loader_time: 0.09230 (0.06994) + + + --> STEP: 96/234 -- GLOBAL_STEP: 31920 + | > loss: -0.09288 (-0.09721) + | > log_mle: -0.24949 (-0.24036) + | > loss_dur: 0.15661 (0.14315) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.51611 (11.19044) + | > current_lr: 0.00003 + | > step_time: 8.00350 (2.89135) + | > loader_time: 0.00430 (0.06814) + + + --> STEP: 101/234 -- GLOBAL_STEP: 31925 + | > loss: -0.12599 (-0.09829) + | > log_mle: -0.31100 (-0.24267) + | > loss_dur: 0.18501 (0.14438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.42344 (11.70192) + | > current_lr: 0.00003 + | > step_time: 2.69310 (2.85204) + | > loader_time: 0.00290 (0.06574) + + + --> STEP: 106/234 -- GLOBAL_STEP: 31930 + | > loss: -0.10706 (-0.09951) + | > log_mle: -0.31057 (-0.24552) + | > loss_dur: 0.20350 (0.14600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.54515 (12.19083) + | > current_lr: 0.00003 + | > step_time: 3.88560 (2.84470) + | > loader_time: 0.00270 (0.06435) + + + --> STEP: 111/234 -- GLOBAL_STEP: 31935 + | > loss: -0.14213 (-0.10053) + | > log_mle: -0.35868 (-0.24845) + | > loss_dur: 0.21655 (0.14792) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.19524 (12.75670) + | > current_lr: 0.00003 + | > step_time: 1.51780 (2.79351) + | > loader_time: 0.07850 (0.06468) + + + --> STEP: 116/234 -- GLOBAL_STEP: 31940 + | > loss: -0.11353 (-0.10171) + | > log_mle: -0.32380 (-0.25137) + | > loss_dur: 0.21027 (0.14966) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.69453 (13.34253) + | > current_lr: 0.00003 + | > step_time: 1.46860 (2.76081) + | > loader_time: 0.00230 (0.06298) + + + --> STEP: 121/234 -- GLOBAL_STEP: 31945 + | > loss: -0.07319 (-0.10256) + | > log_mle: -0.23485 (-0.25334) + | > loss_dur: 0.16166 (0.15079) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.98558 (13.69839) + | > current_lr: 0.00003 + | > step_time: 2.30130 (2.70780) + | > loader_time: 0.08610 (0.06264) + + + --> STEP: 126/234 -- GLOBAL_STEP: 31950 + | > loss: -0.17040 (-0.10397) + | > log_mle: -0.37390 (-0.25599) + | > loss_dur: 0.20350 (0.15202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.67754 (14.14987) + | > current_lr: 0.00003 + | > step_time: 1.19290 (2.69485) + | > loader_time: 0.00330 (0.06317) + + + --> STEP: 131/234 -- GLOBAL_STEP: 31955 + | > loss: -0.20764 (-0.10609) + | > log_mle: -0.41786 (-0.25964) + | > loss_dur: 0.21021 (0.15355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.80502 (14.90749) + | > current_lr: 0.00003 + | > step_time: 2.09660 (2.65154) + | > loader_time: 0.00270 (0.06148) + + + --> STEP: 136/234 -- GLOBAL_STEP: 31960 + | > loss: -0.20291 (-0.10787) + | > log_mle: -0.43345 (-0.26301) + | > loss_dur: 0.23054 (0.15514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.51221 (15.84565) + | > current_lr: 0.00003 + | > step_time: 3.71450 (2.64827) + | > loader_time: 0.00360 (0.05938) + + + --> STEP: 141/234 -- GLOBAL_STEP: 31965 + | > loss: -0.15303 (-0.10905) + | > log_mle: -0.36139 (-0.26587) + | > loss_dur: 0.20836 (0.15682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.70097 (16.62202) + | > current_lr: 0.00003 + | > step_time: 1.39810 (2.62169) + | > loader_time: 0.00450 (0.05800) + + + --> STEP: 146/234 -- GLOBAL_STEP: 31970 + | > loss: -0.19796 (-0.11170) + | > log_mle: -0.41094 (-0.27059) + | > loss_dur: 0.21298 (0.15889) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.47235 (17.77101) + | > current_lr: 0.00003 + | > step_time: 1.80480 (2.61000) + | > loader_time: 0.00380 (0.05868) + + + --> STEP: 151/234 -- GLOBAL_STEP: 31975 + | > loss: -0.19832 (-0.11431) + | > log_mle: -0.38311 (-0.27445) + | > loss_dur: 0.18479 (0.16014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.99673 (18.76999) + | > current_lr: 0.00003 + | > step_time: 2.51810 (2.66256) + | > loader_time: 0.18690 (0.06005) + + + --> STEP: 156/234 -- GLOBAL_STEP: 31980 + | > loss: -0.22447 (-0.11801) + | > log_mle: -0.42530 (-0.27985) + | > loss_dur: 0.20083 (0.16184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.45550 (20.08194) + | > current_lr: 0.00003 + | > step_time: 1.89710 (2.65496) + | > loader_time: 0.00310 (0.05994) + + + --> STEP: 161/234 -- GLOBAL_STEP: 31985 + | > loss: -0.24183 (-0.12100) + | > log_mle: -0.45135 (-0.28449) + | > loss_dur: 0.20952 (0.16350) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.38051 (21.23419) + | > current_lr: 0.00003 + | > step_time: 9.50230 (2.68808) + | > loader_time: 0.00540 (0.05922) + + + --> STEP: 166/234 -- GLOBAL_STEP: 31990 + | > loss: -0.19675 (-0.12370) + | > log_mle: -0.38896 (-0.28844) + | > loss_dur: 0.19222 (0.16474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.90163 (22.15495) + | > current_lr: 0.00003 + | > step_time: 1.90080 (2.69259) + | > loader_time: 0.00350 (0.05811) + + + --> STEP: 171/234 -- GLOBAL_STEP: 31995 + | > loss: -0.28660 (-0.12725) + | > log_mle: -0.50284 (-0.29392) + | > loss_dur: 0.21624 (0.16667) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.94579 (23.14490) + | > current_lr: 0.00003 + | > step_time: 6.99800 (2.69984) + | > loader_time: 0.10890 (0.05872) + + + --> STEP: 176/234 -- GLOBAL_STEP: 32000 + | > loss: -0.24673 (-0.13078) + | > log_mle: -0.46794 (-0.29922) + | > loss_dur: 0.22121 (0.16844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.06075 (24.50156) + | > current_lr: 0.00003 + | > step_time: 1.99550 (2.67832) + | > loader_time: 0.00560 (0.05765) + + + --> STEP: 181/234 -- GLOBAL_STEP: 32005 + | > loss: -0.18785 (-0.13367) + | > log_mle: -0.40850 (-0.30400) + | > loss_dur: 0.22066 (0.17034) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.25172 (25.38642) + | > current_lr: 0.00003 + | > step_time: 7.38910 (2.76529) + | > loader_time: 0.00350 (0.06211) + + + --> STEP: 186/234 -- GLOBAL_STEP: 32010 + | > loss: -0.20961 (-0.13664) + | > log_mle: -0.44903 (-0.30894) + | > loss_dur: 0.23942 (0.17229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.02081 (26.66425) + | > current_lr: 0.00003 + | > step_time: 3.20350 (2.79043) + | > loader_time: 0.00310 (0.06157) + + + --> STEP: 191/234 -- GLOBAL_STEP: 32015 + | > loss: -0.25178 (-0.13980) + | > log_mle: -0.46581 (-0.31360) + | > loss_dur: 0.21404 (0.17380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.55126 (27.88554) + | > current_lr: 0.00003 + | > step_time: 2.39410 (2.83354) + | > loader_time: 0.09510 (0.06161) + + + --> STEP: 196/234 -- GLOBAL_STEP: 32020 + | > loss: -0.21948 (-0.14304) + | > log_mle: -0.46078 (-0.31841) + | > loss_dur: 0.24130 (0.17537) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.67789 (29.07153) + | > current_lr: 0.00003 + | > step_time: 3.40540 (2.90729) + | > loader_time: 0.09060 (0.06248) + + + --> STEP: 201/234 -- GLOBAL_STEP: 32025 + | > loss: -0.16836 (-0.14561) + | > log_mle: -0.41736 (-0.32265) + | > loss_dur: 0.24900 (0.17703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.84116 (30.19827) + | > current_lr: 0.00003 + | > step_time: 5.69790 (2.93430) + | > loader_time: 0.00330 (0.06195) + + + --> STEP: 206/234 -- GLOBAL_STEP: 32030 + | > loss: -0.27687 (-0.14872) + | > log_mle: -0.52558 (-0.32733) + | > loss_dur: 0.24871 (0.17861) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.19228 (31.30280) + | > current_lr: 0.00003 + | > step_time: 9.49420 (3.00139) + | > loader_time: 0.18950 (0.06197) + + + --> STEP: 211/234 -- GLOBAL_STEP: 32035 + | > loss: -0.32443 (-0.15224) + | > log_mle: -0.59387 (-0.33264) + | > loss_dur: 0.26944 (0.18040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.44234 (32.78743) + | > current_lr: 0.00003 + | > step_time: 3.89320 (3.01879) + | > loader_time: 0.10700 (0.06202) + + + --> STEP: 216/234 -- GLOBAL_STEP: 32040 + | > loss: -0.31980 (-0.15565) + | > log_mle: -0.58677 (-0.33765) + | > loss_dur: 0.26697 (0.18200) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.80505 (34.04737) + | > current_lr: 0.00003 + | > step_time: 4.71320 (3.12998) + | > loader_time: 0.00300 (0.06245) + + + --> STEP: 221/234 -- GLOBAL_STEP: 32045 + | > loss: -0.26213 (-0.15917) + | > log_mle: -0.49931 (-0.34266) + | > loss_dur: 0.23719 (0.18349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.76897 (35.40175) + | > current_lr: 0.00003 + | > step_time: 3.70200 (3.12802) + | > loader_time: 0.09050 (0.06267) + + + --> STEP: 226/234 -- GLOBAL_STEP: 32050 + | > loss: -0.33843 (-0.16302) + | > log_mle: -0.59671 (-0.34826) + | > loss_dur: 0.25828 (0.18524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.50766 (36.68116) + | > current_lr: 0.00003 + | > step_time: 0.24350 (3.10028) + | > loader_time: 0.00350 (0.06173) + + + --> STEP: 231/234 -- GLOBAL_STEP: 32055 + | > loss: -0.26235 (-0.16587) + | > log_mle: -0.66488 (-0.35410) + | > loss_dur: 0.40254 (0.18823) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.20121 (38.16041) + | > current_lr: 0.00003 + | > step_time: 0.27120 (3.03862) + | > loader_time: 0.00390 (0.06049) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17969 (+0.16979) + | > avg_loss: -0.18448 (+0.02437) + | > avg_log_mle: -0.41020 (+0.02289) + | > avg_loss_dur: 0.22571 (+0.00148) + + + > EPOCH: 137/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 13:59:48)  + + --> STEP: 2/234 -- GLOBAL_STEP: 32060 + | > loss: -0.09459 (-0.10484) + | > log_mle: -0.23980 (-0.23980) + | > loss_dur: 0.14521 (0.13496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.71342 (15.20451) + | > current_lr: 0.00003 + | > step_time: 1.68700 (2.49295) + | > loader_time: 0.00530 (0.00559) + + + --> STEP: 7/234 -- GLOBAL_STEP: 32065 + | > loss: -0.11531 (-0.08782) + | > log_mle: -0.24994 (-0.23783) + | > loss_dur: 0.13463 (0.15001) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.64809 (16.59828) + | > current_lr: 0.00003 + | > step_time: 3.40260 (2.95667) + | > loader_time: 0.39100 (0.09829) + + + --> STEP: 12/234 -- GLOBAL_STEP: 32070 + | > loss: -0.09397 (-0.09369) + | > log_mle: -0.24293 (-0.24075) + | > loss_dur: 0.14896 (0.14705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.47453 (14.68732) + | > current_lr: 0.00003 + | > step_time: 1.40320 (4.32475) + | > loader_time: 0.00200 (0.10541) + + + --> STEP: 17/234 -- GLOBAL_STEP: 32075 + | > loss: -0.08343 (-0.09835) + | > log_mle: -0.21885 (-0.23959) + | > loss_dur: 0.13543 (0.14124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.00068 (13.01510) + | > current_lr: 0.00003 + | > step_time: 1.00650 (3.83539) + | > loader_time: 0.00130 (0.08063) + + + --> STEP: 22/234 -- GLOBAL_STEP: 32080 + | > loss: -0.12092 (-0.10022) + | > log_mle: -0.24077 (-0.23752) + | > loss_dur: 0.11984 (0.13730) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.76400 (11.99829) + | > current_lr: 0.00003 + | > step_time: 5.20320 (3.64869) + | > loader_time: 0.19450 (0.07183) + + + --> STEP: 27/234 -- GLOBAL_STEP: 32085 + | > loss: -0.12289 (-0.10282) + | > log_mle: -0.24209 (-0.23696) + | > loss_dur: 0.11920 (0.13414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.73152 (11.29965) + | > current_lr: 0.00003 + | > step_time: 0.86950 (3.60650) + | > loader_time: 0.00190 (0.06546) + + + --> STEP: 32/234 -- GLOBAL_STEP: 32090 + | > loss: -0.13706 (-0.10395) + | > log_mle: -0.24916 (-0.23708) + | > loss_dur: 0.11210 (0.13314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.84855 (10.76188) + | > current_lr: 0.00003 + | > step_time: 3.10080 (3.41571) + | > loader_time: 0.00330 (0.05810) + + + --> STEP: 37/234 -- GLOBAL_STEP: 32095 + | > loss: -0.11159 (-0.10313) + | > log_mle: -0.23202 (-0.23665) + | > loss_dur: 0.12042 (0.13352) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.87841 (10.51910) + | > current_lr: 0.00003 + | > step_time: 1.29180 (3.37833) + | > loader_time: 0.00510 (0.05501) + + + --> STEP: 42/234 -- GLOBAL_STEP: 32100 + | > loss: -0.09363 (-0.10269) + | > log_mle: -0.22280 (-0.23620) + | > loss_dur: 0.12917 (0.13350) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.32452 (10.33092) + | > current_lr: 0.00003 + | > step_time: 1.99450 (3.17340) + | > loader_time: 0.00370 (0.05102) + + + --> STEP: 47/234 -- GLOBAL_STEP: 32105 + | > loss: -0.08962 (-0.10221) + | > log_mle: -0.23034 (-0.23641) + | > loss_dur: 0.14072 (0.13420) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.31789 (10.49388) + | > current_lr: 0.00003 + | > step_time: 2.31380 (3.02028) + | > loader_time: 0.00230 (0.04587) + + + --> STEP: 52/234 -- GLOBAL_STEP: 32110 + | > loss: -0.07910 (-0.10160) + | > log_mle: -0.22846 (-0.23561) + | > loss_dur: 0.14935 (0.13401) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.62608 (10.16599) + | > current_lr: 0.00003 + | > step_time: 1.84490 (2.91527) + | > loader_time: 0.19920 (0.04704) + + + --> STEP: 57/234 -- GLOBAL_STEP: 32115 + | > loss: -0.07674 (-0.10138) + | > log_mle: -0.22035 (-0.23587) + | > loss_dur: 0.14360 (0.13449) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.36314 (10.00460) + | > current_lr: 0.00003 + | > step_time: 1.31040 (2.79042) + | > loader_time: 0.00200 (0.04643) + + + --> STEP: 62/234 -- GLOBAL_STEP: 32120 + | > loss: -0.06843 (-0.10144) + | > log_mle: -0.27123 (-0.23699) + | > loss_dur: 0.20280 (0.13556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.38720 (10.32386) + | > current_lr: 0.00003 + | > step_time: 1.19810 (2.71509) + | > loader_time: 0.00290 (0.04432) + + + --> STEP: 67/234 -- GLOBAL_STEP: 32125 + | > loss: -0.09558 (-0.10071) + | > log_mle: -0.25284 (-0.23679) + | > loss_dur: 0.15726 (0.13608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.93333 (10.22372) + | > current_lr: 0.00003 + | > step_time: 1.30950 (2.62153) + | > loader_time: 0.00280 (0.04249) + + + --> STEP: 72/234 -- GLOBAL_STEP: 32130 + | > loss: -0.08652 (-0.09880) + | > log_mle: -0.23235 (-0.23664) + | > loss_dur: 0.14584 (0.13784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.76137 (10.36228) + | > current_lr: 0.00003 + | > step_time: 1.30110 (2.57867) + | > loader_time: 0.00270 (0.03974) + + + --> STEP: 77/234 -- GLOBAL_STEP: 32135 + | > loss: -0.10433 (-0.09834) + | > log_mle: -0.24535 (-0.23715) + | > loss_dur: 0.14101 (0.13882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.51701 (10.44499) + | > current_lr: 0.00003 + | > step_time: 2.11010 (2.55038) + | > loader_time: 0.08770 (0.04077) + + + --> STEP: 82/234 -- GLOBAL_STEP: 32140 + | > loss: -0.09727 (-0.09827) + | > log_mle: -0.23566 (-0.23718) + | > loss_dur: 0.13840 (0.13891) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.52987 (10.41944) + | > current_lr: 0.00003 + | > step_time: 1.59900 (2.51945) + | > loader_time: 0.08540 (0.04253) + + + --> STEP: 87/234 -- GLOBAL_STEP: 32145 + | > loss: -0.07627 (-0.09802) + | > log_mle: -0.24324 (-0.23779) + | > loss_dur: 0.16697 (0.13977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.96309 (10.48625) + | > current_lr: 0.00003 + | > step_time: 1.61290 (2.47214) + | > loader_time: 0.09310 (0.04132) + + + --> STEP: 92/234 -- GLOBAL_STEP: 32150 + | > loss: -0.13434 (-0.09879) + | > log_mle: -0.29019 (-0.23992) + | > loss_dur: 0.15585 (0.14114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.46006 (10.82800) + | > current_lr: 0.00003 + | > step_time: 3.19870 (2.47473) + | > loader_time: 0.00290 (0.04009) + + + --> STEP: 97/234 -- GLOBAL_STEP: 32155 + | > loss: -0.12192 (-0.10038) + | > log_mle: -0.28069 (-0.24299) + | > loss_dur: 0.15878 (0.14261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.45709 (11.38372) + | > current_lr: 0.00003 + | > step_time: 1.70640 (2.46468) + | > loader_time: 0.08570 (0.03904) + + + --> STEP: 102/234 -- GLOBAL_STEP: 32160 + | > loss: -0.08892 (-0.10100) + | > log_mle: -0.26097 (-0.24501) + | > loss_dur: 0.17205 (0.14401) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.92914 (11.73763) + | > current_lr: 0.00003 + | > step_time: 1.04770 (2.46172) + | > loader_time: 0.00260 (0.03908) + + + --> STEP: 107/234 -- GLOBAL_STEP: 32165 + | > loss: -0.12852 (-0.10251) + | > log_mle: -0.30929 (-0.24815) + | > loss_dur: 0.18078 (0.14564) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.63637 (12.29422) + | > current_lr: 0.00003 + | > step_time: 2.70460 (2.42066) + | > loader_time: 0.08200 (0.03891) + + + --> STEP: 112/234 -- GLOBAL_STEP: 32170 + | > loss: -0.11979 (-0.10314) + | > log_mle: -0.31461 (-0.25082) + | > loss_dur: 0.19483 (0.14768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.78190 (13.12725) + | > current_lr: 0.00003 + | > step_time: 1.15300 (2.39845) + | > loader_time: 0.00260 (0.03731) + + + --> STEP: 117/234 -- GLOBAL_STEP: 32175 + | > loss: -0.13641 (-0.10419) + | > log_mle: -0.31233 (-0.25346) + | > loss_dur: 0.17592 (0.14927) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.07628 (13.69331) + | > current_lr: 0.00003 + | > step_time: 3.29020 (2.37868) + | > loader_time: 0.00150 (0.03584) + + + --> STEP: 122/234 -- GLOBAL_STEP: 32180 + | > loss: -0.12027 (-0.10481) + | > log_mle: -0.28637 (-0.25510) + | > loss_dur: 0.16610 (0.15029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.59730 (13.97942) + | > current_lr: 0.00003 + | > step_time: 2.39590 (2.36069) + | > loader_time: 0.00340 (0.03656) + + + --> STEP: 127/234 -- GLOBAL_STEP: 32185 + | > loss: -0.15181 (-0.10644) + | > log_mle: -0.34406 (-0.25804) + | > loss_dur: 0.19225 (0.15160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.39243 (14.67619) + | > current_lr: 0.00003 + | > step_time: 2.11620 (2.35063) + | > loader_time: 0.08530 (0.03649) + + + --> STEP: 132/234 -- GLOBAL_STEP: 32190 + | > loss: -0.15192 (-0.10845) + | > log_mle: -0.32775 (-0.26144) + | > loss_dur: 0.17583 (0.15298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.98717 (15.41243) + | > current_lr: 0.00003 + | > step_time: 1.60180 (2.32194) + | > loader_time: 0.08530 (0.03662) + + + --> STEP: 137/234 -- GLOBAL_STEP: 32195 + | > loss: -0.13920 (-0.11042) + | > log_mle: -0.34237 (-0.26514) + | > loss_dur: 0.20317 (0.15472) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.29523 (16.01966) + | > current_lr: 0.00003 + | > step_time: 2.20620 (2.33567) + | > loader_time: 0.08470 (0.03734) + + + --> STEP: 142/234 -- GLOBAL_STEP: 32200 + | > loss: -0.14926 (-0.11211) + | > log_mle: -0.35185 (-0.26833) + | > loss_dur: 0.20258 (0.15622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.29947 (16.79339) + | > current_lr: 0.00003 + | > step_time: 1.50570 (2.34782) + | > loader_time: 0.08620 (0.04007) + + + --> STEP: 147/234 -- GLOBAL_STEP: 32205 + | > loss: -0.14464 (-0.11433) + | > log_mle: -0.34492 (-0.27276) + | > loss_dur: 0.20028 (0.15842) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.31659 (18.29741) + | > current_lr: 0.00003 + | > step_time: 0.92810 (2.33185) + | > loader_time: 0.00280 (0.04242) + + + --> STEP: 152/234 -- GLOBAL_STEP: 32210 + | > loss: -0.19608 (-0.11677) + | > log_mle: -0.42584 (-0.27675) + | > loss_dur: 0.22976 (0.15998) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.72206 (19.26919) + | > current_lr: 0.00003 + | > step_time: 1.88530 (2.32377) + | > loader_time: 0.00250 (0.04330) + + + --> STEP: 157/234 -- GLOBAL_STEP: 32215 + | > loss: -0.17310 (-0.11999) + | > log_mle: -0.38003 (-0.28151) + | > loss_dur: 0.20693 (0.16151) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.50263 (20.66905) + | > current_lr: 0.00003 + | > step_time: 4.09930 (2.33831) + | > loader_time: 0.09110 (0.04357) + + + --> STEP: 162/234 -- GLOBAL_STEP: 32220 + | > loss: -0.21618 (-0.12310) + | > log_mle: -0.41484 (-0.28621) + | > loss_dur: 0.19866 (0.16311) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.22375 (21.65039) + | > current_lr: 0.00003 + | > step_time: 4.20120 (2.36139) + | > loader_time: 0.20200 (0.04466) + + + --> STEP: 167/234 -- GLOBAL_STEP: 32225 + | > loss: -0.29510 (-0.12612) + | > log_mle: -0.50510 (-0.29065) + | > loss_dur: 0.21000 (0.16453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.87105 (22.35057) + | > current_lr: 0.00003 + | > step_time: 2.51720 (2.47663) + | > loader_time: 0.09150 (0.04724) + + + --> STEP: 172/234 -- GLOBAL_STEP: 32230 + | > loss: -0.25143 (-0.12938) + | > log_mle: -0.49379 (-0.29590) + | > loss_dur: 0.24236 (0.16651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.01900 (23.47338) + | > current_lr: 0.00003 + | > step_time: 2.02160 (2.45952) + | > loader_time: 0.08540 (0.04645) + + + --> STEP: 177/234 -- GLOBAL_STEP: 32235 + | > loss: -0.22922 (-0.13263) + | > log_mle: -0.45499 (-0.30091) + | > loss_dur: 0.22577 (0.16828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.21501 (24.59696) + | > current_lr: 0.00003 + | > step_time: 2.99600 (2.46972) + | > loader_time: 0.00310 (0.04618) + + + --> STEP: 182/234 -- GLOBAL_STEP: 32240 + | > loss: -0.24780 (-0.13561) + | > log_mle: -0.49892 (-0.30588) + | > loss_dur: 0.25112 (0.17027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.13087 (25.70058) + | > current_lr: 0.00003 + | > step_time: 3.89820 (2.51141) + | > loader_time: 0.00640 (0.04645) + + + --> STEP: 187/234 -- GLOBAL_STEP: 32245 + | > loss: -0.26242 (-0.13881) + | > log_mle: -0.49675 (-0.31084) + | > loss_dur: 0.23433 (0.17202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.87424 (26.88355) + | > current_lr: 0.00003 + | > step_time: 6.20110 (2.53994) + | > loader_time: 0.10210 (0.04687) + + + --> STEP: 192/234 -- GLOBAL_STEP: 32250 + | > loss: -0.30803 (-0.14227) + | > log_mle: -0.52823 (-0.31571) + | > loss_dur: 0.22019 (0.17344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.73355 (27.93715) + | > current_lr: 0.00003 + | > step_time: 2.49560 (2.60762) + | > loader_time: 0.00410 (0.04677) + + + --> STEP: 197/234 -- GLOBAL_STEP: 32255 + | > loss: -0.26981 (-0.14534) + | > log_mle: -0.49262 (-0.32030) + | > loss_dur: 0.22281 (0.17496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.74113 (29.10810) + | > current_lr: 0.00003 + | > step_time: 4.20000 (2.66124) + | > loader_time: 0.30510 (0.04917) + + + --> STEP: 202/234 -- GLOBAL_STEP: 32260 + | > loss: -0.35685 (-0.14830) + | > log_mle: -0.58700 (-0.32484) + | > loss_dur: 0.23015 (0.17654) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.50462 (30.39038) + | > current_lr: 0.00003 + | > step_time: 2.60280 (2.70490) + | > loader_time: 0.01070 (0.05034) + + + --> STEP: 207/234 -- GLOBAL_STEP: 32265 + | > loss: -0.31525 (-0.15128) + | > log_mle: -0.56711 (-0.32943) + | > loss_dur: 0.25186 (0.17815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.05517 (31.43775) + | > current_lr: 0.00003 + | > step_time: 7.09350 (2.72120) + | > loader_time: 0.00380 (0.04923) + + + --> STEP: 212/234 -- GLOBAL_STEP: 32270 + | > loss: -0.30164 (-0.15478) + | > log_mle: -0.54985 (-0.33467) + | > loss_dur: 0.24821 (0.17989) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.53555 (32.62500) + | > current_lr: 0.00003 + | > step_time: 2.79430 (2.78291) + | > loader_time: 0.00610 (0.04911) + + + --> STEP: 217/234 -- GLOBAL_STEP: 32275 + | > loss: -0.32494 (-0.15846) + | > log_mle: -0.57738 (-0.33990) + | > loss_dur: 0.25244 (0.18144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.03753 (33.97729) + | > current_lr: 0.00003 + | > step_time: 3.49490 (2.81701) + | > loader_time: 0.00420 (0.04887) + + + --> STEP: 222/234 -- GLOBAL_STEP: 32280 + | > loss: -0.30963 (-0.16186) + | > log_mle: -0.59254 (-0.34508) + | > loss_dur: 0.28291 (0.18321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.78957 (35.33753) + | > current_lr: 0.00003 + | > step_time: 2.20900 (2.81589) + | > loader_time: 0.00840 (0.07709) + + + --> STEP: 227/234 -- GLOBAL_STEP: 32285 + | > loss: -0.29761 (-0.16575) + | > log_mle: -0.56460 (-0.35067) + | > loss_dur: 0.26698 (0.18492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.27879 (36.68967) + | > current_lr: 0.00003 + | > step_time: 3.00270 (2.80862) + | > loader_time: 0.00400 (0.07618) + + + --> STEP: 232/234 -- GLOBAL_STEP: 32290 + | > loss: -0.23952 (-0.16856) + | > log_mle: -0.75949 (-0.35751) + | > loss_dur: 0.51997 (0.18895) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 181.55083 (38.59519) + | > current_lr: 0.00003 + | > step_time: 0.33240 (2.76281) + | > loader_time: 0.08030 (0.14652) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.02611 (+0.84642) + | > avg_loss: -0.16562 (+0.01886) + | > avg_log_mle: -0.39596 (+0.01423) + | > avg_loss_dur: 0.23034 (+0.00462) + + + > EPOCH: 138/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 14:12:20)  + + --> STEP: 3/234 -- GLOBAL_STEP: 32295 + | > loss: -0.04084 (-0.09224) + | > log_mle: -0.23820 (-0.24011) + | > loss_dur: 0.19736 (0.14788) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.65590 (11.36687) + | > current_lr: 0.00003 + | > step_time: 2.70110 (8.10049) + | > loader_time: 0.19510 (0.29635) + + + --> STEP: 8/234 -- GLOBAL_STEP: 32300 + | > loss: -0.12175 (-0.09727) + | > log_mle: -0.25282 (-0.24080) + | > loss_dur: 0.13106 (0.14353) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.04021 (12.92098) + | > current_lr: 0.00003 + | > step_time: 1.19920 (3.91095) + | > loader_time: 0.00140 (0.11237) + + + --> STEP: 13/234 -- GLOBAL_STEP: 32305 + | > loss: -0.12791 (-0.10175) + | > log_mle: -0.24417 (-0.24206) + | > loss_dur: 0.11627 (0.14031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.13717 (12.36945) + | > current_lr: 0.00003 + | > step_time: 1.70470 (3.16932) + | > loader_time: 0.10000 (0.07748) + + + --> STEP: 18/234 -- GLOBAL_STEP: 32310 + | > loss: -0.10625 (-0.10408) + | > log_mle: -0.23578 (-0.24055) + | > loss_dur: 0.12953 (0.13647) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.30252 (11.72607) + | > current_lr: 0.00003 + | > step_time: 4.19170 (3.07788) + | > loader_time: 0.00110 (0.06748) + + + --> STEP: 23/234 -- GLOBAL_STEP: 32315 + | > loss: -0.12843 (-0.10678) + | > log_mle: -0.24133 (-0.23904) + | > loss_dur: 0.11289 (0.13226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.41897 (11.07339) + | > current_lr: 0.00003 + | > step_time: 1.39520 (3.41647) + | > loader_time: 0.10510 (0.07126) + + + --> STEP: 28/234 -- GLOBAL_STEP: 32320 + | > loss: -0.12791 (-0.10847) + | > log_mle: -0.23518 (-0.23816) + | > loss_dur: 0.10727 (0.12969) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.01563 (10.63134) + | > current_lr: 0.00003 + | > step_time: 3.88710 (3.16376) + | > loader_time: 0.00410 (0.06529) + + + --> STEP: 33/234 -- GLOBAL_STEP: 32325 + | > loss: -0.10521 (-0.10878) + | > log_mle: -0.23109 (-0.23816) + | > loss_dur: 0.12588 (0.12938) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.96812 (10.34791) + | > current_lr: 0.00003 + | > step_time: 1.11200 (3.07119) + | > loader_time: 0.00190 (0.05815) + + + --> STEP: 38/234 -- GLOBAL_STEP: 32330 + | > loss: -0.10276 (-0.10749) + | > log_mle: -0.24449 (-0.23805) + | > loss_dur: 0.14173 (0.13056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.38001 (10.38032) + | > current_lr: 0.00003 + | > step_time: 1.66600 (2.84307) + | > loader_time: 0.00200 (0.05075) + + + --> STEP: 43/234 -- GLOBAL_STEP: 32335 + | > loss: -0.08362 (-0.10582) + | > log_mle: -0.24329 (-0.23744) + | > loss_dur: 0.15967 (0.13161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.77639 (10.21467) + | > current_lr: 0.00003 + | > step_time: 2.29050 (2.74903) + | > loader_time: 0.00220 (0.04707) + + + --> STEP: 48/234 -- GLOBAL_STEP: 32340 + | > loss: -0.10388 (-0.10532) + | > log_mle: -0.22457 (-0.23728) + | > loss_dur: 0.12069 (0.13196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.78393 (10.18228) + | > current_lr: 0.00003 + | > step_time: 1.26540 (2.64697) + | > loader_time: 0.00210 (0.04239) + + + --> STEP: 53/234 -- GLOBAL_STEP: 32345 + | > loss: -0.10411 (-0.10452) + | > log_mle: -0.24505 (-0.23694) + | > loss_dur: 0.14094 (0.13242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.93470 (9.90689) + | > current_lr: 0.00003 + | > step_time: 1.29200 (2.57290) + | > loader_time: 0.00210 (0.04020) + + + --> STEP: 58/234 -- GLOBAL_STEP: 32350 + | > loss: -0.11260 (-0.10458) + | > log_mle: -0.23196 (-0.23702) + | > loss_dur: 0.11936 (0.13244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.42168 (9.72677) + | > current_lr: 0.00003 + | > step_time: 2.01230 (2.50817) + | > loader_time: 0.08310 (0.04123) + + + --> STEP: 63/234 -- GLOBAL_STEP: 32355 + | > loss: -0.07617 (-0.10412) + | > log_mle: -0.24004 (-0.23842) + | > loss_dur: 0.16387 (0.13431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.32794 (9.89221) + | > current_lr: 0.00003 + | > step_time: 1.32730 (2.42880) + | > loader_time: 0.00210 (0.03980) + + + --> STEP: 68/234 -- GLOBAL_STEP: 32360 + | > loss: -0.06444 (-0.10346) + | > log_mle: -0.23283 (-0.23831) + | > loss_dur: 0.16840 (0.13485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.69932 (9.83099) + | > current_lr: 0.00003 + | > step_time: 1.80590 (2.41218) + | > loader_time: 0.00240 (0.03828) + + + --> STEP: 73/234 -- GLOBAL_STEP: 32365 + | > loss: -0.09068 (-0.10193) + | > log_mle: -0.25606 (-0.23862) + | > loss_dur: 0.16538 (0.13669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.76407 (9.91453) + | > current_lr: 0.00003 + | > step_time: 2.38990 (2.38796) + | > loader_time: 0.00470 (0.03588) + + + --> STEP: 78/234 -- GLOBAL_STEP: 32370 + | > loss: -0.08239 (-0.10155) + | > log_mle: -0.23022 (-0.23884) + | > loss_dur: 0.14783 (0.13729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46453 (9.96346) + | > current_lr: 0.00003 + | > step_time: 2.30790 (2.36568) + | > loader_time: 0.00240 (0.03484) + + + --> STEP: 83/234 -- GLOBAL_STEP: 32375 + | > loss: -0.09119 (-0.10145) + | > log_mle: -0.25648 (-0.23924) + | > loss_dur: 0.16529 (0.13779) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.88743 (10.05979) + | > current_lr: 0.00003 + | > step_time: 2.29470 (2.33472) + | > loader_time: 0.00200 (0.03288) + + + --> STEP: 88/234 -- GLOBAL_STEP: 32380 + | > loss: -0.13128 (-0.10151) + | > log_mle: -0.29271 (-0.24020) + | > loss_dur: 0.16143 (0.13868) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.83880 (10.23416) + | > current_lr: 0.00003 + | > step_time: 1.59390 (2.31419) + | > loader_time: 0.00200 (0.03212) + + + --> STEP: 93/234 -- GLOBAL_STEP: 32385 + | > loss: -0.11598 (-0.10214) + | > log_mle: -0.30513 (-0.24243) + | > loss_dur: 0.18915 (0.14029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.94030 (10.66040) + | > current_lr: 0.00003 + | > step_time: 1.50180 (2.31110) + | > loader_time: 0.00250 (0.03256) + + + --> STEP: 98/234 -- GLOBAL_STEP: 32390 + | > loss: -0.08500 (-0.10342) + | > log_mle: -0.23377 (-0.24474) + | > loss_dur: 0.14877 (0.14133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.67782 (11.06156) + | > current_lr: 0.00003 + | > step_time: 1.80770 (2.26845) + | > loader_time: 0.00500 (0.03106) + + + --> STEP: 103/234 -- GLOBAL_STEP: 32395 + | > loss: -0.14887 (-0.10477) + | > log_mle: -0.33626 (-0.24777) + | > loss_dur: 0.18739 (0.14300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.85929 (11.70597) + | > current_lr: 0.00003 + | > step_time: 3.19340 (2.24847) + | > loader_time: 0.08960 (0.03052) + + + --> STEP: 108/234 -- GLOBAL_STEP: 32400 + | > loss: -0.12126 (-0.10587) + | > log_mle: -0.27957 (-0.25029) + | > loss_dur: 0.15830 (0.14442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.45726 (12.20576) + | > current_lr: 0.00003 + | > step_time: 1.70790 (2.24536) + | > loader_time: 0.00260 (0.03001) + + + --> STEP: 113/234 -- GLOBAL_STEP: 32405 + | > loss: -0.15165 (-0.10689) + | > log_mle: -0.32524 (-0.25339) + | > loss_dur: 0.17358 (0.14650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.56277 (13.32551) + | > current_lr: 0.00003 + | > step_time: 1.55830 (2.25197) + | > loader_time: 0.00250 (0.02963) + + + --> STEP: 118/234 -- GLOBAL_STEP: 32410 + | > loss: -0.11523 (-0.10755) + | > log_mle: -0.29663 (-0.25567) + | > loss_dur: 0.18140 (0.14812) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.01667 (13.83598) + | > current_lr: 0.00003 + | > step_time: 2.61360 (2.24884) + | > loader_time: 0.08580 (0.03058) + + + --> STEP: 123/234 -- GLOBAL_STEP: 32415 + | > loss: -0.10826 (-0.10802) + | > log_mle: -0.26773 (-0.25708) + | > loss_dur: 0.15946 (0.14906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.83574 (14.08836) + | > current_lr: 0.00003 + | > step_time: 2.60120 (2.24126) + | > loader_time: 0.00350 (0.02948) + + + --> STEP: 128/234 -- GLOBAL_STEP: 32420 + | > loss: -0.16273 (-0.11008) + | > log_mle: -0.32440 (-0.26050) + | > loss_dur: 0.16168 (0.15042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.34876 (14.95769) + | > current_lr: 0.00003 + | > step_time: 3.10190 (2.24743) + | > loader_time: 0.19610 (0.03059) + + + --> STEP: 133/234 -- GLOBAL_STEP: 32425 + | > loss: -0.15756 (-0.11197) + | > log_mle: -0.35374 (-0.26399) + | > loss_dur: 0.19617 (0.15203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.71631 (15.63165) + | > current_lr: 0.00003 + | > step_time: 1.39110 (2.23494) + | > loader_time: 0.00310 (0.03021) + + + --> STEP: 138/234 -- GLOBAL_STEP: 32430 + | > loss: -0.11329 (-0.11316) + | > log_mle: -0.30366 (-0.26708) + | > loss_dur: 0.19037 (0.15393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.31565 (16.49751) + | > current_lr: 0.00003 + | > step_time: 2.80440 (2.24250) + | > loader_time: 0.00300 (0.02990) + + + --> STEP: 143/234 -- GLOBAL_STEP: 32435 + | > loss: -0.19881 (-0.11528) + | > log_mle: -0.44436 (-0.27114) + | > loss_dur: 0.24555 (0.15586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.22013 (17.40262) + | > current_lr: 0.00003 + | > step_time: 2.20870 (2.24923) + | > loader_time: 0.08360 (0.03017) + + + --> STEP: 148/234 -- GLOBAL_STEP: 32440 + | > loss: -0.18111 (-0.11785) + | > log_mle: -0.36214 (-0.27529) + | > loss_dur: 0.18103 (0.15744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.93306 (18.17563) + | > current_lr: 0.00003 + | > step_time: 1.09960 (2.24308) + | > loader_time: 0.07780 (0.03033) + + + --> STEP: 153/234 -- GLOBAL_STEP: 32445 + | > loss: -0.27801 (-0.12134) + | > log_mle: -0.48987 (-0.28057) + | > loss_dur: 0.21186 (0.15922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.26769 (19.23809) + | > current_lr: 0.00003 + | > step_time: 4.30450 (2.27959) + | > loader_time: 0.00370 (0.03122) + + + --> STEP: 158/234 -- GLOBAL_STEP: 32450 + | > loss: -0.19551 (-0.12434) + | > log_mle: -0.42523 (-0.28523) + | > loss_dur: 0.22972 (0.16089) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.90372 (20.33152) + | > current_lr: 0.00003 + | > step_time: 8.30660 (2.35011) + | > loader_time: 0.19210 (0.03321) + + + --> STEP: 163/234 -- GLOBAL_STEP: 32455 + | > loss: -0.18962 (-0.12745) + | > log_mle: -0.39351 (-0.28978) + | > loss_dur: 0.20389 (0.16233) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.70372 (21.36535) + | > current_lr: 0.00003 + | > step_time: 3.11190 (2.39027) + | > loader_time: 0.00460 (0.03524) + + + --> STEP: 168/234 -- GLOBAL_STEP: 32460 + | > loss: -0.21577 (-0.13059) + | > log_mle: -0.44916 (-0.29447) + | > loss_dur: 0.23339 (0.16388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.99329 (22.44738) + | > current_lr: 0.00003 + | > step_time: 3.09780 (2.38465) + | > loader_time: 0.00870 (0.03481) + + + --> STEP: 173/234 -- GLOBAL_STEP: 32465 + | > loss: -0.22458 (-0.13374) + | > log_mle: -0.44443 (-0.29952) + | > loss_dur: 0.21984 (0.16578) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.60877 (24.07538) + | > current_lr: 0.00003 + | > step_time: 2.69870 (2.36365) + | > loader_time: 0.00450 (0.03437) + + + --> STEP: 178/234 -- GLOBAL_STEP: 32470 + | > loss: -0.25491 (-0.13693) + | > log_mle: -0.50929 (-0.30459) + | > loss_dur: 0.25438 (0.16765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.97301 (25.31887) + | > current_lr: 0.00003 + | > step_time: 1.80590 (2.35840) + | > loader_time: 0.08390 (0.03398) + + + --> STEP: 183/234 -- GLOBAL_STEP: 32475 + | > loss: -0.27457 (-0.13968) + | > log_mle: -0.50745 (-0.30932) + | > loss_dur: 0.23288 (0.16963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.31333 (26.39679) + | > current_lr: 0.00003 + | > step_time: 9.90500 (2.42446) + | > loader_time: 0.19780 (0.03578) + + + --> STEP: 188/234 -- GLOBAL_STEP: 32480 + | > loss: -0.28036 (-0.14252) + | > log_mle: -0.51764 (-0.31403) + | > loss_dur: 0.23728 (0.17150) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.69184 (27.76227) + | > current_lr: 0.00003 + | > step_time: 3.81180 (2.44623) + | > loader_time: 0.00470 (0.03580) + + + --> STEP: 193/234 -- GLOBAL_STEP: 32485 + | > loss: -0.29311 (-0.14570) + | > log_mle: -0.52038 (-0.31865) + | > loss_dur: 0.22727 (0.17295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.85812 (29.02065) + | > current_lr: 0.00003 + | > step_time: 3.00180 (2.45961) + | > loader_time: 0.00420 (0.03945) + + + --> STEP: 198/234 -- GLOBAL_STEP: 32490 + | > loss: -0.25896 (-0.14849) + | > log_mle: -0.49656 (-0.32297) + | > loss_dur: 0.23760 (0.17449) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 123.47141 (30.38729) + | > current_lr: 0.00003 + | > step_time: 6.69200 (2.54913) + | > loader_time: 0.19360 (0.04141) + + + --> STEP: 203/234 -- GLOBAL_STEP: 32495 + | > loss: -0.19817 (-0.15076) + | > log_mle: -0.43123 (-0.32689) + | > loss_dur: 0.23306 (0.17613) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.93892 (31.82195) + | > current_lr: 0.00003 + | > step_time: 5.71410 (2.59279) + | > loader_time: 0.09560 (0.04240) + + + --> STEP: 208/234 -- GLOBAL_STEP: 32500 + | > loss: -0.27773 (-0.15383) + | > log_mle: -0.52495 (-0.33170) + | > loss_dur: 0.24722 (0.17787) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.59093 (32.82638) + | > current_lr: 0.00003 + | > step_time: 4.39710 (2.70945) + | > loader_time: 0.00600 (0.04332) + + + --> STEP: 213/234 -- GLOBAL_STEP: 32505 + | > loss: -0.31188 (-0.15741) + | > log_mle: -0.57612 (-0.33706) + | > loss_dur: 0.26424 (0.17965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.35606 (34.04676) + | > current_lr: 0.00003 + | > step_time: 9.99890 (2.75338) + | > loader_time: 0.00360 (0.04368) + + + --> STEP: 218/234 -- GLOBAL_STEP: 32510 + | > loss: -0.28604 (-0.16074) + | > log_mle: -0.53729 (-0.34194) + | > loss_dur: 0.25126 (0.18120) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.45945 (35.93576) + | > current_lr: 0.00003 + | > step_time: 3.10300 (2.87100) + | > loader_time: 0.00630 (0.04632) + + + --> STEP: 223/234 -- GLOBAL_STEP: 32515 + | > loss: -0.32809 (-0.16437) + | > log_mle: -0.57622 (-0.34723) + | > loss_dur: 0.24813 (0.18285) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.87797 (37.07933) + | > current_lr: 0.00003 + | > step_time: 0.22500 (2.86569) + | > loader_time: 0.00310 (0.04577) + + + --> STEP: 228/234 -- GLOBAL_STEP: 32520 + | > loss: -0.29378 (-0.16778) + | > log_mle: -0.57925 (-0.35255) + | > loss_dur: 0.28547 (0.18478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.39551 (38.30471) + | > current_lr: 0.00003 + | > step_time: 0.25610 (2.80813) + | > loader_time: 0.00330 (0.04486) + + + --> STEP: 233/234 -- GLOBAL_STEP: 32525 + | > loss: 0.31405 (-0.16752) + | > log_mle: -0.52191 (-0.35874) + | > loss_dur: 0.83596 (0.19122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.96302 (40.08716) + | > current_lr: 0.00003 + | > step_time: 0.18890 (2.75346) + | > loader_time: 0.00290 (0.04399) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.14642 (-0.87969) + | > avg_loss: -0.20223 (-0.03660) + | > avg_log_mle: -0.43367 (-0.03771) + | > avg_loss_dur: 0.23144 (+0.00110) + + + > EPOCH: 139/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 14:24:27)  + + --> STEP: 4/234 -- GLOBAL_STEP: 32530 + | > loss: -0.07408 (-0.08850) + | > log_mle: -0.23406 (-0.23865) + | > loss_dur: 0.15998 (0.15015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.06112 (17.54870) + | > current_lr: 0.00003 + | > step_time: 6.09530 (6.54885) + | > loader_time: 0.40890 (0.19882) + + + --> STEP: 9/234 -- GLOBAL_STEP: 32535 + | > loss: -0.08610 (-0.09710) + | > log_mle: -0.25039 (-0.24254) + | > loss_dur: 0.16429 (0.14545) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.15705 (15.10452) + | > current_lr: 0.00003 + | > step_time: 8.60520 (5.34634) + | > loader_time: 0.09750 (0.13079) + + + --> STEP: 14/234 -- GLOBAL_STEP: 32540 + | > loss: -0.10075 (-0.09947) + | > log_mle: -0.24664 (-0.24276) + | > loss_dur: 0.14589 (0.14329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.05747 (13.77828) + | > current_lr: 0.00003 + | > step_time: 1.01720 (4.09446) + | > loader_time: 0.00160 (0.09927) + + + --> STEP: 19/234 -- GLOBAL_STEP: 32545 + | > loss: -0.13581 (-0.10365) + | > log_mle: -0.23442 (-0.24055) + | > loss_dur: 0.09861 (0.13690) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.23456 (12.70081) + | > current_lr: 0.00003 + | > step_time: 0.80410 (3.46969) + | > loader_time: 0.00240 (0.07371) + + + --> STEP: 24/234 -- GLOBAL_STEP: 32550 + | > loss: -0.13628 (-0.10634) + | > log_mle: -0.23439 (-0.23928) + | > loss_dur: 0.09811 (0.13293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.50552 (11.59653) + | > current_lr: 0.00003 + | > step_time: 1.29310 (3.01480) + | > loader_time: 0.00130 (0.05873) + + + --> STEP: 29/234 -- GLOBAL_STEP: 32555 + | > loss: -0.08576 (-0.10772) + | > log_mle: -0.22363 (-0.23816) + | > loss_dur: 0.13787 (0.13044) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79817 (10.97448) + | > current_lr: 0.00003 + | > step_time: 1.45050 (2.73692) + | > loader_time: 0.00180 (0.04900) + + + --> STEP: 34/234 -- GLOBAL_STEP: 32560 + | > loss: -0.08197 (-0.10791) + | > log_mle: -0.23220 (-0.23819) + | > loss_dur: 0.15023 (0.13028) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42810 (10.69328) + | > current_lr: 0.00003 + | > step_time: 1.16090 (2.48370) + | > loader_time: 0.00190 (0.04208) + + + --> STEP: 39/234 -- GLOBAL_STEP: 32565 + | > loss: -0.09897 (-0.10749) + | > log_mle: -0.24211 (-0.23825) + | > loss_dur: 0.14314 (0.13076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99056 (10.86348) + | > current_lr: 0.00003 + | > step_time: 2.79490 (2.38226) + | > loader_time: 0.00270 (0.03911) + + + --> STEP: 44/234 -- GLOBAL_STEP: 32570 + | > loss: -0.13089 (-0.10649) + | > log_mle: -0.23541 (-0.23768) + | > loss_dur: 0.10452 (0.13119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.62970 (10.55132) + | > current_lr: 0.00003 + | > step_time: 1.33590 (2.25748) + | > loader_time: 0.00200 (0.03497) + + + --> STEP: 49/234 -- GLOBAL_STEP: 32575 + | > loss: -0.12498 (-0.10590) + | > log_mle: -0.24336 (-0.23785) + | > loss_dur: 0.11838 (0.13195) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.68853 (10.46577) + | > current_lr: 0.00003 + | > step_time: 2.90580 (2.22066) + | > loader_time: 0.08870 (0.03679) + + + --> STEP: 54/234 -- GLOBAL_STEP: 32580 + | > loss: -0.11929 (-0.10523) + | > log_mle: -0.24846 (-0.23770) + | > loss_dur: 0.12917 (0.13247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.07071 (10.17899) + | > current_lr: 0.00003 + | > step_time: 1.41080 (2.20231) + | > loader_time: 0.19160 (0.03867) + + + --> STEP: 59/234 -- GLOBAL_STEP: 32585 + | > loss: -0.12753 (-0.10502) + | > log_mle: -0.25418 (-0.23787) + | > loss_dur: 0.12664 (0.13285) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.28530 (10.04685) + | > current_lr: 0.00003 + | > step_time: 1.33820 (2.17399) + | > loader_time: 0.00210 (0.03560) + + + --> STEP: 64/234 -- GLOBAL_STEP: 32590 + | > loss: -0.10802 (-0.10469) + | > log_mle: -0.23212 (-0.23887) + | > loss_dur: 0.12410 (0.13418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.96314 (10.06476) + | > current_lr: 0.00003 + | > step_time: 1.20540 (2.14855) + | > loader_time: 0.00210 (0.03308) + + + --> STEP: 69/234 -- GLOBAL_STEP: 32595 + | > loss: -0.07854 (-0.10369) + | > log_mle: -0.21817 (-0.23855) + | > loss_dur: 0.13963 (0.13486) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.70822 (10.06948) + | > current_lr: 0.00003 + | > step_time: 1.76650 (2.13516) + | > loader_time: 0.08250 (0.03205) + + + --> STEP: 74/234 -- GLOBAL_STEP: 32600 + | > loss: -0.09001 (-0.10255) + | > log_mle: -0.22789 (-0.23876) + | > loss_dur: 0.13788 (0.13620) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.62283 (10.48515) + | > current_lr: 0.00003 + | > step_time: 2.20620 (2.13751) + | > loader_time: 0.00310 (0.03015) + + + --> STEP: 79/234 -- GLOBAL_STEP: 32605 + | > loss: -0.10618 (-0.10201) + | > log_mle: -0.24841 (-0.23910) + | > loss_dur: 0.14223 (0.13709) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.63611 (10.47364) + | > current_lr: 0.00003 + | > step_time: 2.00380 (2.18202) + | > loader_time: 0.09800 (0.03162) + + + --> STEP: 84/234 -- GLOBAL_STEP: 32610 + | > loss: -0.08926 (-0.10177) + | > log_mle: -0.24064 (-0.23944) + | > loss_dur: 0.15138 (0.13767) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.64540 (10.51827) + | > current_lr: 0.00003 + | > step_time: 2.49520 (2.15635) + | > loader_time: 0.00290 (0.02988) + + + --> STEP: 89/234 -- GLOBAL_STEP: 32615 + | > loss: -0.11573 (-0.10222) + | > log_mle: -0.27197 (-0.24073) + | > loss_dur: 0.15624 (0.13850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.22104 (10.66621) + | > current_lr: 0.00003 + | > step_time: 2.10950 (2.17659) + | > loader_time: 0.00400 (0.02841) + + + --> STEP: 94/234 -- GLOBAL_STEP: 32620 + | > loss: -0.15346 (-0.10329) + | > log_mle: -0.30809 (-0.24334) + | > loss_dur: 0.15464 (0.14005) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.72221 (11.09132) + | > current_lr: 0.00003 + | > step_time: 1.30200 (2.14790) + | > loader_time: 0.00280 (0.02702) + + + --> STEP: 99/234 -- GLOBAL_STEP: 32625 + | > loss: -0.15083 (-0.10454) + | > log_mle: -0.33937 (-0.24590) + | > loss_dur: 0.18854 (0.14137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.95096 (11.68958) + | > current_lr: 0.00003 + | > step_time: 0.91840 (2.12758) + | > loader_time: 0.00290 (0.02584) + + + --> STEP: 104/234 -- GLOBAL_STEP: 32630 + | > loss: -0.17817 (-0.10595) + | > log_mle: -0.35041 (-0.24884) + | > loss_dur: 0.17224 (0.14289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.13630 (12.32621) + | > current_lr: 0.00003 + | > step_time: 1.01720 (2.10134) + | > loader_time: 0.00320 (0.02648) + + + --> STEP: 109/234 -- GLOBAL_STEP: 32635 + | > loss: -0.09636 (-0.10626) + | > log_mle: -0.31271 (-0.25092) + | > loss_dur: 0.21635 (0.14466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.65894 (13.02957) + | > current_lr: 0.00003 + | > step_time: 1.69890 (2.08672) + | > loader_time: 0.00370 (0.02542) + + + --> STEP: 114/234 -- GLOBAL_STEP: 32640 + | > loss: -0.13551 (-0.10717) + | > log_mle: -0.30179 (-0.25352) + | > loss_dur: 0.16627 (0.14635) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.47509 (13.92961) + | > current_lr: 0.00003 + | > step_time: 3.09950 (2.12245) + | > loader_time: 0.00250 (0.02681) + + + --> STEP: 119/234 -- GLOBAL_STEP: 32645 + | > loss: -0.12672 (-0.10776) + | > log_mle: -0.30326 (-0.25578) + | > loss_dur: 0.17654 (0.14803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.34035 (14.18525) + | > current_lr: 0.00003 + | > step_time: 2.29350 (2.15919) + | > loader_time: 0.00350 (0.02721) + + + --> STEP: 124/234 -- GLOBAL_STEP: 32650 + | > loss: -0.16394 (-0.10853) + | > log_mle: -0.33162 (-0.25743) + | > loss_dur: 0.16768 (0.14890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.36916 (14.36963) + | > current_lr: 0.00003 + | > step_time: 1.40350 (2.15679) + | > loader_time: 0.08910 (0.02777) + + + --> STEP: 129/234 -- GLOBAL_STEP: 32655 + | > loss: -0.13586 (-0.11037) + | > log_mle: -0.32579 (-0.26079) + | > loss_dur: 0.18993 (0.15042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.31997 (14.89734) + | > current_lr: 0.00003 + | > step_time: 1.46960 (2.16699) + | > loader_time: 0.00200 (0.02819) + + + --> STEP: 134/234 -- GLOBAL_STEP: 32660 + | > loss: -0.16045 (-0.11265) + | > log_mle: -0.37319 (-0.26466) + | > loss_dur: 0.21274 (0.15201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.39858 (15.66579) + | > current_lr: 0.00003 + | > step_time: 1.89450 (2.17219) + | > loader_time: 0.00460 (0.02909) + + + --> STEP: 139/234 -- GLOBAL_STEP: 32665 + | > loss: -0.23623 (-0.11461) + | > log_mle: -0.43436 (-0.26832) + | > loss_dur: 0.19813 (0.15371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.22951 (16.34854) + | > current_lr: 0.00003 + | > step_time: 3.36650 (2.18810) + | > loader_time: 0.00340 (0.02886) + + + --> STEP: 144/234 -- GLOBAL_STEP: 32670 + | > loss: -0.19633 (-0.11649) + | > log_mle: -0.40996 (-0.27224) + | > loss_dur: 0.21363 (0.15575) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.70343 (17.27430) + | > current_lr: 0.00003 + | > step_time: 2.29690 (2.20251) + | > loader_time: 0.08300 (0.03104) + + + --> STEP: 149/234 -- GLOBAL_STEP: 32675 + | > loss: -0.23724 (-0.11932) + | > log_mle: -0.45716 (-0.27670) + | > loss_dur: 0.21992 (0.15738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.97837 (18.21873) + | > current_lr: 0.00003 + | > step_time: 4.01230 (2.23145) + | > loader_time: 0.09420 (0.03117) + + + --> STEP: 154/234 -- GLOBAL_STEP: 32680 + | > loss: -0.22462 (-0.12267) + | > log_mle: -0.42230 (-0.28170) + | > loss_dur: 0.19768 (0.15903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.05226 (19.20662) + | > current_lr: 0.00003 + | > step_time: 1.31150 (2.28254) + | > loader_time: 0.08750 (0.03210) + + + --> STEP: 159/234 -- GLOBAL_STEP: 32685 + | > loss: -0.22875 (-0.12564) + | > log_mle: -0.44078 (-0.28642) + | > loss_dur: 0.21203 (0.16078) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.18588 (20.41558) + | > current_lr: 0.00003 + | > step_time: 4.00330 (2.28997) + | > loader_time: 0.00370 (0.03182) + + + --> STEP: 164/234 -- GLOBAL_STEP: 32690 + | > loss: -0.21937 (-0.12869) + | > log_mle: -0.43264 (-0.29096) + | > loss_dur: 0.21327 (0.16227) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.30585 (21.43870) + | > current_lr: 0.00003 + | > step_time: 1.50070 (2.31538) + | > loader_time: 0.00500 (0.03365) + + + --> STEP: 169/234 -- GLOBAL_STEP: 32695 + | > loss: -0.20212 (-0.13176) + | > log_mle: -0.42737 (-0.29569) + | > loss_dur: 0.22525 (0.16393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.68652 (22.57884) + | > current_lr: 0.00003 + | > step_time: 11.49080 (2.38354) + | > loader_time: 0.10530 (0.03447) + + + --> STEP: 174/234 -- GLOBAL_STEP: 32700 + | > loss: -0.29225 (-0.13562) + | > log_mle: -0.51140 (-0.30141) + | > loss_dur: 0.21914 (0.16579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.75810 (23.95146) + | > current_lr: 0.00003 + | > step_time: 11.80360 (2.48929) + | > loader_time: 0.19550 (0.03631) + + + --> STEP: 179/234 -- GLOBAL_STEP: 32705 + | > loss: -0.25211 (-0.13880) + | > log_mle: -0.51109 (-0.30663) + | > loss_dur: 0.25898 (0.16783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.60579 (24.97627) + | > current_lr: 0.00003 + | > step_time: 3.57200 (2.53208) + | > loader_time: 0.00350 (0.03690) + + + --> STEP: 184/234 -- GLOBAL_STEP: 32710 + | > loss: -0.24997 (-0.14174) + | > log_mle: -0.48195 (-0.31130) + | > loss_dur: 0.23198 (0.16956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.02908 (26.02569) + | > current_lr: 0.00003 + | > step_time: 2.92550 (2.54117) + | > loader_time: 0.18640 (0.03903) + + + --> STEP: 189/234 -- GLOBAL_STEP: 32715 + | > loss: -0.24414 (-0.14469) + | > log_mle: -0.48037 (-0.31616) + | > loss_dur: 0.23623 (0.17147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.19741 (27.22908) + | > current_lr: 0.00003 + | > step_time: 8.69080 (2.60516) + | > loader_time: 0.11080 (0.03958) + + + --> STEP: 194/234 -- GLOBAL_STEP: 32720 + | > loss: -0.28628 (-0.14831) + | > log_mle: -0.51326 (-0.32108) + | > loss_dur: 0.22698 (0.17278) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.30984 (28.45477) + | > current_lr: 0.00003 + | > step_time: 1.98900 (2.60950) + | > loader_time: 0.00300 (0.03959) + + + --> STEP: 199/234 -- GLOBAL_STEP: 32725 + | > loss: -0.28209 (-0.15134) + | > log_mle: -0.52331 (-0.32568) + | > loss_dur: 0.24122 (0.17434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.12654 (29.61165) + | > current_lr: 0.00003 + | > step_time: 6.41070 (2.63485) + | > loader_time: 0.10060 (0.03979) + + + --> STEP: 204/234 -- GLOBAL_STEP: 32730 + | > loss: -0.26470 (-0.15368) + | > log_mle: -0.52214 (-0.32969) + | > loss_dur: 0.25743 (0.17601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.56073 (31.57755) + | > current_lr: 0.00003 + | > step_time: 4.80900 (2.71151) + | > loader_time: 0.00350 (0.04270) + + + --> STEP: 209/234 -- GLOBAL_STEP: 32735 + | > loss: -0.24593 (-0.15619) + | > log_mle: -0.49075 (-0.33396) + | > loss_dur: 0.24482 (0.17777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.35181 (32.66785) + | > current_lr: 0.00003 + | > step_time: 3.79410 (2.75235) + | > loader_time: 0.00490 (0.04310) + + + --> STEP: 214/234 -- GLOBAL_STEP: 32740 + | > loss: -0.30824 (-0.15976) + | > log_mle: -0.53161 (-0.33917) + | > loss_dur: 0.22337 (0.17941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.82729 (34.09640) + | > current_lr: 0.00003 + | > step_time: 15.19940 (2.91512) + | > loader_time: 0.19850 (0.04440) + + + --> STEP: 219/234 -- GLOBAL_STEP: 32745 + | > loss: -0.37848 (-0.16330) + | > log_mle: -0.63333 (-0.34437) + | > loss_dur: 0.25485 (0.18106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.57306 (35.41132) + | > current_lr: 0.00003 + | > step_time: 4.10850 (2.96590) + | > loader_time: 0.28780 (0.04605) + + + --> STEP: 224/234 -- GLOBAL_STEP: 32750 + | > loss: -0.32449 (-0.16649) + | > log_mle: -0.58336 (-0.34921) + | > loss_dur: 0.25887 (0.18272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.25922 (36.42863) + | > current_lr: 0.00003 + | > step_time: 0.23720 (2.93359) + | > loader_time: 0.00440 (0.04636) + + + --> STEP: 229/234 -- GLOBAL_STEP: 32755 + | > loss: -0.29887 (-0.16989) + | > log_mle: -0.60837 (-0.35469) + | > loss_dur: 0.30950 (0.18480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 138.70331 (37.67819) + | > current_lr: 0.00003 + | > step_time: 0.26430 (2.87511) + | > loader_time: 0.00440 (0.04543) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.37950 (+0.23308) + | > avg_loss: -0.20390 (-0.00167) + | > avg_log_mle: -0.42671 (+0.00696) + | > avg_loss_dur: 0.22281 (-0.00863) + + + > EPOCH: 140/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 14:36:44)  + + --> STEP: 0/234 -- GLOBAL_STEP: 32760 + | > loss: -0.15070 (-0.15070) + | > log_mle: -0.31504 (-0.31504) + | > loss_dur: 0.16434 (0.16434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.44356 (17.44356) + | > current_lr: 0.00004 + | > step_time: 21.10390 (21.10395) + | > loader_time: 12.39320 (12.39324) + + + --> STEP: 5/234 -- GLOBAL_STEP: 32765 + | > loss: -0.11945 (-0.09508) + | > log_mle: -0.24749 (-0.24108) + | > loss_dur: 0.12804 (0.14600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.80320 (15.79530) + | > current_lr: 0.00004 + | > step_time: 10.90100 (4.96422) + | > loader_time: 0.18780 (0.05514) + + + --> STEP: 10/234 -- GLOBAL_STEP: 32770 + | > loss: -0.08061 (-0.10082) + | > log_mle: -0.24004 (-0.24422) + | > loss_dur: 0.15943 (0.14339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.20982 (14.29887) + | > current_lr: 0.00004 + | > step_time: 4.09970 (3.85192) + | > loader_time: 0.09970 (0.03910) + + + --> STEP: 15/234 -- GLOBAL_STEP: 32775 + | > loss: -0.13505 (-0.10604) + | > log_mle: -0.24511 (-0.24482) + | > loss_dur: 0.11007 (0.13878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.57646 (12.46789) + | > current_lr: 0.00004 + | > step_time: 0.81150 (3.62874) + | > loader_time: 0.08400 (0.05065) + + + --> STEP: 20/234 -- GLOBAL_STEP: 32780 + | > loss: -0.11632 (-0.10749) + | > log_mle: -0.23361 (-0.24227) + | > loss_dur: 0.11729 (0.13477) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.06726 (11.32975) + | > current_lr: 0.00004 + | > step_time: 2.01010 (3.22587) + | > loader_time: 0.00400 (0.05729) + + + --> STEP: 25/234 -- GLOBAL_STEP: 32785 + | > loss: -0.11051 (-0.11024) + | > log_mle: -0.22652 (-0.24061) + | > loss_dur: 0.11601 (0.13037) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.44618 (11.08021) + | > current_lr: 0.00004 + | > step_time: 1.32920 (2.81956) + | > loader_time: 0.00220 (0.04624) + + + --> STEP: 30/234 -- GLOBAL_STEP: 32790 + | > loss: -0.14255 (-0.11236) + | > log_mle: -0.24724 (-0.24030) + | > loss_dur: 0.10469 (0.12794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.36302 (10.70815) + | > current_lr: 0.00004 + | > step_time: 1.69740 (2.60149) + | > loader_time: 0.00220 (0.03886) + + + --> STEP: 35/234 -- GLOBAL_STEP: 32795 + | > loss: -0.09496 (-0.11067) + | > log_mle: -0.23912 (-0.24011) + | > loss_dur: 0.14416 (0.12944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.47656 (10.84773) + | > current_lr: 0.00004 + | > step_time: 1.60710 (2.72211) + | > loader_time: 0.00340 (0.04150) + + + --> STEP: 40/234 -- GLOBAL_STEP: 32800 + | > loss: -0.07894 (-0.10901) + | > log_mle: -0.22105 (-0.23956) + | > loss_dur: 0.14211 (0.13054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.47810 (10.91901) + | > current_lr: 0.00004 + | > step_time: 1.30440 (2.54830) + | > loader_time: 0.00260 (0.03658) + + + --> STEP: 45/234 -- GLOBAL_STEP: 32805 + | > loss: -0.09973 (-0.10870) + | > log_mle: -0.26151 (-0.23970) + | > loss_dur: 0.16178 (0.13101) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.69356 (10.70085) + | > current_lr: 0.00004 + | > step_time: 1.21930 (2.50170) + | > loader_time: 0.00200 (0.03506) + + + --> STEP: 50/234 -- GLOBAL_STEP: 32810 + | > loss: -0.09492 (-0.10799) + | > log_mle: -0.22961 (-0.23918) + | > loss_dur: 0.13468 (0.13118) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.36194 (10.42367) + | > current_lr: 0.00004 + | > step_time: 2.00040 (2.43305) + | > loader_time: 0.00350 (0.03179) + + + --> STEP: 55/234 -- GLOBAL_STEP: 32815 + | > loss: -0.12623 (-0.10779) + | > log_mle: -0.24752 (-0.23937) + | > loss_dur: 0.12129 (0.13158) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79496 (10.27752) + | > current_lr: 0.00004 + | > step_time: 1.10720 (2.36239) + | > loader_time: 0.08650 (0.03217) + + + --> STEP: 60/234 -- GLOBAL_STEP: 32820 + | > loss: -0.11905 (-0.10781) + | > log_mle: -0.26192 (-0.23977) + | > loss_dur: 0.14287 (0.13196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.56362 (10.17241) + | > current_lr: 0.00004 + | > step_time: 3.60980 (2.32715) + | > loader_time: 0.00400 (0.03257) + + + --> STEP: 65/234 -- GLOBAL_STEP: 32825 + | > loss: -0.10534 (-0.10710) + | > log_mle: -0.23887 (-0.24032) + | > loss_dur: 0.13353 (0.13322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.62064 (10.08458) + | > current_lr: 0.00004 + | > step_time: 1.21770 (2.25519) + | > loader_time: 0.07500 (0.03284) + + + --> STEP: 70/234 -- GLOBAL_STEP: 32830 + | > loss: -0.07376 (-0.10540) + | > log_mle: -0.23410 (-0.23995) + | > loss_dur: 0.16035 (0.13455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.66470 (10.09446) + | > current_lr: 0.00004 + | > step_time: 2.42300 (2.29044) + | > loader_time: 0.08770 (0.03419) + + + --> STEP: 75/234 -- GLOBAL_STEP: 32835 + | > loss: -0.08483 (-0.10408) + | > log_mle: -0.24951 (-0.24040) + | > loss_dur: 0.16468 (0.13632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.40159 (10.28567) + | > current_lr: 0.00004 + | > step_time: 1.58740 (2.26673) + | > loader_time: 0.00190 (0.03326) + + + --> STEP: 80/234 -- GLOBAL_STEP: 32840 + | > loss: -0.10096 (-0.10381) + | > log_mle: -0.23042 (-0.24045) + | > loss_dur: 0.12946 (0.13664) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.89279 (10.32346) + | > current_lr: 0.00004 + | > step_time: 1.08350 (2.23844) + | > loader_time: 0.00240 (0.03135) + + + --> STEP: 85/234 -- GLOBAL_STEP: 32845 + | > loss: -0.10991 (-0.10372) + | > log_mle: -0.24730 (-0.24098) + | > loss_dur: 0.13738 (0.13726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.57870 (10.44981) + | > current_lr: 0.00004 + | > step_time: 1.80270 (2.23487) + | > loader_time: 0.00240 (0.03064) + + + --> STEP: 90/234 -- GLOBAL_STEP: 32850 + | > loss: -0.10302 (-0.10387) + | > log_mle: -0.27490 (-0.24260) + | > loss_dur: 0.17188 (0.13873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.68396 (10.72169) + | > current_lr: 0.00004 + | > step_time: 1.91200 (2.23753) + | > loader_time: 0.00360 (0.03025) + + + --> STEP: 95/234 -- GLOBAL_STEP: 32855 + | > loss: -0.17686 (-0.10582) + | > log_mle: -0.36204 (-0.24611) + | > loss_dur: 0.18518 (0.14029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.01549 (11.39201) + | > current_lr: 0.00004 + | > step_time: 1.50840 (2.20612) + | > loader_time: 0.09530 (0.02979) + + + --> STEP: 100/234 -- GLOBAL_STEP: 32860 + | > loss: -0.12527 (-0.10633) + | > log_mle: -0.28837 (-0.24790) + | > loss_dur: 0.16311 (0.14157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.16396 (11.60636) + | > current_lr: 0.00004 + | > step_time: 3.80980 (2.20995) + | > loader_time: 0.28880 (0.03213) + + + --> STEP: 105/234 -- GLOBAL_STEP: 32865 + | > loss: -0.11898 (-0.10779) + | > log_mle: -0.26310 (-0.25079) + | > loss_dur: 0.14413 (0.14299) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.88611 (12.06530) + | > current_lr: 0.00004 + | > step_time: 3.29900 (2.22464) + | > loader_time: 0.00330 (0.03168) + + + --> STEP: 110/234 -- GLOBAL_STEP: 32870 + | > loss: -0.11660 (-0.10839) + | > log_mle: -0.28427 (-0.25311) + | > loss_dur: 0.16768 (0.14472) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.33928 (12.87339) + | > current_lr: 0.00004 + | > step_time: 3.42040 (2.22273) + | > loader_time: 0.08590 (0.03192) + + + --> STEP: 115/234 -- GLOBAL_STEP: 32875 + | > loss: -0.12325 (-0.10952) + | > log_mle: -0.30898 (-0.25616) + | > loss_dur: 0.18573 (0.14664) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.05846 (13.55312) + | > current_lr: 0.00004 + | > step_time: 1.80330 (2.20063) + | > loader_time: 0.00290 (0.03064) + + + --> STEP: 120/234 -- GLOBAL_STEP: 32880 + | > loss: -0.17233 (-0.11045) + | > log_mle: -0.35550 (-0.25881) + | > loss_dur: 0.18317 (0.14836) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.41003 (14.06449) + | > current_lr: 0.00004 + | > step_time: 2.40270 (2.19834) + | > loader_time: 0.08480 (0.03225) + + + --> STEP: 125/234 -- GLOBAL_STEP: 32885 + | > loss: -0.15111 (-0.11094) + | > log_mle: -0.34228 (-0.26026) + | > loss_dur: 0.19117 (0.14931) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.58117 (14.42020) + | > current_lr: 0.00004 + | > step_time: 1.50240 (2.17527) + | > loader_time: 0.08910 (0.03315) + + + --> STEP: 130/234 -- GLOBAL_STEP: 32890 + | > loss: -0.15521 (-0.11266) + | > log_mle: -0.35461 (-0.26354) + | > loss_dur: 0.19940 (0.15088) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.60787 (15.17118) + | > current_lr: 0.00004 + | > step_time: 2.90460 (2.18164) + | > loader_time: 0.00350 (0.03401) + + + --> STEP: 135/234 -- GLOBAL_STEP: 32895 + | > loss: -0.11641 (-0.11445) + | > log_mle: -0.28726 (-0.26681) + | > loss_dur: 0.17085 (0.15236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.17586 (15.90489) + | > current_lr: 0.00004 + | > step_time: 2.80580 (2.18984) + | > loader_time: 0.00760 (0.03407) + + + --> STEP: 140/234 -- GLOBAL_STEP: 32900 + | > loss: -0.12296 (-0.11640) + | > log_mle: -0.31856 (-0.27063) + | > loss_dur: 0.19560 (0.15423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.33385 (16.90281) + | > current_lr: 0.00004 + | > step_time: 6.30540 (2.22457) + | > loader_time: 0.08770 (0.03612) + + + --> STEP: 145/234 -- GLOBAL_STEP: 32905 + | > loss: -0.21282 (-0.11873) + | > log_mle: -0.42339 (-0.27506) + | > loss_dur: 0.21057 (0.15634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.95921 (17.90094) + | > current_lr: 0.00004 + | > step_time: 1.30590 (2.27126) + | > loader_time: 0.00340 (0.03880) + + + --> STEP: 150/234 -- GLOBAL_STEP: 32910 + | > loss: -0.19383 (-0.12122) + | > log_mle: -0.40326 (-0.27916) + | > loss_dur: 0.20943 (0.15794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.32841 (18.79948) + | > current_lr: 0.00004 + | > step_time: 5.71300 (2.34628) + | > loader_time: 0.19180 (0.03999) + + + --> STEP: 155/234 -- GLOBAL_STEP: 32915 + | > loss: -0.24325 (-0.12469) + | > log_mle: -0.46281 (-0.28420) + | > loss_dur: 0.21957 (0.15952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.66899 (19.98251) + | > current_lr: 0.00004 + | > step_time: 3.71300 (2.35026) + | > loader_time: 0.07910 (0.04171) + + + --> STEP: 160/234 -- GLOBAL_STEP: 32920 + | > loss: -0.23666 (-0.12741) + | > log_mle: -0.46049 (-0.28867) + | > loss_dur: 0.22383 (0.16126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.63137 (20.95407) + | > current_lr: 0.00004 + | > step_time: 1.90260 (2.34107) + | > loader_time: 0.00470 (0.04291) + + + --> STEP: 165/234 -- GLOBAL_STEP: 32925 + | > loss: -0.23726 (-0.13038) + | > log_mle: -0.46437 (-0.29310) + | > loss_dur: 0.22711 (0.16271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.27201 (21.76242) + | > current_lr: 0.00004 + | > step_time: 2.70480 (2.35864) + | > loader_time: 0.00400 (0.04456) + + + --> STEP: 170/234 -- GLOBAL_STEP: 32930 + | > loss: -0.25086 (-0.13347) + | > log_mle: -0.49850 (-0.29788) + | > loss_dur: 0.24764 (0.16441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.27499 (22.93198) + | > current_lr: 0.00004 + | > step_time: 5.28950 (2.43692) + | > loader_time: 0.09910 (0.04565) + + + --> STEP: 175/234 -- GLOBAL_STEP: 32935 + | > loss: -0.23216 (-0.13706) + | > log_mle: -0.47661 (-0.30338) + | > loss_dur: 0.24445 (0.16632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.84968 (24.23794) + | > current_lr: 0.00004 + | > step_time: 3.19380 (2.44687) + | > loader_time: 0.00410 (0.04499) + + + --> STEP: 180/234 -- GLOBAL_STEP: 32940 + | > loss: -0.24845 (-0.14017) + | > log_mle: -0.48270 (-0.30849) + | > loss_dur: 0.23426 (0.16832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.00451 (25.48092) + | > current_lr: 0.00004 + | > step_time: 3.80690 (2.49575) + | > loader_time: 0.00450 (0.04643) + + + --> STEP: 185/234 -- GLOBAL_STEP: 32945 + | > loss: -0.21885 (-0.14191) + | > log_mle: -0.47090 (-0.31216) + | > loss_dur: 0.25205 (0.17024) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.52523 (27.62221) + | > current_lr: 0.00004 + | > step_time: 4.39960 (2.55649) + | > loader_time: 0.09300 (0.04624) + + + --> STEP: 190/234 -- GLOBAL_STEP: 32950 + | > loss: -0.25883 (-0.14439) + | > log_mle: -0.47525 (-0.31635) + | > loss_dur: 0.21642 (0.17195) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.42594 (28.39812) + | > current_lr: 0.00004 + | > step_time: 9.00170 (2.68185) + | > loader_time: 0.09730 (0.04649) + + + --> STEP: 195/234 -- GLOBAL_STEP: 32955 + | > loss: -0.25272 (-0.14758) + | > log_mle: -0.49798 (-0.32099) + | > loss_dur: 0.24526 (0.17341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.79220 (29.48102) + | > current_lr: 0.00004 + | > step_time: 8.30440 (2.74024) + | > loader_time: 0.19260 (0.04836) + + + --> STEP: 200/234 -- GLOBAL_STEP: 32960 + | > loss: -0.22058 (-0.14988) + | > log_mle: -0.48044 (-0.32492) + | > loss_dur: 0.25986 (0.17504) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.87277 (31.19710) + | > current_lr: 0.00004 + | > step_time: 5.30660 (2.75724) + | > loader_time: 0.09790 (0.05001) + + + --> STEP: 205/234 -- GLOBAL_STEP: 32965 + | > loss: -0.24194 (-0.15215) + | > log_mle: -0.48059 (-0.32880) + | > loss_dur: 0.23865 (0.17665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.26451 (31.83992) + | > current_lr: 0.00004 + | > step_time: 7.90890 (2.79889) + | > loader_time: 0.00350 (0.04979) + + + --> STEP: 210/234 -- GLOBAL_STEP: 32970 + | > loss: -0.31719 (-0.15518) + | > log_mle: -0.56630 (-0.33354) + | > loss_dur: 0.24911 (0.17836) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.94659 (32.85275) + | > current_lr: 0.00004 + | > step_time: 11.29570 (2.90369) + | > loader_time: 0.00410 (0.05181) + + + --> STEP: 215/234 -- GLOBAL_STEP: 32975 + | > loss: -0.27929 (-0.15863) + | > log_mle: -0.52177 (-0.33858) + | > loss_dur: 0.24248 (0.17996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.89000 (34.03627) + | > current_lr: 0.00004 + | > step_time: 6.40170 (2.95943) + | > loader_time: 0.09170 (0.05381) + + + --> STEP: 220/234 -- GLOBAL_STEP: 32980 + | > loss: -0.32143 (-0.16245) + | > log_mle: -0.57622 (-0.34413) + | > loss_dur: 0.25478 (0.18168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.32581 (34.90208) + | > current_lr: 0.00004 + | > step_time: 2.10170 (2.96455) + | > loader_time: 0.08990 (0.05385) + + + --> STEP: 225/234 -- GLOBAL_STEP: 32985 + | > loss: -0.35752 (-0.16590) + | > log_mle: -0.62940 (-0.34931) + | > loss_dur: 0.27188 (0.18340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.78059 (36.10375) + | > current_lr: 0.00004 + | > step_time: 0.25440 (2.92531) + | > loader_time: 0.00380 (0.05311) + + + --> STEP: 230/234 -- GLOBAL_STEP: 32990 + | > loss: -0.32559 (-0.16905) + | > log_mle: -0.66842 (-0.35490) + | > loss_dur: 0.34283 (0.18585) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 159.72778 (37.53184) + | > current_lr: 0.00004 + | > step_time: 0.28270 (2.86731) + | > loader_time: 0.00380 (0.05204) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.10225 (-0.27725) + | > avg_loss: -0.22135 (-0.01745) + | > avg_log_mle: -0.44667 (-0.01996) + | > avg_loss_dur: 0.22532 (+0.00251) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_32994.pth + + > EPOCH: 141/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 14:49:18)  + + --> STEP: 1/234 -- GLOBAL_STEP: 32995 + | > loss: -0.11980 (-0.11980) + | > log_mle: -0.24672 (-0.24672) + | > loss_dur: 0.12692 (0.12692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.92992 (18.92992) + | > current_lr: 0.00004 + | > step_time: 8.09400 (8.09396) + | > loader_time: 0.10750 (0.10746) + + + --> STEP: 6/234 -- GLOBAL_STEP: 33000 + | > loss: -0.11270 (-0.09980) + | > log_mle: -0.23437 (-0.24253) + | > loss_dur: 0.12167 (0.14273) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.13330 (14.76073) + | > current_lr: 0.00004 + | > step_time: 9.19290 (6.50136) + | > loader_time: 0.00170 (0.03414) + + + --> STEP: 11/234 -- GLOBAL_STEP: 33005 + | > loss: -0.13832 (-0.10722) + | > log_mle: -0.24519 (-0.24644) + | > loss_dur: 0.10687 (0.13922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.56455 (13.41956) + | > current_lr: 0.00004 + | > step_time: 2.81710 (4.45369) + | > loader_time: 0.00830 (0.02780) + + + --> STEP: 16/234 -- GLOBAL_STEP: 33010 + | > loss: -0.14571 (-0.11196) + | > log_mle: -0.24172 (-0.24642) + | > loss_dur: 0.09601 (0.13446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.19503 (12.48141) + | > current_lr: 0.00004 + | > step_time: 1.62280 (3.50661) + | > loader_time: 0.00220 (0.02596) + + + --> STEP: 21/234 -- GLOBAL_STEP: 33015 + | > loss: -0.10761 (-0.11142) + | > log_mle: -0.22147 (-0.24278) + | > loss_dur: 0.11386 (0.13135) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.02868 (11.74653) + | > current_lr: 0.00004 + | > step_time: 1.09770 (3.03790) + | > loader_time: 0.00140 (0.02032) + + + --> STEP: 26/234 -- GLOBAL_STEP: 33020 + | > loss: -0.10440 (-0.11411) + | > log_mle: -0.24000 (-0.24218) + | > loss_dur: 0.13560 (0.12807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.35065 (11.29715) + | > current_lr: 0.00004 + | > step_time: 1.11820 (2.65895) + | > loader_time: 0.00180 (0.01678) + + + --> STEP: 31/234 -- GLOBAL_STEP: 33025 + | > loss: -0.08072 (-0.11504) + | > log_mle: -0.24003 (-0.24201) + | > loss_dur: 0.15930 (0.12697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.40441 (10.83683) + | > current_lr: 0.00004 + | > step_time: 1.07010 (2.51599) + | > loader_time: 0.00210 (0.01454) + + + --> STEP: 36/234 -- GLOBAL_STEP: 33030 + | > loss: -0.08640 (-0.11426) + | > log_mle: -0.24059 (-0.24199) + | > loss_dur: 0.15418 (0.12773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.43338 (10.53715) + | > current_lr: 0.00004 + | > step_time: 0.80590 (2.32658) + | > loader_time: 0.00430 (0.01288) + + + --> STEP: 41/234 -- GLOBAL_STEP: 33035 + | > loss: -0.11993 (-0.11352) + | > log_mle: -0.24080 (-0.24167) + | > loss_dur: 0.12087 (0.12815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.15939 (10.36921) + | > current_lr: 0.00004 + | > step_time: 1.11500 (2.19575) + | > loader_time: 0.00210 (0.01357) + + + --> STEP: 46/234 -- GLOBAL_STEP: 33040 + | > loss: -0.09909 (-0.11309) + | > log_mle: -0.23953 (-0.24186) + | > loss_dur: 0.14044 (0.12877) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.40794 (10.36332) + | > current_lr: 0.00004 + | > step_time: 0.94190 (2.09663) + | > loader_time: 0.00200 (0.01411) + + + --> STEP: 51/234 -- GLOBAL_STEP: 33045 + | > loss: -0.09849 (-0.11250) + | > log_mle: -0.22888 (-0.24125) + | > loss_dur: 0.13039 (0.12875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.17400 (9.98738) + | > current_lr: 0.00004 + | > step_time: 1.44830 (2.04966) + | > loader_time: 0.00220 (0.01297) + + + --> STEP: 56/234 -- GLOBAL_STEP: 33050 + | > loss: -0.09061 (-0.11228) + | > log_mle: -0.24588 (-0.24173) + | > loss_dur: 0.15527 (0.12944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.30802 (9.81846) + | > current_lr: 0.00004 + | > step_time: 1.45450 (2.01966) + | > loader_time: 0.00220 (0.01391) + + + --> STEP: 61/234 -- GLOBAL_STEP: 33055 + | > loss: -0.11977 (-0.11251) + | > log_mle: -0.24322 (-0.24199) + | > loss_dur: 0.12345 (0.12948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.13646 (9.91008) + | > current_lr: 0.00004 + | > step_time: 1.41450 (1.98006) + | > loader_time: 0.08280 (0.01435) + + + --> STEP: 66/234 -- GLOBAL_STEP: 33060 + | > loss: -0.10179 (-0.11150) + | > log_mle: -0.23112 (-0.24224) + | > loss_dur: 0.12933 (0.13075) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.05905 (9.92265) + | > current_lr: 0.00004 + | > step_time: 1.60350 (1.97368) + | > loader_time: 0.00270 (0.01349) + + + --> STEP: 71/234 -- GLOBAL_STEP: 33065 + | > loss: -0.09092 (-0.10962) + | > log_mle: -0.27105 (-0.24236) + | > loss_dur: 0.18014 (0.13274) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.23377 (10.07661) + | > current_lr: 0.00004 + | > step_time: 2.30720 (2.00275) + | > loader_time: 0.00290 (0.01389) + + + --> STEP: 76/234 -- GLOBAL_STEP: 33070 + | > loss: -0.10759 (-0.10851) + | > log_mle: -0.25240 (-0.24253) + | > loss_dur: 0.14481 (0.13402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.19184 (10.21874) + | > current_lr: 0.00004 + | > step_time: 1.40260 (1.98108) + | > loader_time: 0.08690 (0.01535) + + + --> STEP: 81/234 -- GLOBAL_STEP: 33075 + | > loss: -0.11241 (-0.10834) + | > log_mle: -0.26004 (-0.24263) + | > loss_dur: 0.14762 (0.13429) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.48234 (10.19298) + | > current_lr: 0.00004 + | > step_time: 2.80680 (2.00940) + | > loader_time: 0.09270 (0.02003) + + + --> STEP: 86/234 -- GLOBAL_STEP: 33080 + | > loss: -0.09999 (-0.10805) + | > log_mle: -0.25548 (-0.24302) + | > loss_dur: 0.15549 (0.13497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.65373 (10.27229) + | > current_lr: 0.00004 + | > step_time: 1.71090 (2.00750) + | > loader_time: 0.09210 (0.02011) + + + --> STEP: 91/234 -- GLOBAL_STEP: 33085 + | > loss: -0.09878 (-0.10813) + | > log_mle: -0.27005 (-0.24460) + | > loss_dur: 0.17127 (0.13648) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.04978 (10.55637) + | > current_lr: 0.00004 + | > step_time: 1.18240 (2.00364) + | > loader_time: 0.00270 (0.02009) + + + --> STEP: 96/234 -- GLOBAL_STEP: 33090 + | > loss: -0.10839 (-0.10970) + | > log_mle: -0.25732 (-0.24781) + | > loss_dur: 0.14893 (0.13811) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.70528 (11.25773) + | > current_lr: 0.00004 + | > step_time: 2.60330 (2.03685) + | > loader_time: 0.00320 (0.02022) + + + --> STEP: 101/234 -- GLOBAL_STEP: 33095 + | > loss: -0.14106 (-0.11068) + | > log_mle: -0.31444 (-0.25002) + | > loss_dur: 0.17338 (0.13934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.78498 (11.72236) + | > current_lr: 0.00004 + | > step_time: 1.55780 (2.02392) + | > loader_time: 0.00240 (0.01937) + + + --> STEP: 106/234 -- GLOBAL_STEP: 33100 + | > loss: -0.11083 (-0.11156) + | > log_mle: -0.31350 (-0.25271) + | > loss_dur: 0.20266 (0.14115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.30498 (12.22854) + | > current_lr: 0.00004 + | > step_time: 1.50680 (1.99805) + | > loader_time: 0.08120 (0.02083) + + + --> STEP: 111/234 -- GLOBAL_STEP: 33105 + | > loss: -0.14394 (-0.11224) + | > log_mle: -0.36170 (-0.25547) + | > loss_dur: 0.21775 (0.14323) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.04454 (12.77546) + | > current_lr: 0.00004 + | > step_time: 1.40220 (1.97944) + | > loader_time: 0.08890 (0.02237) + + + --> STEP: 116/234 -- GLOBAL_STEP: 33110 + | > loss: -0.11975 (-0.11321) + | > log_mle: -0.32827 (-0.25825) + | > loss_dur: 0.20852 (0.14504) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.68663 (13.35765) + | > current_lr: 0.00004 + | > step_time: 1.39730 (1.98041) + | > loader_time: 0.08880 (0.02380) + + + --> STEP: 121/234 -- GLOBAL_STEP: 33115 + | > loss: -0.07711 (-0.11391) + | > log_mle: -0.24110 (-0.26020) + | > loss_dur: 0.16399 (0.14628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.96938 (13.65613) + | > current_lr: 0.00004 + | > step_time: 2.70570 (1.97541) + | > loader_time: 0.00400 (0.02516) + + + --> STEP: 126/234 -- GLOBAL_STEP: 33120 + | > loss: -0.18231 (-0.11514) + | > log_mle: -0.37827 (-0.26278) + | > loss_dur: 0.19595 (0.14764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.72876 (14.23115) + | > current_lr: 0.00004 + | > step_time: 1.71250 (1.96628) + | > loader_time: 0.08320 (0.02611) + + + --> STEP: 131/234 -- GLOBAL_STEP: 33125 + | > loss: -0.20759 (-0.11705) + | > log_mle: -0.42260 (-0.26645) + | > loss_dur: 0.21501 (0.14939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.75531 (15.03185) + | > current_lr: 0.00004 + | > step_time: 4.02320 (1.97768) + | > loader_time: 0.09010 (0.02657) + + + --> STEP: 136/234 -- GLOBAL_STEP: 33130 + | > loss: -0.23617 (-0.11895) + | > log_mle: -0.46386 (-0.27004) + | > loss_dur: 0.22769 (0.15109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.61701 (15.75445) + | > current_lr: 0.00004 + | > step_time: 5.88820 (2.00209) + | > loader_time: 0.10700 (0.02770) + + + --> STEP: 141/234 -- GLOBAL_STEP: 33135 + | > loss: -0.17233 (-0.12024) + | > log_mle: -0.37290 (-0.27309) + | > loss_dur: 0.20057 (0.15285) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.62854 (16.58314) + | > current_lr: 0.00004 + | > step_time: 2.29880 (2.04604) + | > loader_time: 0.00380 (0.02882) + + + --> STEP: 146/234 -- GLOBAL_STEP: 33140 + | > loss: -0.21770 (-0.12296) + | > log_mle: -0.42507 (-0.27792) + | > loss_dur: 0.20736 (0.15496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.79059 (17.68187) + | > current_lr: 0.00004 + | > step_time: 4.50880 (2.05219) + | > loader_time: 0.19850 (0.03037) + + + --> STEP: 151/234 -- GLOBAL_STEP: 33145 + | > loss: -0.19833 (-0.12554) + | > log_mle: -0.39240 (-0.28194) + | > loss_dur: 0.19407 (0.15640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.59545 (18.47374) + | > current_lr: 0.00004 + | > step_time: 8.11010 (2.13458) + | > loader_time: 0.08670 (0.03201) + + + --> STEP: 156/234 -- GLOBAL_STEP: 33150 + | > loss: -0.22073 (-0.12925) + | > log_mle: -0.43267 (-0.28746) + | > loss_dur: 0.21194 (0.15822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.47968 (19.76518) + | > current_lr: 0.00004 + | > step_time: 1.09630 (2.19058) + | > loader_time: 0.08160 (0.03326) + + + --> STEP: 161/234 -- GLOBAL_STEP: 33155 + | > loss: -0.24853 (-0.13213) + | > log_mle: -0.45481 (-0.29210) + | > loss_dur: 0.20628 (0.15997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.51278 (20.85620) + | > current_lr: 0.00004 + | > step_time: 1.40750 (2.17234) + | > loader_time: 0.10010 (0.03346) + + + --> STEP: 166/234 -- GLOBAL_STEP: 33160 + | > loss: -0.20387 (-0.13459) + | > log_mle: -0.39472 (-0.29602) + | > loss_dur: 0.19085 (0.16142) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.10602 (21.90696) + | > current_lr: 0.00004 + | > step_time: 1.20120 (2.19851) + | > loader_time: 0.00680 (0.03421) + + + --> STEP: 171/234 -- GLOBAL_STEP: 33165 + | > loss: -0.29095 (-0.13805) + | > log_mle: -0.51002 (-0.30143) + | > loss_dur: 0.21907 (0.16338) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.15144 (23.31956) + | > current_lr: 0.00004 + | > step_time: 1.99280 (2.19038) + | > loader_time: 0.09670 (0.03435) + + + --> STEP: 176/234 -- GLOBAL_STEP: 33170 + | > loss: -0.25672 (-0.14136) + | > log_mle: -0.47963 (-0.30670) + | > loss_dur: 0.22291 (0.16535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.37720 (24.64803) + | > current_lr: 0.00004 + | > step_time: 2.71950 (2.22949) + | > loader_time: 0.08360 (0.03720) + + + --> STEP: 181/234 -- GLOBAL_STEP: 33175 + | > loss: -0.19581 (-0.14416) + | > log_mle: -0.41575 (-0.31151) + | > loss_dur: 0.21994 (0.16735) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.04740 (25.65848) + | > current_lr: 0.00004 + | > step_time: 8.80310 (2.28887) + | > loader_time: 0.08480 (0.03782) + + + --> STEP: 186/234 -- GLOBAL_STEP: 33180 + | > loss: -0.20680 (-0.14696) + | > log_mle: -0.45175 (-0.31636) + | > loss_dur: 0.24495 (0.16939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.84375 (26.93618) + | > current_lr: 0.00004 + | > step_time: 1.59430 (2.30850) + | > loader_time: 0.00340 (0.03743) + + + --> STEP: 191/234 -- GLOBAL_STEP: 33185 + | > loss: -0.26811 (-0.15011) + | > log_mle: -0.47811 (-0.32107) + | > loss_dur: 0.21000 (0.17095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.39663 (27.87292) + | > current_lr: 0.00004 + | > step_time: 6.39140 (2.35551) + | > loader_time: 0.19380 (0.03851) + + + --> STEP: 196/234 -- GLOBAL_STEP: 33190 + | > loss: -0.22606 (-0.15332) + | > log_mle: -0.46667 (-0.32586) + | > loss_dur: 0.24061 (0.17254) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.07606 (29.08690) + | > current_lr: 0.00004 + | > step_time: 4.29830 (2.44543) + | > loader_time: 0.00440 (0.03947) + + + --> STEP: 201/234 -- GLOBAL_STEP: 33195 + | > loss: -0.18096 (-0.15591) + | > log_mle: -0.43106 (-0.33017) + | > loss_dur: 0.25010 (0.17425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.80586 (30.24717) + | > current_lr: 0.00004 + | > step_time: 1.59650 (2.43069) + | > loader_time: 0.00390 (0.03951) + + + --> STEP: 206/234 -- GLOBAL_STEP: 33200 + | > loss: -0.28758 (-0.15916) + | > log_mle: -0.53010 (-0.33500) + | > loss_dur: 0.24253 (0.17583) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.37838 (31.47417) + | > current_lr: 0.00004 + | > step_time: 2.40190 (2.46110) + | > loader_time: 0.00760 (0.03911) + + + --> STEP: 211/234 -- GLOBAL_STEP: 33205 + | > loss: -0.32680 (-0.16264) + | > log_mle: -0.59670 (-0.34024) + | > loss_dur: 0.26990 (0.17761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.51611 (32.95732) + | > current_lr: 0.00004 + | > step_time: 7.30000 (2.50406) + | > loader_time: 0.11100 (0.03930) + + + --> STEP: 216/234 -- GLOBAL_STEP: 33210 + | > loss: -0.30882 (-0.16598) + | > log_mle: -0.57877 (-0.34523) + | > loss_dur: 0.26994 (0.17925) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.49452 (34.41892) + | > current_lr: 0.00004 + | > step_time: 9.20790 (2.59508) + | > loader_time: 0.00450 (0.04174) + + + --> STEP: 221/234 -- GLOBAL_STEP: 33215 + | > loss: -0.27071 (-0.16941) + | > log_mle: -0.50796 (-0.35025) + | > loss_dur: 0.23726 (0.18085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.33944 (35.78946) + | > current_lr: 0.00004 + | > step_time: 2.31300 (2.60736) + | > loader_time: 0.09680 (0.04132) + + + --> STEP: 226/234 -- GLOBAL_STEP: 33220 + | > loss: -0.34576 (-0.17312) + | > log_mle: -0.60716 (-0.35580) + | > loss_dur: 0.26140 (0.18267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.14444 (37.39250) + | > current_lr: 0.00004 + | > step_time: 0.47730 (2.58188) + | > loader_time: 0.00540 (0.04122) + + + --> STEP: 231/234 -- GLOBAL_STEP: 33225 + | > loss: -0.26906 (-0.17579) + | > log_mle: -0.67806 (-0.36158) + | > loss_dur: 0.40900 (0.18579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.18245 (39.04229) + | > current_lr: 0.00004 + | > step_time: 0.27050 (2.53163) + | > loader_time: 0.00520 (0.04042) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00313 (-0.09912) + | > avg_loss: -0.20040 (+0.02095) + | > avg_log_mle: -0.42728 (+0.01939) + | > avg_loss_dur: 0.22688 (+0.00156) + + + > EPOCH: 142/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 15:00:23)  + + --> STEP: 2/234 -- GLOBAL_STEP: 33230 + | > loss: -0.10790 (-0.11595) + | > log_mle: -0.24612 (-0.24600) + | > loss_dur: 0.13822 (0.13005) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.62541 (14.93567) + | > current_lr: 0.00004 + | > step_time: 2.79810 (5.09373) + | > loader_time: 0.00230 (0.00368) + + + --> STEP: 7/234 -- GLOBAL_STEP: 33235 + | > loss: -0.12684 (-0.09628) + | > log_mle: -0.25200 (-0.24229) + | > loss_dur: 0.12516 (0.14601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.91752 (21.43508) + | > current_lr: 0.00004 + | > step_time: 10.98410 (4.98342) + | > loader_time: 0.00120 (0.05477) + + + --> STEP: 12/234 -- GLOBAL_STEP: 33240 + | > loss: -0.10648 (-0.10280) + | > log_mle: -0.24393 (-0.24446) + | > loss_dur: 0.13745 (0.14166) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.04597 (17.54596) + | > current_lr: 0.00004 + | > step_time: 1.57290 (4.18047) + | > loader_time: 0.00210 (0.04879) + + + --> STEP: 17/234 -- GLOBAL_STEP: 33245 + | > loss: -0.09888 (-0.10777) + | > log_mle: -0.22509 (-0.24357) + | > loss_dur: 0.12621 (0.13580) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.31600 (14.74939) + | > current_lr: 0.00004 + | > step_time: 1.07860 (3.45945) + | > loader_time: 0.00120 (0.03491) + + + --> STEP: 22/234 -- GLOBAL_STEP: 33250 + | > loss: -0.12148 (-0.10909) + | > log_mle: -0.24586 (-0.24156) + | > loss_dur: 0.12438 (0.13247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.94737 (13.37973) + | > current_lr: 0.00004 + | > step_time: 1.31540 (2.95144) + | > loader_time: 0.08360 (0.03110) + + + --> STEP: 27/234 -- GLOBAL_STEP: 33255 + | > loss: -0.13331 (-0.11197) + | > log_mle: -0.24899 (-0.24127) + | > loss_dur: 0.11569 (0.12930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.82785 (12.33451) + | > current_lr: 0.00004 + | > step_time: 0.91240 (2.66383) + | > loader_time: 0.00270 (0.02876) + + + --> STEP: 32/234 -- GLOBAL_STEP: 33260 + | > loss: -0.15014 (-0.11363) + | > log_mle: -0.25333 (-0.24138) + | > loss_dur: 0.10320 (0.12775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.20610 (11.69267) + | > current_lr: 0.00004 + | > step_time: 1.80550 (2.59725) + | > loader_time: 0.08610 (0.03251) + + + --> STEP: 37/234 -- GLOBAL_STEP: 33265 + | > loss: -0.12525 (-0.11220) + | > log_mle: -0.23565 (-0.24088) + | > loss_dur: 0.11040 (0.12868) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.75288 (11.51385) + | > current_lr: 0.00004 + | > step_time: 0.82020 (2.38437) + | > loader_time: 0.00210 (0.02840) + + + --> STEP: 42/234 -- GLOBAL_STEP: 33270 + | > loss: -0.10424 (-0.11139) + | > log_mle: -0.22838 (-0.24053) + | > loss_dur: 0.12413 (0.12914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.53761 (11.27534) + | > current_lr: 0.00004 + | > step_time: 1.93140 (2.35297) + | > loader_time: 0.00200 (0.02983) + + + --> STEP: 47/234 -- GLOBAL_STEP: 33275 + | > loss: -0.08330 (-0.11026) + | > log_mle: -0.23656 (-0.24088) + | > loss_dur: 0.15326 (0.13063) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.93269 (11.17755) + | > current_lr: 0.00004 + | > step_time: 1.90060 (2.36817) + | > loader_time: 0.00240 (0.02885) + + + --> STEP: 52/234 -- GLOBAL_STEP: 33280 + | > loss: -0.09043 (-0.10958) + | > log_mle: -0.23367 (-0.24023) + | > loss_dur: 0.14324 (0.13066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.68565 (10.83730) + | > current_lr: 0.00004 + | > step_time: 1.69880 (2.28612) + | > loader_time: 0.00270 (0.02782) + + + --> STEP: 57/234 -- GLOBAL_STEP: 33285 + | > loss: -0.08783 (-0.10955) + | > log_mle: -0.22476 (-0.24056) + | > loss_dur: 0.13693 (0.13101) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.63215 (10.60400) + | > current_lr: 0.00004 + | > step_time: 1.19240 (2.27577) + | > loader_time: 0.00200 (0.03086) + + + --> STEP: 62/234 -- GLOBAL_STEP: 33290 + | > loss: -0.06296 (-0.10957) + | > log_mle: -0.27880 (-0.24183) + | > loss_dur: 0.21584 (0.13227) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.71756 (10.72970) + | > current_lr: 0.00004 + | > step_time: 0.90410 (2.23168) + | > loader_time: 0.00290 (0.02998) + + + --> STEP: 67/234 -- GLOBAL_STEP: 33295 + | > loss: -0.10211 (-0.10921) + | > log_mle: -0.26101 (-0.24189) + | > loss_dur: 0.15890 (0.13267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.14051 (10.50976) + | > current_lr: 0.00004 + | > step_time: 0.91510 (2.16534) + | > loader_time: 0.08470 (0.02914) + + + --> STEP: 72/234 -- GLOBAL_STEP: 33300 + | > loss: -0.08816 (-0.10734) + | > log_mle: -0.23721 (-0.24181) + | > loss_dur: 0.14905 (0.13446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.77574 (10.58717) + | > current_lr: 0.00004 + | > step_time: 1.59210 (2.19554) + | > loader_time: 0.00280 (0.02733) + + + --> STEP: 77/234 -- GLOBAL_STEP: 33305 + | > loss: -0.11498 (-0.10665) + | > log_mle: -0.25027 (-0.24222) + | > loss_dur: 0.13529 (0.13557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.05220 (10.59764) + | > current_lr: 0.00004 + | > step_time: 1.00320 (2.14539) + | > loader_time: 0.00260 (0.02678) + + + --> STEP: 82/234 -- GLOBAL_STEP: 33310 + | > loss: -0.09933 (-0.10634) + | > log_mle: -0.23941 (-0.24225) + | > loss_dur: 0.14008 (0.13591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.22426 (10.50086) + | > current_lr: 0.00004 + | > step_time: 1.50260 (2.12932) + | > loader_time: 0.10040 (0.02850) + + + --> STEP: 87/234 -- GLOBAL_STEP: 33315 + | > loss: -0.08709 (-0.10598) + | > log_mle: -0.24937 (-0.24282) + | > loss_dur: 0.16227 (0.13684) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99941 (10.59963) + | > current_lr: 0.00004 + | > step_time: 1.39610 (2.10811) + | > loader_time: 0.00310 (0.02904) + + + --> STEP: 92/234 -- GLOBAL_STEP: 33320 + | > loss: -0.15052 (-0.10699) + | > log_mle: -0.29816 (-0.24508) + | > loss_dur: 0.14764 (0.13809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.51041 (10.84183) + | > current_lr: 0.00004 + | > step_time: 2.51530 (2.12542) + | > loader_time: 0.17590 (0.03042) + + + --> STEP: 97/234 -- GLOBAL_STEP: 33325 + | > loss: -0.12416 (-0.10872) + | > log_mle: -0.28529 (-0.24827) + | > loss_dur: 0.16113 (0.13956) + | > amp_scaler: 4096.00000 (2132.45361) + | > grad_norm: 17.70044 (11.32213) + | > current_lr: 0.00004 + | > step_time: 1.50510 (2.12821) + | > loader_time: 0.00330 (0.02987) + + + --> STEP: 102/234 -- GLOBAL_STEP: 33330 + | > loss: -0.09693 (-0.10933) + | > log_mle: -0.26624 (-0.25041) + | > loss_dur: 0.16930 (0.14108) + | > amp_scaler: 4096.00000 (2228.70588) + | > grad_norm: 16.34811 (11.82622) + | > current_lr: 0.00004 + | > step_time: 3.46160 (2.22861) + | > loader_time: 0.00230 (0.03047) + + + --> STEP: 107/234 -- GLOBAL_STEP: 33335 + | > loss: -0.13134 (-0.11073) + | > log_mle: -0.31452 (-0.25371) + | > loss_dur: 0.18319 (0.14297) + | > amp_scaler: 4096.00000 (2315.96262) + | > grad_norm: 25.57726 (12.47291) + | > current_lr: 0.00004 + | > step_time: 2.60230 (2.21129) + | > loader_time: 0.00320 (0.03005) + + + --> STEP: 112/234 -- GLOBAL_STEP: 33340 + | > loss: -0.12884 (-0.11160) + | > log_mle: -0.32655 (-0.25667) + | > loss_dur: 0.19771 (0.14507) + | > amp_scaler: 4096.00000 (2395.42857) + | > grad_norm: 26.15681 (13.13467) + | > current_lr: 0.00004 + | > step_time: 4.49810 (2.21080) + | > loader_time: 0.09060 (0.03110) + + + --> STEP: 117/234 -- GLOBAL_STEP: 33345 + | > loss: -0.14373 (-0.11268) + | > log_mle: -0.32172 (-0.25950) + | > loss_dur: 0.17799 (0.14682) + | > amp_scaler: 4096.00000 (2468.10256) + | > grad_norm: 22.88542 (13.65593) + | > current_lr: 0.00004 + | > step_time: 3.13250 (2.20015) + | > loader_time: 0.00290 (0.03055) + + + --> STEP: 122/234 -- GLOBAL_STEP: 33350 + | > loss: -0.12380 (-0.11319) + | > log_mle: -0.29533 (-0.26128) + | > loss_dur: 0.17153 (0.14809) + | > amp_scaler: 4096.00000 (2534.81967) + | > grad_norm: 19.73084 (13.98131) + | > current_lr: 0.00004 + | > step_time: 2.71190 (2.19245) + | > loader_time: 0.08760 (0.03013) + + + --> STEP: 127/234 -- GLOBAL_STEP: 33355 + | > loss: -0.15241 (-0.11486) + | > log_mle: -0.34974 (-0.26430) + | > loss_dur: 0.19733 (0.14944) + | > amp_scaler: 4096.00000 (2596.28346) + | > grad_norm: 50.92010 (14.81539) + | > current_lr: 0.00004 + | > step_time: 2.28910 (2.20621) + | > loader_time: 0.00430 (0.03178) + + + --> STEP: 132/234 -- GLOBAL_STEP: 33360 + | > loss: -0.16066 (-0.11671) + | > log_mle: -0.33483 (-0.26773) + | > loss_dur: 0.17417 (0.15102) + | > amp_scaler: 4096.00000 (2653.09091) + | > grad_norm: 30.18229 (15.65242) + | > current_lr: 0.00004 + | > step_time: 1.20270 (2.21097) + | > loader_time: 0.03860 (0.03232) + + + --> STEP: 137/234 -- GLOBAL_STEP: 33365 + | > loss: -0.13513 (-0.11850) + | > log_mle: -0.34527 (-0.27139) + | > loss_dur: 0.21014 (0.15289) + | > amp_scaler: 4096.00000 (2705.75182) + | > grad_norm: 40.14572 (16.54239) + | > current_lr: 0.00004 + | > step_time: 2.19660 (2.21276) + | > loader_time: 0.00840 (0.03382) + + + --> STEP: 142/234 -- GLOBAL_STEP: 33370 + | > loss: -0.15697 (-0.12009) + | > log_mle: -0.36245 (-0.27455) + | > loss_dur: 0.20548 (0.15447) + | > amp_scaler: 4096.00000 (2754.70423) + | > grad_norm: 32.82544 (17.32988) + | > current_lr: 0.00004 + | > step_time: 1.39560 (2.24113) + | > loader_time: 0.00250 (0.03344) + + + --> STEP: 147/234 -- GLOBAL_STEP: 33375 + | > loss: -0.16691 (-0.12280) + | > log_mle: -0.36410 (-0.27929) + | > loss_dur: 0.19719 (0.15649) + | > amp_scaler: 4096.00000 (2800.32653) + | > grad_norm: 32.30696 (18.41444) + | > current_lr: 0.00004 + | > step_time: 1.31440 (2.22218) + | > loader_time: 0.08750 (0.03406) + + + --> STEP: 152/234 -- GLOBAL_STEP: 33380 + | > loss: -0.21774 (-0.12564) + | > log_mle: -0.44488 (-0.28377) + | > loss_dur: 0.22714 (0.15812) + | > amp_scaler: 4096.00000 (2842.94737) + | > grad_norm: 61.09587 (19.26740) + | > current_lr: 0.00004 + | > step_time: 4.10100 (2.33322) + | > loader_time: 0.00190 (0.03708) + + + --> STEP: 157/234 -- GLOBAL_STEP: 33385 + | > loss: -0.18622 (-0.12927) + | > log_mle: -0.39502 (-0.28889) + | > loss_dur: 0.20880 (0.15961) + | > amp_scaler: 4096.00000 (2882.85350) + | > grad_norm: 43.57539 (20.40948) + | > current_lr: 0.00004 + | > step_time: 4.60610 (2.34544) + | > loader_time: 0.10180 (0.03720) + + + --> STEP: 162/234 -- GLOBAL_STEP: 33390 + | > loss: -0.22406 (-0.13261) + | > log_mle: -0.42214 (-0.29371) + | > loss_dur: 0.19808 (0.16110) + | > amp_scaler: 4096.00000 (2920.29630) + | > grad_norm: 63.62653 (21.67808) + | > current_lr: 0.00004 + | > step_time: 1.40480 (2.38866) + | > loader_time: 0.00570 (0.03850) + + + --> STEP: 167/234 -- GLOBAL_STEP: 33395 + | > loss: -0.29302 (-0.13546) + | > log_mle: -0.49918 (-0.29798) + | > loss_dur: 0.20617 (0.16252) + | > amp_scaler: 4096.00000 (2955.49701) + | > grad_norm: 75.03735 (22.82005) + | > current_lr: 0.00004 + | > step_time: 2.69310 (2.40035) + | > loader_time: 0.00330 (0.03807) + + + --> STEP: 172/234 -- GLOBAL_STEP: 33400 + | > loss: -0.25868 (-0.13857) + | > log_mle: -0.49608 (-0.30316) + | > loss_dur: 0.23740 (0.16459) + | > amp_scaler: 4096.00000 (2988.65116) + | > grad_norm: 80.05839 (24.15533) + | > current_lr: 0.00004 + | > step_time: 2.31210 (2.42892) + | > loader_time: 0.00290 (0.03933) + + + --> STEP: 177/234 -- GLOBAL_STEP: 33405 + | > loss: -0.22687 (-0.14179) + | > log_mle: -0.44997 (-0.30806) + | > loss_dur: 0.22310 (0.16628) + | > amp_scaler: 4096.00000 (3019.93220) + | > grad_norm: 60.91994 (25.28541) + | > current_lr: 0.00004 + | > step_time: 11.80210 (2.54742) + | > loader_time: 0.09060 (0.04107) + + + --> STEP: 182/234 -- GLOBAL_STEP: 33410 + | > loss: -0.25382 (-0.14452) + | > log_mle: -0.49978 (-0.31289) + | > loss_dur: 0.24596 (0.16838) + | > amp_scaler: 4096.00000 (3049.49451) + | > grad_norm: 75.17779 (26.42881) + | > current_lr: 0.00004 + | > step_time: 4.29770 (2.63827) + | > loader_time: 0.19280 (0.04274) + + + --> STEP: 187/234 -- GLOBAL_STEP: 33415 + | > loss: -0.26211 (-0.14748) + | > log_mle: -0.49930 (-0.31772) + | > loss_dur: 0.23720 (0.17024) + | > amp_scaler: 4096.00000 (3077.47594) + | > grad_norm: 89.38625 (27.53495) + | > current_lr: 0.00004 + | > step_time: 1.69670 (2.68863) + | > loader_time: 0.00620 (0.04314) + + + --> STEP: 192/234 -- GLOBAL_STEP: 33420 + | > loss: -0.30326 (-0.15076) + | > log_mle: -0.52717 (-0.32249) + | > loss_dur: 0.22390 (0.17173) + | > amp_scaler: 4096.00000 (3104.00000) + | > grad_norm: 89.81600 (28.67048) + | > current_lr: 0.00004 + | > step_time: 6.70800 (2.71875) + | > loader_time: 0.18960 (0.04398) + + + --> STEP: 197/234 -- GLOBAL_STEP: 33425 + | > loss: -0.28424 (-0.15385) + | > log_mle: -0.50001 (-0.32707) + | > loss_dur: 0.21577 (0.17322) + | > amp_scaler: 4096.00000 (3129.17766) + | > grad_norm: 67.62082 (29.72946) + | > current_lr: 0.00004 + | > step_time: 3.68600 (2.78911) + | > loader_time: 0.00340 (0.04347) + + + --> STEP: 202/234 -- GLOBAL_STEP: 33430 + | > loss: -0.35329 (-0.15673) + | > log_mle: -0.58641 (-0.33164) + | > loss_dur: 0.23312 (0.17491) + | > amp_scaler: 4096.00000 (3153.10891) + | > grad_norm: 101.15884 (30.96622) + | > current_lr: 0.00004 + | > step_time: 8.60490 (2.82552) + | > loader_time: 0.10510 (0.04495) + + + --> STEP: 207/234 -- GLOBAL_STEP: 33435 + | > loss: -0.32919 (-0.15965) + | > log_mle: -0.57533 (-0.33624) + | > loss_dur: 0.24613 (0.17659) + | > amp_scaler: 4096.00000 (3175.88406) + | > grad_norm: 83.80355 (32.04293) + | > current_lr: 0.00004 + | > step_time: 3.50320 (2.88500) + | > loader_time: 0.08990 (0.04667) + + + --> STEP: 212/234 -- GLOBAL_STEP: 33440 + | > loss: -0.31726 (-0.16319) + | > log_mle: -0.56127 (-0.34153) + | > loss_dur: 0.24400 (0.17834) + | > amp_scaler: 4096.00000 (3197.58491) + | > grad_norm: 92.47784 (33.33384) + | > current_lr: 0.00004 + | > step_time: 6.39830 (2.97127) + | > loader_time: 0.08540 (0.04880) + + + --> STEP: 217/234 -- GLOBAL_STEP: 33445 + | > loss: -0.32427 (-0.16679) + | > log_mle: -0.58648 (-0.34682) + | > loss_dur: 0.26221 (0.18003) + | > amp_scaler: 4096.00000 (3218.28571) + | > grad_norm: 81.93279 (34.53227) + | > current_lr: 0.00004 + | > step_time: 7.10410 (3.05774) + | > loader_time: 0.39670 (0.05084) + + + --> STEP: 222/234 -- GLOBAL_STEP: 33450 + | > loss: -0.30810 (-0.17034) + | > log_mle: -0.59236 (-0.35205) + | > loss_dur: 0.28426 (0.18171) + | > amp_scaler: 4096.00000 (3238.05405) + | > grad_norm: 111.03572 (35.77861) + | > current_lr: 0.00004 + | > step_time: 0.22710 (3.01147) + | > loader_time: 0.00480 (0.04978) + + + --> STEP: 227/234 -- GLOBAL_STEP: 33455 + | > loss: -0.30265 (-0.17410) + | > log_mle: -0.57130 (-0.35754) + | > loss_dur: 0.26865 (0.18344) + | > amp_scaler: 4096.00000 (3256.95154) + | > grad_norm: 101.53458 (37.53894) + | > current_lr: 0.00004 + | > step_time: 0.24240 (2.95023) + | > loader_time: 0.00390 (0.04876) + + + --> STEP: 232/234 -- GLOBAL_STEP: 33460 + | > loss: -0.26582 (-0.17695) + | > log_mle: -0.77413 (-0.36435) + | > loss_dur: 0.50831 (0.18740) + | > amp_scaler: 2048.00000 (3239.72414) + | > grad_norm: 155.39790 (39.11520) + | > current_lr: 0.00004 + | > step_time: 0.33090 (2.89227) + | > loader_time: 0.01040 (0.04783) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.63088 (+0.62775) + | > avg_loss: -0.18880 (+0.01160) + | > avg_log_mle: -0.41785 (+0.00944) + | > avg_loss_dur: 0.22904 (+0.00216) + + + > EPOCH: 143/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 15:12:58)  + + --> STEP: 3/234 -- GLOBAL_STEP: 33465 + | > loss: -0.05551 (-0.10808) + | > log_mle: -0.23973 (-0.24615) + | > loss_dur: 0.18422 (0.13807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.71406 (17.32447) + | > current_lr: 0.00004 + | > step_time: 5.69060 (4.86498) + | > loader_time: 0.00090 (1.13344) + + + --> STEP: 8/234 -- GLOBAL_STEP: 33470 + | > loss: -0.12720 (-0.11364) + | > log_mle: -0.26109 (-0.24770) + | > loss_dur: 0.13389 (0.13406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.82772 (15.70970) + | > current_lr: 0.00004 + | > step_time: 10.50040 (5.36419) + | > loader_time: 0.18790 (0.48252) + + + --> STEP: 13/234 -- GLOBAL_STEP: 33475 + | > loss: -0.13661 (-0.11383) + | > log_mle: -0.25035 (-0.24845) + | > loss_dur: 0.11374 (0.13462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.93227 (14.17436) + | > current_lr: 0.00004 + | > step_time: 1.90610 (4.70048) + | > loader_time: 0.00170 (0.29892) + + + --> STEP: 18/234 -- GLOBAL_STEP: 33480 + | > loss: -0.11084 (-0.11408) + | > log_mle: -0.24524 (-0.24656) + | > loss_dur: 0.13440 (0.13248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.44567 (13.08164) + | > current_lr: 0.00004 + | > step_time: 4.00060 (5.08467) + | > loader_time: 0.00210 (0.23214) + + + --> STEP: 23/234 -- GLOBAL_STEP: 33485 + | > loss: -0.14726 (-0.11621) + | > log_mle: -0.24713 (-0.24445) + | > loss_dur: 0.09987 (0.12824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.41970 (12.10794) + | > current_lr: 0.00004 + | > step_time: 1.30420 (4.25958) + | > loader_time: 0.10490 (0.19386) + + + --> STEP: 28/234 -- GLOBAL_STEP: 33490 + | > loss: -0.13625 (-0.11774) + | > log_mle: -0.23933 (-0.24344) + | > loss_dur: 0.10308 (0.12570) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.17535 (11.34367) + | > current_lr: 0.00004 + | > step_time: 1.28860 (3.71873) + | > loader_time: 0.00140 (0.16275) + + + --> STEP: 33/234 -- GLOBAL_STEP: 33495 + | > loss: -0.10990 (-0.11757) + | > log_mle: -0.23470 (-0.24323) + | > loss_dur: 0.12480 (0.12566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.20651 (10.98706) + | > current_lr: 0.00004 + | > step_time: 1.17540 (3.36671) + | > loader_time: 0.00160 (0.13836) + + + --> STEP: 38/234 -- GLOBAL_STEP: 33500 + | > loss: -0.10640 (-0.11583) + | > log_mle: -0.25079 (-0.24312) + | > loss_dur: 0.14439 (0.12729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.05536 (10.92612) + | > current_lr: 0.00004 + | > step_time: 1.49640 (3.13791) + | > loader_time: 0.00190 (0.12045) + + + --> STEP: 43/234 -- GLOBAL_STEP: 33505 + | > loss: -0.09903 (-0.11403) + | > log_mle: -0.25010 (-0.24267) + | > loss_dur: 0.15108 (0.12864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.29499 (10.72360) + | > current_lr: 0.00004 + | > step_time: 1.19820 (2.93748) + | > loader_time: 0.00180 (0.10893) + + + --> STEP: 48/234 -- GLOBAL_STEP: 33510 + | > loss: -0.11347 (-0.11372) + | > log_mle: -0.23247 (-0.24265) + | > loss_dur: 0.11900 (0.12893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.53832 (10.57229) + | > current_lr: 0.00004 + | > step_time: 1.88430 (2.78916) + | > loader_time: 0.00210 (0.09781) + + + --> STEP: 53/234 -- GLOBAL_STEP: 33515 + | > loss: -0.10470 (-0.11271) + | > log_mle: -0.25052 (-0.24235) + | > loss_dur: 0.14582 (0.12963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.46854 (10.30467) + | > current_lr: 0.00004 + | > step_time: 1.21790 (2.64725) + | > loader_time: 0.08430 (0.09200) + + + --> STEP: 58/234 -- GLOBAL_STEP: 33520 + | > loss: -0.11417 (-0.11263) + | > log_mle: -0.23754 (-0.24243) + | > loss_dur: 0.12338 (0.12980) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.30411 (10.17519) + | > current_lr: 0.00004 + | > step_time: 0.99980 (2.56873) + | > loader_time: 0.08480 (0.08715) + + + --> STEP: 63/234 -- GLOBAL_STEP: 33525 + | > loss: -0.08977 (-0.11222) + | > log_mle: -0.24355 (-0.24370) + | > loss_dur: 0.15378 (0.13148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.65297 (10.51943) + | > current_lr: 0.00004 + | > step_time: 1.59980 (2.52389) + | > loader_time: 0.00190 (0.08049) + + + --> STEP: 68/234 -- GLOBAL_STEP: 33530 + | > loss: -0.08339 (-0.11156) + | > log_mle: -0.23687 (-0.24358) + | > loss_dur: 0.15348 (0.13202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.65025 (10.43180) + | > current_lr: 0.00004 + | > step_time: 1.12480 (2.44887) + | > loader_time: 0.08690 (0.07599) + + + --> STEP: 73/234 -- GLOBAL_STEP: 33535 + | > loss: -0.09283 (-0.11001) + | > log_mle: -0.25976 (-0.24371) + | > loss_dur: 0.16694 (0.13370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.69193 (10.59375) + | > current_lr: 0.00004 + | > step_time: 1.31030 (2.45524) + | > loader_time: 0.08190 (0.07454) + + + --> STEP: 78/234 -- GLOBAL_STEP: 33540 + | > loss: -0.08087 (-0.10920) + | > log_mle: -0.23079 (-0.24362) + | > loss_dur: 0.14992 (0.13442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.93003 (10.65477) + | > current_lr: 0.00004 + | > step_time: 1.31470 (2.40720) + | > loader_time: 0.08320 (0.07097) + + + --> STEP: 83/234 -- GLOBAL_STEP: 33545 + | > loss: -0.09822 (-0.10907) + | > log_mle: -0.26046 (-0.24395) + | > loss_dur: 0.16224 (0.13489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.83818 (10.73862) + | > current_lr: 0.00004 + | > step_time: 1.48140 (2.36947) + | > loader_time: 0.00200 (0.06987) + + + --> STEP: 88/234 -- GLOBAL_STEP: 33550 + | > loss: -0.13053 (-0.10925) + | > log_mle: -0.29724 (-0.24486) + | > loss_dur: 0.16671 (0.13561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.70477 (10.84035) + | > current_lr: 0.00004 + | > step_time: 1.06260 (2.33237) + | > loader_time: 0.00260 (0.06701) + + + --> STEP: 93/234 -- GLOBAL_STEP: 33555 + | > loss: -0.12869 (-0.10998) + | > log_mle: -0.31132 (-0.24716) + | > loss_dur: 0.18263 (0.13718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.32635 (11.17402) + | > current_lr: 0.00004 + | > step_time: 2.09610 (2.28229) + | > loader_time: 0.00310 (0.06444) + + + --> STEP: 98/234 -- GLOBAL_STEP: 33560 + | > loss: -0.09219 (-0.11127) + | > log_mle: -0.23878 (-0.24948) + | > loss_dur: 0.14659 (0.13821) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.12601 (11.52302) + | > current_lr: 0.00004 + | > step_time: 1.16180 (2.23707) + | > loader_time: 0.00240 (0.06218) + + + --> STEP: 103/234 -- GLOBAL_STEP: 33565 + | > loss: -0.15637 (-0.11274) + | > log_mle: -0.34520 (-0.25264) + | > loss_dur: 0.18882 (0.13990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.98653 (11.96652) + | > current_lr: 0.00004 + | > step_time: 1.49450 (2.21861) + | > loader_time: 0.00300 (0.06089) + + + --> STEP: 108/234 -- GLOBAL_STEP: 33570 + | > loss: -0.12838 (-0.11390) + | > log_mle: -0.28648 (-0.25526) + | > loss_dur: 0.15809 (0.14136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.88842 (12.58988) + | > current_lr: 0.00004 + | > step_time: 4.60900 (2.22058) + | > loader_time: 0.10270 (0.06170) + + + --> STEP: 113/234 -- GLOBAL_STEP: 33575 + | > loss: -0.15605 (-0.11481) + | > log_mle: -0.33522 (-0.25851) + | > loss_dur: 0.17917 (0.14370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.60392 (13.41650) + | > current_lr: 0.00004 + | > step_time: 3.09690 (2.22247) + | > loader_time: 0.00240 (0.06159) + + + --> STEP: 118/234 -- GLOBAL_STEP: 33580 + | > loss: -0.12096 (-0.11555) + | > log_mle: -0.29991 (-0.26094) + | > loss_dur: 0.17895 (0.14539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.45256 (13.94872) + | > current_lr: 0.00004 + | > step_time: 0.98230 (2.20344) + | > loader_time: 0.00280 (0.06001) + + + --> STEP: 123/234 -- GLOBAL_STEP: 33585 + | > loss: -0.10624 (-0.11596) + | > log_mle: -0.27096 (-0.26237) + | > loss_dur: 0.16472 (0.14641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.95941 (14.22539) + | > current_lr: 0.00004 + | > step_time: 2.92110 (2.20694) + | > loader_time: 0.00350 (0.05838) + + + --> STEP: 128/234 -- GLOBAL_STEP: 33590 + | > loss: -0.16095 (-0.11778) + | > log_mle: -0.33190 (-0.26578) + | > loss_dur: 0.17095 (0.14800) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.31193 (15.07028) + | > current_lr: 0.00004 + | > step_time: 2.90500 (2.20082) + | > loader_time: 0.08650 (0.05687) + + + --> STEP: 133/234 -- GLOBAL_STEP: 33595 + | > loss: -0.16623 (-0.11976) + | > log_mle: -0.35690 (-0.26934) + | > loss_dur: 0.19068 (0.14958) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.91936 (16.03580) + | > current_lr: 0.00004 + | > step_time: 2.19520 (2.20152) + | > loader_time: 0.01190 (0.05703) + + + --> STEP: 138/234 -- GLOBAL_STEP: 33600 + | > loss: -0.12724 (-0.12117) + | > log_mle: -0.31161 (-0.27257) + | > loss_dur: 0.18437 (0.15140) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.54162 (16.78517) + | > current_lr: 0.00004 + | > step_time: 8.08440 (2.28480) + | > loader_time: 0.20700 (0.05854) + + + --> STEP: 143/234 -- GLOBAL_STEP: 33605 + | > loss: -0.21026 (-0.12335) + | > log_mle: -0.45504 (-0.27675) + | > loss_dur: 0.24479 (0.15340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.70516 (17.73552) + | > current_lr: 0.00004 + | > step_time: 2.81780 (2.32745) + | > loader_time: 0.10610 (0.05936) + + + --> STEP: 148/234 -- GLOBAL_STEP: 33610 + | > loss: -0.19480 (-0.12602) + | > log_mle: -0.36943 (-0.28098) + | > loss_dur: 0.17463 (0.15497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.46070 (18.59961) + | > current_lr: 0.00004 + | > step_time: 2.60410 (2.34290) + | > loader_time: 0.09110 (0.05986) + + + --> STEP: 153/234 -- GLOBAL_STEP: 33615 + | > loss: -0.28722 (-0.12954) + | > log_mle: -0.49535 (-0.28626) + | > loss_dur: 0.20813 (0.15672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.15197 (19.79233) + | > current_lr: 0.00004 + | > step_time: 1.49240 (2.33895) + | > loader_time: 0.00220 (0.06566) + + + --> STEP: 158/234 -- GLOBAL_STEP: 33620 + | > loss: -0.21622 (-0.13245) + | > log_mle: -0.43214 (-0.29087) + | > loss_dur: 0.21592 (0.15842) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.61457 (21.11559) + | > current_lr: 0.00004 + | > step_time: 3.18820 (2.40094) + | > loader_time: 0.00520 (0.06495) + + + --> STEP: 163/234 -- GLOBAL_STEP: 33625 + | > loss: -0.19406 (-0.13549) + | > log_mle: -0.40179 (-0.29548) + | > loss_dur: 0.20773 (0.15999) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.63744 (22.17346) + | > current_lr: 0.00004 + | > step_time: 3.50740 (2.46738) + | > loader_time: 0.08920 (0.06687) + + + --> STEP: 168/234 -- GLOBAL_STEP: 33630 + | > loss: -0.21653 (-0.13851) + | > log_mle: -0.45928 (-0.30026) + | > loss_dur: 0.24275 (0.16175) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.61801 (23.19707) + | > current_lr: 0.00004 + | > step_time: 3.80820 (2.53448) + | > loader_time: 0.09130 (0.06602) + + + --> STEP: 173/234 -- GLOBAL_STEP: 33635 + | > loss: -0.24603 (-0.14196) + | > log_mle: -0.46887 (-0.30559) + | > loss_dur: 0.22284 (0.16364) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.51437 (24.51765) + | > current_lr: 0.00004 + | > step_time: 1.58980 (2.59017) + | > loader_time: 0.00490 (0.06757) + + + --> STEP: 178/234 -- GLOBAL_STEP: 33640 + | > loss: -0.27400 (-0.14532) + | > log_mle: -0.52305 (-0.31092) + | > loss_dur: 0.24905 (0.16560) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.13225 (25.87289) + | > current_lr: 0.00004 + | > step_time: 3.60360 (2.64039) + | > loader_time: 0.00270 (0.06732) + + + --> STEP: 183/234 -- GLOBAL_STEP: 33645 + | > loss: -0.28609 (-0.14817) + | > log_mle: -0.52511 (-0.31585) + | > loss_dur: 0.23902 (0.16768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.42758 (27.04944) + | > current_lr: 0.00004 + | > step_time: 3.70010 (2.67433) + | > loader_time: 0.00440 (0.06927) + + + --> STEP: 188/234 -- GLOBAL_STEP: 33650 + | > loss: -0.28829 (-0.15116) + | > log_mle: -0.52361 (-0.32070) + | > loss_dur: 0.23532 (0.16954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.22488 (28.34352) + | > current_lr: 0.00004 + | > step_time: 6.49510 (2.74964) + | > loader_time: 0.09790 (0.06942) + + + --> STEP: 193/234 -- GLOBAL_STEP: 33655 + | > loss: -0.29730 (-0.15428) + | > log_mle: -0.52494 (-0.32525) + | > loss_dur: 0.22764 (0.17097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.77400 (29.72426) + | > current_lr: 0.00004 + | > step_time: 4.80120 (2.79342) + | > loader_time: 0.09300 (0.06863) + + + --> STEP: 198/234 -- GLOBAL_STEP: 33660 + | > loss: -0.26515 (-0.15699) + | > log_mle: -0.50276 (-0.32953) + | > loss_dur: 0.23761 (0.17254) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.34399 (31.09972) + | > current_lr: 0.00004 + | > step_time: 4.40400 (2.80370) + | > loader_time: 0.08960 (0.06786) + + + --> STEP: 203/234 -- GLOBAL_STEP: 33665 + | > loss: -0.22303 (-0.15957) + | > log_mle: -0.45275 (-0.33369) + | > loss_dur: 0.22973 (0.17413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.51388 (32.14866) + | > current_lr: 0.00004 + | > step_time: 3.61080 (2.85293) + | > loader_time: 0.08040 (0.06762) + + + --> STEP: 208/234 -- GLOBAL_STEP: 33670 + | > loss: -0.28492 (-0.16272) + | > log_mle: -0.53800 (-0.33865) + | > loss_dur: 0.25309 (0.17593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.17921 (33.08046) + | > current_lr: 0.00004 + | > step_time: 5.48970 (2.88043) + | > loader_time: 0.00330 (0.06732) + + + --> STEP: 213/234 -- GLOBAL_STEP: 33675 + | > loss: -0.33209 (-0.16634) + | > log_mle: -0.58950 (-0.34405) + | > loss_dur: 0.25742 (0.17772) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.02698 (34.16874) + | > current_lr: 0.00004 + | > step_time: 3.50770 (2.98630) + | > loader_time: 0.00470 (0.06806) + + + --> STEP: 218/234 -- GLOBAL_STEP: 33680 + | > loss: -0.29261 (-0.16974) + | > log_mle: -0.54381 (-0.34902) + | > loss_dur: 0.25119 (0.17927) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.76340 (35.33368) + | > current_lr: 0.00004 + | > step_time: 2.69490 (3.08621) + | > loader_time: 0.00570 (0.06740) + + + --> STEP: 223/234 -- GLOBAL_STEP: 33685 + | > loss: -0.34878 (-0.17342) + | > log_mle: -0.59221 (-0.35428) + | > loss_dur: 0.24343 (0.18086) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.00562 (36.54597) + | > current_lr: 0.00004 + | > step_time: 0.24990 (3.05326) + | > loader_time: 0.00390 (0.06632) + + + --> STEP: 228/234 -- GLOBAL_STEP: 33690 + | > loss: -0.29682 (-0.17687) + | > log_mle: -0.57683 (-0.35959) + | > loss_dur: 0.28001 (0.18273) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 125.78122 (38.25648) + | > current_lr: 0.00004 + | > step_time: 0.24020 (2.99155) + | > loader_time: 0.00420 (0.06495) + + + --> STEP: 233/234 -- GLOBAL_STEP: 33695 + | > loss: 0.27639 (-0.17670) + | > log_mle: -0.53073 (-0.36576) + | > loss_dur: 0.80711 (0.18906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.57848 (40.61831) + | > current_lr: 0.00004 + | > step_time: 0.20080 (2.93299) + | > loader_time: 0.00340 (0.06399) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.37645 (-0.25443) + | > avg_loss: -0.21131 (-0.02251) + | > avg_log_mle: -0.43783 (-0.01998) + | > avg_loss_dur: 0.22652 (-0.00252) + + + > EPOCH: 144/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 15:25:35)  + + --> STEP: 4/234 -- GLOBAL_STEP: 33700 + | > loss: -0.08502 (-0.09985) + | > log_mle: -0.24200 (-0.24713) + | > loss_dur: 0.15698 (0.14727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.61824 (14.20942) + | > current_lr: 0.00004 + | > step_time: 6.58920 (6.99363) + | > loader_time: 0.10350 (0.17837) + + + --> STEP: 9/234 -- GLOBAL_STEP: 33705 + | > loss: -0.11292 (-0.11081) + | > log_mle: -0.25777 (-0.25043) + | > loss_dur: 0.14484 (0.13962) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.06272 (12.96697) + | > current_lr: 0.00004 + | > step_time: 3.08400 (6.11727) + | > loader_time: 0.00100 (0.11161) + + + --> STEP: 14/234 -- GLOBAL_STEP: 33710 + | > loss: -0.11943 (-0.11342) + | > log_mle: -0.25349 (-0.25002) + | > loss_dur: 0.13405 (0.13660) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.58122 (12.55164) + | > current_lr: 0.00004 + | > step_time: 2.00460 (4.75497) + | > loader_time: 0.00330 (0.07270) + + + --> STEP: 19/234 -- GLOBAL_STEP: 33715 + | > loss: -0.12136 (-0.11558) + | > log_mle: -0.23795 (-0.24730) + | > loss_dur: 0.11660 (0.13172) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.39987 (12.15111) + | > current_lr: 0.00004 + | > step_time: 0.81530 (4.16880) + | > loader_time: 0.00390 (0.06906) + + + --> STEP: 24/234 -- GLOBAL_STEP: 33720 + | > loss: -0.13961 (-0.11794) + | > log_mle: -0.24053 (-0.24576) + | > loss_dur: 0.10092 (0.12783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.83526 (11.90165) + | > current_lr: 0.00004 + | > step_time: 1.09870 (4.10417) + | > loader_time: 0.00250 (0.06702) + + + --> STEP: 29/234 -- GLOBAL_STEP: 33725 + | > loss: -0.09291 (-0.11750) + | > log_mle: -0.22726 (-0.24461) + | > loss_dur: 0.13435 (0.12711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.76097 (11.59049) + | > current_lr: 0.00004 + | > step_time: 2.90310 (3.82076) + | > loader_time: 0.00300 (0.05881) + + + --> STEP: 34/234 -- GLOBAL_STEP: 33730 + | > loss: -0.09576 (-0.11698) + | > log_mle: -0.23812 (-0.24410) + | > loss_dur: 0.14236 (0.12713) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.45563 (11.33525) + | > current_lr: 0.00004 + | > step_time: 2.40130 (4.02994) + | > loader_time: 0.00190 (0.05661) + + + --> STEP: 39/234 -- GLOBAL_STEP: 33735 + | > loss: -0.10869 (-0.11568) + | > log_mle: -0.24745 (-0.24378) + | > loss_dur: 0.13875 (0.12810) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.41521 (11.63639) + | > current_lr: 0.00004 + | > step_time: 2.80110 (4.02069) + | > loader_time: 0.00160 (0.05462) + + + --> STEP: 44/234 -- GLOBAL_STEP: 33740 + | > loss: -0.13784 (-0.11440) + | > log_mle: -0.23746 (-0.24291) + | > loss_dur: 0.09962 (0.12851) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.88605 (11.30012) + | > current_lr: 0.00004 + | > step_time: 0.96390 (3.69009) + | > loader_time: 0.00360 (0.04872) + + + --> STEP: 49/234 -- GLOBAL_STEP: 33745 + | > loss: -0.13243 (-0.11430) + | > log_mle: -0.24994 (-0.24313) + | > loss_dur: 0.11752 (0.12882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.60667 (11.10524) + | > current_lr: 0.00004 + | > step_time: 0.78260 (3.47025) + | > loader_time: 0.00340 (0.04729) + + + --> STEP: 54/234 -- GLOBAL_STEP: 33750 + | > loss: -0.13029 (-0.11378) + | > log_mle: -0.25419 (-0.24298) + | > loss_dur: 0.12390 (0.12921) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.30707 (10.74780) + | > current_lr: 0.00004 + | > step_time: 1.11060 (3.28256) + | > loader_time: 0.00230 (0.04620) + + + --> STEP: 59/234 -- GLOBAL_STEP: 33755 + | > loss: -0.13030 (-0.11331) + | > log_mle: -0.26106 (-0.24315) + | > loss_dur: 0.13076 (0.12984) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.71788 (10.70317) + | > current_lr: 0.00004 + | > step_time: 3.41420 (3.16040) + | > loader_time: 0.09720 (0.04553) + + + --> STEP: 64/234 -- GLOBAL_STEP: 33760 + | > loss: -0.11886 (-0.11266) + | > log_mle: -0.23863 (-0.24413) + | > loss_dur: 0.11977 (0.13147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.74217 (10.74003) + | > current_lr: 0.00004 + | > step_time: 1.89830 (3.06354) + | > loader_time: 0.00330 (0.04221) + + + --> STEP: 69/234 -- GLOBAL_STEP: 33765 + | > loss: -0.08911 (-0.11173) + | > log_mle: -0.22464 (-0.24384) + | > loss_dur: 0.13553 (0.13211) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.25955 (10.68150) + | > current_lr: 0.00004 + | > step_time: 2.20700 (2.95125) + | > loader_time: 0.00220 (0.04050) + + + --> STEP: 74/234 -- GLOBAL_STEP: 33770 + | > loss: -0.10480 (-0.11052) + | > log_mle: -0.23788 (-0.24422) + | > loss_dur: 0.13309 (0.13370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99265 (10.93210) + | > current_lr: 0.00004 + | > step_time: 2.60340 (2.87748) + | > loader_time: 0.00590 (0.04014) + + + --> STEP: 79/234 -- GLOBAL_STEP: 33775 + | > loss: -0.10989 (-0.11021) + | > log_mle: -0.25369 (-0.24468) + | > loss_dur: 0.14380 (0.13446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.72185 (10.84258) + | > current_lr: 0.00004 + | > step_time: 1.30230 (2.84619) + | > loader_time: 0.00290 (0.03998) + + + --> STEP: 84/234 -- GLOBAL_STEP: 33780 + | > loss: -0.10942 (-0.11019) + | > log_mle: -0.24876 (-0.24507) + | > loss_dur: 0.13934 (0.13487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.75407 (10.79713) + | > current_lr: 0.00004 + | > step_time: 3.80010 (2.80825) + | > loader_time: 0.00600 (0.03881) + + + --> STEP: 89/234 -- GLOBAL_STEP: 33785 + | > loss: -0.12520 (-0.11043) + | > log_mle: -0.27930 (-0.24642) + | > loss_dur: 0.15410 (0.13599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.64382 (10.97711) + | > current_lr: 0.00004 + | > step_time: 3.40410 (2.78183) + | > loader_time: 0.08320 (0.03769) + + + --> STEP: 94/234 -- GLOBAL_STEP: 33790 + | > loss: -0.15815 (-0.11150) + | > log_mle: -0.31611 (-0.24907) + | > loss_dur: 0.15796 (0.13757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.21715 (11.41324) + | > current_lr: 0.00004 + | > step_time: 1.90840 (2.75296) + | > loader_time: 0.00260 (0.03777) + + + --> STEP: 99/234 -- GLOBAL_STEP: 33795 + | > loss: -0.16594 (-0.11273) + | > log_mle: -0.34694 (-0.25168) + | > loss_dur: 0.18100 (0.13896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.96550 (11.87569) + | > current_lr: 0.00004 + | > step_time: 5.40920 (2.72079) + | > loader_time: 0.00510 (0.03679) + + + --> STEP: 104/234 -- GLOBAL_STEP: 33800 + | > loss: -0.18358 (-0.11429) + | > log_mle: -0.36190 (-0.25487) + | > loss_dur: 0.17831 (0.14058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.03156 (12.42521) + | > current_lr: 0.00004 + | > step_time: 1.10080 (2.68097) + | > loader_time: 0.00220 (0.03609) + + + --> STEP: 109/234 -- GLOBAL_STEP: 33805 + | > loss: -0.11527 (-0.11499) + | > log_mle: -0.33054 (-0.25715) + | > loss_dur: 0.21527 (0.14216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.63144 (12.89530) + | > current_lr: 0.00004 + | > step_time: 2.39460 (2.67164) + | > loader_time: 0.00270 (0.03710) + + + --> STEP: 114/234 -- GLOBAL_STEP: 33810 + | > loss: -0.14099 (-0.11613) + | > log_mle: -0.31278 (-0.26023) + | > loss_dur: 0.17180 (0.14409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.06928 (13.62628) + | > current_lr: 0.00004 + | > step_time: 1.79610 (2.64026) + | > loader_time: 0.00320 (0.03632) + + + --> STEP: 119/234 -- GLOBAL_STEP: 33815 + | > loss: -0.13769 (-0.11680) + | > log_mle: -0.30963 (-0.26259) + | > loss_dur: 0.17194 (0.14579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.31304 (14.11465) + | > current_lr: 0.00004 + | > step_time: 1.50930 (2.60667) + | > loader_time: 0.18620 (0.03718) + + + --> STEP: 124/234 -- GLOBAL_STEP: 33820 + | > loss: -0.15826 (-0.11742) + | > log_mle: -0.33319 (-0.26416) + | > loss_dur: 0.17493 (0.14674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.87016 (14.60810) + | > current_lr: 0.00004 + | > step_time: 1.09700 (2.57655) + | > loader_time: 0.00270 (0.03642) + + + --> STEP: 129/234 -- GLOBAL_STEP: 33825 + | > loss: -0.13864 (-0.11894) + | > log_mle: -0.33069 (-0.26741) + | > loss_dur: 0.19205 (0.14847) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.65585 (15.46058) + | > current_lr: 0.00004 + | > step_time: 1.19560 (2.53480) + | > loader_time: 0.00260 (0.03575) + + + --> STEP: 134/234 -- GLOBAL_STEP: 33830 + | > loss: -0.16399 (-0.12116) + | > log_mle: -0.37861 (-0.27130) + | > loss_dur: 0.21462 (0.15014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.76923 (16.35332) + | > current_lr: 0.00004 + | > step_time: 1.50210 (2.50738) + | > loader_time: 0.00430 (0.03515) + + + --> STEP: 139/234 -- GLOBAL_STEP: 33835 + | > loss: -0.23522 (-0.12292) + | > log_mle: -0.43753 (-0.27489) + | > loss_dur: 0.20231 (0.15198) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.24664 (17.15236) + | > current_lr: 0.00004 + | > step_time: 2.31090 (2.50007) + | > loader_time: 0.00270 (0.03526) + + + --> STEP: 144/234 -- GLOBAL_STEP: 33840 + | > loss: -0.20803 (-0.12475) + | > log_mle: -0.41566 (-0.27875) + | > loss_dur: 0.20763 (0.15400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.53262 (18.05281) + | > current_lr: 0.00004 + | > step_time: 4.00050 (2.51739) + | > loader_time: 0.09090 (0.03536) + + + --> STEP: 149/234 -- GLOBAL_STEP: 33845 + | > loss: -0.24599 (-0.12756) + | > log_mle: -0.45959 (-0.28318) + | > loss_dur: 0.21361 (0.15562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.63496 (19.08085) + | > current_lr: 0.00004 + | > step_time: 3.49890 (2.57448) + | > loader_time: 0.08630 (0.03671) + + + --> STEP: 154/234 -- GLOBAL_STEP: 33850 + | > loss: -0.22214 (-0.13072) + | > log_mle: -0.41730 (-0.28798) + | > loss_dur: 0.19517 (0.15726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.18954 (20.16279) + | > current_lr: 0.00004 + | > step_time: 4.09480 (2.66292) + | > loader_time: 0.09180 (0.03926) + + + --> STEP: 159/234 -- GLOBAL_STEP: 33855 + | > loss: -0.22363 (-0.13347) + | > log_mle: -0.43588 (-0.29253) + | > loss_dur: 0.21225 (0.15906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.75415 (21.22886) + | > current_lr: 0.00004 + | > step_time: 5.29510 (2.70387) + | > loader_time: 0.00420 (0.03981) + + + --> STEP: 164/234 -- GLOBAL_STEP: 33860 + | > loss: -0.21986 (-0.13639) + | > log_mle: -0.43383 (-0.29686) + | > loss_dur: 0.21397 (0.16047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.67112 (22.14814) + | > current_lr: 0.00004 + | > step_time: 1.11190 (2.73019) + | > loader_time: 0.00460 (0.04091) + + + --> STEP: 169/234 -- GLOBAL_STEP: 33865 + | > loss: -0.21538 (-0.13930) + | > log_mle: -0.43480 (-0.30146) + | > loss_dur: 0.21943 (0.16216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.09854 (23.22355) + | > current_lr: 0.00004 + | > step_time: 1.70040 (2.71267) + | > loader_time: 0.00530 (0.04026) + + + --> STEP: 174/234 -- GLOBAL_STEP: 33870 + | > loss: -0.30229 (-0.14314) + | > log_mle: -0.52426 (-0.30724) + | > loss_dur: 0.22196 (0.16410) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.96451 (24.45207) + | > current_lr: 0.00004 + | > step_time: 3.91330 (2.73075) + | > loader_time: 0.08630 (0.04133) + + + --> STEP: 179/234 -- GLOBAL_STEP: 33875 + | > loss: -0.26795 (-0.14621) + | > log_mle: -0.51790 (-0.31242) + | > loss_dur: 0.24995 (0.16622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.59118 (25.53047) + | > current_lr: 0.00004 + | > step_time: 2.99910 (2.76671) + | > loader_time: 0.00340 (0.04187) + + + --> STEP: 184/234 -- GLOBAL_STEP: 33880 + | > loss: -0.24087 (-0.14890) + | > log_mle: -0.47426 (-0.31693) + | > loss_dur: 0.23339 (0.16803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.95486 (26.70000) + | > current_lr: 0.00004 + | > step_time: 2.20800 (2.81876) + | > loader_time: 0.09290 (0.04486) + + + --> STEP: 189/234 -- GLOBAL_STEP: 33885 + | > loss: -0.23282 (-0.15159) + | > log_mle: -0.47235 (-0.32145) + | > loss_dur: 0.23954 (0.16986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.92262 (28.10388) + | > current_lr: 0.00004 + | > step_time: 2.20550 (2.83885) + | > loader_time: 0.00460 (0.04380) + + + --> STEP: 194/234 -- GLOBAL_STEP: 33890 + | > loss: -0.27535 (-0.15490) + | > log_mle: -0.50977 (-0.32620) + | > loss_dur: 0.23442 (0.17130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.56674 (29.09259) + | > current_lr: 0.00004 + | > step_time: 4.28810 (2.91137) + | > loader_time: 0.00660 (0.04286) + + + --> STEP: 199/234 -- GLOBAL_STEP: 33895 + | > loss: -0.27548 (-0.15781) + | > log_mle: -0.52038 (-0.33074) + | > loss_dur: 0.24491 (0.17293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.69277 (30.04096) + | > current_lr: 0.00004 + | > step_time: 9.40270 (3.00206) + | > loader_time: 0.09660 (0.04382) + + + --> STEP: 204/234 -- GLOBAL_STEP: 33900 + | > loss: -0.30811 (-0.16055) + | > log_mle: -0.56119 (-0.33518) + | > loss_dur: 0.25308 (0.17463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.69956 (31.07302) + | > current_lr: 0.00004 + | > step_time: 5.98640 (3.06039) + | > loader_time: 0.00510 (0.04424) + + + --> STEP: 209/234 -- GLOBAL_STEP: 33905 + | > loss: -0.27721 (-0.16374) + | > log_mle: -0.51166 (-0.33994) + | > loss_dur: 0.23445 (0.17619) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.67654 (32.15808) + | > current_lr: 0.00004 + | > step_time: 5.69980 (3.10351) + | > loader_time: 0.10910 (0.04695) + + + --> STEP: 214/234 -- GLOBAL_STEP: 33910 + | > loss: -0.32939 (-0.16763) + | > log_mle: -0.54772 (-0.34549) + | > loss_dur: 0.21833 (0.17786) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.22807 (33.55623) + | > current_lr: 0.00004 + | > step_time: 3.19180 (3.12989) + | > loader_time: 0.00440 (0.04639) + + + --> STEP: 219/234 -- GLOBAL_STEP: 33915 + | > loss: -0.40839 (-0.17136) + | > log_mle: -0.65298 (-0.35092) + | > loss_dur: 0.24459 (0.17955) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.79095 (34.92769) + | > current_lr: 0.00004 + | > step_time: 2.51720 (3.12845) + | > loader_time: 0.08380 (0.04705) + + + --> STEP: 224/234 -- GLOBAL_STEP: 33920 + | > loss: -0.34027 (-0.17474) + | > log_mle: -0.60182 (-0.35596) + | > loss_dur: 0.26155 (0.18123) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.84229 (36.27186) + | > current_lr: 0.00004 + | > step_time: 0.23910 (3.09986) + | > loader_time: 0.00380 (0.04649) + + + --> STEP: 229/234 -- GLOBAL_STEP: 33925 + | > loss: -0.30284 (-0.17803) + | > log_mle: -0.62045 (-0.36148) + | > loss_dur: 0.31761 (0.18345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 123.20841 (37.80489) + | > current_lr: 0.00004 + | > step_time: 0.25300 (3.03757) + | > loader_time: 0.01150 (0.04559) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.22640 (+0.84995) + | > avg_loss: -0.17991 (+0.03140) + | > avg_log_mle: -0.40427 (+0.03356) + | > avg_loss_dur: 0.22436 (-0.00216) + + + > EPOCH: 145/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 15:38:31)  + + --> STEP: 0/234 -- GLOBAL_STEP: 33930 + | > loss: -0.17584 (-0.17584) + | > log_mle: -0.32406 (-0.32406) + | > loss_dur: 0.14821 (0.14821) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.18429 (16.18429) + | > current_lr: 0.00004 + | > step_time: 9.69470 (9.69471) + | > loader_time: 12.52800 (12.52799) + + + --> STEP: 5/234 -- GLOBAL_STEP: 33935 + | > loss: -0.12783 (-0.10889) + | > log_mle: -0.25344 (-0.24927) + | > loss_dur: 0.12561 (0.14039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.52386 (16.27094) + | > current_lr: 0.00004 + | > step_time: 2.88980 (6.09437) + | > loader_time: 0.11080 (0.08410) + + + --> STEP: 10/234 -- GLOBAL_STEP: 33940 + | > loss: -0.10990 (-0.11536) + | > log_mle: -0.24526 (-0.25208) + | > loss_dur: 0.13535 (0.13671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.76202 (14.67297) + | > current_lr: 0.00004 + | > step_time: 2.01290 (5.01848) + | > loader_time: 0.09430 (0.08214) + + + --> STEP: 15/234 -- GLOBAL_STEP: 33945 + | > loss: -0.13778 (-0.12040) + | > log_mle: -0.25265 (-0.25236) + | > loss_dur: 0.11487 (0.13196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.89266 (12.84942) + | > current_lr: 0.00004 + | > step_time: 1.32930 (4.64470) + | > loader_time: 0.00290 (0.06260) + + + --> STEP: 20/234 -- GLOBAL_STEP: 33950 + | > loss: -0.11837 (-0.12141) + | > log_mle: -0.24055 (-0.24919) + | > loss_dur: 0.12218 (0.12778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.23788 (11.95320) + | > current_lr: 0.00004 + | > step_time: 0.99840 (3.71047) + | > loader_time: 0.08840 (0.05170) + + + --> STEP: 25/234 -- GLOBAL_STEP: 33955 + | > loss: -0.11503 (-0.12291) + | > log_mle: -0.23260 (-0.24746) + | > loss_dur: 0.11757 (0.12455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.52265 (11.43251) + | > current_lr: 0.00004 + | > step_time: 1.20940 (3.22385) + | > loader_time: 0.00240 (0.04516) + + + --> STEP: 30/234 -- GLOBAL_STEP: 33960 + | > loss: -0.13999 (-0.12328) + | > log_mle: -0.25104 (-0.24695) + | > loss_dur: 0.11105 (0.12367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.10607 (10.99784) + | > current_lr: 0.00004 + | > step_time: 3.51380 (3.08178) + | > loader_time: 0.08440 (0.04071) + + + --> STEP: 35/234 -- GLOBAL_STEP: 33965 + | > loss: -0.11551 (-0.12135) + | > log_mle: -0.24557 (-0.24676) + | > loss_dur: 0.13007 (0.12541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.47289 (10.90051) + | > current_lr: 0.00004 + | > step_time: 2.31040 (3.26438) + | > loader_time: 0.00300 (0.03547) + + + --> STEP: 40/234 -- GLOBAL_STEP: 33970 + | > loss: -0.07981 (-0.11971) + | > log_mle: -0.22759 (-0.24631) + | > loss_dur: 0.14778 (0.12661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.61810 (10.85201) + | > current_lr: 0.00004 + | > step_time: 0.90070 (3.36608) + | > loader_time: 0.00210 (0.03602) + + + --> STEP: 45/234 -- GLOBAL_STEP: 33975 + | > loss: -0.12700 (-0.11982) + | > log_mle: -0.26898 (-0.24653) + | > loss_dur: 0.14198 (0.12672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.11539 (10.72825) + | > current_lr: 0.00004 + | > step_time: 3.10200 (3.19294) + | > loader_time: 0.10130 (0.03631) + + + --> STEP: 50/234 -- GLOBAL_STEP: 33980 + | > loss: -0.10407 (-0.11890) + | > log_mle: -0.23568 (-0.24601) + | > loss_dur: 0.13161 (0.12711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.41945 (10.34680) + | > current_lr: 0.00004 + | > step_time: 2.30110 (3.04988) + | > loader_time: 0.00190 (0.03996) + + + --> STEP: 55/234 -- GLOBAL_STEP: 33985 + | > loss: -0.13295 (-0.11839) + | > log_mle: -0.25335 (-0.24602) + | > loss_dur: 0.12039 (0.12762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.43467 (10.19536) + | > current_lr: 0.00004 + | > step_time: 1.10950 (2.87951) + | > loader_time: 0.00270 (0.03652) + + + --> STEP: 60/234 -- GLOBAL_STEP: 33990 + | > loss: -0.11313 (-0.11785) + | > log_mle: -0.26820 (-0.24628) + | > loss_dur: 0.15506 (0.12843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.15751 (10.22058) + | > current_lr: 0.00004 + | > step_time: 5.59850 (2.93775) + | > loader_time: 0.00920 (0.03526) + + + --> STEP: 65/234 -- GLOBAL_STEP: 33995 + | > loss: -0.10831 (-0.11655) + | > log_mle: -0.24178 (-0.24663) + | > loss_dur: 0.13347 (0.13009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.14279 (10.28739) + | > current_lr: 0.00004 + | > step_time: 1.80160 (2.85960) + | > loader_time: 0.00190 (0.03573) + + + --> STEP: 70/234 -- GLOBAL_STEP: 34000 + | > loss: -0.08782 (-0.11471) + | > log_mle: -0.23657 (-0.24605) + | > loss_dur: 0.14875 (0.13134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.02538 (10.39227) + | > current_lr: 0.00004 + | > step_time: 3.70600 (2.82964) + | > loader_time: 0.08600 (0.03599) + + + --> STEP: 75/234 -- GLOBAL_STEP: 34005 + | > loss: -0.09324 (-0.11345) + | > log_mle: -0.25473 (-0.24645) + | > loss_dur: 0.16149 (0.13300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.28203 (10.65890) + | > current_lr: 0.00004 + | > step_time: 1.11610 (2.77836) + | > loader_time: 0.08500 (0.03841) + + + --> STEP: 80/234 -- GLOBAL_STEP: 34010 + | > loss: -0.10954 (-0.11304) + | > log_mle: -0.23688 (-0.24653) + | > loss_dur: 0.12735 (0.13349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.64071 (10.60174) + | > current_lr: 0.00004 + | > step_time: 1.40120 (2.72201) + | > loader_time: 0.00260 (0.03844) + + + --> STEP: 85/234 -- GLOBAL_STEP: 34015 + | > loss: -0.11323 (-0.11259) + | > log_mle: -0.25356 (-0.24704) + | > loss_dur: 0.14034 (0.13444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.95378 (10.69124) + | > current_lr: 0.00004 + | > step_time: 1.59930 (2.68579) + | > loader_time: 0.00230 (0.03948) + + + --> STEP: 90/234 -- GLOBAL_STEP: 34020 + | > loss: -0.11060 (-0.11276) + | > log_mle: -0.27978 (-0.24862) + | > loss_dur: 0.16918 (0.13586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05835 (10.88338) + | > current_lr: 0.00004 + | > step_time: 2.57880 (2.65228) + | > loader_time: 0.00290 (0.04016) + + + --> STEP: 95/234 -- GLOBAL_STEP: 34025 + | > loss: -0.17905 (-0.11433) + | > log_mle: -0.36574 (-0.25210) + | > loss_dur: 0.18669 (0.13777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.28218 (11.47597) + | > current_lr: 0.00004 + | > step_time: 1.07830 (2.62304) + | > loader_time: 0.00270 (0.04019) + + + --> STEP: 100/234 -- GLOBAL_STEP: 34030 + | > loss: -0.14024 (-0.11483) + | > log_mle: -0.29216 (-0.25386) + | > loss_dur: 0.15192 (0.13903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.68006 (11.66382) + | > current_lr: 0.00004 + | > step_time: 2.19620 (2.58209) + | > loader_time: 0.00270 (0.03833) + + + --> STEP: 105/234 -- GLOBAL_STEP: 34035 + | > loss: -0.12760 (-0.11620) + | > log_mle: -0.26867 (-0.25669) + | > loss_dur: 0.14108 (0.14049) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.48568 (12.24879) + | > current_lr: 0.00004 + | > step_time: 2.39770 (2.56701) + | > loader_time: 0.00240 (0.03833) + + + --> STEP: 110/234 -- GLOBAL_STEP: 34040 + | > loss: -0.13216 (-0.11663) + | > log_mle: -0.29511 (-0.25903) + | > loss_dur: 0.16295 (0.14240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.29887 (12.75807) + | > current_lr: 0.00004 + | > step_time: 1.89560 (2.52926) + | > loader_time: 0.00280 (0.03768) + + + --> STEP: 115/234 -- GLOBAL_STEP: 34045 + | > loss: -0.12942 (-0.11786) + | > log_mle: -0.31421 (-0.26220) + | > loss_dur: 0.18478 (0.14434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.54922 (13.43051) + | > current_lr: 0.00004 + | > step_time: 3.11470 (2.52895) + | > loader_time: 0.09950 (0.03843) + + + --> STEP: 120/234 -- GLOBAL_STEP: 34050 + | > loss: -0.18062 (-0.11887) + | > log_mle: -0.36755 (-0.26493) + | > loss_dur: 0.18693 (0.14606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.14742 (13.98078) + | > current_lr: 0.00004 + | > step_time: 1.38180 (2.53066) + | > loader_time: 0.00330 (0.03700) + + + --> STEP: 125/234 -- GLOBAL_STEP: 34055 + | > loss: -0.15540 (-0.11931) + | > log_mle: -0.34941 (-0.26634) + | > loss_dur: 0.19401 (0.14703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.38983 (14.33129) + | > current_lr: 0.00004 + | > step_time: 1.59310 (2.50734) + | > loader_time: 0.00340 (0.03636) + + + --> STEP: 130/234 -- GLOBAL_STEP: 34060 + | > loss: -0.16671 (-0.12093) + | > log_mle: -0.36360 (-0.26968) + | > loss_dur: 0.19689 (0.14874) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.93823 (14.99632) + | > current_lr: 0.00004 + | > step_time: 1.10250 (2.48630) + | > loader_time: 0.08440 (0.03694) + + + --> STEP: 135/234 -- GLOBAL_STEP: 34065 + | > loss: -0.12371 (-0.12277) + | > log_mle: -0.29334 (-0.27304) + | > loss_dur: 0.16963 (0.15027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.96772 (15.70812) + | > current_lr: 0.00004 + | > step_time: 3.28860 (2.46674) + | > loader_time: 0.00680 (0.03632) + + + --> STEP: 140/234 -- GLOBAL_STEP: 34070 + | > loss: -0.13516 (-0.12477) + | > log_mle: -0.32479 (-0.27692) + | > loss_dur: 0.18962 (0.15216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.91836 (16.63174) + | > current_lr: 0.00004 + | > step_time: 0.99870 (2.45777) + | > loader_time: 0.00480 (0.03652) + + + --> STEP: 145/234 -- GLOBAL_STEP: 34075 + | > loss: -0.22284 (-0.12710) + | > log_mle: -0.42685 (-0.28137) + | > loss_dur: 0.20401 (0.15428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.45377 (17.81392) + | > current_lr: 0.00004 + | > step_time: 7.39760 (2.53355) + | > loader_time: 0.11040 (0.03687) + + + --> STEP: 150/234 -- GLOBAL_STEP: 34080 + | > loss: -0.19639 (-0.12968) + | > log_mle: -0.41138 (-0.28551) + | > loss_dur: 0.21499 (0.15583) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.79452 (18.86737) + | > current_lr: 0.00004 + | > step_time: 1.71380 (2.51164) + | > loader_time: 0.00360 (0.03627) + + + --> STEP: 155/234 -- GLOBAL_STEP: 34085 + | > loss: -0.22043 (-0.13262) + | > log_mle: -0.44885 (-0.29022) + | > loss_dur: 0.22842 (0.15759) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.89858 (21.84922) + | > current_lr: 0.00004 + | > step_time: 1.29280 (2.49811) + | > loader_time: 0.00440 (0.03520) + + + --> STEP: 160/234 -- GLOBAL_STEP: 34090 + | > loss: -0.23797 (-0.13504) + | > log_mle: -0.45810 (-0.29440) + | > loss_dur: 0.22013 (0.15936) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.70279 (22.92323) + | > current_lr: 0.00004 + | > step_time: 8.19250 (2.58756) + | > loader_time: 0.00190 (0.03544) + + + --> STEP: 165/234 -- GLOBAL_STEP: 34095 + | > loss: -0.23642 (-0.13759) + | > log_mle: -0.45686 (-0.29852) + | > loss_dur: 0.22044 (0.16093) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.86353 (23.92496) + | > current_lr: 0.00004 + | > step_time: 2.60610 (2.59840) + | > loader_time: 0.00360 (0.03802) + + + --> STEP: 170/234 -- GLOBAL_STEP: 34100 + | > loss: -0.23439 (-0.14033) + | > log_mle: -0.48902 (-0.30309) + | > loss_dur: 0.25463 (0.16276) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.06590 (25.07124) + | > current_lr: 0.00004 + | > step_time: 6.50300 (2.70380) + | > loader_time: 0.09720 (0.03860) + + + --> STEP: 175/234 -- GLOBAL_STEP: 34105 + | > loss: -0.24308 (-0.14380) + | > log_mle: -0.47966 (-0.30833) + | > loss_dur: 0.23657 (0.16453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.07508 (26.06329) + | > current_lr: 0.00004 + | > step_time: 1.89910 (2.75180) + | > loader_time: 0.00240 (0.03917) + + + --> STEP: 180/234 -- GLOBAL_STEP: 34110 + | > loss: -0.25630 (-0.14681) + | > log_mle: -0.48575 (-0.31341) + | > loss_dur: 0.22944 (0.16659) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.92369 (27.01810) + | > current_lr: 0.00004 + | > step_time: 4.30850 (2.76824) + | > loader_time: 0.09750 (0.03980) + + + --> STEP: 185/234 -- GLOBAL_STEP: 34115 + | > loss: -0.25783 (-0.14954) + | > log_mle: -0.50519 (-0.31805) + | > loss_dur: 0.24735 (0.16851) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.26826 (28.30838) + | > current_lr: 0.00004 + | > step_time: 2.89970 (2.75297) + | > loader_time: 0.08950 (0.03930) + + + --> STEP: 190/234 -- GLOBAL_STEP: 34120 + | > loss: -0.27584 (-0.15249) + | > log_mle: -0.49091 (-0.32264) + | > loss_dur: 0.21507 (0.17015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.94984 (29.17397) + | > current_lr: 0.00004 + | > step_time: 2.49470 (2.80619) + | > loader_time: 0.00460 (0.04150) + + + --> STEP: 195/234 -- GLOBAL_STEP: 34125 + | > loss: -0.27156 (-0.15595) + | > log_mle: -0.50516 (-0.32750) + | > loss_dur: 0.23359 (0.17155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.33651 (30.32476) + | > current_lr: 0.00004 + | > step_time: 7.51730 (2.87342) + | > loader_time: 0.09160 (0.04237) + + + --> STEP: 200/234 -- GLOBAL_STEP: 34130 + | > loss: -0.25430 (-0.15882) + | > log_mle: -0.51389 (-0.33201) + | > loss_dur: 0.25959 (0.17319) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.34112 (31.33344) + | > current_lr: 0.00004 + | > step_time: 3.31510 (2.90213) + | > loader_time: 0.08390 (0.04320) + + + --> STEP: 205/234 -- GLOBAL_STEP: 34135 + | > loss: -0.26685 (-0.16139) + | > log_mle: -0.50044 (-0.33619) + | > loss_dur: 0.23359 (0.17480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.78026 (32.61360) + | > current_lr: 0.00004 + | > step_time: 10.99310 (2.99145) + | > loader_time: 0.00490 (0.04551) + + + --> STEP: 210/234 -- GLOBAL_STEP: 34140 + | > loss: -0.32769 (-0.16476) + | > log_mle: -0.57941 (-0.34122) + | > loss_dur: 0.25172 (0.17646) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 122.70837 (33.93070) + | > current_lr: 0.00004 + | > step_time: 8.19810 (3.07029) + | > loader_time: 0.10590 (0.04722) + + + --> STEP: 215/234 -- GLOBAL_STEP: 34145 + | > loss: -0.26780 (-0.16802) + | > log_mle: -0.51152 (-0.34618) + | > loss_dur: 0.24372 (0.17816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.70542 (35.74300) + | > current_lr: 0.00004 + | > step_time: 5.19230 (3.18501) + | > loader_time: 0.00320 (0.04750) + + + --> STEP: 220/234 -- GLOBAL_STEP: 34150 + | > loss: -0.30812 (-0.17156) + | > log_mle: -0.56225 (-0.35142) + | > loss_dur: 0.25412 (0.17986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.46252 (37.14557) + | > current_lr: 0.00004 + | > step_time: 1.99260 (3.19679) + | > loader_time: 0.00330 (0.04729) + + + --> STEP: 225/234 -- GLOBAL_STEP: 34155 + | > loss: -0.36267 (-0.17480) + | > log_mle: -0.62756 (-0.35641) + | > loss_dur: 0.26489 (0.18161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 154.81807 (38.48645) + | > current_lr: 0.00004 + | > step_time: 0.22870 (3.14230) + | > loader_time: 0.00350 (0.04707) + + + --> STEP: 230/234 -- GLOBAL_STEP: 34160 + | > loss: -0.28172 (-0.17718) + | > log_mle: -0.61887 (-0.36124) + | > loss_dur: 0.33715 (0.18406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 168.51668 (40.92479) + | > current_lr: 0.00004 + | > step_time: 0.26910 (3.07957) + | > loader_time: 0.00680 (0.04615) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.95526 (-0.27115) + | > avg_loss: -0.18249 (-0.00259) + | > avg_log_mle: -0.41276 (-0.00849) + | > avg_loss_dur: 0.23026 (+0.00590) + + + > EPOCH: 146/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 15:51:37)  + + --> STEP: 1/234 -- GLOBAL_STEP: 34165 + | > loss: -0.11659 (-0.11659) + | > log_mle: -0.25059 (-0.25059) + | > loss_dur: 0.13400 (0.13400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.08479 (16.08479) + | > current_lr: 0.00004 + | > step_time: 6.10100 (6.10098) + | > loader_time: 1.40180 (1.40185) + + + --> STEP: 6/234 -- GLOBAL_STEP: 34170 + | > loss: -0.11409 (-0.09756) + | > log_mle: -0.23893 (-0.24591) + | > loss_dur: 0.12484 (0.14835) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.03289 (14.59866) + | > current_lr: 0.00004 + | > step_time: 2.78930 (4.37939) + | > loader_time: 0.00140 (0.51615) + + + --> STEP: 11/234 -- GLOBAL_STEP: 34175 + | > loss: -0.14145 (-0.10799) + | > log_mle: -0.25088 (-0.25054) + | > loss_dur: 0.10944 (0.14255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.22724 (13.28364) + | > current_lr: 0.00004 + | > step_time: 6.40020 (4.80028) + | > loader_time: 0.11220 (0.29265) + + + --> STEP: 16/234 -- GLOBAL_STEP: 34180 + | > loss: -0.14720 (-0.11706) + | > log_mle: -0.24959 (-0.25125) + | > loss_dur: 0.10239 (0.13419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.30515 (12.21202) + | > current_lr: 0.00004 + | > step_time: 12.10290 (5.28895) + | > loader_time: 0.09880 (0.22488) + + + --> STEP: 21/234 -- GLOBAL_STEP: 34185 + | > loss: -0.11037 (-0.11675) + | > log_mle: -0.22817 (-0.24797) + | > loss_dur: 0.11780 (0.13122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.81994 (11.34118) + | > current_lr: 0.00004 + | > step_time: 3.59110 (4.70590) + | > loader_time: 0.00130 (0.18085) + + + --> STEP: 26/234 -- GLOBAL_STEP: 34190 + | > loss: -0.11234 (-0.11861) + | > log_mle: -0.24519 (-0.24723) + | > loss_dur: 0.13285 (0.12862) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.42398 (11.07205) + | > current_lr: 0.00004 + | > step_time: 5.60180 (4.56271) + | > loader_time: 0.00770 (0.15003) + + + --> STEP: 31/234 -- GLOBAL_STEP: 34195 + | > loss: -0.09279 (-0.11935) + | > log_mle: -0.24442 (-0.24672) + | > loss_dur: 0.15163 (0.12737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.27937 (10.95185) + | > current_lr: 0.00004 + | > step_time: 4.61230 (4.43947) + | > loader_time: 0.08660 (0.13494) + + + --> STEP: 36/234 -- GLOBAL_STEP: 34200 + | > loss: -0.10765 (-0.11831) + | > log_mle: -0.24396 (-0.24644) + | > loss_dur: 0.13631 (0.12813) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.35356 (11.02908) + | > current_lr: 0.00004 + | > step_time: 4.59180 (4.37007) + | > loader_time: 0.00490 (0.12235) + + + --> STEP: 41/234 -- GLOBAL_STEP: 34205 + | > loss: -0.12750 (-0.11772) + | > log_mle: -0.24500 (-0.24610) + | > loss_dur: 0.11750 (0.12837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.10910 (10.84400) + | > current_lr: 0.00004 + | > step_time: 0.90170 (4.24000) + | > loader_time: 0.00260 (0.11260) + + + --> STEP: 46/234 -- GLOBAL_STEP: 34210 + | > loss: -0.10352 (-0.11737) + | > log_mle: -0.24348 (-0.24633) + | > loss_dur: 0.13996 (0.12896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.16698 (10.60677) + | > current_lr: 0.00004 + | > step_time: 2.01000 (3.90301) + | > loader_time: 0.08740 (0.10408) + + + --> STEP: 51/234 -- GLOBAL_STEP: 34215 + | > loss: -0.11250 (-0.11725) + | > log_mle: -0.23560 (-0.24577) + | > loss_dur: 0.12310 (0.12853) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.18732 (10.22005) + | > current_lr: 0.00004 + | > step_time: 1.39300 (3.69626) + | > loader_time: 0.00200 (0.09590) + + + --> STEP: 56/234 -- GLOBAL_STEP: 34220 + | > loss: -0.10208 (-0.11700) + | > log_mle: -0.25025 (-0.24622) + | > loss_dur: 0.14817 (0.12922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.58138 (10.04789) + | > current_lr: 0.00004 + | > step_time: 1.91370 (3.48415) + | > loader_time: 0.08700 (0.08906) + + + --> STEP: 61/234 -- GLOBAL_STEP: 34225 + | > loss: -0.11879 (-0.11721) + | > log_mle: -0.24892 (-0.24663) + | > loss_dur: 0.13013 (0.12942) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.40473 (9.91174) + | > current_lr: 0.00004 + | > step_time: 3.11160 (3.34178) + | > loader_time: 0.00280 (0.08338) + + + --> STEP: 66/234 -- GLOBAL_STEP: 34230 + | > loss: -0.11664 (-0.11644) + | > log_mle: -0.23881 (-0.24698) + | > loss_dur: 0.12217 (0.13054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.60436 (9.85175) + | > current_lr: 0.00004 + | > step_time: 1.31330 (3.21883) + | > loader_time: 0.00400 (0.07726) + + + --> STEP: 71/234 -- GLOBAL_STEP: 34235 + | > loss: -0.09465 (-0.11486) + | > log_mle: -0.27998 (-0.24728) + | > loss_dur: 0.18533 (0.13243) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.59858 (9.93757) + | > current_lr: 0.00004 + | > step_time: 1.32310 (3.11830) + | > loader_time: 0.00210 (0.07328) + + + --> STEP: 76/234 -- GLOBAL_STEP: 34240 + | > loss: -0.12153 (-0.11400) + | > log_mle: -0.26031 (-0.24760) + | > loss_dur: 0.13878 (0.13359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.09350 (10.15310) + | > current_lr: 0.00004 + | > step_time: 2.10950 (3.02792) + | > loader_time: 0.08350 (0.06969) + + + --> STEP: 81/234 -- GLOBAL_STEP: 34245 + | > loss: -0.12256 (-0.11402) + | > log_mle: -0.26762 (-0.24787) + | > loss_dur: 0.14506 (0.13385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18864 (10.13367) + | > current_lr: 0.00004 + | > step_time: 1.59840 (2.96295) + | > loader_time: 0.00630 (0.06682) + + + --> STEP: 86/234 -- GLOBAL_STEP: 34250 + | > loss: -0.09762 (-0.11364) + | > log_mle: -0.26253 (-0.24838) + | > loss_dur: 0.16491 (0.13474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.76839 (10.23541) + | > current_lr: 0.00004 + | > step_time: 0.97960 (2.87297) + | > loader_time: 0.00160 (0.06307) + + + --> STEP: 91/234 -- GLOBAL_STEP: 34255 + | > loss: -0.10766 (-0.11392) + | > log_mle: -0.27763 (-0.25008) + | > loss_dur: 0.16997 (0.13616) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.52196 (10.51955) + | > current_lr: 0.00004 + | > step_time: 3.20040 (2.81604) + | > loader_time: 0.00350 (0.05979) + + + --> STEP: 96/234 -- GLOBAL_STEP: 34260 + | > loss: -0.11702 (-0.11578) + | > log_mle: -0.26435 (-0.25339) + | > loss_dur: 0.14733 (0.13761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.51911 (11.07782) + | > current_lr: 0.00004 + | > step_time: 1.30600 (2.78727) + | > loader_time: 0.00220 (0.05772) + + + --> STEP: 101/234 -- GLOBAL_STEP: 34265 + | > loss: -0.14699 (-0.11670) + | > log_mle: -0.32146 (-0.25567) + | > loss_dur: 0.17447 (0.13897) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.88982 (11.44010) + | > current_lr: 0.00004 + | > step_time: 1.59840 (2.73392) + | > loader_time: 0.00390 (0.05499) + + + --> STEP: 106/234 -- GLOBAL_STEP: 34270 + | > loss: -0.11736 (-0.11770) + | > log_mle: -0.31667 (-0.25837) + | > loss_dur: 0.19931 (0.14067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.79586 (12.00604) + | > current_lr: 0.00004 + | > step_time: 3.31130 (2.75019) + | > loader_time: 0.18330 (0.05709) + + + --> STEP: 111/234 -- GLOBAL_STEP: 34275 + | > loss: -0.15070 (-0.11840) + | > log_mle: -0.36726 (-0.26110) + | > loss_dur: 0.21656 (0.14270) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.59659 (12.79083) + | > current_lr: 0.00004 + | > step_time: 3.40140 (2.72982) + | > loader_time: 0.19640 (0.05896) + + + --> STEP: 116/234 -- GLOBAL_STEP: 34280 + | > loss: -0.12775 (-0.11924) + | > log_mle: -0.33539 (-0.26389) + | > loss_dur: 0.20765 (0.14464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.27656 (13.36953) + | > current_lr: 0.00004 + | > step_time: 2.58570 (2.71727) + | > loader_time: 0.00240 (0.05733) + + + --> STEP: 121/234 -- GLOBAL_STEP: 34285 + | > loss: -0.08979 (-0.12007) + | > log_mle: -0.24572 (-0.26579) + | > loss_dur: 0.15593 (0.14572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.14981 (13.70721) + | > current_lr: 0.00004 + | > step_time: 2.21450 (2.69342) + | > loader_time: 0.08510 (0.05726) + + + --> STEP: 126/234 -- GLOBAL_STEP: 34290 + | > loss: -0.17433 (-0.12121) + | > log_mle: -0.37860 (-0.26824) + | > loss_dur: 0.20427 (0.14703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.19799 (14.25838) + | > current_lr: 0.00004 + | > step_time: 2.00470 (2.65410) + | > loader_time: 0.00350 (0.05638) + + + --> STEP: 131/234 -- GLOBAL_STEP: 34295 + | > loss: -0.21334 (-0.12305) + | > log_mle: -0.42695 (-0.27184) + | > loss_dur: 0.21361 (0.14879) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.47760 (14.99870) + | > current_lr: 0.00004 + | > step_time: 1.68940 (2.64351) + | > loader_time: 0.00950 (0.05588) + + + --> STEP: 136/234 -- GLOBAL_STEP: 34300 + | > loss: -0.24335 (-0.12483) + | > log_mle: -0.46630 (-0.27535) + | > loss_dur: 0.22296 (0.15052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.09404 (16.04349) + | > current_lr: 0.00004 + | > step_time: 1.20110 (2.62787) + | > loader_time: 0.08490 (0.05588) + + + --> STEP: 141/234 -- GLOBAL_STEP: 34305 + | > loss: -0.16683 (-0.12622) + | > log_mle: -0.37632 (-0.27840) + | > loss_dur: 0.20950 (0.15217) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.26554 (16.68961) + | > current_lr: 0.00004 + | > step_time: 1.48300 (2.61832) + | > loader_time: 0.00330 (0.05462) + + + --> STEP: 146/234 -- GLOBAL_STEP: 34310 + | > loss: -0.21069 (-0.12877) + | > log_mle: -0.42582 (-0.28306) + | > loss_dur: 0.21514 (0.15429) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.14355 (17.88445) + | > current_lr: 0.00004 + | > step_time: 1.49660 (2.58822) + | > loader_time: 0.00310 (0.05396) + + + --> STEP: 151/234 -- GLOBAL_STEP: 34315 + | > loss: -0.20005 (-0.13118) + | > log_mle: -0.39122 (-0.28686) + | > loss_dur: 0.19117 (0.15568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.83310 (18.74603) + | > current_lr: 0.00004 + | > step_time: 1.51400 (2.56356) + | > loader_time: 0.08840 (0.05438) + + + --> STEP: 156/234 -- GLOBAL_STEP: 34320 + | > loss: -0.22371 (-0.13461) + | > log_mle: -0.42828 (-0.29205) + | > loss_dur: 0.20456 (0.15744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.11660 (20.35629) + | > current_lr: 0.00004 + | > step_time: 3.80640 (2.57104) + | > loader_time: 0.00370 (0.05330) + + + --> STEP: 161/234 -- GLOBAL_STEP: 34325 + | > loss: -0.25244 (-0.13731) + | > log_mle: -0.45923 (-0.29658) + | > loss_dur: 0.20679 (0.15927) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.80077 (21.38696) + | > current_lr: 0.00004 + | > step_time: 1.39770 (2.56808) + | > loader_time: 0.00530 (0.05344) + + + --> STEP: 166/234 -- GLOBAL_STEP: 34330 + | > loss: -0.20260 (-0.13978) + | > log_mle: -0.39930 (-0.30049) + | > loss_dur: 0.19670 (0.16071) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.77565 (22.28685) + | > current_lr: 0.00004 + | > step_time: 6.50400 (2.60836) + | > loader_time: 0.10760 (0.05355) + + + --> STEP: 171/234 -- GLOBAL_STEP: 34335 + | > loss: -0.28900 (-0.14318) + | > log_mle: -0.50472 (-0.30581) + | > loss_dur: 0.21572 (0.16262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.74863 (23.45233) + | > current_lr: 0.00004 + | > step_time: 2.70690 (2.60334) + | > loader_time: 0.08650 (0.05315) + + + --> STEP: 176/234 -- GLOBAL_STEP: 34340 + | > loss: -0.25791 (-0.14667) + | > log_mle: -0.47936 (-0.31112) + | > loss_dur: 0.22145 (0.16445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.64165 (24.70970) + | > current_lr: 0.00004 + | > step_time: 3.49830 (2.59659) + | > loader_time: 0.00530 (0.05430) + + + --> STEP: 181/234 -- GLOBAL_STEP: 34345 + | > loss: -0.20268 (-0.14957) + | > log_mle: -0.41886 (-0.31589) + | > loss_dur: 0.21618 (0.16632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.14605 (25.83333) + | > current_lr: 0.00004 + | > step_time: 1.99980 (2.73140) + | > loader_time: 0.00290 (0.05512) + + + --> STEP: 186/234 -- GLOBAL_STEP: 34350 + | > loss: -0.21436 (-0.15247) + | > log_mle: -0.45678 (-0.32079) + | > loss_dur: 0.24242 (0.16833) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.34195 (27.08459) + | > current_lr: 0.00004 + | > step_time: 1.89950 (2.72451) + | > loader_time: 0.09040 (0.05469) + + + --> STEP: 191/234 -- GLOBAL_STEP: 34355 + | > loss: -0.25994 (-0.15550) + | > log_mle: -0.47608 (-0.32538) + | > loss_dur: 0.21614 (0.16988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.21338 (28.37343) + | > current_lr: 0.00004 + | > step_time: 5.09660 (2.78358) + | > loader_time: 0.19490 (0.05542) + + + --> STEP: 196/234 -- GLOBAL_STEP: 34360 + | > loss: -0.23254 (-0.15869) + | > log_mle: -0.47062 (-0.33013) + | > loss_dur: 0.23808 (0.17144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.67930 (29.65214) + | > current_lr: 0.00004 + | > step_time: 3.58910 (2.78729) + | > loader_time: 0.00290 (0.05519) + + + --> STEP: 201/234 -- GLOBAL_STEP: 34365 + | > loss: -0.19757 (-0.16123) + | > log_mle: -0.43401 (-0.33436) + | > loss_dur: 0.23644 (0.17313) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.13707 (30.88923) + | > current_lr: 0.00004 + | > step_time: 3.79460 (2.79043) + | > loader_time: 0.09980 (0.05490) + + + --> STEP: 206/234 -- GLOBAL_STEP: 34370 + | > loss: -0.28658 (-0.16437) + | > log_mle: -0.53400 (-0.33912) + | > loss_dur: 0.24742 (0.17475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.23074 (32.15156) + | > current_lr: 0.00004 + | > step_time: 2.70350 (2.78828) + | > loader_time: 0.08600 (0.05529) + + + --> STEP: 211/234 -- GLOBAL_STEP: 34375 + | > loss: -0.34774 (-0.16793) + | > log_mle: -0.61066 (-0.34448) + | > loss_dur: 0.26292 (0.17655) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.75568 (33.31811) + | > current_lr: 0.00004 + | > step_time: 6.20710 (2.85257) + | > loader_time: 0.00410 (0.05723) + + + --> STEP: 216/234 -- GLOBAL_STEP: 34380 + | > loss: -0.33816 (-0.17155) + | > log_mle: -0.59934 (-0.34971) + | > loss_dur: 0.26118 (0.17816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.54552 (34.47167) + | > current_lr: 0.00004 + | > step_time: 8.89290 (2.93230) + | > loader_time: 0.00290 (0.05783) + + + --> STEP: 221/234 -- GLOBAL_STEP: 34385 + | > loss: -0.28715 (-0.17518) + | > log_mle: -0.52197 (-0.35493) + | > loss_dur: 0.23482 (0.17974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.40839 (35.69835) + | > current_lr: 0.00004 + | > step_time: 1.61330 (2.96836) + | > loader_time: 0.08470 (0.05817) + + + --> STEP: 226/234 -- GLOBAL_STEP: 34390 + | > loss: -0.38016 (-0.17920) + | > log_mle: -0.63109 (-0.36076) + | > loss_dur: 0.25093 (0.18156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.44631 (36.98128) + | > current_lr: 0.00004 + | > step_time: 0.23430 (2.92134) + | > loader_time: 0.00360 (0.05734) + + + --> STEP: 231/234 -- GLOBAL_STEP: 34395 + | > loss: -0.27456 (-0.18186) + | > log_mle: -0.67526 (-0.36641) + | > loss_dur: 0.40070 (0.18455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 120.18891 (38.82608) + | > current_lr: 0.00004 + | > step_time: 0.29350 (2.86387) + | > loader_time: 0.00650 (0.05620) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.10183 (-0.85343) + | > avg_loss: -0.19575 (-0.01325) + | > avg_log_mle: -0.42937 (-0.01661) + | > avg_loss_dur: 0.23362 (+0.00336) + + + > EPOCH: 147/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 16:03:57)  + + --> STEP: 2/234 -- GLOBAL_STEP: 34400 + | > loss: -0.12856 (-0.12987) + | > log_mle: -0.25650 (-0.25575) + | > loss_dur: 0.12795 (0.12588) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.67658 (10.35472) + | > current_lr: 0.00004 + | > step_time: 14.20170 (8.35541) + | > loader_time: 0.09750 (0.04953) + + + --> STEP: 7/234 -- GLOBAL_STEP: 34405 + | > loss: -0.13433 (-0.10913) + | > log_mle: -0.26243 (-0.25201) + | > loss_dur: 0.12809 (0.14289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.48687 (14.70130) + | > current_lr: 0.00004 + | > step_time: 2.29830 (5.46989) + | > loader_time: 0.08380 (0.02705) + + + --> STEP: 12/234 -- GLOBAL_STEP: 34410 + | > loss: -0.12935 (-0.11532) + | > log_mle: -0.25450 (-0.25405) + | > loss_dur: 0.12515 (0.13873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.60127 (13.51723) + | > current_lr: 0.00004 + | > step_time: 1.30100 (3.68927) + | > loader_time: 0.07720 (0.02907) + + + --> STEP: 17/234 -- GLOBAL_STEP: 34415 + | > loss: -0.10191 (-0.12181) + | > log_mle: -0.22941 (-0.25293) + | > loss_dur: 0.12750 (0.13113) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42804 (12.33873) + | > current_lr: 0.00004 + | > step_time: 3.10950 (3.25871) + | > loader_time: 0.08800 (0.03556) + + + --> STEP: 22/234 -- GLOBAL_STEP: 34420 + | > loss: -0.12752 (-0.12123) + | > log_mle: -0.25167 (-0.25088) + | > loss_dur: 0.12415 (0.12965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.42073 (11.81564) + | > current_lr: 0.00004 + | > step_time: 1.80410 (3.15944) + | > loader_time: 0.00150 (0.04510) + + + --> STEP: 27/234 -- GLOBAL_STEP: 34425 + | > loss: -0.13824 (-0.12331) + | > log_mle: -0.25334 (-0.25015) + | > loss_dur: 0.11510 (0.12685) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.41623 (11.13717) + | > current_lr: 0.00004 + | > step_time: 8.39870 (3.44161) + | > loader_time: 0.00170 (0.05906) + + + --> STEP: 32/234 -- GLOBAL_STEP: 34430 + | > loss: -0.16473 (-0.12406) + | > log_mle: -0.26251 (-0.25006) + | > loss_dur: 0.09778 (0.12600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.41495 (10.85125) + | > current_lr: 0.00004 + | > step_time: 0.59140 (3.10599) + | > loader_time: 0.00210 (0.05275) + + + --> STEP: 37/234 -- GLOBAL_STEP: 34435 + | > loss: -0.13121 (-0.12273) + | > log_mle: -0.24142 (-0.24932) + | > loss_dur: 0.11020 (0.12659) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.55930 (10.91265) + | > current_lr: 0.00004 + | > step_time: 1.21130 (2.86487) + | > loader_time: 0.00140 (0.04587) + + + --> STEP: 42/234 -- GLOBAL_STEP: 34440 + | > loss: -0.10145 (-0.12132) + | > log_mle: -0.23613 (-0.24887) + | > loss_dur: 0.13468 (0.12754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.90318 (10.69291) + | > current_lr: 0.00004 + | > step_time: 1.60240 (2.74513) + | > loader_time: 0.00680 (0.04282) + + + --> STEP: 47/234 -- GLOBAL_STEP: 34445 + | > loss: -0.10293 (-0.12056) + | > log_mle: -0.24487 (-0.24919) + | > loss_dur: 0.14194 (0.12862) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.62716 (10.64369) + | > current_lr: 0.00004 + | > step_time: 3.20530 (2.68741) + | > loader_time: 0.00370 (0.04248) + + + --> STEP: 52/234 -- GLOBAL_STEP: 34450 + | > loss: -0.08771 (-0.11975) + | > log_mle: -0.24095 (-0.24848) + | > loss_dur: 0.15324 (0.12873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.59544 (10.34979) + | > current_lr: 0.00004 + | > step_time: 1.32950 (2.59109) + | > loader_time: 0.00220 (0.04188) + + + --> STEP: 57/234 -- GLOBAL_STEP: 34455 + | > loss: -0.09229 (-0.11944) + | > log_mle: -0.23152 (-0.24867) + | > loss_dur: 0.13923 (0.12923) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.26267 (10.23292) + | > current_lr: 0.00004 + | > step_time: 0.91560 (2.47535) + | > loader_time: 0.00340 (0.03988) + + + --> STEP: 62/234 -- GLOBAL_STEP: 34460 + | > loss: -0.07126 (-0.11942) + | > log_mle: -0.28384 (-0.24967) + | > loss_dur: 0.21258 (0.13025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.45686 (10.61177) + | > current_lr: 0.00004 + | > step_time: 1.20190 (2.43229) + | > loader_time: 0.00190 (0.03844) + + + --> STEP: 67/234 -- GLOBAL_STEP: 34465 + | > loss: -0.11329 (-0.11894) + | > log_mle: -0.26430 (-0.24949) + | > loss_dur: 0.15102 (0.13055) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.69342 (10.50919) + | > current_lr: 0.00004 + | > step_time: 1.68510 (2.41300) + | > loader_time: 0.00210 (0.03715) + + + --> STEP: 72/234 -- GLOBAL_STEP: 34470 + | > loss: -0.09068 (-0.11692) + | > log_mle: -0.24276 (-0.24928) + | > loss_dur: 0.15208 (0.13236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.36362 (10.60901) + | > current_lr: 0.00004 + | > step_time: 1.01530 (2.33744) + | > loader_time: 0.08760 (0.03960) + + + --> STEP: 77/234 -- GLOBAL_STEP: 34475 + | > loss: -0.12689 (-0.11630) + | > log_mle: -0.25745 (-0.24978) + | > loss_dur: 0.13056 (0.13348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.30894 (10.58712) + | > current_lr: 0.00004 + | > step_time: 1.39940 (2.28847) + | > loader_time: 0.08710 (0.04053) + + + --> STEP: 82/234 -- GLOBAL_STEP: 34480 + | > loss: -0.10799 (-0.11598) + | > log_mle: -0.24852 (-0.24982) + | > loss_dur: 0.14053 (0.13384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.67762 (10.54468) + | > current_lr: 0.00004 + | > step_time: 1.60660 (2.25124) + | > loader_time: 0.00310 (0.03924) + + + --> STEP: 87/234 -- GLOBAL_STEP: 34485 + | > loss: -0.10729 (-0.11564) + | > log_mle: -0.25724 (-0.25034) + | > loss_dur: 0.14995 (0.13470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.66929 (10.64240) + | > current_lr: 0.00004 + | > step_time: 1.78680 (2.23064) + | > loader_time: 0.10100 (0.04037) + + + --> STEP: 92/234 -- GLOBAL_STEP: 34490 + | > loss: -0.15741 (-0.11652) + | > log_mle: -0.30717 (-0.25259) + | > loss_dur: 0.14976 (0.13607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.83597 (10.93667) + | > current_lr: 0.00004 + | > step_time: 2.70940 (2.23338) + | > loader_time: 0.09030 (0.04020) + + + --> STEP: 97/234 -- GLOBAL_STEP: 34495 + | > loss: -0.13303 (-0.11790) + | > log_mle: -0.28998 (-0.25569) + | > loss_dur: 0.15694 (0.13779) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.49616 (11.50287) + | > current_lr: 0.00004 + | > step_time: 2.26090 (2.22194) + | > loader_time: 0.00260 (0.03827) + + + --> STEP: 102/234 -- GLOBAL_STEP: 34500 + | > loss: -0.10908 (-0.11857) + | > log_mle: -0.27327 (-0.25775) + | > loss_dur: 0.16419 (0.13919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.29510 (11.96720) + | > current_lr: 0.00004 + | > step_time: 2.11720 (2.19731) + | > loader_time: 0.08680 (0.03822) + + + --> STEP: 107/234 -- GLOBAL_STEP: 34505 + | > loss: -0.13641 (-0.11994) + | > log_mle: -0.31878 (-0.26091) + | > loss_dur: 0.18237 (0.14097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.51622 (12.60661) + | > current_lr: 0.00004 + | > step_time: 1.38430 (2.17978) + | > loader_time: 0.00240 (0.03817) + + + --> STEP: 112/234 -- GLOBAL_STEP: 34510 + | > loss: -0.13237 (-0.12070) + | > log_mle: -0.32863 (-0.26373) + | > loss_dur: 0.19626 (0.14304) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.06342 (13.42868) + | > current_lr: 0.00004 + | > step_time: 1.80340 (2.16734) + | > loader_time: 0.10880 (0.03827) + + + --> STEP: 117/234 -- GLOBAL_STEP: 34515 + | > loss: -0.15486 (-0.12176) + | > log_mle: -0.32486 (-0.26639) + | > loss_dur: 0.17000 (0.14463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.96557 (13.97324) + | > current_lr: 0.00004 + | > step_time: 2.19630 (2.18096) + | > loader_time: 0.00330 (0.03828) + + + --> STEP: 122/234 -- GLOBAL_STEP: 34520 + | > loss: -0.12746 (-0.12222) + | > log_mle: -0.29888 (-0.26804) + | > loss_dur: 0.17142 (0.14583) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.69744 (14.35925) + | > current_lr: 0.00004 + | > step_time: 1.90050 (2.17023) + | > loader_time: 0.00310 (0.03753) + + + --> STEP: 127/234 -- GLOBAL_STEP: 34525 + | > loss: -0.15810 (-0.12366) + | > log_mle: -0.35887 (-0.27102) + | > loss_dur: 0.20077 (0.14736) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.38619 (15.05298) + | > current_lr: 0.00004 + | > step_time: 1.96540 (2.15641) + | > loader_time: 0.00230 (0.03683) + + + --> STEP: 132/234 -- GLOBAL_STEP: 34530 + | > loss: -0.16801 (-0.12553) + | > log_mle: -0.33999 (-0.27446) + | > loss_dur: 0.17197 (0.14893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.62043 (15.67281) + | > current_lr: 0.00004 + | > step_time: 1.98550 (2.18973) + | > loader_time: 0.00370 (0.03622) + + + --> STEP: 137/234 -- GLOBAL_STEP: 34535 + | > loss: -0.13593 (-0.12726) + | > log_mle: -0.35128 (-0.27809) + | > loss_dur: 0.21535 (0.15084) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.82003 (16.69287) + | > current_lr: 0.00004 + | > step_time: 5.49990 (2.26681) + | > loader_time: 0.09720 (0.03852) + + + --> STEP: 142/234 -- GLOBAL_STEP: 34540 + | > loss: -0.16332 (-0.12880) + | > log_mle: -0.36550 (-0.28123) + | > loss_dur: 0.20218 (0.15243) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.79424 (17.49919) + | > current_lr: 0.00004 + | > step_time: 2.80060 (2.29911) + | > loader_time: 0.08460 (0.04047) + + + --> STEP: 147/234 -- GLOBAL_STEP: 34545 + | > loss: -0.17012 (-0.13156) + | > log_mle: -0.36960 (-0.28604) + | > loss_dur: 0.19947 (0.15448) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.06079 (18.52900) + | > current_lr: 0.00004 + | > step_time: 2.19650 (2.32244) + | > loader_time: 0.00350 (0.04047) + + + --> STEP: 152/234 -- GLOBAL_STEP: 34550 + | > loss: -0.22769 (-0.13448) + | > log_mle: -0.45371 (-0.29052) + | > loss_dur: 0.22601 (0.15604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.41850 (19.54043) + | > current_lr: 0.00004 + | > step_time: 1.02030 (2.33893) + | > loader_time: 0.08610 (0.04103) + + + --> STEP: 157/234 -- GLOBAL_STEP: 34555 + | > loss: -0.19052 (-0.13793) + | > log_mle: -0.40158 (-0.29561) + | > loss_dur: 0.21106 (0.15768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.63153 (20.90514) + | > current_lr: 0.00004 + | > step_time: 1.58810 (2.31929) + | > loader_time: 0.00320 (0.03983) + + + --> STEP: 162/234 -- GLOBAL_STEP: 34560 + | > loss: -0.23426 (-0.14119) + | > log_mle: -0.43081 (-0.30041) + | > loss_dur: 0.19655 (0.15922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.77920 (22.10040) + | > current_lr: 0.00004 + | > step_time: 3.71600 (2.32563) + | > loader_time: 0.08880 (0.04088) + + + --> STEP: 167/234 -- GLOBAL_STEP: 34565 + | > loss: -0.30863 (-0.14417) + | > log_mle: -0.51699 (-0.30484) + | > loss_dur: 0.20836 (0.16068) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.69392 (23.11378) + | > current_lr: 0.00004 + | > step_time: 5.99570 (2.42535) + | > loader_time: 0.08610 (0.04317) + + + --> STEP: 172/234 -- GLOBAL_STEP: 34570 + | > loss: -0.27653 (-0.14756) + | > log_mle: -0.51236 (-0.31018) + | > loss_dur: 0.23583 (0.16262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.87460 (24.22331) + | > current_lr: 0.00004 + | > step_time: 2.30460 (2.42248) + | > loader_time: 0.00330 (0.04290) + + + --> STEP: 177/234 -- GLOBAL_STEP: 34575 + | > loss: -0.25048 (-0.15085) + | > log_mle: -0.46830 (-0.31520) + | > loss_dur: 0.21782 (0.16435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.48360 (25.44374) + | > current_lr: 0.00004 + | > step_time: 4.71810 (2.47107) + | > loader_time: 0.08980 (0.04405) + + + --> STEP: 182/234 -- GLOBAL_STEP: 34580 + | > loss: -0.26584 (-0.15378) + | > log_mle: -0.51399 (-0.32021) + | > loss_dur: 0.24815 (0.16643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.12052 (26.72442) + | > current_lr: 0.00004 + | > step_time: 1.10180 (2.45863) + | > loader_time: 0.00340 (0.04441) + + + --> STEP: 187/234 -- GLOBAL_STEP: 34585 + | > loss: -0.28036 (-0.15682) + | > log_mle: -0.51351 (-0.32506) + | > loss_dur: 0.23315 (0.16824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.97651 (28.11314) + | > current_lr: 0.00004 + | > step_time: 2.70740 (2.46835) + | > loader_time: 0.10170 (0.04533) + + + --> STEP: 192/234 -- GLOBAL_STEP: 34590 + | > loss: -0.31732 (-0.16003) + | > log_mle: -0.53851 (-0.32980) + | > loss_dur: 0.22118 (0.16978) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.08144 (29.45692) + | > current_lr: 0.00004 + | > step_time: 2.59630 (2.46215) + | > loader_time: 0.00280 (0.04466) + + + --> STEP: 197/234 -- GLOBAL_STEP: 34595 + | > loss: -0.29748 (-0.16308) + | > log_mle: -0.51047 (-0.33442) + | > loss_dur: 0.21299 (0.17134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.08392 (30.61666) + | > current_lr: 0.00004 + | > step_time: 2.09190 (2.53879) + | > loader_time: 0.00480 (0.04637) + + + --> STEP: 202/234 -- GLOBAL_STEP: 34600 + | > loss: -0.37179 (-0.16611) + | > log_mle: -0.60123 (-0.33911) + | > loss_dur: 0.22944 (0.17300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.31785 (31.86666) + | > current_lr: 0.00004 + | > step_time: 2.90800 (2.56615) + | > loader_time: 0.00460 (0.04622) + + + --> STEP: 207/234 -- GLOBAL_STEP: 34605 + | > loss: -0.32475 (-0.16894) + | > log_mle: -0.57495 (-0.34365) + | > loss_dur: 0.25020 (0.17471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.61634 (32.88562) + | > current_lr: 0.00004 + | > step_time: 3.10270 (2.61102) + | > loader_time: 0.00410 (0.04562) + + + --> STEP: 212/234 -- GLOBAL_STEP: 34610 + | > loss: -0.31086 (-0.17235) + | > log_mle: -0.56629 (-0.34883) + | > loss_dur: 0.25542 (0.17647) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.92274 (34.15435) + | > current_lr: 0.00004 + | > step_time: 11.19800 (2.70421) + | > loader_time: 0.10680 (0.04704) + + + --> STEP: 217/234 -- GLOBAL_STEP: 34615 + | > loss: -0.33743 (-0.17592) + | > log_mle: -0.59196 (-0.35407) + | > loss_dur: 0.25453 (0.17815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.51548 (35.48841) + | > current_lr: 0.00004 + | > step_time: 2.20030 (2.71472) + | > loader_time: 0.09440 (0.04647) + + + --> STEP: 222/234 -- GLOBAL_STEP: 34620 + | > loss: -0.31236 (-0.17919) + | > log_mle: -0.59457 (-0.35908) + | > loss_dur: 0.28221 (0.17988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.49670 (37.04940) + | > current_lr: 0.00004 + | > step_time: 2.50740 (2.75592) + | > loader_time: 0.00310 (0.04762) + + + --> STEP: 227/234 -- GLOBAL_STEP: 34625 + | > loss: -0.30525 (-0.18285) + | > log_mle: -0.57719 (-0.36451) + | > loss_dur: 0.27194 (0.18167) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.67722 (38.32099) + | > current_lr: 0.00004 + | > step_time: 0.24550 (2.71669) + | > loader_time: 0.00390 (0.04665) + + + --> STEP: 232/234 -- GLOBAL_STEP: 34630 + | > loss: -0.25425 (-0.18549) + | > log_mle: -0.77407 (-0.37123) + | > loss_dur: 0.51981 (0.18574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 175.15901 (40.26985) + | > current_lr: 0.00004 + | > step_time: 0.32430 (2.66406) + | > loader_time: 0.00460 (0.04575) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12327 (+0.02144) + | > avg_loss: -0.19277 (+0.00297) + | > avg_log_mle: -0.42250 (+0.00687) + | > avg_loss_dur: 0.22973 (-0.00390) + + + > EPOCH: 148/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 16:15:30)  + + --> STEP: 3/234 -- GLOBAL_STEP: 34635 + | > loss: -0.05807 (-0.10697) + | > log_mle: -0.25151 (-0.25610) + | > loss_dur: 0.19344 (0.14914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.65661 (12.68572) + | > current_lr: 0.00004 + | > step_time: 2.71610 (2.67347) + | > loader_time: 0.29010 (0.93141) + + + --> STEP: 8/234 -- GLOBAL_STEP: 34640 + | > loss: -0.15033 (-0.11900) + | > log_mle: -0.26965 (-0.25631) + | > loss_dur: 0.11932 (0.13731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.15621 (13.66223) + | > current_lr: 0.00004 + | > step_time: 7.89900 (3.96393) + | > loader_time: 0.00190 (0.38732) + + + --> STEP: 13/234 -- GLOBAL_STEP: 34645 + | > loss: -0.14770 (-0.12197) + | > log_mle: -0.26102 (-0.25677) + | > loss_dur: 0.11333 (0.13480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.48084 (12.73477) + | > current_lr: 0.00004 + | > step_time: 4.70440 (3.70850) + | > loader_time: 0.00280 (0.36247) + + + --> STEP: 18/234 -- GLOBAL_STEP: 34650 + | > loss: -0.10589 (-0.12313) + | > log_mle: -0.25052 (-0.25501) + | > loss_dur: 0.14463 (0.13188) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.08030 (11.85409) + | > current_lr: 0.00004 + | > step_time: 2.21100 (3.91260) + | > loader_time: 0.08780 (0.27744) + + + --> STEP: 23/234 -- GLOBAL_STEP: 34655 + | > loss: -0.14899 (-0.12485) + | > log_mle: -0.25593 (-0.25314) + | > loss_dur: 0.10695 (0.12828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.24695 (11.36042) + | > current_lr: 0.00004 + | > step_time: 1.69970 (3.66271) + | > loader_time: 0.00230 (0.22187) + + + --> STEP: 28/234 -- GLOBAL_STEP: 34660 + | > loss: -0.15816 (-0.12640) + | > log_mle: -0.24815 (-0.25214) + | > loss_dur: 0.08999 (0.12574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.49224 (10.91273) + | > current_lr: 0.00004 + | > step_time: 3.60410 (4.00867) + | > loader_time: 0.08250 (0.29541) + + + --> STEP: 33/234 -- GLOBAL_STEP: 34665 + | > loss: -0.11963 (-0.12565) + | > log_mle: -0.24044 (-0.25142) + | > loss_dur: 0.12081 (0.12577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.64158 (10.80017) + | > current_lr: 0.00004 + | > step_time: 8.90010 (4.26494) + | > loader_time: 0.40240 (0.26864) + + + --> STEP: 38/234 -- GLOBAL_STEP: 34670 + | > loss: -0.11415 (-0.12312) + | > log_mle: -0.25910 (-0.25101) + | > loss_dur: 0.14495 (0.12789) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76461 (10.87650) + | > current_lr: 0.00004 + | > step_time: 1.59510 (4.15694) + | > loader_time: 0.00550 (0.24600) + + + --> STEP: 43/234 -- GLOBAL_STEP: 34675 + | > loss: -0.10183 (-0.12121) + | > log_mle: -0.25368 (-0.25021) + | > loss_dur: 0.15185 (0.12900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.94318 (10.77816) + | > current_lr: 0.00004 + | > step_time: 1.69500 (3.85677) + | > loader_time: 0.00180 (0.22000) + + + --> STEP: 48/234 -- GLOBAL_STEP: 34680 + | > loss: -0.12338 (-0.12047) + | > log_mle: -0.23820 (-0.24972) + | > loss_dur: 0.11482 (0.12925) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.29369 (10.91618) + | > current_lr: 0.00004 + | > step_time: 1.30470 (3.60844) + | > loader_time: 0.09420 (0.19917) + + + --> STEP: 53/234 -- GLOBAL_STEP: 34685 + | > loss: -0.10690 (-0.11938) + | > log_mle: -0.25211 (-0.24917) + | > loss_dur: 0.14521 (0.12979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.95412 (10.72509) + | > current_lr: 0.00004 + | > step_time: 2.60640 (3.47925) + | > loader_time: 0.20350 (0.18442) + + + --> STEP: 58/234 -- GLOBAL_STEP: 34690 + | > loss: -0.12846 (-0.11939) + | > log_mle: -0.23969 (-0.24897) + | > loss_dur: 0.11123 (0.12957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.50594 (10.68487) + | > current_lr: 0.00004 + | > step_time: 2.40350 (3.31139) + | > loader_time: 0.09370 (0.17157) + + + --> STEP: 63/234 -- GLOBAL_STEP: 34695 + | > loss: -0.08867 (-0.11890) + | > log_mle: -0.24674 (-0.25012) + | > loss_dur: 0.15807 (0.13122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.31655 (10.89223) + | > current_lr: 0.00004 + | > step_time: 1.28660 (3.20363) + | > loader_time: 0.00180 (0.15951) + + + --> STEP: 68/234 -- GLOBAL_STEP: 34700 + | > loss: -0.08660 (-0.11829) + | > log_mle: -0.24103 (-0.24988) + | > loss_dur: 0.15443 (0.13159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.05838 (10.68263) + | > current_lr: 0.00004 + | > step_time: 1.40150 (3.09451) + | > loader_time: 0.00230 (0.15062) + + + --> STEP: 73/234 -- GLOBAL_STEP: 34705 + | > loss: -0.09771 (-0.11656) + | > log_mle: -0.26594 (-0.24998) + | > loss_dur: 0.16823 (0.13342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.33959 (10.82603) + | > current_lr: 0.00004 + | > step_time: 1.61040 (3.01816) + | > loader_time: 0.08540 (0.14509) + + + --> STEP: 78/234 -- GLOBAL_STEP: 34710 + | > loss: -0.09760 (-0.11615) + | > log_mle: -0.24384 (-0.25018) + | > loss_dur: 0.14624 (0.13404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.84961 (10.81031) + | > current_lr: 0.00004 + | > step_time: 2.29800 (2.94139) + | > loader_time: 0.00280 (0.13699) + + + --> STEP: 83/234 -- GLOBAL_STEP: 34715 + | > loss: -0.09082 (-0.11583) + | > log_mle: -0.26581 (-0.25054) + | > loss_dur: 0.17499 (0.13471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.22115 (10.87351) + | > current_lr: 0.00004 + | > step_time: 2.51490 (2.91110) + | > loader_time: 0.18670 (0.13116) + + + --> STEP: 88/234 -- GLOBAL_STEP: 34720 + | > loss: -0.14051 (-0.11596) + | > log_mle: -0.30483 (-0.25148) + | > loss_dur: 0.16432 (0.13552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.29875 (10.96626) + | > current_lr: 0.00004 + | > step_time: 1.58510 (2.86507) + | > loader_time: 0.00340 (0.12580) + + + --> STEP: 93/234 -- GLOBAL_STEP: 34725 + | > loss: -0.12910 (-0.11687) + | > log_mle: -0.30903 (-0.25367) + | > loss_dur: 0.17993 (0.13680) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.45896 (11.57656) + | > current_lr: 0.00004 + | > step_time: 1.59200 (2.83465) + | > loader_time: 0.00240 (0.12461) + + + --> STEP: 98/234 -- GLOBAL_STEP: 34730 + | > loss: -0.09271 (-0.11779) + | > log_mle: -0.24510 (-0.25580) + | > loss_dur: 0.15239 (0.13801) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.48131 (11.96437) + | > current_lr: 0.00004 + | > step_time: 3.10900 (2.80675) + | > loader_time: 0.00230 (0.11843) + + + --> STEP: 103/234 -- GLOBAL_STEP: 34735 + | > loss: -0.15977 (-0.11911) + | > log_mle: -0.34591 (-0.25871) + | > loss_dur: 0.18614 (0.13959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.83453 (12.56833) + | > current_lr: 0.00004 + | > step_time: 4.17500 (2.81801) + | > loader_time: 0.00420 (0.11370) + + + --> STEP: 108/234 -- GLOBAL_STEP: 34740 + | > loss: -0.14036 (-0.12015) + | > log_mle: -0.29094 (-0.26118) + | > loss_dur: 0.15057 (0.14103) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.00032 (13.23574) + | > current_lr: 0.00004 + | > step_time: 1.57580 (2.77445) + | > loader_time: 0.00300 (0.10945) + + + --> STEP: 113/234 -- GLOBAL_STEP: 34745 + | > loss: -0.16546 (-0.12116) + | > log_mle: -0.34036 (-0.26440) + | > loss_dur: 0.17490 (0.14324) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.97010 (13.92885) + | > current_lr: 0.00004 + | > step_time: 1.19720 (2.71861) + | > loader_time: 0.00390 (0.10473) + + + --> STEP: 118/234 -- GLOBAL_STEP: 34750 + | > loss: -0.12586 (-0.12176) + | > log_mle: -0.30881 (-0.26678) + | > loss_dur: 0.18295 (0.14502) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.61604 (14.36973) + | > current_lr: 0.00004 + | > step_time: 1.39790 (2.72136) + | > loader_time: 0.00290 (0.10427) + + + --> STEP: 123/234 -- GLOBAL_STEP: 34755 + | > loss: -0.11076 (-0.12206) + | > log_mle: -0.27717 (-0.26816) + | > loss_dur: 0.16641 (0.14610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76103 (14.59385) + | > current_lr: 0.00004 + | > step_time: 1.10650 (2.69572) + | > loader_time: 0.00400 (0.10016) + + + --> STEP: 128/234 -- GLOBAL_STEP: 34760 + | > loss: -0.17291 (-0.12392) + | > log_mle: -0.33870 (-0.27156) + | > loss_dur: 0.16579 (0.14764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.40324 (15.31595) + | > current_lr: 0.00004 + | > step_time: 1.78680 (2.72076) + | > loader_time: 0.00440 (0.09855) + + + --> STEP: 133/234 -- GLOBAL_STEP: 34765 + | > loss: -0.17369 (-0.12583) + | > log_mle: -0.36548 (-0.27516) + | > loss_dur: 0.19178 (0.14933) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.53692 (15.93609) + | > current_lr: 0.00004 + | > step_time: 1.40260 (2.67561) + | > loader_time: 0.07500 (0.09555) + + + --> STEP: 138/234 -- GLOBAL_STEP: 34770 + | > loss: -0.13499 (-0.12729) + | > log_mle: -0.31747 (-0.27840) + | > loss_dur: 0.18247 (0.15111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.59360 (16.78407) + | > current_lr: 0.00004 + | > step_time: 2.19370 (2.68211) + | > loader_time: 0.00600 (0.09344) + + + --> STEP: 143/234 -- GLOBAL_STEP: 34775 + | > loss: -0.21457 (-0.12953) + | > log_mle: -0.45633 (-0.28256) + | > loss_dur: 0.24176 (0.15303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.62988 (17.86702) + | > current_lr: 0.00004 + | > step_time: 3.79270 (2.74226) + | > loader_time: 0.00610 (0.09160) + + + --> STEP: 148/234 -- GLOBAL_STEP: 34780 + | > loss: -0.19833 (-0.13219) + | > log_mle: -0.37579 (-0.28674) + | > loss_dur: 0.17747 (0.15454) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.64280 (18.84384) + | > current_lr: 0.00004 + | > step_time: 2.31660 (2.77075) + | > loader_time: 0.00310 (0.09113) + + + --> STEP: 153/234 -- GLOBAL_STEP: 34785 + | > loss: -0.27583 (-0.13543) + | > log_mle: -0.48851 (-0.29172) + | > loss_dur: 0.21268 (0.15629) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.26307 (20.22104) + | > current_lr: 0.00004 + | > step_time: 1.98760 (2.74596) + | > loader_time: 0.00500 (0.08828) + + + --> STEP: 158/234 -- GLOBAL_STEP: 34790 + | > loss: -0.21207 (-0.13822) + | > log_mle: -0.43225 (-0.29615) + | > loss_dur: 0.22018 (0.15793) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.78313 (21.56365) + | > current_lr: 0.00004 + | > step_time: 3.50500 (2.77432) + | > loader_time: 0.00280 (0.08734) + + + --> STEP: 163/234 -- GLOBAL_STEP: 34795 + | > loss: -0.20181 (-0.14126) + | > log_mle: -0.40256 (-0.30067) + | > loss_dur: 0.20075 (0.15941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.85239 (22.55682) + | > current_lr: 0.00004 + | > step_time: 1.30650 (2.85370) + | > loader_time: 0.00250 (0.08825) + + + --> STEP: 168/234 -- GLOBAL_STEP: 34800 + | > loss: -0.22764 (-0.14433) + | > log_mle: -0.46217 (-0.30540) + | > loss_dur: 0.23454 (0.16107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.69563 (23.56925) + | > current_lr: 0.00004 + | > step_time: 2.69710 (2.81909) + | > loader_time: 0.00470 (0.08675) + + + --> STEP: 173/234 -- GLOBAL_STEP: 34805 + | > loss: -0.24772 (-0.14759) + | > log_mle: -0.46615 (-0.31066) + | > loss_dur: 0.21843 (0.16306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.28524 (24.84151) + | > current_lr: 0.00004 + | > step_time: 2.70940 (2.86782) + | > loader_time: 0.09520 (0.08708) + + + --> STEP: 178/234 -- GLOBAL_STEP: 34810 + | > loss: -0.27525 (-0.15084) + | > log_mle: -0.52288 (-0.31579) + | > loss_dur: 0.24763 (0.16494) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.78550 (26.21267) + | > current_lr: 0.00004 + | > step_time: 1.59980 (2.85015) + | > loader_time: 0.08860 (0.08569) + + + --> STEP: 183/234 -- GLOBAL_STEP: 34815 + | > loss: -0.29398 (-0.15374) + | > log_mle: -0.52730 (-0.32064) + | > loss_dur: 0.23332 (0.16689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.01047 (27.26759) + | > current_lr: 0.00004 + | > step_time: 4.41530 (2.96974) + | > loader_time: 0.08960 (0.09022) + + + --> STEP: 188/234 -- GLOBAL_STEP: 34820 + | > loss: -0.30155 (-0.15676) + | > log_mle: -0.53997 (-0.32557) + | > loss_dur: 0.23842 (0.16881) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.33022 (28.40725) + | > current_lr: 0.00004 + | > step_time: 1.69010 (2.94653) + | > loader_time: 0.00390 (0.08892) + + + --> STEP: 193/234 -- GLOBAL_STEP: 34825 + | > loss: -0.31915 (-0.16020) + | > log_mle: -0.54624 (-0.33039) + | > loss_dur: 0.22709 (0.17019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.20541 (29.40266) + | > current_lr: 0.00004 + | > step_time: 2.95340 (3.01097) + | > loader_time: 0.08160 (0.09004) + + + --> STEP: 198/234 -- GLOBAL_STEP: 34830 + | > loss: -0.28659 (-0.16313) + | > log_mle: -0.52482 (-0.33490) + | > loss_dur: 0.23823 (0.17176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.77485 (30.52665) + | > current_lr: 0.00004 + | > step_time: 2.60660 (2.98758) + | > loader_time: 0.00390 (0.08834) + + + --> STEP: 203/234 -- GLOBAL_STEP: 34835 + | > loss: -0.23280 (-0.16574) + | > log_mle: -0.46275 (-0.33917) + | > loss_dur: 0.22995 (0.17343) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.64711 (31.58153) + | > current_lr: 0.00004 + | > step_time: 2.58940 (2.99084) + | > loader_time: 0.00300 (0.08763) + + + --> STEP: 208/234 -- GLOBAL_STEP: 34840 + | > loss: -0.29098 (-0.16899) + | > log_mle: -0.53901 (-0.34414) + | > loss_dur: 0.24803 (0.17515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.11970 (32.78191) + | > current_lr: 0.00004 + | > step_time: 6.10230 (3.05019) + | > loader_time: 0.08580 (0.08641) + + + --> STEP: 213/234 -- GLOBAL_STEP: 34845 + | > loss: -0.33516 (-0.17247) + | > log_mle: -0.59085 (-0.34939) + | > loss_dur: 0.25569 (0.17692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.56978 (34.23351) + | > current_lr: 0.00004 + | > step_time: 3.10020 (3.10675) + | > loader_time: 0.10240 (0.08864) + + + --> STEP: 218/234 -- GLOBAL_STEP: 34850 + | > loss: -0.29740 (-0.17568) + | > log_mle: -0.54392 (-0.35416) + | > loss_dur: 0.24653 (0.17848) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.33892 (35.40614) + | > current_lr: 0.00004 + | > step_time: 4.20510 (3.11994) + | > loader_time: 0.08940 (0.08880) + + + --> STEP: 223/234 -- GLOBAL_STEP: 34855 + | > loss: -0.34156 (-0.17929) + | > log_mle: -0.59436 (-0.35944) + | > loss_dur: 0.25280 (0.18015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.13049 (36.49245) + | > current_lr: 0.00004 + | > step_time: 2.29470 (3.10334) + | > loader_time: 0.00360 (0.08778) + + + --> STEP: 228/234 -- GLOBAL_STEP: 34860 + | > loss: -0.30587 (-0.18277) + | > log_mle: -0.58849 (-0.36483) + | > loss_dur: 0.28263 (0.18205) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.12080 (37.92562) + | > current_lr: 0.00004 + | > step_time: 0.24600 (3.06415) + | > loader_time: 0.00400 (0.08633) + + + --> STEP: 233/234 -- GLOBAL_STEP: 34865 + | > loss: 0.23387 (-0.18317) + | > log_mle: -0.54640 (-0.37136) + | > loss_dur: 0.78027 (0.18818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.72444 (39.58061) + | > current_lr: 0.00004 + | > step_time: 0.18820 (3.00403) + | > loader_time: 0.00310 (0.08457) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00300 (-0.12027) + | > avg_loss: -0.23548 (-0.04271) + | > avg_log_mle: -0.45711 (-0.03461) + | > avg_loss_dur: 0.22163 (-0.00810) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_34866.pth + + > EPOCH: 149/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 16:28:30)  + + --> STEP: 4/234 -- GLOBAL_STEP: 34870 + | > loss: -0.06853 (-0.09818) + | > log_mle: -0.25221 (-0.25635) + | > loss_dur: 0.18368 (0.15817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.96697 (17.19092) + | > current_lr: 0.00004 + | > step_time: 3.49500 (5.95496) + | > loader_time: 0.00120 (2.67446) + + + --> STEP: 9/234 -- GLOBAL_STEP: 34875 + | > loss: -0.10447 (-0.11477) + | > log_mle: -0.26560 (-0.25897) + | > loss_dur: 0.16113 (0.14420) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.96518 (15.05166) + | > current_lr: 0.00004 + | > step_time: 1.40330 (3.22888) + | > loader_time: 0.00220 (1.19987) + + + --> STEP: 14/234 -- GLOBAL_STEP: 34880 + | > loss: -0.11400 (-0.11827) + | > log_mle: -0.26005 (-0.25826) + | > loss_dur: 0.14605 (0.13999) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.97662 (14.37074) + | > current_lr: 0.00004 + | > step_time: 1.48880 (2.85275) + | > loader_time: 0.00130 (0.78487) + + + --> STEP: 19/234 -- GLOBAL_STEP: 34885 + | > loss: -0.13685 (-0.12237) + | > log_mle: -0.24654 (-0.25571) + | > loss_dur: 0.10969 (0.13334) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.10915 (13.04100) + | > current_lr: 0.00004 + | > step_time: 1.03830 (2.94594) + | > loader_time: 0.07940 (0.59728) + + + --> STEP: 24/234 -- GLOBAL_STEP: 34890 + | > loss: -0.15151 (-0.12560) + | > log_mle: -0.24772 (-0.25424) + | > loss_dur: 0.09622 (0.12864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.37311 (12.12225) + | > current_lr: 0.00004 + | > step_time: 1.08730 (2.57370) + | > loader_time: 0.00190 (0.47467) + + + --> STEP: 29/234 -- GLOBAL_STEP: 34895 + | > loss: -0.10381 (-0.12602) + | > log_mle: -0.23596 (-0.25301) + | > loss_dur: 0.13216 (0.12699) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.65113 (11.52697) + | > current_lr: 0.00004 + | > step_time: 1.81310 (2.39172) + | > loader_time: 0.00310 (0.39611) + + + --> STEP: 34/234 -- GLOBAL_STEP: 34900 + | > loss: -0.10397 (-0.12590) + | > log_mle: -0.24502 (-0.25272) + | > loss_dur: 0.14105 (0.12683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.30095 (11.29338) + | > current_lr: 0.00004 + | > step_time: 6.00470 (2.55591) + | > loader_time: 0.29940 (0.35763) + + + --> STEP: 39/234 -- GLOBAL_STEP: 34905 + | > loss: -0.12524 (-0.12501) + | > log_mle: -0.25468 (-0.25258) + | > loss_dur: 0.12944 (0.12756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.37464 (11.33963) + | > current_lr: 0.00004 + | > step_time: 1.14590 (2.88791) + | > loader_time: 0.00190 (0.32502) + + + --> STEP: 44/234 -- GLOBAL_STEP: 34910 + | > loss: -0.14589 (-0.12425) + | > log_mle: -0.24736 (-0.25182) + | > loss_dur: 0.10147 (0.12757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.65113 (10.95767) + | > current_lr: 0.00004 + | > step_time: 1.30290 (2.68526) + | > loader_time: 0.00140 (0.29013) + + + --> STEP: 49/234 -- GLOBAL_STEP: 34915 + | > loss: -0.14234 (-0.12379) + | > log_mle: -0.25522 (-0.25183) + | > loss_dur: 0.11287 (0.12805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.86000 (10.86520) + | > current_lr: 0.00004 + | > step_time: 1.82620 (2.60988) + | > loader_time: 0.00510 (0.26270) + + + --> STEP: 54/234 -- GLOBAL_STEP: 34920 + | > loss: -0.14570 (-0.12303) + | > log_mle: -0.26034 (-0.25157) + | > loss_dur: 0.11464 (0.12854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.35688 (10.57575) + | > current_lr: 0.00004 + | > step_time: 1.09750 (2.50648) + | > loader_time: 0.00260 (0.23859) + + + --> STEP: 59/234 -- GLOBAL_STEP: 34925 + | > loss: -0.14951 (-0.12294) + | > log_mle: -0.26831 (-0.25166) + | > loss_dur: 0.11880 (0.12872) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03580 (10.43296) + | > current_lr: 0.00004 + | > step_time: 0.79580 (2.46355) + | > loader_time: 0.00260 (0.22133) + + + --> STEP: 64/234 -- GLOBAL_STEP: 34930 + | > loss: -0.12866 (-0.12199) + | > log_mle: -0.24560 (-0.25249) + | > loss_dur: 0.11694 (0.13049) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 4.72163 (10.47613) + | > current_lr: 0.00004 + | > step_time: 1.07090 (2.40834) + | > loader_time: 0.00210 (0.20726) + + + --> STEP: 69/234 -- GLOBAL_STEP: 34935 + | > loss: -0.09762 (-0.12094) + | > log_mle: -0.23007 (-0.25203) + | > loss_dur: 0.13245 (0.13110) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.14016 (10.31320) + | > current_lr: 0.00004 + | > step_time: 1.33330 (2.37234) + | > loader_time: 0.00250 (0.19365) + + + --> STEP: 74/234 -- GLOBAL_STEP: 34940 + | > loss: -0.09856 (-0.11947) + | > log_mle: -0.23981 (-0.25216) + | > loss_dur: 0.14125 (0.13270) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.04620 (10.51838) + | > current_lr: 0.00004 + | > step_time: 2.00800 (2.34228) + | > loader_time: 0.00230 (0.18185) + + + --> STEP: 79/234 -- GLOBAL_STEP: 34945 + | > loss: -0.10403 (-0.11853) + | > log_mle: -0.25600 (-0.25230) + | > loss_dur: 0.15197 (0.13376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.38390 (10.53272) + | > current_lr: 0.00004 + | > step_time: 3.41500 (2.32262) + | > loader_time: 0.18760 (0.17388) + + + --> STEP: 84/234 -- GLOBAL_STEP: 34950 + | > loss: -0.11340 (-0.11825) + | > log_mle: -0.25313 (-0.25251) + | > loss_dur: 0.13973 (0.13426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.12466 (10.57640) + | > current_lr: 0.00004 + | > step_time: 6.51960 (2.49289) + | > loader_time: 0.00430 (0.16714) + + + --> STEP: 89/234 -- GLOBAL_STEP: 34955 + | > loss: -0.13924 (-0.11833) + | > log_mle: -0.28417 (-0.25368) + | > loss_dur: 0.14493 (0.13535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.24152 (10.75782) + | > current_lr: 0.00004 + | > step_time: 1.40380 (2.43828) + | > loader_time: 0.00280 (0.15899) + + + --> STEP: 94/234 -- GLOBAL_STEP: 34960 + | > loss: -0.17068 (-0.11943) + | > log_mle: -0.32086 (-0.25622) + | > loss_dur: 0.15019 (0.13679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.57049 (11.27505) + | > current_lr: 0.00004 + | > step_time: 1.29360 (2.39068) + | > loader_time: 0.00210 (0.15249) + + + --> STEP: 99/234 -- GLOBAL_STEP: 34965 + | > loss: -0.18264 (-0.12074) + | > log_mle: -0.35420 (-0.25878) + | > loss_dur: 0.17156 (0.13804) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.29804 (11.81735) + | > current_lr: 0.00004 + | > step_time: 1.79800 (2.34473) + | > loader_time: 0.00220 (0.14678) + + + --> STEP: 104/234 -- GLOBAL_STEP: 34970 + | > loss: -0.18768 (-0.12233) + | > log_mle: -0.36882 (-0.26189) + | > loss_dur: 0.18114 (0.13956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.12397 (12.51028) + | > current_lr: 0.00004 + | > step_time: 1.40260 (2.32545) + | > loader_time: 0.00240 (0.14071) + + + --> STEP: 109/234 -- GLOBAL_STEP: 34975 + | > loss: -0.12438 (-0.12284) + | > log_mle: -0.33546 (-0.26408) + | > loss_dur: 0.21108 (0.14124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.49777 (13.10874) + | > current_lr: 0.00004 + | > step_time: 2.09200 (2.30323) + | > loader_time: 0.00270 (0.13680) + + + --> STEP: 114/234 -- GLOBAL_STEP: 34980 + | > loss: -0.15716 (-0.12393) + | > log_mle: -0.31879 (-0.26705) + | > loss_dur: 0.16163 (0.14312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.76429 (13.96374) + | > current_lr: 0.00004 + | > step_time: 1.59880 (2.26813) + | > loader_time: 0.00330 (0.13237) + + + --> STEP: 119/234 -- GLOBAL_STEP: 34985 + | > loss: -0.13578 (-0.12446) + | > log_mle: -0.31652 (-0.26934) + | > loss_dur: 0.18074 (0.14489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.74549 (14.42570) + | > current_lr: 0.00004 + | > step_time: 2.88800 (2.24090) + | > loader_time: 0.00250 (0.12847) + + + --> STEP: 124/234 -- GLOBAL_STEP: 34990 + | > loss: -0.17700 (-0.12503) + | > log_mle: -0.34605 (-0.27097) + | > loss_dur: 0.16905 (0.14594) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.41602 (14.69132) + | > current_lr: 0.00004 + | > step_time: 2.30600 (2.24025) + | > loader_time: 0.09110 (0.12676) + + + --> STEP: 129/234 -- GLOBAL_STEP: 34995 + | > loss: -0.13316 (-0.12650) + | > log_mle: -0.33544 (-0.27425) + | > loss_dur: 0.20228 (0.14775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.83628 (15.43862) + | > current_lr: 0.00004 + | > step_time: 3.61150 (2.26906) + | > loader_time: 0.00370 (0.12559) + + + --> STEP: 134/234 -- GLOBAL_STEP: 35000 + | > loss: -0.17790 (-0.12875) + | > log_mle: -0.38493 (-0.27816) + | > loss_dur: 0.20703 (0.14941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.31425 (16.41735) + | > current_lr: 0.00004 + | > step_time: 2.30800 (2.26346) + | > loader_time: 0.08730 (0.12236) + + + --> STEP: 139/234 -- GLOBAL_STEP: 35005 + | > loss: -0.24560 (-0.13071) + | > log_mle: -0.44464 (-0.28176) + | > loss_dur: 0.19904 (0.15105) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.35930 (17.42719) + | > current_lr: 0.00004 + | > step_time: 3.80880 (2.29578) + | > loader_time: 0.00440 (0.11873) + + + --> STEP: 144/234 -- GLOBAL_STEP: 35010 + | > loss: -0.20950 (-0.13259) + | > log_mle: -0.41711 (-0.28560) + | > loss_dur: 0.20761 (0.15302) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.27979 (18.34987) + | > current_lr: 0.00004 + | > step_time: 1.30700 (2.28121) + | > loader_time: 0.09440 (0.12010) + + + --> STEP: 149/234 -- GLOBAL_STEP: 35015 + | > loss: -0.25580 (-0.13531) + | > log_mle: -0.47058 (-0.28992) + | > loss_dur: 0.21478 (0.15461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.58586 (19.25328) + | > current_lr: 0.00004 + | > step_time: 3.74960 (2.28218) + | > loader_time: 0.10210 (0.11746) + + + --> STEP: 154/234 -- GLOBAL_STEP: 35020 + | > loss: -0.22275 (-0.13833) + | > log_mle: -0.42717 (-0.29468) + | > loss_dur: 0.20442 (0.15636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.09470 (20.53455) + | > current_lr: 0.00004 + | > step_time: 6.89150 (2.29987) + | > loader_time: 0.00810 (0.11438) + + + --> STEP: 159/234 -- GLOBAL_STEP: 35025 + | > loss: -0.22927 (-0.14112) + | > log_mle: -0.44510 (-0.29922) + | > loss_dur: 0.21582 (0.15810) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.93827 (21.76896) + | > current_lr: 0.00004 + | > step_time: 2.30320 (2.31698) + | > loader_time: 0.00280 (0.11452) + + + --> STEP: 164/234 -- GLOBAL_STEP: 35030 + | > loss: -0.22346 (-0.14403) + | > log_mle: -0.43910 (-0.30361) + | > loss_dur: 0.21564 (0.15958) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.92061 (22.73563) + | > current_lr: 0.00004 + | > step_time: 1.04810 (2.36584) + | > loader_time: 0.00300 (0.11169) + + + --> STEP: 169/234 -- GLOBAL_STEP: 35035 + | > loss: -0.21748 (-0.14700) + | > log_mle: -0.44114 (-0.30827) + | > loss_dur: 0.22366 (0.16127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.31017 (23.92053) + | > current_lr: 0.00004 + | > step_time: 2.70000 (2.36805) + | > loader_time: 0.08310 (0.11111) + + + --> STEP: 174/234 -- GLOBAL_STEP: 35040 + | > loss: -0.30645 (-0.15092) + | > log_mle: -0.53216 (-0.31406) + | > loss_dur: 0.22571 (0.16314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.22441 (25.25508) + | > current_lr: 0.00004 + | > step_time: 6.50170 (2.45355) + | > loader_time: 0.09660 (0.11069) + + + --> STEP: 179/234 -- GLOBAL_STEP: 35045 + | > loss: -0.26967 (-0.15411) + | > log_mle: -0.52876 (-0.31930) + | > loss_dur: 0.25909 (0.16519) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.71165 (26.42104) + | > current_lr: 0.00004 + | > step_time: 6.30400 (2.52591) + | > loader_time: 0.09190 (0.11032) + + + --> STEP: 184/234 -- GLOBAL_STEP: 35050 + | > loss: -0.26177 (-0.15681) + | > log_mle: -0.49381 (-0.32388) + | > loss_dur: 0.23204 (0.16706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.03895 (27.62008) + | > current_lr: 0.00004 + | > step_time: 12.29200 (2.62634) + | > loader_time: 0.00340 (0.10934) + + + --> STEP: 189/234 -- GLOBAL_STEP: 35055 + | > loss: -0.25382 (-0.15976) + | > log_mle: -0.49017 (-0.32871) + | > loss_dur: 0.23636 (0.16895) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.32787 (28.68037) + | > current_lr: 0.00004 + | > step_time: 0.70160 (2.62515) + | > loader_time: 0.00240 (0.10760) + + + --> STEP: 194/234 -- GLOBAL_STEP: 35060 + | > loss: -0.28477 (-0.16324) + | > log_mle: -0.52085 (-0.33356) + | > loss_dur: 0.23608 (0.17031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.45726 (29.85621) + | > current_lr: 0.00004 + | > step_time: 2.89430 (2.62248) + | > loader_time: 0.00300 (0.10621) + + + --> STEP: 199/234 -- GLOBAL_STEP: 35065 + | > loss: -0.28927 (-0.16609) + | > log_mle: -0.52611 (-0.33798) + | > loss_dur: 0.23684 (0.17189) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.59694 (31.01492) + | > current_lr: 0.00004 + | > step_time: 2.38990 (2.66870) + | > loader_time: 0.00310 (0.10505) + + + --> STEP: 204/234 -- GLOBAL_STEP: 35070 + | > loss: -0.30904 (-0.16875) + | > log_mle: -0.56439 (-0.34236) + | > loss_dur: 0.25535 (0.17361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.01116 (32.29567) + | > current_lr: 0.00004 + | > step_time: 10.30320 (2.75913) + | > loader_time: 0.29660 (0.10643) + + + --> STEP: 209/234 -- GLOBAL_STEP: 35075 + | > loss: -0.26974 (-0.17184) + | > log_mle: -0.50814 (-0.34707) + | > loss_dur: 0.23840 (0.17523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.95935 (33.69757) + | > current_lr: 0.00004 + | > step_time: 10.50670 (2.87067) + | > loader_time: 0.00360 (0.10622) + + + --> STEP: 214/234 -- GLOBAL_STEP: 35080 + | > loss: -0.32910 (-0.17563) + | > log_mle: -0.54775 (-0.35252) + | > loss_dur: 0.21865 (0.17689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.87772 (35.08080) + | > current_lr: 0.00004 + | > step_time: 3.89370 (2.94514) + | > loader_time: 0.60200 (0.10754) + + + --> STEP: 219/234 -- GLOBAL_STEP: 35085 + | > loss: -0.40079 (-0.17920) + | > log_mle: -0.65276 (-0.35782) + | > loss_dur: 0.25197 (0.17862) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.52576 (36.32345) + | > current_lr: 0.00004 + | > step_time: 4.10680 (2.99387) + | > loader_time: 0.09700 (0.10875) + + + --> STEP: 224/234 -- GLOBAL_STEP: 35090 + | > loss: -0.35646 (-0.18266) + | > log_mle: -0.60416 (-0.36286) + | > loss_dur: 0.24771 (0.18021) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.05944 (37.57280) + | > current_lr: 0.00004 + | > step_time: 0.23590 (2.95134) + | > loader_time: 0.00420 (0.10712) + + + --> STEP: 229/234 -- GLOBAL_STEP: 35095 + | > loss: -0.32035 (-0.18608) + | > log_mle: -0.63620 (-0.36843) + | > loss_dur: 0.31585 (0.18235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 135.09608 (39.02643) + | > current_lr: 0.00004 + | > step_time: 0.25310 (2.89220) + | > loader_time: 0.00430 (0.10487) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.79592 (+0.79293) + | > avg_loss: -0.22584 (+0.00964) + | > avg_log_mle: -0.45055 (+0.00656) + | > avg_loss_dur: 0.22471 (+0.00308) + + + > EPOCH: 150/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 16:40:55)  + + --> STEP: 0/234 -- GLOBAL_STEP: 35100 + | > loss: -0.18161 (-0.18161) + | > log_mle: -0.33015 (-0.33015) + | > loss_dur: 0.14854 (0.14854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.83287 (18.83287) + | > current_lr: 0.00004 + | > step_time: 2.90670 (2.90665) + | > loader_time: 6.81880 (6.81884) + + + --> STEP: 5/234 -- GLOBAL_STEP: 35105 + | > loss: -0.11998 (-0.10857) + | > log_mle: -0.25730 (-0.25581) + | > loss_dur: 0.13732 (0.14724) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.91774 (16.11370) + | > current_lr: 0.00004 + | > step_time: 7.70540 (6.22235) + | > loader_time: 0.09760 (3.75578) + + + --> STEP: 10/234 -- GLOBAL_STEP: 35110 + | > loss: -0.10336 (-0.11930) + | > log_mle: -0.25156 (-0.25810) + | > loss_dur: 0.14820 (0.13880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.31590 (14.93826) + | > current_lr: 0.00004 + | > step_time: 1.89040 (5.36294) + | > loader_time: 0.00200 (1.97647) + + + --> STEP: 15/234 -- GLOBAL_STEP: 35115 + | > loss: -0.14782 (-0.12591) + | > log_mle: -0.25629 (-0.25832) + | > loss_dur: 0.10847 (0.13241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.91098 (13.83820) + | > current_lr: 0.00004 + | > step_time: 2.00850 (4.30227) + | > loader_time: 0.00230 (1.31829) + + + --> STEP: 20/234 -- GLOBAL_STEP: 35120 + | > loss: -0.11772 (-0.12700) + | > log_mle: -0.24766 (-0.25562) + | > loss_dur: 0.12994 (0.12861) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.27759 (12.75265) + | > current_lr: 0.00004 + | > step_time: 4.00410 (4.03106) + | > loader_time: 0.00150 (1.00367) + + + --> STEP: 25/234 -- GLOBAL_STEP: 35125 + | > loss: -0.12813 (-0.12834) + | > log_mle: -0.23966 (-0.25386) + | > loss_dur: 0.11153 (0.12552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.83587 (12.44921) + | > current_lr: 0.00004 + | > step_time: 1.62500 (3.59142) + | > loader_time: 0.08790 (0.81150) + + + --> STEP: 30/234 -- GLOBAL_STEP: 35130 + | > loss: -0.15492 (-0.12984) + | > log_mle: -0.25711 (-0.25348) + | > loss_dur: 0.10219 (0.12364) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.00200 (11.85552) + | > current_lr: 0.00004 + | > step_time: 1.31650 (3.47048) + | > loader_time: 0.08780 (0.68220) + + + --> STEP: 35/234 -- GLOBAL_STEP: 35135 + | > loss: -0.11108 (-0.12828) + | > log_mle: -0.24950 (-0.25312) + | > loss_dur: 0.13841 (0.12485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.02561 (11.68033) + | > current_lr: 0.00004 + | > step_time: 1.20460 (3.23241) + | > loader_time: 0.08720 (0.59498) + + + --> STEP: 40/234 -- GLOBAL_STEP: 35140 + | > loss: -0.08967 (-0.12704) + | > log_mle: -0.23481 (-0.25257) + | > loss_dur: 0.14514 (0.12552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.45240 (11.59788) + | > current_lr: 0.00004 + | > step_time: 2.09310 (3.00812) + | > loader_time: 0.00220 (0.52299) + + + --> STEP: 45/234 -- GLOBAL_STEP: 35145 + | > loss: -0.12085 (-0.12619) + | > log_mle: -0.26998 (-0.25257) + | > loss_dur: 0.14913 (0.12638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.90095 (11.52739) + | > current_lr: 0.00004 + | > step_time: 2.50150 (2.97342) + | > loader_time: 0.08740 (0.47132) + + + --> STEP: 50/234 -- GLOBAL_STEP: 35150 + | > loss: -0.11063 (-0.12547) + | > log_mle: -0.24126 (-0.25196) + | > loss_dur: 0.13063 (0.12650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.41444 (11.24583) + | > current_lr: 0.00004 + | > step_time: 2.00580 (2.84825) + | > loader_time: 0.00350 (0.42641) + + + --> STEP: 55/234 -- GLOBAL_STEP: 35155 + | > loss: -0.15043 (-0.12538) + | > log_mle: -0.26195 (-0.25212) + | > loss_dur: 0.11152 (0.12673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.39854 (10.95577) + | > current_lr: 0.00004 + | > step_time: 1.99520 (2.76885) + | > loader_time: 0.00230 (0.38787) + + + --> STEP: 60/234 -- GLOBAL_STEP: 35160 + | > loss: -0.13106 (-0.12492) + | > log_mle: -0.27427 (-0.25236) + | > loss_dur: 0.14321 (0.12744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18898 (10.92210) + | > current_lr: 0.00004 + | > step_time: 1.00910 (2.67687) + | > loader_time: 0.00300 (0.36148) + + + --> STEP: 65/234 -- GLOBAL_STEP: 35165 + | > loss: -0.13228 (-0.12429) + | > log_mle: -0.25207 (-0.25279) + | > loss_dur: 0.11979 (0.12850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.43130 (10.91768) + | > current_lr: 0.00004 + | > step_time: 1.39930 (2.65968) + | > loader_time: 0.00240 (0.33395) + + + --> STEP: 70/234 -- GLOBAL_STEP: 35170 + | > loss: -0.09225 (-0.12277) + | > log_mle: -0.24557 (-0.25242) + | > loss_dur: 0.15332 (0.12965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.65253 (10.86760) + | > current_lr: 0.00004 + | > step_time: 1.70000 (2.58782) + | > loader_time: 0.08760 (0.31266) + + + --> STEP: 75/234 -- GLOBAL_STEP: 35175 + | > loss: -0.11470 (-0.12191) + | > log_mle: -0.26449 (-0.25303) + | > loss_dur: 0.14979 (0.13112) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.09106 (10.95986) + | > current_lr: 0.00004 + | > step_time: 1.64000 (2.57645) + | > loader_time: 0.00250 (0.29455) + + + --> STEP: 80/234 -- GLOBAL_STEP: 35180 + | > loss: -0.13498 (-0.12163) + | > log_mle: -0.24428 (-0.25313) + | > loss_dur: 0.10930 (0.13150) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.20153 (10.92581) + | > current_lr: 0.00004 + | > step_time: 1.70200 (2.53297) + | > loader_time: 0.09760 (0.27855) + + + --> STEP: 85/234 -- GLOBAL_STEP: 35185 + | > loss: -0.11762 (-0.12113) + | > log_mle: -0.25609 (-0.25343) + | > loss_dur: 0.13847 (0.13230) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17457 (10.96786) + | > current_lr: 0.00004 + | > step_time: 1.20130 (2.47082) + | > loader_time: 0.00300 (0.26234) + + + --> STEP: 90/234 -- GLOBAL_STEP: 35190 + | > loss: -0.12097 (-0.12105) + | > log_mle: -0.28179 (-0.25481) + | > loss_dur: 0.16082 (0.13376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.91208 (11.20100) + | > current_lr: 0.00004 + | > step_time: 1.69760 (2.45469) + | > loader_time: 0.00420 (0.24999) + + + --> STEP: 95/234 -- GLOBAL_STEP: 35195 + | > loss: -0.16637 (-0.12223) + | > log_mle: -0.35352 (-0.25775) + | > loss_dur: 0.18715 (0.13551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.20382 (12.32196) + | > current_lr: 0.00004 + | > step_time: 2.09950 (2.45585) + | > loader_time: 0.00480 (0.23911) + + + --> STEP: 100/234 -- GLOBAL_STEP: 35200 + | > loss: -0.12886 (-0.12246) + | > log_mle: -0.29195 (-0.25919) + | > loss_dur: 0.16309 (0.13673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.95744 (12.53833) + | > current_lr: 0.00004 + | > step_time: 2.39160 (2.46739) + | > loader_time: 0.00790 (0.22894) + + + --> STEP: 105/234 -- GLOBAL_STEP: 35205 + | > loss: -0.13036 (-0.12349) + | > log_mle: -0.27200 (-0.26181) + | > loss_dur: 0.14164 (0.13832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.31352 (12.95504) + | > current_lr: 0.00004 + | > step_time: 3.21740 (2.43782) + | > loader_time: 0.00240 (0.21898) + + + --> STEP: 110/234 -- GLOBAL_STEP: 35210 + | > loss: -0.13666 (-0.12381) + | > log_mle: -0.29836 (-0.26405) + | > loss_dur: 0.16170 (0.14024) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.51919 (13.43302) + | > current_lr: 0.00004 + | > step_time: 1.29440 (2.40583) + | > loader_time: 0.00330 (0.20915) + + + --> STEP: 115/234 -- GLOBAL_STEP: 35215 + | > loss: -0.13555 (-0.12496) + | > log_mle: -0.31901 (-0.26710) + | > loss_dur: 0.18347 (0.14214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.08464 (14.01056) + | > current_lr: 0.00004 + | > step_time: 1.09590 (2.38921) + | > loader_time: 0.00170 (0.20174) + + + --> STEP: 120/234 -- GLOBAL_STEP: 35220 + | > loss: -0.17891 (-0.12578) + | > log_mle: -0.36523 (-0.26971) + | > loss_dur: 0.18633 (0.14393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.99576 (14.49588) + | > current_lr: 0.00004 + | > step_time: 1.50670 (2.34994) + | > loader_time: 0.07710 (0.19556) + + + --> STEP: 125/234 -- GLOBAL_STEP: 35225 + | > loss: -0.16214 (-0.12612) + | > log_mle: -0.35346 (-0.27113) + | > loss_dur: 0.19132 (0.14501) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.66006 (14.83679) + | > current_lr: 0.00004 + | > step_time: 2.89900 (2.35196) + | > loader_time: 0.19430 (0.19169) + + + --> STEP: 130/234 -- GLOBAL_STEP: 35230 + | > loss: -0.16685 (-0.12780) + | > log_mle: -0.36788 (-0.27446) + | > loss_dur: 0.20103 (0.14667) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.24200 (15.54689) + | > current_lr: 0.00004 + | > step_time: 1.06880 (2.35229) + | > loader_time: 0.00200 (0.18594) + + + --> STEP: 135/234 -- GLOBAL_STEP: 35235 + | > loss: -0.13964 (-0.12969) + | > log_mle: -0.29894 (-0.27781) + | > loss_dur: 0.15930 (0.14813) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.28958 (16.21353) + | > current_lr: 0.00004 + | > step_time: 3.30520 (2.35396) + | > loader_time: 0.09660 (0.18120) + + + --> STEP: 140/234 -- GLOBAL_STEP: 35240 + | > loss: -0.13737 (-0.13158) + | > log_mle: -0.33127 (-0.28165) + | > loss_dur: 0.19390 (0.15007) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.32729 (17.21457) + | > current_lr: 0.00004 + | > step_time: 2.10320 (2.34860) + | > loader_time: 0.08880 (0.17819) + + + --> STEP: 145/234 -- GLOBAL_STEP: 35245 + | > loss: -0.21913 (-0.13383) + | > log_mle: -0.42688 (-0.28610) + | > loss_dur: 0.20775 (0.15227) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.22601 (18.20967) + | > current_lr: 0.00004 + | > step_time: 1.40970 (2.43950) + | > loader_time: 0.09280 (0.17471) + + + --> STEP: 150/234 -- GLOBAL_STEP: 35250 + | > loss: -0.20439 (-0.13637) + | > log_mle: -0.41520 (-0.29030) + | > loss_dur: 0.21081 (0.15393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.60306 (19.09235) + | > current_lr: 0.00004 + | > step_time: 1.58730 (2.42493) + | > loader_time: 0.00370 (0.17096) + + + --> STEP: 155/234 -- GLOBAL_STEP: 35255 + | > loss: -0.24945 (-0.13975) + | > log_mle: -0.47203 (-0.29537) + | > loss_dur: 0.22257 (0.15561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.82505 (20.52124) + | > current_lr: 0.00004 + | > step_time: 1.29190 (2.41014) + | > loader_time: 0.00230 (0.16616) + + + --> STEP: 160/234 -- GLOBAL_STEP: 35260 + | > loss: -0.22197 (-0.14195) + | > log_mle: -0.44080 (-0.29938) + | > loss_dur: 0.21883 (0.15743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.34360 (22.72337) + | > current_lr: 0.00004 + | > step_time: 2.80710 (2.39661) + | > loader_time: 0.00350 (0.16159) + + + --> STEP: 165/234 -- GLOBAL_STEP: 35265 + | > loss: -0.24226 (-0.14420) + | > log_mle: -0.46258 (-0.30322) + | > loss_dur: 0.22032 (0.15902) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.60976 (23.54877) + | > current_lr: 0.00004 + | > step_time: 3.09650 (2.39374) + | > loader_time: 0.01030 (0.15905) + + + --> STEP: 170/234 -- GLOBAL_STEP: 35270 + | > loss: -0.25453 (-0.14663) + | > log_mle: -0.49539 (-0.30755) + | > loss_dur: 0.24086 (0.16092) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.44084 (24.59482) + | > current_lr: 0.00004 + | > step_time: 1.71710 (2.39514) + | > loader_time: 0.08520 (0.15723) + + + --> STEP: 175/234 -- GLOBAL_STEP: 35275 + | > loss: -0.24338 (-0.15008) + | > log_mle: -0.47849 (-0.31280) + | > loss_dur: 0.23511 (0.16271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.13902 (25.65122) + | > current_lr: 0.00004 + | > step_time: 3.89280 (2.39810) + | > loader_time: 0.00550 (0.15285) + + + --> STEP: 180/234 -- GLOBAL_STEP: 35280 + | > loss: -0.26624 (-0.15311) + | > log_mle: -0.48749 (-0.31776) + | > loss_dur: 0.22125 (0.16466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.24951 (26.51036) + | > current_lr: 0.00004 + | > step_time: 2.70840 (2.46148) + | > loader_time: 0.49900 (0.15361) + + + --> STEP: 185/234 -- GLOBAL_STEP: 35285 + | > loss: -0.27104 (-0.15600) + | > log_mle: -0.51830 (-0.32245) + | > loss_dur: 0.24726 (0.16645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.22453 (27.46647) + | > current_lr: 0.00004 + | > step_time: 2.20410 (2.59393) + | > loader_time: 0.09670 (0.15107) + + + --> STEP: 190/234 -- GLOBAL_STEP: 35290 + | > loss: -0.28146 (-0.15889) + | > log_mle: -0.49333 (-0.32707) + | > loss_dur: 0.21188 (0.16818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.72650 (28.49597) + | > current_lr: 0.00004 + | > step_time: 2.09890 (2.62722) + | > loader_time: 0.00370 (0.14764) + + + --> STEP: 195/234 -- GLOBAL_STEP: 35295 + | > loss: -0.26844 (-0.16234) + | > log_mle: -0.51504 (-0.33199) + | > loss_dur: 0.24659 (0.16965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.00177 (29.63245) + | > current_lr: 0.00004 + | > step_time: 12.90380 (2.72301) + | > loader_time: 0.19560 (0.14585) + + + --> STEP: 200/234 -- GLOBAL_STEP: 35300 + | > loss: -0.26608 (-0.16518) + | > log_mle: -0.52103 (-0.33650) + | > loss_dur: 0.25495 (0.17132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.03316 (30.67385) + | > current_lr: 0.00004 + | > step_time: 5.71170 (2.80044) + | > loader_time: 0.29170 (0.14427) + + + --> STEP: 205/234 -- GLOBAL_STEP: 35305 + | > loss: -0.27862 (-0.16780) + | > log_mle: -0.51335 (-0.34080) + | > loss_dur: 0.23472 (0.17301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.89114 (31.67246) + | > current_lr: 0.00004 + | > step_time: 4.59630 (2.82418) + | > loader_time: 0.00710 (0.14175) + + + --> STEP: 210/234 -- GLOBAL_STEP: 35310 + | > loss: -0.33120 (-0.17120) + | > log_mle: -0.58750 (-0.34595) + | > loss_dur: 0.25630 (0.17475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.00142 (32.87428) + | > current_lr: 0.00004 + | > step_time: 8.40960 (2.94652) + | > loader_time: 0.19320 (0.14124) + + + --> STEP: 215/234 -- GLOBAL_STEP: 35315 + | > loss: -0.30028 (-0.17491) + | > log_mle: -0.54404 (-0.35131) + | > loss_dur: 0.24376 (0.17640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.65322 (34.27882) + | > current_lr: 0.00004 + | > step_time: 26.20240 (3.05859) + | > loader_time: 0.89820 (0.14261) + + + --> STEP: 220/234 -- GLOBAL_STEP: 35320 + | > loss: -0.34517 (-0.17889) + | > log_mle: -0.59879 (-0.35701) + | > loss_dur: 0.25362 (0.17812) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.32815 (35.59785) + | > current_lr: 0.00004 + | > step_time: 2.59240 (3.07453) + | > loader_time: 0.00370 (0.13994) + + + --> STEP: 225/234 -- GLOBAL_STEP: 35325 + | > loss: -0.38950 (-0.18242) + | > log_mle: -0.65468 (-0.36233) + | > loss_dur: 0.26518 (0.17990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 128.57394 (37.08074) + | > current_lr: 0.00004 + | > step_time: 1.20290 (3.05953) + | > loader_time: 0.08400 (0.13800) + + + --> STEP: 230/234 -- GLOBAL_STEP: 35330 + | > loss: -0.36414 (-0.18576) + | > log_mle: -0.70313 (-0.36810) + | > loss_dur: 0.33899 (0.18234) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.52371 (38.52189) + | > current_lr: 0.00004 + | > step_time: 0.25960 (3.00125) + | > loader_time: 0.00460 (0.13509) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.14637 (-0.64955) + | > avg_loss: -0.19083 (+0.03501) + | > avg_log_mle: -0.42838 (+0.02217) + | > avg_loss_dur: 0.23755 (+0.01284) + + + > EPOCH: 151/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 16:53:52)  + + --> STEP: 1/234 -- GLOBAL_STEP: 35335 + | > loss: -0.13904 (-0.13904) + | > log_mle: -0.26085 (-0.26085) + | > loss_dur: 0.12181 (0.12181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.68436 (18.68436) + | > current_lr: 0.00004 + | > step_time: 6.79360 (6.79356) + | > loader_time: 14.19840 (14.19844) + + + --> STEP: 6/234 -- GLOBAL_STEP: 35340 + | > loss: -0.13348 (-0.11820) + | > log_mle: -0.24695 (-0.25670) + | > loss_dur: 0.11347 (0.13850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.23843 (17.94295) + | > current_lr: 0.00004 + | > step_time: 1.28900 (4.09354) + | > loader_time: 0.00100 (2.38589) + + + --> STEP: 11/234 -- GLOBAL_STEP: 35345 + | > loss: -0.14257 (-0.12162) + | > log_mle: -0.25915 (-0.25936) + | > loss_dur: 0.11658 (0.13775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.88410 (15.68123) + | > current_lr: 0.00004 + | > step_time: 3.20250 (3.47864) + | > loader_time: 0.00160 (1.34585) + + + --> STEP: 16/234 -- GLOBAL_STEP: 35350 + | > loss: -0.14232 (-0.12603) + | > log_mle: -0.25544 (-0.25934) + | > loss_dur: 0.11312 (0.13331) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.16263 (14.17038) + | > current_lr: 0.00004 + | > step_time: 7.00090 (4.02836) + | > loader_time: 0.00670 (0.93688) + + + --> STEP: 21/234 -- GLOBAL_STEP: 35355 + | > loss: -0.12258 (-0.12727) + | > log_mle: -0.23501 (-0.25581) + | > loss_dur: 0.11243 (0.12854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.90616 (12.85238) + | > current_lr: 0.00004 + | > step_time: 1.81460 (3.69820) + | > loader_time: 0.08760 (0.72729) + + + --> STEP: 26/234 -- GLOBAL_STEP: 35360 + | > loss: -0.11712 (-0.12965) + | > log_mle: -0.25291 (-0.25531) + | > loss_dur: 0.13579 (0.12566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.13156 (12.06027) + | > current_lr: 0.00004 + | > step_time: 0.97760 (3.86803) + | > loader_time: 0.00210 (0.60694) + + + --> STEP: 31/234 -- GLOBAL_STEP: 35365 + | > loss: -0.09499 (-0.13071) + | > log_mle: -0.25204 (-0.25512) + | > loss_dur: 0.15704 (0.12441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.37576 (11.45618) + | > current_lr: 0.00004 + | > step_time: 1.80240 (3.55726) + | > loader_time: 0.00280 (0.51835) + + + --> STEP: 36/234 -- GLOBAL_STEP: 35370 + | > loss: -0.10762 (-0.13021) + | > log_mle: -0.25233 (-0.25492) + | > loss_dur: 0.14471 (0.12471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.48069 (11.33482) + | > current_lr: 0.00004 + | > step_time: 3.10800 (3.70837) + | > loader_time: 0.09350 (0.46152) + + + --> STEP: 41/234 -- GLOBAL_STEP: 35375 + | > loss: -0.14191 (-0.12977) + | > log_mle: -0.25313 (-0.25452) + | > loss_dur: 0.11122 (0.12475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.20825 (11.11657) + | > current_lr: 0.00004 + | > step_time: 0.89830 (3.53046) + | > loader_time: 0.00200 (0.40799) + + + --> STEP: 46/234 -- GLOBAL_STEP: 35380 + | > loss: -0.11431 (-0.12869) + | > log_mle: -0.24953 (-0.25456) + | > loss_dur: 0.13522 (0.12587) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.48719 (11.14571) + | > current_lr: 0.00004 + | > step_time: 1.11830 (3.37305) + | > loader_time: 0.00310 (0.36760) + + + --> STEP: 51/234 -- GLOBAL_STEP: 35385 + | > loss: -0.11466 (-0.12814) + | > log_mle: -0.24182 (-0.25383) + | > loss_dur: 0.12716 (0.12569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.53342 (10.82868) + | > current_lr: 0.00004 + | > step_time: 1.16180 (3.19404) + | > loader_time: 0.00160 (0.33194) + + + --> STEP: 56/234 -- GLOBAL_STEP: 35390 + | > loss: -0.11244 (-0.12796) + | > log_mle: -0.25754 (-0.25426) + | > loss_dur: 0.14510 (0.12630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.87893 (10.67190) + | > current_lr: 0.00004 + | > step_time: 1.71340 (3.08239) + | > loader_time: 0.08670 (0.30704) + + + --> STEP: 61/234 -- GLOBAL_STEP: 35395 + | > loss: -0.11887 (-0.12766) + | > log_mle: -0.25550 (-0.25454) + | > loss_dur: 0.13663 (0.12688) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.65539 (10.58308) + | > current_lr: 0.00004 + | > step_time: 2.40860 (2.99459) + | > loader_time: 0.08400 (0.28339) + + + --> STEP: 66/234 -- GLOBAL_STEP: 35400 + | > loss: -0.13100 (-0.12676) + | > log_mle: -0.24402 (-0.25477) + | > loss_dur: 0.11302 (0.12802) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.12111 (10.64250) + | > current_lr: 0.00004 + | > step_time: 1.61280 (2.92932) + | > loader_time: 0.00230 (0.26468) + + + --> STEP: 71/234 -- GLOBAL_STEP: 35405 + | > loss: -0.10762 (-0.12502) + | > log_mle: -0.28510 (-0.25493) + | > loss_dur: 0.17748 (0.12991) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.72592 (10.76334) + | > current_lr: 0.00004 + | > step_time: 1.70910 (2.82982) + | > loader_time: 0.08050 (0.24847) + + + --> STEP: 76/234 -- GLOBAL_STEP: 35410 + | > loss: -0.12975 (-0.12406) + | > log_mle: -0.26703 (-0.25515) + | > loss_dur: 0.13728 (0.13110) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.91494 (10.89077) + | > current_lr: 0.00004 + | > step_time: 1.29980 (2.80041) + | > loader_time: 0.00360 (0.23455) + + + --> STEP: 81/234 -- GLOBAL_STEP: 35415 + | > loss: -0.12925 (-0.12387) + | > log_mle: -0.27295 (-0.25531) + | > loss_dur: 0.14370 (0.13145) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.15026 (10.86141) + | > current_lr: 0.00004 + | > step_time: 2.48960 (2.73415) + | > loader_time: 0.00270 (0.22124) + + + --> STEP: 86/234 -- GLOBAL_STEP: 35420 + | > loss: -0.11652 (-0.12329) + | > log_mle: -0.26927 (-0.25566) + | > loss_dur: 0.15275 (0.13237) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.34379 (10.98037) + | > current_lr: 0.00004 + | > step_time: 2.50760 (2.68814) + | > loader_time: 0.00220 (0.21048) + + + --> STEP: 91/234 -- GLOBAL_STEP: 35425 + | > loss: -0.11762 (-0.12339) + | > log_mle: -0.28118 (-0.25719) + | > loss_dur: 0.16355 (0.13380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.06602 (11.21082) + | > current_lr: 0.00004 + | > step_time: 2.06100 (2.63770) + | > loader_time: 0.00410 (0.19911) + + + --> STEP: 96/234 -- GLOBAL_STEP: 35430 + | > loss: -0.12390 (-0.12512) + | > log_mle: -0.26866 (-0.26039) + | > loss_dur: 0.14476 (0.13527) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.57299 (11.85810) + | > current_lr: 0.00004 + | > step_time: 3.10030 (2.59835) + | > loader_time: 0.00540 (0.18891) + + + --> STEP: 101/234 -- GLOBAL_STEP: 35435 + | > loss: -0.15519 (-0.12599) + | > log_mle: -0.32726 (-0.26255) + | > loss_dur: 0.17206 (0.13656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.49683 (12.35484) + | > current_lr: 0.00004 + | > step_time: 2.41470 (2.56506) + | > loader_time: 0.00290 (0.18209) + + + --> STEP: 106/234 -- GLOBAL_STEP: 35440 + | > loss: -0.12531 (-0.12688) + | > log_mle: -0.32523 (-0.26518) + | > loss_dur: 0.19992 (0.13829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.14382 (12.85158) + | > current_lr: 0.00004 + | > step_time: 1.50970 (2.53857) + | > loader_time: 0.07560 (0.17503) + + + --> STEP: 111/234 -- GLOBAL_STEP: 35445 + | > loss: -0.15858 (-0.12743) + | > log_mle: -0.37126 (-0.26783) + | > loss_dur: 0.21268 (0.14039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.93282 (13.50293) + | > current_lr: 0.00004 + | > step_time: 1.41160 (2.49658) + | > loader_time: 0.09030 (0.16986) + + + --> STEP: 116/234 -- GLOBAL_STEP: 35450 + | > loss: -0.13081 (-0.12811) + | > log_mle: -0.33981 (-0.27049) + | > loss_dur: 0.20899 (0.14239) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.04632 (14.07582) + | > current_lr: 0.00004 + | > step_time: 1.20580 (2.48132) + | > loader_time: 0.00290 (0.16337) + + + --> STEP: 121/234 -- GLOBAL_STEP: 35455 + | > loss: -0.09420 (-0.12867) + | > log_mle: -0.25153 (-0.27230) + | > loss_dur: 0.15733 (0.14362) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.31936 (14.43781) + | > current_lr: 0.00004 + | > step_time: 2.49520 (2.45394) + | > loader_time: 0.09430 (0.15752) + + + --> STEP: 126/234 -- GLOBAL_STEP: 35460 + | > loss: -0.19007 (-0.12982) + | > log_mle: -0.38477 (-0.27470) + | > loss_dur: 0.19470 (0.14488) + | > amp_scaler: 4096.00000 (2113.01587) + | > grad_norm: 52.46059 (15.16084) + | > current_lr: 0.00004 + | > step_time: 1.80780 (2.42572) + | > loader_time: 0.00350 (0.15213) + + + --> STEP: 131/234 -- GLOBAL_STEP: 35465 + | > loss: -0.22002 (-0.13163) + | > log_mle: -0.42918 (-0.27823) + | > loss_dur: 0.20916 (0.14661) + | > amp_scaler: 4096.00000 (2188.70229) + | > grad_norm: 44.77305 (15.84900) + | > current_lr: 0.00004 + | > step_time: 1.41140 (2.40428) + | > loader_time: 0.08470 (0.14844) + + + --> STEP: 136/234 -- GLOBAL_STEP: 35470 + | > loss: -0.23351 (-0.13342) + | > log_mle: -0.46457 (-0.28168) + | > loss_dur: 0.23106 (0.14827) + | > amp_scaler: 4096.00000 (2258.82353) + | > grad_norm: 61.72546 (16.62185) + | > current_lr: 0.00004 + | > step_time: 2.11040 (2.38204) + | > loader_time: 0.00320 (0.14372) + + + --> STEP: 141/234 -- GLOBAL_STEP: 35475 + | > loss: -0.19003 (-0.13459) + | > log_mle: -0.38155 (-0.28470) + | > loss_dur: 0.19153 (0.15012) + | > amp_scaler: 4096.00000 (2323.97163) + | > grad_norm: 35.83735 (17.30075) + | > current_lr: 0.00004 + | > step_time: 2.40830 (2.36781) + | > loader_time: 0.00460 (0.13947) + + + --> STEP: 146/234 -- GLOBAL_STEP: 35480 + | > loss: -0.22063 (-0.13723) + | > log_mle: -0.43023 (-0.28942) + | > loss_dur: 0.20960 (0.15219) + | > amp_scaler: 4096.00000 (2384.65753) + | > grad_norm: 57.32765 (18.52537) + | > current_lr: 0.00004 + | > step_time: 1.26570 (2.35657) + | > loader_time: 0.00270 (0.13550) + + + --> STEP: 151/234 -- GLOBAL_STEP: 35485 + | > loss: -0.21337 (-0.13970) + | > log_mle: -0.40234 (-0.29335) + | > loss_dur: 0.18897 (0.15366) + | > amp_scaler: 4096.00000 (2441.32450) + | > grad_norm: 39.41413 (19.33675) + | > current_lr: 0.00004 + | > step_time: 2.69800 (2.33633) + | > loader_time: 0.00320 (0.13228) + + + --> STEP: 156/234 -- GLOBAL_STEP: 35490 + | > loss: -0.23649 (-0.14330) + | > log_mle: -0.43750 (-0.29871) + | > loss_dur: 0.20102 (0.15541) + | > amp_scaler: 4096.00000 (2494.35897) + | > grad_norm: 67.28429 (20.85882) + | > current_lr: 0.00004 + | > step_time: 1.70370 (2.31900) + | > loader_time: 0.00380 (0.12869) + + + --> STEP: 161/234 -- GLOBAL_STEP: 35495 + | > loss: -0.26273 (-0.14613) + | > log_mle: -0.46785 (-0.30330) + | > loss_dur: 0.20512 (0.15717) + | > amp_scaler: 4096.00000 (2544.09938) + | > grad_norm: 62.28788 (21.95651) + | > current_lr: 0.00004 + | > step_time: 3.89700 (2.32161) + | > loader_time: 0.09550 (0.12588) + + + --> STEP: 166/234 -- GLOBAL_STEP: 35500 + | > loss: -0.21422 (-0.14864) + | > log_mle: -0.40799 (-0.30727) + | > loss_dur: 0.19377 (0.15863) + | > amp_scaler: 4096.00000 (2590.84337) + | > grad_norm: 46.35788 (22.91869) + | > current_lr: 0.00004 + | > step_time: 2.80650 (2.31688) + | > loader_time: 0.00330 (0.12221) + + + --> STEP: 171/234 -- GLOBAL_STEP: 35505 + | > loss: -0.29888 (-0.15208) + | > log_mle: -0.50913 (-0.31256) + | > loss_dur: 0.21025 (0.16047) + | > amp_scaler: 4096.00000 (2634.85380) + | > grad_norm: 63.48531 (24.33056) + | > current_lr: 0.00004 + | > step_time: 1.49620 (2.33531) + | > loader_time: 0.00280 (0.12081) + + + --> STEP: 176/234 -- GLOBAL_STEP: 35510 + | > loss: -0.27045 (-0.15538) + | > log_mle: -0.48855 (-0.31773) + | > loss_dur: 0.21811 (0.16235) + | > amp_scaler: 4096.00000 (2676.36364) + | > grad_norm: 59.56654 (25.61213) + | > current_lr: 0.00004 + | > step_time: 2.38960 (2.33830) + | > loader_time: 0.00290 (0.11846) + + + --> STEP: 181/234 -- GLOBAL_STEP: 35515 + | > loss: -0.20542 (-0.15815) + | > log_mle: -0.42435 (-0.32242) + | > loss_dur: 0.21892 (0.16427) + | > amp_scaler: 4096.00000 (2715.58011) + | > grad_norm: 39.49730 (26.58046) + | > current_lr: 0.00004 + | > step_time: 4.60100 (2.41944) + | > loader_time: 0.09870 (0.11792) + + + --> STEP: 186/234 -- GLOBAL_STEP: 35520 + | > loss: -0.23086 (-0.16103) + | > log_mle: -0.46779 (-0.32723) + | > loss_dur: 0.23693 (0.16620) + | > amp_scaler: 4096.00000 (2752.68817) + | > grad_norm: 47.86748 (27.67175) + | > current_lr: 0.00004 + | > step_time: 4.00780 (2.50926) + | > loader_time: 0.09330 (0.11694) + + + --> STEP: 191/234 -- GLOBAL_STEP: 35525 + | > loss: -0.25875 (-0.16391) + | > log_mle: -0.47344 (-0.33169) + | > loss_dur: 0.21469 (0.16778) + | > amp_scaler: 4096.00000 (2787.85340) + | > grad_norm: 61.35219 (28.94125) + | > current_lr: 0.00004 + | > step_time: 4.98730 (2.59682) + | > loader_time: 0.30670 (0.11767) + + + --> STEP: 196/234 -- GLOBAL_STEP: 35530 + | > loss: -0.24086 (-0.16695) + | > log_mle: -0.47300 (-0.33631) + | > loss_dur: 0.23214 (0.16936) + | > amp_scaler: 4096.00000 (2821.22449) + | > grad_norm: 59.41204 (30.15186) + | > current_lr: 0.00004 + | > step_time: 3.80410 (2.61072) + | > loader_time: 0.09700 (0.11829) + + + --> STEP: 201/234 -- GLOBAL_STEP: 35535 + | > loss: -0.19597 (-0.16951) + | > log_mle: -0.43412 (-0.34046) + | > loss_dur: 0.23815 (0.17095) + | > amp_scaler: 4096.00000 (2852.93532) + | > grad_norm: 52.07186 (31.21152) + | > current_lr: 0.00004 + | > step_time: 8.29560 (2.75670) + | > loader_time: 0.10950 (0.11758) + + + --> STEP: 206/234 -- GLOBAL_STEP: 35540 + | > loss: -0.29211 (-0.17252) + | > log_mle: -0.53572 (-0.34512) + | > loss_dur: 0.24361 (0.17261) + | > amp_scaler: 4096.00000 (2883.10680) + | > grad_norm: 73.39498 (32.30808) + | > current_lr: 0.00004 + | > step_time: 4.60350 (2.79314) + | > loader_time: 0.08840 (0.11710) + + + --> STEP: 211/234 -- GLOBAL_STEP: 35545 + | > loss: -0.34035 (-0.17594) + | > log_mle: -0.60445 (-0.35032) + | > loss_dur: 0.26410 (0.17438) + | > amp_scaler: 4096.00000 (2911.84834) + | > grad_norm: 113.54592 (33.53329) + | > current_lr: 0.00004 + | > step_time: 6.09200 (2.83215) + | > loader_time: 0.00600 (0.11565) + + + --> STEP: 216/234 -- GLOBAL_STEP: 35550 + | > loss: -0.32766 (-0.17920) + | > log_mle: -0.59650 (-0.35525) + | > loss_dur: 0.26884 (0.17605) + | > amp_scaler: 4096.00000 (2939.25926) + | > grad_norm: 88.90334 (34.68921) + | > current_lr: 0.00004 + | > step_time: 4.79090 (2.89715) + | > loader_time: 0.09480 (0.11390) + + + --> STEP: 221/234 -- GLOBAL_STEP: 35555 + | > loss: -0.29138 (-0.18268) + | > log_mle: -0.52010 (-0.36027) + | > loss_dur: 0.22872 (0.17759) + | > amp_scaler: 4096.00000 (2965.42986) + | > grad_norm: 78.75420 (35.92817) + | > current_lr: 0.00004 + | > step_time: 5.68950 (2.89665) + | > loader_time: 0.00640 (0.11222) + + + --> STEP: 226/234 -- GLOBAL_STEP: 35560 + | > loss: -0.37535 (-0.18654) + | > log_mle: -0.62490 (-0.36592) + | > loss_dur: 0.24954 (0.17938) + | > amp_scaler: 4096.00000 (2990.44248) + | > grad_norm: 103.50549 (37.17431) + | > current_lr: 0.00004 + | > step_time: 1.49170 (2.87981) + | > loader_time: 0.00380 (0.11058) + + + --> STEP: 231/234 -- GLOBAL_STEP: 35565 + | > loss: -0.29157 (-0.18947) + | > log_mle: -0.68792 (-0.37189) + | > loss_dur: 0.39635 (0.18242) + | > amp_scaler: 4096.00000 (3014.37229) + | > grad_norm: 112.42941 (38.50211) + | > current_lr: 0.00004 + | > step_time: 0.28240 (2.82465) + | > loader_time: 0.00360 (0.10828) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.65519 (+0.50882) + | > avg_loss: -0.21758 (-0.02675) + | > avg_log_mle: -0.44404 (-0.01566) + | > avg_loss_dur: 0.22646 (-0.01109) + + + > EPOCH: 152/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 17:06:02)  + + --> STEP: 2/234 -- GLOBAL_STEP: 35570 + | > loss: -0.16207 (-0.15265) + | > log_mle: -0.26657 (-0.26371) + | > loss_dur: 0.10450 (0.11107) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.02484 (15.11854) + | > current_lr: 0.00004 + | > step_time: 10.91330 (6.50461) + | > loader_time: 0.08840 (9.64130) + + + --> STEP: 7/234 -- GLOBAL_STEP: 35575 + | > loss: -0.13766 (-0.12594) + | > log_mle: -0.26838 (-0.25935) + | > loss_dur: 0.13072 (0.13342) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.57341 (16.45164) + | > current_lr: 0.00004 + | > step_time: 0.70420 (4.54249) + | > loader_time: 0.08560 (2.79499) + + + --> STEP: 12/234 -- GLOBAL_STEP: 35580 + | > loss: -0.13485 (-0.12832) + | > log_mle: -0.26074 (-0.26107) + | > loss_dur: 0.12589 (0.13274) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.62103 (15.02628) + | > current_lr: 0.00004 + | > step_time: 1.33740 (3.09607) + | > loader_time: 0.00210 (1.63113) + + + --> STEP: 17/234 -- GLOBAL_STEP: 35585 + | > loss: -0.12370 (-0.13209) + | > log_mle: -0.23819 (-0.25980) + | > loss_dur: 0.11448 (0.12770) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.10542 (13.73616) + | > current_lr: 0.00004 + | > step_time: 1.06100 (2.52591) + | > loader_time: 0.00190 (1.15184) + + + --> STEP: 22/234 -- GLOBAL_STEP: 35590 + | > loss: -0.13776 (-0.13135) + | > log_mle: -0.25903 (-0.25750) + | > loss_dur: 0.12127 (0.12616) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 15.13525 (13.27389) + | > current_lr: 0.00004 + | > step_time: 1.19780 (2.28786) + | > loader_time: 0.00240 (0.89467) + + + --> STEP: 27/234 -- GLOBAL_STEP: 35595 + | > loss: -0.14898 (-0.13349) + | > log_mle: -0.26089 (-0.25704) + | > loss_dur: 0.11191 (0.12354) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.08947 (12.48926) + | > current_lr: 0.00004 + | > step_time: 1.81560 (2.19894) + | > loader_time: 0.00190 (0.73325) + + + --> STEP: 32/234 -- GLOBAL_STEP: 35600 + | > loss: -0.17375 (-0.13405) + | > log_mle: -0.26934 (-0.25692) + | > loss_dur: 0.09559 (0.12287) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.47549 (12.10457) + | > current_lr: 0.00004 + | > step_time: 1.12440 (2.04498) + | > loader_time: 0.00200 (0.62163) + + + --> STEP: 37/234 -- GLOBAL_STEP: 35605 + | > loss: -0.14472 (-0.13242) + | > log_mle: -0.24892 (-0.25612) + | > loss_dur: 0.10420 (0.12369) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.91816 (12.06574) + | > current_lr: 0.00004 + | > step_time: 3.50140 (2.01530) + | > loader_time: 0.00290 (0.54063) + + + --> STEP: 42/234 -- GLOBAL_STEP: 35610 + | > loss: -0.11427 (-0.13097) + | > log_mle: -0.24300 (-0.25566) + | > loss_dur: 0.12873 (0.12469) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.13344 (11.59967) + | > current_lr: 0.00004 + | > step_time: 2.50330 (2.13237) + | > loader_time: 0.08870 (0.48107) + + + --> STEP: 47/234 -- GLOBAL_STEP: 35615 + | > loss: -0.10989 (-0.13008) + | > log_mle: -0.25166 (-0.25600) + | > loss_dur: 0.14177 (0.12592) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.93010 (11.35140) + | > current_lr: 0.00004 + | > step_time: 2.50070 (2.12474) + | > loader_time: 0.09750 (0.43414) + + + --> STEP: 52/234 -- GLOBAL_STEP: 35620 + | > loss: -0.10079 (-0.12899) + | > log_mle: -0.24884 (-0.25528) + | > loss_dur: 0.14805 (0.12630) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.91324 (10.97294) + | > current_lr: 0.00004 + | > step_time: 1.26940 (2.05459) + | > loader_time: 0.00170 (0.39267) + + + --> STEP: 57/234 -- GLOBAL_STEP: 35625 + | > loss: -0.08447 (-0.12826) + | > log_mle: -0.23849 (-0.25549) + | > loss_dur: 0.15402 (0.12723) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.50394 (10.76472) + | > current_lr: 0.00004 + | > step_time: 1.75480 (2.01916) + | > loader_time: 0.00230 (0.35846) + + + --> STEP: 62/234 -- GLOBAL_STEP: 35630 + | > loss: -0.09898 (-0.12808) + | > log_mle: -0.28945 (-0.25654) + | > loss_dur: 0.19047 (0.12845) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.77529 (10.98634) + | > current_lr: 0.00004 + | > step_time: 1.01090 (1.96041) + | > loader_time: 0.00240 (0.33109) + + + --> STEP: 67/234 -- GLOBAL_STEP: 35635 + | > loss: -0.11674 (-0.12754) + | > log_mle: -0.27104 (-0.25644) + | > loss_dur: 0.15430 (0.12890) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.22296 (10.84052) + | > current_lr: 0.00004 + | > step_time: 1.58140 (1.92826) + | > loader_time: 0.00170 (0.30656) + + + --> STEP: 72/234 -- GLOBAL_STEP: 35640 + | > loss: -0.11046 (-0.12583) + | > log_mle: -0.25330 (-0.25635) + | > loss_dur: 0.14284 (0.13051) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.94815 (10.86746) + | > current_lr: 0.00004 + | > step_time: 2.59220 (1.91663) + | > loader_time: 0.00270 (0.28662) + + + --> STEP: 77/234 -- GLOBAL_STEP: 35645 + | > loss: -0.13019 (-0.12539) + | > log_mle: -0.26234 (-0.25682) + | > loss_dur: 0.13215 (0.13143) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.24696 (10.91821) + | > current_lr: 0.00004 + | > step_time: 1.50260 (1.89740) + | > loader_time: 0.00270 (0.26935) + + + --> STEP: 82/234 -- GLOBAL_STEP: 35650 + | > loss: -0.11621 (-0.12497) + | > log_mle: -0.25205 (-0.25678) + | > loss_dur: 0.13584 (0.13181) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.10704 (10.92575) + | > current_lr: 0.00004 + | > step_time: 3.33030 (1.93198) + | > loader_time: 0.08950 (0.25641) + + + --> STEP: 87/234 -- GLOBAL_STEP: 35655 + | > loss: -0.11396 (-0.12446) + | > log_mle: -0.26108 (-0.25713) + | > loss_dur: 0.14712 (0.13267) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.80781 (10.98138) + | > current_lr: 0.00004 + | > step_time: 1.32350 (1.93489) + | > loader_time: 0.00250 (0.24278) + + + --> STEP: 92/234 -- GLOBAL_STEP: 35660 + | > loss: -0.15552 (-0.12507) + | > log_mle: -0.30476 (-0.25906) + | > loss_dur: 0.14924 (0.13400) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.09252 (11.37211) + | > current_lr: 0.00004 + | > step_time: 2.90700 (1.93864) + | > loader_time: 0.10350 (0.23273) + + + --> STEP: 97/234 -- GLOBAL_STEP: 35665 + | > loss: -0.13592 (-0.12626) + | > log_mle: -0.29427 (-0.26184) + | > loss_dur: 0.15835 (0.13558) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.76647 (12.00239) + | > current_lr: 0.00004 + | > step_time: 2.09220 (1.92405) + | > loader_time: 0.00860 (0.22183) + + + --> STEP: 102/234 -- GLOBAL_STEP: 35670 + | > loss: -0.11677 (-0.12663) + | > log_mle: -0.27831 (-0.26383) + | > loss_dur: 0.16154 (0.13720) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.10617 (12.39858) + | > current_lr: 0.00004 + | > step_time: 1.68990 (1.93239) + | > loader_time: 0.00250 (0.21208) + + + --> STEP: 107/234 -- GLOBAL_STEP: 35675 + | > loss: -0.14695 (-0.12786) + | > log_mle: -0.32426 (-0.26692) + | > loss_dur: 0.17731 (0.13907) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 22.87003 (13.01391) + | > current_lr: 0.00004 + | > step_time: 1.59630 (1.95251) + | > loader_time: 0.00300 (0.20314) + + + --> STEP: 112/234 -- GLOBAL_STEP: 35680 + | > loss: -0.14313 (-0.12846) + | > log_mle: -0.33397 (-0.26972) + | > loss_dur: 0.19083 (0.14126) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.43603 (13.67880) + | > current_lr: 0.00004 + | > step_time: 1.90300 (1.97059) + | > loader_time: 0.09410 (0.19582) + + + --> STEP: 117/234 -- GLOBAL_STEP: 35685 + | > loss: -0.15786 (-0.12936) + | > log_mle: -0.33066 (-0.27234) + | > loss_dur: 0.17280 (0.14297) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.72519 (14.40291) + | > current_lr: 0.00004 + | > step_time: 2.01070 (1.99321) + | > loader_time: 0.10010 (0.19007) + + + --> STEP: 122/234 -- GLOBAL_STEP: 35690 + | > loss: -0.13755 (-0.12975) + | > log_mle: -0.30442 (-0.27396) + | > loss_dur: 0.16687 (0.14421) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 20.69117 (14.75506) + | > current_lr: 0.00004 + | > step_time: 1.89630 (1.98341) + | > loader_time: 0.09900 (0.18323) + + + --> STEP: 127/234 -- GLOBAL_STEP: 35695 + | > loss: -0.16851 (-0.13110) + | > log_mle: -0.36392 (-0.27690) + | > loss_dur: 0.19542 (0.14580) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.10097 (15.34862) + | > current_lr: 0.00004 + | > step_time: 2.30830 (1.98575) + | > loader_time: 0.00420 (0.17614) + + + --> STEP: 132/234 -- GLOBAL_STEP: 35700 + | > loss: -0.17753 (-0.13306) + | > log_mle: -0.34461 (-0.28030) + | > loss_dur: 0.16708 (0.14724) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.42474 (16.14154) + | > current_lr: 0.00004 + | > step_time: 1.69480 (1.99914) + | > loader_time: 0.00180 (0.17240) + + + --> STEP: 137/234 -- GLOBAL_STEP: 35705 + | > loss: -0.14363 (-0.13476) + | > log_mle: -0.35971 (-0.28386) + | > loss_dur: 0.21608 (0.14910) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.13857 (17.10538) + | > current_lr: 0.00004 + | > step_time: 2.49540 (2.02536) + | > loader_time: 0.00290 (0.16764) + + + --> STEP: 142/234 -- GLOBAL_STEP: 35710 + | > loss: -0.16547 (-0.13620) + | > log_mle: -0.36656 (-0.28687) + | > loss_dur: 0.20109 (0.15067) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.84838 (17.95898) + | > current_lr: 0.00004 + | > step_time: 4.51500 (2.04444) + | > loader_time: 0.08830 (0.16305) + + + --> STEP: 147/234 -- GLOBAL_STEP: 35715 + | > loss: -0.16839 (-0.13867) + | > log_mle: -0.37060 (-0.29150) + | > loss_dur: 0.20221 (0.15283) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 36.63664 (18.98503) + | > current_lr: 0.00004 + | > step_time: 1.01320 (2.02465) + | > loader_time: 0.18890 (0.15890) + + + --> STEP: 152/234 -- GLOBAL_STEP: 35720 + | > loss: -0.23331 (-0.14153) + | > log_mle: -0.45664 (-0.29592) + | > loss_dur: 0.22333 (0.15439) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 51.48763 (19.82900) + | > current_lr: 0.00004 + | > step_time: 1.90480 (2.02081) + | > loader_time: 0.00450 (0.15380) + + + --> STEP: 157/234 -- GLOBAL_STEP: 35725 + | > loss: -0.20086 (-0.14491) + | > log_mle: -0.40570 (-0.30093) + | > loss_dur: 0.20484 (0.15602) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 33.53244 (21.01698) + | > current_lr: 0.00004 + | > step_time: 5.30460 (2.04834) + | > loader_time: 0.10420 (0.15020) + + + --> STEP: 162/234 -- GLOBAL_STEP: 35730 + | > loss: -0.24870 (-0.14805) + | > log_mle: -0.43901 (-0.30575) + | > loss_dur: 0.19031 (0.15770) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 49.63055 (22.03813) + | > current_lr: 0.00004 + | > step_time: 5.10650 (2.06894) + | > loader_time: 0.09350 (0.14623) + + + --> STEP: 167/234 -- GLOBAL_STEP: 35735 + | > loss: -0.31281 (-0.15097) + | > log_mle: -0.51899 (-0.31010) + | > loss_dur: 0.20619 (0.15914) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.15602 (23.32141) + | > current_lr: 0.00004 + | > step_time: 2.11820 (2.08023) + | > loader_time: 0.08970 (0.14480) + + + --> STEP: 172/234 -- GLOBAL_STEP: 35740 + | > loss: -0.26858 (-0.15391) + | > log_mle: -0.50786 (-0.31515) + | > loss_dur: 0.23928 (0.16124) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.74352 (24.54501) + | > current_lr: 0.00004 + | > step_time: 4.01520 (2.07750) + | > loader_time: 0.08420 (0.14173) + + + --> STEP: 177/234 -- GLOBAL_STEP: 35745 + | > loss: -0.24526 (-0.15707) + | > log_mle: -0.46843 (-0.32007) + | > loss_dur: 0.22317 (0.16300) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 61.01598 (25.62489) + | > current_lr: 0.00004 + | > step_time: 3.99600 (2.08374) + | > loader_time: 0.00370 (0.13881) + + + --> STEP: 182/234 -- GLOBAL_STEP: 35750 + | > loss: -0.26555 (-0.15993) + | > log_mle: -0.51753 (-0.32500) + | > loss_dur: 0.25198 (0.16507) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.05945 (26.77760) + | > current_lr: 0.00004 + | > step_time: 1.20480 (2.08282) + | > loader_time: 0.00270 (0.13749) + + + --> STEP: 187/234 -- GLOBAL_STEP: 35755 + | > loss: -0.28704 (-0.16298) + | > log_mle: -0.51712 (-0.32991) + | > loss_dur: 0.23008 (0.16693) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.18681 (28.00967) + | > current_lr: 0.00004 + | > step_time: 6.11200 (2.10908) + | > loader_time: 0.19560 (0.13537) + + + --> STEP: 192/234 -- GLOBAL_STEP: 35760 + | > loss: -0.30921 (-0.16614) + | > log_mle: -0.52494 (-0.33455) + | > loss_dur: 0.21572 (0.16841) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 119.31173 (29.47908) + | > current_lr: 0.00004 + | > step_time: 5.49220 (2.17912) + | > loader_time: 0.01160 (0.13560) + + + --> STEP: 197/234 -- GLOBAL_STEP: 35765 + | > loss: -0.30450 (-0.16903) + | > log_mle: -0.51271 (-0.33899) + | > loss_dur: 0.20821 (0.16996) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 56.99488 (30.59629) + | > current_lr: 0.00004 + | > step_time: 1.60120 (2.20462) + | > loader_time: 0.00510 (0.13372) + + + --> STEP: 202/234 -- GLOBAL_STEP: 35770 + | > loss: -0.36808 (-0.17182) + | > log_mle: -0.60245 (-0.34356) + | > loss_dur: 0.23437 (0.17174) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 100.36478 (31.86094) + | > current_lr: 0.00004 + | > step_time: 2.50410 (2.20751) + | > loader_time: 0.00480 (0.13095) + + + --> STEP: 207/234 -- GLOBAL_STEP: 35775 + | > loss: -0.33450 (-0.17461) + | > log_mle: -0.58431 (-0.34813) + | > loss_dur: 0.24981 (0.17352) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 87.12498 (33.26329) + | > current_lr: 0.00004 + | > step_time: 5.10430 (2.25769) + | > loader_time: 0.39570 (0.13063) + + + --> STEP: 212/234 -- GLOBAL_STEP: 35780 + | > loss: -0.32747 (-0.17810) + | > log_mle: -0.57023 (-0.35341) + | > loss_dur: 0.24276 (0.17531) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 88.13729 (34.31006) + | > current_lr: 0.00004 + | > step_time: 5.20320 (2.31539) + | > loader_time: 0.18930 (0.12935) + + + --> STEP: 217/234 -- GLOBAL_STEP: 35785 + | > loss: -0.33396 (-0.18169) + | > log_mle: -0.58818 (-0.35866) + | > loss_dur: 0.25422 (0.17696) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 122.17298 (35.72224) + | > current_lr: 0.00004 + | > step_time: 2.00440 (2.37226) + | > loader_time: 0.08610 (0.12805) + + + --> STEP: 222/234 -- GLOBAL_STEP: 35790 + | > loss: -0.33059 (-0.18527) + | > log_mle: -0.60834 (-0.36388) + | > loss_dur: 0.27776 (0.17861) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 89.94884 (37.00917) + | > current_lr: 0.00004 + | > step_time: 2.90530 (2.40040) + | > loader_time: 0.00530 (0.12618) + + + --> STEP: 227/234 -- GLOBAL_STEP: 35795 + | > loss: -0.28898 (-0.18886) + | > log_mle: -0.55618 (-0.36920) + | > loss_dur: 0.26719 (0.18034) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 100.46452 (38.82166) + | > current_lr: 0.00004 + | > step_time: 0.23910 (2.37650) + | > loader_time: 0.00410 (0.12382) + + + --> STEP: 232/234 -- GLOBAL_STEP: 35800 + | > loss: -0.24938 (-0.19124) + | > log_mle: -0.76654 (-0.37562) + | > loss_dur: 0.51716 (0.18438) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 146.50159 (40.41039) + | > current_lr: 0.00004 + | > step_time: 0.35700 (2.33114) + | > loader_time: 0.00870 (0.12127) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05256 (-0.60264) + | > avg_loss: -0.19568 (+0.02190) + | > avg_log_mle: -0.43741 (+0.00664) + | > avg_loss_dur: 0.24173 (+0.01526) + + + > EPOCH: 153/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 17:16:15)  + + --> STEP: 3/234 -- GLOBAL_STEP: 35805 + | > loss: -0.08884 (-0.13130) + | > log_mle: -0.25331 (-0.26030) + | > loss_dur: 0.16447 (0.12900) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.94604 (22.46466) + | > current_lr: 0.00004 + | > step_time: 8.70150 (7.09604) + | > loader_time: 0.11480 (0.07137) + + + --> STEP: 8/234 -- GLOBAL_STEP: 35810 + | > loss: -0.14175 (-0.12628) + | > log_mle: -0.27671 (-0.26135) + | > loss_dur: 0.13496 (0.13507) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 14.23889 (17.98648) + | > current_lr: 0.00004 + | > step_time: 2.70790 (5.02485) + | > loader_time: 0.08370 (0.07311) + + + --> STEP: 13/234 -- GLOBAL_STEP: 35815 + | > loss: -0.15210 (-0.13080) + | > log_mle: -0.26616 (-0.26240) + | > loss_dur: 0.11407 (0.13160) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.85344 (15.49912) + | > current_lr: 0.00004 + | > step_time: 4.01080 (4.89393) + | > loader_time: 0.09330 (0.11975) + + + --> STEP: 18/234 -- GLOBAL_STEP: 35820 + | > loss: -0.12971 (-0.13267) + | > log_mle: -0.25477 (-0.26057) + | > loss_dur: 0.12506 (0.12789) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.25232 (14.23545) + | > current_lr: 0.00004 + | > step_time: 3.00580 (4.50720) + | > loader_time: 0.00190 (0.09297) + + + --> STEP: 23/234 -- GLOBAL_STEP: 35825 + | > loss: -0.16483 (-0.13427) + | > log_mle: -0.26475 (-0.25904) + | > loss_dur: 0.09992 (0.12477) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.01371 (13.26428) + | > current_lr: 0.00004 + | > step_time: 4.20150 (4.38360) + | > loader_time: 0.21100 (0.08238) + + + --> STEP: 28/234 -- GLOBAL_STEP: 35830 + | > loss: -0.16815 (-0.13712) + | > log_mle: -0.25909 (-0.25842) + | > loss_dur: 0.09094 (0.12130) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.12750 (12.47889) + | > current_lr: 0.00004 + | > step_time: 2.91590 (4.46939) + | > loader_time: 0.00440 (0.10326) + + + --> STEP: 33/234 -- GLOBAL_STEP: 35835 + | > loss: -0.13647 (-0.13647) + | > log_mle: -0.25110 (-0.25813) + | > loss_dur: 0.11463 (0.12165) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.56305 (11.90070) + | > current_lr: 0.00004 + | > step_time: 2.34770 (4.01512) + | > loader_time: 0.00190 (0.09047) + + + --> STEP: 38/234 -- GLOBAL_STEP: 35840 + | > loss: -0.13471 (-0.13479) + | > log_mle: -0.26812 (-0.25793) + | > loss_dur: 0.13341 (0.12314) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 5.84209 (11.65365) + | > current_lr: 0.00004 + | > step_time: 1.09240 (3.66597) + | > loader_time: 0.00200 (0.08088) + + + --> STEP: 43/234 -- GLOBAL_STEP: 35845 + | > loss: -0.11121 (-0.13316) + | > log_mle: -0.26433 (-0.25735) + | > loss_dur: 0.15313 (0.12418) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.20895 (11.57063) + | > current_lr: 0.00004 + | > step_time: 0.75750 (3.36858) + | > loader_time: 0.00220 (0.07173) + + + --> STEP: 48/234 -- GLOBAL_STEP: 35850 + | > loss: -0.13619 (-0.13258) + | > log_mle: -0.24783 (-0.25724) + | > loss_dur: 0.11164 (0.12466) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 6.45687 (11.49216) + | > current_lr: 0.00004 + | > step_time: 0.98480 (3.13237) + | > loader_time: 0.00200 (0.06447) + + + --> STEP: 53/234 -- GLOBAL_STEP: 35855 + | > loss: -0.12681 (-0.13182) + | > log_mle: -0.26441 (-0.25686) + | > loss_dur: 0.13760 (0.12503) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.83688 (11.22737) + | > current_lr: 0.00004 + | > step_time: 1.49100 (2.96968) + | > loader_time: 0.00450 (0.06215) + + + --> STEP: 58/234 -- GLOBAL_STEP: 35860 + | > loss: -0.13566 (-0.13179) + | > log_mle: -0.24971 (-0.25681) + | > loss_dur: 0.11405 (0.12502) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.59007 (10.99503) + | > current_lr: 0.00004 + | > step_time: 2.25660 (2.85650) + | > loader_time: 0.00210 (0.05987) + + + --> STEP: 63/234 -- GLOBAL_STEP: 35865 + | > loss: -0.11446 (-0.13139) + | > log_mle: -0.25481 (-0.25783) + | > loss_dur: 0.14036 (0.12644) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.55289 (11.29193) + | > current_lr: 0.00004 + | > step_time: 1.40770 (2.75199) + | > loader_time: 0.00280 (0.05531) + + + --> STEP: 68/234 -- GLOBAL_STEP: 35870 + | > loss: -0.09364 (-0.13039) + | > log_mle: -0.24762 (-0.25751) + | > loss_dur: 0.15398 (0.12712) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.08690 (11.09811) + | > current_lr: 0.00004 + | > step_time: 1.79310 (2.67304) + | > loader_time: 0.00340 (0.05403) + + + --> STEP: 73/234 -- GLOBAL_STEP: 35875 + | > loss: -0.10508 (-0.12877) + | > log_mle: -0.27081 (-0.25766) + | > loss_dur: 0.16573 (0.12890) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.79656 (11.07968) + | > current_lr: 0.00004 + | > step_time: 1.90770 (2.63372) + | > loader_time: 0.09610 (0.05180) + + + --> STEP: 78/234 -- GLOBAL_STEP: 35880 + | > loss: -0.10626 (-0.12797) + | > log_mle: -0.24810 (-0.25769) + | > loss_dur: 0.14183 (0.12972) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.88734 (11.07416) + | > current_lr: 0.00004 + | > step_time: 1.10710 (2.57477) + | > loader_time: 0.00210 (0.04956) + + + --> STEP: 83/234 -- GLOBAL_STEP: 35885 + | > loss: -0.10492 (-0.12773) + | > log_mle: -0.27234 (-0.25803) + | > loss_dur: 0.16742 (0.13029) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.53218 (11.09532) + | > current_lr: 0.00004 + | > step_time: 2.90930 (2.54610) + | > loader_time: 0.00310 (0.04673) + + + --> STEP: 88/234 -- GLOBAL_STEP: 35890 + | > loss: -0.14447 (-0.12748) + | > log_mle: -0.30969 (-0.25892) + | > loss_dur: 0.16522 (0.13144) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 23.23493 (11.24046) + | > current_lr: 0.00004 + | > step_time: 2.39230 (2.55240) + | > loader_time: 0.00330 (0.04536) + + + --> STEP: 93/234 -- GLOBAL_STEP: 35895 + | > loss: -0.14224 (-0.12813) + | > log_mle: -0.32245 (-0.26113) + | > loss_dur: 0.18021 (0.13300) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.43267 (11.68140) + | > current_lr: 0.00004 + | > step_time: 1.70770 (2.52290) + | > loader_time: 0.00380 (0.04709) + + + --> STEP: 98/234 -- GLOBAL_STEP: 35900 + | > loss: -0.10972 (-0.12898) + | > log_mle: -0.25161 (-0.26330) + | > loss_dur: 0.14189 (0.13432) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.85058 (12.32098) + | > current_lr: 0.00004 + | > step_time: 1.61920 (2.49003) + | > loader_time: 0.08820 (0.04571) + + + --> STEP: 103/234 -- GLOBAL_STEP: 35905 + | > loss: -0.16921 (-0.13026) + | > log_mle: -0.35627 (-0.26632) + | > loss_dur: 0.18707 (0.13606) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.73264 (12.85072) + | > current_lr: 0.00004 + | > step_time: 3.00930 (2.45674) + | > loader_time: 0.00470 (0.04366) + + + --> STEP: 108/234 -- GLOBAL_STEP: 35910 + | > loss: -0.14741 (-0.13138) + | > log_mle: -0.29834 (-0.26881) + | > loss_dur: 0.15093 (0.13743) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 16.48768 (13.37969) + | > current_lr: 0.00004 + | > step_time: 3.00080 (2.43831) + | > loader_time: 0.09520 (0.04263) + + + --> STEP: 113/234 -- GLOBAL_STEP: 35915 + | > loss: -0.16162 (-0.13208) + | > log_mle: -0.34374 (-0.27197) + | > loss_dur: 0.18212 (0.13989) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 35.51633 (14.14709) + | > current_lr: 0.00004 + | > step_time: 2.01570 (2.41864) + | > loader_time: 0.08360 (0.04236) + + + --> STEP: 118/234 -- GLOBAL_STEP: 35920 + | > loss: -0.13690 (-0.13264) + | > log_mle: -0.31427 (-0.27430) + | > loss_dur: 0.17737 (0.14166) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 26.38156 (14.56478) + | > current_lr: 0.00004 + | > step_time: 1.59950 (2.41788) + | > loader_time: 0.00600 (0.04381) + + + --> STEP: 123/234 -- GLOBAL_STEP: 35925 + | > loss: -0.12348 (-0.13299) + | > log_mle: -0.28369 (-0.27569) + | > loss_dur: 0.16021 (0.14270) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 19.16545 (14.84501) + | > current_lr: 0.00004 + | > step_time: 2.80800 (2.39280) + | > loader_time: 0.08580 (0.04358) + + + --> STEP: 128/234 -- GLOBAL_STEP: 35930 + | > loss: -0.17585 (-0.13492) + | > log_mle: -0.34332 (-0.27908) + | > loss_dur: 0.16746 (0.14416) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 34.79190 (15.70054) + | > current_lr: 0.00004 + | > step_time: 1.89320 (2.36185) + | > loader_time: 0.10370 (0.04347) + + + --> STEP: 133/234 -- GLOBAL_STEP: 35935 + | > loss: -0.18014 (-0.13667) + | > log_mle: -0.37232 (-0.28267) + | > loss_dur: 0.19218 (0.14599) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.41459 (16.46599) + | > current_lr: 0.00004 + | > step_time: 2.81890 (2.34603) + | > loader_time: 0.00300 (0.04262) + + + --> STEP: 138/234 -- GLOBAL_STEP: 35940 + | > loss: -0.13838 (-0.13784) + | > log_mle: -0.32347 (-0.28592) + | > loss_dur: 0.18509 (0.14808) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.17037 (17.22587) + | > current_lr: 0.00004 + | > step_time: 1.50090 (2.33146) + | > loader_time: 0.08320 (0.04244) + + + --> STEP: 143/234 -- GLOBAL_STEP: 35945 + | > loss: -0.21701 (-0.13975) + | > log_mle: -0.46151 (-0.28995) + | > loss_dur: 0.24451 (0.15021) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 70.18388 (18.41348) + | > current_lr: 0.00004 + | > step_time: 2.30020 (2.33519) + | > loader_time: 0.00290 (0.04111) + + + --> STEP: 148/234 -- GLOBAL_STEP: 35950 + | > loss: -0.19323 (-0.14208) + | > log_mle: -0.37202 (-0.29396) + | > loss_dur: 0.17879 (0.15188) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.91072 (19.55277) + | > current_lr: 0.00004 + | > step_time: 2.10490 (2.33477) + | > loader_time: 0.08740 (0.04308) + + + --> STEP: 153/234 -- GLOBAL_STEP: 35955 + | > loss: -0.29414 (-0.14533) + | > log_mle: -0.50281 (-0.29907) + | > loss_dur: 0.20867 (0.15374) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 76.04417 (20.81793) + | > current_lr: 0.00004 + | > step_time: 2.79800 (2.39311) + | > loader_time: 0.00440 (0.04544) + + + --> STEP: 158/234 -- GLOBAL_STEP: 35960 + | > loss: -0.21907 (-0.14813) + | > log_mle: -0.43918 (-0.30364) + | > loss_dur: 0.22011 (0.15551) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 82.00414 (21.94401) + | > current_lr: 0.00004 + | > step_time: 1.59910 (2.36555) + | > loader_time: 0.00410 (0.04413) + + + --> STEP: 163/234 -- GLOBAL_STEP: 35965 + | > loss: -0.19901 (-0.15075) + | > log_mle: -0.40195 (-0.30778) + | > loss_dur: 0.20295 (0.15703) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 48.46138 (23.15022) + | > current_lr: 0.00004 + | > step_time: 3.69700 (2.37455) + | > loader_time: 0.00250 (0.04345) + + + --> STEP: 168/234 -- GLOBAL_STEP: 35970 + | > loss: -0.22331 (-0.15359) + | > log_mle: -0.46102 (-0.31231) + | > loss_dur: 0.23771 (0.15872) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 66.75534 (24.10721) + | > current_lr: 0.00004 + | > step_time: 0.99400 (2.36637) + | > loader_time: 0.00450 (0.04284) + + + --> STEP: 173/234 -- GLOBAL_STEP: 35975 + | > loss: -0.24735 (-0.15676) + | > log_mle: -0.47233 (-0.31744) + | > loss_dur: 0.22498 (0.16068) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 64.73556 (25.40555) + | > current_lr: 0.00004 + | > step_time: 1.49610 (2.36966) + | > loader_time: 0.00310 (0.04213) + + + --> STEP: 178/234 -- GLOBAL_STEP: 35980 + | > loss: -0.28421 (-0.16001) + | > log_mle: -0.52919 (-0.32263) + | > loss_dur: 0.24498 (0.16262) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 70.78857 (26.54964) + | > current_lr: 0.00004 + | > step_time: 5.69690 (2.40928) + | > loader_time: 0.19790 (0.04321) + + + --> STEP: 183/234 -- GLOBAL_STEP: 35985 + | > loss: -0.30058 (-0.16280) + | > log_mle: -0.52888 (-0.32743) + | > loss_dur: 0.22830 (0.16463) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 74.42390 (27.63882) + | > current_lr: 0.00004 + | > step_time: 2.89810 (2.43963) + | > loader_time: 0.09980 (0.04480) + + + --> STEP: 188/234 -- GLOBAL_STEP: 35990 + | > loss: -0.31023 (-0.16576) + | > log_mle: -0.54174 (-0.33228) + | > loss_dur: 0.23151 (0.16652) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 87.03657 (28.70260) + | > current_lr: 0.00004 + | > step_time: 3.60550 (2.45366) + | > loader_time: 0.09090 (0.04565) + + + --> STEP: 193/234 -- GLOBAL_STEP: 35995 + | > loss: -0.31724 (-0.16901) + | > log_mle: -0.54465 (-0.33701) + | > loss_dur: 0.22742 (0.16799) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 78.42444 (29.83568) + | > current_lr: 0.00004 + | > step_time: 1.79700 (2.44900) + | > loader_time: 0.00330 (0.04507) + + + --> STEP: 198/234 -- GLOBAL_STEP: 36000 + | > loss: -0.30125 (-0.17206) + | > log_mle: -0.53752 (-0.34158) + | > loss_dur: 0.23627 (0.16952) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 83.70452 (30.89346) + | > current_lr: 0.00004 + | > step_time: 1.48800 (2.44583) + | > loader_time: 0.01160 (0.04483) + + + --> STEP: 203/234 -- GLOBAL_STEP: 36005 + | > loss: -0.23716 (-0.17455) + | > log_mle: -0.46992 (-0.34582) + | > loss_dur: 0.23276 (0.17127) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.87236 (32.12081) + | > current_lr: 0.00004 + | > step_time: 4.19810 (2.49445) + | > loader_time: 0.00460 (0.04478) + + + --> STEP: 208/234 -- GLOBAL_STEP: 36010 + | > loss: -0.28825 (-0.17772) + | > log_mle: -0.54170 (-0.35077) + | > loss_dur: 0.25346 (0.17305) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 109.11976 (33.54667) + | > current_lr: 0.00004 + | > step_time: 5.20700 (2.53399) + | > loader_time: 0.00390 (0.04516) + + + --> STEP: 213/234 -- GLOBAL_STEP: 36015 + | > loss: -0.34532 (-0.18142) + | > log_mle: -0.60259 (-0.35625) + | > loss_dur: 0.25726 (0.17483) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 90.91396 (34.91200) + | > current_lr: 0.00004 + | > step_time: 11.49770 (2.61262) + | > loader_time: 0.30600 (0.04676) + + + --> STEP: 218/234 -- GLOBAL_STEP: 36020 + | > loss: -0.31444 (-0.18487) + | > log_mle: -0.55217 (-0.36129) + | > loss_dur: 0.23773 (0.17642) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 112.50871 (36.33064) + | > current_lr: 0.00004 + | > step_time: 5.18680 (2.67745) + | > loader_time: 0.09430 (0.04839) + + + --> STEP: 223/234 -- GLOBAL_STEP: 36025 + | > loss: -0.35369 (-0.18857) + | > log_mle: -0.60116 (-0.36664) + | > loss_dur: 0.24747 (0.17807) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 95.91345 (37.68317) + | > current_lr: 0.00004 + | > step_time: 0.23190 (2.63739) + | > loader_time: 0.00280 (0.04810) + + + --> STEP: 228/234 -- GLOBAL_STEP: 36030 + | > loss: -0.32479 (-0.19218) + | > log_mle: -0.60218 (-0.37214) + | > loss_dur: 0.27739 (0.17996) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 128.88419 (39.27898) + | > current_lr: 0.00004 + | > step_time: 0.25170 (2.58488) + | > loader_time: 0.00340 (0.04713) + + + --> STEP: 233/234 -- GLOBAL_STEP: 36035 + | > loss: 0.28721 (-0.19238) + | > log_mle: -0.53893 (-0.37871) + | > loss_dur: 0.82615 (0.18633) + | > amp_scaler: 2048.00000 (4087.21030) + | > grad_norm: 0.00000 (40.53416) + | > current_lr: 0.00004 + | > step_time: 0.14930 (2.53490) + | > loader_time: 0.00270 (0.04626) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.10190 (+1.04935) + | > avg_loss: -0.18191 (+0.01377) + | > avg_log_mle: -0.40679 (+0.03062) + | > avg_loss_dur: 0.22487 (-0.01685) + + + > EPOCH: 154/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 17:27:23)  + + --> STEP: 4/234 -- GLOBAL_STEP: 36040 + | > loss: -0.07792 (-0.11606) + | > log_mle: -0.24393 (-0.25491) + | > loss_dur: 0.16601 (0.13886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.61206 (23.57845) + | > current_lr: 0.00004 + | > step_time: 5.01220 (8.60107) + | > loader_time: 0.09400 (0.12304) + + + --> STEP: 9/234 -- GLOBAL_STEP: 36045 + | > loss: -0.12006 (-0.12810) + | > log_mle: -0.26700 (-0.26003) + | > loss_dur: 0.14694 (0.13193) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.37320 (18.21611) + | > current_lr: 0.00004 + | > step_time: 5.98730 (5.57908) + | > loader_time: 0.10260 (0.08590) + + + --> STEP: 14/234 -- GLOBAL_STEP: 36050 + | > loss: -0.12984 (-0.13052) + | > log_mle: -0.26463 (-0.26073) + | > loss_dur: 0.13479 (0.13020) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.41861 (16.21199) + | > current_lr: 0.00004 + | > step_time: 0.78310 (4.82890) + | > loader_time: 0.00120 (0.06770) + + + --> STEP: 19/234 -- GLOBAL_STEP: 36055 + | > loss: -0.14581 (-0.13389) + | > log_mle: -0.25288 (-0.25877) + | > loss_dur: 0.10707 (0.12488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.08601 (14.48529) + | > current_lr: 0.00004 + | > step_time: 3.19460 (4.12414) + | > loader_time: 0.00220 (0.05031) + + + --> STEP: 24/234 -- GLOBAL_STEP: 36060 + | > loss: -0.14864 (-0.13550) + | > log_mle: -0.25171 (-0.25749) + | > loss_dur: 0.10306 (0.12199) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.24661 (13.63029) + | > current_lr: 0.00004 + | > step_time: 2.20600 (4.01565) + | > loader_time: 0.00260 (0.04409) + + + --> STEP: 29/234 -- GLOBAL_STEP: 36065 + | > loss: -0.10552 (-0.13601) + | > log_mle: -0.23693 (-0.25652) + | > loss_dur: 0.13142 (0.12051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.85406 (12.74148) + | > current_lr: 0.00004 + | > step_time: 6.30560 (4.07872) + | > loader_time: 0.00240 (0.04652) + + + --> STEP: 34/234 -- GLOBAL_STEP: 36070 + | > loss: -0.10786 (-0.13559) + | > log_mle: -0.24550 (-0.25618) + | > loss_dur: 0.13764 (0.12059) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.83037 (12.14992) + | > current_lr: 0.00004 + | > step_time: 3.49650 (4.01333) + | > loader_time: 0.00770 (0.04877) + + + --> STEP: 39/234 -- GLOBAL_STEP: 36075 + | > loss: -0.13320 (-0.13450) + | > log_mle: -0.25587 (-0.25615) + | > loss_dur: 0.12267 (0.12165) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.38432 (11.91481) + | > current_lr: 0.00004 + | > step_time: 1.50750 (4.19974) + | > loader_time: 0.08300 (0.05193) + + + --> STEP: 44/234 -- GLOBAL_STEP: 36080 + | > loss: -0.14924 (-0.13303) + | > log_mle: -0.24954 (-0.25535) + | > loss_dur: 0.10030 (0.12231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.78599 (11.45228) + | > current_lr: 0.00004 + | > step_time: 1.11510 (3.92878) + | > loader_time: 0.00330 (0.04976) + + + --> STEP: 49/234 -- GLOBAL_STEP: 36085 + | > loss: -0.15017 (-0.13247) + | > log_mle: -0.25983 (-0.25547) + | > loss_dur: 0.10966 (0.12300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.97512 (11.23854) + | > current_lr: 0.00004 + | > step_time: 1.90860 (3.64606) + | > loader_time: 0.07820 (0.04643) + + + --> STEP: 54/234 -- GLOBAL_STEP: 36090 + | > loss: -0.14505 (-0.13157) + | > log_mle: -0.26591 (-0.25538) + | > loss_dur: 0.12086 (0.12381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.86944 (10.95387) + | > current_lr: 0.00004 + | > step_time: 1.57210 (3.47862) + | > loader_time: 0.00260 (0.04555) + + + --> STEP: 59/234 -- GLOBAL_STEP: 36095 + | > loss: -0.15998 (-0.13146) + | > log_mle: -0.27251 (-0.25558) + | > loss_dur: 0.11253 (0.12412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.94380 (10.78540) + | > current_lr: 0.00004 + | > step_time: 1.60490 (3.32123) + | > loader_time: 0.00270 (0.04471) + + + --> STEP: 64/234 -- GLOBAL_STEP: 36100 + | > loss: -0.13695 (-0.13033) + | > log_mle: -0.24946 (-0.25630) + | > loss_dur: 0.11250 (0.12597) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.11305 (11.39348) + | > current_lr: 0.00004 + | > step_time: 1.18300 (3.18184) + | > loader_time: 0.00210 (0.04140) + + + --> STEP: 69/234 -- GLOBAL_STEP: 36105 + | > loss: -0.09723 (-0.12906) + | > log_mle: -0.23837 (-0.25593) + | > loss_dur: 0.14114 (0.12687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.73023 (11.35110) + | > current_lr: 0.00004 + | > step_time: 1.21790 (3.04723) + | > loader_time: 0.00210 (0.03857) + + + --> STEP: 74/234 -- GLOBAL_STEP: 36110 + | > loss: -0.11199 (-0.12758) + | > log_mle: -0.24440 (-0.25620) + | > loss_dur: 0.13241 (0.12862) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.86697 (11.56991) + | > current_lr: 0.00004 + | > step_time: 1.31510 (2.93692) + | > loader_time: 0.08570 (0.03841) + + + --> STEP: 79/234 -- GLOBAL_STEP: 36115 + | > loss: -0.11334 (-0.12694) + | > log_mle: -0.26282 (-0.25652) + | > loss_dur: 0.14948 (0.12958) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.29462 (11.59441) + | > current_lr: 0.00004 + | > step_time: 1.79800 (2.89177) + | > loader_time: 0.00230 (0.03852) + + + --> STEP: 84/234 -- GLOBAL_STEP: 36120 + | > loss: -0.11965 (-0.12669) + | > log_mle: -0.25941 (-0.25686) + | > loss_dur: 0.13976 (0.13017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.47145 (11.61804) + | > current_lr: 0.00004 + | > step_time: 1.40540 (2.81669) + | > loader_time: 0.08690 (0.04071) + + + --> STEP: 89/234 -- GLOBAL_STEP: 36125 + | > loss: -0.14024 (-0.12688) + | > log_mle: -0.29130 (-0.25815) + | > loss_dur: 0.15105 (0.13127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.09893 (11.71209) + | > current_lr: 0.00004 + | > step_time: 1.10030 (2.73948) + | > loader_time: 0.09400 (0.03963) + + + --> STEP: 94/234 -- GLOBAL_STEP: 36130 + | > loss: -0.17008 (-0.12785) + | > log_mle: -0.32543 (-0.26066) + | > loss_dur: 0.15535 (0.13281) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.29340 (12.18382) + | > current_lr: 0.00004 + | > step_time: 1.39620 (2.70661) + | > loader_time: 0.00280 (0.04063) + + + --> STEP: 99/234 -- GLOBAL_STEP: 36135 + | > loss: -0.17256 (-0.12888) + | > log_mle: -0.35711 (-0.26317) + | > loss_dur: 0.18455 (0.13429) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.05714 (12.57898) + | > current_lr: 0.00004 + | > step_time: 1.10870 (2.67530) + | > loader_time: 0.07680 (0.04123) + + + --> STEP: 104/234 -- GLOBAL_STEP: 36140 + | > loss: -0.19676 (-0.13021) + | > log_mle: -0.37084 (-0.26619) + | > loss_dur: 0.17408 (0.13598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.21933 (13.24409) + | > current_lr: 0.00004 + | > step_time: 1.39580 (2.63137) + | > loader_time: 0.00300 (0.03939) + + + --> STEP: 109/234 -- GLOBAL_STEP: 36145 + | > loss: -0.12590 (-0.13050) + | > log_mle: -0.33877 (-0.26833) + | > loss_dur: 0.21287 (0.13783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.12353 (13.68559) + | > current_lr: 0.00004 + | > step_time: 1.28900 (2.60055) + | > loader_time: 0.00230 (0.03846) + + + --> STEP: 114/234 -- GLOBAL_STEP: 36150 + | > loss: -0.15746 (-0.13139) + | > log_mle: -0.32139 (-0.27115) + | > loss_dur: 0.16394 (0.13976) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.23319 (14.31664) + | > current_lr: 0.00004 + | > step_time: 1.49610 (2.60073) + | > loader_time: 0.00360 (0.03762) + + + --> STEP: 119/234 -- GLOBAL_STEP: 36155 + | > loss: -0.14856 (-0.13196) + | > log_mle: -0.32070 (-0.27344) + | > loss_dur: 0.17215 (0.14148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.72263 (14.63180) + | > current_lr: 0.00004 + | > step_time: 2.10390 (2.57710) + | > loader_time: 0.00330 (0.03780) + + + --> STEP: 124/234 -- GLOBAL_STEP: 36160 + | > loss: -0.18446 (-0.13238) + | > log_mle: -0.35010 (-0.27503) + | > loss_dur: 0.16564 (0.14265) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.62745 (14.90036) + | > current_lr: 0.00004 + | > step_time: 3.79720 (2.58045) + | > loader_time: 0.00670 (0.03708) + + + --> STEP: 129/234 -- GLOBAL_STEP: 36165 + | > loss: -0.14798 (-0.13409) + | > log_mle: -0.33869 (-0.27836) + | > loss_dur: 0.19071 (0.14427) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.80980 (15.66446) + | > current_lr: 0.00004 + | > step_time: 1.39180 (2.53846) + | > loader_time: 0.00350 (0.03715) + + + --> STEP: 134/234 -- GLOBAL_STEP: 36170 + | > loss: -0.18209 (-0.13626) + | > log_mle: -0.39480 (-0.28231) + | > loss_dur: 0.21271 (0.14605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.65089 (16.57268) + | > current_lr: 0.00004 + | > step_time: 2.80490 (2.53493) + | > loader_time: 0.09590 (0.03798) + + + --> STEP: 139/234 -- GLOBAL_STEP: 36175 + | > loss: -0.25286 (-0.13820) + | > log_mle: -0.45839 (-0.28601) + | > loss_dur: 0.20553 (0.14781) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.85142 (17.32165) + | > current_lr: 0.00004 + | > step_time: 1.30530 (2.49636) + | > loader_time: 0.00280 (0.03729) + + + --> STEP: 144/234 -- GLOBAL_STEP: 36180 + | > loss: -0.21693 (-0.14003) + | > log_mle: -0.42552 (-0.28986) + | > loss_dur: 0.20858 (0.14983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.15027 (18.47618) + | > current_lr: 0.00004 + | > step_time: 6.00970 (2.55690) + | > loader_time: 0.19150 (0.04144) + + + --> STEP: 149/234 -- GLOBAL_STEP: 36185 + | > loss: -0.26215 (-0.14286) + | > log_mle: -0.47368 (-0.29430) + | > loss_dur: 0.21154 (0.15144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.08558 (19.65047) + | > current_lr: 0.00004 + | > step_time: 5.00190 (2.63758) + | > loader_time: 0.08770 (0.04274) + + + --> STEP: 154/234 -- GLOBAL_STEP: 36190 + | > loss: -0.24473 (-0.14612) + | > log_mle: -0.43312 (-0.29916) + | > loss_dur: 0.18839 (0.15303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.49729 (21.06811) + | > current_lr: 0.00004 + | > step_time: 3.71350 (2.70849) + | > loader_time: 0.08650 (0.04338) + + + --> STEP: 159/234 -- GLOBAL_STEP: 36195 + | > loss: -0.23532 (-0.14875) + | > log_mle: -0.44816 (-0.30366) + | > loss_dur: 0.21284 (0.15491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.30072 (22.40827) + | > current_lr: 0.00004 + | > step_time: 0.88830 (2.69620) + | > loader_time: 0.00360 (0.04213) + + + --> STEP: 164/234 -- GLOBAL_STEP: 36200 + | > loss: -0.23118 (-0.15152) + | > log_mle: -0.44103 (-0.30798) + | > loss_dur: 0.20985 (0.15646) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.72864 (23.28796) + | > current_lr: 0.00004 + | > step_time: 4.40780 (2.72498) + | > loader_time: 0.00530 (0.04199) + + + --> STEP: 169/234 -- GLOBAL_STEP: 36205 + | > loss: -0.21894 (-0.15430) + | > log_mle: -0.43834 (-0.31251) + | > loss_dur: 0.21940 (0.15821) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.51760 (24.45802) + | > current_lr: 0.00004 + | > step_time: 5.01170 (2.75570) + | > loader_time: 0.09880 (0.04246) + + + --> STEP: 174/234 -- GLOBAL_STEP: 36210 + | > loss: -0.30592 (-0.15805) + | > log_mle: -0.52702 (-0.31817) + | > loss_dur: 0.22110 (0.16012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.01258 (25.78194) + | > current_lr: 0.00004 + | > step_time: 6.11060 (2.88688) + | > loader_time: 0.19840 (0.04297) + + + --> STEP: 179/234 -- GLOBAL_STEP: 36215 + | > loss: -0.27429 (-0.16106) + | > log_mle: -0.52501 (-0.32329) + | > loss_dur: 0.25072 (0.16222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.62194 (26.83293) + | > current_lr: 0.00004 + | > step_time: 9.50380 (2.95427) + | > loader_time: 0.28420 (0.04504) + + + --> STEP: 184/234 -- GLOBAL_STEP: 36220 + | > loss: -0.26532 (-0.16380) + | > log_mle: -0.49522 (-0.32791) + | > loss_dur: 0.22990 (0.16411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.08935 (27.99060) + | > current_lr: 0.00004 + | > step_time: 1.71710 (2.96221) + | > loader_time: 0.08860 (0.04535) + + + --> STEP: 189/234 -- GLOBAL_STEP: 36225 + | > loss: -0.25923 (-0.16664) + | > log_mle: -0.49405 (-0.33268) + | > loss_dur: 0.23482 (0.16604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.39727 (29.10557) + | > current_lr: 0.00004 + | > step_time: 4.00190 (2.95677) + | > loader_time: 0.00370 (0.04424) + + + --> STEP: 194/234 -- GLOBAL_STEP: 36230 + | > loss: -0.29661 (-0.17011) + | > log_mle: -0.52039 (-0.33748) + | > loss_dur: 0.22378 (0.16738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.12151 (30.42312) + | > current_lr: 0.00004 + | > step_time: 6.40870 (2.98319) + | > loader_time: 0.19650 (0.04560) + + + --> STEP: 199/234 -- GLOBAL_STEP: 36235 + | > loss: -0.30969 (-0.17297) + | > log_mle: -0.54072 (-0.34200) + | > loss_dur: 0.23103 (0.16903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.86239 (31.65233) + | > current_lr: 0.00004 + | > step_time: 3.59730 (2.97551) + | > loader_time: 0.21050 (0.04657) + + + --> STEP: 204/234 -- GLOBAL_STEP: 36240 + | > loss: -0.31833 (-0.17572) + | > log_mle: -0.56594 (-0.34643) + | > loss_dur: 0.24760 (0.17071) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.51701 (32.82631) + | > current_lr: 0.00004 + | > step_time: 7.59630 (3.03835) + | > loader_time: 0.20660 (0.04987) + + + --> STEP: 209/234 -- GLOBAL_STEP: 36245 + | > loss: -0.29192 (-0.17884) + | > log_mle: -0.52833 (-0.35122) + | > loss_dur: 0.23641 (0.17238) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.20395 (33.99476) + | > current_lr: 0.00004 + | > step_time: 5.70790 (3.04317) + | > loader_time: 0.09500 (0.05018) + + + --> STEP: 214/234 -- GLOBAL_STEP: 36250 + | > loss: -0.33849 (-0.18272) + | > log_mle: -0.56395 (-0.35682) + | > loss_dur: 0.22546 (0.17410) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.57278 (35.43298) + | > current_lr: 0.00004 + | > step_time: 8.29290 (3.14504) + | > loader_time: 0.01100 (0.05138) + + + --> STEP: 219/234 -- GLOBAL_STEP: 36255 + | > loss: -0.38931 (-0.18623) + | > log_mle: -0.65541 (-0.36211) + | > loss_dur: 0.26609 (0.17589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 110.19245 (36.90557) + | > current_lr: 0.00004 + | > step_time: 2.99280 (3.16637) + | > loader_time: 0.00610 (0.05239) + + + --> STEP: 224/234 -- GLOBAL_STEP: 36260 + | > loss: -0.31791 (-0.18871) + | > log_mle: -0.57423 (-0.36633) + | > loss_dur: 0.25632 (0.17762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.20183 (38.83279) + | > current_lr: 0.00004 + | > step_time: 2.39370 (3.15778) + | > loader_time: 0.00350 (0.05169) + + + --> STEP: 229/234 -- GLOBAL_STEP: 36265 + | > loss: -0.30155 (-0.19145) + | > log_mle: -0.60525 (-0.37117) + | > loss_dur: 0.30371 (0.17973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.26877 (40.13507) + | > current_lr: 0.00004 + | > step_time: 0.25770 (3.10559) + | > loader_time: 0.00340 (0.05103) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.19028 (-0.91163) + | > avg_loss: -0.20307 (-0.02116) + | > avg_log_mle: -0.42614 (-0.01936) + | > avg_loss_dur: 0.22307 (-0.00180) + + + > EPOCH: 155/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 17:40:33)  + + --> STEP: 0/234 -- GLOBAL_STEP: 36270 + | > loss: -0.16124 (-0.16124) + | > log_mle: -0.33136 (-0.33136) + | > loss_dur: 0.17012 (0.17012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.91970 (22.91970) + | > current_lr: 0.00004 + | > step_time: 9.69340 (9.69338) + | > loader_time: 2.20910 (2.20915) + + + --> STEP: 5/234 -- GLOBAL_STEP: 36275 + | > loss: -0.12848 (-0.12231) + | > log_mle: -0.26105 (-0.26020) + | > loss_dur: 0.13257 (0.13789) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.34280 (15.70092) + | > current_lr: 0.00004 + | > step_time: 5.50100 (5.50246) + | > loader_time: 0.09370 (3.17797) + + + --> STEP: 10/234 -- GLOBAL_STEP: 36280 + | > loss: -0.10579 (-0.13021) + | > log_mle: -0.25851 (-0.26328) + | > loss_dur: 0.15272 (0.13307) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.20683 (13.43597) + | > current_lr: 0.00004 + | > step_time: 4.30720 (4.39198) + | > loader_time: 0.08630 (1.60789) + + + --> STEP: 15/234 -- GLOBAL_STEP: 36285 + | > loss: -0.15228 (-0.13762) + | > log_mle: -0.26385 (-0.26432) + | > loss_dur: 0.11157 (0.12670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.59960 (12.42398) + | > current_lr: 0.00004 + | > step_time: 0.89170 (4.36100) + | > loader_time: 0.00160 (1.08533) + + + --> STEP: 20/234 -- GLOBAL_STEP: 36290 + | > loss: -0.14072 (-0.13861) + | > log_mle: -0.25414 (-0.26165) + | > loss_dur: 0.11342 (0.12304) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.51816 (11.76274) + | > current_lr: 0.00004 + | > step_time: 9.19210 (4.12173) + | > loader_time: 0.09880 (0.82307) + + + --> STEP: 25/234 -- GLOBAL_STEP: 36295 + | > loss: -0.13920 (-0.13971) + | > log_mle: -0.24504 (-0.25996) + | > loss_dur: 0.10584 (0.12025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.82173 (11.62401) + | > current_lr: 0.00004 + | > step_time: 1.70880 (3.70837) + | > loader_time: 0.30240 (0.67802) + + + --> STEP: 30/234 -- GLOBAL_STEP: 36300 + | > loss: -0.16501 (-0.14109) + | > log_mle: -0.26518 (-0.25975) + | > loss_dur: 0.10017 (0.11866) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.08845 (11.10142) + | > current_lr: 0.00004 + | > step_time: 1.07630 (3.35634) + | > loader_time: 0.00150 (0.56538) + + + --> STEP: 35/234 -- GLOBAL_STEP: 36305 + | > loss: -0.12153 (-0.13947) + | > log_mle: -0.25801 (-0.25945) + | > loss_dur: 0.13647 (0.11998) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.33953 (10.92315) + | > current_lr: 0.00004 + | > step_time: 1.11460 (3.04033) + | > loader_time: 0.08720 (0.49442) + + + --> STEP: 40/234 -- GLOBAL_STEP: 36310 + | > loss: -0.10299 (-0.13819) + | > log_mle: -0.24335 (-0.25909) + | > loss_dur: 0.14036 (0.12090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.05418 (10.84381) + | > current_lr: 0.00004 + | > step_time: 2.61170 (2.96849) + | > loader_time: 0.00290 (0.44000) + + + --> STEP: 45/234 -- GLOBAL_STEP: 36315 + | > loss: -0.13549 (-0.13801) + | > log_mle: -0.27821 (-0.25925) + | > loss_dur: 0.14272 (0.12124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.78276 (10.59840) + | > current_lr: 0.00004 + | > step_time: 1.39880 (2.77998) + | > loader_time: 0.08440 (0.39319) + + + --> STEP: 50/234 -- GLOBAL_STEP: 36320 + | > loss: -0.12803 (-0.13725) + | > log_mle: -0.24810 (-0.25870) + | > loss_dur: 0.12007 (0.12145) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.89712 (10.35520) + | > current_lr: 0.00004 + | > step_time: 1.90800 (2.63933) + | > loader_time: 0.08380 (0.35944) + + + --> STEP: 55/234 -- GLOBAL_STEP: 36325 + | > loss: -0.15788 (-0.13686) + | > log_mle: -0.26760 (-0.25873) + | > loss_dur: 0.10972 (0.12188) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06699 (10.25379) + | > current_lr: 0.00004 + | > step_time: 2.36290 (2.54343) + | > loader_time: 0.00280 (0.32696) + + + --> STEP: 60/234 -- GLOBAL_STEP: 36330 + | > loss: -0.12474 (-0.13604) + | > log_mle: -0.28121 (-0.25901) + | > loss_dur: 0.15648 (0.12296) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.41491 (10.23286) + | > current_lr: 0.00004 + | > step_time: 1.40390 (2.46299) + | > loader_time: 0.00200 (0.29994) + + + --> STEP: 65/234 -- GLOBAL_STEP: 36335 + | > loss: -0.13139 (-0.13473) + | > log_mle: -0.25565 (-0.25938) + | > loss_dur: 0.12427 (0.12465) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.72596 (10.33184) + | > current_lr: 0.00004 + | > step_time: 1.97940 (2.40863) + | > loader_time: 0.00210 (0.27832) + + + --> STEP: 70/234 -- GLOBAL_STEP: 36340 + | > loss: -0.10093 (-0.13310) + | > log_mle: -0.25000 (-0.25890) + | > loss_dur: 0.14907 (0.12580) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.06813 (10.32170) + | > current_lr: 0.00004 + | > step_time: 1.48490 (2.40770) + | > loader_time: 0.00180 (0.25866) + + + --> STEP: 75/234 -- GLOBAL_STEP: 36345 + | > loss: -0.10154 (-0.13166) + | > log_mle: -0.26805 (-0.25939) + | > loss_dur: 0.16651 (0.12774) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.66625 (10.36304) + | > current_lr: 0.00004 + | > step_time: 1.79930 (2.36317) + | > loader_time: 0.00280 (0.24390) + + + --> STEP: 80/234 -- GLOBAL_STEP: 36350 + | > loss: -0.13112 (-0.13125) + | > log_mle: -0.24804 (-0.25944) + | > loss_dur: 0.11692 (0.12819) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.09359 (10.30836) + | > current_lr: 0.00004 + | > step_time: 2.10980 (2.31685) + | > loader_time: 0.00330 (0.22884) + + + --> STEP: 85/234 -- GLOBAL_STEP: 36355 + | > loss: -0.13117 (-0.13084) + | > log_mle: -0.26382 (-0.25985) + | > loss_dur: 0.13265 (0.12901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.70606 (10.39330) + | > current_lr: 0.00004 + | > step_time: 2.48920 (2.29993) + | > loader_time: 0.10510 (0.21675) + + + --> STEP: 90/234 -- GLOBAL_STEP: 36360 + | > loss: -0.13011 (-0.13102) + | > log_mle: -0.29241 (-0.26138) + | > loss_dur: 0.16230 (0.13037) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.95799 (10.62830) + | > current_lr: 0.00004 + | > step_time: 1.30630 (2.26206) + | > loader_time: 0.00300 (0.20668) + + + --> STEP: 95/234 -- GLOBAL_STEP: 36365 + | > loss: -0.19982 (-0.13263) + | > log_mle: -0.37880 (-0.26478) + | > loss_dur: 0.17898 (0.13215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.82592 (11.24780) + | > current_lr: 0.00004 + | > step_time: 1.98780 (2.26723) + | > loader_time: 0.00260 (0.19800) + + + --> STEP: 100/234 -- GLOBAL_STEP: 36370 + | > loss: -0.15299 (-0.13316) + | > log_mle: -0.30291 (-0.26646) + | > loss_dur: 0.14992 (0.13330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.42642 (11.61999) + | > current_lr: 0.00004 + | > step_time: 1.80490 (2.28204) + | > loader_time: 0.08640 (0.19077) + + + --> STEP: 105/234 -- GLOBAL_STEP: 36375 + | > loss: -0.13207 (-0.13404) + | > log_mle: -0.27995 (-0.26927) + | > loss_dur: 0.14789 (0.13522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58538 (12.17597) + | > current_lr: 0.00004 + | > step_time: 2.39300 (2.29811) + | > loader_time: 0.09420 (0.18357) + + + --> STEP: 110/234 -- GLOBAL_STEP: 36380 + | > loss: -0.14056 (-0.13445) + | > log_mle: -0.30634 (-0.27162) + | > loss_dur: 0.16577 (0.13717) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.17744 (12.82459) + | > current_lr: 0.00004 + | > step_time: 2.30460 (2.27921) + | > loader_time: 0.00310 (0.17685) + + + --> STEP: 115/234 -- GLOBAL_STEP: 36385 + | > loss: -0.14567 (-0.13542) + | > log_mle: -0.32599 (-0.27476) + | > loss_dur: 0.18032 (0.13935) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.95811 (13.53951) + | > current_lr: 0.00004 + | > step_time: 1.20720 (2.24092) + | > loader_time: 0.00320 (0.16930) + + + --> STEP: 120/234 -- GLOBAL_STEP: 36390 + | > loss: -0.18836 (-0.13625) + | > log_mle: -0.37413 (-0.27742) + | > loss_dur: 0.18577 (0.14117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.99909 (14.17984) + | > current_lr: 0.00004 + | > step_time: 1.60390 (2.25852) + | > loader_time: 0.08830 (0.16451) + + + --> STEP: 125/234 -- GLOBAL_STEP: 36395 + | > loss: -0.18225 (-0.13664) + | > log_mle: -0.36335 (-0.27886) + | > loss_dur: 0.18110 (0.14222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.17047 (14.54697) + | > current_lr: 0.00004 + | > step_time: 3.28890 (2.28195) + | > loader_time: 0.00340 (0.15952) + + + --> STEP: 130/234 -- GLOBAL_STEP: 36400 + | > loss: -0.18668 (-0.13820) + | > log_mle: -0.37854 (-0.28219) + | > loss_dur: 0.19187 (0.14399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.51745 (15.40782) + | > current_lr: 0.00004 + | > step_time: 2.09450 (2.30816) + | > loader_time: 0.00340 (0.15549) + + + --> STEP: 135/234 -- GLOBAL_STEP: 36405 + | > loss: -0.14514 (-0.13994) + | > log_mle: -0.30330 (-0.28554) + | > loss_dur: 0.15816 (0.14561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.71384 (16.10534) + | > current_lr: 0.00004 + | > step_time: 4.41020 (2.30521) + | > loader_time: 0.09120 (0.15225) + + + --> STEP: 140/234 -- GLOBAL_STEP: 36410 + | > loss: -0.13907 (-0.14171) + | > log_mle: -0.33876 (-0.28936) + | > loss_dur: 0.19969 (0.14765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.64973 (17.21646) + | > current_lr: 0.00004 + | > step_time: 4.61730 (2.36153) + | > loader_time: 0.00360 (0.14882) + + + --> STEP: 145/234 -- GLOBAL_STEP: 36415 + | > loss: -0.22191 (-0.14395) + | > log_mle: -0.43080 (-0.29381) + | > loss_dur: 0.20889 (0.14985) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.25890 (18.64808) + | > current_lr: 0.00004 + | > step_time: 1.30700 (2.37044) + | > loader_time: 0.07910 (0.14489) + + + --> STEP: 150/234 -- GLOBAL_STEP: 36420 + | > loss: -0.21130 (-0.14639) + | > log_mle: -0.41805 (-0.29782) + | > loss_dur: 0.20675 (0.15144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.09916 (19.53126) + | > current_lr: 0.00004 + | > step_time: 2.20020 (2.35608) + | > loader_time: 0.10430 (0.14197) + + + --> STEP: 155/234 -- GLOBAL_STEP: 36425 + | > loss: -0.26665 (-0.14980) + | > log_mle: -0.48554 (-0.30296) + | > loss_dur: 0.21889 (0.15316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.32266 (20.91617) + | > current_lr: 0.00004 + | > step_time: 1.79510 (2.40457) + | > loader_time: 0.00410 (0.14043) + + + --> STEP: 160/234 -- GLOBAL_STEP: 36430 + | > loss: -0.25949 (-0.15244) + | > log_mle: -0.47743 (-0.30745) + | > loss_dur: 0.21794 (0.15501) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.67471 (22.16376) + | > current_lr: 0.00004 + | > step_time: 2.80230 (2.40503) + | > loader_time: 0.00430 (0.13617) + + + --> STEP: 165/234 -- GLOBAL_STEP: 36435 + | > loss: -0.26355 (-0.15511) + | > log_mle: -0.48010 (-0.31167) + | > loss_dur: 0.21655 (0.15655) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.99884 (23.28641) + | > current_lr: 0.00004 + | > step_time: 2.59690 (2.45211) + | > loader_time: 0.09360 (0.13433) + + + --> STEP: 170/234 -- GLOBAL_STEP: 36440 + | > loss: -0.27775 (-0.15794) + | > log_mle: -0.52533 (-0.31644) + | > loss_dur: 0.24758 (0.15850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.25462 (24.45491) + | > current_lr: 0.00004 + | > step_time: 3.40580 (2.52234) + | > loader_time: 0.09080 (0.13216) + + + --> STEP: 175/234 -- GLOBAL_STEP: 36445 + | > loss: -0.24982 (-0.16132) + | > log_mle: -0.48688 (-0.32164) + | > loss_dur: 0.23706 (0.16032) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.54482 (26.27362) + | > current_lr: 0.00004 + | > step_time: 2.49750 (2.54530) + | > loader_time: 0.00480 (0.12939) + + + --> STEP: 180/234 -- GLOBAL_STEP: 36450 + | > loss: -0.27039 (-0.16433) + | > log_mle: -0.49452 (-0.32665) + | > loss_dur: 0.22414 (0.16232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.38352 (27.34550) + | > current_lr: 0.00004 + | > step_time: 2.71080 (2.57680) + | > loader_time: 0.08820 (0.12740) + + + --> STEP: 185/234 -- GLOBAL_STEP: 36455 + | > loss: -0.27879 (-0.16703) + | > log_mle: -0.52089 (-0.33126) + | > loss_dur: 0.24210 (0.16423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.30190 (28.37300) + | > current_lr: 0.00004 + | > step_time: 5.40330 (2.66446) + | > loader_time: 0.00250 (0.12466) + + + --> STEP: 190/234 -- GLOBAL_STEP: 36460 + | > loss: -0.29225 (-0.16994) + | > log_mle: -0.50642 (-0.33587) + | > loss_dur: 0.21416 (0.16594) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.24038 (29.27299) + | > current_lr: 0.00004 + | > step_time: 1.99930 (2.70801) + | > loader_time: 0.08560 (0.12395) + + + --> STEP: 195/234 -- GLOBAL_STEP: 36465 + | > loss: -0.29455 (-0.17347) + | > log_mle: -0.52905 (-0.34087) + | > loss_dur: 0.23450 (0.16740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.67073 (30.19669) + | > current_lr: 0.00004 + | > step_time: 2.78750 (2.72103) + | > loader_time: 0.00440 (0.12130) + + + --> STEP: 200/234 -- GLOBAL_STEP: 36470 + | > loss: -0.28279 (-0.17639) + | > log_mle: -0.53659 (-0.34547) + | > loss_dur: 0.25380 (0.16907) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.86840 (31.52736) + | > current_lr: 0.00004 + | > step_time: 2.49660 (2.79860) + | > loader_time: 0.09980 (0.12073) + + + --> STEP: 205/234 -- GLOBAL_STEP: 36475 + | > loss: -0.28561 (-0.17914) + | > log_mle: -0.52187 (-0.34988) + | > loss_dur: 0.23626 (0.17073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.57595 (32.63525) + | > current_lr: 0.00004 + | > step_time: 2.20620 (2.83625) + | > loader_time: 0.10010 (0.12072) + + + --> STEP: 210/234 -- GLOBAL_STEP: 36480 + | > loss: -0.34294 (-0.18243) + | > log_mle: -0.59171 (-0.35493) + | > loss_dur: 0.24877 (0.17251) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.81904 (34.07557) + | > current_lr: 0.00004 + | > step_time: 4.68480 (2.88774) + | > loader_time: 0.10740 (0.11979) + + + --> STEP: 215/234 -- GLOBAL_STEP: 36485 + | > loss: -0.30522 (-0.18593) + | > log_mle: -0.54984 (-0.36018) + | > loss_dur: 0.24462 (0.17425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.91499 (35.60360) + | > current_lr: 0.00004 + | > step_time: 5.48850 (2.95126) + | > loader_time: 0.00670 (0.11837) + + + --> STEP: 220/234 -- GLOBAL_STEP: 36490 + | > loss: -0.34146 (-0.18974) + | > log_mle: -0.59289 (-0.36578) + | > loss_dur: 0.25142 (0.17604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 110.98410 (37.04966) + | > current_lr: 0.00004 + | > step_time: 2.11160 (3.00290) + | > loader_time: 0.09200 (0.12697) + + + --> STEP: 225/234 -- GLOBAL_STEP: 36495 + | > loss: -0.40154 (-0.19336) + | > log_mle: -0.66268 (-0.37113) + | > loss_dur: 0.26113 (0.17777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 130.10475 (38.33649) + | > current_lr: 0.00004 + | > step_time: 0.24200 (2.95020) + | > loader_time: 0.00330 (0.12463) + + + --> STEP: 230/234 -- GLOBAL_STEP: 36500 + | > loss: -0.35730 (-0.19653) + | > log_mle: -0.69593 (-0.37671) + | > loss_dur: 0.33863 (0.18018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 126.22264 (39.86595) + | > current_lr: 0.00004 + | > step_time: 0.28530 (2.89187) + | > loader_time: 0.00400 (0.12201) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00270 (-0.18758) + | > avg_loss: -0.22915 (-0.02608) + | > avg_log_mle: -0.45279 (-0.02664) + | > avg_loss_dur: 0.22364 (+0.00057) + + + > EPOCH: 156/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 17:53:06)  + + --> STEP: 1/234 -- GLOBAL_STEP: 36505 + | > loss: -0.14530 (-0.14530) + | > log_mle: -0.26645 (-0.26645) + | > loss_dur: 0.12114 (0.12114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.71490 (18.71490) + | > current_lr: 0.00004 + | > step_time: 5.20510 (5.20512) + | > loader_time: 0.09290 (0.09290) + + + --> STEP: 6/234 -- GLOBAL_STEP: 36510 + | > loss: -0.14749 (-0.12709) + | > log_mle: -0.25587 (-0.26196) + | > loss_dur: 0.10838 (0.13487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.62458 (15.89703) + | > current_lr: 0.00004 + | > step_time: 2.09280 (3.56759) + | > loader_time: 0.00150 (0.16510) + + + --> STEP: 11/234 -- GLOBAL_STEP: 36515 + | > loss: -0.16727 (-0.13468) + | > log_mle: -0.26557 (-0.26619) + | > loss_dur: 0.09831 (0.13151) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.44212 (16.06866) + | > current_lr: 0.00004 + | > step_time: 3.01430 (4.87474) + | > loader_time: 0.09290 (0.10767) + + + --> STEP: 16/234 -- GLOBAL_STEP: 36520 + | > loss: -0.15809 (-0.13900) + | > log_mle: -0.26345 (-0.26617) + | > loss_dur: 0.10535 (0.12717) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.25635 (15.18956) + | > current_lr: 0.00004 + | > step_time: 3.83160 (4.27861) + | > loader_time: 0.00170 (0.07475) + + + --> STEP: 21/234 -- GLOBAL_STEP: 36525 + | > loss: -0.13022 (-0.13786) + | > log_mle: -0.23871 (-0.26227) + | > loss_dur: 0.10849 (0.12441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.81587 (14.29624) + | > current_lr: 0.00004 + | > step_time: 3.61160 (3.70462) + | > loader_time: 0.00570 (0.05762) + + + --> STEP: 26/234 -- GLOBAL_STEP: 36530 + | > loss: -0.12275 (-0.13973) + | > log_mle: -0.25892 (-0.26148) + | > loss_dur: 0.13618 (0.12175) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.66442 (13.78872) + | > current_lr: 0.00004 + | > step_time: 2.79740 (3.33496) + | > loader_time: 0.00280 (0.05019) + + + --> STEP: 31/234 -- GLOBAL_STEP: 36535 + | > loss: -0.10616 (-0.14038) + | > log_mle: -0.25616 (-0.26107) + | > loss_dur: 0.15000 (0.12069) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.32693 (13.12199) + | > current_lr: 0.00004 + | > step_time: 0.69050 (3.47224) + | > loader_time: 0.00150 (0.05130) + + + --> STEP: 36/234 -- GLOBAL_STEP: 36540 + | > loss: -0.12477 (-0.13941) + | > log_mle: -0.25514 (-0.26055) + | > loss_dur: 0.13037 (0.12114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.02818 (12.79062) + | > current_lr: 0.00004 + | > step_time: 1.69330 (3.21639) + | > loader_time: 0.09510 (0.04920) + + + --> STEP: 41/234 -- GLOBAL_STEP: 36545 + | > loss: -0.14572 (-0.13833) + | > log_mle: -0.25977 (-0.26004) + | > loss_dur: 0.11406 (0.12171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.34113 (12.39754) + | > current_lr: 0.00004 + | > step_time: 3.80000 (3.12252) + | > loader_time: 0.00560 (0.04953) + + + --> STEP: 46/234 -- GLOBAL_STEP: 36550 + | > loss: -0.12568 (-0.13764) + | > log_mle: -0.25388 (-0.25997) + | > loss_dur: 0.12820 (0.12233) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.47174 (12.21802) + | > current_lr: 0.00004 + | > step_time: 0.91950 (2.92467) + | > loader_time: 0.00220 (0.04435) + + + --> STEP: 51/234 -- GLOBAL_STEP: 36555 + | > loss: -0.12174 (-0.13683) + | > log_mle: -0.24824 (-0.25932) + | > loss_dur: 0.12650 (0.12249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.58512 (11.73605) + | > current_lr: 0.00004 + | > step_time: 1.60570 (2.78564) + | > loader_time: 0.00250 (0.04193) + + + --> STEP: 56/234 -- GLOBAL_STEP: 36560 + | > loss: -0.10589 (-0.13618) + | > log_mle: -0.26202 (-0.25962) + | > loss_dur: 0.15614 (0.12344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.29550 (11.55190) + | > current_lr: 0.00004 + | > step_time: 1.30100 (2.66145) + | > loader_time: 0.08690 (0.03990) + + + --> STEP: 61/234 -- GLOBAL_STEP: 36565 + | > loss: -0.13356 (-0.13623) + | > log_mle: -0.25965 (-0.25990) + | > loss_dur: 0.12609 (0.12366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.83242 (11.41199) + | > current_lr: 0.00004 + | > step_time: 1.57640 (2.59057) + | > loader_time: 0.00210 (0.03956) + + + --> STEP: 66/234 -- GLOBAL_STEP: 36570 + | > loss: -0.13645 (-0.13501) + | > log_mle: -0.25218 (-0.26009) + | > loss_dur: 0.11573 (0.12509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.78342 (11.39352) + | > current_lr: 0.00004 + | > step_time: 2.42120 (2.53837) + | > loader_time: 0.00220 (0.03800) + + + --> STEP: 71/234 -- GLOBAL_STEP: 36575 + | > loss: -0.10627 (-0.13315) + | > log_mle: -0.28762 (-0.26021) + | > loss_dur: 0.18135 (0.12705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.07145 (11.54259) + | > current_lr: 0.00004 + | > step_time: 1.20290 (2.45601) + | > loader_time: 0.00250 (0.03667) + + + --> STEP: 76/234 -- GLOBAL_STEP: 36580 + | > loss: -0.13492 (-0.13203) + | > log_mle: -0.27268 (-0.26035) + | > loss_dur: 0.13776 (0.12832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58102 (11.58266) + | > current_lr: 0.00004 + | > step_time: 1.40190 (2.38388) + | > loader_time: 0.08730 (0.03668) + + + --> STEP: 81/234 -- GLOBAL_STEP: 36585 + | > loss: -0.12751 (-0.13154) + | > log_mle: -0.27704 (-0.26038) + | > loss_dur: 0.14954 (0.12885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.35756 (11.60554) + | > current_lr: 0.00004 + | > step_time: 1.20640 (2.31312) + | > loader_time: 0.00310 (0.03662) + + + --> STEP: 86/234 -- GLOBAL_STEP: 36590 + | > loss: -0.12716 (-0.13105) + | > log_mle: -0.27384 (-0.26073) + | > loss_dur: 0.14668 (0.12968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.44750 (11.68007) + | > current_lr: 0.00004 + | > step_time: 1.17560 (2.27641) + | > loader_time: 0.00210 (0.03559) + + + --> STEP: 91/234 -- GLOBAL_STEP: 36595 + | > loss: -0.12497 (-0.13131) + | > log_mle: -0.28743 (-0.26228) + | > loss_dur: 0.16246 (0.13097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.76025 (11.94659) + | > current_lr: 0.00004 + | > step_time: 2.00370 (2.27233) + | > loader_time: 0.08500 (0.03470) + + + --> STEP: 96/234 -- GLOBAL_STEP: 36600 + | > loss: -0.12303 (-0.13262) + | > log_mle: -0.27304 (-0.26543) + | > loss_dur: 0.15001 (0.13281) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.08800 (12.53707) + | > current_lr: 0.00004 + | > step_time: 1.78980 (2.26215) + | > loader_time: 0.00540 (0.03481) + + + --> STEP: 101/234 -- GLOBAL_STEP: 36605 + | > loss: -0.14567 (-0.13316) + | > log_mle: -0.33068 (-0.26754) + | > loss_dur: 0.18502 (0.13438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.29456 (13.08378) + | > current_lr: 0.00004 + | > step_time: 1.29950 (2.24421) + | > loader_time: 0.08600 (0.03497) + + + --> STEP: 106/234 -- GLOBAL_STEP: 36610 + | > loss: -0.13721 (-0.13429) + | > log_mle: -0.32993 (-0.27023) + | > loss_dur: 0.19272 (0.13594) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.00771 (13.70786) + | > current_lr: 0.00004 + | > step_time: 1.11480 (2.20514) + | > loader_time: 0.08400 (0.03496) + + + --> STEP: 111/234 -- GLOBAL_STEP: 36615 + | > loss: -0.17386 (-0.13494) + | > log_mle: -0.37806 (-0.27290) + | > loss_dur: 0.20420 (0.13796) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.25269 (14.34897) + | > current_lr: 0.00004 + | > step_time: 2.71510 (2.20570) + | > loader_time: 0.08490 (0.03429) + + + --> STEP: 116/234 -- GLOBAL_STEP: 36620 + | > loss: -0.13554 (-0.13571) + | > log_mle: -0.34533 (-0.27566) + | > loss_dur: 0.20979 (0.13995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.24720 (14.89791) + | > current_lr: 0.00004 + | > step_time: 1.40810 (2.17619) + | > loader_time: 0.00250 (0.03438) + + + --> STEP: 121/234 -- GLOBAL_STEP: 36625 + | > loss: -0.10980 (-0.13627) + | > log_mle: -0.25773 (-0.27756) + | > loss_dur: 0.14793 (0.14130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.82367 (15.23614) + | > current_lr: 0.00004 + | > step_time: 1.70620 (2.15147) + | > loader_time: 0.00340 (0.03381) + + + --> STEP: 126/234 -- GLOBAL_STEP: 36630 + | > loss: -0.20246 (-0.13749) + | > log_mle: -0.39567 (-0.28010) + | > loss_dur: 0.19321 (0.14261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.97245 (15.67455) + | > current_lr: 0.00004 + | > step_time: 1.60120 (2.12967) + | > loader_time: 0.00410 (0.03328) + + + --> STEP: 131/234 -- GLOBAL_STEP: 36635 + | > loss: -0.22222 (-0.13900) + | > log_mle: -0.43193 (-0.28343) + | > loss_dur: 0.20972 (0.14443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.59191 (16.79306) + | > current_lr: 0.00004 + | > step_time: 4.31740 (2.13019) + | > loader_time: 0.08230 (0.03343) + + + --> STEP: 136/234 -- GLOBAL_STEP: 36640 + | > loss: -0.26225 (-0.14092) + | > log_mle: -0.48654 (-0.28702) + | > loss_dur: 0.22430 (0.14610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.57302 (17.44181) + | > current_lr: 0.00004 + | > step_time: 1.21260 (2.13285) + | > loader_time: 0.00260 (0.03425) + + + --> STEP: 141/234 -- GLOBAL_STEP: 36645 + | > loss: -0.18842 (-0.14221) + | > log_mle: -0.38773 (-0.29012) + | > loss_dur: 0.19931 (0.14791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.45242 (18.25549) + | > current_lr: 0.00004 + | > step_time: 2.51020 (2.12136) + | > loader_time: 0.00460 (0.03488) + + + --> STEP: 146/234 -- GLOBAL_STEP: 36650 + | > loss: -0.23150 (-0.14473) + | > log_mle: -0.44229 (-0.29494) + | > loss_dur: 0.21079 (0.15022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.71878 (19.16853) + | > current_lr: 0.00004 + | > step_time: 1.79100 (2.10362) + | > loader_time: 0.00660 (0.03510) + + + --> STEP: 151/234 -- GLOBAL_STEP: 36655 + | > loss: -0.21275 (-0.14713) + | > log_mle: -0.40342 (-0.29882) + | > loss_dur: 0.19066 (0.15168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.71624 (20.10626) + | > current_lr: 0.00004 + | > step_time: 1.79310 (2.09967) + | > loader_time: 0.00320 (0.03460) + + + --> STEP: 156/234 -- GLOBAL_STEP: 36660 + | > loss: -0.23727 (-0.15053) + | > log_mle: -0.44756 (-0.30420) + | > loss_dur: 0.21028 (0.15366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.50957 (21.39593) + | > current_lr: 0.00004 + | > step_time: 2.50560 (2.09838) + | > loader_time: 0.00330 (0.03469) + + + --> STEP: 161/234 -- GLOBAL_STEP: 36665 + | > loss: -0.27315 (-0.15329) + | > log_mle: -0.47596 (-0.30884) + | > loss_dur: 0.20280 (0.15555) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.08985 (22.45103) + | > current_lr: 0.00004 + | > step_time: 2.39920 (2.10716) + | > loader_time: 0.09240 (0.03490) + + + --> STEP: 166/234 -- GLOBAL_STEP: 36670 + | > loss: -0.22916 (-0.15585) + | > log_mle: -0.41649 (-0.31287) + | > loss_dur: 0.18733 (0.15701) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.51659 (23.29472) + | > current_lr: 0.00004 + | > step_time: 3.89110 (2.14733) + | > loader_time: 0.00780 (0.03627) + + + --> STEP: 171/234 -- GLOBAL_STEP: 36675 + | > loss: -0.31251 (-0.15944) + | > log_mle: -0.52270 (-0.31836) + | > loss_dur: 0.21019 (0.15892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.55244 (24.63840) + | > current_lr: 0.00004 + | > step_time: 1.52030 (2.15897) + | > loader_time: 0.07570 (0.03631) + + + --> STEP: 176/234 -- GLOBAL_STEP: 36680 + | > loss: -0.28527 (-0.16296) + | > log_mle: -0.49723 (-0.32372) + | > loss_dur: 0.21196 (0.16076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.28488 (25.91205) + | > current_lr: 0.00004 + | > step_time: 1.99690 (2.17208) + | > loader_time: 0.00720 (0.03647) + + + --> STEP: 181/234 -- GLOBAL_STEP: 36685 + | > loss: -0.21071 (-0.16567) + | > log_mle: -0.42706 (-0.32833) + | > loss_dur: 0.21634 (0.16266) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.19393 (27.18710) + | > current_lr: 0.00004 + | > step_time: 1.99730 (2.20220) + | > loader_time: 0.00380 (0.03767) + + + --> STEP: 186/234 -- GLOBAL_STEP: 36690 + | > loss: -0.22799 (-0.16846) + | > log_mle: -0.46596 (-0.33316) + | > loss_dur: 0.23797 (0.16471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.34712 (28.55505) + | > current_lr: 0.00004 + | > step_time: 2.31040 (2.20802) + | > loader_time: 0.00350 (0.03733) + + + --> STEP: 191/234 -- GLOBAL_STEP: 36695 + | > loss: -0.26759 (-0.17140) + | > log_mle: -0.48500 (-0.33777) + | > loss_dur: 0.21741 (0.16636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.60421 (29.83734) + | > current_lr: 0.00004 + | > step_time: 6.99810 (2.27062) + | > loader_time: 0.19290 (0.03841) + + + --> STEP: 196/234 -- GLOBAL_STEP: 36700 + | > loss: -0.26250 (-0.17468) + | > log_mle: -0.48769 (-0.34259) + | > loss_dur: 0.22520 (0.16791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.15970 (30.83010) + | > current_lr: 0.00004 + | > step_time: 5.69170 (2.35594) + | > loader_time: 0.00550 (0.03955) + + + --> STEP: 201/234 -- GLOBAL_STEP: 36705 + | > loss: -0.20843 (-0.17739) + | > log_mle: -0.44446 (-0.34689) + | > loss_dur: 0.23604 (0.16950) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.70344 (31.74057) + | > current_lr: 0.00004 + | > step_time: 2.70270 (2.38046) + | > loader_time: 0.18550 (0.04101) + + + --> STEP: 206/234 -- GLOBAL_STEP: 36710 + | > loss: -0.30698 (-0.18044) + | > log_mle: -0.54721 (-0.35162) + | > loss_dur: 0.24023 (0.17118) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.60088 (32.90689) + | > current_lr: 0.00004 + | > step_time: 1.70740 (2.37609) + | > loader_time: 0.09390 (0.04151) + + + --> STEP: 211/234 -- GLOBAL_STEP: 36715 + | > loss: -0.35924 (-0.18398) + | > log_mle: -0.61464 (-0.35692) + | > loss_dur: 0.25541 (0.17294) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.95107 (34.19081) + | > current_lr: 0.00004 + | > step_time: 7.40160 (2.42356) + | > loader_time: 0.19860 (0.04195) + + + --> STEP: 216/234 -- GLOBAL_STEP: 36720 + | > loss: -0.34546 (-0.18738) + | > log_mle: -0.61142 (-0.36201) + | > loss_dur: 0.26596 (0.17463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.99980 (35.51198) + | > current_lr: 0.00004 + | > step_time: 5.90150 (2.50871) + | > loader_time: 0.00700 (0.04240) + + + --> STEP: 221/234 -- GLOBAL_STEP: 36725 + | > loss: -0.30220 (-0.19109) + | > log_mle: -0.53200 (-0.36724) + | > loss_dur: 0.22980 (0.17615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.04441 (36.65342) + | > current_lr: 0.00004 + | > step_time: 2.10520 (2.52528) + | > loader_time: 0.08720 (0.04404) + + + --> STEP: 226/234 -- GLOBAL_STEP: 36730 + | > loss: -0.37309 (-0.19498) + | > log_mle: -0.63279 (-0.37295) + | > loss_dur: 0.25969 (0.17797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.04929 (38.11645) + | > current_lr: 0.00004 + | > step_time: 0.25660 (2.49547) + | > loader_time: 0.00380 (0.04316) + + + --> STEP: 231/234 -- GLOBAL_STEP: 36735 + | > loss: -0.30462 (-0.19781) + | > log_mle: -0.69998 (-0.37881) + | > loss_dur: 0.39536 (0.18100) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 112.21279 (39.84177) + | > current_lr: 0.00004 + | > step_time: 0.28380 (2.44715) + | > loader_time: 0.00380 (0.04232) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.49347 (+0.49078) + | > avg_loss: -0.20015 (+0.02900) + | > avg_log_mle: -0.44129 (+0.01150) + | > avg_loss_dur: 0.24113 (+0.01749) + + + > EPOCH: 157/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 18:03:47)  + + --> STEP: 2/234 -- GLOBAL_STEP: 36740 + | > loss: -0.16617 (-0.14833) + | > log_mle: -0.27171 (-0.26978) + | > loss_dur: 0.10554 (0.12146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.16076 (15.97413) + | > current_lr: 0.00004 + | > step_time: 4.79740 (3.80281) + | > loader_time: 0.00460 (0.04861) + + + --> STEP: 7/234 -- GLOBAL_STEP: 36745 + | > loss: -0.16399 (-0.13335) + | > log_mle: -0.27160 (-0.26320) + | > loss_dur: 0.10761 (0.12985) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.25910 (20.45864) + | > current_lr: 0.00004 + | > step_time: 5.19220 (4.05897) + | > loader_time: 0.09800 (0.05640) + + + --> STEP: 12/234 -- GLOBAL_STEP: 36750 + | > loss: -0.14686 (-0.13701) + | > log_mle: -0.26621 (-0.26548) + | > loss_dur: 0.11935 (0.12847) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.08041 (17.10604) + | > current_lr: 0.00004 + | > step_time: 2.20430 (5.60322) + | > loader_time: 0.00170 (0.06483) + + + --> STEP: 17/234 -- GLOBAL_STEP: 36755 + | > loss: -0.14305 (-0.14086) + | > log_mle: -0.24460 (-0.26447) + | > loss_dur: 0.10155 (0.12361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.56240 (14.99760) + | > current_lr: 0.00004 + | > step_time: 3.40680 (5.02106) + | > loader_time: 0.09410 (0.06917) + + + --> STEP: 22/234 -- GLOBAL_STEP: 36760 + | > loss: -0.14631 (-0.14117) + | > log_mle: -0.26479 (-0.26249) + | > loss_dur: 0.11848 (0.12132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17786 (13.97800) + | > current_lr: 0.00004 + | > step_time: 3.19610 (4.83016) + | > loader_time: 0.00280 (0.06275) + + + --> STEP: 27/234 -- GLOBAL_STEP: 36765 + | > loss: -0.14591 (-0.14324) + | > log_mle: -0.26546 (-0.26207) + | > loss_dur: 0.11955 (0.11882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.79771 (13.24689) + | > current_lr: 0.00004 + | > step_time: 6.49820 (4.66793) + | > loader_time: 0.00150 (0.05870) + + + --> STEP: 32/234 -- GLOBAL_STEP: 36770 + | > loss: -0.17646 (-0.14465) + | > log_mle: -0.27458 (-0.26221) + | > loss_dur: 0.09812 (0.11756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.36321 (12.49408) + | > current_lr: 0.00004 + | > step_time: 1.39490 (4.24863) + | > loader_time: 0.00140 (0.05259) + + + --> STEP: 37/234 -- GLOBAL_STEP: 36775 + | > loss: -0.15114 (-0.14324) + | > log_mle: -0.25504 (-0.26140) + | > loss_dur: 0.10390 (0.11816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.82100 (12.20652) + | > current_lr: 0.00004 + | > step_time: 1.19080 (3.99107) + | > loader_time: 0.00180 (0.05037) + + + --> STEP: 42/234 -- GLOBAL_STEP: 36780 + | > loss: -0.12630 (-0.14135) + | > log_mle: -0.24732 (-0.26094) + | > loss_dur: 0.12101 (0.11959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.60703 (11.88141) + | > current_lr: 0.00004 + | > step_time: 2.12570 (3.70188) + | > loader_time: 0.00180 (0.04458) + + + --> STEP: 47/234 -- GLOBAL_STEP: 36785 + | > loss: -0.11924 (-0.14043) + | > log_mle: -0.25571 (-0.26118) + | > loss_dur: 0.13647 (0.12075) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.56914 (11.74891) + | > current_lr: 0.00004 + | > step_time: 1.57640 (3.43958) + | > loader_time: 0.00200 (0.04177) + + + --> STEP: 52/234 -- GLOBAL_STEP: 36790 + | > loss: -0.11520 (-0.13965) + | > log_mle: -0.25398 (-0.26045) + | > loss_dur: 0.13878 (0.12080) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.12987 (11.37038) + | > current_lr: 0.00004 + | > step_time: 1.40340 (3.24490) + | > loader_time: 0.00280 (0.03799) + + + --> STEP: 57/234 -- GLOBAL_STEP: 36795 + | > loss: -0.10707 (-0.13895) + | > log_mle: -0.24625 (-0.26065) + | > loss_dur: 0.13918 (0.12170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.94629 (11.07520) + | > current_lr: 0.00004 + | > step_time: 1.15760 (3.08940) + | > loader_time: 0.00200 (0.03484) + + + --> STEP: 62/234 -- GLOBAL_STEP: 36800 + | > loss: -0.10548 (-0.13884) + | > log_mle: -0.29584 (-0.26178) + | > loss_dur: 0.19036 (0.12293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.87646 (11.15476) + | > current_lr: 0.00004 + | > step_time: 2.39760 (2.95761) + | > loader_time: 0.00210 (0.03220) + + + --> STEP: 67/234 -- GLOBAL_STEP: 36805 + | > loss: -0.12589 (-0.13808) + | > log_mle: -0.27710 (-0.26172) + | > loss_dur: 0.15121 (0.12364) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.52238 (10.99012) + | > current_lr: 0.00004 + | > step_time: 1.90980 (2.85169) + | > loader_time: 0.08120 (0.03363) + + + --> STEP: 72/234 -- GLOBAL_STEP: 36810 + | > loss: -0.11399 (-0.13601) + | > log_mle: -0.25633 (-0.26156) + | > loss_dur: 0.14234 (0.12554) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.51288 (11.11572) + | > current_lr: 0.00004 + | > step_time: 1.70520 (2.76844) + | > loader_time: 0.00300 (0.03285) + + + --> STEP: 77/234 -- GLOBAL_STEP: 36815 + | > loss: -0.15084 (-0.13517) + | > log_mle: -0.26912 (-0.26189) + | > loss_dur: 0.11828 (0.12672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.98778 (11.29426) + | > current_lr: 0.00004 + | > step_time: 1.59590 (2.71178) + | > loader_time: 0.00280 (0.03091) + + + --> STEP: 82/234 -- GLOBAL_STEP: 36820 + | > loss: -0.12086 (-0.13456) + | > log_mle: -0.25786 (-0.26189) + | > loss_dur: 0.13700 (0.12733) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.32850 (11.29725) + | > current_lr: 0.00004 + | > step_time: 2.09280 (2.66023) + | > loader_time: 0.00660 (0.03245) + + + --> STEP: 87/234 -- GLOBAL_STEP: 36825 + | > loss: -0.12531 (-0.13376) + | > log_mle: -0.26601 (-0.26220) + | > loss_dur: 0.14070 (0.12844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.74080 (11.41866) + | > current_lr: 0.00004 + | > step_time: 2.61490 (2.61903) + | > loader_time: 0.08530 (0.03168) + + + --> STEP: 92/234 -- GLOBAL_STEP: 36830 + | > loss: -0.16007 (-0.13405) + | > log_mle: -0.31301 (-0.26417) + | > loss_dur: 0.15294 (0.13012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.57065 (11.70150) + | > current_lr: 0.00004 + | > step_time: 2.08710 (2.57106) + | > loader_time: 0.00300 (0.03091) + + + --> STEP: 97/234 -- GLOBAL_STEP: 36835 + | > loss: -0.13974 (-0.13537) + | > log_mle: -0.29913 (-0.26712) + | > loss_dur: 0.15939 (0.13175) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.10535 (12.17400) + | > current_lr: 0.00004 + | > step_time: 1.39360 (2.52700) + | > loader_time: 0.00500 (0.02948) + + + --> STEP: 102/234 -- GLOBAL_STEP: 36840 + | > loss: -0.12207 (-0.13579) + | > log_mle: -0.28307 (-0.26901) + | > loss_dur: 0.16100 (0.13322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.85400 (12.61059) + | > current_lr: 0.00004 + | > step_time: 1.36370 (2.48137) + | > loader_time: 0.00220 (0.02980) + + + --> STEP: 107/234 -- GLOBAL_STEP: 36845 + | > loss: -0.16044 (-0.13701) + | > log_mle: -0.32833 (-0.27202) + | > loss_dur: 0.16789 (0.13501) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.48851 (13.29601) + | > current_lr: 0.00004 + | > step_time: 1.59780 (2.44673) + | > loader_time: 0.00330 (0.02931) + + + --> STEP: 112/234 -- GLOBAL_STEP: 36850 + | > loss: -0.14191 (-0.13747) + | > log_mle: -0.34204 (-0.27476) + | > loss_dur: 0.20013 (0.13729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.37277 (13.84286) + | > current_lr: 0.00004 + | > step_time: 1.35860 (2.42674) + | > loader_time: 0.00240 (0.02888) + + + --> STEP: 117/234 -- GLOBAL_STEP: 36855 + | > loss: -0.16546 (-0.13830) + | > log_mle: -0.33585 (-0.27739) + | > loss_dur: 0.17039 (0.13909) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.49859 (14.37884) + | > current_lr: 0.00004 + | > step_time: 1.38570 (2.39869) + | > loader_time: 0.00280 (0.02777) + + + --> STEP: 122/234 -- GLOBAL_STEP: 36860 + | > loss: -0.14454 (-0.13851) + | > log_mle: -0.30804 (-0.27899) + | > loss_dur: 0.16350 (0.14047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.20588 (14.75530) + | > current_lr: 0.00004 + | > step_time: 1.90310 (2.37828) + | > loader_time: 0.07670 (0.02738) + + + --> STEP: 127/234 -- GLOBAL_STEP: 36865 + | > loss: -0.17655 (-0.13979) + | > log_mle: -0.36648 (-0.28186) + | > loss_dur: 0.18993 (0.14207) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.24528 (15.42260) + | > current_lr: 0.00004 + | > step_time: 0.98660 (2.35170) + | > loader_time: 0.00240 (0.02717) + + + --> STEP: 132/234 -- GLOBAL_STEP: 36870 + | > loss: -0.18230 (-0.14158) + | > log_mle: -0.35273 (-0.28524) + | > loss_dur: 0.17043 (0.14366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.91825 (16.29303) + | > current_lr: 0.00004 + | > step_time: 3.20160 (2.38775) + | > loader_time: 0.00370 (0.02761) + + + --> STEP: 137/234 -- GLOBAL_STEP: 36875 + | > loss: -0.14597 (-0.14316) + | > log_mle: -0.35739 (-0.28882) + | > loss_dur: 0.21142 (0.14566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.86571 (17.22929) + | > current_lr: 0.00004 + | > step_time: 3.85200 (2.46810) + | > loader_time: 0.08840 (0.02941) + + + --> STEP: 142/234 -- GLOBAL_STEP: 36880 + | > loss: -0.17436 (-0.14438) + | > log_mle: -0.37239 (-0.29180) + | > loss_dur: 0.19803 (0.14742) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.01728 (18.17057) + | > current_lr: 0.00004 + | > step_time: 5.70550 (2.48443) + | > loader_time: 0.19230 (0.03042) + + + --> STEP: 147/234 -- GLOBAL_STEP: 36885 + | > loss: -0.17952 (-0.14690) + | > log_mle: -0.36948 (-0.29641) + | > loss_dur: 0.18996 (0.14952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.13130 (19.50123) + | > current_lr: 0.00004 + | > step_time: 1.59680 (2.48621) + | > loader_time: 0.01050 (0.03066) + + + --> STEP: 152/234 -- GLOBAL_STEP: 36890 + | > loss: -0.23571 (-0.14940) + | > log_mle: -0.45751 (-0.30061) + | > loss_dur: 0.22180 (0.15121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.55281 (20.49719) + | > current_lr: 0.00004 + | > step_time: 3.50810 (2.47756) + | > loader_time: 0.19390 (0.03200) + + + --> STEP: 157/234 -- GLOBAL_STEP: 36895 + | > loss: -0.19800 (-0.15258) + | > log_mle: -0.40450 (-0.30544) + | > loss_dur: 0.20650 (0.15286) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.13049 (21.82364) + | > current_lr: 0.00004 + | > step_time: 2.59880 (2.48529) + | > loader_time: 0.00370 (0.03228) + + + --> STEP: 162/234 -- GLOBAL_STEP: 36900 + | > loss: -0.23621 (-0.15536) + | > log_mle: -0.43875 (-0.30998) + | > loss_dur: 0.20253 (0.15462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.84732 (22.96483) + | > current_lr: 0.00004 + | > step_time: 2.91010 (2.49950) + | > loader_time: 0.00340 (0.03302) + + + --> STEP: 167/234 -- GLOBAL_STEP: 36905 + | > loss: -0.32177 (-0.15810) + | > log_mle: -0.52823 (-0.31434) + | > loss_dur: 0.20646 (0.15625) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.01193 (23.85428) + | > current_lr: 0.00004 + | > step_time: 2.39220 (2.48112) + | > loader_time: 0.00290 (0.03262) + + + --> STEP: 172/234 -- GLOBAL_STEP: 36910 + | > loss: -0.27959 (-0.16105) + | > log_mle: -0.51050 (-0.31933) + | > loss_dur: 0.23092 (0.15828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.83349 (25.40226) + | > current_lr: 0.00004 + | > step_time: 1.69710 (2.47377) + | > loader_time: 0.10650 (0.03452) + + + --> STEP: 177/234 -- GLOBAL_STEP: 36915 + | > loss: -0.25609 (-0.16413) + | > log_mle: -0.47065 (-0.32422) + | > loss_dur: 0.21456 (0.16008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.84733 (26.57806) + | > current_lr: 0.00004 + | > step_time: 2.89580 (2.48702) + | > loader_time: 0.00330 (0.03419) + + + --> STEP: 182/234 -- GLOBAL_STEP: 36920 + | > loss: -0.26969 (-0.16687) + | > log_mle: -0.52221 (-0.32909) + | > loss_dur: 0.25253 (0.16222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.03819 (27.83469) + | > current_lr: 0.00004 + | > step_time: 3.30480 (2.49295) + | > loader_time: 0.19190 (0.03491) + + + --> STEP: 187/234 -- GLOBAL_STEP: 36925 + | > loss: -0.29468 (-0.16975) + | > log_mle: -0.52600 (-0.33392) + | > loss_dur: 0.23132 (0.16417) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.14302 (28.95535) + | > current_lr: 0.00004 + | > step_time: 7.20280 (2.53426) + | > loader_time: 0.08520 (0.03548) + + + --> STEP: 192/234 -- GLOBAL_STEP: 36930 + | > loss: -0.32269 (-0.17281) + | > log_mle: -0.54390 (-0.33849) + | > loss_dur: 0.22121 (0.16567) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.49039 (30.15545) + | > current_lr: 0.00004 + | > step_time: 3.79570 (2.62148) + | > loader_time: 0.00670 (0.04174) + + + --> STEP: 197/234 -- GLOBAL_STEP: 36935 + | > loss: -0.30154 (-0.17589) + | > log_mle: -0.51643 (-0.34310) + | > loss_dur: 0.21489 (0.16721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.26302 (31.20816) + | > current_lr: 0.00004 + | > step_time: 0.68040 (2.63761) + | > loader_time: 0.00310 (0.04169) + + + --> STEP: 202/234 -- GLOBAL_STEP: 36940 + | > loss: -0.38182 (-0.17885) + | > log_mle: -0.61142 (-0.34780) + | > loss_dur: 0.22960 (0.16895) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.48386 (32.41890) + | > current_lr: 0.00004 + | > step_time: 6.30590 (2.70265) + | > loader_time: 0.10110 (0.04265) + + + --> STEP: 207/234 -- GLOBAL_STEP: 36945 + | > loss: -0.35273 (-0.18187) + | > log_mle: -0.59456 (-0.35246) + | > loss_dur: 0.24183 (0.17060) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.69494 (33.39520) + | > current_lr: 0.00004 + | > step_time: 11.49600 (2.76941) + | > loader_time: 0.10640 (0.04345) + + + --> STEP: 212/234 -- GLOBAL_STEP: 36950 + | > loss: -0.32650 (-0.18536) + | > log_mle: -0.57542 (-0.35778) + | > loss_dur: 0.24892 (0.17243) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.44458 (34.49355) + | > current_lr: 0.00004 + | > step_time: 3.60540 (2.78249) + | > loader_time: 0.00420 (0.04292) + + + --> STEP: 217/234 -- GLOBAL_STEP: 36955 + | > loss: -0.35642 (-0.18904) + | > log_mle: -0.60648 (-0.36310) + | > loss_dur: 0.25006 (0.17406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.68256 (35.79718) + | > current_lr: 0.00004 + | > step_time: 2.99530 (2.81382) + | > loader_time: 0.00620 (0.04331) + + + --> STEP: 222/234 -- GLOBAL_STEP: 36960 + | > loss: -0.33318 (-0.19261) + | > log_mle: -0.61039 (-0.36835) + | > loss_dur: 0.27722 (0.17574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.03371 (37.11594) + | > current_lr: 0.00004 + | > step_time: 0.23700 (2.78555) + | > loader_time: 0.00270 (0.04241) + + + --> STEP: 227/234 -- GLOBAL_STEP: 36965 + | > loss: -0.31401 (-0.19644) + | > log_mle: -0.58430 (-0.37392) + | > loss_dur: 0.27028 (0.17749) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.04058 (38.65419) + | > current_lr: 0.00004 + | > step_time: 0.24130 (2.72947) + | > loader_time: 0.00420 (0.04156) + + + --> STEP: 232/234 -- GLOBAL_STEP: 36970 + | > loss: -0.27056 (-0.19908) + | > log_mle: -0.78337 (-0.38065) + | > loss_dur: 0.51281 (0.18157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.79546 (40.50856) + | > current_lr: 0.00004 + | > step_time: 0.33440 (2.67672) + | > loader_time: 0.07290 (0.04105) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12067 (-0.37280) + | > avg_loss: -0.19683 (+0.00333) + | > avg_log_mle: -0.42701 (+0.01427) + | > avg_loss_dur: 0.23019 (-0.01095) + + + > EPOCH: 158/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 18:15:24)  + + --> STEP: 3/234 -- GLOBAL_STEP: 36975 + | > loss: -0.08271 (-0.12954) + | > log_mle: -0.26210 (-0.26847) + | > loss_dur: 0.17939 (0.13893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.89996 (17.52155) + | > current_lr: 0.00004 + | > step_time: 4.69550 (3.29043) + | > loader_time: 0.00120 (0.03508) + + + --> STEP: 8/234 -- GLOBAL_STEP: 36980 + | > loss: -0.15456 (-0.13553) + | > log_mle: -0.28174 (-0.26911) + | > loss_dur: 0.12718 (0.13358) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.50276 (15.71601) + | > current_lr: 0.00004 + | > step_time: 3.20190 (4.63506) + | > loader_time: 0.08910 (0.06097) + + + --> STEP: 13/234 -- GLOBAL_STEP: 36985 + | > loss: -0.17036 (-0.13899) + | > log_mle: -0.27533 (-0.26965) + | > loss_dur: 0.10497 (0.13066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.29681 (14.61325) + | > current_lr: 0.00004 + | > step_time: 1.61270 (3.46159) + | > loader_time: 0.00160 (0.04502) + + + --> STEP: 18/234 -- GLOBAL_STEP: 36990 + | > loss: -0.13240 (-0.14017) + | > log_mle: -0.25861 (-0.26749) + | > loss_dur: 0.12621 (0.12732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.33573 (13.96688) + | > current_lr: 0.00004 + | > step_time: 1.60070 (2.90956) + | > loader_time: 0.00160 (0.03297) + + + --> STEP: 23/234 -- GLOBAL_STEP: 36995 + | > loss: -0.16621 (-0.14264) + | > log_mle: -0.26907 (-0.26565) + | > loss_dur: 0.10287 (0.12300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.81585 (12.98265) + | > current_lr: 0.00004 + | > step_time: 1.39700 (2.56471) + | > loader_time: 0.10150 (0.03783) + + + --> STEP: 28/234 -- GLOBAL_STEP: 37000 + | > loss: -0.17529 (-0.14468) + | > log_mle: -0.26846 (-0.26491) + | > loss_dur: 0.09317 (0.12023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.57135 (12.16643) + | > current_lr: 0.00004 + | > step_time: 1.29740 (2.33872) + | > loader_time: 0.00210 (0.03470) + + + --> STEP: 33/234 -- GLOBAL_STEP: 37005 + | > loss: -0.14376 (-0.14486) + | > log_mle: -0.25586 (-0.26433) + | > loss_dur: 0.11210 (0.11947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.58847 (12.00762) + | > current_lr: 0.00004 + | > step_time: 1.25220 (2.22317) + | > loader_time: 0.00350 (0.03579) + + + --> STEP: 38/234 -- GLOBAL_STEP: 37010 + | > loss: -0.14318 (-0.14332) + | > log_mle: -0.27014 (-0.26376) + | > loss_dur: 0.12696 (0.12043) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.68284 (11.94133) + | > current_lr: 0.00004 + | > step_time: 1.32840 (2.08900) + | > loader_time: 0.00180 (0.03136) + + + --> STEP: 43/234 -- GLOBAL_STEP: 37015 + | > loss: -0.12292 (-0.14143) + | > log_mle: -0.26880 (-0.26305) + | > loss_dur: 0.14588 (0.12161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.60534 (11.70252) + | > current_lr: 0.00004 + | > step_time: 1.19700 (1.98901) + | > loader_time: 0.00240 (0.02797) + + + --> STEP: 48/234 -- GLOBAL_STEP: 37020 + | > loss: -0.14098 (-0.14063) + | > log_mle: -0.25061 (-0.26266) + | > loss_dur: 0.10963 (0.12203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.05849 (11.67255) + | > current_lr: 0.00004 + | > step_time: 1.21370 (1.90627) + | > loader_time: 0.07500 (0.02680) + + + --> STEP: 53/234 -- GLOBAL_STEP: 37025 + | > loss: -0.12965 (-0.13958) + | > log_mle: -0.26593 (-0.26203) + | > loss_dur: 0.13628 (0.12245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.41117 (11.43866) + | > current_lr: 0.00004 + | > step_time: 1.78740 (1.87699) + | > loader_time: 0.00230 (0.02631) + + + --> STEP: 58/234 -- GLOBAL_STEP: 37030 + | > loss: -0.14069 (-0.13899) + | > log_mle: -0.25567 (-0.26190) + | > loss_dur: 0.11498 (0.12291) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.06516 (11.20383) + | > current_lr: 0.00004 + | > step_time: 1.18620 (1.83517) + | > loader_time: 0.00210 (0.02424) + + + --> STEP: 63/234 -- GLOBAL_STEP: 37035 + | > loss: -0.10329 (-0.13824) + | > log_mle: -0.25806 (-0.26292) + | > loss_dur: 0.15478 (0.12468) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.43959 (11.40739) + | > current_lr: 0.00004 + | > step_time: 1.18790 (1.81178) + | > loader_time: 0.00260 (0.02379) + + + --> STEP: 68/234 -- GLOBAL_STEP: 37040 + | > loss: -0.10676 (-0.13761) + | > log_mle: -0.25560 (-0.26269) + | > loss_dur: 0.14883 (0.12508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.41450 (11.18950) + | > current_lr: 0.00004 + | > step_time: 1.10110 (1.77854) + | > loader_time: 0.00200 (0.02464) + + + --> STEP: 73/234 -- GLOBAL_STEP: 37045 + | > loss: -0.10578 (-0.13624) + | > log_mle: -0.27596 (-0.26277) + | > loss_dur: 0.17017 (0.12653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.49388 (11.28352) + | > current_lr: 0.00004 + | > step_time: 1.73780 (1.78618) + | > loader_time: 0.08800 (0.02665) + + + --> STEP: 78/234 -- GLOBAL_STEP: 37050 + | > loss: -0.10882 (-0.13548) + | > log_mle: -0.24624 (-0.26271) + | > loss_dur: 0.13742 (0.12724) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.71145 (11.34708) + | > current_lr: 0.00004 + | > step_time: 1.41720 (1.80343) + | > loader_time: 0.08670 (0.02726) + + + --> STEP: 83/234 -- GLOBAL_STEP: 37055 + | > loss: -0.11026 (-0.13505) + | > log_mle: -0.27352 (-0.26291) + | > loss_dur: 0.16325 (0.12786) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.15549 (11.51768) + | > current_lr: 0.00004 + | > step_time: 1.26220 (1.78113) + | > loader_time: 0.00210 (0.02777) + + + --> STEP: 88/234 -- GLOBAL_STEP: 37060 + | > loss: -0.14743 (-0.13485) + | > log_mle: -0.31607 (-0.26378) + | > loss_dur: 0.16864 (0.12894) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.58119 (11.62437) + | > current_lr: 0.00004 + | > step_time: 1.40590 (1.76639) + | > loader_time: 0.00310 (0.02730) + + + --> STEP: 93/234 -- GLOBAL_STEP: 37065 + | > loss: -0.14906 (-0.13539) + | > log_mle: -0.32921 (-0.26600) + | > loss_dur: 0.18015 (0.13061) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.15205 (11.94088) + | > current_lr: 0.00004 + | > step_time: 3.09320 (1.78388) + | > loader_time: 0.00430 (0.02598) + + + --> STEP: 98/234 -- GLOBAL_STEP: 37070 + | > loss: -0.10923 (-0.13620) + | > log_mle: -0.25615 (-0.26821) + | > loss_dur: 0.14692 (0.13201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.99059 (12.37020) + | > current_lr: 0.00004 + | > step_time: 2.21760 (1.78613) + | > loader_time: 0.08320 (0.02725) + + + --> STEP: 103/234 -- GLOBAL_STEP: 37075 + | > loss: -0.17794 (-0.13727) + | > log_mle: -0.36137 (-0.27118) + | > loss_dur: 0.18342 (0.13391) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.64448 (12.96257) + | > current_lr: 0.00004 + | > step_time: 2.69970 (1.79527) + | > loader_time: 0.00260 (0.02867) + + + --> STEP: 108/234 -- GLOBAL_STEP: 37080 + | > loss: -0.15200 (-0.13816) + | > log_mle: -0.30301 (-0.27366) + | > loss_dur: 0.15101 (0.13550) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.12593 (13.42286) + | > current_lr: 0.00004 + | > step_time: 2.19730 (1.78266) + | > loader_time: 0.08410 (0.03057) + + + --> STEP: 113/234 -- GLOBAL_STEP: 37085 + | > loss: -0.17121 (-0.13887) + | > log_mle: -0.34981 (-0.27684) + | > loss_dur: 0.17860 (0.13797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.04169 (14.12906) + | > current_lr: 0.00004 + | > step_time: 2.29690 (1.78173) + | > loader_time: 0.00340 (0.03156) + + + --> STEP: 118/234 -- GLOBAL_STEP: 37090 + | > loss: -0.13997 (-0.13930) + | > log_mle: -0.31897 (-0.27912) + | > loss_dur: 0.17900 (0.13982) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.39248 (14.66839) + | > current_lr: 0.00004 + | > step_time: 2.49880 (1.80543) + | > loader_time: 0.00320 (0.03035) + + + --> STEP: 123/234 -- GLOBAL_STEP: 37095 + | > loss: -0.12557 (-0.13942) + | > log_mle: -0.28738 (-0.28037) + | > loss_dur: 0.16180 (0.14095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.07500 (15.05177) + | > current_lr: 0.00004 + | > step_time: 2.20340 (1.80593) + | > loader_time: 0.00270 (0.03006) + + + --> STEP: 128/234 -- GLOBAL_STEP: 37100 + | > loss: -0.18534 (-0.14115) + | > log_mle: -0.34966 (-0.28371) + | > loss_dur: 0.16432 (0.14257) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.18026 (15.76931) + | > current_lr: 0.00004 + | > step_time: 1.41370 (1.79313) + | > loader_time: 0.08500 (0.03096) + + + --> STEP: 133/234 -- GLOBAL_STEP: 37105 + | > loss: -0.18719 (-0.14296) + | > log_mle: -0.37612 (-0.28730) + | > loss_dur: 0.18893 (0.14434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.76527 (16.48131) + | > current_lr: 0.00004 + | > step_time: 1.79460 (1.80968) + | > loader_time: 0.00260 (0.03055) + + + --> STEP: 138/234 -- GLOBAL_STEP: 37110 + | > loss: -0.14418 (-0.14421) + | > log_mle: -0.32572 (-0.29046) + | > loss_dur: 0.18154 (0.14625) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.67356 (17.37838) + | > current_lr: 0.00004 + | > step_time: 1.57880 (1.82162) + | > loader_time: 0.00270 (0.03365) + + + --> STEP: 143/234 -- GLOBAL_STEP: 37115 + | > loss: -0.23029 (-0.14613) + | > log_mle: -0.46293 (-0.29441) + | > loss_dur: 0.23264 (0.14828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.73262 (18.41909) + | > current_lr: 0.00004 + | > step_time: 1.68810 (1.82932) + | > loader_time: 0.00320 (0.03320) + + + --> STEP: 148/234 -- GLOBAL_STEP: 37120 + | > loss: -0.21565 (-0.14867) + | > log_mle: -0.38233 (-0.29843) + | > loss_dur: 0.16668 (0.14977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.84298 (19.27590) + | > current_lr: 0.00004 + | > step_time: 4.91230 (1.84873) + | > loader_time: 0.08370 (0.03390) + + + --> STEP: 153/234 -- GLOBAL_STEP: 37125 + | > loss: -0.30029 (-0.15194) + | > log_mle: -0.50780 (-0.30364) + | > loss_dur: 0.20751 (0.15169) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.36813 (20.47415) + | > current_lr: 0.00004 + | > step_time: 1.71960 (1.87015) + | > loader_time: 0.08770 (0.03405) + + + --> STEP: 158/234 -- GLOBAL_STEP: 37130 + | > loss: -0.22843 (-0.15474) + | > log_mle: -0.44740 (-0.30825) + | > loss_dur: 0.21897 (0.15351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.24124 (21.51515) + | > current_lr: 0.00004 + | > step_time: 1.16550 (1.89313) + | > loader_time: 0.00260 (0.03372) + + + --> STEP: 163/234 -- GLOBAL_STEP: 37135 + | > loss: -0.21458 (-0.15770) + | > log_mle: -0.41630 (-0.31277) + | > loss_dur: 0.20173 (0.15506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.20383 (22.50808) + | > current_lr: 0.00004 + | > step_time: 2.59200 (1.91419) + | > loader_time: 0.00340 (0.03427) + + + --> STEP: 168/234 -- GLOBAL_STEP: 37140 + | > loss: -0.23251 (-0.16053) + | > log_mle: -0.47015 (-0.31739) + | > loss_dur: 0.23765 (0.15686) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.51956 (23.60226) + | > current_lr: 0.00004 + | > step_time: 2.41960 (1.92403) + | > loader_time: 0.08540 (0.03489) + + + --> STEP: 173/234 -- GLOBAL_STEP: 37145 + | > loss: -0.26459 (-0.16376) + | > log_mle: -0.48265 (-0.32259) + | > loss_dur: 0.21806 (0.15883) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.84323 (24.88819) + | > current_lr: 0.00004 + | > step_time: 3.89690 (1.94358) + | > loader_time: 0.29530 (0.03669) + + + --> STEP: 178/234 -- GLOBAL_STEP: 37150 + | > loss: -0.28880 (-0.16695) + | > log_mle: -0.53023 (-0.32775) + | > loss_dur: 0.24143 (0.16079) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.00452 (26.36683) + | > current_lr: 0.00004 + | > step_time: 1.30510 (1.95085) + | > loader_time: 0.00320 (0.03576) + + + --> STEP: 183/234 -- GLOBAL_STEP: 37155 + | > loss: -0.30858 (-0.16969) + | > log_mle: -0.53434 (-0.33248) + | > loss_dur: 0.22575 (0.16278) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.02264 (27.54594) + | > current_lr: 0.00004 + | > step_time: 2.49680 (1.95561) + | > loader_time: 0.00360 (0.03639) + + + --> STEP: 188/234 -- GLOBAL_STEP: 37160 + | > loss: -0.31225 (-0.17260) + | > log_mle: -0.54327 (-0.33724) + | > loss_dur: 0.23102 (0.16464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.62337 (28.62594) + | > current_lr: 0.00004 + | > step_time: 2.90440 (1.96397) + | > loader_time: 0.00340 (0.03641) + + + --> STEP: 193/234 -- GLOBAL_STEP: 37165 + | > loss: -0.32313 (-0.17580) + | > log_mle: -0.55001 (-0.34191) + | > loss_dur: 0.22688 (0.16612) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.17490 (29.73662) + | > current_lr: 0.00004 + | > step_time: 3.70060 (1.99143) + | > loader_time: 0.10260 (0.03742) + + + --> STEP: 198/234 -- GLOBAL_STEP: 37170 + | > loss: -0.29538 (-0.17861) + | > log_mle: -0.53390 (-0.34636) + | > loss_dur: 0.23852 (0.16776) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.10738 (30.86080) + | > current_lr: 0.00004 + | > step_time: 4.98990 (2.08260) + | > loader_time: 0.20530 (0.03898) + + + --> STEP: 203/234 -- GLOBAL_STEP: 37175 + | > loss: -0.24493 (-0.18114) + | > log_mle: -0.47271 (-0.35061) + | > loss_dur: 0.22778 (0.16947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.18821 (32.10437) + | > current_lr: 0.00004 + | > step_time: 3.40240 (2.15588) + | > loader_time: 0.00380 (0.03955) + + + --> STEP: 208/234 -- GLOBAL_STEP: 37180 + | > loss: -0.29254 (-0.18406) + | > log_mle: -0.53785 (-0.35525) + | > loss_dur: 0.24531 (0.17119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.34254 (33.48567) + | > current_lr: 0.00004 + | > step_time: 2.80310 (2.23333) + | > loader_time: 0.09990 (0.04015) + + + --> STEP: 213/234 -- GLOBAL_STEP: 37185 + | > loss: -0.33142 (-0.18711) + | > log_mle: -0.58289 (-0.36018) + | > loss_dur: 0.25147 (0.17307) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.42548 (34.90282) + | > current_lr: 0.00004 + | > step_time: 9.89930 (2.36031) + | > loader_time: 0.00370 (0.04015) + + + --> STEP: 218/234 -- GLOBAL_STEP: 37190 + | > loss: -0.30067 (-0.19021) + | > log_mle: -0.54816 (-0.36492) + | > loss_dur: 0.24749 (0.17471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.00056 (36.07006) + | > current_lr: 0.00004 + | > step_time: 2.29750 (2.41402) + | > loader_time: 0.07740 (0.04133) + + + --> STEP: 223/234 -- GLOBAL_STEP: 37195 + | > loss: -0.35533 (-0.19373) + | > log_mle: -0.59965 (-0.37011) + | > loss_dur: 0.24432 (0.17638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.25069 (37.28560) + | > current_lr: 0.00004 + | > step_time: 1.09100 (2.38228) + | > loader_time: 0.00400 (0.04049) + + + --> STEP: 228/234 -- GLOBAL_STEP: 37200 + | > loss: -0.32230 (-0.19725) + | > log_mle: -0.60406 (-0.37553) + | > loss_dur: 0.28176 (0.17828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.60253 (38.43279) + | > current_lr: 0.00004 + | > step_time: 0.27270 (2.33762) + | > loader_time: 0.00400 (0.04000) + + + --> STEP: 233/234 -- GLOBAL_STEP: 37205 + | > loss: 0.13298 (-0.19810) + | > log_mle: -0.56727 (-0.38221) + | > loss_dur: 0.70025 (0.18411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.01951 (40.11915) + | > current_lr: 0.00004 + | > step_time: 0.20420 (2.29352) + | > loader_time: 0.00330 (0.03924) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02454 (-0.09613) + | > avg_loss: -0.19815 (-0.00133) + | > avg_log_mle: -0.43584 (-0.00883) + | > avg_loss_dur: 0.23769 (+0.00750) + + + > EPOCH: 159/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 18:25:43)  + + --> STEP: 4/234 -- GLOBAL_STEP: 37210 + | > loss: -0.10927 (-0.13083) + | > log_mle: -0.26210 (-0.26709) + | > loss_dur: 0.15283 (0.13626) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.35766 (19.28724) + | > current_lr: 0.00004 + | > step_time: 3.50050 (3.29785) + | > loader_time: 2.59940 (2.07541) + + + --> STEP: 9/234 -- GLOBAL_STEP: 37215 + | > loss: -0.13511 (-0.14467) + | > log_mle: -0.27759 (-0.27125) + | > loss_dur: 0.14248 (0.12657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.08353 (15.26836) + | > current_lr: 0.00004 + | > step_time: 8.99830 (4.09940) + | > loader_time: 0.00410 (0.94462) + + + --> STEP: 14/234 -- GLOBAL_STEP: 37220 + | > loss: -0.13936 (-0.14730) + | > log_mle: -0.27192 (-0.27114) + | > loss_dur: 0.13256 (0.12384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.39192 (13.84419) + | > current_lr: 0.00004 + | > step_time: 5.80560 (3.98782) + | > loader_time: 0.09060 (0.67649) + + + --> STEP: 19/234 -- GLOBAL_STEP: 37225 + | > loss: -0.16910 (-0.14879) + | > log_mle: -0.26173 (-0.26860) + | > loss_dur: 0.09264 (0.11981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.86561 (12.93100) + | > current_lr: 0.00004 + | > step_time: 0.99190 (4.36642) + | > loader_time: 0.00130 (0.51402) + + + --> STEP: 24/234 -- GLOBAL_STEP: 37230 + | > loss: -0.17097 (-0.15043) + | > log_mle: -0.25930 (-0.26686) + | > loss_dur: 0.08833 (0.11643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.33080 (12.54275) + | > current_lr: 0.00004 + | > step_time: 0.99130 (3.82777) + | > loader_time: 0.00120 (0.41423) + + + --> STEP: 29/234 -- GLOBAL_STEP: 37235 + | > loss: -0.12521 (-0.15023) + | > log_mle: -0.24953 (-0.26577) + | > loss_dur: 0.12432 (0.11555) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.48559 (12.06835) + | > current_lr: 0.00004 + | > step_time: 1.13250 (3.37104) + | > loader_time: 0.00140 (0.34311) + + + --> STEP: 34/234 -- GLOBAL_STEP: 37240 + | > loss: -0.12715 (-0.14953) + | > log_mle: -0.25888 (-0.26572) + | > loss_dur: 0.13174 (0.11619) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.35398 (11.72851) + | > current_lr: 0.00004 + | > step_time: 1.60730 (3.12070) + | > loader_time: 0.08710 (0.29541) + + + --> STEP: 39/234 -- GLOBAL_STEP: 37245 + | > loss: -0.15282 (-0.14859) + | > log_mle: -0.26926 (-0.26575) + | > loss_dur: 0.11644 (0.11716) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.30369 (11.57596) + | > current_lr: 0.00004 + | > step_time: 1.96850 (2.91395) + | > loader_time: 0.00190 (0.25779) + + + --> STEP: 44/234 -- GLOBAL_STEP: 37250 + | > loss: -0.15578 (-0.14683) + | > log_mle: -0.26054 (-0.26506) + | > loss_dur: 0.10477 (0.11822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.95926 (11.19793) + | > current_lr: 0.00004 + | > step_time: 2.21570 (2.80739) + | > loader_time: 0.08380 (0.23260) + + + --> STEP: 49/234 -- GLOBAL_STEP: 37255 + | > loss: -0.16058 (-0.14569) + | > log_mle: -0.26907 (-0.26507) + | > loss_dur: 0.10848 (0.11939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.84191 (11.07898) + | > current_lr: 0.00004 + | > step_time: 1.19210 (2.69622) + | > loader_time: 0.00180 (0.20908) + + + --> STEP: 54/234 -- GLOBAL_STEP: 37260 + | > loss: -0.15832 (-0.14429) + | > log_mle: -0.27491 (-0.26475) + | > loss_dur: 0.11659 (0.12045) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.27761 (10.80998) + | > current_lr: 0.00004 + | > step_time: 2.66550 (2.64593) + | > loader_time: 0.00480 (0.19001) + + + --> STEP: 59/234 -- GLOBAL_STEP: 37265 + | > loss: -0.17090 (-0.14428) + | > log_mle: -0.27926 (-0.26484) + | > loss_dur: 0.10836 (0.12056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.74421 (10.60273) + | > current_lr: 0.00004 + | > step_time: 1.33280 (2.53107) + | > loader_time: 0.00350 (0.17552) + + + --> STEP: 64/234 -- GLOBAL_STEP: 37270 + | > loss: -0.14534 (-0.14305) + | > log_mle: -0.25821 (-0.26555) + | > loss_dur: 0.11286 (0.12250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.19182 (10.74207) + | > current_lr: 0.00004 + | > step_time: 1.29160 (2.46708) + | > loader_time: 0.00220 (0.16456) + + + --> STEP: 69/234 -- GLOBAL_STEP: 37275 + | > loss: -0.11598 (-0.14185) + | > log_mle: -0.24634 (-0.26518) + | > loss_dur: 0.13036 (0.12333) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.88575 (10.64833) + | > current_lr: 0.00004 + | > step_time: 1.39200 (2.43875) + | > loader_time: 0.00220 (0.15414) + + + --> STEP: 74/234 -- GLOBAL_STEP: 37280 + | > loss: -0.12184 (-0.14013) + | > log_mle: -0.25437 (-0.26547) + | > loss_dur: 0.13253 (0.12534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.15963 (10.84540) + | > current_lr: 0.00004 + | > step_time: 1.20170 (2.37522) + | > loader_time: 0.08420 (0.14502) + + + --> STEP: 79/234 -- GLOBAL_STEP: 37285 + | > loss: -0.12598 (-0.13939) + | > log_mle: -0.27147 (-0.26574) + | > loss_dur: 0.14549 (0.12635) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.81351 (10.85189) + | > current_lr: 0.00004 + | > step_time: 1.40880 (2.34049) + | > loader_time: 0.09310 (0.14055) + + + --> STEP: 84/234 -- GLOBAL_STEP: 37290 + | > loss: -0.12837 (-0.13881) + | > log_mle: -0.26615 (-0.26594) + | > loss_dur: 0.13778 (0.12713) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.81275 (10.92128) + | > current_lr: 0.00004 + | > step_time: 2.29850 (2.31766) + | > loader_time: 0.00300 (0.13454) + + + --> STEP: 89/234 -- GLOBAL_STEP: 37295 + | > loss: -0.15454 (-0.13876) + | > log_mle: -0.29656 (-0.26705) + | > loss_dur: 0.14202 (0.12829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.95787 (11.17229) + | > current_lr: 0.00004 + | > step_time: 1.20030 (2.28247) + | > loader_time: 0.08490 (0.12805) + + + --> STEP: 94/234 -- GLOBAL_STEP: 37300 + | > loss: -0.18032 (-0.13941) + | > log_mle: -0.33248 (-0.26956) + | > loss_dur: 0.15216 (0.13015) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.75430 (11.63302) + | > current_lr: 0.00004 + | > step_time: 1.79900 (2.22734) + | > loader_time: 0.00280 (0.12229) + + + --> STEP: 99/234 -- GLOBAL_STEP: 37305 + | > loss: -0.18216 (-0.14026) + | > log_mle: -0.36341 (-0.27195) + | > loss_dur: 0.18125 (0.13169) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.33202 (12.11544) + | > current_lr: 0.00004 + | > step_time: 1.78830 (2.18744) + | > loader_time: 0.00310 (0.11789) + + + --> STEP: 104/234 -- GLOBAL_STEP: 37310 + | > loss: -0.19206 (-0.14137) + | > log_mle: -0.37617 (-0.27488) + | > loss_dur: 0.18411 (0.13351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.62506 (12.67849) + | > current_lr: 0.00004 + | > step_time: 3.70340 (2.16978) + | > loader_time: 0.00290 (0.11311) + + + --> STEP: 109/234 -- GLOBAL_STEP: 37315 + | > loss: -0.12769 (-0.14151) + | > log_mle: -0.34357 (-0.27683) + | > loss_dur: 0.21588 (0.13532) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.34326 (13.29759) + | > current_lr: 0.00004 + | > step_time: 2.56960 (2.19245) + | > loader_time: 0.00250 (0.10975) + + + --> STEP: 114/234 -- GLOBAL_STEP: 37320 + | > loss: -0.15932 (-0.14238) + | > log_mle: -0.32884 (-0.27970) + | > loss_dur: 0.16951 (0.13732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.97574 (14.02165) + | > current_lr: 0.00004 + | > step_time: 2.18610 (2.17845) + | > loader_time: 0.00390 (0.10643) + + + --> STEP: 119/234 -- GLOBAL_STEP: 37325 + | > loss: -0.15400 (-0.14268) + | > log_mle: -0.32738 (-0.28194) + | > loss_dur: 0.17339 (0.13926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.89506 (14.50411) + | > current_lr: 0.00004 + | > step_time: 1.20360 (2.17246) + | > loader_time: 0.00400 (0.10360) + + + --> STEP: 124/234 -- GLOBAL_STEP: 37330 + | > loss: -0.18338 (-0.14305) + | > log_mle: -0.35653 (-0.28352) + | > loss_dur: 0.17315 (0.14047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.57227 (14.73246) + | > current_lr: 0.00004 + | > step_time: 2.10910 (2.18583) + | > loader_time: 0.00380 (0.10100) + + + --> STEP: 129/234 -- GLOBAL_STEP: 37335 + | > loss: -0.15869 (-0.14451) + | > log_mle: -0.34689 (-0.28673) + | > loss_dur: 0.18820 (0.14223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.29479 (15.57562) + | > current_lr: 0.00004 + | > step_time: 2.20620 (2.17400) + | > loader_time: 0.08960 (0.09853) + + + --> STEP: 134/234 -- GLOBAL_STEP: 37340 + | > loss: -0.18326 (-0.14658) + | > log_mle: -0.39924 (-0.29064) + | > loss_dur: 0.21598 (0.14406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.04289 (16.48047) + | > current_lr: 0.00004 + | > step_time: 4.71190 (2.20655) + | > loader_time: 0.08500 (0.09766) + + + --> STEP: 139/234 -- GLOBAL_STEP: 37345 + | > loss: -0.26096 (-0.14834) + | > log_mle: -0.46311 (-0.29428) + | > loss_dur: 0.20215 (0.14594) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.91330 (17.39222) + | > current_lr: 0.00004 + | > step_time: 2.61490 (2.20042) + | > loader_time: 0.08340 (0.09548) + + + --> STEP: 144/234 -- GLOBAL_STEP: 37350 + | > loss: -0.24018 (-0.15022) + | > log_mle: -0.43846 (-0.29820) + | > loss_dur: 0.19828 (0.14799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.22143 (18.30900) + | > current_lr: 0.00004 + | > step_time: 1.20350 (2.18520) + | > loader_time: 0.00210 (0.09327) + + + --> STEP: 149/234 -- GLOBAL_STEP: 37355 + | > loss: -0.26501 (-0.15303) + | > log_mle: -0.47430 (-0.30256) + | > loss_dur: 0.20929 (0.14954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.27546 (19.36956) + | > current_lr: 0.00004 + | > step_time: 1.59970 (2.18436) + | > loader_time: 0.00250 (0.09024) + + + --> STEP: 154/234 -- GLOBAL_STEP: 37360 + | > loss: -0.24405 (-0.15607) + | > log_mle: -0.43951 (-0.30734) + | > loss_dur: 0.19546 (0.15127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.57213 (20.80740) + | > current_lr: 0.00004 + | > step_time: 2.30990 (2.20765) + | > loader_time: 0.08330 (0.08853) + + + --> STEP: 159/234 -- GLOBAL_STEP: 37365 + | > loss: -0.24848 (-0.15879) + | > log_mle: -0.45990 (-0.31194) + | > loss_dur: 0.21142 (0.15314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.47030 (22.04660) + | > current_lr: 0.00004 + | > step_time: 12.60250 (2.29946) + | > loader_time: 0.39260 (0.08940) + + + --> STEP: 164/234 -- GLOBAL_STEP: 37370 + | > loss: -0.22923 (-0.16145) + | > log_mle: -0.44836 (-0.31630) + | > loss_dur: 0.21913 (0.15485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.91126 (23.18524) + | > current_lr: 0.00004 + | > step_time: 1.60700 (2.31232) + | > loader_time: 0.09600 (0.08848) + + + --> STEP: 169/234 -- GLOBAL_STEP: 37375 + | > loss: -0.23640 (-0.16428) + | > log_mle: -0.45143 (-0.32095) + | > loss_dur: 0.21503 (0.15666) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.46603 (24.18996) + | > current_lr: 0.00004 + | > step_time: 2.60850 (2.32575) + | > loader_time: 0.00530 (0.08748) + + + --> STEP: 174/234 -- GLOBAL_STEP: 37380 + | > loss: -0.31773 (-0.16811) + | > log_mle: -0.54173 (-0.32672) + | > loss_dur: 0.22401 (0.15862) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.81449 (25.38765) + | > current_lr: 0.00004 + | > step_time: 2.30510 (2.35050) + | > loader_time: 0.00290 (0.08665) + + + --> STEP: 179/234 -- GLOBAL_STEP: 37385 + | > loss: -0.28013 (-0.17123) + | > log_mle: -0.53645 (-0.33189) + | > loss_dur: 0.25632 (0.16066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.01128 (26.58172) + | > current_lr: 0.00004 + | > step_time: 1.80320 (2.35523) + | > loader_time: 0.09050 (0.08526) + + + --> STEP: 184/234 -- GLOBAL_STEP: 37390 + | > loss: -0.26635 (-0.17396) + | > log_mle: -0.49498 (-0.33649) + | > loss_dur: 0.22863 (0.16252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.06335 (27.84175) + | > current_lr: 0.00004 + | > step_time: 2.80350 (2.38213) + | > loader_time: 0.00500 (0.08412) + + + --> STEP: 189/234 -- GLOBAL_STEP: 37395 + | > loss: -0.27105 (-0.17684) + | > log_mle: -0.50211 (-0.34127) + | > loss_dur: 0.23106 (0.16442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.80879 (29.00734) + | > current_lr: 0.00004 + | > step_time: 2.60980 (2.48870) + | > loader_time: 0.09520 (0.08497) + + + --> STEP: 194/234 -- GLOBAL_STEP: 37400 + | > loss: -0.31732 (-0.18030) + | > log_mle: -0.53819 (-0.34614) + | > loss_dur: 0.22086 (0.16584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.19093 (30.29336) + | > current_lr: 0.00004 + | > step_time: 2.29630 (2.49160) + | > loader_time: 0.00340 (0.08387) + + + --> STEP: 199/234 -- GLOBAL_STEP: 37405 + | > loss: -0.29767 (-0.18317) + | > log_mle: -0.53654 (-0.35064) + | > loss_dur: 0.23888 (0.16747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.25748 (31.55650) + | > current_lr: 0.00004 + | > step_time: 3.58070 (2.59710) + | > loader_time: 0.00470 (0.08333) + + + --> STEP: 204/234 -- GLOBAL_STEP: 37410 + | > loss: -0.32733 (-0.18575) + | > log_mle: -0.57482 (-0.35498) + | > loss_dur: 0.24748 (0.16923) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.95589 (32.65545) + | > current_lr: 0.00004 + | > step_time: 3.58800 (2.59178) + | > loader_time: 0.00380 (0.08426) + + + --> STEP: 209/234 -- GLOBAL_STEP: 37415 + | > loss: -0.30003 (-0.18883) + | > log_mle: -0.53635 (-0.35974) + | > loss_dur: 0.23632 (0.17091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.28474 (33.91854) + | > current_lr: 0.00004 + | > step_time: 4.99870 (2.62984) + | > loader_time: 0.00670 (0.08326) + + + --> STEP: 214/234 -- GLOBAL_STEP: 37420 + | > loss: -0.34260 (-0.19263) + | > log_mle: -0.56263 (-0.36529) + | > loss_dur: 0.22003 (0.17266) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 122.78660 (35.64734) + | > current_lr: 0.00004 + | > step_time: 4.70140 (2.70588) + | > loader_time: 0.00280 (0.08316) + + + --> STEP: 219/234 -- GLOBAL_STEP: 37425 + | > loss: -0.42162 (-0.19617) + | > log_mle: -0.66389 (-0.37054) + | > loss_dur: 0.24228 (0.17437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.36242 (37.01845) + | > current_lr: 0.00004 + | > step_time: 2.40310 (2.75649) + | > loader_time: 0.00300 (0.08480) + + + --> STEP: 224/234 -- GLOBAL_STEP: 37430 + | > loss: -0.37024 (-0.19950) + | > log_mle: -0.62184 (-0.37554) + | > loss_dur: 0.25160 (0.17603) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.39191 (38.39484) + | > current_lr: 0.00004 + | > step_time: 0.35200 (2.73626) + | > loader_time: 0.00300 (0.08383) + + + --> STEP: 229/234 -- GLOBAL_STEP: 37435 + | > loss: -0.34409 (-0.20294) + | > log_mle: -0.65649 (-0.38114) + | > loss_dur: 0.31240 (0.17820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.65519 (39.76386) + | > current_lr: 0.00004 + | > step_time: 0.24390 (2.68175) + | > loader_time: 0.00450 (0.08208) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.52679 (+0.50225) + | > avg_loss: -0.19359 (+0.00457) + | > avg_log_mle: -0.43301 (+0.00284) + | > avg_loss_dur: 0.23942 (+0.00173) + + + > EPOCH: 160/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 18:37:07)  + + --> STEP: 0/234 -- GLOBAL_STEP: 37440 + | > loss: -0.18281 (-0.18281) + | > log_mle: -0.34519 (-0.34519) + | > loss_dur: 0.16238 (0.16238) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.93367 (18.93367) + | > current_lr: 0.00004 + | > step_time: 12.59320 (12.59317) + | > loader_time: 5.10650 (5.10649) + + + --> STEP: 5/234 -- GLOBAL_STEP: 37445 + | > loss: -0.15615 (-0.13402) + | > log_mle: -0.27209 (-0.26999) + | > loss_dur: 0.11595 (0.13597) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.39071 (18.30973) + | > current_lr: 0.00004 + | > step_time: 4.71070 (4.34068) + | > loader_time: 0.09090 (0.71967) + + + --> STEP: 10/234 -- GLOBAL_STEP: 37450 + | > loss: -0.13437 (-0.14206) + | > log_mle: -0.26881 (-0.27233) + | > loss_dur: 0.13443 (0.13027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.93558 (16.76767) + | > current_lr: 0.00004 + | > step_time: 3.20620 (4.58063) + | > loader_time: 0.08990 (0.39984) + + + --> STEP: 15/234 -- GLOBAL_STEP: 37455 + | > loss: -0.16792 (-0.14942) + | > log_mle: -0.27156 (-0.27282) + | > loss_dur: 0.10364 (0.12341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.42181 (15.13269) + | > current_lr: 0.00004 + | > step_time: 1.09160 (4.71323) + | > loader_time: 0.00130 (0.29281) + + + --> STEP: 20/234 -- GLOBAL_STEP: 37460 + | > loss: -0.15196 (-0.14979) + | > log_mle: -0.26183 (-0.27002) + | > loss_dur: 0.10987 (0.12023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.02667 (13.73439) + | > current_lr: 0.00004 + | > step_time: 3.99890 (4.34469) + | > loader_time: 0.09950 (0.22948) + + + --> STEP: 25/234 -- GLOBAL_STEP: 37465 + | > loss: -0.14446 (-0.15090) + | > log_mle: -0.25238 (-0.26811) + | > loss_dur: 0.10792 (0.11721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.80930 (12.93901) + | > current_lr: 0.00004 + | > step_time: 1.41030 (3.95790) + | > loader_time: 0.08190 (0.18714) + + + --> STEP: 30/234 -- GLOBAL_STEP: 37470 + | > loss: -0.16684 (-0.15164) + | > log_mle: -0.26953 (-0.26758) + | > loss_dur: 0.10268 (0.11593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.77326 (12.48590) + | > current_lr: 0.00004 + | > step_time: 3.80750 (3.71167) + | > loader_time: 0.09330 (0.16211) + + + --> STEP: 35/234 -- GLOBAL_STEP: 37475 + | > loss: -0.13279 (-0.14978) + | > log_mle: -0.26030 (-0.26686) + | > loss_dur: 0.12752 (0.11708) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.58941 (12.48310) + | > current_lr: 0.00004 + | > step_time: 1.50290 (3.40086) + | > loader_time: 0.00180 (0.14443) + + + --> STEP: 40/234 -- GLOBAL_STEP: 37480 + | > loss: -0.10447 (-0.14817) + | > log_mle: -0.24667 (-0.26612) + | > loss_dur: 0.14220 (0.11795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.45029 (12.50938) + | > current_lr: 0.00004 + | > step_time: 1.80500 (3.18335) + | > loader_time: 0.00180 (0.12910) + + + --> STEP: 45/234 -- GLOBAL_STEP: 37485 + | > loss: -0.13066 (-0.14709) + | > log_mle: -0.28438 (-0.26617) + | > loss_dur: 0.15372 (0.11908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.81430 (12.21088) + | > current_lr: 0.00004 + | > step_time: 0.99100 (2.97115) + | > loader_time: 0.00260 (0.12092) + + + --> STEP: 50/234 -- GLOBAL_STEP: 37490 + | > loss: -0.12720 (-0.14587) + | > log_mle: -0.25370 (-0.26543) + | > loss_dur: 0.12651 (0.11955) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.33695 (11.93699) + | > current_lr: 0.00004 + | > step_time: 2.67550 (2.85704) + | > loader_time: 0.00200 (0.10903) + + + --> STEP: 55/234 -- GLOBAL_STEP: 37495 + | > loss: -0.16961 (-0.14543) + | > log_mle: -0.27401 (-0.26539) + | > loss_dur: 0.10440 (0.11996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.27794 (11.77669) + | > current_lr: 0.00004 + | > step_time: 2.02080 (2.79982) + | > loader_time: 0.00350 (0.10097) + + + --> STEP: 60/234 -- GLOBAL_STEP: 37500 + | > loss: -0.13683 (-0.14458) + | > log_mle: -0.28611 (-0.26570) + | > loss_dur: 0.14928 (0.12111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.95813 (11.65310) + | > current_lr: 0.00004 + | > step_time: 1.30850 (2.70401) + | > loader_time: 0.00270 (0.09278) + + + --> STEP: 65/234 -- GLOBAL_STEP: 37505 + | > loss: -0.14249 (-0.14315) + | > log_mle: -0.26337 (-0.26599) + | > loss_dur: 0.12088 (0.12284) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.95434 (11.61255) + | > current_lr: 0.00004 + | > step_time: 1.40120 (2.60644) + | > loader_time: 0.00290 (0.08586) + + + --> STEP: 70/234 -- GLOBAL_STEP: 37510 + | > loss: -0.10414 (-0.14153) + | > log_mle: -0.25550 (-0.26549) + | > loss_dur: 0.15136 (0.12396) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.41006 (11.53046) + | > current_lr: 0.00004 + | > step_time: 2.00950 (2.55051) + | > loader_time: 0.08580 (0.08347) + + + --> STEP: 75/234 -- GLOBAL_STEP: 37515 + | > loss: -0.12271 (-0.14041) + | > log_mle: -0.27689 (-0.26601) + | > loss_dur: 0.15419 (0.12561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.23891 (11.56972) + | > current_lr: 0.00004 + | > step_time: 4.50390 (2.54229) + | > loader_time: 0.00220 (0.07924) + + + --> STEP: 80/234 -- GLOBAL_STEP: 37520 + | > loss: -0.13943 (-0.13989) + | > log_mle: -0.25650 (-0.26604) + | > loss_dur: 0.11707 (0.12615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.43834 (11.46780) + | > current_lr: 0.00004 + | > step_time: 1.71730 (2.51780) + | > loader_time: 0.00230 (0.07555) + + + --> STEP: 85/234 -- GLOBAL_STEP: 37525 + | > loss: -0.13844 (-0.13924) + | > log_mle: -0.26993 (-0.26642) + | > loss_dur: 0.13149 (0.12718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.88492 (11.45258) + | > current_lr: 0.00004 + | > step_time: 2.19110 (2.47642) + | > loader_time: 0.10850 (0.07353) + + + --> STEP: 90/234 -- GLOBAL_STEP: 37530 + | > loss: -0.12940 (-0.13923) + | > log_mle: -0.29495 (-0.26772) + | > loss_dur: 0.16555 (0.12849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.61535 (11.80491) + | > current_lr: 0.00004 + | > step_time: 1.78030 (2.44130) + | > loader_time: 0.00260 (0.07137) + + + --> STEP: 95/234 -- GLOBAL_STEP: 37535 + | > loss: -0.19619 (-0.14042) + | > log_mle: -0.37434 (-0.27081) + | > loss_dur: 0.17816 (0.13039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.45138 (12.40117) + | > current_lr: 0.00004 + | > step_time: 1.36510 (2.40081) + | > loader_time: 0.00270 (0.06775) + + + --> STEP: 100/234 -- GLOBAL_STEP: 37540 + | > loss: -0.15330 (-0.14059) + | > log_mle: -0.30568 (-0.27227) + | > loss_dur: 0.15239 (0.13168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.19069 (12.74653) + | > current_lr: 0.00004 + | > step_time: 3.19820 (2.38814) + | > loader_time: 0.00170 (0.06448) + + + --> STEP: 105/234 -- GLOBAL_STEP: 37545 + | > loss: -0.14240 (-0.14154) + | > log_mle: -0.28382 (-0.27485) + | > loss_dur: 0.14142 (0.13331) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.33025 (13.25373) + | > current_lr: 0.00004 + | > step_time: 1.30280 (2.34972) + | > loader_time: 0.09030 (0.06242) + + + --> STEP: 110/234 -- GLOBAL_STEP: 37550 + | > loss: -0.14169 (-0.14169) + | > log_mle: -0.30791 (-0.27705) + | > loss_dur: 0.16623 (0.13536) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.72268 (13.70942) + | > current_lr: 0.00004 + | > step_time: 2.09390 (2.32088) + | > loader_time: 0.00460 (0.06047) + + + --> STEP: 115/234 -- GLOBAL_STEP: 37555 + | > loss: -0.15044 (-0.14260) + | > log_mle: -0.33067 (-0.28008) + | > loss_dur: 0.18023 (0.13748) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.15359 (14.29728) + | > current_lr: 0.00004 + | > step_time: 2.59950 (2.31469) + | > loader_time: 0.00580 (0.05804) + + + --> STEP: 120/234 -- GLOBAL_STEP: 37560 + | > loss: -0.19085 (-0.14336) + | > log_mle: -0.37947 (-0.28266) + | > loss_dur: 0.18862 (0.13930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.84918 (14.86875) + | > current_lr: 0.00004 + | > step_time: 1.99830 (2.34823) + | > loader_time: 0.00330 (0.05582) + + + --> STEP: 125/234 -- GLOBAL_STEP: 37565 + | > loss: -0.17741 (-0.14361) + | > log_mle: -0.36650 (-0.28407) + | > loss_dur: 0.18909 (0.14046) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.25125 (15.15505) + | > current_lr: 0.00004 + | > step_time: 1.80430 (2.35037) + | > loader_time: 0.00340 (0.05667) + + + --> STEP: 130/234 -- GLOBAL_STEP: 37570 + | > loss: -0.19558 (-0.14518) + | > log_mle: -0.37997 (-0.28734) + | > loss_dur: 0.18439 (0.14216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.24295 (15.93648) + | > current_lr: 0.00004 + | > step_time: 1.90500 (2.34394) + | > loader_time: 0.18450 (0.05665) + + + --> STEP: 135/234 -- GLOBAL_STEP: 37575 + | > loss: -0.14854 (-0.14689) + | > log_mle: -0.30531 (-0.29050) + | > loss_dur: 0.15678 (0.14361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.20862 (16.79479) + | > current_lr: 0.00004 + | > step_time: 1.79750 (2.32670) + | > loader_time: 0.08990 (0.05600) + + + --> STEP: 140/234 -- GLOBAL_STEP: 37580 + | > loss: -0.15320 (-0.14862) + | > log_mle: -0.34227 (-0.29420) + | > loss_dur: 0.18907 (0.14558) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.85708 (17.65293) + | > current_lr: 0.00004 + | > step_time: 2.70670 (2.34589) + | > loader_time: 0.00350 (0.05550) + + + --> STEP: 145/234 -- GLOBAL_STEP: 37585 + | > loss: -0.23915 (-0.15076) + | > log_mle: -0.44511 (-0.29861) + | > loss_dur: 0.20596 (0.14784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.58274 (18.67790) + | > current_lr: 0.00004 + | > step_time: 4.89960 (2.44011) + | > loader_time: 0.00270 (0.05502) + + + --> STEP: 150/234 -- GLOBAL_STEP: 37590 + | > loss: -0.22321 (-0.15325) + | > log_mle: -0.42647 (-0.30267) + | > loss_dur: 0.20326 (0.14941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.75482 (19.78221) + | > current_lr: 0.00004 + | > step_time: 1.70470 (2.41088) + | > loader_time: 0.00350 (0.05642) + + + --> STEP: 155/234 -- GLOBAL_STEP: 37595 + | > loss: -0.27262 (-0.15674) + | > log_mle: -0.48650 (-0.30783) + | > loss_dur: 0.21388 (0.15109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.40443 (21.00772) + | > current_lr: 0.00004 + | > step_time: 1.90550 (2.40272) + | > loader_time: 0.08720 (0.05640) + + + --> STEP: 160/234 -- GLOBAL_STEP: 37600 + | > loss: -0.27727 (-0.15934) + | > log_mle: -0.48954 (-0.31231) + | > loss_dur: 0.21227 (0.15297) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.94392 (22.14417) + | > current_lr: 0.00004 + | > step_time: 3.11000 (2.40586) + | > loader_time: 0.00450 (0.05535) + + + --> STEP: 165/234 -- GLOBAL_STEP: 37605 + | > loss: -0.27149 (-0.16205) + | > log_mle: -0.48852 (-0.31666) + | > loss_dur: 0.21703 (0.15461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.25114 (23.14031) + | > current_lr: 0.00004 + | > step_time: 3.21440 (2.42143) + | > loader_time: 0.09860 (0.05495) + + + --> STEP: 170/234 -- GLOBAL_STEP: 37610 + | > loss: -0.28944 (-0.16501) + | > log_mle: -0.52554 (-0.32144) + | > loss_dur: 0.23610 (0.15642) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.92558 (24.33083) + | > current_lr: 0.00004 + | > step_time: 1.30220 (2.40487) + | > loader_time: 0.00300 (0.05484) + + + --> STEP: 175/234 -- GLOBAL_STEP: 37615 + | > loss: -0.25428 (-0.16839) + | > log_mle: -0.48884 (-0.32680) + | > loss_dur: 0.23456 (0.15840) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.85263 (26.04492) + | > current_lr: 0.00004 + | > step_time: 2.69370 (2.40939) + | > loader_time: 0.00470 (0.05439) + + + --> STEP: 180/234 -- GLOBAL_STEP: 37620 + | > loss: -0.27494 (-0.17125) + | > log_mle: -0.49280 (-0.33165) + | > loss_dur: 0.21786 (0.16040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.62829 (27.68066) + | > current_lr: 0.00004 + | > step_time: 2.80360 (2.44024) + | > loader_time: 0.08920 (0.05449) + + + --> STEP: 185/234 -- GLOBAL_STEP: 37625 + | > loss: -0.28422 (-0.17378) + | > log_mle: -0.52559 (-0.33614) + | > loss_dur: 0.24137 (0.16236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.52154 (28.99826) + | > current_lr: 0.00004 + | > step_time: 2.00000 (2.44409) + | > loader_time: 0.00410 (0.05367) + + + --> STEP: 190/234 -- GLOBAL_STEP: 37630 + | > loss: -0.28795 (-0.17653) + | > log_mle: -0.50231 (-0.34065) + | > loss_dur: 0.21436 (0.16412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.81340 (29.96956) + | > current_lr: 0.00004 + | > step_time: 6.79130 (2.56139) + | > loader_time: 0.29740 (0.05683) + + + --> STEP: 195/234 -- GLOBAL_STEP: 37635 + | > loss: -0.27294 (-0.17979) + | > log_mle: -0.51722 (-0.34548) + | > loss_dur: 0.24428 (0.16569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.22003 (31.05709) + | > current_lr: 0.00004 + | > step_time: 6.08410 (2.62216) + | > loader_time: 0.00930 (0.05700) + + + --> STEP: 200/234 -- GLOBAL_STEP: 37640 + | > loss: -0.27878 (-0.18229) + | > log_mle: -0.53372 (-0.34968) + | > loss_dur: 0.25494 (0.16739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.91320 (32.55468) + | > current_lr: 0.00004 + | > step_time: 10.09880 (2.67619) + | > loader_time: 0.20060 (0.05803) + + + --> STEP: 205/234 -- GLOBAL_STEP: 37645 + | > loss: -0.25541 (-0.18435) + | > log_mle: -0.50079 (-0.35350) + | > loss_dur: 0.24538 (0.16915) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.08913 (33.92916) + | > current_lr: 0.00004 + | > step_time: 3.40280 (2.72721) + | > loader_time: 0.00430 (0.05906) + + + --> STEP: 210/234 -- GLOBAL_STEP: 37650 + | > loss: -0.32850 (-0.18723) + | > log_mle: -0.57950 (-0.35819) + | > loss_dur: 0.25100 (0.17095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.02737 (34.98515) + | > current_lr: 0.00004 + | > step_time: 14.29620 (2.81027) + | > loader_time: 0.10440 (0.06015) + + + --> STEP: 215/234 -- GLOBAL_STEP: 37655 + | > loss: -0.29375 (-0.19021) + | > log_mle: -0.53987 (-0.36292) + | > loss_dur: 0.24612 (0.17271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.11250 (36.04732) + | > current_lr: 0.00004 + | > step_time: 5.00150 (2.86390) + | > loader_time: 0.00480 (0.06020) + + + --> STEP: 220/234 -- GLOBAL_STEP: 37660 + | > loss: -0.34485 (-0.19374) + | > log_mle: -0.59032 (-0.36822) + | > loss_dur: 0.24548 (0.17448) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.20875 (37.01062) + | > current_lr: 0.00004 + | > step_time: 2.78260 (2.86828) + | > loader_time: 0.00310 (0.06505) + + + --> STEP: 225/234 -- GLOBAL_STEP: 37665 + | > loss: -0.39879 (-0.19708) + | > log_mle: -0.66156 (-0.37334) + | > loss_dur: 0.26277 (0.17627) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.83728 (38.05046) + | > current_lr: 0.00004 + | > step_time: 0.24340 (2.86121) + | > loader_time: 0.00410 (0.06536) + + + --> STEP: 230/234 -- GLOBAL_STEP: 37670 + | > loss: -0.35363 (-0.20023) + | > log_mle: -0.68888 (-0.37892) + | > loss_dur: 0.33525 (0.17869) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 163.77011 (39.49520) + | > current_lr: 0.00004 + | > step_time: 0.26530 (2.80447) + | > loader_time: 0.00900 (0.06406) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.29893 (-0.22786) + | > avg_loss: -0.18042 (+0.01317) + | > avg_log_mle: -0.42047 (+0.01253) + | > avg_loss_dur: 0.24006 (+0.00064) + + + > EPOCH: 161/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 18:49:12)  + + --> STEP: 1/234 -- GLOBAL_STEP: 37675 + | > loss: -0.14528 (-0.14528) + | > log_mle: -0.26858 (-0.26858) + | > loss_dur: 0.12330 (0.12330) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.05280 (25.05280) + | > current_lr: 0.00004 + | > step_time: 8.09390 (8.09392) + | > loader_time: 0.00110 (0.00105) + + + --> STEP: 6/234 -- GLOBAL_STEP: 37680 + | > loss: -0.15371 (-0.14057) + | > log_mle: -0.26667 (-0.26971) + | > loss_dur: 0.11296 (0.12914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.15684 (16.93273) + | > current_lr: 0.00004 + | > step_time: 0.60760 (4.90240) + | > loader_time: 0.00300 (0.06330) + + + --> STEP: 11/234 -- GLOBAL_STEP: 37685 + | > loss: -0.17110 (-0.14784) + | > log_mle: -0.27434 (-0.27318) + | > loss_dur: 0.10323 (0.12534) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.85115 (15.34522) + | > current_lr: 0.00004 + | > step_time: 2.10360 (3.33781) + | > loader_time: 0.07410 (0.04290) + + + --> STEP: 16/234 -- GLOBAL_STEP: 37690 + | > loss: -0.16048 (-0.15071) + | > log_mle: -0.26805 (-0.27342) + | > loss_dur: 0.10758 (0.12271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.84575 (13.54311) + | > current_lr: 0.00004 + | > step_time: 3.09120 (2.78717) + | > loader_time: 0.00170 (0.03014) + + + --> STEP: 21/234 -- GLOBAL_STEP: 37695 + | > loss: -0.13848 (-0.14987) + | > log_mle: -0.24971 (-0.26989) + | > loss_dur: 0.11123 (0.12001) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.75657 (12.77716) + | > current_lr: 0.00004 + | > step_time: 4.40540 (3.53342) + | > loader_time: 0.00240 (0.02777) + + + --> STEP: 26/234 -- GLOBAL_STEP: 37700 + | > loss: -0.14267 (-0.15199) + | > log_mle: -0.26622 (-0.26904) + | > loss_dur: 0.12356 (0.11705) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.38022 (12.16211) + | > current_lr: 0.00004 + | > step_time: 2.80610 (3.54975) + | > loader_time: 0.08290 (0.03018) + + + --> STEP: 31/234 -- GLOBAL_STEP: 37705 + | > loss: -0.10647 (-0.15161) + | > log_mle: -0.26236 (-0.26852) + | > loss_dur: 0.15590 (0.11691) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.13305 (11.86440) + | > current_lr: 0.00004 + | > step_time: 2.00360 (3.76143) + | > loader_time: 0.10150 (0.04073) + + + --> STEP: 36/234 -- GLOBAL_STEP: 37710 + | > loss: -0.13108 (-0.15055) + | > log_mle: -0.26438 (-0.26801) + | > loss_dur: 0.13330 (0.11746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.01607 (11.78774) + | > current_lr: 0.00004 + | > step_time: 3.71410 (3.58936) + | > loader_time: 0.09350 (0.04039) + + + --> STEP: 41/234 -- GLOBAL_STEP: 37715 + | > loss: -0.15203 (-0.14959) + | > log_mle: -0.26614 (-0.26747) + | > loss_dur: 0.11411 (0.11789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.43714 (11.53738) + | > current_lr: 0.00004 + | > step_time: 1.69820 (3.41478) + | > loader_time: 0.00200 (0.04272) + + + --> STEP: 46/234 -- GLOBAL_STEP: 37720 + | > loss: -0.12610 (-0.14844) + | > log_mle: -0.26179 (-0.26746) + | > loss_dur: 0.13569 (0.11903) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67379 (11.41706) + | > current_lr: 0.00004 + | > step_time: 1.09310 (3.25406) + | > loader_time: 0.00190 (0.03835) + + + --> STEP: 51/234 -- GLOBAL_STEP: 37725 + | > loss: -0.12881 (-0.14740) + | > log_mle: -0.25446 (-0.26675) + | > loss_dur: 0.12565 (0.11935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.40982 (10.97593) + | > current_lr: 0.00004 + | > step_time: 1.42190 (3.09065) + | > loader_time: 0.00450 (0.03653) + + + --> STEP: 56/234 -- GLOBAL_STEP: 37730 + | > loss: -0.12304 (-0.14680) + | > log_mle: -0.27143 (-0.26710) + | > loss_dur: 0.14838 (0.12030) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.30016 (10.84098) + | > current_lr: 0.00004 + | > step_time: 2.10600 (2.93650) + | > loader_time: 0.00230 (0.03349) + + + --> STEP: 61/234 -- GLOBAL_STEP: 37735 + | > loss: -0.14830 (-0.14673) + | > log_mle: -0.26799 (-0.26737) + | > loss_dur: 0.11969 (0.12064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.28258 (10.82676) + | > current_lr: 0.00004 + | > step_time: 1.90370 (2.86965) + | > loader_time: 0.08540 (0.03507) + + + --> STEP: 66/234 -- GLOBAL_STEP: 37740 + | > loss: -0.13619 (-0.14545) + | > log_mle: -0.25632 (-0.26752) + | > loss_dur: 0.12013 (0.12207) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.89116 (10.99237) + | > current_lr: 0.00004 + | > step_time: 1.56660 (2.86744) + | > loader_time: 0.00210 (0.03692) + + + --> STEP: 71/234 -- GLOBAL_STEP: 37745 + | > loss: -0.11699 (-0.14385) + | > log_mle: -0.29497 (-0.26764) + | > loss_dur: 0.17798 (0.12379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.64639 (11.16227) + | > current_lr: 0.00004 + | > step_time: 3.70120 (2.79893) + | > loader_time: 0.00310 (0.03452) + + + --> STEP: 76/234 -- GLOBAL_STEP: 37750 + | > loss: -0.13526 (-0.14277) + | > log_mle: -0.27919 (-0.26798) + | > loss_dur: 0.14393 (0.12521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.05192 (11.18814) + | > current_lr: 0.00004 + | > step_time: 1.72200 (2.78765) + | > loader_time: 0.00260 (0.03669) + + + --> STEP: 81/234 -- GLOBAL_STEP: 37755 + | > loss: -0.13994 (-0.14245) + | > log_mle: -0.28521 (-0.26812) + | > loss_dur: 0.14527 (0.12567) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.45399 (11.13121) + | > current_lr: 0.00004 + | > step_time: 1.20510 (2.69835) + | > loader_time: 0.00280 (0.03461) + + + --> STEP: 86/234 -- GLOBAL_STEP: 37760 + | > loss: -0.13406 (-0.14177) + | > log_mle: -0.28387 (-0.26850) + | > loss_dur: 0.14980 (0.12673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.79200 (11.19287) + | > current_lr: 0.00004 + | > step_time: 1.19730 (2.64361) + | > loader_time: 0.00270 (0.03381) + + + --> STEP: 91/234 -- GLOBAL_STEP: 37765 + | > loss: -0.13000 (-0.14170) + | > log_mle: -0.29515 (-0.27010) + | > loss_dur: 0.16516 (0.12840) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.34135 (11.41480) + | > current_lr: 0.00004 + | > step_time: 2.10790 (2.61361) + | > loader_time: 0.08570 (0.03525) + + + --> STEP: 96/234 -- GLOBAL_STEP: 37770 + | > loss: -0.13963 (-0.14327) + | > log_mle: -0.28148 (-0.27327) + | > loss_dur: 0.14185 (0.13000) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.87490 (12.07074) + | > current_lr: 0.00004 + | > step_time: 3.89250 (2.59034) + | > loader_time: 0.00240 (0.03436) + + + --> STEP: 101/234 -- GLOBAL_STEP: 37775 + | > loss: -0.16680 (-0.14395) + | > log_mle: -0.33932 (-0.27541) + | > loss_dur: 0.17252 (0.13146) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.28959 (12.56165) + | > current_lr: 0.00004 + | > step_time: 1.74510 (2.54451) + | > loader_time: 0.08280 (0.03360) + + + --> STEP: 106/234 -- GLOBAL_STEP: 37780 + | > loss: -0.14145 (-0.14468) + | > log_mle: -0.33744 (-0.27808) + | > loss_dur: 0.19599 (0.13340) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.50144 (13.18378) + | > current_lr: 0.00004 + | > step_time: 1.79520 (2.49179) + | > loader_time: 0.00300 (0.03290) + + + --> STEP: 111/234 -- GLOBAL_STEP: 37785 + | > loss: -0.16931 (-0.14506) + | > log_mle: -0.37976 (-0.28064) + | > loss_dur: 0.21045 (0.13558) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.73454 (13.85460) + | > current_lr: 0.00004 + | > step_time: 1.69740 (2.46252) + | > loader_time: 0.00290 (0.03238) + + + --> STEP: 116/234 -- GLOBAL_STEP: 37790 + | > loss: -0.13634 (-0.14566) + | > log_mle: -0.34612 (-0.28324) + | > loss_dur: 0.20978 (0.13758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.62026 (14.52791) + | > current_lr: 0.00004 + | > step_time: 1.81130 (2.48328) + | > loader_time: 0.08800 (0.03345) + + + --> STEP: 121/234 -- GLOBAL_STEP: 37795 + | > loss: -0.11911 (-0.14607) + | > log_mle: -0.26171 (-0.28500) + | > loss_dur: 0.14260 (0.13893) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.06664 (14.92821) + | > current_lr: 0.00004 + | > step_time: 2.09910 (2.48725) + | > loader_time: 0.08500 (0.03522) + + + --> STEP: 126/234 -- GLOBAL_STEP: 37800 + | > loss: -0.21090 (-0.14713) + | > log_mle: -0.40145 (-0.28737) + | > loss_dur: 0.19056 (0.14025) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.57667 (15.45605) + | > current_lr: 0.00004 + | > step_time: 2.01160 (2.50309) + | > loader_time: 0.00880 (0.03780) + + + --> STEP: 131/234 -- GLOBAL_STEP: 37805 + | > loss: -0.23415 (-0.14875) + | > log_mle: -0.44077 (-0.29081) + | > loss_dur: 0.20662 (0.14206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.91724 (16.41648) + | > current_lr: 0.00004 + | > step_time: 2.07050 (2.50231) + | > loader_time: 0.00290 (0.03722) + + + --> STEP: 136/234 -- GLOBAL_STEP: 37810 + | > loss: -0.26790 (-0.15051) + | > log_mle: -0.48999 (-0.29429) + | > loss_dur: 0.22209 (0.14378) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.65516 (17.24277) + | > current_lr: 0.00004 + | > step_time: 1.80190 (2.48308) + | > loader_time: 0.08730 (0.03657) + + + --> STEP: 141/234 -- GLOBAL_STEP: 37815 + | > loss: -0.19069 (-0.15167) + | > log_mle: -0.39228 (-0.29726) + | > loss_dur: 0.20159 (0.14559) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.49279 (18.11465) + | > current_lr: 0.00004 + | > step_time: 3.92070 (2.53779) + | > loader_time: 0.09140 (0.03794) + + + --> STEP: 146/234 -- GLOBAL_STEP: 37820 + | > loss: -0.23637 (-0.15406) + | > log_mle: -0.44447 (-0.30194) + | > loss_dur: 0.20810 (0.14789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.47382 (19.36732) + | > current_lr: 0.00004 + | > step_time: 4.63870 (2.55587) + | > loader_time: 0.20150 (0.03957) + + + --> STEP: 151/234 -- GLOBAL_STEP: 37825 + | > loss: -0.22295 (-0.15635) + | > log_mle: -0.41159 (-0.30578) + | > loss_dur: 0.18863 (0.14943) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.80833 (20.32397) + | > current_lr: 0.00004 + | > step_time: 1.69660 (2.52539) + | > loader_time: 0.00360 (0.03901) + + + --> STEP: 156/234 -- GLOBAL_STEP: 37830 + | > loss: -0.24684 (-0.15967) + | > log_mle: -0.44946 (-0.31106) + | > loss_dur: 0.20263 (0.15139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.51447 (21.79886) + | > current_lr: 0.00004 + | > step_time: 2.09610 (2.53236) + | > loader_time: 0.00400 (0.03848) + + + --> STEP: 161/234 -- GLOBAL_STEP: 37835 + | > loss: -0.27073 (-0.16222) + | > log_mle: -0.47112 (-0.31552) + | > loss_dur: 0.20039 (0.15330) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.49092 (22.81240) + | > current_lr: 0.00004 + | > step_time: 1.39120 (2.52806) + | > loader_time: 0.00300 (0.03922) + + + --> STEP: 166/234 -- GLOBAL_STEP: 37840 + | > loss: -0.22794 (-0.16443) + | > log_mle: -0.41269 (-0.31928) + | > loss_dur: 0.18475 (0.15485) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.75768 (23.85987) + | > current_lr: 0.00004 + | > step_time: 3.00010 (2.51574) + | > loader_time: 0.08790 (0.03969) + + + --> STEP: 171/234 -- GLOBAL_STEP: 37845 + | > loss: -0.31246 (-0.16779) + | > log_mle: -0.51735 (-0.32451) + | > loss_dur: 0.20489 (0.15672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.38870 (25.33976) + | > current_lr: 0.00004 + | > step_time: 5.20290 (2.57078) + | > loader_time: 0.00480 (0.03868) + + + --> STEP: 176/234 -- GLOBAL_STEP: 37850 + | > loss: -0.27553 (-0.17105) + | > log_mle: -0.49385 (-0.32970) + | > loss_dur: 0.21832 (0.15865) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.10657 (26.41960) + | > current_lr: 0.00004 + | > step_time: 1.20610 (2.60469) + | > loader_time: 0.00280 (0.04155) + + + --> STEP: 181/234 -- GLOBAL_STEP: 37855 + | > loss: -0.21133 (-0.17361) + | > log_mle: -0.42185 (-0.33424) + | > loss_dur: 0.21052 (0.16062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.42804 (27.56515) + | > current_lr: 0.00004 + | > step_time: 1.70630 (2.61455) + | > loader_time: 0.00390 (0.04094) + + + --> STEP: 186/234 -- GLOBAL_STEP: 37860 + | > loss: -0.21668 (-0.17605) + | > log_mle: -0.46022 (-0.33872) + | > loss_dur: 0.24354 (0.16267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.64100 (28.84175) + | > current_lr: 0.00004 + | > step_time: 10.69850 (2.64858) + | > loader_time: 0.10380 (0.04152) + + + --> STEP: 191/234 -- GLOBAL_STEP: 37865 + | > loss: -0.26887 (-0.17883) + | > log_mle: -0.48757 (-0.34316) + | > loss_dur: 0.21870 (0.16433) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.49054 (29.86338) + | > current_lr: 0.00004 + | > step_time: 4.70760 (2.65992) + | > loader_time: 0.19680 (0.04289) + + + --> STEP: 196/234 -- GLOBAL_STEP: 37870 + | > loss: -0.23087 (-0.18159) + | > log_mle: -0.46891 (-0.34757) + | > loss_dur: 0.23804 (0.16598) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.29363 (31.30003) + | > current_lr: 0.00004 + | > step_time: 3.50020 (2.68289) + | > loader_time: 0.29470 (0.04481) + + + --> STEP: 201/234 -- GLOBAL_STEP: 37875 + | > loss: -0.20580 (-0.18381) + | > log_mle: -0.44038 (-0.35149) + | > loss_dur: 0.23458 (0.16768) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.65009 (32.16619) + | > current_lr: 0.00004 + | > step_time: 5.50540 (2.73711) + | > loader_time: 0.10430 (0.04624) + + + --> STEP: 206/234 -- GLOBAL_STEP: 37880 + | > loss: -0.31180 (-0.18661) + | > log_mle: -0.54503 (-0.35596) + | > loss_dur: 0.23323 (0.16935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.89574 (33.18055) + | > current_lr: 0.00004 + | > step_time: 4.61250 (2.77961) + | > loader_time: 0.08880 (0.04840) + + + --> STEP: 211/234 -- GLOBAL_STEP: 37885 + | > loss: -0.36139 (-0.18995) + | > log_mle: -0.62097 (-0.36111) + | > loss_dur: 0.25958 (0.17116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.28401 (34.50357) + | > current_lr: 0.00004 + | > step_time: 2.41150 (2.81332) + | > loader_time: 0.08910 (0.04919) + + + --> STEP: 216/234 -- GLOBAL_STEP: 37890 + | > loss: -0.33988 (-0.19328) + | > log_mle: -0.60291 (-0.36612) + | > loss_dur: 0.26303 (0.17284) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 107.95153 (35.68084) + | > current_lr: 0.00004 + | > step_time: 13.29460 (2.89775) + | > loader_time: 0.11120 (0.05358) + + + --> STEP: 221/234 -- GLOBAL_STEP: 37895 + | > loss: -0.29143 (-0.19662) + | > log_mle: -0.52174 (-0.37106) + | > loss_dur: 0.23032 (0.17444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.10319 (36.94287) + | > current_lr: 0.00004 + | > step_time: 3.79390 (2.95033) + | > loader_time: 0.00520 (0.05509) + + + --> STEP: 226/234 -- GLOBAL_STEP: 37900 + | > loss: -0.37613 (-0.20032) + | > log_mle: -0.62651 (-0.37655) + | > loss_dur: 0.25038 (0.17623) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.50589 (38.35331) + | > current_lr: 0.00004 + | > step_time: 0.24310 (2.89179) + | > loader_time: 0.00370 (0.05399) + + + --> STEP: 231/234 -- GLOBAL_STEP: 37905 + | > loss: -0.30867 (-0.20306) + | > log_mle: -0.70507 (-0.38242) + | > loss_dur: 0.39640 (0.17936) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.08398 (39.69507) + | > current_lr: 0.00004 + | > step_time: 0.28670 (2.83487) + | > loader_time: 0.00360 (0.05291) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67406 (+0.37513) + | > avg_loss: -0.22094 (-0.04052) + | > avg_log_mle: -0.45851 (-0.03804) + | > avg_loss_dur: 0.23758 (-0.00248) + + + > EPOCH: 162/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:01:34)  + + --> STEP: 2/234 -- GLOBAL_STEP: 37910 + | > loss: -0.16287 (-0.15152) + | > log_mle: -0.27956 (-0.27650) + | > loss_dur: 0.11669 (0.12498) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.69606 (13.35001) + | > current_lr: 0.00004 + | > step_time: 4.60030 (4.25430) + | > loader_time: 0.00200 (0.00310) + + + --> STEP: 7/234 -- GLOBAL_STEP: 37915 + | > loss: -0.16380 (-0.14503) + | > log_mle: -0.28135 (-0.27416) + | > loss_dur: 0.11755 (0.12912) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.66867 (15.17857) + | > current_lr: 0.00004 + | > step_time: 4.79900 (4.03090) + | > loader_time: 0.00380 (0.06827) + + + --> STEP: 12/234 -- GLOBAL_STEP: 37920 + | > loss: -0.16943 (-0.15021) + | > log_mle: -0.27586 (-0.27605) + | > loss_dur: 0.10643 (0.12584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.03895 (14.69002) + | > current_lr: 0.00004 + | > step_time: 7.40150 (4.78711) + | > loader_time: 0.08160 (0.07131) + + + --> STEP: 17/234 -- GLOBAL_STEP: 37925 + | > loss: -0.14673 (-0.15387) + | > log_mle: -0.25346 (-0.27482) + | > loss_dur: 0.10673 (0.12095) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.59839 (13.65621) + | > current_lr: 0.00004 + | > step_time: 5.41050 (4.87800) + | > loader_time: 0.10030 (0.06945) + + + --> STEP: 22/234 -- GLOBAL_STEP: 37930 + | > loss: -0.15694 (-0.15435) + | > log_mle: -0.27076 (-0.27217) + | > loss_dur: 0.11382 (0.11781) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.38647 (13.31948) + | > current_lr: 0.00004 + | > step_time: 1.61740 (4.08018) + | > loader_time: 0.00230 (0.05404) + + + --> STEP: 27/234 -- GLOBAL_STEP: 37935 + | > loss: -0.16901 (-0.15607) + | > log_mle: -0.27220 (-0.27138) + | > loss_dur: 0.10319 (0.11531) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.24492 (12.76048) + | > current_lr: 0.00004 + | > step_time: 3.09620 (3.89401) + | > loader_time: 0.00610 (0.05564) + + + --> STEP: 32/234 -- GLOBAL_STEP: 37940 + | > loss: -0.18208 (-0.15626) + | > log_mle: -0.27854 (-0.27108) + | > loss_dur: 0.09646 (0.11482) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.88826 (12.17699) + | > current_lr: 0.00004 + | > step_time: 1.08890 (3.59813) + | > loader_time: 0.00140 (0.05301) + + + --> STEP: 37/234 -- GLOBAL_STEP: 37945 + | > loss: -0.14540 (-0.15308) + | > log_mle: -0.25938 (-0.26995) + | > loss_dur: 0.11399 (0.11687) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.82446 (12.14449) + | > current_lr: 0.00004 + | > step_time: 1.11810 (3.26769) + | > loader_time: 0.00220 (0.04608) + + + --> STEP: 42/234 -- GLOBAL_STEP: 37950 + | > loss: -0.13910 (-0.15111) + | > log_mle: -0.25518 (-0.26922) + | > loss_dur: 0.11608 (0.11810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.32054 (11.98002) + | > current_lr: 0.00004 + | > step_time: 0.96080 (3.04509) + | > loader_time: 0.00250 (0.04301) + + + --> STEP: 47/234 -- GLOBAL_STEP: 37955 + | > loss: -0.12884 (-0.14970) + | > log_mle: -0.26294 (-0.26935) + | > loss_dur: 0.13410 (0.11965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.87813 (11.86602) + | > current_lr: 0.00004 + | > step_time: 1.54250 (2.90218) + | > loader_time: 0.09920 (0.04073) + + + --> STEP: 52/234 -- GLOBAL_STEP: 37960 + | > loss: -0.12322 (-0.14865) + | > log_mle: -0.25895 (-0.26844) + | > loss_dur: 0.13574 (0.11978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.04394 (11.50255) + | > current_lr: 0.00004 + | > step_time: 1.90460 (2.76022) + | > loader_time: 0.00250 (0.03705) + + + --> STEP: 57/234 -- GLOBAL_STEP: 37965 + | > loss: -0.11548 (-0.14811) + | > log_mle: -0.25019 (-0.26847) + | > loss_dur: 0.13471 (0.12035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.16200 (11.35544) + | > current_lr: 0.00004 + | > step_time: 2.30660 (2.68963) + | > loader_time: 0.00190 (0.03401) + + + --> STEP: 62/234 -- GLOBAL_STEP: 37970 + | > loss: -0.09783 (-0.14751) + | > log_mle: -0.29731 (-0.26929) + | > loss_dur: 0.19949 (0.12178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.03761 (11.69066) + | > current_lr: 0.00004 + | > step_time: 1.42310 (2.63571) + | > loader_time: 0.00230 (0.03151) + + + --> STEP: 67/234 -- GLOBAL_STEP: 37975 + | > loss: -0.13625 (-0.14673) + | > log_mle: -0.27952 (-0.26888) + | > loss_dur: 0.14326 (0.12216) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.59569 (11.52484) + | > current_lr: 0.00004 + | > step_time: 1.95920 (2.58096) + | > loader_time: 0.00230 (0.03074) + + + --> STEP: 72/234 -- GLOBAL_STEP: 37980 + | > loss: -0.12838 (-0.14460) + | > log_mle: -0.25972 (-0.26836) + | > loss_dur: 0.13134 (0.12376) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.15758 (11.56370) + | > current_lr: 0.00004 + | > step_time: 1.19080 (2.52245) + | > loader_time: 0.00670 (0.02994) + + + --> STEP: 77/234 -- GLOBAL_STEP: 37985 + | > loss: -0.13368 (-0.14309) + | > log_mle: -0.26643 (-0.26832) + | > loss_dur: 0.13275 (0.12523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.92293 (11.64946) + | > current_lr: 0.00004 + | > step_time: 1.60840 (2.48372) + | > loader_time: 0.08660 (0.03142) + + + --> STEP: 82/234 -- GLOBAL_STEP: 37990 + | > loss: -0.12661 (-0.14223) + | > log_mle: -0.26018 (-0.26807) + | > loss_dur: 0.13357 (0.12584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.76357 (11.54236) + | > current_lr: 0.00004 + | > step_time: 1.31460 (2.43326) + | > loader_time: 0.00240 (0.03180) + + + --> STEP: 87/234 -- GLOBAL_STEP: 37995 + | > loss: -0.11846 (-0.14121) + | > log_mle: -0.27030 (-0.26831) + | > loss_dur: 0.15184 (0.12710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.07530 (11.54677) + | > current_lr: 0.00004 + | > step_time: 2.79950 (2.38864) + | > loader_time: 0.00300 (0.03013) + + + --> STEP: 92/234 -- GLOBAL_STEP: 38000 + | > loss: -0.17284 (-0.14162) + | > log_mle: -0.31872 (-0.27027) + | > loss_dur: 0.14588 (0.12865) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.86994 (11.80314) + | > current_lr: 0.00004 + | > step_time: 1.70920 (2.35856) + | > loader_time: 0.00280 (0.03050) + + + --> STEP: 97/234 -- GLOBAL_STEP: 38005 + | > loss: -0.15286 (-0.14282) + | > log_mle: -0.30350 (-0.27305) + | > loss_dur: 0.15064 (0.13023) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.30022 (12.46908) + | > current_lr: 0.00004 + | > step_time: 1.99620 (2.32956) + | > loader_time: 0.00230 (0.02993) + + + --> STEP: 102/234 -- GLOBAL_STEP: 38010 + | > loss: -0.12206 (-0.14314) + | > log_mle: -0.28565 (-0.27485) + | > loss_dur: 0.16359 (0.13171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.19601 (13.06147) + | > current_lr: 0.00004 + | > step_time: 2.99860 (2.33484) + | > loader_time: 0.09180 (0.03127) + + + --> STEP: 107/234 -- GLOBAL_STEP: 38015 + | > loss: -0.15808 (-0.14407) + | > log_mle: -0.33119 (-0.27776) + | > loss_dur: 0.17311 (0.13369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.22861 (13.77253) + | > current_lr: 0.00004 + | > step_time: 1.50660 (2.30861) + | > loader_time: 0.08570 (0.03070) + + + --> STEP: 112/234 -- GLOBAL_STEP: 38020 + | > loss: -0.14893 (-0.14447) + | > log_mle: -0.34277 (-0.28039) + | > loss_dur: 0.19383 (0.13591) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.09180 (14.23118) + | > current_lr: 0.00004 + | > step_time: 1.38870 (2.27860) + | > loader_time: 0.00250 (0.03087) + + + --> STEP: 117/234 -- GLOBAL_STEP: 38025 + | > loss: -0.16968 (-0.14508) + | > log_mle: -0.33808 (-0.28291) + | > loss_dur: 0.16839 (0.13784) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.36403 (14.76323) + | > current_lr: 0.00004 + | > step_time: 1.99700 (2.27499) + | > loader_time: 0.00270 (0.02967) + + + --> STEP: 122/234 -- GLOBAL_STEP: 38030 + | > loss: -0.14148 (-0.14530) + | > log_mle: -0.31414 (-0.28445) + | > loss_dur: 0.17266 (0.13915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.75036 (15.05932) + | > current_lr: 0.00004 + | > step_time: 1.68260 (2.28558) + | > loader_time: 0.01080 (0.03009) + + + --> STEP: 127/234 -- GLOBAL_STEP: 38035 + | > loss: -0.18274 (-0.14653) + | > log_mle: -0.37286 (-0.28733) + | > loss_dur: 0.19012 (0.14080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.12689 (15.73705) + | > current_lr: 0.00004 + | > step_time: 1.58720 (2.30979) + | > loader_time: 0.00430 (0.03048) + + + --> STEP: 132/234 -- GLOBAL_STEP: 38040 + | > loss: -0.18919 (-0.14818) + | > log_mle: -0.35636 (-0.29064) + | > loss_dur: 0.16717 (0.14246) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.60882 (16.45707) + | > current_lr: 0.00004 + | > step_time: 2.02040 (2.29596) + | > loader_time: 0.08780 (0.03073) + + + --> STEP: 137/234 -- GLOBAL_STEP: 38045 + | > loss: -0.15590 (-0.14955) + | > log_mle: -0.36397 (-0.29407) + | > loss_dur: 0.20807 (0.14453) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.14190 (17.44250) + | > current_lr: 0.00004 + | > step_time: 1.11200 (2.27908) + | > loader_time: 0.00240 (0.03041) + + + --> STEP: 142/234 -- GLOBAL_STEP: 38050 + | > loss: -0.17004 (-0.15065) + | > log_mle: -0.36609 (-0.29677) + | > loss_dur: 0.19605 (0.14612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.95856 (18.77199) + | > current_lr: 0.00004 + | > step_time: 2.89410 (2.29688) + | > loader_time: 0.10390 (0.03185) + + + --> STEP: 147/234 -- GLOBAL_STEP: 38055 + | > loss: -0.17438 (-0.15264) + | > log_mle: -0.37183 (-0.30097) + | > loss_dur: 0.19746 (0.14832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.97535 (19.72703) + | > current_lr: 0.00004 + | > step_time: 2.48300 (2.29547) + | > loader_time: 0.00460 (0.03150) + + + --> STEP: 152/234 -- GLOBAL_STEP: 38060 + | > loss: -0.23003 (-0.15494) + | > log_mle: -0.45515 (-0.30508) + | > loss_dur: 0.22512 (0.15014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.85531 (20.56358) + | > current_lr: 0.00004 + | > step_time: 1.29940 (2.27777) + | > loader_time: 0.00480 (0.03106) + + + --> STEP: 157/234 -- GLOBAL_STEP: 38065 + | > loss: -0.20259 (-0.15798) + | > log_mle: -0.41182 (-0.30988) + | > loss_dur: 0.20922 (0.15190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.81174 (21.72323) + | > current_lr: 0.00004 + | > step_time: 3.80830 (2.29120) + | > loader_time: 0.00290 (0.03019) + + + --> STEP: 162/234 -- GLOBAL_STEP: 38070 + | > loss: -0.24585 (-0.16090) + | > log_mle: -0.44446 (-0.31454) + | > loss_dur: 0.19861 (0.15364) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.40029 (22.59431) + | > current_lr: 0.00004 + | > step_time: 3.18030 (2.30372) + | > loader_time: 0.10740 (0.03174) + + + --> STEP: 167/234 -- GLOBAL_STEP: 38075 + | > loss: -0.32932 (-0.16365) + | > log_mle: -0.53521 (-0.31893) + | > loss_dur: 0.20589 (0.15527) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.14339 (23.59026) + | > current_lr: 0.00004 + | > step_time: 2.80630 (2.29726) + | > loader_time: 0.00360 (0.03134) + + + --> STEP: 172/234 -- GLOBAL_STEP: 38080 + | > loss: -0.29404 (-0.16687) + | > log_mle: -0.52668 (-0.32427) + | > loss_dur: 0.23263 (0.15739) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.61186 (24.59327) + | > current_lr: 0.00004 + | > step_time: 6.21300 (2.34219) + | > loader_time: 0.09170 (0.03159) + + + --> STEP: 177/234 -- GLOBAL_STEP: 38085 + | > loss: -0.26131 (-0.17005) + | > log_mle: -0.47671 (-0.32926) + | > loss_dur: 0.21540 (0.15921) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.33590 (25.73137) + | > current_lr: 0.00004 + | > step_time: 3.30730 (2.43940) + | > loader_time: 0.60260 (0.03465) + + + --> STEP: 182/234 -- GLOBAL_STEP: 38090 + | > loss: -0.28465 (-0.17290) + | > log_mle: -0.52699 (-0.33421) + | > loss_dur: 0.24235 (0.16131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.27763 (26.80426) + | > current_lr: 0.00004 + | > step_time: 4.10710 (2.51304) + | > loader_time: 0.29100 (0.03585) + + + --> STEP: 187/234 -- GLOBAL_STEP: 38095 + | > loss: -0.29432 (-0.17580) + | > log_mle: -0.52675 (-0.33902) + | > loss_dur: 0.23243 (0.16322) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.87993 (27.95012) + | > current_lr: 0.00004 + | > step_time: 6.81650 (2.55242) + | > loader_time: 0.09570 (0.03643) + + + --> STEP: 192/234 -- GLOBAL_STEP: 38100 + | > loss: -0.33609 (-0.17895) + | > log_mle: -0.54986 (-0.34368) + | > loss_dur: 0.21377 (0.16473) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.39484 (29.23460) + | > current_lr: 0.00004 + | > step_time: 4.30610 (2.71680) + | > loader_time: 0.08720 (0.03935) + + + --> STEP: 197/234 -- GLOBAL_STEP: 38105 + | > loss: -0.31696 (-0.18198) + | > log_mle: -0.52833 (-0.34832) + | > loss_dur: 0.21137 (0.16634) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.92781 (30.23746) + | > current_lr: 0.00004 + | > step_time: 6.30560 (2.75295) + | > loader_time: 0.00450 (0.04087) + + + --> STEP: 202/234 -- GLOBAL_STEP: 38110 + | > loss: -0.38406 (-0.18492) + | > log_mle: -0.61746 (-0.35307) + | > loss_dur: 0.23339 (0.16815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.89897 (31.37204) + | > current_lr: 0.00004 + | > step_time: 1.30970 (2.83231) + | > loader_time: 0.08070 (0.04328) + + + --> STEP: 207/234 -- GLOBAL_STEP: 38115 + | > loss: -0.35687 (-0.18795) + | > log_mle: -0.59742 (-0.35778) + | > loss_dur: 0.24055 (0.16983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.35490 (32.65777) + | > current_lr: 0.00004 + | > step_time: 3.39840 (2.84780) + | > loader_time: 0.08800 (0.04323) + + + --> STEP: 212/234 -- GLOBAL_STEP: 38120 + | > loss: -0.33693 (-0.19141) + | > log_mle: -0.57912 (-0.36301) + | > loss_dur: 0.24219 (0.17160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.85583 (34.13237) + | > current_lr: 0.00004 + | > step_time: 4.19750 (2.88974) + | > loader_time: 0.20710 (0.04490) + + + --> STEP: 217/234 -- GLOBAL_STEP: 38125 + | > loss: -0.34637 (-0.19492) + | > log_mle: -0.59646 (-0.36818) + | > loss_dur: 0.25009 (0.17326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 118.54634 (35.66528) + | > current_lr: 0.00004 + | > step_time: 6.80090 (2.94440) + | > loader_time: 0.09920 (0.04515) + + + --> STEP: 222/234 -- GLOBAL_STEP: 38130 + | > loss: -0.35467 (-0.19845) + | > log_mle: -0.62011 (-0.37337) + | > loss_dur: 0.26543 (0.17492) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.83856 (36.72850) + | > current_lr: 0.00004 + | > step_time: 2.28890 (2.96140) + | > loader_time: 0.09660 (0.04583) + + + --> STEP: 227/234 -- GLOBAL_STEP: 38135 + | > loss: -0.32519 (-0.20218) + | > log_mle: -0.58972 (-0.37887) + | > loss_dur: 0.26453 (0.17670) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.13848 (38.12078) + | > current_lr: 0.00004 + | > step_time: 0.24600 (2.90146) + | > loader_time: 0.00400 (0.04490) + + + --> STEP: 232/234 -- GLOBAL_STEP: 38140 + | > loss: -0.29595 (-0.20504) + | > log_mle: -0.79955 (-0.38574) + | > loss_dur: 0.50360 (0.18070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 133.79787 (39.72941) + | > current_lr: 0.00004 + | > step_time: 0.33570 (2.84494) + | > loader_time: 0.00540 (0.04404) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22457 (-0.44948) + | > avg_loss: -0.24817 (-0.02723) + | > avg_log_mle: -0.46674 (-0.00823) + | > avg_loss_dur: 0.21858 (-0.01900) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_38142.pth + + > EPOCH: 163/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:13:49)  + + --> STEP: 3/234 -- GLOBAL_STEP: 38145 + | > loss: -0.08820 (-0.13311) + | > log_mle: -0.26623 (-0.27367) + | > loss_dur: 0.17803 (0.14056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.48603 (18.21162) + | > current_lr: 0.00004 + | > step_time: 3.40040 (3.15855) + | > loader_time: 0.29600 (1.90081) + + + --> STEP: 8/234 -- GLOBAL_STEP: 38150 + | > loss: -0.16019 (-0.14371) + | > log_mle: -0.28938 (-0.27518) + | > loss_dur: 0.12919 (0.13148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.68151 (15.62665) + | > current_lr: 0.00004 + | > step_time: 1.59330 (5.29509) + | > loader_time: 0.00520 (0.75178) + + + --> STEP: 13/234 -- GLOBAL_STEP: 38155 + | > loss: -0.16404 (-0.14825) + | > log_mle: -0.28102 (-0.27616) + | > loss_dur: 0.11698 (0.12791) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.78689 (14.68908) + | > current_lr: 0.00004 + | > step_time: 1.11360 (3.97601) + | > loader_time: 0.00210 (0.47637) + + + --> STEP: 18/234 -- GLOBAL_STEP: 38160 + | > loss: -0.13687 (-0.14937) + | > log_mle: -0.26679 (-0.27408) + | > loss_dur: 0.12993 (0.12471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.05789 (13.78569) + | > current_lr: 0.00004 + | > step_time: 1.69440 (3.19805) + | > loader_time: 0.00200 (0.34457) + + + --> STEP: 23/234 -- GLOBAL_STEP: 38165 + | > loss: -0.18667 (-0.15205) + | > log_mle: -0.27520 (-0.27189) + | > loss_dur: 0.08853 (0.11984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.99496 (13.47055) + | > current_lr: 0.00004 + | > step_time: 1.59080 (2.86216) + | > loader_time: 0.00180 (0.27347) + + + --> STEP: 28/234 -- GLOBAL_STEP: 38170 + | > loss: -0.19323 (-0.15392) + | > log_mle: -0.27483 (-0.27101) + | > loss_dur: 0.08159 (0.11709) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.52749 (12.92984) + | > current_lr: 0.00004 + | > step_time: 2.29350 (2.65745) + | > loader_time: 0.00180 (0.23130) + + + --> STEP: 33/234 -- GLOBAL_STEP: 38175 + | > loss: -0.15785 (-0.15273) + | > log_mle: -0.26265 (-0.27038) + | > loss_dur: 0.10480 (0.11765) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.12015 (12.57941) + | > current_lr: 0.00004 + | > step_time: 6.20830 (2.82541) + | > loader_time: 0.09560 (0.20753) + + + --> STEP: 38/234 -- GLOBAL_STEP: 38180 + | > loss: -0.15263 (-0.15097) + | > log_mle: -0.27714 (-0.26976) + | > loss_dur: 0.12451 (0.11879) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.84015 (12.42755) + | > current_lr: 0.00004 + | > step_time: 1.40680 (2.90962) + | > loader_time: 0.00260 (0.19050) + + + --> STEP: 43/234 -- GLOBAL_STEP: 38185 + | > loss: -0.14501 (-0.14956) + | > log_mle: -0.27441 (-0.26903) + | > loss_dur: 0.12940 (0.11947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.26161 (12.13752) + | > current_lr: 0.00004 + | > step_time: 6.39970 (2.88108) + | > loader_time: 0.00260 (0.17020) + + + --> STEP: 48/234 -- GLOBAL_STEP: 38190 + | > loss: -0.15809 (-0.14901) + | > log_mle: -0.26074 (-0.26879) + | > loss_dur: 0.10265 (0.11978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.45385 (11.88051) + | > current_lr: 0.00004 + | > step_time: 2.00780 (2.75188) + | > loader_time: 0.00350 (0.15639) + + + --> STEP: 53/234 -- GLOBAL_STEP: 38195 + | > loss: -0.14130 (-0.14809) + | > log_mle: -0.27280 (-0.26831) + | > loss_dur: 0.13150 (0.12021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.54744 (11.49629) + | > current_lr: 0.00004 + | > step_time: 1.57020 (2.67482) + | > loader_time: 0.08660 (0.14504) + + + --> STEP: 58/234 -- GLOBAL_STEP: 38200 + | > loss: -0.14368 (-0.14768) + | > log_mle: -0.26089 (-0.26818) + | > loss_dur: 0.11721 (0.12050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.13367 (11.25612) + | > current_lr: 0.00004 + | > step_time: 1.80770 (2.56711) + | > loader_time: 0.05160 (0.13356) + + + --> STEP: 63/234 -- GLOBAL_STEP: 38205 + | > loss: -0.11056 (-0.14661) + | > log_mle: -0.26302 (-0.26907) + | > loss_dur: 0.15246 (0.12246) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.33397 (11.50210) + | > current_lr: 0.00004 + | > step_time: 1.60280 (2.48828) + | > loader_time: 0.08730 (0.12770) + + + --> STEP: 68/234 -- GLOBAL_STEP: 38210 + | > loss: -0.10227 (-0.14575) + | > log_mle: -0.25842 (-0.26868) + | > loss_dur: 0.15615 (0.12293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.10482 (11.38402) + | > current_lr: 0.00004 + | > step_time: 2.30510 (2.44054) + | > loader_time: 0.08610 (0.12097) + + + --> STEP: 73/234 -- GLOBAL_STEP: 38215 + | > loss: -0.11690 (-0.14399) + | > log_mle: -0.28236 (-0.26873) + | > loss_dur: 0.16546 (0.12474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.46870 (11.58012) + | > current_lr: 0.00004 + | > step_time: 1.77280 (2.42799) + | > loader_time: 0.00230 (0.11425) + + + --> STEP: 78/234 -- GLOBAL_STEP: 38220 + | > loss: -0.12907 (-0.14337) + | > log_mle: -0.25772 (-0.26876) + | > loss_dur: 0.12865 (0.12539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.67977 (11.63971) + | > current_lr: 0.00004 + | > step_time: 1.23640 (2.37822) + | > loader_time: 0.00210 (0.10828) + + + --> STEP: 83/234 -- GLOBAL_STEP: 38225 + | > loss: -0.11743 (-0.14266) + | > log_mle: -0.28171 (-0.26905) + | > loss_dur: 0.16428 (0.12638) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.62077 (11.67296) + | > current_lr: 0.00004 + | > step_time: 1.26740 (2.37400) + | > loader_time: 0.00240 (0.10274) + + + --> STEP: 88/234 -- GLOBAL_STEP: 38230 + | > loss: -0.15237 (-0.14255) + | > log_mle: -0.31778 (-0.26985) + | > loss_dur: 0.16541 (0.12730) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.72283 (11.80507) + | > current_lr: 0.00004 + | > step_time: 1.91360 (2.33655) + | > loader_time: 0.08720 (0.09989) + + + --> STEP: 93/234 -- GLOBAL_STEP: 38235 + | > loss: -0.14322 (-0.14279) + | > log_mle: -0.32812 (-0.27192) + | > loss_dur: 0.18491 (0.12913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.60373 (12.29569) + | > current_lr: 0.00004 + | > step_time: 1.69720 (2.38159) + | > loader_time: 0.00310 (0.09577) + + + --> STEP: 98/234 -- GLOBAL_STEP: 38240 + | > loss: -0.12115 (-0.14352) + | > log_mle: -0.26104 (-0.27396) + | > loss_dur: 0.13989 (0.13044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.26509 (12.74509) + | > current_lr: 0.00004 + | > step_time: 1.09110 (2.35567) + | > loader_time: 0.00230 (0.09475) + + + --> STEP: 103/234 -- GLOBAL_STEP: 38245 + | > loss: -0.16668 (-0.14416) + | > log_mle: -0.36303 (-0.27678) + | > loss_dur: 0.19635 (0.13263) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.33567 (13.41502) + | > current_lr: 0.00004 + | > step_time: 2.02230 (2.31815) + | > loader_time: 0.00200 (0.09028) + + + --> STEP: 108/234 -- GLOBAL_STEP: 38250 + | > loss: -0.15518 (-0.14491) + | > log_mle: -0.30619 (-0.27903) + | > loss_dur: 0.15101 (0.13412) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.99456 (14.24110) + | > current_lr: 0.00004 + | > step_time: 2.99200 (2.29523) + | > loader_time: 0.00330 (0.08767) + + + --> STEP: 113/234 -- GLOBAL_STEP: 38255 + | > loss: -0.17516 (-0.14557) + | > log_mle: -0.35390 (-0.28213) + | > loss_dur: 0.17875 (0.13656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.66813 (14.83618) + | > current_lr: 0.00004 + | > step_time: 1.55890 (2.31570) + | > loader_time: 0.00230 (0.08596) + + + --> STEP: 118/234 -- GLOBAL_STEP: 38260 + | > loss: -0.13968 (-0.14611) + | > log_mle: -0.32230 (-0.28442) + | > loss_dur: 0.18262 (0.13831) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.64205 (15.10122) + | > current_lr: 0.00004 + | > step_time: 2.09870 (2.30743) + | > loader_time: 0.00310 (0.08479) + + + --> STEP: 123/234 -- GLOBAL_STEP: 38265 + | > loss: -0.12509 (-0.14609) + | > log_mle: -0.29096 (-0.28565) + | > loss_dur: 0.16588 (0.13957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.42930 (15.33643) + | > current_lr: 0.00004 + | > step_time: 1.70280 (2.27780) + | > loader_time: 0.00430 (0.08288) + + + --> STEP: 128/234 -- GLOBAL_STEP: 38270 + | > loss: -0.18421 (-0.14769) + | > log_mle: -0.35312 (-0.28884) + | > loss_dur: 0.16891 (0.14114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.14620 (16.24617) + | > current_lr: 0.00004 + | > step_time: 1.95710 (2.29741) + | > loader_time: 0.00240 (0.08114) + + + --> STEP: 133/234 -- GLOBAL_STEP: 38275 + | > loss: -0.18286 (-0.14931) + | > log_mle: -0.37784 (-0.29230) + | > loss_dur: 0.19498 (0.14299) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.50071 (17.07574) + | > current_lr: 0.00004 + | > step_time: 3.49480 (2.29612) + | > loader_time: 0.00170 (0.07884) + + + --> STEP: 138/234 -- GLOBAL_STEP: 38280 + | > loss: -0.15323 (-0.15058) + | > log_mle: -0.33077 (-0.29546) + | > loss_dur: 0.17754 (0.14488) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.41884 (17.69871) + | > current_lr: 0.00004 + | > step_time: 3.10640 (2.27955) + | > loader_time: 0.08910 (0.07672) + + + --> STEP: 143/234 -- GLOBAL_STEP: 38285 + | > loss: -0.23054 (-0.15236) + | > log_mle: -0.46458 (-0.29929) + | > loss_dur: 0.23403 (0.14694) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.25932 (18.73451) + | > current_lr: 0.00004 + | > step_time: 2.20600 (2.28593) + | > loader_time: 0.00200 (0.07531) + + + --> STEP: 148/234 -- GLOBAL_STEP: 38290 + | > loss: -0.22233 (-0.15472) + | > log_mle: -0.38479 (-0.30328) + | > loss_dur: 0.16246 (0.14855) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.66538 (19.50638) + | > current_lr: 0.00004 + | > step_time: 1.08810 (2.31394) + | > loader_time: 0.00290 (0.07411) + + + --> STEP: 153/234 -- GLOBAL_STEP: 38295 + | > loss: -0.29522 (-0.15788) + | > log_mle: -0.51171 (-0.30842) + | > loss_dur: 0.21650 (0.15054) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.70393 (20.65235) + | > current_lr: 0.00004 + | > step_time: 5.01060 (2.33638) + | > loader_time: 0.00530 (0.07183) + + + --> STEP: 158/234 -- GLOBAL_STEP: 38300 + | > loss: -0.22786 (-0.16048) + | > log_mle: -0.44827 (-0.31288) + | > loss_dur: 0.22041 (0.15239) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.80174 (21.85389) + | > current_lr: 0.00004 + | > step_time: 2.10760 (2.37963) + | > loader_time: 0.00480 (0.07146) + + + --> STEP: 163/234 -- GLOBAL_STEP: 38305 + | > loss: -0.22293 (-0.16339) + | > log_mle: -0.41993 (-0.31739) + | > loss_dur: 0.19701 (0.15401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.60289 (22.86583) + | > current_lr: 0.00004 + | > step_time: 1.97680 (2.37480) + | > loader_time: 0.00270 (0.06990) + + + --> STEP: 168/234 -- GLOBAL_STEP: 38310 + | > loss: -0.24378 (-0.16629) + | > log_mle: -0.47847 (-0.32205) + | > loss_dur: 0.23469 (0.15577) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.86031 (24.01511) + | > current_lr: 0.00004 + | > step_time: 2.69660 (2.36191) + | > loader_time: 0.00700 (0.06844) + + + --> STEP: 173/234 -- GLOBAL_STEP: 38315 + | > loss: -0.27581 (-0.16957) + | > log_mle: -0.48736 (-0.32730) + | > loss_dur: 0.21155 (0.15773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.93200 (25.15592) + | > current_lr: 0.00004 + | > step_time: 1.20320 (2.35031) + | > loader_time: 0.00430 (0.06754) + + + --> STEP: 178/234 -- GLOBAL_STEP: 38320 + | > loss: -0.30731 (-0.17287) + | > log_mle: -0.54759 (-0.33259) + | > loss_dur: 0.24028 (0.15971) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.91416 (26.34476) + | > current_lr: 0.00004 + | > step_time: 3.79750 (2.39159) + | > loader_time: 0.09970 (0.06678) + + + --> STEP: 183/234 -- GLOBAL_STEP: 38325 + | > loss: -0.32501 (-0.17580) + | > log_mle: -0.54955 (-0.33757) + | > loss_dur: 0.22455 (0.16177) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.88505 (27.39859) + | > current_lr: 0.00004 + | > step_time: 5.20080 (2.44171) + | > loader_time: 0.08680 (0.06707) + + + --> STEP: 188/234 -- GLOBAL_STEP: 38330 + | > loss: -0.33283 (-0.17884) + | > log_mle: -0.56075 (-0.34254) + | > loss_dur: 0.22792 (0.16369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.35561 (28.53010) + | > current_lr: 0.00004 + | > step_time: 2.79530 (2.49656) + | > loader_time: 0.79060 (0.07313) + + + --> STEP: 193/234 -- GLOBAL_STEP: 38335 + | > loss: -0.33099 (-0.18215) + | > log_mle: -0.56005 (-0.34732) + | > loss_dur: 0.22907 (0.16517) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.21355 (29.80812) + | > current_lr: 0.00004 + | > step_time: 3.10550 (2.48122) + | > loader_time: 0.00390 (0.07222) + + + --> STEP: 198/234 -- GLOBAL_STEP: 38340 + | > loss: -0.31605 (-0.18525) + | > log_mle: -0.54839 (-0.35197) + | > loss_dur: 0.23234 (0.16672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.89233 (30.93947) + | > current_lr: 0.00004 + | > step_time: 4.09410 (2.60336) + | > loader_time: 0.00460 (0.07405) + + + --> STEP: 203/234 -- GLOBAL_STEP: 38345 + | > loss: -0.25555 (-0.18801) + | > log_mle: -0.48273 (-0.35638) + | > loss_dur: 0.22719 (0.16837) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.07793 (31.97445) + | > current_lr: 0.00004 + | > step_time: 2.41230 (2.66149) + | > loader_time: 0.00420 (0.07369) + + + --> STEP: 208/234 -- GLOBAL_STEP: 38350 + | > loss: -0.32874 (-0.19124) + | > log_mle: -0.56375 (-0.36140) + | > loss_dur: 0.23500 (0.17015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.98400 (33.30690) + | > current_lr: 0.00004 + | > step_time: 10.88350 (2.77686) + | > loader_time: 0.00270 (0.07329) + + + --> STEP: 213/234 -- GLOBAL_STEP: 38355 + | > loss: -0.34591 (-0.19487) + | > log_mle: -0.60053 (-0.36685) + | > loss_dur: 0.25462 (0.17197) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.60406 (34.78725) + | > current_lr: 0.00004 + | > step_time: 2.89260 (2.88344) + | > loader_time: 0.00820 (0.07453) + + + --> STEP: 218/234 -- GLOBAL_STEP: 38360 + | > loss: -0.33121 (-0.19834) + | > log_mle: -0.57075 (-0.37187) + | > loss_dur: 0.23954 (0.17352) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.06738 (36.01348) + | > current_lr: 0.00004 + | > step_time: 2.20570 (2.93663) + | > loader_time: 0.00680 (0.07422) + + + --> STEP: 223/234 -- GLOBAL_STEP: 38365 + | > loss: -0.37108 (-0.20198) + | > log_mle: -0.61110 (-0.37708) + | > loss_dur: 0.24003 (0.17511) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.40569 (37.25869) + | > current_lr: 0.00004 + | > step_time: 0.84590 (2.92661) + | > loader_time: 0.00290 (0.07346) + + + --> STEP: 228/234 -- GLOBAL_STEP: 38370 + | > loss: -0.33327 (-0.20548) + | > log_mle: -0.60621 (-0.38247) + | > loss_dur: 0.27294 (0.17700) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.31503 (38.67686) + | > current_lr: 0.00004 + | > step_time: 0.24490 (2.86773) + | > loader_time: 0.00390 (0.07193) + + + --> STEP: 233/234 -- GLOBAL_STEP: 38375 + | > loss: 0.19216 (-0.20594) + | > log_mle: -0.57804 (-0.38904) + | > loss_dur: 0.77019 (0.18310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.55486 (40.37014) + | > current_lr: 0.00004 + | > step_time: 0.19390 (2.81177) + | > loader_time: 0.00340 (0.07049) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32579 (+0.10122) + | > avg_loss: -0.23792 (+0.01025) + | > avg_log_mle: -0.46525 (+0.00150) + | > avg_loss_dur: 0.22733 (+0.00875) + + + > EPOCH: 164/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:26:10)  + + --> STEP: 4/234 -- GLOBAL_STEP: 38380 + | > loss: -0.12586 (-0.13974) + | > log_mle: -0.27055 (-0.27498) + | > loss_dur: 0.14469 (0.13524) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.73698 (17.29487) + | > current_lr: 0.00004 + | > step_time: 3.59080 (4.39756) + | > loader_time: 0.00100 (0.02351) + + + --> STEP: 9/234 -- GLOBAL_STEP: 38385 + | > loss: -0.14115 (-0.15250) + | > log_mle: -0.28476 (-0.27832) + | > loss_dur: 0.14360 (0.12582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.41670 (15.23543) + | > current_lr: 0.00004 + | > step_time: 3.09010 (5.35273) + | > loader_time: 0.19760 (0.04269) + + + --> STEP: 14/234 -- GLOBAL_STEP: 38390 + | > loss: -0.14944 (-0.15535) + | > log_mle: -0.28081 (-0.27836) + | > loss_dur: 0.13138 (0.12301) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.42677 (14.50947) + | > current_lr: 0.00004 + | > step_time: 3.58360 (5.66107) + | > loader_time: 0.00100 (0.05535) + + + --> STEP: 19/234 -- GLOBAL_STEP: 38395 + | > loss: -0.16628 (-0.15624) + | > log_mle: -0.26663 (-0.27576) + | > loss_dur: 0.10035 (0.11952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.11844 (13.34646) + | > current_lr: 0.00004 + | > step_time: 4.10400 (4.87847) + | > loader_time: 0.00280 (0.05157) + + + --> STEP: 24/234 -- GLOBAL_STEP: 38400 + | > loss: -0.17775 (-0.15836) + | > log_mle: -0.26880 (-0.27426) + | > loss_dur: 0.09105 (0.11590) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.89252 (12.32635) + | > current_lr: 0.00004 + | > step_time: 1.60590 (4.73673) + | > loader_time: 0.00250 (0.04906) + + + --> STEP: 29/234 -- GLOBAL_STEP: 38405 + | > loss: -0.13988 (-0.15936) + | > log_mle: -0.25638 (-0.27326) + | > loss_dur: 0.11650 (0.11390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.82470 (11.88713) + | > current_lr: 0.00004 + | > step_time: 1.05280 (4.40368) + | > loader_time: 0.00190 (0.04116) + + + --> STEP: 34/234 -- GLOBAL_STEP: 38410 + | > loss: -0.13819 (-0.15848) + | > log_mle: -0.26603 (-0.27305) + | > loss_dur: 0.12784 (0.11457) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.84942 (11.61159) + | > current_lr: 0.00004 + | > step_time: 1.11580 (3.94015) + | > loader_time: 0.00270 (0.03539) + + + --> STEP: 39/234 -- GLOBAL_STEP: 38415 + | > loss: -0.14640 (-0.15696) + | > log_mle: -0.27244 (-0.27282) + | > loss_dur: 0.12605 (0.11586) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.27912 (11.82834) + | > current_lr: 0.00004 + | > step_time: 1.47530 (3.59223) + | > loader_time: 0.00200 (0.03112) + + + --> STEP: 44/234 -- GLOBAL_STEP: 38420 + | > loss: -0.16289 (-0.15549) + | > log_mle: -0.26501 (-0.27197) + | > loss_dur: 0.10212 (0.11648) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.65901 (11.58580) + | > current_lr: 0.00004 + | > step_time: 0.98110 (3.31363) + | > loader_time: 0.00200 (0.02794) + + + --> STEP: 49/234 -- GLOBAL_STEP: 38425 + | > loss: -0.16856 (-0.15465) + | > log_mle: -0.27367 (-0.27198) + | > loss_dur: 0.10511 (0.11733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.87453 (11.58592) + | > current_lr: 0.00004 + | > step_time: 1.51140 (3.13255) + | > loader_time: 0.00250 (0.02754) + + + --> STEP: 54/234 -- GLOBAL_STEP: 38430 + | > loss: -0.15888 (-0.15319) + | > log_mle: -0.27530 (-0.27130) + | > loss_dur: 0.11642 (0.11812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.94308 (11.36742) + | > current_lr: 0.00004 + | > step_time: 1.37650 (2.95488) + | > loader_time: 0.00210 (0.02518) + + + --> STEP: 59/234 -- GLOBAL_STEP: 38435 + | > loss: -0.16996 (-0.15260) + | > log_mle: -0.28461 (-0.27116) + | > loss_dur: 0.11465 (0.11856) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.16801 (11.27032) + | > current_lr: 0.00004 + | > step_time: 2.20230 (2.83063) + | > loader_time: 0.09980 (0.02632) + + + --> STEP: 64/234 -- GLOBAL_STEP: 38440 + | > loss: -0.14578 (-0.15130) + | > log_mle: -0.26285 (-0.27164) + | > loss_dur: 0.11707 (0.12034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.67609 (11.53523) + | > current_lr: 0.00004 + | > step_time: 1.90790 (2.72774) + | > loader_time: 0.08830 (0.02704) + + + --> STEP: 69/234 -- GLOBAL_STEP: 38445 + | > loss: -0.11690 (-0.14989) + | > log_mle: -0.24869 (-0.27088) + | > loss_dur: 0.13179 (0.12100) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.92058 (11.53842) + | > current_lr: 0.00004 + | > step_time: 2.59870 (2.66448) + | > loader_time: 0.09990 (0.02666) + + + --> STEP: 74/234 -- GLOBAL_STEP: 38450 + | > loss: -0.12504 (-0.14760) + | > log_mle: -0.25601 (-0.27078) + | > loss_dur: 0.13098 (0.12318) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.02340 (11.81011) + | > current_lr: 0.00004 + | > step_time: 1.60240 (2.57487) + | > loader_time: 0.00350 (0.02504) + + + --> STEP: 79/234 -- GLOBAL_STEP: 38455 + | > loss: -0.12587 (-0.14652) + | > log_mle: -0.27371 (-0.27078) + | > loss_dur: 0.14785 (0.12426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.76997 (11.84501) + | > current_lr: 0.00004 + | > step_time: 2.99450 (2.57547) + | > loader_time: 0.29790 (0.02842) + + + --> STEP: 84/234 -- GLOBAL_STEP: 38460 + | > loss: -0.13680 (-0.14580) + | > log_mle: -0.27028 (-0.27089) + | > loss_dur: 0.13348 (0.12509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.11566 (11.91087) + | > current_lr: 0.00004 + | > step_time: 1.16400 (2.52169) + | > loader_time: 0.00250 (0.02794) + + + --> STEP: 89/234 -- GLOBAL_STEP: 38465 + | > loss: -0.15622 (-0.14573) + | > log_mle: -0.29850 (-0.27192) + | > loss_dur: 0.14228 (0.12620) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.51330 (12.09490) + | > current_lr: 0.00004 + | > step_time: 2.10930 (2.49912) + | > loader_time: 0.00350 (0.02735) + + + --> STEP: 94/234 -- GLOBAL_STEP: 38470 + | > loss: -0.18058 (-0.14623) + | > log_mle: -0.33143 (-0.27417) + | > loss_dur: 0.15085 (0.12793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.29124 (12.64524) + | > current_lr: 0.00004 + | > step_time: 2.30140 (2.44189) + | > loader_time: 0.00220 (0.02693) + + + --> STEP: 99/234 -- GLOBAL_STEP: 38475 + | > loss: -0.18502 (-0.14690) + | > log_mle: -0.36763 (-0.27643) + | > loss_dur: 0.18261 (0.12953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.38333 (12.98003) + | > current_lr: 0.00004 + | > step_time: 1.17010 (2.39508) + | > loader_time: 0.00250 (0.02571) + + + --> STEP: 104/234 -- GLOBAL_STEP: 38480 + | > loss: -0.20598 (-0.14802) + | > log_mle: -0.37928 (-0.27934) + | > loss_dur: 0.17330 (0.13132) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.03430 (13.51545) + | > current_lr: 0.00004 + | > step_time: 1.60200 (2.34997) + | > loader_time: 0.00490 (0.02543) + + + --> STEP: 109/234 -- GLOBAL_STEP: 38485 + | > loss: -0.14184 (-0.14824) + | > log_mle: -0.35005 (-0.28139) + | > loss_dur: 0.20821 (0.13315) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.96768 (13.97859) + | > current_lr: 0.00004 + | > step_time: 1.61340 (2.30537) + | > loader_time: 0.00300 (0.02440) + + + --> STEP: 114/234 -- GLOBAL_STEP: 38490 + | > loss: -0.17195 (-0.14902) + | > log_mle: -0.33546 (-0.28431) + | > loss_dur: 0.16351 (0.13529) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.27467 (14.52840) + | > current_lr: 0.00004 + | > step_time: 1.21660 (2.25791) + | > loader_time: 0.08400 (0.02551) + + + --> STEP: 119/234 -- GLOBAL_STEP: 38495 + | > loss: -0.15778 (-0.14952) + | > log_mle: -0.33315 (-0.28660) + | > loss_dur: 0.17537 (0.13708) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.93755 (14.79449) + | > current_lr: 0.00004 + | > step_time: 1.80780 (2.26155) + | > loader_time: 0.00360 (0.02670) + + + --> STEP: 124/234 -- GLOBAL_STEP: 38500 + | > loss: -0.19379 (-0.15001) + | > log_mle: -0.36083 (-0.28813) + | > loss_dur: 0.16704 (0.13812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.71265 (15.13873) + | > current_lr: 0.00004 + | > step_time: 2.19430 (2.24467) + | > loader_time: 0.00300 (0.02644) + + + --> STEP: 129/234 -- GLOBAL_STEP: 38505 + | > loss: -0.16136 (-0.15130) + | > log_mle: -0.34830 (-0.29134) + | > loss_dur: 0.18694 (0.14004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.79103 (15.91102) + | > current_lr: 0.00004 + | > step_time: 1.06800 (2.21270) + | > loader_time: 0.00250 (0.02635) + + + --> STEP: 134/234 -- GLOBAL_STEP: 38510 + | > loss: -0.19610 (-0.15311) + | > log_mle: -0.40245 (-0.29513) + | > loss_dur: 0.20635 (0.14202) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.37106 (16.80037) + | > current_lr: 0.00004 + | > step_time: 1.09770 (2.18985) + | > loader_time: 0.00260 (0.02548) + + + --> STEP: 139/234 -- GLOBAL_STEP: 38515 + | > loss: -0.26577 (-0.15483) + | > log_mle: -0.46540 (-0.29875) + | > loss_dur: 0.19963 (0.14392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.82878 (17.59238) + | > current_lr: 0.00004 + | > step_time: 4.40310 (2.20327) + | > loader_time: 0.19320 (0.02788) + + + --> STEP: 144/234 -- GLOBAL_STEP: 38520 + | > loss: -0.22444 (-0.15625) + | > log_mle: -0.42479 (-0.30243) + | > loss_dur: 0.20035 (0.14617) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.71331 (18.74767) + | > current_lr: 0.00004 + | > step_time: 3.09690 (2.23455) + | > loader_time: 0.08770 (0.02942) + + + --> STEP: 149/234 -- GLOBAL_STEP: 38525 + | > loss: -0.27417 (-0.15883) + | > log_mle: -0.47967 (-0.30656) + | > loss_dur: 0.20550 (0.14773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.62985 (19.67391) + | > current_lr: 0.00004 + | > step_time: 2.11120 (2.23095) + | > loader_time: 0.09950 (0.03084) + + + --> STEP: 154/234 -- GLOBAL_STEP: 38530 + | > loss: -0.24524 (-0.16170) + | > log_mle: -0.43786 (-0.31124) + | > loss_dur: 0.19262 (0.14954) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.82230 (20.86106) + | > current_lr: 0.00004 + | > step_time: 2.20440 (2.27601) + | > loader_time: 0.00380 (0.03313) + + + --> STEP: 159/234 -- GLOBAL_STEP: 38535 + | > loss: -0.26239 (-0.16454) + | > log_mle: -0.46502 (-0.31582) + | > loss_dur: 0.20264 (0.15128) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.83075 (21.98004) + | > current_lr: 0.00004 + | > step_time: 2.31470 (2.27565) + | > loader_time: 0.00400 (0.03329) + + + --> STEP: 164/234 -- GLOBAL_STEP: 38540 + | > loss: -0.22689 (-0.16703) + | > log_mle: -0.44154 (-0.31999) + | > loss_dur: 0.21464 (0.15296) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.82815 (23.24074) + | > current_lr: 0.00004 + | > step_time: 3.10120 (2.28487) + | > loader_time: 0.08880 (0.03340) + + + --> STEP: 169/234 -- GLOBAL_STEP: 38545 + | > loss: -0.23074 (-0.16937) + | > log_mle: -0.44902 (-0.32422) + | > loss_dur: 0.21828 (0.15485) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.55200 (24.28262) + | > current_lr: 0.00004 + | > step_time: 5.20070 (2.35271) + | > loader_time: 0.18290 (0.03644) + + + --> STEP: 174/234 -- GLOBAL_STEP: 38550 + | > loss: -0.33381 (-0.17307) + | > log_mle: -0.54623 (-0.32984) + | > loss_dur: 0.21242 (0.15677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.32583 (25.44166) + | > current_lr: 0.00004 + | > step_time: 1.68680 (2.35747) + | > loader_time: 0.02000 (0.03719) + + + --> STEP: 179/234 -- GLOBAL_STEP: 38555 + | > loss: -0.27870 (-0.17597) + | > log_mle: -0.53376 (-0.33492) + | > loss_dur: 0.25506 (0.15895) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.04453 (26.67759) + | > current_lr: 0.00004 + | > step_time: 3.08990 (2.35873) + | > loader_time: 0.20020 (0.03736) + + + --> STEP: 184/234 -- GLOBAL_STEP: 38560 + | > loss: -0.27971 (-0.17875) + | > log_mle: -0.50396 (-0.33952) + | > loss_dur: 0.22425 (0.16077) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.70174 (27.66219) + | > current_lr: 0.00004 + | > step_time: 3.28670 (2.36568) + | > loader_time: 0.00300 (0.03750) + + + --> STEP: 189/234 -- GLOBAL_STEP: 38565 + | > loss: -0.26403 (-0.18156) + | > log_mle: -0.49255 (-0.34428) + | > loss_dur: 0.22852 (0.16273) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 97.65460 (28.89108) + | > current_lr: 0.00004 + | > step_time: 4.70300 (2.39890) + | > loader_time: 0.09280 (0.03848) + + + --> STEP: 194/234 -- GLOBAL_STEP: 38570 + | > loss: -0.30855 (-0.18484) + | > log_mle: -0.53325 (-0.34901) + | > loss_dur: 0.22471 (0.16417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.05436 (30.03914) + | > current_lr: 0.00004 + | > step_time: 1.68430 (2.43443) + | > loader_time: 0.00290 (0.03953) + + + --> STEP: 199/234 -- GLOBAL_STEP: 38575 + | > loss: -0.31199 (-0.18768) + | > log_mle: -0.54465 (-0.35352) + | > loss_dur: 0.23266 (0.16583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.97198 (31.15956) + | > current_lr: 0.00004 + | > step_time: 1.70070 (2.41758) + | > loader_time: 0.00400 (0.03949) + + + --> STEP: 204/234 -- GLOBAL_STEP: 38580 + | > loss: -0.31423 (-0.19022) + | > log_mle: -0.56850 (-0.35785) + | > loss_dur: 0.25427 (0.16763) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 119.39001 (32.55497) + | > current_lr: 0.00004 + | > step_time: 3.39980 (2.48043) + | > loader_time: 0.09930 (0.04193) + + + --> STEP: 209/234 -- GLOBAL_STEP: 38585 + | > loss: -0.30770 (-0.19324) + | > log_mle: -0.53644 (-0.36259) + | > loss_dur: 0.22873 (0.16935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.88831 (33.63040) + | > current_lr: 0.00004 + | > step_time: 4.19540 (2.56841) + | > loader_time: 0.08840 (0.04427) + + + --> STEP: 214/234 -- GLOBAL_STEP: 38590 + | > loss: -0.34750 (-0.19699) + | > log_mle: -0.56492 (-0.36810) + | > loss_dur: 0.21742 (0.17111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.52913 (35.09803) + | > current_lr: 0.00004 + | > step_time: 7.09180 (2.66304) + | > loader_time: 0.10610 (0.04468) + + + --> STEP: 219/234 -- GLOBAL_STEP: 38595 + | > loss: -0.41556 (-0.20063) + | > log_mle: -0.66612 (-0.37346) + | > loss_dur: 0.25056 (0.17283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 116.86742 (36.43224) + | > current_lr: 0.00004 + | > step_time: 5.79140 (2.73286) + | > loader_time: 0.30660 (0.04630) + + + --> STEP: 224/234 -- GLOBAL_STEP: 38600 + | > loss: -0.36600 (-0.20398) + | > log_mle: -0.61803 (-0.37847) + | > loss_dur: 0.25203 (0.17449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.26048 (37.51609) + | > current_lr: 0.00004 + | > step_time: 0.23130 (2.69814) + | > loader_time: 0.00400 (0.04574) + + + --> STEP: 229/234 -- GLOBAL_STEP: 38605 + | > loss: -0.35731 (-0.20743) + | > log_mle: -0.67032 (-0.38408) + | > loss_dur: 0.31301 (0.17665) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.25301 (38.80427) + | > current_lr: 0.00004 + | > step_time: 0.24750 (2.64458) + | > loader_time: 0.00380 (0.04484) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.13268 (-0.19312) + | > avg_loss: -0.25493 (-0.01702) + | > avg_log_mle: -0.47420 (-0.00896) + | > avg_loss_dur: 0.21927 (-0.00806) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_38610.pth + + > EPOCH: 165/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:37:31)  + + --> STEP: 0/234 -- GLOBAL_STEP: 38610 + | > loss: -0.16862 (-0.16862) + | > log_mle: -0.35261 (-0.35261) + | > loss_dur: 0.18399 (0.18399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.25799 (22.25799) + | > current_lr: 0.00004 + | > step_time: 3.58840 (3.58837) + | > loader_time: 13.07700 (13.07704) + + + --> STEP: 5/234 -- GLOBAL_STEP: 38615 + | > loss: -0.15518 (-0.14553) + | > log_mle: -0.27717 (-0.27559) + | > loss_dur: 0.12199 (0.13006) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.36078 (18.48476) + | > current_lr: 0.00004 + | > step_time: 3.40980 (5.39922) + | > loader_time: 0.19190 (0.58056) + + + --> STEP: 10/234 -- GLOBAL_STEP: 38620 + | > loss: -0.13293 (-0.15142) + | > log_mle: -0.27320 (-0.27780) + | > loss_dur: 0.14027 (0.12638) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.40015 (16.19018) + | > current_lr: 0.00004 + | > step_time: 3.21330 (4.37278) + | > loader_time: 0.08290 (0.33761) + + + --> STEP: 15/234 -- GLOBAL_STEP: 38625 + | > loss: -0.17846 (-0.15627) + | > log_mle: -0.27750 (-0.27843) + | > loss_dur: 0.09904 (0.12216) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63165 (14.67695) + | > current_lr: 0.00004 + | > step_time: 0.88870 (3.91311) + | > loader_time: 0.00120 (0.24522) + + + --> STEP: 20/234 -- GLOBAL_STEP: 38630 + | > loss: -0.16457 (-0.15716) + | > log_mle: -0.27001 (-0.27592) + | > loss_dur: 0.10544 (0.11876) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.14778 (13.12156) + | > current_lr: 0.00004 + | > step_time: 4.01210 (3.70280) + | > loader_time: 0.19480 (0.21223) + + + --> STEP: 25/234 -- GLOBAL_STEP: 38635 + | > loss: -0.15572 (-0.15825) + | > log_mle: -0.26219 (-0.27442) + | > loss_dur: 0.10646 (0.11617) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.55662 (12.40985) + | > current_lr: 0.00004 + | > step_time: 1.40550 (3.35148) + | > loader_time: 0.00170 (0.17344) + + + --> STEP: 30/234 -- GLOBAL_STEP: 38640 + | > loss: -0.17284 (-0.15876) + | > log_mle: -0.27629 (-0.27382) + | > loss_dur: 0.10345 (0.11506) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.68633 (12.39075) + | > current_lr: 0.00004 + | > step_time: 1.89070 (3.00928) + | > loader_time: 0.00260 (0.14732) + + + --> STEP: 35/234 -- GLOBAL_STEP: 38645 + | > loss: -0.14658 (-0.15717) + | > log_mle: -0.26948 (-0.27316) + | > loss_dur: 0.12291 (0.11598) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.95583 (12.33622) + | > current_lr: 0.00004 + | > step_time: 3.80920 (2.95955) + | > loader_time: 0.00660 (0.12955) + + + --> STEP: 40/234 -- GLOBAL_STEP: 38650 + | > loss: -0.10595 (-0.15493) + | > log_mle: -0.25195 (-0.27212) + | > loss_dur: 0.14600 (0.11720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.18161 (12.35574) + | > current_lr: 0.00004 + | > step_time: 2.11100 (2.78889) + | > loader_time: 0.08240 (0.11775) + + + --> STEP: 45/234 -- GLOBAL_STEP: 38655 + | > loss: -0.14993 (-0.15466) + | > log_mle: -0.29049 (-0.27215) + | > loss_dur: 0.14056 (0.11749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.80948 (12.16299) + | > current_lr: 0.00004 + | > step_time: 1.09720 (2.62774) + | > loader_time: 0.00190 (0.10656) + + + --> STEP: 50/234 -- GLOBAL_STEP: 38660 + | > loss: -0.14313 (-0.15379) + | > log_mle: -0.26182 (-0.27152) + | > loss_dur: 0.11869 (0.11773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.57445 (11.69186) + | > current_lr: 0.00004 + | > step_time: 1.40300 (2.53105) + | > loader_time: 0.00580 (0.09626) + + + --> STEP: 55/234 -- GLOBAL_STEP: 38665 + | > loss: -0.16609 (-0.15299) + | > log_mle: -0.27698 (-0.27142) + | > loss_dur: 0.11088 (0.11844) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.01608 (11.52060) + | > current_lr: 0.00004 + | > step_time: 1.61360 (2.44117) + | > loader_time: 0.09220 (0.09088) + + + --> STEP: 60/234 -- GLOBAL_STEP: 38670 + | > loss: -0.14770 (-0.15252) + | > log_mle: -0.29120 (-0.27150) + | > loss_dur: 0.14350 (0.11898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.60430 (11.46195) + | > current_lr: 0.00004 + | > step_time: 1.73980 (2.39035) + | > loader_time: 0.00280 (0.08670) + + + --> STEP: 65/234 -- GLOBAL_STEP: 38675 + | > loss: -0.14662 (-0.15083) + | > log_mle: -0.26442 (-0.27152) + | > loss_dur: 0.11781 (0.12069) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.55968 (11.63335) + | > current_lr: 0.00004 + | > step_time: 1.98790 (2.35927) + | > loader_time: 0.00220 (0.08279) + + + --> STEP: 70/234 -- GLOBAL_STEP: 38680 + | > loss: -0.10893 (-0.14878) + | > log_mle: -0.26088 (-0.27091) + | > loss_dur: 0.15195 (0.12213) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.73141 (11.57340) + | > current_lr: 0.00004 + | > step_time: 1.20550 (2.27911) + | > loader_time: 0.08460 (0.07825) + + + --> STEP: 75/234 -- GLOBAL_STEP: 38685 + | > loss: -0.11576 (-0.14719) + | > log_mle: -0.27656 (-0.27113) + | > loss_dur: 0.16080 (0.12394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.32931 (11.68997) + | > current_lr: 0.00004 + | > step_time: 2.27570 (2.28929) + | > loader_time: 0.00250 (0.07562) + + + --> STEP: 80/234 -- GLOBAL_STEP: 38690 + | > loss: -0.13977 (-0.14642) + | > log_mle: -0.25981 (-0.27097) + | > loss_dur: 0.12003 (0.12455) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.92035 (11.63680) + | > current_lr: 0.00004 + | > step_time: 1.38750 (2.25891) + | > loader_time: 0.00210 (0.07335) + + + --> STEP: 85/234 -- GLOBAL_STEP: 38695 + | > loss: -0.13807 (-0.14567) + | > log_mle: -0.27238 (-0.27119) + | > loss_dur: 0.13431 (0.12552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.50441 (11.84623) + | > current_lr: 0.00004 + | > step_time: 1.19830 (2.23088) + | > loader_time: 0.00220 (0.07007) + + + --> STEP: 90/234 -- GLOBAL_STEP: 38700 + | > loss: -0.14694 (-0.14572) + | > log_mle: -0.29884 (-0.27243) + | > loss_dur: 0.15190 (0.12671) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.02716 (12.18922) + | > current_lr: 0.00004 + | > step_time: 1.18820 (2.17482) + | > loader_time: 0.00230 (0.06723) + + + --> STEP: 95/234 -- GLOBAL_STEP: 38705 + | > loss: -0.20534 (-0.14714) + | > log_mle: -0.38554 (-0.27569) + | > loss_dur: 0.18020 (0.12854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.14591 (12.79983) + | > current_lr: 0.00004 + | > step_time: 1.58570 (2.12926) + | > loader_time: 0.00200 (0.06385) + + + --> STEP: 100/234 -- GLOBAL_STEP: 38710 + | > loss: -0.16652 (-0.14751) + | > log_mle: -0.31377 (-0.27728) + | > loss_dur: 0.14725 (0.12977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.25484 (12.97340) + | > current_lr: 0.00004 + | > step_time: 2.71590 (2.14088) + | > loader_time: 0.08490 (0.06429) + + + --> STEP: 105/234 -- GLOBAL_STEP: 38715 + | > loss: -0.15563 (-0.14863) + | > log_mle: -0.29065 (-0.28002) + | > loss_dur: 0.13502 (0.13139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.03746 (13.44935) + | > current_lr: 0.00004 + | > step_time: 1.48560 (2.13292) + | > loader_time: 0.00340 (0.06398) + + + --> STEP: 110/234 -- GLOBAL_STEP: 38720 + | > loss: -0.15546 (-0.14874) + | > log_mle: -0.31406 (-0.28228) + | > loss_dur: 0.15859 (0.13354) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.11085 (13.97942) + | > current_lr: 0.00004 + | > step_time: 2.90870 (2.14607) + | > loader_time: 0.00290 (0.06271) + + + --> STEP: 115/234 -- GLOBAL_STEP: 38725 + | > loss: -0.14782 (-0.14963) + | > log_mle: -0.33645 (-0.28539) + | > loss_dur: 0.18863 (0.13576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.38655 (14.63310) + | > current_lr: 0.00004 + | > step_time: 1.17170 (2.12204) + | > loader_time: 0.00380 (0.06011) + + + --> STEP: 120/234 -- GLOBAL_STEP: 38730 + | > loss: -0.21222 (-0.15041) + | > log_mle: -0.38497 (-0.28805) + | > loss_dur: 0.17275 (0.13764) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.11864 (15.08072) + | > current_lr: 0.00004 + | > step_time: 1.80610 (2.10785) + | > loader_time: 0.00360 (0.05847) + + + --> STEP: 125/234 -- GLOBAL_STEP: 38735 + | > loss: -0.18051 (-0.15078) + | > log_mle: -0.36821 (-0.28938) + | > loss_dur: 0.18770 (0.13861) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.66835 (15.44923) + | > current_lr: 0.00004 + | > step_time: 1.73580 (2.11984) + | > loader_time: 0.00220 (0.05837) + + + --> STEP: 130/234 -- GLOBAL_STEP: 38740 + | > loss: -0.19682 (-0.15229) + | > log_mle: -0.38345 (-0.29260) + | > loss_dur: 0.18663 (0.14031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.18365 (16.32057) + | > current_lr: 0.00004 + | > step_time: 2.30670 (2.10175) + | > loader_time: 0.00330 (0.05687) + + + --> STEP: 135/234 -- GLOBAL_STEP: 38745 + | > loss: -0.14565 (-0.15394) + | > log_mle: -0.30934 (-0.29587) + | > loss_dur: 0.16368 (0.14193) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.21432 (17.19143) + | > current_lr: 0.00004 + | > step_time: 1.89730 (2.12169) + | > loader_time: 0.08830 (0.05617) + + + --> STEP: 140/234 -- GLOBAL_STEP: 38750 + | > loss: -0.15576 (-0.15559) + | > log_mle: -0.34654 (-0.29957) + | > loss_dur: 0.19078 (0.14398) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.74235 (18.15806) + | > current_lr: 0.00004 + | > step_time: 1.10170 (2.09605) + | > loader_time: 0.08300 (0.05661) + + + --> STEP: 145/234 -- GLOBAL_STEP: 38755 + | > loss: -0.25461 (-0.15791) + | > log_mle: -0.45002 (-0.30406) + | > loss_dur: 0.19541 (0.14615) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.35327 (19.13368) + | > current_lr: 0.00004 + | > step_time: 2.00540 (2.08785) + | > loader_time: 0.08730 (0.05538) + + + --> STEP: 150/234 -- GLOBAL_STEP: 38760 + | > loss: -0.21703 (-0.16023) + | > log_mle: -0.42743 (-0.30815) + | > loss_dur: 0.21040 (0.14792) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.72176 (20.44319) + | > current_lr: 0.00004 + | > step_time: 3.09950 (2.10015) + | > loader_time: 0.00330 (0.05418) + + + --> STEP: 155/234 -- GLOBAL_STEP: 38765 + | > loss: -0.27394 (-0.16334) + | > log_mle: -0.48869 (-0.31320) + | > loss_dur: 0.21475 (0.14985) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.42487 (21.61058) + | > current_lr: 0.00004 + | > step_time: 1.70240 (2.08796) + | > loader_time: 0.08710 (0.05361) + + + --> STEP: 160/234 -- GLOBAL_STEP: 38770 + | > loss: -0.27437 (-0.16577) + | > log_mle: -0.49041 (-0.31762) + | > loss_dur: 0.21604 (0.15185) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.30949 (22.68046) + | > current_lr: 0.00004 + | > step_time: 1.30700 (2.15293) + | > loader_time: 0.00380 (0.05447) + + + --> STEP: 165/234 -- GLOBAL_STEP: 38775 + | > loss: -0.27086 (-0.16841) + | > log_mle: -0.49195 (-0.32196) + | > loss_dur: 0.22109 (0.15355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.34311 (23.71025) + | > current_lr: 0.00004 + | > step_time: 2.59660 (2.14628) + | > loader_time: 0.00220 (0.05299) + + + --> STEP: 170/234 -- GLOBAL_STEP: 38780 + | > loss: -0.29344 (-0.17138) + | > log_mle: -0.53247 (-0.32678) + | > loss_dur: 0.23903 (0.15540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.04312 (24.88670) + | > current_lr: 0.00004 + | > step_time: 1.70370 (2.13723) + | > loader_time: 0.00540 (0.05254) + + + --> STEP: 175/234 -- GLOBAL_STEP: 38785 + | > loss: -0.26676 (-0.17501) + | > log_mle: -0.51196 (-0.33232) + | > loss_dur: 0.24520 (0.15731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.10601 (26.27158) + | > current_lr: 0.00004 + | > step_time: 6.09930 (2.16998) + | > loader_time: 0.10280 (0.05326) + + + --> STEP: 180/234 -- GLOBAL_STEP: 38790 + | > loss: -0.28794 (-0.17791) + | > log_mle: -0.50619 (-0.33726) + | > loss_dur: 0.21825 (0.15935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.65833 (27.62678) + | > current_lr: 0.00004 + | > step_time: 7.30160 (2.23466) + | > loader_time: 0.10200 (0.05500) + + + --> STEP: 185/234 -- GLOBAL_STEP: 38795 + | > loss: -0.29720 (-0.18066) + | > log_mle: -0.53821 (-0.34189) + | > loss_dur: 0.24102 (0.16123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.28421 (28.62650) + | > current_lr: 0.00004 + | > step_time: 3.30160 (2.27856) + | > loader_time: 0.19420 (0.05582) + + + --> STEP: 190/234 -- GLOBAL_STEP: 38800 + | > loss: -0.29056 (-0.18344) + | > log_mle: -0.50964 (-0.34649) + | > loss_dur: 0.21907 (0.16306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.11201 (29.64835) + | > current_lr: 0.00004 + | > step_time: 2.00300 (2.30956) + | > loader_time: 0.08370 (0.05636) + + + --> STEP: 195/234 -- GLOBAL_STEP: 38805 + | > loss: -0.29330 (-0.18685) + | > log_mle: -0.53671 (-0.35144) + | > loss_dur: 0.24341 (0.16458) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.03739 (30.66911) + | > current_lr: 0.00004 + | > step_time: 3.99520 (2.33646) + | > loader_time: 0.00430 (0.05656) + + + --> STEP: 200/234 -- GLOBAL_STEP: 38810 + | > loss: -0.29770 (-0.18985) + | > log_mle: -0.54402 (-0.35602) + | > loss_dur: 0.24632 (0.16618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.87498 (31.80702) + | > current_lr: 0.00004 + | > step_time: 2.19670 (2.36253) + | > loader_time: 0.09880 (0.05767) + + + --> STEP: 205/234 -- GLOBAL_STEP: 38815 + | > loss: -0.27783 (-0.19224) + | > log_mle: -0.51933 (-0.36022) + | > loss_dur: 0.24150 (0.16798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.73273 (32.98960) + | > current_lr: 0.00004 + | > step_time: 2.79080 (2.45847) + | > loader_time: 0.11130 (0.05828) + + + --> STEP: 210/234 -- GLOBAL_STEP: 38820 + | > loss: -0.36813 (-0.19563) + | > log_mle: -0.61145 (-0.36531) + | > loss_dur: 0.24332 (0.16968) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.03866 (34.10477) + | > current_lr: 0.00004 + | > step_time: 3.40720 (2.49859) + | > loader_time: 0.08760 (0.06003) + + + --> STEP: 215/234 -- GLOBAL_STEP: 38825 + | > loss: -0.31585 (-0.19930) + | > log_mle: -0.55764 (-0.37065) + | > loss_dur: 0.24179 (0.17135) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.85188 (35.43744) + | > current_lr: 0.00004 + | > step_time: 11.30630 (2.65305) + | > loader_time: 0.09700 (0.05996) + + + --> STEP: 220/234 -- GLOBAL_STEP: 38830 + | > loss: -0.36336 (-0.20316) + | > log_mle: -0.60883 (-0.37626) + | > loss_dur: 0.24547 (0.17310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.85769 (36.79848) + | > current_lr: 0.00004 + | > step_time: 1.21030 (2.66655) + | > loader_time: 0.00470 (0.05872) + + + --> STEP: 225/234 -- GLOBAL_STEP: 38835 + | > loss: -0.42654 (-0.20693) + | > log_mle: -0.68999 (-0.38172) + | > loss_dur: 0.26345 (0.17479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.82388 (37.95800) + | > current_lr: 0.00004 + | > step_time: 0.23260 (2.61253) + | > loader_time: 0.00380 (0.05749) + + + --> STEP: 230/234 -- GLOBAL_STEP: 38840 + | > loss: -0.38675 (-0.21037) + | > log_mle: -0.72412 (-0.38759) + | > loss_dur: 0.33737 (0.17722) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 141.64969 (39.64824) + | > current_lr: 0.00004 + | > step_time: 0.24710 (2.56121) + | > loader_time: 0.00340 (0.05633) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02602 (-0.10665) + | > avg_loss: -0.21064 (+0.04430) + | > avg_log_mle: -0.44518 (+0.02903) + | > avg_loss_dur: 0.23454 (+0.01527) + + + > EPOCH: 166/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:48:39)  + + --> STEP: 1/234 -- GLOBAL_STEP: 38845 + | > loss: -0.15288 (-0.15288) + | > log_mle: -0.27917 (-0.27917) + | > loss_dur: 0.12629 (0.12629) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.39132 (23.39132) + | > current_lr: 0.00004 + | > step_time: 2.69570 (2.69574) + | > loader_time: 0.10160 (0.10161) + + + --> STEP: 6/234 -- GLOBAL_STEP: 38850 + | > loss: -0.17396 (-0.13739) + | > log_mle: -0.27182 (-0.27629) + | > loss_dur: 0.09786 (0.13891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.43204 (20.74450) + | > current_lr: 0.00004 + | > step_time: 6.10120 (5.85077) + | > loader_time: 0.00280 (0.11509) + + + --> STEP: 11/234 -- GLOBAL_STEP: 38855 + | > loss: -0.18461 (-0.15061) + | > log_mle: -0.28180 (-0.28028) + | > loss_dur: 0.09719 (0.12968) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.22796 (17.41468) + | > current_lr: 0.00004 + | > step_time: 5.80040 (5.50162) + | > loader_time: 0.09810 (0.12657) + + + --> STEP: 16/234 -- GLOBAL_STEP: 38860 + | > loss: -0.18156 (-0.15655) + | > log_mle: -0.27782 (-0.28078) + | > loss_dur: 0.09625 (0.12423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.94665 (15.37701) + | > current_lr: 0.00004 + | > step_time: 1.59450 (4.26892) + | > loader_time: 0.00390 (0.09983) + + + --> STEP: 21/234 -- GLOBAL_STEP: 38865 + | > loss: -0.14732 (-0.15604) + | > log_mle: -0.25616 (-0.27718) + | > loss_dur: 0.10884 (0.12114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.79969 (13.83439) + | > current_lr: 0.00004 + | > step_time: 1.76420 (3.60405) + | > loader_time: 0.00570 (0.08445) + + + --> STEP: 26/234 -- GLOBAL_STEP: 38870 + | > loss: -0.14513 (-0.15748) + | > log_mle: -0.26955 (-0.27599) + | > loss_dur: 0.12442 (0.11851) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.81573 (13.26975) + | > current_lr: 0.00004 + | > step_time: 1.57820 (3.14620) + | > loader_time: 0.00140 (0.06848) + + + --> STEP: 31/234 -- GLOBAL_STEP: 38875 + | > loss: -0.11912 (-0.15731) + | > log_mle: -0.26776 (-0.27511) + | > loss_dur: 0.14864 (0.11780) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.46608 (12.88394) + | > current_lr: 0.00004 + | > step_time: 1.02510 (2.84711) + | > loader_time: 0.00190 (0.05774) + + + --> STEP: 36/234 -- GLOBAL_STEP: 38880 + | > loss: -0.13142 (-0.15609) + | > log_mle: -0.26626 (-0.27444) + | > loss_dur: 0.13484 (0.11834) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.63101 (12.70285) + | > current_lr: 0.00004 + | > step_time: 1.70220 (2.63025) + | > loader_time: 0.00260 (0.04999) + + + --> STEP: 41/234 -- GLOBAL_STEP: 38885 + | > loss: -0.16637 (-0.15531) + | > log_mle: -0.27296 (-0.27390) + | > loss_dur: 0.10659 (0.11859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.94149 (12.29607) + | > current_lr: 0.00004 + | > step_time: 1.49780 (2.49224) + | > loader_time: 0.00200 (0.04690) + + + --> STEP: 46/234 -- GLOBAL_STEP: 38890 + | > loss: -0.12965 (-0.15420) + | > log_mle: -0.26600 (-0.27365) + | > loss_dur: 0.13635 (0.11945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.65874 (12.36342) + | > current_lr: 0.00004 + | > step_time: 1.39640 (2.38410) + | > loader_time: 0.00280 (0.04383) + + + --> STEP: 51/234 -- GLOBAL_STEP: 38895 + | > loss: -0.14329 (-0.15360) + | > log_mle: -0.25936 (-0.27292) + | > loss_dur: 0.11607 (0.11931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.38399 (11.97002) + | > current_lr: 0.00004 + | > step_time: 0.98740 (2.26768) + | > loader_time: 0.00190 (0.04134) + + + --> STEP: 56/234 -- GLOBAL_STEP: 38900 + | > loss: -0.12917 (-0.15305) + | > log_mle: -0.27571 (-0.27318) + | > loss_dur: 0.14654 (0.12013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.39101 (11.77096) + | > current_lr: 0.00004 + | > step_time: 4.10890 (2.24217) + | > loader_time: 0.08320 (0.04259) + + + --> STEP: 61/234 -- GLOBAL_STEP: 38905 + | > loss: -0.15770 (-0.15301) + | > log_mle: -0.27235 (-0.27334) + | > loss_dur: 0.11465 (0.12033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.13983 (11.65746) + | > current_lr: 0.00004 + | > step_time: 0.98810 (2.16917) + | > loader_time: 0.00210 (0.04064) + + + --> STEP: 66/234 -- GLOBAL_STEP: 38910 + | > loss: -0.14819 (-0.15208) + | > log_mle: -0.26081 (-0.27342) + | > loss_dur: 0.11262 (0.12134) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.19173 (11.55873) + | > current_lr: 0.00004 + | > step_time: 1.19750 (2.12737) + | > loader_time: 0.00370 (0.03779) + + + --> STEP: 71/234 -- GLOBAL_STEP: 38915 + | > loss: -0.12642 (-0.15024) + | > log_mle: -0.29633 (-0.27333) + | > loss_dur: 0.16991 (0.12309) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.99993 (11.60513) + | > current_lr: 0.00004 + | > step_time: 1.14140 (2.11537) + | > loader_time: 0.00230 (0.03531) + + + --> STEP: 76/234 -- GLOBAL_STEP: 38920 + | > loss: -0.14575 (-0.14889) + | > log_mle: -0.28034 (-0.27323) + | > loss_dur: 0.13459 (0.12435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.50917 (11.71474) + | > current_lr: 0.00004 + | > step_time: 3.05600 (2.09689) + | > loader_time: 0.18520 (0.03795) + + + --> STEP: 81/234 -- GLOBAL_STEP: 38925 + | > loss: -0.13995 (-0.14832) + | > log_mle: -0.28760 (-0.27326) + | > loss_dur: 0.14764 (0.12494) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.05548 (11.57948) + | > current_lr: 0.00004 + | > step_time: 1.29550 (2.06144) + | > loader_time: 0.00220 (0.03574) + + + --> STEP: 86/234 -- GLOBAL_STEP: 38930 + | > loss: -0.14645 (-0.14771) + | > log_mle: -0.28626 (-0.27352) + | > loss_dur: 0.13981 (0.12581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.97588 (11.63733) + | > current_lr: 0.00004 + | > step_time: 2.00660 (2.06589) + | > loader_time: 0.08660 (0.03479) + + + --> STEP: 91/234 -- GLOBAL_STEP: 38935 + | > loss: -0.13265 (-0.14774) + | > log_mle: -0.29900 (-0.27502) + | > loss_dur: 0.16634 (0.12728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.65085 (11.96237) + | > current_lr: 0.00004 + | > step_time: 1.01110 (2.02800) + | > loader_time: 0.08330 (0.03395) + + + --> STEP: 96/234 -- GLOBAL_STEP: 38940 + | > loss: -0.13254 (-0.14896) + | > log_mle: -0.28619 (-0.27816) + | > loss_dur: 0.15364 (0.12920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.66609 (12.61290) + | > current_lr: 0.00004 + | > step_time: 1.19240 (2.00940) + | > loader_time: 0.00210 (0.03231) + + + --> STEP: 101/234 -- GLOBAL_STEP: 38945 + | > loss: -0.16645 (-0.14952) + | > log_mle: -0.34376 (-0.28034) + | > loss_dur: 0.17731 (0.13082) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.73130 (13.03328) + | > current_lr: 0.00004 + | > step_time: 1.78690 (1.99279) + | > loader_time: 0.00220 (0.03259) + + + --> STEP: 106/234 -- GLOBAL_STEP: 38950 + | > loss: -0.14034 (-0.15023) + | > log_mle: -0.33968 (-0.28299) + | > loss_dur: 0.19934 (0.13276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.85285 (13.67404) + | > current_lr: 0.00004 + | > step_time: 1.71140 (1.99623) + | > loader_time: 0.09330 (0.03539) + + + --> STEP: 111/234 -- GLOBAL_STEP: 38955 + | > loss: -0.18207 (-0.15072) + | > log_mle: -0.39081 (-0.28568) + | > loss_dur: 0.20874 (0.13496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.00394 (14.42935) + | > current_lr: 0.00004 + | > step_time: 1.36450 (1.98088) + | > loader_time: 0.00230 (0.03484) + + + --> STEP: 116/234 -- GLOBAL_STEP: 38960 + | > loss: -0.14749 (-0.15131) + | > log_mle: -0.35748 (-0.28836) + | > loss_dur: 0.20999 (0.13704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.80093 (15.18721) + | > current_lr: 0.00004 + | > step_time: 3.89640 (2.00500) + | > loader_time: 0.09860 (0.03577) + + + --> STEP: 121/234 -- GLOBAL_STEP: 38965 + | > loss: -0.12011 (-0.15187) + | > log_mle: -0.26908 (-0.29021) + | > loss_dur: 0.14896 (0.13834) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.54557 (15.59531) + | > current_lr: 0.00004 + | > step_time: 1.61310 (2.04481) + | > loader_time: 0.09900 (0.03917) + + + --> STEP: 126/234 -- GLOBAL_STEP: 38970 + | > loss: -0.21534 (-0.15290) + | > log_mle: -0.40615 (-0.29269) + | > loss_dur: 0.19081 (0.13979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.68925 (16.18471) + | > current_lr: 0.00004 + | > step_time: 1.89460 (2.02622) + | > loader_time: 0.00340 (0.03773) + + + --> STEP: 131/234 -- GLOBAL_STEP: 38975 + | > loss: -0.24126 (-0.15449) + | > log_mle: -0.44877 (-0.29618) + | > loss_dur: 0.20751 (0.14169) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.85463 (16.99189) + | > current_lr: 0.00004 + | > step_time: 1.10090 (2.01063) + | > loader_time: 0.00300 (0.03703) + + + --> STEP: 136/234 -- GLOBAL_STEP: 38980 + | > loss: -0.27454 (-0.15641) + | > log_mle: -0.49331 (-0.29968) + | > loss_dur: 0.21877 (0.14327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.32855 (17.72961) + | > current_lr: 0.00004 + | > step_time: 1.70130 (2.00110) + | > loader_time: 0.00510 (0.03745) + + + --> STEP: 141/234 -- GLOBAL_STEP: 38985 + | > loss: -0.21074 (-0.15774) + | > log_mle: -0.40253 (-0.30272) + | > loss_dur: 0.19179 (0.14498) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.46828 (18.40582) + | > current_lr: 0.00004 + | > step_time: 1.60700 (1.97900) + | > loader_time: 0.09250 (0.03687) + + + --> STEP: 146/234 -- GLOBAL_STEP: 38990 + | > loss: -0.24744 (-0.16039) + | > log_mle: -0.45065 (-0.30743) + | > loss_dur: 0.20321 (0.14704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.74122 (19.48065) + | > current_lr: 0.00004 + | > step_time: 3.40700 (1.99082) + | > loader_time: 0.09390 (0.03739) + + + --> STEP: 151/234 -- GLOBAL_STEP: 38995 + | > loss: -0.23744 (-0.16280) + | > log_mle: -0.41940 (-0.31135) + | > loss_dur: 0.18196 (0.14854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.00175 (20.42542) + | > current_lr: 0.00004 + | > step_time: 3.79940 (1.99902) + | > loader_time: 0.00360 (0.03677) + + + --> STEP: 156/234 -- GLOBAL_STEP: 39000 + | > loss: -0.26638 (-0.16636) + | > log_mle: -0.46373 (-0.31682) + | > loss_dur: 0.19735 (0.15046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.92435 (21.81222) + | > current_lr: 0.00004 + | > step_time: 1.98540 (1.98618) + | > loader_time: 0.00370 (0.03746) + + + --> STEP: 161/234 -- GLOBAL_STEP: 39005 + | > loss: -0.28203 (-0.16904) + | > log_mle: -0.48380 (-0.32130) + | > loss_dur: 0.20177 (0.15227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.88617 (23.08262) + | > current_lr: 0.00004 + | > step_time: 5.11030 (2.01146) + | > loader_time: 0.08870 (0.03747) + + + --> STEP: 166/234 -- GLOBAL_STEP: 39010 + | > loss: -0.23260 (-0.17127) + | > log_mle: -0.42129 (-0.32516) + | > loss_dur: 0.18870 (0.15389) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.70344 (24.00212) + | > current_lr: 0.00004 + | > step_time: 1.40470 (2.01429) + | > loader_time: 0.00350 (0.03786) + + + --> STEP: 171/234 -- GLOBAL_STEP: 39015 + | > loss: -0.33525 (-0.17474) + | > log_mle: -0.53769 (-0.33055) + | > loss_dur: 0.20244 (0.15581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.95471 (25.30018) + | > current_lr: 0.00004 + | > step_time: 4.10890 (2.03010) + | > loader_time: 0.19490 (0.03908) + + + --> STEP: 176/234 -- GLOBAL_STEP: 39020 + | > loss: -0.29284 (-0.17814) + | > log_mle: -0.50849 (-0.33585) + | > loss_dur: 0.21565 (0.15771) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.88902 (26.69336) + | > current_lr: 0.00004 + | > step_time: 1.60970 (2.03734) + | > loader_time: 0.00380 (0.03995) + + + --> STEP: 181/234 -- GLOBAL_STEP: 39025 + | > loss: -0.22031 (-0.18080) + | > log_mle: -0.44006 (-0.34050) + | > loss_dur: 0.21976 (0.15970) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.63765 (27.96600) + | > current_lr: 0.00004 + | > step_time: 2.90660 (2.06879) + | > loader_time: 0.00290 (0.04105) + + + --> STEP: 186/234 -- GLOBAL_STEP: 39030 + | > loss: -0.23268 (-0.18360) + | > log_mle: -0.47237 (-0.34530) + | > loss_dur: 0.23969 (0.16170) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.75902 (29.30017) + | > current_lr: 0.00004 + | > step_time: 2.50990 (2.08220) + | > loader_time: 0.00280 (0.04053) + + + --> STEP: 191/234 -- GLOBAL_STEP: 39035 + | > loss: -0.27584 (-0.18657) + | > log_mle: -0.48922 (-0.34984) + | > loss_dur: 0.21337 (0.16326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.11024 (30.46078) + | > current_lr: 0.00004 + | > step_time: 5.30110 (2.10944) + | > loader_time: 0.20180 (0.04105) + + + --> STEP: 196/234 -- GLOBAL_STEP: 39040 + | > loss: -0.25633 (-0.18958) + | > log_mle: -0.48944 (-0.35444) + | > loss_dur: 0.23311 (0.16486) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.30732 (31.54721) + | > current_lr: 0.00004 + | > step_time: 3.38890 (2.15562) + | > loader_time: 0.49640 (0.04442) + + + --> STEP: 201/234 -- GLOBAL_STEP: 39045 + | > loss: -0.21687 (-0.19221) + | > log_mle: -0.45532 (-0.35867) + | > loss_dur: 0.23846 (0.16646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.06753 (32.50480) + | > current_lr: 0.00004 + | > step_time: 2.90470 (2.17568) + | > loader_time: 0.18400 (0.04618) + + + --> STEP: 206/234 -- GLOBAL_STEP: 39050 + | > loss: -0.33042 (-0.19538) + | > log_mle: -0.56813 (-0.36344) + | > loss_dur: 0.23771 (0.16806) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.19058 (33.59966) + | > current_lr: 0.00004 + | > step_time: 9.29720 (2.25879) + | > loader_time: 0.00520 (0.04704) + + + --> STEP: 211/234 -- GLOBAL_STEP: 39055 + | > loss: -0.37666 (-0.19894) + | > log_mle: -0.62983 (-0.36881) + | > loss_dur: 0.25317 (0.16987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 115.39562 (34.82708) + | > current_lr: 0.00004 + | > step_time: 3.50180 (2.32051) + | > loader_time: 0.00570 (0.04650) + + + --> STEP: 216/234 -- GLOBAL_STEP: 39060 + | > loss: -0.33047 (-0.20186) + | > log_mle: -0.59529 (-0.37341) + | > loss_dur: 0.26482 (0.17155) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.36332 (36.57153) + | > current_lr: 0.00004 + | > step_time: 5.89920 (2.36079) + | > loader_time: 0.09700 (0.04728) + + + --> STEP: 221/234 -- GLOBAL_STEP: 39065 + | > loss: -0.29418 (-0.20472) + | > log_mle: -0.52202 (-0.37789) + | > loss_dur: 0.22784 (0.17317) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.36314 (37.67258) + | > current_lr: 0.00004 + | > step_time: 2.51230 (2.36577) + | > loader_time: 0.00350 (0.04706) + + + --> STEP: 226/234 -- GLOBAL_STEP: 39070 + | > loss: -0.37713 (-0.20834) + | > log_mle: -0.63346 (-0.38334) + | > loss_dur: 0.25632 (0.17500) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 96.73272 (38.84726) + | > current_lr: 0.00004 + | > step_time: 1.40860 (2.38924) + | > loader_time: 0.00510 (0.04725) + + + --> STEP: 231/234 -- GLOBAL_STEP: 39075 + | > loss: -0.32092 (-0.21124) + | > log_mle: -0.70884 (-0.38926) + | > loss_dur: 0.38792 (0.17802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.52190 (40.20369) + | > current_lr: 0.00004 + | > step_time: 0.27950 (2.34388) + | > loader_time: 0.00400 (0.04633) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.93095 (+0.90493) + | > avg_loss: -0.24497 (-0.03433) + | > avg_log_mle: -0.46390 (-0.01873) + | > avg_loss_dur: 0.21893 (-0.01561) + + + > EPOCH: 167/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 19:58:52)  + + --> STEP: 2/234 -- GLOBAL_STEP: 39080 + | > loss: -0.18735 (-0.16920) + | > log_mle: -0.28863 (-0.28534) + | > loss_dur: 0.10128 (0.11614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.43014 (12.37076) + | > current_lr: 0.00004 + | > step_time: 12.29860 (12.09681) + | > loader_time: 0.20070 (0.10151) + + + --> STEP: 7/234 -- GLOBAL_STEP: 39085 + | > loss: -0.17632 (-0.15281) + | > log_mle: -0.28638 (-0.27942) + | > loss_dur: 0.11005 (0.12660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45414 (16.40068) + | > current_lr: 0.00004 + | > step_time: 0.70220 (5.07366) + | > loader_time: 0.00120 (0.08452) + + + --> STEP: 12/234 -- GLOBAL_STEP: 39090 + | > loss: -0.16060 (-0.15879) + | > log_mle: -0.28275 (-0.28136) + | > loss_dur: 0.12216 (0.12258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.98953 (15.28705) + | > current_lr: 0.00004 + | > step_time: 1.98860 (3.87371) + | > loader_time: 0.00110 (0.07389) + + + --> STEP: 17/234 -- GLOBAL_STEP: 39095 + | > loss: -0.15053 (-0.16278) + | > log_mle: -0.26040 (-0.28073) + | > loss_dur: 0.10986 (0.11795) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.04884 (14.03391) + | > current_lr: 0.00004 + | > step_time: 1.31670 (3.27154) + | > loader_time: 0.00170 (0.05286) + + + --> STEP: 22/234 -- GLOBAL_STEP: 39100 + | > loss: -0.17264 (-0.16425) + | > log_mle: -0.27739 (-0.27849) + | > loss_dur: 0.10475 (0.11424) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.73558 (13.16967) + | > current_lr: 0.00004 + | > step_time: 2.89660 (2.85090) + | > loader_time: 0.00560 (0.04176) + + + --> STEP: 27/234 -- GLOBAL_STEP: 39105 + | > loss: -0.17528 (-0.16539) + | > log_mle: -0.27755 (-0.27769) + | > loss_dur: 0.10227 (0.11230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.66215 (12.59861) + | > current_lr: 0.00004 + | > step_time: 1.40050 (2.60788) + | > loader_time: 0.00200 (0.03440) + + + --> STEP: 32/234 -- GLOBAL_STEP: 39110 + | > loss: -0.18377 (-0.16537) + | > log_mle: -0.28382 (-0.27715) + | > loss_dur: 0.10005 (0.11178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.84791 (12.41184) + | > current_lr: 0.00004 + | > step_time: 1.29050 (2.39830) + | > loader_time: 0.00190 (0.02937) + + + --> STEP: 37/234 -- GLOBAL_STEP: 39115 + | > loss: -0.15925 (-0.16255) + | > log_mle: -0.26421 (-0.27590) + | > loss_dur: 0.10496 (0.11335) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.34320 (12.32005) + | > current_lr: 0.00004 + | > step_time: 1.11030 (2.23927) + | > loader_time: 0.00190 (0.02567) + + + --> STEP: 42/234 -- GLOBAL_STEP: 39120 + | > loss: -0.12905 (-0.16048) + | > log_mle: -0.25859 (-0.27507) + | > loss_dur: 0.12955 (0.11459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.52198 (11.95466) + | > current_lr: 0.00004 + | > step_time: 1.38710 (2.16853) + | > loader_time: 0.00220 (0.02499) + + + --> STEP: 47/234 -- GLOBAL_STEP: 39125 + | > loss: -0.13030 (-0.15914) + | > log_mle: -0.26815 (-0.27502) + | > loss_dur: 0.13786 (0.11589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.45718 (11.85300) + | > current_lr: 0.00004 + | > step_time: 1.01280 (2.07112) + | > loader_time: 0.00230 (0.02428) + + + --> STEP: 52/234 -- GLOBAL_STEP: 39130 + | > loss: -0.13024 (-0.15834) + | > log_mle: -0.26557 (-0.27419) + | > loss_dur: 0.13533 (0.11585) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.58934 (11.50106) + | > current_lr: 0.00004 + | > step_time: 1.40100 (2.03017) + | > loader_time: 0.08690 (0.02695) + + + --> STEP: 57/234 -- GLOBAL_STEP: 39135 + | > loss: -0.12169 (-0.15749) + | > log_mle: -0.25302 (-0.27416) + | > loss_dur: 0.13133 (0.11667) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.72871 (11.38141) + | > current_lr: 0.00004 + | > step_time: 1.40200 (1.99620) + | > loader_time: 0.08470 (0.02775) + + + --> STEP: 62/234 -- GLOBAL_STEP: 39140 + | > loss: -0.09948 (-0.15645) + | > log_mle: -0.30343 (-0.27480) + | > loss_dur: 0.20395 (0.11835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.79175 (11.68630) + | > current_lr: 0.00004 + | > step_time: 1.44440 (1.97692) + | > loader_time: 0.00210 (0.02833) + + + --> STEP: 67/234 -- GLOBAL_STEP: 39145 + | > loss: -0.14091 (-0.15544) + | > log_mle: -0.28484 (-0.27444) + | > loss_dur: 0.14393 (0.11901) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.06449 (11.58407) + | > current_lr: 0.00004 + | > step_time: 1.49980 (1.93805) + | > loader_time: 0.00260 (0.02777) + + + --> STEP: 72/234 -- GLOBAL_STEP: 39150 + | > loss: -0.12364 (-0.15307) + | > log_mle: -0.25962 (-0.27393) + | > loss_dur: 0.13597 (0.12086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.97423 (12.07574) + | > current_lr: 0.00004 + | > step_time: 1.50150 (1.96718) + | > loader_time: 0.08390 (0.02984) + + + --> STEP: 77/234 -- GLOBAL_STEP: 39155 + | > loss: -0.14925 (-0.15149) + | > log_mle: -0.27619 (-0.27397) + | > loss_dur: 0.12694 (0.12248) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.31690 (12.05945) + | > current_lr: 0.00004 + | > step_time: 0.99550 (1.94404) + | > loader_time: 0.00230 (0.02804) + + + --> STEP: 82/234 -- GLOBAL_STEP: 39160 + | > loss: -0.13878 (-0.15066) + | > log_mle: -0.27100 (-0.27386) + | > loss_dur: 0.13222 (0.12320) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.22977 (11.93232) + | > current_lr: 0.00004 + | > step_time: 1.32870 (1.91629) + | > loader_time: 0.00210 (0.02764) + + + --> STEP: 87/234 -- GLOBAL_STEP: 39165 + | > loss: -0.12861 (-0.14959) + | > log_mle: -0.27459 (-0.27415) + | > loss_dur: 0.14599 (0.12456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.29092 (12.00793) + | > current_lr: 0.00004 + | > step_time: 3.02130 (1.93262) + | > loader_time: 0.08860 (0.02922) + + + --> STEP: 92/234 -- GLOBAL_STEP: 39170 + | > loss: -0.18057 (-0.14989) + | > log_mle: -0.32413 (-0.27609) + | > loss_dur: 0.14357 (0.12620) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.84032 (12.26092) + | > current_lr: 0.00004 + | > step_time: 1.70760 (1.92586) + | > loader_time: 0.08300 (0.02862) + + + --> STEP: 97/234 -- GLOBAL_STEP: 39175 + | > loss: -0.15833 (-0.15101) + | > log_mle: -0.31177 (-0.27898) + | > loss_dur: 0.15344 (0.12796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.85010 (12.76362) + | > current_lr: 0.00004 + | > step_time: 1.88420 (1.92310) + | > loader_time: 0.00240 (0.02728) + + + --> STEP: 102/234 -- GLOBAL_STEP: 39180 + | > loss: -0.12850 (-0.15129) + | > log_mle: -0.29336 (-0.28089) + | > loss_dur: 0.16487 (0.12960) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.03965 (13.13562) + | > current_lr: 0.00004 + | > step_time: 1.11610 (1.92179) + | > loader_time: 0.00260 (0.02785) + + + --> STEP: 107/234 -- GLOBAL_STEP: 39185 + | > loss: -0.16618 (-0.15232) + | > log_mle: -0.33892 (-0.28386) + | > loss_dur: 0.17274 (0.13154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.57459 (13.79153) + | > current_lr: 0.00004 + | > step_time: 1.01220 (1.92089) + | > loader_time: 0.00350 (0.02911) + + + --> STEP: 112/234 -- GLOBAL_STEP: 39190 + | > loss: -0.15633 (-0.15257) + | > log_mle: -0.34947 (-0.28648) + | > loss_dur: 0.19314 (0.13390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.80666 (14.72443) + | > current_lr: 0.00004 + | > step_time: 2.00300 (1.93082) + | > loader_time: 0.00520 (0.02871) + + + --> STEP: 117/234 -- GLOBAL_STEP: 39195 + | > loss: -0.18311 (-0.15340) + | > log_mle: -0.34357 (-0.28896) + | > loss_dur: 0.16045 (0.13556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.36133 (15.42462) + | > current_lr: 0.00004 + | > step_time: 0.92140 (1.90754) + | > loader_time: 0.10350 (0.02916) + + + --> STEP: 122/234 -- GLOBAL_STEP: 39200 + | > loss: -0.15257 (-0.15353) + | > log_mle: -0.31913 (-0.29052) + | > loss_dur: 0.16656 (0.13699) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.82222 (15.71304) + | > current_lr: 0.00004 + | > step_time: 1.39170 (1.89386) + | > loader_time: 0.00360 (0.02870) + + + --> STEP: 127/234 -- GLOBAL_STEP: 39205 + | > loss: -0.19169 (-0.15472) + | > log_mle: -0.37885 (-0.29343) + | > loss_dur: 0.18716 (0.13871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.21476 (16.44172) + | > current_lr: 0.00004 + | > step_time: 2.39140 (1.89809) + | > loader_time: 0.00340 (0.02841) + + + --> STEP: 132/234 -- GLOBAL_STEP: 39210 + | > loss: -0.19548 (-0.15644) + | > log_mle: -0.35948 (-0.29677) + | > loss_dur: 0.16400 (0.14032) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.23495 (17.27187) + | > current_lr: 0.00004 + | > step_time: 3.89510 (1.92929) + | > loader_time: 0.00290 (0.02943) + + + --> STEP: 137/234 -- GLOBAL_STEP: 39215 + | > loss: -0.16937 (-0.15803) + | > log_mle: -0.37439 (-0.30037) + | > loss_dur: 0.20502 (0.14234) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.39488 (18.21976) + | > current_lr: 0.00004 + | > step_time: 1.39190 (1.90996) + | > loader_time: 0.00330 (0.02985) + + + --> STEP: 142/234 -- GLOBAL_STEP: 39220 + | > loss: -0.18233 (-0.15930) + | > log_mle: -0.38349 (-0.30335) + | > loss_dur: 0.20116 (0.14405) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.79266 (19.01760) + | > current_lr: 0.00004 + | > step_time: 2.00340 (1.90056) + | > loader_time: 0.00390 (0.02949) + + + --> STEP: 147/234 -- GLOBAL_STEP: 39225 + | > loss: -0.19061 (-0.16142) + | > log_mle: -0.38500 (-0.30777) + | > loss_dur: 0.19439 (0.14636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.32545 (20.28158) + | > current_lr: 0.00004 + | > step_time: 2.61050 (1.90892) + | > loader_time: 0.00360 (0.03039) + + + --> STEP: 152/234 -- GLOBAL_STEP: 39230 + | > loss: -0.22463 (-0.16365) + | > log_mle: -0.45818 (-0.31186) + | > loss_dur: 0.23355 (0.14821) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.34060 (21.51155) + | > current_lr: 0.00004 + | > step_time: 1.71410 (1.90682) + | > loader_time: 0.08650 (0.03059) + + + --> STEP: 157/234 -- GLOBAL_STEP: 39235 + | > loss: -0.20265 (-0.16660) + | > log_mle: -0.41328 (-0.31661) + | > loss_dur: 0.21064 (0.15001) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.48222 (22.66908) + | > current_lr: 0.00004 + | > step_time: 1.80310 (1.93588) + | > loader_time: 0.00370 (0.03100) + + + --> STEP: 162/234 -- GLOBAL_STEP: 39240 + | > loss: -0.25421 (-0.16924) + | > log_mle: -0.44487 (-0.32107) + | > loss_dur: 0.19065 (0.15182) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.08160 (23.67414) + | > current_lr: 0.00004 + | > step_time: 2.00230 (1.95572) + | > loader_time: 0.10000 (0.03142) + + + --> STEP: 167/234 -- GLOBAL_STEP: 39245 + | > loss: -0.32827 (-0.17190) + | > log_mle: -0.52954 (-0.32528) + | > loss_dur: 0.20127 (0.15337) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.14477 (24.65786) + | > current_lr: 0.00004 + | > step_time: 4.30320 (2.03970) + | > loader_time: 0.10150 (0.03346) + + + --> STEP: 172/234 -- GLOBAL_STEP: 39250 + | > loss: -0.29934 (-0.17497) + | > log_mle: -0.52827 (-0.33045) + | > loss_dur: 0.22894 (0.15548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.27766 (25.69807) + | > current_lr: 0.00004 + | > step_time: 2.09380 (2.05545) + | > loader_time: 0.00470 (0.03306) + + + --> STEP: 177/234 -- GLOBAL_STEP: 39255 + | > loss: -0.27426 (-0.17789) + | > log_mle: -0.48902 (-0.33532) + | > loss_dur: 0.21476 (0.15743) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.20641 (26.74721) + | > current_lr: 0.00004 + | > step_time: 2.99520 (2.09859) + | > loader_time: 0.00390 (0.03332) + + + --> STEP: 182/234 -- GLOBAL_STEP: 39260 + | > loss: -0.28314 (-0.18058) + | > log_mle: -0.52727 (-0.34010) + | > loss_dur: 0.24412 (0.15952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.20579 (27.90153) + | > current_lr: 0.00004 + | > step_time: 5.40000 (2.19310) + | > loader_time: 0.21520 (0.03418) + + + --> STEP: 187/234 -- GLOBAL_STEP: 39265 + | > loss: -0.30206 (-0.18347) + | > log_mle: -0.53220 (-0.34491) + | > loss_dur: 0.23014 (0.16144) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.31796 (28.94143) + | > current_lr: 0.00004 + | > step_time: 12.99310 (2.28367) + | > loader_time: 0.18880 (0.03848) + + + --> STEP: 192/234 -- GLOBAL_STEP: 39270 + | > loss: -0.34098 (-0.18647) + | > log_mle: -0.55693 (-0.34946) + | > loss_dur: 0.21596 (0.16298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.85052 (30.19661) + | > current_lr: 0.00004 + | > step_time: 2.98660 (2.30213) + | > loader_time: 0.00260 (0.03899) + + + --> STEP: 197/234 -- GLOBAL_STEP: 39275 + | > loss: -0.31848 (-0.18928) + | > log_mle: -0.52704 (-0.35385) + | > loss_dur: 0.20857 (0.16457) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.17583 (31.27473) + | > current_lr: 0.00004 + | > step_time: 1.79620 (2.38475) + | > loader_time: 0.00370 (0.03813) + + + --> STEP: 202/234 -- GLOBAL_STEP: 39280 + | > loss: -0.37963 (-0.19211) + | > log_mle: -0.61068 (-0.35842) + | > loss_dur: 0.23105 (0.16631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.99991 (32.48346) + | > current_lr: 0.00004 + | > step_time: 6.11460 (2.49913) + | > loader_time: 0.09330 (0.04062) + + + --> STEP: 207/234 -- GLOBAL_STEP: 39285 + | > loss: -0.34654 (-0.19462) + | > log_mle: -0.58923 (-0.36269) + | > loss_dur: 0.24269 (0.16807) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.04809 (34.09851) + | > current_lr: 0.00004 + | > step_time: 6.21500 (2.53937) + | > loader_time: 0.29320 (0.04244) + + + --> STEP: 212/234 -- GLOBAL_STEP: 39290 + | > loss: -0.33642 (-0.19787) + | > log_mle: -0.58156 (-0.36773) + | > loss_dur: 0.24513 (0.16985) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.43047 (35.23700) + | > current_lr: 0.00004 + | > step_time: 8.19060 (2.64931) + | > loader_time: 0.01810 (0.04538) + + + --> STEP: 217/234 -- GLOBAL_STEP: 39295 + | > loss: -0.36732 (-0.20138) + | > log_mle: -0.61520 (-0.37292) + | > loss_dur: 0.24788 (0.17154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.16364 (36.45797) + | > current_lr: 0.00004 + | > step_time: 5.31620 (2.68062) + | > loader_time: 0.18520 (0.04694) + + + --> STEP: 222/234 -- GLOBAL_STEP: 39300 + | > loss: -0.35186 (-0.20490) + | > log_mle: -0.62283 (-0.37808) + | > loss_dur: 0.27097 (0.17319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.66830 (37.47144) + | > current_lr: 0.00004 + | > step_time: 0.23130 (2.64275) + | > loader_time: 0.00340 (0.04712) + + + --> STEP: 227/234 -- GLOBAL_STEP: 39305 + | > loss: -0.32476 (-0.20863) + | > log_mle: -0.58413 (-0.38356) + | > loss_dur: 0.25937 (0.17493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 127.24739 (38.90979) + | > current_lr: 0.00004 + | > step_time: 0.23900 (2.58974) + | > loader_time: 0.00570 (0.04618) + + + --> STEP: 232/234 -- GLOBAL_STEP: 39310 + | > loss: -0.25370 (-0.21099) + | > log_mle: -0.75429 (-0.38995) + | > loss_dur: 0.50059 (0.17896) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 208.22321 (41.06191) + | > current_lr: 0.00004 + | > step_time: 0.32620 (2.53965) + | > loader_time: 0.01250 (0.04531) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00186 (-0.92909) + | > avg_loss: -0.24731 (-0.00234) + | > avg_log_mle: -0.47373 (-0.00983) + | > avg_loss_dur: 0.22641 (+0.00749) + + + > EPOCH: 168/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 20:09:58)  + + --> STEP: 3/234 -- GLOBAL_STEP: 39315 + | > loss: -0.09005 (-0.13888) + | > log_mle: -0.27247 (-0.28011) + | > loss_dur: 0.18242 (0.14123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.30659 (19.89534) + | > current_lr: 0.00004 + | > step_time: 1.18890 (7.33659) + | > loader_time: 0.10160 (0.10030) + + + --> STEP: 8/234 -- GLOBAL_STEP: 39320 + | > loss: -0.15401 (-0.15163) + | > log_mle: -0.29467 (-0.28113) + | > loss_dur: 0.14066 (0.12950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.42337 (17.04976) + | > current_lr: 0.00004 + | > step_time: 9.60690 (4.85150) + | > loader_time: 0.08440 (0.05919) + + + --> STEP: 13/234 -- GLOBAL_STEP: 39325 + | > loss: -0.19122 (-0.15750) + | > log_mle: -0.28924 (-0.28241) + | > loss_dur: 0.09802 (0.12492) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.05656 (15.63747) + | > current_lr: 0.00004 + | > step_time: 1.50770 (3.88658) + | > loader_time: 0.07620 (0.06444) + + + --> STEP: 18/234 -- GLOBAL_STEP: 39330 + | > loss: -0.14877 (-0.16015) + | > log_mle: -0.27380 (-0.28058) + | > loss_dur: 0.12504 (0.12043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.88451 (14.63923) + | > current_lr: 0.00004 + | > step_time: 2.02170 (3.30656) + | > loader_time: 0.00160 (0.04744) + + + --> STEP: 23/234 -- GLOBAL_STEP: 39335 + | > loss: -0.18451 (-0.16273) + | > log_mle: -0.28362 (-0.27928) + | > loss_dur: 0.09911 (0.11654) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.25755 (13.59296) + | > current_lr: 0.00004 + | > step_time: 0.88150 (2.83572) + | > loader_time: 0.00140 (0.04119) + + + --> STEP: 28/234 -- GLOBAL_STEP: 39340 + | > loss: -0.20049 (-0.16402) + | > log_mle: -0.28390 (-0.27868) + | > loss_dur: 0.08341 (0.11466) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.22552 (12.78650) + | > current_lr: 0.00004 + | > step_time: 4.10420 (2.64854) + | > loader_time: 0.00430 (0.03422) + + + --> STEP: 33/234 -- GLOBAL_STEP: 39345 + | > loss: -0.16167 (-0.16317) + | > log_mle: -0.26996 (-0.27792) + | > loss_dur: 0.10829 (0.11474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.77641 (12.41431) + | > current_lr: 0.00004 + | > step_time: 1.16080 (2.59305) + | > loader_time: 0.00190 (0.03199) + + + --> STEP: 38/234 -- GLOBAL_STEP: 39350 + | > loss: -0.16517 (-0.16180) + | > log_mle: -0.28729 (-0.27758) + | > loss_dur: 0.12212 (0.11578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.31340 (11.98530) + | > current_lr: 0.00004 + | > step_time: 1.65550 (2.54535) + | > loader_time: 0.00190 (0.03283) + + + --> STEP: 43/234 -- GLOBAL_STEP: 39355 + | > loss: -0.14611 (-0.16084) + | > log_mle: -0.28272 (-0.27688) + | > loss_dur: 0.13661 (0.11604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.00073 (11.86042) + | > current_lr: 0.00004 + | > step_time: 1.66610 (2.39631) + | > loader_time: 0.00190 (0.02927) + + + --> STEP: 48/234 -- GLOBAL_STEP: 39360 + | > loss: -0.17040 (-0.16023) + | > log_mle: -0.26739 (-0.27658) + | > loss_dur: 0.09699 (0.11635) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.31678 (11.72892) + | > current_lr: 0.00004 + | > step_time: 1.60060 (2.30936) + | > loader_time: 0.00200 (0.02836) + + + --> STEP: 53/234 -- GLOBAL_STEP: 39365 + | > loss: -0.13485 (-0.15894) + | > log_mle: -0.28106 (-0.27609) + | > loss_dur: 0.14621 (0.11715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.31003 (11.45955) + | > current_lr: 0.00004 + | > step_time: 1.20430 (2.25277) + | > loader_time: 0.00220 (0.02588) + + + --> STEP: 58/234 -- GLOBAL_STEP: 39370 + | > loss: -0.15102 (-0.15778) + | > log_mle: -0.27057 (-0.27594) + | > loss_dur: 0.11955 (0.11816) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.88076 (11.26324) + | > current_lr: 0.00004 + | > step_time: 1.50910 (2.18889) + | > loader_time: 0.08690 (0.02536) + + + --> STEP: 63/234 -- GLOBAL_STEP: 39375 + | > loss: -0.12363 (-0.15653) + | > log_mle: -0.27043 (-0.27675) + | > loss_dur: 0.14680 (0.12021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.35182 (11.60672) + | > current_lr: 0.00004 + | > step_time: 1.19610 (2.20421) + | > loader_time: 0.00230 (0.02830) + + + --> STEP: 68/234 -- GLOBAL_STEP: 39380 + | > loss: -0.12196 (-0.15562) + | > log_mle: -0.26472 (-0.27634) + | > loss_dur: 0.14276 (0.12071) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.46870 (11.48057) + | > current_lr: 0.00004 + | > step_time: 1.81670 (2.14775) + | > loader_time: 0.08570 (0.02762) + + + --> STEP: 73/234 -- GLOBAL_STEP: 39385 + | > loss: -0.12731 (-0.15391) + | > log_mle: -0.28813 (-0.27625) + | > loss_dur: 0.16082 (0.12234) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.20102 (11.56224) + | > current_lr: 0.00004 + | > step_time: 2.58100 (2.14387) + | > loader_time: 0.00290 (0.02591) + + + --> STEP: 78/234 -- GLOBAL_STEP: 39390 + | > loss: -0.12181 (-0.15287) + | > log_mle: -0.26277 (-0.27605) + | > loss_dur: 0.14096 (0.12318) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.26959 (11.78195) + | > current_lr: 0.00004 + | > step_time: 1.12960 (2.10376) + | > loader_time: 0.00260 (0.02438) + + + --> STEP: 83/234 -- GLOBAL_STEP: 39395 + | > loss: -0.12330 (-0.15215) + | > log_mle: -0.28778 (-0.27618) + | > loss_dur: 0.16448 (0.12403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.59648 (11.87811) + | > current_lr: 0.00004 + | > step_time: 2.10320 (2.07933) + | > loader_time: 0.00490 (0.02504) + + + --> STEP: 88/234 -- GLOBAL_STEP: 39400 + | > loss: -0.16481 (-0.15170) + | > log_mle: -0.32201 (-0.27681) + | > loss_dur: 0.15720 (0.12512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.45937 (11.94489) + | > current_lr: 0.00004 + | > step_time: 1.28940 (2.04546) + | > loader_time: 0.00210 (0.02377) + + + --> STEP: 93/234 -- GLOBAL_STEP: 39405 + | > loss: -0.16240 (-0.15210) + | > log_mle: -0.33485 (-0.27876) + | > loss_dur: 0.17245 (0.12667) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.84621 (12.27692) + | > current_lr: 0.00004 + | > step_time: 1.63290 (2.03104) + | > loader_time: 0.00230 (0.02523) + + + --> STEP: 98/234 -- GLOBAL_STEP: 39410 + | > loss: -0.12569 (-0.15263) + | > log_mle: -0.26635 (-0.28076) + | > loss_dur: 0.14066 (0.12813) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.37708 (12.59514) + | > current_lr: 0.00004 + | > step_time: 1.63970 (2.03393) + | > loader_time: 0.00330 (0.02408) + + + --> STEP: 103/234 -- GLOBAL_STEP: 39415 + | > loss: -0.18325 (-0.15332) + | > log_mle: -0.36707 (-0.28346) + | > loss_dur: 0.18382 (0.13013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.40255 (13.26505) + | > current_lr: 0.00004 + | > step_time: 1.18490 (2.00934) + | > loader_time: 0.00280 (0.02471) + + + --> STEP: 108/234 -- GLOBAL_STEP: 39420 + | > loss: -0.15820 (-0.15390) + | > log_mle: -0.30767 (-0.28570) + | > loss_dur: 0.14948 (0.13180) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.78381 (13.84566) + | > current_lr: 0.00004 + | > step_time: 1.17810 (2.04094) + | > loader_time: 0.00230 (0.02451) + + + --> STEP: 113/234 -- GLOBAL_STEP: 39425 + | > loss: -0.17475 (-0.15430) + | > log_mle: -0.35739 (-0.28864) + | > loss_dur: 0.18264 (0.13434) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.62187 (14.51983) + | > current_lr: 0.00004 + | > step_time: 1.80770 (2.02176) + | > loader_time: 0.00330 (0.02355) + + + --> STEP: 118/234 -- GLOBAL_STEP: 39430 + | > loss: -0.15662 (-0.15469) + | > log_mle: -0.32567 (-0.29080) + | > loss_dur: 0.16905 (0.13612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.81864 (14.97126) + | > current_lr: 0.00004 + | > step_time: 3.50950 (2.02594) + | > loader_time: 0.18730 (0.02493) + + + --> STEP: 123/234 -- GLOBAL_STEP: 39435 + | > loss: -0.13462 (-0.15456) + | > log_mle: -0.29600 (-0.29195) + | > loss_dur: 0.16138 (0.13739) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.29490 (15.25878) + | > current_lr: 0.00004 + | > step_time: 1.80230 (2.01694) + | > loader_time: 0.00340 (0.02470) + + + --> STEP: 128/234 -- GLOBAL_STEP: 39440 + | > loss: -0.19199 (-0.15613) + | > log_mle: -0.35801 (-0.29524) + | > loss_dur: 0.16602 (0.13911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.21236 (15.91980) + | > current_lr: 0.00004 + | > step_time: 2.39840 (2.02805) + | > loader_time: 0.00300 (0.02450) + + + --> STEP: 133/234 -- GLOBAL_STEP: 39445 + | > loss: -0.19806 (-0.15784) + | > log_mle: -0.38083 (-0.29869) + | > loss_dur: 0.18278 (0.14085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.82578 (16.70002) + | > current_lr: 0.00004 + | > step_time: 1.70790 (2.05020) + | > loader_time: 0.08690 (0.02440) + + + --> STEP: 138/234 -- GLOBAL_STEP: 39450 + | > loss: -0.15841 (-0.15908) + | > log_mle: -0.33606 (-0.30180) + | > loss_dur: 0.17765 (0.14273) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.48716 (17.42182) + | > current_lr: 0.00004 + | > step_time: 1.90390 (2.05636) + | > loader_time: 0.00300 (0.02546) + + + --> STEP: 143/234 -- GLOBAL_STEP: 39455 + | > loss: -0.23512 (-0.16089) + | > log_mle: -0.47856 (-0.30582) + | > loss_dur: 0.24344 (0.14493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.66654 (18.40927) + | > current_lr: 0.00004 + | > step_time: 1.69520 (2.06057) + | > loader_time: 0.00420 (0.02589) + + + --> STEP: 148/234 -- GLOBAL_STEP: 39460 + | > loss: -0.22526 (-0.16325) + | > log_mle: -0.39609 (-0.30986) + | > loss_dur: 0.17083 (0.14662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.65123 (19.22755) + | > current_lr: 0.00004 + | > step_time: 5.59800 (2.12670) + | > loader_time: 0.09990 (0.02693) + + + --> STEP: 153/234 -- GLOBAL_STEP: 39465 + | > loss: -0.31716 (-0.16637) + | > log_mle: -0.51931 (-0.31494) + | > loss_dur: 0.20215 (0.14857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.45786 (20.64783) + | > current_lr: 0.00004 + | > step_time: 3.00010 (2.16240) + | > loader_time: 0.08430 (0.02786) + + + --> STEP: 158/234 -- GLOBAL_STEP: 39470 + | > loss: -0.23349 (-0.16894) + | > log_mle: -0.45305 (-0.31920) + | > loss_dur: 0.21956 (0.15025) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.69466 (22.12366) + | > current_lr: 0.00004 + | > step_time: 1.89780 (2.14835) + | > loader_time: 0.00310 (0.02709) + + + --> STEP: 163/234 -- GLOBAL_STEP: 39475 + | > loss: -0.21832 (-0.17172) + | > log_mle: -0.42290 (-0.32357) + | > loss_dur: 0.20458 (0.15185) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.59106 (23.18050) + | > current_lr: 0.00004 + | > step_time: 6.01940 (2.22545) + | > loader_time: 0.00310 (0.02760) + + + --> STEP: 168/234 -- GLOBAL_STEP: 39480 + | > loss: -0.24361 (-0.17447) + | > log_mle: -0.47796 (-0.32810) + | > loss_dur: 0.23435 (0.15363) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.68568 (24.34652) + | > current_lr: 0.00004 + | > step_time: 1.50390 (2.21145) + | > loader_time: 0.08300 (0.02833) + + + --> STEP: 173/234 -- GLOBAL_STEP: 39485 + | > loss: -0.27195 (-0.17765) + | > log_mle: -0.48781 (-0.33324) + | > loss_dur: 0.21586 (0.15559) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.39114 (25.55912) + | > current_lr: 0.00004 + | > step_time: 4.51240 (2.22046) + | > loader_time: 0.18460 (0.02914) + + + --> STEP: 178/234 -- GLOBAL_STEP: 39490 + | > loss: -0.30789 (-0.18094) + | > log_mle: -0.54955 (-0.33845) + | > loss_dur: 0.24165 (0.15751) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.18008 (26.53140) + | > current_lr: 0.00004 + | > step_time: 2.41320 (2.23729) + | > loader_time: 0.19060 (0.03060) + + + --> STEP: 183/234 -- GLOBAL_STEP: 39495 + | > loss: -0.33405 (-0.18384) + | > log_mle: -0.55312 (-0.34336) + | > loss_dur: 0.21907 (0.15952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.18562 (27.47138) + | > current_lr: 0.00004 + | > step_time: 1.69640 (2.23300) + | > loader_time: 0.00360 (0.03033) + + + --> STEP: 188/234 -- GLOBAL_STEP: 39500 + | > loss: -0.32873 (-0.18667) + | > log_mle: -0.55538 (-0.34821) + | > loss_dur: 0.22665 (0.16153) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.44888 (28.62897) + | > current_lr: 0.00004 + | > step_time: 1.51180 (2.24555) + | > loader_time: 0.00360 (0.03015) + + + --> STEP: 193/234 -- GLOBAL_STEP: 39505 + | > loss: -0.34622 (-0.18992) + | > log_mle: -0.56415 (-0.35295) + | > loss_dur: 0.21793 (0.16304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.23519 (29.89714) + | > current_lr: 0.00004 + | > step_time: 7.60740 (2.30824) + | > loader_time: 0.08310 (0.03194) + + + --> STEP: 198/234 -- GLOBAL_STEP: 39510 + | > loss: -0.32664 (-0.19304) + | > log_mle: -0.56186 (-0.35765) + | > loss_dur: 0.23521 (0.16462) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.67444 (30.90660) + | > current_lr: 0.00004 + | > step_time: 4.10500 (2.33478) + | > loader_time: 0.08140 (0.03163) + + + --> STEP: 203/234 -- GLOBAL_STEP: 39515 + | > loss: -0.25224 (-0.19563) + | > log_mle: -0.47559 (-0.36190) + | > loss_dur: 0.22335 (0.16627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.36185 (32.25264) + | > current_lr: 0.00004 + | > step_time: 3.19350 (2.32711) + | > loader_time: 0.00390 (0.03145) + + + --> STEP: 208/234 -- GLOBAL_STEP: 39520 + | > loss: -0.29546 (-0.19869) + | > log_mle: -0.53857 (-0.36673) + | > loss_dur: 0.24312 (0.16804) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 269.32114 (34.37128) + | > current_lr: 0.00004 + | > step_time: 5.60690 (2.40593) + | > loader_time: 0.09930 (0.03300) + + + --> STEP: 213/234 -- GLOBAL_STEP: 39525 + | > loss: -0.31604 (-0.20102) + | > log_mle: -0.57087 (-0.37090) + | > loss_dur: 0.25483 (0.16988) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.98904 (35.78713) + | > current_lr: 0.00004 + | > step_time: 3.19830 (2.44150) + | > loader_time: 0.09290 (0.03353) + + + --> STEP: 218/234 -- GLOBAL_STEP: 39530 + | > loss: -0.29693 (-0.20369) + | > log_mle: -0.54399 (-0.37525) + | > loss_dur: 0.24706 (0.17156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.49430 (36.74514) + | > current_lr: 0.00004 + | > step_time: 3.41180 (2.53557) + | > loader_time: 0.00500 (0.03455) + + + --> STEP: 223/234 -- GLOBAL_STEP: 39535 + | > loss: -0.35606 (-0.20680) + | > log_mle: -0.59890 (-0.38014) + | > loss_dur: 0.24284 (0.17334) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.22891 (38.00536) + | > current_lr: 0.00004 + | > step_time: 0.24900 (2.53625) + | > loader_time: 0.00640 (0.03389) + + + --> STEP: 228/234 -- GLOBAL_STEP: 39540 + | > loss: -0.32433 (-0.21011) + | > log_mle: -0.60247 (-0.38537) + | > loss_dur: 0.27813 (0.17526) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.24887 (38.93716) + | > current_lr: 0.00004 + | > step_time: 0.25500 (2.48600) + | > loader_time: 0.00460 (0.03322) + + + --> STEP: 233/234 -- GLOBAL_STEP: 39545 + | > loss: 0.14520 (-0.21077) + | > log_mle: -0.57487 (-0.39187) + | > loss_dur: 0.72007 (0.18111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.52431 (40.30421) + | > current_lr: 0.00004 + | > step_time: 0.20620 (2.43840) + | > loader_time: 0.00310 (0.03261) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22265 (+0.22079) + | > avg_loss: -0.27031 (-0.02299) + | > avg_log_mle: -0.48491 (-0.01119) + | > avg_loss_dur: 0.21461 (-0.01181) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_39546.pth + + > EPOCH: 169/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 20:20:30)  + + --> STEP: 4/234 -- GLOBAL_STEP: 39550 + | > loss: -0.13701 (-0.15042) + | > log_mle: -0.27401 (-0.28006) + | > loss_dur: 0.13701 (0.12965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.39441 (20.04795) + | > current_lr: 0.00004 + | > step_time: 1.39520 (3.67690) + | > loader_time: 0.00240 (0.04932) + + + --> STEP: 9/234 -- GLOBAL_STEP: 39555 + | > loss: -0.14137 (-0.15687) + | > log_mle: -0.28949 (-0.28345) + | > loss_dur: 0.14812 (0.12657) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.76775 (16.77621) + | > current_lr: 0.00004 + | > step_time: 5.50600 (4.14705) + | > loader_time: 0.00280 (0.05424) + + + --> STEP: 14/234 -- GLOBAL_STEP: 39560 + | > loss: -0.16472 (-0.16143) + | > log_mle: -0.28610 (-0.28398) + | > loss_dur: 0.12138 (0.12255) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.22794 (15.52891) + | > current_lr: 0.00004 + | > step_time: 6.50220 (4.32294) + | > loader_time: 0.00310 (0.04996) + + + --> STEP: 19/234 -- GLOBAL_STEP: 39565 + | > loss: -0.18014 (-0.16413) + | > log_mle: -0.27633 (-0.28180) + | > loss_dur: 0.09620 (0.11766) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.42532 (13.96191) + | > current_lr: 0.00004 + | > step_time: 1.98380 (3.90083) + | > loader_time: 0.00120 (0.05285) + + + --> STEP: 24/234 -- GLOBAL_STEP: 39570 + | > loss: -0.19685 (-0.16749) + | > log_mle: -0.27716 (-0.28061) + | > loss_dur: 0.08031 (0.11312) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.21262 (12.74544) + | > current_lr: 0.00004 + | > step_time: 4.69630 (3.96349) + | > loader_time: 0.09890 (0.04666) + + + --> STEP: 29/234 -- GLOBAL_STEP: 39575 + | > loss: -0.13716 (-0.16705) + | > log_mle: -0.26227 (-0.27955) + | > loss_dur: 0.12512 (0.11250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.83978 (12.35177) + | > current_lr: 0.00004 + | > step_time: 1.24710 (3.48631) + | > loader_time: 0.00130 (0.03897) + + + --> STEP: 34/234 -- GLOBAL_STEP: 39580 + | > loss: -0.14844 (-0.16657) + | > log_mle: -0.27442 (-0.27909) + | > loss_dur: 0.12597 (0.11252) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.10038 (12.06948) + | > current_lr: 0.00004 + | > step_time: 1.81150 (3.20784) + | > loader_time: 0.08530 (0.03841) + + + --> STEP: 39/234 -- GLOBAL_STEP: 39585 + | > loss: -0.15403 (-0.16507) + | > log_mle: -0.27929 (-0.27876) + | > loss_dur: 0.12526 (0.11369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.61084 (12.20360) + | > current_lr: 0.00004 + | > step_time: 1.29490 (3.12484) + | > loader_time: 0.00150 (0.04092) + + + --> STEP: 44/234 -- GLOBAL_STEP: 39590 + | > loss: -0.17897 (-0.16348) + | > log_mle: -0.27383 (-0.27792) + | > loss_dur: 0.09486 (0.11444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.13752 (11.88924) + | > current_lr: 0.00004 + | > step_time: 1.03270 (2.88842) + | > loader_time: 0.00200 (0.03820) + + + --> STEP: 49/234 -- GLOBAL_STEP: 39595 + | > loss: -0.17262 (-0.16271) + | > log_mle: -0.27830 (-0.27786) + | > loss_dur: 0.10569 (0.11514) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.06115 (11.73160) + | > current_lr: 0.00004 + | > step_time: 1.77960 (2.73671) + | > loader_time: 0.00200 (0.03454) + + + --> STEP: 54/234 -- GLOBAL_STEP: 39600 + | > loss: -0.16558 (-0.16125) + | > log_mle: -0.28471 (-0.27737) + | > loss_dur: 0.11913 (0.11612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.69012 (11.60393) + | > current_lr: 0.00004 + | > step_time: 1.19590 (2.60266) + | > loader_time: 0.00220 (0.03155) + + + --> STEP: 59/234 -- GLOBAL_STEP: 39605 + | > loss: -0.17237 (-0.16058) + | > log_mle: -0.28799 (-0.27728) + | > loss_dur: 0.11562 (0.11670) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.95017 (11.46701) + | > current_lr: 0.00004 + | > step_time: 2.19730 (2.53183) + | > loader_time: 0.00270 (0.03186) + + + --> STEP: 64/234 -- GLOBAL_STEP: 39610 + | > loss: -0.15921 (-0.15921) + | > log_mle: -0.27246 (-0.27791) + | > loss_dur: 0.11325 (0.11871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.43511 (11.50187) + | > current_lr: 0.00004 + | > step_time: 1.20040 (2.46482) + | > loader_time: 0.08690 (0.03227) + + + --> STEP: 69/234 -- GLOBAL_STEP: 39615 + | > loss: -0.12893 (-0.15757) + | > log_mle: -0.25622 (-0.27736) + | > loss_dur: 0.12729 (0.11979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27107 (11.39294) + | > current_lr: 0.00004 + | > step_time: 2.01540 (2.42687) + | > loader_time: 0.00340 (0.03135) + + + --> STEP: 74/234 -- GLOBAL_STEP: 39620 + | > loss: -0.12959 (-0.15566) + | > log_mle: -0.26142 (-0.27739) + | > loss_dur: 0.13183 (0.12173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.42438 (11.68143) + | > current_lr: 0.00004 + | > step_time: 1.25980 (2.39138) + | > loader_time: 0.00360 (0.02943) + + + --> STEP: 79/234 -- GLOBAL_STEP: 39625 + | > loss: -0.14756 (-0.15471) + | > log_mle: -0.28270 (-0.27755) + | > loss_dur: 0.13513 (0.12284) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.04800 (11.69831) + | > current_lr: 0.00004 + | > step_time: 1.60300 (2.33093) + | > loader_time: 0.00360 (0.02773) + + + --> STEP: 84/234 -- GLOBAL_STEP: 39630 + | > loss: -0.13879 (-0.15365) + | > log_mle: -0.27354 (-0.27762) + | > loss_dur: 0.13475 (0.12397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.56670 (11.79550) + | > current_lr: 0.00004 + | > step_time: 4.60690 (2.32468) + | > loader_time: 0.00230 (0.02723) + + + --> STEP: 89/234 -- GLOBAL_STEP: 39635 + | > loss: -0.16877 (-0.15346) + | > log_mle: -0.30534 (-0.27860) + | > loss_dur: 0.13657 (0.12514) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.26373 (12.02433) + | > current_lr: 0.00004 + | > step_time: 1.79230 (2.28376) + | > loader_time: 0.00210 (0.02677) + + + --> STEP: 94/234 -- GLOBAL_STEP: 39640 + | > loss: -0.19006 (-0.15407) + | > log_mle: -0.34150 (-0.28099) + | > loss_dur: 0.15143 (0.12691) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.59995 (12.46498) + | > current_lr: 0.00004 + | > step_time: 2.50610 (2.27824) + | > loader_time: 0.00290 (0.02841) + + + --> STEP: 99/234 -- GLOBAL_STEP: 39645 + | > loss: -0.18251 (-0.15478) + | > log_mle: -0.37210 (-0.28332) + | > loss_dur: 0.18960 (0.12854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.44062 (12.93480) + | > current_lr: 0.00004 + | > step_time: 1.50080 (2.26714) + | > loader_time: 0.00230 (0.02881) + + + --> STEP: 104/234 -- GLOBAL_STEP: 39650 + | > loss: -0.22010 (-0.15584) + | > log_mle: -0.38453 (-0.28608) + | > loss_dur: 0.16443 (0.13025) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.00086 (13.71918) + | > current_lr: 0.00004 + | > step_time: 1.99220 (2.25221) + | > loader_time: 0.00750 (0.02844) + + + --> STEP: 109/234 -- GLOBAL_STEP: 39655 + | > loss: -0.14329 (-0.15580) + | > log_mle: -0.35320 (-0.28800) + | > loss_dur: 0.20991 (0.13220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.27835 (14.23920) + | > current_lr: 0.00004 + | > step_time: 2.60610 (2.24792) + | > loader_time: 0.29440 (0.02994) + + + --> STEP: 114/234 -- GLOBAL_STEP: 39660 + | > loss: -0.17596 (-0.15635) + | > log_mle: -0.33696 (-0.29067) + | > loss_dur: 0.16100 (0.13432) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.59326 (15.10744) + | > current_lr: 0.00004 + | > step_time: 2.40340 (2.23761) + | > loader_time: 0.00270 (0.02973) + + + --> STEP: 119/234 -- GLOBAL_STEP: 39665 + | > loss: -0.16969 (-0.15664) + | > log_mle: -0.33642 (-0.29282) + | > loss_dur: 0.16673 (0.13618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.13235 (15.49891) + | > current_lr: 0.00004 + | > step_time: 1.40900 (2.20002) + | > loader_time: 0.08540 (0.03090) + + + --> STEP: 124/234 -- GLOBAL_STEP: 39670 + | > loss: -0.20553 (-0.15701) + | > log_mle: -0.36601 (-0.29433) + | > loss_dur: 0.16048 (0.13732) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.60071 (15.73244) + | > current_lr: 0.00004 + | > step_time: 2.60550 (2.20028) + | > loader_time: 0.09410 (0.03194) + + + --> STEP: 129/234 -- GLOBAL_STEP: 39675 + | > loss: -0.16390 (-0.15828) + | > log_mle: -0.35306 (-0.29745) + | > loss_dur: 0.18917 (0.13917) + | > amp_scaler: 2048.00000 (1055.75194) + | > grad_norm: 37.42388 (16.47976) + | > current_lr: 0.00004 + | > step_time: 1.80130 (2.19230) + | > loader_time: 0.08710 (0.03296) + + + --> STEP: 134/234 -- GLOBAL_STEP: 39680 + | > loss: -0.19610 (-0.15999) + | > log_mle: -0.40611 (-0.30117) + | > loss_dur: 0.21002 (0.14118) + | > amp_scaler: 2048.00000 (1092.77612) + | > grad_norm: 41.17587 (17.39852) + | > current_lr: 0.00004 + | > step_time: 3.69520 (2.21797) + | > loader_time: 0.00280 (0.03244) + + + --> STEP: 139/234 -- GLOBAL_STEP: 39685 + | > loss: -0.27040 (-0.16165) + | > log_mle: -0.46792 (-0.30471) + | > loss_dur: 0.19752 (0.14306) + | > amp_scaler: 2048.00000 (1127.13669) + | > grad_norm: 56.46742 (18.14823) + | > current_lr: 0.00004 + | > step_time: 1.30660 (2.20941) + | > loader_time: 0.08470 (0.03389) + + + --> STEP: 144/234 -- GLOBAL_STEP: 39690 + | > loss: -0.23910 (-0.16330) + | > log_mle: -0.44352 (-0.30844) + | > loss_dur: 0.20442 (0.14515) + | > amp_scaler: 2048.00000 (1159.11111) + | > grad_norm: 47.30091 (19.18968) + | > current_lr: 0.00004 + | > step_time: 1.99750 (2.21211) + | > loader_time: 0.00440 (0.03398) + + + --> STEP: 149/234 -- GLOBAL_STEP: 39695 + | > loss: -0.28778 (-0.16597) + | > log_mle: -0.49757 (-0.31280) + | > loss_dur: 0.20979 (0.14684) + | > amp_scaler: 2048.00000 (1188.93960) + | > grad_norm: 59.03074 (20.23121) + | > current_lr: 0.00004 + | > step_time: 1.50120 (2.19149) + | > loader_time: 0.00300 (0.03351) + + + --> STEP: 154/234 -- GLOBAL_STEP: 39700 + | > loss: -0.25186 (-0.16904) + | > log_mle: -0.44763 (-0.31768) + | > loss_dur: 0.19577 (0.14865) + | > amp_scaler: 2048.00000 (1216.83117) + | > grad_norm: 77.96133 (21.58119) + | > current_lr: 0.00004 + | > step_time: 1.49880 (2.19513) + | > loader_time: 0.00350 (0.03502) + + + --> STEP: 159/234 -- GLOBAL_STEP: 39705 + | > loss: -0.24781 (-0.17158) + | > log_mle: -0.46423 (-0.32221) + | > loss_dur: 0.21643 (0.15063) + | > amp_scaler: 2048.00000 (1242.96855) + | > grad_norm: 93.70948 (22.89355) + | > current_lr: 0.00004 + | > step_time: 2.69610 (2.19406) + | > loader_time: 0.00390 (0.03403) + + + --> STEP: 164/234 -- GLOBAL_STEP: 39710 + | > loss: -0.22965 (-0.17386) + | > log_mle: -0.44869 (-0.32616) + | > loss_dur: 0.21904 (0.15230) + | > amp_scaler: 2048.00000 (1267.51220) + | > grad_norm: 44.96819 (23.97668) + | > current_lr: 0.00004 + | > step_time: 2.69770 (2.20024) + | > loader_time: 0.00330 (0.03421) + + + --> STEP: 169/234 -- GLOBAL_STEP: 39715 + | > loss: -0.23222 (-0.17623) + | > log_mle: -0.44953 (-0.33037) + | > loss_dur: 0.21730 (0.15414) + | > amp_scaler: 2048.00000 (1290.60355) + | > grad_norm: 73.56641 (25.38294) + | > current_lr: 0.00004 + | > step_time: 2.00310 (2.21024) + | > loader_time: 0.07670 (0.03545) + + + --> STEP: 174/234 -- GLOBAL_STEP: 39720 + | > loss: -0.29824 (-0.17921) + | > log_mle: -0.52461 (-0.33544) + | > loss_dur: 0.22638 (0.15623) + | > amp_scaler: 2048.00000 (1312.36782) + | > grad_norm: 61.45214 (26.95364) + | > current_lr: 0.00004 + | > step_time: 3.51470 (2.23481) + | > loader_time: 0.00520 (0.03557) + + + --> STEP: 179/234 -- GLOBAL_STEP: 39725 + | > loss: -0.28386 (-0.18183) + | > log_mle: -0.53309 (-0.34021) + | > loss_dur: 0.24923 (0.15838) + | > amp_scaler: 2048.00000 (1332.91620) + | > grad_norm: 66.58833 (27.82235) + | > current_lr: 0.00004 + | > step_time: 1.92120 (2.23514) + | > loader_time: 0.09380 (0.03620) + + + --> STEP: 184/234 -- GLOBAL_STEP: 39730 + | > loss: -0.27642 (-0.18449) + | > log_mle: -0.50320 (-0.34463) + | > loss_dur: 0.22678 (0.16015) + | > amp_scaler: 2048.00000 (1352.34783) + | > grad_norm: 65.41406 (28.64619) + | > current_lr: 0.00004 + | > step_time: 3.70460 (2.27438) + | > loader_time: 0.08650 (0.03621) + + + --> STEP: 189/234 -- GLOBAL_STEP: 39735 + | > loss: -0.28298 (-0.18726) + | > log_mle: -0.50834 (-0.34936) + | > loss_dur: 0.22537 (0.16210) + | > amp_scaler: 2048.00000 (1370.75132) + | > grad_norm: 48.81814 (29.40432) + | > current_lr: 0.00004 + | > step_time: 6.49980 (2.32271) + | > loader_time: 0.00370 (0.03728) + + + --> STEP: 194/234 -- GLOBAL_STEP: 39740 + | > loss: -0.32084 (-0.19041) + | > log_mle: -0.54084 (-0.35391) + | > loss_dur: 0.22000 (0.16351) + | > amp_scaler: 2048.00000 (1388.20619) + | > grad_norm: 52.68984 (30.51952) + | > current_lr: 0.00004 + | > step_time: 7.60000 (2.47015) + | > loader_time: 0.09890 (0.03941) + + + --> STEP: 199/234 -- GLOBAL_STEP: 39745 + | > loss: -0.31279 (-0.19329) + | > log_mle: -0.54481 (-0.35833) + | > loss_dur: 0.23202 (0.16504) + | > amp_scaler: 2048.00000 (1404.78392) + | > grad_norm: 101.44260 (31.56501) + | > current_lr: 0.00004 + | > step_time: 6.10520 (2.53486) + | > loader_time: 0.19380 (0.04225) + + + --> STEP: 204/234 -- GLOBAL_STEP: 39750 + | > loss: -0.33534 (-0.19592) + | > log_mle: -0.58148 (-0.36267) + | > loss_dur: 0.24614 (0.16675) + | > amp_scaler: 2048.00000 (1420.54902) + | > grad_norm: 99.67950 (32.59278) + | > current_lr: 0.00004 + | > step_time: 2.90590 (2.59517) + | > loader_time: 0.08890 (0.04232) + + + --> STEP: 209/234 -- GLOBAL_STEP: 39755 + | > loss: -0.30947 (-0.19905) + | > log_mle: -0.53928 (-0.36744) + | > loss_dur: 0.22981 (0.16839) + | > amp_scaler: 2048.00000 (1435.55981) + | > grad_norm: 90.38958 (33.73383) + | > current_lr: 0.00004 + | > step_time: 2.70430 (2.63506) + | > loader_time: 0.08420 (0.04227) + + + --> STEP: 214/234 -- GLOBAL_STEP: 39760 + | > loss: -0.36181 (-0.20274) + | > log_mle: -0.57750 (-0.37288) + | > loss_dur: 0.21569 (0.17014) + | > amp_scaler: 2048.00000 (1449.86916) + | > grad_norm: 80.61322 (35.14407) + | > current_lr: 0.00004 + | > step_time: 2.39600 (2.68845) + | > loader_time: 0.00310 (0.04361) + + + --> STEP: 219/234 -- GLOBAL_STEP: 39765 + | > loss: -0.43183 (-0.20638) + | > log_mle: -0.67427 (-0.37829) + | > loss_dur: 0.24245 (0.17191) + | > amp_scaler: 2048.00000 (1463.52511) + | > grad_norm: 116.54181 (36.48849) + | > current_lr: 0.00004 + | > step_time: 3.19870 (2.79053) + | > loader_time: 0.00430 (0.04534) + + + --> STEP: 224/234 -- GLOBAL_STEP: 39770 + | > loss: -0.36274 (-0.20957) + | > log_mle: -0.61496 (-0.38319) + | > loss_dur: 0.25222 (0.17362) + | > amp_scaler: 2048.00000 (1476.57143) + | > grad_norm: 108.08820 (37.79856) + | > current_lr: 0.00004 + | > step_time: 0.25180 (2.74321) + | > loader_time: 0.00290 (0.04515) + + + --> STEP: 229/234 -- GLOBAL_STEP: 39775 + | > loss: -0.34760 (-0.21285) + | > log_mle: -0.65333 (-0.38860) + | > loss_dur: 0.30573 (0.17575) + | > amp_scaler: 2048.00000 (1489.04803) + | > grad_norm: 104.34367 (39.13033) + | > current_lr: 0.00004 + | > step_time: 0.27210 (2.68910) + | > loader_time: 0.00390 (0.04425) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.37426 (+0.15160) + | > avg_loss: -0.23114 (+0.03917) + | > avg_log_mle: -0.45444 (+0.03047) + | > avg_loss_dur: 0.22331 (+0.00870) + + + > EPOCH: 170/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 20:32:10)  + + --> STEP: 0/234 -- GLOBAL_STEP: 39780 + | > loss: -0.19886 (-0.19886) + | > log_mle: -0.35976 (-0.35976) + | > loss_dur: 0.16090 (0.16090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.12102 (19.12102) + | > current_lr: 0.00004 + | > step_time: 8.60180 (8.60176) + | > loader_time: 11.59100 (11.59100) + + + --> STEP: 5/234 -- GLOBAL_STEP: 39785 + | > loss: -0.17559 (-0.15925) + | > log_mle: -0.28438 (-0.28448) + | > loss_dur: 0.10879 (0.12522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.49961 (16.48774) + | > current_lr: 0.00004 + | > step_time: 6.08940 (7.58165) + | > loader_time: 0.00100 (0.05935) + + + --> STEP: 10/234 -- GLOBAL_STEP: 39790 + | > loss: -0.15073 (-0.16313) + | > log_mle: -0.28351 (-0.28701) + | > loss_dur: 0.13278 (0.12388) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.99625 (14.66874) + | > current_lr: 0.00004 + | > step_time: 1.49900 (4.66985) + | > loader_time: 0.00370 (0.08024) + + + --> STEP: 15/234 -- GLOBAL_STEP: 39795 + | > loss: -0.18400 (-0.16809) + | > log_mle: -0.28592 (-0.28757) + | > loss_dur: 0.10192 (0.11948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.31307 (13.85132) + | > current_lr: 0.00004 + | > step_time: 1.89070 (3.62649) + | > loader_time: 0.00170 (0.05407) + + + --> STEP: 20/234 -- GLOBAL_STEP: 39800 + | > loss: -0.17013 (-0.16933) + | > log_mle: -0.27809 (-0.28485) + | > loss_dur: 0.10796 (0.11552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.02311 (12.82071) + | > current_lr: 0.00004 + | > step_time: 1.10700 (3.01696) + | > loader_time: 0.00160 (0.04100) + + + --> STEP: 25/234 -- GLOBAL_STEP: 39805 + | > loss: -0.15826 (-0.17018) + | > log_mle: -0.26825 (-0.28311) + | > loss_dur: 0.10999 (0.11293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.32244 (12.49445) + | > current_lr: 0.00004 + | > step_time: 1.22990 (2.66693) + | > loader_time: 0.00330 (0.03330) + + + --> STEP: 30/234 -- GLOBAL_STEP: 39810 + | > loss: -0.18940 (-0.17052) + | > log_mle: -0.28578 (-0.28250) + | > loss_dur: 0.09637 (0.11198) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.76278 (12.14130) + | > current_lr: 0.00004 + | > step_time: 1.07840 (2.41596) + | > loader_time: 0.00170 (0.02806) + + + --> STEP: 35/234 -- GLOBAL_STEP: 39815 + | > loss: -0.14211 (-0.16847) + | > log_mle: -0.27579 (-0.28181) + | > loss_dur: 0.13368 (0.11334) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.83689 (12.03113) + | > current_lr: 0.00004 + | > step_time: 1.20630 (2.26327) + | > loader_time: 0.00190 (0.02949) + + + --> STEP: 40/234 -- GLOBAL_STEP: 39820 + | > loss: -0.12382 (-0.16645) + | > log_mle: -0.26105 (-0.28102) + | > loss_dur: 0.13722 (0.11457) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.59995 (12.02874) + | > current_lr: 0.00004 + | > step_time: 1.88670 (2.14850) + | > loader_time: 0.00180 (0.02605) + + + --> STEP: 45/234 -- GLOBAL_STEP: 39825 + | > loss: -0.15983 (-0.16604) + | > log_mle: -0.29816 (-0.28093) + | > loss_dur: 0.13834 (0.11489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.97189 (11.95408) + | > current_lr: 0.00004 + | > step_time: 1.21460 (2.04309) + | > loader_time: 0.00210 (0.02520) + + + --> STEP: 50/234 -- GLOBAL_STEP: 39830 + | > loss: -0.14367 (-0.16502) + | > log_mle: -0.26611 (-0.28020) + | > loss_dur: 0.12244 (0.11518) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.20576 (11.60706) + | > current_lr: 0.00004 + | > step_time: 1.39330 (1.96556) + | > loader_time: 0.00270 (0.02463) + + + --> STEP: 55/234 -- GLOBAL_STEP: 39835 + | > loss: -0.18219 (-0.16408) + | > log_mle: -0.28450 (-0.27989) + | > loss_dur: 0.10232 (0.11581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.95002 (11.46054) + | > current_lr: 0.00004 + | > step_time: 1.04450 (1.91243) + | > loader_time: 0.00210 (0.02259) + + + --> STEP: 60/234 -- GLOBAL_STEP: 39840 + | > loss: -0.15176 (-0.16312) + | > log_mle: -0.29926 (-0.27998) + | > loss_dur: 0.14750 (0.11686) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.43013 (11.29454) + | > current_lr: 0.00004 + | > step_time: 1.81720 (1.87909) + | > loader_time: 0.00210 (0.02238) + + + --> STEP: 65/234 -- GLOBAL_STEP: 39845 + | > loss: -0.15591 (-0.16173) + | > log_mle: -0.27461 (-0.28013) + | > loss_dur: 0.11870 (0.11839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.49481 (11.24611) + | > current_lr: 0.00004 + | > step_time: 1.16370 (1.85353) + | > loader_time: 0.00220 (0.02082) + + + --> STEP: 70/234 -- GLOBAL_STEP: 39850 + | > loss: -0.12394 (-0.15970) + | > log_mle: -0.26699 (-0.27942) + | > loss_dur: 0.14305 (0.11972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.60096 (11.24885) + | > current_lr: 0.00004 + | > step_time: 1.20850 (1.84229) + | > loader_time: 0.00220 (0.01949) + + + --> STEP: 75/234 -- GLOBAL_STEP: 39855 + | > loss: -0.13724 (-0.15834) + | > log_mle: -0.28694 (-0.27971) + | > loss_dur: 0.14970 (0.12137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.27411 (11.33055) + | > current_lr: 0.00004 + | > step_time: 1.36380 (1.81430) + | > loader_time: 0.00220 (0.01838) + + + --> STEP: 80/234 -- GLOBAL_STEP: 39860 + | > loss: -0.14744 (-0.15740) + | > log_mle: -0.26706 (-0.27954) + | > loss_dur: 0.11962 (0.12214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.45326 (11.29166) + | > current_lr: 0.00004 + | > step_time: 1.60790 (1.79998) + | > loader_time: 0.00250 (0.01907) + + + --> STEP: 85/234 -- GLOBAL_STEP: 39865 + | > loss: -0.15899 (-0.15667) + | > log_mle: -0.27975 (-0.27970) + | > loss_dur: 0.12076 (0.12303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10476 (11.37819) + | > current_lr: 0.00004 + | > step_time: 1.20850 (1.80002) + | > loader_time: 0.00260 (0.01913) + + + --> STEP: 90/234 -- GLOBAL_STEP: 39870 + | > loss: -0.15048 (-0.15625) + | > log_mle: -0.30487 (-0.28088) + | > loss_dur: 0.15439 (0.12463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.46906 (11.65558) + | > current_lr: 0.00004 + | > step_time: 1.09460 (1.76433) + | > loader_time: 0.00270 (0.01911) + + + --> STEP: 95/234 -- GLOBAL_STEP: 39875 + | > loss: -0.21127 (-0.15736) + | > log_mle: -0.39125 (-0.28405) + | > loss_dur: 0.17999 (0.12669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.48130 (12.27077) + | > current_lr: 0.00004 + | > step_time: 1.41040 (1.75237) + | > loader_time: 0.08480 (0.01997) + + + --> STEP: 100/234 -- GLOBAL_STEP: 39880 + | > loss: -0.16517 (-0.15766) + | > log_mle: -0.31551 (-0.28548) + | > loss_dur: 0.15034 (0.12781) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.35051 (12.55281) + | > current_lr: 0.00004 + | > step_time: 1.19620 (1.72676) + | > loader_time: 0.00450 (0.01917) + + + --> STEP: 105/234 -- GLOBAL_STEP: 39885 + | > loss: -0.15738 (-0.15850) + | > log_mle: -0.29550 (-0.28805) + | > loss_dur: 0.13812 (0.12955) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.91468 (13.05622) + | > current_lr: 0.00004 + | > step_time: 1.97720 (1.76178) + | > loader_time: 0.00220 (0.02007) + + + --> STEP: 110/234 -- GLOBAL_STEP: 39890 + | > loss: -0.16539 (-0.15872) + | > log_mle: -0.32075 (-0.29017) + | > loss_dur: 0.15535 (0.13146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.36526 (13.64460) + | > current_lr: 0.00004 + | > step_time: 1.99800 (1.76131) + | > loader_time: 0.08650 (0.02003) + + + --> STEP: 115/234 -- GLOBAL_STEP: 39895 + | > loss: -0.16784 (-0.15956) + | > log_mle: -0.34105 (-0.29310) + | > loss_dur: 0.17321 (0.13354) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.90351 (14.43684) + | > current_lr: 0.00004 + | > step_time: 1.89040 (1.77065) + | > loader_time: 0.00390 (0.02099) + + + --> STEP: 120/234 -- GLOBAL_STEP: 39900 + | > loss: -0.20345 (-0.16004) + | > log_mle: -0.38904 (-0.29561) + | > loss_dur: 0.18559 (0.13557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.25166 (15.08003) + | > current_lr: 0.00004 + | > step_time: 4.62560 (1.80206) + | > loader_time: 0.19280 (0.02335) + + + --> STEP: 125/234 -- GLOBAL_STEP: 39905 + | > loss: -0.19724 (-0.16015) + | > log_mle: -0.37624 (-0.29685) + | > loss_dur: 0.17900 (0.13670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.80012 (15.45669) + | > current_lr: 0.00004 + | > step_time: 1.91010 (1.81093) + | > loader_time: 0.00380 (0.02257) + + + --> STEP: 130/234 -- GLOBAL_STEP: 39910 + | > loss: -0.19843 (-0.16150) + | > log_mle: -0.38562 (-0.29996) + | > loss_dur: 0.18719 (0.13846) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.36616 (16.36036) + | > current_lr: 0.00004 + | > step_time: 1.66890 (1.86283) + | > loader_time: 0.00250 (0.02389) + + + --> STEP: 135/234 -- GLOBAL_STEP: 39915 + | > loss: -0.15033 (-0.16306) + | > log_mle: -0.31811 (-0.30313) + | > loss_dur: 0.16778 (0.14006) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.45774 (17.09240) + | > current_lr: 0.00004 + | > step_time: 1.10750 (1.85167) + | > loader_time: 0.07620 (0.02367) + + + --> STEP: 140/234 -- GLOBAL_STEP: 39920 + | > loss: -0.16430 (-0.16460) + | > log_mle: -0.35241 (-0.30684) + | > loss_dur: 0.18811 (0.14224) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.97211 (17.99918) + | > current_lr: 0.00004 + | > step_time: 3.99960 (1.87335) + | > loader_time: 0.00370 (0.02295) + + + --> STEP: 145/234 -- GLOBAL_STEP: 39925 + | > loss: -0.25667 (-0.16685) + | > log_mle: -0.44923 (-0.31113) + | > loss_dur: 0.19256 (0.14428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.37812 (19.25570) + | > current_lr: 0.00004 + | > step_time: 1.60300 (1.95358) + | > loader_time: 0.00570 (0.02366) + + + --> STEP: 150/234 -- GLOBAL_STEP: 39930 + | > loss: -0.23498 (-0.16922) + | > log_mle: -0.43859 (-0.31520) + | > loss_dur: 0.20361 (0.14598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.95428 (20.15672) + | > current_lr: 0.00004 + | > step_time: 2.41350 (1.94243) + | > loader_time: 0.08590 (0.02407) + + + --> STEP: 155/234 -- GLOBAL_STEP: 39935 + | > loss: -0.28591 (-0.17240) + | > log_mle: -0.49715 (-0.32023) + | > loss_dur: 0.21124 (0.14783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.07515 (21.75551) + | > current_lr: 0.00004 + | > step_time: 1.59690 (1.95013) + | > loader_time: 0.00560 (0.02395) + + + --> STEP: 160/234 -- GLOBAL_STEP: 39940 + | > loss: -0.27962 (-0.17487) + | > log_mle: -0.49693 (-0.32463) + | > loss_dur: 0.21731 (0.14976) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.94610 (22.86686) + | > current_lr: 0.00004 + | > step_time: 2.19470 (1.95351) + | > loader_time: 0.00380 (0.02433) + + + --> STEP: 165/234 -- GLOBAL_STEP: 39945 + | > loss: -0.27356 (-0.17746) + | > log_mle: -0.49556 (-0.32889) + | > loss_dur: 0.22201 (0.15143) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.05885 (23.76556) + | > current_lr: 0.00004 + | > step_time: 1.19860 (1.94703) + | > loader_time: 0.08620 (0.02524) + + + --> STEP: 170/234 -- GLOBAL_STEP: 39950 + | > loss: -0.29249 (-0.18035) + | > log_mle: -0.53494 (-0.33362) + | > loss_dur: 0.24245 (0.15327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.69269 (24.78793) + | > current_lr: 0.00004 + | > step_time: 3.19980 (1.95269) + | > loader_time: 0.00360 (0.02616) + + + --> STEP: 175/234 -- GLOBAL_STEP: 39955 + | > loss: -0.27642 (-0.18375) + | > log_mle: -0.51460 (-0.33901) + | > loss_dur: 0.23818 (0.15526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.89061 (25.93736) + | > current_lr: 0.00004 + | > step_time: 1.69100 (1.96776) + | > loader_time: 0.00340 (0.02666) + + + --> STEP: 180/234 -- GLOBAL_STEP: 39960 + | > loss: -0.29925 (-0.18668) + | > log_mle: -0.51603 (-0.34397) + | > loss_dur: 0.21679 (0.15729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.97778 (27.46669) + | > current_lr: 0.00004 + | > step_time: 2.40120 (1.99156) + | > loader_time: 0.08690 (0.02748) + + + --> STEP: 185/234 -- GLOBAL_STEP: 39965 + | > loss: -0.30098 (-0.18928) + | > log_mle: -0.54119 (-0.34852) + | > loss_dur: 0.24020 (0.15924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.84218 (28.70362) + | > current_lr: 0.00004 + | > step_time: 1.99100 (1.99238) + | > loader_time: 0.11070 (0.02831) + + + --> STEP: 190/234 -- GLOBAL_STEP: 39970 + | > loss: -0.31274 (-0.19203) + | > log_mle: -0.52346 (-0.35300) + | > loss_dur: 0.21073 (0.16097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.90604 (29.69637) + | > current_lr: 0.00004 + | > step_time: 2.29910 (2.00272) + | > loader_time: 0.10640 (0.02909) + + + --> STEP: 195/234 -- GLOBAL_STEP: 39975 + | > loss: -0.29526 (-0.19506) + | > log_mle: -0.53843 (-0.35765) + | > loss_dur: 0.24317 (0.16259) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.42505 (30.80713) + | > current_lr: 0.00004 + | > step_time: 2.00350 (2.02683) + | > loader_time: 0.08220 (0.02981) + + + --> STEP: 200/234 -- GLOBAL_STEP: 39980 + | > loss: -0.28676 (-0.19777) + | > log_mle: -0.54436 (-0.36208) + | > loss_dur: 0.25759 (0.16431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.23749 (31.90011) + | > current_lr: 0.00004 + | > step_time: 5.59310 (2.07511) + | > loader_time: 0.10400 (0.03108) + + + --> STEP: 205/234 -- GLOBAL_STEP: 39985 + | > loss: -0.29641 (-0.20033) + | > log_mle: -0.53290 (-0.36637) + | > loss_dur: 0.23649 (0.16604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.21407 (33.02722) + | > current_lr: 0.00004 + | > step_time: 1.90800 (2.09243) + | > loader_time: 0.07980 (0.03252) + + + --> STEP: 210/234 -- GLOBAL_STEP: 39990 + | > loss: -0.36132 (-0.20376) + | > log_mle: -0.60317 (-0.37145) + | > loss_dur: 0.24185 (0.16769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.76220 (34.23440) + | > current_lr: 0.00004 + | > step_time: 5.18960 (2.16269) + | > loader_time: 0.00440 (0.03366) + + + --> STEP: 215/234 -- GLOBAL_STEP: 39995 + | > loss: -0.31780 (-0.20728) + | > log_mle: -0.56313 (-0.37667) + | > loss_dur: 0.24533 (0.16940) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.04784 (35.44681) + | > current_lr: 0.00004 + | > step_time: 1.89910 (2.22492) + | > loader_time: 0.28930 (0.03659) + + + --> STEP: 220/234 -- GLOBAL_STEP: 40000 + | > loss: -0.36623 (-0.21110) + | > log_mle: -0.61105 (-0.38224) + | > loss_dur: 0.24481 (0.17114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.21060 (36.82531) + | > current_lr: 0.00004 + | > step_time: 2.89600 (2.26617) + | > loader_time: 0.09400 (0.03668) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_40000.pth + + --> STEP: 225/234 -- GLOBAL_STEP: 40005 + | > loss: -0.42584 (-0.21465) + | > log_mle: -0.68417 (-0.38753) + | > loss_dur: 0.25833 (0.17288) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.63298 (38.16902) + | > current_lr: 0.00004 + | > step_time: 0.23650 (2.22146) + | > loader_time: 0.00390 (0.03628) + + + --> STEP: 230/234 -- GLOBAL_STEP: 40010 + | > loss: -0.39222 (-0.21783) + | > log_mle: -0.72412 (-0.39318) + | > loss_dur: 0.33190 (0.17534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 122.90004 (39.69367) + | > current_lr: 0.00004 + | > step_time: 0.24870 (2.17852) + | > loader_time: 0.00410 (0.03558) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67837 (+0.30412) + | > avg_loss: -0.23838 (-0.00724) + | > avg_log_mle: -0.46050 (-0.00605) + | > avg_loss_dur: 0.22212 (-0.00119) + + + > EPOCH: 171/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 20:41:48)  + + --> STEP: 1/234 -- GLOBAL_STEP: 40015 + | > loss: -0.17612 (-0.17612) + | > log_mle: -0.28616 (-0.28616) + | > loss_dur: 0.11004 (0.11004) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.41052 (23.41052) + | > current_lr: 0.00004 + | > step_time: 1.39820 (1.39820) + | > loader_time: 0.20470 (0.20471) + + + --> STEP: 6/234 -- GLOBAL_STEP: 40020 + | > loss: -0.17337 (-0.15903) + | > log_mle: -0.27748 (-0.28428) + | > loss_dur: 0.10412 (0.12525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.61577 (16.95770) + | > current_lr: 0.00004 + | > step_time: 1.98700 (3.35154) + | > loader_time: 0.00110 (0.33285) + + + --> STEP: 11/234 -- GLOBAL_STEP: 40025 + | > loss: -0.20196 (-0.16548) + | > log_mle: -0.28905 (-0.28727) + | > loss_dur: 0.08709 (0.12178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.10526 (15.55437) + | > current_lr: 0.00004 + | > step_time: 5.11230 (3.36516) + | > loader_time: 0.08790 (0.33592) + + + --> STEP: 16/234 -- GLOBAL_STEP: 40030 + | > loss: -0.18402 (-0.16849) + | > log_mle: -0.28224 (-0.28682) + | > loss_dur: 0.09822 (0.11833) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.02365 (15.29962) + | > current_lr: 0.00004 + | > step_time: 6.88670 (4.43178) + | > loader_time: 0.00750 (0.24386) + + + --> STEP: 21/234 -- GLOBAL_STEP: 40035 + | > loss: -0.15997 (-0.16932) + | > log_mle: -0.26247 (-0.28351) + | > loss_dur: 0.10250 (0.11418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.11843 (13.91524) + | > current_lr: 0.00004 + | > step_time: 4.50540 (4.22912) + | > loader_time: 0.09530 (0.19915) + + + --> STEP: 26/234 -- GLOBAL_STEP: 40040 + | > loss: -0.14848 (-0.17069) + | > log_mle: -0.27671 (-0.28253) + | > loss_dur: 0.12822 (0.11184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.45006 (13.17123) + | > current_lr: 0.00004 + | > step_time: 1.99600 (4.21183) + | > loader_time: 0.20140 (0.17956) + + + --> STEP: 31/234 -- GLOBAL_STEP: 40045 + | > loss: -0.12808 (-0.17063) + | > log_mle: -0.27515 (-0.28207) + | > loss_dur: 0.14707 (0.11144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.93449 (12.49986) + | > current_lr: 0.00004 + | > step_time: 1.79390 (4.02885) + | > loader_time: 0.00280 (0.15703) + + + --> STEP: 36/234 -- GLOBAL_STEP: 40050 + | > loss: -0.15119 (-0.16940) + | > log_mle: -0.27550 (-0.28164) + | > loss_dur: 0.12431 (0.11225) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78676 (12.19275) + | > current_lr: 0.00004 + | > step_time: 0.72770 (3.73488) + | > loader_time: 0.00160 (0.14039) + + + --> STEP: 41/234 -- GLOBAL_STEP: 40055 + | > loss: -0.17763 (-0.16761) + | > log_mle: -0.27739 (-0.28091) + | > loss_dur: 0.09976 (0.11330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.33385 (12.02280) + | > current_lr: 0.00004 + | > step_time: 1.11400 (3.43554) + | > loader_time: 0.08460 (0.12946) + + + --> STEP: 46/234 -- GLOBAL_STEP: 40060 + | > loss: -0.14798 (-0.16642) + | > log_mle: -0.27250 (-0.28067) + | > loss_dur: 0.12452 (0.11425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.49012 (11.90355) + | > current_lr: 0.00004 + | > step_time: 1.06410 (3.22424) + | > loader_time: 0.00220 (0.11565) + + + --> STEP: 51/234 -- GLOBAL_STEP: 40065 + | > loss: -0.14774 (-0.16563) + | > log_mle: -0.26470 (-0.27980) + | > loss_dur: 0.11695 (0.11417) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.24865 (11.60141) + | > current_lr: 0.00004 + | > step_time: 2.08820 (3.01996) + | > loader_time: 0.00200 (0.10452) + + + --> STEP: 56/234 -- GLOBAL_STEP: 40070 + | > loss: -0.14226 (-0.16500) + | > log_mle: -0.28050 (-0.27982) + | > loss_dur: 0.13824 (0.11483) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.89543 (11.61707) + | > current_lr: 0.00004 + | > step_time: 2.10160 (2.86965) + | > loader_time: 0.00270 (0.09692) + + + --> STEP: 61/234 -- GLOBAL_STEP: 40075 + | > loss: -0.15596 (-0.16419) + | > log_mle: -0.27684 (-0.27988) + | > loss_dur: 0.12088 (0.11569) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.08063 (11.60000) + | > current_lr: 0.00004 + | > step_time: 1.71930 (2.82797) + | > loader_time: 0.10740 (0.09093) + + + --> STEP: 66/234 -- GLOBAL_STEP: 40080 + | > loss: -0.15388 (-0.16263) + | > log_mle: -0.26817 (-0.27990) + | > loss_dur: 0.11429 (0.11727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79405 (11.55186) + | > current_lr: 0.00004 + | > step_time: 2.65910 (2.72960) + | > loader_time: 0.00380 (0.08554) + + + --> STEP: 71/234 -- GLOBAL_STEP: 40085 + | > loss: -0.14043 (-0.16042) + | > log_mle: -0.30019 (-0.27971) + | > loss_dur: 0.15976 (0.11929) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.48982 (11.60441) + | > current_lr: 0.00004 + | > step_time: 2.88580 (2.65377) + | > loader_time: 0.00860 (0.08010) + + + --> STEP: 76/234 -- GLOBAL_STEP: 40090 + | > loss: -0.14272 (-0.15887) + | > log_mle: -0.28531 (-0.27955) + | > loss_dur: 0.14258 (0.12068) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.68884 (11.61116) + | > current_lr: 0.00004 + | > step_time: 1.07670 (2.57484) + | > loader_time: 0.00210 (0.07500) + + + --> STEP: 81/234 -- GLOBAL_STEP: 40095 + | > loss: -0.14916 (-0.15796) + | > log_mle: -0.29271 (-0.27941) + | > loss_dur: 0.14355 (0.12144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.35143 (11.53139) + | > current_lr: 0.00004 + | > step_time: 1.18530 (2.51189) + | > loader_time: 0.00240 (0.07052) + + + --> STEP: 86/234 -- GLOBAL_STEP: 40100 + | > loss: -0.13629 (-0.15704) + | > log_mle: -0.28729 (-0.27953) + | > loss_dur: 0.15100 (0.12249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.89673 (11.65355) + | > current_lr: 0.00004 + | > step_time: 2.59890 (2.45824) + | > loader_time: 0.10100 (0.06769) + + + --> STEP: 91/234 -- GLOBAL_STEP: 40105 + | > loss: -0.13607 (-0.15677) + | > log_mle: -0.30240 (-0.28086) + | > loss_dur: 0.16633 (0.12410) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.39769 (11.92531) + | > current_lr: 0.00004 + | > step_time: 1.91470 (2.39805) + | > loader_time: 0.08460 (0.06764) + + + --> STEP: 96/234 -- GLOBAL_STEP: 40110 + | > loss: -0.15167 (-0.15782) + | > log_mle: -0.28791 (-0.28369) + | > loss_dur: 0.13624 (0.12587) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.71262 (12.66565) + | > current_lr: 0.00004 + | > step_time: 1.90390 (2.35924) + | > loader_time: 0.00250 (0.06512) + + + --> STEP: 101/234 -- GLOBAL_STEP: 40115 + | > loss: -0.15954 (-0.15814) + | > log_mle: -0.34499 (-0.28567) + | > loss_dur: 0.18545 (0.12754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.44869 (13.04155) + | > current_lr: 0.00004 + | > step_time: 2.29160 (2.32322) + | > loader_time: 0.00720 (0.06207) + + + --> STEP: 106/234 -- GLOBAL_STEP: 40120 + | > loss: -0.14429 (-0.15870) + | > log_mle: -0.34564 (-0.28823) + | > loss_dur: 0.20135 (0.12953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.51768 (13.48539) + | > current_lr: 0.00004 + | > step_time: 2.69660 (2.29553) + | > loader_time: 0.10830 (0.06105) + + + --> STEP: 111/234 -- GLOBAL_STEP: 40125 + | > loss: -0.19504 (-0.15919) + | > log_mle: -0.39654 (-0.29085) + | > loss_dur: 0.20150 (0.13166) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.21613 (14.06830) + | > current_lr: 0.00004 + | > step_time: 4.09990 (2.28659) + | > loader_time: 0.00530 (0.05846) + + + --> STEP: 116/234 -- GLOBAL_STEP: 40130 + | > loss: -0.16080 (-0.15962) + | > log_mle: -0.35419 (-0.29339) + | > loss_dur: 0.19338 (0.13377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.97192 (14.78005) + | > current_lr: 0.00004 + | > step_time: 1.00830 (2.25889) + | > loader_time: 0.08440 (0.05911) + + + --> STEP: 121/234 -- GLOBAL_STEP: 40135 + | > loss: -0.12018 (-0.15978) + | > log_mle: -0.27151 (-0.29508) + | > loss_dur: 0.15133 (0.13530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.15880 (15.15752) + | > current_lr: 0.00004 + | > step_time: 4.20040 (2.27459) + | > loader_time: 0.00340 (0.05911) + + + --> STEP: 126/234 -- GLOBAL_STEP: 40140 + | > loss: -0.21048 (-0.16051) + | > log_mle: -0.40546 (-0.29734) + | > loss_dur: 0.19498 (0.13683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.39693 (15.71897) + | > current_lr: 0.00004 + | > step_time: 1.11180 (2.26300) + | > loader_time: 0.08470 (0.05826) + + + --> STEP: 131/234 -- GLOBAL_STEP: 40145 + | > loss: -0.25227 (-0.16218) + | > log_mle: -0.45146 (-0.30078) + | > loss_dur: 0.19919 (0.13860) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.89976 (16.46178) + | > current_lr: 0.00004 + | > step_time: 3.91300 (2.25484) + | > loader_time: 0.19690 (0.05827) + + + --> STEP: 136/234 -- GLOBAL_STEP: 40150 + | > loss: -0.28187 (-0.16394) + | > log_mle: -0.50309 (-0.30434) + | > loss_dur: 0.22122 (0.14039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.15279 (17.18318) + | > current_lr: 0.00004 + | > step_time: 1.70080 (2.22548) + | > loader_time: 0.00280 (0.05747) + + + --> STEP: 141/234 -- GLOBAL_STEP: 40155 + | > loss: -0.23007 (-0.16520) + | > log_mle: -0.40522 (-0.30736) + | > loss_dur: 0.17515 (0.14216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.86300 (17.88710) + | > current_lr: 0.00004 + | > step_time: 1.30110 (2.21189) + | > loader_time: 0.08490 (0.05671) + + + --> STEP: 146/234 -- GLOBAL_STEP: 40160 + | > loss: -0.25107 (-0.16781) + | > log_mle: -0.45798 (-0.31214) + | > loss_dur: 0.20691 (0.14433) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.55302 (19.04427) + | > current_lr: 0.00004 + | > step_time: 1.87760 (2.20871) + | > loader_time: 0.00200 (0.05718) + + + --> STEP: 151/234 -- GLOBAL_STEP: 40165 + | > loss: -0.24109 (-0.17016) + | > log_mle: -0.42339 (-0.31606) + | > loss_dur: 0.18230 (0.14590) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.23855 (19.84983) + | > current_lr: 0.00004 + | > step_time: 4.13610 (2.22846) + | > loader_time: 0.00270 (0.05597) + + + --> STEP: 156/234 -- GLOBAL_STEP: 40170 + | > loss: -0.26437 (-0.17357) + | > log_mle: -0.46864 (-0.32151) + | > loss_dur: 0.20427 (0.14793) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.89864 (21.01375) + | > current_lr: 0.00004 + | > step_time: 2.39820 (2.22170) + | > loader_time: 0.19880 (0.05553) + + + --> STEP: 161/234 -- GLOBAL_STEP: 40175 + | > loss: -0.28704 (-0.17613) + | > log_mle: -0.48916 (-0.32611) + | > loss_dur: 0.20212 (0.14998) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.99307 (22.19937) + | > current_lr: 0.00004 + | > step_time: 5.10350 (2.26834) + | > loader_time: 0.10380 (0.05622) + + + --> STEP: 166/234 -- GLOBAL_STEP: 40180 + | > loss: -0.24060 (-0.17843) + | > log_mle: -0.42557 (-0.32998) + | > loss_dur: 0.18497 (0.15154) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.56092 (23.31918) + | > current_lr: 0.00004 + | > step_time: 2.20410 (2.25713) + | > loader_time: 0.08380 (0.05569) + + + --> STEP: 171/234 -- GLOBAL_STEP: 40185 + | > loss: -0.32898 (-0.18179) + | > log_mle: -0.53073 (-0.33534) + | > loss_dur: 0.20175 (0.15354) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.83929 (24.64052) + | > current_lr: 0.00004 + | > step_time: 1.90120 (2.25481) + | > loader_time: 0.08860 (0.05465) + + + --> STEP: 176/234 -- GLOBAL_STEP: 40190 + | > loss: -0.28043 (-0.18496) + | > log_mle: -0.50664 (-0.34055) + | > loss_dur: 0.22621 (0.15559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.62262 (26.14849) + | > current_lr: 0.00004 + | > step_time: 5.31250 (2.27307) + | > loader_time: 0.18730 (0.05578) + + + --> STEP: 181/234 -- GLOBAL_STEP: 40195 + | > loss: -0.23267 (-0.18759) + | > log_mle: -0.44267 (-0.34517) + | > loss_dur: 0.21000 (0.15759) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.95575 (27.42842) + | > current_lr: 0.00004 + | > step_time: 5.99930 (2.32081) + | > loader_time: 0.00380 (0.05577) + + + --> STEP: 186/234 -- GLOBAL_STEP: 40200 + | > loss: -0.23966 (-0.19035) + | > log_mle: -0.47820 (-0.34996) + | > loss_dur: 0.23854 (0.15961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.65517 (28.70892) + | > current_lr: 0.00004 + | > step_time: 1.39490 (2.34842) + | > loader_time: 0.00300 (0.05630) + + + --> STEP: 191/234 -- GLOBAL_STEP: 40205 + | > loss: -0.29045 (-0.19308) + | > log_mle: -0.50044 (-0.35440) + | > loss_dur: 0.20999 (0.16132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.38576 (29.68937) + | > current_lr: 0.00004 + | > step_time: 1.08670 (2.35815) + | > loader_time: 0.00510 (0.05679) + + + --> STEP: 196/234 -- GLOBAL_STEP: 40210 + | > loss: -0.22401 (-0.19547) + | > log_mle: -0.46017 (-0.35838) + | > loss_dur: 0.23616 (0.16291) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.43837 (31.53499) + | > current_lr: 0.00004 + | > step_time: 3.19140 (2.36837) + | > loader_time: 0.10120 (0.05729) + + + --> STEP: 201/234 -- GLOBAL_STEP: 40215 + | > loss: -0.20492 (-0.19737) + | > log_mle: -0.43790 (-0.36198) + | > loss_dur: 0.23298 (0.16462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.57704 (32.27490) + | > current_lr: 0.00004 + | > step_time: 11.20710 (2.46303) + | > loader_time: 0.29150 (0.05832) + + + --> STEP: 206/234 -- GLOBAL_STEP: 40220 + | > loss: -0.32055 (-0.20015) + | > log_mle: -0.55503 (-0.36643) + | > loss_dur: 0.23448 (0.16628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.90866 (33.09267) + | > current_lr: 0.00004 + | > step_time: 6.90250 (2.52171) + | > loader_time: 0.00420 (0.05933) + + + --> STEP: 211/234 -- GLOBAL_STEP: 40225 + | > loss: -0.37601 (-0.20349) + | > log_mle: -0.63085 (-0.37161) + | > loss_dur: 0.25485 (0.16812) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.05061 (34.04858) + | > current_lr: 0.00004 + | > step_time: 7.00990 (2.62986) + | > loader_time: 0.09420 (0.06075) + + + --> STEP: 216/234 -- GLOBAL_STEP: 40230 + | > loss: -0.34346 (-0.20663) + | > log_mle: -0.60657 (-0.37645) + | > loss_dur: 0.26312 (0.16983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 119.74086 (35.25682) + | > current_lr: 0.00004 + | > step_time: 5.49710 (2.68380) + | > loader_time: 0.00420 (0.06119) + + + --> STEP: 221/234 -- GLOBAL_STEP: 40235 + | > loss: -0.31425 (-0.20998) + | > log_mle: -0.54232 (-0.38143) + | > loss_dur: 0.22806 (0.17145) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.42550 (36.24712) + | > current_lr: 0.00004 + | > step_time: 2.10490 (2.71133) + | > loader_time: 0.08430 (0.06069) + + + --> STEP: 226/234 -- GLOBAL_STEP: 40240 + | > loss: -0.39194 (-0.21363) + | > log_mle: -0.64085 (-0.38692) + | > loss_dur: 0.24891 (0.17329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.22713 (37.69691) + | > current_lr: 0.00004 + | > step_time: 0.23820 (2.66161) + | > loader_time: 0.00440 (0.05944) + + + --> STEP: 231/234 -- GLOBAL_STEP: 40245 + | > loss: -0.31305 (-0.21642) + | > log_mle: -0.70165 (-0.39271) + | > loss_dur: 0.38860 (0.17629) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 125.00063 (39.10799) + | > current_lr: 0.00004 + | > step_time: 0.27430 (2.60942) + | > loader_time: 0.00500 (0.05825) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32894 (-0.34944) + | > avg_loss: -0.25152 (-0.01314) + | > avg_log_mle: -0.46952 (-0.00902) + | > avg_loss_dur: 0.21801 (-0.00411) + + + > EPOCH: 172/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 20:53:10)  + + --> STEP: 2/234 -- GLOBAL_STEP: 40250 + | > loss: -0.18624 (-0.18255) + | > log_mle: -0.29503 (-0.29130) + | > loss_dur: 0.10879 (0.10875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.00543 (16.75536) + | > current_lr: 0.00004 + | > step_time: 2.39800 (1.80198) + | > loader_time: 2.58320 (1.33301) + + + --> STEP: 7/234 -- GLOBAL_STEP: 40255 + | > loss: -0.17462 (-0.16180) + | > log_mle: -0.29268 (-0.28508) + | > loss_dur: 0.11806 (0.12329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.44330 (18.79358) + | > current_lr: 0.00004 + | > step_time: 3.81660 (3.29065) + | > loader_time: 0.00150 (1.39425) + + + --> STEP: 12/234 -- GLOBAL_STEP: 40260 + | > loss: -0.17451 (-0.16555) + | > log_mle: -0.28814 (-0.28730) + | > loss_dur: 0.11363 (0.12176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.80672 (17.01561) + | > current_lr: 0.00004 + | > step_time: 1.80300 (4.01273) + | > loader_time: 0.08220 (0.85119) + + + --> STEP: 17/234 -- GLOBAL_STEP: 40265 + | > loss: -0.17463 (-0.17030) + | > log_mle: -0.26794 (-0.28654) + | > loss_dur: 0.09332 (0.11623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.78174 (15.09320) + | > current_lr: 0.00004 + | > step_time: 2.10610 (3.59248) + | > loader_time: 0.00160 (0.61783) + + + --> STEP: 22/234 -- GLOBAL_STEP: 40270 + | > loss: -0.18231 (-0.17099) + | > log_mle: -0.28629 (-0.28476) + | > loss_dur: 0.10398 (0.11376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.71258 (13.79055) + | > current_lr: 0.00004 + | > step_time: 8.78600 (3.82526) + | > loader_time: 0.00310 (0.47801) + + + --> STEP: 27/234 -- GLOBAL_STEP: 40275 + | > loss: -0.18285 (-0.17309) + | > log_mle: -0.28442 (-0.28400) + | > loss_dur: 0.10157 (0.11092) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.42386 (13.12335) + | > current_lr: 0.00004 + | > step_time: 1.99990 (3.83991) + | > loader_time: 0.09420 (0.40045) + + + --> STEP: 32/234 -- GLOBAL_STEP: 40280 + | > loss: -0.19139 (-0.17344) + | > log_mle: -0.28724 (-0.28342) + | > loss_dur: 0.09585 (0.10998) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.57940 (12.74070) + | > current_lr: 0.00004 + | > step_time: 3.50180 (4.02234) + | > loader_time: 0.19290 (0.35241) + + + --> STEP: 37/234 -- GLOBAL_STEP: 40285 + | > loss: -0.17134 (-0.17143) + | > log_mle: -0.27316 (-0.28214) + | > loss_dur: 0.10182 (0.11072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.81111 (12.51130) + | > current_lr: 0.00004 + | > step_time: 1.80520 (3.68456) + | > loader_time: 0.00190 (0.31481) + + + --> STEP: 42/234 -- GLOBAL_STEP: 40290 + | > loss: -0.14642 (-0.16947) + | > log_mle: -0.26618 (-0.28133) + | > loss_dur: 0.11976 (0.11187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.39161 (12.25655) + | > current_lr: 0.00004 + | > step_time: 0.99590 (3.43824) + | > loader_time: 0.00270 (0.27961) + + + --> STEP: 47/234 -- GLOBAL_STEP: 40295 + | > loss: -0.14752 (-0.16795) + | > log_mle: -0.27425 (-0.28136) + | > loss_dur: 0.12673 (0.11341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.61980 (12.14527) + | > current_lr: 0.00004 + | > step_time: 1.41140 (3.21426) + | > loader_time: 0.10120 (0.25218) + + + --> STEP: 52/234 -- GLOBAL_STEP: 40300 + | > loss: -0.13772 (-0.16709) + | > log_mle: -0.27177 (-0.28060) + | > loss_dur: 0.13404 (0.11351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.66589 (11.79864) + | > current_lr: 0.00004 + | > step_time: 1.14310 (3.05702) + | > loader_time: 0.00220 (0.22960) + + + --> STEP: 57/234 -- GLOBAL_STEP: 40305 + | > loss: -0.13497 (-0.16586) + | > log_mle: -0.26118 (-0.28057) + | > loss_dur: 0.12621 (0.11471) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.29524 (11.63560) + | > current_lr: 0.00004 + | > step_time: 2.60520 (2.91058) + | > loader_time: 0.00200 (0.20968) + + + --> STEP: 62/234 -- GLOBAL_STEP: 40310 + | > loss: -0.12417 (-0.16527) + | > log_mle: -0.31053 (-0.28138) + | > loss_dur: 0.18636 (0.11611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.50130 (11.85980) + | > current_lr: 0.00004 + | > step_time: 1.20250 (2.78435) + | > loader_time: 0.08340 (0.19430) + + + --> STEP: 67/234 -- GLOBAL_STEP: 40315 + | > loss: -0.14030 (-0.16402) + | > log_mle: -0.29272 (-0.28116) + | > loss_dur: 0.15242 (0.11714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.65463 (11.64494) + | > current_lr: 0.00004 + | > step_time: 1.20660 (2.67939) + | > loader_time: 0.00210 (0.18123) + + + --> STEP: 72/234 -- GLOBAL_STEP: 40320 + | > loss: -0.14806 (-0.16199) + | > log_mle: -0.27654 (-0.28092) + | > loss_dur: 0.12848 (0.11892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.58393 (11.63991) + | > current_lr: 0.00004 + | > step_time: 1.21470 (2.57305) + | > loader_time: 0.08760 (0.17171) + + + --> STEP: 77/234 -- GLOBAL_STEP: 40325 + | > loss: -0.16420 (-0.16086) + | > log_mle: -0.28426 (-0.28116) + | > loss_dur: 0.12006 (0.12030) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.95947 (11.73724) + | > current_lr: 0.00004 + | > step_time: 1.31150 (2.51166) + | > loader_time: 0.00260 (0.16184) + + + --> STEP: 82/234 -- GLOBAL_STEP: 40330 + | > loss: -0.15226 (-0.16011) + | > log_mle: -0.27844 (-0.28106) + | > loss_dur: 0.12618 (0.12095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.71060 (11.65093) + | > current_lr: 0.00004 + | > step_time: 1.11540 (2.45068) + | > loader_time: 0.08970 (0.15319) + + + --> STEP: 87/234 -- GLOBAL_STEP: 40335 + | > loss: -0.14513 (-0.15941) + | > log_mle: -0.28421 (-0.28133) + | > loss_dur: 0.13908 (0.12191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.97876 (11.79344) + | > current_lr: 0.00004 + | > step_time: 2.04640 (2.40374) + | > loader_time: 0.00240 (0.14453) + + + --> STEP: 92/234 -- GLOBAL_STEP: 40340 + | > loss: -0.18336 (-0.15957) + | > log_mle: -0.33042 (-0.28318) + | > loss_dur: 0.14706 (0.12361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.35569 (12.13391) + | > current_lr: 0.00004 + | > step_time: 1.80250 (2.35356) + | > loader_time: 0.00300 (0.13685) + + + --> STEP: 97/234 -- GLOBAL_STEP: 40345 + | > loss: -0.16753 (-0.16053) + | > log_mle: -0.31846 (-0.28597) + | > loss_dur: 0.15094 (0.12545) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.96766 (12.60879) + | > current_lr: 0.00004 + | > step_time: 1.36320 (2.31984) + | > loader_time: 0.00230 (0.13162) + + + --> STEP: 102/234 -- GLOBAL_STEP: 40350 + | > loss: -0.14376 (-0.16061) + | > log_mle: -0.29868 (-0.28768) + | > loss_dur: 0.15492 (0.12707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.08282 (13.11960) + | > current_lr: 0.00004 + | > step_time: 1.31290 (2.28257) + | > loader_time: 0.08740 (0.12698) + + + --> STEP: 107/234 -- GLOBAL_STEP: 40355 + | > loss: -0.17334 (-0.16130) + | > log_mle: -0.33895 (-0.29044) + | > loss_dur: 0.16561 (0.12915) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.07974 (13.80923) + | > current_lr: 0.00004 + | > step_time: 1.60710 (2.25092) + | > loader_time: 0.00260 (0.12284) + + + --> STEP: 112/234 -- GLOBAL_STEP: 40360 + | > loss: -0.16407 (-0.16134) + | > log_mle: -0.35408 (-0.29295) + | > loss_dur: 0.19001 (0.13160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.79866 (14.55341) + | > current_lr: 0.00004 + | > step_time: 1.41150 (2.23257) + | > loader_time: 0.08710 (0.11992) + + + --> STEP: 117/234 -- GLOBAL_STEP: 40365 + | > loss: -0.18671 (-0.16180) + | > log_mle: -0.34967 (-0.29535) + | > loss_dur: 0.16295 (0.13355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.89615 (15.06340) + | > current_lr: 0.00004 + | > step_time: 1.89240 (2.21760) + | > loader_time: 0.00330 (0.11554) + + + --> STEP: 122/234 -- GLOBAL_STEP: 40370 + | > loss: -0.16271 (-0.16168) + | > log_mle: -0.32266 (-0.29679) + | > loss_dur: 0.15995 (0.13512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.19542 (15.34334) + | > current_lr: 0.00004 + | > step_time: 1.24110 (2.19042) + | > loader_time: 0.00260 (0.11230) + + + --> STEP: 127/234 -- GLOBAL_STEP: 40375 + | > loss: -0.19259 (-0.16265) + | > log_mle: -0.38152 (-0.29949) + | > loss_dur: 0.18893 (0.13684) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.65702 (16.03267) + | > current_lr: 0.00004 + | > step_time: 1.20570 (2.17990) + | > loader_time: 0.08500 (0.11079) + + + --> STEP: 132/234 -- GLOBAL_STEP: 40380 + | > loss: -0.20052 (-0.16419) + | > log_mle: -0.35947 (-0.30264) + | > loss_dur: 0.15895 (0.13845) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.44710 (16.72461) + | > current_lr: 0.00004 + | > step_time: 3.10000 (2.16868) + | > loader_time: 0.00340 (0.10735) + + + --> STEP: 137/234 -- GLOBAL_STEP: 40385 + | > loss: -0.16726 (-0.16558) + | > log_mle: -0.37557 (-0.30602) + | > loss_dur: 0.20831 (0.14044) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.77674 (17.50987) + | > current_lr: 0.00004 + | > step_time: 2.11950 (2.15988) + | > loader_time: 0.00350 (0.10427) + + + --> STEP: 142/234 -- GLOBAL_STEP: 40390 + | > loss: -0.19844 (-0.16696) + | > log_mle: -0.39019 (-0.30902) + | > loss_dur: 0.19175 (0.14206) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.67718 (18.12441) + | > current_lr: 0.00004 + | > step_time: 1.40770 (2.15810) + | > loader_time: 0.08720 (0.10189) + + + --> STEP: 147/234 -- GLOBAL_STEP: 40395 + | > loss: -0.19666 (-0.16937) + | > log_mle: -0.39119 (-0.31370) + | > loss_dur: 0.19453 (0.14433) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.71209 (19.27789) + | > current_lr: 0.00004 + | > step_time: 3.29490 (2.17585) + | > loader_time: 0.11080 (0.10106) + + + --> STEP: 152/234 -- GLOBAL_STEP: 40400 + | > loss: -0.26165 (-0.17208) + | > log_mle: -0.48225 (-0.31811) + | > loss_dur: 0.22060 (0.14603) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.99097 (20.27282) + | > current_lr: 0.00004 + | > step_time: 3.19190 (2.18121) + | > loader_time: 0.10580 (0.09853) + + + --> STEP: 157/234 -- GLOBAL_STEP: 40405 + | > loss: -0.22452 (-0.17525) + | > log_mle: -0.42548 (-0.32309) + | > loss_dur: 0.20097 (0.14784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.22506 (21.63971) + | > current_lr: 0.00004 + | > step_time: 1.30350 (2.19115) + | > loader_time: 0.09730 (0.09737) + + + --> STEP: 162/234 -- GLOBAL_STEP: 40410 + | > loss: -0.26425 (-0.17819) + | > log_mle: -0.45783 (-0.32781) + | > loss_dur: 0.19358 (0.14962) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.30361 (22.72295) + | > current_lr: 0.00004 + | > step_time: 1.51900 (2.18288) + | > loader_time: 0.00500 (0.09608) + + + --> STEP: 167/234 -- GLOBAL_STEP: 40415 + | > loss: -0.34321 (-0.18086) + | > log_mle: -0.54247 (-0.33209) + | > loss_dur: 0.19926 (0.15122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.09568 (23.75849) + | > current_lr: 0.00004 + | > step_time: 1.89790 (2.18399) + | > loader_time: 0.00560 (0.09494) + + + --> STEP: 172/234 -- GLOBAL_STEP: 40420 + | > loss: -0.31253 (-0.18399) + | > log_mle: -0.54082 (-0.33735) + | > loss_dur: 0.22828 (0.15336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.03382 (24.89235) + | > current_lr: 0.00004 + | > step_time: 3.41710 (2.18447) + | > loader_time: 0.19490 (0.09449) + + + --> STEP: 177/234 -- GLOBAL_STEP: 40425 + | > loss: -0.27938 (-0.18702) + | > log_mle: -0.49378 (-0.34230) + | > loss_dur: 0.21439 (0.15528) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.46188 (26.18547) + | > current_lr: 0.00004 + | > step_time: 2.40110 (2.20702) + | > loader_time: 0.00350 (0.09410) + + + --> STEP: 182/234 -- GLOBAL_STEP: 40430 + | > loss: -0.28850 (-0.18967) + | > log_mle: -0.53808 (-0.34720) + | > loss_dur: 0.24958 (0.15753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.38158 (27.47347) + | > current_lr: 0.00004 + | > step_time: 1.38870 (2.23047) + | > loader_time: 0.00350 (0.09411) + + + --> STEP: 187/234 -- GLOBAL_STEP: 40435 + | > loss: -0.31367 (-0.19256) + | > log_mle: -0.54066 (-0.35210) + | > loss_dur: 0.22699 (0.15953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.45847 (28.66224) + | > current_lr: 0.00004 + | > step_time: 4.40570 (2.29391) + | > loader_time: 0.19680 (0.09380) + + + --> STEP: 192/234 -- GLOBAL_STEP: 40440 + | > loss: -0.33338 (-0.19563) + | > log_mle: -0.55570 (-0.35670) + | > loss_dur: 0.22232 (0.16107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.07865 (29.97067) + | > current_lr: 0.00004 + | > step_time: 4.39510 (2.32853) + | > loader_time: 0.00660 (0.09337) + + + --> STEP: 197/234 -- GLOBAL_STEP: 40445 + | > loss: -0.32979 (-0.19864) + | > log_mle: -0.53538 (-0.36123) + | > loss_dur: 0.20559 (0.16260) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.44830 (30.88707) + | > current_lr: 0.00004 + | > step_time: 3.81210 (2.39031) + | > loader_time: 0.10460 (0.09402) + + + --> STEP: 202/234 -- GLOBAL_STEP: 40450 + | > loss: -0.41045 (-0.20154) + | > log_mle: -0.62824 (-0.36591) + | > loss_dur: 0.21779 (0.16437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.54454 (32.10526) + | > current_lr: 0.00004 + | > step_time: 5.69870 (2.46924) + | > loader_time: 0.09370 (0.09562) + + + --> STEP: 207/234 -- GLOBAL_STEP: 40455 + | > loss: -0.38104 (-0.20464) + | > log_mle: -0.62176 (-0.37068) + | > loss_dur: 0.24072 (0.16604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.24906 (33.07424) + | > current_lr: 0.00004 + | > step_time: 7.69160 (2.55846) + | > loader_time: 0.00250 (0.09521) + + + --> STEP: 212/234 -- GLOBAL_STEP: 40460 + | > loss: -0.35492 (-0.20828) + | > log_mle: -0.59439 (-0.37607) + | > loss_dur: 0.23948 (0.16778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 127.73545 (34.49734) + | > current_lr: 0.00004 + | > step_time: 3.90500 (2.58208) + | > loader_time: 0.18830 (0.09586) + + + --> STEP: 217/234 -- GLOBAL_STEP: 40465 + | > loss: -0.36451 (-0.21178) + | > log_mle: -0.60961 (-0.38131) + | > loss_dur: 0.24510 (0.16953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 134.50351 (36.05854) + | > current_lr: 0.00004 + | > step_time: 4.69560 (2.67057) + | > loader_time: 0.00580 (0.09561) + + + --> STEP: 222/234 -- GLOBAL_STEP: 40470 + | > loss: -0.36328 (-0.21517) + | > log_mle: -0.63313 (-0.38643) + | > loss_dur: 0.26985 (0.17125) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.45306 (37.37424) + | > current_lr: 0.00004 + | > step_time: 0.23770 (2.64473) + | > loader_time: 0.00420 (0.09395) + + + --> STEP: 227/234 -- GLOBAL_STEP: 40475 + | > loss: -0.31616 (-0.21888) + | > log_mle: -0.57606 (-0.39186) + | > loss_dur: 0.25990 (0.17298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 152.98921 (39.01793) + | > current_lr: 0.00004 + | > step_time: 0.25240 (2.59183) + | > loader_time: 0.00460 (0.09197) + + + --> STEP: 232/234 -- GLOBAL_STEP: 40480 + | > loss: -0.30437 (-0.22144) + | > log_mle: -0.78507 (-0.39843) + | > loss_dur: 0.48070 (0.17699) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 168.07542 (40.80437) + | > current_lr: 0.00004 + | > step_time: 0.32740 (2.54209) + | > loader_time: 0.01300 (0.09012) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.94351 (+0.61457) + | > avg_loss: -0.23018 (+0.02134) + | > avg_log_mle: -0.45512 (+0.01440) + | > avg_loss_dur: 0.22494 (+0.00694) + + + > EPOCH: 173/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 21:04:20)  + + --> STEP: 3/234 -- GLOBAL_STEP: 40485 + | > loss: -0.09661 (-0.15489) + | > log_mle: -0.28200 (-0.28927) + | > loss_dur: 0.18539 (0.13438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.11550 (15.58653) + | > current_lr: 0.00004 + | > step_time: 1.60020 (1.53141) + | > loader_time: 0.00150 (0.00130) + + + --> STEP: 8/234 -- GLOBAL_STEP: 40490 + | > loss: -0.18413 (-0.16821) + | > log_mle: -0.30154 (-0.28936) + | > loss_dur: 0.11741 (0.12115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.94238 (15.29867) + | > current_lr: 0.00004 + | > step_time: 2.00230 (2.35102) + | > loader_time: 0.00370 (0.03657) + + + --> STEP: 13/234 -- GLOBAL_STEP: 40495 + | > loss: -0.19186 (-0.17012) + | > log_mle: -0.29570 (-0.29057) + | > loss_dur: 0.10384 (0.12044) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.49569 (14.13965) + | > current_lr: 0.00004 + | > step_time: 4.10440 (2.28492) + | > loader_time: 0.09780 (0.03094) + + + --> STEP: 18/234 -- GLOBAL_STEP: 40500 + | > loss: -0.16714 (-0.17232) + | > log_mle: -0.28309 (-0.28859) + | > loss_dur: 0.11595 (0.11627) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85894 (13.24409) + | > current_lr: 0.00004 + | > step_time: 2.20270 (2.39032) + | > loader_time: 0.00170 (0.02763) + + + --> STEP: 23/234 -- GLOBAL_STEP: 40505 + | > loss: -0.20157 (-0.17475) + | > log_mle: -0.29009 (-0.28698) + | > loss_dur: 0.08852 (0.11223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.30323 (12.77921) + | > current_lr: 0.00004 + | > step_time: 5.01900 (2.72359) + | > loader_time: 2.49530 (0.13053) + + + --> STEP: 28/234 -- GLOBAL_STEP: 40510 + | > loss: -0.21526 (-0.17616) + | > log_mle: -0.29390 (-0.28623) + | > loss_dur: 0.07865 (0.11007) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.93046 (12.22918) + | > current_lr: 0.00004 + | > step_time: 4.50690 (3.31622) + | > loader_time: 0.18890 (0.12086) + + + --> STEP: 33/234 -- GLOBAL_STEP: 40515 + | > loss: -0.17732 (-0.17511) + | > log_mle: -0.27756 (-0.28549) + | > loss_dur: 0.10024 (0.11039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.56343 (11.79799) + | > current_lr: 0.00004 + | > step_time: 10.20110 (3.76229) + | > loader_time: 0.10000 (0.10878) + + + --> STEP: 38/234 -- GLOBAL_STEP: 40520 + | > loss: -0.16378 (-0.17306) + | > log_mle: -0.29209 (-0.28506) + | > loss_dur: 0.12831 (0.11199) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.82563 (11.62250) + | > current_lr: 0.00004 + | > step_time: 2.49260 (3.90625) + | > loader_time: 0.00150 (0.09963) + + + --> STEP: 43/234 -- GLOBAL_STEP: 40525 + | > loss: -0.15438 (-0.17098) + | > log_mle: -0.29075 (-0.28423) + | > loss_dur: 0.13637 (0.11325) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85499 (11.50918) + | > current_lr: 0.00004 + | > step_time: 2.50790 (3.68699) + | > loader_time: 0.00180 (0.09246) + + + --> STEP: 48/234 -- GLOBAL_STEP: 40530 + | > loss: -0.17210 (-0.17008) + | > log_mle: -0.27369 (-0.28390) + | > loss_dur: 0.10159 (0.11383) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.83799 (11.35672) + | > current_lr: 0.00004 + | > step_time: 2.72310 (3.48902) + | > loader_time: 0.00170 (0.08883) + + + --> STEP: 53/234 -- GLOBAL_STEP: 40535 + | > loss: -0.16996 (-0.16921) + | > log_mle: -0.28682 (-0.28330) + | > loss_dur: 0.11686 (0.11409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81620 (11.11788) + | > current_lr: 0.00004 + | > step_time: 1.71590 (3.33474) + | > loader_time: 0.00260 (0.08069) + + + --> STEP: 58/234 -- GLOBAL_STEP: 40540 + | > loss: -0.16970 (-0.16846) + | > log_mle: -0.27563 (-0.28304) + | > loss_dur: 0.10593 (0.11458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.04816 (10.94595) + | > current_lr: 0.00004 + | > step_time: 1.01180 (3.19615) + | > loader_time: 0.00270 (0.07651) + + + --> STEP: 63/234 -- GLOBAL_STEP: 40545 + | > loss: -0.13184 (-0.16677) + | > log_mle: -0.27820 (-0.28384) + | > loss_dur: 0.14636 (0.11707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.63301 (11.15911) + | > current_lr: 0.00004 + | > step_time: 1.56600 (3.04515) + | > loader_time: 0.00230 (0.07177) + + + --> STEP: 68/234 -- GLOBAL_STEP: 40550 + | > loss: -0.12468 (-0.16568) + | > log_mle: -0.27231 (-0.28345) + | > loss_dur: 0.14763 (0.11777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.02485 (11.07831) + | > current_lr: 0.00004 + | > step_time: 3.42190 (2.99364) + | > loader_time: 0.00370 (0.06682) + + + --> STEP: 73/234 -- GLOBAL_STEP: 40555 + | > loss: -0.12890 (-0.16374) + | > log_mle: -0.29139 (-0.28332) + | > loss_dur: 0.16249 (0.11957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.91609 (11.25909) + | > current_lr: 0.00004 + | > step_time: 2.40490 (2.93775) + | > loader_time: 0.00380 (0.06365) + + + --> STEP: 78/234 -- GLOBAL_STEP: 40560 + | > loss: -0.13337 (-0.16279) + | > log_mle: -0.27107 (-0.28326) + | > loss_dur: 0.13769 (0.12047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.77581 (11.27212) + | > current_lr: 0.00004 + | > step_time: 3.19020 (2.92521) + | > loader_time: 0.00530 (0.06198) + + + --> STEP: 83/234 -- GLOBAL_STEP: 40565 + | > loss: -0.12714 (-0.16212) + | > log_mle: -0.29220 (-0.28342) + | > loss_dur: 0.16506 (0.12130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.66331 (11.33051) + | > current_lr: 0.00004 + | > step_time: 2.00380 (2.85246) + | > loader_time: 0.00290 (0.05939) + + + --> STEP: 88/234 -- GLOBAL_STEP: 40570 + | > loss: -0.16847 (-0.16171) + | > log_mle: -0.32752 (-0.28402) + | > loss_dur: 0.15905 (0.12231) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.11096 (11.52042) + | > current_lr: 0.00004 + | > step_time: 3.29870 (2.81520) + | > loader_time: 0.00310 (0.05798) + + + --> STEP: 93/234 -- GLOBAL_STEP: 40575 + | > loss: -0.16437 (-0.16169) + | > log_mle: -0.34015 (-0.28586) + | > loss_dur: 0.17578 (0.12417) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.27708 (12.04698) + | > current_lr: 0.00004 + | > step_time: 3.00900 (2.77700) + | > loader_time: 0.08750 (0.05679) + + + --> STEP: 98/234 -- GLOBAL_STEP: 40580 + | > loss: -0.14196 (-0.16232) + | > log_mle: -0.27217 (-0.28782) + | > loss_dur: 0.13020 (0.12551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.02493 (12.36972) + | > current_lr: 0.00004 + | > step_time: 1.97360 (2.81786) + | > loader_time: 0.00230 (0.06086) + + + --> STEP: 103/234 -- GLOBAL_STEP: 40585 + | > loss: -0.19284 (-0.16287) + | > log_mle: -0.37470 (-0.29055) + | > loss_dur: 0.18186 (0.12768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.45311 (13.02836) + | > current_lr: 0.00004 + | > step_time: 3.28800 (2.78190) + | > loader_time: 0.00280 (0.05804) + + + --> STEP: 108/234 -- GLOBAL_STEP: 40590 + | > loss: -0.17156 (-0.16361) + | > log_mle: -0.31873 (-0.29285) + | > loss_dur: 0.14718 (0.12924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.93827 (13.48289) + | > current_lr: 0.00004 + | > step_time: 1.60970 (2.74883) + | > loader_time: 0.08720 (0.05867) + + + --> STEP: 113/234 -- GLOBAL_STEP: 40595 + | > loss: -0.19555 (-0.16426) + | > log_mle: -0.36705 (-0.29592) + | > loss_dur: 0.17150 (0.13166) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.62463 (14.25676) + | > current_lr: 0.00004 + | > step_time: 1.51830 (2.72371) + | > loader_time: 0.08720 (0.05853) + + + --> STEP: 118/234 -- GLOBAL_STEP: 40600 + | > loss: -0.14994 (-0.16450) + | > log_mle: -0.33300 (-0.29816) + | > loss_dur: 0.18306 (0.13366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.12749 (14.74059) + | > current_lr: 0.00004 + | > step_time: 2.09620 (2.72033) + | > loader_time: 0.00220 (0.05773) + + + --> STEP: 123/234 -- GLOBAL_STEP: 40605 + | > loss: -0.13990 (-0.16442) + | > log_mle: -0.30297 (-0.29932) + | > loss_dur: 0.16307 (0.13491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.14937 (15.20849) + | > current_lr: 0.00004 + | > step_time: 1.61010 (2.70336) + | > loader_time: 0.08450 (0.05753) + + + --> STEP: 128/234 -- GLOBAL_STEP: 40610 + | > loss: -0.19330 (-0.16584) + | > log_mle: -0.36292 (-0.30258) + | > loss_dur: 0.16962 (0.13673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.61003 (16.09172) + | > current_lr: 0.00004 + | > step_time: 8.70660 (2.77741) + | > loader_time: 0.10090 (0.05773) + + + --> STEP: 133/234 -- GLOBAL_STEP: 40615 + | > loss: -0.20728 (-0.16738) + | > log_mle: -0.39490 (-0.30602) + | > loss_dur: 0.18762 (0.13864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.26325 (16.98527) + | > current_lr: 0.00004 + | > step_time: 2.30000 (2.75638) + | > loader_time: 0.08320 (0.05702) + + + --> STEP: 138/234 -- GLOBAL_STEP: 40620 + | > loss: -0.15957 (-0.16837) + | > log_mle: -0.34154 (-0.30912) + | > loss_dur: 0.18196 (0.14076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.75346 (17.80552) + | > current_lr: 0.00004 + | > step_time: 1.31600 (2.73506) + | > loader_time: 0.08270 (0.05692) + + + --> STEP: 143/234 -- GLOBAL_STEP: 40625 + | > loss: -0.25395 (-0.17021) + | > log_mle: -0.48870 (-0.31304) + | > loss_dur: 0.23475 (0.14283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.63368 (18.94004) + | > current_lr: 0.00004 + | > step_time: 1.29440 (2.73025) + | > loader_time: 0.00220 (0.05616) + + + --> STEP: 148/234 -- GLOBAL_STEP: 40630 + | > loss: -0.23113 (-0.17261) + | > log_mle: -0.39862 (-0.31709) + | > loss_dur: 0.16750 (0.14447) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.78568 (19.79747) + | > current_lr: 0.00004 + | > step_time: 1.20900 (2.71499) + | > loader_time: 0.00370 (0.05502) + + + --> STEP: 153/234 -- GLOBAL_STEP: 40635 + | > loss: -0.32723 (-0.17592) + | > log_mle: -0.52636 (-0.32222) + | > loss_dur: 0.19913 (0.14629) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.11757 (21.36328) + | > current_lr: 0.00004 + | > step_time: 1.57620 (2.69134) + | > loader_time: 0.00170 (0.05389) + + + --> STEP: 158/234 -- GLOBAL_STEP: 40640 + | > loss: -0.23310 (-0.17838) + | > log_mle: -0.45008 (-0.32663) + | > loss_dur: 0.21698 (0.14824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.49645 (22.75288) + | > current_lr: 0.00004 + | > step_time: 5.60350 (2.73015) + | > loader_time: 0.09720 (0.05347) + + + --> STEP: 163/234 -- GLOBAL_STEP: 40645 + | > loss: -0.22048 (-0.18096) + | > log_mle: -0.42477 (-0.33088) + | > loss_dur: 0.20429 (0.14993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.61934 (23.96519) + | > current_lr: 0.00004 + | > step_time: 10.39810 (2.85803) + | > loader_time: 0.19460 (0.05554) + + + --> STEP: 168/234 -- GLOBAL_STEP: 40650 + | > loss: -0.26031 (-0.18371) + | > log_mle: -0.48865 (-0.33542) + | > loss_dur: 0.22833 (0.15171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.94992 (25.06940) + | > current_lr: 0.00004 + | > step_time: 1.91310 (2.86341) + | > loader_time: 0.00890 (0.05577) + + + --> STEP: 173/234 -- GLOBAL_STEP: 40655 + | > loss: -0.27489 (-0.18691) + | > log_mle: -0.49465 (-0.34062) + | > loss_dur: 0.21976 (0.15371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.54507 (26.31028) + | > current_lr: 0.00004 + | > step_time: 2.39430 (2.86151) + | > loader_time: 0.09730 (0.05585) + + + --> STEP: 178/234 -- GLOBAL_STEP: 40660 + | > loss: -0.31724 (-0.19007) + | > log_mle: -0.55743 (-0.34582) + | > loss_dur: 0.24019 (0.15574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.40527 (27.41301) + | > current_lr: 0.00004 + | > step_time: 1.37570 (2.87946) + | > loader_time: 0.00300 (0.05537) + + + --> STEP: 183/234 -- GLOBAL_STEP: 40665 + | > loss: -0.34192 (-0.19275) + | > log_mle: -0.55262 (-0.35051) + | > loss_dur: 0.21070 (0.15776) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.79285 (28.64071) + | > current_lr: 0.00004 + | > step_time: 1.02010 (2.87559) + | > loader_time: 0.08270 (0.05535) + + + --> STEP: 188/234 -- GLOBAL_STEP: 40670 + | > loss: -0.33403 (-0.19556) + | > log_mle: -0.56154 (-0.35527) + | > loss_dur: 0.22751 (0.15971) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.31213 (29.85022) + | > current_lr: 0.00004 + | > step_time: 4.69800 (2.89529) + | > loader_time: 0.00300 (0.05501) + + + --> STEP: 193/234 -- GLOBAL_STEP: 40675 + | > loss: -0.34461 (-0.19870) + | > log_mle: -0.56016 (-0.35991) + | > loss_dur: 0.21555 (0.16122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.06622 (31.04737) + | > current_lr: 0.00004 + | > step_time: 2.20650 (2.94263) + | > loader_time: 0.00680 (0.05564) + + + --> STEP: 198/234 -- GLOBAL_STEP: 40680 + | > loss: -0.32431 (-0.20150) + | > log_mle: -0.55360 (-0.36432) + | > loss_dur: 0.22929 (0.16282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.33958 (31.94817) + | > current_lr: 0.00004 + | > step_time: 11.40330 (3.00728) + | > loader_time: 0.09460 (0.05921) + + + --> STEP: 203/234 -- GLOBAL_STEP: 40685 + | > loss: -0.25044 (-0.20380) + | > log_mle: -0.47503 (-0.36838) + | > loss_dur: 0.22459 (0.16458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.82950 (33.10083) + | > current_lr: 0.00004 + | > step_time: 3.09660 (3.01154) + | > loader_time: 0.00200 (0.05870) + + + --> STEP: 208/234 -- GLOBAL_STEP: 40690 + | > loss: -0.32681 (-0.20698) + | > log_mle: -0.57101 (-0.37339) + | > loss_dur: 0.24419 (0.16641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.73996 (34.13342) + | > current_lr: 0.00004 + | > step_time: 10.30020 (3.13474) + | > loader_time: 0.01330 (0.05744) + + + --> STEP: 213/234 -- GLOBAL_STEP: 40695 + | > loss: -0.35841 (-0.21046) + | > log_mle: -0.60568 (-0.37866) + | > loss_dur: 0.24726 (0.16820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.71056 (35.54894) + | > current_lr: 0.00004 + | > step_time: 1.79930 (3.13533) + | > loader_time: 0.09100 (0.05751) + + + --> STEP: 218/234 -- GLOBAL_STEP: 40700 + | > loss: -0.33584 (-0.21351) + | > log_mle: -0.57301 (-0.38340) + | > loss_dur: 0.23717 (0.16990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.78025 (36.86028) + | > current_lr: 0.00004 + | > step_time: 8.49240 (3.19695) + | > loader_time: 0.10380 (0.05800) + + + --> STEP: 223/234 -- GLOBAL_STEP: 40705 + | > loss: -0.36823 (-0.21696) + | > log_mle: -0.60947 (-0.38859) + | > loss_dur: 0.24125 (0.17162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 119.65790 (38.23488) + | > current_lr: 0.00004 + | > step_time: 0.23840 (3.19683) + | > loader_time: 0.00270 (0.06064) + + + --> STEP: 228/234 -- GLOBAL_STEP: 40710 + | > loss: -0.34632 (-0.22039) + | > log_mle: -0.61908 (-0.39396) + | > loss_dur: 0.27276 (0.17357) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.22374 (39.59685) + | > current_lr: 0.00004 + | > step_time: 0.24470 (3.13185) + | > loader_time: 0.00770 (0.05940) + + + --> STEP: 233/234 -- GLOBAL_STEP: 40715 + | > loss: 0.05896 (-0.22143) + | > log_mle: -0.59055 (-0.40062) + | > loss_dur: 0.64951 (0.17918) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.26354 (40.99860) + | > current_lr: 0.00004 + | > step_time: 0.19250 (3.07024) + | > loader_time: 0.00340 (0.05830) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27457 (-0.66894) + | > avg_loss: -0.27208 (-0.04190) + | > avg_log_mle: -0.48549 (-0.03037) + | > avg_loss_dur: 0.21341 (-0.01153) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_40716.pth + + > EPOCH: 174/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 21:17:51)  + + --> STEP: 4/234 -- GLOBAL_STEP: 40720 + | > loss: -0.15185 (-0.16088) + | > log_mle: -0.28590 (-0.28836) + | > loss_dur: 0.13405 (0.12747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.00713 (24.56441) + | > current_lr: 0.00004 + | > step_time: 3.00820 (4.53164) + | > loader_time: 0.00140 (0.05002) + + + --> STEP: 9/234 -- GLOBAL_STEP: 40725 + | > loss: -0.16696 (-0.17203) + | > log_mle: -0.29901 (-0.29196) + | > loss_dur: 0.13205 (0.11993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.36521 (19.23846) + | > current_lr: 0.00004 + | > step_time: 4.69200 (5.79090) + | > loader_time: 0.00490 (0.02450) + + + --> STEP: 14/234 -- GLOBAL_STEP: 40730 + | > loss: -0.17637 (-0.17604) + | > log_mle: -0.29250 (-0.29245) + | > loss_dur: 0.11613 (0.11641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.21744 (17.68533) + | > current_lr: 0.00004 + | > step_time: 1.48470 (4.47641) + | > loader_time: 0.00320 (0.01704) + + + --> STEP: 19/234 -- GLOBAL_STEP: 40735 + | > loss: -0.19639 (-0.17793) + | > log_mle: -0.28358 (-0.28987) + | > loss_dur: 0.08718 (0.11194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.86540 (15.98443) + | > current_lr: 0.00004 + | > step_time: 0.87820 (3.61342) + | > loader_time: 0.00130 (0.01734) + + + --> STEP: 24/234 -- GLOBAL_STEP: 40740 + | > loss: -0.20541 (-0.17970) + | > log_mle: -0.28491 (-0.28854) + | > loss_dur: 0.07950 (0.10884) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.73829 (14.61996) + | > current_lr: 0.00004 + | > step_time: 1.28330 (3.15454) + | > loader_time: 0.00210 (0.01416) + + + --> STEP: 29/234 -- GLOBAL_STEP: 40745 + | > loss: -0.15123 (-0.17939) + | > log_mle: -0.26932 (-0.28734) + | > loss_dur: 0.11809 (0.10795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.80462 (13.60225) + | > current_lr: 0.00004 + | > step_time: 1.76880 (3.03373) + | > loader_time: 0.08110 (0.01501) + + + --> STEP: 34/234 -- GLOBAL_STEP: 40750 + | > loss: -0.15474 (-0.17768) + | > log_mle: -0.27828 (-0.28695) + | > loss_dur: 0.12354 (0.10926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.64461 (13.12975) + | > current_lr: 0.00004 + | > step_time: 1.63550 (2.76881) + | > loader_time: 0.00240 (0.01310) + + + --> STEP: 39/234 -- GLOBAL_STEP: 40755 + | > loss: -0.15974 (-0.17556) + | > log_mle: -0.28353 (-0.28640) + | > loss_dur: 0.12379 (0.11084) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.61281 (13.12134) + | > current_lr: 0.00004 + | > step_time: 0.90610 (2.64133) + | > loader_time: 0.00190 (0.01387) + + + --> STEP: 44/234 -- GLOBAL_STEP: 40760 + | > loss: -0.18435 (-0.17392) + | > log_mle: -0.27751 (-0.28537) + | > loss_dur: 0.09315 (0.11145) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.74067 (12.78364) + | > current_lr: 0.00004 + | > step_time: 1.27590 (2.59493) + | > loader_time: 0.00250 (0.01678) + + + --> STEP: 49/234 -- GLOBAL_STEP: 40765 + | > loss: -0.18253 (-0.17263) + | > log_mle: -0.28758 (-0.28511) + | > loss_dur: 0.10505 (0.11248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.43344 (12.67609) + | > current_lr: 0.00004 + | > step_time: 2.99040 (2.50084) + | > loader_time: 0.00480 (0.01534) + + + --> STEP: 54/234 -- GLOBAL_STEP: 40770 + | > loss: -0.17491 (-0.17103) + | > log_mle: -0.28963 (-0.28449) + | > loss_dur: 0.11472 (0.11346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.48529 (12.37847) + | > current_lr: 0.00004 + | > step_time: 2.00030 (2.41742) + | > loader_time: 0.08790 (0.01720) + + + --> STEP: 59/234 -- GLOBAL_STEP: 40775 + | > loss: -0.17982 (-0.17018) + | > log_mle: -0.29366 (-0.28429) + | > loss_dur: 0.11384 (0.11411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.28244 (12.26290) + | > current_lr: 0.00004 + | > step_time: 1.70400 (2.39557) + | > loader_time: 0.00310 (0.01883) + + + --> STEP: 64/234 -- GLOBAL_STEP: 40780 + | > loss: -0.16325 (-0.16816) + | > log_mle: -0.27820 (-0.28467) + | > loss_dur: 0.11495 (0.11650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.10103 (12.56474) + | > current_lr: 0.00004 + | > step_time: 3.11360 (2.46554) + | > loader_time: 0.07710 (0.02632) + + + --> STEP: 69/234 -- GLOBAL_STEP: 40785 + | > loss: -0.14564 (-0.16690) + | > log_mle: -0.26335 (-0.28410) + | > loss_dur: 0.11771 (0.11720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.14908 (12.41946) + | > current_lr: 0.00004 + | > step_time: 1.88570 (2.42462) + | > loader_time: 0.00220 (0.02581) + + + --> STEP: 74/234 -- GLOBAL_STEP: 40790 + | > loss: -0.14461 (-0.16520) + | > log_mle: -0.26781 (-0.28406) + | > loss_dur: 0.12320 (0.11886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.68644 (12.59398) + | > current_lr: 0.00004 + | > step_time: 0.99840 (2.36767) + | > loader_time: 0.00360 (0.02548) + + + --> STEP: 79/234 -- GLOBAL_STEP: 40795 + | > loss: -0.15718 (-0.16402) + | > log_mle: -0.28714 (-0.28411) + | > loss_dur: 0.12996 (0.12009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.63466 (12.49042) + | > current_lr: 0.00004 + | > step_time: 2.11180 (2.33100) + | > loader_time: 0.08170 (0.02594) + + + --> STEP: 84/234 -- GLOBAL_STEP: 40800 + | > loss: -0.15280 (-0.16336) + | > log_mle: -0.28189 (-0.28416) + | > loss_dur: 0.12909 (0.12080) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.48341 (12.44734) + | > current_lr: 0.00004 + | > step_time: 1.61340 (2.29050) + | > loader_time: 0.00230 (0.02555) + + + --> STEP: 89/234 -- GLOBAL_STEP: 40805 + | > loss: -0.16888 (-0.16327) + | > log_mle: -0.31049 (-0.28508) + | > loss_dur: 0.14162 (0.12181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.17319 (12.57653) + | > current_lr: 0.00004 + | > step_time: 2.26600 (2.29368) + | > loader_time: 0.00670 (0.02520) + + + --> STEP: 94/234 -- GLOBAL_STEP: 40810 + | > loss: -0.20185 (-0.16374) + | > log_mle: -0.34618 (-0.28737) + | > loss_dur: 0.14432 (0.12363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.50324 (12.94850) + | > current_lr: 0.00004 + | > step_time: 1.72390 (2.30232) + | > loader_time: 0.00400 (0.02485) + + + --> STEP: 99/234 -- GLOBAL_STEP: 40815 + | > loss: -0.19843 (-0.16437) + | > log_mle: -0.38304 (-0.28970) + | > loss_dur: 0.18461 (0.12534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.51939 (13.30525) + | > current_lr: 0.00004 + | > step_time: 1.40200 (2.28247) + | > loader_time: 0.00300 (0.02377) + + + --> STEP: 104/234 -- GLOBAL_STEP: 40820 + | > loss: -0.22134 (-0.16523) + | > log_mle: -0.39297 (-0.29254) + | > loss_dur: 0.17162 (0.12731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.61591 (13.86126) + | > current_lr: 0.00004 + | > step_time: 4.49430 (2.32660) + | > loader_time: 0.00560 (0.02461) + + + --> STEP: 109/234 -- GLOBAL_STEP: 40825 + | > loss: -0.16069 (-0.16527) + | > log_mle: -0.35885 (-0.29443) + | > loss_dur: 0.19817 (0.12917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.46336 (14.29537) + | > current_lr: 0.00004 + | > step_time: 1.19480 (2.33276) + | > loader_time: 0.00250 (0.02704) + + + --> STEP: 114/234 -- GLOBAL_STEP: 40830 + | > loss: -0.18350 (-0.16590) + | > log_mle: -0.34027 (-0.29717) + | > loss_dur: 0.15677 (0.13126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.43185 (15.04507) + | > current_lr: 0.00004 + | > step_time: 1.80780 (2.33005) + | > loader_time: 0.00350 (0.02737) + + + --> STEP: 119/234 -- GLOBAL_STEP: 40835 + | > loss: -0.16804 (-0.16592) + | > log_mle: -0.33314 (-0.29912) + | > loss_dur: 0.16510 (0.13320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.46131 (15.62384) + | > current_lr: 0.00004 + | > step_time: 1.99730 (2.33210) + | > loader_time: 0.09180 (0.02709) + + + --> STEP: 124/234 -- GLOBAL_STEP: 40840 + | > loss: -0.19174 (-0.16583) + | > log_mle: -0.36514 (-0.30034) + | > loss_dur: 0.17341 (0.13451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.98291 (15.96722) + | > current_lr: 0.00004 + | > step_time: 1.72460 (2.34678) + | > loader_time: 0.00200 (0.02612) + + + --> STEP: 129/234 -- GLOBAL_STEP: 40845 + | > loss: -0.16309 (-0.16677) + | > log_mle: -0.35623 (-0.30329) + | > loss_dur: 0.19314 (0.13652) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.04432 (16.75275) + | > current_lr: 0.00004 + | > step_time: 1.18980 (2.35422) + | > loader_time: 0.09630 (0.02745) + + + --> STEP: 134/234 -- GLOBAL_STEP: 40850 + | > loss: -0.20581 (-0.16863) + | > log_mle: -0.40940 (-0.30699) + | > loss_dur: 0.20359 (0.13836) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.40948 (17.45321) + | > current_lr: 0.00004 + | > step_time: 1.47000 (2.37638) + | > loader_time: 0.00270 (0.02803) + + + --> STEP: 139/234 -- GLOBAL_STEP: 40855 + | > loss: -0.27183 (-0.17020) + | > log_mle: -0.47072 (-0.31048) + | > loss_dur: 0.19888 (0.14029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.08596 (18.22180) + | > current_lr: 0.00004 + | > step_time: 4.59650 (2.37368) + | > loader_time: 0.00440 (0.02777) + + + --> STEP: 144/234 -- GLOBAL_STEP: 40860 + | > loss: -0.25473 (-0.17180) + | > log_mle: -0.45067 (-0.31428) + | > loss_dur: 0.19594 (0.14248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.24088 (19.02760) + | > current_lr: 0.00004 + | > step_time: 1.87850 (2.37044) + | > loader_time: 0.00210 (0.02748) + + + --> STEP: 149/234 -- GLOBAL_STEP: 40865 + | > loss: -0.29483 (-0.17438) + | > log_mle: -0.49940 (-0.31863) + | > loss_dur: 0.20458 (0.14425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.58486 (19.97958) + | > current_lr: 0.00004 + | > step_time: 2.40720 (2.35201) + | > loader_time: 0.07820 (0.02780) + + + --> STEP: 154/234 -- GLOBAL_STEP: 40870 + | > loss: -0.26324 (-0.17741) + | > log_mle: -0.45788 (-0.32349) + | > loss_dur: 0.19464 (0.14608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.54716 (21.17825) + | > current_lr: 0.00004 + | > step_time: 1.49630 (2.32883) + | > loader_time: 0.08630 (0.02934) + + + --> STEP: 159/234 -- GLOBAL_STEP: 40875 + | > loss: -0.26726 (-0.18004) + | > log_mle: -0.47824 (-0.32812) + | > loss_dur: 0.21098 (0.14809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.24459 (22.23947) + | > current_lr: 0.00004 + | > step_time: 2.40920 (2.33923) + | > loader_time: 0.08810 (0.03022) + + + --> STEP: 164/234 -- GLOBAL_STEP: 40880 + | > loss: -0.25210 (-0.18259) + | > log_mle: -0.46523 (-0.33241) + | > loss_dur: 0.21313 (0.14982) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.51382 (23.37527) + | > current_lr: 0.00004 + | > step_time: 0.68560 (2.37712) + | > loader_time: 0.00290 (0.03118) + + + --> STEP: 169/234 -- GLOBAL_STEP: 40885 + | > loss: -0.25313 (-0.18541) + | > log_mle: -0.46838 (-0.33702) + | > loss_dur: 0.21525 (0.15161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.41559 (24.42860) + | > current_lr: 0.00004 + | > step_time: 2.49880 (2.36178) + | > loader_time: 0.09140 (0.03138) + + + --> STEP: 174/234 -- GLOBAL_STEP: 40890 + | > loss: -0.33165 (-0.18898) + | > log_mle: -0.54982 (-0.34253) + | > loss_dur: 0.21817 (0.15355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.59901 (26.06587) + | > current_lr: 0.00004 + | > step_time: 4.59550 (2.37674) + | > loader_time: 0.18680 (0.03265) + + + --> STEP: 179/234 -- GLOBAL_STEP: 40895 + | > loss: -0.29809 (-0.19181) + | > log_mle: -0.55020 (-0.34755) + | > loss_dur: 0.25211 (0.15574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.42626 (27.27617) + | > current_lr: 0.00004 + | > step_time: 1.31820 (2.45788) + | > loader_time: 0.00370 (0.03342) + + + --> STEP: 184/234 -- GLOBAL_STEP: 40900 + | > loss: -0.28858 (-0.19440) + | > log_mle: -0.51626 (-0.35206) + | > loss_dur: 0.22768 (0.15766) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.38700 (28.41030) + | > current_lr: 0.00004 + | > step_time: 4.30070 (2.49763) + | > loader_time: 0.08480 (0.03353) + + + --> STEP: 189/234 -- GLOBAL_STEP: 40905 + | > loss: -0.28486 (-0.19712) + | > log_mle: -0.50330 (-0.35674) + | > loss_dur: 0.21844 (0.15963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.16980 (29.62494) + | > current_lr: 0.00004 + | > step_time: 5.89870 (2.54852) + | > loader_time: 0.18900 (0.03474) + + + --> STEP: 194/234 -- GLOBAL_STEP: 40910 + | > loss: -0.32552 (-0.20042) + | > log_mle: -0.54040 (-0.36146) + | > loss_dur: 0.21488 (0.16104) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.28549 (30.71808) + | > current_lr: 0.00004 + | > step_time: 7.78690 (2.59414) + | > loader_time: 0.00930 (0.03547) + + + --> STEP: 199/234 -- GLOBAL_STEP: 40915 + | > loss: -0.30556 (-0.20316) + | > log_mle: -0.54414 (-0.36585) + | > loss_dur: 0.23859 (0.16269) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.35904 (31.84920) + | > current_lr: 0.00004 + | > step_time: 7.11720 (2.63470) + | > loader_time: 0.17610 (0.03599) + + + --> STEP: 204/234 -- GLOBAL_STEP: 40920 + | > loss: -0.34707 (-0.20566) + | > log_mle: -0.59424 (-0.37016) + | > loss_dur: 0.24717 (0.16451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.98172 (32.76484) + | > current_lr: 0.00004 + | > step_time: 4.40180 (2.64465) + | > loader_time: 0.20390 (0.03620) + + + --> STEP: 209/234 -- GLOBAL_STEP: 40925 + | > loss: -0.32330 (-0.20873) + | > log_mle: -0.54763 (-0.37491) + | > loss_dur: 0.22432 (0.16618) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.37646 (33.68250) + | > current_lr: 0.00004 + | > step_time: 8.29300 (2.74112) + | > loader_time: 0.00580 (0.03689) + + + --> STEP: 214/234 -- GLOBAL_STEP: 40930 + | > loss: -0.36425 (-0.21250) + | > log_mle: -0.57672 (-0.38040) + | > loss_dur: 0.21247 (0.16790) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.23624 (35.09803) + | > current_lr: 0.00004 + | > step_time: 3.80140 (2.82850) + | > loader_time: 0.00710 (0.03881) + + + --> STEP: 219/234 -- GLOBAL_STEP: 40935 + | > loss: -0.43345 (-0.21611) + | > log_mle: -0.67394 (-0.38575) + | > loss_dur: 0.24049 (0.16964) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 121.35466 (36.47016) + | > current_lr: 0.00004 + | > step_time: 3.90530 (2.86898) + | > loader_time: 0.08770 (0.03971) + + + --> STEP: 224/234 -- GLOBAL_STEP: 40940 + | > loss: -0.38852 (-0.21950) + | > log_mle: -0.63616 (-0.39083) + | > loss_dur: 0.24764 (0.17133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.39082 (37.54572) + | > current_lr: 0.00004 + | > step_time: 0.99790 (2.87740) + | > loader_time: 0.00310 (0.04102) + + + --> STEP: 229/234 -- GLOBAL_STEP: 40945 + | > loss: -0.37068 (-0.22299) + | > log_mle: -0.67657 (-0.39647) + | > loss_dur: 0.30589 (0.17348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.60110 (39.00406) + | > current_lr: 0.00004 + | > step_time: 0.24630 (2.81989) + | > loader_time: 0.00500 (0.04022) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02989 (-0.24467) + | > avg_loss: -0.24208 (+0.03000) + | > avg_log_mle: -0.46645 (+0.01904) + | > avg_loss_dur: 0.22437 (+0.01096) + + + > EPOCH: 175/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 21:30:01)  + + --> STEP: 0/234 -- GLOBAL_STEP: 40950 + | > loss: -0.22756 (-0.22756) + | > log_mle: -0.36547 (-0.36547) + | > loss_dur: 0.13791 (0.13791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.34784 (26.34784) + | > current_lr: 0.00004 + | > step_time: 4.90360 (4.90358) + | > loader_time: 12.51200 (12.51195) + + + --> STEP: 5/234 -- GLOBAL_STEP: 40955 + | > loss: -0.17374 (-0.16268) + | > log_mle: -0.28968 (-0.28928) + | > loss_dur: 0.11593 (0.12660) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.12267 (20.13127) + | > current_lr: 0.00004 + | > step_time: 0.67280 (7.13639) + | > loader_time: 0.00120 (0.06309) + + + --> STEP: 10/234 -- GLOBAL_STEP: 40960 + | > loss: -0.16206 (-0.16719) + | > log_mle: -0.28852 (-0.29149) + | > loss_dur: 0.12646 (0.12430) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.16302 (17.47883) + | > current_lr: 0.00004 + | > step_time: 2.41350 (4.65935) + | > loader_time: 0.00310 (0.04076) + + + --> STEP: 15/234 -- GLOBAL_STEP: 40965 + | > loss: -0.20748 (-0.17442) + | > log_mle: -0.29417 (-0.29249) + | > loss_dur: 0.08669 (0.11807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.89992 (15.34542) + | > current_lr: 0.00004 + | > step_time: 4.60960 (3.89292) + | > loader_time: 0.08430 (0.03375) + + + --> STEP: 20/234 -- GLOBAL_STEP: 40970 + | > loss: -0.18191 (-0.17594) + | > log_mle: -0.28365 (-0.28995) + | > loss_dur: 0.10174 (0.11400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.99783 (14.41194) + | > current_lr: 0.00004 + | > step_time: 3.90930 (3.86985) + | > loader_time: 0.08620 (0.03538) + + + --> STEP: 25/234 -- GLOBAL_STEP: 40975 + | > loss: -0.17007 (-0.17790) + | > log_mle: -0.27314 (-0.28807) + | > loss_dur: 0.10307 (0.11017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.42701 (13.72194) + | > current_lr: 0.00004 + | > step_time: 2.29450 (3.67989) + | > loader_time: 0.00290 (0.03565) + + + --> STEP: 30/234 -- GLOBAL_STEP: 40980 + | > loss: -0.19773 (-0.17935) + | > log_mle: -0.29098 (-0.28774) + | > loss_dur: 0.09326 (0.10839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.23816 (13.24144) + | > current_lr: 0.00004 + | > step_time: 2.90090 (3.66304) + | > loader_time: 0.08470 (0.03897) + + + --> STEP: 35/234 -- GLOBAL_STEP: 40985 + | > loss: -0.14820 (-0.17726) + | > log_mle: -0.28410 (-0.28718) + | > loss_dur: 0.13589 (0.10992) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.88289 (12.86647) + | > current_lr: 0.00004 + | > step_time: 2.62150 (3.92015) + | > loader_time: 0.00250 (0.04207) + + + --> STEP: 40/234 -- GLOBAL_STEP: 40990 + | > loss: -0.13109 (-0.17514) + | > log_mle: -0.26677 (-0.28647) + | > loss_dur: 0.13568 (0.11133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.68148 (12.63490) + | > current_lr: 0.00004 + | > step_time: 0.87350 (3.58429) + | > loader_time: 0.00200 (0.03709) + + + --> STEP: 45/234 -- GLOBAL_STEP: 40995 + | > loss: -0.17121 (-0.17487) + | > log_mle: -0.30076 (-0.28635) + | > loss_dur: 0.12955 (0.11147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.56192 (12.37361) + | > current_lr: 0.00004 + | > step_time: 1.56940 (3.32092) + | > loader_time: 0.00230 (0.03644) + + + --> STEP: 50/234 -- GLOBAL_STEP: 41000 + | > loss: -0.15190 (-0.17337) + | > log_mle: -0.27395 (-0.28563) + | > loss_dur: 0.12205 (0.11226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.93994 (11.98448) + | > current_lr: 0.00004 + | > step_time: 1.05480 (3.15190) + | > loader_time: 0.00270 (0.03309) + + + --> STEP: 55/234 -- GLOBAL_STEP: 41005 + | > loss: -0.19552 (-0.17265) + | > log_mle: -0.29113 (-0.28547) + | > loss_dur: 0.09560 (0.11282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.44021 (11.76786) + | > current_lr: 0.00004 + | > step_time: 1.50610 (3.05319) + | > loader_time: 0.00230 (0.03190) + + + --> STEP: 60/234 -- GLOBAL_STEP: 41010 + | > loss: -0.16352 (-0.17124) + | > log_mle: -0.30350 (-0.28551) + | > loss_dur: 0.13998 (0.11427) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.57729 (11.75698) + | > current_lr: 0.00004 + | > step_time: 3.03690 (2.95872) + | > loader_time: 0.08090 (0.03209) + + + --> STEP: 65/234 -- GLOBAL_STEP: 41015 + | > loss: -0.16452 (-0.16959) + | > log_mle: -0.28158 (-0.28560) + | > loss_dur: 0.11706 (0.11601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.07971 (11.88236) + | > current_lr: 0.00004 + | > step_time: 2.81320 (2.86788) + | > loader_time: 0.00290 (0.02979) + + + --> STEP: 70/234 -- GLOBAL_STEP: 41020 + | > loss: -0.13715 (-0.16802) + | > log_mle: -0.27366 (-0.28495) + | > loss_dur: 0.13651 (0.11693) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.33774 (11.84066) + | > current_lr: 0.00004 + | > step_time: 1.80210 (2.77829) + | > loader_time: 0.08820 (0.02908) + + + --> STEP: 75/234 -- GLOBAL_STEP: 41025 + | > loss: -0.14203 (-0.16662) + | > log_mle: -0.29256 (-0.28531) + | > loss_dur: 0.15053 (0.11869) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.72552 (11.92312) + | > current_lr: 0.00004 + | > step_time: 3.19890 (2.76234) + | > loader_time: 0.00440 (0.02854) + + + --> STEP: 80/234 -- GLOBAL_STEP: 41030 + | > loss: -0.15570 (-0.16585) + | > log_mle: -0.27379 (-0.28522) + | > loss_dur: 0.11809 (0.11938) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.87722 (11.80024) + | > current_lr: 0.00004 + | > step_time: 1.08020 (2.72339) + | > loader_time: 0.00220 (0.02907) + + + --> STEP: 85/234 -- GLOBAL_STEP: 41035 + | > loss: -0.15843 (-0.16486) + | > log_mle: -0.28436 (-0.28538) + | > loss_dur: 0.12593 (0.12052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.62637 (11.93488) + | > current_lr: 0.00004 + | > step_time: 1.58010 (2.64886) + | > loader_time: 0.00240 (0.02861) + + + --> STEP: 90/234 -- GLOBAL_STEP: 41040 + | > loss: -0.14973 (-0.16452) + | > log_mle: -0.30782 (-0.28650) + | > loss_dur: 0.15809 (0.12198) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.51806 (12.17731) + | > current_lr: 0.00004 + | > step_time: 7.40390 (2.72647) + | > loader_time: 0.20270 (0.03037) + + + --> STEP: 95/234 -- GLOBAL_STEP: 41045 + | > loss: -0.20087 (-0.16543) + | > log_mle: -0.38168 (-0.28942) + | > loss_dur: 0.18082 (0.12399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.18969 (13.00613) + | > current_lr: 0.00004 + | > step_time: 1.61410 (2.67608) + | > loader_time: 0.00600 (0.03073) + + + --> STEP: 100/234 -- GLOBAL_STEP: 41050 + | > loss: -0.17328 (-0.16536) + | > log_mle: -0.31963 (-0.29067) + | > loss_dur: 0.14635 (0.12532) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.70066 (13.16947) + | > current_lr: 0.00004 + | > step_time: 1.30800 (2.66718) + | > loader_time: 0.09690 (0.03105) + + + --> STEP: 105/234 -- GLOBAL_STEP: 41055 + | > loss: -0.17164 (-0.16624) + | > log_mle: -0.29980 (-0.29314) + | > loss_dur: 0.12816 (0.12690) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.21128 (13.66307) + | > current_lr: 0.00004 + | > step_time: 2.89490 (2.69930) + | > loader_time: 0.10120 (0.03062) + + + --> STEP: 110/234 -- GLOBAL_STEP: 41060 + | > loss: -0.16765 (-0.16600) + | > log_mle: -0.32252 (-0.29517) + | > loss_dur: 0.15487 (0.12917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.94809 (14.21377) + | > current_lr: 0.00004 + | > step_time: 2.31370 (2.67954) + | > loader_time: 0.07750 (0.03094) + + + --> STEP: 115/234 -- GLOBAL_STEP: 41065 + | > loss: -0.16545 (-0.16666) + | > log_mle: -0.34241 (-0.29797) + | > loss_dur: 0.17697 (0.13130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.54730 (14.87969) + | > current_lr: 0.00004 + | > step_time: 1.84290 (2.63378) + | > loader_time: 0.00200 (0.03039) + + + --> STEP: 120/234 -- GLOBAL_STEP: 41070 + | > loss: -0.21425 (-0.16722) + | > log_mle: -0.39491 (-0.30039) + | > loss_dur: 0.18066 (0.13317) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.82239 (15.34054) + | > current_lr: 0.00004 + | > step_time: 2.37160 (2.63729) + | > loader_time: 0.00340 (0.03011) + + + --> STEP: 125/234 -- GLOBAL_STEP: 41075 + | > loss: -0.19494 (-0.16718) + | > log_mle: -0.37846 (-0.30162) + | > loss_dur: 0.18351 (0.13445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.88768 (15.66949) + | > current_lr: 0.00004 + | > step_time: 2.70220 (2.60934) + | > loader_time: 0.08770 (0.02975) + + + --> STEP: 130/234 -- GLOBAL_STEP: 41080 + | > loss: -0.20936 (-0.16853) + | > log_mle: -0.39240 (-0.30479) + | > loss_dur: 0.18304 (0.13627) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.33714 (16.35267) + | > current_lr: 0.00004 + | > step_time: 2.63650 (2.59864) + | > loader_time: 0.08670 (0.03144) + + + --> STEP: 135/234 -- GLOBAL_STEP: 41085 + | > loss: -0.16098 (-0.17008) + | > log_mle: -0.32206 (-0.30798) + | > loss_dur: 0.16108 (0.13790) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.62292 (17.06379) + | > current_lr: 0.00004 + | > step_time: 1.89650 (2.57150) + | > loader_time: 0.00260 (0.03099) + + + --> STEP: 140/234 -- GLOBAL_STEP: 41090 + | > loss: -0.17271 (-0.17157) + | > log_mle: -0.35895 (-0.31171) + | > loss_dur: 0.18624 (0.14014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.08714 (17.88011) + | > current_lr: 0.00004 + | > step_time: 2.30120 (2.56899) + | > loader_time: 0.00540 (0.03194) + + + --> STEP: 145/234 -- GLOBAL_STEP: 41095 + | > loss: -0.26407 (-0.17389) + | > log_mle: -0.46301 (-0.31621) + | > loss_dur: 0.19895 (0.14232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.55821 (18.84509) + | > current_lr: 0.00004 + | > step_time: 3.91480 (2.59766) + | > loader_time: 0.08400 (0.03266) + + + --> STEP: 150/234 -- GLOBAL_STEP: 41100 + | > loss: -0.24314 (-0.17626) + | > log_mle: -0.44625 (-0.32044) + | > loss_dur: 0.20311 (0.14417) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.43005 (19.76680) + | > current_lr: 0.00004 + | > step_time: 3.91380 (2.61330) + | > loader_time: 0.09530 (0.03414) + + + --> STEP: 155/234 -- GLOBAL_STEP: 41105 + | > loss: -0.29481 (-0.17972) + | > log_mle: -0.50811 (-0.32569) + | > loss_dur: 0.21330 (0.14598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.03708 (21.13161) + | > current_lr: 0.00004 + | > step_time: 2.62200 (2.64396) + | > loader_time: 0.09410 (0.03437) + + + --> STEP: 160/234 -- GLOBAL_STEP: 41110 + | > loss: -0.29055 (-0.18222) + | > log_mle: -0.50601 (-0.33016) + | > loss_dur: 0.21546 (0.14794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.92934 (22.38578) + | > current_lr: 0.00004 + | > step_time: 3.10870 (2.69696) + | > loader_time: 0.00270 (0.03517) + + + --> STEP: 165/234 -- GLOBAL_STEP: 41115 + | > loss: -0.28505 (-0.18469) + | > log_mle: -0.50195 (-0.33435) + | > loss_dur: 0.21689 (0.14966) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.59599 (23.61344) + | > current_lr: 0.00004 + | > step_time: 3.08040 (2.73522) + | > loader_time: 0.00200 (0.03597) + + + --> STEP: 170/234 -- GLOBAL_STEP: 41120 + | > loss: -0.30320 (-0.18745) + | > log_mle: -0.53800 (-0.33900) + | > loss_dur: 0.23480 (0.15156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.71529 (25.04695) + | > current_lr: 0.00004 + | > step_time: 4.49800 (2.73121) + | > loader_time: 0.19520 (0.03658) + + + --> STEP: 175/234 -- GLOBAL_STEP: 41125 + | > loss: -0.27248 (-0.19092) + | > log_mle: -0.51063 (-0.34450) + | > loss_dur: 0.23815 (0.15358) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.83572 (26.31902) + | > current_lr: 0.00004 + | > step_time: 1.59940 (2.74570) + | > loader_time: 0.00360 (0.03712) + + + --> STEP: 180/234 -- GLOBAL_STEP: 41130 + | > loss: -0.29384 (-0.19391) + | > log_mle: -0.51923 (-0.34955) + | > loss_dur: 0.22539 (0.15564) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.35567 (27.39854) + | > current_lr: 0.00004 + | > step_time: 1.91350 (2.76625) + | > loader_time: 0.06670 (0.03756) + + + --> STEP: 185/234 -- GLOBAL_STEP: 41135 + | > loss: -0.30588 (-0.19653) + | > log_mle: -0.54501 (-0.35416) + | > loss_dur: 0.23913 (0.15763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.15580 (28.67004) + | > current_lr: 0.00004 + | > step_time: 4.59730 (2.81589) + | > loader_time: 0.00450 (0.03777) + + + --> STEP: 190/234 -- GLOBAL_STEP: 41140 + | > loss: -0.31069 (-0.19931) + | > log_mle: -0.52269 (-0.35873) + | > loss_dur: 0.21200 (0.15942) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.11953 (29.87863) + | > current_lr: 0.00004 + | > step_time: 5.31160 (2.87134) + | > loader_time: 0.08140 (0.03987) + + + --> STEP: 195/234 -- GLOBAL_STEP: 41145 + | > loss: -0.30223 (-0.20242) + | > log_mle: -0.54026 (-0.36340) + | > loss_dur: 0.23803 (0.16098) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.40303 (30.96406) + | > current_lr: 0.00004 + | > step_time: 4.91530 (2.90096) + | > loader_time: 0.00560 (0.04024) + + + --> STEP: 200/234 -- GLOBAL_STEP: 41150 + | > loss: -0.30265 (-0.20514) + | > log_mle: -0.54871 (-0.36789) + | > loss_dur: 0.24605 (0.16275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.60208 (31.95586) + | > current_lr: 0.00004 + | > step_time: 6.49570 (3.00353) + | > loader_time: 0.00810 (0.04069) + + + --> STEP: 205/234 -- GLOBAL_STEP: 41155 + | > loss: -0.30371 (-0.20759) + | > log_mle: -0.53926 (-0.37212) + | > loss_dur: 0.23555 (0.16453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.08717 (33.05913) + | > current_lr: 0.00004 + | > step_time: 7.39060 (3.10725) + | > loader_time: 0.10440 (0.04313) + + + --> STEP: 210/234 -- GLOBAL_STEP: 41160 + | > loss: -0.37325 (-0.21084) + | > log_mle: -0.62008 (-0.37714) + | > loss_dur: 0.24683 (0.16630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.53896 (34.24648) + | > current_lr: 0.00004 + | > step_time: 4.19820 (3.17233) + | > loader_time: 0.09460 (0.04397) + + + --> STEP: 215/234 -- GLOBAL_STEP: 41165 + | > loss: -0.32687 (-0.21428) + | > log_mle: -0.57056 (-0.38235) + | > loss_dur: 0.24369 (0.16807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.01284 (35.55567) + | > current_lr: 0.00004 + | > step_time: 1.70110 (3.21300) + | > loader_time: 0.09080 (0.04660) + + + --> STEP: 220/234 -- GLOBAL_STEP: 41170 + | > loss: -0.36610 (-0.21804) + | > log_mle: -0.61193 (-0.38785) + | > loss_dur: 0.24583 (0.16981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.49721 (36.92432) + | > current_lr: 0.00004 + | > step_time: 3.00660 (3.23091) + | > loader_time: 0.00420 (0.04608) + + + --> STEP: 225/234 -- GLOBAL_STEP: 41175 + | > loss: -0.43045 (-0.22161) + | > log_mle: -0.68553 (-0.39315) + | > loss_dur: 0.25508 (0.17154) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 123.81685 (38.13165) + | > current_lr: 0.00004 + | > step_time: 0.23640 (3.18281) + | > loader_time: 0.00410 (0.04550) + + + --> STEP: 230/234 -- GLOBAL_STEP: 41180 + | > loss: -0.41590 (-0.22497) + | > log_mle: -0.74015 (-0.39898) + | > loss_dur: 0.32425 (0.17401) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 126.35368 (39.79697) + | > current_lr: 0.00004 + | > step_time: 0.25280 (3.11891) + | > loader_time: 0.00380 (0.04460) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.57688 (+0.54699) + | > avg_loss: -0.22881 (+0.01327) + | > avg_log_mle: -0.45676 (+0.00969) + | > avg_loss_dur: 0.22795 (+0.00358) + + + > EPOCH: 176/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 21:43:18)  + + --> STEP: 1/234 -- GLOBAL_STEP: 41185 + | > loss: -0.19417 (-0.19417) + | > log_mle: -0.29158 (-0.29158) + | > loss_dur: 0.09741 (0.09741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.80014 (21.80014) + | > current_lr: 0.00004 + | > step_time: 1.50000 (1.50003) + | > loader_time: 0.00080 (0.00081) + + + --> STEP: 6/234 -- GLOBAL_STEP: 41190 + | > loss: -0.18467 (-0.17341) + | > log_mle: -0.28464 (-0.29147) + | > loss_dur: 0.09997 (0.11805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.22748 (18.01839) + | > current_lr: 0.00004 + | > step_time: 0.99680 (3.93451) + | > loader_time: 0.00090 (1.21766) + + + --> STEP: 11/234 -- GLOBAL_STEP: 41195 + | > loss: -0.20112 (-0.17824) + | > log_mle: -0.29699 (-0.29489) + | > loss_dur: 0.09587 (0.11665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.28815 (16.57344) + | > current_lr: 0.00004 + | > step_time: 5.11130 (5.16728) + | > loader_time: 0.09200 (0.70868) + + + --> STEP: 16/234 -- GLOBAL_STEP: 41200 + | > loss: -0.19390 (-0.18133) + | > log_mle: -0.29087 (-0.29493) + | > loss_dur: 0.09697 (0.11360) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.61199 (15.27488) + | > current_lr: 0.00004 + | > step_time: 4.29870 (4.59525) + | > loader_time: 0.00180 (0.49948) + + + --> STEP: 21/234 -- GLOBAL_STEP: 41205 + | > loss: -0.16913 (-0.18054) + | > log_mle: -0.27094 (-0.29139) + | > loss_dur: 0.10181 (0.11085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.53786 (14.10813) + | > current_lr: 0.00004 + | > step_time: 1.21030 (4.65357) + | > loader_time: 0.00160 (0.39513) + + + --> STEP: 26/234 -- GLOBAL_STEP: 41210 + | > loss: -0.17000 (-0.18241) + | > log_mle: -0.28709 (-0.29055) + | > loss_dur: 0.11709 (0.10814) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.79117 (13.51348) + | > current_lr: 0.00004 + | > step_time: 5.39710 (4.55865) + | > loader_time: 0.29570 (0.33094) + + + --> STEP: 31/234 -- GLOBAL_STEP: 41215 + | > loss: -0.14152 (-0.18176) + | > log_mle: -0.28188 (-0.28977) + | > loss_dur: 0.14037 (0.10801) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.14099 (13.21946) + | > current_lr: 0.00004 + | > step_time: 1.09120 (4.56300) + | > loader_time: 0.00150 (0.28111) + + + --> STEP: 36/234 -- GLOBAL_STEP: 41220 + | > loss: -0.15535 (-0.18025) + | > log_mle: -0.27957 (-0.28904) + | > loss_dur: 0.12422 (0.10878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.41305 (13.15047) + | > current_lr: 0.00004 + | > step_time: 1.20520 (4.24110) + | > loader_time: 0.00320 (0.24698) + + + --> STEP: 41/234 -- GLOBAL_STEP: 41225 + | > loss: -0.18009 (-0.17855) + | > log_mle: -0.28389 (-0.28812) + | > loss_dur: 0.10379 (0.10957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85073 (12.82137) + | > current_lr: 0.00004 + | > step_time: 2.16660 (3.93504) + | > loader_time: 0.00150 (0.21710) + + + --> STEP: 46/234 -- GLOBAL_STEP: 41230 + | > loss: -0.14509 (-0.17661) + | > log_mle: -0.27791 (-0.28768) + | > loss_dur: 0.13283 (0.11107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.29236 (12.86974) + | > current_lr: 0.00004 + | > step_time: 2.02700 (3.67230) + | > loader_time: 0.00340 (0.19381) + + + --> STEP: 51/234 -- GLOBAL_STEP: 41235 + | > loss: -0.15454 (-0.17552) + | > log_mle: -0.27199 (-0.28680) + | > loss_dur: 0.11744 (0.11128) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.93225 (12.38519) + | > current_lr: 0.00004 + | > step_time: 1.24690 (3.45362) + | > loader_time: 0.00210 (0.17502) + + + --> STEP: 56/234 -- GLOBAL_STEP: 41240 + | > loss: -0.15765 (-0.17465) + | > log_mle: -0.28800 (-0.28688) + | > loss_dur: 0.13035 (0.11223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.51982 (12.19133) + | > current_lr: 0.00004 + | > step_time: 3.72300 (3.32054) + | > loader_time: 0.09750 (0.16283) + + + --> STEP: 61/234 -- GLOBAL_STEP: 41245 + | > loss: -0.16344 (-0.17376) + | > log_mle: -0.28269 (-0.28678) + | > loss_dur: 0.11925 (0.11302) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.96138 (12.17760) + | > current_lr: 0.00004 + | > step_time: 1.59420 (3.22987) + | > loader_time: 0.00200 (0.15105) + + + --> STEP: 66/234 -- GLOBAL_STEP: 41250 + | > loss: -0.16624 (-0.17232) + | > log_mle: -0.27540 (-0.28674) + | > loss_dur: 0.10917 (0.11441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.89854 (11.98461) + | > current_lr: 0.00004 + | > step_time: 2.11460 (3.18544) + | > loader_time: 0.00480 (0.14125) + + + --> STEP: 71/234 -- GLOBAL_STEP: 41255 + | > loss: -0.13349 (-0.16975) + | > log_mle: -0.30878 (-0.28644) + | > loss_dur: 0.17529 (0.11669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.91693 (12.27095) + | > current_lr: 0.00004 + | > step_time: 1.30180 (3.09124) + | > loader_time: 0.00220 (0.13165) + + + --> STEP: 76/234 -- GLOBAL_STEP: 41260 + | > loss: -0.15265 (-0.16806) + | > log_mle: -0.29078 (-0.28630) + | > loss_dur: 0.13813 (0.11824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.15753 (12.42798) + | > current_lr: 0.00004 + | > step_time: 2.19740 (3.03099) + | > loader_time: 0.08800 (0.12643) + + + --> STEP: 81/234 -- GLOBAL_STEP: 41265 + | > loss: -0.16233 (-0.16727) + | > log_mle: -0.30000 (-0.28619) + | > loss_dur: 0.13766 (0.11891) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.53077 (12.33264) + | > current_lr: 0.00004 + | > step_time: 1.18520 (2.98310) + | > loader_time: 0.00180 (0.11989) + + + --> STEP: 86/234 -- GLOBAL_STEP: 41270 + | > loss: -0.14991 (-0.16644) + | > log_mle: -0.29464 (-0.28629) + | > loss_dur: 0.14474 (0.11986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.14580 (12.35458) + | > current_lr: 0.00004 + | > step_time: 1.59240 (2.88867) + | > loader_time: 0.00180 (0.11402) + + + --> STEP: 91/234 -- GLOBAL_STEP: 41275 + | > loss: -0.14695 (-0.16602) + | > log_mle: -0.30943 (-0.28761) + | > loss_dur: 0.16248 (0.12158) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.98152 (12.56998) + | > current_lr: 0.00004 + | > step_time: 1.56760 (2.85270) + | > loader_time: 0.09420 (0.10891) + + + --> STEP: 96/234 -- GLOBAL_STEP: 41280 + | > loss: -0.16028 (-0.16711) + | > log_mle: -0.29574 (-0.29055) + | > loss_dur: 0.13547 (0.12344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08684 (13.08346) + | > current_lr: 0.00004 + | > step_time: 2.60690 (2.81259) + | > loader_time: 0.00250 (0.10431) + + + --> STEP: 101/234 -- GLOBAL_STEP: 41285 + | > loss: -0.17657 (-0.16738) + | > log_mle: -0.35251 (-0.29252) + | > loss_dur: 0.17595 (0.12514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.63577 (13.45024) + | > current_lr: 0.00004 + | > step_time: 1.52130 (2.78414) + | > loader_time: 0.00180 (0.10083) + + + --> STEP: 106/234 -- GLOBAL_STEP: 41290 + | > loss: -0.16015 (-0.16802) + | > log_mle: -0.35202 (-0.29503) + | > loss_dur: 0.19187 (0.12701) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.16283 (13.96607) + | > current_lr: 0.00004 + | > step_time: 1.86440 (2.78558) + | > loader_time: 0.00270 (0.09788) + + + --> STEP: 111/234 -- GLOBAL_STEP: 41295 + | > loss: -0.19579 (-0.16824) + | > log_mle: -0.39942 (-0.29746) + | > loss_dur: 0.20363 (0.12922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.54193 (14.65480) + | > current_lr: 0.00004 + | > step_time: 1.69850 (2.74280) + | > loader_time: 0.08340 (0.09669) + + + --> STEP: 116/234 -- GLOBAL_STEP: 41300 + | > loss: -0.15904 (-0.16850) + | > log_mle: -0.36262 (-0.29995) + | > loss_dur: 0.20357 (0.13144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.47925 (15.22300) + | > current_lr: 0.00004 + | > step_time: 3.29750 (2.70934) + | > loader_time: 0.00290 (0.09483) + + + --> STEP: 121/234 -- GLOBAL_STEP: 41305 + | > loss: -0.13272 (-0.16871) + | > log_mle: -0.27657 (-0.30160) + | > loss_dur: 0.14385 (0.13289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.50430 (15.57012) + | > current_lr: 0.00004 + | > step_time: 1.66030 (2.67990) + | > loader_time: 0.00270 (0.09278) + + + --> STEP: 126/234 -- GLOBAL_STEP: 41310 + | > loss: -0.22215 (-0.16938) + | > log_mle: -0.41261 (-0.30392) + | > loss_dur: 0.19046 (0.13454) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.53497 (16.14531) + | > current_lr: 0.00004 + | > step_time: 1.90290 (2.65943) + | > loader_time: 0.08280 (0.09188) + + + --> STEP: 131/234 -- GLOBAL_STEP: 41315 + | > loss: -0.25829 (-0.17096) + | > log_mle: -0.46012 (-0.30740) + | > loss_dur: 0.20182 (0.13644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.82030 (16.96130) + | > current_lr: 0.00004 + | > step_time: 4.08380 (2.65852) + | > loader_time: 0.00250 (0.08913) + + + --> STEP: 136/234 -- GLOBAL_STEP: 41320 + | > loss: -0.29409 (-0.17254) + | > log_mle: -0.50978 (-0.31091) + | > loss_dur: 0.21569 (0.13837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.00256 (17.59783) + | > current_lr: 0.00004 + | > step_time: 2.00960 (2.63003) + | > loader_time: 0.08510 (0.08721) + + + --> STEP: 141/234 -- GLOBAL_STEP: 41325 + | > loss: -0.21768 (-0.17354) + | > log_mle: -0.40711 (-0.31381) + | > loss_dur: 0.18943 (0.14027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.05647 (18.30216) + | > current_lr: 0.00004 + | > step_time: 1.69720 (2.64321) + | > loader_time: 0.00290 (0.08617) + + + --> STEP: 146/234 -- GLOBAL_STEP: 41330 + | > loss: -0.25436 (-0.17585) + | > log_mle: -0.45974 (-0.31834) + | > loss_dur: 0.20538 (0.14250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.96467 (19.30108) + | > current_lr: 0.00004 + | > step_time: 2.39800 (2.65769) + | > loader_time: 0.08980 (0.08448) + + + --> STEP: 151/234 -- GLOBAL_STEP: 41335 + | > loss: -0.24298 (-0.17797) + | > log_mle: -0.42437 (-0.32210) + | > loss_dur: 0.18139 (0.14413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.38414 (20.15659) + | > current_lr: 0.00004 + | > step_time: 1.08510 (2.72544) + | > loader_time: 0.00310 (0.08363) + + + --> STEP: 156/234 -- GLOBAL_STEP: 41340 + | > loss: -0.27306 (-0.18136) + | > log_mle: -0.46979 (-0.32744) + | > loss_dur: 0.19674 (0.14608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.65851 (21.53497) + | > current_lr: 0.00004 + | > step_time: 2.71190 (2.71652) + | > loader_time: 0.00230 (0.08191) + + + --> STEP: 161/234 -- GLOBAL_STEP: 41345 + | > loss: -0.28106 (-0.18390) + | > log_mle: -0.49248 (-0.33198) + | > loss_dur: 0.21142 (0.14808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.43217 (22.65664) + | > current_lr: 0.00004 + | > step_time: 5.59000 (2.75199) + | > loader_time: 0.00930 (0.08179) + + + --> STEP: 166/234 -- GLOBAL_STEP: 41350 + | > loss: -0.24978 (-0.18616) + | > log_mle: -0.43384 (-0.33587) + | > loss_dur: 0.18406 (0.14971) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.81636 (23.67947) + | > current_lr: 0.00004 + | > step_time: 2.40130 (2.78039) + | > loader_time: 0.00240 (0.08108) + + + --> STEP: 171/234 -- GLOBAL_STEP: 41355 + | > loss: -0.32692 (-0.18950) + | > log_mle: -0.53664 (-0.34125) + | > loss_dur: 0.20972 (0.15175) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.16756 (25.12685) + | > current_lr: 0.00004 + | > step_time: 1.70730 (2.75812) + | > loader_time: 0.08650 (0.07930) + + + --> STEP: 176/234 -- GLOBAL_STEP: 41360 + | > loss: -0.30582 (-0.19289) + | > log_mle: -0.52418 (-0.34661) + | > loss_dur: 0.21836 (0.15372) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.24871 (26.27299) + | > current_lr: 0.00004 + | > step_time: 3.60030 (2.75983) + | > loader_time: 0.00500 (0.07774) + + + --> STEP: 181/234 -- GLOBAL_STEP: 41365 + | > loss: -0.24606 (-0.19555) + | > log_mle: -0.45106 (-0.35127) + | > loss_dur: 0.20501 (0.15572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.88139 (27.81174) + | > current_lr: 0.00004 + | > step_time: 4.60950 (2.88247) + | > loader_time: 0.10130 (0.08063) + + + --> STEP: 186/234 -- GLOBAL_STEP: 41370 + | > loss: -0.23952 (-0.19807) + | > log_mle: -0.47507 (-0.35588) + | > loss_dur: 0.23555 (0.15781) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.39364 (29.16537) + | > current_lr: 0.00004 + | > step_time: 4.06790 (2.88777) + | > loader_time: 0.08470 (0.08026) + + + --> STEP: 191/234 -- GLOBAL_STEP: 41375 + | > loss: -0.29313 (-0.20077) + | > log_mle: -0.50375 (-0.36028) + | > loss_dur: 0.21062 (0.15951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.23567 (30.06863) + | > current_lr: 0.00004 + | > step_time: 7.09160 (2.91539) + | > loader_time: 0.10630 (0.07925) + + + --> STEP: 196/234 -- GLOBAL_STEP: 41380 + | > loss: -0.27376 (-0.20376) + | > log_mle: -0.49913 (-0.36489) + | > loss_dur: 0.22538 (0.16113) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.14363 (31.28781) + | > current_lr: 0.00004 + | > step_time: 4.39480 (2.97474) + | > loader_time: 0.29200 (0.08115) + + + --> STEP: 201/234 -- GLOBAL_STEP: 41385 + | > loss: -0.23109 (-0.20616) + | > log_mle: -0.46568 (-0.36903) + | > loss_dur: 0.23459 (0.16287) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.61039 (32.29842) + | > current_lr: 0.00004 + | > step_time: 5.28640 (3.02462) + | > loader_time: 0.31210 (0.08284) + + + --> STEP: 206/234 -- GLOBAL_STEP: 41390 + | > loss: -0.34913 (-0.20924) + | > log_mle: -0.57752 (-0.37381) + | > loss_dur: 0.22839 (0.16456) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.70081 (33.25183) + | > current_lr: 0.00004 + | > step_time: 6.20020 (3.04458) + | > loader_time: 0.20060 (0.08318) + + + --> STEP: 211/234 -- GLOBAL_STEP: 41395 + | > loss: -0.38567 (-0.21265) + | > log_mle: -0.64259 (-0.37907) + | > loss_dur: 0.25692 (0.16642) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.44310 (34.61000) + | > current_lr: 0.00004 + | > step_time: 2.70740 (3.09089) + | > loader_time: 0.08710 (0.08439) + + + --> STEP: 216/234 -- GLOBAL_STEP: 41400 + | > loss: -0.37640 (-0.21611) + | > log_mle: -0.63707 (-0.38421) + | > loss_dur: 0.26067 (0.16810) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.81480 (35.70275) + | > current_lr: 0.00004 + | > step_time: 3.90160 (3.14992) + | > loader_time: 0.09110 (0.08339) + + + --> STEP: 221/234 -- GLOBAL_STEP: 41405 + | > loss: -0.33271 (-0.21967) + | > log_mle: -0.55453 (-0.38935) + | > loss_dur: 0.22182 (0.16967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.08273 (37.18515) + | > current_lr: 0.00004 + | > step_time: 4.61120 (3.21310) + | > loader_time: 0.09310 (0.08418) + + + --> STEP: 226/234 -- GLOBAL_STEP: 41410 + | > loss: -0.39917 (-0.22359) + | > log_mle: -0.65780 (-0.39515) + | > loss_dur: 0.25863 (0.17156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.08611 (38.57845) + | > current_lr: 0.00004 + | > step_time: 0.23330 (3.17271) + | > loader_time: 0.00330 (0.08241) + + + --> STEP: 231/234 -- GLOBAL_STEP: 41415 + | > loss: -0.32595 (-0.22641) + | > log_mle: -0.71531 (-0.40093) + | > loss_dur: 0.38936 (0.17452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 121.82128 (40.39531) + | > current_lr: 0.00004 + | > step_time: 0.29070 (3.10977) + | > loader_time: 0.00420 (0.08072) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.30385 (-0.27304) + | > avg_loss: -0.24413 (-0.01531) + | > avg_log_mle: -0.46624 (-0.00948) + | > avg_loss_dur: 0.22212 (-0.00584) + + + > EPOCH: 177/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 21:56:28)  + + --> STEP: 2/234 -- GLOBAL_STEP: 41420 + | > loss: -0.20936 (-0.19606) + | > log_mle: -0.30323 (-0.29840) + | > loss_dur: 0.09387 (0.10235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.86206 (15.11721) + | > current_lr: 0.00004 + | > step_time: 0.90000 (1.30373) + | > loader_time: 0.00240 (0.04788) + + + --> STEP: 7/234 -- GLOBAL_STEP: 41425 + | > loss: -0.19366 (-0.17893) + | > log_mle: -0.30142 (-0.29389) + | > loss_dur: 0.10775 (0.11496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.91289 (15.86356) + | > current_lr: 0.00004 + | > step_time: 0.88630 (0.95338) + | > loader_time: 0.00210 (0.01489) + + + --> STEP: 12/234 -- GLOBAL_STEP: 41430 + | > loss: -0.17316 (-0.17929) + | > log_mle: -0.29657 (-0.29582) + | > loss_dur: 0.12341 (0.11653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.66152 (14.57453) + | > current_lr: 0.00004 + | > step_time: 1.68320 (1.27214) + | > loader_time: 0.00130 (1.63235) + + + --> STEP: 17/234 -- GLOBAL_STEP: 41435 + | > loss: -0.17558 (-0.18238) + | > log_mle: -0.27065 (-0.29449) + | > loss_dur: 0.09506 (0.11211) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.04226 (13.69864) + | > current_lr: 0.00004 + | > step_time: 0.90210 (1.42929) + | > loader_time: 0.00130 (2.17346) + + + --> STEP: 22/234 -- GLOBAL_STEP: 41440 + | > loss: -0.17557 (-0.18112) + | > log_mle: -0.28958 (-0.29198) + | > loss_dur: 0.11401 (0.11087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.73980 (13.32992) + | > current_lr: 0.00004 + | > step_time: 8.71430 (1.95007) + | > loader_time: 0.08170 (1.68758) + + + --> STEP: 27/234 -- GLOBAL_STEP: 41445 + | > loss: -0.17962 (-0.18196) + | > log_mle: -0.28978 (-0.29096) + | > loss_dur: 0.11016 (0.10900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.33011 (12.83185) + | > current_lr: 0.00004 + | > step_time: 1.29880 (2.14903) + | > loader_time: 0.00520 (1.38207) + + + --> STEP: 32/234 -- GLOBAL_STEP: 41450 + | > loss: -0.20503 (-0.18281) + | > log_mle: -0.29847 (-0.29060) + | > loss_dur: 0.09344 (0.10779) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.73648 (12.55992) + | > current_lr: 0.00004 + | > step_time: 5.58540 (2.42514) + | > loader_time: 0.01030 (1.16682) + + + --> STEP: 37/234 -- GLOBAL_STEP: 41455 + | > loss: -0.18473 (-0.18070) + | > log_mle: -0.27944 (-0.28934) + | > loss_dur: 0.09472 (0.10864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.56292 (12.56496) + | > current_lr: 0.00004 + | > step_time: 1.01080 (2.43751) + | > loader_time: 0.08240 (1.01421) + + + --> STEP: 42/234 -- GLOBAL_STEP: 41460 + | > loss: -0.15521 (-0.17860) + | > log_mle: -0.27001 (-0.28843) + | > loss_dur: 0.11480 (0.10983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.04092 (12.26291) + | > current_lr: 0.00004 + | > step_time: 1.19370 (2.38853) + | > loader_time: 0.00210 (0.89992) + + + --> STEP: 47/234 -- GLOBAL_STEP: 41465 + | > loss: -0.13944 (-0.17650) + | > log_mle: -0.28000 (-0.28836) + | > loss_dur: 0.14057 (0.11186) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.33871 (12.23656) + | > current_lr: 0.00004 + | > step_time: 1.19630 (2.27886) + | > loader_time: 0.00350 (0.80621) + + + --> STEP: 52/234 -- GLOBAL_STEP: 41470 + | > loss: -0.14706 (-0.17528) + | > log_mle: -0.27710 (-0.28747) + | > loss_dur: 0.13004 (0.11219) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.22271 (11.87630) + | > current_lr: 0.00004 + | > step_time: 1.09850 (2.22717) + | > loader_time: 0.00180 (0.73216) + + + --> STEP: 57/234 -- GLOBAL_STEP: 41475 + | > loss: -0.13839 (-0.17463) + | > log_mle: -0.26959 (-0.28738) + | > loss_dur: 0.13120 (0.11275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.09926 (11.70656) + | > current_lr: 0.00004 + | > step_time: 3.29880 (2.24578) + | > loader_time: 0.00330 (0.67290) + + + --> STEP: 62/234 -- GLOBAL_STEP: 41480 + | > loss: -0.12076 (-0.17361) + | > log_mle: -0.31637 (-0.28816) + | > loss_dur: 0.19560 (0.11455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.53413 (11.75986) + | > current_lr: 0.00004 + | > step_time: 1.72540 (2.23128) + | > loader_time: 0.00240 (0.62043) + + + --> STEP: 67/234 -- GLOBAL_STEP: 41485 + | > loss: -0.15836 (-0.17255) + | > log_mle: -0.29554 (-0.28764) + | > loss_dur: 0.13718 (0.11509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.08299 (11.61840) + | > current_lr: 0.00004 + | > step_time: 2.31470 (2.21311) + | > loader_time: 0.09340 (0.57850) + + + --> STEP: 72/234 -- GLOBAL_STEP: 41490 + | > loss: -0.15046 (-0.17022) + | > log_mle: -0.27570 (-0.28691) + | > loss_dur: 0.12525 (0.11669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.87331 (11.84156) + | > current_lr: 0.00004 + | > step_time: 2.47320 (2.21289) + | > loader_time: 0.00180 (0.53975) + + + --> STEP: 77/234 -- GLOBAL_STEP: 41495 + | > loss: -0.15871 (-0.16859) + | > log_mle: -0.28497 (-0.28667) + | > loss_dur: 0.12626 (0.11808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.69907 (12.19039) + | > current_lr: 0.00004 + | > step_time: 3.08790 (2.21439) + | > loader_time: 0.00270 (0.50487) + + + --> STEP: 82/234 -- GLOBAL_STEP: 41500 + | > loss: -0.14392 (-0.16742) + | > log_mle: -0.27734 (-0.28630) + | > loss_dur: 0.13342 (0.11888) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.39193 (12.31962) + | > current_lr: 0.00004 + | > step_time: 1.90920 (2.18157) + | > loader_time: 0.08580 (0.47750) + + + --> STEP: 87/234 -- GLOBAL_STEP: 41505 + | > loss: -0.13758 (-0.16611) + | > log_mle: -0.28384 (-0.28637) + | > loss_dur: 0.14627 (0.12026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.39579 (12.43422) + | > current_lr: 0.00004 + | > step_time: 1.61170 (2.17913) + | > loader_time: 0.00300 (0.45028) + + + --> STEP: 92/234 -- GLOBAL_STEP: 41510 + | > loss: -0.18894 (-0.16629) + | > log_mle: -0.33252 (-0.28814) + | > loss_dur: 0.14358 (0.12185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.65141 (12.73390) + | > current_lr: 0.00004 + | > step_time: 1.50670 (2.14984) + | > loader_time: 0.09610 (0.42698) + + + --> STEP: 97/234 -- GLOBAL_STEP: 41515 + | > loss: -0.16763 (-0.16713) + | > log_mle: -0.32192 (-0.29085) + | > loss_dur: 0.15429 (0.12373) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.01155 (13.27426) + | > current_lr: 0.00004 + | > step_time: 1.60600 (2.13479) + | > loader_time: 0.08420 (0.40690) + + + --> STEP: 102/234 -- GLOBAL_STEP: 41520 + | > loss: -0.13844 (-0.16693) + | > log_mle: -0.30317 (-0.29259) + | > loss_dur: 0.16474 (0.12566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.43106 (13.54492) + | > current_lr: 0.00004 + | > step_time: 2.31270 (2.14619) + | > loader_time: 0.00210 (0.38713) + + + --> STEP: 107/234 -- GLOBAL_STEP: 41525 + | > loss: -0.18396 (-0.16789) + | > log_mle: -0.34628 (-0.29541) + | > loss_dur: 0.16231 (0.12753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.29379 (14.03394) + | > current_lr: 0.00004 + | > step_time: 1.99410 (2.13958) + | > loader_time: 0.00180 (0.36914) + + + --> STEP: 112/234 -- GLOBAL_STEP: 41530 + | > loss: -0.17425 (-0.16834) + | > log_mle: -0.35895 (-0.29803) + | > loss_dur: 0.18470 (0.12969) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.06924 (14.57367) + | > current_lr: 0.00004 + | > step_time: 1.45790 (2.15320) + | > loader_time: 0.00260 (0.35281) + + + --> STEP: 117/234 -- GLOBAL_STEP: 41535 + | > loss: -0.18415 (-0.16883) + | > log_mle: -0.35461 (-0.30050) + | > loss_dur: 0.17046 (0.13167) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.65442 (15.09290) + | > current_lr: 0.00004 + | > step_time: 1.59560 (2.18772) + | > loader_time: 0.01090 (0.34542) + + + --> STEP: 122/234 -- GLOBAL_STEP: 41540 + | > loss: -0.16645 (-0.16880) + | > log_mle: -0.32961 (-0.30195) + | > loss_dur: 0.16316 (0.13316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.15488 (15.50679) + | > current_lr: 0.00004 + | > step_time: 1.05670 (2.20361) + | > loader_time: 0.00250 (0.33140) + + + --> STEP: 127/234 -- GLOBAL_STEP: 41545 + | > loss: -0.21304 (-0.16989) + | > log_mle: -0.39072 (-0.30475) + | > loss_dur: 0.17768 (0.13485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.58646 (16.31021) + | > current_lr: 0.00004 + | > step_time: 1.26940 (2.17814) + | > loader_time: 0.00200 (0.31846) + + + --> STEP: 132/234 -- GLOBAL_STEP: 41550 + | > loss: -0.21582 (-0.17154) + | > log_mle: -0.37340 (-0.30815) + | > loss_dur: 0.15758 (0.13661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.49917 (16.96615) + | > current_lr: 0.00004 + | > step_time: 3.00850 (2.17747) + | > loader_time: 0.00350 (0.30713) + + + --> STEP: 137/234 -- GLOBAL_STEP: 41555 + | > loss: -0.16971 (-0.17295) + | > log_mle: -0.37729 (-0.31167) + | > loss_dur: 0.20758 (0.13871) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.15022 (18.02539) + | > current_lr: 0.00004 + | > step_time: 1.67160 (2.20222) + | > loader_time: 0.00220 (0.29612) + + + --> STEP: 142/234 -- GLOBAL_STEP: 41560 + | > loss: -0.20059 (-0.17393) + | > log_mle: -0.39402 (-0.31444) + | > loss_dur: 0.19343 (0.14051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.04298 (18.81670) + | > current_lr: 0.00004 + | > step_time: 2.79650 (2.19658) + | > loader_time: 0.00510 (0.28695) + + + --> STEP: 147/234 -- GLOBAL_STEP: 41565 + | > loss: -0.20333 (-0.17620) + | > log_mle: -0.39523 (-0.31898) + | > loss_dur: 0.19190 (0.14279) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.74820 (19.82416) + | > current_lr: 0.00004 + | > step_time: 3.23880 (2.23491) + | > loader_time: 0.09830 (0.27937) + + + --> STEP: 152/234 -- GLOBAL_STEP: 41570 + | > loss: -0.25464 (-0.17872) + | > log_mle: -0.47923 (-0.32330) + | > loss_dur: 0.22459 (0.14458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.09881 (20.76629) + | > current_lr: 0.00004 + | > step_time: 3.89590 (2.24469) + | > loader_time: 0.00490 (0.27088) + + + --> STEP: 157/234 -- GLOBAL_STEP: 41575 + | > loss: -0.21900 (-0.18182) + | > log_mle: -0.43160 (-0.32826) + | > loss_dur: 0.21260 (0.14645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.27305 (21.85183) + | > current_lr: 0.00004 + | > step_time: 4.00020 (2.27381) + | > loader_time: 0.00410 (0.26421) + + + --> STEP: 162/234 -- GLOBAL_STEP: 41580 + | > loss: -0.26976 (-0.18451) + | > log_mle: -0.46242 (-0.33289) + | > loss_dur: 0.19266 (0.14839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.71006 (23.15242) + | > current_lr: 0.00004 + | > step_time: 2.69750 (2.28749) + | > loader_time: 0.00290 (0.25671) + + + --> STEP: 167/234 -- GLOBAL_STEP: 41585 + | > loss: -0.34328 (-0.18705) + | > log_mle: -0.54680 (-0.33707) + | > loss_dur: 0.20353 (0.15002) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.71526 (24.22331) + | > current_lr: 0.00004 + | > step_time: 2.40840 (2.28355) + | > loader_time: 0.08680 (0.25021) + + + --> STEP: 172/234 -- GLOBAL_STEP: 41590 + | > loss: -0.30316 (-0.19013) + | > log_mle: -0.53294 (-0.34221) + | > loss_dur: 0.22978 (0.15208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.69411 (25.75185) + | > current_lr: 0.00004 + | > step_time: 2.31220 (2.28188) + | > loader_time: 0.09300 (0.24455) + + + --> STEP: 177/234 -- GLOBAL_STEP: 41595 + | > loss: -0.27790 (-0.19303) + | > log_mle: -0.49494 (-0.34708) + | > loss_dur: 0.21704 (0.15405) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.98221 (26.89654) + | > current_lr: 0.00004 + | > step_time: 6.61560 (2.29606) + | > loader_time: 0.10150 (0.23888) + + + --> STEP: 182/234 -- GLOBAL_STEP: 41600 + | > loss: -0.28339 (-0.19560) + | > log_mle: -0.53842 (-0.35191) + | > loss_dur: 0.25503 (0.15631) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.03196 (28.16687) + | > current_lr: 0.00004 + | > step_time: 2.59390 (2.31149) + | > loader_time: 0.10060 (0.23347) + + + --> STEP: 187/234 -- GLOBAL_STEP: 41605 + | > loss: -0.30791 (-0.19845) + | > log_mle: -0.53805 (-0.35670) + | > loss_dur: 0.23014 (0.15825) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.50120 (29.45770) + | > current_lr: 0.00004 + | > step_time: 4.63270 (2.33710) + | > loader_time: 0.00370 (0.22835) + + + --> STEP: 192/234 -- GLOBAL_STEP: 41610 + | > loss: -0.35951 (-0.20165) + | > log_mle: -0.57158 (-0.36144) + | > loss_dur: 0.21207 (0.15980) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.93359 (30.52336) + | > current_lr: 0.00004 + | > step_time: 4.77900 (2.38546) + | > loader_time: 0.01260 (0.22459) + + + --> STEP: 197/234 -- GLOBAL_STEP: 41615 + | > loss: -0.33363 (-0.20454) + | > log_mle: -0.53418 (-0.36596) + | > loss_dur: 0.20056 (0.16142) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.98145 (31.66698) + | > current_lr: 0.00004 + | > step_time: 2.09110 (2.41670) + | > loader_time: 0.00370 (0.22000) + + + --> STEP: 202/234 -- GLOBAL_STEP: 41620 + | > loss: -0.39819 (-0.20733) + | > log_mle: -0.63096 (-0.37062) + | > loss_dur: 0.23276 (0.16329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.96280 (32.78432) + | > current_lr: 0.00004 + | > step_time: 5.99290 (2.53695) + | > loader_time: 0.00560 (0.21960) + + + --> STEP: 207/234 -- GLOBAL_STEP: 41625 + | > loss: -0.38513 (-0.21035) + | > log_mle: -0.62280 (-0.37535) + | > loss_dur: 0.23767 (0.16500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.16542 (33.92160) + | > current_lr: 0.00004 + | > step_time: 3.89740 (2.56121) + | > loader_time: 0.10220 (0.21664) + + + --> STEP: 212/234 -- GLOBAL_STEP: 41630 + | > loss: -0.36394 (-0.21386) + | > log_mle: -0.60084 (-0.38068) + | > loss_dur: 0.23690 (0.16682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.20344 (35.29783) + | > current_lr: 0.00004 + | > step_time: 4.98770 (2.64266) + | > loader_time: 0.10210 (0.21351) + + + --> STEP: 217/234 -- GLOBAL_STEP: 41635 + | > loss: -0.38148 (-0.21745) + | > log_mle: -0.62259 (-0.38598) + | > loss_dur: 0.24110 (0.16853) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.11057 (36.63482) + | > current_lr: 0.00004 + | > step_time: 8.89000 (2.72355) + | > loader_time: 0.11420 (0.21058) + + + --> STEP: 222/234 -- GLOBAL_STEP: 41640 + | > loss: -0.36943 (-0.22101) + | > log_mle: -0.64076 (-0.39124) + | > loss_dur: 0.27133 (0.17024) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.97398 (37.89027) + | > current_lr: 0.00004 + | > step_time: 0.23230 (2.74134) + | > loader_time: 0.00390 (0.20633) + + + --> STEP: 227/234 -- GLOBAL_STEP: 41645 + | > loss: -0.34212 (-0.22486) + | > log_mle: -0.60317 (-0.39687) + | > loss_dur: 0.26105 (0.17201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.89854 (39.37100) + | > current_lr: 0.00004 + | > step_time: 0.24120 (2.68618) + | > loader_time: 0.00380 (0.20185) + + + --> STEP: 232/234 -- GLOBAL_STEP: 41650 + | > loss: -0.32121 (-0.22764) + | > log_mle: -0.81149 (-0.40368) + | > loss_dur: 0.49028 (0.17604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 183.24104 (41.35168) + | > current_lr: 0.00004 + | > step_time: 0.33240 (2.63418) + | > loader_time: 0.00540 (0.19762) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03878 (-0.26507) + | > avg_loss: -0.25360 (-0.00948) + | > avg_log_mle: -0.48016 (-0.01392) + | > avg_loss_dur: 0.22656 (+0.00444) + + + > EPOCH: 178/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 22:08:26)  + + --> STEP: 3/234 -- GLOBAL_STEP: 41655 + | > loss: -0.13593 (-0.17920) + | > log_mle: -0.29355 (-0.29939) + | > loss_dur: 0.15762 (0.12019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.28115 (15.58838) + | > current_lr: 0.00004 + | > step_time: 9.90490 (4.77101) + | > loader_time: 4.29380 (1.45992) + + + --> STEP: 8/234 -- GLOBAL_STEP: 41660 + | > loss: -0.18890 (-0.18528) + | > log_mle: -0.31183 (-0.29929) + | > loss_dur: 0.12293 (0.11401) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.18788 (15.02238) + | > current_lr: 0.00004 + | > step_time: 2.69220 (4.79984) + | > loader_time: 0.00130 (0.64668) + + + --> STEP: 13/234 -- GLOBAL_STEP: 41665 + | > loss: -0.19754 (-0.18481) + | > log_mle: -0.30361 (-0.29951) + | > loss_dur: 0.10606 (0.11469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.69984 (14.56567) + | > current_lr: 0.00004 + | > step_time: 4.79720 (4.46242) + | > loader_time: 0.10050 (0.43488) + + + --> STEP: 18/234 -- GLOBAL_STEP: 41670 + | > loss: -0.17496 (-0.18528) + | > log_mle: -0.28804 (-0.29701) + | > loss_dur: 0.11308 (0.11173) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.50900 (13.88923) + | > current_lr: 0.00004 + | > step_time: 2.36690 (4.64927) + | > loader_time: 0.07970 (0.33520) + + + --> STEP: 23/234 -- GLOBAL_STEP: 41675 + | > loss: -0.20752 (-0.18723) + | > log_mle: -0.30057 (-0.29518) + | > loss_dur: 0.09305 (0.10795) + | > amp_scaler: 4096.00000 (2404.17391) + | > grad_norm: 7.15597 (13.05466) + | > current_lr: 0.00004 + | > step_time: 1.10350 (3.84718) + | > loader_time: 0.00170 (0.26280) + + + --> STEP: 28/234 -- GLOBAL_STEP: 41680 + | > loss: -0.22476 (-0.18835) + | > log_mle: -0.29969 (-0.29422) + | > loss_dur: 0.07493 (0.10586) + | > amp_scaler: 4096.00000 (2706.28571) + | > grad_norm: 6.07460 (12.31795) + | > current_lr: 0.00004 + | > step_time: 9.99260 (3.94311) + | > loader_time: 0.20650 (0.23633) + + + --> STEP: 33/234 -- GLOBAL_STEP: 41685 + | > loss: -0.18523 (-0.18640) + | > log_mle: -0.28409 (-0.29304) + | > loss_dur: 0.09887 (0.10664) + | > amp_scaler: 4096.00000 (2916.84848) + | > grad_norm: 7.20296 (11.94881) + | > current_lr: 0.00004 + | > step_time: 3.88300 (3.81763) + | > loader_time: 0.00140 (0.20649) + + + --> STEP: 38/234 -- GLOBAL_STEP: 41690 + | > loss: -0.17571 (-0.18421) + | > log_mle: -0.29814 (-0.29215) + | > loss_dur: 0.12243 (0.10795) + | > amp_scaler: 4096.00000 (3072.00000) + | > grad_norm: 8.38435 (12.02711) + | > current_lr: 0.00004 + | > step_time: 2.59910 (3.59274) + | > loader_time: 0.00260 (0.18182) + + + --> STEP: 43/234 -- GLOBAL_STEP: 41695 + | > loss: -0.16736 (-0.18226) + | > log_mle: -0.29721 (-0.29121) + | > loss_dur: 0.12984 (0.10895) + | > amp_scaler: 4096.00000 (3191.06977) + | > grad_norm: 9.90198 (11.92625) + | > current_lr: 0.00004 + | > step_time: 1.28000 (3.81648) + | > loader_time: 0.00120 (0.16975) + + + --> STEP: 48/234 -- GLOBAL_STEP: 41700 + | > loss: -0.18494 (-0.18043) + | > log_mle: -0.27923 (-0.29057) + | > loss_dur: 0.09429 (0.11015) + | > amp_scaler: 4096.00000 (3285.33333) + | > grad_norm: 8.12728 (12.10997) + | > current_lr: 0.00004 + | > step_time: 0.68800 (3.61684) + | > loader_time: 0.00210 (0.15425) + + + --> STEP: 53/234 -- GLOBAL_STEP: 41705 + | > loss: -0.16816 (-0.17858) + | > log_mle: -0.28890 (-0.28973) + | > loss_dur: 0.12074 (0.11115) + | > amp_scaler: 4096.00000 (3361.81132) + | > grad_norm: 14.40392 (12.08148) + | > current_lr: 0.00004 + | > step_time: 1.26090 (3.41316) + | > loader_time: 0.00170 (0.14132) + + + --> STEP: 58/234 -- GLOBAL_STEP: 41710 + | > loss: -0.16522 (-0.17760) + | > log_mle: -0.27943 (-0.28930) + | > loss_dur: 0.11421 (0.11170) + | > amp_scaler: 4096.00000 (3425.10345) + | > grad_norm: 11.32403 (11.95931) + | > current_lr: 0.00004 + | > step_time: 1.59000 (3.26540) + | > loader_time: 0.08050 (0.13071) + + + --> STEP: 63/234 -- GLOBAL_STEP: 41715 + | > loss: -0.14303 (-0.17579) + | > log_mle: -0.28278 (-0.28998) + | > loss_dur: 0.13975 (0.11419) + | > amp_scaler: 4096.00000 (3478.34921) + | > grad_norm: 8.75564 (12.08835) + | > current_lr: 0.00004 + | > step_time: 1.97980 (3.15470) + | > loader_time: 0.00150 (0.12168) + + + --> STEP: 68/234 -- GLOBAL_STEP: 41720 + | > loss: -0.12207 (-0.17454) + | > log_mle: -0.27758 (-0.28955) + | > loss_dur: 0.15551 (0.11501) + | > amp_scaler: 4096.00000 (3523.76471) + | > grad_norm: 12.18374 (11.88852) + | > current_lr: 0.00004 + | > step_time: 1.34820 (3.08100) + | > loader_time: 0.00180 (0.11421) + + + --> STEP: 73/234 -- GLOBAL_STEP: 41725 + | > loss: -0.13797 (-0.17242) + | > log_mle: -0.29817 (-0.28939) + | > loss_dur: 0.16020 (0.11696) + | > amp_scaler: 4096.00000 (3562.95890) + | > grad_norm: 14.19137 (12.00465) + | > current_lr: 0.00004 + | > step_time: 5.32080 (3.04864) + | > loader_time: 0.28960 (0.11162) + + + --> STEP: 78/234 -- GLOBAL_STEP: 41730 + | > loss: -0.13546 (-0.17114) + | > log_mle: -0.27701 (-0.28929) + | > loss_dur: 0.14155 (0.11814) + | > amp_scaler: 4096.00000 (3597.12821) + | > grad_norm: 11.19732 (11.93517) + | > current_lr: 0.00004 + | > step_time: 2.21480 (2.97786) + | > loader_time: 0.09310 (0.10684) + + + --> STEP: 83/234 -- GLOBAL_STEP: 41735 + | > loss: -0.13806 (-0.17031) + | > log_mle: -0.29899 (-0.28942) + | > loss_dur: 0.16092 (0.11911) + | > amp_scaler: 4096.00000 (3627.18072) + | > grad_norm: 18.59740 (11.95865) + | > current_lr: 0.00004 + | > step_time: 2.08950 (2.95109) + | > loader_time: 0.00250 (0.10167) + + + --> STEP: 88/234 -- GLOBAL_STEP: 41740 + | > loss: -0.18120 (-0.16988) + | > log_mle: -0.33732 (-0.29002) + | > loss_dur: 0.15612 (0.12014) + | > amp_scaler: 4096.00000 (3653.81818) + | > grad_norm: 19.82466 (12.08653) + | > current_lr: 0.00004 + | > step_time: 3.33610 (2.89662) + | > loader_time: 0.00820 (0.09971) + + + --> STEP: 93/234 -- GLOBAL_STEP: 41745 + | > loss: -0.17315 (-0.17010) + | > log_mle: -0.34887 (-0.29195) + | > loss_dur: 0.17572 (0.12185) + | > amp_scaler: 4096.00000 (3677.59140) + | > grad_norm: 29.28567 (12.47950) + | > current_lr: 0.00004 + | > step_time: 3.21120 (2.85772) + | > loader_time: 0.09340 (0.09760) + + + --> STEP: 98/234 -- GLOBAL_STEP: 41750 + | > loss: -0.13950 (-0.17042) + | > log_mle: -0.27905 (-0.29387) + | > loss_dur: 0.13955 (0.12345) + | > amp_scaler: 4096.00000 (3698.93878) + | > grad_norm: 9.13149 (13.03187) + | > current_lr: 0.00004 + | > step_time: 1.80910 (2.81268) + | > loader_time: 0.00280 (0.09276) + + + --> STEP: 103/234 -- GLOBAL_STEP: 41755 + | > loss: -0.20301 (-0.17100) + | > log_mle: -0.37759 (-0.29655) + | > loss_dur: 0.17458 (0.12555) + | > amp_scaler: 4096.00000 (3718.21359) + | > grad_norm: 37.93592 (13.77234) + | > current_lr: 0.00004 + | > step_time: 2.93860 (2.78947) + | > loader_time: 0.00200 (0.09002) + + + --> STEP: 108/234 -- GLOBAL_STEP: 41760 + | > loss: -0.17711 (-0.17172) + | > log_mle: -0.32244 (-0.29882) + | > loss_dur: 0.14533 (0.12709) + | > amp_scaler: 4096.00000 (3735.70370) + | > grad_norm: 16.70218 (14.18405) + | > current_lr: 0.00004 + | > step_time: 1.27280 (2.75649) + | > loader_time: 0.00210 (0.08776) + + + --> STEP: 113/234 -- GLOBAL_STEP: 41765 + | > loss: -0.18208 (-0.17164) + | > log_mle: -0.36135 (-0.30137) + | > loss_dur: 0.17927 (0.12973) + | > amp_scaler: 4096.00000 (3751.64602) + | > grad_norm: 29.56508 (15.18367) + | > current_lr: 0.00004 + | > step_time: 3.01690 (2.73645) + | > loader_time: 0.08810 (0.08622) + + + --> STEP: 118/234 -- GLOBAL_STEP: 41770 + | > loss: -0.16018 (-0.17166) + | > log_mle: -0.33611 (-0.30331) + | > loss_dur: 0.17593 (0.13166) + | > amp_scaler: 4096.00000 (3766.23729) + | > grad_norm: 22.22215 (15.58909) + | > current_lr: 0.00004 + | > step_time: 1.79240 (2.70252) + | > loader_time: 0.00330 (0.08342) + + + --> STEP: 123/234 -- GLOBAL_STEP: 41775 + | > loss: -0.14588 (-0.17152) + | > log_mle: -0.30626 (-0.30444) + | > loss_dur: 0.16038 (0.13292) + | > amp_scaler: 4096.00000 (3779.64228) + | > grad_norm: 19.58559 (15.80553) + | > current_lr: 0.00004 + | > step_time: 5.39690 (2.70569) + | > loader_time: 0.10730 (0.08164) + + + --> STEP: 128/234 -- GLOBAL_STEP: 41780 + | > loss: -0.20620 (-0.17283) + | > log_mle: -0.36835 (-0.30763) + | > loss_dur: 0.16216 (0.13480) + | > amp_scaler: 4096.00000 (3792.00000) + | > grad_norm: 31.47627 (16.59361) + | > current_lr: 0.00004 + | > step_time: 1.77600 (2.66077) + | > loader_time: 0.00360 (0.07857) + + + --> STEP: 133/234 -- GLOBAL_STEP: 41785 + | > loss: -0.21794 (-0.17462) + | > log_mle: -0.39687 (-0.31109) + | > loss_dur: 0.17893 (0.13647) + | > amp_scaler: 4096.00000 (3803.42857) + | > grad_norm: 37.77157 (17.31082) + | > current_lr: 0.00004 + | > step_time: 4.69980 (2.66447) + | > loader_time: 0.00350 (0.07636) + + + --> STEP: 138/234 -- GLOBAL_STEP: 41790 + | > loss: -0.17240 (-0.17573) + | > log_mle: -0.34721 (-0.31425) + | > loss_dur: 0.17481 (0.13851) + | > amp_scaler: 4096.00000 (3814.02899) + | > grad_norm: 27.94802 (18.04099) + | > current_lr: 0.00004 + | > step_time: 0.99430 (2.62232) + | > loader_time: 0.00220 (0.07436) + + + --> STEP: 143/234 -- GLOBAL_STEP: 41795 + | > loss: -0.26613 (-0.17754) + | > log_mle: -0.49759 (-0.31827) + | > loss_dur: 0.23145 (0.14074) + | > amp_scaler: 4096.00000 (3823.88811) + | > grad_norm: 60.37175 (19.04638) + | > current_lr: 0.00004 + | > step_time: 3.60220 (2.65245) + | > loader_time: 0.00700 (0.07423) + + + --> STEP: 148/234 -- GLOBAL_STEP: 41800 + | > loss: -0.24075 (-0.17970) + | > log_mle: -0.40362 (-0.32218) + | > loss_dur: 0.16287 (0.14248) + | > amp_scaler: 4096.00000 (3833.08108) + | > grad_norm: 39.17380 (20.20857) + | > current_lr: 0.00004 + | > step_time: 1.71130 (2.63593) + | > loader_time: 0.00230 (0.07247) + + + --> STEP: 153/234 -- GLOBAL_STEP: 41805 + | > loss: -0.33068 (-0.18283) + | > log_mle: -0.53247 (-0.32730) + | > loss_dur: 0.20180 (0.14447) + | > amp_scaler: 4096.00000 (3841.67320) + | > grad_norm: 80.52332 (21.56728) + | > current_lr: 0.00004 + | > step_time: 5.39560 (2.65184) + | > loader_time: 0.00350 (0.07134) + + + --> STEP: 158/234 -- GLOBAL_STEP: 41810 + | > loss: -0.24546 (-0.18538) + | > log_mle: -0.46531 (-0.33175) + | > loss_dur: 0.21984 (0.14637) + | > amp_scaler: 4096.00000 (3849.72152) + | > grad_norm: 63.09336 (22.98750) + | > current_lr: 0.00004 + | > step_time: 5.79980 (2.73320) + | > loader_time: 0.40090 (0.07531) + + + --> STEP: 163/234 -- GLOBAL_STEP: 41815 + | > loss: -0.24750 (-0.18810) + | > log_mle: -0.43811 (-0.33614) + | > loss_dur: 0.19061 (0.14805) + | > amp_scaler: 4096.00000 (3857.27607) + | > grad_norm: 41.49736 (24.19032) + | > current_lr: 0.00004 + | > step_time: 2.20180 (2.78136) + | > loader_time: 0.10170 (0.07601) + + + --> STEP: 168/234 -- GLOBAL_STEP: 41820 + | > loss: -0.26470 (-0.19092) + | > log_mle: -0.49485 (-0.34078) + | > loss_dur: 0.23015 (0.14986) + | > amp_scaler: 4096.00000 (3864.38095) + | > grad_norm: 68.06810 (25.48382) + | > current_lr: 0.00004 + | > step_time: 3.29670 (2.77641) + | > loader_time: 0.18540 (0.07548) + + + --> STEP: 173/234 -- GLOBAL_STEP: 41825 + | > loss: -0.28407 (-0.19416) + | > log_mle: -0.49910 (-0.34601) + | > loss_dur: 0.21503 (0.15186) + | > amp_scaler: 4096.00000 (3871.07514) + | > grad_norm: 68.05621 (27.09001) + | > current_lr: 0.00004 + | > step_time: 4.50070 (2.77645) + | > loader_time: 0.10100 (0.07446) + + + --> STEP: 178/234 -- GLOBAL_STEP: 41830 + | > loss: -0.31043 (-0.19709) + | > log_mle: -0.55811 (-0.35112) + | > loss_dur: 0.24767 (0.15403) + | > amp_scaler: 2048.00000 (3819.86517) + | > grad_norm: 70.68824 (27.84213) + | > current_lr: 0.00004 + | > step_time: 1.26550 (2.82081) + | > loader_time: 0.00270 (0.07361) + + + --> STEP: 183/234 -- GLOBAL_STEP: 41835 + | > loss: -0.33450 (-0.19988) + | > log_mle: -0.55539 (-0.35586) + | > loss_dur: 0.22089 (0.15599) + | > amp_scaler: 2048.00000 (3771.45355) + | > grad_norm: 69.14219 (29.10749) + | > current_lr: 0.00004 + | > step_time: 1.80150 (2.79340) + | > loader_time: 0.00330 (0.07257) + + + --> STEP: 188/234 -- GLOBAL_STEP: 41840 + | > loss: -0.34071 (-0.20242) + | > log_mle: -0.56961 (-0.36047) + | > loss_dur: 0.22890 (0.15805) + | > amp_scaler: 2048.00000 (3725.61702) + | > grad_norm: 66.26875 (30.25436) + | > current_lr: 0.00004 + | > step_time: 8.00060 (2.87393) + | > loader_time: 0.10390 (0.07702) + + + --> STEP: 193/234 -- GLOBAL_STEP: 41845 + | > loss: -0.34461 (-0.20549) + | > log_mle: -0.56543 (-0.36505) + | > loss_dur: 0.22083 (0.15957) + | > amp_scaler: 2048.00000 (3682.15544) + | > grad_norm: 90.57199 (31.30525) + | > current_lr: 0.00004 + | > step_time: 3.68900 (2.94444) + | > loader_time: 0.10070 (0.07757) + + + --> STEP: 198/234 -- GLOBAL_STEP: 41850 + | > loss: -0.34209 (-0.20837) + | > log_mle: -0.56718 (-0.36953) + | > loss_dur: 0.22509 (0.16116) + | > amp_scaler: 2048.00000 (3640.88889) + | > grad_norm: 64.55137 (32.22912) + | > current_lr: 0.00004 + | > step_time: 9.40530 (3.01547) + | > loader_time: 0.00670 (0.07578) + + + --> STEP: 203/234 -- GLOBAL_STEP: 41855 + | > loss: -0.27245 (-0.21070) + | > log_mle: -0.49024 (-0.37364) + | > loss_dur: 0.21779 (0.16294) + | > amp_scaler: 2048.00000 (3601.65517) + | > grad_norm: 58.07531 (33.31372) + | > current_lr: 0.00004 + | > step_time: 5.49450 (3.04653) + | > loader_time: 0.00310 (0.07400) + + + --> STEP: 208/234 -- GLOBAL_STEP: 41860 + | > loss: -0.33414 (-0.21386) + | > log_mle: -0.57423 (-0.37856) + | > loss_dur: 0.24009 (0.16470) + | > amp_scaler: 2048.00000 (3564.30769) + | > grad_norm: 81.93747 (34.43516) + | > current_lr: 0.00004 + | > step_time: 9.30460 (3.08774) + | > loader_time: 0.09860 (0.07452) + + + --> STEP: 213/234 -- GLOBAL_STEP: 41865 + | > loss: -0.37365 (-0.21741) + | > log_mle: -0.62219 (-0.38394) + | > loss_dur: 0.24853 (0.16653) + | > amp_scaler: 2048.00000 (3528.71362) + | > grad_norm: 102.56973 (35.93003) + | > current_lr: 0.00004 + | > step_time: 5.30010 (3.20540) + | > loader_time: 0.09780 (0.07555) + + + --> STEP: 218/234 -- GLOBAL_STEP: 41870 + | > loss: -0.35289 (-0.22082) + | > log_mle: -0.58805 (-0.38896) + | > loss_dur: 0.23516 (0.16814) + | > amp_scaler: 2048.00000 (3494.75229) + | > grad_norm: 75.72089 (37.01055) + | > current_lr: 0.00004 + | > step_time: 4.39380 (3.29148) + | > loader_time: 0.20290 (0.07528) + + + --> STEP: 223/234 -- GLOBAL_STEP: 41875 + | > loss: -0.39170 (-0.22445) + | > log_mle: -0.63340 (-0.39433) + | > loss_dur: 0.24170 (0.16988) + | > amp_scaler: 2048.00000 (3462.31390) + | > grad_norm: 92.52082 (38.27605) + | > current_lr: 0.00004 + | > step_time: 0.22760 (3.23856) + | > loader_time: 0.00340 (0.07404) + + + --> STEP: 228/234 -- GLOBAL_STEP: 41880 + | > loss: -0.33268 (-0.22779) + | > log_mle: -0.62470 (-0.39977) + | > loss_dur: 0.29202 (0.17197) + | > amp_scaler: 2048.00000 (3431.29825) + | > grad_norm: 113.33042 (39.95469) + | > current_lr: 0.00004 + | > step_time: 0.24830 (3.17274) + | > loader_time: 0.00300 (0.07249) + + + --> STEP: 233/234 -- GLOBAL_STEP: 41885 + | > loss: 0.06735 (-0.22874) + | > log_mle: -0.59337 (-0.40636) + | > loss_dur: 0.66073 (0.17761) + | > amp_scaler: 2048.00000 (3401.61373) + | > grad_norm: 101.66300 (41.69202) + | > current_lr: 0.00004 + | > step_time: 0.18920 (3.11032) + | > loader_time: 0.00290 (0.07103) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.54896 (+0.51019) + | > avg_loss: -0.23914 (+0.01446) + | > avg_log_mle: -0.46405 (+0.01611) + | > avg_loss_dur: 0.22491 (-0.00165) + + + > EPOCH: 179/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 22:21:55)  + + --> STEP: 4/234 -- GLOBAL_STEP: 41890 + | > loss: -0.16834 (-0.16621) + | > log_mle: -0.29286 (-0.29666) + | > loss_dur: 0.12452 (0.13045) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.19537 (20.65009) + | > current_lr: 0.00004 + | > step_time: 1.31690 (3.58013) + | > loader_time: 0.00180 (0.07073) + + + --> STEP: 9/234 -- GLOBAL_STEP: 41895 + | > loss: -0.16787 (-0.18312) + | > log_mle: -0.30386 (-0.29945) + | > loss_dur: 0.13599 (0.11633) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.17844 (17.73277) + | > current_lr: 0.00004 + | > step_time: 2.50390 (4.82592) + | > loader_time: 0.00350 (0.07591) + + + --> STEP: 14/234 -- GLOBAL_STEP: 41900 + | > loss: -0.17271 (-0.18620) + | > log_mle: -0.29922 (-0.29961) + | > loss_dur: 0.12651 (0.11341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.07486 (16.00491) + | > current_lr: 0.00004 + | > step_time: 1.35020 (4.04330) + | > loader_time: 0.00120 (0.07101) + + + --> STEP: 19/234 -- GLOBAL_STEP: 41905 + | > loss: -0.19697 (-0.18745) + | > log_mle: -0.28857 (-0.29701) + | > loss_dur: 0.09160 (0.10956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.64345 (14.29118) + | > current_lr: 0.00004 + | > step_time: 1.26380 (3.32627) + | > loader_time: 0.00120 (0.06708) + + + --> STEP: 24/234 -- GLOBAL_STEP: 41910 + | > loss: -0.22135 (-0.18843) + | > log_mle: -0.29227 (-0.29560) + | > loss_dur: 0.07092 (0.10717) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.77679 (13.42142) + | > current_lr: 0.00004 + | > step_time: 1.53930 (3.20902) + | > loader_time: 0.08630 (0.05697) + + + --> STEP: 29/234 -- GLOBAL_STEP: 41915 + | > loss: -0.16478 (-0.18825) + | > log_mle: -0.27508 (-0.29415) + | > loss_dur: 0.11031 (0.10590) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.44815 (13.22163) + | > current_lr: 0.00004 + | > step_time: 1.04860 (2.89575) + | > loader_time: 0.00220 (0.04745) + + + --> STEP: 34/234 -- GLOBAL_STEP: 41920 + | > loss: -0.15969 (-0.18706) + | > log_mle: -0.28412 (-0.29351) + | > loss_dur: 0.12444 (0.10645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.16879 (12.88805) + | > current_lr: 0.00004 + | > step_time: 2.48900 (2.74341) + | > loader_time: 0.00590 (0.04092) + + + --> STEP: 39/234 -- GLOBAL_STEP: 41925 + | > loss: -0.17967 (-0.18537) + | > log_mle: -0.29201 (-0.29291) + | > loss_dur: 0.11234 (0.10754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.40209 (12.94102) + | > current_lr: 0.00004 + | > step_time: 1.50410 (2.61858) + | > loader_time: 0.00290 (0.04014) + + + --> STEP: 44/234 -- GLOBAL_STEP: 41930 + | > loss: -0.18298 (-0.18363) + | > log_mle: -0.28236 (-0.29170) + | > loss_dur: 0.09937 (0.10807) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01090 (12.62583) + | > current_lr: 0.00004 + | > step_time: 1.48860 (2.49508) + | > loader_time: 0.00240 (0.03769) + + + --> STEP: 49/234 -- GLOBAL_STEP: 41935 + | > loss: -0.19668 (-0.18225) + | > log_mle: -0.29359 (-0.29145) + | > loss_dur: 0.09691 (0.10920) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.76841 (12.43117) + | > current_lr: 0.00004 + | > step_time: 2.00680 (2.42537) + | > loader_time: 0.08680 (0.03775) + + + --> STEP: 54/234 -- GLOBAL_STEP: 41940 + | > loss: -0.18605 (-0.18075) + | > log_mle: -0.29634 (-0.29086) + | > loss_dur: 0.11029 (0.11011) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.53747 (12.16460) + | > current_lr: 0.00004 + | > step_time: 1.57200 (2.35470) + | > loader_time: 0.00230 (0.03589) + + + --> STEP: 59/234 -- GLOBAL_STEP: 41945 + | > loss: -0.18698 (-0.17992) + | > log_mle: -0.30031 (-0.29053) + | > loss_dur: 0.11333 (0.11061) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.43865 (12.10630) + | > current_lr: 0.00004 + | > step_time: 1.10040 (2.27635) + | > loader_time: 0.00350 (0.03311) + + + --> STEP: 64/234 -- GLOBAL_STEP: 41950 + | > loss: -0.18080 (-0.17808) + | > log_mle: -0.28346 (-0.29088) + | > loss_dur: 0.10266 (0.11280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.75241 (12.39058) + | > current_lr: 0.00004 + | > step_time: 6.80200 (2.34847) + | > loader_time: 0.10000 (0.03516) + + + --> STEP: 69/234 -- GLOBAL_STEP: 41955 + | > loss: -0.14809 (-0.17643) + | > log_mle: -0.26910 (-0.29023) + | > loss_dur: 0.12102 (0.11380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.73699 (12.24286) + | > current_lr: 0.00004 + | > step_time: 3.20320 (2.31152) + | > loader_time: 0.10070 (0.03544) + + + --> STEP: 74/234 -- GLOBAL_STEP: 41960 + | > loss: -0.14358 (-0.17416) + | > log_mle: -0.27485 (-0.29005) + | > loss_dur: 0.13127 (0.11589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.57884 (12.53551) + | > current_lr: 0.00004 + | > step_time: 2.91990 (2.37329) + | > loader_time: 0.08480 (0.03688) + + + --> STEP: 79/234 -- GLOBAL_STEP: 41965 + | > loss: -0.15862 (-0.17304) + | > log_mle: -0.29367 (-0.29007) + | > loss_dur: 0.13505 (0.11703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.25379 (12.52167) + | > current_lr: 0.00004 + | > step_time: 1.96790 (2.34562) + | > loader_time: 0.00230 (0.03567) + + + --> STEP: 84/234 -- GLOBAL_STEP: 41970 + | > loss: -0.15921 (-0.17209) + | > log_mle: -0.28755 (-0.29012) + | > loss_dur: 0.12833 (0.11803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.68634 (12.49137) + | > current_lr: 0.00004 + | > step_time: 3.03850 (2.32760) + | > loader_time: 0.00300 (0.03577) + + + --> STEP: 89/234 -- GLOBAL_STEP: 41975 + | > loss: -0.17903 (-0.17179) + | > log_mle: -0.31575 (-0.29105) + | > loss_dur: 0.13672 (0.11926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.04142 (12.73128) + | > current_lr: 0.00004 + | > step_time: 2.30790 (2.30671) + | > loader_time: 0.07660 (0.03660) + + + --> STEP: 94/234 -- GLOBAL_STEP: 41980 + | > loss: -0.20059 (-0.17211) + | > log_mle: -0.35017 (-0.29328) + | > loss_dur: 0.14958 (0.12117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.38596 (13.15081) + | > current_lr: 0.00004 + | > step_time: 2.27690 (2.31491) + | > loader_time: 0.00190 (0.03479) + + + --> STEP: 99/234 -- GLOBAL_STEP: 41985 + | > loss: -0.19956 (-0.17253) + | > log_mle: -0.38267 (-0.29545) + | > loss_dur: 0.18311 (0.12292) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.33180 (13.69661) + | > current_lr: 0.00004 + | > step_time: 3.00340 (2.31180) + | > loader_time: 0.08280 (0.03563) + + + --> STEP: 104/234 -- GLOBAL_STEP: 41990 + | > loss: -0.21738 (-0.17297) + | > log_mle: -0.37796 (-0.29782) + | > loss_dur: 0.16058 (0.12485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.68713 (14.61452) + | > current_lr: 0.00004 + | > step_time: 2.39470 (2.38423) + | > loader_time: 0.00860 (0.03682) + + + --> STEP: 109/234 -- GLOBAL_STEP: 41995 + | > loss: -0.15892 (-0.17255) + | > log_mle: -0.35809 (-0.29942) + | > loss_dur: 0.19917 (0.12686) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.97103 (14.92316) + | > current_lr: 0.00004 + | > step_time: 3.34190 (2.37734) + | > loader_time: 0.10940 (0.03696) + + + --> STEP: 114/234 -- GLOBAL_STEP: 42000 + | > loss: -0.18590 (-0.17296) + | > log_mle: -0.34471 (-0.30191) + | > loss_dur: 0.15881 (0.12896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.44731 (15.52595) + | > current_lr: 0.00004 + | > step_time: 2.30650 (2.34260) + | > loader_time: 0.00190 (0.03544) + + + --> STEP: 119/234 -- GLOBAL_STEP: 42005 + | > loss: -0.17007 (-0.17294) + | > log_mle: -0.34491 (-0.30389) + | > loss_dur: 0.17484 (0.13095) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.21927 (15.88399) + | > current_lr: 0.00004 + | > step_time: 3.61010 (2.38980) + | > loader_time: 0.09980 (0.03645) + + + --> STEP: 124/234 -- GLOBAL_STEP: 42010 + | > loss: -0.20978 (-0.17305) + | > log_mle: -0.37359 (-0.30519) + | > loss_dur: 0.16381 (0.13214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.37884 (16.20692) + | > current_lr: 0.00004 + | > step_time: 4.00440 (2.42662) + | > loader_time: 0.08610 (0.03781) + + + --> STEP: 129/234 -- GLOBAL_STEP: 42015 + | > loss: -0.17017 (-0.17421) + | > log_mle: -0.36577 (-0.30831) + | > loss_dur: 0.19560 (0.13409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.24855 (16.87356) + | > current_lr: 0.00004 + | > step_time: 4.81290 (2.53284) + | > loader_time: 0.09390 (0.04170) + + + --> STEP: 134/234 -- GLOBAL_STEP: 42020 + | > loss: -0.20804 (-0.17605) + | > log_mle: -0.41383 (-0.31214) + | > loss_dur: 0.20579 (0.13609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.38010 (17.75819) + | > current_lr: 0.00004 + | > step_time: 1.69360 (2.59419) + | > loader_time: 0.00200 (0.04238) + + + --> STEP: 139/234 -- GLOBAL_STEP: 42025 + | > loss: -0.28137 (-0.17757) + | > log_mle: -0.48051 (-0.31565) + | > loss_dur: 0.19914 (0.13808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.82674 (18.59386) + | > current_lr: 0.00004 + | > step_time: 2.50210 (2.57439) + | > loader_time: 0.00360 (0.04159) + + + --> STEP: 144/234 -- GLOBAL_STEP: 42030 + | > loss: -0.25834 (-0.17916) + | > log_mle: -0.45341 (-0.31943) + | > loss_dur: 0.19507 (0.14027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.12234 (19.45576) + | > current_lr: 0.00004 + | > step_time: 1.91830 (2.56773) + | > loader_time: 0.08760 (0.04206) + + + --> STEP: 149/234 -- GLOBAL_STEP: 42035 + | > loss: -0.30623 (-0.18175) + | > log_mle: -0.51099 (-0.32380) + | > loss_dur: 0.20476 (0.14205) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.60880 (20.49434) + | > current_lr: 0.00004 + | > step_time: 4.90350 (2.56580) + | > loader_time: 0.09600 (0.04136) + + + --> STEP: 154/234 -- GLOBAL_STEP: 42040 + | > loss: -0.27075 (-0.18455) + | > log_mle: -0.46435 (-0.32858) + | > loss_dur: 0.19360 (0.14402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.52537 (21.74536) + | > current_lr: 0.00004 + | > step_time: 2.00000 (2.57686) + | > loader_time: 0.08290 (0.04239) + + + --> STEP: 159/234 -- GLOBAL_STEP: 42045 + | > loss: -0.27900 (-0.18700) + | > log_mle: -0.48304 (-0.33312) + | > loss_dur: 0.20404 (0.14611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.39795 (22.94974) + | > current_lr: 0.00004 + | > step_time: 1.41640 (2.56838) + | > loader_time: 0.08790 (0.04273) + + + --> STEP: 164/234 -- GLOBAL_STEP: 42050 + | > loss: -0.27029 (-0.18979) + | > log_mle: -0.47554 (-0.33753) + | > loss_dur: 0.20524 (0.14774) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.98949 (24.02804) + | > current_lr: 0.00004 + | > step_time: 3.59710 (2.59443) + | > loader_time: 0.00740 (0.04210) + + + --> STEP: 169/234 -- GLOBAL_STEP: 42055 + | > loss: -0.25988 (-0.19276) + | > log_mle: -0.47576 (-0.34227) + | > loss_dur: 0.21588 (0.14951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.56464 (25.10787) + | > current_lr: 0.00004 + | > step_time: 2.99910 (2.66444) + | > loader_time: 0.00350 (0.04266) + + + --> STEP: 174/234 -- GLOBAL_STEP: 42060 + | > loss: -0.34622 (-0.19640) + | > log_mle: -0.55623 (-0.34791) + | > loss_dur: 0.21002 (0.15151) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.76028 (26.78275) + | > current_lr: 0.00004 + | > step_time: 3.40950 (2.80341) + | > loader_time: 0.08560 (0.04466) + + + --> STEP: 179/234 -- GLOBAL_STEP: 42065 + | > loss: -0.29826 (-0.19921) + | > log_mle: -0.54417 (-0.35291) + | > loss_dur: 0.24591 (0.15371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.80370 (28.21515) + | > current_lr: 0.00004 + | > step_time: 1.70580 (2.83918) + | > loader_time: 0.08040 (0.04586) + + + --> STEP: 184/234 -- GLOBAL_STEP: 42070 + | > loss: -0.27827 (-0.20154) + | > log_mle: -0.50453 (-0.35712) + | > loss_dur: 0.22626 (0.15558) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.48141 (29.32610) + | > current_lr: 0.00004 + | > step_time: 2.59360 (2.85758) + | > loader_time: 0.10360 (0.04636) + + + --> STEP: 189/234 -- GLOBAL_STEP: 42075 + | > loss: -0.28888 (-0.20408) + | > log_mle: -0.51382 (-0.36164) + | > loss_dur: 0.22494 (0.15756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.30688 (30.38822) + | > current_lr: 0.00004 + | > step_time: 3.90170 (2.96672) + | > loader_time: 0.00540 (0.04631) + + + --> STEP: 194/234 -- GLOBAL_STEP: 42080 + | > loss: -0.33435 (-0.20739) + | > log_mle: -0.55317 (-0.36645) + | > loss_dur: 0.21882 (0.15906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.76740 (31.36850) + | > current_lr: 0.00004 + | > step_time: 2.39530 (2.98302) + | > loader_time: 0.00320 (0.04924) + + + --> STEP: 199/234 -- GLOBAL_STEP: 42085 + | > loss: -0.33742 (-0.21019) + | > log_mle: -0.56719 (-0.37095) + | > loss_dur: 0.22977 (0.16076) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.45419 (32.46510) + | > current_lr: 0.00004 + | > step_time: 5.80260 (3.04778) + | > loader_time: 0.29370 (0.05402) + + + --> STEP: 204/234 -- GLOBAL_STEP: 42090 + | > loss: -0.35797 (-0.21279) + | > log_mle: -0.60197 (-0.37538) + | > loss_dur: 0.24399 (0.16258) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.81272 (33.71784) + | > current_lr: 0.00004 + | > step_time: 5.59360 (3.09032) + | > loader_time: 0.20190 (0.05538) + + + --> STEP: 209/234 -- GLOBAL_STEP: 42095 + | > loss: -0.32198 (-0.21586) + | > log_mle: -0.55037 (-0.38010) + | > loss_dur: 0.22840 (0.16424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.60633 (34.87607) + | > current_lr: 0.00004 + | > step_time: 9.29700 (3.11679) + | > loader_time: 0.08680 (0.05457) + + + --> STEP: 214/234 -- GLOBAL_STEP: 42100 + | > loss: -0.35863 (-0.21945) + | > log_mle: -0.58003 (-0.38556) + | > loss_dur: 0.22140 (0.16610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.82374 (36.28242) + | > current_lr: 0.00004 + | > step_time: 3.70180 (3.17170) + | > loader_time: 0.09450 (0.05815) + + + --> STEP: 219/234 -- GLOBAL_STEP: 42105 + | > loss: -0.43283 (-0.22296) + | > log_mle: -0.68089 (-0.39088) + | > loss_dur: 0.24807 (0.16792) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.69956 (37.42217) + | > current_lr: 0.00004 + | > step_time: 4.59460 (3.20987) + | > loader_time: 0.08770 (0.05809) + + + --> STEP: 224/234 -- GLOBAL_STEP: 42110 + | > loss: -0.39340 (-0.22632) + | > log_mle: -0.63879 (-0.39593) + | > loss_dur: 0.24539 (0.16961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.10265 (38.64721) + | > current_lr: 0.00004 + | > step_time: 3.90890 (3.22040) + | > loader_time: 0.09810 (0.05774) + + + --> STEP: 229/234 -- GLOBAL_STEP: 42115 + | > loss: -0.37323 (-0.22979) + | > log_mle: -0.68399 (-0.40160) + | > loss_dur: 0.31077 (0.17181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.78869 (39.81871) + | > current_lr: 0.00004 + | > step_time: 0.27520 (3.19306) + | > loader_time: 0.00370 (0.05726) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.40297 (-0.14599) + | > avg_loss: -0.26399 (-0.02485) + | > avg_log_mle: -0.48164 (-0.01759) + | > avg_loss_dur: 0.21765 (-0.00726) + + + > EPOCH: 180/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 22:35:44)  + + --> STEP: 0/234 -- GLOBAL_STEP: 42120 + | > loss: -0.22086 (-0.22086) + | > log_mle: -0.37472 (-0.37472) + | > loss_dur: 0.15386 (0.15386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.57441 (26.57441) + | > current_lr: 0.00005 + | > step_time: 3.21910 (3.21912) + | > loader_time: 15.19280 (15.19281) + + + --> STEP: 5/234 -- GLOBAL_STEP: 42125 + | > loss: -0.18534 (-0.17341) + | > log_mle: -0.29860 (-0.29767) + | > loss_dur: 0.11327 (0.12426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.27310 (19.25204) + | > current_lr: 0.00005 + | > step_time: 8.21030 (5.04889) + | > loader_time: 0.09750 (0.07551) + + + --> STEP: 10/234 -- GLOBAL_STEP: 42130 + | > loss: -0.16675 (-0.18023) + | > log_mle: -0.29570 (-0.29965) + | > loss_dur: 0.12894 (0.11942) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.00153 (17.07215) + | > current_lr: 0.00005 + | > step_time: 2.91680 (4.45705) + | > loader_time: 0.00130 (0.06760) + + + --> STEP: 15/234 -- GLOBAL_STEP: 42135 + | > loss: -0.20740 (-0.18696) + | > log_mle: -0.30157 (-0.30020) + | > loss_dur: 0.09417 (0.11324) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.27007 (15.91749) + | > current_lr: 0.00005 + | > step_time: 10.88660 (4.78515) + | > loader_time: 0.00540 (0.05183) + + + --> STEP: 20/234 -- GLOBAL_STEP: 42140 + | > loss: -0.20447 (-0.18822) + | > log_mle: -0.29311 (-0.29773) + | > loss_dur: 0.08864 (0.10951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.35027 (14.43902) + | > current_lr: 0.00005 + | > step_time: 2.51200 (4.05781) + | > loader_time: 0.07370 (0.05758) + + + --> STEP: 25/234 -- GLOBAL_STEP: 42145 + | > loss: -0.17613 (-0.18986) + | > log_mle: -0.27947 (-0.29607) + | > loss_dur: 0.10335 (0.10621) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06930 (13.51787) + | > current_lr: 0.00005 + | > step_time: 3.69850 (3.66994) + | > loader_time: 0.00320 (0.04752) + + + --> STEP: 30/234 -- GLOBAL_STEP: 42150 + | > loss: -0.20455 (-0.19040) + | > log_mle: -0.29574 (-0.29554) + | > loss_dur: 0.09119 (0.10514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57709 (13.06795) + | > current_lr: 0.00005 + | > step_time: 3.19510 (3.95467) + | > loader_time: 0.00370 (0.05011) + + + --> STEP: 35/234 -- GLOBAL_STEP: 42155 + | > loss: -0.15775 (-0.18786) + | > log_mle: -0.28724 (-0.29473) + | > loss_dur: 0.12949 (0.10687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.91402 (12.99361) + | > current_lr: 0.00005 + | > step_time: 3.69290 (4.07239) + | > loader_time: 0.10800 (0.04918) + + + --> STEP: 40/234 -- GLOBAL_STEP: 42160 + | > loss: -0.12557 (-0.18500) + | > log_mle: -0.27170 (-0.29349) + | > loss_dur: 0.14613 (0.10849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.11821 (13.04776) + | > current_lr: 0.00005 + | > step_time: 1.64760 (3.95480) + | > loader_time: 0.00180 (0.04583) + + + --> STEP: 45/234 -- GLOBAL_STEP: 42165 + | > loss: -0.16453 (-0.18362) + | > log_mle: -0.30388 (-0.29300) + | > loss_dur: 0.13936 (0.10939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.67118 (12.87489) + | > current_lr: 0.00005 + | > step_time: 0.86700 (3.62166) + | > loader_time: 0.00380 (0.04359) + + + --> STEP: 50/234 -- GLOBAL_STEP: 42170 + | > loss: -0.16373 (-0.18256) + | > log_mle: -0.27786 (-0.29206) + | > loss_dur: 0.11413 (0.10949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.87729 (12.44834) + | > current_lr: 0.00005 + | > step_time: 2.11510 (3.50750) + | > loader_time: 0.00240 (0.04094) + + + --> STEP: 55/234 -- GLOBAL_STEP: 42175 + | > loss: -0.19437 (-0.18136) + | > log_mle: -0.29528 (-0.29169) + | > loss_dur: 0.10091 (0.11032) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.63349 (12.18427) + | > current_lr: 0.00005 + | > step_time: 0.61640 (3.35849) + | > loader_time: 0.00240 (0.04041) + + + --> STEP: 60/234 -- GLOBAL_STEP: 42180 + | > loss: -0.17557 (-0.18028) + | > log_mle: -0.30881 (-0.29159) + | > loss_dur: 0.13324 (0.11131) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.75585 (12.08667) + | > current_lr: 0.00005 + | > step_time: 1.88880 (3.23668) + | > loader_time: 0.00190 (0.03866) + + + --> STEP: 65/234 -- GLOBAL_STEP: 42185 + | > loss: -0.16929 (-0.17858) + | > log_mle: -0.28557 (-0.29165) + | > loss_dur: 0.11628 (0.11306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46070 (12.10662) + | > current_lr: 0.00005 + | > step_time: 2.28690 (3.14133) + | > loader_time: 0.00910 (0.04036) + + + --> STEP: 70/234 -- GLOBAL_STEP: 42190 + | > loss: -0.13509 (-0.17643) + | > log_mle: -0.27483 (-0.29082) + | > loss_dur: 0.13974 (0.11439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.79868 (12.06815) + | > current_lr: 0.00005 + | > step_time: 1.69800 (3.02475) + | > loader_time: 0.00280 (0.03766) + + + --> STEP: 75/234 -- GLOBAL_STEP: 42195 + | > loss: -0.15213 (-0.17448) + | > log_mle: -0.29446 (-0.29082) + | > loss_dur: 0.14232 (0.11634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.73008 (12.15270) + | > current_lr: 0.00005 + | > step_time: 3.81540 (3.03127) + | > loader_time: 0.00280 (0.03541) + + + --> STEP: 80/234 -- GLOBAL_STEP: 42200 + | > loss: -0.16027 (-0.17328) + | > log_mle: -0.27769 (-0.29051) + | > loss_dur: 0.11742 (0.11723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.95038 (12.10193) + | > current_lr: 0.00005 + | > step_time: 1.78900 (2.93920) + | > loader_time: 0.00250 (0.03426) + + + --> STEP: 85/234 -- GLOBAL_STEP: 42205 + | > loss: -0.15659 (-0.17221) + | > log_mle: -0.28941 (-0.29058) + | > loss_dur: 0.13282 (0.11837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.11224 (12.16049) + | > current_lr: 0.00005 + | > step_time: 1.59590 (2.86023) + | > loader_time: 0.00260 (0.03243) + + + --> STEP: 90/234 -- GLOBAL_STEP: 42210 + | > loss: -0.14835 (-0.17145) + | > log_mle: -0.30999 (-0.29163) + | > loss_dur: 0.16164 (0.12018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.59251 (12.55788) + | > current_lr: 0.00005 + | > step_time: 2.30240 (2.81060) + | > loader_time: 0.08410 (0.03253) + + + --> STEP: 95/234 -- GLOBAL_STEP: 42215 + | > loss: -0.21259 (-0.17210) + | > log_mle: -0.39735 (-0.29456) + | > loss_dur: 0.18475 (0.12246) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.87005 (13.35590) + | > current_lr: 0.00005 + | > step_time: 1.30470 (2.76043) + | > loader_time: 0.09510 (0.03291) + + + --> STEP: 100/234 -- GLOBAL_STEP: 42220 + | > loss: -0.18045 (-0.17206) + | > log_mle: -0.32205 (-0.29584) + | > loss_dur: 0.14160 (0.12378) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.78493 (13.68869) + | > current_lr: 0.00005 + | > step_time: 1.90220 (2.76064) + | > loader_time: 0.00260 (0.03495) + + + --> STEP: 105/234 -- GLOBAL_STEP: 42225 + | > loss: -0.16985 (-0.17254) + | > log_mle: -0.30427 (-0.29824) + | > loss_dur: 0.13442 (0.12570) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.76896 (14.25142) + | > current_lr: 0.00005 + | > step_time: 1.40040 (2.74836) + | > loader_time: 0.00290 (0.03530) + + + --> STEP: 110/234 -- GLOBAL_STEP: 42230 + | > loss: -0.17088 (-0.17254) + | > log_mle: -0.32849 (-0.30031) + | > loss_dur: 0.15761 (0.12777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.07336 (14.77542) + | > current_lr: 0.00005 + | > step_time: 2.58950 (2.70285) + | > loader_time: 0.10660 (0.03475) + + + --> STEP: 115/234 -- GLOBAL_STEP: 42235 + | > loss: -0.17570 (-0.17331) + | > log_mle: -0.34987 (-0.30320) + | > loss_dur: 0.17417 (0.12990) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.26938 (15.38657) + | > current_lr: 0.00005 + | > step_time: 2.79590 (2.68343) + | > loader_time: 0.00360 (0.03337) + + + --> STEP: 120/234 -- GLOBAL_STEP: 42240 + | > loss: -0.22420 (-0.17375) + | > log_mle: -0.39607 (-0.30560) + | > loss_dur: 0.17187 (0.13185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.53767 (15.85880) + | > current_lr: 0.00005 + | > step_time: 2.20050 (2.68248) + | > loader_time: 0.00320 (0.03288) + + + --> STEP: 125/234 -- GLOBAL_STEP: 42245 + | > loss: -0.20698 (-0.17375) + | > log_mle: -0.38642 (-0.30685) + | > loss_dur: 0.17943 (0.13310) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.13012 (16.17105) + | > current_lr: 0.00005 + | > step_time: 2.51300 (2.66186) + | > loader_time: 0.09380 (0.03242) + + + --> STEP: 130/234 -- GLOBAL_STEP: 42250 + | > loss: -0.21274 (-0.17506) + | > log_mle: -0.39895 (-0.31000) + | > loss_dur: 0.18621 (0.13494) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.66500 (16.97402) + | > current_lr: 0.00005 + | > step_time: 2.28740 (2.66610) + | > loader_time: 0.00340 (0.03276) + + + --> STEP: 135/234 -- GLOBAL_STEP: 42255 + | > loss: -0.16849 (-0.17648) + | > log_mle: -0.32781 (-0.31317) + | > loss_dur: 0.15932 (0.13670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.54003 (17.72786) + | > current_lr: 0.00005 + | > step_time: 2.78960 (2.67155) + | > loader_time: 0.09370 (0.03363) + + + --> STEP: 140/234 -- GLOBAL_STEP: 42260 + | > loss: -0.17355 (-0.17816) + | > log_mle: -0.35735 (-0.31694) + | > loss_dur: 0.18380 (0.13878) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.59956 (18.66588) + | > current_lr: 0.00005 + | > step_time: 2.70190 (2.65894) + | > loader_time: 0.00600 (0.03319) + + + --> STEP: 145/234 -- GLOBAL_STEP: 42265 + | > loss: -0.26002 (-0.18001) + | > log_mle: -0.45482 (-0.32109) + | > loss_dur: 0.19479 (0.14107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.66120 (19.78466) + | > current_lr: 0.00005 + | > step_time: 1.29740 (2.73136) + | > loader_time: 0.00280 (0.03349) + + + --> STEP: 150/234 -- GLOBAL_STEP: 42270 + | > loss: -0.23329 (-0.18212) + | > log_mle: -0.43967 (-0.32502) + | > loss_dur: 0.20638 (0.14290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.85221 (20.79729) + | > current_lr: 0.00005 + | > step_time: 3.30690 (2.70508) + | > loader_time: 0.00230 (0.03302) + + + --> STEP: 155/234 -- GLOBAL_STEP: 42275 + | > loss: -0.28292 (-0.18507) + | > log_mle: -0.49453 (-0.32996) + | > loss_dur: 0.21161 (0.14489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.88634 (22.52238) + | > current_lr: 0.00005 + | > step_time: 1.59510 (2.66954) + | > loader_time: 0.00330 (0.03207) + + + --> STEP: 160/234 -- GLOBAL_STEP: 42280 + | > loss: -0.29189 (-0.18735) + | > log_mle: -0.50355 (-0.33417) + | > loss_dur: 0.21166 (0.14682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.14951 (23.55817) + | > current_lr: 0.00005 + | > step_time: 5.79830 (2.67748) + | > loader_time: 0.10090 (0.03236) + + + --> STEP: 165/234 -- GLOBAL_STEP: 42285 + | > loss: -0.26925 (-0.18948) + | > log_mle: -0.49153 (-0.33807) + | > loss_dur: 0.22228 (0.14859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.84044 (24.68407) + | > current_lr: 0.00005 + | > step_time: 1.30720 (2.66740) + | > loader_time: 0.08260 (0.03349) + + + --> STEP: 170/234 -- GLOBAL_STEP: 42290 + | > loss: -0.29202 (-0.19186) + | > log_mle: -0.52934 (-0.34236) + | > loss_dur: 0.23732 (0.15050) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.85313 (25.92658) + | > current_lr: 0.00005 + | > step_time: 6.08260 (2.68540) + | > loader_time: 0.10880 (0.03370) + + + --> STEP: 175/234 -- GLOBAL_STEP: 42295 + | > loss: -0.25897 (-0.19475) + | > log_mle: -0.49739 (-0.34727) + | > loss_dur: 0.23842 (0.15252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.51561 (27.23958) + | > current_lr: 0.00005 + | > step_time: 1.80480 (2.66230) + | > loader_time: 0.00410 (0.03285) + + + --> STEP: 180/234 -- GLOBAL_STEP: 42300 + | > loss: -0.29141 (-0.19731) + | > log_mle: -0.50454 (-0.35187) + | > loss_dur: 0.21313 (0.15457) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.77740 (28.54132) + | > current_lr: 0.00005 + | > step_time: 1.59920 (2.65284) + | > loader_time: 0.09440 (0.03409) + + + --> STEP: 185/234 -- GLOBAL_STEP: 42305 + | > loss: -0.31813 (-0.19980) + | > log_mle: -0.54984 (-0.35629) + | > loss_dur: 0.23171 (0.15649) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.56528 (29.44453) + | > current_lr: 0.00005 + | > step_time: 2.79680 (2.65642) + | > loader_time: 0.00480 (0.03475) + + + --> STEP: 190/234 -- GLOBAL_STEP: 42310 + | > loss: -0.31073 (-0.20242) + | > log_mle: -0.52582 (-0.36069) + | > loss_dur: 0.21508 (0.15828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.44312 (30.40354) + | > current_lr: 0.00005 + | > step_time: 2.70470 (2.68974) + | > loader_time: 0.08750 (0.03678) + + + --> STEP: 195/234 -- GLOBAL_STEP: 42315 + | > loss: -0.30725 (-0.20557) + | > log_mle: -0.53874 (-0.36544) + | > loss_dur: 0.23149 (0.15987) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.79651 (31.46313) + | > current_lr: 0.00005 + | > step_time: 8.49530 (2.73868) + | > loader_time: 0.00440 (0.03637) + + + --> STEP: 200/234 -- GLOBAL_STEP: 42320 + | > loss: -0.30097 (-0.20825) + | > log_mle: -0.54924 (-0.36982) + | > loss_dur: 0.24827 (0.16157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.44793 (32.46129) + | > current_lr: 0.00005 + | > step_time: 4.40100 (2.83524) + | > loader_time: 0.09150 (0.03789) + + + --> STEP: 205/234 -- GLOBAL_STEP: 42325 + | > loss: -0.30770 (-0.21075) + | > log_mle: -0.54074 (-0.37410) + | > loss_dur: 0.23304 (0.16335) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.86703 (33.31744) + | > current_lr: 0.00005 + | > step_time: 4.40170 (2.85010) + | > loader_time: 0.10460 (0.03938) + + + --> STEP: 210/234 -- GLOBAL_STEP: 42330 + | > loss: -0.38587 (-0.21419) + | > log_mle: -0.62541 (-0.37921) + | > loss_dur: 0.23954 (0.16502) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.85260 (34.49462) + | > current_lr: 0.00005 + | > step_time: 3.39930 (2.89229) + | > loader_time: 0.29260 (0.04117) + + + --> STEP: 215/234 -- GLOBAL_STEP: 42335 + | > loss: -0.33214 (-0.21783) + | > log_mle: -0.57308 (-0.38454) + | > loss_dur: 0.24095 (0.16671) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.47183 (35.84766) + | > current_lr: 0.00005 + | > step_time: 7.89960 (2.91009) + | > loader_time: 0.19400 (0.04290) + + + --> STEP: 220/234 -- GLOBAL_STEP: 42340 + | > loss: -0.38234 (-0.22174) + | > log_mle: -0.62475 (-0.39024) + | > loss_dur: 0.24240 (0.16849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.49687 (37.14649) + | > current_lr: 0.00005 + | > step_time: 4.79910 (2.97621) + | > loader_time: 0.00590 (0.04610) + + + --> STEP: 225/234 -- GLOBAL_STEP: 42345 + | > loss: -0.44421 (-0.22544) + | > log_mle: -0.69864 (-0.39565) + | > loss_dur: 0.25443 (0.17021) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.36317 (38.43744) + | > current_lr: 0.00005 + | > step_time: 0.23440 (2.93521) + | > loader_time: 0.00410 (0.04551) + + + --> STEP: 230/234 -- GLOBAL_STEP: 42350 + | > loss: -0.41326 (-0.22898) + | > log_mle: -0.74391 (-0.40161) + | > loss_dur: 0.33065 (0.17264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 140.52017 (39.86521) + | > current_lr: 0.00005 + | > step_time: 0.26390 (2.87678) + | > loader_time: 0.00300 (0.04461) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.29591 (-0.10706) + | > avg_loss: -0.27343 (-0.00944) + | > avg_log_mle: -0.49367 (-0.01203) + | > avg_loss_dur: 0.22025 (+0.00260) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_42354.pth + + > EPOCH: 181/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 22:48:02)  + + --> STEP: 1/234 -- GLOBAL_STEP: 42355 + | > loss: -0.18244 (-0.18244) + | > log_mle: -0.29767 (-0.29767) + | > loss_dur: 0.11523 (0.11523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.02416 (25.02416) + | > current_lr: 0.00005 + | > step_time: 2.89790 (2.89793) + | > loader_time: 0.28700 (0.28702) + + + --> STEP: 6/234 -- GLOBAL_STEP: 42360 + | > loss: -0.20210 (-0.18273) + | > log_mle: -0.29320 (-0.29761) + | > loss_dur: 0.09110 (0.11487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.43855 (18.26941) + | > current_lr: 0.00005 + | > step_time: 3.99210 (4.78387) + | > loader_time: 0.00180 (0.24083) + + + --> STEP: 11/234 -- GLOBAL_STEP: 42365 + | > loss: -0.20441 (-0.18803) + | > log_mle: -0.30009 (-0.30073) + | > loss_dur: 0.09568 (0.11270) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.31230 (17.06759) + | > current_lr: 0.00005 + | > step_time: 5.79620 (5.17257) + | > loader_time: 0.00490 (0.20439) + + + --> STEP: 16/234 -- GLOBAL_STEP: 42370 + | > loss: -0.20574 (-0.19061) + | > log_mle: -0.29737 (-0.30063) + | > loss_dur: 0.09163 (0.11003) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.93735 (15.89330) + | > current_lr: 0.00005 + | > step_time: 1.59200 (4.59908) + | > loader_time: 0.01020 (0.15314) + + + --> STEP: 21/234 -- GLOBAL_STEP: 42375 + | > loss: -0.17541 (-0.18959) + | > log_mle: -0.27474 (-0.29728) + | > loss_dur: 0.09933 (0.10769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.66189 (14.49827) + | > current_lr: 0.00005 + | > step_time: 2.20520 (4.61250) + | > loader_time: 0.00180 (0.12762) + + + --> STEP: 26/234 -- GLOBAL_STEP: 42380 + | > loss: -0.17941 (-0.19043) + | > log_mle: -0.28846 (-0.29624) + | > loss_dur: 0.10904 (0.10581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.31888 (13.89934) + | > current_lr: 0.00005 + | > step_time: 2.61810 (4.26611) + | > loader_time: 0.00220 (0.10997) + + + --> STEP: 31/234 -- GLOBAL_STEP: 42385 + | > loss: -0.14853 (-0.18941) + | > log_mle: -0.28443 (-0.29497) + | > loss_dur: 0.13590 (0.10556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.56879 (13.54703) + | > current_lr: 0.00005 + | > step_time: 1.39600 (3.83526) + | > loader_time: 0.00140 (0.09270) + + + --> STEP: 36/234 -- GLOBAL_STEP: 42390 + | > loss: -0.17034 (-0.18692) + | > log_mle: -0.28354 (-0.29365) + | > loss_dur: 0.11320 (0.10673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.79844 (13.42928) + | > current_lr: 0.00005 + | > step_time: 3.90590 (3.59058) + | > loader_time: 0.10080 (0.08289) + + + --> STEP: 41/234 -- GLOBAL_STEP: 42395 + | > loss: -0.18435 (-0.18512) + | > log_mle: -0.29037 (-0.29285) + | > loss_dur: 0.10602 (0.10773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.86835 (12.92491) + | > current_lr: 0.00005 + | > step_time: 1.20210 (3.41833) + | > loader_time: 0.00330 (0.07514) + + + --> STEP: 46/234 -- GLOBAL_STEP: 42400 + | > loss: -0.13966 (-0.18267) + | > log_mle: -0.27614 (-0.29205) + | > loss_dur: 0.13648 (0.10938) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.44960 (13.01367) + | > current_lr: 0.00005 + | > step_time: 1.01740 (3.25852) + | > loader_time: 0.00290 (0.06926) + + + --> STEP: 51/234 -- GLOBAL_STEP: 42405 + | > loss: -0.16004 (-0.18127) + | > log_mle: -0.27626 (-0.29095) + | > loss_dur: 0.11622 (0.10968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.70137 (12.56641) + | > current_lr: 0.00005 + | > step_time: 1.06800 (3.11938) + | > loader_time: 0.00220 (0.06603) + + + --> STEP: 56/234 -- GLOBAL_STEP: 42410 + | > loss: -0.16014 (-0.18020) + | > log_mle: -0.29043 (-0.29088) + | > loss_dur: 0.13029 (0.11068) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.62505 (12.38074) + | > current_lr: 0.00005 + | > step_time: 1.26730 (2.94926) + | > loader_time: 0.00230 (0.06033) + + + --> STEP: 61/234 -- GLOBAL_STEP: 42415 + | > loss: -0.17142 (-0.17936) + | > log_mle: -0.28816 (-0.29082) + | > loss_dur: 0.11674 (0.11146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.94136 (12.20902) + | > current_lr: 0.00005 + | > step_time: 1.50100 (2.84008) + | > loader_time: 0.00290 (0.05565) + + + --> STEP: 66/234 -- GLOBAL_STEP: 42420 + | > loss: -0.17321 (-0.17775) + | > log_mle: -0.27969 (-0.29068) + | > loss_dur: 0.10648 (0.11293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.31223 (12.25855) + | > current_lr: 0.00005 + | > step_time: 1.21020 (2.78997) + | > loader_time: 0.07850 (0.05405) + + + --> STEP: 71/234 -- GLOBAL_STEP: 42425 + | > loss: -0.15126 (-0.17548) + | > log_mle: -0.31420 (-0.29045) + | > loss_dur: 0.16294 (0.11497) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.82566 (12.35408) + | > current_lr: 0.00005 + | > step_time: 2.11980 (2.70851) + | > loader_time: 0.08530 (0.05272) + + + --> STEP: 76/234 -- GLOBAL_STEP: 42430 + | > loss: -0.16673 (-0.17387) + | > log_mle: -0.30054 (-0.29060) + | > loss_dur: 0.13381 (0.11672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.47669 (12.27239) + | > current_lr: 0.00005 + | > step_time: 1.62000 (2.64612) + | > loader_time: 0.08510 (0.05160) + + + --> STEP: 81/234 -- GLOBAL_STEP: 42435 + | > loss: -0.16614 (-0.17314) + | > log_mle: -0.30362 (-0.29060) + | > loss_dur: 0.13748 (0.11745) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.79659 (12.17344) + | > current_lr: 0.00005 + | > step_time: 1.73300 (2.58529) + | > loader_time: 0.00230 (0.04856) + + + --> STEP: 86/234 -- GLOBAL_STEP: 42440 + | > loss: -0.15884 (-0.17233) + | > log_mle: -0.30060 (-0.29078) + | > loss_dur: 0.14175 (0.11845) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.56901 (12.16227) + | > current_lr: 0.00005 + | > step_time: 1.56860 (2.55956) + | > loader_time: 0.00230 (0.04891) + + + --> STEP: 91/234 -- GLOBAL_STEP: 42445 + | > loss: -0.15344 (-0.17185) + | > log_mle: -0.31338 (-0.29211) + | > loss_dur: 0.15994 (0.12026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.41926 (12.44735) + | > current_lr: 0.00005 + | > step_time: 1.31440 (2.53094) + | > loader_time: 0.08890 (0.04731) + + + --> STEP: 96/234 -- GLOBAL_STEP: 42450 + | > loss: -0.16505 (-0.17291) + | > log_mle: -0.30039 (-0.29506) + | > loss_dur: 0.13534 (0.12215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.42747 (12.99113) + | > current_lr: 0.00005 + | > step_time: 3.20170 (2.51356) + | > loader_time: 0.08520 (0.04585) + + + --> STEP: 101/234 -- GLOBAL_STEP: 42455 + | > loss: -0.18834 (-0.17327) + | > log_mle: -0.35647 (-0.29698) + | > loss_dur: 0.16813 (0.12371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.38039 (13.59896) + | > current_lr: 0.00005 + | > step_time: 1.97240 (2.49706) + | > loader_time: 0.00260 (0.04371) + + + --> STEP: 106/234 -- GLOBAL_STEP: 42460 + | > loss: -0.16400 (-0.17386) + | > log_mle: -0.35461 (-0.29946) + | > loss_dur: 0.19061 (0.12560) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.79306 (14.19247) + | > current_lr: 0.00005 + | > step_time: 2.99120 (2.46705) + | > loader_time: 0.10210 (0.04421) + + + --> STEP: 111/234 -- GLOBAL_STEP: 42465 + | > loss: -0.20201 (-0.17402) + | > log_mle: -0.40279 (-0.30193) + | > loss_dur: 0.20077 (0.12791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.18299 (14.96369) + | > current_lr: 0.00005 + | > step_time: 1.84150 (2.42511) + | > loader_time: 0.00350 (0.04310) + + + --> STEP: 116/234 -- GLOBAL_STEP: 42470 + | > loss: -0.17036 (-0.17452) + | > log_mle: -0.37012 (-0.30448) + | > loss_dur: 0.19977 (0.12996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.02917 (15.56312) + | > current_lr: 0.00005 + | > step_time: 3.10880 (2.44195) + | > loader_time: 0.00600 (0.04442) + + + --> STEP: 121/234 -- GLOBAL_STEP: 42475 + | > loss: -0.13835 (-0.17489) + | > log_mle: -0.28217 (-0.30619) + | > loss_dur: 0.14381 (0.13130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05743 (15.91183) + | > current_lr: 0.00005 + | > step_time: 4.88940 (2.43576) + | > loader_time: 0.00270 (0.04272) + + + --> STEP: 126/234 -- GLOBAL_STEP: 42480 + | > loss: -0.22476 (-0.17537) + | > log_mle: -0.41628 (-0.30836) + | > loss_dur: 0.19152 (0.13299) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.44074 (16.62316) + | > current_lr: 0.00005 + | > step_time: 2.00080 (2.42455) + | > loader_time: 0.00300 (0.04117) + + + --> STEP: 131/234 -- GLOBAL_STEP: 42485 + | > loss: -0.25787 (-0.17674) + | > log_mle: -0.46455 (-0.31175) + | > loss_dur: 0.20668 (0.13500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.50795 (17.46115) + | > current_lr: 0.00005 + | > step_time: 4.70330 (2.42273) + | > loader_time: 0.00460 (0.03973) + + + --> STEP: 136/234 -- GLOBAL_STEP: 42490 + | > loss: -0.28317 (-0.17828) + | > log_mle: -0.50871 (-0.31516) + | > loss_dur: 0.22554 (0.13689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.88277 (18.22045) + | > current_lr: 0.00005 + | > step_time: 2.98880 (2.46595) + | > loader_time: 0.00560 (0.04036) + + + --> STEP: 141/234 -- GLOBAL_STEP: 42495 + | > loss: -0.22553 (-0.17936) + | > log_mle: -0.41377 (-0.31810) + | > loss_dur: 0.18824 (0.13875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.85163 (18.94890) + | > current_lr: 0.00005 + | > step_time: 1.98900 (2.44287) + | > loader_time: 0.00320 (0.03957) + + + --> STEP: 146/234 -- GLOBAL_STEP: 42500 + | > loss: -0.27209 (-0.18183) + | > log_mle: -0.46537 (-0.32283) + | > loss_dur: 0.19328 (0.14100) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.26460 (19.94197) + | > current_lr: 0.00005 + | > step_time: 3.98900 (2.43179) + | > loader_time: 0.00350 (0.04001) + + + --> STEP: 151/234 -- GLOBAL_STEP: 42505 + | > loss: -0.24496 (-0.18400) + | > log_mle: -0.42812 (-0.32666) + | > loss_dur: 0.18316 (0.14266) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.21849 (20.95309) + | > current_lr: 0.00005 + | > step_time: 1.24310 (2.41422) + | > loader_time: 0.00180 (0.04110) + + + --> STEP: 156/234 -- GLOBAL_STEP: 42510 + | > loss: -0.28222 (-0.18748) + | > log_mle: -0.47772 (-0.33204) + | > loss_dur: 0.19550 (0.14455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.97133 (22.22822) + | > current_lr: 0.00005 + | > step_time: 7.40910 (2.46758) + | > loader_time: 0.29090 (0.04307) + + + --> STEP: 161/234 -- GLOBAL_STEP: 42515 + | > loss: -0.29610 (-0.19004) + | > log_mle: -0.49885 (-0.33660) + | > loss_dur: 0.20275 (0.14656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.54287 (23.22905) + | > current_lr: 0.00005 + | > step_time: 1.99920 (2.52077) + | > loader_time: 0.08940 (0.04477) + + + --> STEP: 166/234 -- GLOBAL_STEP: 42520 + | > loss: -0.25492 (-0.19229) + | > log_mle: -0.44191 (-0.34053) + | > loss_dur: 0.18699 (0.14824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.01185 (24.14779) + | > current_lr: 0.00005 + | > step_time: 8.49800 (2.61659) + | > loader_time: 0.00390 (0.04533) + + + --> STEP: 171/234 -- GLOBAL_STEP: 42525 + | > loss: -0.34807 (-0.19572) + | > log_mle: -0.55137 (-0.34600) + | > loss_dur: 0.20330 (0.15028) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.87268 (25.42332) + | > current_lr: 0.00005 + | > step_time: 14.50890 (2.70671) + | > loader_time: 0.08810 (0.04681) + + + --> STEP: 176/234 -- GLOBAL_STEP: 42530 + | > loss: -0.30685 (-0.19895) + | > log_mle: -0.52372 (-0.35132) + | > loss_dur: 0.21688 (0.15236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.24437 (26.84583) + | > current_lr: 0.00005 + | > step_time: 2.59490 (2.71150) + | > loader_time: 0.09450 (0.04662) + + + --> STEP: 181/234 -- GLOBAL_STEP: 42535 + | > loss: -0.23594 (-0.20164) + | > log_mle: -0.44789 (-0.35605) + | > loss_dur: 0.21195 (0.15441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.68304 (28.04706) + | > current_lr: 0.00005 + | > step_time: 3.10030 (2.73402) + | > loader_time: 0.09780 (0.04848) + + + --> STEP: 186/234 -- GLOBAL_STEP: 42540 + | > loss: -0.25862 (-0.20436) + | > log_mle: -0.49485 (-0.36084) + | > loss_dur: 0.23624 (0.15648) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.54908 (29.17669) + | > current_lr: 0.00005 + | > step_time: 7.10520 (2.80426) + | > loader_time: 0.19110 (0.04980) + + + --> STEP: 191/234 -- GLOBAL_STEP: 42545 + | > loss: -0.30091 (-0.20735) + | > log_mle: -0.51383 (-0.36551) + | > loss_dur: 0.21292 (0.15816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.88598 (30.37981) + | > current_lr: 0.00005 + | > step_time: 8.60110 (2.86913) + | > loader_time: 0.11200 (0.05117) + + + --> STEP: 196/234 -- GLOBAL_STEP: 42550 + | > loss: -0.28174 (-0.21057) + | > log_mle: -0.51287 (-0.37036) + | > loss_dur: 0.23113 (0.15979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.23109 (31.47335) + | > current_lr: 0.00005 + | > step_time: 5.20540 (2.95873) + | > loader_time: 0.18070 (0.05274) + + + --> STEP: 201/234 -- GLOBAL_STEP: 42555 + | > loss: -0.24394 (-0.21319) + | > log_mle: -0.47188 (-0.37464) + | > loss_dur: 0.22793 (0.16146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.59810 (32.55690) + | > current_lr: 0.00005 + | > step_time: 3.50200 (2.97472) + | > loader_time: 0.09750 (0.05342) + + + --> STEP: 206/234 -- GLOBAL_STEP: 42560 + | > loss: -0.35381 (-0.21641) + | > log_mle: -0.57984 (-0.37948) + | > loss_dur: 0.22603 (0.16306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.81277 (33.84250) + | > current_lr: 0.00005 + | > step_time: 2.60640 (3.01520) + | > loader_time: 0.00510 (0.05684) + + + --> STEP: 211/234 -- GLOBAL_STEP: 42565 + | > loss: -0.38653 (-0.21982) + | > log_mle: -0.63919 (-0.38469) + | > loss_dur: 0.25266 (0.16487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.71214 (35.29094) + | > current_lr: 0.00005 + | > step_time: 3.60580 (3.07083) + | > loader_time: 0.00980 (0.05652) + + + --> STEP: 216/234 -- GLOBAL_STEP: 42570 + | > loss: -0.37099 (-0.22298) + | > log_mle: -0.62629 (-0.38963) + | > loss_dur: 0.25530 (0.16664) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 121.11886 (36.67737) + | > current_lr: 0.00005 + | > step_time: 9.91030 (3.25444) + | > loader_time: 0.08370 (0.05962) + + + --> STEP: 221/234 -- GLOBAL_STEP: 42575 + | > loss: -0.32843 (-0.22629) + | > log_mle: -0.55305 (-0.39464) + | > loss_dur: 0.22462 (0.16834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.79913 (37.93334) + | > current_lr: 0.00005 + | > step_time: 1.39040 (3.27171) + | > loader_time: 0.00360 (0.05951) + + + --> STEP: 226/234 -- GLOBAL_STEP: 42580 + | > loss: -0.41204 (-0.23006) + | > log_mle: -0.66276 (-0.40028) + | > loss_dur: 0.25072 (0.17022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.07558 (39.40189) + | > current_lr: 0.00005 + | > step_time: 0.24230 (3.21122) + | > loader_time: 0.00420 (0.05863) + + + --> STEP: 231/234 -- GLOBAL_STEP: 42585 + | > loss: -0.33129 (-0.23287) + | > log_mle: -0.71687 (-0.40614) + | > loss_dur: 0.38558 (0.17327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.25223 (40.98143) + | > current_lr: 0.00005 + | > step_time: 0.28890 (3.14744) + | > loader_time: 0.00760 (0.05745) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15001 (-0.14590) + | > avg_loss: -0.26655 (+0.00687) + | > avg_log_mle: -0.48254 (+0.01113) + | > avg_loss_dur: 0.21599 (-0.00426) + + + > EPOCH: 182/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 23:01:30)  + + --> STEP: 2/234 -- GLOBAL_STEP: 42590 + | > loss: -0.21949 (-0.21255) + | > log_mle: -0.31139 (-0.30644) + | > loss_dur: 0.09190 (0.09389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.59685 (15.78058) + | > current_lr: 0.00005 + | > step_time: 2.50810 (5.95299) + | > loader_time: 0.00240 (0.80380) + + + --> STEP: 7/234 -- GLOBAL_STEP: 42595 + | > loss: -0.20462 (-0.18774) + | > log_mle: -0.30458 (-0.30198) + | > loss_dur: 0.09996 (0.11423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.55153 (17.79720) + | > current_lr: 0.00005 + | > step_time: 5.19730 (3.83106) + | > loader_time: 0.00230 (0.24507) + + + --> STEP: 12/234 -- GLOBAL_STEP: 42600 + | > loss: -0.18435 (-0.18877) + | > log_mle: -0.29724 (-0.30207) + | > loss_dur: 0.11289 (0.11330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.82934 (17.81666) + | > current_lr: 0.00005 + | > step_time: 2.80150 (3.82419) + | > loader_time: 0.08200 (0.28276) + + + --> STEP: 17/234 -- GLOBAL_STEP: 42605 + | > loss: -0.19259 (-0.19206) + | > log_mle: -0.27878 (-0.30050) + | > loss_dur: 0.08619 (0.10844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.89713 (16.04430) + | > current_lr: 0.00005 + | > step_time: 2.38640 (3.59223) + | > loader_time: 0.00130 (0.20510) + + + --> STEP: 22/234 -- GLOBAL_STEP: 42610 + | > loss: -0.19271 (-0.19163) + | > log_mle: -0.29384 (-0.29807) + | > loss_dur: 0.10113 (0.10645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.29156 (15.39060) + | > current_lr: 0.00005 + | > step_time: 2.40810 (3.22591) + | > loader_time: 0.07970 (0.16647) + + + --> STEP: 27/234 -- GLOBAL_STEP: 42615 + | > loss: -0.19087 (-0.19222) + | > log_mle: -0.29231 (-0.29674) + | > loss_dur: 0.10144 (0.10452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.68939 (14.65130) + | > current_lr: 0.00005 + | > step_time: 3.98900 (3.43844) + | > loader_time: 0.00190 (0.13992) + + + --> STEP: 32/234 -- GLOBAL_STEP: 42620 + | > loss: -0.20564 (-0.19163) + | > log_mle: -0.30148 (-0.29607) + | > loss_dur: 0.09584 (0.10444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.89169 (13.98332) + | > current_lr: 0.00005 + | > step_time: 6.10390 (3.59585) + | > loader_time: 0.09020 (0.12688) + + + --> STEP: 37/234 -- GLOBAL_STEP: 42625 + | > loss: -0.18942 (-0.18972) + | > log_mle: -0.28397 (-0.29487) + | > loss_dur: 0.09454 (0.10515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85117 (13.59972) + | > current_lr: 0.00005 + | > step_time: 1.69430 (3.53761) + | > loader_time: 0.00220 (0.11518) + + + --> STEP: 42/234 -- GLOBAL_STEP: 42630 + | > loss: -0.15648 (-0.18731) + | > log_mle: -0.27565 (-0.29408) + | > loss_dur: 0.11917 (0.10677) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.28579 (13.07209) + | > current_lr: 0.00005 + | > step_time: 2.50680 (3.33720) + | > loader_time: 0.08250 (0.10832) + + + --> STEP: 47/234 -- GLOBAL_STEP: 42635 + | > loss: -0.15676 (-0.18538) + | > log_mle: -0.28564 (-0.29397) + | > loss_dur: 0.12888 (0.10859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.15159 (12.97062) + | > current_lr: 0.00005 + | > step_time: 1.58940 (3.13934) + | > loader_time: 0.00220 (0.09880) + + + --> STEP: 52/234 -- GLOBAL_STEP: 42640 + | > loss: -0.14621 (-0.18427) + | > log_mle: -0.28466 (-0.29318) + | > loss_dur: 0.13845 (0.10890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.35704 (12.53032) + | > current_lr: 0.00005 + | > step_time: 1.80220 (2.99724) + | > loader_time: 0.00250 (0.09114) + + + --> STEP: 57/234 -- GLOBAL_STEP: 42645 + | > loss: -0.14727 (-0.18310) + | > log_mle: -0.27398 (-0.29310) + | > loss_dur: 0.12671 (0.11000) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.24267 (12.30996) + | > current_lr: 0.00005 + | > step_time: 2.80380 (2.92703) + | > loader_time: 0.00400 (0.08666) + + + --> STEP: 62/234 -- GLOBAL_STEP: 42650 + | > loss: -0.13432 (-0.18212) + | > log_mle: -0.31922 (-0.29384) + | > loss_dur: 0.18490 (0.11173) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.42421 (12.47084) + | > current_lr: 0.00005 + | > step_time: 1.20530 (2.85314) + | > loader_time: 0.00260 (0.08002) + + + --> STEP: 67/234 -- GLOBAL_STEP: 42655 + | > loss: -0.16685 (-0.18114) + | > log_mle: -0.29954 (-0.29344) + | > loss_dur: 0.13269 (0.11230) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.51628 (12.32202) + | > current_lr: 0.00005 + | > step_time: 1.09120 (2.77569) + | > loader_time: 0.00310 (0.07718) + + + --> STEP: 72/234 -- GLOBAL_STEP: 42660 + | > loss: -0.16197 (-0.17877) + | > log_mle: -0.28467 (-0.29290) + | > loss_dur: 0.12270 (0.11412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.47140 (12.38597) + | > current_lr: 0.00005 + | > step_time: 1.66750 (2.69376) + | > loader_time: 0.00140 (0.07195) + + + --> STEP: 77/234 -- GLOBAL_STEP: 42665 + | > loss: -0.17306 (-0.17746) + | > log_mle: -0.29231 (-0.29301) + | > loss_dur: 0.11925 (0.11554) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.31253 (12.42351) + | > current_lr: 0.00005 + | > step_time: 3.38330 (2.65822) + | > loader_time: 0.00210 (0.06863) + + + --> STEP: 82/234 -- GLOBAL_STEP: 42670 + | > loss: -0.15782 (-0.17667) + | > log_mle: -0.28520 (-0.29283) + | > loss_dur: 0.12738 (0.11615) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.00051 (12.34452) + | > current_lr: 0.00005 + | > step_time: 1.40080 (2.61681) + | > loader_time: 0.00220 (0.06467) + + + --> STEP: 87/234 -- GLOBAL_STEP: 42675 + | > loss: -0.14558 (-0.17543) + | > log_mle: -0.28982 (-0.29286) + | > loss_dur: 0.14424 (0.11743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.25192 (12.43922) + | > current_lr: 0.00005 + | > step_time: 4.90670 (2.65635) + | > loader_time: 0.09850 (0.06435) + + + --> STEP: 92/234 -- GLOBAL_STEP: 42680 + | > loss: -0.18954 (-0.17523) + | > log_mle: -0.33820 (-0.29448) + | > loss_dur: 0.14866 (0.11926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.97198 (12.76019) + | > current_lr: 0.00005 + | > step_time: 1.94310 (2.63466) + | > loader_time: 0.08960 (0.06290) + + + --> STEP: 97/234 -- GLOBAL_STEP: 42685 + | > loss: -0.18172 (-0.17590) + | > log_mle: -0.32484 (-0.29707) + | > loss_dur: 0.14313 (0.12117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.99033 (13.28670) + | > current_lr: 0.00005 + | > step_time: 2.20760 (2.58487) + | > loader_time: 0.00280 (0.05982) + + + --> STEP: 102/234 -- GLOBAL_STEP: 42690 + | > loss: -0.15013 (-0.17569) + | > log_mle: -0.30863 (-0.29877) + | > loss_dur: 0.15850 (0.12308) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.77281 (13.65026) + | > current_lr: 0.00005 + | > step_time: 4.60000 (2.57576) + | > loader_time: 0.09930 (0.05799) + + + --> STEP: 107/234 -- GLOBAL_STEP: 42695 + | > loss: -0.18594 (-0.17641) + | > log_mle: -0.35288 (-0.30159) + | > loss_dur: 0.16695 (0.12518) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.71430 (14.34638) + | > current_lr: 0.00005 + | > step_time: 1.10150 (2.62741) + | > loader_time: 0.00210 (0.05702) + + + --> STEP: 112/234 -- GLOBAL_STEP: 42700 + | > loss: -0.18385 (-0.17639) + | > log_mle: -0.36568 (-0.30413) + | > loss_dur: 0.18183 (0.12773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.35079 (15.00263) + | > current_lr: 0.00005 + | > step_time: 1.52250 (2.57484) + | > loader_time: 0.07860 (0.05595) + + + --> STEP: 117/234 -- GLOBAL_STEP: 42705 + | > loss: -0.19630 (-0.17679) + | > log_mle: -0.35937 (-0.30655) + | > loss_dur: 0.16307 (0.12976) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.56552 (15.64214) + | > current_lr: 0.00005 + | > step_time: 1.31910 (2.53253) + | > loader_time: 0.00480 (0.05370) + + + --> STEP: 122/234 -- GLOBAL_STEP: 42710 + | > loss: -0.16832 (-0.17681) + | > log_mle: -0.33336 (-0.30803) + | > loss_dur: 0.16503 (0.13122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.50775 (15.95023) + | > current_lr: 0.00005 + | > step_time: 1.51300 (2.50075) + | > loader_time: 0.08330 (0.05227) + + + --> STEP: 127/234 -- GLOBAL_STEP: 42715 + | > loss: -0.20796 (-0.17787) + | > log_mle: -0.39427 (-0.31086) + | > loss_dur: 0.18631 (0.13299) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.37674 (16.66288) + | > current_lr: 0.00005 + | > step_time: 1.70160 (2.49279) + | > loader_time: 0.08960 (0.05106) + + + --> STEP: 132/234 -- GLOBAL_STEP: 42720 + | > loss: -0.21045 (-0.17938) + | > log_mle: -0.37504 (-0.31411) + | > loss_dur: 0.16458 (0.13473) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.08676 (17.50282) + | > current_lr: 0.00005 + | > step_time: 1.81290 (2.50994) + | > loader_time: 0.00260 (0.05120) + + + --> STEP: 137/234 -- GLOBAL_STEP: 42725 + | > loss: -0.17354 (-0.18077) + | > log_mle: -0.38444 (-0.31771) + | > loss_dur: 0.21090 (0.13694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.91804 (18.30765) + | > current_lr: 0.00005 + | > step_time: 1.04860 (2.48655) + | > loader_time: 0.00260 (0.05006) + + + --> STEP: 142/234 -- GLOBAL_STEP: 42730 + | > loss: -0.20746 (-0.18192) + | > log_mle: -0.39678 (-0.32059) + | > loss_dur: 0.18931 (0.13867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.47306 (19.16088) + | > current_lr: 0.00005 + | > step_time: 2.37510 (2.48078) + | > loader_time: 0.00250 (0.04970) + + + --> STEP: 147/234 -- GLOBAL_STEP: 42735 + | > loss: -0.21399 (-0.18428) + | > log_mle: -0.40398 (-0.32519) + | > loss_dur: 0.18999 (0.14091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.63532 (20.14857) + | > current_lr: 0.00005 + | > step_time: 2.01170 (2.48444) + | > loader_time: 0.00270 (0.04988) + + + --> STEP: 152/234 -- GLOBAL_STEP: 42740 + | > loss: -0.26273 (-0.18673) + | > log_mle: -0.48415 (-0.32944) + | > loss_dur: 0.22142 (0.14272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.10436 (21.37765) + | > current_lr: 0.00005 + | > step_time: 1.39610 (2.46666) + | > loader_time: 0.00410 (0.05016) + + + --> STEP: 157/234 -- GLOBAL_STEP: 42745 + | > loss: -0.22102 (-0.18975) + | > log_mle: -0.43050 (-0.33440) + | > loss_dur: 0.20947 (0.14464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.70655 (22.50888) + | > current_lr: 0.00005 + | > step_time: 1.31100 (2.46523) + | > loader_time: 0.00340 (0.04920) + + + --> STEP: 162/234 -- GLOBAL_STEP: 42750 + | > loss: -0.27237 (-0.19241) + | > log_mle: -0.46473 (-0.33897) + | > loss_dur: 0.19237 (0.14656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.38670 (23.77341) + | > current_lr: 0.00005 + | > step_time: 5.09610 (2.48794) + | > loader_time: 0.00900 (0.04886) + + + --> STEP: 167/234 -- GLOBAL_STEP: 42755 + | > loss: -0.34954 (-0.19491) + | > log_mle: -0.55213 (-0.34318) + | > loss_dur: 0.20259 (0.14827) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.96296 (24.91374) + | > current_lr: 0.00005 + | > step_time: 3.99920 (2.65471) + | > loader_time: 0.08590 (0.05273) + + + --> STEP: 172/234 -- GLOBAL_STEP: 42760 + | > loss: -0.31148 (-0.19794) + | > log_mle: -0.53684 (-0.34831) + | > loss_dur: 0.22537 (0.15037) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.60422 (26.17257) + | > current_lr: 0.00005 + | > step_time: 3.93010 (2.75269) + | > loader_time: 0.00400 (0.05350) + + + --> STEP: 177/234 -- GLOBAL_STEP: 42765 + | > loss: -0.29281 (-0.20078) + | > log_mle: -0.50078 (-0.35316) + | > loss_dur: 0.20797 (0.15237) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.69241 (27.29092) + | > current_lr: 0.00005 + | > step_time: 3.30760 (2.75054) + | > loader_time: 0.00290 (0.05261) + + + --> STEP: 182/234 -- GLOBAL_STEP: 42770 + | > loss: -0.29707 (-0.20336) + | > log_mle: -0.54365 (-0.35791) + | > loss_dur: 0.24658 (0.15456) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.59206 (28.55939) + | > current_lr: 0.00005 + | > step_time: 4.19330 (2.74374) + | > loader_time: 0.00270 (0.05252) + + + --> STEP: 187/234 -- GLOBAL_STEP: 42775 + | > loss: -0.31291 (-0.20597) + | > log_mle: -0.53496 (-0.36242) + | > loss_dur: 0.22205 (0.15645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.11913 (29.87945) + | > current_lr: 0.00005 + | > step_time: 7.20220 (2.82656) + | > loader_time: 0.10580 (0.05425) + + + --> STEP: 192/234 -- GLOBAL_STEP: 42780 + | > loss: -0.33813 (-0.20874) + | > log_mle: -0.55690 (-0.36688) + | > loss_dur: 0.21877 (0.15814) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.76228 (31.11595) + | > current_lr: 0.00005 + | > step_time: 3.40610 (2.86028) + | > loader_time: 0.00310 (0.05497) + + + --> STEP: 197/234 -- GLOBAL_STEP: 42785 + | > loss: -0.33508 (-0.21157) + | > log_mle: -0.54282 (-0.37128) + | > loss_dur: 0.20775 (0.15972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.25423 (32.14781) + | > current_lr: 0.00005 + | > step_time: 5.89290 (2.94911) + | > loader_time: 0.00460 (0.05562) + + + --> STEP: 202/234 -- GLOBAL_STEP: 42790 + | > loss: -0.41485 (-0.21429) + | > log_mle: -0.63707 (-0.37584) + | > loss_dur: 0.22222 (0.16155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.73294 (33.10746) + | > current_lr: 0.00005 + | > step_time: 1.70470 (2.96335) + | > loader_time: 0.00440 (0.05699) + + + --> STEP: 207/234 -- GLOBAL_STEP: 42795 + | > loss: -0.38945 (-0.21722) + | > log_mle: -0.62338 (-0.38045) + | > loss_dur: 0.23392 (0.16323) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.04687 (34.02980) + | > current_lr: 0.00005 + | > step_time: 4.40190 (2.98153) + | > loader_time: 0.09740 (0.05802) + + + --> STEP: 212/234 -- GLOBAL_STEP: 42800 + | > loss: -0.36476 (-0.22052) + | > log_mle: -0.60457 (-0.38565) + | > loss_dur: 0.23981 (0.16513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.49063 (35.40522) + | > current_lr: 0.00005 + | > step_time: 4.60090 (3.01035) + | > loader_time: 0.00540 (0.05853) + + + --> STEP: 217/234 -- GLOBAL_STEP: 42805 + | > loss: -0.39116 (-0.22406) + | > log_mle: -0.63096 (-0.39093) + | > loss_dur: 0.23980 (0.16687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.47301 (36.62196) + | > current_lr: 0.00005 + | > step_time: 4.10050 (3.04882) + | > loader_time: 0.29570 (0.05866) + + + --> STEP: 222/234 -- GLOBAL_STEP: 42810 + | > loss: -0.36387 (-0.22749) + | > log_mle: -0.63182 (-0.39605) + | > loss_dur: 0.26795 (0.16857) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 132.66663 (38.15280) + | > current_lr: 0.00005 + | > step_time: 3.20000 (3.13020) + | > loader_time: 0.19040 (0.05990) + + + --> STEP: 227/234 -- GLOBAL_STEP: 42815 + | > loss: -0.34725 (-0.23096) + | > log_mle: -0.60632 (-0.40141) + | > loss_dur: 0.25908 (0.17045) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.19822 (39.82566) + | > current_lr: 0.00005 + | > step_time: 0.24590 (3.06826) + | > loader_time: 0.00360 (0.05902) + + + --> STEP: 232/234 -- GLOBAL_STEP: 42820 + | > loss: -0.32196 (-0.23355) + | > log_mle: -0.80546 (-0.40795) + | > loss_dur: 0.48350 (0.17440) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 142.26289 (41.56393) + | > current_lr: 0.00005 + | > step_time: 0.33470 (3.00795) + | > loader_time: 0.00610 (0.05785) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.87975 (+0.72974) + | > avg_loss: -0.26558 (+0.00097) + | > avg_log_mle: -0.48990 (-0.00736) + | > avg_loss_dur: 0.22433 (+0.00834) + + + > EPOCH: 183/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 23:14:31)  + + --> STEP: 3/234 -- GLOBAL_STEP: 42825 + | > loss: -0.16025 (-0.17770) + | > log_mle: -0.29821 (-0.30256) + | > loss_dur: 0.13796 (0.12487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.25303 (20.06476) + | > current_lr: 0.00005 + | > step_time: 2.19210 (3.20345) + | > loader_time: 0.00120 (0.06294) + + + --> STEP: 8/234 -- GLOBAL_STEP: 42830 + | > loss: -0.21888 (-0.19322) + | > log_mle: -0.31816 (-0.30372) + | > loss_dur: 0.09928 (0.11050) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.23814 (16.86966) + | > current_lr: 0.00005 + | > step_time: 2.40750 (2.41764) + | > loader_time: 0.00240 (0.04628) + + + --> STEP: 13/234 -- GLOBAL_STEP: 42835 + | > loss: -0.20779 (-0.19051) + | > log_mle: -0.30860 (-0.30413) + | > loss_dur: 0.10081 (0.11362) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.50035 (15.83628) + | > current_lr: 0.00005 + | > step_time: 3.80530 (3.02490) + | > loader_time: 0.09340 (0.07469) + + + --> STEP: 18/234 -- GLOBAL_STEP: 42840 + | > loss: -0.18110 (-0.19231) + | > log_mle: -0.29656 (-0.30232) + | > loss_dur: 0.11545 (0.11002) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.56891 (14.94657) + | > current_lr: 0.00005 + | > step_time: 4.00190 (3.48380) + | > loader_time: 0.01220 (0.06503) + + + --> STEP: 23/234 -- GLOBAL_STEP: 42845 + | > loss: -0.20992 (-0.19391) + | > log_mle: -0.30464 (-0.30066) + | > loss_dur: 0.09472 (0.10676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.95057 (14.03499) + | > current_lr: 0.00005 + | > step_time: 11.29750 (4.11427) + | > loader_time: 0.10140 (0.06757) + + + --> STEP: 28/234 -- GLOBAL_STEP: 42850 + | > loss: -0.23863 (-0.19501) + | > log_mle: -0.30764 (-0.29966) + | > loss_dur: 0.06901 (0.10464) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.27312 (13.37001) + | > current_lr: 0.00005 + | > step_time: 2.70670 (4.17600) + | > loader_time: 0.00160 (0.06950) + + + --> STEP: 33/234 -- GLOBAL_STEP: 42855 + | > loss: -0.19357 (-0.19359) + | > log_mle: -0.28911 (-0.29848) + | > loss_dur: 0.09554 (0.10489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.36909 (12.84500) + | > current_lr: 0.00005 + | > step_time: 3.19720 (3.93661) + | > loader_time: 0.10450 (0.06532) + + + --> STEP: 38/234 -- GLOBAL_STEP: 42860 + | > loss: -0.18493 (-0.19140) + | > log_mle: -0.30200 (-0.29748) + | > loss_dur: 0.11706 (0.10608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.48475 (12.98475) + | > current_lr: 0.00005 + | > step_time: 1.70000 (3.68129) + | > loader_time: 0.08320 (0.06393) + + + --> STEP: 43/234 -- GLOBAL_STEP: 42865 + | > loss: -0.16633 (-0.18934) + | > log_mle: -0.29931 (-0.29645) + | > loss_dur: 0.13298 (0.10711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.28460 (12.90270) + | > current_lr: 0.00005 + | > step_time: 3.90780 (3.72298) + | > loader_time: 0.18970 (0.06527) + + + --> STEP: 48/234 -- GLOBAL_STEP: 42870 + | > loss: -0.18599 (-0.18841) + | > log_mle: -0.28583 (-0.29593) + | > loss_dur: 0.09984 (0.10752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.67868 (12.62511) + | > current_lr: 0.00005 + | > step_time: 1.09990 (3.66654) + | > loader_time: 0.08750 (0.06457) + + + --> STEP: 53/234 -- GLOBAL_STEP: 42875 + | > loss: -0.17011 (-0.18684) + | > log_mle: -0.29182 (-0.29511) + | > loss_dur: 0.12170 (0.10827) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.35202 (12.45777) + | > current_lr: 0.00005 + | > step_time: 1.39270 (3.49380) + | > loader_time: 0.00170 (0.05886) + + + --> STEP: 58/234 -- GLOBAL_STEP: 42880 + | > loss: -0.17280 (-0.18541) + | > log_mle: -0.28298 (-0.29455) + | > loss_dur: 0.11019 (0.10914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.72507 (12.28612) + | > current_lr: 0.00005 + | > step_time: 2.21460 (3.33779) + | > loader_time: 0.00170 (0.05398) + + + --> STEP: 63/234 -- GLOBAL_STEP: 42885 + | > loss: -0.14643 (-0.18353) + | > log_mle: -0.28585 (-0.29512) + | > loss_dur: 0.13943 (0.11159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.17693 (12.43979) + | > current_lr: 0.00005 + | > step_time: 1.71070 (3.19514) + | > loader_time: 0.00480 (0.04993) + + + --> STEP: 68/234 -- GLOBAL_STEP: 42890 + | > loss: -0.13545 (-0.18209) + | > log_mle: -0.28210 (-0.29460) + | > loss_dur: 0.14665 (0.11251) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.48372 (12.22309) + | > current_lr: 0.00005 + | > step_time: 1.99420 (3.13838) + | > loader_time: 0.00240 (0.04791) + + + --> STEP: 73/234 -- GLOBAL_STEP: 42895 + | > loss: -0.14066 (-0.17979) + | > log_mle: -0.30012 (-0.29422) + | > loss_dur: 0.15945 (0.11443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.03482 (12.29476) + | > current_lr: 0.00005 + | > step_time: 1.49450 (3.01160) + | > loader_time: 0.00520 (0.04602) + + + --> STEP: 78/234 -- GLOBAL_STEP: 42900 + | > loss: -0.14797 (-0.17845) + | > log_mle: -0.27796 (-0.29383) + | > loss_dur: 0.12999 (0.11538) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.92919 (12.26633) + | > current_lr: 0.00005 + | > step_time: 1.65970 (2.95053) + | > loader_time: 0.00260 (0.04424) + + + --> STEP: 83/234 -- GLOBAL_STEP: 42905 + | > loss: -0.14652 (-0.17738) + | > log_mle: -0.29960 (-0.29383) + | > loss_dur: 0.15307 (0.11644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.17522 (12.27901) + | > current_lr: 0.00005 + | > step_time: 3.01690 (2.87294) + | > loader_time: 0.00210 (0.04355) + + + --> STEP: 88/234 -- GLOBAL_STEP: 42910 + | > loss: -0.17972 (-0.17686) + | > log_mle: -0.33653 (-0.29429) + | > loss_dur: 0.15681 (0.11743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.47187 (12.35468) + | > current_lr: 0.00005 + | > step_time: 1.89380 (2.81769) + | > loader_time: 0.00220 (0.04345) + + + --> STEP: 93/234 -- GLOBAL_STEP: 42915 + | > loss: -0.18942 (-0.17680) + | > log_mle: -0.35053 (-0.29606) + | > loss_dur: 0.16111 (0.11926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.51304 (12.70105) + | > current_lr: 0.00005 + | > step_time: 1.89850 (2.75618) + | > loader_time: 0.00360 (0.04213) + + + --> STEP: 98/234 -- GLOBAL_STEP: 42920 + | > loss: -0.14803 (-0.17708) + | > log_mle: -0.28288 (-0.29795) + | > loss_dur: 0.13485 (0.12087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.26610 (12.97541) + | > current_lr: 0.00005 + | > step_time: 1.39460 (2.69001) + | > loader_time: 0.00270 (0.04107) + + + --> STEP: 103/234 -- GLOBAL_STEP: 42925 + | > loss: -0.20316 (-0.17742) + | > log_mle: -0.38227 (-0.30065) + | > loss_dur: 0.17911 (0.12322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.57767 (13.57085) + | > current_lr: 0.00005 + | > step_time: 2.50250 (2.69634) + | > loader_time: 0.08760 (0.04352) + + + --> STEP: 108/234 -- GLOBAL_STEP: 42930 + | > loss: -0.17918 (-0.17791) + | > log_mle: -0.32501 (-0.30282) + | > loss_dur: 0.14583 (0.12492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76870 (14.08563) + | > current_lr: 0.00005 + | > step_time: 2.59080 (2.65407) + | > loader_time: 0.00850 (0.04313) + + + --> STEP: 113/234 -- GLOBAL_STEP: 42935 + | > loss: -0.19866 (-0.17813) + | > log_mle: -0.37157 (-0.30568) + | > loss_dur: 0.17291 (0.12755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.56093 (14.85920) + | > current_lr: 0.00005 + | > step_time: 1.79840 (2.63939) + | > loader_time: 0.00230 (0.04213) + + + --> STEP: 118/234 -- GLOBAL_STEP: 42940 + | > loss: -0.16630 (-0.17805) + | > log_mle: -0.34146 (-0.30770) + | > loss_dur: 0.17516 (0.12964) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.84856 (15.35495) + | > current_lr: 0.00005 + | > step_time: 1.49130 (2.59588) + | > loader_time: 0.00230 (0.04047) + + + --> STEP: 123/234 -- GLOBAL_STEP: 42945 + | > loss: -0.15497 (-0.17765) + | > log_mle: -0.31109 (-0.30879) + | > loss_dur: 0.15612 (0.13114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.78864 (15.59670) + | > current_lr: 0.00005 + | > step_time: 1.97890 (2.60429) + | > loader_time: 0.00260 (0.04106) + + + --> STEP: 128/234 -- GLOBAL_STEP: 42950 + | > loss: -0.21859 (-0.17912) + | > log_mle: -0.37113 (-0.31200) + | > loss_dur: 0.15253 (0.13288) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.15381 (16.26907) + | > current_lr: 0.00005 + | > step_time: 1.19820 (2.58117) + | > loader_time: 0.00240 (0.04026) + + + --> STEP: 133/234 -- GLOBAL_STEP: 42955 + | > loss: -0.21843 (-0.18075) + | > log_mle: -0.39627 (-0.31540) + | > loss_dur: 0.17784 (0.13465) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.86335 (16.95718) + | > current_lr: 0.00005 + | > step_time: 2.99730 (2.58338) + | > loader_time: 0.09340 (0.04064) + + + --> STEP: 138/234 -- GLOBAL_STEP: 42960 + | > loss: -0.17228 (-0.18179) + | > log_mle: -0.34928 (-0.31844) + | > loss_dur: 0.17701 (0.13665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.46501 (17.78867) + | > current_lr: 0.00005 + | > step_time: 1.30350 (2.60196) + | > loader_time: 0.00310 (0.03998) + + + --> STEP: 143/234 -- GLOBAL_STEP: 42965 + | > loss: -0.26269 (-0.18357) + | > log_mle: -0.50244 (-0.32245) + | > loss_dur: 0.23975 (0.13888) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.25726 (18.69564) + | > current_lr: 0.00005 + | > step_time: 1.78260 (2.59152) + | > loader_time: 0.00220 (0.03931) + + + --> STEP: 148/234 -- GLOBAL_STEP: 42970 + | > loss: -0.23173 (-0.18563) + | > log_mle: -0.40469 (-0.32633) + | > loss_dur: 0.17296 (0.14070) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.07995 (19.68469) + | > current_lr: 0.00005 + | > step_time: 5.20810 (2.61162) + | > loader_time: 0.09920 (0.03934) + + + --> STEP: 153/234 -- GLOBAL_STEP: 42975 + | > loss: -0.33768 (-0.18873) + | > log_mle: -0.53919 (-0.33140) + | > loss_dur: 0.20151 (0.14268) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.01913 (21.08824) + | > current_lr: 0.00005 + | > step_time: 1.80380 (2.65309) + | > loader_time: 0.00290 (0.03871) + + + --> STEP: 158/234 -- GLOBAL_STEP: 42980 + | > loss: -0.25583 (-0.19123) + | > log_mle: -0.47304 (-0.33589) + | > loss_dur: 0.21721 (0.14467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.06411 (22.31547) + | > current_lr: 0.00005 + | > step_time: 1.51340 (2.66041) + | > loader_time: 0.08230 (0.04050) + + + --> STEP: 163/234 -- GLOBAL_STEP: 42985 + | > loss: -0.24157 (-0.19392) + | > log_mle: -0.43896 (-0.34038) + | > loss_dur: 0.19739 (0.14646) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.66649 (23.33607) + | > current_lr: 0.00005 + | > step_time: 4.01170 (2.71079) + | > loader_time: 0.08720 (0.04144) + + + --> STEP: 168/234 -- GLOBAL_STEP: 42990 + | > loss: -0.26414 (-0.19662) + | > log_mle: -0.49687 (-0.34499) + | > loss_dur: 0.23273 (0.14837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.29006 (24.65274) + | > current_lr: 0.00005 + | > step_time: 3.20840 (2.72139) + | > loader_time: 0.09430 (0.04257) + + + --> STEP: 173/234 -- GLOBAL_STEP: 42995 + | > loss: -0.29486 (-0.19989) + | > log_mle: -0.50931 (-0.35031) + | > loss_dur: 0.21446 (0.15042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.22765 (25.92720) + | > current_lr: 0.00005 + | > step_time: 2.79100 (2.77108) + | > loader_time: 0.00820 (0.04602) + + + --> STEP: 178/234 -- GLOBAL_STEP: 43000 + | > loss: -0.32250 (-0.20306) + | > log_mle: -0.56211 (-0.35553) + | > loss_dur: 0.23961 (0.15247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.70638 (27.19383) + | > current_lr: 0.00005 + | > step_time: 0.99950 (2.80963) + | > loader_time: 0.00420 (0.04698) + + + --> STEP: 183/234 -- GLOBAL_STEP: 43005 + | > loss: -0.35001 (-0.20586) + | > log_mle: -0.56838 (-0.36042) + | > loss_dur: 0.21837 (0.15455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.81126 (28.31422) + | > current_lr: 0.00005 + | > step_time: 4.30110 (2.84214) + | > loader_time: 0.09130 (0.04722) + + + --> STEP: 188/234 -- GLOBAL_STEP: 43010 + | > loss: -0.35369 (-0.20872) + | > log_mle: -0.57675 (-0.36523) + | > loss_dur: 0.22306 (0.15652) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.73509 (29.66141) + | > current_lr: 0.00005 + | > step_time: 1.61240 (2.81355) + | > loader_time: 0.08300 (0.04733) + + + --> STEP: 193/234 -- GLOBAL_STEP: 43015 + | > loss: -0.35152 (-0.21185) + | > log_mle: -0.57164 (-0.36994) + | > loss_dur: 0.22012 (0.15809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.91090 (30.78051) + | > current_lr: 0.00005 + | > step_time: 2.39960 (2.91993) + | > loader_time: 0.09380 (0.04763) + + + --> STEP: 198/234 -- GLOBAL_STEP: 43020 + | > loss: -0.34071 (-0.21473) + | > log_mle: -0.57220 (-0.37450) + | > loss_dur: 0.23149 (0.15977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.43697 (31.85408) + | > current_lr: 0.00005 + | > step_time: 6.20980 (2.98132) + | > loader_time: 0.09480 (0.04873) + + + --> STEP: 203/234 -- GLOBAL_STEP: 43025 + | > loss: -0.27969 (-0.21736) + | > log_mle: -0.49479 (-0.37888) + | > loss_dur: 0.21509 (0.16152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.39607 (32.94957) + | > current_lr: 0.00005 + | > step_time: 2.80000 (3.02319) + | > loader_time: 0.00480 (0.04948) + + + --> STEP: 208/234 -- GLOBAL_STEP: 43030 + | > loss: -0.32837 (-0.22043) + | > log_mle: -0.57473 (-0.38381) + | > loss_dur: 0.24636 (0.16338) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.29962 (34.49345) + | > current_lr: 0.00005 + | > step_time: 1.10880 (3.10928) + | > loader_time: 0.08820 (0.05150) + + + --> STEP: 213/234 -- GLOBAL_STEP: 43035 + | > loss: -0.38181 (-0.22404) + | > log_mle: -0.62925 (-0.38928) + | > loss_dur: 0.24745 (0.16524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.92741 (36.03579) + | > current_lr: 0.00005 + | > step_time: 4.19220 (3.10761) + | > loader_time: 0.00250 (0.05079) + + + --> STEP: 218/234 -- GLOBAL_STEP: 43040 + | > loss: -0.36029 (-0.22748) + | > log_mle: -0.59714 (-0.39436) + | > loss_dur: 0.23684 (0.16688) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.98726 (37.15385) + | > current_lr: 0.00005 + | > step_time: 3.79140 (3.13712) + | > loader_time: 0.00860 (0.05387) + + + --> STEP: 223/234 -- GLOBAL_STEP: 43045 + | > loss: -0.36137 (-0.23081) + | > log_mle: -0.62673 (-0.39956) + | > loss_dur: 0.26537 (0.16875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 117.48129 (38.66882) + | > current_lr: 0.00005 + | > step_time: 1.58700 (3.11512) + | > loader_time: 0.00380 (0.05348) + + + --> STEP: 228/234 -- GLOBAL_STEP: 43050 + | > loss: -0.34580 (-0.23406) + | > log_mle: -0.62779 (-0.40485) + | > loss_dur: 0.28200 (0.17079) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.55652 (40.15749) + | > current_lr: 0.00005 + | > step_time: 0.25340 (3.06322) + | > loader_time: 0.00430 (0.05310) + + + --> STEP: 233/234 -- GLOBAL_STEP: 43055 + | > loss: 0.12849 (-0.23479) + | > log_mle: -0.60710 (-0.41158) + | > loss_dur: 0.73559 (0.17679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.20135 (41.72465) + | > current_lr: 0.00005 + | > step_time: 0.19560 (3.00312) + | > loader_time: 0.00350 (0.05206) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27509 (-0.60466) + | > avg_loss: -0.23689 (+0.02869) + | > avg_log_mle: -0.47203 (+0.01787) + | > avg_loss_dur: 0.23514 (+0.01081) + + + > EPOCH: 184/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 23:27:39)  + + --> STEP: 4/234 -- GLOBAL_STEP: 43060 + | > loss: -0.18799 (-0.18685) + | > log_mle: -0.30227 (-0.30426) + | > loss_dur: 0.11428 (0.11741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.62277 (16.86444) + | > current_lr: 0.00005 + | > step_time: 2.18940 (6.47481) + | > loader_time: 0.00100 (1.57041) + + + --> STEP: 9/234 -- GLOBAL_STEP: 43065 + | > loss: -0.16789 (-0.19528) + | > log_mle: -0.30924 (-0.30640) + | > loss_dur: 0.14135 (0.11112) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.70851 (15.72397) + | > current_lr: 0.00005 + | > step_time: 1.88910 (5.31993) + | > loader_time: 0.09910 (0.75401) + + + --> STEP: 14/234 -- GLOBAL_STEP: 43070 + | > loss: -0.18917 (-0.19544) + | > log_mle: -0.30443 (-0.30600) + | > loss_dur: 0.11525 (0.11056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.04364 (14.96829) + | > current_lr: 0.00005 + | > step_time: 1.21750 (4.60296) + | > loader_time: 0.07230 (0.49763) + + + --> STEP: 19/234 -- GLOBAL_STEP: 43075 + | > loss: -0.21135 (-0.19659) + | > log_mle: -0.29565 (-0.30329) + | > loss_dur: 0.08430 (0.10670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01398 (14.35349) + | > current_lr: 0.00005 + | > step_time: 4.42070 (4.07283) + | > loader_time: 0.08670 (0.37621) + + + --> STEP: 24/234 -- GLOBAL_STEP: 43080 + | > loss: -0.22496 (-0.19708) + | > log_mle: -0.29583 (-0.30149) + | > loss_dur: 0.07087 (0.10442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.20774 (14.16140) + | > current_lr: 0.00005 + | > step_time: 2.91790 (3.55389) + | > loader_time: 0.00470 (0.29836) + + + --> STEP: 29/234 -- GLOBAL_STEP: 43085 + | > loss: -0.16318 (-0.19594) + | > log_mle: -0.28091 (-0.29996) + | > loss_dur: 0.11773 (0.10402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.53476 (13.59339) + | > current_lr: 0.00005 + | > step_time: 2.19390 (3.22830) + | > loader_time: 0.00370 (0.25503) + + + --> STEP: 34/234 -- GLOBAL_STEP: 43090 + | > loss: -0.17481 (-0.19427) + | > log_mle: -0.29082 (-0.29918) + | > loss_dur: 0.11601 (0.10491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.00922 (13.17848) + | > current_lr: 0.00005 + | > step_time: 5.60270 (3.52470) + | > loader_time: 0.10670 (0.22576) + + + --> STEP: 39/234 -- GLOBAL_STEP: 43095 + | > loss: -0.17053 (-0.19222) + | > log_mle: -0.29801 (-0.29858) + | > loss_dur: 0.12748 (0.10636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.09511 (13.19850) + | > current_lr: 0.00005 + | > step_time: 1.50780 (3.25035) + | > loader_time: 0.00230 (0.19722) + + + --> STEP: 44/234 -- GLOBAL_STEP: 43100 + | > loss: -0.20168 (-0.19077) + | > log_mle: -0.28851 (-0.29747) + | > loss_dur: 0.08682 (0.10669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.29946 (12.76646) + | > current_lr: 0.00005 + | > step_time: 0.89990 (3.01729) + | > loader_time: 0.00180 (0.17699) + + + --> STEP: 49/234 -- GLOBAL_STEP: 43105 + | > loss: -0.19942 (-0.18977) + | > log_mle: -0.30014 (-0.29736) + | > loss_dur: 0.10073 (0.10758) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.77825 (12.52333) + | > current_lr: 0.00005 + | > step_time: 2.16540 (2.87955) + | > loader_time: 0.01490 (0.16116) + + + --> STEP: 54/234 -- GLOBAL_STEP: 43110 + | > loss: -0.19369 (-0.18843) + | > log_mle: -0.30203 (-0.29677) + | > loss_dur: 0.10834 (0.10834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.61665 (12.22531) + | > current_lr: 0.00005 + | > step_time: 0.91630 (2.77853) + | > loader_time: 0.07510 (0.15070) + + + --> STEP: 59/234 -- GLOBAL_STEP: 43115 + | > loss: -0.19876 (-0.18787) + | > log_mle: -0.30657 (-0.29657) + | > loss_dur: 0.10780 (0.10870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.80329 (12.02639) + | > current_lr: 0.00005 + | > step_time: 2.79800 (2.69205) + | > loader_time: 0.00210 (0.13812) + + + --> STEP: 64/234 -- GLOBAL_STEP: 43120 + | > loss: -0.18355 (-0.18590) + | > log_mle: -0.28959 (-0.29703) + | > loss_dur: 0.10604 (0.11113) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.94151 (12.10007) + | > current_lr: 0.00005 + | > step_time: 2.02150 (2.66508) + | > loader_time: 0.08240 (0.13172) + + + --> STEP: 69/234 -- GLOBAL_STEP: 43125 + | > loss: -0.16117 (-0.18403) + | > log_mle: -0.27460 (-0.29628) + | > loss_dur: 0.11343 (0.11225) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.08437 (11.99835) + | > current_lr: 0.00005 + | > step_time: 1.32750 (2.60769) + | > loader_time: 0.00240 (0.12372) + + + --> STEP: 74/234 -- GLOBAL_STEP: 43130 + | > loss: -0.15509 (-0.18202) + | > log_mle: -0.28272 (-0.29617) + | > loss_dur: 0.12763 (0.11415) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.19430 (12.14209) + | > current_lr: 0.00005 + | > step_time: 2.35560 (2.60480) + | > loader_time: 0.00620 (0.11671) + + + --> STEP: 79/234 -- GLOBAL_STEP: 43135 + | > loss: -0.16215 (-0.18092) + | > log_mle: -0.30060 (-0.29630) + | > loss_dur: 0.13845 (0.11538) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.38652 (12.05806) + | > current_lr: 0.00005 + | > step_time: 2.90130 (2.57903) + | > loader_time: 0.09390 (0.11062) + + + --> STEP: 84/234 -- GLOBAL_STEP: 43140 + | > loss: -0.16402 (-0.17980) + | > log_mle: -0.29231 (-0.29626) + | > loss_dur: 0.12830 (0.11646) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.26393 (12.11907) + | > current_lr: 0.00005 + | > step_time: 4.02780 (2.57943) + | > loader_time: 0.00370 (0.10756) + + + --> STEP: 89/234 -- GLOBAL_STEP: 43145 + | > loss: -0.17804 (-0.17934) + | > log_mle: -0.31981 (-0.29708) + | > loss_dur: 0.14177 (0.11774) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.55890 (12.28782) + | > current_lr: 0.00005 + | > step_time: 1.06020 (2.54115) + | > loader_time: 0.00240 (0.10525) + + + --> STEP: 94/234 -- GLOBAL_STEP: 43150 + | > loss: -0.21240 (-0.17948) + | > log_mle: -0.35358 (-0.29921) + | > loss_dur: 0.14118 (0.11974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.20859 (12.66290) + | > current_lr: 0.00005 + | > step_time: 4.20730 (2.54029) + | > loader_time: 0.19380 (0.10467) + + + --> STEP: 99/234 -- GLOBAL_STEP: 43155 + | > loss: -0.21283 (-0.17994) + | > log_mle: -0.38983 (-0.30139) + | > loss_dur: 0.17700 (0.12144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.49790 (13.11256) + | > current_lr: 0.00005 + | > step_time: 2.50490 (2.53163) + | > loader_time: 0.00230 (0.10046) + + + --> STEP: 104/234 -- GLOBAL_STEP: 43160 + | > loss: -0.23678 (-0.18044) + | > log_mle: -0.40035 (-0.30407) + | > loss_dur: 0.16356 (0.12363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.60819 (13.84833) + | > current_lr: 0.00005 + | > step_time: 1.49400 (2.51083) + | > loader_time: 0.00350 (0.09606) + + + --> STEP: 109/234 -- GLOBAL_STEP: 43165 + | > loss: -0.15987 (-0.18005) + | > log_mle: -0.37069 (-0.30588) + | > loss_dur: 0.21082 (0.12583) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.03669 (14.41670) + | > current_lr: 0.00005 + | > step_time: 3.40200 (2.51227) + | > loader_time: 0.00390 (0.09332) + + + --> STEP: 114/234 -- GLOBAL_STEP: 43170 + | > loss: -0.19147 (-0.18049) + | > log_mle: -0.35327 (-0.30853) + | > loss_dur: 0.16179 (0.12805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.55688 (15.20434) + | > current_lr: 0.00005 + | > step_time: 1.70030 (2.53324) + | > loader_time: 0.00220 (0.09096) + + + --> STEP: 119/234 -- GLOBAL_STEP: 43175 + | > loss: -0.18726 (-0.18056) + | > log_mle: -0.35187 (-0.31061) + | > loss_dur: 0.16461 (0.13005) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.19054 (15.61238) + | > current_lr: 0.00005 + | > step_time: 3.10840 (2.51176) + | > loader_time: 0.00310 (0.08853) + + + --> STEP: 124/234 -- GLOBAL_STEP: 43180 + | > loss: -0.20181 (-0.18050) + | > log_mle: -0.37334 (-0.31190) + | > loss_dur: 0.17153 (0.13140) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.12011 (16.09741) + | > current_lr: 0.00005 + | > step_time: 1.10520 (2.49761) + | > loader_time: 0.00250 (0.08900) + + + --> STEP: 129/234 -- GLOBAL_STEP: 43185 + | > loss: -0.18316 (-0.18131) + | > log_mle: -0.36846 (-0.31473) + | > loss_dur: 0.18529 (0.13342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.74624 (17.07961) + | > current_lr: 0.00005 + | > step_time: 1.61430 (2.51155) + | > loader_time: 0.00180 (0.08778) + + + --> STEP: 134/234 -- GLOBAL_STEP: 43190 + | > loss: -0.20574 (-0.18292) + | > log_mle: -0.41495 (-0.31827) + | > loss_dur: 0.20921 (0.13535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.38086 (18.32686) + | > current_lr: 0.00005 + | > step_time: 4.51600 (2.52711) + | > loader_time: 0.10590 (0.08726) + + + --> STEP: 139/234 -- GLOBAL_STEP: 43195 + | > loss: -0.28058 (-0.18423) + | > log_mle: -0.47754 (-0.32156) + | > loss_dur: 0.19695 (0.13733) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.88088 (19.07831) + | > current_lr: 0.00005 + | > step_time: 1.41130 (2.50105) + | > loader_time: 0.00320 (0.08487) + + + --> STEP: 144/234 -- GLOBAL_STEP: 43200 + | > loss: -0.25223 (-0.18556) + | > log_mle: -0.44941 (-0.32505) + | > loss_dur: 0.19718 (0.13949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.47759 (19.99706) + | > current_lr: 0.00005 + | > step_time: 3.99910 (2.51862) + | > loader_time: 0.18720 (0.08394) + + + --> STEP: 149/234 -- GLOBAL_STEP: 43205 + | > loss: -0.29392 (-0.18800) + | > log_mle: -0.50628 (-0.32924) + | > loss_dur: 0.21236 (0.14124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.34782 (20.86856) + | > current_lr: 0.00005 + | > step_time: 1.98540 (2.53889) + | > loader_time: 0.00200 (0.08347) + + + --> STEP: 154/234 -- GLOBAL_STEP: 43210 + | > loss: -0.26280 (-0.19079) + | > log_mle: -0.45967 (-0.33397) + | > loss_dur: 0.19687 (0.14318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.82072 (22.01626) + | > current_lr: 0.00005 + | > step_time: 3.30420 (2.53956) + | > loader_time: 0.19830 (0.08218) + + + --> STEP: 159/234 -- GLOBAL_STEP: 43215 + | > loss: -0.27815 (-0.19334) + | > log_mle: -0.48610 (-0.33848) + | > loss_dur: 0.20794 (0.14514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.27744 (23.06225) + | > current_lr: 0.00005 + | > step_time: 7.00340 (2.58427) + | > loader_time: 0.00360 (0.08078) + + + --> STEP: 164/234 -- GLOBAL_STEP: 43220 + | > loss: -0.26844 (-0.19592) + | > log_mle: -0.48147 (-0.34289) + | > loss_dur: 0.21303 (0.14696) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.42168 (24.00454) + | > current_lr: 0.00005 + | > step_time: 2.81110 (2.64766) + | > loader_time: 0.08610 (0.08009) + + + --> STEP: 169/234 -- GLOBAL_STEP: 43225 + | > loss: -0.25883 (-0.19858) + | > log_mle: -0.47420 (-0.34741) + | > loss_dur: 0.21537 (0.14883) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.36463 (25.23296) + | > current_lr: 0.00005 + | > step_time: 4.50800 (2.67257) + | > loader_time: 0.28850 (0.08154) + + + --> STEP: 174/234 -- GLOBAL_STEP: 43230 + | > loss: -0.31397 (-0.20169) + | > log_mle: -0.52921 (-0.35250) + | > loss_dur: 0.21524 (0.15081) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.70081 (27.32292) + | > current_lr: 0.00005 + | > step_time: 7.60860 (2.80329) + | > loader_time: 0.00440 (0.07990) + + + --> STEP: 179/234 -- GLOBAL_STEP: 43235 + | > loss: -0.29326 (-0.20382) + | > log_mle: -0.54298 (-0.35696) + | > loss_dur: 0.24972 (0.15314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.34821 (28.30976) + | > current_lr: 0.00005 + | > step_time: 7.89690 (2.91931) + | > loader_time: 0.00260 (0.07997) + + + --> STEP: 184/234 -- GLOBAL_STEP: 43240 + | > loss: -0.28408 (-0.20601) + | > log_mle: -0.51144 (-0.36112) + | > loss_dur: 0.22736 (0.15510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.06921 (29.42131) + | > current_lr: 0.00005 + | > step_time: 2.00160 (2.94225) + | > loader_time: 0.00280 (0.07975) + + + --> STEP: 189/234 -- GLOBAL_STEP: 43245 + | > loss: -0.28600 (-0.20856) + | > log_mle: -0.51145 (-0.36563) + | > loss_dur: 0.22545 (0.15707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.41647 (30.34505) + | > current_lr: 0.00005 + | > step_time: 7.19680 (2.97921) + | > loader_time: 0.09980 (0.07975) + + + --> STEP: 194/234 -- GLOBAL_STEP: 43250 + | > loss: -0.33006 (-0.21168) + | > log_mle: -0.55025 (-0.37027) + | > loss_dur: 0.22019 (0.15859) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.02245 (31.31699) + | > current_lr: 0.00005 + | > step_time: 4.00840 (3.04374) + | > loader_time: 0.09380 (0.08021) + + + --> STEP: 199/234 -- GLOBAL_STEP: 43255 + | > loss: -0.33204 (-0.21439) + | > log_mle: -0.56643 (-0.37465) + | > loss_dur: 0.23439 (0.16026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.72714 (32.45753) + | > current_lr: 0.00005 + | > step_time: 12.60490 (3.15637) + | > loader_time: 0.30370 (0.08262) + + + --> STEP: 204/234 -- GLOBAL_STEP: 43260 + | > loss: -0.34541 (-0.21670) + | > log_mle: -0.59149 (-0.37881) + | > loss_dur: 0.24608 (0.16211) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.38193 (33.52617) + | > current_lr: 0.00005 + | > step_time: 3.50130 (3.19909) + | > loader_time: 0.10520 (0.08218) + + + --> STEP: 209/234 -- GLOBAL_STEP: 43265 + | > loss: -0.32414 (-0.21957) + | > log_mle: -0.55148 (-0.38343) + | > loss_dur: 0.22734 (0.16386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.76404 (34.60545) + | > current_lr: 0.00005 + | > step_time: 4.40220 (3.24807) + | > loader_time: 0.10400 (0.08236) + + + --> STEP: 214/234 -- GLOBAL_STEP: 43270 + | > loss: -0.38661 (-0.22345) + | > log_mle: -0.59606 (-0.38903) + | > loss_dur: 0.20945 (0.16558) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.51237 (35.87791) + | > current_lr: 0.00005 + | > step_time: 3.19280 (3.26273) + | > loader_time: 0.19810 (0.08328) + + + --> STEP: 219/234 -- GLOBAL_STEP: 43275 + | > loss: -0.45417 (-0.22725) + | > log_mle: -0.69836 (-0.39457) + | > loss_dur: 0.24418 (0.16732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 117.15676 (37.15653) + | > current_lr: 0.00005 + | > step_time: 5.89370 (3.34622) + | > loader_time: 0.09680 (0.08358) + + + --> STEP: 224/234 -- GLOBAL_STEP: 43280 + | > loss: -0.40609 (-0.23065) + | > log_mle: -0.64535 (-0.39968) + | > loss_dur: 0.23926 (0.16903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.37717 (38.42928) + | > current_lr: 0.00005 + | > step_time: 0.25690 (3.31244) + | > loader_time: 0.00340 (0.08256) + + + --> STEP: 229/234 -- GLOBAL_STEP: 43285 + | > loss: -0.36755 (-0.23403) + | > log_mle: -0.66773 (-0.40522) + | > loss_dur: 0.30018 (0.17119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 173.88077 (40.21315) + | > current_lr: 0.00005 + | > step_time: 0.24600 (3.24573) + | > loader_time: 0.00370 (0.08085) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.41993 (+0.14484) + | > avg_loss: -0.26940 (-0.03250) + | > avg_log_mle: -0.49326 (-0.02123) + | > avg_loss_dur: 0.22387 (-0.01128) + + + > EPOCH: 185/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 23:41:26)  + + --> STEP: 0/234 -- GLOBAL_STEP: 43290 + | > loss: -0.23310 (-0.23310) + | > log_mle: -0.38149 (-0.38149) + | > loss_dur: 0.14839 (0.14839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.20659 (26.20659) + | > current_lr: 0.00005 + | > step_time: 12.99520 (12.99516) + | > loader_time: 16.39220 (16.39224) + + + --> STEP: 5/234 -- GLOBAL_STEP: 43295 + | > loss: -0.18449 (-0.18108) + | > log_mle: -0.30136 (-0.30398) + | > loss_dur: 0.11688 (0.12290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.94533 (19.43992) + | > current_lr: 0.00005 + | > step_time: 2.52420 (4.76100) + | > loader_time: 0.09130 (0.18131) + + + --> STEP: 10/234 -- GLOBAL_STEP: 43300 + | > loss: -0.17384 (-0.19170) + | > log_mle: -0.29853 (-0.30459) + | > loss_dur: 0.12469 (0.11289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.08197 (17.89817) + | > current_lr: 0.00005 + | > step_time: 0.27990 (2.76764) + | > loader_time: 0.00100 (0.09869) + + + --> STEP: 15/234 -- GLOBAL_STEP: 43305 + | > loss: -0.21043 (-0.19563) + | > log_mle: -0.30813 (-0.30540) + | > loss_dur: 0.09771 (0.10977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.56074 (15.95209) + | > current_lr: 0.00005 + | > step_time: 1.93550 (2.34795) + | > loader_time: 0.00120 (0.06632) + + + --> STEP: 20/234 -- GLOBAL_STEP: 43310 + | > loss: -0.20282 (-0.19677) + | > log_mle: -0.29972 (-0.30324) + | > loss_dur: 0.09691 (0.10646) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.40674 (14.31730) + | > current_lr: 0.00005 + | > step_time: 1.15390 (2.13109) + | > loader_time: 0.00330 (0.05035) + + + --> STEP: 25/234 -- GLOBAL_STEP: 43315 + | > loss: -0.19156 (-0.19706) + | > log_mle: -0.28547 (-0.30122) + | > loss_dur: 0.09391 (0.10416) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.32492 (13.58652) + | > current_lr: 0.00005 + | > step_time: 0.88970 (2.14032) + | > loader_time: 0.00200 (0.04770) + + + --> STEP: 30/234 -- GLOBAL_STEP: 43320 + | > loss: -0.20639 (-0.19740) + | > log_mle: -0.30107 (-0.30057) + | > loss_dur: 0.09468 (0.10318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.63874 (13.49328) + | > current_lr: 0.00005 + | > step_time: 4.40080 (2.44726) + | > loader_time: 0.09210 (0.05241) + + + --> STEP: 35/234 -- GLOBAL_STEP: 43325 + | > loss: -0.16828 (-0.19509) + | > log_mle: -0.29449 (-0.29989) + | > loss_dur: 0.12621 (0.10481) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05407 (13.26551) + | > current_lr: 0.00005 + | > step_time: 2.70490 (2.44619) + | > loader_time: 0.09370 (0.05035) + + + --> STEP: 40/234 -- GLOBAL_STEP: 43330 + | > loss: -0.15781 (-0.19315) + | > log_mle: -0.28038 (-0.29905) + | > loss_dur: 0.12257 (0.10590) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.75874 (13.10079) + | > current_lr: 0.00005 + | > step_time: 1.35510 (2.64410) + | > loader_time: 0.00210 (0.05143) + + + --> STEP: 45/234 -- GLOBAL_STEP: 43335 + | > loss: -0.17695 (-0.19235) + | > log_mle: -0.31209 (-0.29880) + | > loss_dur: 0.13515 (0.10645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.55462 (12.96800) + | > current_lr: 0.00005 + | > step_time: 1.10570 (2.47821) + | > loader_time: 0.00200 (0.04595) + + + --> STEP: 50/234 -- GLOBAL_STEP: 43340 + | > loss: -0.17660 (-0.19117) + | > log_mle: -0.28456 (-0.29791) + | > loss_dur: 0.10796 (0.10675) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.64575 (12.60626) + | > current_lr: 0.00005 + | > step_time: 1.06730 (2.42564) + | > loader_time: 0.00200 (0.04156) + + + --> STEP: 55/234 -- GLOBAL_STEP: 43345 + | > loss: -0.19739 (-0.19001) + | > log_mle: -0.30075 (-0.29748) + | > loss_dur: 0.10336 (0.10747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.72456 (12.58125) + | > current_lr: 0.00005 + | > step_time: 2.21150 (2.38514) + | > loader_time: 0.00200 (0.03804) + + + --> STEP: 60/234 -- GLOBAL_STEP: 43350 + | > loss: -0.17633 (-0.18857) + | > log_mle: -0.31255 (-0.29743) + | > loss_dur: 0.13622 (0.10886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.90777 (12.49161) + | > current_lr: 0.00005 + | > step_time: 2.30700 (2.35101) + | > loader_time: 0.00280 (0.03507) + + + --> STEP: 65/234 -- GLOBAL_STEP: 43355 + | > loss: -0.17237 (-0.18687) + | > log_mle: -0.29182 (-0.29747) + | > loss_dur: 0.11945 (0.11060) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.34908 (12.48879) + | > current_lr: 0.00005 + | > step_time: 1.07780 (2.33463) + | > loader_time: 0.00220 (0.03260) + + + --> STEP: 70/234 -- GLOBAL_STEP: 43360 + | > loss: -0.14031 (-0.18487) + | > log_mle: -0.28204 (-0.29667) + | > loss_dur: 0.14173 (0.11180) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.92162 (12.35955) + | > current_lr: 0.00005 + | > step_time: 2.37260 (2.31277) + | > loader_time: 0.00220 (0.03045) + + + --> STEP: 75/234 -- GLOBAL_STEP: 43365 + | > loss: -0.15561 (-0.18298) + | > log_mle: -0.30164 (-0.29678) + | > loss_dur: 0.14603 (0.11380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.42816 (12.48511) + | > current_lr: 0.00005 + | > step_time: 1.89340 (2.27008) + | > loader_time: 0.00320 (0.02860) + + + --> STEP: 80/234 -- GLOBAL_STEP: 43370 + | > loss: -0.17820 (-0.18224) + | > log_mle: -0.28407 (-0.29659) + | > loss_dur: 0.10587 (0.11434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.70954 (12.38870) + | > current_lr: 0.00005 + | > step_time: 2.90200 (2.30984) + | > loader_time: 0.08910 (0.02908) + + + --> STEP: 85/234 -- GLOBAL_STEP: 43375 + | > loss: -0.17591 (-0.18119) + | > log_mle: -0.29615 (-0.29671) + | > loss_dur: 0.12024 (0.11553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.36208 (12.37757) + | > current_lr: 0.00005 + | > step_time: 1.49300 (2.28481) + | > loader_time: 0.00370 (0.02950) + + + --> STEP: 90/234 -- GLOBAL_STEP: 43380 + | > loss: -0.16776 (-0.18061) + | > log_mle: -0.32086 (-0.29782) + | > loss_dur: 0.15310 (0.11721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.15903 (12.61390) + | > current_lr: 0.00005 + | > step_time: 2.40700 (2.31230) + | > loader_time: 0.00350 (0.02903) + + + --> STEP: 95/234 -- GLOBAL_STEP: 43385 + | > loss: -0.23390 (-0.18148) + | > log_mle: -0.41004 (-0.30096) + | > loss_dur: 0.17615 (0.11948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.14127 (13.13253) + | > current_lr: 0.00005 + | > step_time: 3.93360 (2.33092) + | > loader_time: 0.07800 (0.02929) + + + --> STEP: 100/234 -- GLOBAL_STEP: 43390 + | > loss: -0.18583 (-0.18143) + | > log_mle: -0.33177 (-0.30233) + | > loss_dur: 0.14593 (0.12090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76732 (13.39475) + | > current_lr: 0.00005 + | > step_time: 1.68260 (2.30900) + | > loader_time: 0.09080 (0.03038) + + + --> STEP: 105/234 -- GLOBAL_STEP: 43395 + | > loss: -0.17767 (-0.18181) + | > log_mle: -0.30953 (-0.30469) + | > loss_dur: 0.13186 (0.12287) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.71082 (14.16491) + | > current_lr: 0.00005 + | > step_time: 4.88120 (2.33583) + | > loader_time: 0.18790 (0.03173) + + + --> STEP: 110/234 -- GLOBAL_STEP: 43400 + | > loss: -0.16635 (-0.18133) + | > log_mle: -0.32927 (-0.30653) + | > loss_dur: 0.16292 (0.12519) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.85704 (14.75682) + | > current_lr: 0.00005 + | > step_time: 2.19160 (2.31636) + | > loader_time: 0.01480 (0.03125) + + + --> STEP: 115/234 -- GLOBAL_STEP: 43405 + | > loss: -0.17565 (-0.18168) + | > log_mle: -0.35291 (-0.30920) + | > loss_dur: 0.17726 (0.12751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.08365 (15.49505) + | > current_lr: 0.00005 + | > step_time: 1.19880 (2.30274) + | > loader_time: 0.00650 (0.03074) + + + --> STEP: 120/234 -- GLOBAL_STEP: 43410 + | > loss: -0.21815 (-0.18197) + | > log_mle: -0.39993 (-0.31153) + | > loss_dur: 0.18178 (0.12956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.91296 (16.01744) + | > current_lr: 0.00005 + | > step_time: 2.79540 (2.28726) + | > loader_time: 0.00300 (0.02958) + + + --> STEP: 125/234 -- GLOBAL_STEP: 43415 + | > loss: -0.20138 (-0.18166) + | > log_mle: -0.38660 (-0.31263) + | > loss_dur: 0.18521 (0.13097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.55163 (16.30368) + | > current_lr: 0.00005 + | > step_time: 2.60290 (2.31568) + | > loader_time: 0.00590 (0.03068) + + + --> STEP: 130/234 -- GLOBAL_STEP: 43420 + | > loss: -0.21684 (-0.18270) + | > log_mle: -0.39849 (-0.31569) + | > loss_dur: 0.18165 (0.13298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.20248 (17.10440) + | > current_lr: 0.00005 + | > step_time: 2.31610 (2.32107) + | > loader_time: 0.08400 (0.03095) + + + --> STEP: 135/234 -- GLOBAL_STEP: 43425 + | > loss: -0.17678 (-0.18419) + | > log_mle: -0.33173 (-0.31882) + | > loss_dur: 0.15495 (0.13463) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.97109 (17.81499) + | > current_lr: 0.00005 + | > step_time: 4.19620 (2.35062) + | > loader_time: 0.10350 (0.03129) + + + --> STEP: 140/234 -- GLOBAL_STEP: 43430 + | > loss: -0.17887 (-0.18575) + | > log_mle: -0.36460 (-0.32255) + | > loss_dur: 0.18573 (0.13680) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.09753 (18.71532) + | > current_lr: 0.00005 + | > step_time: 1.10270 (2.33957) + | > loader_time: 0.00930 (0.03094) + + + --> STEP: 145/234 -- GLOBAL_STEP: 43435 + | > loss: -0.27074 (-0.18788) + | > log_mle: -0.47562 (-0.32698) + | > loss_dur: 0.20489 (0.13910) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.10591 (19.55515) + | > current_lr: 0.00005 + | > step_time: 3.60370 (2.37492) + | > loader_time: 0.08250 (0.03244) + + + --> STEP: 150/234 -- GLOBAL_STEP: 43440 + | > loss: -0.25098 (-0.19022) + | > log_mle: -0.45439 (-0.33103) + | > loss_dur: 0.20341 (0.14081) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.19378 (20.61500) + | > current_lr: 0.00005 + | > step_time: 1.61330 (2.46989) + | > loader_time: 0.00320 (0.03330) + + + --> STEP: 155/234 -- GLOBAL_STEP: 43445 + | > loss: -0.31303 (-0.19351) + | > log_mle: -0.51406 (-0.33618) + | > loss_dur: 0.20103 (0.14267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.07218 (22.20398) + | > current_lr: 0.00005 + | > step_time: 2.79970 (2.46620) + | > loader_time: 0.00390 (0.03233) + + + --> STEP: 160/234 -- GLOBAL_STEP: 43450 + | > loss: -0.29995 (-0.19589) + | > log_mle: -0.51208 (-0.34059) + | > loss_dur: 0.21213 (0.14470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.13067 (23.32411) + | > current_lr: 0.00005 + | > step_time: 1.49140 (2.47397) + | > loader_time: 0.29710 (0.03329) + + + --> STEP: 165/234 -- GLOBAL_STEP: 43455 + | > loss: -0.29839 (-0.19831) + | > log_mle: -0.51583 (-0.34482) + | > loss_dur: 0.21744 (0.14650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.12585 (24.34566) + | > current_lr: 0.00005 + | > step_time: 1.79920 (2.51838) + | > loader_time: 0.08000 (0.03514) + + + --> STEP: 170/234 -- GLOBAL_STEP: 43460 + | > loss: -0.30718 (-0.20097) + | > log_mle: -0.54283 (-0.34940) + | > loss_dur: 0.23565 (0.14843) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.58201 (25.97774) + | > current_lr: 0.00005 + | > step_time: 1.99530 (2.52037) + | > loader_time: 0.00540 (0.03573) + + + --> STEP: 175/234 -- GLOBAL_STEP: 43465 + | > loss: -0.28163 (-0.20435) + | > log_mle: -0.51646 (-0.35473) + | > loss_dur: 0.23483 (0.15039) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.10600 (27.52797) + | > current_lr: 0.00005 + | > step_time: 2.99310 (2.55292) + | > loader_time: 0.00800 (0.03540) + + + --> STEP: 180/234 -- GLOBAL_STEP: 43470 + | > loss: -0.28517 (-0.20657) + | > log_mle: -0.51326 (-0.35919) + | > loss_dur: 0.22809 (0.15262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.08008 (28.67449) + | > current_lr: 0.00005 + | > step_time: 4.41030 (2.55017) + | > loader_time: 0.00270 (0.03453) + + + --> STEP: 185/234 -- GLOBAL_STEP: 43475 + | > loss: -0.30936 (-0.20887) + | > log_mle: -0.54843 (-0.36356) + | > loss_dur: 0.23907 (0.15469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.64854 (29.60736) + | > current_lr: 0.00005 + | > step_time: 4.10300 (2.62016) + | > loader_time: 0.08310 (0.03517) + + + --> STEP: 190/234 -- GLOBAL_STEP: 43480 + | > loss: -0.30880 (-0.21143) + | > log_mle: -0.52867 (-0.36801) + | > loss_dur: 0.21987 (0.15658) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.20387 (30.63665) + | > current_lr: 0.00005 + | > step_time: 3.59830 (2.63073) + | > loader_time: 0.00390 (0.03538) + + + --> STEP: 195/234 -- GLOBAL_STEP: 43485 + | > loss: -0.32138 (-0.21468) + | > log_mle: -0.55255 (-0.37285) + | > loss_dur: 0.23117 (0.15817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.13355 (31.67956) + | > current_lr: 0.00005 + | > step_time: 2.20450 (2.69760) + | > loader_time: 0.00490 (0.03513) + + + --> STEP: 200/234 -- GLOBAL_STEP: 43490 + | > loss: -0.30039 (-0.21727) + | > log_mle: -0.55627 (-0.37720) + | > loss_dur: 0.25588 (0.15993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.57552 (32.82775) + | > current_lr: 0.00005 + | > step_time: 10.89960 (2.80923) + | > loader_time: 0.10240 (0.03630) + + + --> STEP: 205/234 -- GLOBAL_STEP: 43495 + | > loss: -0.30733 (-0.21980) + | > log_mle: -0.54411 (-0.38146) + | > loss_dur: 0.23678 (0.16167) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.68858 (33.99561) + | > current_lr: 0.00005 + | > step_time: 4.49730 (2.85202) + | > loader_time: 0.10280 (0.03981) + + + --> STEP: 210/234 -- GLOBAL_STEP: 43500 + | > loss: -0.37621 (-0.22312) + | > log_mle: -0.61914 (-0.38652) + | > loss_dur: 0.24293 (0.16340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.01593 (35.08876) + | > current_lr: 0.00005 + | > step_time: 5.59510 (2.89651) + | > loader_time: 0.10510 (0.04168) + + + --> STEP: 215/234 -- GLOBAL_STEP: 43505 + | > loss: -0.33552 (-0.22653) + | > log_mle: -0.56851 (-0.39163) + | > loss_dur: 0.23298 (0.16510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.01582 (36.27809) + | > current_lr: 0.00005 + | > step_time: 5.90470 (2.98919) + | > loader_time: 0.00350 (0.05047) + + + --> STEP: 220/234 -- GLOBAL_STEP: 43510 + | > loss: -0.38394 (-0.23022) + | > log_mle: -0.62087 (-0.39710) + | > loss_dur: 0.23693 (0.16688) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.16433 (37.48859) + | > current_lr: 0.00005 + | > step_time: 7.50390 (3.03088) + | > loader_time: 0.09210 (0.05066) + + + --> STEP: 225/234 -- GLOBAL_STEP: 43515 + | > loss: -0.43610 (-0.23353) + | > log_mle: -0.69625 (-0.40228) + | > loss_dur: 0.26014 (0.16875) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 110.37735 (38.75957) + | > current_lr: 0.00005 + | > step_time: 1.18460 (3.01475) + | > loader_time: 0.00540 (0.04993) + + + --> STEP: 230/234 -- GLOBAL_STEP: 43520 + | > loss: -0.40822 (-0.23674) + | > log_mle: -0.73630 (-0.40794) + | > loss_dur: 0.32808 (0.17121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 131.89461 (40.36412) + | > current_lr: 0.00005 + | > step_time: 0.25020 (2.96106) + | > loader_time: 0.00370 (0.04894) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15179 (-0.26814) + | > avg_loss: -0.27487 (-0.00547) + | > avg_log_mle: -0.48936 (+0.00390) + | > avg_loss_dur: 0.21449 (-0.00937) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_43524.pth + + > EPOCH: 186/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-28 23:54:15)  + + --> STEP: 1/234 -- GLOBAL_STEP: 43525 + | > loss: -0.18649 (-0.18649) + | > log_mle: -0.30730 (-0.30730) + | > loss_dur: 0.12081 (0.12081) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.22679 (22.22679) + | > current_lr: 0.00005 + | > step_time: 2.40540 (2.40544) + | > loader_time: 0.00150 (0.00149) + + + --> STEP: 6/234 -- GLOBAL_STEP: 43530 + | > loss: -0.21045 (-0.19174) + | > log_mle: -0.30207 (-0.30597) + | > loss_dur: 0.09162 (0.11423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57721 (18.24338) + | > current_lr: 0.00005 + | > step_time: 9.59980 (4.46845) + | > loader_time: 0.00650 (0.06406) + + + --> STEP: 11/234 -- GLOBAL_STEP: 43535 + | > loss: -0.22825 (-0.19823) + | > log_mle: -0.30909 (-0.30849) + | > loss_dur: 0.08084 (0.11026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.85518 (17.09489) + | > current_lr: 0.00005 + | > step_time: 5.90340 (4.68355) + | > loader_time: 0.00180 (0.13577) + + + --> STEP: 16/234 -- GLOBAL_STEP: 43540 + | > loss: -0.21550 (-0.20192) + | > log_mle: -0.30434 (-0.30876) + | > loss_dur: 0.08884 (0.10684) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.25509 (15.47060) + | > current_lr: 0.00005 + | > step_time: 5.90800 (4.55757) + | > loader_time: 0.00110 (0.09506) + + + --> STEP: 21/234 -- GLOBAL_STEP: 43545 + | > loss: -0.18195 (-0.20156) + | > log_mle: -0.28100 (-0.30538) + | > loss_dur: 0.09905 (0.10381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.27339 (14.54360) + | > current_lr: 0.00005 + | > step_time: 2.70420 (4.07704) + | > loader_time: 0.08950 (0.07702) + + + --> STEP: 26/234 -- GLOBAL_STEP: 43550 + | > loss: -0.18380 (-0.20135) + | > log_mle: -0.29608 (-0.30388) + | > loss_dur: 0.11229 (0.10253) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.37370 (13.87662) + | > current_lr: 0.00005 + | > step_time: 1.80860 (4.23198) + | > loader_time: 0.28970 (0.08519) + + + --> STEP: 31/234 -- GLOBAL_STEP: 43555 + | > loss: -0.15842 (-0.20014) + | > log_mle: -0.29177 (-0.30284) + | > loss_dur: 0.13334 (0.10269) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.41232 (13.45446) + | > current_lr: 0.00005 + | > step_time: 0.98630 (4.31636) + | > loader_time: 0.00200 (0.07765) + + + --> STEP: 36/234 -- GLOBAL_STEP: 43560 + | > loss: -0.17085 (-0.19803) + | > log_mle: -0.29236 (-0.30187) + | > loss_dur: 0.12150 (0.10384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.63476 (13.22091) + | > current_lr: 0.00005 + | > step_time: 1.80140 (4.16324) + | > loader_time: 0.08850 (0.07259) + + + --> STEP: 41/234 -- GLOBAL_STEP: 43565 + | > loss: -0.20261 (-0.19641) + | > log_mle: -0.29820 (-0.30095) + | > loss_dur: 0.09559 (0.10453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.30790 (12.84620) + | > current_lr: 0.00005 + | > step_time: 0.79670 (3.87532) + | > loader_time: 0.00320 (0.06816) + + + --> STEP: 46/234 -- GLOBAL_STEP: 43570 + | > loss: -0.14876 (-0.19383) + | > log_mle: -0.28174 (-0.30004) + | > loss_dur: 0.13298 (0.10620) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.09132 (12.86377) + | > current_lr: 0.00005 + | > step_time: 3.20570 (3.70613) + | > loader_time: 0.00230 (0.06101) + + + --> STEP: 51/234 -- GLOBAL_STEP: 43575 + | > loss: -0.16664 (-0.19224) + | > log_mle: -0.28269 (-0.29873) + | > loss_dur: 0.11605 (0.10649) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.65079 (12.44409) + | > current_lr: 0.00005 + | > step_time: 2.49740 (3.59323) + | > loader_time: 0.00340 (0.05888) + + + --> STEP: 56/234 -- GLOBAL_STEP: 43580 + | > loss: -0.17652 (-0.19094) + | > log_mle: -0.29809 (-0.29849) + | > loss_dur: 0.12157 (0.10755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08994 (12.28930) + | > current_lr: 0.00005 + | > step_time: 2.54880 (3.44114) + | > loader_time: 0.10320 (0.05731) + + + --> STEP: 61/234 -- GLOBAL_STEP: 43585 + | > loss: -0.16778 (-0.18956) + | > log_mle: -0.29263 (-0.29828) + | > loss_dur: 0.12485 (0.10872) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.71373 (12.21643) + | > current_lr: 0.00005 + | > step_time: 1.60190 (3.34837) + | > loader_time: 0.08320 (0.05555) + + + --> STEP: 66/234 -- GLOBAL_STEP: 43590 + | > loss: -0.17422 (-0.18747) + | > log_mle: -0.28481 (-0.29808) + | > loss_dur: 0.11059 (0.11061) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.88521 (12.41505) + | > current_lr: 0.00005 + | > step_time: 1.37300 (3.22164) + | > loader_time: 0.00220 (0.05301) + + + --> STEP: 71/234 -- GLOBAL_STEP: 43595 + | > loss: -0.15084 (-0.18511) + | > log_mle: -0.31863 (-0.29778) + | > loss_dur: 0.16779 (0.11267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.39267 (12.56702) + | > current_lr: 0.00005 + | > step_time: 1.75840 (3.10462) + | > loader_time: 0.00280 (0.05165) + + + --> STEP: 76/234 -- GLOBAL_STEP: 43600 + | > loss: -0.17369 (-0.18351) + | > log_mle: -0.30272 (-0.29770) + | > loss_dur: 0.12902 (0.11419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.95341 (12.61586) + | > current_lr: 0.00005 + | > step_time: 1.71320 (3.04845) + | > loader_time: 0.07480 (0.04943) + + + --> STEP: 81/234 -- GLOBAL_STEP: 43605 + | > loss: -0.17051 (-0.18259) + | > log_mle: -0.30884 (-0.29750) + | > loss_dur: 0.13832 (0.11491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.49042 (12.58783) + | > current_lr: 0.00005 + | > step_time: 2.14320 (3.00046) + | > loader_time: 0.01200 (0.04881) + + + --> STEP: 86/234 -- GLOBAL_STEP: 43610 + | > loss: -0.16996 (-0.18145) + | > log_mle: -0.30538 (-0.29751) + | > loss_dur: 0.13541 (0.11606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.38482 (12.74570) + | > current_lr: 0.00005 + | > step_time: 2.95000 (2.95012) + | > loader_time: 0.19770 (0.05250) + + + --> STEP: 91/234 -- GLOBAL_STEP: 43615 + | > loss: -0.16467 (-0.18091) + | > log_mle: -0.31799 (-0.29871) + | > loss_dur: 0.15332 (0.11780) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.55027 (12.89038) + | > current_lr: 0.00005 + | > step_time: 3.29420 (2.88739) + | > loader_time: 0.00420 (0.05072) + + + --> STEP: 96/234 -- GLOBAL_STEP: 43620 + | > loss: -0.16056 (-0.18165) + | > log_mle: -0.30295 (-0.30148) + | > loss_dur: 0.14239 (0.11983) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.32694 (13.48135) + | > current_lr: 0.00005 + | > step_time: 1.09730 (2.88904) + | > loader_time: 0.00310 (0.05276) + + + --> STEP: 101/234 -- GLOBAL_STEP: 43625 + | > loss: -0.19506 (-0.18192) + | > log_mle: -0.35992 (-0.30329) + | > loss_dur: 0.16487 (0.12137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.55720 (13.92990) + | > current_lr: 0.00005 + | > step_time: 1.90010 (2.83300) + | > loader_time: 0.08430 (0.05183) + + + --> STEP: 106/234 -- GLOBAL_STEP: 43630 + | > loss: -0.17283 (-0.18219) + | > log_mle: -0.35836 (-0.30565) + | > loss_dur: 0.18554 (0.12346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.03595 (14.48821) + | > current_lr: 0.00005 + | > step_time: 2.92600 (2.79954) + | > loader_time: 0.00280 (0.05038) + + + --> STEP: 111/234 -- GLOBAL_STEP: 43635 + | > loss: -0.20464 (-0.18232) + | > log_mle: -0.41100 (-0.30815) + | > loss_dur: 0.20636 (0.12582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.78018 (15.01249) + | > current_lr: 0.00005 + | > step_time: 1.72760 (2.76325) + | > loader_time: 0.07370 (0.04893) + + + --> STEP: 116/234 -- GLOBAL_STEP: 43640 + | > loss: -0.18156 (-0.18265) + | > log_mle: -0.37393 (-0.31062) + | > loss_dur: 0.19237 (0.12798) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.83573 (15.71889) + | > current_lr: 0.00005 + | > step_time: 3.90290 (2.73592) + | > loader_time: 0.00200 (0.04694) + + + --> STEP: 121/234 -- GLOBAL_STEP: 43645 + | > loss: -0.14805 (-0.18278) + | > log_mle: -0.28783 (-0.31229) + | > loss_dur: 0.13978 (0.12951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.99820 (16.12314) + | > current_lr: 0.00005 + | > step_time: 1.06710 (2.72533) + | > loader_time: 0.00320 (0.04574) + + + --> STEP: 126/234 -- GLOBAL_STEP: 43650 + | > loss: -0.22992 (-0.18326) + | > log_mle: -0.42375 (-0.31450) + | > loss_dur: 0.19383 (0.13124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.80886 (16.66123) + | > current_lr: 0.00005 + | > step_time: 2.31100 (2.67979) + | > loader_time: 0.00270 (0.04472) + + + --> STEP: 131/234 -- GLOBAL_STEP: 43655 + | > loss: -0.27383 (-0.18469) + | > log_mle: -0.46838 (-0.31786) + | > loss_dur: 0.19455 (0.13318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.21071 (17.58021) + | > current_lr: 0.00005 + | > step_time: 1.59890 (2.64512) + | > loader_time: 0.00300 (0.04439) + + + --> STEP: 136/234 -- GLOBAL_STEP: 43660 + | > loss: -0.29980 (-0.18614) + | > log_mle: -0.51675 (-0.32127) + | > loss_dur: 0.21695 (0.13513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.71404 (18.57860) + | > current_lr: 0.00005 + | > step_time: 2.39500 (2.63739) + | > loader_time: 0.00730 (0.04291) + + + --> STEP: 141/234 -- GLOBAL_STEP: 43665 + | > loss: -0.23304 (-0.18735) + | > log_mle: -0.41919 (-0.32424) + | > loss_dur: 0.18615 (0.13690) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.65206 (19.31465) + | > current_lr: 0.00005 + | > step_time: 2.40040 (2.62284) + | > loader_time: 0.00550 (0.04326) + + + --> STEP: 146/234 -- GLOBAL_STEP: 43670 + | > loss: -0.26532 (-0.18948) + | > log_mle: -0.46899 (-0.32883) + | > loss_dur: 0.20367 (0.13935) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.28088 (20.64513) + | > current_lr: 0.00005 + | > step_time: 3.41810 (2.60981) + | > loader_time: 0.00280 (0.04313) + + + --> STEP: 151/234 -- GLOBAL_STEP: 43675 + | > loss: -0.25553 (-0.19168) + | > log_mle: -0.43669 (-0.33264) + | > loss_dur: 0.18116 (0.14097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.86249 (21.60190) + | > current_lr: 0.00005 + | > step_time: 1.18970 (2.58625) + | > loader_time: 0.00280 (0.04182) + + + --> STEP: 156/234 -- GLOBAL_STEP: 43680 + | > loss: -0.28628 (-0.19503) + | > log_mle: -0.48278 (-0.33801) + | > loss_dur: 0.19650 (0.14298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.71746 (23.04925) + | > current_lr: 0.00005 + | > step_time: 4.01730 (2.61565) + | > loader_time: 0.00320 (0.04481) + + + --> STEP: 161/234 -- GLOBAL_STEP: 43685 + | > loss: -0.29904 (-0.19742) + | > log_mle: -0.49157 (-0.34237) + | > loss_dur: 0.19254 (0.14495) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.31924 (24.53816) + | > current_lr: 0.00005 + | > step_time: 1.79940 (2.59642) + | > loader_time: 0.00280 (0.04469) + + + --> STEP: 166/234 -- GLOBAL_STEP: 43690 + | > loss: -0.25234 (-0.19926) + | > log_mle: -0.43592 (-0.34583) + | > loss_dur: 0.18358 (0.14657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.35932 (25.33946) + | > current_lr: 0.00005 + | > step_time: 1.89080 (2.58021) + | > loader_time: 0.00370 (0.04349) + + + --> STEP: 171/234 -- GLOBAL_STEP: 43695 + | > loss: -0.34351 (-0.20241) + | > log_mle: -0.54532 (-0.35096) + | > loss_dur: 0.20181 (0.14855) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.38205 (26.35148) + | > current_lr: 0.00005 + | > step_time: 3.99770 (2.57420) + | > loader_time: 0.10310 (0.04289) + + + --> STEP: 176/234 -- GLOBAL_STEP: 43700 + | > loss: -0.31270 (-0.20542) + | > log_mle: -0.52148 (-0.35598) + | > loss_dur: 0.20878 (0.15056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.87182 (27.46186) + | > current_lr: 0.00005 + | > step_time: 2.01970 (2.58464) + | > loader_time: 0.08890 (0.04272) + + + --> STEP: 181/234 -- GLOBAL_STEP: 43705 + | > loss: -0.24251 (-0.20792) + | > log_mle: -0.45653 (-0.36053) + | > loss_dur: 0.21402 (0.15261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.67390 (28.39985) + | > current_lr: 0.00005 + | > step_time: 2.60580 (2.58562) + | > loader_time: 0.10220 (0.04219) + + + --> STEP: 186/234 -- GLOBAL_STEP: 43710 + | > loss: -0.25455 (-0.21046) + | > log_mle: -0.49396 (-0.36520) + | > loss_dur: 0.23941 (0.15475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.55764 (29.52780) + | > current_lr: 0.00005 + | > step_time: 1.60060 (2.57250) + | > loader_time: 0.00480 (0.04117) + + + --> STEP: 191/234 -- GLOBAL_STEP: 43715 + | > loss: -0.30337 (-0.21306) + | > log_mle: -0.51335 (-0.36957) + | > loss_dur: 0.20998 (0.15652) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.46162 (30.69418) + | > current_lr: 0.00005 + | > step_time: 10.69550 (2.65715) + | > loader_time: 0.00600 (0.04361) + + + --> STEP: 196/234 -- GLOBAL_STEP: 43720 + | > loss: -0.29524 (-0.21609) + | > log_mle: -0.51844 (-0.37425) + | > loss_dur: 0.22321 (0.15816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.27253 (31.86707) + | > current_lr: 0.00005 + | > step_time: 1.87200 (2.74017) + | > loader_time: 0.01840 (0.04372) + + + --> STEP: 201/234 -- GLOBAL_STEP: 43725 + | > loss: -0.23504 (-0.21859) + | > log_mle: -0.46446 (-0.37838) + | > loss_dur: 0.22942 (0.15979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.98052 (33.14313) + | > current_lr: 0.00005 + | > step_time: 3.89860 (2.78741) + | > loader_time: 0.00330 (0.04419) + + + --> STEP: 206/234 -- GLOBAL_STEP: 43730 + | > loss: -0.35320 (-0.22161) + | > log_mle: -0.58274 (-0.38308) + | > loss_dur: 0.22954 (0.16147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.77106 (34.14358) + | > current_lr: 0.00005 + | > step_time: 2.10720 (2.82065) + | > loader_time: 0.00530 (0.04408) + + + --> STEP: 211/234 -- GLOBAL_STEP: 43735 + | > loss: -0.40120 (-0.22508) + | > log_mle: -0.65396 (-0.38843) + | > loss_dur: 0.25276 (0.16335) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.99285 (35.45584) + | > current_lr: 0.00005 + | > step_time: 3.11130 (2.88653) + | > loader_time: 0.00580 (0.04448) + + + --> STEP: 216/234 -- GLOBAL_STEP: 43740 + | > loss: -0.39090 (-0.22832) + | > log_mle: -0.64087 (-0.39339) + | > loss_dur: 0.24996 (0.16507) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.14302 (36.81679) + | > current_lr: 0.00005 + | > step_time: 6.09790 (2.94288) + | > loader_time: 0.00310 (0.04800) + + + --> STEP: 221/234 -- GLOBAL_STEP: 43745 + | > loss: -0.33466 (-0.23166) + | > log_mle: -0.54959 (-0.39835) + | > loss_dur: 0.21493 (0.16669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.63634 (37.93823) + | > current_lr: 0.00005 + | > step_time: 1.78720 (2.98220) + | > loader_time: 0.00240 (0.04746) + + + --> STEP: 226/234 -- GLOBAL_STEP: 43750 + | > loss: -0.41060 (-0.23539) + | > log_mle: -0.65760 (-0.40390) + | > loss_dur: 0.24700 (0.16852) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.47504 (39.29037) + | > current_lr: 0.00005 + | > step_time: 0.26630 (2.95759) + | > loader_time: 0.00630 (0.04683) + + + --> STEP: 231/234 -- GLOBAL_STEP: 43755 + | > loss: -0.34920 (-0.23827) + | > log_mle: -0.72999 (-0.40987) + | > loss_dur: 0.38078 (0.17160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 120.71679 (40.84024) + | > current_lr: 0.00005 + | > step_time: 0.27200 (2.89911) + | > loader_time: 0.00500 (0.04590) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.47900 (+0.32721) + | > avg_loss: -0.25545 (+0.01941) + | > avg_log_mle: -0.47922 (+0.01014) + | > avg_loss_dur: 0.22377 (+0.00928) + + + > EPOCH: 187/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 00:06:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 43760 + | > loss: -0.21514 (-0.20436) + | > log_mle: -0.31709 (-0.31219) + | > loss_dur: 0.10195 (0.10783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.48267 (14.84182) + | > current_lr: 0.00005 + | > step_time: 2.20230 (9.75079) + | > loader_time: 0.69700 (4.00376) + + + --> STEP: 7/234 -- GLOBAL_STEP: 43765 + | > loss: -0.21694 (-0.19249) + | > log_mle: -0.31283 (-0.30738) + | > loss_dur: 0.09588 (0.11489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.25247 (16.65743) + | > current_lr: 0.00005 + | > step_time: 6.08880 (5.44458) + | > loader_time: 0.00640 (1.15902) + + + --> STEP: 12/234 -- GLOBAL_STEP: 43770 + | > loss: -0.19854 (-0.19686) + | > log_mle: -0.30553 (-0.30864) + | > loss_dur: 0.10699 (0.11178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.80260 (16.52009) + | > current_lr: 0.00005 + | > step_time: 0.84480 (4.53289) + | > loader_time: 0.00140 (0.73203) + + + --> STEP: 17/234 -- GLOBAL_STEP: 43775 + | > loss: -0.19988 (-0.19986) + | > log_mle: -0.29059 (-0.30803) + | > loss_dur: 0.09072 (0.10818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.45101 (14.76067) + | > current_lr: 0.00005 + | > step_time: 1.19150 (3.59243) + | > loader_time: 0.00400 (0.52692) + + + --> STEP: 22/234 -- GLOBAL_STEP: 43780 + | > loss: -0.18858 (-0.19948) + | > log_mle: -0.30272 (-0.30577) + | > loss_dur: 0.11414 (0.10629) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.15287 (14.15051) + | > current_lr: 0.00005 + | > step_time: 1.18360 (3.22526) + | > loader_time: 0.00680 (0.41632) + + + --> STEP: 27/234 -- GLOBAL_STEP: 43785 + | > loss: -0.20342 (-0.20046) + | > log_mle: -0.30135 (-0.30446) + | > loss_dur: 0.09793 (0.10400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.98899 (13.84933) + | > current_lr: 0.00005 + | > step_time: 0.79110 (2.84141) + | > loader_time: 0.00190 (0.33959) + + + --> STEP: 32/234 -- GLOBAL_STEP: 43790 + | > loss: -0.20918 (-0.19981) + | > log_mle: -0.30825 (-0.30362) + | > loss_dur: 0.09907 (0.10381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.50849 (13.47315) + | > current_lr: 0.00005 + | > step_time: 1.48260 (2.60235) + | > loader_time: 0.00220 (0.28923) + + + --> STEP: 37/234 -- GLOBAL_STEP: 43795 + | > loss: -0.20089 (-0.19743) + | > log_mle: -0.29120 (-0.30219) + | > loss_dur: 0.09031 (0.10476) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.73812 (13.20489) + | > current_lr: 0.00005 + | > step_time: 1.31360 (2.44508) + | > loader_time: 0.07550 (0.25448) + + + --> STEP: 42/234 -- GLOBAL_STEP: 43800 + | > loss: -0.17008 (-0.19550) + | > log_mle: -0.28334 (-0.30132) + | > loss_dur: 0.11326 (0.10581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42563 (12.86633) + | > current_lr: 0.00005 + | > step_time: 1.80850 (2.37319) + | > loader_time: 0.08360 (0.22643) + + + --> STEP: 47/234 -- GLOBAL_STEP: 43805 + | > loss: -0.15972 (-0.19409) + | > log_mle: -0.29210 (-0.30112) + | > loss_dur: 0.13237 (0.10703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.18777 (12.72443) + | > current_lr: 0.00005 + | > step_time: 1.40990 (2.29074) + | > loader_time: 0.00260 (0.20259) + + + --> STEP: 52/234 -- GLOBAL_STEP: 43810 + | > loss: -0.15774 (-0.19281) + | > log_mle: -0.28898 (-0.30010) + | > loss_dur: 0.13124 (0.10729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.44815 (12.29038) + | > current_lr: 0.00005 + | > step_time: 2.29490 (2.23197) + | > loader_time: 0.00250 (0.18333) + + + --> STEP: 57/234 -- GLOBAL_STEP: 43815 + | > loss: -0.14952 (-0.19159) + | > log_mle: -0.27847 (-0.29983) + | > loss_dur: 0.12895 (0.10824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.21595 (12.21775) + | > current_lr: 0.00005 + | > step_time: 3.09200 (2.22333) + | > loader_time: 0.00820 (0.17281) + + + --> STEP: 62/234 -- GLOBAL_STEP: 43820 + | > loss: -0.12748 (-0.19008) + | > log_mle: -0.32452 (-0.30037) + | > loss_dur: 0.19704 (0.11028) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.65045 (12.54763) + | > current_lr: 0.00005 + | > step_time: 1.30600 (2.21666) + | > loader_time: 0.08100 (0.16192) + + + --> STEP: 67/234 -- GLOBAL_STEP: 43825 + | > loss: -0.16738 (-0.18900) + | > log_mle: -0.30532 (-0.29982) + | > loss_dur: 0.13794 (0.11082) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.12852 (12.46796) + | > current_lr: 0.00005 + | > step_time: 0.80680 (2.17930) + | > loader_time: 0.00250 (0.15005) + + + --> STEP: 72/234 -- GLOBAL_STEP: 43830 + | > loss: -0.16728 (-0.18683) + | > log_mle: -0.29113 (-0.29924) + | > loss_dur: 0.12385 (0.11240) + | > amp_scaler: 4096.00000 (2190.22222) + | > grad_norm: 10.26103 (12.65932) + | > current_lr: 0.00005 + | > step_time: 1.49400 (2.22516) + | > loader_time: 0.00330 (0.14132) + + + --> STEP: 77/234 -- GLOBAL_STEP: 43835 + | > loss: -0.18359 (-0.18564) + | > log_mle: -0.30132 (-0.29929) + | > loss_dur: 0.11773 (0.11366) + | > amp_scaler: 4096.00000 (2313.97403) + | > grad_norm: 11.12177 (12.82423) + | > current_lr: 0.00005 + | > step_time: 2.09610 (2.17449) + | > loader_time: 0.00180 (0.13229) + + + --> STEP: 82/234 -- GLOBAL_STEP: 43840 + | > loss: -0.17186 (-0.18461) + | > log_mle: -0.29279 (-0.29903) + | > loss_dur: 0.12094 (0.11442) + | > amp_scaler: 4096.00000 (2422.63415) + | > grad_norm: 14.50478 (12.83568) + | > current_lr: 0.00005 + | > step_time: 2.08140 (2.15156) + | > loader_time: 0.00280 (0.12546) + + + --> STEP: 87/234 -- GLOBAL_STEP: 43845 + | > loss: -0.16221 (-0.18346) + | > log_mle: -0.29695 (-0.29907) + | > loss_dur: 0.13474 (0.11561) + | > amp_scaler: 4096.00000 (2518.80460) + | > grad_norm: 10.60157 (12.82794) + | > current_lr: 0.00005 + | > step_time: 3.59240 (2.15915) + | > loader_time: 0.00280 (0.12035) + + + --> STEP: 92/234 -- GLOBAL_STEP: 43850 + | > loss: -0.20623 (-0.18345) + | > log_mle: -0.34587 (-0.30081) + | > loss_dur: 0.13964 (0.11736) + | > amp_scaler: 4096.00000 (2604.52174) + | > grad_norm: 23.50874 (13.04542) + | > current_lr: 0.00005 + | > step_time: 3.90720 (2.17752) + | > loader_time: 0.00510 (0.11491) + + + --> STEP: 97/234 -- GLOBAL_STEP: 43855 + | > loss: -0.18075 (-0.18423) + | > log_mle: -0.33141 (-0.30347) + | > loss_dur: 0.15066 (0.11924) + | > amp_scaler: 4096.00000 (2681.40206) + | > grad_norm: 17.75059 (13.52516) + | > current_lr: 0.00005 + | > step_time: 2.40960 (2.23648) + | > loader_time: 0.00370 (0.11096) + + + --> STEP: 102/234 -- GLOBAL_STEP: 43860 + | > loss: -0.15065 (-0.18408) + | > log_mle: -0.31440 (-0.30517) + | > loss_dur: 0.16375 (0.12109) + | > amp_scaler: 4096.00000 (2750.74510) + | > grad_norm: 16.27331 (13.94805) + | > current_lr: 0.00005 + | > step_time: 1.10650 (2.20634) + | > loader_time: 0.00320 (0.10647) + + + --> STEP: 107/234 -- GLOBAL_STEP: 43865 + | > loss: -0.18876 (-0.18470) + | > log_mle: -0.35884 (-0.30805) + | > loss_dur: 0.17008 (0.12335) + | > amp_scaler: 4096.00000 (2813.60748) + | > grad_norm: 26.52700 (14.50909) + | > current_lr: 0.00005 + | > step_time: 1.60120 (2.22275) + | > loader_time: 0.00310 (0.10242) + + + --> STEP: 112/234 -- GLOBAL_STEP: 43870 + | > loss: -0.19251 (-0.18481) + | > log_mle: -0.37153 (-0.31061) + | > loss_dur: 0.17903 (0.12579) + | > amp_scaler: 4096.00000 (2870.85714) + | > grad_norm: 26.05057 (15.20510) + | > current_lr: 0.00005 + | > step_time: 4.42390 (2.24966) + | > loader_time: 0.09580 (0.10026) + + + --> STEP: 117/234 -- GLOBAL_STEP: 43875 + | > loss: -0.20074 (-0.18511) + | > log_mle: -0.36213 (-0.31292) + | > loss_dur: 0.16139 (0.12781) + | > amp_scaler: 4096.00000 (2923.21368) + | > grad_norm: 31.92149 (15.95818) + | > current_lr: 0.00005 + | > step_time: 2.31710 (2.24322) + | > loader_time: 0.18740 (0.09930) + + + --> STEP: 122/234 -- GLOBAL_STEP: 43880 + | > loss: -0.18405 (-0.18505) + | > log_mle: -0.33653 (-0.31433) + | > loss_dur: 0.15248 (0.12927) + | > amp_scaler: 4096.00000 (2971.27869) + | > grad_norm: 20.80304 (16.24882) + | > current_lr: 0.00005 + | > step_time: 2.59000 (2.23713) + | > loader_time: 0.00210 (0.09610) + + + --> STEP: 127/234 -- GLOBAL_STEP: 43885 + | > loss: -0.21425 (-0.18587) + | > log_mle: -0.39887 (-0.31697) + | > loss_dur: 0.18462 (0.13111) + | > amp_scaler: 4096.00000 (3015.55906) + | > grad_norm: 34.43619 (17.04395) + | > current_lr: 0.00005 + | > step_time: 1.32610 (2.26221) + | > loader_time: 0.00320 (0.09474) + + + --> STEP: 132/234 -- GLOBAL_STEP: 43890 + | > loss: -0.21710 (-0.18727) + | > log_mle: -0.37768 (-0.32021) + | > loss_dur: 0.16058 (0.13295) + | > amp_scaler: 4096.00000 (3056.48485) + | > grad_norm: 33.15666 (17.69692) + | > current_lr: 0.00005 + | > step_time: 3.50320 (2.27058) + | > loader_time: 0.09280 (0.09276) + + + --> STEP: 137/234 -- GLOBAL_STEP: 43895 + | > loss: -0.19086 (-0.18862) + | > log_mle: -0.39480 (-0.32368) + | > loss_dur: 0.20395 (0.13506) + | > amp_scaler: 4096.00000 (3094.42336) + | > grad_norm: 39.11824 (18.53582) + | > current_lr: 0.00005 + | > step_time: 1.08640 (2.26087) + | > loader_time: 0.00270 (0.09198) + + + --> STEP: 142/234 -- GLOBAL_STEP: 43900 + | > loss: -0.20445 (-0.18960) + | > log_mle: -0.40483 (-0.32655) + | > loss_dur: 0.20038 (0.13694) + | > amp_scaler: 4096.00000 (3129.69014) + | > grad_norm: 40.15891 (19.40043) + | > current_lr: 0.00005 + | > step_time: 7.61040 (2.31649) + | > loader_time: 0.08550 (0.09131) + + + --> STEP: 147/234 -- GLOBAL_STEP: 43905 + | > loss: -0.22389 (-0.19181) + | > log_mle: -0.40889 (-0.33102) + | > loss_dur: 0.18500 (0.13921) + | > amp_scaler: 4096.00000 (3162.55782) + | > grad_norm: 33.56722 (20.54029) + | > current_lr: 0.00005 + | > step_time: 0.76880 (2.35865) + | > loader_time: 0.00280 (0.09096) + + + --> STEP: 152/234 -- GLOBAL_STEP: 43910 + | > loss: -0.26825 (-0.19428) + | > log_mle: -0.48892 (-0.33531) + | > loss_dur: 0.22067 (0.14103) + | > amp_scaler: 4096.00000 (3193.26316) + | > grad_norm: 62.14816 (21.47604) + | > current_lr: 0.00005 + | > step_time: 2.39210 (2.37271) + | > loader_time: 0.10480 (0.08929) + + + --> STEP: 157/234 -- GLOBAL_STEP: 43915 + | > loss: -0.23197 (-0.19727) + | > log_mle: -0.43418 (-0.34022) + | > loss_dur: 0.20222 (0.14295) + | > amp_scaler: 4096.00000 (3222.01274) + | > grad_norm: 56.51508 (22.82487) + | > current_lr: 0.00005 + | > step_time: 3.10380 (2.37936) + | > loader_time: 0.08620 (0.08817) + + + --> STEP: 162/234 -- GLOBAL_STEP: 43920 + | > loss: -0.28674 (-0.19996) + | > log_mle: -0.47084 (-0.34474) + | > loss_dur: 0.18410 (0.14478) + | > amp_scaler: 4096.00000 (3248.98765) + | > grad_norm: 50.05509 (23.89751) + | > current_lr: 0.00005 + | > step_time: 2.69110 (2.38973) + | > loader_time: 0.00350 (0.08558) + + + --> STEP: 167/234 -- GLOBAL_STEP: 43925 + | > loss: -0.36676 (-0.20262) + | > log_mle: -0.56410 (-0.34904) + | > loss_dur: 0.19735 (0.14643) + | > amp_scaler: 4096.00000 (3274.34731) + | > grad_norm: 78.00880 (24.99037) + | > current_lr: 0.00005 + | > step_time: 2.62650 (2.40758) + | > loader_time: 0.00280 (0.08426) + + + --> STEP: 172/234 -- GLOBAL_STEP: 43930 + | > loss: -0.32977 (-0.20564) + | > log_mle: -0.55324 (-0.35426) + | > loss_dur: 0.22347 (0.14862) + | > amp_scaler: 4096.00000 (3298.23256) + | > grad_norm: 73.95501 (26.28918) + | > current_lr: 0.00005 + | > step_time: 5.09190 (2.40316) + | > loader_time: 0.00340 (0.08190) + + + --> STEP: 177/234 -- GLOBAL_STEP: 43935 + | > loss: -0.29403 (-0.20850) + | > log_mle: -0.50568 (-0.35917) + | > loss_dur: 0.21165 (0.15067) + | > amp_scaler: 4096.00000 (3320.76836) + | > grad_norm: 83.63057 (27.72298) + | > current_lr: 0.00005 + | > step_time: 2.40840 (2.40489) + | > loader_time: 0.00710 (0.08018) + + + --> STEP: 182/234 -- GLOBAL_STEP: 43940 + | > loss: -0.29805 (-0.21112) + | > log_mle: -0.54572 (-0.36405) + | > loss_dur: 0.24767 (0.15293) + | > amp_scaler: 4096.00000 (3342.06593) + | > grad_norm: 90.05213 (28.91758) + | > current_lr: 0.00005 + | > step_time: 1.70510 (2.40814) + | > loader_time: 0.08410 (0.07944) + + + --> STEP: 187/234 -- GLOBAL_STEP: 43945 + | > loss: -0.31580 (-0.21376) + | > log_mle: -0.54159 (-0.36865) + | > loss_dur: 0.22579 (0.15490) + | > amp_scaler: 4096.00000 (3362.22460) + | > grad_norm: 77.97733 (30.11992) + | > current_lr: 0.00005 + | > step_time: 1.91360 (2.40908) + | > loader_time: 0.09440 (0.07893) + + + --> STEP: 192/234 -- GLOBAL_STEP: 43950 + | > loss: -0.37191 (-0.21685) + | > log_mle: -0.58131 (-0.37336) + | > loss_dur: 0.20940 (0.15651) + | > amp_scaler: 4096.00000 (3381.33333) + | > grad_norm: 94.79945 (31.28165) + | > current_lr: 0.00005 + | > step_time: 4.00300 (2.41462) + | > loader_time: 0.08370 (0.07840) + + + --> STEP: 197/234 -- GLOBAL_STEP: 43955 + | > loss: -0.34327 (-0.21982) + | > log_mle: -0.54972 (-0.37793) + | > loss_dur: 0.20645 (0.15811) + | > amp_scaler: 4096.00000 (3399.47208) + | > grad_norm: 76.98507 (32.47960) + | > current_lr: 0.00005 + | > step_time: 1.18670 (2.45242) + | > loader_time: 0.00490 (0.07736) + + + --> STEP: 202/234 -- GLOBAL_STEP: 43960 + | > loss: -0.42334 (-0.22257) + | > log_mle: -0.64727 (-0.38249) + | > loss_dur: 0.22394 (0.15992) + | > amp_scaler: 4096.00000 (3416.71287) + | > grad_norm: 83.64680 (33.76134) + | > current_lr: 0.00005 + | > step_time: 3.30140 (2.48281) + | > loader_time: 0.00670 (0.07789) + + + --> STEP: 207/234 -- GLOBAL_STEP: 43965 + | > loss: -0.37441 (-0.22517) + | > log_mle: -0.62011 (-0.38685) + | > loss_dur: 0.24569 (0.16168) + | > amp_scaler: 4096.00000 (3433.12077) + | > grad_norm: 80.89318 (34.89317) + | > current_lr: 0.00005 + | > step_time: 4.99270 (2.56094) + | > loader_time: 0.11550 (0.07842) + + + --> STEP: 212/234 -- GLOBAL_STEP: 43970 + | > loss: -0.36336 (-0.22842) + | > log_mle: -0.60420 (-0.39197) + | > loss_dur: 0.24084 (0.16355) + | > amp_scaler: 4096.00000 (3448.75472) + | > grad_norm: 87.08375 (36.05175) + | > current_lr: 0.00005 + | > step_time: 4.70240 (2.58734) + | > loader_time: 0.09730 (0.07710) + + + --> STEP: 217/234 -- GLOBAL_STEP: 43975 + | > loss: -0.38548 (-0.23173) + | > log_mle: -0.62594 (-0.39698) + | > loss_dur: 0.24046 (0.16525) + | > amp_scaler: 4096.00000 (3463.66820) + | > grad_norm: 108.20532 (37.35842) + | > current_lr: 0.00005 + | > step_time: 4.10890 (2.67802) + | > loader_time: 0.08720 (0.07746) + + + --> STEP: 222/234 -- GLOBAL_STEP: 43980 + | > loss: -0.36052 (-0.23489) + | > log_mle: -0.63543 (-0.40196) + | > loss_dur: 0.27491 (0.16707) + | > amp_scaler: 4096.00000 (3477.90991) + | > grad_norm: 82.01594 (38.42469) + | > current_lr: 0.00005 + | > step_time: 2.39800 (2.67043) + | > loader_time: 0.09290 (0.07841) + + + --> STEP: 227/234 -- GLOBAL_STEP: 43985 + | > loss: -0.35648 (-0.23842) + | > log_mle: -0.61487 (-0.40732) + | > loss_dur: 0.25840 (0.16890) + | > amp_scaler: 4096.00000 (3491.52423) + | > grad_norm: 93.27836 (39.66280) + | > current_lr: 0.00005 + | > step_time: 0.24010 (2.64350) + | > loader_time: 0.00340 (0.07788) + + + --> STEP: 232/234 -- GLOBAL_STEP: 43990 + | > loss: -0.33893 (-0.24122) + | > log_mle: -0.82632 (-0.41411) + | > loss_dur: 0.48739 (0.17289) + | > amp_scaler: 4096.00000 (3504.55172) + | > grad_norm: 151.12119 (41.32526) + | > current_lr: 0.00005 + | > step_time: 0.33460 (2.59243) + | > loader_time: 0.00620 (0.07630) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.64996 (+0.17097) + | > avg_loss: -0.26040 (-0.00494) + | > avg_log_mle: -0.47925 (-0.00003) + | > avg_loss_dur: 0.21886 (-0.00491) + + + > EPOCH: 188/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 00:18:00)  + + --> STEP: 3/234 -- GLOBAL_STEP: 43995 + | > loss: -0.12588 (-0.18327) + | > log_mle: -0.29712 (-0.30751) + | > loss_dur: 0.17124 (0.12424) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 31.79591 (24.86494) + | > current_lr: 0.00005 + | > step_time: 5.78940 (9.33194) + | > loader_time: 0.10040 (0.06912) + + + --> STEP: 8/234 -- GLOBAL_STEP: 44000 + | > loss: -0.22592 (-0.19727) + | > log_mle: -0.32086 (-0.30814) + | > loss_dur: 0.09494 (0.11087) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 12.81069 (20.61507) + | > current_lr: 0.00005 + | > step_time: 0.91600 (5.30093) + | > loader_time: 0.00180 (0.07552) + + + --> STEP: 13/234 -- GLOBAL_STEP: 44005 + | > loss: -0.22730 (-0.20105) + | > log_mle: -0.31911 (-0.30981) + | > loss_dur: 0.09181 (0.10876) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.89475 (18.24033) + | > current_lr: 0.00005 + | > step_time: 2.18580 (3.88520) + | > loader_time: 0.00110 (0.04734) + + + --> STEP: 18/234 -- GLOBAL_STEP: 44010 + | > loss: -0.18860 (-0.20284) + | > log_mle: -0.30269 (-0.30872) + | > loss_dur: 0.11410 (0.10587) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.53977 (16.40517) + | > current_lr: 0.00005 + | > step_time: 1.64920 (3.25993) + | > loader_time: 0.00140 (0.03870) + + + --> STEP: 23/234 -- GLOBAL_STEP: 44015 + | > loss: -0.22693 (-0.20373) + | > log_mle: -0.31100 (-0.30711) + | > loss_dur: 0.08407 (0.10337) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.07585 (15.19545) + | > current_lr: 0.00005 + | > step_time: 4.90530 (2.99974) + | > loader_time: 0.00210 (0.03196) + + + --> STEP: 28/234 -- GLOBAL_STEP: 44020 + | > loss: -0.24493 (-0.20529) + | > log_mle: -0.31292 (-0.30604) + | > loss_dur: 0.06799 (0.10075) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 7.63751 (14.21914) + | > current_lr: 0.00005 + | > step_time: 5.59160 (3.32787) + | > loader_time: 0.00110 (0.04051) + + + --> STEP: 33/234 -- GLOBAL_STEP: 44025 + | > loss: -0.19857 (-0.20369) + | > log_mle: -0.29557 (-0.30478) + | > loss_dur: 0.09700 (0.10110) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.20404 (13.91752) + | > current_lr: 0.00005 + | > step_time: 1.81520 (3.07270) + | > loader_time: 0.09170 (0.04255) + + + --> STEP: 38/234 -- GLOBAL_STEP: 44030 + | > loss: -0.19365 (-0.20154) + | > log_mle: -0.30918 (-0.30377) + | > loss_dur: 0.11553 (0.10223) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.11948 (13.83021) + | > current_lr: 0.00005 + | > step_time: 2.85590 (2.94129) + | > loader_time: 0.08640 (0.04186) + + + --> STEP: 43/234 -- GLOBAL_STEP: 44035 + | > loss: -0.17746 (-0.19885) + | > log_mle: -0.30447 (-0.30261) + | > loss_dur: 0.12702 (0.10376) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.27606 (13.52813) + | > current_lr: 0.00005 + | > step_time: 0.91140 (2.76997) + | > loader_time: 0.00160 (0.03725) + + + --> STEP: 48/234 -- GLOBAL_STEP: 44040 + | > loss: -0.19753 (-0.19748) + | > log_mle: -0.29243 (-0.30219) + | > loss_dur: 0.09490 (0.10471) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.75011 (13.19774) + | > current_lr: 0.00005 + | > step_time: 0.78240 (2.68121) + | > loader_time: 0.00220 (0.03565) + + + --> STEP: 53/234 -- GLOBAL_STEP: 44045 + | > loss: -0.18697 (-0.19628) + | > log_mle: -0.30090 (-0.30144) + | > loss_dur: 0.11394 (0.10516) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.50512 (12.83169) + | > current_lr: 0.00005 + | > step_time: 2.70560 (2.64955) + | > loader_time: 0.00280 (0.03252) + + + --> STEP: 58/234 -- GLOBAL_STEP: 44050 + | > loss: -0.18480 (-0.19529) + | > log_mle: -0.29268 (-0.30098) + | > loss_dur: 0.10788 (0.10569) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 9.88558 (12.73854) + | > current_lr: 0.00005 + | > step_time: 1.80920 (2.60846) + | > loader_time: 0.00340 (0.02995) + + + --> STEP: 63/234 -- GLOBAL_STEP: 44055 + | > loss: -0.16862 (-0.19328) + | > log_mle: -0.29372 (-0.30156) + | > loss_dur: 0.12510 (0.10828) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 10.90783 (12.98195) + | > current_lr: 0.00005 + | > step_time: 3.30980 (2.57741) + | > loader_time: 0.08540 (0.03187) + + + --> STEP: 68/234 -- GLOBAL_STEP: 44060 + | > loss: -0.15039 (-0.19175) + | > log_mle: -0.28980 (-0.30101) + | > loss_dur: 0.13940 (0.10925) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 11.87008 (12.80011) + | > current_lr: 0.00005 + | > step_time: 0.68850 (2.55724) + | > loader_time: 0.00190 (0.03258) + + + --> STEP: 73/234 -- GLOBAL_STEP: 44065 + | > loss: -0.15144 (-0.18944) + | > log_mle: -0.30567 (-0.30074) + | > loss_dur: 0.15423 (0.11130) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.88828 (12.82682) + | > current_lr: 0.00005 + | > step_time: 1.71050 (2.51510) + | > loader_time: 0.00230 (0.03054) + + + --> STEP: 78/234 -- GLOBAL_STEP: 44070 + | > loss: -0.14806 (-0.18784) + | > log_mle: -0.28473 (-0.30041) + | > loss_dur: 0.13667 (0.11258) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 13.61154 (12.84157) + | > current_lr: 0.00005 + | > step_time: 1.98600 (2.49112) + | > loader_time: 0.00280 (0.02880) + + + --> STEP: 83/234 -- GLOBAL_STEP: 44075 + | > loss: -0.14405 (-0.18666) + | > log_mle: -0.30818 (-0.30041) + | > loss_dur: 0.16412 (0.11375) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.41656 (12.83099) + | > current_lr: 0.00005 + | > step_time: 3.50100 (2.48418) + | > loader_time: 0.00280 (0.02729) + + + --> STEP: 88/234 -- GLOBAL_STEP: 44080 + | > loss: -0.18959 (-0.18600) + | > log_mle: -0.34192 (-0.30088) + | > loss_dur: 0.15233 (0.11488) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 17.68445 (12.82981) + | > current_lr: 0.00005 + | > step_time: 0.75190 (2.46125) + | > loader_time: 0.00230 (0.02912) + + + --> STEP: 93/234 -- GLOBAL_STEP: 44085 + | > loss: -0.18472 (-0.18580) + | > log_mle: -0.35346 (-0.30262) + | > loss_dur: 0.16874 (0.11682) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 28.06371 (13.18191) + | > current_lr: 0.00005 + | > step_time: 3.41420 (2.44517) + | > loader_time: 0.00280 (0.02850) + + + --> STEP: 98/234 -- GLOBAL_STEP: 44090 + | > loss: -0.16304 (-0.18612) + | > log_mle: -0.28699 (-0.30431) + | > loss_dur: 0.12394 (0.11819) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 8.90733 (13.61792) + | > current_lr: 0.00005 + | > step_time: 2.30020 (2.45625) + | > loader_time: 0.00320 (0.02814) + + + --> STEP: 103/234 -- GLOBAL_STEP: 44095 + | > loss: -0.20687 (-0.18630) + | > log_mle: -0.38875 (-0.30688) + | > loss_dur: 0.18188 (0.12058) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 27.31282 (14.17723) + | > current_lr: 0.00005 + | > step_time: 2.99060 (2.48907) + | > loader_time: 0.00640 (0.02799) + + + --> STEP: 108/234 -- GLOBAL_STEP: 44100 + | > loss: -0.18731 (-0.18666) + | > log_mle: -0.33150 (-0.30904) + | > loss_dur: 0.14419 (0.12238) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 18.37155 (14.64272) + | > current_lr: 0.00005 + | > step_time: 1.08050 (2.45224) + | > loader_time: 0.00230 (0.02773) + + + --> STEP: 113/234 -- GLOBAL_STEP: 44105 + | > loss: -0.20277 (-0.18680) + | > log_mle: -0.37676 (-0.31193) + | > loss_dur: 0.17399 (0.12512) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 43.91432 (15.46057) + | > current_lr: 0.00005 + | > step_time: 1.62410 (2.42271) + | > loader_time: 0.08390 (0.02804) + + + --> STEP: 118/234 -- GLOBAL_STEP: 44110 + | > loss: -0.17327 (-0.18646) + | > log_mle: -0.34459 (-0.31386) + | > loss_dur: 0.17132 (0.12740) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 29.44165 (15.97950) + | > current_lr: 0.00005 + | > step_time: 1.79990 (2.42948) + | > loader_time: 0.00840 (0.02840) + + + --> STEP: 123/234 -- GLOBAL_STEP: 44115 + | > loss: -0.15670 (-0.18612) + | > log_mle: -0.31268 (-0.31484) + | > loss_dur: 0.15598 (0.12872) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.24557 (16.23043) + | > current_lr: 0.00005 + | > step_time: 0.76760 (2.41019) + | > loader_time: 0.00210 (0.02737) + + + --> STEP: 128/234 -- GLOBAL_STEP: 44120 + | > loss: -0.20615 (-0.18712) + | > log_mle: -0.37259 (-0.31781) + | > loss_dur: 0.16643 (0.13069) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 32.95022 (17.01212) + | > current_lr: 0.00005 + | > step_time: 1.74620 (2.38213) + | > loader_time: 0.08240 (0.02764) + + + --> STEP: 133/234 -- GLOBAL_STEP: 44125 + | > loss: -0.22189 (-0.18857) + | > log_mle: -0.40445 (-0.32113) + | > loss_dur: 0.18256 (0.13256) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 30.04734 (17.60225) + | > current_lr: 0.00005 + | > step_time: 1.56820 (2.36897) + | > loader_time: 0.00200 (0.02814) + + + --> STEP: 138/234 -- GLOBAL_STEP: 44130 + | > loss: -0.18158 (-0.18955) + | > log_mle: -0.35412 (-0.32410) + | > loss_dur: 0.17254 (0.13455) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 21.42612 (18.40682) + | > current_lr: 0.00005 + | > step_time: 2.50640 (2.38820) + | > loader_time: 0.09540 (0.02791) + + + --> STEP: 143/234 -- GLOBAL_STEP: 44135 + | > loss: -0.27135 (-0.19115) + | > log_mle: -0.50169 (-0.32793) + | > loss_dur: 0.23034 (0.13678) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 52.69482 (19.35619) + | > current_lr: 0.00005 + | > step_time: 1.60650 (2.40702) + | > loader_time: 0.00460 (0.03018) + + + --> STEP: 148/234 -- GLOBAL_STEP: 44140 + | > loss: -0.24446 (-0.19326) + | > log_mle: -0.41024 (-0.33190) + | > loss_dur: 0.16578 (0.13864) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 44.77299 (20.27748) + | > current_lr: 0.00005 + | > step_time: 0.89170 (2.39840) + | > loader_time: 0.00270 (0.02992) + + + --> STEP: 153/234 -- GLOBAL_STEP: 44145 + | > loss: -0.33992 (-0.19628) + | > log_mle: -0.54323 (-0.33703) + | > loss_dur: 0.20331 (0.14075) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 58.63773 (21.60236) + | > current_lr: 0.00005 + | > step_time: 2.90690 (2.40882) + | > loader_time: 0.00300 (0.02907) + + + --> STEP: 158/234 -- GLOBAL_STEP: 44150 + | > loss: -0.25524 (-0.19846) + | > log_mle: -0.46792 (-0.34122) + | > loss_dur: 0.21268 (0.14276) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.08889 (22.86344) + | > current_lr: 0.00005 + | > step_time: 0.99650 (2.42634) + | > loader_time: 0.00460 (0.02888) + + + --> STEP: 163/234 -- GLOBAL_STEP: 44155 + | > loss: -0.25043 (-0.20110) + | > log_mle: -0.44476 (-0.34556) + | > loss_dur: 0.19433 (0.14446) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 42.60333 (23.70907) + | > current_lr: 0.00005 + | > step_time: 1.29880 (2.44888) + | > loader_time: 0.00380 (0.02968) + + + --> STEP: 168/234 -- GLOBAL_STEP: 44160 + | > loss: -0.27306 (-0.20378) + | > log_mle: -0.49842 (-0.35010) + | > loss_dur: 0.22536 (0.14633) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 57.13249 (24.81971) + | > current_lr: 0.00005 + | > step_time: 2.30380 (2.43262) + | > loader_time: 0.00380 (0.02936) + + + --> STEP: 173/234 -- GLOBAL_STEP: 44165 + | > loss: -0.29806 (-0.20684) + | > log_mle: -0.51473 (-0.35533) + | > loss_dur: 0.21667 (0.14849) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 60.21726 (25.87062) + | > current_lr: 0.00005 + | > step_time: 2.90320 (2.42120) + | > loader_time: 0.09060 (0.02968) + + + --> STEP: 178/234 -- GLOBAL_STEP: 44170 + | > loss: -0.32797 (-0.20994) + | > log_mle: -0.56848 (-0.36054) + | > loss_dur: 0.24051 (0.15060) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 80.04040 (27.15076) + | > current_lr: 0.00005 + | > step_time: 8.81200 (2.47565) + | > loader_time: 0.08770 (0.03039) + + + --> STEP: 183/234 -- GLOBAL_STEP: 44175 + | > loss: -0.34911 (-0.21259) + | > log_mle: -0.56795 (-0.36531) + | > loss_dur: 0.21884 (0.15273) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 89.09502 (28.39597) + | > current_lr: 0.00005 + | > step_time: 3.70870 (2.48564) + | > loader_time: 0.00450 (0.03121) + + + --> STEP: 188/234 -- GLOBAL_STEP: 44180 + | > loss: -0.32966 (-0.21522) + | > log_mle: -0.56240 (-0.37004) + | > loss_dur: 0.23274 (0.15482) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 107.07726 (29.86635) + | > current_lr: 0.00005 + | > step_time: 6.00760 (2.53700) + | > loader_time: 0.08770 (0.03360) + + + --> STEP: 193/234 -- GLOBAL_STEP: 44185 + | > loss: -0.34760 (-0.21788) + | > log_mle: -0.56984 (-0.37436) + | > loss_dur: 0.22225 (0.15647) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.35436 (30.81514) + | > current_lr: 0.00005 + | > step_time: 5.20300 (2.63599) + | > loader_time: 0.10100 (0.03375) + + + --> STEP: 198/234 -- GLOBAL_STEP: 44190 + | > loss: -0.33381 (-0.22046) + | > log_mle: -0.56632 (-0.37867) + | > loss_dur: 0.23252 (0.15821) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 72.74339 (31.58426) + | > current_lr: 0.00005 + | > step_time: 12.70490 (2.71235) + | > loader_time: 0.10890 (0.03545) + + + --> STEP: 203/234 -- GLOBAL_STEP: 44195 + | > loss: -0.27669 (-0.22281) + | > log_mle: -0.49715 (-0.38277) + | > loss_dur: 0.22046 (0.15996) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 62.46312 (32.68486) + | > current_lr: 0.00005 + | > step_time: 1.79560 (2.77231) + | > loader_time: 0.00500 (0.03564) + + + --> STEP: 208/234 -- GLOBAL_STEP: 44200 + | > loss: -0.34491 (-0.22593) + | > log_mle: -0.58503 (-0.38774) + | > loss_dur: 0.24012 (0.16180) + | > amp_scaler: 4096.00000 (4096.00000) + | > grad_norm: 80.36905 (33.82364) + | > current_lr: 0.00005 + | > step_time: 11.50540 (2.83985) + | > loader_time: 0.00550 (0.03873) + + + --> STEP: 213/234 -- GLOBAL_STEP: 44205 + | > loss: -0.37670 (-0.22942) + | > log_mle: -0.62863 (-0.39315) + | > loss_dur: 0.25194 (0.16373) + | > amp_scaler: 2048.00000 (4057.53991) + | > grad_norm: 128.56703 (34.96822) + | > current_lr: 0.00005 + | > step_time: 5.39250 (2.91637) + | > loader_time: 0.00520 (0.03888) + + + --> STEP: 218/234 -- GLOBAL_STEP: 44210 + | > loss: -0.35590 (-0.23271) + | > log_mle: -0.59056 (-0.39813) + | > loss_dur: 0.23466 (0.16542) + | > amp_scaler: 2048.00000 (4011.44954) + | > grad_norm: 84.75265 (36.25925) + | > current_lr: 0.00005 + | > step_time: 2.60110 (2.97619) + | > loader_time: 0.09730 (0.04019) + + + --> STEP: 223/234 -- GLOBAL_STEP: 44215 + | > loss: -0.39802 (-0.23620) + | > log_mle: -0.64162 (-0.40342) + | > loss_dur: 0.24360 (0.16722) + | > amp_scaler: 2048.00000 (3967.42601) + | > grad_norm: 86.57508 (37.56047) + | > current_lr: 0.00005 + | > step_time: 0.83830 (2.96250) + | > loader_time: 0.00450 (0.04012) + + + --> STEP: 228/234 -- GLOBAL_STEP: 44220 + | > loss: -0.35698 (-0.23967) + | > log_mle: -0.63168 (-0.40886) + | > loss_dur: 0.27471 (0.16920) + | > amp_scaler: 2048.00000 (3925.33333) + | > grad_norm: 114.48914 (39.18661) + | > current_lr: 0.00005 + | > step_time: 0.24120 (2.90287) + | > loader_time: 0.00290 (0.03932) + + + --> STEP: 233/234 -- GLOBAL_STEP: 44225 + | > loss: 0.06483 (-0.24079) + | > log_mle: -0.60640 (-0.41568) + | > loss_dur: 0.67123 (0.17489) + | > amp_scaler: 2048.00000 (3885.04721) + | > grad_norm: 120.64896 (41.11398) + | > current_lr: 0.00005 + | > step_time: 0.19620 (2.84631) + | > loader_time: 0.00270 (0.03876) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22867 (-0.42129) + | > avg_loss: -0.27409 (-0.01369) + | > avg_log_mle: -0.50171 (-0.02245) + | > avg_loss_dur: 0.22762 (+0.00876) + + + > EPOCH: 189/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 00:30:17)  + + --> STEP: 4/234 -- GLOBAL_STEP: 44230 + | > loss: -0.18111 (-0.18620) + | > log_mle: -0.30780 (-0.31045) + | > loss_dur: 0.12669 (0.12426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.27723 (19.03873) + | > current_lr: 0.00005 + | > step_time: 1.70100 (3.06716) + | > loader_time: 0.19800 (0.12570) + + + --> STEP: 9/234 -- GLOBAL_STEP: 44235 + | > loss: -0.19025 (-0.19977) + | > log_mle: -0.31884 (-0.31286) + | > loss_dur: 0.12859 (0.11310) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.28250 (16.23017) + | > current_lr: 0.00005 + | > step_time: 4.33250 (2.53209) + | > loader_time: 0.00140 (0.06558) + + + --> STEP: 14/234 -- GLOBAL_STEP: 44240 + | > loss: -0.19812 (-0.20249) + | > log_mle: -0.31537 (-0.31297) + | > loss_dur: 0.11725 (0.11048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.87031 (15.35977) + | > current_lr: 0.00005 + | > step_time: 11.58470 (3.22133) + | > loader_time: 0.10600 (0.05622) + + + --> STEP: 19/234 -- GLOBAL_STEP: 44245 + | > loss: -0.21393 (-0.20536) + | > log_mle: -0.30607 (-0.31098) + | > loss_dur: 0.09214 (0.10562) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.04709 (14.53651) + | > current_lr: 0.00005 + | > step_time: 4.10340 (3.38954) + | > loader_time: 0.00300 (0.04715) + + + --> STEP: 24/234 -- GLOBAL_STEP: 44250 + | > loss: -0.22942 (-0.20693) + | > log_mle: -0.30446 (-0.30925) + | > loss_dur: 0.07505 (0.10232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.56108 (13.50448) + | > current_lr: 0.00005 + | > step_time: 1.48340 (3.24646) + | > loader_time: 0.00190 (0.04844) + + + --> STEP: 29/234 -- GLOBAL_STEP: 44255 + | > loss: -0.17659 (-0.20622) + | > log_mle: -0.28428 (-0.30759) + | > loss_dur: 0.10770 (0.10138) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.50983 (13.11185) + | > current_lr: 0.00005 + | > step_time: 5.19370 (3.32466) + | > loader_time: 0.00150 (0.04725) + + + --> STEP: 34/234 -- GLOBAL_STEP: 44260 + | > loss: -0.17379 (-0.20383) + | > log_mle: -0.29312 (-0.30602) + | > loss_dur: 0.11933 (0.10219) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.68293 (12.87626) + | > current_lr: 0.00005 + | > step_time: 4.29620 (3.54727) + | > loader_time: 0.00230 (0.04355) + + + --> STEP: 39/234 -- GLOBAL_STEP: 44265 + | > loss: -0.17689 (-0.20161) + | > log_mle: -0.29867 (-0.30508) + | > loss_dur: 0.12178 (0.10346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.44772 (12.88137) + | > current_lr: 0.00005 + | > step_time: 2.10800 (3.66537) + | > loader_time: 0.07690 (0.05247) + + + --> STEP: 44/234 -- GLOBAL_STEP: 44270 + | > loss: -0.20780 (-0.19998) + | > log_mle: -0.29749 (-0.30383) + | > loss_dur: 0.08968 (0.10386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.59990 (12.62834) + | > current_lr: 0.00005 + | > step_time: 2.01530 (3.46252) + | > loader_time: 0.00410 (0.04860) + + + --> STEP: 49/234 -- GLOBAL_STEP: 44275 + | > loss: -0.19667 (-0.19795) + | > log_mle: -0.30166 (-0.30330) + | > loss_dur: 0.10498 (0.10535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.74506 (12.76062) + | > current_lr: 0.00005 + | > step_time: 1.08510 (3.23842) + | > loader_time: 0.00170 (0.04549) + + + --> STEP: 54/234 -- GLOBAL_STEP: 44280 + | > loss: -0.19515 (-0.19580) + | > log_mle: -0.30386 (-0.30232) + | > loss_dur: 0.10872 (0.10652) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.71859 (12.63164) + | > current_lr: 0.00005 + | > step_time: 2.60630 (3.12197) + | > loader_time: 0.00260 (0.04163) + + + --> STEP: 59/234 -- GLOBAL_STEP: 44285 + | > loss: -0.19550 (-0.19458) + | > log_mle: -0.30685 (-0.30185) + | > loss_dur: 0.11135 (0.10727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.50906 (12.49701) + | > current_lr: 0.00005 + | > step_time: 2.00090 (3.05788) + | > loader_time: 0.08720 (0.03973) + + + --> STEP: 64/234 -- GLOBAL_STEP: 44290 + | > loss: -0.18693 (-0.19204) + | > log_mle: -0.29199 (-0.30201) + | > loss_dur: 0.10507 (0.10997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.24457 (12.70057) + | > current_lr: 0.00005 + | > step_time: 1.49990 (2.93530) + | > loader_time: 0.08440 (0.03928) + + + --> STEP: 69/234 -- GLOBAL_STEP: 44295 + | > loss: -0.16342 (-0.19045) + | > log_mle: -0.27677 (-0.30114) + | > loss_dur: 0.11335 (0.11069) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.35453 (12.58847) + | > current_lr: 0.00005 + | > step_time: 1.28580 (2.89510) + | > loader_time: 0.00940 (0.04051) + + + --> STEP: 74/234 -- GLOBAL_STEP: 44300 + | > loss: -0.15676 (-0.18830) + | > log_mle: -0.28476 (-0.30090) + | > loss_dur: 0.12800 (0.11260) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.05229 (12.77823) + | > current_lr: 0.00005 + | > step_time: 1.70290 (2.84648) + | > loader_time: 0.00210 (0.03794) + + + --> STEP: 79/234 -- GLOBAL_STEP: 44305 + | > loss: -0.17545 (-0.18705) + | > log_mle: -0.30416 (-0.30084) + | > loss_dur: 0.12872 (0.11379) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.23931 (12.73301) + | > current_lr: 0.00005 + | > step_time: 2.39640 (2.83166) + | > loader_time: 0.10200 (0.03816) + + + --> STEP: 84/234 -- GLOBAL_STEP: 44310 + | > loss: -0.17389 (-0.18596) + | > log_mle: -0.29610 (-0.30075) + | > loss_dur: 0.12221 (0.11479) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.62520 (12.71902) + | > current_lr: 0.00005 + | > step_time: 2.81070 (2.81551) + | > loader_time: 0.08800 (0.03898) + + + --> STEP: 89/234 -- GLOBAL_STEP: 44315 + | > loss: -0.19895 (-0.18570) + | > log_mle: -0.32520 (-0.30155) + | > loss_dur: 0.12625 (0.11584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.06231 (12.90739) + | > current_lr: 0.00005 + | > step_time: 1.47630 (2.80175) + | > loader_time: 0.07790 (0.04079) + + + --> STEP: 94/234 -- GLOBAL_STEP: 44320 + | > loss: -0.20921 (-0.18565) + | > log_mle: -0.35860 (-0.30359) + | > loss_dur: 0.14939 (0.11793) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.56704 (13.52888) + | > current_lr: 0.00005 + | > step_time: 1.27840 (2.73739) + | > loader_time: 0.00180 (0.03875) + + + --> STEP: 99/234 -- GLOBAL_STEP: 44325 + | > loss: -0.20538 (-0.18588) + | > log_mle: -0.39146 (-0.30562) + | > loss_dur: 0.18608 (0.11974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.93828 (14.12684) + | > current_lr: 0.00005 + | > step_time: 1.44320 (2.69844) + | > loader_time: 0.07810 (0.03854) + + + --> STEP: 104/234 -- GLOBAL_STEP: 44330 + | > loss: -0.23597 (-0.18639) + | > log_mle: -0.39716 (-0.30817) + | > loss_dur: 0.16119 (0.12178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.59960 (14.79761) + | > current_lr: 0.00005 + | > step_time: 1.99730 (2.66623) + | > loader_time: 0.00370 (0.03683) + + + --> STEP: 109/234 -- GLOBAL_STEP: 44335 + | > loss: -0.16077 (-0.18592) + | > log_mle: -0.36918 (-0.30989) + | > loss_dur: 0.20841 (0.12397) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.46668 (15.17478) + | > current_lr: 0.00005 + | > step_time: 1.50840 (2.72204) + | > loader_time: 0.08360 (0.03967) + + + --> STEP: 114/234 -- GLOBAL_STEP: 44340 + | > loss: -0.20228 (-0.18628) + | > log_mle: -0.35660 (-0.31250) + | > loss_dur: 0.15432 (0.12622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.74172 (15.74382) + | > current_lr: 0.00005 + | > step_time: 0.99030 (2.68486) + | > loader_time: 0.00210 (0.03893) + + + --> STEP: 119/234 -- GLOBAL_STEP: 44345 + | > loss: -0.18598 (-0.18624) + | > log_mle: -0.35174 (-0.31452) + | > loss_dur: 0.16576 (0.12828) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.28675 (16.09737) + | > current_lr: 0.00005 + | > step_time: 2.07780 (2.64328) + | > loader_time: 0.00140 (0.03741) + + + --> STEP: 124/234 -- GLOBAL_STEP: 44350 + | > loss: -0.22430 (-0.18635) + | > log_mle: -0.38335 (-0.31584) + | > loss_dur: 0.15904 (0.12949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.05479 (16.44552) + | > current_lr: 0.00005 + | > step_time: 1.70680 (2.60873) + | > loader_time: 0.00570 (0.03676) + + + --> STEP: 129/234 -- GLOBAL_STEP: 44355 + | > loss: -0.19010 (-0.18728) + | > log_mle: -0.37466 (-0.31889) + | > loss_dur: 0.18457 (0.13161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.83044 (17.28718) + | > current_lr: 0.00005 + | > step_time: 6.19450 (2.67440) + | > loader_time: 0.00350 (0.03978) + + + --> STEP: 134/234 -- GLOBAL_STEP: 44360 + | > loss: -0.22193 (-0.18912) + | > log_mle: -0.42947 (-0.32271) + | > loss_dur: 0.20755 (0.13359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.61919 (18.10028) + | > current_lr: 0.00005 + | > step_time: 0.98810 (2.72071) + | > loader_time: 0.00220 (0.04127) + + + --> STEP: 139/234 -- GLOBAL_STEP: 44365 + | > loss: -0.29415 (-0.19056) + | > log_mle: -0.48851 (-0.32621) + | > loss_dur: 0.19436 (0.13565) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.50914 (19.09273) + | > current_lr: 0.00005 + | > step_time: 1.67790 (2.68503) + | > loader_time: 0.00180 (0.03989) + + + --> STEP: 144/234 -- GLOBAL_STEP: 44370 + | > loss: -0.26879 (-0.19216) + | > log_mle: -0.46605 (-0.33000) + | > loss_dur: 0.19726 (0.13785) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.32093 (20.07415) + | > current_lr: 0.00005 + | > step_time: 1.77790 (2.74298) + | > loader_time: 0.00270 (0.04004) + + + --> STEP: 149/234 -- GLOBAL_STEP: 44375 + | > loss: -0.31532 (-0.19479) + | > log_mle: -0.52035 (-0.33442) + | > loss_dur: 0.20503 (0.13963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.78455 (21.03461) + | > current_lr: 0.00005 + | > step_time: 3.30530 (2.71330) + | > loader_time: 0.00370 (0.03936) + + + --> STEP: 154/234 -- GLOBAL_STEP: 44380 + | > loss: -0.27551 (-0.19768) + | > log_mle: -0.46854 (-0.33928) + | > loss_dur: 0.19303 (0.14160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.82882 (22.23446) + | > current_lr: 0.00005 + | > step_time: 1.89840 (2.71103) + | > loader_time: 0.00300 (0.03883) + + + --> STEP: 159/234 -- GLOBAL_STEP: 44385 + | > loss: -0.27805 (-0.20002) + | > log_mle: -0.48782 (-0.34374) + | > loss_dur: 0.20977 (0.14372) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.68659 (23.58809) + | > current_lr: 0.00005 + | > step_time: 2.69680 (2.70832) + | > loader_time: 0.00310 (0.03880) + + + --> STEP: 164/234 -- GLOBAL_STEP: 44390 + | > loss: -0.26797 (-0.20254) + | > log_mle: -0.47999 (-0.34806) + | > loss_dur: 0.21203 (0.14552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.05173 (24.70043) + | > current_lr: 0.00005 + | > step_time: 2.70240 (2.68672) + | > loader_time: 0.00560 (0.03945) + + + --> STEP: 169/234 -- GLOBAL_STEP: 44395 + | > loss: -0.26334 (-0.20536) + | > log_mle: -0.47950 (-0.35267) + | > loss_dur: 0.21616 (0.14731) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.34140 (25.80180) + | > current_lr: 0.00005 + | > step_time: 5.39980 (2.77046) + | > loader_time: 0.00370 (0.03908) + + + --> STEP: 174/234 -- GLOBAL_STEP: 44400 + | > loss: -0.35537 (-0.20892) + | > log_mle: -0.57218 (-0.35835) + | > loss_dur: 0.21680 (0.14943) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.63255 (27.07633) + | > current_lr: 0.00005 + | > step_time: 1.49890 (2.75980) + | > loader_time: 0.00620 (0.03860) + + + --> STEP: 179/234 -- GLOBAL_STEP: 44405 + | > loss: -0.31328 (-0.21183) + | > log_mle: -0.56232 (-0.36349) + | > loss_dur: 0.24904 (0.15166) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.31152 (28.28532) + | > current_lr: 0.00005 + | > step_time: 3.48510 (2.75411) + | > loader_time: 0.00350 (0.03808) + + + --> STEP: 184/234 -- GLOBAL_STEP: 44410 + | > loss: -0.30945 (-0.21444) + | > log_mle: -0.52673 (-0.36801) + | > loss_dur: 0.21728 (0.15356) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.65125 (29.41393) + | > current_lr: 0.00005 + | > step_time: 1.89870 (2.72985) + | > loader_time: 0.00410 (0.03755) + + + --> STEP: 189/234 -- GLOBAL_STEP: 44415 + | > loss: -0.29152 (-0.21706) + | > log_mle: -0.51637 (-0.37259) + | > loss_dur: 0.22485 (0.15553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.93578 (30.71746) + | > current_lr: 0.00005 + | > step_time: 8.28890 (2.75706) + | > loader_time: 0.01050 (0.03714) + + + --> STEP: 194/234 -- GLOBAL_STEP: 44420 + | > loss: -0.33695 (-0.22021) + | > log_mle: -0.55443 (-0.37728) + | > loss_dur: 0.21748 (0.15707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.86428 (31.90487) + | > current_lr: 0.00005 + | > step_time: 12.20010 (2.86119) + | > loader_time: 0.10460 (0.03931) + + + --> STEP: 199/234 -- GLOBAL_STEP: 44425 + | > loss: -0.32376 (-0.22273) + | > log_mle: -0.55576 (-0.38149) + | > loss_dur: 0.23200 (0.15876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.76839 (33.22226) + | > current_lr: 0.00005 + | > step_time: 3.51740 (2.88536) + | > loader_time: 0.08830 (0.04068) + + + --> STEP: 204/234 -- GLOBAL_STEP: 44430 + | > loss: -0.36540 (-0.22514) + | > log_mle: -0.60740 (-0.38581) + | > loss_dur: 0.24200 (0.16067) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.85544 (34.06864) + | > current_lr: 0.00005 + | > step_time: 1.90270 (2.89609) + | > loader_time: 0.10150 (0.04302) + + + --> STEP: 209/234 -- GLOBAL_STEP: 44435 + | > loss: -0.32619 (-0.22796) + | > log_mle: -0.55545 (-0.39039) + | > loss_dur: 0.22926 (0.16243) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.17159 (35.19329) + | > current_lr: 0.00005 + | > step_time: 5.20170 (2.97611) + | > loader_time: 0.10250 (0.04351) + + + --> STEP: 214/234 -- GLOBAL_STEP: 44440 + | > loss: -0.39112 (-0.23158) + | > log_mle: -0.59937 (-0.39581) + | > loss_dur: 0.20825 (0.16423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.59164 (36.43415) + | > current_lr: 0.00005 + | > step_time: 5.60770 (3.02959) + | > loader_time: 0.08370 (0.04372) + + + --> STEP: 219/234 -- GLOBAL_STEP: 44445 + | > loss: -0.45613 (-0.23510) + | > log_mle: -0.69291 (-0.40111) + | > loss_dur: 0.23679 (0.16600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 121.26505 (37.80927) + | > current_lr: 0.00005 + | > step_time: 4.99820 (3.11116) + | > loader_time: 0.07920 (0.04578) + + + --> STEP: 224/234 -- GLOBAL_STEP: 44450 + | > loss: -0.40632 (-0.23834) + | > log_mle: -0.64393 (-0.40602) + | > loss_dur: 0.23760 (0.16768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.15721 (38.94136) + | > current_lr: 0.00005 + | > step_time: 0.25420 (3.06983) + | > loader_time: 0.00300 (0.04591) + + + --> STEP: 229/234 -- GLOBAL_STEP: 44455 + | > loss: -0.38004 (-0.24173) + | > log_mle: -0.68878 (-0.41158) + | > loss_dur: 0.30873 (0.16984) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.39745 (40.13341) + | > current_lr: 0.00005 + | > step_time: 0.24640 (3.00803) + | > loader_time: 0.00430 (0.04498) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.25785 (+1.02917) + | > avg_loss: -0.23921 (+0.03488) + | > avg_log_mle: -0.47330 (+0.02841) + | > avg_loss_dur: 0.23409 (+0.00647) + + + > EPOCH: 190/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 00:43:15)  + + --> STEP: 0/234 -- GLOBAL_STEP: 44460 + | > loss: -0.23215 (-0.23215) + | > log_mle: -0.38699 (-0.38699) + | > loss_dur: 0.15484 (0.15484) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.11184 (23.11184) + | > current_lr: 0.00005 + | > step_time: 5.61010 (5.61012) + | > loader_time: 12.96400 (12.96399) + + + --> STEP: 5/234 -- GLOBAL_STEP: 44465 + | > loss: -0.20694 (-0.18464) + | > log_mle: -0.31167 (-0.31024) + | > loss_dur: 0.10474 (0.12561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.65927 (19.69923) + | > current_lr: 0.00005 + | > step_time: 6.90420 (5.06096) + | > loader_time: 0.79820 (0.20000) + + + --> STEP: 10/234 -- GLOBAL_STEP: 44470 + | > loss: -0.19739 (-0.19755) + | > log_mle: -0.30439 (-0.31236) + | > loss_dur: 0.10701 (0.11481) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.55347 (17.56348) + | > current_lr: 0.00005 + | > step_time: 7.40550 (4.61064) + | > loader_time: 0.00520 (0.13106) + + + --> STEP: 15/234 -- GLOBAL_STEP: 44475 + | > loss: -0.21844 (-0.20439) + | > log_mle: -0.31485 (-0.31299) + | > loss_dur: 0.09641 (0.10860) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.92523 (15.88689) + | > current_lr: 0.00005 + | > step_time: 4.90880 (5.26113) + | > loader_time: 0.19100 (0.14002) + + + --> STEP: 20/234 -- GLOBAL_STEP: 44480 + | > loss: -0.21646 (-0.20664) + | > log_mle: -0.30790 (-0.31070) + | > loss_dur: 0.09144 (0.10407) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.51919 (14.45444) + | > current_lr: 0.00005 + | > step_time: 3.09070 (5.19116) + | > loader_time: 0.00230 (0.10568) + + + --> STEP: 25/234 -- GLOBAL_STEP: 44485 + | > loss: -0.20298 (-0.20766) + | > log_mle: -0.29053 (-0.30875) + | > loss_dur: 0.08755 (0.10109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.17477 (13.82232) + | > current_lr: 0.00005 + | > step_time: 1.68790 (4.72507) + | > loader_time: 0.00170 (0.10056) + + + --> STEP: 30/234 -- GLOBAL_STEP: 44490 + | > loss: -0.21601 (-0.20755) + | > log_mle: -0.30331 (-0.30776) + | > loss_dur: 0.08730 (0.10021) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.89620 (13.37458) + | > current_lr: 0.00005 + | > step_time: 5.48100 (4.83183) + | > loader_time: 0.00120 (0.09242) + + + --> STEP: 35/234 -- GLOBAL_STEP: 44495 + | > loss: -0.17756 (-0.20508) + | > log_mle: -0.29661 (-0.30656) + | > loss_dur: 0.11905 (0.10148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.04255 (13.39497) + | > current_lr: 0.00005 + | > step_time: 6.80100 (4.71540) + | > loader_time: 0.00380 (0.09347) + + + --> STEP: 40/234 -- GLOBAL_STEP: 44500 + | > loss: -0.15694 (-0.20250) + | > log_mle: -0.28440 (-0.30535) + | > loss_dur: 0.12746 (0.10285) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.48257 (13.32361) + | > current_lr: 0.00005 + | > step_time: 0.85050 (4.37571) + | > loader_time: 0.00220 (0.08218) + + + --> STEP: 45/234 -- GLOBAL_STEP: 44505 + | > loss: -0.19097 (-0.20128) + | > log_mle: -0.31783 (-0.30502) + | > loss_dur: 0.12686 (0.10373) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.44479 (13.11332) + | > current_lr: 0.00005 + | > step_time: 0.89260 (4.06444) + | > loader_time: 0.00240 (0.07331) + + + --> STEP: 50/234 -- GLOBAL_STEP: 44510 + | > loss: -0.18381 (-0.20001) + | > log_mle: -0.28952 (-0.30413) + | > loss_dur: 0.10571 (0.10412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.17289 (12.59710) + | > current_lr: 0.00005 + | > step_time: 0.88260 (3.80835) + | > loader_time: 0.00450 (0.06949) + + + --> STEP: 55/234 -- GLOBAL_STEP: 44515 + | > loss: -0.20077 (-0.19848) + | > log_mle: -0.30428 (-0.30352) + | > loss_dur: 0.10352 (0.10503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.88244 (12.46447) + | > current_lr: 0.00005 + | > step_time: 2.40800 (3.61394) + | > loader_time: 0.07540 (0.06758) + + + --> STEP: 60/234 -- GLOBAL_STEP: 44520 + | > loss: -0.17900 (-0.19670) + | > log_mle: -0.31747 (-0.30332) + | > loss_dur: 0.13847 (0.10661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.97968 (12.38416) + | > current_lr: 0.00005 + | > step_time: 3.30390 (3.52336) + | > loader_time: 0.00310 (0.06368) + + + --> STEP: 65/234 -- GLOBAL_STEP: 44525 + | > loss: -0.17173 (-0.19467) + | > log_mle: -0.29572 (-0.30328) + | > loss_dur: 0.12398 (0.10861) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.36350 (12.35668) + | > current_lr: 0.00005 + | > step_time: 1.54650 (3.38804) + | > loader_time: 0.09050 (0.06158) + + + --> STEP: 70/234 -- GLOBAL_STEP: 44530 + | > loss: -0.15037 (-0.19255) + | > log_mle: -0.28487 (-0.30235) + | > loss_dur: 0.13450 (0.10981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.10555 (12.30980) + | > current_lr: 0.00005 + | > step_time: 3.96020 (3.28157) + | > loader_time: 0.00360 (0.05736) + + + --> STEP: 75/234 -- GLOBAL_STEP: 44535 + | > loss: -0.15812 (-0.19032) + | > log_mle: -0.29681 (-0.30200) + | > loss_dur: 0.13869 (0.11168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.05144 (12.49261) + | > current_lr: 0.00005 + | > step_time: 1.17080 (3.17212) + | > loader_time: 0.00230 (0.05371) + + + --> STEP: 80/234 -- GLOBAL_STEP: 44540 + | > loss: -0.17018 (-0.18907) + | > log_mle: -0.28455 (-0.30147) + | > loss_dur: 0.11436 (0.11240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.84441 (12.42199) + | > current_lr: 0.00005 + | > step_time: 1.09190 (3.06984) + | > loader_time: 0.00270 (0.05050) + + + --> STEP: 85/234 -- GLOBAL_STEP: 44545 + | > loss: -0.17677 (-0.18803) + | > log_mle: -0.29918 (-0.30148) + | > loss_dur: 0.12240 (0.11345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.69672 (12.51920) + | > current_lr: 0.00005 + | > step_time: 1.29420 (2.98680) + | > loader_time: 0.00230 (0.04870) + + + --> STEP: 90/234 -- GLOBAL_STEP: 44550 + | > loss: -0.17271 (-0.18716) + | > log_mle: -0.32170 (-0.30238) + | > loss_dur: 0.14899 (0.11522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.40657 (12.87349) + | > current_lr: 0.00005 + | > step_time: 2.18050 (2.91411) + | > loader_time: 0.00230 (0.04715) + + + --> STEP: 95/234 -- GLOBAL_STEP: 44555 + | > loss: -0.22202 (-0.18774) + | > log_mle: -0.40461 (-0.30519) + | > loss_dur: 0.18259 (0.11745) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.18379 (13.59244) + | > current_lr: 0.00005 + | > step_time: 1.30150 (2.87267) + | > loader_time: 0.00450 (0.04576) + + + --> STEP: 100/234 -- GLOBAL_STEP: 44560 + | > loss: -0.19221 (-0.18741) + | > log_mle: -0.33448 (-0.30641) + | > loss_dur: 0.14227 (0.11901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.87045 (13.80826) + | > current_lr: 0.00005 + | > step_time: 2.19460 (2.86423) + | > loader_time: 0.00300 (0.04632) + + + --> STEP: 105/234 -- GLOBAL_STEP: 44565 + | > loss: -0.18221 (-0.18793) + | > log_mle: -0.31402 (-0.30889) + | > loss_dur: 0.13181 (0.12096) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18195 (14.28265) + | > current_lr: 0.00005 + | > step_time: 4.82400 (2.92729) + | > loader_time: 0.19440 (0.04966) + + + --> STEP: 110/234 -- GLOBAL_STEP: 44570 + | > loss: -0.17389 (-0.18768) + | > log_mle: -0.33890 (-0.31098) + | > loss_dur: 0.16501 (0.12330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.84477 (14.70054) + | > current_lr: 0.00005 + | > step_time: 5.29870 (2.91884) + | > loader_time: 0.09420 (0.04837) + + + --> STEP: 115/234 -- GLOBAL_STEP: 44575 + | > loss: -0.17973 (-0.18822) + | > log_mle: -0.36082 (-0.31388) + | > loss_dur: 0.18109 (0.12566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.56203 (15.40361) + | > current_lr: 0.00005 + | > step_time: 1.41780 (2.91802) + | > loader_time: 0.00640 (0.04795) + + + --> STEP: 120/234 -- GLOBAL_STEP: 44580 + | > loss: -0.22990 (-0.18859) + | > log_mle: -0.40959 (-0.31633) + | > loss_dur: 0.17969 (0.12774) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.19343 (16.04288) + | > current_lr: 0.00005 + | > step_time: 4.59230 (2.89696) + | > loader_time: 0.10260 (0.04764) + + + --> STEP: 125/234 -- GLOBAL_STEP: 44585 + | > loss: -0.20750 (-0.18839) + | > log_mle: -0.39362 (-0.31749) + | > loss_dur: 0.18612 (0.12910) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.80224 (16.38716) + | > current_lr: 0.00005 + | > step_time: 2.00450 (2.85384) + | > loader_time: 0.08520 (0.04654) + + + --> STEP: 130/234 -- GLOBAL_STEP: 44590 + | > loss: -0.22738 (-0.18945) + | > log_mle: -0.40782 (-0.32051) + | > loss_dur: 0.18043 (0.13106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.97811 (17.34128) + | > current_lr: 0.00005 + | > step_time: 3.99890 (2.83027) + | > loader_time: 0.00480 (0.04675) + + + --> STEP: 135/234 -- GLOBAL_STEP: 44595 + | > loss: -0.18308 (-0.19086) + | > log_mle: -0.33556 (-0.32367) + | > loss_dur: 0.15249 (0.13281) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.00736 (18.01031) + | > current_lr: 0.00005 + | > step_time: 1.39710 (2.81359) + | > loader_time: 0.00320 (0.04656) + + + --> STEP: 140/234 -- GLOBAL_STEP: 44600 + | > loss: -0.18498 (-0.19234) + | > log_mle: -0.37267 (-0.32741) + | > loss_dur: 0.18769 (0.13507) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.10720 (18.89421) + | > current_lr: 0.00005 + | > step_time: 2.01480 (2.79112) + | > loader_time: 0.08730 (0.04622) + + + --> STEP: 145/234 -- GLOBAL_STEP: 44605 + | > loss: -0.27970 (-0.19448) + | > log_mle: -0.47434 (-0.33184) + | > loss_dur: 0.19464 (0.13736) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.36227 (20.03379) + | > current_lr: 0.00005 + | > step_time: 1.39530 (2.76932) + | > loader_time: 0.00200 (0.04653) + + + --> STEP: 150/234 -- GLOBAL_STEP: 44610 + | > loss: -0.25115 (-0.19682) + | > log_mle: -0.45365 (-0.33587) + | > loss_dur: 0.20250 (0.13905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.82542 (21.01930) + | > current_lr: 0.00005 + | > step_time: 2.78890 (2.77817) + | > loader_time: 0.00760 (0.04574) + + + --> STEP: 155/234 -- GLOBAL_STEP: 44615 + | > loss: -0.31034 (-0.19982) + | > log_mle: -0.50895 (-0.34084) + | > loss_dur: 0.19861 (0.14102) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.88756 (22.31705) + | > current_lr: 0.00005 + | > step_time: 1.69370 (2.75100) + | > loader_time: 0.00290 (0.04491) + + + --> STEP: 160/234 -- GLOBAL_STEP: 44620 + | > loss: -0.30559 (-0.20209) + | > log_mle: -0.51731 (-0.34518) + | > loss_dur: 0.21172 (0.14310) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.38791 (23.31140) + | > current_lr: 0.00005 + | > step_time: 1.50590 (2.73691) + | > loader_time: 0.00290 (0.04472) + + + --> STEP: 165/234 -- GLOBAL_STEP: 44625 + | > loss: -0.30073 (-0.20446) + | > log_mle: -0.51395 (-0.34935) + | > loss_dur: 0.21322 (0.14490) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.27295 (24.33681) + | > current_lr: 0.00005 + | > step_time: 3.59070 (2.72894) + | > loader_time: 0.00250 (0.04347) + + + --> STEP: 170/234 -- GLOBAL_STEP: 44630 + | > loss: -0.32221 (-0.20724) + | > log_mle: -0.55318 (-0.35405) + | > loss_dur: 0.23097 (0.14681) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.82967 (25.50520) + | > current_lr: 0.00005 + | > step_time: 4.60340 (2.73867) + | > loader_time: 0.00400 (0.04232) + + + --> STEP: 175/234 -- GLOBAL_STEP: 44635 + | > loss: -0.29288 (-0.21052) + | > log_mle: -0.52692 (-0.35943) + | > loss_dur: 0.23405 (0.14891) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.97977 (26.80286) + | > current_lr: 0.00005 + | > step_time: 3.89390 (2.75116) + | > loader_time: 0.00300 (0.04341) + + + --> STEP: 180/234 -- GLOBAL_STEP: 44640 + | > loss: -0.32080 (-0.21342) + | > log_mle: -0.53901 (-0.36448) + | > loss_dur: 0.21821 (0.15105) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.24839 (27.94953) + | > current_lr: 0.00005 + | > step_time: 2.29720 (2.77748) + | > loader_time: 0.10200 (0.04345) + + + --> STEP: 185/234 -- GLOBAL_STEP: 44645 + | > loss: -0.33391 (-0.21597) + | > log_mle: -0.56750 (-0.36902) + | > loss_dur: 0.23359 (0.15305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.62789 (29.14931) + | > current_lr: 0.00005 + | > step_time: 4.89390 (2.85312) + | > loader_time: 0.19920 (0.04502) + + + --> STEP: 190/234 -- GLOBAL_STEP: 44650 + | > loss: -0.32828 (-0.21849) + | > log_mle: -0.53508 (-0.37338) + | > loss_dur: 0.20680 (0.15489) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.16793 (30.35494) + | > current_lr: 0.00005 + | > step_time: 2.80840 (2.92929) + | > loader_time: 0.08890 (0.04645) + + + --> STEP: 195/234 -- GLOBAL_STEP: 44655 + | > loss: -0.32538 (-0.22152) + | > log_mle: -0.55922 (-0.37809) + | > loss_dur: 0.23384 (0.15657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.94650 (31.43990) + | > current_lr: 0.00005 + | > step_time: 6.79030 (2.99176) + | > loader_time: 0.40050 (0.04933) + + + --> STEP: 200/234 -- GLOBAL_STEP: 44660 + | > loss: -0.30623 (-0.22422) + | > log_mle: -0.55273 (-0.38254) + | > loss_dur: 0.24650 (0.15832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.34792 (32.79094) + | > current_lr: 0.00005 + | > step_time: 4.30780 (3.00450) + | > loader_time: 0.09390 (0.05017) + + + --> STEP: 205/234 -- GLOBAL_STEP: 44665 + | > loss: -0.31573 (-0.22664) + | > log_mle: -0.55084 (-0.38674) + | > loss_dur: 0.23511 (0.16009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.39571 (33.72119) + | > current_lr: 0.00005 + | > step_time: 11.00380 (3.10478) + | > loader_time: 0.19200 (0.05141) + + + --> STEP: 210/234 -- GLOBAL_STEP: 44670 + | > loss: -0.39243 (-0.22989) + | > log_mle: -0.63403 (-0.39178) + | > loss_dur: 0.24160 (0.16189) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.38429 (34.88050) + | > current_lr: 0.00005 + | > step_time: 3.90420 (3.14519) + | > loader_time: 0.08560 (0.05332) + + + --> STEP: 215/234 -- GLOBAL_STEP: 44675 + | > loss: -0.34725 (-0.23347) + | > log_mle: -0.58611 (-0.39709) + | > loss_dur: 0.23886 (0.16363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.46929 (36.05799) + | > current_lr: 0.00005 + | > step_time: 4.20070 (3.21949) + | > loader_time: 0.69600 (0.05671) + + + --> STEP: 220/234 -- GLOBAL_STEP: 44680 + | > loss: -0.38977 (-0.23732) + | > log_mle: -0.63527 (-0.40274) + | > loss_dur: 0.24550 (0.16542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.52116 (37.31587) + | > current_lr: 0.00005 + | > step_time: 3.01830 (3.26795) + | > loader_time: 0.09040 (0.05630) + + + --> STEP: 225/234 -- GLOBAL_STEP: 44685 + | > loss: -0.44057 (-0.24078) + | > log_mle: -0.69951 (-0.40801) + | > loss_dur: 0.25894 (0.16723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.36851 (38.66460) + | > current_lr: 0.00005 + | > step_time: 0.23680 (3.22780) + | > loader_time: 0.00500 (0.05516) + + + --> STEP: 230/234 -- GLOBAL_STEP: 44690 + | > loss: -0.42998 (-0.24377) + | > log_mle: -0.74920 (-0.41348) + | > loss_dur: 0.31922 (0.16971) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.94465 (40.21861) + | > current_lr: 0.00005 + | > step_time: 0.25530 (3.16297) + | > loader_time: 0.00310 (0.05403) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.02347 (-0.23438) + | > avg_loss: -0.26577 (-0.02656) + | > avg_log_mle: -0.47893 (-0.00563) + | > avg_loss_dur: 0.21316 (-0.02093) + + + > EPOCH: 191/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 00:56:45)  + + --> STEP: 1/234 -- GLOBAL_STEP: 44695 + | > loss: -0.19569 (-0.19569) + | > log_mle: -0.30886 (-0.30886) + | > loss_dur: 0.11318 (0.11318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.84603 (19.84603) + | > current_lr: 0.00005 + | > step_time: 8.09380 (8.09375) + | > loader_time: 0.00150 (0.00146) + + + --> STEP: 6/234 -- GLOBAL_STEP: 44700 + | > loss: -0.21533 (-0.19801) + | > log_mle: -0.30497 (-0.30965) + | > loss_dur: 0.08964 (0.11164) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.22030 (17.87932) + | > current_lr: 0.00005 + | > step_time: 1.69340 (4.68338) + | > loader_time: 0.00240 (0.02674) + + + --> STEP: 11/234 -- GLOBAL_STEP: 44705 + | > loss: -0.22666 (-0.20328) + | > log_mle: -0.31512 (-0.31301) + | > loss_dur: 0.08846 (0.10973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.58357 (16.26713) + | > current_lr: 0.00005 + | > step_time: 0.66910 (2.89799) + | > loader_time: 0.00130 (0.02241) + + + --> STEP: 16/234 -- GLOBAL_STEP: 44710 + | > loss: -0.21987 (-0.20640) + | > log_mle: -0.30978 (-0.31352) + | > loss_dur: 0.08991 (0.10712) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.73080 (15.11019) + | > current_lr: 0.00005 + | > step_time: 6.01300 (3.27539) + | > loader_time: 0.08760 (0.02711) + + + --> STEP: 21/234 -- GLOBAL_STEP: 44715 + | > loss: -0.19836 (-0.20751) + | > log_mle: -0.28609 (-0.31031) + | > loss_dur: 0.08773 (0.10280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.59188 (14.19294) + | > current_lr: 0.00005 + | > step_time: 2.28980 (2.84950) + | > loader_time: 0.00260 (0.02120) + + + --> STEP: 26/234 -- GLOBAL_STEP: 44720 + | > loss: -0.19700 (-0.20854) + | > log_mle: -0.30429 (-0.30926) + | > loss_dur: 0.10730 (0.10072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18775 (13.43782) + | > current_lr: 0.00005 + | > step_time: 2.89400 (2.78128) + | > loader_time: 0.09600 (0.02462) + + + --> STEP: 31/234 -- GLOBAL_STEP: 44725 + | > loss: -0.16699 (-0.20785) + | > log_mle: -0.29724 (-0.30821) + | > loss_dur: 0.13025 (0.10036) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.64396 (13.05292) + | > current_lr: 0.00005 + | > step_time: 0.84480 (2.71456) + | > loader_time: 0.00210 (0.03682) + + + --> STEP: 36/234 -- GLOBAL_STEP: 44730 + | > loss: -0.17635 (-0.20526) + | > log_mle: -0.29156 (-0.30687) + | > loss_dur: 0.11521 (0.10161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.01277 (13.21090) + | > current_lr: 0.00005 + | > step_time: 1.78280 (2.63225) + | > loader_time: 0.00190 (0.03214) + + + --> STEP: 41/234 -- GLOBAL_STEP: 44735 + | > loss: -0.20528 (-0.20309) + | > log_mle: -0.30034 (-0.30571) + | > loss_dur: 0.09506 (0.10262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.42135 (12.95274) + | > current_lr: 0.00005 + | > step_time: 1.19690 (2.58466) + | > loader_time: 0.00140 (0.03965) + + + --> STEP: 46/234 -- GLOBAL_STEP: 44740 + | > loss: -0.17972 (-0.20130) + | > log_mle: -0.29652 (-0.30523) + | > loss_dur: 0.11680 (0.10393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.32683 (12.74032) + | > current_lr: 0.00005 + | > step_time: 1.14100 (2.46726) + | > loader_time: 0.00180 (0.03558) + + + --> STEP: 51/234 -- GLOBAL_STEP: 44745 + | > loss: -0.17766 (-0.20024) + | > log_mle: -0.28927 (-0.30433) + | > loss_dur: 0.11161 (0.10409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.92264 (12.29960) + | > current_lr: 0.00005 + | > step_time: 3.15300 (2.46367) + | > loader_time: 0.18980 (0.03762) + + + --> STEP: 56/234 -- GLOBAL_STEP: 44750 + | > loss: -0.19067 (-0.19902) + | > log_mle: -0.30448 (-0.30416) + | > loss_dur: 0.11381 (0.10515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.86973 (12.19613) + | > current_lr: 0.00005 + | > step_time: 1.66120 (2.41529) + | > loader_time: 0.00220 (0.03443) + + + --> STEP: 61/234 -- GLOBAL_STEP: 44755 + | > loss: -0.18827 (-0.19802) + | > log_mle: -0.29851 (-0.30394) + | > loss_dur: 0.11024 (0.10592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.12497 (12.14065) + | > current_lr: 0.00005 + | > step_time: 2.09370 (2.38032) + | > loader_time: 0.00210 (0.03177) + + + --> STEP: 66/234 -- GLOBAL_STEP: 44760 + | > loss: -0.18955 (-0.19608) + | > log_mle: -0.29183 (-0.30376) + | > loss_dur: 0.10228 (0.10768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76399 (12.11832) + | > current_lr: 0.00005 + | > step_time: 3.43660 (2.35796) + | > loader_time: 0.10010 (0.03389) + + + --> STEP: 71/234 -- GLOBAL_STEP: 44765 + | > loss: -0.15784 (-0.19359) + | > log_mle: -0.31768 (-0.30328) + | > loss_dur: 0.15984 (0.10968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.98045 (12.25469) + | > current_lr: 0.00005 + | > step_time: 3.10490 (2.38302) + | > loader_time: 0.00180 (0.03427) + + + --> STEP: 76/234 -- GLOBAL_STEP: 44770 + | > loss: -0.17525 (-0.19160) + | > log_mle: -0.30214 (-0.30295) + | > loss_dur: 0.12689 (0.11135) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.76158 (12.44893) + | > current_lr: 0.00005 + | > step_time: 1.90250 (2.34913) + | > loader_time: 0.00350 (0.03218) + + + --> STEP: 81/234 -- GLOBAL_STEP: 44775 + | > loss: -0.18036 (-0.19064) + | > log_mle: -0.31374 (-0.30266) + | > loss_dur: 0.13339 (0.11201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.28946 (12.30737) + | > current_lr: 0.00005 + | > step_time: 2.03190 (2.33352) + | > loader_time: 0.00160 (0.03127) + + + --> STEP: 86/234 -- GLOBAL_STEP: 44780 + | > loss: -0.16867 (-0.18940) + | > log_mle: -0.30930 (-0.30263) + | > loss_dur: 0.14062 (0.11323) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.49244 (12.36513) + | > current_lr: 0.00005 + | > step_time: 1.00400 (2.28070) + | > loader_time: 0.08760 (0.03058) + + + --> STEP: 91/234 -- GLOBAL_STEP: 44785 + | > loss: -0.16284 (-0.18864) + | > log_mle: -0.32303 (-0.30377) + | > loss_dur: 0.16019 (0.11513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.17790 (12.55598) + | > current_lr: 0.00005 + | > step_time: 2.60260 (2.28627) + | > loader_time: 0.07550 (0.03174) + + + --> STEP: 96/234 -- GLOBAL_STEP: 44790 + | > loss: -0.17037 (-0.18932) + | > log_mle: -0.30847 (-0.30648) + | > loss_dur: 0.13810 (0.11716) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.86533 (13.05827) + | > current_lr: 0.00005 + | > step_time: 1.80060 (2.33113) + | > loader_time: 0.00210 (0.03368) + + + --> STEP: 101/234 -- GLOBAL_STEP: 44795 + | > loss: -0.19330 (-0.18921) + | > log_mle: -0.36576 (-0.30832) + | > loss_dur: 0.17246 (0.11911) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.82044 (13.43438) + | > current_lr: 0.00005 + | > step_time: 1.68020 (2.35136) + | > loader_time: 0.00210 (0.03689) + + + --> STEP: 106/234 -- GLOBAL_STEP: 44800 + | > loss: -0.18103 (-0.18941) + | > log_mle: -0.36146 (-0.31068) + | > loss_dur: 0.18043 (0.12127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.64132 (13.96317) + | > current_lr: 0.00005 + | > step_time: 1.30080 (2.32061) + | > loader_time: 0.00200 (0.03533) + + + --> STEP: 111/234 -- GLOBAL_STEP: 44805 + | > loss: -0.21398 (-0.18932) + | > log_mle: -0.41361 (-0.31307) + | > loss_dur: 0.19963 (0.12375) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.41756 (14.62012) + | > current_lr: 0.00005 + | > step_time: 3.30850 (2.31882) + | > loader_time: 0.00490 (0.03389) + + + --> STEP: 116/234 -- GLOBAL_STEP: 44810 + | > loss: -0.18371 (-0.18939) + | > log_mle: -0.37963 (-0.31555) + | > loss_dur: 0.19592 (0.12616) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.35750 (15.24279) + | > current_lr: 0.00005 + | > step_time: 1.20360 (2.31032) + | > loader_time: 0.08790 (0.03487) + + + --> STEP: 121/234 -- GLOBAL_STEP: 44815 + | > loss: -0.15434 (-0.18960) + | > log_mle: -0.29399 (-0.31729) + | > loss_dur: 0.13965 (0.12769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.58322 (15.52221) + | > current_lr: 0.00005 + | > step_time: 1.41600 (2.27259) + | > loader_time: 0.08470 (0.03494) + + + --> STEP: 126/234 -- GLOBAL_STEP: 44820 + | > loss: -0.23269 (-0.19015) + | > log_mle: -0.43102 (-0.31963) + | > loss_dur: 0.19833 (0.12948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.60760 (16.12190) + | > current_lr: 0.00005 + | > step_time: 0.99350 (2.29840) + | > loader_time: 0.00390 (0.03517) + + + --> STEP: 131/234 -- GLOBAL_STEP: 44825 + | > loss: -0.28773 (-0.19156) + | > log_mle: -0.48210 (-0.32314) + | > loss_dur: 0.19437 (0.13159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.93820 (17.00214) + | > current_lr: 0.00005 + | > step_time: 5.14600 (2.30205) + | > loader_time: 0.09510 (0.03527) + + + --> STEP: 136/234 -- GLOBAL_STEP: 44830 + | > loss: -0.30898 (-0.19310) + | > log_mle: -0.52157 (-0.32645) + | > loss_dur: 0.21260 (0.13335) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.75993 (18.07110) + | > current_lr: 0.00005 + | > step_time: 1.66880 (2.29417) + | > loader_time: 0.00450 (0.03484) + + + --> STEP: 141/234 -- GLOBAL_STEP: 44835 + | > loss: -0.24441 (-0.19406) + | > log_mle: -0.42397 (-0.32933) + | > loss_dur: 0.17956 (0.13527) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.14130 (18.79011) + | > current_lr: 0.00005 + | > step_time: 4.49490 (2.29771) + | > loader_time: 0.00750 (0.03429) + + + --> STEP: 146/234 -- GLOBAL_STEP: 44840 + | > loss: -0.28106 (-0.19649) + | > log_mle: -0.47815 (-0.33404) + | > loss_dur: 0.19710 (0.13755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.90780 (19.87692) + | > current_lr: 0.00005 + | > step_time: 2.60550 (2.30417) + | > loader_time: 0.09230 (0.03570) + + + --> STEP: 151/234 -- GLOBAL_STEP: 44845 + | > loss: -0.25772 (-0.19857) + | > log_mle: -0.44112 (-0.33790) + | > loss_dur: 0.18341 (0.13933) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.58232 (20.87700) + | > current_lr: 0.00005 + | > step_time: 1.58640 (2.31063) + | > loader_time: 0.00460 (0.03582) + + + --> STEP: 156/234 -- GLOBAL_STEP: 44850 + | > loss: -0.29525 (-0.20191) + | > log_mle: -0.48837 (-0.34325) + | > loss_dur: 0.19313 (0.14134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.95191 (22.41661) + | > current_lr: 0.00005 + | > step_time: 2.40380 (2.29231) + | > loader_time: 0.00370 (0.03477) + + + --> STEP: 161/234 -- GLOBAL_STEP: 44855 + | > loss: -0.31200 (-0.20436) + | > log_mle: -0.50734 (-0.34780) + | > loss_dur: 0.19535 (0.14344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.10995 (23.72284) + | > current_lr: 0.00005 + | > step_time: 3.01700 (2.30453) + | > loader_time: 0.00590 (0.03489) + + + --> STEP: 166/234 -- GLOBAL_STEP: 44860 + | > loss: -0.26577 (-0.20646) + | > log_mle: -0.44904 (-0.35161) + | > loss_dur: 0.18326 (0.14514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.83910 (24.65824) + | > current_lr: 0.00005 + | > step_time: 2.39460 (2.31087) + | > loader_time: 0.19110 (0.03565) + + + --> STEP: 171/234 -- GLOBAL_STEP: 44865 + | > loss: -0.34703 (-0.20984) + | > log_mle: -0.55483 (-0.35699) + | > loss_dur: 0.20780 (0.14715) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.57507 (26.01255) + | > current_lr: 0.00005 + | > step_time: 3.19830 (2.38548) + | > loader_time: 0.01060 (0.03643) + + + --> STEP: 176/234 -- GLOBAL_STEP: 44870 + | > loss: -0.31528 (-0.21294) + | > log_mle: -0.52599 (-0.36220) + | > loss_dur: 0.21072 (0.14925) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.91643 (27.25003) + | > current_lr: 0.00005 + | > step_time: 2.11940 (2.47524) + | > loader_time: 0.00380 (0.03823) + + + --> STEP: 181/234 -- GLOBAL_STEP: 44875 + | > loss: -0.25562 (-0.21554) + | > log_mle: -0.46470 (-0.36682) + | > loss_dur: 0.20908 (0.15128) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.02442 (28.27670) + | > current_lr: 0.00005 + | > step_time: 3.50520 (2.46606) + | > loader_time: 0.00450 (0.03729) + + + --> STEP: 186/234 -- GLOBAL_STEP: 44880 + | > loss: -0.27001 (-0.21830) + | > log_mle: -0.50696 (-0.37175) + | > loss_dur: 0.23696 (0.15345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.91078 (29.61759) + | > current_lr: 0.00005 + | > step_time: 2.59130 (2.56805) + | > loader_time: 0.00460 (0.04056) + + + --> STEP: 191/234 -- GLOBAL_STEP: 44885 + | > loss: -0.30638 (-0.22115) + | > log_mle: -0.51955 (-0.37629) + | > loss_dur: 0.21316 (0.15514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.25605 (30.96905) + | > current_lr: 0.00005 + | > step_time: 3.59680 (2.64648) + | > loader_time: 0.00440 (0.04305) + + + --> STEP: 196/234 -- GLOBAL_STEP: 44890 + | > loss: -0.28976 (-0.22416) + | > log_mle: -0.51894 (-0.38101) + | > loss_dur: 0.22918 (0.15684) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.27228 (32.02211) + | > current_lr: 0.00005 + | > step_time: 1.88600 (2.67548) + | > loader_time: 0.00360 (0.04440) + + + --> STEP: 201/234 -- GLOBAL_STEP: 44895 + | > loss: -0.25088 (-0.22665) + | > log_mle: -0.47658 (-0.38522) + | > loss_dur: 0.22570 (0.15858) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.02014 (33.08268) + | > current_lr: 0.00005 + | > step_time: 5.39480 (2.69501) + | > loader_time: 0.00620 (0.04571) + + + --> STEP: 206/234 -- GLOBAL_STEP: 44900 + | > loss: -0.37027 (-0.22977) + | > log_mle: -0.59749 (-0.39004) + | > loss_dur: 0.22722 (0.16026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.07789 (34.10365) + | > current_lr: 0.00005 + | > step_time: 6.89500 (2.81158) + | > loader_time: 0.19750 (0.04698) + + + --> STEP: 211/234 -- GLOBAL_STEP: 44905 + | > loss: -0.42604 (-0.23337) + | > log_mle: -0.67797 (-0.39550) + | > loss_dur: 0.25193 (0.16213) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.48602 (35.56364) + | > current_lr: 0.00005 + | > step_time: 1.48790 (2.91403) + | > loader_time: 0.00670 (0.04694) + + + --> STEP: 216/234 -- GLOBAL_STEP: 44910 + | > loss: -0.39185 (-0.23673) + | > log_mle: -0.64263 (-0.40053) + | > loss_dur: 0.25079 (0.16380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 125.05930 (37.13349) + | > current_lr: 0.00005 + | > step_time: 4.29850 (2.94063) + | > loader_time: 0.00930 (0.04597) + + + --> STEP: 221/234 -- GLOBAL_STEP: 44915 + | > loss: -0.34264 (-0.24014) + | > log_mle: -0.56846 (-0.40568) + | > loss_dur: 0.22582 (0.16554) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.26486 (38.27887) + | > current_lr: 0.00005 + | > step_time: 3.59400 (2.96463) + | > loader_time: 0.09920 (0.04754) + + + --> STEP: 226/234 -- GLOBAL_STEP: 44920 + | > loss: -0.41784 (-0.24396) + | > log_mle: -0.66939 (-0.41139) + | > loss_dur: 0.25154 (0.16743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.80558 (39.65211) + | > current_lr: 0.00005 + | > step_time: 1.30300 (2.95443) + | > loader_time: 0.08400 (0.04899) + + + --> STEP: 231/234 -- GLOBAL_STEP: 44925 + | > loss: -0.34451 (-0.24681) + | > log_mle: -0.73039 (-0.41738) + | > loss_dur: 0.38589 (0.17057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 144.43704 (41.35507) + | > current_lr: 0.00005 + | > step_time: 0.28410 (2.89882) + | > loader_time: 0.00460 (0.04839) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17357 (-0.84990) + | > avg_loss: -0.24455 (+0.02122) + | > avg_log_mle: -0.48109 (-0.00216) + | > avg_loss_dur: 0.23654 (+0.02338) + + + > EPOCH: 192/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 01:09:08)  + + --> STEP: 2/234 -- GLOBAL_STEP: 44930 + | > loss: -0.22234 (-0.21325) + | > log_mle: -0.32730 (-0.32051) + | > loss_dur: 0.10496 (0.10726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.72001 (19.72677) + | > current_lr: 0.00005 + | > step_time: 4.00100 (4.30355) + | > loader_time: 0.00560 (0.00371) + + + --> STEP: 7/234 -- GLOBAL_STEP: 44935 + | > loss: -0.22540 (-0.20240) + | > log_mle: -0.31872 (-0.31359) + | > loss_dur: 0.09332 (0.11119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.50906 (20.73060) + | > current_lr: 0.00005 + | > step_time: 0.90620 (5.97336) + | > loader_time: 0.00180 (0.02858) + + + --> STEP: 12/234 -- GLOBAL_STEP: 44940 + | > loss: -0.20275 (-0.20619) + | > log_mle: -0.31414 (-0.31549) + | > loss_dur: 0.11139 (0.10930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.53319 (18.37886) + | > current_lr: 0.00005 + | > step_time: 0.99130 (4.07541) + | > loader_time: 0.00300 (0.02993) + + + --> STEP: 17/234 -- GLOBAL_STEP: 44945 + | > loss: -0.21078 (-0.20944) + | > log_mle: -0.29555 (-0.31472) + | > loss_dur: 0.08478 (0.10528) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.61152 (16.05502) + | > current_lr: 0.00005 + | > step_time: 5.19430 (4.01203) + | > loader_time: 0.00410 (0.02638) + + + --> STEP: 22/234 -- GLOBAL_STEP: 44950 + | > loss: -0.21771 (-0.21012) + | > log_mle: -0.31172 (-0.31273) + | > loss_dur: 0.09401 (0.10261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.25318 (14.71035) + | > current_lr: 0.00005 + | > step_time: 2.60220 (4.18722) + | > loader_time: 0.10000 (0.02946) + + + --> STEP: 27/234 -- GLOBAL_STEP: 44955 + | > loss: -0.20958 (-0.21095) + | > log_mle: -0.30836 (-0.31149) + | > loss_dur: 0.09878 (0.10054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.30362 (14.01845) + | > current_lr: 0.00005 + | > step_time: 0.86970 (3.75620) + | > loader_time: 0.00300 (0.03090) + + + --> STEP: 32/234 -- GLOBAL_STEP: 44960 + | > loss: -0.21266 (-0.20983) + | > log_mle: -0.30621 (-0.31001) + | > loss_dur: 0.09355 (0.10018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.48531 (13.56838) + | > current_lr: 0.00005 + | > step_time: 2.01370 (3.52392) + | > loader_time: 0.00240 (0.03410) + + + --> STEP: 37/234 -- GLOBAL_STEP: 44965 + | > loss: -0.20388 (-0.20679) + | > log_mle: -0.29232 (-0.30784) + | > loss_dur: 0.08844 (0.10106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.15354 (13.64922) + | > current_lr: 0.00005 + | > step_time: 3.80550 (3.42090) + | > loader_time: 0.00330 (0.03458) + + + --> STEP: 42/234 -- GLOBAL_STEP: 44970 + | > loss: -0.17365 (-0.20427) + | > log_mle: -0.28572 (-0.30651) + | > loss_dur: 0.11206 (0.10224) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.65149 (13.30446) + | > current_lr: 0.00005 + | > step_time: 2.31600 (3.33011) + | > loader_time: 0.08120 (0.03952) + + + --> STEP: 47/234 -- GLOBAL_STEP: 44975 + | > loss: -0.17163 (-0.20240) + | > log_mle: -0.29579 (-0.30611) + | > loss_dur: 0.12416 (0.10370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.96888 (13.10186) + | > current_lr: 0.00005 + | > step_time: 2.00150 (3.30919) + | > loader_time: 0.08180 (0.04375) + + + --> STEP: 52/234 -- GLOBAL_STEP: 44980 + | > loss: -0.16439 (-0.20099) + | > log_mle: -0.29238 (-0.30507) + | > loss_dur: 0.12799 (0.10408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.35516 (12.67686) + | > current_lr: 0.00005 + | > step_time: 1.36380 (3.16598) + | > loader_time: 0.00240 (0.04171) + + + --> STEP: 57/234 -- GLOBAL_STEP: 44985 + | > loss: -0.15731 (-0.19954) + | > log_mle: -0.28350 (-0.30475) + | > loss_dur: 0.12619 (0.10521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.87126 (12.38640) + | > current_lr: 0.00005 + | > step_time: 1.08230 (3.05651) + | > loader_time: 0.00180 (0.04134) + + + --> STEP: 62/234 -- GLOBAL_STEP: 44990 + | > loss: -0.14689 (-0.19799) + | > log_mle: -0.32458 (-0.30522) + | > loss_dur: 0.17769 (0.10723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.75336 (12.59331) + | > current_lr: 0.00005 + | > step_time: 1.08490 (2.95811) + | > loader_time: 0.00180 (0.04090) + + + --> STEP: 67/234 -- GLOBAL_STEP: 44995 + | > loss: -0.17819 (-0.19677) + | > log_mle: -0.31239 (-0.30475) + | > loss_dur: 0.13421 (0.10799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.91647 (12.36284) + | > current_lr: 0.00005 + | > step_time: 1.31190 (2.86303) + | > loader_time: 0.00270 (0.04081) + + + --> STEP: 72/234 -- GLOBAL_STEP: 45000 + | > loss: -0.17095 (-0.19423) + | > log_mle: -0.28973 (-0.30389) + | > loss_dur: 0.11878 (0.10966) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.81066 (12.51522) + | > current_lr: 0.00005 + | > step_time: 0.89340 (2.80144) + | > loader_time: 0.00150 (0.04090) + + + --> STEP: 77/234 -- GLOBAL_STEP: 45005 + | > loss: -0.18534 (-0.19253) + | > log_mle: -0.30301 (-0.30372) + | > loss_dur: 0.11767 (0.11119) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.66687 (12.57433) + | > current_lr: 0.00005 + | > step_time: 2.10940 (2.74164) + | > loader_time: 0.08500 (0.04056) + + + --> STEP: 82/234 -- GLOBAL_STEP: 45010 + | > loss: -0.17606 (-0.19117) + | > log_mle: -0.29387 (-0.30328) + | > loss_dur: 0.11781 (0.11211) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.05054 (12.67339) + | > current_lr: 0.00005 + | > step_time: 1.52000 (2.69955) + | > loader_time: 0.00240 (0.04027) + + + --> STEP: 87/234 -- GLOBAL_STEP: 45015 + | > loss: -0.16708 (-0.18975) + | > log_mle: -0.30001 (-0.30327) + | > loss_dur: 0.13294 (0.11352) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.77108 (12.80277) + | > current_lr: 0.00005 + | > step_time: 1.90990 (2.64323) + | > loader_time: 0.00170 (0.03907) + + + --> STEP: 92/234 -- GLOBAL_STEP: 45020 + | > loss: -0.21289 (-0.18964) + | > log_mle: -0.34642 (-0.30486) + | > loss_dur: 0.13353 (0.11522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.87992 (13.23145) + | > current_lr: 0.00005 + | > step_time: 1.11820 (2.59715) + | > loader_time: 0.08360 (0.03891) + + + --> STEP: 97/234 -- GLOBAL_STEP: 45025 + | > loss: -0.18714 (-0.19011) + | > log_mle: -0.33264 (-0.30735) + | > loss_dur: 0.14550 (0.11723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.25364 (13.81649) + | > current_lr: 0.00005 + | > step_time: 1.10990 (2.54888) + | > loader_time: 0.00220 (0.03893) + + + --> STEP: 102/234 -- GLOBAL_STEP: 45030 + | > loss: -0.15626 (-0.18966) + | > log_mle: -0.31499 (-0.30879) + | > loss_dur: 0.15873 (0.11913) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.39772 (14.28527) + | > current_lr: 0.00005 + | > step_time: 1.41250 (2.51306) + | > loader_time: 0.18680 (0.03895) + + + --> STEP: 107/234 -- GLOBAL_STEP: 45035 + | > loss: -0.19445 (-0.19000) + | > log_mle: -0.35495 (-0.31133) + | > loss_dur: 0.16050 (0.12134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.12129 (15.03239) + | > current_lr: 0.00005 + | > step_time: 2.50670 (2.48168) + | > loader_time: 0.00430 (0.03727) + + + --> STEP: 112/234 -- GLOBAL_STEP: 45040 + | > loss: -0.18856 (-0.18964) + | > log_mle: -0.36785 (-0.31358) + | > loss_dur: 0.17929 (0.12394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.57673 (15.62843) + | > current_lr: 0.00005 + | > step_time: 1.61550 (2.46011) + | > loader_time: 0.00800 (0.03580) + + + --> STEP: 117/234 -- GLOBAL_STEP: 45045 + | > loss: -0.20481 (-0.18980) + | > log_mle: -0.36173 (-0.31578) + | > loss_dur: 0.15693 (0.12598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.48979 (16.09190) + | > current_lr: 0.00005 + | > step_time: 2.01270 (2.43296) + | > loader_time: 0.00270 (0.03573) + + + --> STEP: 122/234 -- GLOBAL_STEP: 45050 + | > loss: -0.17983 (-0.18970) + | > log_mle: -0.33824 (-0.31709) + | > loss_dur: 0.15840 (0.12739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.26139 (16.36161) + | > current_lr: 0.00005 + | > step_time: 2.16340 (2.43057) + | > loader_time: 0.00360 (0.03516) + + + --> STEP: 127/234 -- GLOBAL_STEP: 45055 + | > loss: -0.20723 (-0.19033) + | > log_mle: -0.39735 (-0.31959) + | > loss_dur: 0.19011 (0.12926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.63015 (17.02014) + | > current_lr: 0.00005 + | > step_time: 1.70540 (2.44218) + | > loader_time: 0.00270 (0.03509) + + + --> STEP: 132/234 -- GLOBAL_STEP: 45060 + | > loss: -0.21518 (-0.19148) + | > log_mle: -0.37846 (-0.32261) + | > loss_dur: 0.16328 (0.13113) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.69877 (17.80010) + | > current_lr: 0.00005 + | > step_time: 1.29560 (2.41839) + | > loader_time: 0.00330 (0.03390) + + + --> STEP: 137/234 -- GLOBAL_STEP: 45065 + | > loss: -0.19470 (-0.19256) + | > log_mle: -0.39631 (-0.32603) + | > loss_dur: 0.20160 (0.13346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.67826 (18.40707) + | > current_lr: 0.00005 + | > step_time: 1.90370 (2.40157) + | > loader_time: 0.08950 (0.03406) + + + --> STEP: 142/234 -- GLOBAL_STEP: 45070 + | > loss: -0.20815 (-0.19361) + | > log_mle: -0.40934 (-0.32897) + | > loss_dur: 0.20119 (0.13535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.01784 (19.04804) + | > current_lr: 0.00005 + | > step_time: 2.69730 (2.40718) + | > loader_time: 0.00400 (0.03358) + + + --> STEP: 147/234 -- GLOBAL_STEP: 45075 + | > loss: -0.22152 (-0.19604) + | > log_mle: -0.41253 (-0.33370) + | > loss_dur: 0.19101 (0.13765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.44492 (20.13739) + | > current_lr: 0.00005 + | > step_time: 2.51320 (2.45883) + | > loader_time: 0.00400 (0.03505) + + + --> STEP: 152/234 -- GLOBAL_STEP: 45080 + | > loss: -0.28036 (-0.19861) + | > log_mle: -0.49309 (-0.33809) + | > loss_dur: 0.21272 (0.13949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.49433 (21.27828) + | > current_lr: 0.00005 + | > step_time: 2.40460 (2.47272) + | > loader_time: 0.08340 (0.03633) + + + --> STEP: 157/234 -- GLOBAL_STEP: 45085 + | > loss: -0.23747 (-0.20144) + | > log_mle: -0.44134 (-0.34290) + | > loss_dur: 0.20387 (0.14146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.92959 (22.94214) + | > current_lr: 0.00005 + | > step_time: 3.80230 (2.49979) + | > loader_time: 0.00410 (0.03699) + + + --> STEP: 162/234 -- GLOBAL_STEP: 45090 + | > loss: -0.27555 (-0.20406) + | > log_mle: -0.47330 (-0.34753) + | > loss_dur: 0.19776 (0.14347) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.95275 (23.93523) + | > current_lr: 0.00005 + | > step_time: 9.27630 (2.60776) + | > loader_time: 0.00440 (0.03719) + + + --> STEP: 167/234 -- GLOBAL_STEP: 45095 + | > loss: -0.36654 (-0.20663) + | > log_mle: -0.56858 (-0.35187) + | > loss_dur: 0.20204 (0.14524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.19435 (24.73352) + | > current_lr: 0.00005 + | > step_time: 3.09800 (2.58392) + | > loader_time: 0.00690 (0.03620) + + + --> STEP: 172/234 -- GLOBAL_STEP: 45100 + | > loss: -0.33334 (-0.20957) + | > log_mle: -0.55328 (-0.35699) + | > loss_dur: 0.21994 (0.14743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.70842 (25.91309) + | > current_lr: 0.00005 + | > step_time: 1.93110 (2.64263) + | > loader_time: 0.00310 (0.03622) + + + --> STEP: 177/234 -- GLOBAL_STEP: 45105 + | > loss: -0.29605 (-0.21254) + | > log_mle: -0.51064 (-0.36195) + | > loss_dur: 0.21458 (0.14941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.66339 (26.96585) + | > current_lr: 0.00005 + | > step_time: 4.30460 (2.63025) + | > loader_time: 0.00770 (0.03677) + + + --> STEP: 182/234 -- GLOBAL_STEP: 45110 + | > loss: -0.30156 (-0.21505) + | > log_mle: -0.55089 (-0.36674) + | > loss_dur: 0.24932 (0.15169) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.93941 (28.44007) + | > current_lr: 0.00005 + | > step_time: 1.14650 (2.64478) + | > loader_time: 0.00390 (0.03892) + + + --> STEP: 187/234 -- GLOBAL_STEP: 45115 + | > loss: -0.32700 (-0.21772) + | > log_mle: -0.55289 (-0.37144) + | > loss_dur: 0.22588 (0.15372) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.70055 (29.69149) + | > current_lr: 0.00005 + | > step_time: 4.39830 (2.64572) + | > loader_time: 0.28980 (0.04007) + + + --> STEP: 192/234 -- GLOBAL_STEP: 45120 + | > loss: -0.33533 (-0.22064) + | > log_mle: -0.54612 (-0.37595) + | > loss_dur: 0.21079 (0.15531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 146.98254 (31.12737) + | > current_lr: 0.00005 + | > step_time: 3.71640 (2.68934) + | > loader_time: 0.09640 (0.04062) + + + --> STEP: 197/234 -- GLOBAL_STEP: 45125 + | > loss: -0.33473 (-0.22303) + | > log_mle: -0.53672 (-0.38000) + | > loss_dur: 0.20199 (0.15697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.19308 (32.27986) + | > current_lr: 0.00005 + | > step_time: 4.69740 (2.75357) + | > loader_time: 0.00310 (0.04047) + + + --> STEP: 202/234 -- GLOBAL_STEP: 45130 + | > loss: -0.41224 (-0.22557) + | > log_mle: -0.63855 (-0.38442) + | > loss_dur: 0.22631 (0.15885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.65058 (33.07637) + | > current_lr: 0.00005 + | > step_time: 6.08830 (2.81701) + | > loader_time: 0.11110 (0.04156) + + + --> STEP: 207/234 -- GLOBAL_STEP: 45135 + | > loss: -0.38422 (-0.22830) + | > log_mle: -0.62378 (-0.38888) + | > loss_dur: 0.23957 (0.16058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.05798 (34.14055) + | > current_lr: 0.00005 + | > step_time: 2.98790 (2.94019) + | > loader_time: 0.01060 (0.04353) + + + --> STEP: 212/234 -- GLOBAL_STEP: 45140 + | > loss: -0.37187 (-0.23163) + | > log_mle: -0.60826 (-0.39403) + | > loss_dur: 0.23639 (0.16240) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.54955 (35.26675) + | > current_lr: 0.00005 + | > step_time: 6.38680 (3.00386) + | > loader_time: 0.20830 (0.04672) + + + --> STEP: 217/234 -- GLOBAL_STEP: 45145 + | > loss: -0.37833 (-0.23507) + | > log_mle: -0.62762 (-0.39920) + | > loss_dur: 0.24929 (0.16413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.67297 (36.52877) + | > current_lr: 0.00005 + | > step_time: 11.49770 (3.09911) + | > loader_time: 0.00430 (0.04714) + + + --> STEP: 222/234 -- GLOBAL_STEP: 45150 + | > loss: -0.39814 (-0.23856) + | > log_mle: -0.65678 (-0.40440) + | > loss_dur: 0.25864 (0.16584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.40259 (37.62086) + | > current_lr: 0.00005 + | > step_time: 2.59470 (3.09141) + | > loader_time: 0.00330 (0.04654) + + + --> STEP: 227/234 -- GLOBAL_STEP: 45155 + | > loss: -0.35877 (-0.24240) + | > log_mle: -0.61833 (-0.41005) + | > loss_dur: 0.25955 (0.16766) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.73322 (38.88311) + | > current_lr: 0.00005 + | > step_time: 0.26080 (3.05471) + | > loader_time: 0.00660 (0.04630) + + + --> STEP: 232/234 -- GLOBAL_STEP: 45160 + | > loss: -0.33197 (-0.24530) + | > log_mle: -0.81303 (-0.41692) + | > loss_dur: 0.48106 (0.17162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 196.25484 (40.77988) + | > current_lr: 0.00005 + | > step_time: 0.34600 (2.99512) + | > loader_time: 0.10810 (0.04584) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00199 (-0.17157) + | > avg_loss: -0.26947 (-0.02492) + | > avg_log_mle: -0.49222 (-0.01114) + | > avg_loss_dur: 0.22276 (-0.01378) + + + > EPOCH: 193/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 01:21:56)  + + --> STEP: 3/234 -- GLOBAL_STEP: 45165 + | > loss: -0.13548 (-0.19078) + | > log_mle: -0.30601 (-0.31537) + | > loss_dur: 0.17053 (0.12459) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.81165 (19.50246) + | > current_lr: 0.00005 + | > step_time: 1.79750 (2.65664) + | > loader_time: 0.00150 (2.56910) + + + --> STEP: 8/234 -- GLOBAL_STEP: 45170 + | > loss: -0.23462 (-0.20976) + | > log_mle: -0.32993 (-0.31759) + | > loss_dur: 0.09531 (0.10784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.00873 (17.54992) + | > current_lr: 0.00005 + | > step_time: 3.29300 (2.60647) + | > loader_time: 0.00220 (1.25044) + + + --> STEP: 13/234 -- GLOBAL_STEP: 45175 + | > loss: -0.24104 (-0.21253) + | > log_mle: -0.32682 (-0.31829) + | > loss_dur: 0.08578 (0.10576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.50878 (16.28156) + | > current_lr: 0.00005 + | > step_time: 5.69540 (2.81232) + | > loader_time: 13.28820 (1.79953) + + + --> STEP: 18/234 -- GLOBAL_STEP: 45180 + | > loss: -0.20072 (-0.21350) + | > log_mle: -0.30773 (-0.31610) + | > loss_dur: 0.10701 (0.10260) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.75874 (15.37330) + | > current_lr: 0.00005 + | > step_time: 7.80900 (3.59904) + | > loader_time: 0.09250 (1.31563) + + + --> STEP: 23/234 -- GLOBAL_STEP: 45185 + | > loss: -0.22650 (-0.21458) + | > log_mle: -0.31349 (-0.31388) + | > loss_dur: 0.08699 (0.09930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.28157 (14.42089) + | > current_lr: 0.00005 + | > step_time: 7.30330 (3.60721) + | > loader_time: 0.09530 (1.03447) + + + --> STEP: 28/234 -- GLOBAL_STEP: 45190 + | > loss: -0.24636 (-0.21539) + | > log_mle: -0.31741 (-0.31244) + | > loss_dur: 0.07105 (0.09704) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.49590 (13.67387) + | > current_lr: 0.00005 + | > step_time: 1.29570 (3.59910) + | > loader_time: 0.00260 (0.86086) + + + --> STEP: 33/234 -- GLOBAL_STEP: 45195 + | > loss: -0.19634 (-0.21181) + | > log_mle: -0.29493 (-0.31013) + | > loss_dur: 0.09858 (0.09832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.93143 (13.54150) + | > current_lr: 0.00005 + | > step_time: 2.40710 (3.66626) + | > loader_time: 0.00170 (0.75156) + + + --> STEP: 38/234 -- GLOBAL_STEP: 45200 + | > loss: -0.20034 (-0.20859) + | > log_mle: -0.31051 (-0.30840) + | > loss_dur: 0.11017 (0.09981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.20753 (13.53408) + | > current_lr: 0.00005 + | > step_time: 0.89910 (3.35165) + | > loader_time: 0.00170 (0.65496) + + + --> STEP: 43/234 -- GLOBAL_STEP: 45205 + | > loss: -0.17256 (-0.20483) + | > log_mle: -0.30570 (-0.30675) + | > loss_dur: 0.13314 (0.10193) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.77723 (13.38169) + | > current_lr: 0.00005 + | > step_time: 1.77280 (3.12657) + | > loader_time: 0.00210 (0.58096) + + + --> STEP: 48/234 -- GLOBAL_STEP: 45210 + | > loss: -0.20548 (-0.20346) + | > log_mle: -0.29793 (-0.30619) + | > loss_dur: 0.09245 (0.10273) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.62206 (13.15304) + | > current_lr: 0.00005 + | > step_time: 2.01870 (3.03023) + | > loader_time: 0.00230 (0.52242) + + + --> STEP: 53/234 -- GLOBAL_STEP: 45215 + | > loss: -0.18400 (-0.20150) + | > log_mle: -0.30356 (-0.30536) + | > loss_dur: 0.11957 (0.10386) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.76195 (12.92568) + | > current_lr: 0.00005 + | > step_time: 1.70280 (2.88569) + | > loader_time: 0.00340 (0.47496) + + + --> STEP: 58/234 -- GLOBAL_STEP: 45220 + | > loss: -0.19904 (-0.20039) + | > log_mle: -0.29790 (-0.30490) + | > loss_dur: 0.09886 (0.10452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.29466 (12.66334) + | > current_lr: 0.00005 + | > step_time: 1.69580 (2.77388) + | > loader_time: 0.00300 (0.43565) + + + --> STEP: 63/234 -- GLOBAL_STEP: 45225 + | > loss: -0.16953 (-0.19884) + | > log_mle: -0.29515 (-0.30537) + | > loss_dur: 0.12562 (0.10653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.34588 (12.83992) + | > current_lr: 0.00005 + | > step_time: 1.49430 (2.71723) + | > loader_time: 0.00290 (0.40393) + + + --> STEP: 68/234 -- GLOBAL_STEP: 45230 + | > loss: -0.14980 (-0.19728) + | > log_mle: -0.28734 (-0.30476) + | > loss_dur: 0.13753 (0.10748) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.31805 (12.63521) + | > current_lr: 0.00005 + | > step_time: 1.40840 (2.68386) + | > loader_time: 0.00200 (0.37569) + + + --> STEP: 73/234 -- GLOBAL_STEP: 45235 + | > loss: -0.15897 (-0.19493) + | > log_mle: -0.30868 (-0.30434) + | > loss_dur: 0.14971 (0.10941) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.05501 (12.62785) + | > current_lr: 0.00005 + | > step_time: 2.26670 (2.64476) + | > loader_time: 0.00230 (0.35259) + + + --> STEP: 78/234 -- GLOBAL_STEP: 45240 + | > loss: -0.15356 (-0.19346) + | > log_mle: -0.28691 (-0.30382) + | > loss_dur: 0.13335 (0.11036) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.01326 (12.61026) + | > current_lr: 0.00005 + | > step_time: 1.84930 (2.58870) + | > loader_time: 0.00250 (0.33020) + + + --> STEP: 83/234 -- GLOBAL_STEP: 45245 + | > loss: -0.14825 (-0.19200) + | > log_mle: -0.30975 (-0.30370) + | > loss_dur: 0.16150 (0.11170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.27971 (12.67660) + | > current_lr: 0.00005 + | > step_time: 1.09680 (2.54012) + | > loader_time: 0.00260 (0.31272) + + + --> STEP: 88/234 -- GLOBAL_STEP: 45250 + | > loss: -0.18570 (-0.19109) + | > log_mle: -0.34672 (-0.30415) + | > loss_dur: 0.16102 (0.11306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.39916 (12.68772) + | > current_lr: 0.00005 + | > step_time: 4.12210 (2.56979) + | > loader_time: 0.00250 (0.29911) + + + --> STEP: 93/234 -- GLOBAL_STEP: 45255 + | > loss: -0.18617 (-0.19092) + | > log_mle: -0.35744 (-0.30591) + | > loss_dur: 0.17127 (0.11499) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.15244 (13.03687) + | > current_lr: 0.00005 + | > step_time: 2.58850 (2.54398) + | > loader_time: 0.00280 (0.28322) + + + --> STEP: 98/234 -- GLOBAL_STEP: 45260 + | > loss: -0.16127 (-0.19091) + | > log_mle: -0.29085 (-0.30772) + | > loss_dur: 0.12958 (0.11681) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.14401 (13.34945) + | > current_lr: 0.00005 + | > step_time: 0.96250 (2.52923) + | > loader_time: 0.00240 (0.26981) + + + --> STEP: 103/234 -- GLOBAL_STEP: 45265 + | > loss: -0.21386 (-0.19110) + | > log_mle: -0.39158 (-0.31038) + | > loss_dur: 0.17772 (0.11928) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.95191 (13.99158) + | > current_lr: 0.00005 + | > step_time: 1.98710 (2.56079) + | > loader_time: 0.00870 (0.25870) + + + --> STEP: 108/234 -- GLOBAL_STEP: 45270 + | > loss: -0.18567 (-0.19125) + | > log_mle: -0.33682 (-0.31257) + | > loss_dur: 0.15115 (0.12133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.98682 (14.47330) + | > current_lr: 0.00005 + | > step_time: 1.45660 (2.53939) + | > loader_time: 0.00270 (0.24756) + + + --> STEP: 113/234 -- GLOBAL_STEP: 45275 + | > loss: -0.20524 (-0.19146) + | > log_mle: -0.37833 (-0.31552) + | > loss_dur: 0.17309 (0.12406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.40603 (15.19107) + | > current_lr: 0.00005 + | > step_time: 2.01750 (2.51943) + | > loader_time: 0.00250 (0.23884) + + + --> STEP: 118/234 -- GLOBAL_STEP: 45280 + | > loss: -0.18244 (-0.19146) + | > log_mle: -0.35117 (-0.31759) + | > loss_dur: 0.16873 (0.12613) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.27579 (15.68696) + | > current_lr: 0.00005 + | > step_time: 1.78970 (2.47154) + | > loader_time: 0.00170 (0.22882) + + + --> STEP: 123/234 -- GLOBAL_STEP: 45285 + | > loss: -0.16116 (-0.19124) + | > log_mle: -0.32057 (-0.31870) + | > loss_dur: 0.15941 (0.12746) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.07284 (15.96123) + | > current_lr: 0.00005 + | > step_time: 2.70010 (2.45473) + | > loader_time: 0.09270 (0.22099) + + + --> STEP: 128/234 -- GLOBAL_STEP: 45290 + | > loss: -0.22780 (-0.19239) + | > log_mle: -0.38200 (-0.32185) + | > loss_dur: 0.15420 (0.12946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.55883 (16.78491) + | > current_lr: 0.00005 + | > step_time: 2.21670 (2.44881) + | > loader_time: 0.08060 (0.21445) + + + --> STEP: 133/234 -- GLOBAL_STEP: 45295 + | > loss: -0.24248 (-0.19401) + | > log_mle: -0.41321 (-0.32524) + | > loss_dur: 0.17074 (0.13123) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.36988 (17.56318) + | > current_lr: 0.00005 + | > step_time: 2.89860 (2.44990) + | > loader_time: 0.00320 (0.20717) + + + --> STEP: 138/234 -- GLOBAL_STEP: 45300 + | > loss: -0.19332 (-0.19512) + | > log_mle: -0.36204 (-0.32835) + | > loss_dur: 0.16871 (0.13322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.04803 (18.34121) + | > current_lr: 0.00005 + | > step_time: 3.20580 (2.44619) + | > loader_time: 0.09560 (0.20166) + + + --> STEP: 143/234 -- GLOBAL_STEP: 45305 + | > loss: -0.27996 (-0.19689) + | > log_mle: -0.51368 (-0.33240) + | > loss_dur: 0.23372 (0.13552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.96629 (19.29956) + | > current_lr: 0.00005 + | > step_time: 1.38720 (2.42719) + | > loader_time: 0.00320 (0.19687) + + + --> STEP: 148/234 -- GLOBAL_STEP: 45310 + | > loss: -0.25097 (-0.19909) + | > log_mle: -0.41967 (-0.33647) + | > loss_dur: 0.16870 (0.13738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.76945 (20.33549) + | > current_lr: 0.00005 + | > step_time: 2.79840 (2.42944) + | > loader_time: 0.09890 (0.19287) + + + --> STEP: 153/234 -- GLOBAL_STEP: 45315 + | > loss: -0.34303 (-0.20191) + | > log_mle: -0.54354 (-0.34149) + | > loss_dur: 0.20051 (0.13957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.28973 (21.65906) + | > current_lr: 0.00005 + | > step_time: 4.60580 (2.45736) + | > loader_time: 0.09270 (0.18784) + + + --> STEP: 158/234 -- GLOBAL_STEP: 45320 + | > loss: -0.24677 (-0.20386) + | > log_mle: -0.46738 (-0.34558) + | > loss_dur: 0.22061 (0.14172) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.28422 (23.02364) + | > current_lr: 0.00005 + | > step_time: 3.29650 (2.51198) + | > loader_time: 0.00640 (0.18371) + + + --> STEP: 163/234 -- GLOBAL_STEP: 45325 + | > loss: -0.23643 (-0.20613) + | > log_mle: -0.43981 (-0.34976) + | > loss_dur: 0.20338 (0.14363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.40869 (23.96095) + | > current_lr: 0.00005 + | > step_time: 1.41430 (2.49160) + | > loader_time: 0.08620 (0.17971) + + + --> STEP: 168/234 -- GLOBAL_STEP: 45330 + | > loss: -0.27630 (-0.20866) + | > log_mle: -0.50052 (-0.35414) + | > loss_dur: 0.22422 (0.14547) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.95611 (24.81463) + | > current_lr: 0.00005 + | > step_time: 2.48460 (2.58648) + | > loader_time: 0.00420 (0.17782) + + + --> STEP: 173/234 -- GLOBAL_STEP: 45335 + | > loss: -0.29821 (-0.21163) + | > log_mle: -0.51126 (-0.35920) + | > loss_dur: 0.21305 (0.14757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.77491 (25.81891) + | > current_lr: 0.00005 + | > step_time: 2.20260 (2.62201) + | > loader_time: 0.00290 (0.17394) + + + --> STEP: 178/234 -- GLOBAL_STEP: 45340 + | > loss: -0.31691 (-0.21450) + | > log_mle: -0.55741 (-0.36420) + | > loss_dur: 0.24050 (0.14970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.88277 (27.01447) + | > current_lr: 0.00005 + | > step_time: 8.19110 (2.67641) + | > loader_time: 0.00230 (0.17060) + + + --> STEP: 183/234 -- GLOBAL_STEP: 45345 + | > loss: -0.34718 (-0.21701) + | > log_mle: -0.56427 (-0.36885) + | > loss_dur: 0.21709 (0.15185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.30152 (28.08441) + | > current_lr: 0.00005 + | > step_time: 4.81740 (2.67448) + | > loader_time: 0.19160 (0.16806) + + + --> STEP: 188/234 -- GLOBAL_STEP: 45350 + | > loss: -0.36119 (-0.21969) + | > log_mle: -0.58147 (-0.37353) + | > loss_dur: 0.22028 (0.15384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.16129 (29.32263) + | > current_lr: 0.00005 + | > step_time: 2.19810 (2.70995) + | > loader_time: 0.00270 (0.16929) + + + --> STEP: 193/234 -- GLOBAL_STEP: 45355 + | > loss: -0.36199 (-0.22263) + | > log_mle: -0.57561 (-0.37804) + | > loss_dur: 0.21362 (0.15541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.25149 (30.52200) + | > current_lr: 0.00005 + | > step_time: 9.60960 (2.77775) + | > loader_time: 0.08670 (0.16786) + + + --> STEP: 198/234 -- GLOBAL_STEP: 45360 + | > loss: -0.34169 (-0.22536) + | > log_mle: -0.57457 (-0.38245) + | > loss_dur: 0.23288 (0.15709) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.99729 (31.53664) + | > current_lr: 0.00005 + | > step_time: 4.00580 (2.89711) + | > loader_time: 0.10850 (0.16625) + + + --> STEP: 203/234 -- GLOBAL_STEP: 45365 + | > loss: -0.29240 (-0.22784) + | > log_mle: -0.50765 (-0.38669) + | > loss_dur: 0.21525 (0.15885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.55844 (32.69874) + | > current_lr: 0.00005 + | > step_time: 7.29870 (2.94258) + | > loader_time: 0.00650 (0.16320) + + + --> STEP: 208/234 -- GLOBAL_STEP: 45370 + | > loss: -0.35287 (-0.23094) + | > log_mle: -0.59257 (-0.39166) + | > loss_dur: 0.23970 (0.16071) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.21468 (33.86808) + | > current_lr: 0.00005 + | > step_time: 3.90280 (2.96853) + | > loader_time: 0.11220 (0.16176) + + + --> STEP: 213/234 -- GLOBAL_STEP: 45375 + | > loss: -0.37413 (-0.23419) + | > log_mle: -0.62306 (-0.39678) + | > loss_dur: 0.24893 (0.16258) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.05872 (35.34600) + | > current_lr: 0.00005 + | > step_time: 4.89620 (3.06559) + | > loader_time: 0.08880 (0.16128) + + + --> STEP: 218/234 -- GLOBAL_STEP: 45380 + | > loss: -0.36326 (-0.23742) + | > log_mle: -0.59890 (-0.40173) + | > loss_dur: 0.23564 (0.16431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.16846 (36.37001) + | > current_lr: 0.00005 + | > step_time: 4.28800 (3.13562) + | > loader_time: 0.10380 (0.16027) + + + --> STEP: 223/234 -- GLOBAL_STEP: 45385 + | > loss: -0.40231 (-0.24085) + | > log_mle: -0.64151 (-0.40697) + | > loss_dur: 0.23919 (0.16612) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.57169 (37.47397) + | > current_lr: 0.00005 + | > step_time: 0.22010 (3.10596) + | > loader_time: 0.00330 (0.15756) + + + --> STEP: 228/234 -- GLOBAL_STEP: 45390 + | > loss: -0.36838 (-0.24432) + | > log_mle: -0.63800 (-0.41237) + | > loss_dur: 0.26962 (0.16805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.41741 (38.79150) + | > current_lr: 0.00005 + | > step_time: 0.24400 (3.04292) + | > loader_time: 0.00440 (0.15419) + + + --> STEP: 233/234 -- GLOBAL_STEP: 45395 + | > loss: 0.06408 (-0.24526) + | > log_mle: -0.59252 (-0.41891) + | > loss_dur: 0.65659 (0.17365) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.51239 (40.68347) + | > current_lr: 0.00005 + | > step_time: 0.19530 (2.98314) + | > loader_time: 0.00360 (0.15097) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15256 (+0.15056) + | > avg_loss: -0.26229 (+0.00718) + | > avg_log_mle: -0.48362 (+0.00860) + | > avg_loss_dur: 0.22133 (-0.00143) + + + > EPOCH: 194/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 01:35:06)  + + --> STEP: 4/234 -- GLOBAL_STEP: 45400 + | > loss: -0.20982 (-0.20217) + | > log_mle: -0.31248 (-0.31487) + | > loss_dur: 0.10266 (0.11270) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.06809 (21.08037) + | > current_lr: 0.00005 + | > step_time: 6.61100 (6.19958) + | > loader_time: 0.29540 (0.07500) + + + --> STEP: 9/234 -- GLOBAL_STEP: 45405 + | > loss: -0.18722 (-0.20863) + | > log_mle: -0.32611 (-0.31834) + | > loss_dur: 0.13889 (0.10972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.80033 (17.53565) + | > current_lr: 0.00005 + | > step_time: 5.90000 (5.13322) + | > loader_time: 0.19560 (0.08603) + + + --> STEP: 14/234 -- GLOBAL_STEP: 45410 + | > loss: -0.21309 (-0.21280) + | > log_mle: -0.31976 (-0.31855) + | > loss_dur: 0.10667 (0.10575) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.93171 (16.68661) + | > current_lr: 0.00005 + | > step_time: 1.79920 (5.19380) + | > loader_time: 0.00280 (0.07691) + + + --> STEP: 19/234 -- GLOBAL_STEP: 45415 + | > loss: -0.22229 (-0.21399) + | > log_mle: -0.31112 (-0.31635) + | > loss_dur: 0.08882 (0.10236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.66135 (15.45059) + | > current_lr: 0.00005 + | > step_time: 2.39290 (4.13664) + | > loader_time: 0.00130 (0.05711) + + + --> STEP: 24/234 -- GLOBAL_STEP: 45420 + | > loss: -0.23709 (-0.21662) + | > log_mle: -0.30752 (-0.31496) + | > loss_dur: 0.07043 (0.09834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10636 (14.38673) + | > current_lr: 0.00005 + | > step_time: 1.80010 (3.78660) + | > loader_time: 0.00130 (0.05390) + + + --> STEP: 29/234 -- GLOBAL_STEP: 45425 + | > loss: -0.17892 (-0.21561) + | > log_mle: -0.29143 (-0.31330) + | > loss_dur: 0.11250 (0.09769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03746 (13.79691) + | > current_lr: 0.00005 + | > step_time: 1.11280 (3.42108) + | > loader_time: 0.08750 (0.04803) + + + --> STEP: 34/234 -- GLOBAL_STEP: 45430 + | > loss: -0.19103 (-0.21346) + | > log_mle: -0.30155 (-0.31213) + | > loss_dur: 0.11051 (0.09868) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.52504 (13.42683) + | > current_lr: 0.00005 + | > step_time: 1.73910 (3.27965) + | > loader_time: 0.00270 (0.04134) + + + --> STEP: 39/234 -- GLOBAL_STEP: 45435 + | > loss: -0.19817 (-0.21107) + | > log_mle: -0.31074 (-0.31127) + | > loss_dur: 0.11257 (0.10019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03779 (13.21464) + | > current_lr: 0.00005 + | > step_time: 2.41470 (3.06291) + | > loader_time: 0.00220 (0.03629) + + + --> STEP: 44/234 -- GLOBAL_STEP: 45440 + | > loss: -0.22138 (-0.20914) + | > log_mle: -0.30263 (-0.31001) + | > loss_dur: 0.08125 (0.10087) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.73118 (12.86207) + | > current_lr: 0.00005 + | > step_time: 1.36310 (2.89160) + | > loader_time: 0.00200 (0.03413) + + + --> STEP: 49/234 -- GLOBAL_STEP: 45445 + | > loss: -0.21547 (-0.20766) + | > log_mle: -0.31221 (-0.30978) + | > loss_dur: 0.09674 (0.10212) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.77050 (12.74866) + | > current_lr: 0.00005 + | > step_time: 1.15250 (2.71782) + | > loader_time: 0.00350 (0.03087) + + + --> STEP: 54/234 -- GLOBAL_STEP: 45450 + | > loss: -0.20419 (-0.20544) + | > log_mle: -0.30764 (-0.30890) + | > loss_dur: 0.10345 (0.10346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.42310 (12.60999) + | > current_lr: 0.00005 + | > step_time: 1.39380 (2.62058) + | > loader_time: 0.00260 (0.02983) + + + --> STEP: 59/234 -- GLOBAL_STEP: 45455 + | > loss: -0.20629 (-0.20416) + | > log_mle: -0.31480 (-0.30850) + | > loss_dur: 0.10852 (0.10434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.33248 (12.45326) + | > current_lr: 0.00005 + | > step_time: 1.64930 (2.58462) + | > loader_time: 0.00240 (0.03074) + + + --> STEP: 64/234 -- GLOBAL_STEP: 45460 + | > loss: -0.20027 (-0.20206) + | > log_mle: -0.30012 (-0.30870) + | > loss_dur: 0.09985 (0.10664) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.74456 (12.56058) + | > current_lr: 0.00005 + | > step_time: 1.53910 (2.51907) + | > loader_time: 0.00210 (0.03113) + + + --> STEP: 69/234 -- GLOBAL_STEP: 45465 + | > loss: -0.17394 (-0.20039) + | > log_mle: -0.28612 (-0.30791) + | > loss_dur: 0.11219 (0.10753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.95239 (12.35312) + | > current_lr: 0.00005 + | > step_time: 1.30310 (2.45824) + | > loader_time: 0.08400 (0.03145) + + + --> STEP: 74/234 -- GLOBAL_STEP: 45470 + | > loss: -0.16116 (-0.19788) + | > log_mle: -0.28800 (-0.30748) + | > loss_dur: 0.12684 (0.10960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.61140 (12.56139) + | > current_lr: 0.00005 + | > step_time: 2.02370 (2.54801) + | > loader_time: 0.09230 (0.03309) + + + --> STEP: 79/234 -- GLOBAL_STEP: 45475 + | > loss: -0.17157 (-0.19633) + | > log_mle: -0.30499 (-0.30718) + | > loss_dur: 0.13342 (0.11085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.34901 (12.61303) + | > current_lr: 0.00005 + | > step_time: 1.01640 (2.52212) + | > loader_time: 0.00280 (0.03238) + + + --> STEP: 84/234 -- GLOBAL_STEP: 45480 + | > loss: -0.17946 (-0.19520) + | > log_mle: -0.30136 (-0.30700) + | > loss_dur: 0.12190 (0.11179) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.38200 (12.61262) + | > current_lr: 0.00005 + | > step_time: 2.38740 (2.52683) + | > loader_time: 0.00720 (0.03286) + + + --> STEP: 89/234 -- GLOBAL_STEP: 45485 + | > loss: -0.19869 (-0.19452) + | > log_mle: -0.32958 (-0.30775) + | > loss_dur: 0.13090 (0.11323) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.51424 (12.75815) + | > current_lr: 0.00005 + | > step_time: 2.38870 (2.47882) + | > loader_time: 0.00260 (0.03115) + + + --> STEP: 94/234 -- GLOBAL_STEP: 45490 + | > loss: -0.21504 (-0.19447) + | > log_mle: -0.36542 (-0.30977) + | > loss_dur: 0.15038 (0.11531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.97171 (13.21540) + | > current_lr: 0.00005 + | > step_time: 1.99270 (2.46300) + | > loader_time: 0.00220 (0.03048) + + + --> STEP: 99/234 -- GLOBAL_STEP: 45495 + | > loss: -0.21358 (-0.19458) + | > log_mle: -0.39601 (-0.31177) + | > loss_dur: 0.18243 (0.11719) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.53057 (13.74742) + | > current_lr: 0.00005 + | > step_time: 2.00090 (2.44373) + | > loader_time: 0.00290 (0.02919) + + + --> STEP: 104/234 -- GLOBAL_STEP: 45500 + | > loss: -0.23867 (-0.19480) + | > log_mle: -0.40709 (-0.31433) + | > loss_dur: 0.16842 (0.11953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.48362 (14.39902) + | > current_lr: 0.00005 + | > step_time: 1.19000 (2.44831) + | > loader_time: 0.00310 (0.02879) + + + --> STEP: 109/234 -- GLOBAL_STEP: 45505 + | > loss: -0.18209 (-0.19440) + | > log_mle: -0.37411 (-0.31600) + | > loss_dur: 0.19203 (0.12160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.39377 (14.97411) + | > current_lr: 0.00005 + | > step_time: 2.00660 (2.44060) + | > loader_time: 0.00320 (0.02840) + + + --> STEP: 114/234 -- GLOBAL_STEP: 45510 + | > loss: -0.20893 (-0.19486) + | > log_mle: -0.36303 (-0.31862) + | > loss_dur: 0.15410 (0.12377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.37404 (15.61348) + | > current_lr: 0.00005 + | > step_time: 2.19100 (2.42399) + | > loader_time: 0.00250 (0.02864) + + + --> STEP: 119/234 -- GLOBAL_STEP: 45515 + | > loss: -0.18942 (-0.19476) + | > log_mle: -0.35841 (-0.32061) + | > loss_dur: 0.16899 (0.12585) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.25828 (16.05367) + | > current_lr: 0.00005 + | > step_time: 3.01950 (2.43883) + | > loader_time: 0.09690 (0.02839) + + + --> STEP: 124/234 -- GLOBAL_STEP: 45520 + | > loss: -0.22330 (-0.19471) + | > log_mle: -0.38610 (-0.32187) + | > loss_dur: 0.16280 (0.12717) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.87294 (16.34698) + | > current_lr: 0.00005 + | > step_time: 2.55500 (2.42827) + | > loader_time: 0.00190 (0.02737) + + + --> STEP: 129/234 -- GLOBAL_STEP: 45525 + | > loss: -0.19418 (-0.19566) + | > log_mle: -0.37559 (-0.32481) + | > loss_dur: 0.18141 (0.12914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.88763 (17.21126) + | > current_lr: 0.00005 + | > step_time: 2.23270 (2.43384) + | > loader_time: 0.00280 (0.02786) + + + --> STEP: 134/234 -- GLOBAL_STEP: 45530 + | > loss: -0.22094 (-0.19725) + | > log_mle: -0.42881 (-0.32847) + | > loss_dur: 0.20786 (0.13122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.17929 (18.01710) + | > current_lr: 0.00005 + | > step_time: 3.70390 (2.45906) + | > loader_time: 0.00600 (0.02763) + + + --> STEP: 139/234 -- GLOBAL_STEP: 45535 + | > loss: -0.29469 (-0.19855) + | > log_mle: -0.49204 (-0.33189) + | > loss_dur: 0.19735 (0.13334) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.57569 (18.78607) + | > current_lr: 0.00005 + | > step_time: 9.40490 (2.55324) + | > loader_time: 0.20490 (0.03015) + + + --> STEP: 144/234 -- GLOBAL_STEP: 45540 + | > loss: -0.26499 (-0.19980) + | > log_mle: -0.46376 (-0.33548) + | > loss_dur: 0.19877 (0.13568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.82083 (19.72539) + | > current_lr: 0.00005 + | > step_time: 1.60900 (2.53618) + | > loader_time: 0.08910 (0.03046) + + + --> STEP: 149/234 -- GLOBAL_STEP: 45545 + | > loss: -0.31156 (-0.20211) + | > log_mle: -0.51694 (-0.33968) + | > loss_dur: 0.20538 (0.13757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.18394 (20.64827) + | > current_lr: 0.00005 + | > step_time: 1.08450 (2.52625) + | > loader_time: 0.00260 (0.03070) + + + --> STEP: 154/234 -- GLOBAL_STEP: 45550 + | > loss: -0.26883 (-0.20486) + | > log_mle: -0.46439 (-0.34434) + | > loss_dur: 0.19556 (0.13948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.27860 (21.93212) + | > current_lr: 0.00005 + | > step_time: 1.78860 (2.52538) + | > loader_time: 0.00280 (0.02980) + + + --> STEP: 159/234 -- GLOBAL_STEP: 45555 + | > loss: -0.28873 (-0.20726) + | > log_mle: -0.49486 (-0.34881) + | > loss_dur: 0.20613 (0.14155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.63832 (23.05237) + | > current_lr: 0.00005 + | > step_time: 9.50660 (2.60837) + | > loader_time: 0.10600 (0.03072) + + + --> STEP: 164/234 -- GLOBAL_STEP: 45560 + | > loss: -0.27473 (-0.20973) + | > log_mle: -0.48160 (-0.35307) + | > loss_dur: 0.20687 (0.14334) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.75598 (24.11787) + | > current_lr: 0.00005 + | > step_time: 2.00020 (2.64035) + | > loader_time: 0.00300 (0.03169) + + + --> STEP: 169/234 -- GLOBAL_STEP: 45565 + | > loss: -0.26578 (-0.21235) + | > log_mle: -0.48308 (-0.35755) + | > loss_dur: 0.21730 (0.14521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.60778 (25.19862) + | > current_lr: 0.00005 + | > step_time: 2.71690 (2.63038) + | > loader_time: 0.19080 (0.03304) + + + --> STEP: 174/234 -- GLOBAL_STEP: 45570 + | > loss: -0.33742 (-0.21580) + | > log_mle: -0.56109 (-0.36317) + | > loss_dur: 0.22367 (0.14737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.14838 (26.75054) + | > current_lr: 0.00005 + | > step_time: 3.30570 (2.64296) + | > loader_time: 0.08820 (0.03321) + + + --> STEP: 179/234 -- GLOBAL_STEP: 45575 + | > loss: -0.32267 (-0.21857) + | > log_mle: -0.57034 (-0.36827) + | > loss_dur: 0.24767 (0.14970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.08052 (28.14068) + | > current_lr: 0.00005 + | > step_time: 3.69310 (2.64732) + | > loader_time: 0.00440 (0.03240) + + + --> STEP: 184/234 -- GLOBAL_STEP: 45580 + | > loss: -0.31807 (-0.22124) + | > log_mle: -0.53467 (-0.37283) + | > loss_dur: 0.21660 (0.15159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.44128 (29.16580) + | > current_lr: 0.00005 + | > step_time: 6.09230 (2.67809) + | > loader_time: 0.00430 (0.03221) + + + --> STEP: 189/234 -- GLOBAL_STEP: 45585 + | > loss: -0.31303 (-0.22383) + | > log_mle: -0.53413 (-0.37749) + | > loss_dur: 0.22109 (0.15367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.86890 (30.44133) + | > current_lr: 0.00005 + | > step_time: 1.79720 (2.73115) + | > loader_time: 0.00530 (0.03338) + + + --> STEP: 194/234 -- GLOBAL_STEP: 45590 + | > loss: -0.34647 (-0.22679) + | > log_mle: -0.55593 (-0.38200) + | > loss_dur: 0.20946 (0.15521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.95979 (31.88850) + | > current_lr: 0.00005 + | > step_time: 5.09850 (2.78606) + | > loader_time: 0.08520 (0.03551) + + + --> STEP: 199/234 -- GLOBAL_STEP: 45595 + | > loss: -0.34215 (-0.22933) + | > log_mle: -0.56640 (-0.38627) + | > loss_dur: 0.22425 (0.15694) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.26817 (32.94517) + | > current_lr: 0.00005 + | > step_time: 3.91240 (2.83940) + | > loader_time: 0.08290 (0.03705) + + + --> STEP: 204/234 -- GLOBAL_STEP: 45600 + | > loss: -0.35430 (-0.23160) + | > log_mle: -0.60263 (-0.39043) + | > loss_dur: 0.24833 (0.15882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.40431 (34.18402) + | > current_lr: 0.00005 + | > step_time: 5.49620 (2.91439) + | > loader_time: 0.10660 (0.03766) + + + --> STEP: 209/234 -- GLOBAL_STEP: 45605 + | > loss: -0.32407 (-0.23414) + | > log_mle: -0.55171 (-0.39470) + | > loss_dur: 0.22764 (0.16057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.25467 (35.13762) + | > current_lr: 0.00005 + | > step_time: 2.98630 (2.98246) + | > loader_time: 0.00240 (0.04008) + + + --> STEP: 214/234 -- GLOBAL_STEP: 45610 + | > loss: -0.37781 (-0.23752) + | > log_mle: -0.58843 (-0.39988) + | > loss_dur: 0.21062 (0.16235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.80217 (36.29812) + | > current_lr: 0.00005 + | > step_time: 3.00760 (3.02811) + | > loader_time: 0.00480 (0.04062) + + + --> STEP: 219/234 -- GLOBAL_STEP: 45615 + | > loss: -0.45735 (-0.24090) + | > log_mle: -0.69536 (-0.40512) + | > loss_dur: 0.23801 (0.16421) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 128.87804 (37.54044) + | > current_lr: 0.00005 + | > step_time: 6.01060 (3.06550) + | > loader_time: 0.00660 (0.04058) + + + --> STEP: 224/234 -- GLOBAL_STEP: 45620 + | > loss: -0.40722 (-0.24415) + | > log_mle: -0.64989 (-0.41008) + | > loss_dur: 0.24268 (0.16594) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.64481 (38.60876) + | > current_lr: 0.00005 + | > step_time: 2.20080 (3.11710) + | > loader_time: 0.00420 (0.04022) + + + --> STEP: 229/234 -- GLOBAL_STEP: 45625 + | > loss: -0.38383 (-0.24750) + | > log_mle: -0.68518 (-0.41567) + | > loss_dur: 0.30136 (0.16816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 124.04053 (40.10573) + | > current_lr: 0.00005 + | > step_time: 0.25280 (3.05534) + | > loader_time: 0.00310 (0.03978) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.12880 (+0.97625) + | > avg_loss: -0.24687 (+0.01542) + | > avg_log_mle: -0.47716 (+0.00646) + | > avg_loss_dur: 0.23029 (+0.00896) + + + > EPOCH: 195/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 01:48:06)  + + --> STEP: 0/234 -- GLOBAL_STEP: 45630 + | > loss: -0.24655 (-0.24655) + | > log_mle: -0.39333 (-0.39333) + | > loss_dur: 0.14678 (0.14678) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.67861 (24.67861) + | > current_lr: 0.00005 + | > step_time: 10.70290 (10.70287) + | > loader_time: 5.78290 (5.78292) + + + --> STEP: 5/234 -- GLOBAL_STEP: 45635 + | > loss: -0.21945 (-0.19448) + | > log_mle: -0.31856 (-0.31595) + | > loss_dur: 0.09912 (0.12147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.14813 (21.23063) + | > current_lr: 0.00005 + | > step_time: 4.50770 (4.28831) + | > loader_time: 0.00320 (0.07083) + + + --> STEP: 10/234 -- GLOBAL_STEP: 45640 + | > loss: -0.20947 (-0.20632) + | > log_mle: -0.31166 (-0.31775) + | > loss_dur: 0.10219 (0.11143) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.46661 (19.27186) + | > current_lr: 0.00005 + | > step_time: 6.00080 (4.77335) + | > loader_time: 0.00300 (0.03737) + + + --> STEP: 15/234 -- GLOBAL_STEP: 45645 + | > loss: -0.23348 (-0.21267) + | > log_mle: -0.32166 (-0.31906) + | > loss_dur: 0.08819 (0.10638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.59385 (17.56894) + | > current_lr: 0.00005 + | > step_time: 3.70760 (4.41655) + | > loader_time: 0.00230 (0.04598) + + + --> STEP: 20/234 -- GLOBAL_STEP: 45650 + | > loss: -0.23170 (-0.21450) + | > log_mle: -0.31384 (-0.31670) + | > loss_dur: 0.08214 (0.10220) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.80311 (15.85093) + | > current_lr: 0.00005 + | > step_time: 5.01380 (4.63846) + | > loader_time: 0.38670 (0.05859) + + + --> STEP: 25/234 -- GLOBAL_STEP: 45655 + | > loss: -0.20566 (-0.21439) + | > log_mle: -0.29233 (-0.31398) + | > loss_dur: 0.08668 (0.09959) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.10198 (15.37147) + | > current_lr: 0.00005 + | > step_time: 2.41860 (4.57917) + | > loader_time: 0.08800 (0.05453) + + + --> STEP: 30/234 -- GLOBAL_STEP: 45660 + | > loss: -0.21181 (-0.21396) + | > log_mle: -0.30673 (-0.31287) + | > loss_dur: 0.09492 (0.09892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.37500 (14.61352) + | > current_lr: 0.00005 + | > step_time: 1.40470 (4.55033) + | > loader_time: 0.07520 (0.06686) + + + --> STEP: 35/234 -- GLOBAL_STEP: 45665 + | > loss: -0.17395 (-0.21062) + | > log_mle: -0.30158 (-0.31178) + | > loss_dur: 0.12763 (0.10117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.94925 (14.16408) + | > current_lr: 0.00005 + | > step_time: 3.61430 (4.23323) + | > loader_time: 0.29270 (0.06586) + + + --> STEP: 40/234 -- GLOBAL_STEP: 45670 + | > loss: -0.15598 (-0.20786) + | > log_mle: -0.28920 (-0.31041) + | > loss_dur: 0.13322 (0.10255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46759 (14.04068) + | > current_lr: 0.00005 + | > step_time: 3.20390 (3.97898) + | > loader_time: 0.00450 (0.05799) + + + --> STEP: 45/234 -- GLOBAL_STEP: 45675 + | > loss: -0.18499 (-0.20676) + | > log_mle: -0.31958 (-0.30996) + | > loss_dur: 0.13460 (0.10320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.70853 (13.80884) + | > current_lr: 0.00005 + | > step_time: 1.29580 (3.65603) + | > loader_time: 0.00220 (0.05178) + + + --> STEP: 50/234 -- GLOBAL_STEP: 45680 + | > loss: -0.19733 (-0.20602) + | > log_mle: -0.29737 (-0.30922) + | > loss_dur: 0.10004 (0.10320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.81159 (13.31979) + | > current_lr: 0.00005 + | > step_time: 1.21930 (3.46306) + | > loader_time: 0.00240 (0.04848) + + + --> STEP: 55/234 -- GLOBAL_STEP: 45685 + | > loss: -0.21797 (-0.20511) + | > log_mle: -0.31047 (-0.30872) + | > loss_dur: 0.09250 (0.10361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.04970 (12.98788) + | > current_lr: 0.00005 + | > step_time: 1.21560 (3.27427) + | > loader_time: 0.00200 (0.04426) + + + --> STEP: 60/234 -- GLOBAL_STEP: 45690 + | > loss: -0.19423 (-0.20386) + | > log_mle: -0.32367 (-0.30866) + | > loss_dur: 0.12943 (0.10480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.45041 (12.78961) + | > current_lr: 0.00005 + | > step_time: 2.31110 (3.16072) + | > loader_time: 0.08590 (0.04212) + + + --> STEP: 65/234 -- GLOBAL_STEP: 45695 + | > loss: -0.19623 (-0.20203) + | > log_mle: -0.30311 (-0.30867) + | > loss_dur: 0.10688 (0.10664) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.11536 (12.71506) + | > current_lr: 0.00005 + | > step_time: 1.93080 (3.10020) + | > loader_time: 0.00220 (0.03913) + + + --> STEP: 70/234 -- GLOBAL_STEP: 45700 + | > loss: -0.15120 (-0.19987) + | > log_mle: -0.29018 (-0.30778) + | > loss_dur: 0.13897 (0.10791) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.22586 (12.63669) + | > current_lr: 0.00005 + | > step_time: 1.49810 (3.00696) + | > loader_time: 0.00320 (0.03916) + + + --> STEP: 75/234 -- GLOBAL_STEP: 45705 + | > loss: -0.17134 (-0.19780) + | > log_mle: -0.30929 (-0.30767) + | > loss_dur: 0.13795 (0.10988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.45879 (12.81955) + | > current_lr: 0.00005 + | > step_time: 3.10620 (2.98105) + | > loader_time: 0.00350 (0.03789) + + + --> STEP: 80/234 -- GLOBAL_STEP: 45710 + | > loss: -0.18133 (-0.19666) + | > log_mle: -0.29109 (-0.30729) + | > loss_dur: 0.10976 (0.11062) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.76510 (12.71399) + | > current_lr: 0.00005 + | > step_time: 1.68910 (2.98978) + | > loader_time: 0.00210 (0.03804) + + + --> STEP: 85/234 -- GLOBAL_STEP: 45715 + | > loss: -0.18991 (-0.19545) + | > log_mle: -0.30327 (-0.30724) + | > loss_dur: 0.11336 (0.11179) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.94700 (12.76815) + | > current_lr: 0.00005 + | > step_time: 2.19640 (2.96198) + | > loader_time: 0.00200 (0.03687) + + + --> STEP: 90/234 -- GLOBAL_STEP: 45720 + | > loss: -0.18061 (-0.19465) + | > log_mle: -0.32890 (-0.30824) + | > loss_dur: 0.14828 (0.11358) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.70422 (13.02726) + | > current_lr: 0.00005 + | > step_time: 5.01060 (2.94097) + | > loader_time: 0.09200 (0.03703) + + + --> STEP: 95/234 -- GLOBAL_STEP: 45725 + | > loss: -0.23550 (-0.19518) + | > log_mle: -0.41276 (-0.31116) + | > loss_dur: 0.17727 (0.11598) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.18855 (13.66083) + | > current_lr: 0.00005 + | > step_time: 0.91410 (2.99342) + | > loader_time: 0.08580 (0.03905) + + + --> STEP: 100/234 -- GLOBAL_STEP: 45730 + | > loss: -0.20616 (-0.19501) + | > log_mle: -0.34120 (-0.31241) + | > loss_dur: 0.13504 (0.11740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.08176 (13.96231) + | > current_lr: 0.00005 + | > step_time: 2.09720 (2.94681) + | > loader_time: 0.00220 (0.03825) + + + --> STEP: 105/234 -- GLOBAL_STEP: 45735 + | > loss: -0.18077 (-0.19528) + | > log_mle: -0.31898 (-0.31484) + | > loss_dur: 0.13821 (0.11956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.91323 (14.59599) + | > current_lr: 0.00005 + | > step_time: 1.50170 (2.87977) + | > loader_time: 0.00390 (0.03657) + + + --> STEP: 110/234 -- GLOBAL_STEP: 45740 + | > loss: -0.18205 (-0.19471) + | > log_mle: -0.33991 (-0.31667) + | > loss_dur: 0.15786 (0.12196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.94756 (15.36110) + | > current_lr: 0.00005 + | > step_time: 5.30680 (2.91079) + | > loader_time: 0.00300 (0.03586) + + + --> STEP: 115/234 -- GLOBAL_STEP: 45745 + | > loss: -0.18294 (-0.19509) + | > log_mle: -0.36334 (-0.31943) + | > loss_dur: 0.18040 (0.12433) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.00154 (16.00251) + | > current_lr: 0.00005 + | > step_time: 1.80190 (2.96754) + | > loader_time: 0.00260 (0.03773) + + + --> STEP: 120/234 -- GLOBAL_STEP: 45750 + | > loss: -0.23772 (-0.19544) + | > log_mle: -0.41333 (-0.32182) + | > loss_dur: 0.17561 (0.12637) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.07870 (16.42164) + | > current_lr: 0.00005 + | > step_time: 2.90850 (2.93894) + | > loader_time: 0.09450 (0.03849) + + + --> STEP: 125/234 -- GLOBAL_STEP: 45755 + | > loss: -0.21661 (-0.19528) + | > log_mle: -0.39625 (-0.32292) + | > loss_dur: 0.17964 (0.12763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.72017 (16.74937) + | > current_lr: 0.00005 + | > step_time: 2.59470 (2.91009) + | > loader_time: 0.00310 (0.03856) + + + --> STEP: 130/234 -- GLOBAL_STEP: 45760 + | > loss: -0.22378 (-0.19602) + | > log_mle: -0.40462 (-0.32572) + | > loss_dur: 0.18084 (0.12970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.17649 (17.78266) + | > current_lr: 0.00005 + | > step_time: 2.50090 (2.88376) + | > loader_time: 0.00360 (0.03853) + + + --> STEP: 135/234 -- GLOBAL_STEP: 45765 + | > loss: -0.18861 (-0.19722) + | > log_mle: -0.33767 (-0.32861) + | > loss_dur: 0.14905 (0.13139) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.64443 (18.51252) + | > current_lr: 0.00005 + | > step_time: 2.89660 (2.87182) + | > loader_time: 0.00440 (0.03845) + + + --> STEP: 140/234 -- GLOBAL_STEP: 45770 + | > loss: -0.18935 (-0.19832) + | > log_mle: -0.37078 (-0.33206) + | > loss_dur: 0.18143 (0.13373) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.81573 (19.28535) + | > current_lr: 0.00005 + | > step_time: 2.02990 (2.87944) + | > loader_time: 0.09660 (0.03922) + + + --> STEP: 145/234 -- GLOBAL_STEP: 45775 + | > loss: -0.28176 (-0.20023) + | > log_mle: -0.47446 (-0.33624) + | > loss_dur: 0.19270 (0.13601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.20967 (20.17788) + | > current_lr: 0.00005 + | > step_time: 3.50350 (2.84989) + | > loader_time: 0.19590 (0.03967) + + + --> STEP: 150/234 -- GLOBAL_STEP: 45780 + | > loss: -0.25518 (-0.20233) + | > log_mle: -0.45678 (-0.34023) + | > loss_dur: 0.20160 (0.13790) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.41237 (21.10921) + | > current_lr: 0.00005 + | > step_time: 1.60120 (2.80768) + | > loader_time: 0.07630 (0.04009) + + + --> STEP: 155/234 -- GLOBAL_STEP: 45785 + | > loss: -0.31063 (-0.20525) + | > log_mle: -0.52224 (-0.34526) + | > loss_dur: 0.21161 (0.14001) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.34046 (22.36192) + | > current_lr: 0.00005 + | > step_time: 1.80520 (2.78310) + | > loader_time: 0.00400 (0.03997) + + + --> STEP: 160/234 -- GLOBAL_STEP: 45790 + | > loss: -0.30181 (-0.20756) + | > log_mle: -0.51694 (-0.34965) + | > loss_dur: 0.21513 (0.14208) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.80942 (23.53423) + | > current_lr: 0.00005 + | > step_time: 1.70280 (2.76673) + | > loader_time: 0.09730 (0.04165) + + + --> STEP: 165/234 -- GLOBAL_STEP: 45795 + | > loss: -0.30857 (-0.20994) + | > log_mle: -0.51938 (-0.35385) + | > loss_dur: 0.21081 (0.14391) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.59040 (24.42514) + | > current_lr: 0.00005 + | > step_time: 1.99460 (2.77429) + | > loader_time: 0.00360 (0.04282) + + + --> STEP: 170/234 -- GLOBAL_STEP: 45800 + | > loss: -0.32908 (-0.21263) + | > log_mle: -0.56225 (-0.35852) + | > loss_dur: 0.23317 (0.14589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.84303 (25.67890) + | > current_lr: 0.00005 + | > step_time: 1.80650 (2.75174) + | > loader_time: 0.00300 (0.04311) + + + --> STEP: 175/234 -- GLOBAL_STEP: 45805 + | > loss: -0.29445 (-0.21593) + | > log_mle: -0.53437 (-0.36392) + | > loss_dur: 0.23993 (0.14799) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.65247 (27.11907) + | > current_lr: 0.00005 + | > step_time: 1.91080 (2.73370) + | > loader_time: 0.00430 (0.04246) + + + --> STEP: 180/234 -- GLOBAL_STEP: 45810 + | > loss: -0.33135 (-0.21876) + | > log_mle: -0.54101 (-0.36894) + | > loss_dur: 0.20966 (0.15017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.63483 (28.33677) + | > current_lr: 0.00005 + | > step_time: 1.19080 (2.71563) + | > loader_time: 0.00410 (0.04235) + + + --> STEP: 185/234 -- GLOBAL_STEP: 45815 + | > loss: -0.31686 (-0.22107) + | > log_mle: -0.56340 (-0.37337) + | > loss_dur: 0.24654 (0.15230) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.67259 (29.69262) + | > current_lr: 0.00005 + | > step_time: 2.10890 (2.70497) + | > loader_time: 0.09280 (0.04272) + + + --> STEP: 190/234 -- GLOBAL_STEP: 45820 + | > loss: -0.33137 (-0.22368) + | > log_mle: -0.54316 (-0.37786) + | > loss_dur: 0.21179 (0.15418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.87663 (30.81477) + | > current_lr: 0.00005 + | > step_time: 2.20440 (2.71170) + | > loader_time: 0.10000 (0.04319) + + + --> STEP: 195/234 -- GLOBAL_STEP: 45825 + | > loss: -0.33574 (-0.22693) + | > log_mle: -0.56488 (-0.38277) + | > loss_dur: 0.22913 (0.15584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.67908 (31.87405) + | > current_lr: 0.00005 + | > step_time: 6.38890 (2.72501) + | > loader_time: 0.00780 (0.04293) + + + --> STEP: 200/234 -- GLOBAL_STEP: 45830 + | > loss: -0.31381 (-0.22931) + | > log_mle: -0.55971 (-0.38686) + | > loss_dur: 0.24591 (0.15756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.52367 (32.89993) + | > current_lr: 0.00005 + | > step_time: 9.10390 (2.81953) + | > loader_time: 0.09950 (0.04379) + + + --> STEP: 205/234 -- GLOBAL_STEP: 45835 + | > loss: -0.32212 (-0.23176) + | > log_mle: -0.55136 (-0.39102) + | > loss_dur: 0.22924 (0.15925) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.54748 (33.73767) + | > current_lr: 0.00005 + | > step_time: 5.21080 (2.87622) + | > loader_time: 0.29960 (0.04513) + + + --> STEP: 210/234 -- GLOBAL_STEP: 45840 + | > loss: -0.39420 (-0.23513) + | > log_mle: -0.63647 (-0.39614) + | > loss_dur: 0.24228 (0.16101) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.87649 (34.69474) + | > current_lr: 0.00005 + | > step_time: 8.19550 (2.98063) + | > loader_time: 0.30380 (0.04769) + + + --> STEP: 215/234 -- GLOBAL_STEP: 45845 + | > loss: -0.35657 (-0.23880) + | > log_mle: -0.59128 (-0.40153) + | > loss_dur: 0.23471 (0.16273) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.33625 (35.79620) + | > current_lr: 0.00005 + | > step_time: 5.88680 (3.03493) + | > loader_time: 0.00330 (0.04759) + + + --> STEP: 220/234 -- GLOBAL_STEP: 45850 + | > loss: -0.39994 (-0.24259) + | > log_mle: -0.64464 (-0.40712) + | > loss_dur: 0.24471 (0.16453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.68285 (37.32190) + | > current_lr: 0.00005 + | > step_time: 2.49270 (3.06183) + | > loader_time: 0.00370 (0.05248) + + + --> STEP: 225/234 -- GLOBAL_STEP: 45855 + | > loss: -0.46502 (-0.24612) + | > log_mle: -0.71231 (-0.41236) + | > loss_dur: 0.24729 (0.16624) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.61662 (38.78949) + | > current_lr: 0.00005 + | > step_time: 0.23540 (3.04075) + | > loader_time: 0.00330 (0.05254) + + + --> STEP: 230/234 -- GLOBAL_STEP: 45860 + | > loss: -0.42555 (-0.24927) + | > log_mle: -0.75258 (-0.41798) + | > loss_dur: 0.32703 (0.16871) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.76093 (40.29581) + | > current_lr: 0.00005 + | > step_time: 0.26100 (2.98010) + | > loader_time: 0.00500 (0.05150) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.39738 (-0.73143) + | > avg_loss: -0.28178 (-0.03491) + | > avg_log_mle: -0.50239 (-0.02523) + | > avg_loss_dur: 0.22061 (-0.00968) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_45864.pth + + > EPOCH: 196/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 02:00:49)  + + --> STEP: 1/234 -- GLOBAL_STEP: 45865 + | > loss: -0.22572 (-0.22572) + | > log_mle: -0.31574 (-0.31574) + | > loss_dur: 0.09002 (0.09002) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.39841 (24.39841) + | > current_lr: 0.00005 + | > step_time: 4.50500 (4.50495) + | > loader_time: 0.00150 (0.00154) + + + --> STEP: 6/234 -- GLOBAL_STEP: 45870 + | > loss: -0.22970 (-0.20654) + | > log_mle: -0.31302 (-0.31717) + | > loss_dur: 0.08332 (0.11063) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.99371 (19.35849) + | > current_lr: 0.00005 + | > step_time: 6.40170 (5.98935) + | > loader_time: 0.19870 (0.10033) + + + --> STEP: 11/234 -- GLOBAL_STEP: 45875 + | > loss: -0.24331 (-0.21553) + | > log_mle: -0.32091 (-0.32005) + | > loss_dur: 0.07760 (0.10451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.94493 (17.46058) + | > current_lr: 0.00005 + | > step_time: 1.70020 (4.30254) + | > loader_time: 0.09280 (0.06454) + + + --> STEP: 16/234 -- GLOBAL_STEP: 45880 + | > loss: -0.22607 (-0.21846) + | > log_mle: -0.31672 (-0.32030) + | > loss_dur: 0.09066 (0.10184) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.22957 (16.43292) + | > current_lr: 0.00005 + | > step_time: 2.99600 (3.83245) + | > loader_time: 0.00400 (0.06298) + + + --> STEP: 21/234 -- GLOBAL_STEP: 45885 + | > loss: -0.20452 (-0.21796) + | > log_mle: -0.29588 (-0.31718) + | > loss_dur: 0.09136 (0.09922) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.07605 (15.18096) + | > current_lr: 0.00005 + | > step_time: 8.39580 (4.47626) + | > loader_time: 0.19400 (0.06668) + + + --> STEP: 26/234 -- GLOBAL_STEP: 45890 + | > loss: -0.20852 (-0.21859) + | > log_mle: -0.31243 (-0.31591) + | > loss_dur: 0.10391 (0.09732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.00598 (14.37535) + | > current_lr: 0.00005 + | > step_time: 2.09190 (4.53851) + | > loader_time: 0.00310 (0.07249) + + + --> STEP: 31/234 -- GLOBAL_STEP: 45895 + | > loss: -0.17143 (-0.21690) + | > log_mle: -0.30180 (-0.31460) + | > loss_dur: 0.13037 (0.09770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.51303 (13.98631) + | > current_lr: 0.00005 + | > step_time: 3.09370 (4.18695) + | > loader_time: 0.00670 (0.06366) + + + --> STEP: 36/234 -- GLOBAL_STEP: 45900 + | > loss: -0.18621 (-0.21437) + | > log_mle: -0.30126 (-0.31341) + | > loss_dur: 0.11505 (0.09904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.16735 (13.67726) + | > current_lr: 0.00005 + | > step_time: 3.30400 (4.15272) + | > loader_time: 0.00240 (0.06038) + + + --> STEP: 41/234 -- GLOBAL_STEP: 45905 + | > loss: -0.21465 (-0.21219) + | > log_mle: -0.30882 (-0.31229) + | > loss_dur: 0.09417 (0.10011) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.07638 (13.37031) + | > current_lr: 0.00005 + | > step_time: 1.43060 (4.08347) + | > loader_time: 0.00130 (0.05806) + + + --> STEP: 46/234 -- GLOBAL_STEP: 45910 + | > loss: -0.18553 (-0.21017) + | > log_mle: -0.30367 (-0.31171) + | > loss_dur: 0.11814 (0.10154) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.72956 (13.18962) + | > current_lr: 0.00005 + | > step_time: 2.71550 (3.83197) + | > loader_time: 0.08330 (0.05371) + + + --> STEP: 51/234 -- GLOBAL_STEP: 45915 + | > loss: -0.18539 (-0.20948) + | > log_mle: -0.29654 (-0.31094) + | > loss_dur: 0.11114 (0.10146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.94113 (12.76173) + | > current_lr: 0.00005 + | > step_time: 2.01280 (3.63607) + | > loader_time: 0.00310 (0.05176) + + + --> STEP: 56/234 -- GLOBAL_STEP: 45920 + | > loss: -0.19082 (-0.20819) + | > log_mle: -0.30948 (-0.31077) + | > loss_dur: 0.11866 (0.10257) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.66627 (12.69915) + | > current_lr: 0.00005 + | > step_time: 0.80470 (3.49359) + | > loader_time: 0.00340 (0.04891) + + + --> STEP: 61/234 -- GLOBAL_STEP: 45925 + | > loss: -0.18184 (-0.20662) + | > log_mle: -0.30263 (-0.31044) + | > loss_dur: 0.12079 (0.10382) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.29404 (12.84554) + | > current_lr: 0.00005 + | > step_time: 1.15100 (3.34166) + | > loader_time: 0.00260 (0.04519) + + + --> STEP: 66/234 -- GLOBAL_STEP: 45930 + | > loss: -0.18907 (-0.20446) + | > log_mle: -0.29236 (-0.30997) + | > loss_dur: 0.10330 (0.10551) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.77487 (13.08747) + | > current_lr: 0.00005 + | > step_time: 4.01370 (3.25312) + | > loader_time: 0.08690 (0.04324) + + + --> STEP: 71/234 -- GLOBAL_STEP: 45935 + | > loss: -0.16758 (-0.20190) + | > log_mle: -0.32191 (-0.30925) + | > loss_dur: 0.15432 (0.10735) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.78045 (13.31867) + | > current_lr: 0.00005 + | > step_time: 1.20740 (3.14962) + | > loader_time: 0.08980 (0.04281) + + + --> STEP: 76/234 -- GLOBAL_STEP: 45940 + | > loss: -0.17963 (-0.19985) + | > log_mle: -0.31106 (-0.30889) + | > loss_dur: 0.13143 (0.10905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.80455 (13.39748) + | > current_lr: 0.00005 + | > step_time: 1.40320 (3.10824) + | > loader_time: 0.00280 (0.04270) + + + --> STEP: 81/234 -- GLOBAL_STEP: 45945 + | > loss: -0.18620 (-0.19879) + | > log_mle: -0.31806 (-0.30861) + | > loss_dur: 0.13186 (0.10982) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.00063 (13.27815) + | > current_lr: 0.00005 + | > step_time: 2.38740 (3.03998) + | > loader_time: 0.00290 (0.04343) + + + --> STEP: 86/234 -- GLOBAL_STEP: 45950 + | > loss: -0.17524 (-0.19748) + | > log_mle: -0.31878 (-0.30862) + | > loss_dur: 0.14354 (0.11114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.74099 (13.24040) + | > current_lr: 0.00005 + | > step_time: 1.09320 (3.00614) + | > loader_time: 0.00270 (0.04424) + + + --> STEP: 91/234 -- GLOBAL_STEP: 45955 + | > loss: -0.17564 (-0.19672) + | > log_mle: -0.32770 (-0.30974) + | > loss_dur: 0.15206 (0.11302) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.66287 (13.34014) + | > current_lr: 0.00005 + | > step_time: 0.66210 (2.91875) + | > loader_time: 0.00310 (0.04202) + + + --> STEP: 96/234 -- GLOBAL_STEP: 45960 + | > loss: -0.18132 (-0.19731) + | > log_mle: -0.31313 (-0.31243) + | > loss_dur: 0.13181 (0.11512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.06145 (13.91326) + | > current_lr: 0.00005 + | > step_time: 2.59450 (2.88333) + | > loader_time: 0.00320 (0.04059) + + + --> STEP: 101/234 -- GLOBAL_STEP: 45965 + | > loss: -0.19368 (-0.19726) + | > log_mle: -0.36835 (-0.31421) + | > loss_dur: 0.17467 (0.11695) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.35260 (14.23616) + | > current_lr: 0.00005 + | > step_time: 2.30000 (2.84501) + | > loader_time: 0.00220 (0.04204) + + + --> STEP: 106/234 -- GLOBAL_STEP: 45970 + | > loss: -0.17758 (-0.19713) + | > log_mle: -0.36212 (-0.31645) + | > loss_dur: 0.18454 (0.11931) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.03180 (14.89810) + | > current_lr: 0.00005 + | > step_time: 2.70470 (2.83042) + | > loader_time: 0.08890 (0.04190) + + + --> STEP: 111/234 -- GLOBAL_STEP: 45975 + | > loss: -0.19981 (-0.19657) + | > log_mle: -0.40973 (-0.31866) + | > loss_dur: 0.20992 (0.12209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.89252 (15.51680) + | > current_lr: 0.00005 + | > step_time: 2.32210 (2.81130) + | > loader_time: 0.00230 (0.04107) + + + --> STEP: 116/234 -- GLOBAL_STEP: 45980 + | > loss: -0.17927 (-0.19645) + | > log_mle: -0.38008 (-0.32092) + | > loss_dur: 0.20081 (0.12447) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.48987 (16.02752) + | > current_lr: 0.00005 + | > step_time: 2.60170 (2.79793) + | > loader_time: 0.07540 (0.04155) + + + --> STEP: 121/234 -- GLOBAL_STEP: 45985 + | > loss: -0.15320 (-0.19641) + | > log_mle: -0.29626 (-0.32246) + | > loss_dur: 0.14306 (0.12606) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.66072 (16.28957) + | > current_lr: 0.00005 + | > step_time: 3.10070 (2.78921) + | > loader_time: 0.18880 (0.04377) + + + --> STEP: 126/234 -- GLOBAL_STEP: 45990 + | > loss: -0.24273 (-0.19699) + | > log_mle: -0.42943 (-0.32460) + | > loss_dur: 0.18670 (0.12761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.53329 (16.76506) + | > current_lr: 0.00005 + | > step_time: 2.00050 (2.77530) + | > loader_time: 0.00390 (0.04217) + + + --> STEP: 131/234 -- GLOBAL_STEP: 45995 + | > loss: -0.28623 (-0.19828) + | > log_mle: -0.48101 (-0.32791) + | > loss_dur: 0.19478 (0.12963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.46311 (17.64862) + | > current_lr: 0.00005 + | > step_time: 1.99130 (2.76646) + | > loader_time: 0.00440 (0.04276) + + + --> STEP: 136/234 -- GLOBAL_STEP: 46000 + | > loss: -0.30732 (-0.19965) + | > log_mle: -0.52682 (-0.33125) + | > loss_dur: 0.21950 (0.13160) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.94376 (18.53053) + | > current_lr: 0.00005 + | > step_time: 1.50270 (2.73165) + | > loader_time: 0.00210 (0.04194) + + + --> STEP: 141/234 -- GLOBAL_STEP: 46005 + | > loss: -0.24296 (-0.20052) + | > log_mle: -0.42730 (-0.33413) + | > loss_dur: 0.18434 (0.13361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.33184 (19.30623) + | > current_lr: 0.00005 + | > step_time: 1.90740 (2.70146) + | > loader_time: 0.08200 (0.04112) + + + --> STEP: 146/234 -- GLOBAL_STEP: 46010 + | > loss: -0.28447 (-0.20265) + | > log_mle: -0.47576 (-0.33861) + | > loss_dur: 0.19129 (0.13597) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.42584 (20.51638) + | > current_lr: 0.00005 + | > step_time: 1.89860 (2.68565) + | > loader_time: 0.00330 (0.04103) + + + --> STEP: 151/234 -- GLOBAL_STEP: 46015 + | > loss: -0.26401 (-0.20449) + | > log_mle: -0.44513 (-0.34228) + | > loss_dur: 0.18112 (0.13780) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.11528 (21.34695) + | > current_lr: 0.00005 + | > step_time: 2.09360 (2.68352) + | > loader_time: 0.00200 (0.04095) + + + --> STEP: 156/234 -- GLOBAL_STEP: 46020 + | > loss: -0.27751 (-0.20760) + | > log_mle: -0.47778 (-0.34751) + | > loss_dur: 0.20027 (0.13991) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.62852 (22.84027) + | > current_lr: 0.00005 + | > step_time: 1.59270 (2.65890) + | > loader_time: 0.00360 (0.04033) + + + --> STEP: 161/234 -- GLOBAL_STEP: 46025 + | > loss: -0.32257 (-0.21001) + | > log_mle: -0.51409 (-0.35187) + | > loss_dur: 0.19153 (0.14186) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.29912 (23.87980) + | > current_lr: 0.00005 + | > step_time: 5.19700 (2.70125) + | > loader_time: 0.00390 (0.04024) + + + --> STEP: 166/234 -- GLOBAL_STEP: 46030 + | > loss: -0.27398 (-0.21201) + | > log_mle: -0.44839 (-0.35557) + | > loss_dur: 0.17441 (0.14356) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.55787 (24.74958) + | > current_lr: 0.00005 + | > step_time: 3.20370 (2.74898) + | > loader_time: 0.00490 (0.04080) + + + --> STEP: 171/234 -- GLOBAL_STEP: 46035 + | > loss: -0.36260 (-0.21533) + | > log_mle: -0.56499 (-0.36094) + | > loss_dur: 0.20239 (0.14560) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.76034 (25.81113) + | > current_lr: 0.00005 + | > step_time: 1.80220 (2.76128) + | > loader_time: 0.00320 (0.04068) + + + --> STEP: 176/234 -- GLOBAL_STEP: 46040 + | > loss: -0.31960 (-0.21841) + | > log_mle: -0.53537 (-0.36621) + | > loss_dur: 0.21577 (0.14779) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.83395 (27.10786) + | > current_lr: 0.00005 + | > step_time: 2.40100 (2.77719) + | > loader_time: 0.09160 (0.04170) + + + --> STEP: 181/234 -- GLOBAL_STEP: 46045 + | > loss: -0.25463 (-0.22107) + | > log_mle: -0.46630 (-0.37088) + | > loss_dur: 0.21167 (0.14981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.77873 (28.21190) + | > current_lr: 0.00005 + | > step_time: 6.11130 (2.81541) + | > loader_time: 0.19660 (0.04222) + + + --> STEP: 186/234 -- GLOBAL_STEP: 46050 + | > loss: -0.27414 (-0.22369) + | > log_mle: -0.50986 (-0.37566) + | > loss_dur: 0.23571 (0.15197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.11737 (29.57902) + | > current_lr: 0.00005 + | > step_time: 1.49680 (2.87246) + | > loader_time: 0.00550 (0.04325) + + + --> STEP: 191/234 -- GLOBAL_STEP: 46055 + | > loss: -0.31697 (-0.22664) + | > log_mle: -0.52818 (-0.38034) + | > loss_dur: 0.21121 (0.15370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.34195 (30.86308) + | > current_lr: 0.00005 + | > step_time: 7.30900 (2.97381) + | > loader_time: 0.00390 (0.04417) + + + --> STEP: 196/234 -- GLOBAL_STEP: 46060 + | > loss: -0.28554 (-0.22963) + | > log_mle: -0.52517 (-0.38509) + | > loss_dur: 0.23963 (0.15546) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.59173 (32.06200) + | > current_lr: 0.00005 + | > step_time: 2.45400 (3.00233) + | > loader_time: 0.00590 (0.04509) + + + --> STEP: 201/234 -- GLOBAL_STEP: 46065 + | > loss: -0.25761 (-0.23201) + | > log_mle: -0.48029 (-0.38919) + | > loss_dur: 0.22268 (0.15718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.97826 (33.20611) + | > current_lr: 0.00005 + | > step_time: 4.60800 (2.99353) + | > loader_time: 0.11590 (0.04540) + + + --> STEP: 206/234 -- GLOBAL_STEP: 46070 + | > loss: -0.36234 (-0.23495) + | > log_mle: -0.58819 (-0.39388) + | > loss_dur: 0.22585 (0.15892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.31700 (34.34471) + | > current_lr: 0.00005 + | > step_time: 2.39130 (2.98338) + | > loader_time: 0.10210 (0.04487) + + + --> STEP: 211/234 -- GLOBAL_STEP: 46075 + | > loss: -0.41618 (-0.23858) + | > log_mle: -0.67045 (-0.39937) + | > loss_dur: 0.25427 (0.16079) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.25780 (35.49762) + | > current_lr: 0.00005 + | > step_time: 5.79810 (3.09607) + | > loader_time: 0.19800 (0.04613) + + + --> STEP: 216/234 -- GLOBAL_STEP: 46080 + | > loss: -0.40011 (-0.24195) + | > log_mle: -0.65019 (-0.40450) + | > loss_dur: 0.25008 (0.16255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.06757 (36.76449) + | > current_lr: 0.00005 + | > step_time: 8.09990 (3.16886) + | > loader_time: 0.11150 (0.04843) + + + --> STEP: 221/234 -- GLOBAL_STEP: 46085 + | > loss: -0.35062 (-0.24547) + | > log_mle: -0.57259 (-0.40966) + | > loss_dur: 0.22197 (0.16419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.19035 (38.02785) + | > current_lr: 0.00005 + | > step_time: 2.41000 (3.18732) + | > loader_time: 0.00350 (0.04910) + + + --> STEP: 226/234 -- GLOBAL_STEP: 46090 + | > loss: -0.41471 (-0.24916) + | > log_mle: -0.66333 (-0.41525) + | > loss_dur: 0.24862 (0.16609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.95680 (39.51516) + | > current_lr: 0.00005 + | > step_time: 0.24180 (3.13244) + | > loader_time: 0.00470 (0.04811) + + + --> STEP: 231/234 -- GLOBAL_STEP: 46095 + | > loss: -0.36870 (-0.25214) + | > log_mle: -0.74930 (-0.42126) + | > loss_dur: 0.38060 (0.16912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.24367 (40.91146) + | > current_lr: 0.00005 + | > step_time: 0.28940 (3.07034) + | > loader_time: 0.00380 (0.04716) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.82208 (+0.42471) + | > avg_loss: -0.28583 (-0.00405) + | > avg_log_mle: -0.49971 (+0.00268) + | > avg_loss_dur: 0.21388 (-0.00673) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_46098.pth + + > EPOCH: 197/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 02:14:06)  + + --> STEP: 2/234 -- GLOBAL_STEP: 46100 + | > loss: -0.22860 (-0.21162) + | > log_mle: -0.33225 (-0.32488) + | > loss_dur: 0.10365 (0.11326) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.55594 (18.12164) + | > current_lr: 0.00005 + | > step_time: 3.51170 (5.90401) + | > loader_time: 0.09400 (0.09589) + + + --> STEP: 7/234 -- GLOBAL_STEP: 46105 + | > loss: -0.22689 (-0.20561) + | > log_mle: -0.32415 (-0.31968) + | > loss_dur: 0.09726 (0.11407) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.24326 (19.16374) + | > current_lr: 0.00005 + | > step_time: 1.31390 (6.02185) + | > loader_time: 0.10560 (0.12925) + + + --> STEP: 12/234 -- GLOBAL_STEP: 46110 + | > loss: -0.20897 (-0.21273) + | > log_mle: -0.31836 (-0.32078) + | > loss_dur: 0.10939 (0.10805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.04224 (18.26611) + | > current_lr: 0.00005 + | > step_time: 0.58020 (3.96848) + | > loader_time: 0.00120 (0.07619) + + + --> STEP: 17/234 -- GLOBAL_STEP: 46115 + | > loss: -0.21707 (-0.21633) + | > log_mle: -0.30346 (-0.32056) + | > loss_dur: 0.08640 (0.10424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.53662 (16.17976) + | > current_lr: 0.00005 + | > step_time: 2.50460 (3.23283) + | > loader_time: 0.00260 (0.05440) + + + --> STEP: 22/234 -- GLOBAL_STEP: 46120 + | > loss: -0.21015 (-0.21695) + | > log_mle: -0.31673 (-0.31843) + | > loss_dur: 0.10658 (0.10148) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01085 (15.02995) + | > current_lr: 0.00005 + | > step_time: 4.40010 (2.95362) + | > loader_time: 0.00160 (0.04657) + + + --> STEP: 27/234 -- GLOBAL_STEP: 46125 + | > loss: -0.22452 (-0.21805) + | > log_mle: -0.31359 (-0.31715) + | > loss_dur: 0.08907 (0.09910) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.58453 (14.17380) + | > current_lr: 0.00005 + | > step_time: 3.01670 (3.28726) + | > loader_time: 0.09250 (0.04532) + + + --> STEP: 32/234 -- GLOBAL_STEP: 46130 + | > loss: -0.23945 (-0.21762) + | > log_mle: -0.31890 (-0.31616) + | > loss_dur: 0.07945 (0.09854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.70311 (13.58169) + | > current_lr: 0.00005 + | > step_time: 3.10070 (3.37992) + | > loader_time: 0.01150 (0.04704) + + + --> STEP: 37/234 -- GLOBAL_STEP: 46135 + | > loss: -0.21086 (-0.21506) + | > log_mle: -0.29888 (-0.31429) + | > loss_dur: 0.08802 (0.09923) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.00826 (13.49005) + | > current_lr: 0.00005 + | > step_time: 1.69000 (3.21134) + | > loader_time: 0.00790 (0.04120) + + + --> STEP: 42/234 -- GLOBAL_STEP: 46140 + | > loss: -0.18417 (-0.21272) + | > log_mle: -0.29079 (-0.31295) + | > loss_dur: 0.10662 (0.10023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.37014 (13.41146) + | > current_lr: 0.00005 + | > step_time: 1.04460 (2.99035) + | > loader_time: 0.00240 (0.04046) + + + --> STEP: 47/234 -- GLOBAL_STEP: 46145 + | > loss: -0.19119 (-0.21096) + | > log_mle: -0.30277 (-0.31249) + | > loss_dur: 0.11158 (0.10152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.22136 (13.40795) + | > current_lr: 0.00005 + | > step_time: 2.73030 (2.90132) + | > loader_time: 0.09380 (0.03838) + + + --> STEP: 52/234 -- GLOBAL_STEP: 46150 + | > loss: -0.17988 (-0.20951) + | > log_mle: -0.29760 (-0.31144) + | > loss_dur: 0.11772 (0.10193) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.66846 (13.10740) + | > current_lr: 0.00005 + | > step_time: 1.21480 (2.79116) + | > loader_time: 0.08400 (0.03978) + + + --> STEP: 57/234 -- GLOBAL_STEP: 46155 + | > loss: -0.17469 (-0.20801) + | > log_mle: -0.29050 (-0.31091) + | > loss_dur: 0.11581 (0.10290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.40666 (12.99146) + | > current_lr: 0.00005 + | > step_time: 1.41030 (2.66008) + | > loader_time: 0.08730 (0.03823) + + + --> STEP: 62/234 -- GLOBAL_STEP: 46160 + | > loss: -0.14789 (-0.20622) + | > log_mle: -0.33529 (-0.31140) + | > loss_dur: 0.18740 (0.10519) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.59239 (13.02454) + | > current_lr: 0.00005 + | > step_time: 3.90300 (2.60177) + | > loader_time: 0.00280 (0.03537) + + + --> STEP: 67/234 -- GLOBAL_STEP: 46165 + | > loss: -0.18430 (-0.20485) + | > log_mle: -0.31527 (-0.31085) + | > loss_dur: 0.13098 (0.10600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.97566 (12.74084) + | > current_lr: 0.00005 + | > step_time: 1.83780 (2.56477) + | > loader_time: 0.00310 (0.03816) + + + --> STEP: 72/234 -- GLOBAL_STEP: 46170 + | > loss: -0.17849 (-0.20237) + | > log_mle: -0.29769 (-0.31006) + | > loss_dur: 0.11920 (0.10769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.78964 (12.72909) + | > current_lr: 0.00005 + | > step_time: 1.89310 (2.51191) + | > loader_time: 0.00230 (0.03816) + + + --> STEP: 77/234 -- GLOBAL_STEP: 46175 + | > loss: -0.17880 (-0.20045) + | > log_mle: -0.30686 (-0.30975) + | > loss_dur: 0.12806 (0.10930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.76679 (12.74414) + | > current_lr: 0.00005 + | > step_time: 2.10080 (2.44433) + | > loader_time: 0.00200 (0.03789) + + + --> STEP: 82/234 -- GLOBAL_STEP: 46180 + | > loss: -0.18457 (-0.19916) + | > log_mle: -0.30046 (-0.30933) + | > loss_dur: 0.11589 (0.11017) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.42730 (12.74656) + | > current_lr: 0.00005 + | > step_time: 1.76070 (2.44793) + | > loader_time: 0.00210 (0.03791) + + + --> STEP: 87/234 -- GLOBAL_STEP: 46185 + | > loss: -0.17664 (-0.19787) + | > log_mle: -0.30699 (-0.30933) + | > loss_dur: 0.13035 (0.11146) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.74964 (12.79692) + | > current_lr: 0.00005 + | > step_time: 3.10460 (2.44563) + | > loader_time: 0.08290 (0.03867) + + + --> STEP: 92/234 -- GLOBAL_STEP: 46190 + | > loss: -0.21712 (-0.19781) + | > log_mle: -0.35452 (-0.31098) + | > loss_dur: 0.13740 (0.11317) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.42503 (13.03598) + | > current_lr: 0.00005 + | > step_time: 1.50140 (2.41702) + | > loader_time: 0.00460 (0.03869) + + + --> STEP: 97/234 -- GLOBAL_STEP: 46195 + | > loss: -0.19034 (-0.19807) + | > log_mle: -0.33684 (-0.31342) + | > loss_dur: 0.14651 (0.11535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.04996 (13.59074) + | > current_lr: 0.00005 + | > step_time: 2.28170 (2.42013) + | > loader_time: 0.00320 (0.03759) + + + --> STEP: 102/234 -- GLOBAL_STEP: 46200 + | > loss: -0.15652 (-0.19771) + | > log_mle: -0.32137 (-0.31496) + | > loss_dur: 0.16485 (0.11726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.12691 (13.96939) + | > current_lr: 0.00005 + | > step_time: 2.72600 (2.41923) + | > loader_time: 0.08380 (0.03668) + + + --> STEP: 107/234 -- GLOBAL_STEP: 46205 + | > loss: -0.19590 (-0.19783) + | > log_mle: -0.36446 (-0.31756) + | > loss_dur: 0.16856 (0.11973) + | > amp_scaler: 4096.00000 (2124.56075) + | > grad_norm: 22.32236 (14.64168) + | > current_lr: 0.00005 + | > step_time: 1.69720 (2.38217) + | > loader_time: 0.00240 (0.03510) + + + --> STEP: 112/234 -- GLOBAL_STEP: 46210 + | > loss: -0.20369 (-0.19776) + | > log_mle: -0.37824 (-0.32001) + | > loss_dur: 0.17455 (0.12225) + | > amp_scaler: 4096.00000 (2212.57143) + | > grad_norm: 31.26230 (15.21424) + | > current_lr: 0.00005 + | > step_time: 1.77010 (2.37381) + | > loader_time: 0.09060 (0.03531) + + + --> STEP: 117/234 -- GLOBAL_STEP: 46215 + | > loss: -0.21769 (-0.19787) + | > log_mle: -0.37225 (-0.32227) + | > loss_dur: 0.15456 (0.12440) + | > amp_scaler: 4096.00000 (2293.05983) + | > grad_norm: 29.02746 (15.83705) + | > current_lr: 0.00005 + | > step_time: 5.22110 (2.42156) + | > loader_time: 0.10300 (0.03548) + + + --> STEP: 122/234 -- GLOBAL_STEP: 46220 + | > loss: -0.19343 (-0.19779) + | > log_mle: -0.34454 (-0.32363) + | > loss_dur: 0.15111 (0.12584) + | > amp_scaler: 4096.00000 (2366.95082) + | > grad_norm: 19.99341 (16.15315) + | > current_lr: 0.00005 + | > step_time: 2.49570 (2.42693) + | > loader_time: 0.09130 (0.03490) + + + --> STEP: 127/234 -- GLOBAL_STEP: 46225 + | > loss: -0.21052 (-0.19843) + | > log_mle: -0.40573 (-0.32631) + | > loss_dur: 0.19520 (0.12788) + | > amp_scaler: 4096.00000 (2435.02362) + | > grad_norm: 38.42023 (16.75706) + | > current_lr: 0.00005 + | > step_time: 3.29240 (2.41704) + | > loader_time: 0.00270 (0.03435) + + + --> STEP: 132/234 -- GLOBAL_STEP: 46230 + | > loss: -0.22258 (-0.19965) + | > log_mle: -0.38564 (-0.32945) + | > loss_dur: 0.16306 (0.12980) + | > amp_scaler: 4096.00000 (2497.93939) + | > grad_norm: 37.39428 (17.56044) + | > current_lr: 0.00005 + | > step_time: 1.91120 (2.41872) + | > loader_time: 0.08730 (0.03732) + + + --> STEP: 137/234 -- GLOBAL_STEP: 46235 + | > loss: -0.20865 (-0.20094) + | > log_mle: -0.40327 (-0.33292) + | > loss_dur: 0.19462 (0.13198) + | > amp_scaler: 4096.00000 (2556.26277) + | > grad_norm: 33.29564 (18.37147) + | > current_lr: 0.00005 + | > step_time: 2.10530 (2.40128) + | > loader_time: 0.00320 (0.03608) + + + --> STEP: 142/234 -- GLOBAL_STEP: 46240 + | > loss: -0.21624 (-0.20184) + | > log_mle: -0.41189 (-0.33575) + | > loss_dur: 0.19564 (0.13391) + | > amp_scaler: 4096.00000 (2610.47887) + | > grad_norm: 35.87743 (19.20980) + | > current_lr: 0.00005 + | > step_time: 2.59890 (2.38516) + | > loader_time: 0.09010 (0.03653) + + + --> STEP: 147/234 -- GLOBAL_STEP: 46245 + | > loss: -0.22701 (-0.20418) + | > log_mle: -0.41623 (-0.34037) + | > loss_dur: 0.18922 (0.13620) + | > amp_scaler: 4096.00000 (2661.00680) + | > grad_norm: 37.48883 (20.20877) + | > current_lr: 0.00005 + | > step_time: 4.28710 (2.42730) + | > loader_time: 0.00230 (0.03712) + + + --> STEP: 152/234 -- GLOBAL_STEP: 46250 + | > loss: -0.25589 (-0.20620) + | > log_mle: -0.48234 (-0.34439) + | > loss_dur: 0.22646 (0.13819) + | > amp_scaler: 4096.00000 (2708.21053) + | > grad_norm: 59.71375 (21.54789) + | > current_lr: 0.00005 + | > step_time: 2.05800 (2.48702) + | > loader_time: 0.00470 (0.03837) + + + --> STEP: 157/234 -- GLOBAL_STEP: 46255 + | > loss: -0.24735 (-0.20881) + | > log_mle: -0.44256 (-0.34905) + | > loss_dur: 0.19521 (0.14024) + | > amp_scaler: 4096.00000 (2752.40764) + | > grad_norm: 47.33182 (22.55079) + | > current_lr: 0.00005 + | > step_time: 1.50370 (2.46609) + | > loader_time: 0.00280 (0.03726) + + + --> STEP: 162/234 -- GLOBAL_STEP: 46260 + | > loss: -0.28377 (-0.21133) + | > log_mle: -0.47744 (-0.35358) + | > loss_dur: 0.19367 (0.14225) + | > amp_scaler: 4096.00000 (2793.87654) + | > grad_norm: 52.93080 (23.58344) + | > current_lr: 0.00005 + | > step_time: 1.38850 (2.53205) + | > loader_time: 0.00300 (0.03915) + + + --> STEP: 167/234 -- GLOBAL_STEP: 46265 + | > loss: -0.36999 (-0.21383) + | > log_mle: -0.56868 (-0.35782) + | > loss_dur: 0.19869 (0.14398) + | > amp_scaler: 4096.00000 (2832.86228) + | > grad_norm: 78.60908 (24.61624) + | > current_lr: 0.00005 + | > step_time: 6.41170 (2.58447) + | > loader_time: 0.09950 (0.03927) + + + --> STEP: 172/234 -- GLOBAL_STEP: 46270 + | > loss: -0.34383 (-0.21692) + | > log_mle: -0.56253 (-0.36308) + | > loss_dur: 0.21870 (0.14617) + | > amp_scaler: 4096.00000 (2869.58140) + | > grad_norm: 88.88320 (25.88408) + | > current_lr: 0.00005 + | > step_time: 4.20520 (2.67751) + | > loader_time: 0.00330 (0.04097) + + + --> STEP: 177/234 -- GLOBAL_STEP: 46275 + | > loss: -0.30694 (-0.21965) + | > log_mle: -0.51651 (-0.36785) + | > loss_dur: 0.20957 (0.14820) + | > amp_scaler: 4096.00000 (2904.22599) + | > grad_norm: 58.04802 (27.37426) + | > current_lr: 0.00005 + | > step_time: 3.89510 (2.68328) + | > loader_time: 0.00320 (0.04142) + + + --> STEP: 182/234 -- GLOBAL_STEP: 46280 + | > loss: -0.31258 (-0.22213) + | > log_mle: -0.56383 (-0.37260) + | > loss_dur: 0.25125 (0.15047) + | > amp_scaler: 4096.00000 (2936.96703) + | > grad_norm: 67.25043 (28.54706) + | > current_lr: 0.00005 + | > step_time: 2.59470 (2.74527) + | > loader_time: 0.09180 (0.04552) + + + --> STEP: 187/234 -- GLOBAL_STEP: 46285 + | > loss: -0.33347 (-0.22494) + | > log_mle: -0.56391 (-0.37743) + | > loss_dur: 0.23044 (0.15249) + | > amp_scaler: 2048.00000 (2935.10160) + | > grad_norm: 71.54890 (29.35859) + | > current_lr: 0.00005 + | > step_time: 3.80250 (2.83561) + | > loader_time: 0.19320 (0.04801) + + + --> STEP: 192/234 -- GLOBAL_STEP: 46290 + | > loss: -0.38341 (-0.22810) + | > log_mle: -0.59458 (-0.38222) + | > loss_dur: 0.21117 (0.15413) + | > amp_scaler: 2048.00000 (2912.00000) + | > grad_norm: 76.58231 (30.43867) + | > current_lr: 0.00005 + | > step_time: 5.30150 (2.91489) + | > loader_time: 0.29640 (0.05240) + + + --> STEP: 197/234 -- GLOBAL_STEP: 46295 + | > loss: -0.34041 (-0.23089) + | > log_mle: -0.55210 (-0.38672) + | > loss_dur: 0.21170 (0.15583) + | > amp_scaler: 2048.00000 (2890.07107) + | > grad_norm: 76.20108 (31.46131) + | > current_lr: 0.00005 + | > step_time: 5.20720 (2.98502) + | > loader_time: 0.00320 (0.05263) + + + --> STEP: 202/234 -- GLOBAL_STEP: 46300 + | > loss: -0.42073 (-0.23352) + | > log_mle: -0.64514 (-0.39118) + | > loss_dur: 0.22442 (0.15766) + | > amp_scaler: 2048.00000 (2869.22772) + | > grad_norm: 107.78796 (32.79098) + | > current_lr: 0.00005 + | > step_time: 5.59240 (3.00461) + | > loader_time: 0.00670 (0.05194) + + + --> STEP: 207/234 -- GLOBAL_STEP: 46305 + | > loss: -0.39993 (-0.23633) + | > log_mle: -0.63429 (-0.39581) + | > loss_dur: 0.23436 (0.15948) + | > amp_scaler: 2048.00000 (2849.39130) + | > grad_norm: 112.06022 (33.89689) + | > current_lr: 0.00005 + | > step_time: 4.99400 (2.99873) + | > loader_time: 0.00390 (0.05178) + + + --> STEP: 212/234 -- GLOBAL_STEP: 46310 + | > loss: -0.36891 (-0.23960) + | > log_mle: -0.60842 (-0.40097) + | > loss_dur: 0.23951 (0.16137) + | > amp_scaler: 2048.00000 (2830.49057) + | > grad_norm: 95.85521 (35.08805) + | > current_lr: 0.00005 + | > step_time: 8.49500 (3.04591) + | > loader_time: 0.00530 (0.05247) + + + --> STEP: 217/234 -- GLOBAL_STEP: 46315 + | > loss: -0.39740 (-0.24302) + | > log_mle: -0.64374 (-0.40616) + | > loss_dur: 0.24634 (0.16314) + | > amp_scaler: 2048.00000 (2812.46083) + | > grad_norm: 99.73935 (36.31974) + | > current_lr: 0.00005 + | > step_time: 5.59790 (3.10781) + | > loader_time: 0.70000 (0.05643) + + + --> STEP: 222/234 -- GLOBAL_STEP: 46320 + | > loss: -0.39040 (-0.24655) + | > log_mle: -0.65843 (-0.41141) + | > loss_dur: 0.26803 (0.16486) + | > amp_scaler: 2048.00000 (2795.24324) + | > grad_norm: 105.89842 (37.62152) + | > current_lr: 0.00005 + | > step_time: 0.26860 (3.11213) + | > loader_time: 0.00320 (0.05642) + + + --> STEP: 227/234 -- GLOBAL_STEP: 46325 + | > loss: -0.37327 (-0.25041) + | > log_mle: -0.63527 (-0.41704) + | > loss_dur: 0.26200 (0.16663) + | > amp_scaler: 2048.00000 (2778.78414) + | > grad_norm: 92.52077 (39.12730) + | > current_lr: 0.00005 + | > step_time: 0.25270 (3.04916) + | > loader_time: 0.00450 (0.05530) + + + --> STEP: 232/234 -- GLOBAL_STEP: 46330 + | > loss: -0.34878 (-0.25330) + | > log_mle: -0.84640 (-0.42396) + | > loss_dur: 0.49761 (0.17066) + | > amp_scaler: 2048.00000 (2763.03448) + | > grad_norm: 111.62146 (40.83485) + | > current_lr: 0.00005 + | > step_time: 0.33600 (2.98936) + | > loader_time: 0.01780 (0.05425) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.54896 (-0.27312) + | > avg_loss: -0.24780 (+0.03803) + | > avg_log_mle: -0.47220 (+0.02751) + | > avg_loss_dur: 0.22440 (+0.01052) + + + > EPOCH: 198/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 02:27:01)  + + --> STEP: 3/234 -- GLOBAL_STEP: 46335 + | > loss: -0.15688 (-0.20111) + | > log_mle: -0.31443 (-0.32217) + | > loss_dur: 0.15754 (0.12106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.53947 (17.12622) + | > current_lr: 0.00005 + | > step_time: 1.29690 (1.32879) + | > loader_time: 0.00190 (0.03066) + + + --> STEP: 8/234 -- GLOBAL_STEP: 46340 + | > loss: -0.23888 (-0.21830) + | > log_mle: -0.33124 (-0.32166) + | > loss_dur: 0.09236 (0.10336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.49222 (20.39604) + | > current_lr: 0.00005 + | > step_time: 16.30110 (4.72420) + | > loader_time: 0.00540 (0.79641) + + + --> STEP: 13/234 -- GLOBAL_STEP: 46345 + | > loss: -0.24820 (-0.21983) + | > log_mle: -0.33122 (-0.32221) + | > loss_dur: 0.08303 (0.10238) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.87594 (18.39888) + | > current_lr: 0.00005 + | > step_time: 1.25700 (3.74781) + | > loader_time: 0.00310 (0.49115) + + + --> STEP: 18/234 -- GLOBAL_STEP: 46350 + | > loss: -0.19952 (-0.22057) + | > log_mle: -0.31265 (-0.32083) + | > loss_dur: 0.11313 (0.10026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.18687 (16.73406) + | > current_lr: 0.00005 + | > step_time: 1.06580 (2.99446) + | > loader_time: 0.00110 (0.35512) + + + --> STEP: 23/234 -- GLOBAL_STEP: 46355 + | > loss: -0.23865 (-0.22257) + | > log_mle: -0.32428 (-0.31976) + | > loss_dur: 0.08562 (0.09718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.11287 (15.19017) + | > current_lr: 0.00005 + | > step_time: 0.98200 (2.80464) + | > loader_time: 0.00100 (0.28661) + + + --> STEP: 28/234 -- GLOBAL_STEP: 46360 + | > loss: -0.26204 (-0.22372) + | > log_mle: -0.32746 (-0.31870) + | > loss_dur: 0.06541 (0.09498) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.58737 (14.21636) + | > current_lr: 0.00005 + | > step_time: 4.01150 (2.66397) + | > loader_time: 0.08400 (0.24192) + + + --> STEP: 33/234 -- GLOBAL_STEP: 46365 + | > loss: -0.21737 (-0.22139) + | > log_mle: -0.30743 (-0.31738) + | > loss_dur: 0.09006 (0.09599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.44997 (13.73741) + | > current_lr: 0.00005 + | > step_time: 0.78880 (2.45040) + | > loader_time: 0.00180 (0.20572) + + + --> STEP: 38/234 -- GLOBAL_STEP: 46370 + | > loss: -0.21213 (-0.21863) + | > log_mle: -0.31811 (-0.31607) + | > loss_dur: 0.10598 (0.09744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.91531 (13.40253) + | > current_lr: 0.00005 + | > step_time: 1.91550 (2.39028) + | > loader_time: 0.09230 (0.18353) + + + --> STEP: 43/234 -- GLOBAL_STEP: 46375 + | > loss: -0.19558 (-0.21556) + | > log_mle: -0.31279 (-0.31467) + | > loss_dur: 0.11722 (0.09912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.37153 (13.44500) + | > current_lr: 0.00005 + | > step_time: 2.20240 (2.30153) + | > loader_time: 0.00350 (0.16846) + + + --> STEP: 48/234 -- GLOBAL_STEP: 46380 + | > loss: -0.21186 (-0.21384) + | > log_mle: -0.30364 (-0.31397) + | > loss_dur: 0.09178 (0.10014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.19999 (13.38029) + | > current_lr: 0.00005 + | > step_time: 1.40440 (2.33744) + | > loader_time: 0.00190 (0.15314) + + + --> STEP: 53/234 -- GLOBAL_STEP: 46385 + | > loss: -0.18839 (-0.21205) + | > log_mle: -0.31019 (-0.31303) + | > loss_dur: 0.12180 (0.10098) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.15904 (13.14273) + | > current_lr: 0.00005 + | > step_time: 2.23220 (2.31537) + | > loader_time: 0.00360 (0.13894) + + + --> STEP: 58/234 -- GLOBAL_STEP: 46390 + | > loss: -0.19287 (-0.21045) + | > log_mle: -0.30418 (-0.31243) + | > loss_dur: 0.11131 (0.10199) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.79113 (12.91900) + | > current_lr: 0.00005 + | > step_time: 1.98780 (2.29552) + | > loader_time: 0.00300 (0.12880) + + + --> STEP: 63/234 -- GLOBAL_STEP: 46395 + | > loss: -0.17169 (-0.20803) + | > log_mle: -0.29624 (-0.31243) + | > loss_dur: 0.12455 (0.10441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.86830 (13.31047) + | > current_lr: 0.00005 + | > step_time: 5.33200 (2.31855) + | > loader_time: 0.10420 (0.12039) + + + --> STEP: 68/234 -- GLOBAL_STEP: 46400 + | > loss: -0.15857 (-0.20641) + | > log_mle: -0.29532 (-0.31164) + | > loss_dur: 0.13675 (0.10523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03568 (13.10900) + | > current_lr: 0.00005 + | > step_time: 2.61820 (2.29326) + | > loader_time: 0.00310 (0.11174) + + + --> STEP: 73/234 -- GLOBAL_STEP: 46405 + | > loss: -0.18090 (-0.20427) + | > log_mle: -0.31591 (-0.31118) + | > loss_dur: 0.13500 (0.10691) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.74383 (13.14667) + | > current_lr: 0.00005 + | > step_time: 3.12820 (2.29502) + | > loader_time: 0.19080 (0.10795) + + + --> STEP: 78/234 -- GLOBAL_STEP: 46410 + | > loss: -0.16357 (-0.20239) + | > log_mle: -0.29325 (-0.31071) + | > loss_dur: 0.12968 (0.10832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.13220 (13.09461) + | > current_lr: 0.00005 + | > step_time: 2.29850 (2.25871) + | > loader_time: 0.08510 (0.10332) + + + --> STEP: 83/234 -- GLOBAL_STEP: 46415 + | > loss: -0.16220 (-0.20088) + | > log_mle: -0.31493 (-0.31056) + | > loss_dur: 0.15273 (0.10968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.93055 (13.07434) + | > current_lr: 0.00005 + | > step_time: 3.20080 (2.25781) + | > loader_time: 0.00320 (0.09928) + + + --> STEP: 88/234 -- GLOBAL_STEP: 46420 + | > loss: -0.18632 (-0.19993) + | > log_mle: -0.34915 (-0.31084) + | > loss_dur: 0.16283 (0.11091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.07997 (13.18665) + | > current_lr: 0.00005 + | > step_time: 1.21330 (2.23447) + | > loader_time: 0.00270 (0.09382) + + + --> STEP: 93/234 -- GLOBAL_STEP: 46425 + | > loss: -0.19734 (-0.19964) + | > log_mle: -0.36111 (-0.31245) + | > loss_dur: 0.16376 (0.11280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.45255 (13.61449) + | > current_lr: 0.00005 + | > step_time: 3.30520 (2.24785) + | > loader_time: 0.08630 (0.09077) + + + --> STEP: 98/234 -- GLOBAL_STEP: 46430 + | > loss: -0.16790 (-0.19953) + | > log_mle: -0.29556 (-0.31413) + | > loss_dur: 0.12766 (0.11460) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.52219 (13.91036) + | > current_lr: 0.00005 + | > step_time: 1.13810 (2.28180) + | > loader_time: 0.08440 (0.09083) + + + --> STEP: 103/234 -- GLOBAL_STEP: 46435 + | > loss: -0.22556 (-0.19970) + | > log_mle: -0.39775 (-0.31665) + | > loss_dur: 0.17219 (0.11695) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.72878 (14.41267) + | > current_lr: 0.00005 + | > step_time: 1.20010 (2.23511) + | > loader_time: 0.07580 (0.08727) + + + --> STEP: 108/234 -- GLOBAL_STEP: 46440 + | > loss: -0.20309 (-0.19989) + | > log_mle: -0.34239 (-0.31879) + | > loss_dur: 0.13930 (0.11890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.81259 (14.80168) + | > current_lr: 0.00005 + | > step_time: 1.40570 (2.21877) + | > loader_time: 0.08360 (0.08585) + + + --> STEP: 113/234 -- GLOBAL_STEP: 46445 + | > loss: -0.21755 (-0.19987) + | > log_mle: -0.38693 (-0.32157) + | > loss_dur: 0.16938 (0.12170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.34171 (15.68376) + | > current_lr: 0.00005 + | > step_time: 2.91340 (2.22536) + | > loader_time: 0.08280 (0.08533) + + + --> STEP: 118/234 -- GLOBAL_STEP: 46450 + | > loss: -0.18736 (-0.19962) + | > log_mle: -0.35408 (-0.32345) + | > loss_dur: 0.16672 (0.12383) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.65743 (16.35240) + | > current_lr: 0.00005 + | > step_time: 7.10420 (2.30577) + | > loader_time: 0.18360 (0.08622) + + + --> STEP: 123/234 -- GLOBAL_STEP: 46455 + | > loss: -0.16740 (-0.19920) + | > log_mle: -0.32421 (-0.32446) + | > loss_dur: 0.15681 (0.12526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.17871 (16.65295) + | > current_lr: 0.00005 + | > step_time: 1.89150 (2.28437) + | > loader_time: 0.00870 (0.08451) + + + --> STEP: 128/234 -- GLOBAL_STEP: 46460 + | > loss: -0.22761 (-0.20029) + | > log_mle: -0.38246 (-0.32747) + | > loss_dur: 0.15485 (0.12718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.68375 (17.36940) + | > current_lr: 0.00005 + | > step_time: 1.70120 (2.28042) + | > loader_time: 0.00290 (0.08419) + + + --> STEP: 133/234 -- GLOBAL_STEP: 46465 + | > loss: -0.23984 (-0.20170) + | > log_mle: -0.41507 (-0.33080) + | > loss_dur: 0.17523 (0.12911) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.83705 (18.10571) + | > current_lr: 0.00005 + | > step_time: 4.28860 (2.28569) + | > loader_time: 0.00500 (0.08245) + + + --> STEP: 138/234 -- GLOBAL_STEP: 46470 + | > loss: -0.18622 (-0.20253) + | > log_mle: -0.36073 (-0.33383) + | > loss_dur: 0.17451 (0.13130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.47244 (18.80588) + | > current_lr: 0.00005 + | > step_time: 4.30560 (2.30131) + | > loader_time: 0.10460 (0.08091) + + + --> STEP: 143/234 -- GLOBAL_STEP: 46475 + | > loss: -0.27002 (-0.20398) + | > log_mle: -0.50581 (-0.33761) + | > loss_dur: 0.23578 (0.13363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.66035 (19.78139) + | > current_lr: 0.00005 + | > step_time: 2.21060 (2.33985) + | > loader_time: 0.00320 (0.08016) + + + --> STEP: 148/234 -- GLOBAL_STEP: 46480 + | > loss: -0.26228 (-0.20610) + | > log_mle: -0.42320 (-0.34155) + | > loss_dur: 0.16093 (0.13544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.16602 (20.56089) + | > current_lr: 0.00005 + | > step_time: 1.64610 (2.39113) + | > loader_time: 0.00290 (0.07816) + + + --> STEP: 153/234 -- GLOBAL_STEP: 46485 + | > loss: -0.35622 (-0.20912) + | > log_mle: -0.55857 (-0.34672) + | > loss_dur: 0.20235 (0.13761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.48303 (21.57038) + | > current_lr: 0.00005 + | > step_time: 3.48670 (2.42488) + | > loader_time: 0.00400 (0.07748) + + + --> STEP: 158/234 -- GLOBAL_STEP: 46490 + | > loss: -0.25612 (-0.21119) + | > log_mle: -0.47649 (-0.35090) + | > loss_dur: 0.22037 (0.13971) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.40740 (22.91338) + | > current_lr: 0.00005 + | > step_time: 0.91830 (2.43138) + | > loader_time: 0.08540 (0.07684) + + + --> STEP: 163/234 -- GLOBAL_STEP: 46495 + | > loss: -0.25798 (-0.21368) + | > log_mle: -0.45570 (-0.35529) + | > loss_dur: 0.19773 (0.14161) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.18370 (23.81288) + | > current_lr: 0.00005 + | > step_time: 3.10700 (2.42910) + | > loader_time: 0.08710 (0.07511) + + + --> STEP: 168/234 -- GLOBAL_STEP: 46500 + | > loss: -0.27000 (-0.21627) + | > log_mle: -0.49847 (-0.35978) + | > loss_dur: 0.22847 (0.14351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.39090 (24.94056) + | > current_lr: 0.00005 + | > step_time: 5.58890 (2.48185) + | > loader_time: 0.00450 (0.07744) + + + --> STEP: 173/234 -- GLOBAL_STEP: 46505 + | > loss: -0.29740 (-0.21902) + | > log_mle: -0.51035 (-0.36462) + | > loss_dur: 0.21295 (0.14561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.32196 (26.17261) + | > current_lr: 0.00005 + | > step_time: 2.40640 (2.54186) + | > loader_time: 0.07870 (0.07628) + + + --> STEP: 178/234 -- GLOBAL_STEP: 46510 + | > loss: -0.33824 (-0.22193) + | > log_mle: -0.57738 (-0.36969) + | > loss_dur: 0.23914 (0.14777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.38579 (27.14495) + | > current_lr: 0.00005 + | > step_time: 3.29640 (2.54573) + | > loader_time: 0.00650 (0.07428) + + + --> STEP: 183/234 -- GLOBAL_STEP: 46515 + | > loss: -0.35674 (-0.22453) + | > log_mle: -0.57248 (-0.37439) + | > loss_dur: 0.21574 (0.14986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.71375 (28.32606) + | > current_lr: 0.00005 + | > step_time: 4.11290 (2.58566) + | > loader_time: 0.49630 (0.07770) + + + --> STEP: 188/234 -- GLOBAL_STEP: 46520 + | > loss: -0.36582 (-0.22728) + | > log_mle: -0.58808 (-0.37921) + | > loss_dur: 0.22226 (0.15194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.15847 (29.48178) + | > current_lr: 0.00005 + | > step_time: 1.59020 (2.62485) + | > loader_time: 0.09250 (0.07816) + + + --> STEP: 193/234 -- GLOBAL_STEP: 46525 + | > loss: -0.37051 (-0.23028) + | > log_mle: -0.59062 (-0.38389) + | > loss_dur: 0.22011 (0.15361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.29494 (30.79525) + | > current_lr: 0.00005 + | > step_time: 3.69670 (2.62770) + | > loader_time: 0.09880 (0.07772) + + + --> STEP: 198/234 -- GLOBAL_STEP: 46530 + | > loss: -0.36007 (-0.23315) + | > log_mle: -0.58339 (-0.38842) + | > loss_dur: 0.22333 (0.15528) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.00350 (31.99325) + | > current_lr: 0.00005 + | > step_time: 2.29240 (2.62792) + | > loader_time: 0.00330 (0.07734) + + + --> STEP: 203/234 -- GLOBAL_STEP: 46535 + | > loss: -0.29561 (-0.23547) + | > log_mle: -0.51137 (-0.39258) + | > loss_dur: 0.21577 (0.15711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.03657 (33.53789) + | > current_lr: 0.00005 + | > step_time: 2.80330 (2.68780) + | > loader_time: 0.10050 (0.07689) + + + --> STEP: 208/234 -- GLOBAL_STEP: 46540 + | > loss: -0.34752 (-0.23828) + | > log_mle: -0.57738 (-0.39728) + | > loss_dur: 0.22986 (0.15900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.11313 (34.86753) + | > current_lr: 0.00005 + | > step_time: 4.71400 (2.72332) + | > loader_time: 0.18920 (0.07605) + + + --> STEP: 213/234 -- GLOBAL_STEP: 46545 + | > loss: -0.39125 (-0.24152) + | > log_mle: -0.64016 (-0.40243) + | > loss_dur: 0.24890 (0.16091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.59840 (35.92709) + | > current_lr: 0.00005 + | > step_time: 6.80520 (2.86884) + | > loader_time: 0.49980 (0.07853) + + + --> STEP: 218/234 -- GLOBAL_STEP: 46550 + | > loss: -0.36646 (-0.24457) + | > log_mle: -0.59958 (-0.40720) + | > loss_dur: 0.23311 (0.16264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.09629 (37.11569) + | > current_lr: 0.00005 + | > step_time: 2.40850 (2.92249) + | > loader_time: 0.00420 (0.08026) + + + --> STEP: 223/234 -- GLOBAL_STEP: 46555 + | > loss: -0.40073 (-0.24801) + | > log_mle: -0.63733 (-0.41238) + | > loss_dur: 0.23661 (0.16437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.89049 (38.26697) + | > current_lr: 0.00005 + | > step_time: 0.23580 (2.88115) + | > loader_time: 0.00320 (0.07931) + + + --> STEP: 228/234 -- GLOBAL_STEP: 46560 + | > loss: -0.35402 (-0.25114) + | > log_mle: -0.62433 (-0.41752) + | > loss_dur: 0.27031 (0.16638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.49788 (39.52913) + | > current_lr: 0.00005 + | > step_time: 0.23770 (2.82315) + | > loader_time: 0.00420 (0.07765) + + + --> STEP: 233/234 -- GLOBAL_STEP: 46565 + | > loss: 0.15391 (-0.25142) + | > log_mle: -0.60762 (-0.42392) + | > loss_dur: 0.76153 (0.17250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.00735 (40.94645) + | > current_lr: 0.00005 + | > step_time: 0.19650 (2.76828) + | > loader_time: 0.00310 (0.07609) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.34732 (-0.20164) + | > avg_loss: -0.28066 (-0.03286) + | > avg_log_mle: -0.50174 (-0.02954) + | > avg_loss_dur: 0.22108 (-0.00332) + + + > EPOCH: 199/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 02:39:19)  + + --> STEP: 4/234 -- GLOBAL_STEP: 46570 + | > loss: -0.20032 (-0.19794) + | > log_mle: -0.31851 (-0.32205) + | > loss_dur: 0.11818 (0.12411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.74214 (19.14098) + | > current_lr: 0.00005 + | > step_time: 1.60700 (7.25635) + | > loader_time: 0.00520 (0.04912) + + + --> STEP: 9/234 -- GLOBAL_STEP: 46575 + | > loss: -0.20340 (-0.21490) + | > log_mle: -0.32234 (-0.32325) + | > loss_dur: 0.11893 (0.10836) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.03873 (18.61649) + | > current_lr: 0.00005 + | > step_time: 8.48920 (5.89129) + | > loader_time: 0.10590 (0.05488) + + + --> STEP: 14/234 -- GLOBAL_STEP: 46580 + | > loss: -0.21302 (-0.21752) + | > log_mle: -0.32396 (-0.32267) + | > loss_dur: 0.11094 (0.10514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.79973 (17.69158) + | > current_lr: 0.00005 + | > step_time: 3.09890 (4.86630) + | > loader_time: 0.00140 (0.04915) + + + --> STEP: 19/234 -- GLOBAL_STEP: 46585 + | > loss: -0.24324 (-0.22067) + | > log_mle: -0.31891 (-0.32126) + | > loss_dur: 0.07567 (0.10059) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.93195 (16.00970) + | > current_lr: 0.00005 + | > step_time: 1.90160 (3.94179) + | > loader_time: 0.08330 (0.04096) + + + --> STEP: 24/234 -- GLOBAL_STEP: 46590 + | > loss: -0.24524 (-0.22280) + | > log_mle: -0.31513 (-0.32024) + | > loss_dur: 0.06989 (0.09744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.74602 (14.71230) + | > current_lr: 0.00005 + | > step_time: 2.01560 (3.82141) + | > loader_time: 0.10160 (0.04790) + + + --> STEP: 29/234 -- GLOBAL_STEP: 46595 + | > loss: -0.18840 (-0.22210) + | > log_mle: -0.29735 (-0.31883) + | > loss_dur: 0.10895 (0.09673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10988 (14.02793) + | > current_lr: 0.00005 + | > step_time: 2.19570 (3.82191) + | > loader_time: 0.00240 (0.04645) + + + --> STEP: 34/234 -- GLOBAL_STEP: 46600 + | > loss: -0.19851 (-0.22082) + | > log_mle: -0.30562 (-0.31758) + | > loss_dur: 0.10711 (0.09676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.45960 (13.62362) + | > current_lr: 0.00005 + | > step_time: 9.89310 (4.02427) + | > loader_time: 0.21020 (0.05162) + + + --> STEP: 39/234 -- GLOBAL_STEP: 46605 + | > loss: -0.19856 (-0.21800) + | > log_mle: -0.31112 (-0.31636) + | > loss_dur: 0.11255 (0.09837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.86389 (13.69628) + | > current_lr: 0.00005 + | > step_time: 3.09770 (4.01560) + | > loader_time: 0.00570 (0.05012) + + + --> STEP: 44/234 -- GLOBAL_STEP: 46610 + | > loss: -0.22468 (-0.21549) + | > log_mle: -0.30810 (-0.31505) + | > loss_dur: 0.08342 (0.09956) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.85416 (13.22802) + | > current_lr: 0.00005 + | > step_time: 1.43010 (3.73623) + | > loader_time: 0.00250 (0.04480) + + + --> STEP: 49/234 -- GLOBAL_STEP: 46615 + | > loss: -0.22307 (-0.21419) + | > log_mle: -0.31689 (-0.31474) + | > loss_dur: 0.09382 (0.10055) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.44248 (12.91578) + | > current_lr: 0.00005 + | > step_time: 1.40130 (3.61478) + | > loader_time: 0.00180 (0.04630) + + + --> STEP: 54/234 -- GLOBAL_STEP: 46620 + | > loss: -0.21093 (-0.21244) + | > log_mle: -0.31633 (-0.31403) + | > loss_dur: 0.10540 (0.10159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.91236 (12.56793) + | > current_lr: 0.00005 + | > step_time: 1.57710 (3.46518) + | > loader_time: 0.08590 (0.04525) + + + --> STEP: 59/234 -- GLOBAL_STEP: 46625 + | > loss: -0.20790 (-0.21145) + | > log_mle: -0.31943 (-0.31368) + | > loss_dur: 0.11153 (0.10222) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.65239 (12.45381) + | > current_lr: 0.00005 + | > step_time: 2.40970 (3.32595) + | > loader_time: 0.07730 (0.04289) + + + --> STEP: 64/234 -- GLOBAL_STEP: 46630 + | > loss: -0.20522 (-0.20960) + | > log_mle: -0.30592 (-0.31384) + | > loss_dur: 0.10070 (0.10424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.33911 (12.67979) + | > current_lr: 0.00005 + | > step_time: 1.06430 (3.24242) + | > loader_time: 0.00190 (0.04504) + + + --> STEP: 69/234 -- GLOBAL_STEP: 46635 + | > loss: -0.17611 (-0.20769) + | > log_mle: -0.29147 (-0.31305) + | > loss_dur: 0.11536 (0.10536) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.87428 (12.60307) + | > current_lr: 0.00005 + | > step_time: 1.19230 (3.09498) + | > loader_time: 0.10570 (0.04343) + + + --> STEP: 74/234 -- GLOBAL_STEP: 46640 + | > loss: -0.17266 (-0.20537) + | > log_mle: -0.29826 (-0.31276) + | > loss_dur: 0.12560 (0.10739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.90445 (12.69390) + | > current_lr: 0.00005 + | > step_time: 1.82570 (3.01852) + | > loader_time: 0.00330 (0.04170) + + + --> STEP: 79/234 -- GLOBAL_STEP: 46645 + | > loss: -0.18565 (-0.20393) + | > log_mle: -0.31299 (-0.31263) + | > loss_dur: 0.12734 (0.10869) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.20985 (12.70631) + | > current_lr: 0.00005 + | > step_time: 1.40670 (2.97825) + | > loader_time: 0.00240 (0.04155) + + + --> STEP: 84/234 -- GLOBAL_STEP: 46650 + | > loss: -0.18936 (-0.20279) + | > log_mle: -0.30812 (-0.31249) + | > loss_dur: 0.11876 (0.10970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.24349 (12.69517) + | > current_lr: 0.00005 + | > step_time: 1.49250 (2.92960) + | > loader_time: 0.00220 (0.04033) + + + --> STEP: 89/234 -- GLOBAL_STEP: 46655 + | > loss: -0.20423 (-0.20207) + | > log_mle: -0.33597 (-0.31317) + | > loss_dur: 0.13174 (0.11111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.90288 (12.82463) + | > current_lr: 0.00005 + | > step_time: 1.59120 (2.88327) + | > loader_time: 0.00220 (0.03928) + + + --> STEP: 94/234 -- GLOBAL_STEP: 46660 + | > loss: -0.21924 (-0.20188) + | > log_mle: -0.36536 (-0.31512) + | > loss_dur: 0.14611 (0.11324) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.99706 (13.26481) + | > current_lr: 0.00005 + | > step_time: 3.00880 (2.87377) + | > loader_time: 0.09530 (0.03946) + + + --> STEP: 99/234 -- GLOBAL_STEP: 46665 + | > loss: -0.22000 (-0.20172) + | > log_mle: -0.39744 (-0.31702) + | > loss_dur: 0.17744 (0.11530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.93021 (13.79149) + | > current_lr: 0.00005 + | > step_time: 1.39250 (2.82785) + | > loader_time: 0.00330 (0.03851) + + + --> STEP: 104/234 -- GLOBAL_STEP: 46670 + | > loss: -0.25454 (-0.20199) + | > log_mle: -0.41023 (-0.31949) + | > loss_dur: 0.15569 (0.11750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.94341 (14.39786) + | > current_lr: 0.00005 + | > step_time: 2.29440 (2.76685) + | > loader_time: 0.00210 (0.03682) + + + --> STEP: 109/234 -- GLOBAL_STEP: 46675 + | > loss: -0.18229 (-0.20150) + | > log_mle: -0.38389 (-0.32118) + | > loss_dur: 0.20159 (0.11968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.41936 (14.75639) + | > current_lr: 0.00005 + | > step_time: 1.20180 (2.75917) + | > loader_time: 0.00340 (0.03601) + + + --> STEP: 114/234 -- GLOBAL_STEP: 46680 + | > loss: -0.21392 (-0.20171) + | > log_mle: -0.36761 (-0.32374) + | > loss_dur: 0.15369 (0.12204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.22503 (15.38220) + | > current_lr: 0.00005 + | > step_time: 1.50300 (2.71337) + | > loader_time: 0.00300 (0.03530) + + + --> STEP: 119/234 -- GLOBAL_STEP: 46685 + | > loss: -0.19916 (-0.20136) + | > log_mle: -0.36298 (-0.32561) + | > loss_dur: 0.16382 (0.12425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.33453 (15.74456) + | > current_lr: 0.00005 + | > step_time: 1.17310 (2.66645) + | > loader_time: 0.00250 (0.03535) + + + --> STEP: 124/234 -- GLOBAL_STEP: 46690 + | > loss: -0.22356 (-0.20117) + | > log_mle: -0.39017 (-0.32684) + | > loss_dur: 0.16661 (0.12568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.30980 (16.08706) + | > current_lr: 0.00005 + | > step_time: 2.50030 (2.65407) + | > loader_time: 0.00520 (0.03484) + + + --> STEP: 129/234 -- GLOBAL_STEP: 46695 + | > loss: -0.19735 (-0.20183) + | > log_mle: -0.37967 (-0.32969) + | > loss_dur: 0.18232 (0.12786) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.89375 (16.82163) + | > current_lr: 0.00005 + | > step_time: 1.61510 (2.62963) + | > loader_time: 0.08560 (0.03492) + + + --> STEP: 134/234 -- GLOBAL_STEP: 46700 + | > loss: -0.23950 (-0.20342) + | > log_mle: -0.43623 (-0.33335) + | > loss_dur: 0.19674 (0.12993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.30187 (17.65582) + | > current_lr: 0.00005 + | > step_time: 1.69970 (2.64163) + | > loader_time: 0.00250 (0.03574) + + + --> STEP: 139/234 -- GLOBAL_STEP: 46705 + | > loss: -0.29668 (-0.20469) + | > log_mle: -0.49864 (-0.33672) + | > loss_dur: 0.20196 (0.13203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.94094 (18.66249) + | > current_lr: 0.00005 + | > step_time: 1.49270 (2.61620) + | > loader_time: 0.00340 (0.03459) + + + --> STEP: 144/234 -- GLOBAL_STEP: 46710 + | > loss: -0.26779 (-0.20594) + | > log_mle: -0.46543 (-0.34032) + | > loss_dur: 0.19763 (0.13438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.85215 (19.66254) + | > current_lr: 0.00005 + | > step_time: 2.80980 (2.60604) + | > loader_time: 0.00550 (0.03471) + + + --> STEP: 149/234 -- GLOBAL_STEP: 46715 + | > loss: -0.31958 (-0.20821) + | > log_mle: -0.52234 (-0.34445) + | > loss_dur: 0.20276 (0.13624) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.59294 (20.60928) + | > current_lr: 0.00005 + | > step_time: 2.39490 (2.58634) + | > loader_time: 0.00250 (0.03367) + + + --> STEP: 154/234 -- GLOBAL_STEP: 46720 + | > loss: -0.29137 (-0.21093) + | > log_mle: -0.47637 (-0.34909) + | > loss_dur: 0.18500 (0.13816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.93339 (21.84753) + | > current_lr: 0.00005 + | > step_time: 3.59210 (2.59123) + | > loader_time: 0.00300 (0.03452) + + + --> STEP: 159/234 -- GLOBAL_STEP: 46725 + | > loss: -0.29438 (-0.21317) + | > log_mle: -0.49972 (-0.35348) + | > loss_dur: 0.20534 (0.14031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.25812 (23.23991) + | > current_lr: 0.00005 + | > step_time: 1.90170 (2.56942) + | > loader_time: 0.00510 (0.03407) + + + --> STEP: 164/234 -- GLOBAL_STEP: 46730 + | > loss: -0.27756 (-0.21561) + | > log_mle: -0.48243 (-0.35768) + | > loss_dur: 0.20487 (0.14207) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.06911 (24.22698) + | > current_lr: 0.00005 + | > step_time: 3.26920 (2.62196) + | > loader_time: 0.22370 (0.03560) + + + --> STEP: 169/234 -- GLOBAL_STEP: 46735 + | > loss: -0.27381 (-0.21810) + | > log_mle: -0.49078 (-0.36207) + | > loss_dur: 0.21698 (0.14398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.19421 (25.29475) + | > current_lr: 0.00005 + | > step_time: 1.70260 (2.61085) + | > loader_time: 0.00330 (0.03516) + + + --> STEP: 174/234 -- GLOBAL_STEP: 46740 + | > loss: -0.36565 (-0.22140) + | > log_mle: -0.57412 (-0.36757) + | > loss_dur: 0.20847 (0.14617) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.96500 (26.45378) + | > current_lr: 0.00005 + | > step_time: 2.61510 (2.61752) + | > loader_time: 0.28410 (0.03923) + + + --> STEP: 179/234 -- GLOBAL_STEP: 46745 + | > loss: -0.32836 (-0.22409) + | > log_mle: -0.57480 (-0.37259) + | > loss_dur: 0.24644 (0.14850) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.33549 (27.69189) + | > current_lr: 0.00005 + | > step_time: 4.40380 (2.71836) + | > loader_time: 0.19650 (0.04303) + + + --> STEP: 184/234 -- GLOBAL_STEP: 46750 + | > loss: -0.30414 (-0.22654) + | > log_mle: -0.52837 (-0.37708) + | > loss_dur: 0.22423 (0.15054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.51047 (28.65979) + | > current_lr: 0.00005 + | > step_time: 1.49720 (2.72600) + | > loader_time: 0.00320 (0.04439) + + + --> STEP: 189/234 -- GLOBAL_STEP: 46755 + | > loss: -0.26286 (-0.22813) + | > log_mle: -0.48671 (-0.38070) + | > loss_dur: 0.22384 (0.15257) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.50694 (30.75798) + | > current_lr: 0.00005 + | > step_time: 2.01080 (2.73009) + | > loader_time: 0.00350 (0.04333) + + + --> STEP: 194/234 -- GLOBAL_STEP: 46760 + | > loss: -0.32716 (-0.23019) + | > log_mle: -0.54188 (-0.38442) + | > loss_dur: 0.21473 (0.15423) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.84550 (31.55342) + | > current_lr: 0.00005 + | > step_time: 2.70370 (2.73248) + | > loader_time: 0.08740 (0.04455) + + + --> STEP: 199/234 -- GLOBAL_STEP: 46765 + | > loss: -0.33722 (-0.23249) + | > log_mle: -0.56170 (-0.38840) + | > loss_dur: 0.22448 (0.15591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.13001 (32.41999) + | > current_lr: 0.00005 + | > step_time: 6.69530 (2.79608) + | > loader_time: 0.40080 (0.04789) + + + --> STEP: 204/234 -- GLOBAL_STEP: 46770 + | > loss: -0.36701 (-0.23474) + | > log_mle: -0.61074 (-0.39254) + | > loss_dur: 0.24373 (0.15780) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.98782 (33.08638) + | > current_lr: 0.00005 + | > step_time: 6.80190 (2.87706) + | > loader_time: 0.19680 (0.05205) + + + --> STEP: 209/234 -- GLOBAL_STEP: 46775 + | > loss: -0.34064 (-0.23762) + | > log_mle: -0.56062 (-0.39712) + | > loss_dur: 0.21998 (0.15950) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.94193 (34.03419) + | > current_lr: 0.00005 + | > step_time: 7.11150 (2.96419) + | > loader_time: 0.08960 (0.05228) + + + --> STEP: 214/234 -- GLOBAL_STEP: 46780 + | > loss: -0.39054 (-0.24129) + | > log_mle: -0.60171 (-0.40255) + | > loss_dur: 0.21117 (0.16126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.93353 (35.36235) + | > current_lr: 0.00005 + | > step_time: 2.59030 (2.99870) + | > loader_time: 0.08540 (0.05248) + + + --> STEP: 219/234 -- GLOBAL_STEP: 46785 + | > loss: -0.46445 (-0.24478) + | > log_mle: -0.70618 (-0.40792) + | > loss_dur: 0.24173 (0.16314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 131.71867 (36.85133) + | > current_lr: 0.00005 + | > step_time: 7.31550 (3.05264) + | > loader_time: 0.09420 (0.05308) + + + --> STEP: 224/234 -- GLOBAL_STEP: 46790 + | > loss: -0.41085 (-0.24815) + | > log_mle: -0.64980 (-0.41302) + | > loss_dur: 0.23895 (0.16488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.06127 (37.89943) + | > current_lr: 0.00005 + | > step_time: 0.25150 (3.02153) + | > loader_time: 0.00330 (0.05240) + + + --> STEP: 229/234 -- GLOBAL_STEP: 46795 + | > loss: -0.39709 (-0.25159) + | > log_mle: -0.69198 (-0.41862) + | > loss_dur: 0.29489 (0.16703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.51672 (39.34884) + | > current_lr: 0.00005 + | > step_time: 0.27140 (2.96088) + | > loader_time: 0.00560 (0.05135) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.60358 (+0.25626) + | > avg_loss: -0.28833 (-0.00767) + | > avg_log_mle: -0.50707 (-0.00533) + | > avg_loss_dur: 0.21874 (-0.00234) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_46800.pth + + > EPOCH: 200/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 02:52:00)  + + --> STEP: 0/234 -- GLOBAL_STEP: 46800 + | > loss: -0.26607 (-0.26607) + | > log_mle: -0.39908 (-0.39908) + | > loss_dur: 0.13301 (0.13301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.32605 (23.32605) + | > current_lr: 0.00005 + | > step_time: 7.60490 (7.60487) + | > loader_time: 11.46280 (11.46277) + + + --> STEP: 5/234 -- GLOBAL_STEP: 46805 + | > loss: -0.21498 (-0.20297) + | > log_mle: -0.32369 (-0.32164) + | > loss_dur: 0.10871 (0.11867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.79573 (21.09796) + | > current_lr: 0.00005 + | > step_time: 1.21100 (4.80242) + | > loader_time: 0.00170 (0.02266) + + + --> STEP: 10/234 -- GLOBAL_STEP: 46810 + | > loss: -0.21299 (-0.21120) + | > log_mle: -0.32248 (-0.32476) + | > loss_dur: 0.10949 (0.11356) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.89979 (17.79231) + | > current_lr: 0.00005 + | > step_time: 1.18580 (3.34864) + | > loader_time: 0.00150 (0.01266) + + + --> STEP: 15/234 -- GLOBAL_STEP: 46815 + | > loss: -0.23155 (-0.21930) + | > log_mle: -0.32975 (-0.32564) + | > loss_dur: 0.09820 (0.10633) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.66860 (16.84108) + | > current_lr: 0.00005 + | > step_time: 3.81930 (3.17308) + | > loader_time: 0.08190 (0.02104) + + + --> STEP: 20/234 -- GLOBAL_STEP: 46820 + | > loss: -0.24512 (-0.22262) + | > log_mle: -0.32417 (-0.32378) + | > loss_dur: 0.07905 (0.10116) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.08546 (15.29992) + | > current_lr: 0.00005 + | > step_time: 3.90020 (3.45063) + | > loader_time: 0.00430 (0.02601) + + + --> STEP: 25/234 -- GLOBAL_STEP: 46825 + | > loss: -0.21705 (-0.22473) + | > log_mle: -0.30668 (-0.32223) + | > loss_dur: 0.08963 (0.09750) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.74322 (14.28133) + | > current_lr: 0.00005 + | > step_time: 2.38270 (3.42445) + | > loader_time: 0.00460 (0.02967) + + + --> STEP: 30/234 -- GLOBAL_STEP: 46830 + | > loss: -0.22989 (-0.22419) + | > log_mle: -0.31839 (-0.32125) + | > loss_dur: 0.08849 (0.09706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.55046 (13.62077) + | > current_lr: 0.00005 + | > step_time: 1.60120 (3.15691) + | > loader_time: 0.08410 (0.02786) + + + --> STEP: 35/234 -- GLOBAL_STEP: 46835 + | > loss: -0.19797 (-0.22217) + | > log_mle: -0.31137 (-0.32008) + | > loss_dur: 0.11339 (0.09792) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.68922 (13.14128) + | > current_lr: 0.00005 + | > step_time: 1.03570 (3.25270) + | > loader_time: 0.00210 (0.04424) + + + --> STEP: 40/234 -- GLOBAL_STEP: 46840 + | > loss: -0.17517 (-0.21966) + | > log_mle: -0.29548 (-0.31871) + | > loss_dur: 0.12031 (0.09905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.81916 (12.91628) + | > current_lr: 0.00005 + | > step_time: 1.90550 (3.03668) + | > loader_time: 0.00180 (0.03895) + + + --> STEP: 45/234 -- GLOBAL_STEP: 46845 + | > loss: -0.20046 (-0.21846) + | > log_mle: -0.32903 (-0.31814) + | > loss_dur: 0.12858 (0.09968) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.22972 (12.71803) + | > current_lr: 0.00005 + | > step_time: 1.41930 (2.86027) + | > loader_time: 0.00330 (0.03840) + + + --> STEP: 50/234 -- GLOBAL_STEP: 46850 + | > loss: -0.19717 (-0.21690) + | > log_mle: -0.30260 (-0.31716) + | > loss_dur: 0.10542 (0.10026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.79046 (12.43162) + | > current_lr: 0.00005 + | > step_time: 2.08710 (2.74338) + | > loader_time: 0.00140 (0.03650) + + + --> STEP: 55/234 -- GLOBAL_STEP: 46855 + | > loss: -0.21942 (-0.21526) + | > log_mle: -0.31522 (-0.31633) + | > loss_dur: 0.09580 (0.10108) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.16290 (12.48203) + | > current_lr: 0.00005 + | > step_time: 1.28580 (2.71891) + | > loader_time: 0.00210 (0.03478) + + + --> STEP: 60/234 -- GLOBAL_STEP: 46860 + | > loss: -0.20273 (-0.21373) + | > log_mle: -0.32850 (-0.31600) + | > loss_dur: 0.12576 (0.10227) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.03226 (12.36855) + | > current_lr: 0.00005 + | > step_time: 3.40170 (2.67922) + | > loader_time: 0.00370 (0.03353) + + + --> STEP: 65/234 -- GLOBAL_STEP: 46865 + | > loss: -0.20230 (-0.21158) + | > log_mle: -0.30801 (-0.31574) + | > loss_dur: 0.10571 (0.10417) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46871 (12.44932) + | > current_lr: 0.00005 + | > step_time: 1.21130 (2.60592) + | > loader_time: 0.00260 (0.03368) + + + --> STEP: 70/234 -- GLOBAL_STEP: 46870 + | > loss: -0.15877 (-0.20939) + | > log_mle: -0.29574 (-0.31477) + | > loss_dur: 0.13696 (0.10537) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.49026 (12.33372) + | > current_lr: 0.00005 + | > step_time: 1.38800 (2.54073) + | > loader_time: 0.00220 (0.03372) + + + --> STEP: 75/234 -- GLOBAL_STEP: 46875 + | > loss: -0.16686 (-0.20700) + | > log_mle: -0.30896 (-0.31453) + | > loss_dur: 0.14209 (0.10753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.07524 (12.55097) + | > current_lr: 0.00005 + | > step_time: 1.81910 (2.52245) + | > loader_time: 0.18670 (0.03669) + + + --> STEP: 80/234 -- GLOBAL_STEP: 46880 + | > loss: -0.18807 (-0.20564) + | > log_mle: -0.29769 (-0.31399) + | > loss_dur: 0.10962 (0.10835) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.04141 (12.41350) + | > current_lr: 0.00005 + | > step_time: 4.03240 (2.49535) + | > loader_time: 0.10130 (0.03685) + + + --> STEP: 85/234 -- GLOBAL_STEP: 46885 + | > loss: -0.18706 (-0.20436) + | > log_mle: -0.30860 (-0.31378) + | > loss_dur: 0.12154 (0.10942) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.53808 (12.45091) + | > current_lr: 0.00005 + | > step_time: 1.40680 (2.46722) + | > loader_time: 0.08840 (0.03700) + + + --> STEP: 90/234 -- GLOBAL_STEP: 46890 + | > loss: -0.18684 (-0.20354) + | > log_mle: -0.33207 (-0.31460) + | > loss_dur: 0.14523 (0.11106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.37073 (12.65598) + | > current_lr: 0.00005 + | > step_time: 2.06850 (2.44071) + | > loader_time: 0.02540 (0.03646) + + + --> STEP: 95/234 -- GLOBAL_STEP: 46895 + | > loss: -0.23681 (-0.20411) + | > log_mle: -0.41491 (-0.31740) + | > loss_dur: 0.17810 (0.11329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.97469 (13.23970) + | > current_lr: 0.00005 + | > step_time: 2.00750 (2.42196) + | > loader_time: 0.00240 (0.03554) + + + --> STEP: 100/234 -- GLOBAL_STEP: 46900 + | > loss: -0.20802 (-0.20378) + | > log_mle: -0.34748 (-0.31858) + | > loss_dur: 0.13946 (0.11480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.34287 (13.50121) + | > current_lr: 0.00005 + | > step_time: 2.77150 (2.42126) + | > loader_time: 0.08740 (0.03475) + + + --> STEP: 105/234 -- GLOBAL_STEP: 46905 + | > loss: -0.19108 (-0.20393) + | > log_mle: -0.32435 (-0.32086) + | > loss_dur: 0.13326 (0.11692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.94602 (14.19437) + | > current_lr: 0.00005 + | > step_time: 3.41030 (2.42426) + | > loader_time: 0.08010 (0.03572) + + + --> STEP: 110/234 -- GLOBAL_STEP: 46910 + | > loss: -0.19937 (-0.20347) + | > log_mle: -0.34829 (-0.32271) + | > loss_dur: 0.14892 (0.11924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.49625 (14.84964) + | > current_lr: 0.00005 + | > step_time: 4.29970 (2.41786) + | > loader_time: 0.10270 (0.03737) + + + --> STEP: 115/234 -- GLOBAL_STEP: 46915 + | > loss: -0.19036 (-0.20367) + | > log_mle: -0.36995 (-0.32542) + | > loss_dur: 0.17959 (0.12175) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.80164 (15.55663) + | > current_lr: 0.00005 + | > step_time: 1.52540 (2.39565) + | > loader_time: 0.00250 (0.03647) + + + --> STEP: 120/234 -- GLOBAL_STEP: 46920 + | > loss: -0.24210 (-0.20383) + | > log_mle: -0.41681 (-0.32767) + | > loss_dur: 0.17471 (0.12385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.70245 (16.25634) + | > current_lr: 0.00005 + | > step_time: 2.51230 (2.37799) + | > loader_time: 0.08450 (0.03646) + + + --> STEP: 125/234 -- GLOBAL_STEP: 46925 + | > loss: -0.22200 (-0.20348) + | > log_mle: -0.40292 (-0.32875) + | > loss_dur: 0.18092 (0.12526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.10641 (16.55680) + | > current_lr: 0.00005 + | > step_time: 3.18740 (2.37807) + | > loader_time: 0.00330 (0.03515) + + + --> STEP: 130/234 -- GLOBAL_STEP: 46930 + | > loss: -0.23513 (-0.20446) + | > log_mle: -0.41732 (-0.33180) + | > loss_dur: 0.18220 (0.12734) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.70832 (17.32335) + | > current_lr: 0.00005 + | > step_time: 1.70230 (2.36722) + | > loader_time: 0.00270 (0.03393) + + + --> STEP: 135/234 -- GLOBAL_STEP: 46935 + | > loss: -0.18995 (-0.20550) + | > log_mle: -0.34256 (-0.33467) + | > loss_dur: 0.15262 (0.12917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.03746 (18.12968) + | > current_lr: 0.00005 + | > step_time: 1.60480 (2.33967) + | > loader_time: 0.08170 (0.03396) + + + --> STEP: 140/234 -- GLOBAL_STEP: 46940 + | > loss: -0.19158 (-0.20672) + | > log_mle: -0.37400 (-0.33814) + | > loss_dur: 0.18242 (0.13142) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.51396 (18.95924) + | > current_lr: 0.00005 + | > step_time: 3.19600 (2.33987) + | > loader_time: 0.00380 (0.03354) + + + --> STEP: 145/234 -- GLOBAL_STEP: 46945 + | > loss: -0.28939 (-0.20853) + | > log_mle: -0.47728 (-0.34230) + | > loss_dur: 0.18789 (0.13376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.72575 (19.88352) + | > current_lr: 0.00005 + | > step_time: 2.18790 (2.34669) + | > loader_time: 0.00720 (0.03379) + + + --> STEP: 150/234 -- GLOBAL_STEP: 46950 + | > loss: -0.25426 (-0.21053) + | > log_mle: -0.46260 (-0.34620) + | > loss_dur: 0.20834 (0.13567) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.50028 (20.79797) + | > current_lr: 0.00005 + | > step_time: 1.61600 (2.35795) + | > loader_time: 0.17670 (0.03700) + + + --> STEP: 155/234 -- GLOBAL_STEP: 46955 + | > loss: -0.31566 (-0.21348) + | > log_mle: -0.53109 (-0.35123) + | > loss_dur: 0.21544 (0.13775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.19296 (21.91716) + | > current_lr: 0.00005 + | > step_time: 3.80950 (2.36657) + | > loader_time: 0.09630 (0.03749) + + + --> STEP: 160/234 -- GLOBAL_STEP: 46960 + | > loss: -0.31551 (-0.21578) + | > log_mle: -0.52893 (-0.35558) + | > loss_dur: 0.21341 (0.13980) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.98597 (23.01657) + | > current_lr: 0.00005 + | > step_time: 10.00040 (2.41516) + | > loader_time: 0.00920 (0.03871) + + + --> STEP: 165/234 -- GLOBAL_STEP: 46965 + | > loss: -0.30868 (-0.21791) + | > log_mle: -0.52598 (-0.35973) + | > loss_dur: 0.21730 (0.14182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.35670 (24.06772) + | > current_lr: 0.00005 + | > step_time: 1.70410 (2.39950) + | > loader_time: 0.00300 (0.03854) + + + --> STEP: 170/234 -- GLOBAL_STEP: 46970 + | > loss: -0.32900 (-0.22051) + | > log_mle: -0.55895 (-0.36437) + | > loss_dur: 0.22995 (0.14385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.38781 (25.18624) + | > current_lr: 0.00005 + | > step_time: 5.50190 (2.51071) + | > loader_time: 0.09750 (0.04323) + + + --> STEP: 175/234 -- GLOBAL_STEP: 46975 + | > loss: -0.29939 (-0.22381) + | > log_mle: -0.53528 (-0.36982) + | > loss_dur: 0.23590 (0.14601) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.50043 (26.31546) + | > current_lr: 0.00005 + | > step_time: 2.20310 (2.61735) + | > loader_time: 0.00310 (0.04423) + + + --> STEP: 180/234 -- GLOBAL_STEP: 46980 + | > loss: -0.33170 (-0.22674) + | > log_mle: -0.54175 (-0.37486) + | > loss_dur: 0.21005 (0.14813) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.11980 (27.45575) + | > current_lr: 0.00005 + | > step_time: 5.40140 (2.66637) + | > loader_time: 0.20610 (0.04625) + + + --> STEP: 185/234 -- GLOBAL_STEP: 46985 + | > loss: -0.34801 (-0.22940) + | > log_mle: -0.57776 (-0.37954) + | > loss_dur: 0.22975 (0.15014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.09651 (28.60036) + | > current_lr: 0.00005 + | > step_time: 4.20510 (2.65881) + | > loader_time: 0.10280 (0.04652) + + + --> STEP: 190/234 -- GLOBAL_STEP: 46990 + | > loss: -0.34252 (-0.23209) + | > log_mle: -0.55392 (-0.38414) + | > loss_dur: 0.21140 (0.15206) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.03256 (29.73240) + | > current_lr: 0.00005 + | > step_time: 1.70280 (2.74749) + | > loader_time: 0.08670 (0.04769) + + + --> STEP: 195/234 -- GLOBAL_STEP: 46995 + | > loss: -0.33618 (-0.23509) + | > log_mle: -0.56745 (-0.38893) + | > loss_dur: 0.23127 (0.15384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.40675 (30.93906) + | > current_lr: 0.00005 + | > step_time: 2.50720 (2.76330) + | > loader_time: 0.00330 (0.04797) + + + --> STEP: 200/234 -- GLOBAL_STEP: 47000 + | > loss: -0.32630 (-0.23781) + | > log_mle: -0.57211 (-0.39342) + | > loss_dur: 0.24582 (0.15561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.11916 (31.93023) + | > current_lr: 0.00005 + | > step_time: 2.80700 (2.76580) + | > loader_time: 0.08830 (0.04921) + + + --> STEP: 205/234 -- GLOBAL_STEP: 47005 + | > loss: -0.33292 (-0.24040) + | > log_mle: -0.56156 (-0.39774) + | > loss_dur: 0.22865 (0.15734) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.53236 (32.98349) + | > current_lr: 0.00005 + | > step_time: 5.29740 (2.78814) + | > loader_time: 0.09660 (0.04940) + + + --> STEP: 210/234 -- GLOBAL_STEP: 47010 + | > loss: -0.39284 (-0.24363) + | > log_mle: -0.63457 (-0.40277) + | > loss_dur: 0.24172 (0.15914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.71647 (34.33244) + | > current_lr: 0.00005 + | > step_time: 9.09480 (2.87167) + | > loader_time: 0.09610 (0.20153) + + + --> STEP: 215/234 -- GLOBAL_STEP: 47015 + | > loss: -0.35470 (-0.24710) + | > log_mle: -0.58749 (-0.40794) + | > loss_dur: 0.23279 (0.16084) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.83088 (35.72416) + | > current_lr: 0.00005 + | > step_time: 9.90710 (2.98218) + | > loader_time: 0.08900 (0.20092) + + + --> STEP: 220/234 -- GLOBAL_STEP: 47020 + | > loss: -0.39401 (-0.25071) + | > log_mle: -0.64083 (-0.41338) + | > loss_dur: 0.24681 (0.16267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.98432 (37.09486) + | > current_lr: 0.00005 + | > step_time: 2.28750 (3.01761) + | > loader_time: 0.01190 (0.19728) + + + --> STEP: 225/234 -- GLOBAL_STEP: 47025 + | > loss: -0.44577 (-0.25410) + | > log_mle: -0.69954 (-0.41855) + | > loss_dur: 0.25377 (0.16445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 117.24872 (38.44470) + | > current_lr: 0.00005 + | > step_time: 1.70270 (2.99725) + | > loader_time: 0.08710 (0.19335) + + + --> STEP: 230/234 -- GLOBAL_STEP: 47030 + | > loss: -0.43188 (-0.25729) + | > log_mle: -0.75918 (-0.42429) + | > loss_dur: 0.32730 (0.16700) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.24564 (39.77051) + | > current_lr: 0.00005 + | > step_time: 0.26320 (2.95313) + | > loader_time: 0.00310 (0.19030) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00515 (-0.59843) + | > avg_loss: -0.27464 (+0.01369) + | > avg_log_mle: -0.49824 (+0.00882) + | > avg_loss_dur: 0.22360 (+0.00487) + + + > EPOCH: 201/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 03:05:09)  + + --> STEP: 1/234 -- GLOBAL_STEP: 47035 + | > loss: -0.22504 (-0.22504) + | > log_mle: -0.32707 (-0.32707) + | > loss_dur: 0.10203 (0.10203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.23253 (18.23253) + | > current_lr: 0.00005 + | > step_time: 5.80380 (5.80384) + | > loader_time: 2.69920 (2.69921) + + + --> STEP: 6/234 -- GLOBAL_STEP: 47040 + | > loss: -0.23454 (-0.21862) + | > log_mle: -0.31924 (-0.32416) + | > loss_dur: 0.08469 (0.10555) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.97982 (19.81486) + | > current_lr: 0.00005 + | > step_time: 1.40300 (6.31502) + | > loader_time: 0.08470 (0.52845) + + + --> STEP: 11/234 -- GLOBAL_STEP: 47045 + | > loss: -0.24253 (-0.22344) + | > log_mle: -0.32668 (-0.32652) + | > loss_dur: 0.08415 (0.10309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.63570 (18.38866) + | > current_lr: 0.00005 + | > step_time: 3.19340 (4.25269) + | > loader_time: 0.10600 (0.32363) + + + --> STEP: 16/234 -- GLOBAL_STEP: 47050 + | > loss: -0.23919 (-0.22561) + | > log_mle: -0.32423 (-0.32646) + | > loss_dur: 0.08504 (0.10086) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.90946 (17.77326) + | > current_lr: 0.00005 + | > step_time: 3.58860 (3.86088) + | > loader_time: 0.00290 (0.26487) + + + --> STEP: 21/234 -- GLOBAL_STEP: 47055 + | > loss: -0.21188 (-0.22454) + | > log_mle: -0.30211 (-0.32358) + | > loss_dur: 0.09024 (0.09904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.41970 (16.11700) + | > current_lr: 0.00005 + | > step_time: 1.08160 (4.09339) + | > loader_time: 0.00100 (0.20240) + + + --> STEP: 26/234 -- GLOBAL_STEP: 47060 + | > loss: -0.21795 (-0.22575) + | > log_mle: -0.31618 (-0.32272) + | > loss_dur: 0.09823 (0.09697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.72150 (15.05911) + | > current_lr: 0.00005 + | > step_time: 3.01460 (4.20313) + | > loader_time: 0.08060 (0.17776) + + + --> STEP: 31/234 -- GLOBAL_STEP: 47065 + | > loss: -0.18104 (-0.22507) + | > log_mle: -0.30750 (-0.32168) + | > loss_dur: 0.12646 (0.09661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01329 (14.32421) + | > current_lr: 0.00005 + | > step_time: 1.79970 (4.17419) + | > loader_time: 0.00140 (0.15580) + + + --> STEP: 36/234 -- GLOBAL_STEP: 47070 + | > loss: -0.20280 (-0.22321) + | > log_mle: -0.30792 (-0.32053) + | > loss_dur: 0.10512 (0.09732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.94825 (14.00941) + | > current_lr: 0.00005 + | > step_time: 3.30380 (4.20889) + | > loader_time: 0.08340 (0.14715) + + + --> STEP: 41/234 -- GLOBAL_STEP: 47075 + | > loss: -0.23084 (-0.22129) + | > log_mle: -0.31733 (-0.31946) + | > loss_dur: 0.08649 (0.09817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.01878 (13.58327) + | > current_lr: 0.00005 + | > step_time: 3.38200 (3.96870) + | > loader_time: 0.00170 (0.13594) + + + --> STEP: 46/234 -- GLOBAL_STEP: 47080 + | > loss: -0.18019 (-0.21902) + | > log_mle: -0.30714 (-0.31857) + | > loss_dur: 0.12695 (0.09954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05903 (13.64412) + | > current_lr: 0.00005 + | > step_time: 2.11880 (3.71099) + | > loader_time: 0.01330 (0.12165) + + + --> STEP: 51/234 -- GLOBAL_STEP: 47085 + | > loss: -0.18447 (-0.21744) + | > log_mle: -0.29914 (-0.31744) + | > loss_dur: 0.11467 (0.10000) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.82931 (13.21085) + | > current_lr: 0.00005 + | > step_time: 2.32780 (3.55575) + | > loader_time: 0.00160 (0.11163) + + + --> STEP: 56/234 -- GLOBAL_STEP: 47090 + | > loss: -0.19411 (-0.21588) + | > log_mle: -0.31600 (-0.31708) + | > loss_dur: 0.12189 (0.10120) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.50000 (13.01736) + | > current_lr: 0.00005 + | > step_time: 0.62530 (3.43599) + | > loader_time: 0.00180 (0.10354) + + + --> STEP: 61/234 -- GLOBAL_STEP: 47095 + | > loss: -0.19718 (-0.21459) + | > log_mle: -0.31224 (-0.31686) + | > loss_dur: 0.11506 (0.10227) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46897 (12.82279) + | > current_lr: 0.00005 + | > step_time: 2.71230 (3.37390) + | > loader_time: 0.00320 (0.09807) + + + --> STEP: 66/234 -- GLOBAL_STEP: 47100 + | > loss: -0.20694 (-0.21288) + | > log_mle: -0.30396 (-0.31657) + | > loss_dur: 0.09702 (0.10368) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.63472 (12.78806) + | > current_lr: 0.00005 + | > step_time: 2.60290 (3.26552) + | > loader_time: 0.00220 (0.09083) + + + --> STEP: 71/234 -- GLOBAL_STEP: 47105 + | > loss: -0.18408 (-0.21038) + | > log_mle: -0.33054 (-0.31601) + | > loss_dur: 0.14646 (0.10563) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.75666 (12.95962) + | > current_lr: 0.00005 + | > step_time: 2.40360 (3.16498) + | > loader_time: 0.00210 (0.08460) + + + --> STEP: 76/234 -- GLOBAL_STEP: 47110 + | > loss: -0.19476 (-0.20858) + | > log_mle: -0.31871 (-0.31574) + | > loss_dur: 0.12395 (0.10716) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.76228 (12.86773) + | > current_lr: 0.00005 + | > step_time: 1.60290 (3.12343) + | > loader_time: 0.00200 (0.07927) + + + --> STEP: 81/234 -- GLOBAL_STEP: 47115 + | > loss: -0.18934 (-0.20732) + | > log_mle: -0.32164 (-0.31534) + | > loss_dur: 0.13230 (0.10802) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.49828 (12.90115) + | > current_lr: 0.00005 + | > step_time: 1.99580 (3.01706) + | > loader_time: 0.00130 (0.07549) + + + --> STEP: 86/234 -- GLOBAL_STEP: 47120 + | > loss: -0.18274 (-0.20588) + | > log_mle: -0.32066 (-0.31511) + | > loss_dur: 0.13792 (0.10924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.61087 (13.10631) + | > current_lr: 0.00005 + | > step_time: 1.18910 (2.96125) + | > loader_time: 0.01750 (0.07148) + + + --> STEP: 91/234 -- GLOBAL_STEP: 47125 + | > loss: -0.18325 (-0.20495) + | > log_mle: -0.33199 (-0.31614) + | > loss_dur: 0.14874 (0.11118) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.60934 (13.32740) + | > current_lr: 0.00005 + | > step_time: 2.28180 (2.92509) + | > loader_time: 0.00210 (0.06855) + + + --> STEP: 96/234 -- GLOBAL_STEP: 47130 + | > loss: -0.19170 (-0.20556) + | > log_mle: -0.31800 (-0.31872) + | > loss_dur: 0.12629 (0.11317) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.94994 (13.99044) + | > current_lr: 0.00005 + | > step_time: 1.19540 (2.87993) + | > loader_time: 0.00200 (0.06687) + + + --> STEP: 101/234 -- GLOBAL_STEP: 47135 + | > loss: -0.20436 (-0.20533) + | > log_mle: -0.37342 (-0.32039) + | > loss_dur: 0.16906 (0.11506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.50275 (14.34050) + | > current_lr: 0.00005 + | > step_time: 2.00750 (2.84573) + | > loader_time: 0.00180 (0.06523) + + + --> STEP: 106/234 -- GLOBAL_STEP: 47140 + | > loss: -0.17965 (-0.20526) + | > log_mle: -0.36720 (-0.32251) + | > loss_dur: 0.18755 (0.11724) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.92105 (14.98236) + | > current_lr: 0.00005 + | > step_time: 1.90700 (2.79971) + | > loader_time: 0.07700 (0.06436) + + + --> STEP: 111/234 -- GLOBAL_STEP: 47145 + | > loss: -0.22066 (-0.20498) + | > log_mle: -0.42008 (-0.32474) + | > loss_dur: 0.19942 (0.11976) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.06863 (15.59989) + | > current_lr: 0.00005 + | > step_time: 1.80610 (2.77107) + | > loader_time: 0.00280 (0.06236) + + + --> STEP: 116/234 -- GLOBAL_STEP: 47150 + | > loss: -0.18673 (-0.20498) + | > log_mle: -0.38616 (-0.32699) + | > loss_dur: 0.19943 (0.12201) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.74244 (16.16077) + | > current_lr: 0.00005 + | > step_time: 1.79680 (2.72828) + | > loader_time: 0.00500 (0.06052) + + + --> STEP: 121/234 -- GLOBAL_STEP: 47155 + | > loss: -0.15897 (-0.20486) + | > log_mle: -0.30210 (-0.32853) + | > loss_dur: 0.14313 (0.12367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.86141 (16.40971) + | > current_lr: 0.00005 + | > step_time: 2.30540 (2.70339) + | > loader_time: 0.00630 (0.05819) + + + --> STEP: 126/234 -- GLOBAL_STEP: 47160 + | > loss: -0.23689 (-0.20514) + | > log_mle: -0.43492 (-0.33063) + | > loss_dur: 0.19803 (0.12550) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.33733 (16.81017) + | > current_lr: 0.00005 + | > step_time: 1.17440 (2.68961) + | > loader_time: 0.00220 (0.05601) + + + --> STEP: 131/234 -- GLOBAL_STEP: 47165 + | > loss: -0.29706 (-0.20634) + | > log_mle: -0.48353 (-0.33389) + | > loss_dur: 0.18647 (0.12755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.85889 (17.51926) + | > current_lr: 0.00005 + | > step_time: 3.09190 (2.66940) + | > loader_time: 0.09690 (0.05666) + + + --> STEP: 136/234 -- GLOBAL_STEP: 47170 + | > loss: -0.31229 (-0.20772) + | > log_mle: -0.53427 (-0.33721) + | > loss_dur: 0.22198 (0.12949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.83550 (18.23265) + | > current_lr: 0.00005 + | > step_time: 1.69370 (2.65220) + | > loader_time: 0.00330 (0.05611) + + + --> STEP: 141/234 -- GLOBAL_STEP: 47175 + | > loss: -0.24675 (-0.20846) + | > log_mle: -0.42481 (-0.33993) + | > loss_dur: 0.17806 (0.13147) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.76871 (19.09401) + | > current_lr: 0.00005 + | > step_time: 2.52420 (2.66024) + | > loader_time: 0.00400 (0.05489) + + + --> STEP: 146/234 -- GLOBAL_STEP: 47180 + | > loss: -0.29014 (-0.21059) + | > log_mle: -0.48715 (-0.34450) + | > loss_dur: 0.19701 (0.13391) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.76928 (20.12298) + | > current_lr: 0.00005 + | > step_time: 2.85480 (2.63918) + | > loader_time: 0.00260 (0.05369) + + + --> STEP: 151/234 -- GLOBAL_STEP: 47185 + | > loss: -0.26702 (-0.21254) + | > log_mle: -0.45396 (-0.34830) + | > loss_dur: 0.18695 (0.13576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.65736 (20.87600) + | > current_lr: 0.00005 + | > step_time: 1.99540 (2.62635) + | > loader_time: 0.09510 (0.05265) + + + --> STEP: 156/234 -- GLOBAL_STEP: 47190 + | > loss: -0.29333 (-0.21528) + | > log_mle: -0.48924 (-0.35328) + | > loss_dur: 0.19591 (0.13800) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.58605 (22.69389) + | > current_lr: 0.00005 + | > step_time: 2.70690 (2.62502) + | > loader_time: 0.08550 (0.05229) + + + --> STEP: 161/234 -- GLOBAL_STEP: 47195 + | > loss: -0.32494 (-0.21744) + | > log_mle: -0.51526 (-0.35756) + | > loss_dur: 0.19033 (0.14012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.05314 (23.63637) + | > current_lr: 0.00005 + | > step_time: 2.09750 (2.65366) + | > loader_time: 0.09770 (0.05368) + + + --> STEP: 166/234 -- GLOBAL_STEP: 47200 + | > loss: -0.26741 (-0.21940) + | > log_mle: -0.45415 (-0.36131) + | > loss_dur: 0.18674 (0.14191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.48264 (24.27584) + | > current_lr: 0.00005 + | > step_time: 4.68650 (2.69176) + | > loader_time: 0.49570 (0.05629) + + + --> STEP: 171/234 -- GLOBAL_STEP: 47205 + | > loss: -0.35786 (-0.22254) + | > log_mle: -0.56278 (-0.36659) + | > loss_dur: 0.20493 (0.14405) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.44083 (25.52613) + | > current_lr: 0.00005 + | > step_time: 1.98140 (2.66626) + | > loader_time: 0.00320 (0.05475) + + + --> STEP: 176/234 -- GLOBAL_STEP: 47210 + | > loss: -0.31720 (-0.22554) + | > log_mle: -0.53914 (-0.37178) + | > loss_dur: 0.22193 (0.14624) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.46548 (26.75326) + | > current_lr: 0.00005 + | > step_time: 2.50460 (2.68865) + | > loader_time: 0.08710 (0.05431) + + + --> STEP: 181/234 -- GLOBAL_STEP: 47215 + | > loss: -0.27270 (-0.22805) + | > log_mle: -0.47139 (-0.37635) + | > loss_dur: 0.19869 (0.14830) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.74051 (27.92492) + | > current_lr: 0.00005 + | > step_time: 3.69430 (2.72004) + | > loader_time: 0.01130 (0.05449) + + + --> STEP: 186/234 -- GLOBAL_STEP: 47220 + | > loss: -0.27965 (-0.23070) + | > log_mle: -0.51330 (-0.38113) + | > loss_dur: 0.23365 (0.15043) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.35040 (29.12662) + | > current_lr: 0.00005 + | > step_time: 2.61440 (2.79808) + | > loader_time: 0.00380 (0.05577) + + + --> STEP: 191/234 -- GLOBAL_STEP: 47225 + | > loss: -0.31852 (-0.23341) + | > log_mle: -0.53058 (-0.38567) + | > loss_dur: 0.21206 (0.15226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.82221 (30.23347) + | > current_lr: 0.00005 + | > step_time: 4.19750 (2.80767) + | > loader_time: 0.08690 (0.05674) + + + --> STEP: 196/234 -- GLOBAL_STEP: 47230 + | > loss: -0.28677 (-0.23626) + | > log_mle: -0.52079 (-0.39032) + | > loss_dur: 0.23402 (0.15406) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.46804 (31.49491) + | > current_lr: 0.00005 + | > step_time: 7.20220 (2.90954) + | > loader_time: 0.00450 (0.05594) + + + --> STEP: 201/234 -- GLOBAL_STEP: 47235 + | > loss: -0.25100 (-0.23857) + | > log_mle: -0.48084 (-0.39439) + | > loss_dur: 0.22985 (0.15581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.78712 (32.47827) + | > current_lr: 0.00005 + | > step_time: 3.70500 (2.96409) + | > loader_time: 0.08520 (0.05635) + + + --> STEP: 206/234 -- GLOBAL_STEP: 47240 + | > loss: -0.36755 (-0.24157) + | > log_mle: -0.59381 (-0.39908) + | > loss_dur: 0.22626 (0.15751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.93504 (33.59891) + | > current_lr: 0.00005 + | > step_time: 1.59870 (3.01743) + | > loader_time: 0.00220 (0.05680) + + + --> STEP: 211/234 -- GLOBAL_STEP: 47245 + | > loss: -0.39793 (-0.24475) + | > log_mle: -0.64903 (-0.40415) + | > loss_dur: 0.25110 (0.15939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 109.14863 (35.08406) + | > current_lr: 0.00005 + | > step_time: 7.19350 (3.01149) + | > loader_time: 0.10010 (0.05735) + + + --> STEP: 216/234 -- GLOBAL_STEP: 47250 + | > loss: -0.40841 (-0.24801) + | > log_mle: -0.65832 (-0.40916) + | > loss_dur: 0.24991 (0.16115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.42207 (36.12937) + | > current_lr: 0.00005 + | > step_time: 4.88890 (3.02781) + | > loader_time: 0.00320 (0.05614) + + + --> STEP: 221/234 -- GLOBAL_STEP: 47255 + | > loss: -0.35400 (-0.25136) + | > log_mle: -0.57381 (-0.41423) + | > loss_dur: 0.21981 (0.16287) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.24982 (37.41748) + | > current_lr: 0.00005 + | > step_time: 2.39900 (3.03633) + | > loader_time: 0.00370 (0.05573) + + + --> STEP: 226/234 -- GLOBAL_STEP: 47260 + | > loss: -0.41208 (-0.25480) + | > log_mle: -0.66127 (-0.41956) + | > loss_dur: 0.24919 (0.16475) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.87594 (38.98612) + | > current_lr: 0.00005 + | > step_time: 0.23900 (3.00638) + | > loader_time: 0.00310 (0.05459) + + + --> STEP: 231/234 -- GLOBAL_STEP: 47265 + | > loss: -0.35599 (-0.25768) + | > log_mle: -0.73746 (-0.42551) + | > loss_dur: 0.38147 (0.16783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 130.74060 (40.26892) + | > current_lr: 0.00005 + | > step_time: 0.27240 (2.94670) + | > loader_time: 0.00370 (0.05349) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00225 (-0.00290) + | > avg_loss: -0.29213 (-0.01749) + | > avg_log_mle: -0.50343 (-0.00519) + | > avg_loss_dur: 0.21130 (-0.01231) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_47268.pth + + > EPOCH: 202/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 03:17:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 47270 + | > loss: -0.23107 (-0.23275) + | > log_mle: -0.33098 (-0.32620) + | > loss_dur: 0.09991 (0.09345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.29667 (21.12194) + | > current_lr: 0.00005 + | > step_time: 12.61040 (7.20917) + | > loader_time: 0.49710 (0.24913) + + + --> STEP: 7/234 -- GLOBAL_STEP: 47275 + | > loss: -0.25140 (-0.21980) + | > log_mle: -0.33027 (-0.32228) + | > loss_dur: 0.07886 (0.10247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.84766 (21.66961) + | > current_lr: 0.00005 + | > step_time: 15.90290 (6.10339) + | > loader_time: 0.40720 (0.15820) + + + --> STEP: 12/234 -- GLOBAL_STEP: 47280 + | > loss: -0.23186 (-0.22403) + | > log_mle: -0.32639 (-0.32496) + | > loss_dur: 0.09454 (0.10094) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.56237 (18.76080) + | > current_lr: 0.00005 + | > step_time: 7.20870 (5.30843) + | > loader_time: 0.08830 (0.10757) + + + --> STEP: 17/234 -- GLOBAL_STEP: 47285 + | > loss: -0.22679 (-0.22791) + | > log_mle: -0.31126 (-0.32543) + | > loss_dur: 0.08448 (0.09753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.52173 (16.35355) + | > current_lr: 0.00005 + | > step_time: 1.30720 (4.31049) + | > loader_time: 0.00250 (0.07661) + + + --> STEP: 22/234 -- GLOBAL_STEP: 47290 + | > loss: -0.22478 (-0.22744) + | > log_mle: -0.32541 (-0.32404) + | > loss_dur: 0.10063 (0.09660) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81502 (15.35060) + | > current_lr: 0.00005 + | > step_time: 1.08120 (4.05629) + | > loader_time: 0.00110 (0.07284) + + + --> STEP: 27/234 -- GLOBAL_STEP: 47295 + | > loss: -0.23204 (-0.22861) + | > log_mle: -0.32162 (-0.32348) + | > loss_dur: 0.08958 (0.09487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.29582 (14.50627) + | > current_lr: 0.00005 + | > step_time: 1.81720 (3.60924) + | > loader_time: 0.08120 (0.07201) + + + --> STEP: 32/234 -- GLOBAL_STEP: 47300 + | > loss: -0.24219 (-0.22856) + | > log_mle: -0.32825 (-0.32294) + | > loss_dur: 0.08606 (0.09438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.62551 (13.82080) + | > current_lr: 0.00005 + | > step_time: 3.79310 (3.35210) + | > loader_time: 0.00320 (0.06364) + + + --> STEP: 37/234 -- GLOBAL_STEP: 47305 + | > loss: -0.22131 (-0.22578) + | > log_mle: -0.30707 (-0.32123) + | > loss_dur: 0.08576 (0.09545) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.88456 (13.58168) + | > current_lr: 0.00005 + | > step_time: 1.49710 (3.58820) + | > loader_time: 0.00140 (0.05820) + + + --> STEP: 42/234 -- GLOBAL_STEP: 47310 + | > loss: -0.20110 (-0.22359) + | > log_mle: -0.30019 (-0.32004) + | > loss_dur: 0.09909 (0.09645) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.10835 (13.31346) + | > current_lr: 0.00005 + | > step_time: 2.61360 (3.43538) + | > loader_time: 0.07650 (0.05329) + + + --> STEP: 47/234 -- GLOBAL_STEP: 47315 + | > loss: -0.19497 (-0.22150) + | > log_mle: -0.31139 (-0.31965) + | > loss_dur: 0.11642 (0.09814) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.81367 (13.12904) + | > current_lr: 0.00005 + | > step_time: 2.98950 (3.42333) + | > loader_time: 0.00620 (0.04980) + + + --> STEP: 52/234 -- GLOBAL_STEP: 47320 + | > loss: -0.19249 (-0.22015) + | > log_mle: -0.30554 (-0.31860) + | > loss_dur: 0.11305 (0.09845) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.07404 (12.73155) + | > current_lr: 0.00005 + | > step_time: 6.00560 (3.42901) + | > loader_time: 0.10780 (0.04902) + + + --> STEP: 57/234 -- GLOBAL_STEP: 47325 + | > loss: -0.18698 (-0.21891) + | > log_mle: -0.30020 (-0.31825) + | > loss_dur: 0.11323 (0.09934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.16558 (12.53505) + | > current_lr: 0.00005 + | > step_time: 2.40200 (3.27550) + | > loader_time: 0.00180 (0.04827) + + + --> STEP: 62/234 -- GLOBAL_STEP: 47330 + | > loss: -0.15266 (-0.21732) + | > log_mle: -0.33889 (-0.31875) + | > loss_dur: 0.18623 (0.10143) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.26026 (12.61120) + | > current_lr: 0.00005 + | > step_time: 0.99280 (3.14147) + | > loader_time: 0.00170 (0.05098) + + + --> STEP: 67/234 -- GLOBAL_STEP: 47335 + | > loss: -0.19808 (-0.21619) + | > log_mle: -0.32405 (-0.31828) + | > loss_dur: 0.12597 (0.10210) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.79009 (12.36423) + | > current_lr: 0.00005 + | > step_time: 1.49440 (3.04932) + | > loader_time: 0.00130 (0.05105) + + + --> STEP: 72/234 -- GLOBAL_STEP: 47340 + | > loss: -0.19240 (-0.21353) + | > log_mle: -0.30833 (-0.31753) + | > loss_dur: 0.11593 (0.10400) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.73772 (12.44864) + | > current_lr: 0.00005 + | > step_time: 2.01220 (2.95569) + | > loader_time: 0.00280 (0.04874) + + + --> STEP: 77/234 -- GLOBAL_STEP: 47345 + | > loss: -0.19977 (-0.21154) + | > log_mle: -0.31320 (-0.31720) + | > loss_dur: 0.11343 (0.10566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.61423 (12.58906) + | > current_lr: 0.00005 + | > step_time: 2.20150 (2.88648) + | > loader_time: 0.09260 (0.05003) + + + --> STEP: 82/234 -- GLOBAL_STEP: 47350 + | > loss: -0.18669 (-0.20998) + | > log_mle: -0.30659 (-0.31673) + | > loss_dur: 0.11989 (0.10674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.61116 (12.60498) + | > current_lr: 0.00005 + | > step_time: 1.91550 (2.86435) + | > loader_time: 0.00140 (0.04926) + + + --> STEP: 87/234 -- GLOBAL_STEP: 47355 + | > loss: -0.17726 (-0.20837) + | > log_mle: -0.31000 (-0.31658) + | > loss_dur: 0.13273 (0.10821) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.52660 (12.66586) + | > current_lr: 0.00005 + | > step_time: 1.70360 (2.82675) + | > loader_time: 0.00300 (0.04928) + + + --> STEP: 92/234 -- GLOBAL_STEP: 47360 + | > loss: -0.21822 (-0.20801) + | > log_mle: -0.35764 (-0.31804) + | > loss_dur: 0.13942 (0.11004) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.58287 (13.02534) + | > current_lr: 0.00005 + | > step_time: 1.71170 (2.77745) + | > loader_time: 0.00910 (0.04877) + + + --> STEP: 97/234 -- GLOBAL_STEP: 47365 + | > loss: -0.19335 (-0.20794) + | > log_mle: -0.34242 (-0.32041) + | > loss_dur: 0.14907 (0.11247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.01911 (13.57871) + | > current_lr: 0.00005 + | > step_time: 3.82110 (2.74245) + | > loader_time: 0.00320 (0.04829) + + + --> STEP: 102/234 -- GLOBAL_STEP: 47370 + | > loss: -0.16717 (-0.20716) + | > log_mle: -0.32103 (-0.32173) + | > loss_dur: 0.15386 (0.11457) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.31907 (14.21128) + | > current_lr: 0.00005 + | > step_time: 4.39230 (2.72471) + | > loader_time: 0.00690 (0.04934) + + + --> STEP: 107/234 -- GLOBAL_STEP: 47375 + | > loss: -0.20510 (-0.20699) + | > log_mle: -0.36789 (-0.32411) + | > loss_dur: 0.16279 (0.11713) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.65429 (14.83793) + | > current_lr: 0.00005 + | > step_time: 1.22710 (2.70971) + | > loader_time: 0.00240 (0.04885) + + + --> STEP: 112/234 -- GLOBAL_STEP: 47380 + | > loss: -0.20086 (-0.20646) + | > log_mle: -0.37860 (-0.32639) + | > loss_dur: 0.17774 (0.11992) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.52597 (15.38665) + | > current_lr: 0.00005 + | > step_time: 1.29960 (2.67430) + | > loader_time: 0.08690 (0.04754) + + + --> STEP: 117/234 -- GLOBAL_STEP: 47385 + | > loss: -0.21209 (-0.20634) + | > log_mle: -0.37498 (-0.32849) + | > loss_dur: 0.16290 (0.12215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.48470 (15.96017) + | > current_lr: 0.00005 + | > step_time: 0.63000 (2.70038) + | > loader_time: 0.08610 (0.04720) + + + --> STEP: 122/234 -- GLOBAL_STEP: 47390 + | > loss: -0.18791 (-0.20594) + | > log_mle: -0.34629 (-0.32968) + | > loss_dur: 0.15839 (0.12374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.43756 (16.28094) + | > current_lr: 0.00005 + | > step_time: 2.50290 (2.68885) + | > loader_time: 0.08820 (0.04684) + + + --> STEP: 127/234 -- GLOBAL_STEP: 47395 + | > loss: -0.22024 (-0.20625) + | > log_mle: -0.40733 (-0.33208) + | > loss_dur: 0.18710 (0.12584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.16509 (16.94145) + | > current_lr: 0.00005 + | > step_time: 1.92150 (2.66518) + | > loader_time: 0.00210 (0.04713) + + + --> STEP: 132/234 -- GLOBAL_STEP: 47400 + | > loss: -0.23374 (-0.20726) + | > log_mle: -0.38892 (-0.33502) + | > loss_dur: 0.15517 (0.12776) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.82246 (17.69172) + | > current_lr: 0.00005 + | > step_time: 1.08320 (2.61694) + | > loader_time: 0.00160 (0.04544) + + + --> STEP: 137/234 -- GLOBAL_STEP: 47405 + | > loss: -0.20505 (-0.20833) + | > log_mle: -0.40622 (-0.33828) + | > loss_dur: 0.20117 (0.12995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.49082 (18.40614) + | > current_lr: 0.00005 + | > step_time: 5.51810 (2.62944) + | > loader_time: 0.08900 (0.04527) + + + --> STEP: 142/234 -- GLOBAL_STEP: 47410 + | > loss: -0.21816 (-0.20921) + | > log_mle: -0.41088 (-0.34103) + | > loss_dur: 0.19272 (0.13182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.53701 (19.14091) + | > current_lr: 0.00005 + | > step_time: 2.09820 (2.59810) + | > loader_time: 0.00380 (0.04437) + + + --> STEP: 147/234 -- GLOBAL_STEP: 47415 + | > loss: -0.22845 (-0.21126) + | > log_mle: -0.41562 (-0.34534) + | > loss_dur: 0.18717 (0.13408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.43011 (20.16291) + | > current_lr: 0.00005 + | > step_time: 2.00290 (2.58678) + | > loader_time: 0.08520 (0.04453) + + + --> STEP: 152/234 -- GLOBAL_STEP: 47420 + | > loss: -0.27946 (-0.21342) + | > log_mle: -0.50076 (-0.34944) + | > loss_dur: 0.22130 (0.13602) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.18394 (21.11236) + | > current_lr: 0.00005 + | > step_time: 2.99010 (2.58768) + | > loader_time: 0.00270 (0.04487) + + + --> STEP: 157/234 -- GLOBAL_STEP: 47425 + | > loss: -0.23285 (-0.21538) + | > log_mle: -0.43141 (-0.35343) + | > loss_dur: 0.19856 (0.13805) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.10828 (22.89127) + | > current_lr: 0.00005 + | > step_time: 3.29880 (2.57666) + | > loader_time: 0.00720 (0.04507) + + + --> STEP: 162/234 -- GLOBAL_STEP: 47430 + | > loss: -0.27474 (-0.21702) + | > log_mle: -0.46458 (-0.35716) + | > loss_dur: 0.18984 (0.14014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.36184 (23.67713) + | > current_lr: 0.00005 + | > step_time: 2.49000 (2.60150) + | > loader_time: 0.00690 (0.04539) + + + --> STEP: 167/234 -- GLOBAL_STEP: 47435 + | > loss: -0.31972 (-0.21875) + | > log_mle: -0.51989 (-0.36075) + | > loss_dur: 0.20017 (0.14200) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.26987 (24.98261) + | > current_lr: 0.00005 + | > step_time: 2.70510 (2.61225) + | > loader_time: 0.08030 (0.04515) + + + --> STEP: 172/234 -- GLOBAL_STEP: 47440 + | > loss: -0.31171 (-0.22080) + | > log_mle: -0.53350 (-0.36502) + | > loss_dur: 0.22179 (0.14422) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.14856 (25.76715) + | > current_lr: 0.00005 + | > step_time: 2.17830 (2.59537) + | > loader_time: 0.00210 (0.04498) + + + --> STEP: 177/234 -- GLOBAL_STEP: 47445 + | > loss: -0.28950 (-0.22304) + | > log_mle: -0.50257 (-0.36936) + | > loss_dur: 0.21307 (0.14632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.37514 (26.56133) + | > current_lr: 0.00005 + | > step_time: 1.69720 (2.62946) + | > loader_time: 0.00430 (0.04641) + + + --> STEP: 182/234 -- GLOBAL_STEP: 47450 + | > loss: -0.30646 (-0.22515) + | > log_mle: -0.55291 (-0.37382) + | > loss_dur: 0.24645 (0.14867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.15643 (27.52348) + | > current_lr: 0.00005 + | > step_time: 8.08550 (2.73632) + | > loader_time: 0.10840 (0.04669) + + + --> STEP: 187/234 -- GLOBAL_STEP: 47455 + | > loss: -0.32550 (-0.22747) + | > log_mle: -0.54765 (-0.37818) + | > loss_dur: 0.22215 (0.15070) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.73984 (28.64699) + | > current_lr: 0.00005 + | > step_time: 6.61140 (2.77813) + | > loader_time: 0.18770 (0.04804) + + + --> STEP: 192/234 -- GLOBAL_STEP: 47460 + | > loss: -0.36247 (-0.23028) + | > log_mle: -0.58335 (-0.38269) + | > loss_dur: 0.22088 (0.15242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.66518 (29.48951) + | > current_lr: 0.00005 + | > step_time: 2.89330 (2.82135) + | > loader_time: 0.10080 (0.05148) + + + --> STEP: 197/234 -- GLOBAL_STEP: 47465 + | > loss: -0.35716 (-0.23307) + | > log_mle: -0.55700 (-0.38710) + | > loss_dur: 0.19985 (0.15403) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.17987 (30.48756) + | > current_lr: 0.00005 + | > step_time: 7.09690 (2.88021) + | > loader_time: 0.00380 (0.05168) + + + --> STEP: 202/234 -- GLOBAL_STEP: 47470 + | > loss: -0.42801 (-0.23570) + | > log_mle: -0.65121 (-0.39165) + | > loss_dur: 0.22321 (0.15596) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.72043 (31.46717) + | > current_lr: 0.00005 + | > step_time: 4.30240 (2.89033) + | > loader_time: 0.10550 (0.05180) + + + --> STEP: 207/234 -- GLOBAL_STEP: 47475 + | > loss: -0.40575 (-0.23854) + | > log_mle: -0.64014 (-0.39628) + | > loss_dur: 0.23439 (0.15773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.88068 (32.45150) + | > current_lr: 0.00005 + | > step_time: 5.69010 (2.91121) + | > loader_time: 0.00710 (0.05113) + + + --> STEP: 212/234 -- GLOBAL_STEP: 47480 + | > loss: -0.36996 (-0.24174) + | > log_mle: -0.60000 (-0.40135) + | > loss_dur: 0.23005 (0.15962) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.07484 (33.90176) + | > current_lr: 0.00005 + | > step_time: 4.20250 (2.97875) + | > loader_time: 0.09910 (0.05135) + + + --> STEP: 217/234 -- GLOBAL_STEP: 47485 + | > loss: -0.38969 (-0.24490) + | > log_mle: -0.63030 (-0.40634) + | > loss_dur: 0.24061 (0.16144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.58337 (35.20683) + | > current_lr: 0.00005 + | > step_time: 4.20530 (3.02537) + | > loader_time: 0.08670 (0.05106) + + + --> STEP: 222/234 -- GLOBAL_STEP: 47490 + | > loss: -0.37735 (-0.24809) + | > log_mle: -0.65037 (-0.41133) + | > loss_dur: 0.27302 (0.16324) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.91397 (36.30796) + | > current_lr: 0.00005 + | > step_time: 1.70260 (3.01445) + | > loader_time: 0.00490 (0.05088) + + + --> STEP: 227/234 -- GLOBAL_STEP: 47495 + | > loss: -0.36623 (-0.25175) + | > log_mle: -0.62572 (-0.41683) + | > loss_dur: 0.25949 (0.16508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.90005 (37.36626) + | > current_lr: 0.00005 + | > step_time: 1.50510 (2.98551) + | > loader_time: 0.39550 (0.05230) + + + --> STEP: 232/234 -- GLOBAL_STEP: 47500 + | > loss: -0.38132 (-0.25483) + | > log_mle: -0.85871 (-0.42389) + | > loss_dur: 0.47738 (0.16906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.78129 (38.80520) + | > current_lr: 0.00005 + | > step_time: 0.35660 (2.94344) + | > loader_time: 0.09320 (0.05236) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22225 (+0.22000) + | > avg_loss: -0.27110 (+0.02103) + | > avg_log_mle: -0.49217 (+0.01126) + | > avg_loss_dur: 0.22107 (+0.00977) + + + > EPOCH: 203/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 03:30:19)  + + --> STEP: 3/234 -- GLOBAL_STEP: 47505 + | > loss: -0.12746 (-0.19281) + | > log_mle: -0.31907 (-0.32673) + | > loss_dur: 0.19161 (0.13392) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.46025 (22.03403) + | > current_lr: 0.00005 + | > step_time: 2.59350 (3.43802) + | > loader_time: 0.00150 (1.39669) + + + --> STEP: 8/234 -- GLOBAL_STEP: 47510 + | > loss: -0.23194 (-0.21698) + | > log_mle: -0.34041 (-0.32872) + | > loss_dur: 0.10847 (0.11174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.72264 (19.45084) + | > current_lr: 0.00005 + | > step_time: 9.48200 (4.31259) + | > loader_time: 0.00080 (0.54523) + + + --> STEP: 13/234 -- GLOBAL_STEP: 47515 + | > loss: -0.24325 (-0.21988) + | > log_mle: -0.33584 (-0.32889) + | > loss_dur: 0.09260 (0.10901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.32713 (18.90302) + | > current_lr: 0.00005 + | > step_time: 2.20860 (4.43978) + | > loader_time: 0.00120 (0.37333) + + + --> STEP: 18/234 -- GLOBAL_STEP: 47520 + | > loss: -0.20763 (-0.22335) + | > log_mle: -0.31951 (-0.32772) + | > loss_dur: 0.11188 (0.10437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.12568 (17.02650) + | > current_lr: 0.00005 + | > step_time: 3.91590 (4.37393) + | > loader_time: 0.00150 (0.28227) + + + --> STEP: 23/234 -- GLOBAL_STEP: 47525 + | > loss: -0.25037 (-0.22631) + | > log_mle: -0.33172 (-0.32672) + | > loss_dur: 0.08135 (0.10040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.53465 (16.03375) + | > current_lr: 0.00005 + | > step_time: 1.07320 (3.83510) + | > loader_time: 0.00220 (0.22588) + + + --> STEP: 28/234 -- GLOBAL_STEP: 47530 + | > loss: -0.27778 (-0.22872) + | > log_mle: -0.33727 (-0.32588) + | > loss_dur: 0.05949 (0.09716) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.35154 (15.16559) + | > current_lr: 0.00005 + | > step_time: 1.34890 (3.40343) + | > loader_time: 0.00270 (0.18866) + + + --> STEP: 33/234 -- GLOBAL_STEP: 47535 + | > loss: -0.22876 (-0.22778) + | > log_mle: -0.31610 (-0.32446) + | > loss_dur: 0.08733 (0.09668) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.07677 (14.39689) + | > current_lr: 0.00005 + | > step_time: 1.56950 (3.28397) + | > loader_time: 0.08100 (0.16580) + + + --> STEP: 38/234 -- GLOBAL_STEP: 47540 + | > loss: -0.21537 (-0.22531) + | > log_mle: -0.32359 (-0.32296) + | > loss_dur: 0.10823 (0.09765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.91277 (14.14779) + | > current_lr: 0.00005 + | > step_time: 3.11650 (3.06501) + | > loader_time: 0.08220 (0.14651) + + + --> STEP: 43/234 -- GLOBAL_STEP: 47545 + | > loss: -0.20204 (-0.22280) + | > log_mle: -0.31968 (-0.32160) + | > loss_dur: 0.11764 (0.09880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.47004 (13.82038) + | > current_lr: 0.00005 + | > step_time: 2.11070 (2.91577) + | > loader_time: 0.00240 (0.13321) + + + --> STEP: 48/234 -- GLOBAL_STEP: 47550 + | > loss: -0.21951 (-0.22098) + | > log_mle: -0.30881 (-0.32086) + | > loss_dur: 0.08930 (0.09988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.30883 (13.54914) + | > current_lr: 0.00005 + | > step_time: 1.23490 (2.77877) + | > loader_time: 0.00170 (0.11961) + + + --> STEP: 53/234 -- GLOBAL_STEP: 47555 + | > loss: -0.18658 (-0.21908) + | > log_mle: -0.31488 (-0.31986) + | > loss_dur: 0.12830 (0.10078) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.54922 (13.25723) + | > current_lr: 0.00005 + | > step_time: 1.51580 (2.66649) + | > loader_time: 0.00170 (0.11144) + + + --> STEP: 58/234 -- GLOBAL_STEP: 47560 + | > loss: -0.20767 (-0.21786) + | > log_mle: -0.31058 (-0.31928) + | > loss_dur: 0.10291 (0.10141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.53397 (13.02166) + | > current_lr: 0.00005 + | > step_time: 1.80180 (2.69737) + | > loader_time: 0.00230 (0.10519) + + + --> STEP: 63/234 -- GLOBAL_STEP: 47565 + | > loss: -0.17427 (-0.21582) + | > log_mle: -0.30325 (-0.31929) + | > loss_dur: 0.12898 (0.10348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.18641 (13.60538) + | > current_lr: 0.00005 + | > step_time: 1.39400 (2.62462) + | > loader_time: 0.00270 (0.09837) + + + --> STEP: 68/234 -- GLOBAL_STEP: 47570 + | > loss: -0.15881 (-0.21437) + | > log_mle: -0.29920 (-0.31856) + | > loss_dur: 0.14039 (0.10419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.65830 (13.38882) + | > current_lr: 0.00005 + | > step_time: 2.20370 (2.58416) + | > loader_time: 0.08740 (0.09255) + + + --> STEP: 73/234 -- GLOBAL_STEP: 47575 + | > loss: -0.16433 (-0.21174) + | > log_mle: -0.31979 (-0.31803) + | > loss_dur: 0.15546 (0.10630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.64371 (13.30652) + | > current_lr: 0.00005 + | > step_time: 1.39210 (2.54505) + | > loader_time: 0.00760 (0.08890) + + + --> STEP: 78/234 -- GLOBAL_STEP: 47580 + | > loss: -0.17621 (-0.21013) + | > log_mle: -0.30066 (-0.31752) + | > loss_dur: 0.12445 (0.10740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.33252 (13.25818) + | > current_lr: 0.00005 + | > step_time: 2.51300 (2.54877) + | > loader_time: 0.00230 (0.08547) + + + --> STEP: 83/234 -- GLOBAL_STEP: 47585 + | > loss: -0.16286 (-0.20865) + | > log_mle: -0.31833 (-0.31730) + | > loss_dur: 0.15547 (0.10865) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.68457 (13.27208) + | > current_lr: 0.00005 + | > step_time: 1.64060 (2.49943) + | > loader_time: 0.08270 (0.08146) + + + --> STEP: 88/234 -- GLOBAL_STEP: 47590 + | > loss: -0.19493 (-0.20763) + | > log_mle: -0.35180 (-0.31756) + | > loss_dur: 0.15687 (0.10993) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.29345 (13.36488) + | > current_lr: 0.00005 + | > step_time: 1.28980 (2.46166) + | > loader_time: 0.00260 (0.07699) + + + --> STEP: 93/234 -- GLOBAL_STEP: 47595 + | > loss: -0.20852 (-0.20719) + | > log_mle: -0.36511 (-0.31896) + | > loss_dur: 0.15659 (0.11177) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.74902 (13.74634) + | > current_lr: 0.00005 + | > step_time: 1.31010 (2.43124) + | > loader_time: 0.00240 (0.07480) + + + --> STEP: 98/234 -- GLOBAL_STEP: 47600 + | > loss: -0.17968 (-0.20690) + | > log_mle: -0.30015 (-0.32049) + | > loss_dur: 0.12047 (0.11359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.58354 (14.00846) + | > current_lr: 0.00005 + | > step_time: 3.71290 (2.44335) + | > loader_time: 0.00670 (0.07368) + + + --> STEP: 103/234 -- GLOBAL_STEP: 47605 + | > loss: -0.21943 (-0.20671) + | > log_mle: -0.39213 (-0.32275) + | > loss_dur: 0.17269 (0.11604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.82873 (14.68213) + | > current_lr: 0.00005 + | > step_time: 2.22340 (2.41615) + | > loader_time: 0.09170 (0.07113) + + + --> STEP: 108/234 -- GLOBAL_STEP: 47610 + | > loss: -0.19546 (-0.20652) + | > log_mle: -0.34035 (-0.32450) + | > loss_dur: 0.14489 (0.11798) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.22048 (15.04257) + | > current_lr: 0.00005 + | > step_time: 1.71390 (2.39383) + | > loader_time: 0.09180 (0.07029) + + + --> STEP: 113/234 -- GLOBAL_STEP: 47615 + | > loss: -0.21890 (-0.20628) + | > log_mle: -0.38372 (-0.32700) + | > loss_dur: 0.16481 (0.12072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.49924 (15.72753) + | > current_lr: 0.00005 + | > step_time: 1.11000 (2.36280) + | > loader_time: 0.00260 (0.06802) + + + --> STEP: 118/234 -- GLOBAL_STEP: 47620 + | > loss: -0.19118 (-0.20593) + | > log_mle: -0.35824 (-0.32882) + | > loss_dur: 0.16706 (0.12289) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.40953 (16.02783) + | > current_lr: 0.00005 + | > step_time: 2.20740 (2.35012) + | > loader_time: 0.00290 (0.06669) + + + --> STEP: 123/234 -- GLOBAL_STEP: 47625 + | > loss: -0.16349 (-0.20536) + | > log_mle: -0.32425 (-0.32974) + | > loss_dur: 0.16076 (0.12438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.32364 (16.25451) + | > current_lr: 0.00005 + | > step_time: 1.20320 (2.32045) + | > loader_time: 0.07710 (0.06545) + + + --> STEP: 128/234 -- GLOBAL_STEP: 47630 + | > loss: -0.22765 (-0.20631) + | > log_mle: -0.38475 (-0.33268) + | > loss_dur: 0.15709 (0.12638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.57943 (16.96892) + | > current_lr: 0.00005 + | > step_time: 3.50970 (2.30505) + | > loader_time: 0.00250 (0.06368) + + + --> STEP: 133/234 -- GLOBAL_STEP: 47635 + | > loss: -0.24173 (-0.20759) + | > log_mle: -0.42018 (-0.33599) + | > loss_dur: 0.17844 (0.12839) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.59131 (17.61839) + | > current_lr: 0.00005 + | > step_time: 3.49600 (2.40395) + | > loader_time: 0.00300 (0.06291) + + + --> STEP: 138/234 -- GLOBAL_STEP: 47640 + | > loss: -0.19859 (-0.20851) + | > log_mle: -0.36583 (-0.33893) + | > loss_dur: 0.16723 (0.13042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.13461 (18.31224) + | > current_lr: 0.00005 + | > step_time: 1.01150 (2.39950) + | > loader_time: 0.00160 (0.06146) + + + --> STEP: 143/234 -- GLOBAL_STEP: 47645 + | > loss: -0.28809 (-0.20998) + | > log_mle: -0.51357 (-0.34272) + | > loss_dur: 0.22549 (0.13274) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.78445 (19.43403) + | > current_lr: 0.00005 + | > step_time: 4.80630 (2.40085) + | > loader_time: 0.19260 (0.06199) + + + --> STEP: 148/234 -- GLOBAL_STEP: 47650 + | > loss: -0.26680 (-0.21204) + | > log_mle: -0.42831 (-0.34662) + | > loss_dur: 0.16151 (0.13459) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.50571 (20.28293) + | > current_lr: 0.00005 + | > step_time: 1.50040 (2.37375) + | > loader_time: 0.00270 (0.05999) + + + --> STEP: 153/234 -- GLOBAL_STEP: 47655 + | > loss: -0.34480 (-0.21478) + | > log_mle: -0.54671 (-0.35158) + | > loss_dur: 0.20191 (0.13680) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.84703 (21.56791) + | > current_lr: 0.00005 + | > step_time: 4.93660 (2.38310) + | > loader_time: 0.19740 (0.05941) + + + --> STEP: 158/234 -- GLOBAL_STEP: 47660 + | > loss: -0.26959 (-0.21701) + | > log_mle: -0.48706 (-0.35590) + | > loss_dur: 0.21747 (0.13889) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.35939 (22.68444) + | > current_lr: 0.00005 + | > step_time: 1.91700 (2.37491) + | > loader_time: 0.09100 (0.05885) + + + --> STEP: 163/234 -- GLOBAL_STEP: 47665 + | > loss: -0.25670 (-0.21960) + | > log_mle: -0.45532 (-0.36034) + | > loss_dur: 0.19862 (0.14074) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.73631 (23.59164) + | > current_lr: 0.00005 + | > step_time: 2.80550 (2.41565) + | > loader_time: 0.08550 (0.05983) + + + --> STEP: 168/234 -- GLOBAL_STEP: 47670 + | > loss: -0.28866 (-0.22210) + | > log_mle: -0.51327 (-0.36486) + | > loss_dur: 0.22461 (0.14277) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.99806 (24.87294) + | > current_lr: 0.00005 + | > step_time: 4.10570 (2.57112) + | > loader_time: 0.00210 (0.05989) + + + --> STEP: 173/234 -- GLOBAL_STEP: 47675 + | > loss: -0.31815 (-0.22527) + | > log_mle: -0.52266 (-0.37008) + | > loss_dur: 0.20451 (0.14481) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.06200 (25.96882) + | > current_lr: 0.00005 + | > step_time: 3.02010 (2.61889) + | > loader_time: 0.00240 (0.05829) + + + --> STEP: 178/234 -- GLOBAL_STEP: 47680 + | > loss: -0.33589 (-0.22808) + | > log_mle: -0.57238 (-0.37512) + | > loss_dur: 0.23649 (0.14704) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.88673 (27.13703) + | > current_lr: 0.00005 + | > step_time: 2.40240 (2.63408) + | > loader_time: 0.00240 (0.05767) + + + --> STEP: 183/234 -- GLOBAL_STEP: 47685 + | > loss: -0.36235 (-0.23057) + | > log_mle: -0.57716 (-0.37975) + | > loss_dur: 0.21481 (0.14918) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.33294 (28.22811) + | > current_lr: 0.00005 + | > step_time: 8.10400 (2.67264) + | > loader_time: 0.06910 (0.05807) + + + --> STEP: 188/234 -- GLOBAL_STEP: 47690 + | > loss: -0.36135 (-0.23319) + | > log_mle: -0.58673 (-0.38445) + | > loss_dur: 0.22539 (0.15126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.22334 (29.23730) + | > current_lr: 0.00005 + | > step_time: 2.10700 (2.69520) + | > loader_time: 0.00370 (0.05974) + + + --> STEP: 193/234 -- GLOBAL_STEP: 47695 + | > loss: -0.37640 (-0.23622) + | > log_mle: -0.58994 (-0.38905) + | > loss_dur: 0.21354 (0.15283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.91303 (30.26013) + | > current_lr: 0.00005 + | > step_time: 11.09520 (2.76641) + | > loader_time: 0.19850 (0.06027) + + + --> STEP: 198/234 -- GLOBAL_STEP: 47700 + | > loss: -0.35835 (-0.23897) + | > log_mle: -0.58806 (-0.39347) + | > loss_dur: 0.22971 (0.15450) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.82272 (31.47289) + | > current_lr: 0.00005 + | > step_time: 4.79790 (2.84765) + | > loader_time: 0.41530 (0.06235) + + + --> STEP: 203/234 -- GLOBAL_STEP: 47705 + | > loss: -0.29423 (-0.24138) + | > log_mle: -0.50997 (-0.39763) + | > loss_dur: 0.21574 (0.15625) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.54891 (32.40642) + | > current_lr: 0.00005 + | > step_time: 2.49970 (2.89236) + | > loader_time: 0.00410 (0.06173) + + + --> STEP: 208/234 -- GLOBAL_STEP: 47710 + | > loss: -0.36108 (-0.24441) + | > log_mle: -0.59837 (-0.40258) + | > loss_dur: 0.23729 (0.15817) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.02869 (33.53528) + | > current_lr: 0.00005 + | > step_time: 7.08650 (2.93639) + | > loader_time: 0.19620 (0.06214) + + + --> STEP: 213/234 -- GLOBAL_STEP: 47715 + | > loss: -0.41606 (-0.24793) + | > log_mle: -0.65633 (-0.40800) + | > loss_dur: 0.24027 (0.16007) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.18069 (34.74988) + | > current_lr: 0.00005 + | > step_time: 1.89760 (3.03734) + | > loader_time: 0.09430 (0.06254) + + + --> STEP: 218/234 -- GLOBAL_STEP: 47720 + | > loss: -0.38260 (-0.25113) + | > log_mle: -0.61213 (-0.41290) + | > loss_dur: 0.22953 (0.16176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.66923 (35.92622) + | > current_lr: 0.00005 + | > step_time: 3.80710 (3.05968) + | > loader_time: 0.00550 (0.06250) + + + --> STEP: 223/234 -- GLOBAL_STEP: 47725 + | > loss: -0.40252 (-0.25463) + | > log_mle: -0.63410 (-0.41812) + | > loss_dur: 0.23157 (0.16349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.82697 (37.33220) + | > current_lr: 0.00005 + | > step_time: 0.48400 (3.05205) + | > loader_time: 0.00410 (0.06248) + + + --> STEP: 228/234 -- GLOBAL_STEP: 47730 + | > loss: -0.37453 (-0.25780) + | > log_mle: -0.64183 (-0.42325) + | > loss_dur: 0.26730 (0.16544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.42950 (38.68897) + | > current_lr: 0.00005 + | > step_time: 0.24380 (2.99030) + | > loader_time: 0.00300 (0.06118) + + + --> STEP: 233/234 -- GLOBAL_STEP: 47735 + | > loss: 0.00288 (-0.25896) + | > log_mle: -0.61924 (-0.42981) + | > loss_dur: 0.62212 (0.17085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.57084 (40.26414) + | > current_lr: 0.00005 + | > step_time: 0.19100 (2.93163) + | > loader_time: 0.00350 (0.05996) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.28037 (+0.05812) + | > avg_loss: -0.29410 (-0.02300) + | > avg_log_mle: -0.50919 (-0.01702) + | > avg_loss_dur: 0.21509 (-0.00598) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_47736.pth + + > EPOCH: 204/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 03:42:54)  + + --> STEP: 4/234 -- GLOBAL_STEP: 47740 + | > loss: -0.20317 (-0.20558) + | > log_mle: -0.32392 (-0.32609) + | > loss_dur: 0.12074 (0.12051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.24864 (19.58674) + | > current_lr: 0.00005 + | > step_time: 4.09150 (5.00134) + | > loader_time: 0.00200 (4.62534) + + + --> STEP: 9/234 -- GLOBAL_STEP: 47745 + | > loss: -0.21711 (-0.22015) + | > log_mle: -0.33393 (-0.32914) + | > loss_dur: 0.11683 (0.10899) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.58602 (17.07494) + | > current_lr: 0.00005 + | > step_time: 9.28740 (4.83704) + | > loader_time: 0.00290 (2.10973) + + + --> STEP: 14/234 -- GLOBAL_STEP: 47750 + | > loss: -0.22773 (-0.22357) + | > log_mle: -0.33107 (-0.32958) + | > loss_dur: 0.10334 (0.10600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.87457 (16.41576) + | > current_lr: 0.00005 + | > step_time: 3.89940 (4.38933) + | > loader_time: 0.00220 (1.36373) + + + --> STEP: 19/234 -- GLOBAL_STEP: 47755 + | > loss: -0.24204 (-0.22771) + | > log_mle: -0.32224 (-0.32766) + | > loss_dur: 0.08020 (0.09995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.34017 (15.27125) + | > current_lr: 0.00005 + | > step_time: 2.31320 (3.93506) + | > loader_time: 0.00120 (1.00537) + + + --> STEP: 24/234 -- GLOBAL_STEP: 47760 + | > loss: -0.25411 (-0.23012) + | > log_mle: -0.32048 (-0.32654) + | > loss_dur: 0.06637 (0.09642) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.55095 (14.43478) + | > current_lr: 0.00005 + | > step_time: 1.09700 (3.48207) + | > loader_time: 0.00110 (0.79623) + + + --> STEP: 29/234 -- GLOBAL_STEP: 47765 + | > loss: -0.20433 (-0.22991) + | > log_mle: -0.30561 (-0.32514) + | > loss_dur: 0.10128 (0.09523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.71995 (13.79760) + | > current_lr: 0.00005 + | > step_time: 4.19550 (3.51655) + | > loader_time: 0.06830 (0.67368) + + + --> STEP: 34/234 -- GLOBAL_STEP: 47770 + | > loss: -0.20990 (-0.22911) + | > log_mle: -0.31285 (-0.32420) + | > loss_dur: 0.10294 (0.09508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.70177 (13.35167) + | > current_lr: 0.00005 + | > step_time: 7.40670 (3.65876) + | > loader_time: 0.49080 (0.59685) + + + --> STEP: 39/234 -- GLOBAL_STEP: 47775 + | > loss: -0.20485 (-0.22691) + | > log_mle: -0.31866 (-0.32314) + | > loss_dur: 0.11381 (0.09623) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.14842 (13.13536) + | > current_lr: 0.00005 + | > step_time: 0.76710 (3.33971) + | > loader_time: 0.00170 (0.52074) + + + --> STEP: 44/234 -- GLOBAL_STEP: 47780 + | > loss: -0.22492 (-0.22440) + | > log_mle: -0.31192 (-0.32173) + | > loss_dur: 0.08700 (0.09733) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.19306 (12.75747) + | > current_lr: 0.00005 + | > step_time: 1.58980 (3.18923) + | > loader_time: 0.19360 (0.46645) + + + --> STEP: 49/234 -- GLOBAL_STEP: 47785 + | > loss: -0.22999 (-0.22283) + | > log_mle: -0.32283 (-0.32135) + | > loss_dur: 0.09285 (0.09852) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.61690 (12.47446) + | > current_lr: 0.00005 + | > step_time: 1.74030 (3.06730) + | > loader_time: 0.00170 (0.42267) + + + --> STEP: 54/234 -- GLOBAL_STEP: 47790 + | > loss: -0.22054 (-0.22079) + | > log_mle: -0.32129 (-0.32050) + | > loss_dur: 0.10075 (0.09971) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.13947 (12.25825) + | > current_lr: 0.00005 + | > step_time: 1.04660 (2.90263) + | > loader_time: 0.00170 (0.38707) + + + --> STEP: 59/234 -- GLOBAL_STEP: 47795 + | > loss: -0.22592 (-0.21977) + | > log_mle: -0.32696 (-0.32011) + | > loss_dur: 0.10104 (0.10034) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.82696 (12.05184) + | > current_lr: 0.00005 + | > step_time: 2.01250 (2.78702) + | > loader_time: 0.08710 (0.35849) + + + --> STEP: 64/234 -- GLOBAL_STEP: 47800 + | > loss: -0.21523 (-0.21768) + | > log_mle: -0.31282 (-0.32024) + | > loss_dur: 0.09759 (0.10256) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.40265 (12.14911) + | > current_lr: 0.00005 + | > step_time: 3.51010 (2.73672) + | > loader_time: 0.10110 (0.33489) + + + --> STEP: 69/234 -- GLOBAL_STEP: 47805 + | > loss: -0.18464 (-0.21596) + | > log_mle: -0.29629 (-0.31939) + | > loss_dur: 0.11164 (0.10343) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85448 (12.09531) + | > current_lr: 0.00005 + | > step_time: 1.15480 (2.65826) + | > loader_time: 0.00230 (0.31181) + + + --> STEP: 74/234 -- GLOBAL_STEP: 47810 + | > loss: -0.18044 (-0.21357) + | > log_mle: -0.29890 (-0.31896) + | > loss_dur: 0.11846 (0.10539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.74958 (12.39977) + | > current_lr: 0.00005 + | > step_time: 1.69460 (2.57569) + | > loader_time: 0.00260 (0.29453) + + + --> STEP: 79/234 -- GLOBAL_STEP: 47815 + | > loss: -0.18454 (-0.21198) + | > log_mle: -0.31594 (-0.31868) + | > loss_dur: 0.13140 (0.10669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.84257 (12.45442) + | > current_lr: 0.00005 + | > step_time: 1.09010 (2.59522) + | > loader_time: 0.00220 (0.27706) + + + --> STEP: 84/234 -- GLOBAL_STEP: 47820 + | > loss: -0.20247 (-0.21082) + | > log_mle: -0.31245 (-0.31845) + | > loss_dur: 0.10998 (0.10764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.56747 (12.49305) + | > current_lr: 0.00005 + | > step_time: 1.18250 (2.55458) + | > loader_time: 0.00210 (0.26276) + + + --> STEP: 89/234 -- GLOBAL_STEP: 47825 + | > loss: -0.21421 (-0.21017) + | > log_mle: -0.33900 (-0.31902) + | > loss_dur: 0.12479 (0.10885) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.05590 (12.81869) + | > current_lr: 0.00005 + | > step_time: 1.22830 (2.50280) + | > loader_time: 0.00130 (0.25017) + + + --> STEP: 94/234 -- GLOBAL_STEP: 47830 + | > loss: -0.22639 (-0.20998) + | > log_mle: -0.37185 (-0.32093) + | > loss_dur: 0.14545 (0.11094) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.46038 (13.30993) + | > current_lr: 0.00005 + | > step_time: 4.30440 (2.50144) + | > loader_time: 0.00750 (0.23704) + + + --> STEP: 99/234 -- GLOBAL_STEP: 47835 + | > loss: -0.22657 (-0.20987) + | > log_mle: -0.40466 (-0.32290) + | > loss_dur: 0.17809 (0.11303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.67033 (13.70995) + | > current_lr: 0.00005 + | > step_time: 1.67690 (2.48414) + | > loader_time: 0.00150 (0.22597) + + + --> STEP: 104/234 -- GLOBAL_STEP: 47840 + | > loss: -0.26326 (-0.21007) + | > log_mle: -0.41934 (-0.32543) + | > loss_dur: 0.15608 (0.11537) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.57284 (14.24825) + | > current_lr: 0.00005 + | > step_time: 1.17610 (2.48463) + | > loader_time: 0.00250 (0.21712) + + + --> STEP: 109/234 -- GLOBAL_STEP: 47845 + | > loss: -0.18651 (-0.20959) + | > log_mle: -0.39046 (-0.32716) + | > loss_dur: 0.20396 (0.11757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.83392 (14.63907) + | > current_lr: 0.00005 + | > step_time: 5.54560 (2.52334) + | > loader_time: 0.19330 (0.21140) + + + --> STEP: 114/234 -- GLOBAL_STEP: 47850 + | > loss: -0.21454 (-0.20993) + | > log_mle: -0.37135 (-0.32977) + | > loss_dur: 0.15681 (0.11985) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.70954 (15.30926) + | > current_lr: 0.00005 + | > step_time: 4.00000 (2.52893) + | > loader_time: 0.09120 (0.20380) + + + --> STEP: 119/234 -- GLOBAL_STEP: 47855 + | > loss: -0.21544 (-0.20961) + | > log_mle: -0.36731 (-0.33163) + | > loss_dur: 0.15186 (0.12202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.57018 (15.80912) + | > current_lr: 0.00005 + | > step_time: 1.45280 (2.49810) + | > loader_time: 0.00230 (0.19680) + + + --> STEP: 124/234 -- GLOBAL_STEP: 47860 + | > loss: -0.22965 (-0.20930) + | > log_mle: -0.39543 (-0.33280) + | > loss_dur: 0.16578 (0.12350) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.39399 (16.21186) + | > current_lr: 0.00005 + | > step_time: 1.40820 (2.50316) + | > loader_time: 0.00300 (0.18978) + + + --> STEP: 129/234 -- GLOBAL_STEP: 47865 + | > loss: -0.20821 (-0.20997) + | > log_mle: -0.38602 (-0.33568) + | > loss_dur: 0.17780 (0.12570) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.38321 (16.97539) + | > current_lr: 0.00005 + | > step_time: 3.71550 (2.49211) + | > loader_time: 0.00460 (0.18317) + + + --> STEP: 134/234 -- GLOBAL_STEP: 47870 + | > loss: -0.24581 (-0.21146) + | > log_mle: -0.44322 (-0.33923) + | > loss_dur: 0.19741 (0.12777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.77168 (17.93514) + | > current_lr: 0.00005 + | > step_time: 2.50590 (2.47511) + | > loader_time: 0.00460 (0.17776) + + + --> STEP: 139/234 -- GLOBAL_STEP: 47875 + | > loss: -0.30358 (-0.21254) + | > log_mle: -0.49404 (-0.34241) + | > loss_dur: 0.19046 (0.12986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.57654 (18.80449) + | > current_lr: 0.00005 + | > step_time: 1.21290 (2.45687) + | > loader_time: 0.08460 (0.17390) + + + --> STEP: 144/234 -- GLOBAL_STEP: 47880 + | > loss: -0.27346 (-0.21360) + | > log_mle: -0.47206 (-0.34589) + | > loss_dur: 0.19860 (0.13229) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.30987 (19.60088) + | > current_lr: 0.00005 + | > step_time: 4.90780 (2.48060) + | > loader_time: 0.08580 (0.16975) + + + --> STEP: 149/234 -- GLOBAL_STEP: 47885 + | > loss: -0.33138 (-0.21579) + | > log_mle: -0.52431 (-0.34994) + | > loss_dur: 0.19293 (0.13414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.67213 (20.39257) + | > current_lr: 0.00005 + | > step_time: 2.37870 (2.47504) + | > loader_time: 0.01740 (0.16435) + + + --> STEP: 154/234 -- GLOBAL_STEP: 47890 + | > loss: -0.28329 (-0.21815) + | > log_mle: -0.47740 (-0.35447) + | > loss_dur: 0.19411 (0.13632) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.91959 (21.51526) + | > current_lr: 0.00005 + | > step_time: 3.33130 (2.49176) + | > loader_time: 0.08450 (0.16145) + + + --> STEP: 159/234 -- GLOBAL_STEP: 47895 + | > loss: -0.29036 (-0.22035) + | > log_mle: -0.50329 (-0.35886) + | > loss_dur: 0.21293 (0.13852) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.54949 (22.66200) + | > current_lr: 0.00005 + | > step_time: 1.91380 (2.51120) + | > loader_time: 0.00440 (0.15815) + + + --> STEP: 164/234 -- GLOBAL_STEP: 47900 + | > loss: -0.28444 (-0.22275) + | > log_mle: -0.48641 (-0.36300) + | > loss_dur: 0.20197 (0.14025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.93680 (23.69485) + | > current_lr: 0.00005 + | > step_time: 1.89090 (2.51624) + | > loader_time: 0.00280 (0.15343) + + + --> STEP: 169/234 -- GLOBAL_STEP: 47905 + | > loss: -0.28263 (-0.22536) + | > log_mle: -0.49147 (-0.36745) + | > loss_dur: 0.20884 (0.14209) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.71109 (24.84298) + | > current_lr: 0.00005 + | > step_time: 2.49440 (2.49573) + | > loader_time: 0.00320 (0.14902) + + + --> STEP: 174/234 -- GLOBAL_STEP: 47910 + | > loss: -0.35531 (-0.22872) + | > log_mle: -0.57166 (-0.37298) + | > loss_dur: 0.21636 (0.14426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 101.22701 (26.36264) + | > current_lr: 0.00005 + | > step_time: 3.09800 (2.49894) + | > loader_time: 0.10100 (0.14696) + + + --> STEP: 179/234 -- GLOBAL_STEP: 47915 + | > loss: -0.32173 (-0.23126) + | > log_mle: -0.57375 (-0.37792) + | > loss_dur: 0.25202 (0.14667) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.75256 (27.63244) + | > current_lr: 0.00005 + | > step_time: 1.99900 (2.48435) + | > loader_time: 0.00290 (0.14297) + + + --> STEP: 184/234 -- GLOBAL_STEP: 47920 + | > loss: -0.32602 (-0.23376) + | > log_mle: -0.54045 (-0.38233) + | > loss_dur: 0.21442 (0.14858) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.24270 (28.77308) + | > current_lr: 0.00005 + | > step_time: 3.11240 (2.50759) + | > loader_time: 0.08930 (0.14019) + + + --> STEP: 189/234 -- GLOBAL_STEP: 47925 + | > loss: -0.31849 (-0.23645) + | > log_mle: -0.53471 (-0.38703) + | > loss_dur: 0.21622 (0.15058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.12086 (29.92570) + | > current_lr: 0.00005 + | > step_time: 5.29670 (2.59037) + | > loader_time: 0.00510 (0.13917) + + + --> STEP: 194/234 -- GLOBAL_STEP: 47930 + | > loss: -0.36863 (-0.23962) + | > log_mle: -0.57606 (-0.39177) + | > loss_dur: 0.20743 (0.15215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.43822 (30.91859) + | > current_lr: 0.00005 + | > step_time: 2.70190 (2.62980) + | > loader_time: 0.00630 (0.13613) + + + --> STEP: 199/234 -- GLOBAL_STEP: 47935 + | > loss: -0.35369 (-0.24221) + | > log_mle: -0.58359 (-0.39619) + | > loss_dur: 0.22990 (0.15398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.68552 (31.88911) + | > current_lr: 0.00005 + | > step_time: 12.71430 (2.77035) + | > loader_time: 0.00320 (0.13378) + + + --> STEP: 204/234 -- GLOBAL_STEP: 47940 + | > loss: -0.39120 (-0.24477) + | > log_mle: -0.62963 (-0.40061) + | > loss_dur: 0.23842 (0.15584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.43166 (32.79999) + | > current_lr: 0.00005 + | > step_time: 2.69140 (2.88613) + | > loader_time: 0.00430 (0.13594) + + + --> STEP: 209/234 -- GLOBAL_STEP: 47945 + | > loss: -0.35541 (-0.24769) + | > log_mle: -0.58052 (-0.40534) + | > loss_dur: 0.22511 (0.15765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.70559 (33.91587) + | > current_lr: 0.00005 + | > step_time: 8.20940 (2.95549) + | > loader_time: 0.09890 (0.13410) + + + --> STEP: 214/234 -- GLOBAL_STEP: 47950 + | > loss: -0.40192 (-0.25131) + | > log_mle: -0.61450 (-0.41079) + | > loss_dur: 0.21259 (0.15948) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.17220 (35.65252) + | > current_lr: 0.00005 + | > step_time: 7.50670 (3.00889) + | > loader_time: 0.09790 (0.13347) + + + --> STEP: 219/234 -- GLOBAL_STEP: 47955 + | > loss: -0.48010 (-0.25481) + | > log_mle: -0.71953 (-0.41615) + | > loss_dur: 0.23944 (0.16135) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.99468 (36.98504) + | > current_lr: 0.00005 + | > step_time: 1.89230 (3.06809) + | > loader_time: 0.00460 (0.13309) + + + --> STEP: 224/234 -- GLOBAL_STEP: 47960 + | > loss: -0.43590 (-0.25813) + | > log_mle: -0.67726 (-0.42127) + | > loss_dur: 0.24136 (0.16314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.02154 (38.09470) + | > current_lr: 0.00005 + | > step_time: 0.22210 (3.02164) + | > loader_time: 0.00260 (0.13124) + + + --> STEP: 229/234 -- GLOBAL_STEP: 47965 + | > loss: -0.39609 (-0.26138) + | > log_mle: -0.69814 (-0.42677) + | > loss_dur: 0.30205 (0.16538) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 112.42274 (39.62316) + | > current_lr: 0.00005 + | > step_time: 0.25030 (2.96083) + | > loader_time: 0.00300 (0.12845) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12852 (-0.15185) + | > avg_loss: -0.26827 (+0.02583) + | > avg_log_mle: -0.49151 (+0.01768) + | > avg_loss_dur: 0.22324 (+0.00816) + + + > EPOCH: 205/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 03:55:35)  + + --> STEP: 0/234 -- GLOBAL_STEP: 47970 + | > loss: -0.25692 (-0.25692) + | > log_mle: -0.40957 (-0.40957) + | > loss_dur: 0.15265 (0.15265) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.01797 (25.01797) + | > current_lr: 0.00005 + | > step_time: 7.49590 (7.49587) + | > loader_time: 8.86340 (8.86336) + + + --> STEP: 5/234 -- GLOBAL_STEP: 47975 + | > loss: -0.22512 (-0.21208) + | > log_mle: -0.32893 (-0.32613) + | > loss_dur: 0.10381 (0.11404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.02180 (24.07316) + | > current_lr: 0.00005 + | > step_time: 7.08950 (7.47715) + | > loader_time: 0.10480 (0.12124) + + + --> STEP: 10/234 -- GLOBAL_STEP: 47980 + | > loss: -0.21711 (-0.22266) + | > log_mle: -0.32425 (-0.32857) + | > loss_dur: 0.10714 (0.10592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.58459 (20.63432) + | > current_lr: 0.00005 + | > step_time: 0.70880 (5.65126) + | > loader_time: 0.09420 (0.09874) + + + --> STEP: 15/234 -- GLOBAL_STEP: 47985 + | > loss: -0.24155 (-0.22730) + | > log_mle: -0.33230 (-0.32962) + | > loss_dur: 0.09076 (0.10232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.69047 (18.31153) + | > current_lr: 0.00005 + | > step_time: 4.71630 (4.46903) + | > loader_time: 0.09530 (0.07759) + + + --> STEP: 20/234 -- GLOBAL_STEP: 47990 + | > loss: -0.24863 (-0.23020) + | > log_mle: -0.32976 (-0.32797) + | > loss_dur: 0.08113 (0.09776) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.27361 (16.42135) + | > current_lr: 0.00005 + | > step_time: 3.10460 (4.04669) + | > loader_time: 0.10000 (0.06351) + + + --> STEP: 25/234 -- GLOBAL_STEP: 47995 + | > loss: -0.22637 (-0.23188) + | > log_mle: -0.31172 (-0.32650) + | > loss_dur: 0.08535 (0.09462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.00782 (15.29146) + | > current_lr: 0.00005 + | > step_time: 3.50730 (3.58239) + | > loader_time: 0.00400 (0.05460) + + + --> STEP: 30/234 -- GLOBAL_STEP: 48000 + | > loss: -0.23410 (-0.23187) + | > log_mle: -0.32115 (-0.32560) + | > loss_dur: 0.08705 (0.09373) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.32547 (14.81555) + | > current_lr: 0.00005 + | > step_time: 1.70450 (3.70531) + | > loader_time: 0.00370 (0.05267) + + + --> STEP: 35/234 -- GLOBAL_STEP: 48005 + | > loss: -0.19509 (-0.22957) + | > log_mle: -0.31333 (-0.32442) + | > loss_dur: 0.11824 (0.09485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.60658 (14.40402) + | > current_lr: 0.00005 + | > step_time: 3.10810 (3.55059) + | > loader_time: 0.08580 (0.05067) + + + --> STEP: 40/234 -- GLOBAL_STEP: 48010 + | > loss: -0.17766 (-0.22655) + | > log_mle: -0.30003 (-0.32290) + | > loss_dur: 0.12237 (0.09635) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.55128 (14.29248) + | > current_lr: 0.00005 + | > step_time: 1.60880 (3.34438) + | > loader_time: 0.08670 (0.04686) + + + --> STEP: 45/234 -- GLOBAL_STEP: 48015 + | > loss: -0.21030 (-0.22520) + | > log_mle: -0.33446 (-0.32227) + | > loss_dur: 0.12416 (0.09707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.31897 (13.91705) + | > current_lr: 0.00005 + | > step_time: 1.09960 (3.14190) + | > loader_time: 0.00180 (0.04197) + + + --> STEP: 50/234 -- GLOBAL_STEP: 48020 + | > loss: -0.20491 (-0.22367) + | > log_mle: -0.30567 (-0.32131) + | > loss_dur: 0.10075 (0.09764) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.96866 (13.47377) + | > current_lr: 0.00005 + | > step_time: 1.20240 (3.02406) + | > loader_time: 0.00120 (0.03801) + + + --> STEP: 55/234 -- GLOBAL_STEP: 48025 + | > loss: -0.22885 (-0.22236) + | > log_mle: -0.32284 (-0.32068) + | > loss_dur: 0.09398 (0.09832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.56024 (13.18213) + | > current_lr: 0.00005 + | > step_time: 1.22370 (2.89971) + | > loader_time: 0.00200 (0.03651) + + + --> STEP: 60/234 -- GLOBAL_STEP: 48030 + | > loss: -0.20880 (-0.22106) + | > log_mle: -0.33361 (-0.32050) + | > loss_dur: 0.12481 (0.09944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.74896 (12.91846) + | > current_lr: 0.00005 + | > step_time: 1.46210 (2.79593) + | > loader_time: 0.08920 (0.03510) + + + --> STEP: 65/234 -- GLOBAL_STEP: 48035 + | > loss: -0.21138 (-0.21903) + | > log_mle: -0.31270 (-0.32024) + | > loss_dur: 0.10132 (0.10120) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.42129 (12.91166) + | > current_lr: 0.00005 + | > step_time: 2.32020 (2.74493) + | > loader_time: 0.07530 (0.03620) + + + --> STEP: 70/234 -- GLOBAL_STEP: 48040 + | > loss: -0.15854 (-0.21667) + | > log_mle: -0.29997 (-0.31926) + | > loss_dur: 0.14144 (0.10259) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.72214 (12.76226) + | > current_lr: 0.00005 + | > step_time: 1.41290 (2.66341) + | > loader_time: 0.08740 (0.03982) + + + --> STEP: 75/234 -- GLOBAL_STEP: 48045 + | > loss: -0.18163 (-0.21419) + | > log_mle: -0.31920 (-0.31906) + | > loss_dur: 0.13757 (0.10487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.78078 (12.86831) + | > current_lr: 0.00005 + | > step_time: 0.91510 (2.58341) + | > loader_time: 0.00240 (0.03991) + + + --> STEP: 80/234 -- GLOBAL_STEP: 48050 + | > loss: -0.19379 (-0.21277) + | > log_mle: -0.30406 (-0.31865) + | > loss_dur: 0.11027 (0.10588) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.36976 (12.72647) + | > current_lr: 0.00005 + | > step_time: 1.97900 (2.55564) + | > loader_time: 0.10820 (0.03983) + + + --> STEP: 85/234 -- GLOBAL_STEP: 48055 + | > loss: -0.19734 (-0.21159) + | > log_mle: -0.31516 (-0.31856) + | > loss_dur: 0.11782 (0.10697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.89433 (12.70106) + | > current_lr: 0.00005 + | > step_time: 1.91250 (2.51793) + | > loader_time: 0.00210 (0.03862) + + + --> STEP: 90/234 -- GLOBAL_STEP: 48060 + | > loss: -0.18960 (-0.21065) + | > log_mle: -0.33566 (-0.31944) + | > loss_dur: 0.14606 (0.10880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.54708 (12.92477) + | > current_lr: 0.00005 + | > step_time: 1.59880 (2.48325) + | > loader_time: 0.00190 (0.03660) + + + --> STEP: 95/234 -- GLOBAL_STEP: 48065 + | > loss: -0.24609 (-0.21109) + | > log_mle: -0.42317 (-0.32226) + | > loss_dur: 0.17708 (0.11116) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.43724 (13.58590) + | > current_lr: 0.00005 + | > step_time: 2.06040 (2.48063) + | > loader_time: 0.00190 (0.03865) + + + --> STEP: 100/234 -- GLOBAL_STEP: 48070 + | > loss: -0.21103 (-0.21081) + | > log_mle: -0.34881 (-0.32341) + | > loss_dur: 0.13778 (0.11260) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.56307 (13.95421) + | > current_lr: 0.00005 + | > step_time: 3.71110 (2.47077) + | > loader_time: 0.00610 (0.03771) + + + --> STEP: 105/234 -- GLOBAL_STEP: 48075 + | > loss: -0.18619 (-0.21098) + | > log_mle: -0.32751 (-0.32568) + | > loss_dur: 0.14131 (0.11470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.88472 (14.55744) + | > current_lr: 0.00005 + | > step_time: 2.79950 (2.44605) + | > loader_time: 0.00560 (0.03782) + + + --> STEP: 110/234 -- GLOBAL_STEP: 48080 + | > loss: -0.20535 (-0.21037) + | > log_mle: -0.35202 (-0.32745) + | > loss_dur: 0.14667 (0.11708) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.57159 (15.07650) + | > current_lr: 0.00005 + | > step_time: 1.50960 (2.45385) + | > loader_time: 0.08310 (0.03699) + + + --> STEP: 115/234 -- GLOBAL_STEP: 48085 + | > loss: -0.18965 (-0.21038) + | > log_mle: -0.36879 (-0.32993) + | > loss_dur: 0.17914 (0.11955) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.74633 (16.14477) + | > current_lr: 0.00005 + | > step_time: 4.10600 (2.44527) + | > loader_time: 0.00720 (0.03623) + + + --> STEP: 120/234 -- GLOBAL_STEP: 48090 + | > loss: -0.25290 (-0.21037) + | > log_mle: -0.42120 (-0.33215) + | > loss_dur: 0.16830 (0.12178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.22741 (16.53367) + | > current_lr: 0.00005 + | > step_time: 1.62560 (2.42276) + | > loader_time: 0.09260 (0.03786) + + + --> STEP: 125/234 -- GLOBAL_STEP: 48095 + | > loss: -0.22185 (-0.20980) + | > log_mle: -0.39988 (-0.33310) + | > loss_dur: 0.17803 (0.12330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.50748 (16.88120) + | > current_lr: 0.00005 + | > step_time: 1.28560 (2.44029) + | > loader_time: 0.00250 (0.03792) + + + --> STEP: 130/234 -- GLOBAL_STEP: 48100 + | > loss: -0.23080 (-0.21045) + | > log_mle: -0.42157 (-0.33604) + | > loss_dur: 0.19078 (0.12559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.24524 (17.57678) + | > current_lr: 0.00005 + | > step_time: 1.71580 (2.43972) + | > loader_time: 0.19240 (0.04012) + + + --> STEP: 135/234 -- GLOBAL_STEP: 48105 + | > loss: -0.19339 (-0.21167) + | > log_mle: -0.34615 (-0.33907) + | > loss_dur: 0.15276 (0.12740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.33932 (18.35465) + | > current_lr: 0.00005 + | > step_time: 4.11260 (2.45465) + | > loader_time: 0.08610 (0.04210) + + + --> STEP: 140/234 -- GLOBAL_STEP: 48110 + | > loss: -0.19548 (-0.21270) + | > log_mle: -0.38052 (-0.34249) + | > loss_dur: 0.18504 (0.12979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.06817 (19.27106) + | > current_lr: 0.00005 + | > step_time: 3.50190 (2.44392) + | > loader_time: 0.08970 (0.04134) + + + --> STEP: 145/234 -- GLOBAL_STEP: 48115 + | > loss: -0.29355 (-0.21448) + | > log_mle: -0.48400 (-0.34666) + | > loss_dur: 0.19045 (0.13219) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.54556 (20.10705) + | > current_lr: 0.00005 + | > step_time: 1.99940 (2.46247) + | > loader_time: 0.07950 (0.04176) + + + --> STEP: 150/234 -- GLOBAL_STEP: 48120 + | > loss: -0.26685 (-0.21651) + | > log_mle: -0.47323 (-0.35065) + | > loss_dur: 0.20637 (0.13414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.44871 (21.01337) + | > current_lr: 0.00005 + | > step_time: 2.09410 (2.45168) + | > loader_time: 0.00300 (0.04106) + + + --> STEP: 155/234 -- GLOBAL_STEP: 48125 + | > loss: -0.31888 (-0.21944) + | > log_mle: -0.52807 (-0.35561) + | > loss_dur: 0.20918 (0.13618) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.75674 (22.42538) + | > current_lr: 0.00005 + | > step_time: 7.38780 (2.59864) + | > loader_time: 0.08450 (0.04408) + + + --> STEP: 160/234 -- GLOBAL_STEP: 48130 + | > loss: -0.28968 (-0.22119) + | > log_mle: -0.50914 (-0.35954) + | > loss_dur: 0.21946 (0.13834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.34039 (23.52756) + | > current_lr: 0.00005 + | > step_time: 0.69870 (2.65044) + | > loader_time: 0.00230 (0.04528) + + + --> STEP: 165/234 -- GLOBAL_STEP: 48135 + | > loss: -0.31762 (-0.22338) + | > log_mle: -0.52723 (-0.36356) + | > loss_dur: 0.20961 (0.14018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.14997 (24.28277) + | > current_lr: 0.00005 + | > step_time: 3.91320 (2.65459) + | > loader_time: 0.09370 (0.04504) + + + --> STEP: 170/234 -- GLOBAL_STEP: 48140 + | > loss: -0.34836 (-0.22600) + | > log_mle: -0.56780 (-0.36819) + | > loss_dur: 0.21944 (0.14218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.64120 (25.42378) + | > current_lr: 0.00005 + | > step_time: 2.08930 (2.67178) + | > loader_time: 0.00380 (0.04539) + + + --> STEP: 175/234 -- GLOBAL_STEP: 48145 + | > loss: -0.31260 (-0.22926) + | > log_mle: -0.54689 (-0.37368) + | > loss_dur: 0.23428 (0.14442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.85580 (26.47252) + | > current_lr: 0.00005 + | > step_time: 11.90590 (2.74287) + | > loader_time: 0.09300 (0.04518) + + + --> STEP: 180/234 -- GLOBAL_STEP: 48150 + | > loss: -0.34005 (-0.23218) + | > log_mle: -0.55013 (-0.37881) + | > loss_dur: 0.21007 (0.14663) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.42975 (27.73765) + | > current_lr: 0.00005 + | > step_time: 3.41310 (2.75851) + | > loader_time: 0.08640 (0.04494) + + + --> STEP: 185/234 -- GLOBAL_STEP: 48155 + | > loss: -0.34035 (-0.23470) + | > log_mle: -0.57374 (-0.38346) + | > loss_dur: 0.23339 (0.14876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.02869 (29.04821) + | > current_lr: 0.00005 + | > step_time: 7.50480 (2.80345) + | > loader_time: 0.09470 (0.04534) + + + --> STEP: 190/234 -- GLOBAL_STEP: 48160 + | > loss: -0.34141 (-0.23727) + | > log_mle: -0.55051 (-0.38800) + | > loss_dur: 0.20909 (0.15072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.74810 (30.17914) + | > current_lr: 0.00005 + | > step_time: 5.50960 (2.85087) + | > loader_time: 0.09270 (0.04677) + + + --> STEP: 195/234 -- GLOBAL_STEP: 48165 + | > loss: -0.33666 (-0.24026) + | > log_mle: -0.56701 (-0.39267) + | > loss_dur: 0.23034 (0.15241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.84811 (31.31588) + | > current_lr: 0.00005 + | > step_time: 13.51060 (2.91989) + | > loader_time: 0.18510 (0.04807) + + + --> STEP: 200/234 -- GLOBAL_STEP: 48170 + | > loss: -0.32788 (-0.24282) + | > log_mle: -0.57565 (-0.39701) + | > loss_dur: 0.24776 (0.15419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.32334 (32.36607) + | > current_lr: 0.00005 + | > step_time: 2.00680 (2.91353) + | > loader_time: 0.08920 (0.05068) + + + --> STEP: 205/234 -- GLOBAL_STEP: 48175 + | > loss: -0.33125 (-0.24526) + | > log_mle: -0.56518 (-0.40121) + | > loss_dur: 0.23393 (0.15595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.44527 (33.30725) + | > current_lr: 0.00005 + | > step_time: 3.00300 (2.91661) + | > loader_time: 0.00520 (0.05043) + + + --> STEP: 210/234 -- GLOBAL_STEP: 48180 + | > loss: -0.40762 (-0.24858) + | > log_mle: -0.64750 (-0.40629) + | > loss_dur: 0.23988 (0.15771) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.62051 (34.68379) + | > current_lr: 0.00005 + | > step_time: 6.29900 (2.93200) + | > loader_time: 0.00300 (0.05019) + + + --> STEP: 215/234 -- GLOBAL_STEP: 48185 + | > loss: -0.36046 (-0.25208) + | > log_mle: -0.59241 (-0.41158) + | > loss_dur: 0.23195 (0.15950) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.51832 (36.06889) + | > current_lr: 0.00005 + | > step_time: 9.11370 (3.04496) + | > loader_time: 0.08360 (0.05292) + + + --> STEP: 220/234 -- GLOBAL_STEP: 48190 + | > loss: -0.39929 (-0.25572) + | > log_mle: -0.64332 (-0.41704) + | > loss_dur: 0.24404 (0.16132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.75772 (37.51406) + | > current_lr: 0.00005 + | > step_time: 2.10210 (3.06763) + | > loader_time: 0.00830 (0.05354) + + + --> STEP: 225/234 -- GLOBAL_STEP: 48195 + | > loss: -0.46995 (-0.25913) + | > log_mle: -0.71981 (-0.42228) + | > loss_dur: 0.24986 (0.16315) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.82765 (38.74054) + | > current_lr: 0.00005 + | > step_time: 0.24310 (3.02072) + | > loader_time: 0.00440 (0.05279) + + + --> STEP: 230/234 -- GLOBAL_STEP: 48200 + | > loss: -0.43209 (-0.26232) + | > log_mle: -0.75484 (-0.42802) + | > loss_dur: 0.32275 (0.16571) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 122.29430 (40.17208) + | > current_lr: 0.00005 + | > step_time: 0.25130 (2.96031) + | > loader_time: 0.00430 (0.05172) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07534 (-0.05319) + | > avg_loss: -0.28920 (-0.02093) + | > avg_log_mle: -0.51206 (-0.02054) + | > avg_loss_dur: 0.22286 (-0.00039) + + + > EPOCH: 206/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 04:08:14)  + + --> STEP: 1/234 -- GLOBAL_STEP: 48205 + | > loss: -0.21589 (-0.21589) + | > log_mle: -0.32715 (-0.32715) + | > loss_dur: 0.11126 (0.11126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.57585 (23.57585) + | > current_lr: 0.00005 + | > step_time: 7.49930 (7.49930) + | > loader_time: 1.89440 (1.89439) + + + --> STEP: 6/234 -- GLOBAL_STEP: 48210 + | > loss: -0.25341 (-0.22113) + | > log_mle: -0.32402 (-0.32851) + | > loss_dur: 0.07061 (0.10738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.81532 (19.28929) + | > current_lr: 0.00005 + | > step_time: 13.13020 (7.20388) + | > loader_time: 0.10130 (0.39938) + + + --> STEP: 11/234 -- GLOBAL_STEP: 48215 + | > loss: -0.25319 (-0.22902) + | > log_mle: -0.33141 (-0.33183) + | > loss_dur: 0.07822 (0.10282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.34857 (18.08010) + | > current_lr: 0.00005 + | > step_time: 1.39170 (4.73719) + | > loader_time: 0.00150 (0.24204) + + + --> STEP: 16/234 -- GLOBAL_STEP: 48220 + | > loss: -0.24188 (-0.23200) + | > log_mle: -0.33016 (-0.33240) + | > loss_dur: 0.08827 (0.10040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85086 (16.90231) + | > current_lr: 0.00005 + | > step_time: 3.30060 (4.05029) + | > loader_time: 0.00210 (0.17836) + + + --> STEP: 21/234 -- GLOBAL_STEP: 48225 + | > loss: -0.22354 (-0.23210) + | > log_mle: -0.30996 (-0.32953) + | > loss_dur: 0.08642 (0.09742) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.33358 (15.59263) + | > current_lr: 0.00005 + | > step_time: 3.00670 (4.21002) + | > loader_time: 0.10190 (0.16492) + + + --> STEP: 26/234 -- GLOBAL_STEP: 48230 + | > loss: -0.22923 (-0.23365) + | > log_mle: -0.32193 (-0.32867) + | > loss_dur: 0.09270 (0.09501) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.72426 (14.41788) + | > current_lr: 0.00005 + | > step_time: 5.70660 (4.38959) + | > loader_time: 0.09220 (0.15467) + + + --> STEP: 31/234 -- GLOBAL_STEP: 48235 + | > loss: -0.19399 (-0.23260) + | > log_mle: -0.31870 (-0.32768) + | > loss_dur: 0.12471 (0.09508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.52548 (13.95201) + | > current_lr: 0.00005 + | > step_time: 5.02140 (4.31468) + | > loader_time: 0.00140 (0.13943) + + + --> STEP: 36/234 -- GLOBAL_STEP: 48240 + | > loss: -0.21929 (-0.23082) + | > log_mle: -0.31375 (-0.32657) + | > loss_dur: 0.09446 (0.09575) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.72891 (13.58492) + | > current_lr: 0.00005 + | > step_time: 2.69160 (4.04568) + | > loader_time: 0.00170 (0.12521) + + + --> STEP: 41/234 -- GLOBAL_STEP: 48245 + | > loss: -0.23545 (-0.22899) + | > log_mle: -0.32055 (-0.32542) + | > loss_dur: 0.08509 (0.09643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.43017 (13.18727) + | > current_lr: 0.00005 + | > step_time: 2.30050 (3.90623) + | > loader_time: 0.00890 (0.11480) + + + --> STEP: 46/234 -- GLOBAL_STEP: 48250 + | > loss: -0.20244 (-0.22707) + | > log_mle: -0.31313 (-0.32464) + | > loss_dur: 0.11069 (0.09757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.08723 (13.10448) + | > current_lr: 0.00005 + | > step_time: 2.71830 (3.90543) + | > loader_time: 0.17180 (0.10880) + + + --> STEP: 51/234 -- GLOBAL_STEP: 48255 + | > loss: -0.20169 (-0.22602) + | > log_mle: -0.30438 (-0.32356) + | > loss_dur: 0.10269 (0.09753) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.88404 (12.76202) + | > current_lr: 0.00005 + | > step_time: 1.51230 (3.65369) + | > loader_time: 0.00260 (0.10198) + + + --> STEP: 56/234 -- GLOBAL_STEP: 48260 + | > loss: -0.21129 (-0.22467) + | > log_mle: -0.32122 (-0.32316) + | > loss_dur: 0.10993 (0.09849) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.29645 (12.63426) + | > current_lr: 0.00005 + | > step_time: 0.89960 (3.41061) + | > loader_time: 0.00460 (0.09307) + + + --> STEP: 61/234 -- GLOBAL_STEP: 48265 + | > loss: -0.20617 (-0.22278) + | > log_mle: -0.31156 (-0.32245) + | > loss_dur: 0.10540 (0.09966) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.04677 (12.79074) + | > current_lr: 0.00005 + | > step_time: 1.90000 (3.30010) + | > loader_time: 0.00220 (0.08821) + + + --> STEP: 66/234 -- GLOBAL_STEP: 48270 + | > loss: -0.21074 (-0.22062) + | > log_mle: -0.30568 (-0.32188) + | > loss_dur: 0.09493 (0.10126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.84939 (12.75260) + | > current_lr: 0.00005 + | > step_time: 2.31450 (3.17515) + | > loader_time: 0.08640 (0.08295) + + + --> STEP: 71/234 -- GLOBAL_STEP: 48275 + | > loss: -0.17792 (-0.21753) + | > log_mle: -0.33079 (-0.32114) + | > loss_dur: 0.15287 (0.10361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.54519 (12.80230) + | > current_lr: 0.00005 + | > step_time: 1.29100 (3.07749) + | > loader_time: 0.00270 (0.07728) + + + --> STEP: 76/234 -- GLOBAL_STEP: 48280 + | > loss: -0.19333 (-0.21490) + | > log_mle: -0.31533 (-0.32042) + | > loss_dur: 0.12200 (0.10552) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.78512 (12.85907) + | > current_lr: 0.00005 + | > step_time: 1.73670 (3.01244) + | > loader_time: 0.00280 (0.07343) + + + --> STEP: 81/234 -- GLOBAL_STEP: 48285 + | > loss: -0.20475 (-0.21332) + | > log_mle: -0.32952 (-0.31988) + | > loss_dur: 0.12477 (0.10656) + | > amp_scaler: 4096.00000 (2123.85185) + | > grad_norm: 14.01009 (12.79980) + | > current_lr: 0.00005 + | > step_time: 1.10840 (2.92752) + | > loader_time: 0.00260 (0.07003) + + + --> STEP: 86/234 -- GLOBAL_STEP: 48290 + | > loss: -0.19181 (-0.21166) + | > log_mle: -0.32220 (-0.31957) + | > loss_dur: 0.13039 (0.10792) + | > amp_scaler: 4096.00000 (2238.51163) + | > grad_norm: 16.05664 (12.85452) + | > current_lr: 0.00005 + | > step_time: 2.48060 (2.88166) + | > loader_time: 0.00900 (0.06718) + + + --> STEP: 91/234 -- GLOBAL_STEP: 48295 + | > loss: -0.19130 (-0.21079) + | > log_mle: -0.33698 (-0.32045) + | > loss_dur: 0.14567 (0.10966) + | > amp_scaler: 4096.00000 (2340.57143) + | > grad_norm: 14.67027 (13.06235) + | > current_lr: 0.00005 + | > step_time: 1.70300 (2.85216) + | > loader_time: 0.08340 (0.06751) + + + --> STEP: 96/234 -- GLOBAL_STEP: 48300 + | > loss: -0.19630 (-0.21124) + | > log_mle: -0.32045 (-0.32287) + | > loss_dur: 0.12415 (0.11163) + | > amp_scaler: 4096.00000 (2432.00000) + | > grad_norm: 13.87628 (13.71192) + | > current_lr: 0.00005 + | > step_time: 1.31130 (2.81112) + | > loader_time: 0.00280 (0.06605) + + + --> STEP: 101/234 -- GLOBAL_STEP: 48305 + | > loss: -0.19721 (-0.21089) + | > log_mle: -0.37439 (-0.32450) + | > loss_dur: 0.17717 (0.11360) + | > amp_scaler: 4096.00000 (2514.37624) + | > grad_norm: 32.91339 (14.15261) + | > current_lr: 0.00005 + | > step_time: 1.50330 (2.77701) + | > loader_time: 0.00240 (0.06363) + + + --> STEP: 106/234 -- GLOBAL_STEP: 48310 + | > loss: -0.20342 (-0.21084) + | > log_mle: -0.37727 (-0.32672) + | > loss_dur: 0.17384 (0.11588) + | > amp_scaler: 4096.00000 (2588.98113) + | > grad_norm: 27.82295 (14.74161) + | > current_lr: 0.00005 + | > step_time: 1.91290 (2.77652) + | > loader_time: 0.09360 (0.06224) + + + --> STEP: 111/234 -- GLOBAL_STEP: 48315 + | > loss: -0.23921 (-0.21079) + | > log_mle: -0.42766 (-0.32906) + | > loss_dur: 0.18845 (0.11827) + | > amp_scaler: 4096.00000 (2656.86486) + | > grad_norm: 46.28737 (15.34653) + | > current_lr: 0.00005 + | > step_time: 2.19250 (2.77504) + | > loader_time: 0.00320 (0.06124) + + + --> STEP: 116/234 -- GLOBAL_STEP: 48320 + | > loss: -0.21253 (-0.21087) + | > log_mle: -0.39475 (-0.33144) + | > loss_dur: 0.18221 (0.12057) + | > amp_scaler: 4096.00000 (2718.89655) + | > grad_norm: 29.00480 (15.87766) + | > current_lr: 0.00005 + | > step_time: 4.11310 (2.75232) + | > loader_time: 0.00360 (0.06015) + + + --> STEP: 121/234 -- GLOBAL_STEP: 48325 + | > loss: -0.16871 (-0.21073) + | > log_mle: -0.30544 (-0.33304) + | > loss_dur: 0.13673 (0.12231) + | > amp_scaler: 4096.00000 (2775.80165) + | > grad_norm: 17.09420 (16.27542) + | > current_lr: 0.00005 + | > step_time: 1.21220 (2.70230) + | > loader_time: 0.09680 (0.05991) + + + --> STEP: 126/234 -- GLOBAL_STEP: 48330 + | > loss: -0.25351 (-0.21091) + | > log_mle: -0.43732 (-0.33512) + | > loss_dur: 0.18381 (0.12421) + | > amp_scaler: 4096.00000 (2828.19048) + | > grad_norm: 55.16021 (16.98594) + | > current_lr: 0.00005 + | > step_time: 2.89180 (2.69739) + | > loader_time: 0.00190 (0.05975) + + + --> STEP: 131/234 -- GLOBAL_STEP: 48335 + | > loss: -0.29507 (-0.21210) + | > log_mle: -0.48627 (-0.33839) + | > loss_dur: 0.19121 (0.12629) + | > amp_scaler: 4096.00000 (2876.58015) + | > grad_norm: 46.95826 (17.72691) + | > current_lr: 0.00005 + | > step_time: 3.51090 (2.68460) + | > loader_time: 0.09410 (0.05951) + + + --> STEP: 136/234 -- GLOBAL_STEP: 48340 + | > loss: -0.31041 (-0.21334) + | > log_mle: -0.53379 (-0.34163) + | > loss_dur: 0.22338 (0.12829) + | > amp_scaler: 4096.00000 (2921.41176) + | > grad_norm: 63.66843 (18.62409) + | > current_lr: 0.00005 + | > step_time: 4.18970 (2.76310) + | > loader_time: 0.00810 (0.06032) + + + --> STEP: 141/234 -- GLOBAL_STEP: 48345 + | > loss: -0.25738 (-0.21404) + | > log_mle: -0.43788 (-0.34442) + | > loss_dur: 0.18050 (0.13037) + | > amp_scaler: 4096.00000 (2963.06383) + | > grad_norm: 34.00073 (19.37385) + | > current_lr: 0.00005 + | > step_time: 5.42450 (2.78367) + | > loader_time: 0.38550 (0.06276) + + + --> STEP: 146/234 -- GLOBAL_STEP: 48350 + | > loss: -0.25573 (-0.21468) + | > log_mle: -0.45482 (-0.34774) + | > loss_dur: 0.19909 (0.13306) + | > amp_scaler: 4096.00000 (3001.86301) + | > grad_norm: 43.63121 (20.94052) + | > current_lr: 0.00005 + | > step_time: 2.89840 (2.75770) + | > loader_time: 0.00410 (0.06116) + + + --> STEP: 151/234 -- GLOBAL_STEP: 48355 + | > loss: -0.25568 (-0.21608) + | > log_mle: -0.43733 (-0.35086) + | > loss_dur: 0.18165 (0.13477) + | > amp_scaler: 4096.00000 (3038.09272) + | > grad_norm: 45.62696 (21.56241) + | > current_lr: 0.00005 + | > step_time: 2.10400 (2.78119) + | > loader_time: 0.09370 (0.06213) + + + --> STEP: 156/234 -- GLOBAL_STEP: 48360 + | > loss: -0.29868 (-0.21882) + | > log_mle: -0.49412 (-0.35576) + | > loss_dur: 0.19544 (0.13694) + | > amp_scaler: 4096.00000 (3072.00000) + | > grad_norm: 44.16348 (22.56296) + | > current_lr: 0.00005 + | > step_time: 5.29050 (2.82151) + | > loader_time: 0.00680 (0.06208) + + + --> STEP: 161/234 -- GLOBAL_STEP: 48365 + | > loss: -0.32468 (-0.22099) + | > log_mle: -0.51725 (-0.36007) + | > loss_dur: 0.19257 (0.13907) + | > amp_scaler: 4096.00000 (3103.80124) + | > grad_norm: 51.12380 (23.36856) + | > current_lr: 0.00005 + | > step_time: 1.99460 (2.83725) + | > loader_time: 0.00610 (0.06243) + + + --> STEP: 166/234 -- GLOBAL_STEP: 48370 + | > loss: -0.26548 (-0.22279) + | > log_mle: -0.45268 (-0.36371) + | > loss_dur: 0.18720 (0.14092) + | > amp_scaler: 4096.00000 (3133.68675) + | > grad_norm: 41.65717 (24.36146) + | > current_lr: 0.00005 + | > step_time: 3.69850 (2.81408) + | > loader_time: 0.00350 (0.06163) + + + --> STEP: 171/234 -- GLOBAL_STEP: 48375 + | > loss: -0.35427 (-0.22591) + | > log_mle: -0.56052 (-0.36893) + | > loss_dur: 0.20625 (0.14302) + | > amp_scaler: 4096.00000 (3161.82456) + | > grad_norm: 74.42744 (25.67344) + | > current_lr: 0.00005 + | > step_time: 9.88430 (2.87954) + | > loader_time: 0.00190 (0.06045) + + + --> STEP: 176/234 -- GLOBAL_STEP: 48380 + | > loss: -0.33583 (-0.22902) + | > log_mle: -0.54780 (-0.37419) + | > loss_dur: 0.21197 (0.14517) + | > amp_scaler: 4096.00000 (3188.36364) + | > grad_norm: 54.25559 (26.84653) + | > current_lr: 0.00005 + | > step_time: 3.60820 (2.88807) + | > loader_time: 0.08560 (0.05986) + + + --> STEP: 181/234 -- GLOBAL_STEP: 48385 + | > loss: -0.26798 (-0.23141) + | > log_mle: -0.47896 (-0.37870) + | > loss_dur: 0.21098 (0.14729) + | > amp_scaler: 2048.00000 (3179.49171) + | > grad_norm: 41.61753 (27.72523) + | > current_lr: 0.00005 + | > step_time: 2.99980 (2.89005) + | > loader_time: 0.00340 (0.06149) + + + --> STEP: 186/234 -- GLOBAL_STEP: 48390 + | > loss: -0.25829 (-0.23367) + | > log_mle: -0.50370 (-0.38318) + | > loss_dur: 0.24541 (0.14952) + | > amp_scaler: 2048.00000 (3149.07527) + | > grad_norm: 60.40764 (29.10684) + | > current_lr: 0.00005 + | > step_time: 8.49970 (2.94228) + | > loader_time: 0.09360 (0.06090) + + + --> STEP: 191/234 -- GLOBAL_STEP: 48395 + | > loss: -0.31702 (-0.23634) + | > log_mle: -0.52829 (-0.38759) + | > loss_dur: 0.21127 (0.15125) + | > amp_scaler: 2048.00000 (3120.25131) + | > grad_norm: 62.07495 (29.98295) + | > current_lr: 0.00005 + | > step_time: 3.43990 (3.03721) + | > loader_time: 0.09720 (0.06345) + + + --> STEP: 196/234 -- GLOBAL_STEP: 48400 + | > loss: -0.30330 (-0.23937) + | > log_mle: -0.53109 (-0.39231) + | > loss_dur: 0.22779 (0.15294) + | > amp_scaler: 2048.00000 (3092.89796) + | > grad_norm: 56.81766 (31.00764) + | > current_lr: 0.00005 + | > step_time: 7.50470 (3.09183) + | > loader_time: 0.09060 (0.06273) + + + --> STEP: 201/234 -- GLOBAL_STEP: 48405 + | > loss: -0.25271 (-0.24172) + | > log_mle: -0.48386 (-0.39650) + | > loss_dur: 0.23116 (0.15477) + | > amp_scaler: 2048.00000 (3066.90547) + | > grad_norm: 56.01092 (31.96154) + | > current_lr: 0.00005 + | > step_time: 6.58920 (3.12277) + | > loader_time: 0.00210 (0.06509) + + + --> STEP: 206/234 -- GLOBAL_STEP: 48410 + | > loss: -0.37435 (-0.24471) + | > log_mle: -0.60207 (-0.40123) + | > loss_dur: 0.22772 (0.15652) + | > amp_scaler: 2048.00000 (3042.17476) + | > grad_norm: 66.13232 (32.87581) + | > current_lr: 0.00005 + | > step_time: 2.36930 (3.15851) + | > loader_time: 0.01900 (0.06367) + + + --> STEP: 211/234 -- GLOBAL_STEP: 48415 + | > loss: -0.43326 (-0.24826) + | > log_mle: -0.67809 (-0.40666) + | > loss_dur: 0.24483 (0.15840) + | > amp_scaler: 2048.00000 (3018.61611) + | > grad_norm: 112.32756 (34.26012) + | > current_lr: 0.00005 + | > step_time: 5.39720 (3.22476) + | > loader_time: 0.29990 (0.06554) + + + --> STEP: 216/234 -- GLOBAL_STEP: 48420 + | > loss: -0.40045 (-0.25165) + | > log_mle: -0.65630 (-0.41187) + | > loss_dur: 0.25585 (0.16021) + | > amp_scaler: 2048.00000 (2996.14815) + | > grad_norm: 134.25366 (35.79885) + | > current_lr: 0.00005 + | > step_time: 13.39340 (3.36624) + | > loader_time: 0.09880 (0.06813) + + + --> STEP: 221/234 -- GLOBAL_STEP: 48425 + | > loss: -0.36912 (-0.25521) + | > log_mle: -0.58676 (-0.41713) + | > loss_dur: 0.21764 (0.16192) + | > amp_scaler: 2048.00000 (2974.69683) + | > grad_norm: 66.29385 (36.92863) + | > current_lr: 0.00005 + | > step_time: 1.79750 (3.34383) + | > loader_time: 0.07680 (0.06783) + + + --> STEP: 226/234 -- GLOBAL_STEP: 48430 + | > loss: -0.43940 (-0.25906) + | > log_mle: -0.68512 (-0.42287) + | > loss_dur: 0.24572 (0.16381) + | > amp_scaler: 2048.00000 (2954.19469) + | > grad_norm: 101.50042 (38.31350) + | > current_lr: 0.00005 + | > step_time: 0.23400 (3.28100) + | > loader_time: 0.00270 (0.06641) + + + --> STEP: 231/234 -- GLOBAL_STEP: 48435 + | > loss: -0.36420 (-0.26199) + | > log_mle: -0.74623 (-0.42888) + | > loss_dur: 0.38203 (0.16689) + | > amp_scaler: 2048.00000 (2934.58009) + | > grad_norm: 168.51894 (40.06461) + | > current_lr: 0.00005 + | > step_time: 0.26720 (3.21547) + | > loader_time: 0.00370 (0.06505) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.56054 (+0.48520) + | > avg_loss: -0.26876 (+0.02043) + | > avg_log_mle: -0.49984 (+0.01221) + | > avg_loss_dur: 0.23108 (+0.00822) + + + > EPOCH: 207/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 04:21:59)  + + --> STEP: 2/234 -- GLOBAL_STEP: 48440 + | > loss: -0.24363 (-0.23592) + | > log_mle: -0.34203 (-0.33583) + | > loss_dur: 0.09840 (0.09992) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.52865 (18.49918) + | > current_lr: 0.00005 + | > step_time: 1.81440 (1.95338) + | > loader_time: 0.08510 (0.04485) + + + --> STEP: 7/234 -- GLOBAL_STEP: 48445 + | > loss: -0.24490 (-0.22759) + | > log_mle: -0.33779 (-0.33242) + | > loss_dur: 0.09289 (0.10482) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.72909 (17.25164) + | > current_lr: 0.00005 + | > step_time: 8.79910 (5.39919) + | > loader_time: 0.00310 (0.08174) + + + --> STEP: 12/234 -- GLOBAL_STEP: 48450 + | > loss: -0.22301 (-0.23094) + | > log_mle: -0.33037 (-0.33425) + | > loss_dur: 0.10736 (0.10332) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.85358 (16.48500) + | > current_lr: 0.00005 + | > step_time: 1.24060 (4.27302) + | > loader_time: 0.00170 (0.07368) + + + --> STEP: 17/234 -- GLOBAL_STEP: 48455 + | > loss: -0.24666 (-0.23565) + | > log_mle: -0.31680 (-0.33368) + | > loss_dur: 0.07014 (0.09803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.70746 (15.23031) + | > current_lr: 0.00005 + | > step_time: 2.39170 (3.62232) + | > loader_time: 0.10770 (0.06485) + + + --> STEP: 22/234 -- GLOBAL_STEP: 48460 + | > loss: -0.24481 (-0.23640) + | > log_mle: -0.33168 (-0.33165) + | > loss_dur: 0.08688 (0.09524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.29223 (14.60153) + | > current_lr: 0.00005 + | > step_time: 1.38020 (3.39209) + | > loader_time: 0.00110 (0.05516) + + + --> STEP: 27/234 -- GLOBAL_STEP: 48465 + | > loss: -0.23091 (-0.23724) + | > log_mle: -0.32442 (-0.33053) + | > loss_dur: 0.09351 (0.09329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.86909 (14.08208) + | > current_lr: 0.00005 + | > step_time: 2.20080 (3.28661) + | > loader_time: 0.00490 (0.05928) + + + --> STEP: 32/234 -- GLOBAL_STEP: 48470 + | > loss: -0.23228 (-0.23474) + | > log_mle: -0.32214 (-0.32870) + | > loss_dur: 0.08986 (0.09396) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.92778 (13.74537) + | > current_lr: 0.00005 + | > step_time: 4.00950 (3.30815) + | > loader_time: 0.08540 (0.05974) + + + --> STEP: 37/234 -- GLOBAL_STEP: 48475 + | > loss: -0.21873 (-0.23102) + | > log_mle: -0.30691 (-0.32627) + | > loss_dur: 0.08818 (0.09526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.63082 (13.54943) + | > current_lr: 0.00005 + | > step_time: 1.25770 (3.53020) + | > loader_time: 0.00180 (0.06126) + + + --> STEP: 42/234 -- GLOBAL_STEP: 48480 + | > loss: -0.19751 (-0.22858) + | > log_mle: -0.30110 (-0.32460) + | > loss_dur: 0.10359 (0.09603) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.51332 (13.24359) + | > current_lr: 0.00005 + | > step_time: 1.18270 (3.25664) + | > loader_time: 0.00120 (0.05422) + + + --> STEP: 47/234 -- GLOBAL_STEP: 48485 + | > loss: -0.20188 (-0.22644) + | > log_mle: -0.31011 (-0.32376) + | > loss_dur: 0.10822 (0.09732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17896 (13.30293) + | > current_lr: 0.00005 + | > step_time: 2.66010 (3.09919) + | > loader_time: 0.00180 (0.04870) + + + --> STEP: 52/234 -- GLOBAL_STEP: 48490 + | > loss: -0.19104 (-0.22493) + | > log_mle: -0.30925 (-0.32256) + | > loss_dur: 0.11821 (0.09763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.84292 (12.95150) + | > current_lr: 0.00005 + | > step_time: 1.69820 (2.99469) + | > loader_time: 0.00190 (0.04435) + + + --> STEP: 57/234 -- GLOBAL_STEP: 48495 + | > loss: -0.18726 (-0.22342) + | > log_mle: -0.29853 (-0.32196) + | > loss_dur: 0.11126 (0.09854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.85676 (12.81666) + | > current_lr: 0.00005 + | > step_time: 1.88780 (2.87371) + | > loader_time: 0.00240 (0.04064) + + + --> STEP: 62/234 -- GLOBAL_STEP: 48500 + | > loss: -0.15401 (-0.22157) + | > log_mle: -0.33839 (-0.32216) + | > loss_dur: 0.18438 (0.10058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.77617 (13.03064) + | > current_lr: 0.00005 + | > step_time: 1.04990 (2.76898) + | > loader_time: 0.00180 (0.03885) + + + --> STEP: 67/234 -- GLOBAL_STEP: 48505 + | > loss: -0.19897 (-0.22038) + | > log_mle: -0.32575 (-0.32162) + | > loss_dur: 0.12678 (0.10124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.24380 (12.79719) + | > current_lr: 0.00005 + | > step_time: 2.10560 (2.67873) + | > loader_time: 0.00180 (0.03735) + + + --> STEP: 72/234 -- GLOBAL_STEP: 48510 + | > loss: -0.19952 (-0.21760) + | > log_mle: -0.31258 (-0.32091) + | > loss_dur: 0.11306 (0.10331) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.27006 (12.75287) + | > current_lr: 0.00005 + | > step_time: 1.29830 (2.60189) + | > loader_time: 0.00320 (0.03491) + + + --> STEP: 77/234 -- GLOBAL_STEP: 48515 + | > loss: -0.20891 (-0.21590) + | > log_mle: -0.31858 (-0.32070) + | > loss_dur: 0.10967 (0.10480) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.30725 (12.69424) + | > current_lr: 0.00005 + | > step_time: 2.93760 (2.57025) + | > loader_time: 0.00250 (0.03382) + + + --> STEP: 82/234 -- GLOBAL_STEP: 48520 + | > loss: -0.19936 (-0.21465) + | > log_mle: -0.31289 (-0.32030) + | > loss_dur: 0.11353 (0.10565) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.60343 (12.59498) + | > current_lr: 0.00005 + | > step_time: 1.77510 (2.54423) + | > loader_time: 0.00940 (0.03335) + + + --> STEP: 87/234 -- GLOBAL_STEP: 48525 + | > loss: -0.17827 (-0.21295) + | > log_mle: -0.31378 (-0.32010) + | > loss_dur: 0.13551 (0.10715) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.36700 (12.68648) + | > current_lr: 0.00005 + | > step_time: 2.90870 (2.56726) + | > loader_time: 0.00740 (0.03474) + + + --> STEP: 92/234 -- GLOBAL_STEP: 48530 + | > loss: -0.22086 (-0.21246) + | > log_mle: -0.36062 (-0.32152) + | > loss_dur: 0.13977 (0.10907) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.78536 (12.97822) + | > current_lr: 0.00005 + | > step_time: 3.06700 (2.53084) + | > loader_time: 0.01360 (0.03405) + + + --> STEP: 97/234 -- GLOBAL_STEP: 48535 + | > loss: -0.19577 (-0.21245) + | > log_mle: -0.34752 (-0.32388) + | > loss_dur: 0.15176 (0.11144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.47901 (13.52571) + | > current_lr: 0.00005 + | > step_time: 1.69980 (2.49999) + | > loader_time: 0.00260 (0.03330) + + + --> STEP: 102/234 -- GLOBAL_STEP: 48540 + | > loss: -0.18615 (-0.21190) + | > log_mle: -0.33074 (-0.32536) + | > loss_dur: 0.14458 (0.11346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.04375 (14.03913) + | > current_lr: 0.00005 + | > step_time: 1.22850 (2.48008) + | > loader_time: 0.00150 (0.03496) + + + --> STEP: 107/234 -- GLOBAL_STEP: 48545 + | > loss: -0.20292 (-0.21192) + | > log_mle: -0.37004 (-0.32778) + | > loss_dur: 0.16712 (0.11586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.15092 (14.83241) + | > current_lr: 0.00005 + | > step_time: 1.11290 (2.43207) + | > loader_time: 0.08670 (0.03423) + + + --> STEP: 112/234 -- GLOBAL_STEP: 48550 + | > loss: -0.20844 (-0.21146) + | > log_mle: -0.38564 (-0.32999) + | > loss_dur: 0.17719 (0.11854) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.81304 (15.43116) + | > current_lr: 0.00005 + | > step_time: 1.69670 (2.40029) + | > loader_time: 0.00230 (0.03445) + + + --> STEP: 117/234 -- GLOBAL_STEP: 48555 + | > loss: -0.21381 (-0.21126) + | > log_mle: -0.37611 (-0.33211) + | > loss_dur: 0.16230 (0.12085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.35017 (15.89743) + | > current_lr: 0.00005 + | > step_time: 3.39460 (2.41472) + | > loader_time: 0.00850 (0.03539) + + + --> STEP: 122/234 -- GLOBAL_STEP: 48560 + | > loss: -0.19447 (-0.21093) + | > log_mle: -0.35081 (-0.33329) + | > loss_dur: 0.15634 (0.12236) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.18181 (16.11746) + | > current_lr: 0.00005 + | > step_time: 2.40680 (2.40110) + | > loader_time: 0.09660 (0.03546) + + + --> STEP: 127/234 -- GLOBAL_STEP: 48565 + | > loss: -0.23399 (-0.21151) + | > log_mle: -0.41742 (-0.33583) + | > loss_dur: 0.18344 (0.12431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.15563 (16.78987) + | > current_lr: 0.00005 + | > step_time: 1.45240 (2.41162) + | > loader_time: 0.00270 (0.03428) + + + --> STEP: 132/234 -- GLOBAL_STEP: 48570 + | > loss: -0.23419 (-0.21277) + | > log_mle: -0.39723 (-0.33892) + | > loss_dur: 0.16304 (0.12614) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.38905 (17.55855) + | > current_lr: 0.00005 + | > step_time: 1.61240 (2.38231) + | > loader_time: 0.07430 (0.03412) + + + --> STEP: 137/234 -- GLOBAL_STEP: 48575 + | > loss: -0.21241 (-0.21388) + | > log_mle: -0.40718 (-0.34218) + | > loss_dur: 0.19478 (0.12829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.21398 (18.39596) + | > current_lr: 0.00005 + | > step_time: 3.60490 (2.38147) + | > loader_time: 0.09470 (0.03486) + + + --> STEP: 142/234 -- GLOBAL_STEP: 48580 + | > loss: -0.22960 (-0.21462) + | > log_mle: -0.41801 (-0.34488) + | > loss_dur: 0.18840 (0.13026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.15831 (19.12770) + | > current_lr: 0.00005 + | > step_time: 3.80040 (2.39047) + | > loader_time: 0.19840 (0.03512) + + + --> STEP: 147/234 -- GLOBAL_STEP: 48585 + | > loss: -0.23841 (-0.21674) + | > log_mle: -0.42471 (-0.34946) + | > loss_dur: 0.18630 (0.13272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.71059 (20.03222) + | > current_lr: 0.00005 + | > step_time: 2.62450 (2.42071) + | > loader_time: 0.08010 (0.03563) + + + --> STEP: 152/234 -- GLOBAL_STEP: 48590 + | > loss: -0.29328 (-0.21910) + | > log_mle: -0.51804 (-0.35384) + | > loss_dur: 0.22476 (0.13473) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.62451 (20.95853) + | > current_lr: 0.00005 + | > step_time: 1.00910 (2.40429) + | > loader_time: 0.00270 (0.03455) + + + --> STEP: 157/234 -- GLOBAL_STEP: 48595 + | > loss: -0.25539 (-0.22191) + | > log_mle: -0.45542 (-0.35874) + | > loss_dur: 0.20003 (0.13683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.81971 (22.23921) + | > current_lr: 0.00005 + | > step_time: 3.00650 (2.39492) + | > loader_time: 0.00360 (0.03413) + + + --> STEP: 162/234 -- GLOBAL_STEP: 48600 + | > loss: -0.29220 (-0.22451) + | > log_mle: -0.48372 (-0.36330) + | > loss_dur: 0.19152 (0.13880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.65672 (23.43072) + | > current_lr: 0.00005 + | > step_time: 1.54350 (2.38637) + | > loader_time: 0.00170 (0.03361) + + + --> STEP: 167/234 -- GLOBAL_STEP: 48605 + | > loss: -0.37906 (-0.22697) + | > log_mle: -0.58122 (-0.36759) + | > loss_dur: 0.20215 (0.14062) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.88284 (24.45909) + | > current_lr: 0.00005 + | > step_time: 3.40820 (2.37232) + | > loader_time: 0.00360 (0.03318) + + + --> STEP: 172/234 -- GLOBAL_STEP: 48610 + | > loss: -0.34457 (-0.22983) + | > log_mle: -0.56241 (-0.37267) + | > loss_dur: 0.21784 (0.14284) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.15515 (26.07320) + | > current_lr: 0.00005 + | > step_time: 2.02080 (2.37213) + | > loader_time: 0.08730 (0.03379) + + + --> STEP: 177/234 -- GLOBAL_STEP: 48615 + | > loss: -0.29997 (-0.23244) + | > log_mle: -0.52119 (-0.37750) + | > loss_dur: 0.22123 (0.14506) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.28977 (27.10983) + | > current_lr: 0.00005 + | > step_time: 1.70230 (2.38818) + | > loader_time: 0.00360 (0.03509) + + + --> STEP: 182/234 -- GLOBAL_STEP: 48620 + | > loss: -0.32045 (-0.23485) + | > log_mle: -0.56752 (-0.38226) + | > loss_dur: 0.24707 (0.14741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.90694 (28.33388) + | > current_lr: 0.00005 + | > step_time: 15.29890 (2.49844) + | > loader_time: 0.00430 (0.03473) + + + --> STEP: 187/234 -- GLOBAL_STEP: 48625 + | > loss: -0.34150 (-0.23746) + | > log_mle: -0.56565 (-0.38698) + | > loss_dur: 0.22415 (0.14952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.81481 (29.38037) + | > current_lr: 0.00005 + | > step_time: 4.89660 (2.52644) + | > loader_time: 0.00640 (0.03430) + + + --> STEP: 192/234 -- GLOBAL_STEP: 48630 + | > loss: -0.38427 (-0.24040) + | > log_mle: -0.59480 (-0.39155) + | > loss_dur: 0.21052 (0.15115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.97599 (30.60037) + | > current_lr: 0.00005 + | > step_time: 4.99610 (2.59299) + | > loader_time: 0.00350 (0.03538) + + + --> STEP: 197/234 -- GLOBAL_STEP: 48635 + | > loss: -0.36442 (-0.24323) + | > log_mle: -0.56734 (-0.39604) + | > loss_dur: 0.20291 (0.15281) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.17276 (31.70750) + | > current_lr: 0.00005 + | > step_time: 1.71130 (2.59383) + | > loader_time: 0.00630 (0.03626) + + + --> STEP: 202/234 -- GLOBAL_STEP: 48640 + | > loss: -0.43479 (-0.24596) + | > log_mle: -0.66097 (-0.40065) + | > loss_dur: 0.22618 (0.15470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.48414 (32.81621) + | > current_lr: 0.00005 + | > step_time: 2.69890 (2.59695) + | > loader_time: 0.00480 (0.03599) + + + --> STEP: 207/234 -- GLOBAL_STEP: 48645 + | > loss: -0.40559 (-0.24872) + | > log_mle: -0.64252 (-0.40525) + | > loss_dur: 0.23693 (0.15653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.07055 (33.84862) + | > current_lr: 0.00005 + | > step_time: 4.10730 (2.64672) + | > loader_time: 0.09780 (0.03949) + + + --> STEP: 212/234 -- GLOBAL_STEP: 48650 + | > loss: -0.40129 (-0.25214) + | > log_mle: -0.63251 (-0.41058) + | > loss_dur: 0.23122 (0.15844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.64423 (35.00566) + | > current_lr: 0.00005 + | > step_time: 2.38060 (2.72097) + | > loader_time: 0.11840 (0.03966) + + + --> STEP: 217/234 -- GLOBAL_STEP: 48655 + | > loss: -0.40620 (-0.25562) + | > log_mle: -0.65667 (-0.41590) + | > loss_dur: 0.25048 (0.16027) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.38306 (36.55591) + | > current_lr: 0.00005 + | > step_time: 3.41610 (2.75415) + | > loader_time: 0.08470 (0.04131) + + + --> STEP: 222/234 -- GLOBAL_STEP: 48660 + | > loss: -0.41076 (-0.25904) + | > log_mle: -0.67503 (-0.42111) + | > loss_dur: 0.26428 (0.16207) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.71311 (37.83808) + | > current_lr: 0.00005 + | > step_time: 1.57460 (2.77079) + | > loader_time: 0.00330 (0.04127) + + + --> STEP: 227/234 -- GLOBAL_STEP: 48665 + | > loss: -0.37984 (-0.26266) + | > log_mle: -0.63462 (-0.42657) + | > loss_dur: 0.25479 (0.16391) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.58426 (39.25435) + | > current_lr: 0.00005 + | > step_time: 0.42070 (2.73870) + | > loader_time: 0.00500 (0.04117) + + + --> STEP: 232/234 -- GLOBAL_STEP: 48670 + | > loss: -0.37104 (-0.26547) + | > log_mle: -0.84585 (-0.43347) + | > loss_dur: 0.47481 (0.16801) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 151.19243 (40.88867) + | > current_lr: 0.00005 + | > step_time: 0.32480 (2.68547) + | > loader_time: 0.00410 (0.04038) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.20912 (-0.35142) + | > avg_loss: -0.27972 (-0.01096) + | > avg_log_mle: -0.50358 (-0.00374) + | > avg_loss_dur: 0.22386 (-0.00722) + + + > EPOCH: 208/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 04:33:30)  + + --> STEP: 3/234 -- GLOBAL_STEP: 48675 + | > loss: -0.18153 (-0.21882) + | > log_mle: -0.32259 (-0.33089) + | > loss_dur: 0.14106 (0.11207) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.70777 (23.59667) + | > current_lr: 0.00005 + | > step_time: 1.98990 (5.10247) + | > loader_time: 0.00380 (0.06679) + + + --> STEP: 8/234 -- GLOBAL_STEP: 48680 + | > loss: -0.24883 (-0.23059) + | > log_mle: -0.34807 (-0.33280) + | > loss_dur: 0.09924 (0.10221) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.90809 (19.89221) + | > current_lr: 0.00005 + | > step_time: 2.49090 (3.06558) + | > loader_time: 0.00140 (0.04757) + + + --> STEP: 13/234 -- GLOBAL_STEP: 48685 + | > loss: -0.26279 (-0.23292) + | > log_mle: -0.34465 (-0.33430) + | > loss_dur: 0.08186 (0.10138) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.51605 (18.59343) + | > current_lr: 0.00005 + | > step_time: 9.40400 (4.28820) + | > loader_time: 0.00170 (0.38898) + + + --> STEP: 18/234 -- GLOBAL_STEP: 48690 + | > loss: -0.21800 (-0.23525) + | > log_mle: -0.32702 (-0.33358) + | > loss_dur: 0.10902 (0.09832) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.15011 (16.61975) + | > current_lr: 0.00005 + | > step_time: 2.40620 (4.07624) + | > loader_time: 0.00160 (0.29680) + + + --> STEP: 23/234 -- GLOBAL_STEP: 48695 + | > loss: -0.25424 (-0.23695) + | > log_mle: -0.33639 (-0.33244) + | > loss_dur: 0.08215 (0.09549) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.30086 (15.55599) + | > current_lr: 0.00005 + | > step_time: 2.30700 (4.09497) + | > loader_time: 0.09060 (0.25749) + + + --> STEP: 28/234 -- GLOBAL_STEP: 48700 + | > loss: -0.27503 (-0.23815) + | > log_mle: -0.34053 (-0.33135) + | > loss_dur: 0.06550 (0.09321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.40249 (14.72584) + | > current_lr: 0.00005 + | > step_time: 1.01700 (3.63290) + | > loader_time: 0.00370 (0.21711) + + + --> STEP: 33/234 -- GLOBAL_STEP: 48705 + | > loss: -0.23458 (-0.23625) + | > log_mle: -0.31882 (-0.32966) + | > loss_dur: 0.08424 (0.09340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.09667 (14.26674) + | > current_lr: 0.00005 + | > step_time: 1.31620 (3.54402) + | > loader_time: 0.09490 (0.19254) + + + --> STEP: 38/234 -- GLOBAL_STEP: 48710 + | > loss: -0.22368 (-0.23369) + | > log_mle: -0.32907 (-0.32818) + | > loss_dur: 0.10538 (0.09449) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.08867 (14.02613) + | > current_lr: 0.00005 + | > step_time: 0.95800 (3.25183) + | > loader_time: 0.00230 (0.16745) + + + --> STEP: 43/234 -- GLOBAL_STEP: 48715 + | > loss: -0.21144 (-0.23139) + | > log_mle: -0.32593 (-0.32678) + | > loss_dur: 0.11450 (0.09540) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.56689 (13.75356) + | > current_lr: 0.00005 + | > step_time: 1.54280 (3.02909) + | > loader_time: 0.08440 (0.15192) + + + --> STEP: 48/234 -- GLOBAL_STEP: 48720 + | > loss: -0.23535 (-0.22999) + | > log_mle: -0.31679 (-0.32610) + | > loss_dur: 0.08144 (0.09611) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.28675 (13.47318) + | > current_lr: 0.00005 + | > step_time: 1.80920 (2.94852) + | > loader_time: 0.00160 (0.13990) + + + --> STEP: 53/234 -- GLOBAL_STEP: 48725 + | > loss: -0.19768 (-0.22783) + | > log_mle: -0.32044 (-0.32513) + | > loss_dur: 0.12276 (0.09729) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.16783 (13.21821) + | > current_lr: 0.00005 + | > step_time: 1.37590 (2.78969) + | > loader_time: 0.00210 (0.12688) + + + --> STEP: 58/234 -- GLOBAL_STEP: 48730 + | > loss: -0.22261 (-0.22629) + | > log_mle: -0.31603 (-0.32445) + | > loss_dur: 0.09343 (0.09816) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06674 (13.05296) + | > current_lr: 0.00005 + | > step_time: 2.36580 (2.67801) + | > loader_time: 0.00580 (0.11915) + + + --> STEP: 63/234 -- GLOBAL_STEP: 48735 + | > loss: -0.17855 (-0.22374) + | > log_mle: -0.31216 (-0.32464) + | > loss_dur: 0.13361 (0.10090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.11347 (13.22729) + | > current_lr: 0.00005 + | > step_time: 1.55710 (2.60993) + | > loader_time: 0.00180 (0.11102) + + + --> STEP: 68/234 -- GLOBAL_STEP: 48740 + | > loss: -0.17672 (-0.22236) + | > log_mle: -0.30667 (-0.32397) + | > loss_dur: 0.12995 (0.10162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.94978 (13.08042) + | > current_lr: 0.00005 + | > step_time: 1.37620 (2.53917) + | > loader_time: 0.00240 (0.10301) + + + --> STEP: 73/234 -- GLOBAL_STEP: 48745 + | > loss: -0.17559 (-0.21987) + | > log_mle: -0.32369 (-0.32340) + | > loss_dur: 0.14810 (0.10353) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.71138 (13.04177) + | > current_lr: 0.00005 + | > step_time: 1.20190 (2.48940) + | > loader_time: 0.00220 (0.09615) + + + --> STEP: 78/234 -- GLOBAL_STEP: 48750 + | > loss: -0.17587 (-0.21792) + | > log_mle: -0.30456 (-0.32288) + | > loss_dur: 0.12869 (0.10496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.05305 (12.99210) + | > current_lr: 0.00005 + | > step_time: 2.16640 (2.46661) + | > loader_time: 0.00190 (0.09012) + + + --> STEP: 83/234 -- GLOBAL_STEP: 48755 + | > loss: -0.17472 (-0.21654) + | > log_mle: -0.32602 (-0.32270) + | > loss_dur: 0.15131 (0.10616) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.62499 (12.95663) + | > current_lr: 0.00005 + | > step_time: 2.40600 (2.41118) + | > loader_time: 0.00280 (0.08588) + + + --> STEP: 88/234 -- GLOBAL_STEP: 48760 + | > loss: -0.21411 (-0.21557) + | > log_mle: -0.35878 (-0.32295) + | > loss_dur: 0.14467 (0.10738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.85272 (13.05346) + | > current_lr: 0.00005 + | > step_time: 2.09500 (2.46614) + | > loader_time: 0.00360 (0.08554) + + + --> STEP: 93/234 -- GLOBAL_STEP: 48765 + | > loss: -0.20510 (-0.21493) + | > log_mle: -0.37424 (-0.32447) + | > loss_dur: 0.16914 (0.10954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.85943 (13.42349) + | > current_lr: 0.00005 + | > step_time: 2.39930 (2.44347) + | > loader_time: 0.00550 (0.08283) + + + --> STEP: 98/234 -- GLOBAL_STEP: 48770 + | > loss: -0.18605 (-0.21461) + | > log_mle: -0.30655 (-0.32597) + | > loss_dur: 0.12050 (0.11136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.70239 (13.84353) + | > current_lr: 0.00005 + | > step_time: 1.21500 (2.42192) + | > loader_time: 0.07450 (0.08044) + + + --> STEP: 103/234 -- GLOBAL_STEP: 48775 + | > loss: -0.23040 (-0.21452) + | > log_mle: -0.40473 (-0.32831) + | > loss_dur: 0.17433 (0.11379) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.75504 (14.40278) + | > current_lr: 0.00005 + | > step_time: 2.39590 (2.38475) + | > loader_time: 0.10010 (0.07848) + + + --> STEP: 108/234 -- GLOBAL_STEP: 48780 + | > loss: -0.19863 (-0.21437) + | > log_mle: -0.34597 (-0.33024) + | > loss_dur: 0.14734 (0.11587) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.31325 (14.76168) + | > current_lr: 0.00005 + | > step_time: 2.67200 (2.40849) + | > loader_time: 0.00460 (0.07738) + + + --> STEP: 113/234 -- GLOBAL_STEP: 48785 + | > loss: -0.23677 (-0.21411) + | > log_mle: -0.39306 (-0.33275) + | > loss_dur: 0.15629 (0.11864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.72138 (15.49369) + | > current_lr: 0.00005 + | > step_time: 1.00550 (2.41072) + | > loader_time: 0.10050 (0.07499) + + + --> STEP: 118/234 -- GLOBAL_STEP: 48790 + | > loss: -0.19501 (-0.21382) + | > log_mle: -0.36380 (-0.33456) + | > loss_dur: 0.16879 (0.12074) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.89551 (15.96433) + | > current_lr: 0.00005 + | > step_time: 2.48410 (2.39070) + | > loader_time: 0.00250 (0.07265) + + + --> STEP: 123/234 -- GLOBAL_STEP: 48795 + | > loss: -0.17571 (-0.21330) + | > log_mle: -0.33451 (-0.33547) + | > loss_dur: 0.15880 (0.12217) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.62583 (16.16482) + | > current_lr: 0.00005 + | > step_time: 1.84560 (2.38318) + | > loader_time: 0.00170 (0.07102) + + + --> STEP: 128/234 -- GLOBAL_STEP: 48800 + | > loss: -0.23301 (-0.21411) + | > log_mle: -0.38958 (-0.33836) + | > loss_dur: 0.15657 (0.12426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.30225 (16.96140) + | > current_lr: 0.00005 + | > step_time: 1.39430 (2.39240) + | > loader_time: 0.00620 (0.06888) + + + --> STEP: 133/234 -- GLOBAL_STEP: 48805 + | > loss: -0.24661 (-0.21528) + | > log_mle: -0.42116 (-0.34157) + | > loss_dur: 0.17454 (0.12628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.91745 (17.69480) + | > current_lr: 0.00005 + | > step_time: 1.90460 (2.38619) + | > loader_time: 0.00360 (0.06850) + + + --> STEP: 138/234 -- GLOBAL_STEP: 48810 + | > loss: -0.19661 (-0.21603) + | > log_mle: -0.37074 (-0.34444) + | > loss_dur: 0.17413 (0.12841) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.68799 (18.59229) + | > current_lr: 0.00005 + | > step_time: 2.70290 (2.40319) + | > loader_time: 0.00160 (0.06744) + + + --> STEP: 143/234 -- GLOBAL_STEP: 48815 + | > loss: -0.29370 (-0.21766) + | > log_mle: -0.51905 (-0.34830) + | > loss_dur: 0.22536 (0.13064) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.52315 (19.64805) + | > current_lr: 0.00005 + | > step_time: 0.95840 (2.41619) + | > loader_time: 0.00190 (0.06711) + + + --> STEP: 148/234 -- GLOBAL_STEP: 48820 + | > loss: -0.26952 (-0.21955) + | > log_mle: -0.42906 (-0.35213) + | > loss_dur: 0.15954 (0.13257) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.23175 (20.66150) + | > current_lr: 0.00005 + | > step_time: 1.69520 (2.42782) + | > loader_time: 0.05950 (0.06670) + + + --> STEP: 153/234 -- GLOBAL_STEP: 48825 + | > loss: -0.35011 (-0.22224) + | > log_mle: -0.55729 (-0.35708) + | > loss_dur: 0.20718 (0.13485) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.64117 (21.81020) + | > current_lr: 0.00005 + | > step_time: 3.20940 (2.43424) + | > loader_time: 0.00410 (0.06513) + + + --> STEP: 158/234 -- GLOBAL_STEP: 48830 + | > loss: -0.27111 (-0.22440) + | > log_mle: -0.49380 (-0.36147) + | > loss_dur: 0.22269 (0.13707) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.37352 (22.82371) + | > current_lr: 0.00005 + | > step_time: 2.29280 (2.41531) + | > loader_time: 0.00370 (0.06367) + + + --> STEP: 163/234 -- GLOBAL_STEP: 48835 + | > loss: -0.25787 (-0.22671) + | > log_mle: -0.45685 (-0.36573) + | > loss_dur: 0.19898 (0.13901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.68318 (24.08043) + | > current_lr: 0.00005 + | > step_time: 3.20780 (2.47072) + | > loader_time: 0.09490 (0.06293) + + + --> STEP: 168/234 -- GLOBAL_STEP: 48840 + | > loss: -0.29145 (-0.22930) + | > log_mle: -0.51795 (-0.37027) + | > loss_dur: 0.22649 (0.14097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.63332 (25.05613) + | > current_lr: 0.00005 + | > step_time: 1.99280 (2.47991) + | > loader_time: 0.00400 (0.06168) + + + --> STEP: 173/234 -- GLOBAL_STEP: 48845 + | > loss: -0.31524 (-0.23235) + | > log_mle: -0.52811 (-0.37547) + | > loss_dur: 0.21286 (0.14311) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.10432 (26.23085) + | > current_lr: 0.00005 + | > step_time: 2.90880 (2.49559) + | > loader_time: 0.00330 (0.06096) + + + --> STEP: 178/234 -- GLOBAL_STEP: 48850 + | > loss: -0.35125 (-0.23541) + | > log_mle: -0.58649 (-0.38066) + | > loss_dur: 0.23523 (0.14525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.61031 (27.49883) + | > current_lr: 0.00005 + | > step_time: 6.49500 (2.54524) + | > loader_time: 0.10710 (0.06144) + + + --> STEP: 183/234 -- GLOBAL_STEP: 48855 + | > loss: -0.37597 (-0.23792) + | > log_mle: -0.59287 (-0.38543) + | > loss_dur: 0.21689 (0.14751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.44973 (28.68190) + | > current_lr: 0.00005 + | > step_time: 8.29770 (2.63189) + | > loader_time: 0.20140 (0.06187) + + + --> STEP: 188/234 -- GLOBAL_STEP: 48860 + | > loss: -0.38127 (-0.24063) + | > log_mle: -0.59994 (-0.39016) + | > loss_dur: 0.21868 (0.14954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.60497 (29.92925) + | > current_lr: 0.00005 + | > step_time: 1.90870 (2.70339) + | > loader_time: 0.08490 (0.06312) + + + --> STEP: 193/234 -- GLOBAL_STEP: 48865 + | > loss: -0.39941 (-0.24389) + | > log_mle: -0.60844 (-0.39493) + | > loss_dur: 0.20902 (0.15103) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.63714 (30.87047) + | > current_lr: 0.00005 + | > step_time: 9.50660 (2.81215) + | > loader_time: 0.19370 (0.06355) + + + --> STEP: 198/234 -- GLOBAL_STEP: 48870 + | > loss: -0.35687 (-0.24659) + | > log_mle: -0.59191 (-0.39942) + | > loss_dur: 0.23504 (0.15282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.00401 (32.18430) + | > current_lr: 0.00005 + | > step_time: 2.10540 (2.81899) + | > loader_time: 0.00620 (0.06439) + + + --> STEP: 203/234 -- GLOBAL_STEP: 48875 + | > loss: -0.29832 (-0.24895) + | > log_mle: -0.51309 (-0.40360) + | > loss_dur: 0.21477 (0.15465) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.62187 (33.18517) + | > current_lr: 0.00005 + | > step_time: 2.91840 (2.82157) + | > loader_time: 0.09100 (0.06374) + + + --> STEP: 208/234 -- GLOBAL_STEP: 48880 + | > loss: -0.34816 (-0.25198) + | > log_mle: -0.59400 (-0.40849) + | > loss_dur: 0.24584 (0.15651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.75519 (34.44505) + | > current_lr: 0.00005 + | > step_time: 3.39700 (2.85989) + | > loader_time: 0.00520 (0.06328) + + + --> STEP: 213/234 -- GLOBAL_STEP: 48885 + | > loss: -0.39193 (-0.25520) + | > log_mle: -0.64883 (-0.41372) + | > loss_dur: 0.25689 (0.15853) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.41743 (35.93490) + | > current_lr: 0.00005 + | > step_time: 4.11090 (2.89936) + | > loader_time: 0.09590 (0.06276) + + + --> STEP: 218/234 -- GLOBAL_STEP: 48890 + | > loss: -0.32874 (-0.25764) + | > log_mle: -0.58498 (-0.41806) + | > loss_dur: 0.25624 (0.16042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.34846 (37.42648) + | > current_lr: 0.00005 + | > step_time: 4.19220 (2.94351) + | > loader_time: 0.00310 (0.06316) + + + --> STEP: 223/234 -- GLOBAL_STEP: 48895 + | > loss: -0.41565 (-0.26072) + | > log_mle: -0.65385 (-0.42307) + | > loss_dur: 0.23820 (0.16235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.40279 (38.31231) + | > current_lr: 0.00005 + | > step_time: 2.40190 (2.92786) + | > loader_time: 0.00510 (0.06225) + + + --> STEP: 228/234 -- GLOBAL_STEP: 48900 + | > loss: -0.38569 (-0.26414) + | > log_mle: -0.65620 (-0.42845) + | > loss_dur: 0.27051 (0.16431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.17599 (39.44441) + | > current_lr: 0.00005 + | > step_time: 4.09340 (2.93554) + | > loader_time: 0.00660 (0.06178) + + + --> STEP: 233/234 -- GLOBAL_STEP: 48905 + | > loss: 0.01495 (-0.26502) + | > log_mle: -0.61254 (-0.43488) + | > loss_dur: 0.62749 (0.16986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.36980 (41.08865) + | > current_lr: 0.00005 + | > step_time: 0.19950 (2.87863) + | > loader_time: 0.00350 (0.06126) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.19503 (-0.01409) + | > avg_loss: -0.29739 (-0.01767) + | > avg_log_mle: -0.50779 (-0.00420) + | > avg_loss_dur: 0.21039 (-0.01347) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_48906.pth + + > EPOCH: 209/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 04:46:10)  + + --> STEP: 4/234 -- GLOBAL_STEP: 48910 + | > loss: -0.21996 (-0.21967) + | > log_mle: -0.33259 (-0.33320) + | > loss_dur: 0.11262 (0.11353) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.29490 (18.80327) + | > current_lr: 0.00005 + | > step_time: 6.90250 (8.87648) + | > loader_time: 0.00880 (0.05131) + + + --> STEP: 9/234 -- GLOBAL_STEP: 48915 + | > loss: -0.21123 (-0.23249) + | > log_mle: -0.34051 (-0.33683) + | > loss_dur: 0.12928 (0.10433) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.10431 (17.38607) + | > current_lr: 0.00005 + | > step_time: 4.89830 (5.79087) + | > loader_time: 0.10510 (0.07627) + + + --> STEP: 14/234 -- GLOBAL_STEP: 48920 + | > loss: -0.25284 (-0.23663) + | > log_mle: -0.34085 (-0.33714) + | > loss_dur: 0.08801 (0.10051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.26545 (16.51947) + | > current_lr: 0.00005 + | > step_time: 2.30200 (4.37182) + | > loader_time: 0.00210 (0.05622) + + + --> STEP: 19/234 -- GLOBAL_STEP: 48925 + | > loss: -0.25778 (-0.23956) + | > log_mle: -0.33041 (-0.33558) + | > loss_dur: 0.07262 (0.09602) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46044 (15.39824) + | > current_lr: 0.00005 + | > step_time: 6.20500 (3.98125) + | > loader_time: 0.49170 (0.07179) + + + --> STEP: 24/234 -- GLOBAL_STEP: 48930 + | > loss: -0.26461 (-0.24162) + | > log_mle: -0.32955 (-0.33455) + | > loss_dur: 0.06494 (0.09293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.92860 (14.56512) + | > current_lr: 0.00005 + | > step_time: 1.29620 (4.08976) + | > loader_time: 0.00120 (0.06179) + + + --> STEP: 29/234 -- GLOBAL_STEP: 48935 + | > loss: -0.21038 (-0.24087) + | > log_mle: -0.30909 (-0.33282) + | > loss_dur: 0.09870 (0.09195) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.67997 (14.13791) + | > current_lr: 0.00005 + | > step_time: 2.91350 (3.73246) + | > loader_time: 0.07850 (0.05425) + + + --> STEP: 34/234 -- GLOBAL_STEP: 48940 + | > loss: -0.22385 (-0.23927) + | > log_mle: -0.32007 (-0.33141) + | > loss_dur: 0.09621 (0.09215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81791 (13.82530) + | > current_lr: 0.00005 + | > step_time: 0.70740 (3.61898) + | > loader_time: 0.00170 (0.05468) + + + --> STEP: 39/234 -- GLOBAL_STEP: 48945 + | > loss: -0.22481 (-0.23656) + | > log_mle: -0.32529 (-0.33005) + | > loss_dur: 0.10048 (0.09349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.73119 (13.71062) + | > current_lr: 0.00005 + | > step_time: 2.69720 (3.39597) + | > loader_time: 0.00200 (0.05002) + + + --> STEP: 44/234 -- GLOBAL_STEP: 48950 + | > loss: -0.23716 (-0.23429) + | > log_mle: -0.31488 (-0.32833) + | > loss_dur: 0.07772 (0.09404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.80710 (13.40377) + | > current_lr: 0.00005 + | > step_time: 1.29950 (3.21081) + | > loader_time: 0.00110 (0.04835) + + + --> STEP: 49/234 -- GLOBAL_STEP: 48955 + | > loss: -0.23361 (-0.23243) + | > log_mle: -0.32719 (-0.32774) + | > loss_dur: 0.09358 (0.09531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.28074 (13.46002) + | > current_lr: 0.00005 + | > step_time: 1.04330 (3.04503) + | > loader_time: 0.00210 (0.04364) + + + --> STEP: 54/234 -- GLOBAL_STEP: 48960 + | > loss: -0.22156 (-0.23016) + | > log_mle: -0.32398 (-0.32658) + | > loss_dur: 0.10241 (0.09642) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17052 (13.28915) + | > current_lr: 0.00005 + | > step_time: 1.58550 (2.91505) + | > loader_time: 0.00250 (0.04169) + + + --> STEP: 59/234 -- GLOBAL_STEP: 48965 + | > loss: -0.22382 (-0.22875) + | > log_mle: -0.32990 (-0.32602) + | > loss_dur: 0.10608 (0.09727) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.26199 (13.11884) + | > current_lr: 0.00005 + | > step_time: 1.76160 (2.82788) + | > loader_time: 0.00190 (0.03833) + + + --> STEP: 64/234 -- GLOBAL_STEP: 48970 + | > loss: -0.21910 (-0.22619) + | > log_mle: -0.31648 (-0.32598) + | > loss_dur: 0.09738 (0.09979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.68572 (13.09495) + | > current_lr: 0.00005 + | > step_time: 4.01520 (2.79306) + | > loader_time: 0.19580 (0.03980) + + + --> STEP: 69/234 -- GLOBAL_STEP: 48975 + | > loss: -0.19316 (-0.22442) + | > log_mle: -0.30262 (-0.32514) + | > loss_dur: 0.10946 (0.10072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.11674 (12.87605) + | > current_lr: 0.00005 + | > step_time: 2.03280 (2.74438) + | > loader_time: 0.09470 (0.03841) + + + --> STEP: 74/234 -- GLOBAL_STEP: 48980 + | > loss: -0.18067 (-0.22162) + | > log_mle: -0.30381 (-0.32463) + | > loss_dur: 0.12314 (0.10300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.80313 (13.02666) + | > current_lr: 0.00005 + | > step_time: 3.11900 (2.74235) + | > loader_time: 0.00190 (0.03821) + + + --> STEP: 79/234 -- GLOBAL_STEP: 48985 + | > loss: -0.18874 (-0.21993) + | > log_mle: -0.32180 (-0.32434) + | > loss_dur: 0.13306 (0.10441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.24881 (12.96892) + | > current_lr: 0.00005 + | > step_time: 3.19840 (2.70494) + | > loader_time: 0.10130 (0.03720) + + + --> STEP: 84/234 -- GLOBAL_STEP: 48990 + | > loss: -0.20582 (-0.21863) + | > log_mle: -0.31776 (-0.32409) + | > loss_dur: 0.11194 (0.10546) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.49456 (12.94113) + | > current_lr: 0.00005 + | > step_time: 1.85620 (2.67255) + | > loader_time: 0.00180 (0.03642) + + + --> STEP: 89/234 -- GLOBAL_STEP: 48995 + | > loss: -0.21566 (-0.21784) + | > log_mle: -0.34500 (-0.32469) + | > loss_dur: 0.12933 (0.10685) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.50151 (13.03505) + | > current_lr: 0.00005 + | > step_time: 1.29520 (2.60052) + | > loader_time: 0.00260 (0.03450) + + + --> STEP: 94/234 -- GLOBAL_STEP: 49000 + | > loss: -0.22553 (-0.21742) + | > log_mle: -0.37615 (-0.32655) + | > loss_dur: 0.15062 (0.10913) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.91689 (13.52842) + | > current_lr: 0.00005 + | > step_time: 2.12510 (2.58261) + | > loader_time: 0.00300 (0.03550) + + + --> STEP: 99/234 -- GLOBAL_STEP: 49005 + | > loss: -0.24066 (-0.21717) + | > log_mle: -0.40812 (-0.32839) + | > loss_dur: 0.16745 (0.11122) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.33206 (13.99643) + | > current_lr: 0.00005 + | > step_time: 1.30110 (2.53098) + | > loader_time: 0.00270 (0.03471) + + + --> STEP: 104/234 -- GLOBAL_STEP: 49010 + | > loss: -0.26442 (-0.21717) + | > log_mle: -0.42089 (-0.33078) + | > loss_dur: 0.15646 (0.11361) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.14524 (14.63826) + | > current_lr: 0.00005 + | > step_time: 2.28930 (2.51693) + | > loader_time: 0.00660 (0.03406) + + + --> STEP: 109/234 -- GLOBAL_STEP: 49015 + | > loss: -0.19223 (-0.21630) + | > log_mle: -0.38483 (-0.33225) + | > loss_dur: 0.19260 (0.11595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.80531 (15.16392) + | > current_lr: 0.00005 + | > step_time: 7.09130 (2.64571) + | > loader_time: 0.26530 (0.03757) + + + --> STEP: 114/234 -- GLOBAL_STEP: 49020 + | > loss: -0.22280 (-0.21636) + | > log_mle: -0.37220 (-0.33461) + | > loss_dur: 0.14940 (0.11825) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.68186 (15.77158) + | > current_lr: 0.00005 + | > step_time: 2.90330 (2.63058) + | > loader_time: 0.00880 (0.03683) + + + --> STEP: 119/234 -- GLOBAL_STEP: 49025 + | > loss: -0.21267 (-0.21578) + | > log_mle: -0.36909 (-0.33633) + | > loss_dur: 0.15642 (0.12056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.85549 (16.18669) + | > current_lr: 0.00005 + | > step_time: 1.90920 (2.59913) + | > loader_time: 0.00330 (0.03610) + + + --> STEP: 124/234 -- GLOBAL_STEP: 49030 + | > loss: -0.23356 (-0.21538) + | > log_mle: -0.39848 (-0.33741) + | > loss_dur: 0.16493 (0.12204) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.35088 (16.43076) + | > current_lr: 0.00005 + | > step_time: 1.19710 (2.57255) + | > loader_time: 0.00250 (0.03548) + + + --> STEP: 129/234 -- GLOBAL_STEP: 49035 + | > loss: -0.20545 (-0.21599) + | > log_mle: -0.38555 (-0.34018) + | > loss_dur: 0.18010 (0.12419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.55504 (17.32016) + | > current_lr: 0.00005 + | > step_time: 6.99840 (2.61842) + | > loader_time: 0.10010 (0.03646) + + + --> STEP: 134/234 -- GLOBAL_STEP: 49040 + | > loss: -0.23962 (-0.21706) + | > log_mle: -0.43535 (-0.34344) + | > loss_dur: 0.19573 (0.12638) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.33804 (18.11722) + | > current_lr: 0.00005 + | > step_time: 2.70460 (2.63659) + | > loader_time: 0.11120 (0.03741) + + + --> STEP: 139/234 -- GLOBAL_STEP: 49045 + | > loss: -0.30410 (-0.21801) + | > log_mle: -0.50246 (-0.34659) + | > loss_dur: 0.19836 (0.12858) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.95105 (18.93653) + | > current_lr: 0.00005 + | > step_time: 2.19650 (2.60649) + | > loader_time: 0.00230 (0.03617) + + + --> STEP: 144/234 -- GLOBAL_STEP: 49050 + | > loss: -0.27573 (-0.21915) + | > log_mle: -0.47603 (-0.35005) + | > loss_dur: 0.20030 (0.13090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.01832 (19.74004) + | > current_lr: 0.00005 + | > step_time: 2.81470 (2.65134) + | > loader_time: 0.09020 (0.03573) + + + --> STEP: 149/234 -- GLOBAL_STEP: 49055 + | > loss: -0.32714 (-0.22128) + | > log_mle: -0.52874 (-0.35415) + | > loss_dur: 0.20160 (0.13287) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.98680 (20.57209) + | > current_lr: 0.00005 + | > step_time: 3.29960 (2.65757) + | > loader_time: 0.09200 (0.03862) + + + --> STEP: 154/234 -- GLOBAL_STEP: 49060 + | > loss: -0.29135 (-0.22379) + | > log_mle: -0.48560 (-0.35883) + | > loss_dur: 0.19425 (0.13504) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.09243 (21.73006) + | > current_lr: 0.00005 + | > step_time: 4.81110 (2.70634) + | > loader_time: 0.07830 (0.03983) + + + --> STEP: 159/234 -- GLOBAL_STEP: 49065 + | > loss: -0.30319 (-0.22605) + | > log_mle: -0.50680 (-0.36328) + | > loss_dur: 0.20361 (0.13723) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.38418 (22.77007) + | > current_lr: 0.00005 + | > step_time: 3.51020 (2.69822) + | > loader_time: 0.00270 (0.04028) + + + --> STEP: 164/234 -- GLOBAL_STEP: 49070 + | > loss: -0.28232 (-0.22808) + | > log_mle: -0.49336 (-0.36727) + | > loss_dur: 0.21104 (0.13920) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.54776 (24.03097) + | > current_lr: 0.00005 + | > step_time: 6.40070 (2.72759) + | > loader_time: 0.10670 (0.04035) + + + --> STEP: 169/234 -- GLOBAL_STEP: 49075 + | > loss: -0.29295 (-0.23061) + | > log_mle: -0.50209 (-0.37173) + | > loss_dur: 0.20914 (0.14112) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.53453 (25.04220) + | > current_lr: 0.00005 + | > step_time: 2.79000 (2.72193) + | > loader_time: 0.00690 (0.03993) + + + --> STEP: 174/234 -- GLOBAL_STEP: 49080 + | > loss: -0.37129 (-0.23417) + | > log_mle: -0.59143 (-0.37747) + | > loss_dur: 0.22014 (0.14330) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.30703 (26.35235) + | > current_lr: 0.00005 + | > step_time: 9.19650 (2.74492) + | > loader_time: 0.10580 (0.04043) + + + --> STEP: 179/234 -- GLOBAL_STEP: 49085 + | > loss: -0.33966 (-0.23698) + | > log_mle: -0.58258 (-0.38263) + | > loss_dur: 0.24292 (0.14565) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.13044 (27.45725) + | > current_lr: 0.00005 + | > step_time: 1.38530 (2.75742) + | > loader_time: 0.00280 (0.04040) + + + --> STEP: 184/234 -- GLOBAL_STEP: 49090 + | > loss: -0.33098 (-0.23939) + | > log_mle: -0.54710 (-0.38708) + | > loss_dur: 0.21612 (0.14768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.83097 (28.71208) + | > current_lr: 0.00005 + | > step_time: 4.10080 (2.75266) + | > loader_time: 0.00430 (0.04073) + + + --> STEP: 189/234 -- GLOBAL_STEP: 49095 + | > loss: -0.32039 (-0.24204) + | > log_mle: -0.54292 (-0.39181) + | > loss_dur: 0.22253 (0.14977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.26072 (29.86597) + | > current_lr: 0.00005 + | > step_time: 3.11030 (2.78562) + | > loader_time: 0.08650 (0.04108) + + + --> STEP: 194/234 -- GLOBAL_STEP: 49100 + | > loss: -0.35183 (-0.24508) + | > log_mle: -0.56747 (-0.39647) + | > loss_dur: 0.21564 (0.15139) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 167.83406 (31.57162) + | > current_lr: 0.00005 + | > step_time: 10.30380 (2.80057) + | > loader_time: 0.09240 (0.04103) + + + --> STEP: 199/234 -- GLOBAL_STEP: 49105 + | > loss: -0.32988 (-0.24686) + | > log_mle: -0.55760 (-0.40003) + | > loss_dur: 0.22772 (0.15318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.45597 (32.44895) + | > current_lr: 0.00005 + | > step_time: 7.60350 (2.92721) + | > loader_time: 0.00340 (0.04154) + + + --> STEP: 204/234 -- GLOBAL_STEP: 49110 + | > loss: -0.37507 (-0.24894) + | > log_mle: -0.61574 (-0.40401) + | > loss_dur: 0.24067 (0.15508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.76925 (33.11600) + | > current_lr: 0.00005 + | > step_time: 5.20430 (2.98981) + | > loader_time: 0.09550 (0.04297) + + + --> STEP: 209/234 -- GLOBAL_STEP: 49115 + | > loss: -0.33746 (-0.25115) + | > log_mle: -0.55503 (-0.40807) + | > loss_dur: 0.21757 (0.15692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.74389 (34.41603) + | > current_lr: 0.00005 + | > step_time: 3.49380 (3.02399) + | > loader_time: 0.09830 (0.04435) + + + --> STEP: 214/234 -- GLOBAL_STEP: 49120 + | > loss: -0.39073 (-0.25437) + | > log_mle: -0.60458 (-0.41317) + | > loss_dur: 0.21385 (0.15880) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.26797 (35.36015) + | > current_lr: 0.00005 + | > step_time: 7.00010 (3.06323) + | > loader_time: 0.00390 (0.04506) + + + --> STEP: 219/234 -- GLOBAL_STEP: 49125 + | > loss: -0.42006 (-0.25730) + | > log_mle: -0.65432 (-0.41803) + | > loss_dur: 0.23426 (0.16072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 151.33856 (37.31339) + | > current_lr: 0.00005 + | > step_time: 5.01230 (3.12197) + | > loader_time: 0.09960 (0.04730) + + + --> STEP: 224/234 -- GLOBAL_STEP: 49130 + | > loss: -0.39786 (-0.25993) + | > log_mle: -0.64290 (-0.42252) + | > loss_dur: 0.24504 (0.16259) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.37298 (38.38322) + | > current_lr: 0.00005 + | > step_time: 0.23090 (3.06606) + | > loader_time: 0.00370 (0.04633) + + + --> STEP: 229/234 -- GLOBAL_STEP: 49135 + | > loss: -0.39238 (-0.26303) + | > log_mle: -0.69032 (-0.42787) + | > loss_dur: 0.29793 (0.16484) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.56830 (39.27325) + | > current_lr: 0.00005 + | > step_time: 0.24220 (3.00428) + | > loader_time: 0.00300 (0.04539) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.30444 (+0.10941) + | > avg_loss: -0.27273 (+0.02466) + | > avg_log_mle: -0.50947 (-0.00168) + | > avg_loss_dur: 0.23674 (+0.02635) + + + > EPOCH: 210/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 04:58:49)  + + --> STEP: 0/234 -- GLOBAL_STEP: 49140 + | > loss: -0.27333 (-0.27333) + | > log_mle: -0.41353 (-0.41353) + | > loss_dur: 0.14020 (0.14020) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.43252 (25.43252) + | > current_lr: 0.00005 + | > step_time: 13.40020 (13.40019) + | > loader_time: 12.96500 (12.96495) + + + --> STEP: 5/234 -- GLOBAL_STEP: 49145 + | > loss: -0.25125 (-0.23443) + | > log_mle: -0.33589 (-0.33415) + | > loss_dur: 0.08464 (0.09972) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.30794 (20.56368) + | > current_lr: 0.00005 + | > step_time: 1.20720 (3.69730) + | > loader_time: 0.00240 (0.00357) + + + --> STEP: 10/234 -- GLOBAL_STEP: 49150 + | > loss: -0.22765 (-0.23877) + | > log_mle: -0.33199 (-0.33635) + | > loss_dur: 0.10433 (0.09758) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.13713 (18.85554) + | > current_lr: 0.00005 + | > step_time: 6.59450 (4.12994) + | > loader_time: 0.00090 (0.01236) + + + --> STEP: 15/234 -- GLOBAL_STEP: 49155 + | > loss: -0.25698 (-0.24243) + | > log_mle: -0.34250 (-0.33767) + | > loss_dur: 0.08552 (0.09525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.45588 (17.17466) + | > current_lr: 0.00005 + | > step_time: 7.11190 (4.88821) + | > loader_time: 0.10490 (0.03520) + + + --> STEP: 20/234 -- GLOBAL_STEP: 49160 + | > loss: -0.25547 (-0.24400) + | > log_mle: -0.33671 (-0.33609) + | > loss_dur: 0.08124 (0.09210) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.85525 (15.37541) + | > current_lr: 0.00005 + | > step_time: 4.60040 (4.48325) + | > loader_time: 0.09540 (0.03578) + + + --> STEP: 25/234 -- GLOBAL_STEP: 49165 + | > loss: -0.23369 (-0.24414) + | > log_mle: -0.32034 (-0.33454) + | > loss_dur: 0.08665 (0.09040) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.11400 (14.58762) + | > current_lr: 0.00005 + | > step_time: 2.61840 (4.15507) + | > loader_time: 0.00370 (0.03256) + + + --> STEP: 30/234 -- GLOBAL_STEP: 49170 + | > loss: -0.23598 (-0.24339) + | > log_mle: -0.32664 (-0.33348) + | > loss_dur: 0.09066 (0.09009) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.23455 (13.90545) + | > current_lr: 0.00005 + | > step_time: 4.81560 (3.83816) + | > loader_time: 0.10600 (0.03089) + + + --> STEP: 35/234 -- GLOBAL_STEP: 49175 + | > loss: -0.20877 (-0.24028) + | > log_mle: -0.32249 (-0.33227) + | > loss_dur: 0.11372 (0.09199) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.41129 (13.55871) + | > current_lr: 0.00005 + | > step_time: 1.39010 (3.48307) + | > loader_time: 0.00200 (0.02673) + + + --> STEP: 40/234 -- GLOBAL_STEP: 49180 + | > loss: -0.18799 (-0.23706) + | > log_mle: -0.30888 (-0.33076) + | > loss_dur: 0.12089 (0.09370) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.21925 (13.51626) + | > current_lr: 0.00005 + | > step_time: 1.89720 (3.27567) + | > loader_time: 0.00240 (0.02363) + + + --> STEP: 45/234 -- GLOBAL_STEP: 49185 + | > loss: -0.21328 (-0.23498) + | > log_mle: -0.33687 (-0.32954) + | > loss_dur: 0.12359 (0.09456) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.67482 (13.79161) + | > current_lr: 0.00005 + | > step_time: 1.06580 (3.07107) + | > loader_time: 0.00220 (0.02126) + + + --> STEP: 50/234 -- GLOBAL_STEP: 49190 + | > loss: -0.21207 (-0.23298) + | > log_mle: -0.31132 (-0.32820) + | > loss_dur: 0.09924 (0.09523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.93331 (13.71137) + | > current_lr: 0.00005 + | > step_time: 2.06530 (2.94521) + | > loader_time: 0.00210 (0.02112) + + + --> STEP: 55/234 -- GLOBAL_STEP: 49195 + | > loss: -0.23197 (-0.23117) + | > log_mle: -0.32664 (-0.32743) + | > loss_dur: 0.09466 (0.09625) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.23697 (13.37201) + | > current_lr: 0.00005 + | > step_time: 2.54250 (2.85713) + | > loader_time: 0.00250 (0.01938) + + + --> STEP: 60/234 -- GLOBAL_STEP: 49200 + | > loss: -0.20718 (-0.22947) + | > log_mle: -0.33841 (-0.32706) + | > loss_dur: 0.13123 (0.09759) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.01890 (13.17214) + | > current_lr: 0.00005 + | > step_time: 1.76390 (2.79577) + | > loader_time: 0.00180 (0.02008) + + + --> STEP: 65/234 -- GLOBAL_STEP: 49205 + | > loss: -0.21269 (-0.22736) + | > log_mle: -0.31971 (-0.32682) + | > loss_dur: 0.10702 (0.09946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.71910 (13.04085) + | > current_lr: 0.00005 + | > step_time: 2.35360 (2.74924) + | > loader_time: 0.00230 (0.01878) + + + --> STEP: 70/234 -- GLOBAL_STEP: 49210 + | > loss: -0.17261 (-0.22501) + | > log_mle: -0.30609 (-0.32583) + | > loss_dur: 0.13348 (0.10082) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.37768 (12.88009) + | > current_lr: 0.00005 + | > step_time: 1.80060 (2.66688) + | > loader_time: 0.08460 (0.02007) + + + --> STEP: 75/234 -- GLOBAL_STEP: 49215 + | > loss: -0.19266 (-0.22268) + | > log_mle: -0.32756 (-0.32570) + | > loss_dur: 0.13491 (0.10303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.27899 (12.90001) + | > current_lr: 0.00005 + | > step_time: 2.99990 (2.64945) + | > loader_time: 0.00260 (0.02244) + + + --> STEP: 80/234 -- GLOBAL_STEP: 49220 + | > loss: -0.19792 (-0.22131) + | > log_mle: -0.30899 (-0.32516) + | > loss_dur: 0.11108 (0.10385) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.51034 (12.92036) + | > current_lr: 0.00005 + | > step_time: 2.40880 (2.60733) + | > loader_time: 0.00240 (0.02119) + + + --> STEP: 85/234 -- GLOBAL_STEP: 49225 + | > loss: -0.20588 (-0.21984) + | > log_mle: -0.31652 (-0.32492) + | > loss_dur: 0.11064 (0.10508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.22919 (13.25857) + | > current_lr: 0.00005 + | > step_time: 1.95710 (2.56171) + | > loader_time: 0.00380 (0.02016) + + + --> STEP: 90/234 -- GLOBAL_STEP: 49230 + | > loss: -0.18983 (-0.21863) + | > log_mle: -0.34073 (-0.32566) + | > loss_dur: 0.15090 (0.10703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.67062 (13.57443) + | > current_lr: 0.00005 + | > step_time: 1.59400 (2.53118) + | > loader_time: 0.00320 (0.01921) + + + --> STEP: 95/234 -- GLOBAL_STEP: 49235 + | > loss: -0.23581 (-0.21861) + | > log_mle: -0.42513 (-0.32826) + | > loss_dur: 0.18931 (0.10965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.66949 (14.37579) + | > current_lr: 0.00005 + | > step_time: 1.86820 (2.53756) + | > loader_time: 0.00320 (0.01942) + + + --> STEP: 100/234 -- GLOBAL_STEP: 49240 + | > loss: -0.21072 (-0.21787) + | > log_mle: -0.35311 (-0.32932) + | > loss_dur: 0.14239 (0.11145) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.18067 (14.56020) + | > current_lr: 0.00005 + | > step_time: 3.21640 (2.52780) + | > loader_time: 0.09180 (0.01949) + + + --> STEP: 105/234 -- GLOBAL_STEP: 49245 + | > loss: -0.19463 (-0.21779) + | > log_mle: -0.33170 (-0.33145) + | > loss_dur: 0.13706 (0.11366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.14138 (15.00330) + | > current_lr: 0.00005 + | > step_time: 3.09500 (2.56375) + | > loader_time: 0.00370 (0.01974) + + + --> STEP: 110/234 -- GLOBAL_STEP: 49250 + | > loss: -0.19303 (-0.21707) + | > log_mle: -0.35544 (-0.33324) + | > loss_dur: 0.16240 (0.11616) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.72858 (15.48471) + | > current_lr: 0.00005 + | > step_time: 1.91560 (2.54556) + | > loader_time: 0.09410 (0.02053) + + + --> STEP: 115/234 -- GLOBAL_STEP: 49255 + | > loss: -0.19733 (-0.21712) + | > log_mle: -0.37709 (-0.33586) + | > loss_dur: 0.17975 (0.11874) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.25583 (15.98930) + | > current_lr: 0.00005 + | > step_time: 1.38560 (2.52429) + | > loader_time: 0.00230 (0.01976) + + + --> STEP: 120/234 -- GLOBAL_STEP: 49260 + | > loss: -0.24886 (-0.21704) + | > log_mle: -0.42287 (-0.33803) + | > loss_dur: 0.17401 (0.12099) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.85184 (16.53227) + | > current_lr: 0.00005 + | > step_time: 2.30290 (2.50567) + | > loader_time: 0.00450 (0.02139) + + + --> STEP: 125/234 -- GLOBAL_STEP: 49265 + | > loss: -0.23296 (-0.21650) + | > log_mle: -0.40987 (-0.33897) + | > loss_dur: 0.17691 (0.12247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.60207 (16.77173) + | > current_lr: 0.00005 + | > step_time: 2.02090 (2.49200) + | > loader_time: 0.00260 (0.02070) + + + --> STEP: 130/234 -- GLOBAL_STEP: 49270 + | > loss: -0.24904 (-0.21719) + | > log_mle: -0.42809 (-0.34191) + | > loss_dur: 0.17905 (0.12473) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.12459 (17.50117) + | > current_lr: 0.00005 + | > step_time: 2.78130 (2.49180) + | > loader_time: 0.00600 (0.02071) + + + --> STEP: 135/234 -- GLOBAL_STEP: 49275 + | > loss: -0.21026 (-0.21835) + | > log_mle: -0.35210 (-0.34495) + | > loss_dur: 0.14183 (0.12660) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.33804 (18.26305) + | > current_lr: 0.00005 + | > step_time: 1.31060 (2.46046) + | > loader_time: 0.00230 (0.02126) + + + --> STEP: 140/234 -- GLOBAL_STEP: 49280 + | > loss: -0.20680 (-0.21953) + | > log_mle: -0.38715 (-0.34851) + | > loss_dur: 0.18035 (0.12898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.04939 (19.12109) + | > current_lr: 0.00005 + | > step_time: 1.59190 (2.46268) + | > loader_time: 0.00830 (0.02186) + + + --> STEP: 145/234 -- GLOBAL_STEP: 49285 + | > loss: -0.30316 (-0.22133) + | > log_mle: -0.49049 (-0.35268) + | > loss_dur: 0.18733 (0.13136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.81605 (20.24942) + | > current_lr: 0.00005 + | > step_time: 1.21430 (2.43214) + | > loader_time: 0.08400 (0.02181) + + + --> STEP: 150/234 -- GLOBAL_STEP: 49290 + | > loss: -0.27152 (-0.22327) + | > log_mle: -0.47443 (-0.35669) + | > loss_dur: 0.20291 (0.13341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.15909 (21.05214) + | > current_lr: 0.00005 + | > step_time: 1.70330 (2.44601) + | > loader_time: 0.07780 (0.02351) + + + --> STEP: 155/234 -- GLOBAL_STEP: 49295 + | > loss: -0.33066 (-0.22604) + | > log_mle: -0.52772 (-0.36148) + | > loss_dur: 0.19707 (0.13544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.60986 (22.42597) + | > current_lr: 0.00005 + | > step_time: 2.09660 (2.47494) + | > loader_time: 0.10760 (0.02534) + + + --> STEP: 160/234 -- GLOBAL_STEP: 49300 + | > loss: -0.32173 (-0.22804) + | > log_mle: -0.52709 (-0.36559) + | > loss_dur: 0.20536 (0.13756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.69413 (23.40728) + | > current_lr: 0.00005 + | > step_time: 5.50910 (2.53072) + | > loader_time: 0.09930 (0.02650) + + + --> STEP: 165/234 -- GLOBAL_STEP: 49305 + | > loss: -0.32028 (-0.23019) + | > log_mle: -0.52967 (-0.36963) + | > loss_dur: 0.20939 (0.13944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.08144 (24.30754) + | > current_lr: 0.00005 + | > step_time: 2.20800 (2.53108) + | > loader_time: 0.08440 (0.02628) + + + --> STEP: 170/234 -- GLOBAL_STEP: 49310 + | > loss: -0.33647 (-0.23265) + | > log_mle: -0.56854 (-0.37419) + | > loss_dur: 0.23206 (0.14154) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.25871 (25.31314) + | > current_lr: 0.00005 + | > step_time: 2.20420 (2.53356) + | > loader_time: 0.00290 (0.02684) + + + --> STEP: 175/234 -- GLOBAL_STEP: 49315 + | > loss: -0.28900 (-0.23544) + | > log_mle: -0.52565 (-0.37927) + | > loss_dur: 0.23665 (0.14384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.48406 (27.00261) + | > current_lr: 0.00005 + | > step_time: 4.29430 (2.54971) + | > loader_time: 0.00610 (0.02721) + + + --> STEP: 180/234 -- GLOBAL_STEP: 49320 + | > loss: -0.31603 (-0.23728) + | > log_mle: -0.52983 (-0.38336) + | > loss_dur: 0.21380 (0.14608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.31668 (28.45530) + | > current_lr: 0.00005 + | > step_time: 3.39250 (2.57276) + | > loader_time: 0.09850 (0.02713) + + + --> STEP: 185/234 -- GLOBAL_STEP: 49325 + | > loss: -0.33092 (-0.23930) + | > log_mle: -0.56326 (-0.38752) + | > loss_dur: 0.23233 (0.14822) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.60633 (29.30914) + | > current_lr: 0.00005 + | > step_time: 4.60570 (2.58284) + | > loader_time: 0.00700 (0.02778) + + + --> STEP: 190/234 -- GLOBAL_STEP: 49330 + | > loss: -0.33673 (-0.24163) + | > log_mle: -0.54565 (-0.39175) + | > loss_dur: 0.20892 (0.15013) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.55111 (30.12894) + | > current_lr: 0.00005 + | > step_time: 5.10600 (2.63653) + | > loader_time: 0.39190 (0.03027) + + + --> STEP: 195/234 -- GLOBAL_STEP: 49335 + | > loss: -0.34410 (-0.24449) + | > log_mle: -0.57313 (-0.39632) + | > loss_dur: 0.22903 (0.15183) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.55943 (31.02273) + | > current_lr: 0.00005 + | > step_time: 3.69280 (2.66994) + | > loader_time: 0.09380 (0.03199) + + + --> STEP: 200/234 -- GLOBAL_STEP: 49340 + | > loss: -0.31791 (-0.24695) + | > log_mle: -0.57018 (-0.40061) + | > loss_dur: 0.25227 (0.15367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.59053 (32.15596) + | > current_lr: 0.00005 + | > step_time: 3.51470 (2.76478) + | > loader_time: 0.09050 (0.03309) + + + --> STEP: 205/234 -- GLOBAL_STEP: 49345 + | > loss: -0.33550 (-0.24928) + | > log_mle: -0.56024 (-0.40468) + | > loss_dur: 0.22474 (0.15540) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.10245 (33.24406) + | > current_lr: 0.00005 + | > step_time: 4.20800 (2.80772) + | > loader_time: 0.00340 (0.03289) + + + --> STEP: 210/234 -- GLOBAL_STEP: 49350 + | > loss: -0.39605 (-0.25215) + | > log_mle: -0.63894 (-0.40950) + | > loss_dur: 0.24289 (0.15735) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.08958 (34.26119) + | > current_lr: 0.00005 + | > step_time: 4.99280 (2.89023) + | > loader_time: 0.19800 (0.03363) + + + --> STEP: 215/234 -- GLOBAL_STEP: 49355 + | > loss: -0.36363 (-0.25551) + | > log_mle: -0.59811 (-0.41463) + | > loss_dur: 0.23447 (0.15912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.74641 (35.23809) + | > current_lr: 0.00005 + | > step_time: 8.70700 (2.96769) + | > loader_time: 0.09720 (0.03556) + + + --> STEP: 220/234 -- GLOBAL_STEP: 49360 + | > loss: -0.42213 (-0.25913) + | > log_mle: -0.66054 (-0.42012) + | > loss_dur: 0.23841 (0.16099) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.32992 (36.52348) + | > current_lr: 0.00005 + | > step_time: 4.40220 (3.04854) + | > loader_time: 0.00970 (0.03699) + + + --> STEP: 225/234 -- GLOBAL_STEP: 49365 + | > loss: -0.45802 (-0.26223) + | > log_mle: -0.70614 (-0.42505) + | > loss_dur: 0.24812 (0.16282) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 141.71368 (38.07829) + | > current_lr: 0.00005 + | > step_time: 0.23780 (2.99399) + | > loader_time: 0.00290 (0.03626) + + + --> STEP: 230/234 -- GLOBAL_STEP: 49370 + | > loss: -0.42761 (-0.26473) + | > log_mle: -0.74862 (-0.43015) + | > loss_dur: 0.32101 (0.16542) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.39857 (39.97131) + | > current_lr: 0.00005 + | > step_time: 0.25340 (2.93417) + | > loader_time: 0.00330 (0.03555) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15306 (-0.15138) + | > avg_loss: -0.27445 (-0.00172) + | > avg_log_mle: -0.51389 (-0.00442) + | > avg_loss_dur: 0.23944 (+0.00270) + + + > EPOCH: 211/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 05:11:18)  + + --> STEP: 1/234 -- GLOBAL_STEP: 49375 + | > loss: -0.23737 (-0.23737) + | > log_mle: -0.33145 (-0.33145) + | > loss_dur: 0.09408 (0.09408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.01267 (23.01267) + | > current_lr: 0.00005 + | > step_time: 8.11520 (8.11525) + | > loader_time: 17.39360 (17.39363) + + + --> STEP: 6/234 -- GLOBAL_STEP: 49380 + | > loss: -0.25466 (-0.22927) + | > log_mle: -0.33000 (-0.33364) + | > loss_dur: 0.07534 (0.10437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.03440 (18.76660) + | > current_lr: 0.00005 + | > step_time: 4.40580 (4.36879) + | > loader_time: 0.08140 (2.94390) + + + --> STEP: 11/234 -- GLOBAL_STEP: 49385 + | > loss: -0.26980 (-0.23746) + | > log_mle: -0.33933 (-0.33704) + | > loss_dur: 0.06953 (0.09957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.17504 (17.18450) + | > current_lr: 0.00005 + | > step_time: 2.28890 (3.48186) + | > loader_time: 0.00160 (1.62343) + + + --> STEP: 16/234 -- GLOBAL_STEP: 49390 + | > loss: -0.24906 (-0.24020) + | > log_mle: -0.33502 (-0.33767) + | > loss_dur: 0.08595 (0.09747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.29910 (16.18015) + | > current_lr: 0.00005 + | > step_time: 2.19380 (3.47318) + | > loader_time: 0.00200 (1.12847) + + + --> STEP: 21/234 -- GLOBAL_STEP: 49395 + | > loss: -0.23041 (-0.24197) + | > log_mle: -0.31156 (-0.33523) + | > loss_dur: 0.08115 (0.09327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.76733 (14.93001) + | > current_lr: 0.00005 + | > step_time: 6.40700 (4.08492) + | > loader_time: 0.00600 (0.86555) + + + --> STEP: 26/234 -- GLOBAL_STEP: 49400 + | > loss: -0.23829 (-0.24292) + | > log_mle: -0.32589 (-0.33426) + | > loss_dur: 0.08760 (0.09134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.64076 (14.07209) + | > current_lr: 0.00005 + | > step_time: 1.83600 (3.55121) + | > loader_time: 0.00110 (0.70348) + + + --> STEP: 31/234 -- GLOBAL_STEP: 49405 + | > loss: -0.21176 (-0.24223) + | > log_mle: -0.32112 (-0.33282) + | > loss_dur: 0.10936 (0.09059) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.67438 (13.79183) + | > current_lr: 0.00005 + | > step_time: 5.19620 (3.35599) + | > loader_time: 0.08760 (0.59854) + + + --> STEP: 36/234 -- GLOBAL_STEP: 49410 + | > loss: -0.21493 (-0.24023) + | > log_mle: -0.31941 (-0.33158) + | > loss_dur: 0.10448 (0.09136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17711 (13.29857) + | > current_lr: 0.00005 + | > step_time: 2.26240 (3.21623) + | > loader_time: 0.00200 (0.51562) + + + --> STEP: 41/234 -- GLOBAL_STEP: 49415 + | > loss: -0.23050 (-0.23760) + | > log_mle: -0.32134 (-0.33021) + | > loss_dur: 0.09084 (0.09261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.28129 (13.06559) + | > current_lr: 0.00005 + | > step_time: 1.81120 (3.01327) + | > loader_time: 0.00280 (0.45501) + + + --> STEP: 46/234 -- GLOBAL_STEP: 49420 + | > loss: -0.20026 (-0.23490) + | > log_mle: -0.31690 (-0.32917) + | > loss_dur: 0.11664 (0.09428) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.94163 (13.00747) + | > current_lr: 0.00005 + | > step_time: 1.61600 (2.87887) + | > loader_time: 0.18400 (0.41178) + + + --> STEP: 51/234 -- GLOBAL_STEP: 49425 + | > loss: -0.20525 (-0.23288) + | > log_mle: -0.30995 (-0.32779) + | > loss_dur: 0.10470 (0.09491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.20450 (12.80973) + | > current_lr: 0.00005 + | > step_time: 1.28560 (2.80784) + | > loader_time: 0.00170 (0.37176) + + + --> STEP: 56/234 -- GLOBAL_STEP: 49430 + | > loss: -0.20202 (-0.23089) + | > log_mle: -0.32194 (-0.32713) + | > loss_dur: 0.11992 (0.09624) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.02484 (12.84642) + | > current_lr: 0.00005 + | > step_time: 1.79390 (2.70690) + | > loader_time: 0.00120 (0.33886) + + + --> STEP: 61/234 -- GLOBAL_STEP: 49435 + | > loss: -0.21189 (-0.22890) + | > log_mle: -0.31973 (-0.32669) + | > loss_dur: 0.10784 (0.09779) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.96393 (12.67736) + | > current_lr: 0.00005 + | > step_time: 2.11380 (2.62258) + | > loader_time: 0.08590 (0.31411) + + + --> STEP: 66/234 -- GLOBAL_STEP: 49440 + | > loss: -0.23015 (-0.22727) + | > log_mle: -0.31605 (-0.32629) + | > loss_dur: 0.08590 (0.09902) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.58637 (12.68956) + | > current_lr: 0.00005 + | > step_time: 2.67870 (2.63033) + | > loader_time: 0.00230 (0.29196) + + + --> STEP: 71/234 -- GLOBAL_STEP: 49445 + | > loss: -0.18384 (-0.22422) + | > log_mle: -0.33831 (-0.32559) + | > loss_dur: 0.15448 (0.10137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.56085 (12.79066) + | > current_lr: 0.00005 + | > step_time: 2.00130 (2.59753) + | > loader_time: 0.00260 (0.27283) + + + --> STEP: 76/234 -- GLOBAL_STEP: 49450 + | > loss: -0.19968 (-0.22207) + | > log_mle: -0.32296 (-0.32512) + | > loss_dur: 0.12328 (0.10305) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57728 (12.77505) + | > current_lr: 0.00005 + | > step_time: 0.99360 (2.57669) + | > loader_time: 0.00260 (0.25732) + + + --> STEP: 81/234 -- GLOBAL_STEP: 49455 + | > loss: -0.20244 (-0.22056) + | > log_mle: -0.33254 (-0.32466) + | > loss_dur: 0.13010 (0.10410) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.75413 (12.68494) + | > current_lr: 0.00005 + | > step_time: 3.60120 (2.60409) + | > loader_time: 0.09970 (0.24629) + + + --> STEP: 86/234 -- GLOBAL_STEP: 49460 + | > loss: -0.19949 (-0.21897) + | > log_mle: -0.32919 (-0.32446) + | > loss_dur: 0.12970 (0.10549) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57354 (12.67462) + | > current_lr: 0.00005 + | > step_time: 0.76540 (2.57800) + | > loader_time: 0.00170 (0.23317) + + + --> STEP: 91/234 -- GLOBAL_STEP: 49465 + | > loss: -0.19575 (-0.21800) + | > log_mle: -0.34059 (-0.32540) + | > loss_dur: 0.14485 (0.10740) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78433 (12.82438) + | > current_lr: 0.00005 + | > step_time: 1.68270 (2.55462) + | > loader_time: 0.00190 (0.22165) + + + --> STEP: 96/234 -- GLOBAL_STEP: 49470 + | > loss: -0.19501 (-0.21819) + | > log_mle: -0.32797 (-0.32800) + | > loss_dur: 0.13296 (0.10981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.25616 (13.32128) + | > current_lr: 0.00005 + | > step_time: 3.30350 (2.55193) + | > loader_time: 0.00610 (0.21120) + + + --> STEP: 101/234 -- GLOBAL_STEP: 49475 + | > loss: -0.22296 (-0.21783) + | > log_mle: -0.38268 (-0.32965) + | > loss_dur: 0.15972 (0.11182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.74132 (13.69631) + | > current_lr: 0.00005 + | > step_time: 2.61680 (2.55202) + | > loader_time: 0.09340 (0.20368) + + + --> STEP: 106/234 -- GLOBAL_STEP: 49480 + | > loss: -0.19139 (-0.21777) + | > log_mle: -0.37987 (-0.33186) + | > loss_dur: 0.18849 (0.11409) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.41910 (14.24852) + | > current_lr: 0.00005 + | > step_time: 1.61480 (2.54410) + | > loader_time: 0.07730 (0.19746) + + + --> STEP: 111/234 -- GLOBAL_STEP: 49485 + | > loss: -0.23344 (-0.21733) + | > log_mle: -0.43343 (-0.33407) + | > loss_dur: 0.19999 (0.11674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.18343 (14.94584) + | > current_lr: 0.00005 + | > step_time: 1.78460 (2.51223) + | > loader_time: 0.00150 (0.19019) + + + --> STEP: 116/234 -- GLOBAL_STEP: 49490 + | > loss: -0.19509 (-0.21713) + | > log_mle: -0.39322 (-0.33633) + | > loss_dur: 0.19813 (0.11921) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.70773 (15.63191) + | > current_lr: 0.00005 + | > step_time: 1.06280 (2.50618) + | > loader_time: 0.00190 (0.18290) + + + --> STEP: 121/234 -- GLOBAL_STEP: 49495 + | > loss: -0.17531 (-0.21705) + | > log_mle: -0.30896 (-0.33783) + | > loss_dur: 0.13365 (0.12078) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.44572 (15.95392) + | > current_lr: 0.00005 + | > step_time: 2.09320 (2.50831) + | > loader_time: 0.00150 (0.17853) + + + --> STEP: 126/234 -- GLOBAL_STEP: 49500 + | > loss: -0.26048 (-0.21732) + | > log_mle: -0.44627 (-0.33993) + | > loss_dur: 0.18578 (0.12262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.94987 (16.52084) + | > current_lr: 0.00005 + | > step_time: 3.49530 (2.57538) + | > loader_time: 0.10390 (0.17389) + + + --> STEP: 131/234 -- GLOBAL_STEP: 49505 + | > loss: -0.30337 (-0.21846) + | > log_mle: -0.49566 (-0.34323) + | > loss_dur: 0.19229 (0.12478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.15199 (17.39196) + | > current_lr: 0.00005 + | > step_time: 6.40770 (2.61623) + | > loader_time: 0.09090 (0.17000) + + + --> STEP: 136/234 -- GLOBAL_STEP: 49510 + | > loss: -0.31683 (-0.21968) + | > log_mle: -0.53497 (-0.34649) + | > loss_dur: 0.21814 (0.12681) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.30951 (18.28465) + | > current_lr: 0.00005 + | > step_time: 4.41020 (2.64895) + | > loader_time: 0.08990 (0.16653) + + + --> STEP: 141/234 -- GLOBAL_STEP: 49515 + | > loss: -0.24718 (-0.22019) + | > log_mle: -0.42460 (-0.34905) + | > loss_dur: 0.17742 (0.12886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.44307 (19.23680) + | > current_lr: 0.00005 + | > step_time: 2.21160 (2.61364) + | > loader_time: 0.08830 (0.16310) + + + --> STEP: 146/234 -- GLOBAL_STEP: 49520 + | > loss: -0.27591 (-0.22152) + | > log_mle: -0.47709 (-0.35294) + | > loss_dur: 0.20118 (0.13141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.98197 (20.44505) + | > current_lr: 0.00005 + | > step_time: 2.10550 (2.64628) + | > loader_time: 0.08980 (0.16013) + + + --> STEP: 151/234 -- GLOBAL_STEP: 49525 + | > loss: -0.27528 (-0.22318) + | > log_mle: -0.44954 (-0.35635) + | > loss_dur: 0.17426 (0.13317) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.33208 (21.13435) + | > current_lr: 0.00005 + | > step_time: 2.29990 (2.66731) + | > loader_time: 0.09290 (0.15671) + + + --> STEP: 156/234 -- GLOBAL_STEP: 49530 + | > loss: -0.30471 (-0.22622) + | > log_mle: -0.49842 (-0.36146) + | > loss_dur: 0.19371 (0.13524) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.94960 (22.35386) + | > current_lr: 0.00005 + | > step_time: 2.29820 (2.70563) + | > loader_time: 0.01030 (0.15412) + + + --> STEP: 161/234 -- GLOBAL_STEP: 49535 + | > loss: -0.34502 (-0.22866) + | > log_mle: -0.52564 (-0.36590) + | > loss_dur: 0.18062 (0.13724) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.01762 (23.24891) + | > current_lr: 0.00005 + | > step_time: 5.32440 (2.70826) + | > loader_time: 0.08280 (0.14997) + + + --> STEP: 166/234 -- GLOBAL_STEP: 49540 + | > loss: -0.27785 (-0.23050) + | > log_mle: -0.45762 (-0.36956) + | > loss_dur: 0.17977 (0.13906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.33529 (24.36688) + | > current_lr: 0.00005 + | > step_time: 1.59610 (2.68050) + | > loader_time: 0.00190 (0.14607) + + + --> STEP: 171/234 -- GLOBAL_STEP: 49545 + | > loss: -0.36020 (-0.23354) + | > log_mle: -0.57262 (-0.37476) + | > loss_dur: 0.21242 (0.14123) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.82263 (25.58397) + | > current_lr: 0.00005 + | > step_time: 5.49480 (2.75349) + | > loader_time: 0.10120 (0.14348) + + + --> STEP: 176/234 -- GLOBAL_STEP: 49550 + | > loss: -0.33408 (-0.23637) + | > log_mle: -0.54253 (-0.37986) + | > loss_dur: 0.20845 (0.14349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.97451 (26.71135) + | > current_lr: 0.00005 + | > step_time: 2.99750 (2.75667) + | > loader_time: 0.07710 (0.14176) + + + --> STEP: 181/234 -- GLOBAL_STEP: 49555 + | > loss: -0.26854 (-0.23864) + | > log_mle: -0.47992 (-0.38440) + | > loss_dur: 0.21138 (0.14576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.11188 (27.73251) + | > current_lr: 0.00005 + | > step_time: 2.90990 (2.75963) + | > loader_time: 0.00320 (0.13989) + + + --> STEP: 186/234 -- GLOBAL_STEP: 49560 + | > loss: -0.28561 (-0.24112) + | > log_mle: -0.51920 (-0.38915) + | > loss_dur: 0.23359 (0.14803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.87875 (28.94025) + | > current_lr: 0.00005 + | > step_time: 4.10440 (2.80013) + | > loader_time: 0.10390 (0.13812) + + + --> STEP: 191/234 -- GLOBAL_STEP: 49565 + | > loss: -0.32988 (-0.24389) + | > log_mle: -0.54390 (-0.39376) + | > loss_dur: 0.21402 (0.14986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.44233 (29.93998) + | > current_lr: 0.00005 + | > step_time: 5.38740 (2.82992) + | > loader_time: 0.00390 (0.13704) + + + --> STEP: 196/234 -- GLOBAL_STEP: 49570 + | > loss: -0.31798 (-0.24705) + | > log_mle: -0.53781 (-0.39862) + | > loss_dur: 0.21983 (0.15157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.19167 (31.26408) + | > current_lr: 0.00005 + | > step_time: 2.30390 (2.87158) + | > loader_time: 0.00360 (0.13502) + + + --> STEP: 201/234 -- GLOBAL_STEP: 49575 + | > loss: -0.26498 (-0.24931) + | > log_mle: -0.49502 (-0.40275) + | > loss_dur: 0.23005 (0.15344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.65509 (32.57790) + | > current_lr: 0.00005 + | > step_time: 12.49430 (2.96467) + | > loader_time: 0.09260 (0.13413) + + + --> STEP: 206/234 -- GLOBAL_STEP: 49580 + | > loss: -0.37897 (-0.25204) + | > log_mle: -0.60195 (-0.40729) + | > loss_dur: 0.22297 (0.15526) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.82621 (33.70438) + | > current_lr: 0.00005 + | > step_time: 2.01440 (3.03552) + | > loader_time: 0.00300 (0.13624) + + + --> STEP: 211/234 -- GLOBAL_STEP: 49585 + | > loss: -0.43837 (-0.25542) + | > log_mle: -0.67908 (-0.41257) + | > loss_dur: 0.24071 (0.15715) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 115.84626 (34.92914) + | > current_lr: 0.00005 + | > step_time: 10.99270 (3.09568) + | > loader_time: 0.10950 (0.13493) + + + --> STEP: 216/234 -- GLOBAL_STEP: 49590 + | > loss: -0.41566 (-0.25871) + | > log_mle: -0.66666 (-0.41761) + | > loss_dur: 0.25100 (0.15890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.20098 (36.44517) + | > current_lr: 0.00005 + | > step_time: 4.00710 (3.11573) + | > loader_time: 0.08360 (0.13315) + + + --> STEP: 221/234 -- GLOBAL_STEP: 49595 + | > loss: -0.36154 (-0.26218) + | > log_mle: -0.57991 (-0.42275) + | > loss_dur: 0.21837 (0.16057) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.67203 (37.47824) + | > current_lr: 0.00005 + | > step_time: 4.18930 (3.13619) + | > loader_time: 0.01090 (0.13110) + + + --> STEP: 226/234 -- GLOBAL_STEP: 49600 + | > loss: -0.44492 (-0.26609) + | > log_mle: -0.68719 (-0.42850) + | > loss_dur: 0.24226 (0.16241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.57140 (38.55276) + | > current_lr: 0.00005 + | > step_time: 0.24550 (3.11709) + | > loader_time: 0.00290 (0.12874) + + + --> STEP: 231/234 -- GLOBAL_STEP: 49605 + | > loss: -0.37811 (-0.26898) + | > log_mle: -0.75662 (-0.43452) + | > loss_dur: 0.37851 (0.16554) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 121.15762 (40.22933) + | > current_lr: 0.00005 + | > step_time: 0.28950 (3.05528) + | > loader_time: 0.00420 (0.12603) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.19832 (+0.04527) + | > avg_loss: -0.29149 (-0.01703) + | > avg_log_mle: -0.51715 (-0.00326) + | > avg_loss_dur: 0.22566 (-0.01378) + + + > EPOCH: 212/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 05:24:37)  + + --> STEP: 2/234 -- GLOBAL_STEP: 49610 + | > loss: -0.24461 (-0.24161) + | > log_mle: -0.35035 (-0.34228) + | > loss_dur: 0.10574 (0.10066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.59829 (17.20261) + | > current_lr: 0.00005 + | > step_time: 3.90820 (6.50103) + | > loader_time: 0.00320 (1.15403) + + + --> STEP: 7/234 -- GLOBAL_STEP: 49615 + | > loss: -0.25397 (-0.23317) + | > log_mle: -0.34035 (-0.33813) + | > loss_dur: 0.08638 (0.10496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.81460 (17.82218) + | > current_lr: 0.00005 + | > step_time: 6.19820 (6.29820) + | > loader_time: 0.09620 (0.34573) + + + --> STEP: 12/234 -- GLOBAL_STEP: 49620 + | > loss: -0.22483 (-0.23793) + | > log_mle: -0.33481 (-0.34037) + | > loss_dur: 0.10998 (0.10244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.82756 (16.60691) + | > current_lr: 0.00005 + | > step_time: 0.92340 (5.35059) + | > loader_time: 0.08200 (0.26001) + + + --> STEP: 17/234 -- GLOBAL_STEP: 49625 + | > loss: -0.24679 (-0.24147) + | > log_mle: -0.32573 (-0.33981) + | > loss_dur: 0.07894 (0.09834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.29558 (15.59220) + | > current_lr: 0.00005 + | > step_time: 5.98260 (4.51888) + | > loader_time: 0.00110 (0.18861) + + + --> STEP: 22/234 -- GLOBAL_STEP: 49630 + | > loss: -0.23917 (-0.24175) + | > log_mle: -0.33516 (-0.33751) + | > loss_dur: 0.09599 (0.09577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.44612 (15.19719) + | > current_lr: 0.00005 + | > step_time: 4.29690 (4.23702) + | > loader_time: 0.00130 (0.15574) + + + --> STEP: 27/234 -- GLOBAL_STEP: 49635 + | > loss: -0.23932 (-0.24238) + | > log_mle: -0.33001 (-0.33594) + | > loss_dur: 0.09068 (0.09357) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.63429 (14.42834) + | > current_lr: 0.00005 + | > step_time: 1.76530 (3.75038) + | > loader_time: 0.00160 (0.13001) + + + --> STEP: 32/234 -- GLOBAL_STEP: 49640 + | > loss: -0.25295 (-0.24171) + | > log_mle: -0.33834 (-0.33480) + | > loss_dur: 0.08539 (0.09309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.97536 (13.96028) + | > current_lr: 0.00005 + | > step_time: 7.81190 (3.65305) + | > loader_time: 0.08400 (0.12150) + + + --> STEP: 37/234 -- GLOBAL_STEP: 49645 + | > loss: -0.22753 (-0.23914) + | > log_mle: -0.31397 (-0.33257) + | > loss_dur: 0.08645 (0.09342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.59830 (13.83420) + | > current_lr: 0.00005 + | > step_time: 1.08990 (3.34773) + | > loader_time: 0.00490 (0.10975) + + + --> STEP: 42/234 -- GLOBAL_STEP: 49650 + | > loss: -0.20761 (-0.23654) + | > log_mle: -0.30716 (-0.33088) + | > loss_dur: 0.09955 (0.09434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.93665 (13.51458) + | > current_lr: 0.00005 + | > step_time: 3.11540 (3.18043) + | > loader_time: 0.08430 (0.10328) + + + --> STEP: 47/234 -- GLOBAL_STEP: 49655 + | > loss: -0.20675 (-0.23382) + | > log_mle: -0.31621 (-0.32990) + | > loss_dur: 0.10945 (0.09608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.83252 (13.33405) + | > current_lr: 0.00005 + | > step_time: 1.60870 (2.99990) + | > loader_time: 0.08690 (0.09590) + + + --> STEP: 52/234 -- GLOBAL_STEP: 49660 + | > loss: -0.19973 (-0.23262) + | > log_mle: -0.31299 (-0.32866) + | > loss_dur: 0.11326 (0.09604) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03250 (12.88899) + | > current_lr: 0.00005 + | > step_time: 0.80710 (2.84652) + | > loader_time: 0.00250 (0.08861) + + + --> STEP: 57/234 -- GLOBAL_STEP: 49665 + | > loss: -0.18669 (-0.23109) + | > log_mle: -0.30583 (-0.32798) + | > loss_dur: 0.11914 (0.09689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.00645 (12.62885) + | > current_lr: 0.00005 + | > step_time: 5.80880 (2.93578) + | > loader_time: 0.20240 (0.08772) + + + --> STEP: 62/234 -- GLOBAL_STEP: 49670 + | > loss: -0.16819 (-0.22920) + | > log_mle: -0.34185 (-0.32817) + | > loss_dur: 0.17365 (0.09896) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.34603 (12.80560) + | > current_lr: 0.00005 + | > step_time: 3.41240 (2.88605) + | > loader_time: 0.07730 (0.08206) + + + --> STEP: 67/234 -- GLOBAL_STEP: 49675 + | > loss: -0.19848 (-0.22768) + | > log_mle: -0.32712 (-0.32745) + | > loss_dur: 0.12864 (0.09977) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.13490 (12.60119) + | > current_lr: 0.00005 + | > step_time: 5.70950 (2.89625) + | > loader_time: 0.09990 (0.07758) + + + --> STEP: 72/234 -- GLOBAL_STEP: 49680 + | > loss: -0.21036 (-0.22516) + | > log_mle: -0.31590 (-0.32668) + | > loss_dur: 0.10554 (0.10152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.82877 (12.62702) + | > current_lr: 0.00005 + | > step_time: 2.19660 (2.80354) + | > loader_time: 0.00310 (0.07239) + + + --> STEP: 77/234 -- GLOBAL_STEP: 49685 + | > loss: -0.21567 (-0.22324) + | > log_mle: -0.32147 (-0.32639) + | > loss_dur: 0.10580 (0.10316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.45544 (12.65854) + | > current_lr: 0.00005 + | > step_time: 2.08620 (2.78124) + | > loader_time: 0.00170 (0.07113) + + + --> STEP: 82/234 -- GLOBAL_STEP: 49690 + | > loss: -0.20360 (-0.22189) + | > log_mle: -0.31794 (-0.32587) + | > loss_dur: 0.11434 (0.10398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.98546 (12.57842) + | > current_lr: 0.00005 + | > step_time: 1.69340 (2.71138) + | > loader_time: 0.00260 (0.06696) + + + --> STEP: 87/234 -- GLOBAL_STEP: 49695 + | > loss: -0.19578 (-0.22058) + | > log_mle: -0.31944 (-0.32571) + | > loss_dur: 0.12366 (0.10513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08748 (12.62630) + | > current_lr: 0.00005 + | > step_time: 1.60400 (2.67651) + | > loader_time: 0.00290 (0.06422) + + + --> STEP: 92/234 -- GLOBAL_STEP: 49700 + | > loss: -0.22544 (-0.21991) + | > log_mle: -0.36338 (-0.32713) + | > loss_dur: 0.13794 (0.10722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.37930 (12.90526) + | > current_lr: 0.00005 + | > step_time: 1.48810 (2.63764) + | > loader_time: 0.00720 (0.06353) + + + --> STEP: 97/234 -- GLOBAL_STEP: 49705 + | > loss: -0.20446 (-0.22000) + | > log_mle: -0.34944 (-0.32936) + | > loss_dur: 0.14498 (0.10937) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.01820 (13.41919) + | > current_lr: 0.00005 + | > step_time: 1.78400 (2.61859) + | > loader_time: 0.00220 (0.06212) + + + --> STEP: 102/234 -- GLOBAL_STEP: 49710 + | > loss: -0.18159 (-0.21937) + | > log_mle: -0.33450 (-0.33078) + | > loss_dur: 0.15291 (0.11141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.77253 (13.80395) + | > current_lr: 0.00005 + | > step_time: 2.09800 (2.57325) + | > loader_time: 0.00170 (0.05920) + + + --> STEP: 107/234 -- GLOBAL_STEP: 49715 + | > loss: -0.21058 (-0.21943) + | > log_mle: -0.37609 (-0.33327) + | > loss_dur: 0.16550 (0.11384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.19270 (14.48499) + | > current_lr: 0.00005 + | > step_time: 1.35960 (2.54972) + | > loader_time: 0.10220 (0.06102) + + + --> STEP: 112/234 -- GLOBAL_STEP: 49720 + | > loss: -0.21150 (-0.21906) + | > log_mle: -0.39017 (-0.33559) + | > loss_dur: 0.17867 (0.11653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.81166 (15.04071) + | > current_lr: 0.00005 + | > step_time: 2.40770 (2.52434) + | > loader_time: 0.09100 (0.05920) + + + --> STEP: 117/234 -- GLOBAL_STEP: 49725 + | > loss: -0.23218 (-0.21898) + | > log_mle: -0.38434 (-0.33771) + | > loss_dur: 0.15216 (0.11873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.33023 (15.64139) + | > current_lr: 0.00005 + | > step_time: 1.60700 (2.49510) + | > loader_time: 0.00260 (0.05681) + + + --> STEP: 122/234 -- GLOBAL_STEP: 49730 + | > loss: -0.20067 (-0.21836) + | > log_mle: -0.35435 (-0.33882) + | > loss_dur: 0.15368 (0.12046) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.43197 (15.98228) + | > current_lr: 0.00005 + | > step_time: 1.79560 (2.49310) + | > loader_time: 0.00220 (0.05608) + + + --> STEP: 127/234 -- GLOBAL_STEP: 49735 + | > loss: -0.23226 (-0.21871) + | > log_mle: -0.42000 (-0.34133) + | > loss_dur: 0.18774 (0.12262) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.97740 (16.62419) + | > current_lr: 0.00005 + | > step_time: 1.60400 (2.49434) + | > loader_time: 0.00570 (0.05601) + + + --> STEP: 132/234 -- GLOBAL_STEP: 49740 + | > loss: -0.23559 (-0.21963) + | > log_mle: -0.39633 (-0.34436) + | > loss_dur: 0.16074 (0.12473) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.98907 (17.47149) + | > current_lr: 0.00005 + | > step_time: 5.40940 (2.52415) + | > loader_time: 0.08880 (0.05685) + + + --> STEP: 137/234 -- GLOBAL_STEP: 49745 + | > loss: -0.22677 (-0.22073) + | > log_mle: -0.41467 (-0.34769) + | > loss_dur: 0.18791 (0.12696) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.80694 (18.21485) + | > current_lr: 0.00005 + | > step_time: 5.91660 (2.58318) + | > loader_time: 0.09320 (0.05766) + + + --> STEP: 142/234 -- GLOBAL_STEP: 49750 + | > loss: -0.23529 (-0.22161) + | > log_mle: -0.42785 (-0.35054) + | > loss_dur: 0.19255 (0.12893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.52868 (18.90779) + | > current_lr: 0.00005 + | > step_time: 3.81720 (2.57420) + | > loader_time: 0.07680 (0.05673) + + + --> STEP: 147/234 -- GLOBAL_STEP: 49755 + | > loss: -0.24119 (-0.22370) + | > log_mle: -0.42827 (-0.35510) + | > loss_dur: 0.18709 (0.13140) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.06732 (20.13580) + | > current_lr: 0.00005 + | > step_time: 1.20970 (2.56702) + | > loader_time: 0.00210 (0.05557) + + + --> STEP: 152/234 -- GLOBAL_STEP: 49760 + | > loss: -0.30137 (-0.22606) + | > log_mle: -0.51752 (-0.35939) + | > loss_dur: 0.21615 (0.13332) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.63424 (21.04443) + | > current_lr: 0.00005 + | > step_time: 2.50860 (2.54893) + | > loader_time: 0.08510 (0.05444) + + + --> STEP: 157/234 -- GLOBAL_STEP: 49765 + | > loss: -0.25951 (-0.22886) + | > log_mle: -0.46362 (-0.36431) + | > loss_dur: 0.20411 (0.13544) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.48225 (22.20800) + | > current_lr: 0.00005 + | > step_time: 3.90500 (2.57742) + | > loader_time: 0.09430 (0.05403) + + + --> STEP: 162/234 -- GLOBAL_STEP: 49770 + | > loss: -0.30844 (-0.23144) + | > log_mle: -0.49371 (-0.36903) + | > loss_dur: 0.18527 (0.13759) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.30783 (23.23371) + | > current_lr: 0.00005 + | > step_time: 3.18440 (2.55472) + | > loader_time: 0.00570 (0.05300) + + + --> STEP: 167/234 -- GLOBAL_STEP: 49775 + | > loss: -0.39493 (-0.23390) + | > log_mle: -0.59017 (-0.37339) + | > loss_dur: 0.19524 (0.13949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.53465 (24.47260) + | > current_lr: 0.00005 + | > step_time: 1.59770 (2.56646) + | > loader_time: 0.08840 (0.05304) + + + --> STEP: 172/234 -- GLOBAL_STEP: 49780 + | > loss: -0.34650 (-0.23681) + | > log_mle: -0.56970 (-0.37857) + | > loss_dur: 0.22320 (0.14177) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.45106 (25.95572) + | > current_lr: 0.00005 + | > step_time: 2.39860 (2.59371) + | > loader_time: 0.01110 (0.05322) + + + --> STEP: 177/234 -- GLOBAL_STEP: 49785 + | > loss: -0.32022 (-0.23950) + | > log_mle: -0.52989 (-0.38343) + | > loss_dur: 0.20967 (0.14393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.05299 (27.04663) + | > current_lr: 0.00005 + | > step_time: 2.61180 (2.59298) + | > loader_time: 0.08710 (0.05331) + + + --> STEP: 182/234 -- GLOBAL_STEP: 49790 + | > loss: -0.32838 (-0.24197) + | > log_mle: -0.57523 (-0.38826) + | > loss_dur: 0.24686 (0.14629) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.91920 (28.17270) + | > current_lr: 0.00005 + | > step_time: 2.11700 (2.62291) + | > loader_time: 0.08670 (0.05341) + + + --> STEP: 187/234 -- GLOBAL_STEP: 49795 + | > loss: -0.34604 (-0.24467) + | > log_mle: -0.58145 (-0.39307) + | > loss_dur: 0.23541 (0.14840) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.87381 (29.45613) + | > current_lr: 0.00005 + | > step_time: 1.90250 (2.66394) + | > loader_time: 0.00270 (0.05305) + + + --> STEP: 192/234 -- GLOBAL_STEP: 49800 + | > loss: -0.39553 (-0.24767) + | > log_mle: -0.60921 (-0.39780) + | > loss_dur: 0.21368 (0.15014) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.37257 (30.55271) + | > current_lr: 0.00005 + | > step_time: 2.60220 (2.70243) + | > loader_time: 0.00440 (0.05273) + + + --> STEP: 197/234 -- GLOBAL_STEP: 49805 + | > loss: -0.37874 (-0.25068) + | > log_mle: -0.58043 (-0.40245) + | > loss_dur: 0.20169 (0.15177) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.67503 (31.55966) + | > current_lr: 0.00005 + | > step_time: 6.38890 (2.83040) + | > loader_time: 0.01200 (0.05352) + + + --> STEP: 202/234 -- GLOBAL_STEP: 49810 + | > loss: -0.44509 (-0.25334) + | > log_mle: -0.66809 (-0.40703) + | > loss_dur: 0.22299 (0.15369) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 108.75077 (32.89645) + | > current_lr: 0.00005 + | > step_time: 3.00060 (2.89558) + | > loader_time: 0.00420 (0.05414) + + + --> STEP: 207/234 -- GLOBAL_STEP: 49815 + | > loss: -0.42068 (-0.25620) + | > log_mle: -0.65498 (-0.41167) + | > loss_dur: 0.23430 (0.15547) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.54364 (33.92659) + | > current_lr: 0.00005 + | > step_time: 11.33700 (2.95872) + | > loader_time: 0.09060 (0.05502) + + + --> STEP: 212/234 -- GLOBAL_STEP: 49820 + | > loss: -0.39383 (-0.25957) + | > log_mle: -0.63553 (-0.41698) + | > loss_dur: 0.24170 (0.15741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.55839 (35.05132) + | > current_lr: 0.00005 + | > step_time: 1.50910 (3.01535) + | > loader_time: 0.00530 (0.05748) + + + --> STEP: 217/234 -- GLOBAL_STEP: 49825 + | > loss: -0.41178 (-0.26293) + | > log_mle: -0.65505 (-0.42220) + | > loss_dur: 0.24326 (0.15926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 125.05248 (36.53228) + | > current_lr: 0.00005 + | > step_time: 2.29750 (3.01962) + | > loader_time: 0.09130 (0.05827) + + + --> STEP: 222/234 -- GLOBAL_STEP: 49830 + | > loss: -0.40931 (-0.26624) + | > log_mle: -0.67202 (-0.42733) + | > loss_dur: 0.26271 (0.16109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.81371 (37.89197) + | > current_lr: 0.00005 + | > step_time: 5.70070 (3.05067) + | > loader_time: 0.00770 (0.05876) + + + --> STEP: 227/234 -- GLOBAL_STEP: 49835 + | > loss: -0.38050 (-0.26995) + | > log_mle: -0.64716 (-0.43295) + | > loss_dur: 0.26665 (0.16300) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.13740 (39.17764) + | > current_lr: 0.00005 + | > step_time: 2.60840 (3.04208) + | > loader_time: 0.00670 (0.05834) + + + --> STEP: 232/234 -- GLOBAL_STEP: 49840 + | > loss: -0.37477 (-0.27257) + | > log_mle: -0.85705 (-0.43966) + | > loss_dur: 0.48227 (0.16709) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 118.51661 (41.09569) + | > current_lr: 0.00005 + | > step_time: 0.33240 (2.99098) + | > loader_time: 0.00590 (0.05718) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.99876 (+0.80044) + | > avg_loss: -0.28061 (+0.01088) + | > avg_log_mle: -0.50399 (+0.01316) + | > avg_loss_dur: 0.22338 (-0.00228) + + + > EPOCH: 213/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 05:37:25)  + + --> STEP: 3/234 -- GLOBAL_STEP: 49845 + | > loss: -0.18378 (-0.22949) + | > log_mle: -0.33094 (-0.33971) + | > loss_dur: 0.14717 (0.11022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.22615 (21.90738) + | > current_lr: 0.00005 + | > step_time: 5.70070 (5.93369) + | > loader_time: 0.28800 (0.32790) + + + --> STEP: 8/234 -- GLOBAL_STEP: 49850 + | > loss: -0.26061 (-0.23989) + | > log_mle: -0.35410 (-0.34061) + | > loss_dur: 0.09349 (0.10072) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.92997 (21.30777) + | > current_lr: 0.00005 + | > step_time: 0.71410 (3.99350) + | > loader_time: 0.08170 (0.15850) + + + --> STEP: 13/234 -- GLOBAL_STEP: 49855 + | > loss: -0.27434 (-0.24226) + | > log_mle: -0.34844 (-0.34102) + | > loss_dur: 0.07410 (0.09876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.38860 (19.64796) + | > current_lr: 0.00005 + | > step_time: 1.90820 (3.24284) + | > loader_time: 0.00170 (0.11067) + + + --> STEP: 18/234 -- GLOBAL_STEP: 49860 + | > loss: -0.22188 (-0.24280) + | > log_mle: -0.32571 (-0.33941) + | > loss_dur: 0.10382 (0.09661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.60603 (17.73453) + | > current_lr: 0.00005 + | > step_time: 1.49430 (2.81757) + | > loader_time: 0.00160 (0.08502) + + + --> STEP: 23/234 -- GLOBAL_STEP: 49865 + | > loss: -0.27209 (-0.24603) + | > log_mle: -0.34107 (-0.33829) + | > loss_dur: 0.06898 (0.09226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.57853 (16.20606) + | > current_lr: 0.00005 + | > step_time: 4.02170 (2.70573) + | > loader_time: 0.00260 (0.06711) + + + --> STEP: 28/234 -- GLOBAL_STEP: 49870 + | > loss: -0.28278 (-0.24769) + | > log_mle: -0.34905 (-0.33744) + | > loss_dur: 0.06627 (0.08974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 5.32272 (15.01356) + | > current_lr: 0.00005 + | > step_time: 1.79330 (2.51002) + | > loader_time: 0.00280 (0.05560) + + + --> STEP: 33/234 -- GLOBAL_STEP: 49875 + | > loss: -0.24302 (-0.24528) + | > log_mle: -0.32703 (-0.33563) + | > loss_dur: 0.08401 (0.09035) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.24142 (14.46826) + | > current_lr: 0.00005 + | > step_time: 1.56240 (2.71766) + | > loader_time: 0.00210 (0.04779) + + + --> STEP: 38/234 -- GLOBAL_STEP: 49880 + | > loss: -0.23683 (-0.24223) + | > log_mle: -0.33519 (-0.33397) + | > loss_dur: 0.09836 (0.09174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08720 (14.25300) + | > current_lr: 0.00005 + | > step_time: 1.56370 (2.60007) + | > loader_time: 0.00220 (0.04399) + + + --> STEP: 43/234 -- GLOBAL_STEP: 49885 + | > loss: -0.20679 (-0.23873) + | > log_mle: -0.32754 (-0.33228) + | > loss_dur: 0.12074 (0.09355) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.63272 (13.93011) + | > current_lr: 0.00005 + | > step_time: 1.52100 (2.50396) + | > loader_time: 0.08270 (0.04295) + + + --> STEP: 48/234 -- GLOBAL_STEP: 49890 + | > loss: -0.23585 (-0.23676) + | > log_mle: -0.32250 (-0.33142) + | > loss_dur: 0.08666 (0.09466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.47909 (13.56531) + | > current_lr: 0.00005 + | > step_time: 0.85910 (2.38691) + | > loader_time: 0.00190 (0.04222) + + + --> STEP: 53/234 -- GLOBAL_STEP: 49895 + | > loss: -0.21087 (-0.23451) + | > log_mle: -0.32504 (-0.33040) + | > loss_dur: 0.11417 (0.09589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.42325 (13.17924) + | > current_lr: 0.00005 + | > step_time: 2.78810 (2.36170) + | > loader_time: 0.00290 (0.04164) + + + --> STEP: 58/234 -- GLOBAL_STEP: 49900 + | > loss: -0.22061 (-0.23311) + | > log_mle: -0.32168 (-0.32969) + | > loss_dur: 0.10107 (0.09658) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.53851 (13.09548) + | > current_lr: 0.00005 + | > step_time: 1.69150 (2.40284) + | > loader_time: 0.00260 (0.04107) + + + --> STEP: 63/234 -- GLOBAL_STEP: 49905 + | > loss: -0.20165 (-0.23093) + | > log_mle: -0.31705 (-0.32981) + | > loss_dur: 0.11540 (0.09887) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.09112 (13.22228) + | > current_lr: 0.00005 + | > step_time: 1.68390 (2.34015) + | > loader_time: 0.00210 (0.03928) + + + --> STEP: 68/234 -- GLOBAL_STEP: 49910 + | > loss: -0.18185 (-0.22945) + | > log_mle: -0.31252 (-0.32909) + | > loss_dur: 0.13068 (0.09964) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.66848 (13.02154) + | > current_lr: 0.00005 + | > step_time: 2.79420 (2.35742) + | > loader_time: 0.00260 (0.03783) + + + --> STEP: 73/234 -- GLOBAL_STEP: 49915 + | > loss: -0.19476 (-0.22721) + | > log_mle: -0.33097 (-0.32862) + | > loss_dur: 0.13620 (0.10141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.06785 (12.97116) + | > current_lr: 0.00005 + | > step_time: 3.99400 (2.35740) + | > loader_time: 0.00280 (0.03540) + + + --> STEP: 78/234 -- GLOBAL_STEP: 49920 + | > loss: -0.19067 (-0.22537) + | > log_mle: -0.31037 (-0.32811) + | > loss_dur: 0.11970 (0.10274) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.86127 (12.97288) + | > current_lr: 0.00005 + | > step_time: 1.07410 (2.31628) + | > loader_time: 0.00200 (0.03556) + + + --> STEP: 83/234 -- GLOBAL_STEP: 49925 + | > loss: -0.17322 (-0.22387) + | > log_mle: -0.32669 (-0.32784) + | > loss_dur: 0.15348 (0.10397) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.25812 (12.99739) + | > current_lr: 0.00005 + | > step_time: 1.21650 (2.27061) + | > loader_time: 0.08660 (0.03458) + + + --> STEP: 88/234 -- GLOBAL_STEP: 49930 + | > loss: -0.21736 (-0.22291) + | > log_mle: -0.36366 (-0.32808) + | > loss_dur: 0.14630 (0.10517) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.23987 (12.99610) + | > current_lr: 0.00005 + | > step_time: 1.29880 (2.25493) + | > loader_time: 0.00350 (0.03280) + + + --> STEP: 93/234 -- GLOBAL_STEP: 49935 + | > loss: -0.20606 (-0.22208) + | > log_mle: -0.37174 (-0.32942) + | > loss_dur: 0.16569 (0.10734) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.76554 (13.58852) + | > current_lr: 0.00005 + | > step_time: 2.18620 (2.26460) + | > loader_time: 0.00420 (0.03125) + + + --> STEP: 98/234 -- GLOBAL_STEP: 49940 + | > loss: -0.18788 (-0.22189) + | > log_mle: -0.31026 (-0.33094) + | > loss_dur: 0.12238 (0.10904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.17724 (14.00269) + | > current_lr: 0.00005 + | > step_time: 1.80020 (2.26511) + | > loader_time: 0.00200 (0.02979) + + + --> STEP: 103/234 -- GLOBAL_STEP: 49945 + | > loss: -0.22667 (-0.22143) + | > log_mle: -0.40948 (-0.33320) + | > loss_dur: 0.18281 (0.11177) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.84486 (14.51506) + | > current_lr: 0.00005 + | > step_time: 2.49930 (2.24944) + | > loader_time: 0.00370 (0.02930) + + + --> STEP: 108/234 -- GLOBAL_STEP: 49950 + | > loss: -0.21206 (-0.22114) + | > log_mle: -0.35208 (-0.33503) + | > loss_dur: 0.14002 (0.11389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.72871 (14.95956) + | > current_lr: 0.00005 + | > step_time: 1.56390 (2.21325) + | > loader_time: 0.00260 (0.02804) + + + --> STEP: 113/234 -- GLOBAL_STEP: 49955 + | > loss: -0.23259 (-0.22095) + | > log_mle: -0.39804 (-0.33760) + | > loss_dur: 0.16546 (0.11665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.84927 (15.55090) + | > current_lr: 0.00005 + | > step_time: 2.37890 (2.21264) + | > loader_time: 0.00250 (0.02844) + + + --> STEP: 118/234 -- GLOBAL_STEP: 49960 + | > loss: -0.20125 (-0.22055) + | > log_mle: -0.36850 (-0.33941) + | > loss_dur: 0.16725 (0.11886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.08749 (16.01000) + | > current_lr: 0.00005 + | > step_time: 4.61150 (2.23775) + | > loader_time: 0.09870 (0.02963) + + + --> STEP: 123/234 -- GLOBAL_STEP: 49965 + | > loss: -0.16944 (-0.21977) + | > log_mle: -0.33444 (-0.34028) + | > loss_dur: 0.16501 (0.12051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.38031 (16.30769) + | > current_lr: 0.00005 + | > step_time: 3.39600 (2.23821) + | > loader_time: 0.00600 (0.02858) + + + --> STEP: 128/234 -- GLOBAL_STEP: 49970 + | > loss: -0.23667 (-0.22052) + | > log_mle: -0.39561 (-0.34316) + | > loss_dur: 0.15894 (0.12264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.30221 (17.01579) + | > current_lr: 0.00005 + | > step_time: 2.01430 (2.26021) + | > loader_time: 0.08670 (0.02908) + + + --> STEP: 133/234 -- GLOBAL_STEP: 49975 + | > loss: -0.24875 (-0.22160) + | > log_mle: -0.42276 (-0.34628) + | > loss_dur: 0.17401 (0.12469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.87204 (17.78991) + | > current_lr: 0.00005 + | > step_time: 2.46270 (2.28269) + | > loader_time: 0.00210 (0.02996) + + + --> STEP: 138/234 -- GLOBAL_STEP: 49980 + | > loss: -0.20168 (-0.22226) + | > log_mle: -0.37263 (-0.34907) + | > loss_dur: 0.17095 (0.12682) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.46627 (18.53498) + | > current_lr: 0.00005 + | > step_time: 2.80890 (2.27840) + | > loader_time: 0.07810 (0.03011) + + + --> STEP: 143/234 -- GLOBAL_STEP: 49985 + | > loss: -0.26602 (-0.22351) + | > log_mle: -0.49627 (-0.35268) + | > loss_dur: 0.23026 (0.12917) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 130.55547 (19.95634) + | > current_lr: 0.00005 + | > step_time: 1.38680 (2.27557) + | > loader_time: 0.10500 (0.03101) + + + --> STEP: 148/234 -- GLOBAL_STEP: 49990 + | > loss: -0.25957 (-0.22492) + | > log_mle: -0.42348 (-0.35612) + | > loss_dur: 0.16392 (0.13120) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.80858 (20.80584) + | > current_lr: 0.00005 + | > step_time: 1.70360 (2.27218) + | > loader_time: 0.00310 (0.03072) + + + --> STEP: 153/234 -- GLOBAL_STEP: 49995 + | > loss: -0.36072 (-0.22749) + | > log_mle: -0.55570 (-0.36090) + | > loss_dur: 0.19498 (0.13341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.04990 (21.84653) + | > current_lr: 0.00005 + | > step_time: 1.47720 (2.26265) + | > loader_time: 0.00320 (0.03043) + + + --> STEP: 158/234 -- GLOBAL_STEP: 50000 + | > loss: -0.27493 (-0.22953) + | > log_mle: -0.49086 (-0.36509) + | > loss_dur: 0.21593 (0.13556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.31927 (22.95506) + | > current_lr: 0.00005 + | > step_time: 5.70000 (2.26382) + | > loader_time: 0.00340 (0.03063) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_50000.pth + + --> STEP: 163/234 -- GLOBAL_STEP: 50005 + | > loss: -0.26792 (-0.23189) + | > log_mle: -0.46667 (-0.36941) + | > loss_dur: 0.19875 (0.13752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.32767 (23.74338) + | > current_lr: 0.00005 + | > step_time: 2.88330 (2.27590) + | > loader_time: 0.19280 (0.03178) + + + --> STEP: 168/234 -- GLOBAL_STEP: 50010 + | > loss: -0.29664 (-0.23444) + | > log_mle: -0.52090 (-0.37391) + | > loss_dur: 0.22426 (0.13947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.32310 (25.04657) + | > current_lr: 0.00005 + | > step_time: 2.00940 (2.27016) + | > loader_time: 0.00340 (0.03152) + + + --> STEP: 173/234 -- GLOBAL_STEP: 50015 + | > loss: -0.31921 (-0.23738) + | > log_mle: -0.53199 (-0.37902) + | > loss_dur: 0.21278 (0.14164) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.44602 (26.33287) + | > current_lr: 0.00005 + | > step_time: 5.59940 (2.30418) + | > loader_time: 0.19400 (0.03280) + + + --> STEP: 178/234 -- GLOBAL_STEP: 50020 + | > loss: -0.36942 (-0.24046) + | > log_mle: -0.59832 (-0.38422) + | > loss_dur: 0.22890 (0.14376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.52340 (27.34945) + | > current_lr: 0.00005 + | > step_time: 0.99200 (2.28499) + | > loader_time: 0.00470 (0.03245) + + + --> STEP: 183/234 -- GLOBAL_STEP: 50025 + | > loss: -0.37520 (-0.24261) + | > log_mle: -0.58421 (-0.38862) + | > loss_dur: 0.20901 (0.14600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.82769 (28.66726) + | > current_lr: 0.00005 + | > step_time: 2.31080 (2.28281) + | > loader_time: 0.00280 (0.03322) + + + --> STEP: 188/234 -- GLOBAL_STEP: 50030 + | > loss: -0.37380 (-0.24505) + | > log_mle: -0.59562 (-0.39318) + | > loss_dur: 0.22182 (0.14813) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.03237 (29.64438) + | > current_lr: 0.00005 + | > step_time: 1.59660 (2.36788) + | > loader_time: 0.00250 (0.03397) + + + --> STEP: 193/234 -- GLOBAL_STEP: 50035 + | > loss: -0.36697 (-0.24766) + | > log_mle: -0.58552 (-0.39755) + | > loss_dur: 0.21855 (0.14989) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.55700 (30.90217) + | > current_lr: 0.00005 + | > step_time: 7.09120 (2.45736) + | > loader_time: 0.10110 (0.03477) + + + --> STEP: 198/234 -- GLOBAL_STEP: 50040 + | > loss: -0.35732 (-0.25015) + | > log_mle: -0.58450 (-0.40177) + | > loss_dur: 0.22717 (0.15163) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.60429 (31.99039) + | > current_lr: 0.00005 + | > step_time: 6.89150 (2.55800) + | > loader_time: 0.09940 (0.03680) + + + --> STEP: 203/234 -- GLOBAL_STEP: 50045 + | > loss: -0.29957 (-0.25228) + | > log_mle: -0.51960 (-0.40578) + | > loss_dur: 0.22004 (0.15351) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.99769 (33.05929) + | > current_lr: 0.00005 + | > step_time: 3.61140 (2.59006) + | > loader_time: 0.08350 (0.03768) + + + --> STEP: 208/234 -- GLOBAL_STEP: 50050 + | > loss: -0.35055 (-0.25476) + | > log_mle: -0.59659 (-0.41032) + | > loss_dur: 0.24604 (0.15556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.01579 (34.38753) + | > current_lr: 0.00005 + | > step_time: 1.69720 (2.61301) + | > loader_time: 0.00380 (0.03908) + + + --> STEP: 213/234 -- GLOBAL_STEP: 50055 + | > loss: -0.37296 (-0.25774) + | > log_mle: -0.62912 (-0.41535) + | > loss_dur: 0.25616 (0.15761) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.86963 (35.57454) + | > current_lr: 0.00005 + | > step_time: 3.20610 (2.63061) + | > loader_time: 0.10160 (0.03915) + + + --> STEP: 218/234 -- GLOBAL_STEP: 50060 + | > loss: -0.36653 (-0.26066) + | > log_mle: -0.60743 (-0.42009) + | > loss_dur: 0.24090 (0.15944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.41429 (36.50633) + | > current_lr: 0.00005 + | > step_time: 3.40350 (2.63816) + | > loader_time: 0.01350 (0.03916) + + + --> STEP: 223/234 -- GLOBAL_STEP: 50065 + | > loss: -0.41590 (-0.26405) + | > log_mle: -0.65696 (-0.42529) + | > loss_dur: 0.24106 (0.16124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.03153 (37.53807) + | > current_lr: 0.00005 + | > step_time: 0.32100 (2.61414) + | > loader_time: 0.00360 (0.03910) + + + --> STEP: 228/234 -- GLOBAL_STEP: 50070 + | > loss: -0.37934 (-0.26738) + | > log_mle: -0.64906 (-0.43066) + | > loss_dur: 0.26972 (0.16327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.73753 (39.14529) + | > current_lr: 0.00005 + | > step_time: 0.24930 (2.56214) + | > loader_time: 0.00370 (0.03833) + + + --> STEP: 233/234 -- GLOBAL_STEP: 50075 + | > loss: 0.12344 (-0.26797) + | > log_mle: -0.62360 (-0.43721) + | > loss_dur: 0.74704 (0.16924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.94987 (41.07510) + | > current_lr: 0.00005 + | > step_time: 0.22300 (2.51311) + | > loader_time: 0.00260 (0.03777) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02203 (-0.97674) + | > avg_loss: -0.28038 (+0.00023) + | > avg_log_mle: -0.49766 (+0.00633) + | > avg_loss_dur: 0.21728 (-0.00610) + + + > EPOCH: 214/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 05:48:32)  + + --> STEP: 4/234 -- GLOBAL_STEP: 50080 + | > loss: -0.23363 (-0.22540) + | > log_mle: -0.33719 (-0.34063) + | > loss_dur: 0.10356 (0.11523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.88258 (17.81083) + | > current_lr: 0.00005 + | > step_time: 13.41230 (6.34964) + | > loader_time: 0.09950 (0.04857) + + + --> STEP: 9/234 -- GLOBAL_STEP: 50085 + | > loss: -0.20893 (-0.23820) + | > log_mle: -0.34876 (-0.34287) + | > loss_dur: 0.13982 (0.10467) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.80499 (15.84781) + | > current_lr: 0.00005 + | > step_time: 1.81870 (6.53793) + | > loader_time: 0.09260 (0.05401) + + + --> STEP: 14/234 -- GLOBAL_STEP: 50090 + | > loss: -0.24973 (-0.24427) + | > log_mle: -0.34330 (-0.34329) + | > loss_dur: 0.09357 (0.09901) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.89139 (15.10675) + | > current_lr: 0.00005 + | > step_time: 5.79560 (5.18845) + | > loader_time: 0.09920 (0.04927) + + + --> STEP: 19/234 -- GLOBAL_STEP: 50095 + | > loss: -0.27046 (-0.24733) + | > log_mle: -0.33762 (-0.34153) + | > loss_dur: 0.06716 (0.09420) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.04046 (14.27725) + | > current_lr: 0.00005 + | > step_time: 1.50220 (4.24890) + | > loader_time: 0.07460 (0.04435) + + + --> STEP: 24/234 -- GLOBAL_STEP: 50100 + | > loss: -0.27245 (-0.24889) + | > log_mle: -0.33482 (-0.34021) + | > loss_dur: 0.06238 (0.09132) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.17193 (13.77827) + | > current_lr: 0.00005 + | > step_time: 1.68290 (3.78900) + | > loader_time: 0.00120 (0.04283) + + + --> STEP: 29/234 -- GLOBAL_STEP: 50105 + | > loss: -0.21350 (-0.24841) + | > log_mle: -0.31653 (-0.33861) + | > loss_dur: 0.10302 (0.09019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.94879 (13.25672) + | > current_lr: 0.00005 + | > step_time: 1.80000 (3.44484) + | > loader_time: 0.00250 (0.03600) + + + --> STEP: 34/234 -- GLOBAL_STEP: 50110 + | > loss: -0.23576 (-0.24656) + | > log_mle: -0.32546 (-0.33697) + | > loss_dur: 0.08970 (0.09042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.65067 (13.09255) + | > current_lr: 0.00005 + | > step_time: 1.40440 (3.18354) + | > loader_time: 0.04440 (0.03463) + + + --> STEP: 39/234 -- GLOBAL_STEP: 50115 + | > loss: -0.21795 (-0.24305) + | > log_mle: -0.32405 (-0.33470) + | > loss_dur: 0.10610 (0.09165) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.95265 (13.63831) + | > current_lr: 0.00005 + | > step_time: 2.59050 (3.01916) + | > loader_time: 0.00140 (0.03471) + + + --> STEP: 44/234 -- GLOBAL_STEP: 50120 + | > loss: -0.23800 (-0.24005) + | > log_mle: -0.31487 (-0.33261) + | > loss_dur: 0.07687 (0.09256) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.72144 (13.30944) + | > current_lr: 0.00005 + | > step_time: 2.11710 (2.94408) + | > loader_time: 0.08620 (0.03720) + + + --> STEP: 49/234 -- GLOBAL_STEP: 50125 + | > loss: -0.23513 (-0.23815) + | > log_mle: -0.32790 (-0.33193) + | > loss_dur: 0.09276 (0.09377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.78324 (13.01064) + | > current_lr: 0.00005 + | > step_time: 3.71800 (2.89072) + | > loader_time: 0.00460 (0.03568) + + + --> STEP: 54/234 -- GLOBAL_STEP: 50130 + | > loss: -0.22857 (-0.23580) + | > log_mle: -0.32888 (-0.33095) + | > loss_dur: 0.10030 (0.09515) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.19028 (12.62154) + | > current_lr: 0.00005 + | > step_time: 1.70740 (2.75331) + | > loader_time: 0.08320 (0.03409) + + + --> STEP: 59/234 -- GLOBAL_STEP: 50135 + | > loss: -0.23234 (-0.23448) + | > log_mle: -0.33225 (-0.33039) + | > loss_dur: 0.09991 (0.09591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.67033 (12.52218) + | > current_lr: 0.00005 + | > step_time: 1.33690 (2.65994) + | > loader_time: 0.00210 (0.03143) + + + --> STEP: 64/234 -- GLOBAL_STEP: 50140 + | > loss: -0.22470 (-0.23216) + | > log_mle: -0.31982 (-0.33036) + | > loss_dur: 0.09512 (0.09820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.20245 (12.58959) + | > current_lr: 0.00005 + | > step_time: 2.90840 (2.62659) + | > loader_time: 0.00280 (0.03178) + + + --> STEP: 69/234 -- GLOBAL_STEP: 50145 + | > loss: -0.20354 (-0.23056) + | > log_mle: -0.30574 (-0.32953) + | > loss_dur: 0.10220 (0.09897) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.77819 (12.44696) + | > current_lr: 0.00005 + | > step_time: 2.48580 (2.56168) + | > loader_time: 0.00210 (0.03085) + + + --> STEP: 74/234 -- GLOBAL_STEP: 50150 + | > loss: -0.18522 (-0.22795) + | > log_mle: -0.30999 (-0.32909) + | > loss_dur: 0.12477 (0.10114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.37591 (12.63366) + | > current_lr: 0.00005 + | > step_time: 1.67090 (2.53136) + | > loader_time: 0.00200 (0.03023) + + + --> STEP: 79/234 -- GLOBAL_STEP: 50155 + | > loss: -0.20742 (-0.22632) + | > log_mle: -0.32835 (-0.32884) + | > loss_dur: 0.12093 (0.10252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.90562 (12.68451) + | > current_lr: 0.00005 + | > step_time: 1.19820 (2.50560) + | > loader_time: 0.00230 (0.02845) + + + --> STEP: 84/234 -- GLOBAL_STEP: 50160 + | > loss: -0.21100 (-0.22484) + | > log_mle: -0.32277 (-0.32858) + | > loss_dur: 0.11177 (0.10374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.47448 (12.74229) + | > current_lr: 0.00005 + | > step_time: 3.69810 (2.49953) + | > loader_time: 0.00690 (0.02898) + + + --> STEP: 89/234 -- GLOBAL_STEP: 50165 + | > loss: -0.21924 (-0.22366) + | > log_mle: -0.34638 (-0.32897) + | > loss_dur: 0.12713 (0.10530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.67942 (12.83058) + | > current_lr: 0.00005 + | > step_time: 1.69170 (2.46996) + | > loader_time: 0.00380 (0.02758) + + + --> STEP: 94/234 -- GLOBAL_STEP: 50170 + | > loss: -0.22990 (-0.22309) + | > log_mle: -0.37806 (-0.33065) + | > loss_dur: 0.14815 (0.10756) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.69512 (13.21457) + | > current_lr: 0.00005 + | > step_time: 2.10470 (2.46021) + | > loader_time: 0.00190 (0.02820) + + + --> STEP: 99/234 -- GLOBAL_STEP: 50175 + | > loss: -0.23331 (-0.22280) + | > log_mle: -0.41337 (-0.33249) + | > loss_dur: 0.18006 (0.10969) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.55789 (13.60939) + | > current_lr: 0.00005 + | > step_time: 1.91340 (2.41481) + | > loader_time: 0.00190 (0.02778) + + + --> STEP: 104/234 -- GLOBAL_STEP: 50180 + | > loss: -0.26282 (-0.22293) + | > log_mle: -0.42415 (-0.33486) + | > loss_dur: 0.16133 (0.11193) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.46971 (14.22303) + | > current_lr: 0.00005 + | > step_time: 2.10370 (2.38241) + | > loader_time: 0.00360 (0.02741) + + + --> STEP: 109/234 -- GLOBAL_STEP: 50185 + | > loss: -0.19488 (-0.22206) + | > log_mle: -0.39577 (-0.33640) + | > loss_dur: 0.20090 (0.11434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.47449 (14.64235) + | > current_lr: 0.00005 + | > step_time: 2.29740 (2.38770) + | > loader_time: 0.08440 (0.02793) + + + --> STEP: 114/234 -- GLOBAL_STEP: 50190 + | > loss: -0.23077 (-0.22196) + | > log_mle: -0.37689 (-0.33881) + | > loss_dur: 0.14612 (0.11685) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.69315 (15.37065) + | > current_lr: 0.00005 + | > step_time: 1.22260 (2.36316) + | > loader_time: 0.00330 (0.02973) + + + --> STEP: 119/234 -- GLOBAL_STEP: 50195 + | > loss: -0.21659 (-0.22149) + | > log_mle: -0.37521 (-0.34065) + | > loss_dur: 0.15862 (0.11916) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.14845 (15.90234) + | > current_lr: 0.00005 + | > step_time: 2.01710 (2.39100) + | > loader_time: 0.00300 (0.03089) + + + --> STEP: 124/234 -- GLOBAL_STEP: 50200 + | > loss: -0.24021 (-0.22080) + | > log_mle: -0.39970 (-0.34166) + | > loss_dur: 0.15948 (0.12085) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.59021 (16.30165) + | > current_lr: 0.00005 + | > step_time: 2.81160 (2.39360) + | > loader_time: 0.19460 (0.03271) + + + --> STEP: 129/234 -- GLOBAL_STEP: 50205 + | > loss: -0.21349 (-0.22134) + | > log_mle: -0.39245 (-0.34435) + | > loss_dur: 0.17896 (0.12301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.42407 (17.12390) + | > current_lr: 0.00005 + | > step_time: 1.19850 (2.38849) + | > loader_time: 0.00240 (0.03210) + + + --> STEP: 134/234 -- GLOBAL_STEP: 50210 + | > loss: -0.23908 (-0.22221) + | > log_mle: -0.42880 (-0.34740) + | > loss_dur: 0.18972 (0.12519) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.03618 (18.39193) + | > current_lr: 0.00005 + | > step_time: 1.48510 (2.37988) + | > loader_time: 0.00180 (0.03308) + + + --> STEP: 139/234 -- GLOBAL_STEP: 50215 + | > loss: -0.29705 (-0.22299) + | > log_mle: -0.49943 (-0.35030) + | > loss_dur: 0.20238 (0.12730) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.72515 (19.02171) + | > current_lr: 0.00005 + | > step_time: 5.29140 (2.47927) + | > loader_time: 0.10510 (0.03471) + + + --> STEP: 144/234 -- GLOBAL_STEP: 50220 + | > loss: -0.28220 (-0.22384) + | > log_mle: -0.48035 (-0.35365) + | > loss_dur: 0.19815 (0.12981) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.04870 (19.70303) + | > current_lr: 0.00005 + | > step_time: 1.28810 (2.44036) + | > loader_time: 0.00190 (0.03475) + + + --> STEP: 149/234 -- GLOBAL_STEP: 50225 + | > loss: -0.33061 (-0.22571) + | > log_mle: -0.52602 (-0.35754) + | > loss_dur: 0.19541 (0.13182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.74883 (20.65423) + | > current_lr: 0.00005 + | > step_time: 3.61280 (2.48145) + | > loader_time: 0.19830 (0.03692) + + + --> STEP: 154/234 -- GLOBAL_STEP: 50230 + | > loss: -0.28693 (-0.22811) + | > log_mle: -0.47816 (-0.36204) + | > loss_dur: 0.19123 (0.13393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.82076 (21.95340) + | > current_lr: 0.00005 + | > step_time: 2.13390 (2.49915) + | > loader_time: 0.07110 (0.03682) + + + --> STEP: 159/234 -- GLOBAL_STEP: 50235 + | > loss: -0.30161 (-0.23001) + | > log_mle: -0.50444 (-0.36614) + | > loss_dur: 0.20283 (0.13613) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.60323 (23.05314) + | > current_lr: 0.00005 + | > step_time: 1.45510 (2.52461) + | > loader_time: 0.00240 (0.03808) + + + --> STEP: 164/234 -- GLOBAL_STEP: 50240 + | > loss: -0.28556 (-0.23210) + | > log_mle: -0.49962 (-0.37025) + | > loss_dur: 0.21406 (0.13815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.35220 (23.73090) + | > current_lr: 0.00005 + | > step_time: 2.29560 (2.51235) + | > loader_time: 0.00300 (0.03703) + + + --> STEP: 169/234 -- GLOBAL_STEP: 50245 + | > loss: -0.29326 (-0.23466) + | > log_mle: -0.50038 (-0.37470) + | > loss_dur: 0.20712 (0.14004) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.99033 (24.64418) + | > current_lr: 0.00005 + | > step_time: 3.41130 (2.54872) + | > loader_time: 0.09990 (0.03771) + + + --> STEP: 174/234 -- GLOBAL_STEP: 50250 + | > loss: -0.36427 (-0.23783) + | > log_mle: -0.58105 (-0.38020) + | > loss_dur: 0.21678 (0.14237) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.71645 (25.95077) + | > current_lr: 0.00005 + | > step_time: 3.80130 (2.54621) + | > loader_time: 0.19880 (0.03878) + + + --> STEP: 179/234 -- GLOBAL_STEP: 50255 + | > loss: -0.34824 (-0.24045) + | > log_mle: -0.58886 (-0.38523) + | > loss_dur: 0.24062 (0.14478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.36970 (27.12068) + | > current_lr: 0.00005 + | > step_time: 1.90300 (2.56319) + | > loader_time: 0.08910 (0.03939) + + + --> STEP: 184/234 -- GLOBAL_STEP: 50260 + | > loss: -0.33159 (-0.24291) + | > log_mle: -0.54548 (-0.38972) + | > loss_dur: 0.21389 (0.14680) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.10120 (28.21965) + | > current_lr: 0.00005 + | > step_time: 5.89320 (2.59200) + | > loader_time: 0.01020 (0.03978) + + + --> STEP: 189/234 -- GLOBAL_STEP: 50265 + | > loss: -0.32860 (-0.24554) + | > log_mle: -0.54724 (-0.39448) + | > loss_dur: 0.21864 (0.14893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.82935 (29.26641) + | > current_lr: 0.00005 + | > step_time: 3.80870 (2.61015) + | > loader_time: 0.00580 (0.03977) + + + --> STEP: 194/234 -- GLOBAL_STEP: 50270 + | > loss: -0.37834 (-0.24886) + | > log_mle: -0.59149 (-0.39940) + | > loss_dur: 0.21315 (0.15053) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.30752 (30.28931) + | > current_lr: 0.00005 + | > step_time: 6.60660 (2.64813) + | > loader_time: 0.19610 (0.04125) + + + --> STEP: 199/234 -- GLOBAL_STEP: 50275 + | > loss: -0.37991 (-0.25177) + | > log_mle: -0.59719 (-0.40400) + | > loss_dur: 0.21729 (0.15223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.50912 (31.60259) + | > current_lr: 0.00005 + | > step_time: 6.01300 (2.72525) + | > loader_time: 0.30430 (0.04331) + + + --> STEP: 204/234 -- GLOBAL_STEP: 50280 + | > loss: -0.39500 (-0.25427) + | > log_mle: -0.63714 (-0.40845) + | > loss_dur: 0.24214 (0.15418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.77937 (32.75936) + | > current_lr: 0.00005 + | > step_time: 1.99810 (2.76003) + | > loader_time: 0.18520 (0.04511) + + + --> STEP: 209/234 -- GLOBAL_STEP: 50285 + | > loss: -0.35969 (-0.25729) + | > log_mle: -0.58670 (-0.41322) + | > loss_dur: 0.22701 (0.15593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.10614 (34.05238) + | > current_lr: 0.00005 + | > step_time: 12.40840 (2.85688) + | > loader_time: 0.09900 (0.04791) + + + --> STEP: 214/234 -- GLOBAL_STEP: 50290 + | > loss: -0.39496 (-0.26065) + | > log_mle: -0.60394 (-0.41848) + | > loss_dur: 0.20898 (0.15783) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.18750 (35.50270) + | > current_lr: 0.00005 + | > step_time: 7.18690 (2.87840) + | > loader_time: 0.11170 (0.04957) + + + --> STEP: 219/234 -- GLOBAL_STEP: 50295 + | > loss: -0.49118 (-0.26411) + | > log_mle: -0.72795 (-0.42381) + | > loss_dur: 0.23676 (0.15970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.47653 (36.62483) + | > current_lr: 0.00005 + | > step_time: 2.58860 (2.91220) + | > loader_time: 0.00420 (0.04975) + + + --> STEP: 224/234 -- GLOBAL_STEP: 50300 + | > loss: -0.42977 (-0.26732) + | > log_mle: -0.67523 (-0.42887) + | > loss_dur: 0.24547 (0.16155) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.96749 (37.86504) + | > current_lr: 0.00005 + | > step_time: 1.52090 (2.90962) + | > loader_time: 0.07990 (0.05033) + + + --> STEP: 229/234 -- GLOBAL_STEP: 50305 + | > loss: -0.36400 (-0.26993) + | > log_mle: -0.66211 (-0.43366) + | > loss_dur: 0.29811 (0.16373) + | > amp_scaler: 1024.00000 (2043.52838) + | > grad_norm: 0.00000 (39.33916) + | > current_lr: 0.00005 + | > step_time: 0.23330 (2.85887) + | > loader_time: 0.00280 (0.04965) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.22876 (+1.20673) + | > avg_loss: -0.28559 (-0.00521) + | > avg_log_mle: -0.51043 (-0.01276) + | > avg_loss_dur: 0.22483 (+0.00755) + + + > EPOCH: 215/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 06:00:43)  + + --> STEP: 0/234 -- GLOBAL_STEP: 50310 + | > loss: -0.28503 (-0.28503) + | > log_mle: -0.41522 (-0.41522) + | > loss_dur: 0.13020 (0.13020) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.46261 (30.46261) + | > current_lr: 0.00005 + | > step_time: 1.39940 (1.39941) + | > loader_time: 8.12370 (8.12372) + + + --> STEP: 5/234 -- GLOBAL_STEP: 50315 + | > loss: -0.25029 (-0.22939) + | > log_mle: -0.34386 (-0.34066) + | > loss_dur: 0.09356 (0.11127) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.80456 (19.03492) + | > current_lr: 0.00005 + | > step_time: 2.50080 (4.75850) + | > loader_time: 0.01150 (2.21441) + + + --> STEP: 10/234 -- GLOBAL_STEP: 50320 + | > loss: -0.23061 (-0.23950) + | > log_mle: -0.33937 (-0.34305) + | > loss_dur: 0.10877 (0.10355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.37991 (17.14638) + | > current_lr: 0.00005 + | > step_time: 4.42800 (5.29207) + | > loader_time: 0.07260 (1.13664) + + + --> STEP: 15/234 -- GLOBAL_STEP: 50325 + | > loss: -0.26634 (-0.24728) + | > log_mle: -0.34452 (-0.34393) + | > loss_dur: 0.07818 (0.09665) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.82691 (16.65738) + | > current_lr: 0.00005 + | > step_time: 2.40120 (4.64062) + | > loader_time: 0.00240 (0.75917) + + + --> STEP: 20/234 -- GLOBAL_STEP: 50330 + | > loss: -0.26338 (-0.24892) + | > log_mle: -0.33923 (-0.34168) + | > loss_dur: 0.07585 (0.09276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.21911 (15.88846) + | > current_lr: 0.00005 + | > step_time: 4.30820 (4.54606) + | > loader_time: 0.00680 (0.57958) + + + --> STEP: 25/234 -- GLOBAL_STEP: 50335 + | > loss: -0.23400 (-0.24880) + | > log_mle: -0.32317 (-0.33964) + | > loss_dur: 0.08917 (0.09085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.86840 (15.06693) + | > current_lr: 0.00005 + | > step_time: 3.50660 (4.59530) + | > loader_time: 0.00740 (0.48484) + + + --> STEP: 30/234 -- GLOBAL_STEP: 50340 + | > loss: -0.22331 (-0.24835) + | > log_mle: -0.32141 (-0.33818) + | > loss_dur: 0.09810 (0.08984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.15756 (14.28116) + | > current_lr: 0.00005 + | > step_time: 2.39150 (4.46902) + | > loader_time: 0.00570 (0.41761) + + + --> STEP: 35/234 -- GLOBAL_STEP: 50345 + | > loss: -0.20631 (-0.24621) + | > log_mle: -0.32186 (-0.33662) + | > loss_dur: 0.11555 (0.09040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.06318 (13.80204) + | > current_lr: 0.00005 + | > step_time: 2.42480 (4.37456) + | > loader_time: 0.05930 (0.36727) + + + --> STEP: 40/234 -- GLOBAL_STEP: 50350 + | > loss: -0.19473 (-0.24317) + | > log_mle: -0.31110 (-0.33467) + | > loss_dur: 0.11637 (0.09150) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.73226 (13.51054) + | > current_lr: 0.00005 + | > step_time: 1.68710 (4.17973) + | > loader_time: 0.00240 (0.32461) + + + --> STEP: 45/234 -- GLOBAL_STEP: 50355 + | > loss: -0.21662 (-0.24054) + | > log_mle: -0.34078 (-0.33358) + | > loss_dur: 0.12416 (0.09304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.06256 (13.29426) + | > current_lr: 0.00005 + | > step_time: 1.89360 (4.08352) + | > loader_time: 0.07590 (0.29673) + + + --> STEP: 50/234 -- GLOBAL_STEP: 50360 + | > loss: -0.22016 (-0.23890) + | > log_mle: -0.32019 (-0.33258) + | > loss_dur: 0.10003 (0.09368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.64758 (12.82250) + | > current_lr: 0.00005 + | > step_time: 3.08760 (3.94466) + | > loader_time: 0.00800 (0.26918) + + + --> STEP: 55/234 -- GLOBAL_STEP: 50365 + | > loss: -0.23828 (-0.23718) + | > log_mle: -0.33133 (-0.33189) + | > loss_dur: 0.09305 (0.09472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.31692 (12.53539) + | > current_lr: 0.00005 + | > step_time: 5.41450 (3.84805) + | > loader_time: 0.10070 (0.25126) + + + --> STEP: 60/234 -- GLOBAL_STEP: 50370 + | > loss: -0.21864 (-0.23541) + | > log_mle: -0.34205 (-0.33159) + | > loss_dur: 0.12341 (0.09618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.35760 (12.46346) + | > current_lr: 0.00005 + | > step_time: 1.98290 (3.82544) + | > loader_time: 0.08460 (0.23652) + + + --> STEP: 65/234 -- GLOBAL_STEP: 50375 + | > loss: -0.22294 (-0.23335) + | > log_mle: -0.32329 (-0.33134) + | > loss_dur: 0.10035 (0.09799) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.16027 (12.48717) + | > current_lr: 0.00005 + | > step_time: 3.52150 (3.76415) + | > loader_time: 0.27840 (0.22410) + + + --> STEP: 70/234 -- GLOBAL_STEP: 50380 + | > loss: -0.17607 (-0.23068) + | > log_mle: -0.30806 (-0.33012) + | > loss_dur: 0.13200 (0.09944) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.06544 (12.64329) + | > current_lr: 0.00005 + | > step_time: 1.21160 (3.72200) + | > loader_time: 0.00720 (0.20974) + + + --> STEP: 75/234 -- GLOBAL_STEP: 50385 + | > loss: -0.19948 (-0.22847) + | > log_mle: -0.32841 (-0.32984) + | > loss_dur: 0.12893 (0.10137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.23652 (12.87548) + | > current_lr: 0.00005 + | > step_time: 2.90520 (3.70306) + | > loader_time: 0.08880 (0.19954) + + + --> STEP: 80/234 -- GLOBAL_STEP: 50390 + | > loss: -0.20481 (-0.22689) + | > log_mle: -0.31024 (-0.32921) + | > loss_dur: 0.10543 (0.10232) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.54229 (12.84532) + | > current_lr: 0.00005 + | > step_time: 2.01310 (3.67806) + | > loader_time: 0.10050 (0.19240) + + + --> STEP: 85/234 -- GLOBAL_STEP: 50395 + | > loss: -0.20373 (-0.22547) + | > log_mle: -0.31970 (-0.32890) + | > loss_dur: 0.11597 (0.10343) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.28709 (13.04573) + | > current_lr: 0.00005 + | > step_time: 2.50100 (3.67007) + | > loader_time: 0.00840 (0.18362) + + + --> STEP: 90/234 -- GLOBAL_STEP: 50400 + | > loss: -0.20580 (-0.22449) + | > log_mle: -0.34417 (-0.32969) + | > loss_dur: 0.13837 (0.10520) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.54928 (13.20227) + | > current_lr: 0.00005 + | > step_time: 4.08830 (3.61717) + | > loader_time: 0.10420 (0.17686) + + + --> STEP: 95/234 -- GLOBAL_STEP: 50405 + | > loss: -0.25353 (-0.22465) + | > log_mle: -0.42851 (-0.33226) + | > loss_dur: 0.17498 (0.10762) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.71386 (13.77780) + | > current_lr: 0.00005 + | > step_time: 2.20470 (3.57334) + | > loader_time: 0.00360 (0.17026) + + + --> STEP: 100/234 -- GLOBAL_STEP: 50410 + | > loss: -0.22174 (-0.22400) + | > log_mle: -0.35709 (-0.33331) + | > loss_dur: 0.13535 (0.10931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.26855 (14.01046) + | > current_lr: 0.00005 + | > step_time: 6.53550 (3.53003) + | > loader_time: 0.11930 (0.16493) + + + --> STEP: 105/234 -- GLOBAL_STEP: 50415 + | > loss: -0.20400 (-0.22380) + | > log_mle: -0.33615 (-0.33553) + | > loss_dur: 0.13215 (0.11173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61184 (14.44747) + | > current_lr: 0.00005 + | > step_time: 3.19340 (3.47804) + | > loader_time: 0.09690 (0.15943) + + + --> STEP: 110/234 -- GLOBAL_STEP: 50420 + | > loss: -0.20659 (-0.22306) + | > log_mle: -0.35868 (-0.33732) + | > loss_dur: 0.15209 (0.11425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.39874 (14.90780) + | > current_lr: 0.00005 + | > step_time: 3.69340 (3.45371) + | > loader_time: 0.20450 (0.15744) + + + --> STEP: 115/234 -- GLOBAL_STEP: 50425 + | > loss: -0.20346 (-0.22298) + | > log_mle: -0.37918 (-0.33992) + | > loss_dur: 0.17572 (0.11694) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.58916 (15.59013) + | > current_lr: 0.00005 + | > step_time: 3.58780 (3.43940) + | > loader_time: 0.00230 (0.15316) + + + --> STEP: 120/234 -- GLOBAL_STEP: 50430 + | > loss: -0.26343 (-0.22300) + | > log_mle: -0.43223 (-0.34218) + | > loss_dur: 0.16880 (0.11917) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.83784 (16.11469) + | > current_lr: 0.00005 + | > step_time: 3.30120 (3.40011) + | > loader_time: 0.09770 (0.14849) + + + --> STEP: 125/234 -- GLOBAL_STEP: 50435 + | > loss: -0.23900 (-0.22248) + | > log_mle: -0.41718 (-0.34319) + | > loss_dur: 0.17818 (0.12071) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.48184 (16.46780) + | > current_lr: 0.00005 + | > step_time: 2.96990 (3.39534) + | > loader_time: 0.13270 (0.14510) + + + --> STEP: 130/234 -- GLOBAL_STEP: 50440 + | > loss: -0.24820 (-0.22327) + | > log_mle: -0.42381 (-0.34617) + | > loss_dur: 0.17561 (0.12289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.07119 (17.25282) + | > current_lr: 0.00005 + | > step_time: 2.69450 (3.41131) + | > loader_time: 0.00770 (0.14237) + + + --> STEP: 135/234 -- GLOBAL_STEP: 50445 + | > loss: -0.20556 (-0.22417) + | > log_mle: -0.35226 (-0.34899) + | > loss_dur: 0.14670 (0.12482) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.35992 (18.06709) + | > current_lr: 0.00005 + | > step_time: 3.01580 (3.39639) + | > loader_time: 0.07160 (0.13934) + + + --> STEP: 140/234 -- GLOBAL_STEP: 50450 + | > loss: -0.20816 (-0.22529) + | > log_mle: -0.38921 (-0.35244) + | > loss_dur: 0.18106 (0.12714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.65870 (18.99730) + | > current_lr: 0.00005 + | > step_time: 4.18920 (3.40055) + | > loader_time: 0.00160 (0.13928) + + + --> STEP: 145/234 -- GLOBAL_STEP: 50455 + | > loss: -0.30448 (-0.22701) + | > log_mle: -0.49008 (-0.35661) + | > loss_dur: 0.18559 (0.12960) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.96940 (20.09665) + | > current_lr: 0.00005 + | > step_time: 2.07670 (3.37960) + | > loader_time: 0.00480 (0.13530) + + + --> STEP: 150/234 -- GLOBAL_STEP: 50460 + | > loss: -0.27807 (-0.22900) + | > log_mle: -0.48070 (-0.36054) + | > loss_dur: 0.20263 (0.13154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.73737 (20.87662) + | > current_lr: 0.00005 + | > step_time: 2.20150 (3.36510) + | > loader_time: 0.07650 (0.13340) + + + --> STEP: 155/234 -- GLOBAL_STEP: 50465 + | > loss: -0.32996 (-0.23173) + | > log_mle: -0.53899 (-0.36548) + | > loss_dur: 0.20903 (0.13375) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.21495 (22.23619) + | > current_lr: 0.00005 + | > step_time: 4.90300 (3.37961) + | > loader_time: 0.11830 (0.13192) + + + --> STEP: 160/234 -- GLOBAL_STEP: 50470 + | > loss: -0.32713 (-0.23374) + | > log_mle: -0.54067 (-0.36980) + | > loss_dur: 0.21353 (0.13606) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.72087 (23.28354) + | > current_lr: 0.00005 + | > step_time: 2.78200 (3.39004) + | > loader_time: 0.08020 (0.13061) + + + --> STEP: 165/234 -- GLOBAL_STEP: 50475 + | > loss: -0.31967 (-0.23579) + | > log_mle: -0.53473 (-0.37388) + | > loss_dur: 0.21506 (0.13809) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.86897 (24.44506) + | > current_lr: 0.00005 + | > step_time: 0.90030 (3.38253) + | > loader_time: 0.00420 (0.12842) + + + --> STEP: 170/234 -- GLOBAL_STEP: 50480 + | > loss: -0.35137 (-0.23831) + | > log_mle: -0.58058 (-0.37852) + | > loss_dur: 0.22921 (0.14021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.37904 (25.48650) + | > current_lr: 0.00005 + | > step_time: 4.60400 (3.37428) + | > loader_time: 0.08830 (0.12634) + + + --> STEP: 175/234 -- GLOBAL_STEP: 50485 + | > loss: -0.32496 (-0.24160) + | > log_mle: -0.55667 (-0.38400) + | > loss_dur: 0.23171 (0.14240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.37682 (26.42391) + | > current_lr: 0.00005 + | > step_time: 3.11410 (3.38553) + | > loader_time: 0.07570 (0.12445) + + + --> STEP: 180/234 -- GLOBAL_STEP: 50490 + | > loss: -0.35286 (-0.24456) + | > log_mle: -0.56356 (-0.38913) + | > loss_dur: 0.21070 (0.14457) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.81384 (27.72586) + | > current_lr: 0.00005 + | > step_time: 2.39140 (3.39929) + | > loader_time: 0.00220 (0.12329) + + + --> STEP: 185/234 -- GLOBAL_STEP: 50495 + | > loss: -0.35382 (-0.24712) + | > log_mle: -0.58450 (-0.39377) + | > loss_dur: 0.23068 (0.14665) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.08898 (29.00820) + | > current_lr: 0.00005 + | > step_time: 3.50250 (3.44044) + | > loader_time: 0.07940 (0.12237) + + + --> STEP: 190/234 -- GLOBAL_STEP: 50500 + | > loss: -0.35815 (-0.24956) + | > log_mle: -0.56634 (-0.39828) + | > loss_dur: 0.20819 (0.14871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.66125 (30.13927) + | > current_lr: 0.00005 + | > step_time: 3.69330 (3.49772) + | > loader_time: 0.01610 (0.12195) + + + --> STEP: 195/234 -- GLOBAL_STEP: 50505 + | > loss: -0.36320 (-0.25283) + | > log_mle: -0.58880 (-0.40324) + | > loss_dur: 0.22560 (0.15041) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.55927 (31.25696) + | > current_lr: 0.00005 + | > step_time: 3.19530 (3.50536) + | > loader_time: 0.18530 (0.12037) + + + --> STEP: 200/234 -- GLOBAL_STEP: 50510 + | > loss: -0.34544 (-0.25552) + | > log_mle: -0.59112 (-0.40775) + | > loss_dur: 0.24567 (0.15223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.73491 (32.46601) + | > current_lr: 0.00005 + | > step_time: 2.27400 (3.51109) + | > loader_time: 0.08240 (0.11889) + + + --> STEP: 205/234 -- GLOBAL_STEP: 50515 + | > loss: -0.35415 (-0.25813) + | > log_mle: -0.57511 (-0.41209) + | > loss_dur: 0.22096 (0.15397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.28905 (33.48877) + | > current_lr: 0.00005 + | > step_time: 2.80990 (3.51956) + | > loader_time: 0.09910 (0.11744) + + + --> STEP: 210/234 -- GLOBAL_STEP: 50520 + | > loss: -0.40818 (-0.26134) + | > log_mle: -0.64794 (-0.41710) + | > loss_dur: 0.23975 (0.15576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 125.66806 (35.10233) + | > current_lr: 0.00005 + | > step_time: 6.21570 (3.53443) + | > loader_time: 0.40070 (0.11884) + + + --> STEP: 215/234 -- GLOBAL_STEP: 50525 + | > loss: -0.36119 (-0.26463) + | > log_mle: -0.60061 (-0.42225) + | > loss_dur: 0.23942 (0.15761) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.68076 (36.33727) + | > current_lr: 0.00005 + | > step_time: 4.40880 (3.56162) + | > loader_time: 0.18960 (0.11979) + + + --> STEP: 220/234 -- GLOBAL_STEP: 50530 + | > loss: -0.40254 (-0.26792) + | > log_mle: -0.64609 (-0.42745) + | > loss_dur: 0.24354 (0.15953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.28497 (37.81576) + | > current_lr: 0.00005 + | > step_time: 2.39400 (3.56927) + | > loader_time: 0.09070 (0.11933) + + + --> STEP: 225/234 -- GLOBAL_STEP: 50535 + | > loss: -0.46387 (-0.27099) + | > log_mle: -0.71056 (-0.43235) + | > loss_dur: 0.24669 (0.16135) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 114.51630 (39.20486) + | > current_lr: 0.00005 + | > step_time: 0.25020 (3.52308) + | > loader_time: 0.00680 (0.11747) + + + --> STEP: 230/234 -- GLOBAL_STEP: 50540 + | > loss: -0.43549 (-0.27383) + | > log_mle: -0.75065 (-0.43776) + | > loss_dur: 0.31516 (0.16393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 112.59873 (40.45842) + | > current_lr: 0.00005 + | > step_time: 0.29630 (3.45233) + | > loader_time: 0.00260 (0.11499) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.65005 (-0.57870) + | > avg_loss: -0.29871 (-0.01312) + | > avg_log_mle: -0.51640 (-0.00597) + | > avg_loss_dur: 0.21768 (-0.00715) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_50544.pth + + > EPOCH: 216/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 06:15:30)  + + --> STEP: 1/234 -- GLOBAL_STEP: 50545 + | > loss: -0.25733 (-0.25733) + | > log_mle: -0.33980 (-0.33980) + | > loss_dur: 0.08247 (0.08247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.86417 (17.86417) + | > current_lr: 0.00005 + | > step_time: 14.10620 (14.10624) + | > loader_time: 0.00490 (0.00494) + + + --> STEP: 6/234 -- GLOBAL_STEP: 50550 + | > loss: -0.27251 (-0.24583) + | > log_mle: -0.33989 (-0.34221) + | > loss_dur: 0.06738 (0.09638) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.18148 (17.73250) + | > current_lr: 0.00005 + | > step_time: 3.50520 (5.98414) + | > loader_time: 0.00140 (0.03129) + + + --> STEP: 11/234 -- GLOBAL_STEP: 50555 + | > loss: -0.28130 (-0.24996) + | > log_mle: -0.34947 (-0.34567) + | > loss_dur: 0.06817 (0.09572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.14976 (17.28281) + | > current_lr: 0.00005 + | > step_time: 2.00150 (5.13869) + | > loader_time: 0.00160 (0.03452) + + + --> STEP: 16/234 -- GLOBAL_STEP: 50560 + | > loss: -0.26427 (-0.25251) + | > log_mle: -0.34148 (-0.34552) + | > loss_dur: 0.07721 (0.09302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.63772 (17.24872) + | > current_lr: 0.00005 + | > step_time: 5.30200 (5.30161) + | > loader_time: 0.00230 (0.03589) + + + --> STEP: 21/234 -- GLOBAL_STEP: 50565 + | > loss: -0.23796 (-0.25229) + | > log_mle: -0.31901 (-0.34220) + | > loss_dur: 0.08105 (0.08991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.50589 (15.70366) + | > current_lr: 0.00005 + | > step_time: 4.79690 (5.05413) + | > loader_time: 0.00230 (0.03634) + + + --> STEP: 26/234 -- GLOBAL_STEP: 50570 + | > loss: -0.24729 (-0.25281) + | > log_mle: -0.33356 (-0.34101) + | > loss_dur: 0.08626 (0.08820) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.31950 (14.85759) + | > current_lr: 0.00005 + | > step_time: 2.90450 (4.73251) + | > loader_time: 0.00190 (0.03715) + + + --> STEP: 31/234 -- GLOBAL_STEP: 50575 + | > loss: -0.21189 (-0.25098) + | > log_mle: -0.32579 (-0.33960) + | > loss_dur: 0.11390 (0.08863) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.02456 (14.51100) + | > current_lr: 0.00005 + | > step_time: 2.20040 (4.52081) + | > loader_time: 0.09090 (0.03756) + + + --> STEP: 36/234 -- GLOBAL_STEP: 50580 + | > loss: -0.21883 (-0.24840) + | > log_mle: -0.32214 (-0.33815) + | > loss_dur: 0.10331 (0.08975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.76580 (14.03070) + | > current_lr: 0.00005 + | > step_time: 8.18800 (4.46521) + | > loader_time: 0.00910 (0.03773) + + + --> STEP: 41/234 -- GLOBAL_STEP: 50585 + | > loss: -0.24193 (-0.24567) + | > log_mle: -0.32973 (-0.33674) + | > loss_dur: 0.08780 (0.09106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.97291 (13.56983) + | > current_lr: 0.00005 + | > step_time: 5.89180 (4.50576) + | > loader_time: 0.10280 (0.03591) + + + --> STEP: 46/234 -- GLOBAL_STEP: 50590 + | > loss: -0.20450 (-0.24251) + | > log_mle: -0.32225 (-0.33565) + | > loss_dur: 0.11775 (0.09314) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.19736 (13.52282) + | > current_lr: 0.00005 + | > step_time: 0.88610 (4.28949) + | > loader_time: 0.00200 (0.03437) + + + --> STEP: 51/234 -- GLOBAL_STEP: 50595 + | > loss: -0.22242 (-0.24157) + | > log_mle: -0.32243 (-0.33461) + | > loss_dur: 0.10001 (0.09303) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.09417 (13.13291) + | > current_lr: 0.00005 + | > step_time: 0.88410 (3.98947) + | > loader_time: 0.00240 (0.03123) + + + --> STEP: 56/234 -- GLOBAL_STEP: 50600 + | > loss: -0.21486 (-0.23994) + | > log_mle: -0.33166 (-0.33418) + | > loss_dur: 0.11681 (0.09424) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.36779 (12.97428) + | > current_lr: 0.00005 + | > step_time: 2.07640 (3.85286) + | > loader_time: 0.00200 (0.03336) + + + --> STEP: 61/234 -- GLOBAL_STEP: 50605 + | > loss: -0.21459 (-0.23814) + | > log_mle: -0.32595 (-0.33378) + | > loss_dur: 0.11136 (0.09564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.09110 (12.90815) + | > current_lr: 0.00005 + | > step_time: 1.49140 (3.67583) + | > loader_time: 0.00170 (0.03082) + + + --> STEP: 66/234 -- GLOBAL_STEP: 50610 + | > loss: -0.23133 (-0.23614) + | > log_mle: -0.32064 (-0.33337) + | > loss_dur: 0.08931 (0.09723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.84244 (12.89110) + | > current_lr: 0.00005 + | > step_time: 2.69860 (3.60966) + | > loader_time: 0.00730 (0.03143) + + + --> STEP: 71/234 -- GLOBAL_STEP: 50615 + | > loss: -0.18891 (-0.23312) + | > log_mle: -0.34506 (-0.33258) + | > loss_dur: 0.15615 (0.09946) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.45988 (12.95770) + | > current_lr: 0.00005 + | > step_time: 2.80330 (3.50587) + | > loader_time: 0.08770 (0.03304) + + + --> STEP: 76/234 -- GLOBAL_STEP: 50620 + | > loss: -0.20807 (-0.23067) + | > log_mle: -0.32846 (-0.33195) + | > loss_dur: 0.12039 (0.10129) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.80842 (12.99585) + | > current_lr: 0.00005 + | > step_time: 2.39850 (3.47624) + | > loader_time: 0.00210 (0.03363) + + + --> STEP: 81/234 -- GLOBAL_STEP: 50625 + | > loss: -0.21308 (-0.22918) + | > log_mle: -0.33727 (-0.33137) + | > loss_dur: 0.12419 (0.10219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.91248 (12.97251) + | > current_lr: 0.00005 + | > step_time: 1.42370 (3.34549) + | > loader_time: 0.08380 (0.03278) + + + --> STEP: 86/234 -- GLOBAL_STEP: 50630 + | > loss: -0.21008 (-0.22759) + | > log_mle: -0.33495 (-0.33108) + | > loss_dur: 0.12487 (0.10349) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.80669 (13.00264) + | > current_lr: 0.00005 + | > step_time: 3.21520 (3.29869) + | > loader_time: 0.00200 (0.03286) + + + --> STEP: 91/234 -- GLOBAL_STEP: 50635 + | > loss: -0.19429 (-0.22639) + | > log_mle: -0.34480 (-0.33184) + | > loss_dur: 0.15051 (0.10545) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.41286 (13.14765) + | > current_lr: 0.00005 + | > step_time: 1.06960 (3.23377) + | > loader_time: 0.00190 (0.03119) + + + --> STEP: 96/234 -- GLOBAL_STEP: 50640 + | > loss: -0.19869 (-0.22654) + | > log_mle: -0.33296 (-0.33416) + | > loss_dur: 0.13427 (0.10762) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.54449 (13.86073) + | > current_lr: 0.00005 + | > step_time: 3.29590 (3.18321) + | > loader_time: 0.00250 (0.03159) + + + --> STEP: 101/234 -- GLOBAL_STEP: 50645 + | > loss: -0.21767 (-0.22607) + | > log_mle: -0.38743 (-0.33571) + | > loss_dur: 0.16976 (0.10965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.91056 (14.22461) + | > current_lr: 0.00005 + | > step_time: 3.81620 (3.15103) + | > loader_time: 0.08760 (0.03337) + + + --> STEP: 106/234 -- GLOBAL_STEP: 50650 + | > loss: -0.19552 (-0.22580) + | > log_mle: -0.38342 (-0.33781) + | > loss_dur: 0.18789 (0.11202) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.95364 (14.72429) + | > current_lr: 0.00005 + | > step_time: 3.52280 (3.10708) + | > loader_time: 0.17790 (0.03412) + + + --> STEP: 111/234 -- GLOBAL_STEP: 50655 + | > loss: -0.24143 (-0.22540) + | > log_mle: -0.43838 (-0.34001) + | > loss_dur: 0.19695 (0.11461) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.10588 (15.24515) + | > current_lr: 0.00005 + | > step_time: 0.83230 (3.06072) + | > loader_time: 0.00280 (0.03368) + + + --> STEP: 116/234 -- GLOBAL_STEP: 50660 + | > loss: -0.20662 (-0.22500) + | > log_mle: -0.40213 (-0.34220) + | > loss_dur: 0.19551 (0.11719) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.94399 (15.93773) + | > current_lr: 0.00005 + | > step_time: 1.41320 (2.99345) + | > loader_time: 0.00330 (0.03379) + + + --> STEP: 121/234 -- GLOBAL_STEP: 50665 + | > loss: -0.16983 (-0.22471) + | > log_mle: -0.31103 (-0.34352) + | > loss_dur: 0.14120 (0.11881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.69104 (16.45302) + | > current_lr: 0.00005 + | > step_time: 2.00400 (2.97549) + | > loader_time: 0.10010 (0.03485) + + + --> STEP: 126/234 -- GLOBAL_STEP: 50670 + | > loss: -0.25619 (-0.22464) + | > log_mle: -0.44576 (-0.34536) + | > loss_dur: 0.18958 (0.12071) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.11134 (17.02826) + | > current_lr: 0.00005 + | > step_time: 2.20080 (2.95664) + | > loader_time: 0.00350 (0.03426) + + + --> STEP: 131/234 -- GLOBAL_STEP: 50675 + | > loss: -0.29745 (-0.22564) + | > log_mle: -0.49495 (-0.34845) + | > loss_dur: 0.19751 (0.12280) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.32227 (17.91318) + | > current_lr: 0.00005 + | > step_time: 2.28090 (2.91729) + | > loader_time: 0.07360 (0.03417) + + + --> STEP: 136/234 -- GLOBAL_STEP: 50680 + | > loss: -0.33606 (-0.22686) + | > log_mle: -0.54866 (-0.35163) + | > loss_dur: 0.21260 (0.12477) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.55725 (18.60252) + | > current_lr: 0.00005 + | > step_time: 2.71550 (2.88370) + | > loader_time: 0.08540 (0.03363) + + + --> STEP: 141/234 -- GLOBAL_STEP: 50685 + | > loss: -0.26918 (-0.22721) + | > log_mle: -0.43926 (-0.35397) + | > loss_dur: 0.17008 (0.12677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.88077 (19.39712) + | > current_lr: 0.00005 + | > step_time: 1.30190 (2.84318) + | > loader_time: 0.00330 (0.03395) + + + --> STEP: 146/234 -- GLOBAL_STEP: 50690 + | > loss: -0.30188 (-0.22904) + | > log_mle: -0.49324 (-0.35834) + | > loss_dur: 0.19135 (0.12930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.05192 (20.30876) + | > current_lr: 0.00005 + | > step_time: 2.99650 (2.83288) + | > loader_time: 0.08510 (0.03401) + + + --> STEP: 151/234 -- GLOBAL_STEP: 50695 + | > loss: -0.27663 (-0.23072) + | > log_mle: -0.45873 (-0.36192) + | > loss_dur: 0.18210 (0.13120) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.17735 (21.02553) + | > current_lr: 0.00005 + | > step_time: 2.17010 (2.92661) + | > loader_time: 0.01120 (0.03756) + + + --> STEP: 156/234 -- GLOBAL_STEP: 50700 + | > loss: -0.29748 (-0.23353) + | > log_mle: -0.49949 (-0.36700) + | > loss_dur: 0.20201 (0.13347) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.79500 (22.23038) + | > current_lr: 0.00005 + | > step_time: 4.61500 (2.92848) + | > loader_time: 0.08630 (0.03759) + + + --> STEP: 161/234 -- GLOBAL_STEP: 50705 + | > loss: -0.33517 (-0.23566) + | > log_mle: -0.52864 (-0.37129) + | > loss_dur: 0.19347 (0.13562) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.87786 (23.15493) + | > current_lr: 0.00005 + | > step_time: 4.28290 (3.03437) + | > loader_time: 0.00210 (0.03940) + + + --> STEP: 166/234 -- GLOBAL_STEP: 50710 + | > loss: -0.28727 (-0.23756) + | > log_mle: -0.46760 (-0.37500) + | > loss_dur: 0.18034 (0.13744) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.46057 (23.97470) + | > current_lr: 0.00005 + | > step_time: 2.00170 (3.02975) + | > loader_time: 0.00800 (0.03835) + + + --> STEP: 171/234 -- GLOBAL_STEP: 50715 + | > loss: -0.37537 (-0.24069) + | > log_mle: -0.58082 (-0.38031) + | > loss_dur: 0.20545 (0.13962) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.33602 (25.34814) + | > current_lr: 0.00005 + | > step_time: 3.40370 (3.04165) + | > loader_time: 0.09570 (0.03906) + + + --> STEP: 176/234 -- GLOBAL_STEP: 50720 + | > loss: -0.34568 (-0.24368) + | > log_mle: -0.55504 (-0.38556) + | > loss_dur: 0.20935 (0.14188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.49222 (26.51584) + | > current_lr: 0.00005 + | > step_time: 4.80280 (3.04628) + | > loader_time: 0.09510 (0.03955) + + + --> STEP: 181/234 -- GLOBAL_STEP: 50725 + | > loss: -0.27481 (-0.24618) + | > log_mle: -0.48633 (-0.39028) + | > loss_dur: 0.21152 (0.14410) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.47622 (27.64996) + | > current_lr: 0.00005 + | > step_time: 6.39830 (3.09423) + | > loader_time: 0.00560 (0.03952) + + + --> STEP: 186/234 -- GLOBAL_STEP: 50730 + | > loss: -0.29097 (-0.24866) + | > log_mle: -0.52672 (-0.39512) + | > loss_dur: 0.23576 (0.14646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.17266 (29.05501) + | > current_lr: 0.00005 + | > step_time: 2.00770 (3.07199) + | > loader_time: 0.00430 (0.03973) + + + --> STEP: 191/234 -- GLOBAL_STEP: 50735 + | > loss: -0.33698 (-0.25136) + | > log_mle: -0.54867 (-0.39971) + | > loss_dur: 0.21169 (0.14835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.93427 (30.27895) + | > current_lr: 0.00005 + | > step_time: 3.79050 (3.06381) + | > loader_time: 0.00290 (0.04019) + + + --> STEP: 196/234 -- GLOBAL_STEP: 50740 + | > loss: -0.32018 (-0.25429) + | > log_mle: -0.54550 (-0.40443) + | > loss_dur: 0.22533 (0.15014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.40761 (31.28882) + | > current_lr: 0.00005 + | > step_time: 2.68580 (3.04283) + | > loader_time: 0.00270 (0.03967) + + + --> STEP: 201/234 -- GLOBAL_STEP: 50745 + | > loss: -0.27283 (-0.25664) + | > log_mle: -0.50759 (-0.40869) + | > loss_dur: 0.23476 (0.15205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.02849 (32.28186) + | > current_lr: 0.00005 + | > step_time: 2.50120 (3.03325) + | > loader_time: 0.09000 (0.04011) + + + --> STEP: 206/234 -- GLOBAL_STEP: 50750 + | > loss: -0.38703 (-0.25948) + | > log_mle: -0.61465 (-0.41333) + | > loss_dur: 0.22762 (0.15385) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.30505 (33.51882) + | > current_lr: 0.00005 + | > step_time: 13.30920 (3.10789) + | > loader_time: 0.08590 (0.04324) + + + --> STEP: 211/234 -- GLOBAL_STEP: 50755 + | > loss: -0.44782 (-0.26286) + | > log_mle: -0.69047 (-0.41868) + | > loss_dur: 0.24264 (0.15582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 99.93334 (34.82841) + | > current_lr: 0.00005 + | > step_time: 5.60800 (3.16126) + | > loader_time: 0.09240 (0.04400) + + + --> STEP: 216/234 -- GLOBAL_STEP: 50760 + | > loss: -0.39940 (-0.26610) + | > log_mle: -0.66071 (-0.42376) + | > loss_dur: 0.26130 (0.15767) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 141.69954 (36.24945) + | > current_lr: 0.00005 + | > step_time: 2.20040 (3.19636) + | > loader_time: 0.00530 (0.04477) + + + --> STEP: 221/234 -- GLOBAL_STEP: 50765 + | > loss: -0.36392 (-0.26936) + | > log_mle: -0.57980 (-0.42875) + | > loss_dur: 0.21588 (0.15939) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.33470 (37.42041) + | > current_lr: 0.00005 + | > step_time: 2.20360 (3.23743) + | > loader_time: 0.07570 (0.05126) + + + --> STEP: 226/234 -- GLOBAL_STEP: 50770 + | > loss: -0.44620 (-0.27295) + | > log_mle: -0.69258 (-0.43433) + | > loss_dur: 0.24638 (0.16137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.69100 (38.70956) + | > current_lr: 0.00005 + | > step_time: 0.24530 (3.19413) + | > loader_time: 0.00310 (0.05029) + + + --> STEP: 231/234 -- GLOBAL_STEP: 50775 + | > loss: -0.37818 (-0.27581) + | > log_mle: -0.75655 (-0.44030) + | > loss_dur: 0.37837 (0.16449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 110.59769 (40.01441) + | > current_lr: 0.00005 + | > step_time: 0.28860 (3.13072) + | > loader_time: 0.00330 (0.04928) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05054 (-0.59952) + | > avg_loss: -0.31479 (-0.01608) + | > avg_log_mle: -0.53054 (-0.01414) + | > avg_loss_dur: 0.21574 (-0.00194) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_50778.pth + + > EPOCH: 217/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 06:28:59)  + + --> STEP: 2/234 -- GLOBAL_STEP: 50780 + | > loss: -0.26510 (-0.25933) + | > log_mle: -0.35553 (-0.34910) + | > loss_dur: 0.09043 (0.08977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.01852 (18.90999) + | > current_lr: 0.00005 + | > step_time: 4.20020 (5.65522) + | > loader_time: 0.01230 (0.65714) + + + --> STEP: 7/234 -- GLOBAL_STEP: 50785 + | > loss: -0.25824 (-0.24559) + | > log_mle: -0.34563 (-0.34413) + | > loss_dur: 0.08739 (0.09854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.24721 (18.68847) + | > current_lr: 0.00005 + | > step_time: 1.79860 (5.88428) + | > loader_time: 0.00130 (0.24604) + + + --> STEP: 12/234 -- GLOBAL_STEP: 50790 + | > loss: -0.24587 (-0.24939) + | > log_mle: -0.33956 (-0.34594) + | > loss_dur: 0.09368 (0.09654) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.65259 (17.99604) + | > current_lr: 0.00005 + | > step_time: 2.20050 (5.09581) + | > loader_time: 0.10580 (0.18563) + + + --> STEP: 17/234 -- GLOBAL_STEP: 50795 + | > loss: -0.25626 (-0.25248) + | > log_mle: -0.33193 (-0.34524) + | > loss_dur: 0.07567 (0.09277) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.57065 (16.89911) + | > current_lr: 0.00005 + | > step_time: 3.57900 (4.31329) + | > loader_time: 0.00240 (0.14358) + + + --> STEP: 22/234 -- GLOBAL_STEP: 50800 + | > loss: -0.24854 (-0.25166) + | > log_mle: -0.34340 (-0.34314) + | > loss_dur: 0.09485 (0.09148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.91891 (15.75688) + | > current_lr: 0.00005 + | > step_time: 7.68830 (4.24052) + | > loader_time: 0.18870 (0.12679) + + + --> STEP: 27/234 -- GLOBAL_STEP: 50805 + | > loss: -0.25557 (-0.25342) + | > log_mle: -0.33738 (-0.34205) + | > loss_dur: 0.08181 (0.08863) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74022 (14.70925) + | > current_lr: 0.00005 + | > step_time: 2.18300 (4.09969) + | > loader_time: 0.09380 (0.11713) + + + --> STEP: 32/234 -- GLOBAL_STEP: 50810 + | > loss: -0.26101 (-0.25257) + | > log_mle: -0.34501 (-0.34114) + | > loss_dur: 0.08401 (0.08857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.05093 (14.01937) + | > current_lr: 0.00005 + | > step_time: 4.20550 (4.01246) + | > loader_time: 0.10470 (0.10607) + + + --> STEP: 37/234 -- GLOBAL_STEP: 50815 + | > loss: -0.23364 (-0.24957) + | > log_mle: -0.31705 (-0.33899) + | > loss_dur: 0.08341 (0.08943) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.40473 (13.82100) + | > current_lr: 0.00005 + | > step_time: 3.61380 (3.98614) + | > loader_time: 0.09680 (0.09680) + + + --> STEP: 42/234 -- GLOBAL_STEP: 50820 + | > loss: -0.21117 (-0.24686) + | > log_mle: -0.31032 (-0.33717) + | > loss_dur: 0.09914 (0.09031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.08087 (13.55895) + | > current_lr: 0.00005 + | > step_time: 1.21900 (3.89589) + | > loader_time: 0.07150 (0.09674) + + + --> STEP: 47/234 -- GLOBAL_STEP: 50825 + | > loss: -0.22044 (-0.24401) + | > log_mle: -0.32271 (-0.33619) + | > loss_dur: 0.10228 (0.09218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.58097 (13.36491) + | > current_lr: 0.00005 + | > step_time: 1.90240 (3.76043) + | > loader_time: 0.00710 (0.09408) + + + --> STEP: 52/234 -- GLOBAL_STEP: 50830 + | > loss: -0.19879 (-0.24231) + | > log_mle: -0.32103 (-0.33498) + | > loss_dur: 0.12224 (0.09267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.27147 (12.96429) + | > current_lr: 0.00005 + | > step_time: 2.49950 (3.71598) + | > loader_time: 0.08500 (0.09379) + + + --> STEP: 57/234 -- GLOBAL_STEP: 50835 + | > loss: -0.20560 (-0.24083) + | > log_mle: -0.31412 (-0.33441) + | > loss_dur: 0.10853 (0.09358) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56875 (12.62965) + | > current_lr: 0.00005 + | > step_time: 1.39280 (3.58957) + | > loader_time: 0.00620 (0.08957) + + + --> STEP: 62/234 -- GLOBAL_STEP: 50840 + | > loss: -0.18695 (-0.23906) + | > log_mle: -0.35276 (-0.33466) + | > loss_dur: 0.16582 (0.09560) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.55761 (12.74046) + | > current_lr: 0.00005 + | > step_time: 5.80560 (3.56151) + | > loader_time: 0.09100 (0.08819) + + + --> STEP: 67/234 -- GLOBAL_STEP: 50845 + | > loss: -0.20930 (-0.23764) + | > log_mle: -0.33435 (-0.33403) + | > loss_dur: 0.12505 (0.09639) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.70965 (12.55563) + | > current_lr: 0.00005 + | > step_time: 1.77930 (3.45146) + | > loader_time: 0.01210 (0.08454) + + + --> STEP: 72/234 -- GLOBAL_STEP: 50850 + | > loss: -0.20546 (-0.23495) + | > log_mle: -0.32093 (-0.33321) + | > loss_dur: 0.11546 (0.09826) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.83038 (12.59093) + | > current_lr: 0.00005 + | > step_time: 2.11440 (3.41017) + | > loader_time: 0.01600 (0.08162) + + + --> STEP: 77/234 -- GLOBAL_STEP: 50855 + | > loss: -0.20621 (-0.23259) + | > log_mle: -0.32849 (-0.33293) + | > loss_dur: 0.12228 (0.10034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.57999 (12.64820) + | > current_lr: 0.00005 + | > step_time: 3.41200 (3.38422) + | > loader_time: 0.00690 (0.07872) + + + --> STEP: 82/234 -- GLOBAL_STEP: 50860 + | > loss: -0.20920 (-0.23103) + | > log_mle: -0.32242 (-0.33237) + | > loss_dur: 0.11321 (0.10134) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.99150 (12.64560) + | > current_lr: 0.00005 + | > step_time: 5.81350 (3.35255) + | > loader_time: 0.10770 (0.07819) + + + --> STEP: 87/234 -- GLOBAL_STEP: 50865 + | > loss: -0.19764 (-0.22932) + | > log_mle: -0.32295 (-0.33206) + | > loss_dur: 0.12531 (0.10274) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.26635 (12.77203) + | > current_lr: 0.00005 + | > step_time: 4.08710 (3.31957) + | > loader_time: 0.02880 (0.07566) + + + --> STEP: 92/234 -- GLOBAL_STEP: 50870 + | > loss: -0.23046 (-0.22839) + | > log_mle: -0.36761 (-0.33328) + | > loss_dur: 0.13716 (0.10489) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.59224 (13.06762) + | > current_lr: 0.00005 + | > step_time: 2.59440 (3.27381) + | > loader_time: 0.10450 (0.07585) + + + --> STEP: 97/234 -- GLOBAL_STEP: 50875 + | > loss: -0.21303 (-0.22833) + | > log_mle: -0.35336 (-0.33544) + | > loss_dur: 0.14033 (0.10711) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.99521 (13.57760) + | > current_lr: 0.00005 + | > step_time: 2.12240 (3.25799) + | > loader_time: 0.08240 (0.07315) + + + --> STEP: 102/234 -- GLOBAL_STEP: 50880 + | > loss: -0.18309 (-0.22746) + | > log_mle: -0.33883 (-0.33674) + | > loss_dur: 0.15574 (0.10928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.39283 (13.96829) + | > current_lr: 0.00005 + | > step_time: 4.33900 (3.23946) + | > loader_time: 0.08910 (0.07174) + + + --> STEP: 107/234 -- GLOBAL_STEP: 50885 + | > loss: -0.21524 (-0.22735) + | > log_mle: -0.38077 (-0.33916) + | > loss_dur: 0.16553 (0.11181) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.00372 (14.66745) + | > current_lr: 0.00005 + | > step_time: 4.22040 (3.22280) + | > loader_time: 0.09440 (0.07086) + + + --> STEP: 112/234 -- GLOBAL_STEP: 50890 + | > loss: -0.22402 (-0.22694) + | > log_mle: -0.39522 (-0.34140) + | > loss_dur: 0.17120 (0.11446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.11052 (15.43065) + | > current_lr: 0.00005 + | > step_time: 3.41350 (3.17799) + | > loader_time: 0.00340 (0.06982) + + + --> STEP: 117/234 -- GLOBAL_STEP: 50895 + | > loss: -0.23140 (-0.22669) + | > log_mle: -0.39145 (-0.34354) + | > loss_dur: 0.16004 (0.11685) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.47888 (16.00797) + | > current_lr: 0.00005 + | > step_time: 4.39850 (3.18769) + | > loader_time: 0.09450 (0.07093) + + + --> STEP: 122/234 -- GLOBAL_STEP: 50900 + | > loss: -0.21005 (-0.22613) + | > log_mle: -0.36077 (-0.34469) + | > loss_dur: 0.15071 (0.11857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.91259 (16.40121) + | > current_lr: 0.00005 + | > step_time: 2.00400 (3.16128) + | > loader_time: 0.00270 (0.06952) + + + --> STEP: 127/234 -- GLOBAL_STEP: 50905 + | > loss: -0.23482 (-0.22632) + | > log_mle: -0.42059 (-0.34715) + | > loss_dur: 0.18577 (0.12083) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.57905 (17.04627) + | > current_lr: 0.00005 + | > step_time: 2.48620 (3.14153) + | > loader_time: 0.00320 (0.06916) + + + --> STEP: 132/234 -- GLOBAL_STEP: 50910 + | > loss: -0.23910 (-0.22705) + | > log_mle: -0.39532 (-0.34978) + | > loss_dur: 0.15622 (0.12274) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.44748 (17.98355) + | > current_lr: 0.00005 + | > step_time: 4.82300 (3.18450) + | > loader_time: 0.09270 (0.06946) + + + --> STEP: 137/234 -- GLOBAL_STEP: 50915 + | > loss: -0.21634 (-0.22779) + | > log_mle: -0.41256 (-0.35284) + | > loss_dur: 0.19622 (0.12505) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.75370 (18.61872) + | > current_lr: 0.00005 + | > step_time: 4.12790 (3.16472) + | > loader_time: 0.08660 (0.06844) + + + --> STEP: 142/234 -- GLOBAL_STEP: 50920 + | > loss: -0.23000 (-0.22816) + | > log_mle: -0.42399 (-0.35536) + | > loss_dur: 0.19398 (0.12720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.58911 (19.25594) + | > current_lr: 0.00005 + | > step_time: 4.09520 (3.20985) + | > loader_time: 0.11830 (0.07444) + + + --> STEP: 147/234 -- GLOBAL_STEP: 50925 + | > loss: -0.25921 (-0.23015) + | > log_mle: -0.43313 (-0.35971) + | > loss_dur: 0.17392 (0.12956) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.20304 (19.95294) + | > current_lr: 0.00005 + | > step_time: 3.79420 (3.20761) + | > loader_time: 0.09860 (0.07448) + + + --> STEP: 152/234 -- GLOBAL_STEP: 50930 + | > loss: -0.30986 (-0.23230) + | > log_mle: -0.52061 (-0.36386) + | > loss_dur: 0.21075 (0.13156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.61446 (21.03707) + | > current_lr: 0.00005 + | > step_time: 3.89790 (3.24319) + | > loader_time: 0.06880 (0.07390) + + + --> STEP: 157/234 -- GLOBAL_STEP: 50935 + | > loss: -0.26873 (-0.23500) + | > log_mle: -0.46408 (-0.36861) + | > loss_dur: 0.19536 (0.13362) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.81828 (22.45608) + | > current_lr: 0.00005 + | > step_time: 1.77560 (3.24339) + | > loader_time: 0.11630 (0.07562) + + + --> STEP: 162/234 -- GLOBAL_STEP: 50940 + | > loss: -0.29988 (-0.23746) + | > log_mle: -0.49153 (-0.37315) + | > loss_dur: 0.19165 (0.13568) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.95728 (23.70537) + | > current_lr: 0.00005 + | > step_time: 8.48880 (3.26113) + | > loader_time: 0.09840 (0.07451) + + + --> STEP: 167/234 -- GLOBAL_STEP: 50945 + | > loss: -0.38773 (-0.23984) + | > log_mle: -0.58846 (-0.37740) + | > loss_dur: 0.20073 (0.13756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.79866 (24.69303) + | > current_lr: 0.00005 + | > step_time: 1.51410 (3.29306) + | > loader_time: 0.08340 (0.07578) + + + --> STEP: 172/234 -- GLOBAL_STEP: 50950 + | > loss: -0.35102 (-0.24255) + | > log_mle: -0.56908 (-0.38240) + | > loss_dur: 0.21807 (0.13984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.63422 (26.21639) + | > current_lr: 0.00005 + | > step_time: 3.19960 (3.30441) + | > loader_time: 0.00950 (0.07536) + + + --> STEP: 177/234 -- GLOBAL_STEP: 50955 + | > loss: -0.32390 (-0.24511) + | > log_mle: -0.53608 (-0.38720) + | > loss_dur: 0.21218 (0.14208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.03813 (27.29312) + | > current_lr: 0.00005 + | > step_time: 4.50410 (3.27599) + | > loader_time: 0.19120 (0.07525) + + + --> STEP: 182/234 -- GLOBAL_STEP: 50960 + | > loss: -0.33666 (-0.24755) + | > log_mle: -0.58322 (-0.39204) + | > loss_dur: 0.24656 (0.14450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.27803 (28.34693) + | > current_lr: 0.00005 + | > step_time: 3.59530 (3.29682) + | > loader_time: 0.07720 (0.07481) + + + --> STEP: 187/234 -- GLOBAL_STEP: 50965 + | > loss: -0.35520 (-0.25013) + | > log_mle: -0.58063 (-0.39675) + | > loss_dur: 0.22543 (0.14662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.66437 (29.51148) + | > current_lr: 0.00005 + | > step_time: 2.80340 (3.32221) + | > loader_time: 0.18680 (0.07634) + + + --> STEP: 192/234 -- GLOBAL_STEP: 50970 + | > loss: -0.37373 (-0.25280) + | > log_mle: -0.59117 (-0.40124) + | > loss_dur: 0.21744 (0.14845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.74811 (30.76489) + | > current_lr: 0.00005 + | > step_time: 4.10190 (3.33781) + | > loader_time: 0.09920 (0.07562) + + + --> STEP: 197/234 -- GLOBAL_STEP: 50975 + | > loss: -0.37620 (-0.25542) + | > log_mle: -0.57529 (-0.40558) + | > loss_dur: 0.19909 (0.15016) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.87319 (31.69180) + | > current_lr: 0.00005 + | > step_time: 4.10490 (3.36494) + | > loader_time: 0.00790 (0.07672) + + + --> STEP: 202/234 -- GLOBAL_STEP: 50980 + | > loss: -0.45019 (-0.25802) + | > log_mle: -0.66702 (-0.41012) + | > loss_dur: 0.21683 (0.15209) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 104.98601 (32.70801) + | > current_lr: 0.00005 + | > step_time: 4.78810 (3.36329) + | > loader_time: 0.59500 (0.07866) + + + --> STEP: 207/234 -- GLOBAL_STEP: 50985 + | > loss: -0.41686 (-0.26054) + | > log_mle: -0.64866 (-0.41447) + | > loss_dur: 0.23180 (0.15393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.76255 (33.86930) + | > current_lr: 0.00005 + | > step_time: 3.89250 (3.37304) + | > loader_time: 0.19120 (0.07995) + + + --> STEP: 212/234 -- GLOBAL_STEP: 50990 + | > loss: -0.39727 (-0.26369) + | > log_mle: -0.63301 (-0.41958) + | > loss_dur: 0.23574 (0.15589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 96.20686 (35.03187) + | > current_lr: 0.00005 + | > step_time: 7.18780 (3.41516) + | > loader_time: 0.12280 (0.07952) + + + --> STEP: 217/234 -- GLOBAL_STEP: 50995 + | > loss: -0.41596 (-0.26695) + | > log_mle: -0.65823 (-0.42468) + | > loss_dur: 0.24227 (0.15773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.79992 (36.21891) + | > current_lr: 0.00005 + | > step_time: 5.04780 (3.41468) + | > loader_time: 0.19290 (0.07985) + + + --> STEP: 222/234 -- GLOBAL_STEP: 51000 + | > loss: -0.39477 (-0.27010) + | > log_mle: -0.66471 (-0.42973) + | > loss_dur: 0.26994 (0.15963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.59725 (37.28482) + | > current_lr: 0.00005 + | > step_time: 1.78340 (3.40397) + | > loader_time: 0.00470 (0.07934) + + + --> STEP: 227/234 -- GLOBAL_STEP: 51005 + | > loss: -0.38160 (-0.27370) + | > log_mle: -0.64431 (-0.43521) + | > loss_dur: 0.26270 (0.16151) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.25918 (38.62535) + | > current_lr: 0.00005 + | > step_time: 0.25820 (3.34281) + | > loader_time: 0.00310 (0.07832) + + + --> STEP: 232/234 -- GLOBAL_STEP: 51010 + | > loss: -0.34120 (-0.27604) + | > log_mle: -0.83126 (-0.44167) + | > loss_dur: 0.49006 (0.16563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 173.17992 (40.71767) + | > current_lr: 0.00005 + | > step_time: 0.36080 (3.27712) + | > loader_time: 0.05990 (0.07695) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03042 (-0.02012) + | > avg_loss: -0.29353 (+0.02127) + | > avg_log_mle: -0.51171 (+0.01882) + | > avg_loss_dur: 0.21819 (+0.00244) + + + > EPOCH: 218/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 06:42:56)  + + --> STEP: 3/234 -- GLOBAL_STEP: 51015 + | > loss: -0.17907 (-0.23099) + | > log_mle: -0.33538 (-0.34549) + | > loss_dur: 0.15631 (0.11450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.84252 (19.62890) + | > current_lr: 0.00005 + | > step_time: 3.00560 (3.50880) + | > loader_time: 0.08670 (6.15797) + + + --> STEP: 8/234 -- GLOBAL_STEP: 51020 + | > loss: -0.27051 (-0.24567) + | > log_mle: -0.35961 (-0.34679) + | > loss_dur: 0.08910 (0.10112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.91392 (18.51620) + | > current_lr: 0.00005 + | > step_time: 2.31130 (4.55416) + | > loader_time: 0.00190 (2.33762) + + + --> STEP: 13/234 -- GLOBAL_STEP: 51025 + | > loss: -0.27362 (-0.24838) + | > log_mle: -0.35456 (-0.34754) + | > loss_dur: 0.08095 (0.09916) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.82541 (17.80523) + | > current_lr: 0.00005 + | > step_time: 1.29490 (3.78826) + | > loader_time: 0.00640 (1.44079) + + + --> STEP: 18/234 -- GLOBAL_STEP: 51030 + | > loss: -0.23537 (-0.25058) + | > log_mle: -0.33484 (-0.34579) + | > loss_dur: 0.09947 (0.09521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.87776 (16.14847) + | > current_lr: 0.00005 + | > step_time: 3.62410 (3.70565) + | > loader_time: 0.09520 (1.04709) + + + --> STEP: 23/234 -- GLOBAL_STEP: 51035 + | > loss: -0.27486 (-0.25301) + | > log_mle: -0.34876 (-0.34468) + | > loss_dur: 0.07390 (0.09167) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.96849 (14.61118) + | > current_lr: 0.00005 + | > step_time: 3.22320 (3.72797) + | > loader_time: 0.08170 (0.83066) + + + --> STEP: 28/234 -- GLOBAL_STEP: 51040 + | > loss: -0.30583 (-0.25618) + | > log_mle: -0.35804 (-0.34421) + | > loss_dur: 0.05221 (0.08803) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.55962 (13.58754) + | > current_lr: 0.00005 + | > step_time: 1.92340 (3.72334) + | > loader_time: 0.00300 (0.69249) + + + --> STEP: 33/234 -- GLOBAL_STEP: 51045 + | > loss: -0.24665 (-0.25412) + | > log_mle: -0.33215 (-0.34251) + | > loss_dur: 0.08550 (0.08839) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.38670 (13.16125) + | > current_lr: 0.00005 + | > step_time: 3.61170 (3.89645) + | > loader_time: 0.09230 (0.60227) + + + --> STEP: 38/234 -- GLOBAL_STEP: 51050 + | > loss: -0.23544 (-0.25130) + | > log_mle: -0.34165 (-0.34083) + | > loss_dur: 0.10621 (0.08953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.83882 (12.95801) + | > current_lr: 0.00005 + | > step_time: 3.90600 (3.82346) + | > loader_time: 0.01250 (0.53054) + + + --> STEP: 43/234 -- GLOBAL_STEP: 51055 + | > loss: -0.20974 (-0.24777) + | > log_mle: -0.33504 (-0.33895) + | > loss_dur: 0.12530 (0.09118) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.70367 (12.92905) + | > current_lr: 0.00005 + | > step_time: 1.71580 (3.68755) + | > loader_time: 0.08570 (0.47480) + + + --> STEP: 48/234 -- GLOBAL_STEP: 51060 + | > loss: -0.23943 (-0.24541) + | > log_mle: -0.32718 (-0.33788) + | > loss_dur: 0.08775 (0.09247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.08503 (12.78683) + | > current_lr: 0.00005 + | > step_time: 3.59510 (3.60745) + | > loader_time: 0.04770 (0.43094) + + + --> STEP: 53/234 -- GLOBAL_STEP: 51065 + | > loss: -0.21603 (-0.24344) + | > log_mle: -0.33299 (-0.33688) + | > loss_dur: 0.11696 (0.09344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.20210 (12.44819) + | > current_lr: 0.00005 + | > step_time: 1.38310 (3.50945) + | > loader_time: 0.00170 (0.39723) + + + --> STEP: 58/234 -- GLOBAL_STEP: 51070 + | > loss: -0.23130 (-0.24193) + | > log_mle: -0.32841 (-0.33620) + | > loss_dur: 0.09712 (0.09426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.61815 (12.28504) + | > current_lr: 0.00005 + | > step_time: 2.57520 (3.43808) + | > loader_time: 0.00270 (0.36434) + + + --> STEP: 63/234 -- GLOBAL_STEP: 51075 + | > loss: -0.20241 (-0.23933) + | > log_mle: -0.32149 (-0.33620) + | > loss_dur: 0.11908 (0.09687) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.25848 (12.56754) + | > current_lr: 0.00005 + | > step_time: 2.50400 (3.38305) + | > loader_time: 0.00210 (0.33988) + + + --> STEP: 68/234 -- GLOBAL_STEP: 51080 + | > loss: -0.18814 (-0.23762) + | > log_mle: -0.31677 (-0.33530) + | > loss_dur: 0.12863 (0.09769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.68284 (12.50818) + | > current_lr: 0.00005 + | > step_time: 2.72310 (3.34219) + | > loader_time: 0.07240 (0.31980) + + + --> STEP: 73/234 -- GLOBAL_STEP: 51085 + | > loss: -0.18660 (-0.23486) + | > log_mle: -0.33513 (-0.33462) + | > loss_dur: 0.14853 (0.09977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.21426 (12.58286) + | > current_lr: 0.00005 + | > step_time: 2.59940 (3.33880) + | > loader_time: 0.00450 (0.29956) + + + --> STEP: 78/234 -- GLOBAL_STEP: 51090 + | > loss: -0.19553 (-0.23300) + | > log_mle: -0.31584 (-0.33395) + | > loss_dur: 0.12031 (0.10095) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.58584 (12.58517) + | > current_lr: 0.00005 + | > step_time: 3.41210 (3.29841) + | > loader_time: 0.09350 (0.28387) + + + --> STEP: 83/234 -- GLOBAL_STEP: 51095 + | > loss: -0.18947 (-0.23123) + | > log_mle: -0.33370 (-0.33361) + | > loss_dur: 0.14423 (0.10238) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.74628 (12.54313) + | > current_lr: 0.00005 + | > step_time: 5.93340 (3.27749) + | > loader_time: 0.10230 (0.27099) + + + --> STEP: 88/234 -- GLOBAL_STEP: 51100 + | > loss: -0.21219 (-0.23000) + | > log_mle: -0.37184 (-0.33381) + | > loss_dur: 0.15965 (0.10380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.74119 (12.63834) + | > current_lr: 0.00005 + | > step_time: 3.71860 (3.24052) + | > loader_time: 0.09070 (0.25783) + + + --> STEP: 93/234 -- GLOBAL_STEP: 51105 + | > loss: -0.22222 (-0.22928) + | > log_mle: -0.38101 (-0.33522) + | > loss_dur: 0.15880 (0.10594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.35902 (13.10430) + | > current_lr: 0.00005 + | > step_time: 2.31690 (3.18247) + | > loader_time: 0.07720 (0.24706) + + + --> STEP: 98/234 -- GLOBAL_STEP: 51110 + | > loss: -0.19677 (-0.22903) + | > log_mle: -0.31567 (-0.33669) + | > loss_dur: 0.11890 (0.10767) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.70545 (13.58198) + | > current_lr: 0.00005 + | > step_time: 3.22320 (3.14993) + | > loader_time: 0.01140 (0.23550) + + + --> STEP: 103/234 -- GLOBAL_STEP: 51115 + | > loss: -0.23604 (-0.22866) + | > log_mle: -0.40831 (-0.33886) + | > loss_dur: 0.17227 (0.11020) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.42210 (14.33431) + | > current_lr: 0.00005 + | > step_time: 2.60360 (3.11957) + | > loader_time: 0.07620 (0.22745) + + + --> STEP: 108/234 -- GLOBAL_STEP: 51120 + | > loss: -0.21829 (-0.22832) + | > log_mle: -0.35745 (-0.34072) + | > loss_dur: 0.13916 (0.11240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.30273 (14.72333) + | > current_lr: 0.00005 + | > step_time: 1.68350 (3.08231) + | > loader_time: 0.09920 (0.22137) + + + --> STEP: 113/234 -- GLOBAL_STEP: 51125 + | > loss: -0.23935 (-0.22812) + | > log_mle: -0.40246 (-0.34331) + | > loss_dur: 0.16311 (0.11518) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.25630 (15.51913) + | > current_lr: 0.00005 + | > step_time: 2.11340 (3.07635) + | > loader_time: 0.06660 (0.21552) + + + --> STEP: 118/234 -- GLOBAL_STEP: 51130 + | > loss: -0.21224 (-0.22767) + | > log_mle: -0.37291 (-0.34513) + | > loss_dur: 0.16068 (0.11746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.00644 (15.89162) + | > current_lr: 0.00005 + | > step_time: 1.69050 (3.08329) + | > loader_time: 0.09070 (0.20952) + + + --> STEP: 123/234 -- GLOBAL_STEP: 51135 + | > loss: -0.18471 (-0.22700) + | > log_mle: -0.33987 (-0.34594) + | > loss_dur: 0.15517 (0.11894) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.25699 (16.21173) + | > current_lr: 0.00005 + | > step_time: 3.49220 (3.06111) + | > loader_time: 0.09550 (0.20488) + + + --> STEP: 128/234 -- GLOBAL_STEP: 51140 + | > loss: -0.23830 (-0.22760) + | > log_mle: -0.39379 (-0.34878) + | > loss_dur: 0.15549 (0.12118) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.54731 (17.17758) + | > current_lr: 0.00005 + | > step_time: 3.42380 (3.08684) + | > loader_time: 0.08820 (0.19860) + + + --> STEP: 133/234 -- GLOBAL_STEP: 51145 + | > loss: -0.25232 (-0.22851) + | > log_mle: -0.42827 (-0.35182) + | > loss_dur: 0.17595 (0.12331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.16877 (17.90448) + | > current_lr: 0.00005 + | > step_time: 4.68100 (3.13296) + | > loader_time: 0.01130 (0.19357) + + + --> STEP: 138/234 -- GLOBAL_STEP: 51150 + | > loss: -0.21588 (-0.22917) + | > log_mle: -0.37484 (-0.35464) + | > loss_dur: 0.15896 (0.12547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.24677 (18.68786) + | > current_lr: 0.00005 + | > step_time: 2.47270 (3.11141) + | > loader_time: 0.00260 (0.18973) + + + --> STEP: 143/234 -- GLOBAL_STEP: 51155 + | > loss: -0.28920 (-0.23031) + | > log_mle: -0.52609 (-0.35826) + | > loss_dur: 0.23689 (0.12795) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.28930 (19.73014) + | > current_lr: 0.00005 + | > step_time: 2.79860 (3.11648) + | > loader_time: 0.00670 (0.18580) + + + --> STEP: 148/234 -- GLOBAL_STEP: 51160 + | > loss: -0.26356 (-0.23172) + | > log_mle: -0.41921 (-0.36151) + | > loss_dur: 0.15565 (0.12979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.59321 (20.98557) + | > current_lr: 0.00005 + | > step_time: 5.78770 (3.13369) + | > loader_time: 0.09450 (0.18316) + + + --> STEP: 153/234 -- GLOBAL_STEP: 51165 + | > loss: -0.35158 (-0.23403) + | > log_mle: -0.55378 (-0.36600) + | > loss_dur: 0.20220 (0.13198) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.06459 (21.85850) + | > current_lr: 0.00005 + | > step_time: 2.21160 (3.11425) + | > loader_time: 0.08020 (0.17877) + + + --> STEP: 158/234 -- GLOBAL_STEP: 51170 + | > loss: -0.27805 (-0.23588) + | > log_mle: -0.49164 (-0.37001) + | > loss_dur: 0.21359 (0.13413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.14288 (22.78147) + | > current_lr: 0.00005 + | > step_time: 4.80200 (3.11070) + | > loader_time: 0.09260 (0.17541) + + + --> STEP: 163/234 -- GLOBAL_STEP: 51175 + | > loss: -0.27147 (-0.23806) + | > log_mle: -0.46544 (-0.37414) + | > loss_dur: 0.19397 (0.13608) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.39569 (23.51777) + | > current_lr: 0.00005 + | > step_time: 3.49830 (3.12055) + | > loader_time: 0.00860 (0.17213) + + + --> STEP: 168/234 -- GLOBAL_STEP: 51180 + | > loss: -0.30012 (-0.24053) + | > log_mle: -0.52621 (-0.37863) + | > loss_dur: 0.22609 (0.13810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.06754 (24.32718) + | > current_lr: 0.00005 + | > step_time: 3.71390 (3.15264) + | > loader_time: 0.00880 (0.16942) + + + --> STEP: 173/234 -- GLOBAL_STEP: 51185 + | > loss: -0.32066 (-0.24332) + | > log_mle: -0.53104 (-0.38359) + | > loss_dur: 0.21037 (0.14027) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.22604 (25.80234) + | > current_lr: 0.00005 + | > step_time: 2.11780 (3.15478) + | > loader_time: 0.07900 (0.16617) + + + --> STEP: 178/234 -- GLOBAL_STEP: 51190 + | > loss: -0.36372 (-0.24612) + | > log_mle: -0.59503 (-0.38863) + | > loss_dur: 0.23131 (0.14251) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.14595 (26.82716) + | > current_lr: 0.00005 + | > step_time: 5.90800 (3.22342) + | > loader_time: 0.10180 (0.16426) + + + --> STEP: 183/234 -- GLOBAL_STEP: 51195 + | > loss: -0.37493 (-0.24844) + | > log_mle: -0.59389 (-0.39323) + | > loss_dur: 0.21896 (0.14479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.27490 (27.93052) + | > current_lr: 0.00005 + | > step_time: 2.31890 (3.22656) + | > loader_time: 0.00790 (0.16161) + + + --> STEP: 188/234 -- GLOBAL_STEP: 51200 + | > loss: -0.37524 (-0.25076) + | > log_mle: -0.59326 (-0.39767) + | > loss_dur: 0.21801 (0.14690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 93.91447 (29.29715) + | > current_lr: 0.00005 + | > step_time: 4.29860 (3.25535) + | > loader_time: 0.09490 (0.15955) + + + --> STEP: 193/234 -- GLOBAL_STEP: 51205 + | > loss: -0.37230 (-0.25340) + | > log_mle: -0.59064 (-0.40207) + | > loss_dur: 0.21834 (0.14867) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.24889 (30.30140) + | > current_lr: 0.00005 + | > step_time: 11.11390 (3.31860) + | > loader_time: 0.10770 (0.15755) + + + --> STEP: 198/234 -- GLOBAL_STEP: 51210 + | > loss: -0.36248 (-0.25601) + | > log_mle: -0.59726 (-0.40641) + | > loss_dur: 0.23477 (0.15041) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.40497 (31.17883) + | > current_lr: 0.00005 + | > step_time: 3.79600 (3.33167) + | > loader_time: 0.10190 (0.15563) + + + --> STEP: 203/234 -- GLOBAL_STEP: 51215 + | > loss: -0.31515 (-0.25839) + | > log_mle: -0.52981 (-0.41065) + | > loss_dur: 0.21466 (0.15227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.46591 (32.03455) + | > current_lr: 0.00005 + | > step_time: 2.60210 (3.33792) + | > loader_time: 0.01900 (0.15245) + + + --> STEP: 208/234 -- GLOBAL_STEP: 51220 + | > loss: -0.37157 (-0.26154) + | > log_mle: -0.61208 (-0.41575) + | > loss_dur: 0.24051 (0.15422) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.80206 (33.29811) + | > current_lr: 0.00005 + | > step_time: 4.19580 (3.34165) + | > loader_time: 0.07610 (0.15153) + + + --> STEP: 213/234 -- GLOBAL_STEP: 51225 + | > loss: -0.41635 (-0.26499) + | > log_mle: -0.65868 (-0.42117) + | > loss_dur: 0.24234 (0.15618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 131.73338 (35.29200) + | > current_lr: 0.00005 + | > step_time: 4.70190 (3.32790) + | > loader_time: 0.08910 (0.14897) + + + --> STEP: 218/234 -- GLOBAL_STEP: 51230 + | > loss: -0.38114 (-0.26817) + | > log_mle: -0.62003 (-0.42620) + | > loss_dur: 0.23889 (0.15803) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.09914 (36.42027) + | > current_lr: 0.00005 + | > step_time: 2.41180 (3.31161) + | > loader_time: 0.08020 (0.14686) + + + --> STEP: 223/234 -- GLOBAL_STEP: 51235 + | > loss: -0.43001 (-0.27174) + | > log_mle: -0.66687 (-0.43159) + | > loss_dur: 0.23687 (0.15986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 97.33419 (37.57728) + | > current_lr: 0.00005 + | > step_time: 2.08180 (3.31487) + | > loader_time: 0.09810 (0.14491) + + + --> STEP: 228/234 -- GLOBAL_STEP: 51240 + | > loss: -0.40435 (-0.27533) + | > log_mle: -0.67682 (-0.43723) + | > loss_dur: 0.27247 (0.16190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.58344 (38.76288) + | > current_lr: 0.00005 + | > step_time: 0.27950 (3.29441) + | > loader_time: 0.00300 (0.14183) + + + --> STEP: 233/234 -- GLOBAL_STEP: 51245 + | > loss: 0.12507 (-0.27606) + | > log_mle: -0.61317 (-0.44391) + | > loss_dur: 0.73824 (0.16785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 137.99846 (40.66099) + | > current_lr: 0.00005 + | > step_time: 0.20700 (3.22961) + | > loader_time: 0.00870 (0.13889) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.93152 (+0.90111) + | > avg_loss: -0.27471 (+0.01882) + | > avg_log_mle: -0.50519 (+0.00653) + | > avg_loss_dur: 0.23048 (+0.01230) + + + > EPOCH: 219/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 06:56:55)  + + --> STEP: 4/234 -- GLOBAL_STEP: 51250 + | > loss: -0.23261 (-0.22881) + | > log_mle: -0.34225 (-0.34228) + | > loss_dur: 0.10964 (0.11347) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.32771 (27.25506) + | > current_lr: 0.00005 + | > step_time: 3.70990 (3.04378) + | > loader_time: 0.07670 (0.04269) + + + --> STEP: 9/234 -- GLOBAL_STEP: 51255 + | > loss: -0.23981 (-0.24647) + | > log_mle: -0.35620 (-0.34670) + | > loss_dur: 0.11639 (0.10023) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.57505 (22.01277) + | > current_lr: 0.00005 + | > step_time: 10.10380 (4.49632) + | > loader_time: 0.18040 (0.09424) + + + --> STEP: 14/234 -- GLOBAL_STEP: 51260 + | > loss: -0.25633 (-0.25182) + | > log_mle: -0.35014 (-0.34773) + | > loss_dur: 0.09380 (0.09591) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.45026 (19.15391) + | > current_lr: 0.00005 + | > step_time: 3.30680 (4.69101) + | > loader_time: 0.08920 (0.10047) + + + --> STEP: 19/234 -- GLOBAL_STEP: 51265 + | > loss: -0.26322 (-0.25462) + | > log_mle: -0.34207 (-0.34645) + | > loss_dur: 0.07885 (0.09182) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.07262 (16.95556) + | > current_lr: 0.00005 + | > step_time: 2.38310 (4.02492) + | > loader_time: 0.00810 (0.08452) + + + --> STEP: 24/234 -- GLOBAL_STEP: 51270 + | > loss: -0.27446 (-0.25749) + | > log_mle: -0.34164 (-0.34591) + | > loss_dur: 0.06718 (0.08842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.70046 (15.48689) + | > current_lr: 0.00005 + | > step_time: 1.40760 (4.12338) + | > loader_time: 0.00250 (0.07992) + + + --> STEP: 29/234 -- GLOBAL_STEP: 51275 + | > loss: -0.21300 (-0.25667) + | > log_mle: -0.32168 (-0.34477) + | > loss_dur: 0.10868 (0.08810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.17922 (14.52722) + | > current_lr: 0.00005 + | > step_time: 9.39350 (4.17450) + | > loader_time: 0.11240 (0.07604) + + + --> STEP: 34/234 -- GLOBAL_STEP: 51280 + | > loss: -0.25010 (-0.25505) + | > log_mle: -0.33367 (-0.34357) + | > loss_dur: 0.08356 (0.08852) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.34814 (13.91939) + | > current_lr: 0.00005 + | > step_time: 4.21090 (4.05874) + | > loader_time: 0.00630 (0.07979) + + + --> STEP: 39/234 -- GLOBAL_STEP: 51285 + | > loss: -0.22446 (-0.25168) + | > log_mle: -0.32796 (-0.34165) + | > loss_dur: 0.10350 (0.08996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.65591 (14.04673) + | > current_lr: 0.00005 + | > step_time: 2.00100 (3.82554) + | > loader_time: 0.01860 (0.07736) + + + --> STEP: 44/234 -- GLOBAL_STEP: 51290 + | > loss: -0.24548 (-0.24815) + | > log_mle: -0.32212 (-0.33938) + | > loss_dur: 0.07664 (0.09123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61640 (13.72869) + | > current_lr: 0.00005 + | > step_time: 1.60370 (3.65727) + | > loader_time: 0.00170 (0.07094) + + + --> STEP: 49/234 -- GLOBAL_STEP: 51295 + | > loss: -0.24096 (-0.24575) + | > log_mle: -0.33315 (-0.33845) + | > loss_dur: 0.09219 (0.09270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.37791 (13.50086) + | > current_lr: 0.00005 + | > step_time: 1.88600 (3.58787) + | > loader_time: 0.00120 (0.06727) + + + --> STEP: 54/234 -- GLOBAL_STEP: 51300 + | > loss: -0.23621 (-0.24366) + | > log_mle: -0.33468 (-0.33741) + | > loss_dur: 0.09847 (0.09375) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.78913 (13.16211) + | > current_lr: 0.00005 + | > step_time: 1.87880 (3.48844) + | > loader_time: 0.10690 (0.06624) + + + --> STEP: 59/234 -- GLOBAL_STEP: 51305 + | > loss: -0.23607 (-0.24241) + | > log_mle: -0.33785 (-0.33680) + | > loss_dur: 0.10178 (0.09439) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.76176 (12.95922) + | > current_lr: 0.00005 + | > step_time: 3.19380 (3.42690) + | > loader_time: 0.00370 (0.06491) + + + --> STEP: 64/234 -- GLOBAL_STEP: 51310 + | > loss: -0.22968 (-0.23966) + | > log_mle: -0.32331 (-0.33659) + | > loss_dur: 0.09363 (0.09693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.60769 (13.03737) + | > current_lr: 0.00005 + | > step_time: 1.61650 (3.35042) + | > loader_time: 0.00300 (0.06297) + + + --> STEP: 69/234 -- GLOBAL_STEP: 51315 + | > loss: -0.20668 (-0.23785) + | > log_mle: -0.31179 (-0.33563) + | > loss_dur: 0.10511 (0.09778) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.26436 (12.84516) + | > current_lr: 0.00005 + | > step_time: 2.09350 (3.33709) + | > loader_time: 0.00300 (0.06214) + + + --> STEP: 74/234 -- GLOBAL_STEP: 51320 + | > loss: -0.18584 (-0.23504) + | > log_mle: -0.31217 (-0.33502) + | > loss_dur: 0.12633 (0.09998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.01393 (13.08364) + | > current_lr: 0.00005 + | > step_time: 2.38750 (3.31962) + | > loader_time: 0.11250 (0.06196) + + + --> STEP: 79/234 -- GLOBAL_STEP: 51325 + | > loss: -0.21022 (-0.23300) + | > log_mle: -0.32897 (-0.33449) + | > loss_dur: 0.11874 (0.10149) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.03256 (13.11073) + | > current_lr: 0.00005 + | > step_time: 2.00420 (3.25929) + | > loader_time: 0.07530 (0.06209) + + + --> STEP: 84/234 -- GLOBAL_STEP: 51330 + | > loss: -0.21720 (-0.23133) + | > log_mle: -0.32450 (-0.33404) + | > loss_dur: 0.10729 (0.10272) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27867 (13.11996) + | > current_lr: 0.00005 + | > step_time: 1.49020 (3.17375) + | > loader_time: 0.00330 (0.06143) + + + --> STEP: 89/234 -- GLOBAL_STEP: 51335 + | > loss: -0.22256 (-0.22997) + | > log_mle: -0.35105 (-0.33448) + | > loss_dur: 0.12848 (0.10450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.62481 (13.25013) + | > current_lr: 0.00005 + | > step_time: 2.70250 (3.19336) + | > loader_time: 0.09800 (0.06033) + + + --> STEP: 94/234 -- GLOBAL_STEP: 51340 + | > loss: -0.24366 (-0.22951) + | > log_mle: -0.38356 (-0.33620) + | > loss_dur: 0.13990 (0.10668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.17969 (13.61326) + | > current_lr: 0.00005 + | > step_time: 3.19190 (3.15953) + | > loader_time: 0.80780 (0.06686) + + + --> STEP: 99/234 -- GLOBAL_STEP: 51345 + | > loss: -0.23762 (-0.22916) + | > log_mle: -0.41550 (-0.33797) + | > loss_dur: 0.17789 (0.10880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.57070 (14.05323) + | > current_lr: 0.00005 + | > step_time: 2.90760 (3.13326) + | > loader_time: 0.09120 (0.06943) + + + --> STEP: 104/234 -- GLOBAL_STEP: 51350 + | > loss: -0.28061 (-0.22928) + | > log_mle: -0.43167 (-0.34037) + | > loss_dur: 0.15106 (0.11109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.33493 (14.67596) + | > current_lr: 0.00005 + | > step_time: 4.19360 (3.13413) + | > loader_time: 0.10240 (0.07066) + + + --> STEP: 109/234 -- GLOBAL_STEP: 51355 + | > loss: -0.20263 (-0.22839) + | > log_mle: -0.40134 (-0.34192) + | > loss_dur: 0.19871 (0.11353) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.21672 (15.23260) + | > current_lr: 0.00005 + | > step_time: 3.20290 (3.15229) + | > loader_time: 0.29250 (0.07263) + + + --> STEP: 114/234 -- GLOBAL_STEP: 51360 + | > loss: -0.23395 (-0.22853) + | > log_mle: -0.38216 (-0.34437) + | > loss_dur: 0.14822 (0.11584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.29458 (16.04076) + | > current_lr: 0.00005 + | > step_time: 2.29550 (3.16152) + | > loader_time: 0.00400 (0.07133) + + + --> STEP: 119/234 -- GLOBAL_STEP: 51365 + | > loss: -0.22147 (-0.22784) + | > log_mle: -0.38011 (-0.34603) + | > loss_dur: 0.15864 (0.11819) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.22569 (16.71442) + | > current_lr: 0.00005 + | > step_time: 4.27060 (3.14311) + | > loader_time: 0.09870 (0.07143) + + + --> STEP: 124/234 -- GLOBAL_STEP: 51370 + | > loss: -0.24747 (-0.22734) + | > log_mle: -0.40488 (-0.34700) + | > loss_dur: 0.15741 (0.11966) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.12596 (17.01419) + | > current_lr: 0.00005 + | > step_time: 2.87590 (3.13554) + | > loader_time: 0.00720 (0.07055) + + + --> STEP: 129/234 -- GLOBAL_STEP: 51375 + | > loss: -0.22534 (-0.22786) + | > log_mle: -0.39868 (-0.34972) + | > loss_dur: 0.17333 (0.12185) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.66598 (17.56433) + | > current_lr: 0.00005 + | > step_time: 1.49910 (3.11868) + | > loader_time: 0.08830 (0.07096) + + + --> STEP: 134/234 -- GLOBAL_STEP: 51380 + | > loss: -0.25627 (-0.22923) + | > log_mle: -0.45249 (-0.35318) + | > loss_dur: 0.19622 (0.12395) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.45452 (18.33368) + | > current_lr: 0.00005 + | > step_time: 5.29650 (3.15978) + | > loader_time: 0.29520 (0.07283) + + + --> STEP: 139/234 -- GLOBAL_STEP: 51385 + | > loss: -0.32263 (-0.23028) + | > log_mle: -0.51191 (-0.35641) + | > loss_dur: 0.18928 (0.12612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.20490 (19.12114) + | > current_lr: 0.00005 + | > step_time: 1.48110 (3.13671) + | > loader_time: 0.09480 (0.07212) + + + --> STEP: 144/234 -- GLOBAL_STEP: 51390 + | > loss: -0.27589 (-0.23126) + | > log_mle: -0.48447 (-0.35990) + | > loss_dur: 0.20859 (0.12864) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.08274 (19.91101) + | > current_lr: 0.00005 + | > step_time: 1.81000 (3.16902) + | > loader_time: 0.09000 (0.07367) + + + --> STEP: 149/234 -- GLOBAL_STEP: 51395 + | > loss: -0.33430 (-0.23334) + | > log_mle: -0.53728 (-0.36395) + | > loss_dur: 0.20298 (0.13062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.51447 (20.77007) + | > current_lr: 0.00005 + | > step_time: 10.30140 (3.21876) + | > loader_time: 0.11590 (0.07339) + + + --> STEP: 154/234 -- GLOBAL_STEP: 51400 + | > loss: -0.30285 (-0.23573) + | > log_mle: -0.49446 (-0.36849) + | > loss_dur: 0.19161 (0.13276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.79186 (22.09432) + | > current_lr: 0.00005 + | > step_time: 3.61670 (3.23831) + | > loader_time: 0.17790 (0.07278) + + + --> STEP: 159/234 -- GLOBAL_STEP: 51405 + | > loss: -0.31421 (-0.23780) + | > log_mle: -0.50996 (-0.37279) + | > loss_dur: 0.19575 (0.13499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.35318 (23.13451) + | > current_lr: 0.00005 + | > step_time: 4.20260 (3.24082) + | > loader_time: 0.00630 (0.07347) + + + --> STEP: 164/234 -- GLOBAL_STEP: 51410 + | > loss: -0.29281 (-0.24002) + | > log_mle: -0.49972 (-0.37691) + | > loss_dur: 0.20691 (0.13688) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.32903 (24.26200) + | > current_lr: 0.00005 + | > step_time: 1.08110 (3.23566) + | > loader_time: 0.01390 (0.07247) + + + --> STEP: 169/234 -- GLOBAL_STEP: 51415 + | > loss: -0.30489 (-0.24254) + | > log_mle: -0.51283 (-0.38138) + | > loss_dur: 0.20794 (0.13884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.64189 (25.23034) + | > current_lr: 0.00005 + | > step_time: 2.61930 (3.24253) + | > loader_time: 0.00740 (0.07109) + + + --> STEP: 174/234 -- GLOBAL_STEP: 51420 + | > loss: -0.39135 (-0.24614) + | > log_mle: -0.60243 (-0.38710) + | > loss_dur: 0.21108 (0.14096) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.90015 (26.48849) + | > current_lr: 0.00005 + | > step_time: 3.81730 (3.24837) + | > loader_time: 0.18840 (0.07123) + + + --> STEP: 179/234 -- GLOBAL_STEP: 51425 + | > loss: -0.35544 (-0.24885) + | > log_mle: -0.59719 (-0.39225) + | > loss_dur: 0.24175 (0.14340) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.45089 (27.67395) + | > current_lr: 0.00005 + | > step_time: 2.67670 (3.26040) + | > loader_time: 0.00360 (0.06989) + + + --> STEP: 184/234 -- GLOBAL_STEP: 51430 + | > loss: -0.33025 (-0.25102) + | > log_mle: -0.55151 (-0.39657) + | > loss_dur: 0.22125 (0.14555) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.77484 (28.80799) + | > current_lr: 0.00005 + | > step_time: 2.77750 (3.26092) + | > loader_time: 0.01640 (0.07017) + + + --> STEP: 189/234 -- GLOBAL_STEP: 51435 + | > loss: -0.33299 (-0.25354) + | > log_mle: -0.55193 (-0.40111) + | > loss_dur: 0.21894 (0.14758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.10322 (29.84646) + | > current_lr: 0.00005 + | > step_time: 2.70910 (3.27761) + | > loader_time: 0.08340 (0.07026) + + + --> STEP: 194/234 -- GLOBAL_STEP: 51440 + | > loss: -0.36857 (-0.25653) + | > log_mle: -0.58285 (-0.40579) + | > loss_dur: 0.21428 (0.14926) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.70912 (30.96201) + | > current_lr: 0.00005 + | > step_time: 4.78150 (3.29597) + | > loader_time: 0.00240 (0.06938) + + + --> STEP: 199/234 -- GLOBAL_STEP: 51445 + | > loss: -0.37364 (-0.25911) + | > log_mle: -0.59757 (-0.41014) + | > loss_dur: 0.22393 (0.15103) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.81277 (32.22247) + | > current_lr: 0.00005 + | > step_time: 3.39790 (3.31119) + | > loader_time: 0.09650 (0.06920) + + + --> STEP: 204/234 -- GLOBAL_STEP: 51450 + | > loss: -0.39530 (-0.26156) + | > log_mle: -0.63050 (-0.41448) + | > loss_dur: 0.23520 (0.15293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 93.40157 (33.31252) + | > current_lr: 0.00005 + | > step_time: 6.80240 (3.36050) + | > loader_time: 0.00350 (0.07005) + + + --> STEP: 209/234 -- GLOBAL_STEP: 51455 + | > loss: -0.35707 (-0.26438) + | > log_mle: -0.58235 (-0.41906) + | > loss_dur: 0.22528 (0.15468) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.91981 (34.49281) + | > current_lr: 0.00005 + | > step_time: 5.68540 (3.40716) + | > loader_time: 0.02290 (0.06955) + + + --> STEP: 214/234 -- GLOBAL_STEP: 51460 + | > loss: -0.38308 (-0.26771) + | > log_mle: -0.59426 (-0.42422) + | > loss_dur: 0.21118 (0.15651) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 158.71078 (36.31121) + | > current_lr: 0.00005 + | > step_time: 1.59940 (3.44924) + | > loader_time: 0.01760 (0.06973) + + + --> STEP: 219/234 -- GLOBAL_STEP: 51465 + | > loss: -0.46121 (-0.27030) + | > log_mle: -0.69234 (-0.42872) + | > loss_dur: 0.23114 (0.15842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.90572 (37.19220) + | > current_lr: 0.00005 + | > step_time: 2.28800 (3.45899) + | > loader_time: 0.10310 (0.07047) + + + --> STEP: 224/234 -- GLOBAL_STEP: 51470 + | > loss: -0.41298 (-0.27282) + | > log_mle: -0.65071 (-0.43311) + | > loss_dur: 0.23772 (0.16029) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.45996 (37.94497) + | > current_lr: 0.00005 + | > step_time: 1.58600 (3.42732) + | > loader_time: 0.08680 (0.06978) + + + --> STEP: 229/234 -- GLOBAL_STEP: 51475 + | > loss: -0.38837 (-0.27555) + | > log_mle: -0.68910 (-0.43820) + | > loss_dur: 0.30073 (0.16265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.03178 (39.03415) + | > current_lr: 0.00005 + | > step_time: 0.26730 (3.37265) + | > loader_time: 0.01300 (0.06874) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04952 (-0.88201) + | > avg_loss: -0.29245 (-0.01775) + | > avg_log_mle: -0.51045 (-0.00527) + | > avg_loss_dur: 0.21800 (-0.01248) + + + > EPOCH: 220/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 07:11:13)  + + --> STEP: 0/234 -- GLOBAL_STEP: 51480 + | > loss: -0.29600 (-0.29600) + | > log_mle: -0.42448 (-0.42448) + | > loss_dur: 0.12848 (0.12848) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.16840 (30.16840) + | > current_lr: 0.00006 + | > step_time: 4.29930 (4.29927) + | > loader_time: 16.83510 (16.83511) + + + --> STEP: 5/234 -- GLOBAL_STEP: 51485 + | > loss: -0.25853 (-0.24302) + | > log_mle: -0.34477 (-0.34445) + | > loss_dur: 0.08623 (0.10143) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.89902 (22.33315) + | > current_lr: 0.00006 + | > step_time: 2.90170 (6.70077) + | > loader_time: 0.10450 (0.02459) + + + --> STEP: 10/234 -- GLOBAL_STEP: 51490 + | > loss: -0.25338 (-0.25061) + | > log_mle: -0.34292 (-0.34689) + | > loss_dur: 0.08954 (0.09629) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.49899 (19.46410) + | > current_lr: 0.00006 + | > step_time: 1.19760 (4.23835) + | > loader_time: 0.00100 (0.01342) + + + --> STEP: 15/234 -- GLOBAL_STEP: 51495 + | > loss: -0.26025 (-0.25527) + | > log_mle: -0.34872 (-0.34821) + | > loss_dur: 0.08846 (0.09294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.35764 (17.43286) + | > current_lr: 0.00006 + | > step_time: 3.39440 (3.62370) + | > loader_time: 0.09370 (0.02182) + + + --> STEP: 20/234 -- GLOBAL_STEP: 51500 + | > loss: -0.27836 (-0.25750) + | > log_mle: -0.34912 (-0.34697) + | > loss_dur: 0.07076 (0.08947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.99179 (15.39250) + | > current_lr: 0.00006 + | > step_time: 6.10180 (3.74176) + | > loader_time: 0.10020 (0.02573) + + + --> STEP: 25/234 -- GLOBAL_STEP: 51505 + | > loss: -0.24861 (-0.25851) + | > log_mle: -0.33328 (-0.34591) + | > loss_dur: 0.08467 (0.08740) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.37720 (14.24834) + | > current_lr: 0.00006 + | > step_time: 1.39670 (3.51557) + | > loader_time: 0.00320 (0.02448) + + + --> STEP: 30/234 -- GLOBAL_STEP: 51510 + | > loss: -0.23095 (-0.25758) + | > log_mle: -0.32356 (-0.34455) + | > loss_dur: 0.09261 (0.08697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.04989 (13.81798) + | > current_lr: 0.00006 + | > step_time: 2.08900 (3.22292) + | > loader_time: 0.00270 (0.02609) + + + --> STEP: 35/234 -- GLOBAL_STEP: 51515 + | > loss: -0.20354 (-0.25388) + | > log_mle: -0.32130 (-0.34233) + | > loss_dur: 0.11776 (0.08845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.26207 (13.70864) + | > current_lr: 0.00006 + | > step_time: 2.10000 (3.27263) + | > loader_time: 0.00230 (0.02302) + + + --> STEP: 40/234 -- GLOBAL_STEP: 51520 + | > loss: -0.19303 (-0.24954) + | > log_mle: -0.31328 (-0.33996) + | > loss_dur: 0.12025 (0.09043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.77343 (13.60026) + | > current_lr: 0.00006 + | > step_time: 1.15270 (3.02902) + | > loader_time: 0.00200 (0.02439) + + + --> STEP: 45/234 -- GLOBAL_STEP: 51525 + | > loss: -0.22707 (-0.24683) + | > log_mle: -0.34460 (-0.33870) + | > loss_dur: 0.11753 (0.09187) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.66761 (13.33855) + | > current_lr: 0.00006 + | > step_time: 1.59280 (2.91926) + | > loader_time: 0.00170 (0.02211) + + + --> STEP: 50/234 -- GLOBAL_STEP: 51530 + | > loss: -0.22784 (-0.24508) + | > log_mle: -0.32338 (-0.33759) + | > loss_dur: 0.09553 (0.09252) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.05512 (12.86156) + | > current_lr: 0.00006 + | > step_time: 1.99200 (2.79277) + | > loader_time: 0.00130 (0.02165) + + + --> STEP: 55/234 -- GLOBAL_STEP: 51535 + | > loss: -0.25147 (-0.24360) + | > log_mle: -0.33603 (-0.33679) + | > loss_dur: 0.08455 (0.09319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.62183 (12.64253) + | > current_lr: 0.00006 + | > step_time: 1.09730 (2.70476) + | > loader_time: 0.00210 (0.02139) + + + --> STEP: 60/234 -- GLOBAL_STEP: 51540 + | > loss: -0.20945 (-0.24157) + | > log_mle: -0.34165 (-0.33630) + | > loss_dur: 0.13220 (0.09474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.92124 (12.71268) + | > current_lr: 0.00006 + | > step_time: 4.80640 (2.66204) + | > loader_time: 0.08530 (0.02251) + + + --> STEP: 65/234 -- GLOBAL_STEP: 51545 + | > loss: -0.22825 (-0.23944) + | > log_mle: -0.32621 (-0.33577) + | > loss_dur: 0.09795 (0.09633) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.89265 (12.85752) + | > current_lr: 0.00006 + | > step_time: 1.81870 (2.61151) + | > loader_time: 0.09740 (0.02245) + + + --> STEP: 70/234 -- GLOBAL_STEP: 51550 + | > loss: -0.18305 (-0.23699) + | > log_mle: -0.31138 (-0.33464) + | > loss_dur: 0.12833 (0.09765) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.06821 (12.77541) + | > current_lr: 0.00006 + | > step_time: 1.79610 (2.57656) + | > loader_time: 0.00820 (0.02370) + + + --> STEP: 75/234 -- GLOBAL_STEP: 51555 + | > loss: -0.19940 (-0.23419) + | > log_mle: -0.32971 (-0.33421) + | > loss_dur: 0.13031 (0.10003) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.88293 (12.90174) + | > current_lr: 0.00006 + | > step_time: 1.60040 (2.51422) + | > loader_time: 0.08900 (0.02457) + + + --> STEP: 80/234 -- GLOBAL_STEP: 51560 + | > loss: -0.20663 (-0.23237) + | > log_mle: -0.31402 (-0.33358) + | > loss_dur: 0.10740 (0.10121) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.88227 (12.76833) + | > current_lr: 0.00006 + | > step_time: 2.01190 (2.50861) + | > loader_time: 0.00400 (0.02334) + + + --> STEP: 85/234 -- GLOBAL_STEP: 51565 + | > loss: -0.22174 (-0.23084) + | > log_mle: -0.32679 (-0.33335) + | > loss_dur: 0.10505 (0.10250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.34069 (12.73834) + | > current_lr: 0.00006 + | > step_time: 0.90120 (2.44184) + | > loader_time: 0.00250 (0.02210) + + + --> STEP: 90/234 -- GLOBAL_STEP: 51570 + | > loss: -0.20438 (-0.22993) + | > log_mle: -0.34984 (-0.33417) + | > loss_dur: 0.14546 (0.10423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.27159 (12.84877) + | > current_lr: 0.00006 + | > step_time: 2.49940 (2.41078) + | > loader_time: 0.00710 (0.02409) + + + --> STEP: 95/234 -- GLOBAL_STEP: 51575 + | > loss: -0.25336 (-0.23000) + | > log_mle: -0.43485 (-0.33686) + | > loss_dur: 0.18149 (0.10686) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.16751 (13.43216) + | > current_lr: 0.00006 + | > step_time: 1.09340 (2.39141) + | > loader_time: 0.00820 (0.02405) + + + --> STEP: 100/234 -- GLOBAL_STEP: 51580 + | > loss: -0.22813 (-0.22944) + | > log_mle: -0.36042 (-0.33789) + | > loss_dur: 0.13229 (0.10845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.83148 (13.85417) + | > current_lr: 0.00006 + | > step_time: 1.79940 (2.37478) + | > loader_time: 0.08840 (0.02556) + + + --> STEP: 105/234 -- GLOBAL_STEP: 51585 + | > loss: -0.20660 (-0.22935) + | > log_mle: -0.34023 (-0.34009) + | > loss_dur: 0.13362 (0.11074) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.26028 (14.43204) + | > current_lr: 0.00006 + | > step_time: 2.80140 (2.44864) + | > loader_time: 0.08030 (0.02855) + + + --> STEP: 110/234 -- GLOBAL_STEP: 51590 + | > loss: -0.21150 (-0.22846) + | > log_mle: -0.36121 (-0.34177) + | > loss_dur: 0.14971 (0.11331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.96807 (15.05221) + | > current_lr: 0.00006 + | > step_time: 5.00370 (2.47183) + | > loader_time: 0.09900 (0.03067) + + + --> STEP: 115/234 -- GLOBAL_STEP: 51595 + | > loss: -0.20709 (-0.22830) + | > log_mle: -0.38088 (-0.34425) + | > loss_dur: 0.17380 (0.11594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.58413 (15.81110) + | > current_lr: 0.00006 + | > step_time: 2.21070 (2.53677) + | > loader_time: 0.09860 (0.03339) + + + --> STEP: 120/234 -- GLOBAL_STEP: 51600 + | > loss: -0.25849 (-0.22816) + | > log_mle: -0.43014 (-0.34632) + | > loss_dur: 0.17165 (0.11816) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.81513 (16.27006) + | > current_lr: 0.00006 + | > step_time: 2.72920 (2.56128) + | > loader_time: 0.17310 (0.03519) + + + --> STEP: 125/234 -- GLOBAL_STEP: 51605 + | > loss: -0.24457 (-0.22747) + | > log_mle: -0.41993 (-0.34723) + | > loss_dur: 0.17536 (0.11975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.44777 (16.47701) + | > current_lr: 0.00006 + | > step_time: 1.61930 (2.58298) + | > loader_time: 0.07550 (0.03651) + + + --> STEP: 130/234 -- GLOBAL_STEP: 51610 + | > loss: -0.25335 (-0.22815) + | > log_mle: -0.43357 (-0.35007) + | > loss_dur: 0.18022 (0.12192) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.87010 (17.19178) + | > current_lr: 0.00006 + | > step_time: 4.08070 (2.57447) + | > loader_time: 0.11940 (0.03612) + + + --> STEP: 135/234 -- GLOBAL_STEP: 51615 + | > loss: -0.21022 (-0.22930) + | > log_mle: -0.36094 (-0.35310) + | > loss_dur: 0.15073 (0.12380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.79053 (17.81156) + | > current_lr: 0.00006 + | > step_time: 2.38240 (2.59904) + | > loader_time: 0.00710 (0.03630) + + + --> STEP: 140/234 -- GLOBAL_STEP: 51620 + | > loss: -0.21563 (-0.23048) + | > log_mle: -0.39689 (-0.35670) + | > loss_dur: 0.18126 (0.12621) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.54022 (18.69625) + | > current_lr: 0.00006 + | > step_time: 7.71660 (2.69507) + | > loader_time: 0.30380 (0.03870) + + + --> STEP: 145/234 -- GLOBAL_STEP: 51625 + | > loss: -0.31519 (-0.23226) + | > log_mle: -0.50254 (-0.36096) + | > loss_dur: 0.18736 (0.12870) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.11372 (19.86875) + | > current_lr: 0.00006 + | > step_time: 3.41680 (2.72142) + | > loader_time: 0.08920 (0.03817) + + + --> STEP: 150/234 -- GLOBAL_STEP: 51630 + | > loss: -0.28156 (-0.23423) + | > log_mle: -0.48871 (-0.36498) + | > loss_dur: 0.20715 (0.13075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.95057 (20.81540) + | > current_lr: 0.00006 + | > step_time: 1.72520 (2.73685) + | > loader_time: 0.06380 (0.03874) + + + --> STEP: 155/234 -- GLOBAL_STEP: 51635 + | > loss: -0.35069 (-0.23725) + | > log_mle: -0.55591 (-0.37010) + | > loss_dur: 0.20521 (0.13286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.46164 (22.20581) + | > current_lr: 0.00006 + | > step_time: 3.29470 (2.75302) + | > loader_time: 0.09380 (0.03956) + + + --> STEP: 160/234 -- GLOBAL_STEP: 51640 + | > loss: -0.32995 (-0.23944) + | > log_mle: -0.54536 (-0.37450) + | > loss_dur: 0.21541 (0.13506) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.51825 (23.48401) + | > current_lr: 0.00006 + | > step_time: 2.30550 (2.78824) + | > loader_time: 0.00530 (0.04086) + + + --> STEP: 165/234 -- GLOBAL_STEP: 51645 + | > loss: -0.34837 (-0.24179) + | > log_mle: -0.54917 (-0.37872) + | > loss_dur: 0.20080 (0.13693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.96026 (24.60468) + | > current_lr: 0.00006 + | > step_time: 5.80810 (2.81998) + | > loader_time: 0.09900 (0.04146) + + + --> STEP: 170/234 -- GLOBAL_STEP: 51650 + | > loss: -0.36867 (-0.24453) + | > log_mle: -0.59113 (-0.38349) + | > loss_dur: 0.22246 (0.13896) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.90697 (25.67216) + | > current_lr: 0.00006 + | > step_time: 5.40320 (2.85490) + | > loader_time: 0.10420 (0.04268) + + + --> STEP: 175/234 -- GLOBAL_STEP: 51655 + | > loss: -0.32710 (-0.24773) + | > log_mle: -0.55699 (-0.38887) + | > loss_dur: 0.22988 (0.14114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.47848 (26.79494) + | > current_lr: 0.00006 + | > step_time: 9.90650 (2.90634) + | > loader_time: 0.09920 (0.04318) + + + --> STEP: 180/234 -- GLOBAL_STEP: 51660 + | > loss: -0.35250 (-0.25052) + | > log_mle: -0.56236 (-0.39385) + | > loss_dur: 0.20986 (0.14333) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.47500 (27.94563) + | > current_lr: 0.00006 + | > step_time: 2.79070 (2.93966) + | > loader_time: 0.09750 (0.04407) + + + --> STEP: 185/234 -- GLOBAL_STEP: 51665 + | > loss: -0.35233 (-0.25282) + | > log_mle: -0.58840 (-0.39836) + | > loss_dur: 0.23607 (0.14554) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.34058 (29.22172) + | > current_lr: 0.00006 + | > step_time: 4.70450 (2.98135) + | > loader_time: 0.09610 (0.04661) + + + --> STEP: 190/234 -- GLOBAL_STEP: 51670 + | > loss: -0.36140 (-0.25538) + | > log_mle: -0.56978 (-0.40286) + | > loss_dur: 0.20838 (0.14748) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.55772 (30.32988) + | > current_lr: 0.00006 + | > step_time: 6.08990 (3.04387) + | > loader_time: 0.09210 (0.04719) + + + --> STEP: 195/234 -- GLOBAL_STEP: 51675 + | > loss: -0.35309 (-0.25846) + | > log_mle: -0.58355 (-0.40770) + | > loss_dur: 0.23046 (0.14924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.95798 (31.51311) + | > current_lr: 0.00006 + | > step_time: 4.82280 (3.07267) + | > loader_time: 0.17880 (0.04896) + + + --> STEP: 200/234 -- GLOBAL_STEP: 51680 + | > loss: -0.35748 (-0.26110) + | > log_mle: -0.59794 (-0.41220) + | > loss_dur: 0.24045 (0.15110) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.10715 (32.54228) + | > current_lr: 0.00006 + | > step_time: 3.41030 (3.14259) + | > loader_time: 0.11090 (0.05160) + + + --> STEP: 205/234 -- GLOBAL_STEP: 51685 + | > loss: -0.36029 (-0.26366) + | > log_mle: -0.58511 (-0.41655) + | > loss_dur: 0.22482 (0.15289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.74181 (33.52486) + | > current_lr: 0.00006 + | > step_time: 3.99860 (3.15922) + | > loader_time: 0.00920 (0.05214) + + + --> STEP: 210/234 -- GLOBAL_STEP: 51690 + | > loss: -0.42615 (-0.26697) + | > log_mle: -0.66575 (-0.42169) + | > loss_dur: 0.23960 (0.15472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.00555 (35.06427) + | > current_lr: 0.00006 + | > step_time: 4.49080 (3.20688) + | > loader_time: 0.01060 (0.05375) + + + --> STEP: 215/234 -- GLOBAL_STEP: 51695 + | > loss: -0.37339 (-0.27024) + | > log_mle: -0.60993 (-0.42685) + | > loss_dur: 0.23654 (0.15660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.53508 (36.55613) + | > current_lr: 0.00006 + | > step_time: 4.30490 (3.23507) + | > loader_time: 0.00760 (0.05335) + + + --> STEP: 220/234 -- GLOBAL_STEP: 51700 + | > loss: -0.42057 (-0.27383) + | > log_mle: -0.65813 (-0.43231) + | > loss_dur: 0.23756 (0.15849) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.54759 (38.11749) + | > current_lr: 0.00006 + | > step_time: 2.92040 (3.26048) + | > loader_time: 0.08580 (0.05383) + + + --> STEP: 225/234 -- GLOBAL_STEP: 51705 + | > loss: -0.47509 (-0.27722) + | > log_mle: -0.72785 (-0.43756) + | > loss_dur: 0.25276 (0.16034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 115.84122 (39.59933) + | > current_lr: 0.00006 + | > step_time: 0.36000 (3.22369) + | > loader_time: 0.07230 (0.05525) + + + --> STEP: 230/234 -- GLOBAL_STEP: 51710 + | > loss: -0.44864 (-0.28022) + | > log_mle: -0.76056 (-0.44307) + | > loss_dur: 0.31192 (0.16285) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 109.24829 (41.14721) + | > current_lr: 0.00006 + | > step_time: 0.27690 (3.15931) + | > loader_time: 0.00300 (0.05413) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.42030 (+0.37078) + | > avg_loss: -0.31018 (-0.01773) + | > avg_log_mle: -0.52624 (-0.01579) + | > avg_loss_dur: 0.21606 (-0.00194) + + + > EPOCH: 221/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 07:24:42)  + + --> STEP: 1/234 -- GLOBAL_STEP: 51715 + | > loss: -0.25081 (-0.25081) + | > log_mle: -0.34648 (-0.34648) + | > loss_dur: 0.09567 (0.09567) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.18603 (26.18603) + | > current_lr: 0.00006 + | > step_time: 5.80060 (5.80061) + | > loader_time: 0.08970 (0.08972) + + + --> STEP: 6/234 -- GLOBAL_STEP: 51720 + | > loss: -0.27489 (-0.23990) + | > log_mle: -0.34703 (-0.34887) + | > loss_dur: 0.07213 (0.10897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.12327 (19.71287) + | > current_lr: 0.00006 + | > step_time: 2.82490 (3.37123) + | > loader_time: 0.00200 (0.06502) + + + --> STEP: 11/234 -- GLOBAL_STEP: 51725 + | > loss: -0.27454 (-0.24971) + | > log_mle: -0.35512 (-0.35149) + | > loss_dur: 0.08058 (0.10178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.48574 (18.94307) + | > current_lr: 0.00006 + | > step_time: 2.77090 (3.54313) + | > loader_time: 0.00130 (0.05071) + + + --> STEP: 16/234 -- GLOBAL_STEP: 51730 + | > loss: -0.27617 (-0.25400) + | > log_mle: -0.35525 (-0.35193) + | > loss_dur: 0.07909 (0.09793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.60072 (17.07352) + | > current_lr: 0.00006 + | > step_time: 3.70820 (3.64114) + | > loader_time: 0.09760 (0.04630) + + + --> STEP: 21/234 -- GLOBAL_STEP: 51735 + | > loss: -0.25396 (-0.25614) + | > log_mle: -0.33122 (-0.34946) + | > loss_dur: 0.07726 (0.09332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.93465 (15.57341) + | > current_lr: 0.00006 + | > step_time: 2.60370 (3.92785) + | > loader_time: 0.00250 (0.04458) + + + --> STEP: 26/234 -- GLOBAL_STEP: 51740 + | > loss: -0.25272 (-0.25751) + | > log_mle: -0.34095 (-0.34840) + | > loss_dur: 0.08823 (0.09089) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.97879 (14.87722) + | > current_lr: 0.00006 + | > step_time: 3.70190 (4.18453) + | > loader_time: 0.09350 (0.05033) + + + --> STEP: 31/234 -- GLOBAL_STEP: 51745 + | > loss: -0.21166 (-0.25598) + | > log_mle: -0.33031 (-0.34668) + | > loss_dur: 0.11865 (0.09070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.64983 (14.55961) + | > current_lr: 0.00006 + | > step_time: 2.09140 (4.05707) + | > loader_time: 0.08190 (0.04515) + + + --> STEP: 36/234 -- GLOBAL_STEP: 51750 + | > loss: -0.21264 (-0.25364) + | > log_mle: -0.32382 (-0.34492) + | > loss_dur: 0.11118 (0.09128) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.95597 (14.17363) + | > current_lr: 0.00006 + | > step_time: 2.68910 (3.81841) + | > loader_time: 0.09620 (0.05170) + + + --> STEP: 41/234 -- GLOBAL_STEP: 51755 + | > loss: -0.25142 (-0.25065) + | > log_mle: -0.33431 (-0.34309) + | > loss_dur: 0.08289 (0.09244) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.07338 (13.69840) + | > current_lr: 0.00006 + | > step_time: 2.50310 (3.87060) + | > loader_time: 0.09120 (0.05210) + + + --> STEP: 46/234 -- GLOBAL_STEP: 51760 + | > loss: -0.21579 (-0.24766) + | > log_mle: -0.32847 (-0.34169) + | > loss_dur: 0.11268 (0.09403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.15425 (13.50127) + | > current_lr: 0.00006 + | > step_time: 4.20040 (3.83343) + | > loader_time: 0.10370 (0.05864) + + + --> STEP: 51/234 -- GLOBAL_STEP: 51765 + | > loss: -0.22549 (-0.24619) + | > log_mle: -0.32492 (-0.34047) + | > loss_dur: 0.09943 (0.09428) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.94860 (13.07330) + | > current_lr: 0.00006 + | > step_time: 3.68990 (3.73611) + | > loader_time: 0.09860 (0.05999) + + + --> STEP: 56/234 -- GLOBAL_STEP: 51770 + | > loss: -0.22182 (-0.24461) + | > log_mle: -0.33250 (-0.33977) + | > loss_dur: 0.11067 (0.09516) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.08530 (12.89781) + | > current_lr: 0.00006 + | > step_time: 4.20860 (3.64169) + | > loader_time: 0.08960 (0.05976) + + + --> STEP: 61/234 -- GLOBAL_STEP: 51775 + | > loss: -0.22057 (-0.24246) + | > log_mle: -0.32734 (-0.33904) + | > loss_dur: 0.10677 (0.09658) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.00147 (12.79034) + | > current_lr: 0.00006 + | > step_time: 1.97790 (3.57127) + | > loader_time: 0.00250 (0.05773) + + + --> STEP: 66/234 -- GLOBAL_STEP: 51780 + | > loss: -0.22833 (-0.24049) + | > log_mle: -0.32193 (-0.33839) + | > loss_dur: 0.09361 (0.09789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.95508 (12.65586) + | > current_lr: 0.00006 + | > step_time: 5.08550 (3.54422) + | > loader_time: 0.09690 (0.05905) + + + --> STEP: 71/234 -- GLOBAL_STEP: 51785 + | > loss: -0.20267 (-0.23784) + | > log_mle: -0.34828 (-0.33757) + | > loss_dur: 0.14561 (0.09973) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.50617 (12.67981) + | > current_lr: 0.00006 + | > step_time: 1.39000 (3.43841) + | > loader_time: 0.00430 (0.05625) + + + --> STEP: 76/234 -- GLOBAL_STEP: 51790 + | > loss: -0.20934 (-0.23528) + | > log_mle: -0.33252 (-0.33697) + | > loss_dur: 0.12317 (0.10169) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.04648 (12.78029) + | > current_lr: 0.00006 + | > step_time: 1.81420 (3.41798) + | > loader_time: 0.08900 (0.05641) + + + --> STEP: 81/234 -- GLOBAL_STEP: 51795 + | > loss: -0.21299 (-0.23341) + | > log_mle: -0.33801 (-0.33625) + | > loss_dur: 0.12502 (0.10284) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.15886 (12.81504) + | > current_lr: 0.00006 + | > step_time: 3.50850 (3.38154) + | > loader_time: 0.09770 (0.05916) + + + --> STEP: 86/234 -- GLOBAL_STEP: 51800 + | > loss: -0.21457 (-0.23191) + | > log_mle: -0.33912 (-0.33583) + | > loss_dur: 0.12455 (0.10392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.41564 (12.92169) + | > current_lr: 0.00006 + | > step_time: 1.16410 (3.38049) + | > loader_time: 0.00170 (0.05915) + + + --> STEP: 91/234 -- GLOBAL_STEP: 51805 + | > loss: -0.20028 (-0.23071) + | > log_mle: -0.34753 (-0.33653) + | > loss_dur: 0.14725 (0.10583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.41278 (13.11349) + | > current_lr: 0.00006 + | > step_time: 4.81600 (3.35667) + | > loader_time: 0.18970 (0.05923) + + + --> STEP: 96/234 -- GLOBAL_STEP: 51810 + | > loss: -0.21154 (-0.23079) + | > log_mle: -0.33484 (-0.33882) + | > loss_dur: 0.12330 (0.10802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.43562 (13.67749) + | > current_lr: 0.00006 + | > step_time: 4.99670 (3.37479) + | > loader_time: 0.00140 (0.05892) + + + --> STEP: 101/234 -- GLOBAL_STEP: 51815 + | > loss: -0.22642 (-0.23036) + | > log_mle: -0.38707 (-0.34023) + | > loss_dur: 0.16065 (0.10987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.36807 (14.03445) + | > current_lr: 0.00006 + | > step_time: 2.58260 (3.34120) + | > loader_time: 0.00440 (0.05956) + + + --> STEP: 106/234 -- GLOBAL_STEP: 51820 + | > loss: -0.20041 (-0.22988) + | > log_mle: -0.38534 (-0.34213) + | > loss_dur: 0.18493 (0.11225) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.44232 (14.55814) + | > current_lr: 0.00006 + | > step_time: 3.20300 (3.30910) + | > loader_time: 0.00570 (0.05772) + + + --> STEP: 111/234 -- GLOBAL_STEP: 51825 + | > loss: -0.24022 (-0.22926) + | > log_mle: -0.43602 (-0.34406) + | > loss_dur: 0.19580 (0.11479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.19541 (15.08883) + | > current_lr: 0.00006 + | > step_time: 2.30120 (3.28573) + | > loader_time: 0.01080 (0.05692) + + + --> STEP: 116/234 -- GLOBAL_STEP: 51830 + | > loss: -0.20707 (-0.22878) + | > log_mle: -0.40108 (-0.34613) + | > loss_dur: 0.19401 (0.11736) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.01457 (15.57162) + | > current_lr: 0.00006 + | > step_time: 0.75150 (3.32105) + | > loader_time: 0.00540 (0.05620) + + + --> STEP: 121/234 -- GLOBAL_STEP: 51835 + | > loss: -0.18243 (-0.22832) + | > log_mle: -0.31556 (-0.34746) + | > loss_dur: 0.13313 (0.11914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.25835 (15.95781) + | > current_lr: 0.00006 + | > step_time: 5.82850 (3.33273) + | > loader_time: 0.09910 (0.05621) + + + --> STEP: 126/234 -- GLOBAL_STEP: 51840 + | > loss: -0.25562 (-0.22820) + | > log_mle: -0.45151 (-0.34934) + | > loss_dur: 0.19589 (0.12113) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.39814 (16.65103) + | > current_lr: 0.00006 + | > step_time: 6.90490 (3.32748) + | > loader_time: 0.09630 (0.05626) + + + --> STEP: 131/234 -- GLOBAL_STEP: 51845 + | > loss: -0.30775 (-0.22914) + | > log_mle: -0.49476 (-0.35241) + | > loss_dur: 0.18700 (0.12326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.14745 (17.51456) + | > current_lr: 0.00006 + | > step_time: 1.48230 (3.28454) + | > loader_time: 0.09310 (0.05674) + + + --> STEP: 136/234 -- GLOBAL_STEP: 51850 + | > loss: -0.33633 (-0.23034) + | > log_mle: -0.54379 (-0.35553) + | > loss_dur: 0.20745 (0.12520) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.43808 (18.39911) + | > current_lr: 0.00006 + | > step_time: 1.80610 (3.33252) + | > loader_time: 0.00360 (0.05889) + + + --> STEP: 141/234 -- GLOBAL_STEP: 51855 + | > loss: -0.28210 (-0.23100) + | > log_mle: -0.44812 (-0.35824) + | > loss_dur: 0.16602 (0.12725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.78640 (19.08394) + | > current_lr: 0.00006 + | > step_time: 2.20650 (3.35118) + | > loader_time: 0.01550 (0.05814) + + + --> STEP: 146/234 -- GLOBAL_STEP: 51860 + | > loss: -0.29441 (-0.23291) + | > log_mle: -0.49338 (-0.36266) + | > loss_dur: 0.19897 (0.12975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.29823 (20.38159) + | > current_lr: 0.00006 + | > step_time: 2.42770 (3.39844) + | > loader_time: 0.09460 (0.05887) + + + --> STEP: 151/234 -- GLOBAL_STEP: 51865 + | > loss: -0.29160 (-0.23478) + | > log_mle: -0.46887 (-0.36635) + | > loss_dur: 0.17727 (0.13156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.72966 (21.12979) + | > current_lr: 0.00006 + | > step_time: 3.01750 (3.38868) + | > loader_time: 0.09160 (0.05930) + + + --> STEP: 156/234 -- GLOBAL_STEP: 51870 + | > loss: -0.32760 (-0.23804) + | > log_mle: -0.51899 (-0.37173) + | > loss_dur: 0.19139 (0.13369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.40418 (22.51349) + | > current_lr: 0.00006 + | > step_time: 2.72310 (3.36051) + | > loader_time: 0.10120 (0.05865) + + + --> STEP: 161/234 -- GLOBAL_STEP: 51875 + | > loss: -0.33182 (-0.24024) + | > log_mle: -0.52341 (-0.37596) + | > loss_dur: 0.19159 (0.13572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.34243 (23.66348) + | > current_lr: 0.00006 + | > step_time: 3.99950 (3.38109) + | > loader_time: 0.08640 (0.05806) + + + --> STEP: 166/234 -- GLOBAL_STEP: 51880 + | > loss: -0.27966 (-0.24191) + | > log_mle: -0.46491 (-0.37948) + | > loss_dur: 0.18525 (0.13757) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.89740 (24.55545) + | > current_lr: 0.00006 + | > step_time: 3.18390 (3.39780) + | > loader_time: 0.11670 (0.05816) + + + --> STEP: 171/234 -- GLOBAL_STEP: 51885 + | > loss: -0.37618 (-0.24497) + | > log_mle: -0.57222 (-0.38462) + | > loss_dur: 0.19604 (0.13964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.01369 (25.87050) + | > current_lr: 0.00006 + | > step_time: 3.51360 (3.39783) + | > loader_time: 0.10520 (0.05870) + + + --> STEP: 176/234 -- GLOBAL_STEP: 51890 + | > loss: -0.33491 (-0.24783) + | > log_mle: -0.55052 (-0.38967) + | > loss_dur: 0.21561 (0.14184) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.32809 (26.94464) + | > current_lr: 0.00006 + | > step_time: 3.59070 (3.42171) + | > loader_time: 0.09940 (0.05881) + + + --> STEP: 181/234 -- GLOBAL_STEP: 51895 + | > loss: -0.28028 (-0.25003) + | > log_mle: -0.48409 (-0.39410) + | > loss_dur: 0.20382 (0.14407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.50081 (28.07869) + | > current_lr: 0.00006 + | > step_time: 4.61210 (3.47705) + | > loader_time: 0.00920 (0.05955) + + + --> STEP: 186/234 -- GLOBAL_STEP: 51900 + | > loss: -0.28285 (-0.25224) + | > log_mle: -0.52438 (-0.39860) + | > loss_dur: 0.24153 (0.14636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.00841 (29.35440) + | > current_lr: 0.00006 + | > step_time: 5.09710 (3.48032) + | > loader_time: 0.00410 (0.05905) + + + --> STEP: 191/234 -- GLOBAL_STEP: 51905 + | > loss: -0.34067 (-0.25482) + | > log_mle: -0.54446 (-0.40294) + | > loss_dur: 0.20379 (0.14812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.91174 (30.53382) + | > current_lr: 0.00006 + | > step_time: 5.60380 (3.53803) + | > loader_time: 0.09400 (0.06268) + + + --> STEP: 196/234 -- GLOBAL_STEP: 51910 + | > loss: -0.31406 (-0.25779) + | > log_mle: -0.54115 (-0.40765) + | > loss_dur: 0.22709 (0.14986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.99705 (31.48403) + | > current_lr: 0.00006 + | > step_time: 3.68930 (3.57787) + | > loader_time: 0.00840 (0.06172) + + + --> STEP: 201/234 -- GLOBAL_STEP: 51915 + | > loss: -0.27609 (-0.26015) + | > log_mle: -0.50205 (-0.41184) + | > loss_dur: 0.22596 (0.15168) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.61618 (32.51170) + | > current_lr: 0.00006 + | > step_time: 3.89040 (3.58445) + | > loader_time: 0.18220 (0.06255) + + + --> STEP: 206/234 -- GLOBAL_STEP: 51920 + | > loss: -0.39377 (-0.26325) + | > log_mle: -0.61522 (-0.41664) + | > loss_dur: 0.22145 (0.15339) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.28119 (33.52864) + | > current_lr: 0.00006 + | > step_time: 3.10890 (3.57310) + | > loader_time: 0.09410 (0.06245) + + + --> STEP: 211/234 -- GLOBAL_STEP: 51925 + | > loss: -0.43854 (-0.26655) + | > log_mle: -0.68160 (-0.42189) + | > loss_dur: 0.24306 (0.15534) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 97.50265 (34.82517) + | > current_lr: 0.00006 + | > step_time: 2.97330 (3.58937) + | > loader_time: 0.00460 (0.06173) + + + --> STEP: 216/234 -- GLOBAL_STEP: 51930 + | > loss: -0.43813 (-0.26986) + | > log_mle: -0.68105 (-0.42706) + | > loss_dur: 0.24292 (0.15719) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.50925 (36.03224) + | > current_lr: 0.00006 + | > step_time: 3.92190 (3.59858) + | > loader_time: 0.08130 (0.06198) + + + --> STEP: 221/234 -- GLOBAL_STEP: 51935 + | > loss: -0.37797 (-0.27332) + | > log_mle: -0.59528 (-0.43227) + | > loss_dur: 0.21732 (0.15895) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.72417 (37.12425) + | > current_lr: 0.00006 + | > step_time: 2.31910 (3.59195) + | > loader_time: 0.07520 (0.06134) + + + --> STEP: 226/234 -- GLOBAL_STEP: 51940 + | > loss: -0.44558 (-0.27704) + | > log_mle: -0.69625 (-0.43800) + | > loss_dur: 0.25067 (0.16095) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 124.31948 (38.67564) + | > current_lr: 0.00006 + | > step_time: 2.89340 (3.58779) + | > loader_time: 0.00580 (0.06010) + + + --> STEP: 231/234 -- GLOBAL_STEP: 51945 + | > loss: -0.38793 (-0.28016) + | > log_mle: -0.76733 (-0.44415) + | > loss_dur: 0.37940 (0.16399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 131.27101 (40.22962) + | > current_lr: 0.00006 + | > step_time: 0.28940 (3.53855) + | > loader_time: 0.00340 (0.06563) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.31053 (-0.10976) + | > avg_loss: -0.28820 (+0.02198) + | > avg_log_mle: -0.51172 (+0.01452) + | > avg_loss_dur: 0.22352 (+0.00746) + + + > EPOCH: 222/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 07:39:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 51950 + | > loss: -0.27638 (-0.26174) + | > log_mle: -0.36108 (-0.35334) + | > loss_dur: 0.08470 (0.09160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.35990 (18.88599) + | > current_lr: 0.00006 + | > step_time: 5.11730 (3.41757) + | > loader_time: 0.08230 (0.04161) + + + --> STEP: 7/234 -- GLOBAL_STEP: 51955 + | > loss: -0.27264 (-0.24555) + | > log_mle: -0.35284 (-0.34866) + | > loss_dur: 0.08020 (0.10311) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.74600 (20.94660) + | > current_lr: 0.00006 + | > step_time: 5.78780 (4.00876) + | > loader_time: 0.18930 (0.06494) + + + --> STEP: 12/234 -- GLOBAL_STEP: 51960 + | > loss: -0.25567 (-0.25074) + | > log_mle: -0.34763 (-0.35156) + | > loss_dur: 0.09196 (0.10081) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.43212 (18.60616) + | > current_lr: 0.00006 + | > step_time: 2.60190 (3.55059) + | > loader_time: 0.09460 (0.06795) + + + --> STEP: 17/234 -- GLOBAL_STEP: 51965 + | > loss: -0.27223 (-0.25732) + | > log_mle: -0.33859 (-0.35188) + | > loss_dur: 0.06636 (0.09456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.16929 (16.77361) + | > current_lr: 0.00006 + | > step_time: 2.40790 (4.01157) + | > loader_time: 0.00170 (0.07260) + + + --> STEP: 22/234 -- GLOBAL_STEP: 51970 + | > loss: -0.25550 (-0.25842) + | > log_mle: -0.34963 (-0.35018) + | > loss_dur: 0.09413 (0.09176) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.60074 (15.52779) + | > current_lr: 0.00006 + | > step_time: 4.49400 (3.85444) + | > loader_time: 0.01550 (0.06923) + + + --> STEP: 27/234 -- GLOBAL_STEP: 51975 + | > loss: -0.26641 (-0.26043) + | > log_mle: -0.34553 (-0.34926) + | > loss_dur: 0.07912 (0.08883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.57246 (14.55341) + | > current_lr: 0.00006 + | > step_time: 4.49420 (3.74195) + | > loader_time: 0.20170 (0.06431) + + + --> STEP: 32/234 -- GLOBAL_STEP: 51980 + | > loss: -0.26751 (-0.25967) + | > log_mle: -0.34974 (-0.34812) + | > loss_dur: 0.08223 (0.08845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.03674 (13.98284) + | > current_lr: 0.00006 + | > step_time: 2.41360 (3.70174) + | > loader_time: 0.08870 (0.07471) + + + --> STEP: 37/234 -- GLOBAL_STEP: 51985 + | > loss: -0.24366 (-0.25695) + | > log_mle: -0.32734 (-0.34608) + | > loss_dur: 0.08368 (0.08913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.81140 (13.57283) + | > current_lr: 0.00006 + | > step_time: 5.30180 (3.74461) + | > loader_time: 0.10420 (0.07254) + + + --> STEP: 42/234 -- GLOBAL_STEP: 51990 + | > loss: -0.21697 (-0.25352) + | > log_mle: -0.31950 (-0.34437) + | > loss_dur: 0.10253 (0.09085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.21322 (13.40645) + | > current_lr: 0.00006 + | > step_time: 3.39150 (3.69373) + | > loader_time: 0.00330 (0.07043) + + + --> STEP: 47/234 -- GLOBAL_STEP: 51995 + | > loss: -0.21659 (-0.25050) + | > log_mle: -0.32712 (-0.34326) + | > loss_dur: 0.11053 (0.09276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.86149 (13.37300) + | > current_lr: 0.00006 + | > step_time: 2.61660 (3.59114) + | > loader_time: 0.09880 (0.07921) + + + --> STEP: 52/234 -- GLOBAL_STEP: 52000 + | > loss: -0.21107 (-0.24911) + | > log_mle: -0.32720 (-0.34206) + | > loss_dur: 0.11613 (0.09294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.41276 (12.98585) + | > current_lr: 0.00006 + | > step_time: 2.09800 (3.54437) + | > loader_time: 0.09890 (0.07587) + + + --> STEP: 57/234 -- GLOBAL_STEP: 52005 + | > loss: -0.21336 (-0.24745) + | > log_mle: -0.31819 (-0.34124) + | > loss_dur: 0.10483 (0.09379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.24497 (12.86109) + | > current_lr: 0.00006 + | > step_time: 5.00120 (3.49158) + | > loader_time: 0.00180 (0.07473) + + + --> STEP: 62/234 -- GLOBAL_STEP: 52010 + | > loss: -0.17595 (-0.24497) + | > log_mle: -0.35021 (-0.34120) + | > loss_dur: 0.17426 (0.09623) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.10126 (12.94672) + | > current_lr: 0.00006 + | > step_time: 3.28830 (3.45619) + | > loader_time: 0.09440 (0.07199) + + + --> STEP: 67/234 -- GLOBAL_STEP: 52015 + | > loss: -0.22384 (-0.24328) + | > log_mle: -0.33787 (-0.34021) + | > loss_dur: 0.11403 (0.09693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.68239 (12.72311) + | > current_lr: 0.00006 + | > step_time: 2.00360 (3.40454) + | > loader_time: 0.19500 (0.07218) + + + --> STEP: 72/234 -- GLOBAL_STEP: 52020 + | > loss: -0.21743 (-0.24060) + | > log_mle: -0.32560 (-0.33911) + | > loss_dur: 0.10817 (0.09850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.84754 (12.78243) + | > current_lr: 0.00006 + | > step_time: 1.77410 (3.39122) + | > loader_time: 0.00620 (0.07287) + + + --> STEP: 77/234 -- GLOBAL_STEP: 52025 + | > loss: -0.22515 (-0.23807) + | > log_mle: -0.32972 (-0.33842) + | > loss_dur: 0.10457 (0.10035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.41450 (13.03197) + | > current_lr: 0.00006 + | > step_time: 2.58240 (3.36964) + | > loader_time: 0.19530 (0.07194) + + + --> STEP: 82/234 -- GLOBAL_STEP: 52030 + | > loss: -0.20208 (-0.23608) + | > log_mle: -0.32492 (-0.33766) + | > loss_dur: 0.12284 (0.10158) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.75274 (12.97376) + | > current_lr: 0.00006 + | > step_time: 4.89220 (3.30206) + | > loader_time: 0.11000 (0.07092) + + + --> STEP: 87/234 -- GLOBAL_STEP: 52035 + | > loss: -0.20259 (-0.23435) + | > log_mle: -0.32962 (-0.33732) + | > loss_dur: 0.12703 (0.10298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.36200 (12.97009) + | > current_lr: 0.00006 + | > step_time: 2.01320 (3.21782) + | > loader_time: 0.00180 (0.06705) + + + --> STEP: 92/234 -- GLOBAL_STEP: 52040 + | > loss: -0.23227 (-0.23329) + | > log_mle: -0.37168 (-0.33846) + | > loss_dur: 0.13941 (0.10517) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.02577 (13.29838) + | > current_lr: 0.00006 + | > step_time: 1.41080 (3.12680) + | > loader_time: 0.00220 (0.06352) + + + --> STEP: 97/234 -- GLOBAL_STEP: 52045 + | > loss: -0.22333 (-0.23302) + | > log_mle: -0.35586 (-0.34039) + | > loss_dur: 0.13253 (0.10737) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.52415 (13.89949) + | > current_lr: 0.00006 + | > step_time: 2.69220 (3.06166) + | > loader_time: 0.10590 (0.06235) + + + --> STEP: 102/234 -- GLOBAL_STEP: 52050 + | > loss: -0.19668 (-0.23204) + | > log_mle: -0.34128 (-0.34153) + | > loss_dur: 0.14460 (0.10949) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.22207 (14.28345) + | > current_lr: 0.00006 + | > step_time: 2.71460 (3.11214) + | > loader_time: 0.09270 (0.06303) + + + --> STEP: 107/234 -- GLOBAL_STEP: 52055 + | > loss: -0.22014 (-0.23182) + | > log_mle: -0.37971 (-0.34371) + | > loss_dur: 0.15957 (0.11188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.71787 (14.93342) + | > current_lr: 0.00006 + | > step_time: 2.10800 (3.12699) + | > loader_time: 0.06330 (0.06189) + + + --> STEP: 112/234 -- GLOBAL_STEP: 52060 + | > loss: -0.22203 (-0.23130) + | > log_mle: -0.39741 (-0.34580) + | > loss_dur: 0.17539 (0.11450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.46013 (15.49436) + | > current_lr: 0.00006 + | > step_time: 3.30620 (3.13515) + | > loader_time: 0.09100 (0.06258) + + + --> STEP: 117/234 -- GLOBAL_STEP: 52065 + | > loss: -0.22602 (-0.23106) + | > log_mle: -0.39042 (-0.34781) + | > loss_dur: 0.16440 (0.11676) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.47300 (15.99857) + | > current_lr: 0.00006 + | > step_time: 2.88660 (3.14647) + | > loader_time: 0.00250 (0.06513) + + + --> STEP: 122/234 -- GLOBAL_STEP: 52070 + | > loss: -0.20332 (-0.23052) + | > log_mle: -0.36336 (-0.34891) + | > loss_dur: 0.16004 (0.11839) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.04512 (16.26295) + | > current_lr: 0.00006 + | > step_time: 2.37790 (3.19195) + | > loader_time: 0.00350 (0.06724) + + + --> STEP: 127/234 -- GLOBAL_STEP: 52075 + | > loss: -0.23932 (-0.23055) + | > log_mle: -0.42674 (-0.35131) + | > loss_dur: 0.18742 (0.12075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.22940 (17.08932) + | > current_lr: 0.00006 + | > step_time: 6.79050 (3.21599) + | > loader_time: 0.09980 (0.06623) + + + --> STEP: 132/234 -- GLOBAL_STEP: 52080 + | > loss: -0.24683 (-0.23163) + | > log_mle: -0.40649 (-0.35428) + | > loss_dur: 0.15966 (0.12265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.16877 (17.79548) + | > current_lr: 0.00006 + | > step_time: 0.88800 (3.24217) + | > loader_time: 0.00740 (0.06723) + + + --> STEP: 137/234 -- GLOBAL_STEP: 52085 + | > loss: -0.23014 (-0.23259) + | > log_mle: -0.42639 (-0.35758) + | > loss_dur: 0.19625 (0.12499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.75828 (18.61997) + | > current_lr: 0.00006 + | > step_time: 3.00650 (3.22592) + | > loader_time: 0.08410 (0.06621) + + + --> STEP: 142/234 -- GLOBAL_STEP: 52090 + | > loss: -0.24016 (-0.23328) + | > log_mle: -0.43153 (-0.36021) + | > loss_dur: 0.19137 (0.12693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.86265 (19.56231) + | > current_lr: 0.00006 + | > step_time: 4.40630 (3.29081) + | > loader_time: 0.18420 (0.06769) + + + --> STEP: 147/234 -- GLOBAL_STEP: 52095 + | > loss: -0.24372 (-0.23518) + | > log_mle: -0.43663 (-0.36466) + | > loss_dur: 0.19292 (0.12948) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.88365 (20.64632) + | > current_lr: 0.00006 + | > step_time: 3.00050 (3.28428) + | > loader_time: 0.19390 (0.06821) + + + --> STEP: 152/234 -- GLOBAL_STEP: 52100 + | > loss: -0.30535 (-0.23729) + | > log_mle: -0.51866 (-0.36881) + | > loss_dur: 0.21331 (0.13152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.87715 (21.57844) + | > current_lr: 0.00006 + | > step_time: 4.48190 (3.30964) + | > loader_time: 0.11580 (0.06872) + + + --> STEP: 157/234 -- GLOBAL_STEP: 52105 + | > loss: -0.27371 (-0.24011) + | > log_mle: -0.47083 (-0.37360) + | > loss_dur: 0.19712 (0.13350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.12662 (22.80839) + | > current_lr: 0.00006 + | > step_time: 4.78380 (3.37988) + | > loader_time: 0.00210 (0.07004) + + + --> STEP: 162/234 -- GLOBAL_STEP: 52110 + | > loss: -0.30326 (-0.24247) + | > log_mle: -0.49581 (-0.37810) + | > loss_dur: 0.19255 (0.13563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.38119 (24.06419) + | > current_lr: 0.00006 + | > step_time: 1.83050 (3.36378) + | > loader_time: 0.07520 (0.06952) + + + --> STEP: 167/234 -- GLOBAL_STEP: 52115 + | > loss: -0.38808 (-0.24473) + | > log_mle: -0.58615 (-0.38222) + | > loss_dur: 0.19807 (0.13749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.24229 (25.02652) + | > current_lr: 0.00006 + | > step_time: 8.01840 (3.42489) + | > loader_time: 0.17890 (0.07111) + + + --> STEP: 172/234 -- GLOBAL_STEP: 52120 + | > loss: -0.35299 (-0.24732) + | > log_mle: -0.57204 (-0.38707) + | > loss_dur: 0.21905 (0.13975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.74839 (26.31230) + | > current_lr: 0.00006 + | > step_time: 6.60340 (3.44700) + | > loader_time: 0.28460 (0.07409) + + + --> STEP: 177/234 -- GLOBAL_STEP: 52125 + | > loss: -0.30797 (-0.24973) + | > log_mle: -0.52696 (-0.39173) + | > loss_dur: 0.21899 (0.14200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.39396 (27.52478) + | > current_lr: 0.00006 + | > step_time: 4.21100 (3.46033) + | > loader_time: 0.10380 (0.07439) + + + --> STEP: 182/234 -- GLOBAL_STEP: 52130 + | > loss: -0.34248 (-0.25217) + | > log_mle: -0.58247 (-0.39644) + | > loss_dur: 0.23998 (0.14428) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.05177 (28.57082) + | > current_lr: 0.00006 + | > step_time: 5.68230 (3.50076) + | > loader_time: 0.10080 (0.07479) + + + --> STEP: 187/234 -- GLOBAL_STEP: 52135 + | > loss: -0.34440 (-0.25460) + | > log_mle: -0.56997 (-0.40101) + | > loss_dur: 0.22558 (0.14641) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.18085 (29.77866) + | > current_lr: 0.00006 + | > step_time: 1.81120 (3.51899) + | > loader_time: 0.00830 (0.07503) + + + --> STEP: 192/234 -- GLOBAL_STEP: 52140 + | > loss: -0.39690 (-0.25711) + | > log_mle: -0.60823 (-0.40537) + | > loss_dur: 0.21133 (0.14826) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.17743 (30.93404) + | > current_lr: 0.00006 + | > step_time: 3.10260 (3.51553) + | > loader_time: 0.00620 (0.07377) + + + --> STEP: 197/234 -- GLOBAL_STEP: 52145 + | > loss: -0.35381 (-0.25977) + | > log_mle: -0.55316 (-0.40968) + | > loss_dur: 0.19936 (0.14991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.62634 (32.07916) + | > current_lr: 0.00006 + | > step_time: 5.19470 (3.56330) + | > loader_time: 0.09310 (0.07286) + + + --> STEP: 202/234 -- GLOBAL_STEP: 52150 + | > loss: -0.43821 (-0.26188) + | > log_mle: -0.65616 (-0.41378) + | > loss_dur: 0.21795 (0.15190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.42386 (33.06199) + | > current_lr: 0.00006 + | > step_time: 6.89470 (3.64207) + | > loader_time: 0.11070 (0.07366) + + + --> STEP: 207/234 -- GLOBAL_STEP: 52155 + | > loss: -0.39051 (-0.26421) + | > log_mle: -0.63584 (-0.41799) + | > loss_dur: 0.24533 (0.15379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 110.78547 (34.07882) + | > current_lr: 0.00006 + | > step_time: 8.41350 (3.69475) + | > loader_time: 0.07880 (0.07516) + + + --> STEP: 212/234 -- GLOBAL_STEP: 52160 + | > loss: -0.38415 (-0.26703) + | > log_mle: -0.62648 (-0.42286) + | > loss_dur: 0.24233 (0.15583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.03508 (35.24285) + | > current_lr: 0.00006 + | > step_time: 4.40990 (3.72844) + | > loader_time: 0.18670 (0.07620) + + + --> STEP: 217/234 -- GLOBAL_STEP: 52165 + | > loss: -0.41352 (-0.27017) + | > log_mle: -0.65600 (-0.42782) + | > loss_dur: 0.24248 (0.15765) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.14394 (36.26323) + | > current_lr: 0.00006 + | > step_time: 5.21220 (3.73144) + | > loader_time: 0.08890 (0.07720) + + + --> STEP: 222/234 -- GLOBAL_STEP: 52170 + | > loss: -0.41411 (-0.27344) + | > log_mle: -0.67357 (-0.43289) + | > loss_dur: 0.25946 (0.15945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.96049 (37.38063) + | > current_lr: 0.00006 + | > step_time: 4.07210 (3.73340) + | > loader_time: 0.00460 (0.07667) + + + --> STEP: 227/234 -- GLOBAL_STEP: 52175 + | > loss: -0.38046 (-0.27697) + | > log_mle: -0.63672 (-0.43834) + | > loss_dur: 0.25626 (0.16136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.25002 (38.63757) + | > current_lr: 0.00006 + | > step_time: 0.26340 (3.67287) + | > loader_time: 0.00260 (0.07583) + + + --> STEP: 232/234 -- GLOBAL_STEP: 52180 + | > loss: -0.39357 (-0.27980) + | > log_mle: -0.85487 (-0.44508) + | > loss_dur: 0.46130 (0.16528) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 135.93546 (40.31942) + | > current_lr: 0.00006 + | > step_time: 0.36770 (3.60005) + | > loader_time: 0.02450 (0.07437) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.18209 (-0.12844) + | > avg_loss: -0.26762 (+0.02058) + | > avg_log_mle: -0.49486 (+0.01687) + | > avg_loss_dur: 0.22724 (+0.00371) + + + > EPOCH: 223/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 07:55:23)  + + --> STEP: 3/234 -- GLOBAL_STEP: 52185 + | > loss: -0.15962 (-0.22312) + | > log_mle: -0.33802 (-0.34814) + | > loss_dur: 0.17840 (0.12502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.85603 (20.08514) + | > current_lr: 0.00006 + | > step_time: 1.49630 (2.38922) + | > loader_time: 0.01230 (0.03774) + + + --> STEP: 8/234 -- GLOBAL_STEP: 52190 + | > loss: -0.27474 (-0.25102) + | > log_mle: -0.36659 (-0.35133) + | > loss_dur: 0.09185 (0.10031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.83489 (17.84521) + | > current_lr: 0.00006 + | > step_time: 5.13260 (3.45008) + | > loader_time: 0.18310 (0.03924) + + + --> STEP: 13/234 -- GLOBAL_STEP: 52195 + | > loss: -0.28676 (-0.25654) + | > log_mle: -0.36233 (-0.35287) + | > loss_dur: 0.07558 (0.09633) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.77021 (17.06006) + | > current_lr: 0.00006 + | > step_time: 7.88760 (4.84460) + | > loader_time: 0.00200 (0.05676) + + + --> STEP: 18/234 -- GLOBAL_STEP: 52200 + | > loss: -0.25700 (-0.25965) + | > log_mle: -0.34338 (-0.35188) + | > loss_dur: 0.08638 (0.09223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.48741 (15.70263) + | > current_lr: 0.00006 + | > step_time: 2.30310 (4.32110) + | > loader_time: 0.00850 (0.07344) + + + --> STEP: 23/234 -- GLOBAL_STEP: 52205 + | > loss: -0.28012 (-0.26131) + | > log_mle: -0.35954 (-0.35122) + | > loss_dur: 0.07942 (0.08991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.83060 (14.75657) + | > current_lr: 0.00006 + | > step_time: 2.49380 (4.06840) + | > loader_time: 0.00150 (0.06553) + + + --> STEP: 28/234 -- GLOBAL_STEP: 52210 + | > loss: -0.30492 (-0.26291) + | > log_mle: -0.36377 (-0.35000) + | > loss_dur: 0.05886 (0.08709) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.27830 (14.33779) + | > current_lr: 0.00006 + | > step_time: 5.00070 (3.89640) + | > loader_time: 0.00220 (0.05804) + + + --> STEP: 33/234 -- GLOBAL_STEP: 52215 + | > loss: -0.24503 (-0.25945) + | > log_mle: -0.33193 (-0.34729) + | > loss_dur: 0.08690 (0.08784) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.09638 (14.35170) + | > current_lr: 0.00006 + | > step_time: 3.60600 (4.06018) + | > loader_time: 0.09570 (0.05842) + + + --> STEP: 38/234 -- GLOBAL_STEP: 52220 + | > loss: -0.23866 (-0.25542) + | > log_mle: -0.33650 (-0.34454) + | > loss_dur: 0.09784 (0.08912) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.58727 (14.41125) + | > current_lr: 0.00006 + | > step_time: 7.28840 (4.22339) + | > loader_time: 0.19560 (0.06123) + + + --> STEP: 43/234 -- GLOBAL_STEP: 52225 + | > loss: -0.21434 (-0.25161) + | > log_mle: -0.33204 (-0.34249) + | > loss_dur: 0.11770 (0.09089) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.45475 (13.96436) + | > current_lr: 0.00006 + | > step_time: 1.50700 (4.05991) + | > loader_time: 0.00410 (0.05666) + + + --> STEP: 48/234 -- GLOBAL_STEP: 52230 + | > loss: -0.24631 (-0.24947) + | > log_mle: -0.33090 (-0.34134) + | > loss_dur: 0.08458 (0.09188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.17698 (13.46241) + | > current_lr: 0.00006 + | > step_time: 4.08390 (3.93298) + | > loader_time: 0.09800 (0.06027) + + + --> STEP: 53/234 -- GLOBAL_STEP: 52235 + | > loss: -0.22215 (-0.24770) + | > log_mle: -0.33392 (-0.34029) + | > loss_dur: 0.11177 (0.09258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.59382 (13.02392) + | > current_lr: 0.00006 + | > step_time: 4.02400 (3.83250) + | > loader_time: 0.00190 (0.05996) + + + --> STEP: 58/234 -- GLOBAL_STEP: 52240 + | > loss: -0.23835 (-0.24629) + | > log_mle: -0.33233 (-0.33950) + | > loss_dur: 0.09399 (0.09321) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.86447 (12.84806) + | > current_lr: 0.00006 + | > step_time: 4.81640 (3.85318) + | > loader_time: 0.07870 (0.05967) + + + --> STEP: 63/234 -- GLOBAL_STEP: 52245 + | > loss: -0.21466 (-0.24371) + | > log_mle: -0.32608 (-0.33949) + | > loss_dur: 0.11142 (0.09579) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.70701 (12.99224) + | > current_lr: 0.00006 + | > step_time: 3.41800 (3.76905) + | > loader_time: 0.19650 (0.06134) + + + --> STEP: 68/234 -- GLOBAL_STEP: 52250 + | > loss: -0.19095 (-0.24192) + | > log_mle: -0.32056 (-0.33869) + | > loss_dur: 0.12961 (0.09677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.70359 (12.79500) + | > current_lr: 0.00006 + | > step_time: 2.82320 (3.73487) + | > loader_time: 0.09720 (0.06424) + + + --> STEP: 73/234 -- GLOBAL_STEP: 52255 + | > loss: -0.20275 (-0.23935) + | > log_mle: -0.33726 (-0.33801) + | > loss_dur: 0.13451 (0.09866) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.18539 (12.86398) + | > current_lr: 0.00006 + | > step_time: 1.61570 (3.69982) + | > loader_time: 0.08240 (0.06645) + + + --> STEP: 78/234 -- GLOBAL_STEP: 52260 + | > loss: -0.20035 (-0.23705) + | > log_mle: -0.31607 (-0.33727) + | > loss_dur: 0.11572 (0.10022) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.32677 (13.03838) + | > current_lr: 0.00006 + | > step_time: 3.61130 (3.70307) + | > loader_time: 0.00210 (0.06472) + + + --> STEP: 83/234 -- GLOBAL_STEP: 52265 + | > loss: -0.18424 (-0.23529) + | > log_mle: -0.33767 (-0.33689) + | > loss_dur: 0.15343 (0.10160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.55431 (13.08076) + | > current_lr: 0.00006 + | > step_time: 6.31600 (3.69702) + | > loader_time: 0.00950 (0.06241) + + + --> STEP: 88/234 -- GLOBAL_STEP: 52270 + | > loss: -0.22505 (-0.23428) + | > log_mle: -0.37216 (-0.33714) + | > loss_dur: 0.14710 (0.10286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.86238 (13.02209) + | > current_lr: 0.00006 + | > step_time: 3.78690 (3.68196) + | > loader_time: 0.08330 (0.06332) + + + --> STEP: 93/234 -- GLOBAL_STEP: 52275 + | > loss: -0.23660 (-0.23376) + | > log_mle: -0.38655 (-0.33860) + | > loss_dur: 0.14995 (0.10484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.83808 (13.30744) + | > current_lr: 0.00006 + | > step_time: 4.18520 (3.65570) + | > loader_time: 0.10040 (0.06398) + + + --> STEP: 98/234 -- GLOBAL_STEP: 52280 + | > loss: -0.19977 (-0.23346) + | > log_mle: -0.31907 (-0.34007) + | > loss_dur: 0.11930 (0.10662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.20119 (13.74205) + | > current_lr: 0.00006 + | > step_time: 3.00200 (3.68675) + | > loader_time: 0.10050 (0.06501) + + + --> STEP: 103/234 -- GLOBAL_STEP: 52285 + | > loss: -0.23749 (-0.23311) + | > log_mle: -0.41535 (-0.34231) + | > loss_dur: 0.17786 (0.10920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.18369 (14.45492) + | > current_lr: 0.00006 + | > step_time: 1.41070 (3.68102) + | > loader_time: 0.00700 (0.06447) + + + --> STEP: 108/234 -- GLOBAL_STEP: 52290 + | > loss: -0.22349 (-0.23297) + | > log_mle: -0.36095 (-0.34420) + | > loss_dur: 0.13747 (0.11123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.71506 (14.82399) + | > current_lr: 0.00006 + | > step_time: 1.50660 (3.63125) + | > loader_time: 0.00360 (0.06403) + + + --> STEP: 113/234 -- GLOBAL_STEP: 52295 + | > loss: -0.23901 (-0.23260) + | > log_mle: -0.40449 (-0.34674) + | > loss_dur: 0.16548 (0.11414) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.34300 (15.65833) + | > current_lr: 0.00006 + | > step_time: 2.41780 (3.63079) + | > loader_time: 0.07190 (0.06367) + + + --> STEP: 118/234 -- GLOBAL_STEP: 52300 + | > loss: -0.20802 (-0.23197) + | > log_mle: -0.37576 (-0.34852) + | > loss_dur: 0.16774 (0.11656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.80344 (15.97099) + | > current_lr: 0.00006 + | > step_time: 3.62090 (3.62989) + | > loader_time: 0.07290 (0.06254) + + + --> STEP: 123/234 -- GLOBAL_STEP: 52305 + | > loss: -0.18712 (-0.23121) + | > log_mle: -0.34470 (-0.34939) + | > loss_dur: 0.15759 (0.11817) + | > amp_scaler: 2048.00000 (1032.32520) + | > grad_norm: 20.54430 (16.21819) + | > current_lr: 0.00006 + | > step_time: 2.09770 (3.61236) + | > loader_time: 0.17860 (0.06447) + + + --> STEP: 128/234 -- GLOBAL_STEP: 52310 + | > loss: -0.23995 (-0.23192) + | > log_mle: -0.39982 (-0.35221) + | > loss_dur: 0.15986 (0.12029) + | > amp_scaler: 2048.00000 (1072.00000) + | > grad_norm: 38.23828 (17.05515) + | > current_lr: 0.00006 + | > step_time: 7.32680 (3.66697) + | > loader_time: 0.09930 (0.06552) + + + --> STEP: 133/234 -- GLOBAL_STEP: 52315 + | > loss: -0.25615 (-0.23284) + | > log_mle: -0.43154 (-0.35526) + | > loss_dur: 0.17539 (0.12242) + | > amp_scaler: 2048.00000 (1108.69173) + | > grad_norm: 34.86447 (17.93653) + | > current_lr: 0.00006 + | > step_time: 3.50070 (3.65078) + | > loader_time: 0.08060 (0.06595) + + + --> STEP: 138/234 -- GLOBAL_STEP: 52320 + | > loss: -0.20829 (-0.23349) + | > log_mle: -0.37859 (-0.35807) + | > loss_dur: 0.17030 (0.12457) + | > amp_scaler: 2048.00000 (1142.72464) + | > grad_norm: 28.30306 (18.70385) + | > current_lr: 0.00006 + | > step_time: 2.90600 (3.61665) + | > loader_time: 0.10210 (0.06776) + + + --> STEP: 143/234 -- GLOBAL_STEP: 52325 + | > loss: -0.29114 (-0.23473) + | > log_mle: -0.52249 (-0.36169) + | > loss_dur: 0.23135 (0.12696) + | > amp_scaler: 2048.00000 (1174.37762) + | > grad_norm: 56.46130 (19.58791) + | > current_lr: 0.00006 + | > step_time: 7.91800 (3.65394) + | > loader_time: 0.19780 (0.07007) + + + --> STEP: 148/234 -- GLOBAL_STEP: 52330 + | > loss: -0.27912 (-0.23638) + | > log_mle: -0.43756 (-0.36532) + | > loss_dur: 0.15844 (0.12895) + | > amp_scaler: 2048.00000 (1203.89189) + | > grad_norm: 35.05389 (20.39439) + | > current_lr: 0.00006 + | > step_time: 4.11950 (3.67113) + | > loader_time: 0.07960 (0.07079) + + + --> STEP: 153/234 -- GLOBAL_STEP: 52335 + | > loss: -0.37103 (-0.23912) + | > log_mle: -0.57151 (-0.37027) + | > loss_dur: 0.20048 (0.13115) + | > amp_scaler: 2048.00000 (1231.47712) + | > grad_norm: 75.30254 (21.44994) + | > current_lr: 0.00006 + | > step_time: 4.58010 (3.66320) + | > loader_time: 0.09890 (0.07078) + + + --> STEP: 158/234 -- GLOBAL_STEP: 52340 + | > loss: -0.29605 (-0.24122) + | > log_mle: -0.50379 (-0.37459) + | > loss_dur: 0.20773 (0.13337) + | > amp_scaler: 2048.00000 (1257.31646) + | > grad_norm: 51.58325 (22.57510) + | > current_lr: 0.00006 + | > step_time: 4.19430 (3.64246) + | > loader_time: 0.00310 (0.07071) + + + --> STEP: 163/234 -- GLOBAL_STEP: 52345 + | > loss: -0.27578 (-0.24356) + | > log_mle: -0.47260 (-0.37888) + | > loss_dur: 0.19683 (0.13532) + | > amp_scaler: 2048.00000 (1281.57055) + | > grad_norm: 51.08821 (23.69557) + | > current_lr: 0.00006 + | > step_time: 5.41780 (3.70202) + | > loader_time: 0.18920 (0.07166) + + + --> STEP: 168/234 -- GLOBAL_STEP: 52350 + | > loss: -0.30792 (-0.24587) + | > log_mle: -0.52637 (-0.38324) + | > loss_dur: 0.21846 (0.13737) + | > amp_scaler: 2048.00000 (1304.38095) + | > grad_norm: 87.48227 (25.01698) + | > current_lr: 0.00006 + | > step_time: 6.98670 (3.69718) + | > loader_time: 0.10240 (0.07181) + + + --> STEP: 173/234 -- GLOBAL_STEP: 52355 + | > loss: -0.31740 (-0.24845) + | > log_mle: -0.53497 (-0.38812) + | > loss_dur: 0.21757 (0.13967) + | > amp_scaler: 2048.00000 (1325.87283) + | > grad_norm: 70.91328 (26.35367) + | > current_lr: 0.00006 + | > step_time: 8.68900 (3.73826) + | > loader_time: 0.21470 (0.07224) + + + --> STEP: 178/234 -- GLOBAL_STEP: 52360 + | > loss: -0.36489 (-0.25126) + | > log_mle: -0.59936 (-0.39318) + | > loss_dur: 0.23447 (0.14192) + | > amp_scaler: 2048.00000 (1346.15730) + | > grad_norm: 57.55754 (27.41003) + | > current_lr: 0.00006 + | > step_time: 5.38970 (3.76721) + | > loader_time: 0.10450 (0.07391) + + + --> STEP: 183/234 -- GLOBAL_STEP: 52365 + | > loss: -0.37389 (-0.25350) + | > log_mle: -0.59263 (-0.39765) + | > loss_dur: 0.21874 (0.14414) + | > amp_scaler: 2048.00000 (1365.33333) + | > grad_norm: 62.73517 (28.53810) + | > current_lr: 0.00006 + | > step_time: 6.88700 (3.79638) + | > loader_time: 0.10870 (0.07851) + + + --> STEP: 188/234 -- GLOBAL_STEP: 52370 + | > loss: -0.39533 (-0.25594) + | > log_mle: -0.60785 (-0.40221) + | > loss_dur: 0.21252 (0.14628) + | > amp_scaler: 2048.00000 (1383.48936) + | > grad_norm: 82.78552 (29.50577) + | > current_lr: 0.00006 + | > step_time: 1.60220 (3.79161) + | > loader_time: 0.00320 (0.07850) + + + --> STEP: 193/234 -- GLOBAL_STEP: 52375 + | > loss: -0.38689 (-0.25868) + | > log_mle: -0.60838 (-0.40667) + | > loss_dur: 0.22149 (0.14799) + | > amp_scaler: 2048.00000 (1400.70466) + | > grad_norm: 73.24723 (30.69510) + | > current_lr: 0.00006 + | > step_time: 10.39370 (3.84657) + | > loader_time: 0.08570 (0.07841) + + + --> STEP: 198/234 -- GLOBAL_STEP: 52380 + | > loss: -0.37392 (-0.26133) + | > log_mle: -0.59887 (-0.41104) + | > loss_dur: 0.22496 (0.14972) + | > amp_scaler: 2048.00000 (1417.05051) + | > grad_norm: 79.99033 (31.65265) + | > current_lr: 0.00006 + | > step_time: 4.69160 (3.87115) + | > loader_time: 0.00470 (0.07889) + + + --> STEP: 203/234 -- GLOBAL_STEP: 52385 + | > loss: -0.31219 (-0.26360) + | > log_mle: -0.52479 (-0.41520) + | > loss_dur: 0.21260 (0.15160) + | > amp_scaler: 2048.00000 (1432.59113) + | > grad_norm: 66.75938 (32.72841) + | > current_lr: 0.00006 + | > step_time: 2.59360 (3.87123) + | > loader_time: 0.00440 (0.08002) + + + --> STEP: 208/234 -- GLOBAL_STEP: 52390 + | > loss: -0.37485 (-0.26670) + | > log_mle: -0.61054 (-0.42022) + | > loss_dur: 0.23569 (0.15353) + | > amp_scaler: 2048.00000 (1447.38462) + | > grad_norm: 93.30862 (33.85049) + | > current_lr: 0.00006 + | > step_time: 3.58860 (3.89676) + | > loader_time: 0.19970 (0.08089) + + + --> STEP: 213/234 -- GLOBAL_STEP: 52395 + | > loss: -0.42985 (-0.27010) + | > log_mle: -0.66935 (-0.42559) + | > loss_dur: 0.23951 (0.15550) + | > amp_scaler: 2048.00000 (1461.48357) + | > grad_norm: 94.00314 (35.06924) + | > current_lr: 0.00006 + | > step_time: 4.69070 (3.90569) + | > loader_time: 0.00500 (0.08011) + + + --> STEP: 218/234 -- GLOBAL_STEP: 52400 + | > loss: -0.38516 (-0.27319) + | > log_mle: -0.62123 (-0.43048) + | > loss_dur: 0.23607 (0.15729) + | > amp_scaler: 2048.00000 (1474.93578) + | > grad_norm: 105.85996 (36.50723) + | > current_lr: 0.00006 + | > step_time: 5.90080 (3.89560) + | > loader_time: 0.10160 (0.07982) + + + --> STEP: 223/234 -- GLOBAL_STEP: 52405 + | > loss: -0.42493 (-0.27657) + | > log_mle: -0.66638 (-0.43569) + | > loss_dur: 0.24145 (0.15912) + | > amp_scaler: 2048.00000 (1487.78475) + | > grad_norm: 91.85123 (37.83706) + | > current_lr: 0.00006 + | > step_time: 3.98250 (3.87626) + | > loader_time: 1.09050 (0.08559) + + + --> STEP: 228/234 -- GLOBAL_STEP: 52410 + | > loss: -0.40206 (-0.28007) + | > log_mle: -0.66872 (-0.44117) + | > loss_dur: 0.26666 (0.16110) + | > amp_scaler: 2048.00000 (1500.07018) + | > grad_norm: 95.67184 (39.09506) + | > current_lr: 0.00006 + | > step_time: 0.28090 (3.81555) + | > loader_time: 0.00640 (0.08459) + + + --> STEP: 233/234 -- GLOBAL_STEP: 52415 + | > loss: -0.02566 (-0.28148) + | > log_mle: -0.64824 (-0.44808) + | > loss_dur: 0.62258 (0.16660) + | > amp_scaler: 2048.00000 (1511.82833) + | > grad_norm: 105.74664 (40.61325) + | > current_lr: 0.00006 + | > step_time: 0.21360 (3.73983) + | > loader_time: 0.00300 (0.08333) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.09832 (+0.91623) + | > avg_loss: -0.27771 (-0.01010) + | > avg_log_mle: -0.51679 (-0.02194) + | > avg_loss_dur: 0.23908 (+0.01184) + + + > EPOCH: 224/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 08:11:26)  + + --> STEP: 4/234 -- GLOBAL_STEP: 52420 + | > loss: -0.24591 (-0.24731) + | > log_mle: -0.35142 (-0.35097) + | > loss_dur: 0.10551 (0.10366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.98497 (23.14357) + | > current_lr: 0.00006 + | > step_time: 8.30300 (5.22794) + | > loader_time: 0.00620 (0.10304) + + + --> STEP: 9/234 -- GLOBAL_STEP: 52425 + | > loss: -0.24853 (-0.25583) + | > log_mle: -0.36435 (-0.35492) + | > loss_dur: 0.11583 (0.09909) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.81927 (19.12486) + | > current_lr: 0.00006 + | > step_time: 3.41260 (4.77854) + | > loader_time: 0.09200 (0.08105) + + + --> STEP: 14/234 -- GLOBAL_STEP: 52430 + | > loss: -0.27220 (-0.26133) + | > log_mle: -0.35468 (-0.35528) + | > loss_dur: 0.08247 (0.09394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.13659 (17.82451) + | > current_lr: 0.00006 + | > step_time: 2.72600 (4.48729) + | > loader_time: 0.07740 (0.06597) + + + --> STEP: 19/234 -- GLOBAL_STEP: 52435 + | > loss: -0.27532 (-0.26371) + | > log_mle: -0.34755 (-0.35396) + | > loss_dur: 0.07223 (0.09025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.78430 (16.09557) + | > current_lr: 0.00006 + | > step_time: 3.91680 (4.34356) + | > loader_time: 0.20080 (0.06806) + + + --> STEP: 24/234 -- GLOBAL_STEP: 52440 + | > loss: -0.27495 (-0.26573) + | > log_mle: -0.34388 (-0.35299) + | > loss_dur: 0.06894 (0.08726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.93049 (15.08031) + | > current_lr: 0.00006 + | > step_time: 5.61680 (4.36938) + | > loader_time: 0.08870 (0.07283) + + + --> STEP: 29/234 -- GLOBAL_STEP: 52445 + | > loss: -0.23707 (-0.26553) + | > log_mle: -0.32755 (-0.35140) + | > loss_dur: 0.09048 (0.08586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.20147 (14.59152) + | > current_lr: 0.00006 + | > step_time: 4.29660 (4.25394) + | > loader_time: 0.00170 (0.07040) + + + --> STEP: 34/234 -- GLOBAL_STEP: 52450 + | > loss: -0.24321 (-0.26312) + | > log_mle: -0.33702 (-0.34951) + | > loss_dur: 0.09381 (0.08639) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.17751 (14.40605) + | > current_lr: 0.00006 + | > step_time: 2.89140 (4.15724) + | > loader_time: 0.00120 (0.06644) + + + --> STEP: 39/234 -- GLOBAL_STEP: 52455 + | > loss: -0.23261 (-0.25900) + | > log_mle: -0.33464 (-0.34744) + | > loss_dur: 0.10203 (0.08844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.88693 (14.17133) + | > current_lr: 0.00006 + | > step_time: 2.01580 (4.13783) + | > loader_time: 0.07340 (0.06718) + + + --> STEP: 44/234 -- GLOBAL_STEP: 52460 + | > loss: -0.25094 (-0.25639) + | > log_mle: -0.32799 (-0.34550) + | > loss_dur: 0.07705 (0.08912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.33428 (13.59797) + | > current_lr: 0.00006 + | > step_time: 2.02250 (3.92223) + | > loader_time: 0.08040 (0.06399) + + + --> STEP: 49/234 -- GLOBAL_STEP: 52465 + | > loss: -0.25782 (-0.25452) + | > log_mle: -0.34229 (-0.34475) + | > loss_dur: 0.08448 (0.09022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.03307 (13.22068) + | > current_lr: 0.00006 + | > step_time: 1.60290 (3.79009) + | > loader_time: 0.08520 (0.06266) + + + --> STEP: 54/234 -- GLOBAL_STEP: 52470 + | > loss: -0.24455 (-0.25241) + | > log_mle: -0.33699 (-0.34357) + | > loss_dur: 0.09244 (0.09115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.05105 (13.10809) + | > current_lr: 0.00006 + | > step_time: 2.01150 (3.71330) + | > loader_time: 0.09790 (0.06359) + + + --> STEP: 59/234 -- GLOBAL_STEP: 52475 + | > loss: -0.23879 (-0.25060) + | > log_mle: -0.34211 (-0.34264) + | > loss_dur: 0.10332 (0.09203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.71653 (13.21133) + | > current_lr: 0.00006 + | > step_time: 5.71770 (3.64977) + | > loader_time: 0.10070 (0.06265) + + + --> STEP: 64/234 -- GLOBAL_STEP: 52480 + | > loss: -0.22859 (-0.24786) + | > log_mle: -0.32932 (-0.34228) + | > loss_dur: 0.10074 (0.09441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.66487 (13.38279) + | > current_lr: 0.00006 + | > step_time: 2.71230 (3.54086) + | > loader_time: 0.28320 (0.06383) + + + --> STEP: 69/234 -- GLOBAL_STEP: 52485 + | > loss: -0.21758 (-0.24597) + | > log_mle: -0.31784 (-0.34126) + | > loss_dur: 0.10026 (0.09530) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.61970 (13.13356) + | > current_lr: 0.00006 + | > step_time: 1.42320 (3.48191) + | > loader_time: 0.00170 (0.06224) + + + --> STEP: 74/234 -- GLOBAL_STEP: 52490 + | > loss: -0.20227 (-0.24301) + | > log_mle: -0.31847 (-0.34049) + | > loss_dur: 0.11619 (0.09747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.22173 (13.20344) + | > current_lr: 0.00006 + | > step_time: 3.99140 (3.46934) + | > loader_time: 0.00370 (0.05972) + + + --> STEP: 79/234 -- GLOBAL_STEP: 52495 + | > loss: -0.21249 (-0.24081) + | > log_mle: -0.33282 (-0.33983) + | > loss_dur: 0.12033 (0.09902) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.47278 (13.14757) + | > current_lr: 0.00006 + | > step_time: 1.81240 (3.46063) + | > loader_time: 0.00330 (0.05855) + + + --> STEP: 84/234 -- GLOBAL_STEP: 52500 + | > loss: -0.22280 (-0.23910) + | > log_mle: -0.32994 (-0.33929) + | > loss_dur: 0.10715 (0.10019) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.08371 (13.16118) + | > current_lr: 0.00006 + | > step_time: 5.52150 (3.46349) + | > loader_time: 0.09390 (0.05834) + + + --> STEP: 89/234 -- GLOBAL_STEP: 52505 + | > loss: -0.22676 (-0.23786) + | > log_mle: -0.35361 (-0.33965) + | > loss_dur: 0.12685 (0.10179) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.38028 (13.26660) + | > current_lr: 0.00006 + | > step_time: 2.58940 (3.42635) + | > loader_time: 0.00600 (0.05723) + + + --> STEP: 94/234 -- GLOBAL_STEP: 52510 + | > loss: -0.23436 (-0.23697) + | > log_mle: -0.38219 (-0.34108) + | > loss_dur: 0.14783 (0.10411) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.71142 (13.80573) + | > current_lr: 0.00006 + | > step_time: 2.32330 (3.38776) + | > loader_time: 0.08610 (0.05708) + + + --> STEP: 99/234 -- GLOBAL_STEP: 52515 + | > loss: -0.23224 (-0.23614) + | > log_mle: -0.40708 (-0.34255) + | > loss_dur: 0.17484 (0.10641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.24463 (14.31841) + | > current_lr: 0.00006 + | > step_time: 3.69850 (3.37914) + | > loader_time: 0.00250 (0.05620) + + + --> STEP: 104/234 -- GLOBAL_STEP: 52520 + | > loss: -0.26944 (-0.23583) + | > log_mle: -0.42709 (-0.34460) + | > loss_dur: 0.15765 (0.10877) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.66618 (14.94575) + | > current_lr: 0.00006 + | > step_time: 2.71310 (3.37108) + | > loader_time: 0.10310 (0.05656) + + + --> STEP: 109/234 -- GLOBAL_STEP: 52525 + | > loss: -0.21213 (-0.23488) + | > log_mle: -0.40073 (-0.34599) + | > loss_dur: 0.18860 (0.11111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.28396 (15.30747) + | > current_lr: 0.00006 + | > step_time: 5.71730 (3.41828) + | > loader_time: 0.00510 (0.05776) + + + --> STEP: 114/234 -- GLOBAL_STEP: 52530 + | > loss: -0.23670 (-0.23476) + | > log_mle: -0.38309 (-0.34824) + | > loss_dur: 0.14639 (0.11348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.08893 (15.78706) + | > current_lr: 0.00006 + | > step_time: 3.27740 (3.44601) + | > loader_time: 0.00270 (0.05696) + + + --> STEP: 119/234 -- GLOBAL_STEP: 52535 + | > loss: -0.23009 (-0.23408) + | > log_mle: -0.38426 (-0.34995) + | > loss_dur: 0.15417 (0.11587) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.53209 (16.05914) + | > current_lr: 0.00006 + | > step_time: 3.70830 (3.44984) + | > loader_time: 0.01220 (0.05691) + + + --> STEP: 124/234 -- GLOBAL_STEP: 52540 + | > loss: -0.25368 (-0.23352) + | > log_mle: -0.41007 (-0.35089) + | > loss_dur: 0.15639 (0.11737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.74616 (16.42371) + | > current_lr: 0.00006 + | > step_time: 1.71190 (3.45160) + | > loader_time: 0.07410 (0.05681) + + + --> STEP: 129/234 -- GLOBAL_STEP: 52545 + | > loss: -0.22853 (-0.23398) + | > log_mle: -0.40326 (-0.35357) + | > loss_dur: 0.17473 (0.11960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.91509 (17.08066) + | > current_lr: 0.00006 + | > step_time: 5.00730 (3.45541) + | > loader_time: 0.11650 (0.05776) + + + --> STEP: 134/234 -- GLOBAL_STEP: 52550 + | > loss: -0.26375 (-0.23522) + | > log_mle: -0.45406 (-0.35703) + | > loss_dur: 0.19031 (0.12182) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.03241 (18.02898) + | > current_lr: 0.00006 + | > step_time: 4.10660 (3.42891) + | > loader_time: 0.09840 (0.05863) + + + --> STEP: 139/234 -- GLOBAL_STEP: 52555 + | > loss: -0.33346 (-0.23614) + | > log_mle: -0.52249 (-0.36026) + | > loss_dur: 0.18904 (0.12412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.49989 (18.81085) + | > current_lr: 0.00006 + | > step_time: 2.01730 (3.39941) + | > loader_time: 0.09020 (0.05887) + + + --> STEP: 144/234 -- GLOBAL_STEP: 52560 + | > loss: -0.28576 (-0.23697) + | > log_mle: -0.48387 (-0.36350) + | > loss_dur: 0.19811 (0.12653) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.79490 (19.85175) + | > current_lr: 0.00006 + | > step_time: 1.29120 (3.39854) + | > loader_time: 0.02400 (0.06158) + + + --> STEP: 149/234 -- GLOBAL_STEP: 52565 + | > loss: -0.34391 (-0.23891) + | > log_mle: -0.54330 (-0.36743) + | > loss_dur: 0.19939 (0.12852) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.89268 (20.68644) + | > current_lr: 0.00006 + | > step_time: 5.10530 (3.40017) + | > loader_time: 0.00380 (0.06153) + + + --> STEP: 154/234 -- GLOBAL_STEP: 52570 + | > loss: -0.29508 (-0.24115) + | > log_mle: -0.48437 (-0.37188) + | > loss_dur: 0.18929 (0.13073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 169.14146 (22.50843) + | > current_lr: 0.00006 + | > step_time: 2.39640 (3.39253) + | > loader_time: 0.10450 (0.06328) + + + --> STEP: 159/234 -- GLOBAL_STEP: 52575 + | > loss: -0.29154 (-0.24184) + | > log_mle: -0.49100 (-0.37490) + | > loss_dur: 0.19946 (0.13306) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.13776 (23.60551) + | > current_lr: 0.00006 + | > step_time: 1.70660 (3.43325) + | > loader_time: 0.00520 (0.06301) + + + --> STEP: 164/234 -- GLOBAL_STEP: 52580 + | > loss: -0.28713 (-0.24352) + | > log_mle: -0.49388 (-0.37857) + | > loss_dur: 0.20675 (0.13505) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.38884 (24.14981) + | > current_lr: 0.00006 + | > step_time: 7.20090 (3.47479) + | > loader_time: 0.20630 (0.06422) + + + --> STEP: 169/234 -- GLOBAL_STEP: 52585 + | > loss: -0.29333 (-0.24573) + | > log_mle: -0.50373 (-0.38270) + | > loss_dur: 0.21040 (0.13697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.91166 (24.97492) + | > current_lr: 0.00006 + | > step_time: 2.39450 (3.45372) + | > loader_time: 0.00240 (0.06362) + + + --> STEP: 174/234 -- GLOBAL_STEP: 52590 + | > loss: -0.38660 (-0.24896) + | > log_mle: -0.59757 (-0.38812) + | > loss_dur: 0.21097 (0.13916) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.30182 (25.83706) + | > current_lr: 0.00006 + | > step_time: 1.39920 (3.47777) + | > loader_time: 0.00430 (0.06335) + + + --> STEP: 179/234 -- GLOBAL_STEP: 52595 + | > loss: -0.35047 (-0.25147) + | > log_mle: -0.59526 (-0.39305) + | > loss_dur: 0.24478 (0.14158) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.41062 (27.01862) + | > current_lr: 0.00006 + | > step_time: 3.88880 (3.49115) + | > loader_time: 0.00300 (0.06324) + + + --> STEP: 184/234 -- GLOBAL_STEP: 52600 + | > loss: -0.33571 (-0.25390) + | > log_mle: -0.55567 (-0.39754) + | > loss_dur: 0.21995 (0.14364) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.14630 (27.92384) + | > current_lr: 0.00006 + | > step_time: 1.92280 (3.50544) + | > loader_time: 0.07350 (0.06878) + + + --> STEP: 189/234 -- GLOBAL_STEP: 52605 + | > loss: -0.33782 (-0.25641) + | > log_mle: -0.55592 (-0.40227) + | > loss_dur: 0.21809 (0.14586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.18700 (29.01116) + | > current_lr: 0.00006 + | > step_time: 6.83130 (3.53518) + | > loader_time: 0.09940 (0.06918) + + + --> STEP: 194/234 -- GLOBAL_STEP: 52610 + | > loss: -0.37770 (-0.25943) + | > log_mle: -0.58615 (-0.40698) + | > loss_dur: 0.20845 (0.14755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.41788 (30.21280) + | > current_lr: 0.00006 + | > step_time: 2.28080 (3.53534) + | > loader_time: 0.00330 (0.06886) + + + --> STEP: 199/234 -- GLOBAL_STEP: 52615 + | > loss: -0.37879 (-0.26205) + | > log_mle: -0.60111 (-0.41144) + | > loss_dur: 0.22232 (0.14939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.11539 (31.21042) + | > current_lr: 0.00006 + | > step_time: 5.89570 (3.52939) + | > loader_time: 0.00600 (0.06803) + + + --> STEP: 204/234 -- GLOBAL_STEP: 52620 + | > loss: -0.38983 (-0.26435) + | > log_mle: -0.63602 (-0.41568) + | > loss_dur: 0.24619 (0.15134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 149.21628 (32.74841) + | > current_lr: 0.00006 + | > step_time: 4.32070 (3.55529) + | > loader_time: 0.01010 (0.06863) + + + --> STEP: 209/234 -- GLOBAL_STEP: 52625 + | > loss: -0.34486 (-0.26633) + | > log_mle: -0.56374 (-0.41949) + | > loss_dur: 0.21888 (0.15316) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.21555 (33.64634) + | > current_lr: 0.00006 + | > step_time: 5.79000 (3.61707) + | > loader_time: 0.19120 (0.07328) + + + --> STEP: 214/234 -- GLOBAL_STEP: 52630 + | > loss: -0.39971 (-0.26944) + | > log_mle: -0.61391 (-0.42452) + | > loss_dur: 0.21420 (0.15508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.90716 (34.50590) + | > current_lr: 0.00006 + | > step_time: 4.99720 (3.63732) + | > loader_time: 0.10030 (0.07384) + + + --> STEP: 219/234 -- GLOBAL_STEP: 52635 + | > loss: -0.47610 (-0.27243) + | > log_mle: -0.70856 (-0.42946) + | > loss_dur: 0.23246 (0.15704) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 103.88955 (35.80263) + | > current_lr: 0.00006 + | > step_time: 2.79660 (3.65756) + | > loader_time: 0.00320 (0.07424) + + + --> STEP: 224/234 -- GLOBAL_STEP: 52640 + | > loss: -0.44221 (-0.27549) + | > log_mle: -0.67367 (-0.43434) + | > loss_dur: 0.23146 (0.15886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.36523 (36.62947) + | > current_lr: 0.00006 + | > step_time: 1.20020 (3.61787) + | > loader_time: 0.00360 (0.07302) + + + --> STEP: 229/234 -- GLOBAL_STEP: 52645 + | > loss: -0.42014 (-0.27882) + | > log_mle: -0.71526 (-0.43994) + | > loss_dur: 0.29513 (0.16112) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 128.16960 (37.90981) + | > current_lr: 0.00006 + | > step_time: 0.26940 (3.54561) + | > loader_time: 0.00430 (0.07151) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32968 (-0.76864) + | > avg_loss: -0.30344 (-0.02573) + | > avg_log_mle: -0.52258 (-0.00578) + | > avg_loss_dur: 0.21913 (-0.01995) + + + > EPOCH: 225/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 08:26:26)  + + --> STEP: 0/234 -- GLOBAL_STEP: 52650 + | > loss: -0.27956 (-0.27956) + | > log_mle: -0.43154 (-0.43154) + | > loss_dur: 0.15197 (0.15197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.77866 (27.77866) + | > current_lr: 0.00006 + | > step_time: 1.49320 (1.49322) + | > loader_time: 8.62940 (8.62945) + + + --> STEP: 5/234 -- GLOBAL_STEP: 52655 + | > loss: -0.25053 (-0.24180) + | > log_mle: -0.35395 (-0.35082) + | > loss_dur: 0.10341 (0.10903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.80251 (20.56251) + | > current_lr: 0.00006 + | > step_time: 4.79480 (3.87618) + | > loader_time: 0.00200 (1.82748) + + + --> STEP: 10/234 -- GLOBAL_STEP: 52660 + | > loss: -0.25157 (-0.25342) + | > log_mle: -0.35143 (-0.35380) + | > loss_dur: 0.09986 (0.10038) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.46041 (17.88388) + | > current_lr: 0.00006 + | > step_time: 5.10150 (4.84873) + | > loader_time: 0.19260 (1.06071) + + + --> STEP: 15/234 -- GLOBAL_STEP: 52665 + | > loss: -0.26900 (-0.25883) + | > log_mle: -0.35721 (-0.35488) + | > loss_dur: 0.08822 (0.09605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.63102 (16.55844) + | > current_lr: 0.00006 + | > step_time: 9.50780 (5.12539) + | > loader_time: 0.00110 (0.72722) + + + --> STEP: 20/234 -- GLOBAL_STEP: 52670 + | > loss: -0.27846 (-0.26213) + | > log_mle: -0.35659 (-0.35372) + | > loss_dur: 0.07813 (0.09159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06223 (15.48115) + | > current_lr: 0.00006 + | > step_time: 2.40300 (4.55126) + | > loader_time: 0.00270 (0.55412) + + + --> STEP: 25/234 -- GLOBAL_STEP: 52675 + | > loss: -0.25320 (-0.26292) + | > log_mle: -0.32965 (-0.35152) + | > loss_dur: 0.07645 (0.08861) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.42767 (15.12849) + | > current_lr: 0.00006 + | > step_time: 7.50560 (4.48908) + | > loader_time: 0.08050 (0.45356) + + + --> STEP: 30/234 -- GLOBAL_STEP: 52680 + | > loss: -0.24240 (-0.26260) + | > log_mle: -0.33256 (-0.34997) + | > loss_dur: 0.09016 (0.08737) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.51431 (14.57073) + | > current_lr: 0.00006 + | > step_time: 2.08190 (4.32034) + | > loader_time: 0.00120 (0.37903) + + + --> STEP: 35/234 -- GLOBAL_STEP: 52685 + | > loss: -0.21371 (-0.25932) + | > log_mle: -0.33186 (-0.34810) + | > loss_dur: 0.11815 (0.08877) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.79843 (14.27098) + | > current_lr: 0.00006 + | > step_time: 1.08390 (3.92675) + | > loader_time: 0.00150 (0.32731) + + + --> STEP: 40/234 -- GLOBAL_STEP: 52690 + | > loss: -0.21491 (-0.25543) + | > log_mle: -0.32144 (-0.34579) + | > loss_dur: 0.10653 (0.09036) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.17453 (14.00430) + | > current_lr: 0.00006 + | > step_time: 0.88780 (3.66657) + | > loader_time: 0.00320 (0.28904) + + + --> STEP: 45/234 -- GLOBAL_STEP: 52695 + | > loss: -0.22325 (-0.25352) + | > log_mle: -0.35213 (-0.34468) + | > loss_dur: 0.12888 (0.09116) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76125 (13.72296) + | > current_lr: 0.00006 + | > step_time: 1.79540 (3.49436) + | > loader_time: 0.00360 (0.26106) + + + --> STEP: 50/234 -- GLOBAL_STEP: 52700 + | > loss: -0.23419 (-0.25149) + | > log_mle: -0.32786 (-0.34339) + | > loss_dur: 0.09367 (0.09190) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.47832 (13.36676) + | > current_lr: 0.00006 + | > step_time: 4.31390 (3.38722) + | > loader_time: 0.09830 (0.24087) + + + --> STEP: 55/234 -- GLOBAL_STEP: 52705 + | > loss: -0.25156 (-0.24992) + | > log_mle: -0.33704 (-0.34236) + | > loss_dur: 0.08548 (0.09244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.27644 (13.15776) + | > current_lr: 0.00006 + | > step_time: 1.89430 (3.23931) + | > loader_time: 0.00210 (0.21924) + + + --> STEP: 60/234 -- GLOBAL_STEP: 52710 + | > loss: -0.22145 (-0.24814) + | > log_mle: -0.34740 (-0.34173) + | > loss_dur: 0.12595 (0.09359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.27840 (12.95892) + | > current_lr: 0.00006 + | > step_time: 1.61710 (3.11286) + | > loader_time: 0.08580 (0.20403) + + + --> STEP: 65/234 -- GLOBAL_STEP: 52715 + | > loss: -0.23296 (-0.24607) + | > log_mle: -0.33215 (-0.34130) + | > loss_dur: 0.09919 (0.09523) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.76853 (12.82415) + | > current_lr: 0.00006 + | > step_time: 1.09880 (2.98845) + | > loader_time: 0.08350 (0.19235) + + + --> STEP: 70/234 -- GLOBAL_STEP: 52720 + | > loss: -0.18166 (-0.24356) + | > log_mle: -0.31577 (-0.34019) + | > loss_dur: 0.13411 (0.09663) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.00989 (12.73189) + | > current_lr: 0.00006 + | > step_time: 2.08900 (2.88487) + | > loader_time: 0.00250 (0.18001) + + + --> STEP: 75/234 -- GLOBAL_STEP: 52725 + | > loss: -0.20096 (-0.24077) + | > log_mle: -0.33259 (-0.33963) + | > loss_dur: 0.13164 (0.09886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.97748 (12.97258) + | > current_lr: 0.00006 + | > step_time: 1.49330 (2.82579) + | > loader_time: 0.00250 (0.16822) + + + --> STEP: 80/234 -- GLOBAL_STEP: 52730 + | > loss: -0.21034 (-0.23883) + | > log_mle: -0.31845 (-0.33880) + | > loss_dur: 0.10811 (0.09997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.90302 (12.87223) + | > current_lr: 0.00006 + | > step_time: 2.49860 (2.79045) + | > loader_time: 0.07730 (0.15983) + + + --> STEP: 85/234 -- GLOBAL_STEP: 52735 + | > loss: -0.22070 (-0.23720) + | > log_mle: -0.33054 (-0.33848) + | > loss_dur: 0.10984 (0.10127) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.52789 (12.96736) + | > current_lr: 0.00006 + | > step_time: 2.30710 (2.74164) + | > loader_time: 0.00240 (0.15255) + + + --> STEP: 90/234 -- GLOBAL_STEP: 52740 + | > loss: -0.20427 (-0.23612) + | > log_mle: -0.35198 (-0.33921) + | > loss_dur: 0.14771 (0.10309) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.68542 (13.07878) + | > current_lr: 0.00006 + | > step_time: 3.00050 (2.71592) + | > loader_time: 0.00380 (0.14705) + + + --> STEP: 95/234 -- GLOBAL_STEP: 52745 + | > loss: -0.26906 (-0.23617) + | > log_mle: -0.43901 (-0.34177) + | > loss_dur: 0.16995 (0.10560) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.49308 (13.67355) + | > current_lr: 0.00006 + | > step_time: 1.98530 (2.67912) + | > loader_time: 0.00340 (0.14132) + + + --> STEP: 100/234 -- GLOBAL_STEP: 52750 + | > loss: -0.22603 (-0.23538) + | > log_mle: -0.36427 (-0.34270) + | > loss_dur: 0.13824 (0.10732) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.43091 (13.92907) + | > current_lr: 0.00006 + | > step_time: 1.91760 (2.67745) + | > loader_time: 0.08670 (0.13688) + + + --> STEP: 105/234 -- GLOBAL_STEP: 52755 + | > loss: -0.21572 (-0.23511) + | > log_mle: -0.34441 (-0.34477) + | > loss_dur: 0.12868 (0.10967) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.17212 (14.42130) + | > current_lr: 0.00006 + | > step_time: 1.60200 (2.66711) + | > loader_time: 0.00290 (0.13227) + + + --> STEP: 110/234 -- GLOBAL_STEP: 52760 + | > loss: -0.21854 (-0.23425) + | > log_mle: -0.36689 (-0.34643) + | > loss_dur: 0.14835 (0.11218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.03718 (14.92652) + | > current_lr: 0.00006 + | > step_time: 2.32980 (2.67957) + | > loader_time: 0.08330 (0.12881) + + + --> STEP: 115/234 -- GLOBAL_STEP: 52765 + | > loss: -0.21254 (-0.23431) + | > log_mle: -0.38700 (-0.34897) + | > loss_dur: 0.17446 (0.11466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.03613 (15.72267) + | > current_lr: 0.00006 + | > step_time: 1.71250 (2.65042) + | > loader_time: 0.09090 (0.12413) + + + --> STEP: 120/234 -- GLOBAL_STEP: 52770 + | > loss: -0.27000 (-0.23416) + | > log_mle: -0.43310 (-0.35105) + | > loss_dur: 0.16309 (0.11689) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.93756 (16.26597) + | > current_lr: 0.00006 + | > step_time: 1.90690 (2.70269) + | > loader_time: 0.00220 (0.12062) + + + --> STEP: 125/234 -- GLOBAL_STEP: 52775 + | > loss: -0.24872 (-0.23338) + | > log_mle: -0.41896 (-0.35184) + | > loss_dur: 0.17023 (0.11846) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.54320 (16.58555) + | > current_lr: 0.00006 + | > step_time: 0.87970 (2.65571) + | > loader_time: 0.00230 (0.11665) + + + --> STEP: 130/234 -- GLOBAL_STEP: 52780 + | > loss: -0.25884 (-0.23401) + | > log_mle: -0.43497 (-0.35461) + | > loss_dur: 0.17612 (0.12060) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.09487 (17.24175) + | > current_lr: 0.00006 + | > step_time: 1.60450 (2.62978) + | > loader_time: 0.00320 (0.11230) + + + --> STEP: 135/234 -- GLOBAL_STEP: 52785 + | > loss: -0.21195 (-0.23487) + | > log_mle: -0.36383 (-0.35751) + | > loss_dur: 0.15188 (0.12264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.21316 (17.83430) + | > current_lr: 0.00006 + | > step_time: 1.20390 (2.61610) + | > loader_time: 0.00270 (0.10902) + + + --> STEP: 140/234 -- GLOBAL_STEP: 52790 + | > loss: -0.21664 (-0.23597) + | > log_mle: -0.39242 (-0.36089) + | > loss_dur: 0.17578 (0.12492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.72902 (18.85643) + | > current_lr: 0.00006 + | > step_time: 1.81540 (2.60190) + | > loader_time: 0.08450 (0.10652) + + + --> STEP: 145/234 -- GLOBAL_STEP: 52795 + | > loss: -0.30321 (-0.23728) + | > log_mle: -0.49425 (-0.36485) + | > loss_dur: 0.19104 (0.12757) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.88594 (19.94355) + | > current_lr: 0.00006 + | > step_time: 2.81670 (2.64858) + | > loader_time: 0.18460 (0.10674) + + + --> STEP: 150/234 -- GLOBAL_STEP: 52800 + | > loss: -0.28606 (-0.23911) + | > log_mle: -0.48588 (-0.36865) + | > loss_dur: 0.19981 (0.12954) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.68048 (20.73753) + | > current_lr: 0.00006 + | > step_time: 2.58410 (2.70681) + | > loader_time: 0.00350 (0.10517) + + + --> STEP: 155/234 -- GLOBAL_STEP: 52805 + | > loss: -0.35080 (-0.24196) + | > log_mle: -0.55668 (-0.37367) + | > loss_dur: 0.20588 (0.13171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.76219 (21.80675) + | > current_lr: 0.00006 + | > step_time: 3.09800 (2.69630) + | > loader_time: 0.08690 (0.10351) + + + --> STEP: 160/234 -- GLOBAL_STEP: 52810 + | > loss: -0.33377 (-0.24382) + | > log_mle: -0.54568 (-0.37775) + | > loss_dur: 0.21191 (0.13393) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.35895 (22.98013) + | > current_lr: 0.00006 + | > step_time: 1.69520 (2.78453) + | > loader_time: 0.00620 (0.10347) + + + --> STEP: 165/234 -- GLOBAL_STEP: 52815 + | > loss: -0.33496 (-0.24582) + | > log_mle: -0.54374 (-0.38168) + | > loss_dur: 0.20878 (0.13586) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.02201 (24.03564) + | > current_lr: 0.00006 + | > step_time: 0.99470 (2.77633) + | > loader_time: 0.00300 (0.10050) + + + --> STEP: 170/234 -- GLOBAL_STEP: 52820 + | > loss: -0.34886 (-0.24810) + | > log_mle: -0.58222 (-0.38616) + | > loss_dur: 0.23336 (0.13806) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.65293 (25.07457) + | > current_lr: 0.00006 + | > step_time: 11.68960 (2.83467) + | > loader_time: 0.10150 (0.09917) + + + --> STEP: 175/234 -- GLOBAL_STEP: 52825 + | > loss: -0.32285 (-0.25102) + | > log_mle: -0.55477 (-0.39128) + | > loss_dur: 0.23193 (0.14026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.82142 (26.24895) + | > current_lr: 0.00006 + | > step_time: 2.42410 (2.85568) + | > loader_time: 0.08200 (0.09971) + + + --> STEP: 180/234 -- GLOBAL_STEP: 52830 + | > loss: -0.35580 (-0.25367) + | > log_mle: -0.55833 (-0.39611) + | > loss_dur: 0.20253 (0.14244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.17265 (27.22359) + | > current_lr: 0.00006 + | > step_time: 2.59560 (2.84168) + | > loader_time: 0.08950 (0.09792) + + + --> STEP: 185/234 -- GLOBAL_STEP: 52835 + | > loss: -0.36589 (-0.25611) + | > log_mle: -0.58992 (-0.40063) + | > loss_dur: 0.22403 (0.14452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.25352 (28.22199) + | > current_lr: 0.00006 + | > step_time: 2.70770 (2.85793) + | > loader_time: 0.17460 (0.09674) + + + --> STEP: 190/234 -- GLOBAL_STEP: 52840 + | > loss: -0.36294 (-0.25871) + | > log_mle: -0.57088 (-0.40514) + | > loss_dur: 0.20793 (0.14643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.71455 (29.20081) + | > current_lr: 0.00006 + | > step_time: 4.19900 (2.85329) + | > loader_time: 0.09120 (0.09569) + + + --> STEP: 195/234 -- GLOBAL_STEP: 52845 + | > loss: -0.36291 (-0.26161) + | > log_mle: -0.59010 (-0.40979) + | > loss_dur: 0.22719 (0.14818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.29317 (30.37649) + | > current_lr: 0.00006 + | > step_time: 5.39660 (2.90522) + | > loader_time: 0.09640 (0.09587) + + + --> STEP: 200/234 -- GLOBAL_STEP: 52850 + | > loss: -0.35686 (-0.26407) + | > log_mle: -0.60027 (-0.41414) + | > loss_dur: 0.24340 (0.15006) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.37322 (31.55732) + | > current_lr: 0.00006 + | > step_time: 3.59550 (2.98104) + | > loader_time: 0.09730 (0.09663) + + + --> STEP: 205/234 -- GLOBAL_STEP: 52855 + | > loss: -0.36331 (-0.26648) + | > log_mle: -0.58366 (-0.41833) + | > loss_dur: 0.22035 (0.15185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.90068 (32.71494) + | > current_lr: 0.00006 + | > step_time: 2.39980 (3.04693) + | > loader_time: 0.50890 (0.09908) + + + --> STEP: 210/234 -- GLOBAL_STEP: 52860 + | > loss: -0.41363 (-0.26927) + | > log_mle: -0.64990 (-0.42304) + | > loss_dur: 0.23626 (0.15377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.99082 (33.89344) + | > current_lr: 0.00006 + | > step_time: 6.79180 (3.11092) + | > loader_time: 0.29650 (0.09911) + + + --> STEP: 215/234 -- GLOBAL_STEP: 52865 + | > loss: -0.37749 (-0.27256) + | > log_mle: -0.61217 (-0.42812) + | > loss_dur: 0.23468 (0.15556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.79661 (35.08073) + | > current_lr: 0.00006 + | > step_time: 6.80050 (3.20185) + | > loader_time: 0.09420 (0.10233) + + + --> STEP: 220/234 -- GLOBAL_STEP: 52870 + | > loss: -0.40709 (-0.27580) + | > log_mle: -0.64451 (-0.43327) + | > loss_dur: 0.23743 (0.15747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.45242 (36.46993) + | > current_lr: 0.00006 + | > step_time: 3.09460 (3.26089) + | > loader_time: 0.00910 (0.10134) + + + --> STEP: 225/234 -- GLOBAL_STEP: 52875 + | > loss: -0.45892 (-0.27884) + | > log_mle: -0.70770 (-0.43816) + | > loss_dur: 0.24879 (0.15932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.70590 (37.39058) + | > current_lr: 0.00006 + | > step_time: 0.24260 (3.19705) + | > loader_time: 0.00430 (0.09959) + + + --> STEP: 230/234 -- GLOBAL_STEP: 52880 + | > loss: -0.45340 (-0.28180) + | > log_mle: -0.76941 (-0.44371) + | > loss_dur: 0.31601 (0.16191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.24996 (38.40756) + | > current_lr: 0.00006 + | > step_time: 0.25890 (3.13307) + | > loader_time: 0.00480 (0.09750) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17494 (-0.15474) + | > avg_loss: -0.30832 (-0.00488) + | > avg_log_mle: -0.52402 (-0.00144) + | > avg_loss_dur: 0.21570 (-0.00343) + + + > EPOCH: 226/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 08:39:53)  + + --> STEP: 1/234 -- GLOBAL_STEP: 52885 + | > loss: -0.24001 (-0.24001) + | > log_mle: -0.34748 (-0.34748) + | > loss_dur: 0.10747 (0.10747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.86553 (28.86553) + | > current_lr: 0.00006 + | > step_time: 4.99620 (4.99623) + | > loader_time: 0.09470 (0.09470) + + + --> STEP: 6/234 -- GLOBAL_STEP: 52890 + | > loss: -0.26842 (-0.24229) + | > log_mle: -0.34753 (-0.35077) + | > loss_dur: 0.07910 (0.10848) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.84278 (22.04153) + | > current_lr: 0.00006 + | > step_time: 4.40000 (5.33248) + | > loader_time: 0.00160 (0.08298) + + + --> STEP: 11/234 -- GLOBAL_STEP: 52895 + | > loss: -0.29517 (-0.25623) + | > log_mle: -0.36340 (-0.35614) + | > loss_dur: 0.06824 (0.09991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.51717 (19.11911) + | > current_lr: 0.00006 + | > step_time: 2.79970 (3.86904) + | > loader_time: 0.20420 (0.07320) + + + --> STEP: 16/234 -- GLOBAL_STEP: 52900 + | > loss: -0.28827 (-0.26202) + | > log_mle: -0.35906 (-0.35651) + | > loss_dur: 0.07079 (0.09449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.50845 (17.59176) + | > current_lr: 0.00006 + | > step_time: 1.69200 (3.24045) + | > loader_time: 0.00110 (0.05104) + + + --> STEP: 21/234 -- GLOBAL_STEP: 52905 + | > loss: -0.25700 (-0.26364) + | > log_mle: -0.33401 (-0.35369) + | > loss_dur: 0.07701 (0.09005) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.26495 (16.32852) + | > current_lr: 0.00006 + | > step_time: 3.81150 (3.67174) + | > loader_time: 0.69080 (0.09681) + + + --> STEP: 26/234 -- GLOBAL_STEP: 52910 + | > loss: -0.26235 (-0.26438) + | > log_mle: -0.35024 (-0.35262) + | > loss_dur: 0.08789 (0.08824) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.05669 (15.18910) + | > current_lr: 0.00006 + | > step_time: 4.24250 (3.62941) + | > loader_time: 0.09400 (0.08538) + + + --> STEP: 31/234 -- GLOBAL_STEP: 52915 + | > loss: -0.22558 (-0.26255) + | > log_mle: -0.33147 (-0.35066) + | > loss_dur: 0.10589 (0.08811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.80099 (14.51129) + | > current_lr: 0.00006 + | > step_time: 2.20350 (3.37191) + | > loader_time: 0.08900 (0.07736) + + + --> STEP: 36/234 -- GLOBAL_STEP: 52920 + | > loss: -0.22766 (-0.25956) + | > log_mle: -0.32983 (-0.34894) + | > loss_dur: 0.10217 (0.08938) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.17488 (14.18923) + | > current_lr: 0.00006 + | > step_time: 4.00570 (3.37950) + | > loader_time: 0.09190 (0.07420) + + + --> STEP: 41/234 -- GLOBAL_STEP: 52925 + | > loss: -0.25562 (-0.25728) + | > log_mle: -0.34262 (-0.34739) + | > loss_dur: 0.08700 (0.09010) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.27899 (13.73195) + | > current_lr: 0.00006 + | > step_time: 1.00730 (3.17241) + | > loader_time: 0.00780 (0.06558) + + + --> STEP: 46/234 -- GLOBAL_STEP: 52930 + | > loss: -0.22407 (-0.25401) + | > log_mle: -0.33331 (-0.34574) + | > loss_dur: 0.10925 (0.09174) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.92417 (13.90078) + | > current_lr: 0.00006 + | > step_time: 1.31500 (2.98639) + | > loader_time: 0.08620 (0.06226) + + + --> STEP: 51/234 -- GLOBAL_STEP: 52935 + | > loss: -0.22602 (-0.25227) + | > log_mle: -0.32737 (-0.34447) + | > loss_dur: 0.10136 (0.09219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.35633 (13.44400) + | > current_lr: 0.00006 + | > step_time: 1.96700 (2.85858) + | > loader_time: 0.00170 (0.05785) + + + --> STEP: 56/234 -- GLOBAL_STEP: 52940 + | > loss: -0.21617 (-0.25047) + | > log_mle: -0.33630 (-0.34378) + | > loss_dur: 0.12013 (0.09332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.08622 (13.14817) + | > current_lr: 0.00006 + | > step_time: 2.06810 (2.82812) + | > loader_time: 0.00220 (0.05795) + + + --> STEP: 61/234 -- GLOBAL_STEP: 52945 + | > loss: -0.22264 (-0.24860) + | > log_mle: -0.33719 (-0.34331) + | > loss_dur: 0.11455 (0.09471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.73900 (12.92463) + | > current_lr: 0.00006 + | > step_time: 1.28150 (2.73941) + | > loader_time: 0.00210 (0.05346) + + + --> STEP: 66/234 -- GLOBAL_STEP: 52950 + | > loss: -0.23219 (-0.24654) + | > log_mle: -0.32790 (-0.34273) + | > loss_dur: 0.09571 (0.09619) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.31768 (12.83523) + | > current_lr: 0.00006 + | > step_time: 1.52140 (2.66413) + | > loader_time: 0.00310 (0.04962) + + + --> STEP: 71/234 -- GLOBAL_STEP: 52955 + | > loss: -0.20630 (-0.24390) + | > log_mle: -0.35470 (-0.34198) + | > loss_dur: 0.14840 (0.09808) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.86478 (12.91377) + | > current_lr: 0.00006 + | > step_time: 2.10230 (2.63172) + | > loader_time: 0.00230 (0.04629) + + + --> STEP: 76/234 -- GLOBAL_STEP: 52960 + | > loss: -0.20816 (-0.24140) + | > log_mle: -0.33670 (-0.34133) + | > loss_dur: 0.12854 (0.09993) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.74605 (12.92252) + | > current_lr: 0.00006 + | > step_time: 1.91780 (2.62637) + | > loader_time: 0.08330 (0.04689) + + + --> STEP: 81/234 -- GLOBAL_STEP: 52965 + | > loss: -0.22294 (-0.23972) + | > log_mle: -0.34275 (-0.34064) + | > loss_dur: 0.11981 (0.10092) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.45888 (12.88169) + | > current_lr: 0.00006 + | > step_time: 3.59900 (2.59379) + | > loader_time: 0.08880 (0.04524) + + + --> STEP: 86/234 -- GLOBAL_STEP: 52970 + | > loss: -0.22046 (-0.23818) + | > log_mle: -0.34197 (-0.34029) + | > loss_dur: 0.12151 (0.10211) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.98468 (12.91335) + | > current_lr: 0.00006 + | > step_time: 3.03960 (2.58145) + | > loader_time: 0.08600 (0.04490) + + + --> STEP: 91/234 -- GLOBAL_STEP: 52975 + | > loss: -0.20903 (-0.23704) + | > log_mle: -0.35166 (-0.34107) + | > loss_dur: 0.14263 (0.10402) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.73161 (13.07631) + | > current_lr: 0.00006 + | > step_time: 2.02680 (2.54041) + | > loader_time: 0.09610 (0.04363) + + + --> STEP: 96/234 -- GLOBAL_STEP: 52980 + | > loss: -0.21701 (-0.23712) + | > log_mle: -0.33905 (-0.34332) + | > loss_dur: 0.12204 (0.10620) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.47309 (13.52595) + | > current_lr: 0.00006 + | > step_time: 2.01710 (2.53725) + | > loader_time: 0.00220 (0.04319) + + + --> STEP: 101/234 -- GLOBAL_STEP: 52985 + | > loss: -0.23027 (-0.23665) + | > log_mle: -0.39020 (-0.34469) + | > loss_dur: 0.15993 (0.10804) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.09145 (13.88996) + | > current_lr: 0.00006 + | > step_time: 2.00240 (2.53487) + | > loader_time: 0.00220 (0.04222) + + + --> STEP: 106/234 -- GLOBAL_STEP: 52990 + | > loss: -0.19462 (-0.23596) + | > log_mle: -0.38116 (-0.34644) + | > loss_dur: 0.18654 (0.11048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.23232 (14.54545) + | > current_lr: 0.00006 + | > step_time: 2.40550 (2.52100) + | > loader_time: 0.00320 (0.04036) + + + --> STEP: 111/234 -- GLOBAL_STEP: 52995 + | > loss: -0.24487 (-0.23527) + | > log_mle: -0.43867 (-0.34826) + | > loss_dur: 0.19380 (0.11299) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.09204 (15.05903) + | > current_lr: 0.00006 + | > step_time: 2.60910 (2.52454) + | > loader_time: 0.00250 (0.04220) + + + --> STEP: 116/234 -- GLOBAL_STEP: 53000 + | > loss: -0.22190 (-0.23477) + | > log_mle: -0.40505 (-0.35023) + | > loss_dur: 0.18316 (0.11546) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.78243 (15.60265) + | > current_lr: 0.00006 + | > step_time: 1.80080 (2.55029) + | > loader_time: 0.00310 (0.04056) + + + --> STEP: 121/234 -- GLOBAL_STEP: 53005 + | > loss: -0.18103 (-0.23435) + | > log_mle: -0.32018 (-0.35155) + | > loss_dur: 0.13916 (0.11721) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.21735 (15.83524) + | > current_lr: 0.00006 + | > step_time: 1.58820 (2.52923) + | > loader_time: 0.00270 (0.04214) + + + --> STEP: 126/234 -- GLOBAL_STEP: 53010 + | > loss: -0.25956 (-0.23435) + | > log_mle: -0.44992 (-0.35342) + | > loss_dur: 0.19036 (0.11907) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.62968 (16.48963) + | > current_lr: 0.00006 + | > step_time: 1.09700 (2.54260) + | > loader_time: 0.00290 (0.04272) + + + --> STEP: 131/234 -- GLOBAL_STEP: 53015 + | > loss: -0.31329 (-0.23525) + | > log_mle: -0.50225 (-0.35648) + | > loss_dur: 0.18896 (0.12123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.48199 (17.38309) + | > current_lr: 0.00006 + | > step_time: 1.58000 (2.54245) + | > loader_time: 0.00260 (0.04329) + + + --> STEP: 136/234 -- GLOBAL_STEP: 53020 + | > loss: -0.34294 (-0.23640) + | > log_mle: -0.55147 (-0.35967) + | > loss_dur: 0.20854 (0.12327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.53616 (18.17959) + | > current_lr: 0.00006 + | > step_time: 1.50850 (2.51325) + | > loader_time: 0.00530 (0.04246) + + + --> STEP: 141/234 -- GLOBAL_STEP: 53025 + | > loss: -0.27517 (-0.23685) + | > log_mle: -0.44945 (-0.36228) + | > loss_dur: 0.17427 (0.12544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.72215 (18.81453) + | > current_lr: 0.00006 + | > step_time: 1.48410 (2.54873) + | > loader_time: 0.00320 (0.04177) + + + --> STEP: 146/234 -- GLOBAL_STEP: 53030 + | > loss: -0.27079 (-0.23804) + | > log_mle: -0.46966 (-0.36609) + | > loss_dur: 0.19888 (0.12805) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.06962 (20.05698) + | > current_lr: 0.00006 + | > step_time: 1.50850 (2.52265) + | > loader_time: 0.08700 (0.04207) + + + --> STEP: 151/234 -- GLOBAL_STEP: 53035 + | > loss: -0.27388 (-0.23919) + | > log_mle: -0.45552 (-0.36916) + | > loss_dur: 0.18164 (0.12997) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.71015 (20.59573) + | > current_lr: 0.00006 + | > step_time: 1.09040 (2.49761) + | > loader_time: 0.00320 (0.04205) + + + --> STEP: 156/234 -- GLOBAL_STEP: 53040 + | > loss: -0.30351 (-0.24141) + | > log_mle: -0.49934 (-0.37371) + | > loss_dur: 0.19582 (0.13230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.08485 (21.76329) + | > current_lr: 0.00006 + | > step_time: 3.80310 (2.50924) + | > loader_time: 0.00310 (0.04192) + + + --> STEP: 161/234 -- GLOBAL_STEP: 53045 + | > loss: -0.34660 (-0.24330) + | > log_mle: -0.53161 (-0.37783) + | > loss_dur: 0.18501 (0.13453) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.50626 (22.43340) + | > current_lr: 0.00006 + | > step_time: 2.20400 (2.50227) + | > loader_time: 0.07580 (0.04228) + + + --> STEP: 166/234 -- GLOBAL_STEP: 53050 + | > loss: -0.28367 (-0.24471) + | > log_mle: -0.46789 (-0.38116) + | > loss_dur: 0.18422 (0.13645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.87930 (23.28746) + | > current_lr: 0.00006 + | > step_time: 2.89380 (2.49076) + | > loader_time: 0.00340 (0.04217) + + + --> STEP: 171/234 -- GLOBAL_STEP: 53055 + | > loss: -0.37633 (-0.24756) + | > log_mle: -0.58003 (-0.38623) + | > loss_dur: 0.20370 (0.13866) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.89546 (24.34983) + | > current_lr: 0.00006 + | > step_time: 3.39280 (2.53136) + | > loader_time: 0.10360 (0.04377) + + + --> STEP: 176/234 -- GLOBAL_STEP: 53060 + | > loss: -0.35028 (-0.25049) + | > log_mle: -0.55882 (-0.39135) + | > loss_dur: 0.20854 (0.14086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.50007 (25.53970) + | > current_lr: 0.00006 + | > step_time: 2.09780 (2.51800) + | > loader_time: 0.08720 (0.04458) + + + --> STEP: 181/234 -- GLOBAL_STEP: 53065 + | > loss: -0.28656 (-0.25276) + | > log_mle: -0.48936 (-0.39589) + | > loss_dur: 0.20280 (0.14313) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.91953 (26.60914) + | > current_lr: 0.00006 + | > step_time: 3.99810 (2.54133) + | > loader_time: 0.00550 (0.04440) + + + --> STEP: 186/234 -- GLOBAL_STEP: 53070 + | > loss: -0.28443 (-0.25507) + | > log_mle: -0.52800 (-0.40053) + | > loss_dur: 0.24357 (0.14546) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.67349 (27.71006) + | > current_lr: 0.00006 + | > step_time: 10.28760 (2.60363) + | > loader_time: 0.30750 (0.04694) + + + --> STEP: 191/234 -- GLOBAL_STEP: 53075 + | > loss: -0.34767 (-0.25771) + | > log_mle: -0.55243 (-0.40503) + | > loss_dur: 0.20476 (0.14732) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.70057 (28.59892) + | > current_lr: 0.00006 + | > step_time: 5.91230 (2.68736) + | > loader_time: 0.09620 (0.04978) + + + --> STEP: 196/234 -- GLOBAL_STEP: 53080 + | > loss: -0.31099 (-0.26050) + | > log_mle: -0.53815 (-0.40964) + | > loss_dur: 0.22717 (0.14914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.97559 (29.59138) + | > current_lr: 0.00006 + | > step_time: 8.99700 (2.77142) + | > loader_time: 0.19720 (0.05252) + + + --> STEP: 201/234 -- GLOBAL_STEP: 53085 + | > loss: -0.27364 (-0.26262) + | > log_mle: -0.49783 (-0.41362) + | > loss_dur: 0.22418 (0.15099) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.46064 (30.60965) + | > current_lr: 0.00006 + | > step_time: 2.69920 (2.77657) + | > loader_time: 0.00400 (0.05380) + + + --> STEP: 206/234 -- GLOBAL_STEP: 53090 + | > loss: -0.39095 (-0.26548) + | > log_mle: -0.61029 (-0.41824) + | > loss_dur: 0.21934 (0.15276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.43797 (31.56997) + | > current_lr: 0.00006 + | > step_time: 2.09520 (2.81707) + | > loader_time: 0.00310 (0.05342) + + + --> STEP: 211/234 -- GLOBAL_STEP: 53095 + | > loss: -0.43248 (-0.26881) + | > log_mle: -0.67527 (-0.42350) + | > loss_dur: 0.24278 (0.15469) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.90922 (32.80199) + | > current_lr: 0.00006 + | > step_time: 6.69940 (2.87305) + | > loader_time: 0.09950 (0.05450) + + + --> STEP: 216/234 -- GLOBAL_STEP: 53100 + | > loss: -0.40390 (-0.27181) + | > log_mle: -0.65699 (-0.42837) + | > loss_dur: 0.25309 (0.15656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 133.68074 (34.10815) + | > current_lr: 0.00006 + | > step_time: 13.00850 (2.98218) + | > loader_time: 0.18860 (0.05915) + + + --> STEP: 221/234 -- GLOBAL_STEP: 53105 + | > loss: -0.37693 (-0.27500) + | > log_mle: -0.58974 (-0.43330) + | > loss_dur: 0.21280 (0.15830) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.20053 (35.25331) + | > current_lr: 0.00006 + | > step_time: 2.19730 (3.03372) + | > loader_time: 0.00400 (0.05880) + + + --> STEP: 226/234 -- GLOBAL_STEP: 53110 + | > loss: -0.43992 (-0.27861) + | > log_mle: -0.68223 (-0.43885) + | > loss_dur: 0.24231 (0.16024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 118.53218 (36.73464) + | > current_lr: 0.00006 + | > step_time: 0.23010 (2.97529) + | > loader_time: 0.00330 (0.05799) + + + --> STEP: 231/234 -- GLOBAL_STEP: 53115 + | > loss: -0.34399 (-0.28122) + | > log_mle: -0.72982 (-0.44461) + | > loss_dur: 0.38583 (0.16340) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 157.36177 (38.25824) + | > current_lr: 0.00006 + | > step_time: 0.27530 (2.91634) + | > loader_time: 0.00800 (0.05686) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27668 (+0.10174) + | > avg_loss: -0.28942 (+0.01890) + | > avg_log_mle: -0.51422 (+0.00980) + | > avg_loss_dur: 0.22481 (+0.00911) + + + > EPOCH: 227/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 08:52:34)  + + --> STEP: 2/234 -- GLOBAL_STEP: 53120 + | > loss: -0.28229 (-0.26657) + | > log_mle: -0.36654 (-0.35701) + | > loss_dur: 0.08425 (0.09044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.28786 (17.45576) + | > current_lr: 0.00006 + | > step_time: 19.73020 (13.76175) + | > loader_time: 0.01220 (1.75557) + + + --> STEP: 7/234 -- GLOBAL_STEP: 53125 + | > loss: -0.27504 (-0.25492) + | > log_mle: -0.35839 (-0.35311) + | > loss_dur: 0.08335 (0.09820) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.61803 (18.70587) + | > current_lr: 0.00006 + | > step_time: 0.81920 (5.11579) + | > loader_time: 0.00110 (0.51774) + + + --> STEP: 12/234 -- GLOBAL_STEP: 53130 + | > loss: -0.25920 (-0.26096) + | > log_mle: -0.35034 (-0.35543) + | > loss_dur: 0.09113 (0.09447) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.86818 (18.24152) + | > current_lr: 0.00006 + | > step_time: 2.90880 (3.53735) + | > loader_time: 0.07580 (0.30907) + + + --> STEP: 17/234 -- GLOBAL_STEP: 53135 + | > loss: -0.28133 (-0.26712) + | > log_mle: -0.34854 (-0.35673) + | > loss_dur: 0.06720 (0.08961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.49241 (16.15900) + | > current_lr: 0.00006 + | > step_time: 0.91650 (3.05794) + | > loader_time: 0.00190 (0.22883) + + + --> STEP: 22/234 -- GLOBAL_STEP: 53140 + | > loss: -0.26647 (-0.26751) + | > log_mle: -0.35711 (-0.35534) + | > loss_dur: 0.09064 (0.08783) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.41787 (14.78243) + | > current_lr: 0.00006 + | > step_time: 1.25690 (3.09342) + | > loader_time: 0.00680 (0.18177) + + + --> STEP: 27/234 -- GLOBAL_STEP: 53145 + | > loss: -0.26765 (-0.26821) + | > log_mle: -0.35075 (-0.35436) + | > loss_dur: 0.08310 (0.08616) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.35293 (13.97936) + | > current_lr: 0.00006 + | > step_time: 1.37190 (2.77450) + | > loader_time: 0.00290 (0.14859) + + + --> STEP: 32/234 -- GLOBAL_STEP: 53150 + | > loss: -0.27868 (-0.26721) + | > log_mle: -0.35524 (-0.35329) + | > loss_dur: 0.07657 (0.08608) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.20715 (13.44165) + | > current_lr: 0.00006 + | > step_time: 3.89440 (2.66066) + | > loader_time: 0.00260 (0.12825) + + + --> STEP: 37/234 -- GLOBAL_STEP: 53155 + | > loss: -0.24757 (-0.26413) + | > log_mle: -0.32919 (-0.35119) + | > loss_dur: 0.08162 (0.08707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.16649 (13.22310) + | > current_lr: 0.00006 + | > step_time: 1.79940 (2.71531) + | > loader_time: 0.08350 (0.11968) + + + --> STEP: 42/234 -- GLOBAL_STEP: 53160 + | > loss: -0.22189 (-0.26093) + | > log_mle: -0.32323 (-0.34973) + | > loss_dur: 0.10135 (0.08880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.56283 (12.98090) + | > current_lr: 0.00006 + | > step_time: 0.65240 (2.50704) + | > loader_time: 0.00200 (0.10609) + + + --> STEP: 47/234 -- GLOBAL_STEP: 53165 + | > loss: -0.22124 (-0.25808) + | > log_mle: -0.33297 (-0.34861) + | > loss_dur: 0.11173 (0.09053) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.74157 (13.09796) + | > current_lr: 0.00006 + | > step_time: 1.16240 (2.41955) + | > loader_time: 0.00210 (0.09509) + + + --> STEP: 52/234 -- GLOBAL_STEP: 53170 + | > loss: -0.21193 (-0.25602) + | > log_mle: -0.33079 (-0.34729) + | > loss_dur: 0.11886 (0.09126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.57072 (12.89476) + | > current_lr: 0.00006 + | > step_time: 1.58730 (2.36128) + | > loader_time: 0.00230 (0.08615) + + + --> STEP: 57/234 -- GLOBAL_STEP: 53175 + | > loss: -0.21233 (-0.25418) + | > log_mle: -0.32294 (-0.34649) + | > loss_dur: 0.11061 (0.09231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47333 (12.66868) + | > current_lr: 0.00006 + | > step_time: 1.61660 (2.30656) + | > loader_time: 0.00200 (0.08217) + + + --> STEP: 62/234 -- GLOBAL_STEP: 53180 + | > loss: -0.18197 (-0.25182) + | > log_mle: -0.35634 (-0.34646) + | > loss_dur: 0.17438 (0.09464) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.93283 (12.86371) + | > current_lr: 0.00006 + | > step_time: 2.10680 (2.27659) + | > loader_time: 0.00290 (0.07573) + + + --> STEP: 67/234 -- GLOBAL_STEP: 53185 + | > loss: -0.22318 (-0.25005) + | > log_mle: -0.34155 (-0.34544) + | > loss_dur: 0.11837 (0.09539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.95769 (12.78409) + | > current_lr: 0.00006 + | > step_time: 1.80980 (2.27105) + | > loader_time: 0.07860 (0.07414) + + + --> STEP: 72/234 -- GLOBAL_STEP: 53190 + | > loss: -0.21972 (-0.24707) + | > log_mle: -0.32965 (-0.34431) + | > loss_dur: 0.10994 (0.09723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.48735 (12.75653) + | > current_lr: 0.00006 + | > step_time: 4.11520 (2.27128) + | > loader_time: 0.08670 (0.07039) + + + --> STEP: 77/234 -- GLOBAL_STEP: 53195 + | > loss: -0.21775 (-0.24441) + | > log_mle: -0.33527 (-0.34366) + | > loss_dur: 0.11752 (0.09924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.37861 (12.80914) + | > current_lr: 0.00006 + | > step_time: 2.72750 (2.30371) + | > loader_time: 0.00260 (0.06819) + + + --> STEP: 82/234 -- GLOBAL_STEP: 53200 + | > loss: -0.21643 (-0.24281) + | > log_mle: -0.32860 (-0.34288) + | > loss_dur: 0.11217 (0.10007) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.83074 (12.77889) + | > current_lr: 0.00006 + | > step_time: 1.59470 (2.27408) + | > loader_time: 0.00310 (0.06637) + + + --> STEP: 87/234 -- GLOBAL_STEP: 53205 + | > loss: -0.20916 (-0.24122) + | > log_mle: -0.33463 (-0.34247) + | > loss_dur: 0.12548 (0.10125) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.40157 (12.90825) + | > current_lr: 0.00006 + | > step_time: 2.10940 (2.25359) + | > loader_time: 0.00310 (0.06376) + + + --> STEP: 92/234 -- GLOBAL_STEP: 53210 + | > loss: -0.23848 (-0.24050) + | > log_mle: -0.37802 (-0.34366) + | > loss_dur: 0.13954 (0.10316) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.93707 (13.16428) + | > current_lr: 0.00006 + | > step_time: 1.11470 (2.22488) + | > loader_time: 0.08940 (0.06320) + + + --> STEP: 97/234 -- GLOBAL_STEP: 53215 + | > loss: -0.22728 (-0.24053) + | > log_mle: -0.35836 (-0.34572) + | > loss_dur: 0.13108 (0.10520) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.48215 (13.68387) + | > current_lr: 0.00006 + | > step_time: 2.00100 (2.21543) + | > loader_time: 0.00300 (0.06088) + + + --> STEP: 102/234 -- GLOBAL_STEP: 53220 + | > loss: -0.19396 (-0.23948) + | > log_mle: -0.34281 (-0.34673) + | > loss_dur: 0.14885 (0.10725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.80207 (14.32504) + | > current_lr: 0.00006 + | > step_time: 1.91380 (2.18922) + | > loader_time: 0.00290 (0.05804) + + + --> STEP: 107/234 -- GLOBAL_STEP: 53225 + | > loss: -0.22024 (-0.23906) + | > log_mle: -0.38572 (-0.34893) + | > loss_dur: 0.16548 (0.10987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.07109 (14.90601) + | > current_lr: 0.00006 + | > step_time: 1.93620 (2.16458) + | > loader_time: 0.00200 (0.05548) + + + --> STEP: 112/234 -- GLOBAL_STEP: 53230 + | > loss: -0.22438 (-0.23831) + | > log_mle: -0.40065 (-0.35097) + | > loss_dur: 0.17627 (0.11265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.49845 (15.46751) + | > current_lr: 0.00006 + | > step_time: 1.50640 (2.14272) + | > loader_time: 0.00230 (0.05384) + + + --> STEP: 117/234 -- GLOBAL_STEP: 53235 + | > loss: -0.23462 (-0.23788) + | > log_mle: -0.39652 (-0.35289) + | > loss_dur: 0.16191 (0.11501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.69331 (16.04675) + | > current_lr: 0.00006 + | > step_time: 1.40890 (2.12136) + | > loader_time: 0.00270 (0.05241) + + + --> STEP: 122/234 -- GLOBAL_STEP: 53240 + | > loss: -0.22116 (-0.23724) + | > log_mle: -0.36872 (-0.35398) + | > loss_dur: 0.14756 (0.11674) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.91953 (16.32928) + | > current_lr: 0.00006 + | > step_time: 2.28750 (2.14399) + | > loader_time: 0.00360 (0.05184) + + + --> STEP: 127/234 -- GLOBAL_STEP: 53245 + | > loss: -0.24186 (-0.23713) + | > log_mle: -0.43023 (-0.35616) + | > loss_dur: 0.18837 (0.11903) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.11595 (17.13739) + | > current_lr: 0.00006 + | > step_time: 1.20410 (2.14757) + | > loader_time: 0.08430 (0.05060) + + + --> STEP: 132/234 -- GLOBAL_STEP: 53250 + | > loss: -0.24845 (-0.23795) + | > log_mle: -0.40537 (-0.35891) + | > loss_dur: 0.15692 (0.12096) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.25278 (17.88610) + | > current_lr: 0.00006 + | > step_time: 5.09240 (2.18384) + | > loader_time: 0.10450 (0.04957) + + + --> STEP: 137/234 -- GLOBAL_STEP: 53255 + | > loss: -0.23149 (-0.23877) + | > log_mle: -0.42461 (-0.36200) + | > loss_dur: 0.19312 (0.12323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.26585 (18.59508) + | > current_lr: 0.00006 + | > step_time: 2.90430 (2.18287) + | > loader_time: 0.00600 (0.04790) + + + --> STEP: 142/234 -- GLOBAL_STEP: 53260 + | > loss: -0.24184 (-0.23932) + | > log_mle: -0.42787 (-0.36454) + | > loss_dur: 0.18603 (0.12522) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.45465 (19.22513) + | > current_lr: 0.00006 + | > step_time: 2.30170 (2.18137) + | > loader_time: 0.00230 (0.04633) + + + --> STEP: 147/234 -- GLOBAL_STEP: 53265 + | > loss: -0.24124 (-0.24091) + | > log_mle: -0.43172 (-0.36856) + | > loss_dur: 0.19048 (0.12765) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.75326 (20.25582) + | > current_lr: 0.00006 + | > step_time: 2.17830 (2.18573) + | > loader_time: 0.00160 (0.04884) + + + --> STEP: 152/234 -- GLOBAL_STEP: 53270 + | > loss: -0.31518 (-0.24279) + | > log_mle: -0.52084 (-0.37243) + | > loss_dur: 0.20566 (0.12964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.44135 (21.04980) + | > current_lr: 0.00006 + | > step_time: 7.41740 (2.23814) + | > loader_time: 0.09240 (0.04793) + + + --> STEP: 157/234 -- GLOBAL_STEP: 53275 + | > loss: -0.27103 (-0.24533) + | > log_mle: -0.46390 (-0.37705) + | > loss_dur: 0.19287 (0.13172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.66181 (22.15622) + | > current_lr: 0.00006 + | > step_time: 2.19240 (2.23953) + | > loader_time: 0.00270 (0.04711) + + + --> STEP: 162/234 -- GLOBAL_STEP: 53280 + | > loss: -0.29650 (-0.24763) + | > log_mle: -0.49499 (-0.38138) + | > loss_dur: 0.19850 (0.13375) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.24546 (23.19209) + | > current_lr: 0.00006 + | > step_time: 2.40700 (2.23776) + | > loader_time: 0.08720 (0.04686) + + + --> STEP: 167/234 -- GLOBAL_STEP: 53285 + | > loss: -0.38917 (-0.24976) + | > log_mle: -0.59353 (-0.38548) + | > loss_dur: 0.20435 (0.13571) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.78024 (24.18393) + | > current_lr: 0.00006 + | > step_time: 2.69770 (2.33245) + | > loader_time: 0.09360 (0.04949) + + + --> STEP: 172/234 -- GLOBAL_STEP: 53290 + | > loss: -0.37389 (-0.25255) + | > log_mle: -0.58809 (-0.39051) + | > loss_dur: 0.21420 (0.13796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.09990 (25.54308) + | > current_lr: 0.00006 + | > step_time: 1.47980 (2.35991) + | > loader_time: 0.00360 (0.04986) + + + --> STEP: 177/234 -- GLOBAL_STEP: 53295 + | > loss: -0.31638 (-0.25507) + | > log_mle: -0.53620 (-0.39525) + | > loss_dur: 0.21982 (0.14018) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.66214 (26.86334) + | > current_lr: 0.00006 + | > step_time: 5.00490 (2.42818) + | > loader_time: 0.00740 (0.05024) + + + --> STEP: 182/234 -- GLOBAL_STEP: 53300 + | > loss: -0.33932 (-0.25746) + | > log_mle: -0.58332 (-0.40005) + | > loss_dur: 0.24399 (0.14259) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.12868 (27.80781) + | > current_lr: 0.00006 + | > step_time: 4.71300 (2.43788) + | > loader_time: 0.00280 (0.04988) + + + --> STEP: 187/234 -- GLOBAL_STEP: 53305 + | > loss: -0.35867 (-0.25998) + | > log_mle: -0.58756 (-0.40482) + | > loss_dur: 0.22889 (0.14484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.34711 (28.80258) + | > current_lr: 0.00006 + | > step_time: 16.69560 (2.57692) + | > loader_time: 0.30850 (0.05129) + + + --> STEP: 192/234 -- GLOBAL_STEP: 53310 + | > loss: -0.39816 (-0.26275) + | > log_mle: -0.60974 (-0.40938) + | > loss_dur: 0.21158 (0.14663) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.07310 (30.08237) + | > current_lr: 0.00006 + | > step_time: 2.78730 (2.61245) + | > loader_time: 0.10040 (0.05207) + + + --> STEP: 197/234 -- GLOBAL_STEP: 53315 + | > loss: -0.38484 (-0.26555) + | > log_mle: -0.58592 (-0.41391) + | > loss_dur: 0.20108 (0.14836) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.81953 (31.10137) + | > current_lr: 0.00006 + | > step_time: 1.89750 (2.61780) + | > loader_time: 0.00290 (0.05090) + + + --> STEP: 202/234 -- GLOBAL_STEP: 53320 + | > loss: -0.46033 (-0.26822) + | > log_mle: -0.67994 (-0.41850) + | > loss_dur: 0.21961 (0.15028) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.80089 (32.14480) + | > current_lr: 0.00006 + | > step_time: 0.88110 (2.63514) + | > loader_time: 0.00300 (0.05103) + + + --> STEP: 207/234 -- GLOBAL_STEP: 53325 + | > loss: -0.42882 (-0.27074) + | > log_mle: -0.66144 (-0.42294) + | > loss_dur: 0.23263 (0.15220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.55111 (33.25049) + | > current_lr: 0.00006 + | > step_time: 2.80710 (2.66429) + | > loader_time: 0.00540 (0.05115) + + + --> STEP: 212/234 -- GLOBAL_STEP: 53330 + | > loss: -0.40978 (-0.27403) + | > log_mle: -0.64018 (-0.42816) + | > loss_dur: 0.23040 (0.15413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.21175 (34.39334) + | > current_lr: 0.00006 + | > step_time: 5.11410 (2.74405) + | > loader_time: 0.08580 (0.05262) + + + --> STEP: 217/234 -- GLOBAL_STEP: 53335 + | > loss: -0.41773 (-0.27725) + | > log_mle: -0.66224 (-0.43326) + | > loss_dur: 0.24452 (0.15601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 106.03402 (35.87226) + | > current_lr: 0.00006 + | > step_time: 4.39770 (2.77123) + | > loader_time: 0.00360 (0.05192) + + + --> STEP: 222/234 -- GLOBAL_STEP: 53340 + | > loss: -0.40990 (-0.28041) + | > log_mle: -0.67557 (-0.43832) + | > loss_dur: 0.26567 (0.15791) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.99156 (36.85730) + | > current_lr: 0.00006 + | > step_time: 1.10400 (2.75675) + | > loader_time: 0.00570 (0.05156) + + + --> STEP: 227/234 -- GLOBAL_STEP: 53345 + | > loss: -0.39622 (-0.28401) + | > log_mle: -0.65089 (-0.44376) + | > loss_dur: 0.25467 (0.15975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.19801 (38.05581) + | > current_lr: 0.00006 + | > step_time: 0.24280 (2.70269) + | > loader_time: 0.00320 (0.05089) + + + --> STEP: 232/234 -- GLOBAL_STEP: 53350 + | > loss: -0.40739 (-0.28711) + | > log_mle: -0.87765 (-0.45085) + | > loss_dur: 0.47026 (0.16374) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 125.84238 (39.40279) + | > current_lr: 0.00006 + | > step_time: 0.33900 (2.65035) + | > loader_time: 0.04060 (0.05003) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32099 (+0.04431) + | > avg_loss: -0.29378 (-0.00437) + | > avg_log_mle: -0.50998 (+0.00424) + | > avg_loss_dur: 0.21620 (-0.00861) + + + > EPOCH: 228/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 09:03:55)  + + --> STEP: 3/234 -- GLOBAL_STEP: 53355 + | > loss: -0.17108 (-0.23095) + | > log_mle: -0.34510 (-0.35676) + | > loss_dur: 0.17402 (0.12581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.52027 (20.46867) + | > current_lr: 0.00006 + | > step_time: 6.30170 (6.39814) + | > loader_time: 0.08840 (1.26861) + + + --> STEP: 8/234 -- GLOBAL_STEP: 53360 + | > loss: -0.27503 (-0.25870) + | > log_mle: -0.36920 (-0.35871) + | > loss_dur: 0.09418 (0.10001) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.01241 (19.07395) + | > current_lr: 0.00006 + | > step_time: 1.88330 (4.45952) + | > loader_time: 0.00200 (0.49972) + + + --> STEP: 13/234 -- GLOBAL_STEP: 53365 + | > loss: -0.29062 (-0.26280) + | > log_mle: -0.37055 (-0.35958) + | > loss_dur: 0.07993 (0.09678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.50038 (18.75706) + | > current_lr: 0.00006 + | > step_time: 3.00850 (4.32092) + | > loader_time: 0.00130 (0.32400) + + + --> STEP: 18/234 -- GLOBAL_STEP: 53370 + | > loss: -0.25590 (-0.26639) + | > log_mle: -0.34770 (-0.35876) + | > loss_dur: 0.09180 (0.09236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.23910 (17.36905) + | > current_lr: 0.00006 + | > step_time: 2.39680 (4.38218) + | > loader_time: 0.09000 (0.27195) + + + --> STEP: 23/234 -- GLOBAL_STEP: 53375 + | > loss: -0.28940 (-0.26859) + | > log_mle: -0.36572 (-0.35807) + | > loss_dur: 0.07632 (0.08947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.86159 (16.05861) + | > current_lr: 0.00006 + | > step_time: 0.87550 (3.79052) + | > loader_time: 0.00130 (0.21977) + + + --> STEP: 28/234 -- GLOBAL_STEP: 53380 + | > loss: -0.31141 (-0.26955) + | > log_mle: -0.37338 (-0.35713) + | > loss_dur: 0.06197 (0.08758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.12974 (15.12886) + | > current_lr: 0.00006 + | > step_time: 1.10160 (3.33271) + | > loader_time: 0.00140 (0.18392) + + + --> STEP: 33/234 -- GLOBAL_STEP: 53385 + | > loss: -0.25717 (-0.26645) + | > log_mle: -0.34288 (-0.35483) + | > loss_dur: 0.08571 (0.08838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27606 (14.60163) + | > current_lr: 0.00006 + | > step_time: 4.79240 (3.16904) + | > loader_time: 0.00640 (0.15890) + + + --> STEP: 38/234 -- GLOBAL_STEP: 53390 + | > loss: -0.24904 (-0.26315) + | > log_mle: -0.35295 (-0.35293) + | > loss_dur: 0.10392 (0.08978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.58619 (14.30912) + | > current_lr: 0.00006 + | > step_time: 1.01270 (3.12109) + | > loader_time: 0.00960 (0.14070) + + + --> STEP: 43/234 -- GLOBAL_STEP: 53395 + | > loss: -0.23364 (-0.26013) + | > log_mle: -0.34583 (-0.35108) + | > loss_dur: 0.11219 (0.09095) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.95775 (13.92573) + | > current_lr: 0.00006 + | > step_time: 1.22560 (2.92588) + | > loader_time: 0.00200 (0.12647) + + + --> STEP: 48/234 -- GLOBAL_STEP: 53400 + | > loss: -0.25621 (-0.25823) + | > log_mle: -0.33971 (-0.35001) + | > loss_dur: 0.08350 (0.09178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.98703 (13.60075) + | > current_lr: 0.00006 + | > step_time: 2.14920 (2.78615) + | > loader_time: 0.00170 (0.11556) + + + --> STEP: 53/234 -- GLOBAL_STEP: 53405 + | > loss: -0.24040 (-0.25671) + | > log_mle: -0.34393 (-0.34893) + | > loss_dur: 0.10352 (0.09222) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.79579 (13.23642) + | > current_lr: 0.00006 + | > step_time: 1.78980 (2.67317) + | > loader_time: 0.00260 (0.10489) + + + --> STEP: 58/234 -- GLOBAL_STEP: 53410 + | > loss: -0.24763 (-0.25544) + | > log_mle: -0.34004 (-0.34812) + | > loss_dur: 0.09241 (0.09268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.74071 (12.93868) + | > current_lr: 0.00006 + | > step_time: 7.71240 (2.68109) + | > loader_time: 0.28510 (0.10401) + + + --> STEP: 63/234 -- GLOBAL_STEP: 53415 + | > loss: -0.22097 (-0.25295) + | > log_mle: -0.32975 (-0.34789) + | > loss_dur: 0.10878 (0.09495) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.54861 (13.21916) + | > current_lr: 0.00006 + | > step_time: 1.17490 (2.59851) + | > loader_time: 0.00310 (0.10020) + + + --> STEP: 68/234 -- GLOBAL_STEP: 53420 + | > loss: -0.20532 (-0.25094) + | > log_mle: -0.32475 (-0.34682) + | > loss_dur: 0.11942 (0.09588) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.20690 (13.00265) + | > current_lr: 0.00006 + | > step_time: 1.40820 (2.55622) + | > loader_time: 0.00190 (0.09698) + + + --> STEP: 73/234 -- GLOBAL_STEP: 53425 + | > loss: -0.19950 (-0.24817) + | > log_mle: -0.34093 (-0.34596) + | > loss_dur: 0.14143 (0.09780) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.33669 (13.00634) + | > current_lr: 0.00006 + | > step_time: 2.59470 (2.49556) + | > loader_time: 0.00290 (0.09253) + + + --> STEP: 78/234 -- GLOBAL_STEP: 53430 + | > loss: -0.20645 (-0.24578) + | > log_mle: -0.32180 (-0.34505) + | > loss_dur: 0.11535 (0.09927) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.78644 (12.99443) + | > current_lr: 0.00006 + | > step_time: 2.29790 (2.44948) + | > loader_time: 0.00260 (0.08673) + + + --> STEP: 83/234 -- GLOBAL_STEP: 53435 + | > loss: -0.19083 (-0.24375) + | > log_mle: -0.33977 (-0.34448) + | > loss_dur: 0.14894 (0.10073) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.87708 (13.03407) + | > current_lr: 0.00006 + | > step_time: 2.00560 (2.42699) + | > loader_time: 0.00280 (0.08261) + + + --> STEP: 88/234 -- GLOBAL_STEP: 53440 + | > loss: -0.21951 (-0.24260) + | > log_mle: -0.37816 (-0.34457) + | > loss_dur: 0.15865 (0.10197) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.91145 (13.05334) + | > current_lr: 0.00006 + | > step_time: 2.00990 (2.41771) + | > loader_time: 0.00310 (0.07912) + + + --> STEP: 93/234 -- GLOBAL_STEP: 53445 + | > loss: -0.23567 (-0.24188) + | > log_mle: -0.38986 (-0.34591) + | > loss_dur: 0.15419 (0.10403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.62639 (13.24638) + | > current_lr: 0.00006 + | > step_time: 2.30800 (2.38678) + | > loader_time: 0.08770 (0.07688) + + + --> STEP: 98/234 -- GLOBAL_STEP: 53450 + | > loss: -0.20569 (-0.24138) + | > log_mle: -0.32579 (-0.34730) + | > loss_dur: 0.12010 (0.10591) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.03787 (13.64243) + | > current_lr: 0.00006 + | > step_time: 2.70650 (2.39254) + | > loader_time: 0.08800 (0.07587) + + + --> STEP: 103/234 -- GLOBAL_STEP: 53455 + | > loss: -0.25699 (-0.24119) + | > log_mle: -0.42505 (-0.34953) + | > loss_dur: 0.16805 (0.10834) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.21529 (14.37490) + | > current_lr: 0.00006 + | > step_time: 1.37410 (2.34689) + | > loader_time: 0.00240 (0.07312) + + + --> STEP: 108/234 -- GLOBAL_STEP: 53460 + | > loss: -0.23046 (-0.24089) + | > log_mle: -0.36403 (-0.35128) + | > loss_dur: 0.13357 (0.11039) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.02895 (15.05411) + | > current_lr: 0.00006 + | > step_time: 2.00290 (2.34405) + | > loader_time: 0.00250 (0.07226) + + + --> STEP: 113/234 -- GLOBAL_STEP: 53465 + | > loss: -0.24378 (-0.24033) + | > log_mle: -0.40782 (-0.35354) + | > loss_dur: 0.16404 (0.11321) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.41672 (16.12033) + | > current_lr: 0.00006 + | > step_time: 3.80350 (2.39344) + | > loader_time: 0.00340 (0.06997) + + + --> STEP: 118/234 -- GLOBAL_STEP: 53470 + | > loss: -0.22008 (-0.23968) + | > log_mle: -0.37750 (-0.35510) + | > loss_dur: 0.15743 (0.11542) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.15245 (16.72672) + | > current_lr: 0.00006 + | > step_time: 0.47610 (2.41810) + | > loader_time: 0.00260 (0.06804) + + + --> STEP: 123/234 -- GLOBAL_STEP: 53475 + | > loss: -0.19643 (-0.23872) + | > log_mle: -0.34911 (-0.35578) + | > loss_dur: 0.15268 (0.11707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.90284 (16.97242) + | > current_lr: 0.00006 + | > step_time: 2.50010 (2.41718) + | > loader_time: 0.00190 (0.06636) + + + --> STEP: 128/234 -- GLOBAL_STEP: 53480 + | > loss: -0.26185 (-0.23943) + | > log_mle: -0.40746 (-0.35851) + | > loss_dur: 0.14562 (0.11908) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.76204 (17.67346) + | > current_lr: 0.00006 + | > step_time: 2.59430 (2.40839) + | > loader_time: 0.00250 (0.06509) + + + --> STEP: 133/234 -- GLOBAL_STEP: 53485 + | > loss: -0.25523 (-0.24015) + | > log_mle: -0.43634 (-0.36151) + | > loss_dur: 0.18111 (0.12136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.65273 (18.24033) + | > current_lr: 0.00006 + | > step_time: 2.89310 (2.40488) + | > loader_time: 0.00260 (0.06406) + + + --> STEP: 138/234 -- GLOBAL_STEP: 53490 + | > loss: -0.21154 (-0.24056) + | > log_mle: -0.38344 (-0.36426) + | > loss_dur: 0.17190 (0.12371) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.48838 (18.79808) + | > current_lr: 0.00006 + | > step_time: 2.63100 (2.39487) + | > loader_time: 0.09350 (0.06315) + + + --> STEP: 143/234 -- GLOBAL_STEP: 53495 + | > loss: -0.30242 (-0.24168) + | > log_mle: -0.53269 (-0.36784) + | > loss_dur: 0.23027 (0.12615) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.26632 (19.80035) + | > current_lr: 0.00006 + | > step_time: 2.40690 (2.37188) + | > loader_time: 0.08520 (0.06292) + + + --> STEP: 148/234 -- GLOBAL_STEP: 53500 + | > loss: -0.27861 (-0.24316) + | > log_mle: -0.43943 (-0.37130) + | > loss_dur: 0.16082 (0.12814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.21230 (20.65396) + | > current_lr: 0.00006 + | > step_time: 4.51480 (2.39799) + | > loader_time: 0.18500 (0.06335) + + + --> STEP: 153/234 -- GLOBAL_STEP: 53505 + | > loss: -0.38574 (-0.24566) + | > log_mle: -0.57468 (-0.37600) + | > loss_dur: 0.18894 (0.13034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.22260 (22.05278) + | > current_lr: 0.00006 + | > step_time: 0.91680 (2.42224) + | > loader_time: 0.08610 (0.06316) + + + --> STEP: 158/234 -- GLOBAL_STEP: 53510 + | > loss: -0.29092 (-0.24744) + | > log_mle: -0.50014 (-0.37990) + | > loss_dur: 0.20922 (0.13246) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.01566 (23.37746) + | > current_lr: 0.00006 + | > step_time: 11.80240 (2.49005) + | > loader_time: 0.09250 (0.06285) + + + --> STEP: 163/234 -- GLOBAL_STEP: 53515 + | > loss: -0.27833 (-0.24961) + | > log_mle: -0.47701 (-0.38405) + | > loss_dur: 0.19868 (0.13444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.52397 (24.10089) + | > current_lr: 0.00006 + | > step_time: 3.69870 (2.50125) + | > loader_time: 0.10490 (0.06234) + + + --> STEP: 168/234 -- GLOBAL_STEP: 53520 + | > loss: -0.31131 (-0.25192) + | > log_mle: -0.53444 (-0.38845) + | > loss_dur: 0.22313 (0.13653) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.42755 (25.10599) + | > current_lr: 0.00006 + | > step_time: 1.70240 (2.52980) + | > loader_time: 0.08980 (0.06217) + + + --> STEP: 173/234 -- GLOBAL_STEP: 53525 + | > loss: -0.33479 (-0.25482) + | > log_mle: -0.54666 (-0.39362) + | > loss_dur: 0.21187 (0.13880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.66927 (26.22899) + | > current_lr: 0.00006 + | > step_time: 3.19460 (2.57755) + | > loader_time: 0.00330 (0.06156) + + + --> STEP: 178/234 -- GLOBAL_STEP: 53530 + | > loss: -0.37696 (-0.25778) + | > log_mle: -0.60764 (-0.39884) + | > loss_dur: 0.23068 (0.14106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.53749 (27.34363) + | > current_lr: 0.00006 + | > step_time: 1.51220 (2.59115) + | > loader_time: 0.09600 (0.06137) + + + --> STEP: 183/234 -- GLOBAL_STEP: 53535 + | > loss: -0.37881 (-0.26019) + | > log_mle: -0.59150 (-0.40351) + | > loss_dur: 0.21269 (0.14332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.21923 (28.52127) + | > current_lr: 0.00006 + | > step_time: 1.40100 (2.57123) + | > loader_time: 0.00390 (0.05979) + + + --> STEP: 188/234 -- GLOBAL_STEP: 53540 + | > loss: -0.40305 (-0.26278) + | > log_mle: -0.62392 (-0.40827) + | > loss_dur: 0.22087 (0.14549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.89197 (29.56945) + | > current_lr: 0.00006 + | > step_time: 5.10540 (2.60065) + | > loader_time: 0.00480 (0.05980) + + + --> STEP: 193/234 -- GLOBAL_STEP: 53545 + | > loss: -0.39868 (-0.26574) + | > log_mle: -0.61889 (-0.41299) + | > loss_dur: 0.22020 (0.14725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.26817 (30.61674) + | > current_lr: 0.00006 + | > step_time: 5.20580 (2.65666) + | > loader_time: 0.09990 (0.06068) + + + --> STEP: 198/234 -- GLOBAL_STEP: 53550 + | > loss: -0.38502 (-0.26836) + | > log_mle: -0.60678 (-0.41738) + | > loss_dur: 0.22176 (0.14902) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.97156 (31.87736) + | > current_lr: 0.00006 + | > step_time: 4.59530 (2.71136) + | > loader_time: 0.29470 (0.06217) + + + --> STEP: 203/234 -- GLOBAL_STEP: 53555 + | > loss: -0.32476 (-0.27082) + | > log_mle: -0.54233 (-0.42169) + | > loss_dur: 0.21757 (0.15088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.04839 (32.76050) + | > current_lr: 0.00006 + | > step_time: 4.31770 (2.79639) + | > loader_time: 0.09830 (0.06311) + + + --> STEP: 208/234 -- GLOBAL_STEP: 53560 + | > loss: -0.38978 (-0.27392) + | > log_mle: -0.62568 (-0.42672) + | > loss_dur: 0.23590 (0.15280) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.78817 (34.01528) + | > current_lr: 0.00006 + | > step_time: 6.80790 (2.88010) + | > loader_time: 0.09830 (0.06257) + + + --> STEP: 213/234 -- GLOBAL_STEP: 53565 + | > loss: -0.43431 (-0.27744) + | > log_mle: -0.67177 (-0.43216) + | > loss_dur: 0.23747 (0.15472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 105.03502 (35.52131) + | > current_lr: 0.00006 + | > step_time: 4.11100 (2.93885) + | > loader_time: 0.08450 (0.06241) + + + --> STEP: 218/234 -- GLOBAL_STEP: 53570 + | > loss: -0.40049 (-0.28060) + | > log_mle: -0.63230 (-0.43711) + | > loss_dur: 0.23181 (0.15651) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.83581 (36.71667) + | > current_lr: 0.00006 + | > step_time: 6.49250 (3.04762) + | > loader_time: 0.00410 (0.06459) + + + --> STEP: 223/234 -- GLOBAL_STEP: 53575 + | > loss: -0.44132 (-0.28419) + | > log_mle: -0.67688 (-0.44250) + | > loss_dur: 0.23555 (0.15831) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.53580 (37.97447) + | > current_lr: 0.00006 + | > step_time: 2.89810 (3.04709) + | > loader_time: 0.01110 (0.06406) + + + --> STEP: 228/234 -- GLOBAL_STEP: 53580 + | > loss: -0.39465 (-0.28764) + | > log_mle: -0.66854 (-0.44795) + | > loss_dur: 0.27389 (0.16031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.81397 (39.27929) + | > current_lr: 0.00006 + | > step_time: 0.26580 (3.00793) + | > loader_time: 0.00470 (0.06352) + + + --> STEP: 233/234 -- GLOBAL_STEP: 53585 + | > loss: 0.01407 (-0.28853) + | > log_mle: -0.64284 (-0.45450) + | > loss_dur: 0.65691 (0.16597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.89362 (40.82814) + | > current_lr: 0.00006 + | > step_time: 0.19220 (2.94943) + | > loader_time: 0.00270 (0.06239) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.45420 (+1.13321) + | > avg_loss: -0.30205 (-0.00827) + | > avg_log_mle: -0.53023 (-0.02025) + | > avg_loss_dur: 0.22818 (+0.01198) + + + > EPOCH: 229/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 09:16:43)  + + --> STEP: 4/234 -- GLOBAL_STEP: 53590 + | > loss: -0.25193 (-0.24253) + | > log_mle: -0.35546 (-0.35728) + | > loss_dur: 0.10353 (0.11475) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.35518 (21.04537) + | > current_lr: 0.00006 + | > step_time: 3.50270 (7.58384) + | > loader_time: 0.00130 (0.37046) + + + --> STEP: 9/234 -- GLOBAL_STEP: 53595 + | > loss: -0.22739 (-0.25578) + | > log_mle: -0.36725 (-0.35939) + | > loss_dur: 0.13985 (0.10361) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.04895 (19.30737) + | > current_lr: 0.00006 + | > step_time: 4.09860 (6.14988) + | > loader_time: 0.10250 (0.19885) + + + --> STEP: 14/234 -- GLOBAL_STEP: 53600 + | > loss: -0.27114 (-0.26343) + | > log_mle: -0.36047 (-0.35948) + | > loss_dur: 0.08933 (0.09605) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.46140 (18.30959) + | > current_lr: 0.00006 + | > step_time: 3.71930 (4.74522) + | > loader_time: 0.07340 (0.13362) + + + --> STEP: 19/234 -- GLOBAL_STEP: 53605 + | > loss: -0.28077 (-0.26804) + | > log_mle: -0.35514 (-0.35864) + | > loss_dur: 0.07437 (0.09060) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.06190 (16.67234) + | > current_lr: 0.00006 + | > step_time: 5.10110 (4.40271) + | > loader_time: 0.00400 (0.10805) + + + --> STEP: 24/234 -- GLOBAL_STEP: 53610 + | > loss: -0.28571 (-0.27124) + | > log_mle: -0.35068 (-0.35795) + | > loss_dur: 0.06498 (0.08671) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.69960 (15.33363) + | > current_lr: 0.00006 + | > step_time: 1.40350 (3.77188) + | > loader_time: 0.00220 (0.08589) + + + --> STEP: 29/234 -- GLOBAL_STEP: 53615 + | > loss: -0.24319 (-0.27103) + | > log_mle: -0.33734 (-0.35652) + | > loss_dur: 0.09415 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.76711 (14.52598) + | > current_lr: 0.00006 + | > step_time: 2.28760 (3.36732) + | > loader_time: 0.00200 (0.07144) + + + --> STEP: 34/234 -- GLOBAL_STEP: 53620 + | > loss: -0.25707 (-0.26870) + | > log_mle: -0.34495 (-0.35473) + | > loss_dur: 0.08788 (0.08603) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29394 (14.16428) + | > current_lr: 0.00006 + | > step_time: 1.80870 (3.10694) + | > loader_time: 0.08620 (0.06637) + + + --> STEP: 39/234 -- GLOBAL_STEP: 53625 + | > loss: -0.23430 (-0.26478) + | > log_mle: -0.34103 (-0.35274) + | > loss_dur: 0.10673 (0.08797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.82065 (14.11324) + | > current_lr: 0.00006 + | > step_time: 1.33730 (3.04804) + | > loader_time: 0.00190 (0.06616) + + + --> STEP: 44/234 -- GLOBAL_STEP: 53630 + | > loss: -0.24959 (-0.26096) + | > log_mle: -0.33205 (-0.35056) + | > loss_dur: 0.08246 (0.08960) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.59360 (13.79000) + | > current_lr: 0.00006 + | > step_time: 1.52870 (2.89273) + | > loader_time: 0.00300 (0.06263) + + + --> STEP: 49/234 -- GLOBAL_STEP: 53635 + | > loss: -0.25908 (-0.25887) + | > log_mle: -0.34542 (-0.34971) + | > loss_dur: 0.08635 (0.09084) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.36557 (13.45940) + | > current_lr: 0.00006 + | > step_time: 1.22210 (2.78396) + | > loader_time: 0.00220 (0.05802) + + + --> STEP: 54/234 -- GLOBAL_STEP: 53640 + | > loss: -0.25396 (-0.25673) + | > log_mle: -0.34483 (-0.34859) + | > loss_dur: 0.09088 (0.09187) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.31199 (13.11961) + | > current_lr: 0.00006 + | > step_time: 1.15770 (2.67006) + | > loader_time: 0.00230 (0.05288) + + + --> STEP: 59/234 -- GLOBAL_STEP: 53645 + | > loss: -0.25597 (-0.25538) + | > log_mle: -0.34753 (-0.34778) + | > loss_dur: 0.09157 (0.09240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.96678 (12.97539) + | > current_lr: 0.00006 + | > step_time: 2.08900 (2.58198) + | > loader_time: 0.08690 (0.05008) + + + --> STEP: 64/234 -- GLOBAL_STEP: 53650 + | > loss: -0.24019 (-0.25255) + | > log_mle: -0.33498 (-0.34748) + | > loss_dur: 0.09479 (0.09493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.13988 (13.12695) + | > current_lr: 0.00006 + | > step_time: 1.70170 (2.51563) + | > loader_time: 0.00430 (0.04639) + + + --> STEP: 69/234 -- GLOBAL_STEP: 53655 + | > loss: -0.22410 (-0.25057) + | > log_mle: -0.32221 (-0.34642) + | > loss_dur: 0.09811 (0.09585) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.35662 (12.97321) + | > current_lr: 0.00006 + | > step_time: 1.71180 (2.44450) + | > loader_time: 0.00220 (0.04319) + + + --> STEP: 74/234 -- GLOBAL_STEP: 53660 + | > loss: -0.19594 (-0.24749) + | > log_mle: -0.32276 (-0.34568) + | > loss_dur: 0.12682 (0.09818) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.10214 (13.13645) + | > current_lr: 0.00006 + | > step_time: 1.91040 (2.39887) + | > loader_time: 0.00290 (0.04152) + + + --> STEP: 79/234 -- GLOBAL_STEP: 53665 + | > loss: -0.22217 (-0.24564) + | > log_mle: -0.33751 (-0.34503) + | > loss_dur: 0.11534 (0.09939) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.54869 (13.08025) + | > current_lr: 0.00006 + | > step_time: 1.79110 (2.33790) + | > loader_time: 0.00340 (0.03907) + + + --> STEP: 84/234 -- GLOBAL_STEP: 53670 + | > loss: -0.22747 (-0.24379) + | > log_mle: -0.33644 (-0.34456) + | > loss_dur: 0.10897 (0.10077) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.52425 (12.99375) + | > current_lr: 0.00006 + | > step_time: 2.42270 (2.31405) + | > loader_time: 0.00240 (0.03802) + + + --> STEP: 89/234 -- GLOBAL_STEP: 53675 + | > loss: -0.23206 (-0.24253) + | > log_mle: -0.35988 (-0.34488) + | > loss_dur: 0.12782 (0.10235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.18012 (13.13374) + | > current_lr: 0.00006 + | > step_time: 1.30030 (2.27452) + | > loader_time: 0.00240 (0.03799) + + + --> STEP: 94/234 -- GLOBAL_STEP: 53680 + | > loss: -0.23697 (-0.24168) + | > log_mle: -0.38664 (-0.34637) + | > loss_dur: 0.14967 (0.10469) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.17425 (13.61915) + | > current_lr: 0.00006 + | > step_time: 1.13580 (2.24559) + | > loader_time: 0.00270 (0.03787) + + + --> STEP: 99/234 -- GLOBAL_STEP: 53685 + | > loss: -0.25797 (-0.24138) + | > log_mle: -0.42073 (-0.34797) + | > loss_dur: 0.16276 (0.10660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.65958 (14.06413) + | > current_lr: 0.00006 + | > step_time: 2.79700 (2.23214) + | > loader_time: 0.00290 (0.03614) + + + --> STEP: 104/234 -- GLOBAL_STEP: 53690 + | > loss: -0.27099 (-0.24107) + | > log_mle: -0.43156 (-0.34996) + | > loss_dur: 0.16057 (0.10890) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.53217 (14.91435) + | > current_lr: 0.00006 + | > step_time: 2.90480 (2.26390) + | > loader_time: 0.10280 (0.03562) + + + --> STEP: 109/234 -- GLOBAL_STEP: 53695 + | > loss: -0.21127 (-0.24002) + | > log_mle: -0.40169 (-0.35118) + | > loss_dur: 0.19043 (0.11116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.45016 (15.45065) + | > current_lr: 0.00006 + | > step_time: 1.85050 (2.24672) + | > loader_time: 0.00200 (0.03410) + + + --> STEP: 114/234 -- GLOBAL_STEP: 53700 + | > loss: -0.23248 (-0.23974) + | > log_mle: -0.38713 (-0.35339) + | > loss_dur: 0.15464 (0.11366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.36142 (16.20299) + | > current_lr: 0.00006 + | > step_time: 1.49960 (2.22076) + | > loader_time: 0.00280 (0.03418) + + + --> STEP: 119/234 -- GLOBAL_STEP: 53705 + | > loss: -0.22795 (-0.23901) + | > log_mle: -0.38606 (-0.35502) + | > loss_dur: 0.15810 (0.11601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.41339 (16.60081) + | > current_lr: 0.00006 + | > step_time: 1.96960 (2.25006) + | > loader_time: 0.00250 (0.03510) + + + --> STEP: 124/234 -- GLOBAL_STEP: 53710 + | > loss: -0.25199 (-0.23835) + | > log_mle: -0.41551 (-0.35596) + | > loss_dur: 0.16352 (0.11761) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.30926 (16.86804) + | > current_lr: 0.00006 + | > step_time: 4.60490 (2.28772) + | > loader_time: 0.08850 (0.03588) + + + --> STEP: 129/234 -- GLOBAL_STEP: 53715 + | > loss: -0.21413 (-0.23852) + | > log_mle: -0.39824 (-0.35845) + | > loss_dur: 0.18411 (0.11993) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.25191 (17.82571) + | > current_lr: 0.00006 + | > step_time: 2.20990 (2.30670) + | > loader_time: 0.00260 (0.03605) + + + --> STEP: 134/234 -- GLOBAL_STEP: 53720 + | > loss: -0.26582 (-0.23966) + | > log_mle: -0.45496 (-0.36172) + | > loss_dur: 0.18914 (0.12206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.34014 (18.56985) + | > current_lr: 0.00006 + | > step_time: 2.79640 (2.31392) + | > loader_time: 0.09410 (0.03695) + + + --> STEP: 139/234 -- GLOBAL_STEP: 53725 + | > loss: -0.32720 (-0.24060) + | > log_mle: -0.51915 (-0.36483) + | > loss_dur: 0.19195 (0.12424) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.22749 (19.25089) + | > current_lr: 0.00006 + | > step_time: 1.39560 (2.31991) + | > loader_time: 0.00350 (0.03707) + + + --> STEP: 144/234 -- GLOBAL_STEP: 53730 + | > loss: -0.29472 (-0.24145) + | > log_mle: -0.49319 (-0.36818) + | > loss_dur: 0.19847 (0.12673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.84950 (20.00158) + | > current_lr: 0.00006 + | > step_time: 1.39250 (2.31516) + | > loader_time: 0.00220 (0.03641) + + + --> STEP: 149/234 -- GLOBAL_STEP: 53735 + | > loss: -0.35368 (-0.24351) + | > log_mle: -0.54900 (-0.37214) + | > loss_dur: 0.19532 (0.12863) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.99921 (20.97814) + | > current_lr: 0.00006 + | > step_time: 2.30940 (2.34420) + | > loader_time: 0.07790 (0.03581) + + + --> STEP: 154/234 -- GLOBAL_STEP: 53740 + | > loss: -0.31093 (-0.24602) + | > log_mle: -0.50584 (-0.37682) + | > loss_dur: 0.19490 (0.13080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.50013 (21.95117) + | > current_lr: 0.00006 + | > step_time: 4.90350 (2.38635) + | > loader_time: 0.00350 (0.03653) + + + --> STEP: 159/234 -- GLOBAL_STEP: 53745 + | > loss: -0.32893 (-0.24818) + | > log_mle: -0.52388 (-0.38120) + | > loss_dur: 0.19495 (0.13303) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.46256 (23.04898) + | > current_lr: 0.00006 + | > step_time: 6.40510 (2.46280) + | > loader_time: 0.00360 (0.03600) + + + --> STEP: 164/234 -- GLOBAL_STEP: 53750 + | > loss: -0.30428 (-0.25024) + | > log_mle: -0.51243 (-0.38527) + | > loss_dur: 0.20815 (0.13503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.30555 (24.17900) + | > current_lr: 0.00006 + | > step_time: 4.71060 (2.57859) + | > loader_time: 0.09270 (0.03726) + + + --> STEP: 169/234 -- GLOBAL_STEP: 53755 + | > loss: -0.31237 (-0.25273) + | > log_mle: -0.52238 (-0.38973) + | > loss_dur: 0.21000 (0.13700) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.09209 (25.19999) + | > current_lr: 0.00006 + | > step_time: 3.30750 (2.58655) + | > loader_time: 0.00450 (0.03630) + + + --> STEP: 174/234 -- GLOBAL_STEP: 53760 + | > loss: -0.39498 (-0.25608) + | > log_mle: -0.60825 (-0.39531) + | > loss_dur: 0.21327 (0.13923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.87784 (26.49912) + | > current_lr: 0.00006 + | > step_time: 7.79050 (2.61916) + | > loader_time: 0.20320 (0.03855) + + + --> STEP: 179/234 -- GLOBAL_STEP: 53765 + | > loss: -0.35853 (-0.25887) + | > log_mle: -0.60206 (-0.40045) + | > loss_dur: 0.24353 (0.14158) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.58613 (27.52018) + | > current_lr: 0.00006 + | > step_time: 3.91580 (2.69196) + | > loader_time: 0.09320 (0.04073) + + + --> STEP: 184/234 -- GLOBAL_STEP: 53770 + | > loss: -0.33984 (-0.26123) + | > log_mle: -0.55454 (-0.40491) + | > loss_dur: 0.21470 (0.14368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.11914 (28.76932) + | > current_lr: 0.00006 + | > step_time: 5.40540 (2.75535) + | > loader_time: 0.09020 (0.04170) + + + --> STEP: 189/234 -- GLOBAL_STEP: 53775 + | > loss: -0.34343 (-0.26363) + | > log_mle: -0.55935 (-0.40946) + | > loss_dur: 0.21592 (0.14583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.33676 (30.02633) + | > current_lr: 0.00006 + | > step_time: 1.89120 (2.82788) + | > loader_time: 0.00290 (0.04175) + + + --> STEP: 194/234 -- GLOBAL_STEP: 53780 + | > loss: -0.38162 (-0.26674) + | > log_mle: -0.59525 (-0.41419) + | > loss_dur: 0.21363 (0.14744) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.33613 (31.14726) + | > current_lr: 0.00006 + | > step_time: 4.70240 (2.86492) + | > loader_time: 0.00750 (0.04220) + + + --> STEP: 199/234 -- GLOBAL_STEP: 53785 + | > loss: -0.38370 (-0.26922) + | > log_mle: -0.60137 (-0.41845) + | > loss_dur: 0.21767 (0.14923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.18050 (32.34550) + | > current_lr: 0.00006 + | > step_time: 3.61510 (2.97285) + | > loader_time: 0.08390 (0.05161) + + + --> STEP: 204/234 -- GLOBAL_STEP: 53790 + | > loss: -0.39087 (-0.27136) + | > log_mle: -0.62450 (-0.42250) + | > loss_dur: 0.23363 (0.15114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.68216 (33.54109) + | > current_lr: 0.00006 + | > step_time: 4.88870 (3.00533) + | > loader_time: 0.00750 (0.05192) + + + --> STEP: 209/234 -- GLOBAL_STEP: 53795 + | > loss: -0.37671 (-0.27394) + | > log_mle: -0.59432 (-0.42691) + | > loss_dur: 0.21761 (0.15298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.91686 (34.70063) + | > current_lr: 0.00006 + | > step_time: 5.68810 (3.11605) + | > loader_time: 0.10480 (0.05167) + + + --> STEP: 214/234 -- GLOBAL_STEP: 53800 + | > loss: -0.40654 (-0.27739) + | > log_mle: -0.61551 (-0.43222) + | > loss_dur: 0.20897 (0.15483) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.04458 (35.88513) + | > current_lr: 0.00006 + | > step_time: 3.50170 (3.13855) + | > loader_time: 0.09210 (0.05375) + + + --> STEP: 219/234 -- GLOBAL_STEP: 53805 + | > loss: -0.46603 (-0.28032) + | > log_mle: -0.70276 (-0.43708) + | > loss_dur: 0.23673 (0.15675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.86607 (37.40856) + | > current_lr: 0.00006 + | > step_time: 3.51180 (3.24417) + | > loader_time: 0.08480 (0.05468) + + + --> STEP: 224/234 -- GLOBAL_STEP: 53810 + | > loss: -0.42356 (-0.28313) + | > log_mle: -0.66207 (-0.44175) + | > loss_dur: 0.23851 (0.15862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 114.68038 (38.41853) + | > current_lr: 0.00006 + | > step_time: 0.24120 (3.20485) + | > loader_time: 0.00440 (0.05400) + + + --> STEP: 229/234 -- GLOBAL_STEP: 53815 + | > loss: -0.38018 (-0.28554) + | > log_mle: -0.67539 (-0.44639) + | > loss_dur: 0.29521 (0.16085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.20098 (40.27811) + | > current_lr: 0.00006 + | > step_time: 0.25260 (3.14026) + | > loader_time: 0.00720 (0.05294) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32844 (-1.12576) + | > avg_loss: -0.29163 (+0.01043) + | > avg_log_mle: -0.51434 (+0.01590) + | > avg_loss_dur: 0.22271 (-0.00547) + + + > EPOCH: 230/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 09:30:04)  + + --> STEP: 0/234 -- GLOBAL_STEP: 53820 + | > loss: -0.32145 (-0.32145) + | > log_mle: -0.43609 (-0.43609) + | > loss_dur: 0.11464 (0.11464) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.47873 (29.47873) + | > current_lr: 0.00006 + | > step_time: 7.89940 (7.89937) + | > loader_time: 7.64330 (7.64327) + + + --> STEP: 5/234 -- GLOBAL_STEP: 53825 + | > loss: -0.24963 (-0.24546) + | > log_mle: -0.35649 (-0.35614) + | > loss_dur: 0.10686 (0.11068) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.94513 (22.22580) + | > current_lr: 0.00006 + | > step_time: 2.39830 (5.18214) + | > loader_time: 0.00760 (0.82047) + + + --> STEP: 10/234 -- GLOBAL_STEP: 53830 + | > loss: -0.25601 (-0.25836) + | > log_mle: -0.35347 (-0.35820) + | > loss_dur: 0.09747 (0.09985) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.70024 (18.63061) + | > current_lr: 0.00006 + | > step_time: 9.50460 (4.88063) + | > loader_time: 0.20780 (0.44788) + + + --> STEP: 15/234 -- GLOBAL_STEP: 53835 + | > loss: -0.29570 (-0.26738) + | > log_mle: -0.36590 (-0.36013) + | > loss_dur: 0.07020 (0.09275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.47679 (16.70357) + | > current_lr: 0.00006 + | > step_time: 3.80300 (5.02920) + | > loader_time: 0.00680 (0.31253) + + + --> STEP: 20/234 -- GLOBAL_STEP: 53840 + | > loss: -0.29481 (-0.27127) + | > log_mle: -0.36375 (-0.35932) + | > loss_dur: 0.06895 (0.08805) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.49992 (15.43505) + | > current_lr: 0.00006 + | > step_time: 2.71140 (5.11168) + | > loader_time: 0.19200 (0.26373) + + + --> STEP: 25/234 -- GLOBAL_STEP: 53845 + | > loss: -0.26468 (-0.27206) + | > log_mle: -0.33818 (-0.35777) + | > loss_dur: 0.07350 (0.08570) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.33164 (14.76778) + | > current_lr: 0.00006 + | > step_time: 3.40810 (4.50632) + | > loader_time: 0.00190 (0.21841) + + + --> STEP: 30/234 -- GLOBAL_STEP: 53850 + | > loss: -0.25688 (-0.27186) + | > log_mle: -0.33465 (-0.35623) + | > loss_dur: 0.07776 (0.08437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.46915 (14.40067) + | > current_lr: 0.00006 + | > step_time: 2.69310 (4.00764) + | > loader_time: 0.00180 (0.18479) + + + --> STEP: 35/234 -- GLOBAL_STEP: 53855 + | > loss: -0.21718 (-0.26864) + | > log_mle: -0.33474 (-0.35425) + | > loss_dur: 0.11756 (0.08561) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.08842 (13.88491) + | > current_lr: 0.00006 + | > step_time: 3.60320 (4.04708) + | > loader_time: 0.00310 (0.16435) + + + --> STEP: 40/234 -- GLOBAL_STEP: 53860 + | > loss: -0.22318 (-0.26439) + | > log_mle: -0.33100 (-0.35217) + | > loss_dur: 0.10782 (0.08778) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.25615 (13.44475) + | > current_lr: 0.00006 + | > step_time: 1.27890 (3.73919) + | > loader_time: 0.00170 (0.15066) + + + --> STEP: 45/234 -- GLOBAL_STEP: 53865 + | > loss: -0.23385 (-0.26212) + | > log_mle: -0.35682 (-0.35101) + | > loss_dur: 0.12297 (0.08889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.31297 (13.29895) + | > current_lr: 0.00006 + | > step_time: 1.56540 (3.45793) + | > loader_time: 0.00360 (0.13418) + + + --> STEP: 50/234 -- GLOBAL_STEP: 53870 + | > loss: -0.23340 (-0.25991) + | > log_mle: -0.33179 (-0.34975) + | > loss_dur: 0.09839 (0.08984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.59361 (12.97613) + | > current_lr: 0.00006 + | > step_time: 2.31860 (3.30473) + | > loader_time: 0.00320 (0.12451) + + + --> STEP: 55/234 -- GLOBAL_STEP: 53875 + | > loss: -0.25236 (-0.25803) + | > log_mle: -0.34360 (-0.34870) + | > loss_dur: 0.09124 (0.09068) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.77466 (12.92226) + | > current_lr: 0.00006 + | > step_time: 1.79590 (3.14356) + | > loader_time: 0.00200 (0.11507) + + + --> STEP: 60/234 -- GLOBAL_STEP: 53880 + | > loss: -0.21627 (-0.25566) + | > log_mle: -0.34997 (-0.34791) + | > loss_dur: 0.13370 (0.09225) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.75701 (12.96102) + | > current_lr: 0.00006 + | > step_time: 2.60650 (3.04282) + | > loader_time: 0.00310 (0.10570) + + + --> STEP: 65/234 -- GLOBAL_STEP: 53885 + | > loss: -0.23009 (-0.25283) + | > log_mle: -0.33394 (-0.34714) + | > loss_dur: 0.10385 (0.09431) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.21176 (13.00260) + | > current_lr: 0.00006 + | > step_time: 2.50630 (2.97984) + | > loader_time: 0.00340 (0.10047) + + + --> STEP: 70/234 -- GLOBAL_STEP: 53890 + | > loss: -0.20007 (-0.25030) + | > log_mle: -0.32377 (-0.34587) + | > loss_dur: 0.12370 (0.09556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.35628 (12.85096) + | > current_lr: 0.00006 + | > step_time: 2.68450 (2.92665) + | > loader_time: 0.10950 (0.09734) + + + --> STEP: 75/234 -- GLOBAL_STEP: 53895 + | > loss: -0.20308 (-0.24782) + | > log_mle: -0.34251 (-0.34542) + | > loss_dur: 0.13944 (0.09761) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.10669 (12.90713) + | > current_lr: 0.00006 + | > step_time: 2.61590 (2.88219) + | > loader_time: 0.08390 (0.09321) + + + --> STEP: 80/234 -- GLOBAL_STEP: 53900 + | > loss: -0.22892 (-0.24622) + | > log_mle: -0.32635 (-0.34468) + | > loss_dur: 0.09744 (0.09846) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.36215 (12.76157) + | > current_lr: 0.00006 + | > step_time: 2.54430 (2.80914) + | > loader_time: 0.08670 (0.08961) + + + --> STEP: 85/234 -- GLOBAL_STEP: 53905 + | > loss: -0.22858 (-0.24457) + | > log_mle: -0.33328 (-0.34425) + | > loss_dur: 0.10470 (0.09968) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.41615 (12.88467) + | > current_lr: 0.00006 + | > step_time: 2.38750 (2.78795) + | > loader_time: 0.00170 (0.08672) + + + --> STEP: 90/234 -- GLOBAL_STEP: 53910 + | > loss: -0.21443 (-0.24324) + | > log_mle: -0.35378 (-0.34483) + | > loss_dur: 0.13935 (0.10159) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.18551 (13.00444) + | > current_lr: 0.00006 + | > step_time: 1.43640 (2.73017) + | > loader_time: 0.00320 (0.08213) + + + --> STEP: 95/234 -- GLOBAL_STEP: 53915 + | > loss: -0.26762 (-0.24302) + | > log_mle: -0.44015 (-0.34720) + | > loss_dur: 0.17253 (0.10418) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.37761 (13.69107) + | > current_lr: 0.00006 + | > step_time: 1.69380 (2.69840) + | > loader_time: 0.10420 (0.08073) + + + --> STEP: 100/234 -- GLOBAL_STEP: 53920 + | > loss: -0.22977 (-0.24221) + | > log_mle: -0.36620 (-0.34797) + | > loss_dur: 0.13643 (0.10576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.72224 (13.89255) + | > current_lr: 0.00006 + | > step_time: 1.24460 (2.69246) + | > loader_time: 0.00240 (0.08155) + + + --> STEP: 105/234 -- GLOBAL_STEP: 53925 + | > loss: -0.22345 (-0.24188) + | > log_mle: -0.34735 (-0.34985) + | > loss_dur: 0.12390 (0.10797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.30513 (14.33272) + | > current_lr: 0.00006 + | > step_time: 3.49880 (2.70250) + | > loader_time: 0.00330 (0.07862) + + + --> STEP: 110/234 -- GLOBAL_STEP: 53930 + | > loss: -0.21548 (-0.24072) + | > log_mle: -0.36839 (-0.35140) + | > loss_dur: 0.15291 (0.11067) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.86423 (14.70079) + | > current_lr: 0.00006 + | > step_time: 1.12240 (2.67112) + | > loader_time: 0.00210 (0.07592) + + + --> STEP: 115/234 -- GLOBAL_STEP: 53935 + | > loss: -0.21415 (-0.24064) + | > log_mle: -0.39064 (-0.35382) + | > loss_dur: 0.17649 (0.11318) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.43716 (15.20440) + | > current_lr: 0.00006 + | > step_time: 3.95750 (2.64616) + | > loader_time: 0.09420 (0.07428) + + + --> STEP: 120/234 -- GLOBAL_STEP: 53940 + | > loss: -0.26566 (-0.24050) + | > log_mle: -0.44045 (-0.35590) + | > loss_dur: 0.17479 (0.11541) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.57424 (15.69956) + | > current_lr: 0.00006 + | > step_time: 2.69750 (2.61877) + | > loader_time: 0.00460 (0.07280) + + + --> STEP: 125/234 -- GLOBAL_STEP: 53945 + | > loss: -0.24678 (-0.23973) + | > log_mle: -0.42525 (-0.35673) + | > loss_dur: 0.17846 (0.11700) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.20314 (16.01896) + | > current_lr: 0.00006 + | > step_time: 1.10500 (2.60133) + | > loader_time: 0.00370 (0.07224) + + + --> STEP: 130/234 -- GLOBAL_STEP: 53950 + | > loss: -0.26814 (-0.24034) + | > log_mle: -0.44173 (-0.35944) + | > loss_dur: 0.17360 (0.11910) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.03416 (16.87436) + | > current_lr: 0.00006 + | > step_time: 1.49940 (2.56798) + | > loader_time: 0.00280 (0.06962) + + + --> STEP: 135/234 -- GLOBAL_STEP: 53955 + | > loss: -0.22285 (-0.24119) + | > log_mle: -0.36629 (-0.36228) + | > loss_dur: 0.14344 (0.12109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.38091 (17.60033) + | > current_lr: 0.00006 + | > step_time: 1.68510 (2.57589) + | > loader_time: 0.10460 (0.06929) + + + --> STEP: 140/234 -- GLOBAL_STEP: 53960 + | > loss: -0.21590 (-0.24204) + | > log_mle: -0.40340 (-0.36573) + | > loss_dur: 0.18750 (0.12369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.71889 (18.52623) + | > current_lr: 0.00006 + | > step_time: 2.81140 (2.55530) + | > loader_time: 0.08520 (0.06751) + + + --> STEP: 145/234 -- GLOBAL_STEP: 53965 + | > loss: -0.32100 (-0.24364) + | > log_mle: -0.51074 (-0.36988) + | > loss_dur: 0.18974 (0.12625) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.36313 (19.50316) + | > current_lr: 0.00006 + | > step_time: 2.20470 (2.54120) + | > loader_time: 0.00430 (0.06637) + + + --> STEP: 150/234 -- GLOBAL_STEP: 53970 + | > loss: -0.27702 (-0.24523) + | > log_mle: -0.47305 (-0.37351) + | > loss_dur: 0.19603 (0.12828) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.81803 (20.58995) + | > current_lr: 0.00006 + | > step_time: 1.59020 (2.55054) + | > loader_time: 0.00400 (0.06665) + + + --> STEP: 155/234 -- GLOBAL_STEP: 53975 + | > loss: -0.33579 (-0.24763) + | > log_mle: -0.53418 (-0.37805) + | > loss_dur: 0.19839 (0.13043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.89995 (21.61866) + | > current_lr: 0.00006 + | > step_time: 1.10580 (2.53862) + | > loader_time: 0.00510 (0.06464) + + + --> STEP: 160/234 -- GLOBAL_STEP: 53980 + | > loss: -0.32670 (-0.24907) + | > log_mle: -0.54219 (-0.38194) + | > loss_dur: 0.21549 (0.13286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.74214 (22.58056) + | > current_lr: 0.00006 + | > step_time: 2.50100 (2.52681) + | > loader_time: 0.09230 (0.06485) + + + --> STEP: 165/234 -- GLOBAL_STEP: 53985 + | > loss: -0.33984 (-0.25102) + | > log_mle: -0.54569 (-0.38583) + | > loss_dur: 0.20585 (0.13480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.30994 (23.42480) + | > current_lr: 0.00006 + | > step_time: 9.00630 (2.59208) + | > loader_time: 0.08540 (0.06518) + + + --> STEP: 170/234 -- GLOBAL_STEP: 53990 + | > loss: -0.36045 (-0.25337) + | > log_mle: -0.58335 (-0.39023) + | > loss_dur: 0.22290 (0.13686) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.60479 (24.32249) + | > current_lr: 0.00006 + | > step_time: 6.50060 (2.65518) + | > loader_time: 0.00350 (0.06452) + + + --> STEP: 175/234 -- GLOBAL_STEP: 53995 + | > loss: -0.31655 (-0.25619) + | > log_mle: -0.55621 (-0.39539) + | > loss_dur: 0.23966 (0.13920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.78873 (25.68019) + | > current_lr: 0.00006 + | > step_time: 3.89080 (2.74669) + | > loader_time: 0.09060 (0.06713) + + + --> STEP: 180/234 -- GLOBAL_STEP: 54000 + | > loss: -0.35686 (-0.25886) + | > log_mle: -0.56838 (-0.40035) + | > loss_dur: 0.21152 (0.14150) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.77786 (26.65717) + | > current_lr: 0.00006 + | > step_time: 1.41530 (2.80055) + | > loader_time: 0.09550 (0.06787) + + + --> STEP: 185/234 -- GLOBAL_STEP: 54005 + | > loss: -0.36743 (-0.26121) + | > log_mle: -0.59863 (-0.40487) + | > loss_dur: 0.23120 (0.14366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.95995 (27.71780) + | > current_lr: 0.00006 + | > step_time: 2.30290 (2.81203) + | > loader_time: 0.00260 (0.06823) + + + --> STEP: 190/234 -- GLOBAL_STEP: 54010 + | > loss: -0.34162 (-0.26341) + | > log_mle: -0.54128 (-0.40905) + | > loss_dur: 0.19966 (0.14564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.76679 (29.05962) + | > current_lr: 0.00006 + | > step_time: 1.37810 (2.79591) + | > loader_time: 0.00310 (0.06913) + + + --> STEP: 195/234 -- GLOBAL_STEP: 54015 + | > loss: -0.36075 (-0.26619) + | > log_mle: -0.58616 (-0.41353) + | > loss_dur: 0.22541 (0.14734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.45186 (30.02976) + | > current_lr: 0.00006 + | > step_time: 3.69390 (2.83451) + | > loader_time: 0.00380 (0.06926) + + + --> STEP: 200/234 -- GLOBAL_STEP: 54020 + | > loss: -0.34965 (-0.26857) + | > log_mle: -0.59414 (-0.41782) + | > loss_dur: 0.24448 (0.14925) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.54611 (31.10614) + | > current_lr: 0.00006 + | > step_time: 1.28790 (2.93270) + | > loader_time: 0.00450 (0.06852) + + + --> STEP: 205/234 -- GLOBAL_STEP: 54025 + | > loss: -0.35694 (-0.27082) + | > log_mle: -0.58145 (-0.42196) + | > loss_dur: 0.22452 (0.15114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.98605 (31.95992) + | > current_lr: 0.00006 + | > step_time: 5.10990 (2.93782) + | > loader_time: 0.00420 (0.06783) + + + --> STEP: 210/234 -- GLOBAL_STEP: 54030 + | > loss: -0.41989 (-0.27382) + | > log_mle: -0.65965 (-0.42685) + | > loss_dur: 0.23976 (0.15304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.87787 (32.93946) + | > current_lr: 0.00006 + | > step_time: 4.40110 (3.00647) + | > loader_time: 0.00410 (0.06915) + + + --> STEP: 215/234 -- GLOBAL_STEP: 54035 + | > loss: -0.38289 (-0.27695) + | > log_mle: -0.61158 (-0.43174) + | > loss_dur: 0.22869 (0.15479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.09436 (34.55693) + | > current_lr: 0.00006 + | > step_time: 5.59830 (3.06225) + | > loader_time: 0.00990 (0.06806) + + + --> STEP: 220/234 -- GLOBAL_STEP: 54040 + | > loss: -0.42118 (-0.28050) + | > log_mle: -0.65907 (-0.43718) + | > loss_dur: 0.23790 (0.15668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.76155 (35.69146) + | > current_lr: 0.00006 + | > step_time: 2.99630 (3.12806) + | > loader_time: 0.00440 (0.06795) + + + --> STEP: 225/234 -- GLOBAL_STEP: 54045 + | > loss: -0.49021 (-0.28380) + | > log_mle: -0.73954 (-0.44237) + | > loss_dur: 0.24934 (0.15858) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.65067 (36.81469) + | > current_lr: 0.00006 + | > step_time: 0.24280 (3.08666) + | > loader_time: 0.00440 (0.06726) + + + --> STEP: 230/234 -- GLOBAL_STEP: 54050 + | > loss: -0.47688 (-0.28710) + | > log_mle: -0.79363 (-0.44821) + | > loss_dur: 0.31676 (0.16111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 109.64641 (38.05076) + | > current_lr: 0.00006 + | > step_time: 0.26660 (3.02504) + | > loader_time: 0.00370 (0.06588) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.46294 (+0.13451) + | > avg_loss: -0.30827 (-0.01664) + | > avg_log_mle: -0.52630 (-0.01196) + | > avg_loss_dur: 0.21803 (-0.00468) + + + > EPOCH: 231/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 09:42:59)  + + --> STEP: 1/234 -- GLOBAL_STEP: 54055 + | > loss: -0.24099 (-0.24099) + | > log_mle: -0.35184 (-0.35184) + | > loss_dur: 0.11085 (0.11085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.02104 (38.02104) + | > current_lr: 0.00006 + | > step_time: 14.20520 (14.20517) + | > loader_time: 0.00450 (0.00454) + + + --> STEP: 6/234 -- GLOBAL_STEP: 54060 + | > loss: -0.28994 (-0.25181) + | > log_mle: -0.35462 (-0.35412) + | > loss_dur: 0.06468 (0.10231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.20801 (25.47829) + | > current_lr: 0.00006 + | > step_time: 0.71270 (6.73956) + | > loader_time: 0.08220 (0.09225) + + + --> STEP: 11/234 -- GLOBAL_STEP: 54065 + | > loss: -0.29515 (-0.26231) + | > log_mle: -0.36646 (-0.35830) + | > loss_dur: 0.07132 (0.09599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.85678 (22.59360) + | > current_lr: 0.00006 + | > step_time: 9.49790 (5.78581) + | > loader_time: 0.00110 (0.05836) + + + --> STEP: 16/234 -- GLOBAL_STEP: 54070 + | > loss: -0.28764 (-0.26869) + | > log_mle: -0.36667 (-0.36051) + | > loss_dur: 0.07904 (0.09182) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.86809 (19.68856) + | > current_lr: 0.00006 + | > step_time: 4.59280 (5.46606) + | > loader_time: 0.19600 (0.06456) + + + --> STEP: 21/234 -- GLOBAL_STEP: 54075 + | > loss: -0.26665 (-0.27035) + | > log_mle: -0.34450 (-0.35897) + | > loss_dur: 0.07786 (0.08862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.80128 (17.46284) + | > current_lr: 0.00006 + | > step_time: 7.32670 (5.23660) + | > loader_time: 0.09870 (0.05437) + + + --> STEP: 26/234 -- GLOBAL_STEP: 54080 + | > loss: -0.26983 (-0.27203) + | > log_mle: -0.35326 (-0.35848) + | > loss_dur: 0.08343 (0.08645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45547 (16.07097) + | > current_lr: 0.00006 + | > step_time: 1.00960 (4.49422) + | > loader_time: 0.00210 (0.04436) + + + --> STEP: 31/234 -- GLOBAL_STEP: 54085 + | > loss: -0.23305 (-0.27143) + | > log_mle: -0.34161 (-0.35716) + | > loss_dur: 0.10857 (0.08573) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.09616 (15.44782) + | > current_lr: 0.00006 + | > step_time: 3.70030 (4.16013) + | > loader_time: 0.09660 (0.04353) + + + --> STEP: 36/234 -- GLOBAL_STEP: 54090 + | > loss: -0.22536 (-0.26822) + | > log_mle: -0.33598 (-0.35555) + | > loss_dur: 0.11062 (0.08733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.27836 (14.98877) + | > current_lr: 0.00006 + | > step_time: 6.30480 (4.41094) + | > loader_time: 0.09290 (0.04796) + + + --> STEP: 41/234 -- GLOBAL_STEP: 54095 + | > loss: -0.27310 (-0.26581) + | > log_mle: -0.34932 (-0.35404) + | > loss_dur: 0.07622 (0.08824) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.71310 (14.50575) + | > current_lr: 0.00006 + | > step_time: 0.89480 (4.16757) + | > loader_time: 0.00300 (0.04398) + + + --> STEP: 46/234 -- GLOBAL_STEP: 54100 + | > loss: -0.22910 (-0.26267) + | > log_mle: -0.33995 (-0.35265) + | > loss_dur: 0.11084 (0.08998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.82107 (14.22301) + | > current_lr: 0.00006 + | > step_time: 1.07170 (3.97674) + | > loader_time: 0.00190 (0.04138) + + + --> STEP: 51/234 -- GLOBAL_STEP: 54105 + | > loss: -0.23688 (-0.26140) + | > log_mle: -0.33268 (-0.35143) + | > loss_dur: 0.09580 (0.09002) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.14179 (13.71420) + | > current_lr: 0.00006 + | > step_time: 2.29320 (3.76096) + | > loader_time: 0.00500 (0.03964) + + + --> STEP: 56/234 -- GLOBAL_STEP: 54110 + | > loss: -0.22701 (-0.25932) + | > log_mle: -0.34133 (-0.35069) + | > loss_dur: 0.11432 (0.09137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.52040 (13.39953) + | > current_lr: 0.00006 + | > step_time: 0.86190 (3.55022) + | > loader_time: 0.00210 (0.03804) + + + --> STEP: 61/234 -- GLOBAL_STEP: 54115 + | > loss: -0.23998 (-0.25757) + | > log_mle: -0.34049 (-0.35002) + | > loss_dur: 0.10051 (0.09245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.76266 (13.20316) + | > current_lr: 0.00006 + | > step_time: 1.60300 (3.38862) + | > loader_time: 0.00250 (0.03644) + + + --> STEP: 66/234 -- GLOBAL_STEP: 54120 + | > loss: -0.24449 (-0.25499) + | > log_mle: -0.33419 (-0.34925) + | > loss_dur: 0.08969 (0.09426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.75919 (13.19715) + | > current_lr: 0.00006 + | > step_time: 1.58780 (3.28954) + | > loader_time: 0.00220 (0.03626) + + + --> STEP: 71/234 -- GLOBAL_STEP: 54125 + | > loss: -0.21540 (-0.25230) + | > log_mle: -0.35905 (-0.34844) + | > loss_dur: 0.14365 (0.09614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.59328 (13.23483) + | > current_lr: 0.00006 + | > step_time: 2.48740 (3.22489) + | > loader_time: 0.00320 (0.03509) + + + --> STEP: 76/234 -- GLOBAL_STEP: 54130 + | > loss: -0.22610 (-0.24997) + | > log_mle: -0.34331 (-0.34773) + | > loss_dur: 0.11720 (0.09776) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.64154 (13.27287) + | > current_lr: 0.00006 + | > step_time: 1.20770 (3.15000) + | > loader_time: 0.08280 (0.03642) + + + --> STEP: 81/234 -- GLOBAL_STEP: 54135 + | > loss: -0.22708 (-0.24824) + | > log_mle: -0.34730 (-0.34700) + | > loss_dur: 0.12022 (0.09876) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.52063 (13.17562) + | > current_lr: 0.00006 + | > step_time: 1.61170 (3.08402) + | > loader_time: 0.00220 (0.03547) + + + --> STEP: 86/234 -- GLOBAL_STEP: 54140 + | > loss: -0.22863 (-0.24667) + | > log_mle: -0.34793 (-0.34657) + | > loss_dur: 0.11930 (0.09990) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.90054 (13.25539) + | > current_lr: 0.00006 + | > step_time: 4.40270 (3.08487) + | > loader_time: 0.00470 (0.03467) + + + --> STEP: 91/234 -- GLOBAL_STEP: 54145 + | > loss: -0.21516 (-0.24540) + | > log_mle: -0.35835 (-0.34726) + | > loss_dur: 0.14319 (0.10186) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.88196 (13.40068) + | > current_lr: 0.00006 + | > step_time: 4.90990 (3.04496) + | > loader_time: 0.00340 (0.03383) + + + --> STEP: 96/234 -- GLOBAL_STEP: 54150 + | > loss: -0.21657 (-0.24519) + | > log_mle: -0.34277 (-0.34937) + | > loss_dur: 0.12620 (0.10419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.75435 (13.94778) + | > current_lr: 0.00006 + | > step_time: 1.44910 (2.99795) + | > loader_time: 0.00140 (0.03415) + + + --> STEP: 101/234 -- GLOBAL_STEP: 54155 + | > loss: -0.23116 (-0.24424) + | > log_mle: -0.39351 (-0.35059) + | > loss_dur: 0.16235 (0.10635) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.94572 (14.27584) + | > current_lr: 0.00006 + | > step_time: 2.50930 (2.96057) + | > loader_time: 0.00260 (0.03440) + + + --> STEP: 106/234 -- GLOBAL_STEP: 54160 + | > loss: -0.20191 (-0.24357) + | > log_mle: -0.39030 (-0.35240) + | > loss_dur: 0.18838 (0.10883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.39119 (14.67943) + | > current_lr: 0.00006 + | > step_time: 1.01580 (2.91640) + | > loader_time: 0.08800 (0.03371) + + + --> STEP: 111/234 -- GLOBAL_STEP: 54165 + | > loss: -0.25514 (-0.24266) + | > log_mle: -0.44157 (-0.35414) + | > loss_dur: 0.18644 (0.11148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.26112 (15.31569) + | > current_lr: 0.00006 + | > step_time: 1.87530 (2.89906) + | > loader_time: 0.01220 (0.03424) + + + --> STEP: 116/234 -- GLOBAL_STEP: 54170 + | > loss: -0.21625 (-0.24211) + | > log_mle: -0.40501 (-0.35602) + | > loss_dur: 0.18877 (0.11390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.82887 (15.79195) + | > current_lr: 0.00006 + | > step_time: 3.69440 (2.85352) + | > loader_time: 0.00420 (0.03291) + + + --> STEP: 121/234 -- GLOBAL_STEP: 54175 + | > loss: -0.19308 (-0.24145) + | > log_mle: -0.32389 (-0.35720) + | > loss_dur: 0.13081 (0.11575) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.71817 (16.04705) + | > current_lr: 0.00006 + | > step_time: 1.81020 (2.79540) + | > loader_time: 0.00340 (0.03236) + + + --> STEP: 126/234 -- GLOBAL_STEP: 54180 + | > loss: -0.27412 (-0.24138) + | > log_mle: -0.45805 (-0.35897) + | > loss_dur: 0.18393 (0.11759) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.95477 (16.60673) + | > current_lr: 0.00006 + | > step_time: 5.50890 (2.80860) + | > loader_time: 0.19150 (0.03271) + + + --> STEP: 131/234 -- GLOBAL_STEP: 54185 + | > loss: -0.31247 (-0.24218) + | > log_mle: -0.50972 (-0.36200) + | > loss_dur: 0.19725 (0.11982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.45967 (17.39253) + | > current_lr: 0.00006 + | > step_time: 1.82350 (2.77484) + | > loader_time: 0.08740 (0.03223) + + + --> STEP: 136/234 -- GLOBAL_STEP: 54190 + | > loss: -0.33414 (-0.24308) + | > log_mle: -0.54819 (-0.36504) + | > loss_dur: 0.21405 (0.12196) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.03387 (18.18453) + | > current_lr: 0.00006 + | > step_time: 0.94840 (2.78832) + | > loader_time: 0.00290 (0.03395) + + + --> STEP: 141/234 -- GLOBAL_STEP: 54195 + | > loss: -0.27526 (-0.24348) + | > log_mle: -0.45038 (-0.36754) + | > loss_dur: 0.17512 (0.12407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.38156 (18.86988) + | > current_lr: 0.00006 + | > step_time: 8.30710 (2.80007) + | > loader_time: 0.00520 (0.03406) + + + --> STEP: 146/234 -- GLOBAL_STEP: 54200 + | > loss: -0.30675 (-0.24529) + | > log_mle: -0.50597 (-0.37198) + | > loss_dur: 0.19922 (0.12668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.54003 (19.84792) + | > current_lr: 0.00006 + | > step_time: 1.91530 (2.79553) + | > loader_time: 0.10880 (0.03484) + + + --> STEP: 151/234 -- GLOBAL_STEP: 54205 + | > loss: -0.28504 (-0.24702) + | > log_mle: -0.46596 (-0.37553) + | > loss_dur: 0.18092 (0.12851) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.14463 (20.72418) + | > current_lr: 0.00006 + | > step_time: 2.79330 (2.78442) + | > loader_time: 0.00460 (0.03447) + + + --> STEP: 156/234 -- GLOBAL_STEP: 54210 + | > loss: -0.32476 (-0.24990) + | > log_mle: -0.51795 (-0.38065) + | > loss_dur: 0.19319 (0.13075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.80629 (21.99647) + | > current_lr: 0.00006 + | > step_time: 1.34430 (2.83472) + | > loader_time: 0.18550 (0.03587) + + + --> STEP: 161/234 -- GLOBAL_STEP: 54215 + | > loss: -0.35707 (-0.25197) + | > log_mle: -0.54422 (-0.38498) + | > loss_dur: 0.18715 (0.13301) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.84034 (23.00991) + | > current_lr: 0.00006 + | > step_time: 2.19460 (2.80047) + | > loader_time: 0.00280 (0.03548) + + + --> STEP: 166/234 -- GLOBAL_STEP: 54220 + | > loss: -0.28870 (-0.25371) + | > log_mle: -0.47145 (-0.38858) + | > loss_dur: 0.18275 (0.13487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.03646 (24.05232) + | > current_lr: 0.00006 + | > step_time: 6.30910 (2.80167) + | > loader_time: 0.08860 (0.03600) + + + --> STEP: 171/234 -- GLOBAL_STEP: 54225 + | > loss: -0.37490 (-0.25657) + | > log_mle: -0.58421 (-0.39368) + | > loss_dur: 0.20931 (0.13711) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.81853 (25.33765) + | > current_lr: 0.00006 + | > step_time: 2.90660 (2.81219) + | > loader_time: 0.09520 (0.03621) + + + --> STEP: 176/234 -- GLOBAL_STEP: 54230 + | > loss: -0.35722 (-0.25931) + | > log_mle: -0.56241 (-0.39863) + | > loss_dur: 0.20519 (0.13932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.41642 (26.42862) + | > current_lr: 0.00006 + | > step_time: 3.59580 (2.79818) + | > loader_time: 0.00440 (0.03730) + + + --> STEP: 181/234 -- GLOBAL_STEP: 54235 + | > loss: -0.27819 (-0.26115) + | > log_mle: -0.48542 (-0.40276) + | > loss_dur: 0.20723 (0.14160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.69543 (27.61831) + | > current_lr: 0.00006 + | > step_time: 7.90730 (2.91218) + | > loader_time: 0.18370 (0.03996) + + + --> STEP: 186/234 -- GLOBAL_STEP: 54240 + | > loss: -0.29626 (-0.26326) + | > log_mle: -0.52786 (-0.40723) + | > loss_dur: 0.23161 (0.14397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.51144 (28.57223) + | > current_lr: 0.00006 + | > step_time: 1.80010 (2.93073) + | > loader_time: 0.00490 (0.04036) + + + --> STEP: 191/234 -- GLOBAL_STEP: 54245 + | > loss: -0.33826 (-0.26577) + | > log_mle: -0.55223 (-0.41161) + | > loss_dur: 0.21396 (0.14584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.62004 (29.58640) + | > current_lr: 0.00006 + | > step_time: 3.01480 (2.95473) + | > loader_time: 0.19630 (0.04087) + + + --> STEP: 196/234 -- GLOBAL_STEP: 54250 + | > loss: -0.32201 (-0.26856) + | > log_mle: -0.53990 (-0.41618) + | > loss_dur: 0.21788 (0.14763) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.42165 (30.62197) + | > current_lr: 0.00006 + | > step_time: 7.20500 (3.02617) + | > loader_time: 0.20330 (0.04187) + + + --> STEP: 201/234 -- GLOBAL_STEP: 54255 + | > loss: -0.27173 (-0.27066) + | > log_mle: -0.49810 (-0.42020) + | > loss_dur: 0.22637 (0.14954) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.70142 (31.59650) + | > current_lr: 0.00006 + | > step_time: 4.11500 (3.06952) + | > loader_time: 0.58710 (0.04612) + + + --> STEP: 206/234 -- GLOBAL_STEP: 54260 + | > loss: -0.41496 (-0.27353) + | > log_mle: -0.62940 (-0.42483) + | > loss_dur: 0.21444 (0.15131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.56831 (32.59645) + | > current_lr: 0.00006 + | > step_time: 3.40590 (3.08538) + | > loader_time: 0.10310 (0.04733) + + + --> STEP: 211/234 -- GLOBAL_STEP: 54265 + | > loss: -0.45619 (-0.27692) + | > log_mle: -0.70154 (-0.43014) + | > loss_dur: 0.24535 (0.15323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.75494 (33.85546) + | > current_lr: 0.00006 + | > step_time: 4.81090 (3.11802) + | > loader_time: 0.09590 (0.04681) + + + --> STEP: 216/234 -- GLOBAL_STEP: 54270 + | > loss: -0.44794 (-0.28029) + | > log_mle: -0.69135 (-0.43536) + | > loss_dur: 0.24341 (0.15507) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.01711 (35.04816) + | > current_lr: 0.00006 + | > step_time: 4.40160 (3.14688) + | > loader_time: 0.00430 (0.04753) + + + --> STEP: 221/234 -- GLOBAL_STEP: 54275 + | > loss: -0.37041 (-0.28364) + | > log_mle: -0.59267 (-0.44051) + | > loss_dur: 0.22226 (0.15687) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.21507 (36.18055) + | > current_lr: 0.00006 + | > step_time: 1.69460 (3.16525) + | > loader_time: 0.00440 (0.04826) + + + --> STEP: 226/234 -- GLOBAL_STEP: 54280 + | > loss: -0.46159 (-0.28733) + | > log_mle: -0.70045 (-0.44615) + | > loss_dur: 0.23886 (0.15882) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.61613 (37.44809) + | > current_lr: 0.00006 + | > step_time: 0.24700 (3.10886) + | > loader_time: 0.00530 (0.04763) + + + --> STEP: 231/234 -- GLOBAL_STEP: 54285 + | > loss: -0.40162 (-0.29031) + | > log_mle: -0.77950 (-0.45222) + | > loss_dur: 0.37787 (0.16191) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 130.94749 (38.94918) + | > current_lr: 0.00006 + | > step_time: 0.26550 (3.04703) + | > loader_time: 0.00380 (0.04667) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00438 (-0.45856) + | > avg_loss: -0.31932 (-0.01105) + | > avg_log_mle: -0.53971 (-0.01341) + | > avg_loss_dur: 0.22039 (+0.00236) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_54288.pth + + > EPOCH: 232/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 09:55:58)  + + --> STEP: 2/234 -- GLOBAL_STEP: 54290 + | > loss: -0.27577 (-0.27238) + | > log_mle: -0.37468 (-0.36524) + | > loss_dur: 0.09891 (0.09286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.03367 (19.42827) + | > current_lr: 0.00006 + | > step_time: 0.90750 (1.56041) + | > loader_time: 0.00110 (0.00266) + + + --> STEP: 7/234 -- GLOBAL_STEP: 54295 + | > loss: -0.29407 (-0.26092) + | > log_mle: -0.36377 (-0.35963) + | > loss_dur: 0.06969 (0.09871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.86599 (20.48588) + | > current_lr: 0.00006 + | > step_time: 2.99650 (3.54363) + | > loader_time: 0.00290 (0.02859) + + + --> STEP: 12/234 -- GLOBAL_STEP: 54300 + | > loss: -0.27832 (-0.26822) + | > log_mle: -0.36077 (-0.36216) + | > loss_dur: 0.08245 (0.09394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.67428 (18.88950) + | > current_lr: 0.00006 + | > step_time: 0.98260 (3.24691) + | > loader_time: 0.00320 (0.05119) + + + --> STEP: 17/234 -- GLOBAL_STEP: 54305 + | > loss: -0.29219 (-0.27383) + | > log_mle: -0.35548 (-0.36358) + | > loss_dur: 0.06328 (0.08976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.74113 (17.20119) + | > current_lr: 0.00006 + | > step_time: 0.81250 (2.72801) + | > loader_time: 0.00260 (0.04130) + + + --> STEP: 22/234 -- GLOBAL_STEP: 54310 + | > loss: -0.27530 (-0.27458) + | > log_mle: -0.36120 (-0.36181) + | > loss_dur: 0.08590 (0.08723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.99874 (16.67070) + | > current_lr: 0.00006 + | > step_time: 1.61310 (2.81345) + | > loader_time: 0.00340 (0.04416) + + + --> STEP: 27/234 -- GLOBAL_STEP: 54315 + | > loss: -0.27399 (-0.27497) + | > log_mle: -0.35487 (-0.36038) + | > loss_dur: 0.08087 (0.08541) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.12244 (15.81173) + | > current_lr: 0.00006 + | > step_time: 2.40730 (2.67270) + | > loader_time: 0.09600 (0.03978) + + + --> STEP: 32/234 -- GLOBAL_STEP: 54320 + | > loss: -0.27882 (-0.27371) + | > log_mle: -0.35805 (-0.35895) + | > loss_dur: 0.07922 (0.08523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.81423 (15.06898) + | > current_lr: 0.00006 + | > step_time: 4.41170 (2.88375) + | > loader_time: 0.18410 (0.06641) + + + --> STEP: 37/234 -- GLOBAL_STEP: 54325 + | > loss: -0.25523 (-0.26974) + | > log_mle: -0.33613 (-0.35671) + | > loss_dur: 0.08090 (0.08696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.81130 (14.59476) + | > current_lr: 0.00006 + | > step_time: 1.11470 (2.96160) + | > loader_time: 0.00210 (0.06264) + + + --> STEP: 42/234 -- GLOBAL_STEP: 54330 + | > loss: -0.23013 (-0.26688) + | > log_mle: -0.32856 (-0.35498) + | > loss_dur: 0.09844 (0.08810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.91159 (14.24683) + | > current_lr: 0.00006 + | > step_time: 1.28600 (2.82111) + | > loader_time: 0.00780 (0.05559) + + + --> STEP: 47/234 -- GLOBAL_STEP: 54335 + | > loss: -0.23424 (-0.26414) + | > log_mle: -0.33885 (-0.35378) + | > loss_dur: 0.10461 (0.08964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.61452 (14.11097) + | > current_lr: 0.00006 + | > step_time: 1.89360 (2.67716) + | > loader_time: 0.00170 (0.04989) + + + --> STEP: 52/234 -- GLOBAL_STEP: 54340 + | > loss: -0.22526 (-0.26234) + | > log_mle: -0.33969 (-0.35247) + | > loss_dur: 0.11443 (0.09013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.95236 (13.70875) + | > current_lr: 0.00006 + | > step_time: 3.31340 (2.60257) + | > loader_time: 0.00190 (0.04880) + + + --> STEP: 57/234 -- GLOBAL_STEP: 54345 + | > loss: -0.22500 (-0.26069) + | > log_mle: -0.32830 (-0.35158) + | > loss_dur: 0.10330 (0.09088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.65926 (13.45786) + | > current_lr: 0.00006 + | > step_time: 2.40760 (2.50991) + | > loader_time: 0.00340 (0.04615) + + + --> STEP: 62/234 -- GLOBAL_STEP: 54350 + | > loss: -0.18936 (-0.25854) + | > log_mle: -0.36303 (-0.35153) + | > loss_dur: 0.17367 (0.09299) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.51579 (13.46482) + | > current_lr: 0.00006 + | > step_time: 2.40500 (2.51471) + | > loader_time: 0.00540 (0.04688) + + + --> STEP: 67/234 -- GLOBAL_STEP: 54355 + | > loss: -0.22832 (-0.25658) + | > log_mle: -0.34803 (-0.35048) + | > loss_dur: 0.11971 (0.09390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.50280 (13.27024) + | > current_lr: 0.00006 + | > step_time: 2.80960 (2.50177) + | > loader_time: 0.08570 (0.04486) + + + --> STEP: 72/234 -- GLOBAL_STEP: 54360 + | > loss: -0.22634 (-0.25392) + | > log_mle: -0.33580 (-0.34944) + | > loss_dur: 0.10946 (0.09553) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.44611 (13.23950) + | > current_lr: 0.00006 + | > step_time: 1.29960 (2.44867) + | > loader_time: 0.00220 (0.04189) + + + --> STEP: 77/234 -- GLOBAL_STEP: 54365 + | > loss: -0.23058 (-0.25147) + | > log_mle: -0.33783 (-0.34875) + | > loss_dur: 0.10725 (0.09728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.53143 (13.21437) + | > current_lr: 0.00006 + | > step_time: 2.70090 (2.43946) + | > loader_time: 0.08790 (0.04258) + + + --> STEP: 82/234 -- GLOBAL_STEP: 54370 + | > loss: -0.21546 (-0.24947) + | > log_mle: -0.33341 (-0.34793) + | > loss_dur: 0.11794 (0.09845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.26357 (13.12885) + | > current_lr: 0.00006 + | > step_time: 2.60100 (2.42079) + | > loader_time: 0.07850 (0.04107) + + + --> STEP: 87/234 -- GLOBAL_STEP: 54375 + | > loss: -0.21021 (-0.24757) + | > log_mle: -0.33663 (-0.34749) + | > loss_dur: 0.12642 (0.09992) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.93903 (13.16409) + | > current_lr: 0.00006 + | > step_time: 2.29960 (2.40942) + | > loader_time: 0.00260 (0.04051) + + + --> STEP: 92/234 -- GLOBAL_STEP: 54380 + | > loss: -0.24617 (-0.24661) + | > log_mle: -0.37985 (-0.34856) + | > loss_dur: 0.13368 (0.10195) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.71701 (13.50588) + | > current_lr: 0.00006 + | > step_time: 1.86090 (2.42690) + | > loader_time: 0.00240 (0.03850) + + + --> STEP: 97/234 -- GLOBAL_STEP: 54385 + | > loss: -0.23502 (-0.24634) + | > log_mle: -0.36586 (-0.35052) + | > loss_dur: 0.13085 (0.10418) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.05214 (13.90085) + | > current_lr: 0.00006 + | > step_time: 5.39670 (2.46181) + | > loader_time: 0.10440 (0.03945) + + + --> STEP: 102/234 -- GLOBAL_STEP: 54390 + | > loss: -0.19768 (-0.24526) + | > log_mle: -0.34966 (-0.35157) + | > loss_dur: 0.15198 (0.10631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.37547 (14.25011) + | > current_lr: 0.00006 + | > step_time: 9.28850 (2.51264) + | > loader_time: 0.10270 (0.04193) + + + --> STEP: 107/234 -- GLOBAL_STEP: 54395 + | > loss: -0.22417 (-0.24483) + | > log_mle: -0.38360 (-0.35357) + | > loss_dur: 0.15944 (0.10874) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.82005 (14.92415) + | > current_lr: 0.00006 + | > step_time: 1.50100 (2.48952) + | > loader_time: 0.00300 (0.04097) + + + --> STEP: 112/234 -- GLOBAL_STEP: 54400 + | > loss: -0.23033 (-0.24401) + | > log_mle: -0.39895 (-0.35543) + | > loss_dur: 0.16862 (0.11142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.76057 (15.59310) + | > current_lr: 0.00006 + | > step_time: 1.89690 (2.46117) + | > loader_time: 0.00260 (0.03931) + + + --> STEP: 117/234 -- GLOBAL_STEP: 54405 + | > loss: -0.24274 (-0.24348) + | > log_mle: -0.39846 (-0.35731) + | > loss_dur: 0.15573 (0.11382) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.04660 (16.02114) + | > current_lr: 0.00006 + | > step_time: 1.41390 (2.41909) + | > loader_time: 0.00250 (0.03776) + + + --> STEP: 122/234 -- GLOBAL_STEP: 54410 + | > loss: -0.21732 (-0.24282) + | > log_mle: -0.36805 (-0.35830) + | > loss_dur: 0.15074 (0.11548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.13608 (16.18415) + | > current_lr: 0.00006 + | > step_time: 1.41070 (2.39863) + | > loader_time: 0.07630 (0.03773) + + + --> STEP: 127/234 -- GLOBAL_STEP: 54415 + | > loss: -0.23985 (-0.24278) + | > log_mle: -0.42875 (-0.36051) + | > loss_dur: 0.18890 (0.11773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.53769 (16.92064) + | > current_lr: 0.00006 + | > step_time: 1.85780 (2.36087) + | > loader_time: 0.00200 (0.03767) + + + --> STEP: 132/234 -- GLOBAL_STEP: 54420 + | > loss: -0.26152 (-0.24353) + | > log_mle: -0.41292 (-0.36330) + | > loss_dur: 0.15140 (0.11977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.33585 (17.91818) + | > current_lr: 0.00006 + | > step_time: 2.20410 (2.35916) + | > loader_time: 0.07710 (0.03749) + + + --> STEP: 137/234 -- GLOBAL_STEP: 54425 + | > loss: -0.22215 (-0.24426) + | > log_mle: -0.42052 (-0.36635) + | > loss_dur: 0.19836 (0.12209) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.77940 (18.91214) + | > current_lr: 0.00006 + | > step_time: 2.36010 (2.37596) + | > loader_time: 0.00220 (0.03807) + + + --> STEP: 142/234 -- GLOBAL_STEP: 54430 + | > loss: -0.22895 (-0.24415) + | > log_mle: -0.41628 (-0.36828) + | > loss_dur: 0.18733 (0.12413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.77377 (20.09928) + | > current_lr: 0.00006 + | > step_time: 1.99780 (2.35551) + | > loader_time: 0.00350 (0.03803) + + + --> STEP: 147/234 -- GLOBAL_STEP: 54435 + | > loss: -0.25345 (-0.24524) + | > log_mle: -0.43095 (-0.37192) + | > loss_dur: 0.17750 (0.12668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.13288 (20.64718) + | > current_lr: 0.00006 + | > step_time: 0.88100 (2.33978) + | > loader_time: 0.00270 (0.03685) + + + --> STEP: 152/234 -- GLOBAL_STEP: 54440 + | > loss: -0.30199 (-0.24693) + | > log_mle: -0.51862 (-0.37564) + | > loss_dur: 0.21663 (0.12871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.63626 (21.25843) + | > current_lr: 0.00006 + | > step_time: 3.75000 (2.34466) + | > loader_time: 0.00260 (0.03686) + + + --> STEP: 157/234 -- GLOBAL_STEP: 54445 + | > loss: -0.26899 (-0.24931) + | > log_mle: -0.46331 (-0.38014) + | > loss_dur: 0.19432 (0.13083) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.17675 (22.17248) + | > current_lr: 0.00006 + | > step_time: 2.00130 (2.32962) + | > loader_time: 0.00400 (0.03583) + + + --> STEP: 162/234 -- GLOBAL_STEP: 54450 + | > loss: -0.30811 (-0.25138) + | > log_mle: -0.50188 (-0.38441) + | > loss_dur: 0.19377 (0.13303) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.29607 (22.98241) + | > current_lr: 0.00006 + | > step_time: 6.29440 (2.38051) + | > loader_time: 0.09530 (0.03659) + + + --> STEP: 167/234 -- GLOBAL_STEP: 54455 + | > loss: -0.38922 (-0.25334) + | > log_mle: -0.58188 (-0.38834) + | > loss_dur: 0.19266 (0.13500) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.53197 (24.14450) + | > current_lr: 0.00006 + | > step_time: 5.40040 (2.48889) + | > loader_time: 0.00630 (0.03568) + + + --> STEP: 172/234 -- GLOBAL_STEP: 54460 + | > loss: -0.35988 (-0.25597) + | > log_mle: -0.58548 (-0.39333) + | > loss_dur: 0.22560 (0.13736) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.84632 (25.28063) + | > current_lr: 0.00006 + | > step_time: 1.70270 (2.50678) + | > loader_time: 0.00230 (0.03633) + + + --> STEP: 177/234 -- GLOBAL_STEP: 54465 + | > loss: -0.32920 (-0.25853) + | > log_mle: -0.53835 (-0.39811) + | > loss_dur: 0.20915 (0.13958) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.27082 (26.35710) + | > current_lr: 0.00006 + | > step_time: 4.41260 (2.58250) + | > loader_time: 0.28810 (0.03852) + + + --> STEP: 182/234 -- GLOBAL_STEP: 54470 + | > loss: -0.33111 (-0.26057) + | > log_mle: -0.57573 (-0.40261) + | > loss_dur: 0.24462 (0.14204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.76532 (27.46536) + | > current_lr: 0.00006 + | > step_time: 8.59280 (2.63246) + | > loader_time: 0.10900 (0.03972) + + + --> STEP: 187/234 -- GLOBAL_STEP: 54475 + | > loss: -0.36906 (-0.26302) + | > log_mle: -0.58764 (-0.40721) + | > loss_dur: 0.21859 (0.14419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.46436 (28.38980) + | > current_lr: 0.00006 + | > step_time: 2.89460 (2.69344) + | > loader_time: 0.00380 (0.03978) + + + --> STEP: 192/234 -- GLOBAL_STEP: 54480 + | > loss: -0.40801 (-0.26585) + | > log_mle: -0.62110 (-0.41183) + | > loss_dur: 0.21309 (0.14597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.87191 (29.38289) + | > current_lr: 0.00006 + | > step_time: 9.00050 (2.74669) + | > loader_time: 0.09930 (0.04129) + + + --> STEP: 197/234 -- GLOBAL_STEP: 54485 + | > loss: -0.38342 (-0.26834) + | > log_mle: -0.58163 (-0.41616) + | > loss_dur: 0.19822 (0.14782) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.43283 (30.49189) + | > current_lr: 0.00006 + | > step_time: 6.31500 (2.80248) + | > loader_time: 0.19040 (0.04273) + + + --> STEP: 202/234 -- GLOBAL_STEP: 54490 + | > loss: -0.46553 (-0.27104) + | > log_mle: -0.68307 (-0.42072) + | > loss_dur: 0.21754 (0.14969) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.26171 (31.45372) + | > current_lr: 0.00006 + | > step_time: 8.10520 (2.83570) + | > loader_time: 0.09240 (0.04267) + + + --> STEP: 207/234 -- GLOBAL_STEP: 54495 + | > loss: -0.43774 (-0.27374) + | > log_mle: -0.66983 (-0.42534) + | > loss_dur: 0.23209 (0.15160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.07193 (32.48078) + | > current_lr: 0.00006 + | > step_time: 6.49520 (2.89383) + | > loader_time: 0.10240 (0.04370) + + + --> STEP: 212/234 -- GLOBAL_STEP: 54500 + | > loss: -0.38884 (-0.27689) + | > log_mle: -0.62862 (-0.43047) + | > loss_dur: 0.23977 (0.15358) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.51827 (33.39853) + | > current_lr: 0.00006 + | > step_time: 2.19800 (2.93116) + | > loader_time: 0.00380 (0.04455) + + + --> STEP: 217/234 -- GLOBAL_STEP: 54505 + | > loss: -0.41273 (-0.27968) + | > log_mle: -0.64936 (-0.43507) + | > loss_dur: 0.23662 (0.15540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.10022 (34.61255) + | > current_lr: 0.00006 + | > step_time: 10.90100 (3.02176) + | > loader_time: 0.11210 (0.04577) + + + --> STEP: 222/234 -- GLOBAL_STEP: 54510 + | > loss: -0.40268 (-0.28264) + | > log_mle: -0.66782 (-0.43988) + | > loss_dur: 0.26515 (0.15725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.78682 (35.57116) + | > current_lr: 0.00006 + | > step_time: 0.32510 (3.02724) + | > loader_time: 0.00600 (0.04571) + + + --> STEP: 227/234 -- GLOBAL_STEP: 54515 + | > loss: -0.38578 (-0.28609) + | > log_mle: -0.64647 (-0.44527) + | > loss_dur: 0.26070 (0.15918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.73999 (36.63927) + | > current_lr: 0.00006 + | > step_time: 0.24860 (2.96592) + | > loader_time: 0.00380 (0.04479) + + + --> STEP: 232/234 -- GLOBAL_STEP: 54520 + | > loss: -0.41063 (-0.28900) + | > log_mle: -0.88152 (-0.45225) + | > loss_dur: 0.47088 (0.16325) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.15990 (37.98370) + | > current_lr: 0.00006 + | > step_time: 0.34110 (2.90785) + | > loader_time: 0.01070 (0.04394) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.68023 (+0.67585) + | > avg_loss: -0.30922 (+0.01010) + | > avg_log_mle: -0.53022 (+0.00949) + | > avg_loss_dur: 0.22100 (+0.00061) + + + > EPOCH: 233/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 10:08:39)  + + --> STEP: 3/234 -- GLOBAL_STEP: 54525 + | > loss: -0.19283 (-0.25009) + | > log_mle: -0.34663 (-0.35854) + | > loss_dur: 0.15380 (0.10845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.00882 (23.21846) + | > current_lr: 0.00006 + | > step_time: 4.59970 (2.97134) + | > loader_time: 8.08510 (2.72309) + + + --> STEP: 8/234 -- GLOBAL_STEP: 54530 + | > loss: -0.30006 (-0.26864) + | > log_mle: -0.37570 (-0.36183) + | > loss_dur: 0.07564 (0.09319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.02318 (20.67537) + | > current_lr: 0.00006 + | > step_time: 6.18910 (3.64900) + | > loader_time: 0.00280 (1.03529) + + + --> STEP: 13/234 -- GLOBAL_STEP: 54535 + | > loss: -0.31035 (-0.27480) + | > log_mle: -0.37998 (-0.36468) + | > loss_dur: 0.06962 (0.08988) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.40362 (18.19845) + | > current_lr: 0.00006 + | > step_time: 3.70850 (4.53881) + | > loader_time: 0.08160 (0.66622) + + + --> STEP: 18/234 -- GLOBAL_STEP: 54540 + | > loss: -0.26122 (-0.27678) + | > log_mle: -0.35088 (-0.36401) + | > loss_dur: 0.08966 (0.08723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.17634 (17.03380) + | > current_lr: 0.00006 + | > step_time: 1.50880 (4.28894) + | > loader_time: 0.00160 (0.49658) + + + --> STEP: 23/234 -- GLOBAL_STEP: 54545 + | > loss: -0.29536 (-0.27780) + | > log_mle: -0.36949 (-0.36324) + | > loss_dur: 0.07413 (0.08544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61542 (16.03415) + | > current_lr: 0.00006 + | > step_time: 7.20400 (4.33015) + | > loader_time: 0.10080 (0.40557) + + + --> STEP: 28/234 -- GLOBAL_STEP: 54550 + | > loss: -0.32279 (-0.27863) + | > log_mle: -0.38033 (-0.36232) + | > loss_dur: 0.05754 (0.08369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.73382 (15.15412) + | > current_lr: 0.00006 + | > step_time: 1.31560 (4.12183) + | > loader_time: 0.07600 (0.35390) + + + --> STEP: 33/234 -- GLOBAL_STEP: 54555 + | > loss: -0.27220 (-0.27549) + | > log_mle: -0.35095 (-0.36028) + | > loss_dur: 0.07875 (0.08479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.91371 (14.74019) + | > current_lr: 0.00006 + | > step_time: 0.77530 (3.72371) + | > loader_time: 0.00270 (0.30332) + + + --> STEP: 38/234 -- GLOBAL_STEP: 54560 + | > loss: -0.26244 (-0.27283) + | > log_mle: -0.35902 (-0.35833) + | > loss_dur: 0.09658 (0.08550) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.25857 (14.55555) + | > current_lr: 0.00006 + | > step_time: 2.29920 (3.55778) + | > loader_time: 0.00140 (0.27114) + + + --> STEP: 43/234 -- GLOBAL_STEP: 54565 + | > loss: -0.24719 (-0.27000) + | > log_mle: -0.35217 (-0.35663) + | > loss_dur: 0.10499 (0.08663) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.86605 (14.13565) + | > current_lr: 0.00006 + | > step_time: 2.50640 (3.38571) + | > loader_time: 0.10330 (0.24429) + + + --> STEP: 48/234 -- GLOBAL_STEP: 54570 + | > loss: -0.27413 (-0.26829) + | > log_mle: -0.34653 (-0.35558) + | > loss_dur: 0.07240 (0.08729) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.86490 (13.73856) + | > current_lr: 0.00006 + | > step_time: 2.15470 (3.18609) + | > loader_time: 0.00180 (0.22253) + + + --> STEP: 53/234 -- GLOBAL_STEP: 54575 + | > loss: -0.23601 (-0.26633) + | > log_mle: -0.34872 (-0.35445) + | > loss_dur: 0.11270 (0.08812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06746 (13.35131) + | > current_lr: 0.00006 + | > step_time: 2.08230 (3.04959) + | > loader_time: 0.00220 (0.20340) + + + --> STEP: 58/234 -- GLOBAL_STEP: 54580 + | > loss: -0.25136 (-0.26440) + | > log_mle: -0.34571 (-0.35358) + | > loss_dur: 0.09435 (0.08918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.11660 (13.15878) + | > current_lr: 0.00006 + | > step_time: 2.29370 (2.96567) + | > loader_time: 0.00200 (0.18925) + + + --> STEP: 63/234 -- GLOBAL_STEP: 54585 + | > loss: -0.22595 (-0.26162) + | > log_mle: -0.33523 (-0.35332) + | > loss_dur: 0.10928 (0.09171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.91111 (13.32327) + | > current_lr: 0.00006 + | > step_time: 1.28180 (2.89353) + | > loader_time: 0.00210 (0.17578) + + + --> STEP: 68/234 -- GLOBAL_STEP: 54590 + | > loss: -0.21499 (-0.25969) + | > log_mle: -0.33314 (-0.35238) + | > loss_dur: 0.11815 (0.09269) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.38840 (13.13307) + | > current_lr: 0.00006 + | > step_time: 2.31520 (2.80909) + | > loader_time: 0.08190 (0.16653) + + + --> STEP: 73/234 -- GLOBAL_STEP: 54595 + | > loss: -0.20048 (-0.25654) + | > log_mle: -0.34290 (-0.35140) + | > loss_dur: 0.14242 (0.09486) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.23840 (13.17851) + | > current_lr: 0.00006 + | > step_time: 2.41710 (2.79762) + | > loader_time: 0.08600 (0.15866) + + + --> STEP: 78/234 -- GLOBAL_STEP: 54600 + | > loss: -0.21240 (-0.25400) + | > log_mle: -0.32697 (-0.35043) + | > loss_dur: 0.11457 (0.09644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.73396 (13.15366) + | > current_lr: 0.00006 + | > step_time: 1.81100 (2.75172) + | > loader_time: 0.00240 (0.15005) + + + --> STEP: 83/234 -- GLOBAL_STEP: 54605 + | > loss: -0.19404 (-0.25192) + | > log_mle: -0.34445 (-0.34980) + | > loss_dur: 0.15041 (0.09788) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.48487 (13.11760) + | > current_lr: 0.00006 + | > step_time: 1.16270 (2.73424) + | > loader_time: 0.00230 (0.14438) + + + --> STEP: 88/234 -- GLOBAL_STEP: 54610 + | > loss: -0.22695 (-0.25030) + | > log_mle: -0.37931 (-0.34971) + | > loss_dur: 0.15237 (0.09941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.42407 (13.12364) + | > current_lr: 0.00006 + | > step_time: 0.84760 (2.66600) + | > loader_time: 0.00240 (0.13727) + + + --> STEP: 93/234 -- GLOBAL_STEP: 54615 + | > loss: -0.22921 (-0.24926) + | > log_mle: -0.39142 (-0.35088) + | > loss_dur: 0.16221 (0.10162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.02431 (13.38297) + | > current_lr: 0.00006 + | > step_time: 1.27610 (2.63676) + | > loader_time: 0.00420 (0.13275) + + + --> STEP: 98/234 -- GLOBAL_STEP: 54620 + | > loss: -0.21033 (-0.24852) + | > log_mle: -0.32770 (-0.35209) + | > loss_dur: 0.11738 (0.10357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.85841 (13.73046) + | > current_lr: 0.00006 + | > step_time: 2.49750 (2.60502) + | > loader_time: 0.00200 (0.12700) + + + --> STEP: 103/234 -- GLOBAL_STEP: 54625 + | > loss: -0.25265 (-0.24790) + | > log_mle: -0.41692 (-0.35402) + | > loss_dur: 0.16426 (0.10613) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.85493 (14.31940) + | > current_lr: 0.00006 + | > step_time: 4.28730 (2.64457) + | > loader_time: 0.09750 (0.12461) + + + --> STEP: 108/234 -- GLOBAL_STEP: 54630 + | > loss: -0.22787 (-0.24718) + | > log_mle: -0.36418 (-0.35550) + | > loss_dur: 0.13631 (0.10832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.69046 (14.83967) + | > current_lr: 0.00006 + | > step_time: 2.79620 (2.67300) + | > loader_time: 0.00900 (0.12066) + + + --> STEP: 113/234 -- GLOBAL_STEP: 54635 + | > loss: -0.24958 (-0.24647) + | > log_mle: -0.40678 (-0.35769) + | > loss_dur: 0.15721 (0.11122) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.40906 (15.38849) + | > current_lr: 0.00006 + | > step_time: 1.20630 (2.61917) + | > loader_time: 0.00260 (0.11546) + + + --> STEP: 118/234 -- GLOBAL_STEP: 54640 + | > loss: -0.22079 (-0.24565) + | > log_mle: -0.38396 (-0.35916) + | > loss_dur: 0.16317 (0.11351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.87962 (15.91772) + | > current_lr: 0.00006 + | > step_time: 1.19770 (2.62350) + | > loader_time: 0.00930 (0.11224) + + + --> STEP: 123/234 -- GLOBAL_STEP: 54645 + | > loss: -0.19514 (-0.24461) + | > log_mle: -0.35227 (-0.35980) + | > loss_dur: 0.15712 (0.11519) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.28525 (16.17576) + | > current_lr: 0.00006 + | > step_time: 3.19880 (2.60811) + | > loader_time: 0.00280 (0.10924) + + + --> STEP: 128/234 -- GLOBAL_STEP: 54650 + | > loss: -0.26077 (-0.24515) + | > log_mle: -0.40730 (-0.36249) + | > loss_dur: 0.14653 (0.11734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.67682 (16.88996) + | > current_lr: 0.00006 + | > step_time: 1.38520 (2.59907) + | > loader_time: 0.00270 (0.10720) + + + --> STEP: 133/234 -- GLOBAL_STEP: 54655 + | > loss: -0.26622 (-0.24595) + | > log_mle: -0.43624 (-0.36539) + | > loss_dur: 0.17002 (0.11944) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.89136 (17.63925) + | > current_lr: 0.00006 + | > step_time: 2.67490 (2.57344) + | > loader_time: 0.00210 (0.10393) + + + --> STEP: 138/234 -- GLOBAL_STEP: 54660 + | > loss: -0.21584 (-0.24643) + | > log_mle: -0.38278 (-0.36803) + | > loss_dur: 0.16694 (0.12160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.44849 (18.31629) + | > current_lr: 0.00006 + | > step_time: 3.29940 (2.60991) + | > loader_time: 0.00820 (0.10164) + + + --> STEP: 143/234 -- GLOBAL_STEP: 54665 + | > loss: -0.30891 (-0.24738) + | > log_mle: -0.52753 (-0.37144) + | > loss_dur: 0.21862 (0.12406) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.19254 (19.28920) + | > current_lr: 0.00006 + | > step_time: 2.51300 (2.57842) + | > loader_time: 0.08680 (0.09980) + + + --> STEP: 148/234 -- GLOBAL_STEP: 54670 + | > loss: -0.27792 (-0.24881) + | > log_mle: -0.44429 (-0.37502) + | > loss_dur: 0.16637 (0.12621) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.91263 (19.98445) + | > current_lr: 0.00006 + | > step_time: 1.40620 (2.60248) + | > loader_time: 0.18670 (0.09928) + + + --> STEP: 153/234 -- GLOBAL_STEP: 54675 + | > loss: -0.38870 (-0.25142) + | > log_mle: -0.58506 (-0.37998) + | > loss_dur: 0.19636 (0.12857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.64994 (21.00515) + | > current_lr: 0.00006 + | > step_time: 4.00920 (2.62469) + | > loader_time: 0.10260 (0.09731) + + + --> STEP: 158/234 -- GLOBAL_STEP: 54680 + | > loss: -0.28590 (-0.25333) + | > log_mle: -0.50537 (-0.38409) + | > loss_dur: 0.21947 (0.13075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.50343 (22.09146) + | > current_lr: 0.00006 + | > step_time: 2.80650 (2.64097) + | > loader_time: 0.00320 (0.09619) + + + --> STEP: 163/234 -- GLOBAL_STEP: 54685 + | > loss: -0.27934 (-0.25550) + | > log_mle: -0.47766 (-0.38825) + | > loss_dur: 0.19832 (0.13275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.49863 (23.00396) + | > current_lr: 0.00006 + | > step_time: 4.20800 (2.65523) + | > loader_time: 0.18650 (0.09647) + + + --> STEP: 168/234 -- GLOBAL_STEP: 54690 + | > loss: -0.32468 (-0.25788) + | > log_mle: -0.53728 (-0.39267) + | > loss_dur: 0.21260 (0.13478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.06582 (24.05004) + | > current_lr: 0.00006 + | > step_time: 2.79720 (2.64659) + | > loader_time: 0.00360 (0.09594) + + + --> STEP: 173/234 -- GLOBAL_STEP: 54695 + | > loss: -0.33768 (-0.26066) + | > log_mle: -0.54547 (-0.39765) + | > loss_dur: 0.20779 (0.13699) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.75195 (25.21791) + | > current_lr: 0.00006 + | > step_time: 2.10480 (2.62428) + | > loader_time: 0.08890 (0.09426) + + + --> STEP: 178/234 -- GLOBAL_STEP: 54700 + | > loss: -0.37067 (-0.26333) + | > log_mle: -0.59607 (-0.40262) + | > loss_dur: 0.22540 (0.13929) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.18182 (26.35987) + | > current_lr: 0.00006 + | > step_time: 2.01200 (2.63781) + | > loader_time: 0.00340 (0.09279) + + + --> STEP: 183/234 -- GLOBAL_STEP: 54705 + | > loss: -0.38153 (-0.26550) + | > log_mle: -0.59745 (-0.40713) + | > loss_dur: 0.21592 (0.14163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.67916 (27.38355) + | > current_lr: 0.00006 + | > step_time: 5.19120 (2.75369) + | > loader_time: 0.00510 (0.09403) + + + --> STEP: 188/234 -- GLOBAL_STEP: 54710 + | > loss: -0.39740 (-0.26790) + | > log_mle: -0.61554 (-0.41173) + | > loss_dur: 0.21814 (0.14383) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.02699 (28.44288) + | > current_lr: 0.00006 + | > step_time: 7.68980 (2.81493) + | > loader_time: 0.20540 (0.09467) + + + --> STEP: 193/234 -- GLOBAL_STEP: 54715 + | > loss: -0.41316 (-0.27086) + | > log_mle: -0.62151 (-0.41638) + | > loss_dur: 0.20835 (0.14552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.53717 (29.32994) + | > current_lr: 0.00006 + | > step_time: 6.00520 (2.91979) + | > loader_time: 0.00330 (0.09467) + + + --> STEP: 198/234 -- GLOBAL_STEP: 54720 + | > loss: -0.39226 (-0.27356) + | > log_mle: -0.61079 (-0.42081) + | > loss_dur: 0.21853 (0.14725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.13886 (30.39235) + | > current_lr: 0.00006 + | > step_time: 2.80010 (2.96122) + | > loader_time: 0.29410 (0.09532) + + + --> STEP: 203/234 -- GLOBAL_STEP: 54725 + | > loss: -0.32211 (-0.27577) + | > log_mle: -0.53707 (-0.42496) + | > loss_dur: 0.21497 (0.14919) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.11347 (31.34471) + | > current_lr: 0.00006 + | > step_time: 8.70950 (3.03568) + | > loader_time: 0.10320 (0.09496) + + + --> STEP: 208/234 -- GLOBAL_STEP: 54730 + | > loss: -0.39164 (-0.27885) + | > log_mle: -0.62882 (-0.43001) + | > loss_dur: 0.23717 (0.15116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.16894 (32.59425) + | > current_lr: 0.00006 + | > step_time: 8.79590 (3.04112) + | > loader_time: 0.39200 (0.09584) + + + --> STEP: 213/234 -- GLOBAL_STEP: 54735 + | > loss: -0.44473 (-0.28240) + | > log_mle: -0.68193 (-0.43550) + | > loss_dur: 0.23720 (0.15310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.56868 (33.94402) + | > current_lr: 0.00006 + | > step_time: 2.51320 (3.06138) + | > loader_time: 0.09250 (0.09508) + + + --> STEP: 218/234 -- GLOBAL_STEP: 54740 + | > loss: -0.39748 (-0.28543) + | > log_mle: -0.63096 (-0.44037) + | > loss_dur: 0.23348 (0.15494) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.94446 (35.32505) + | > current_lr: 0.00006 + | > step_time: 9.39230 (3.14476) + | > loader_time: 0.09840 (0.09651) + + + --> STEP: 223/234 -- GLOBAL_STEP: 54745 + | > loss: -0.43453 (-0.28878) + | > log_mle: -0.66755 (-0.44556) + | > loss_dur: 0.23302 (0.15678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.82798 (36.73572) + | > current_lr: 0.00006 + | > step_time: 0.71290 (3.12688) + | > loader_time: 0.07710 (0.09520) + + + --> STEP: 228/234 -- GLOBAL_STEP: 54750 + | > loss: -0.39184 (-0.29190) + | > log_mle: -0.65794 (-0.45073) + | > loss_dur: 0.26610 (0.15883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.97543 (38.16443) + | > current_lr: 0.00006 + | > step_time: 0.24370 (3.06362) + | > loader_time: 0.00350 (0.09320) + + + --> STEP: 233/234 -- GLOBAL_STEP: 54755 + | > loss: 0.08458 (-0.29240) + | > log_mle: -0.65036 (-0.45729) + | > loss_dur: 0.73494 (0.16489) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.78371 (39.69156) + | > current_lr: 0.00006 + | > step_time: 0.19810 (3.00353) + | > loader_time: 0.00260 (0.09136) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22773 (-0.45250) + | > avg_loss: -0.32638 (-0.01716) + | > avg_log_mle: -0.53978 (-0.00956) + | > avg_loss_dur: 0.21339 (-0.00760) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_54756.pth + + > EPOCH: 234/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 10:21:51)  + + --> STEP: 4/234 -- GLOBAL_STEP: 54760 + | > loss: -0.26840 (-0.25278) + | > log_mle: -0.36092 (-0.36011) + | > loss_dur: 0.09252 (0.10733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.09951 (20.89174) + | > current_lr: 0.00006 + | > step_time: 1.72930 (3.87960) + | > loader_time: 0.09350 (0.19668) + + + --> STEP: 9/234 -- GLOBAL_STEP: 54765 + | > loss: -0.26804 (-0.26652) + | > log_mle: -0.37274 (-0.36349) + | > loss_dur: 0.10470 (0.09697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.55153 (17.74839) + | > current_lr: 0.00006 + | > step_time: 5.78840 (4.38201) + | > loader_time: 0.11100 (0.15019) + + + --> STEP: 14/234 -- GLOBAL_STEP: 54770 + | > loss: -0.28570 (-0.27471) + | > log_mle: -0.36797 (-0.36565) + | > loss_dur: 0.08226 (0.09094) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.85550 (16.61108) + | > current_lr: 0.00006 + | > step_time: 2.19810 (4.35827) + | > loader_time: 0.00300 (0.11886) + + + --> STEP: 19/234 -- GLOBAL_STEP: 54775 + | > loss: -0.28223 (-0.27749) + | > log_mle: -0.35906 (-0.36471) + | > loss_dur: 0.07683 (0.08722) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.69474 (15.18519) + | > current_lr: 0.00006 + | > step_time: 7.90000 (4.66398) + | > loader_time: 0.00370 (0.10341) + + + --> STEP: 24/234 -- GLOBAL_STEP: 54780 + | > loss: -0.28950 (-0.27884) + | > log_mle: -0.35510 (-0.36381) + | > loss_dur: 0.06560 (0.08497) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.95304 (14.44525) + | > current_lr: 0.00006 + | > step_time: 3.10060 (4.26676) + | > loader_time: 0.00550 (0.08247) + + + --> STEP: 29/234 -- GLOBAL_STEP: 54785 + | > loss: -0.25770 (-0.27891) + | > log_mle: -0.34250 (-0.36242) + | > loss_dur: 0.08479 (0.08351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.19565 (13.73091) + | > current_lr: 0.00006 + | > step_time: 2.17130 (3.99023) + | > loader_time: 0.00330 (0.07436) + + + --> STEP: 34/234 -- GLOBAL_STEP: 54790 + | > loss: -0.26614 (-0.27619) + | > log_mle: -0.35262 (-0.36082) + | > loss_dur: 0.08648 (0.08463) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.64760 (13.37590) + | > current_lr: 0.00006 + | > step_time: 1.05740 (3.60499) + | > loader_time: 0.00140 (0.06369) + + + --> STEP: 39/234 -- GLOBAL_STEP: 54795 + | > loss: -0.24692 (-0.27276) + | > log_mle: -0.34873 (-0.35898) + | > loss_dur: 0.10181 (0.08622) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.69228 (13.38935) + | > current_lr: 0.00006 + | > step_time: 1.59420 (3.39753) + | > loader_time: 0.00170 (0.05591) + + + --> STEP: 44/234 -- GLOBAL_STEP: 54800 + | > loss: -0.27123 (-0.27013) + | > log_mle: -0.34166 (-0.35721) + | > loss_dur: 0.07043 (0.08707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.96699 (12.97006) + | > current_lr: 0.00006 + | > step_time: 3.22710 (3.28262) + | > loader_time: 0.08780 (0.05178) + + + --> STEP: 49/234 -- GLOBAL_STEP: 54805 + | > loss: -0.26712 (-0.26876) + | > log_mle: -0.35347 (-0.35658) + | > loss_dur: 0.08635 (0.08782) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.39976 (12.64972) + | > current_lr: 0.00006 + | > step_time: 1.15330 (3.15546) + | > loader_time: 0.00390 (0.05056) + + + --> STEP: 54/234 -- GLOBAL_STEP: 54810 + | > loss: -0.25889 (-0.26684) + | > log_mle: -0.35100 (-0.35538) + | > loss_dur: 0.09210 (0.08854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.10946 (12.44433) + | > current_lr: 0.00006 + | > step_time: 1.91120 (2.99813) + | > loader_time: 0.00250 (0.04759) + + + --> STEP: 59/234 -- GLOBAL_STEP: 54815 + | > loss: -0.25331 (-0.26531) + | > log_mle: -0.35446 (-0.35453) + | > loss_dur: 0.10115 (0.08923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.87962 (12.26035) + | > current_lr: 0.00006 + | > step_time: 1.51190 (2.90856) + | > loader_time: 0.09430 (0.04699) + + + --> STEP: 64/234 -- GLOBAL_STEP: 54820 + | > loss: -0.24035 (-0.26226) + | > log_mle: -0.34095 (-0.35410) + | > loss_dur: 0.10061 (0.09184) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.66833 (12.32168) + | > current_lr: 0.00006 + | > step_time: 3.39530 (2.87644) + | > loader_time: 0.00440 (0.04611) + + + --> STEP: 69/234 -- GLOBAL_STEP: 54825 + | > loss: -0.22652 (-0.26017) + | > log_mle: -0.32912 (-0.35303) + | > loss_dur: 0.10259 (0.09286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.10613 (12.20223) + | > current_lr: 0.00006 + | > step_time: 2.19800 (2.79633) + | > loader_time: 0.00220 (0.04294) + + + --> STEP: 74/234 -- GLOBAL_STEP: 54830 + | > loss: -0.21918 (-0.25728) + | > log_mle: -0.33155 (-0.35227) + | > loss_dur: 0.11237 (0.09499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.65385 (12.38731) + | > current_lr: 0.00006 + | > step_time: 1.08620 (2.74748) + | > loader_time: 0.00230 (0.04023) + + + --> STEP: 79/234 -- GLOBAL_STEP: 54835 + | > loss: -0.22762 (-0.25520) + | > log_mle: -0.34363 (-0.35166) + | > loss_dur: 0.11601 (0.09647) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.84951 (12.41841) + | > current_lr: 0.00006 + | > step_time: 2.60170 (2.74933) + | > loader_time: 0.09710 (0.03911) + + + --> STEP: 84/234 -- GLOBAL_STEP: 54840 + | > loss: -0.23891 (-0.25326) + | > log_mle: -0.34008 (-0.35104) + | > loss_dur: 0.10118 (0.09777) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.96498 (12.48731) + | > current_lr: 0.00006 + | > step_time: 2.99220 (2.71962) + | > loader_time: 0.08920 (0.03795) + + + --> STEP: 89/234 -- GLOBAL_STEP: 54845 + | > loss: -0.24221 (-0.25206) + | > log_mle: -0.36746 (-0.35135) + | > loss_dur: 0.12526 (0.09929) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.77766 (12.60215) + | > current_lr: 0.00006 + | > step_time: 2.00450 (2.70952) + | > loader_time: 0.00180 (0.03686) + + + --> STEP: 94/234 -- GLOBAL_STEP: 54850 + | > loss: -0.25007 (-0.25125) + | > log_mle: -0.39307 (-0.35285) + | > loss_dur: 0.14300 (0.10160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.17028 (13.03170) + | > current_lr: 0.00006 + | > step_time: 2.71470 (2.67056) + | > loader_time: 0.00320 (0.03503) + + + --> STEP: 99/234 -- GLOBAL_STEP: 54855 + | > loss: -0.26158 (-0.25070) + | > log_mle: -0.42965 (-0.35441) + | > loss_dur: 0.16808 (0.10371) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.31047 (13.49731) + | > current_lr: 0.00006 + | > step_time: 2.19410 (2.62407) + | > loader_time: 0.00360 (0.03342) + + + --> STEP: 104/234 -- GLOBAL_STEP: 54860 + | > loss: -0.28562 (-0.25031) + | > log_mle: -0.44274 (-0.35652) + | > loss_dur: 0.15711 (0.10622) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.42095 (14.05729) + | > current_lr: 0.00006 + | > step_time: 5.59440 (2.62264) + | > loader_time: 0.09650 (0.03361) + + + --> STEP: 109/234 -- GLOBAL_STEP: 54865 + | > loss: -0.21090 (-0.24908) + | > log_mle: -0.40732 (-0.35771) + | > loss_dur: 0.19643 (0.10863) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.00338 (14.57037) + | > current_lr: 0.00006 + | > step_time: 2.59780 (2.61982) + | > loader_time: 0.00660 (0.03302) + + + --> STEP: 114/234 -- GLOBAL_STEP: 54870 + | > loss: -0.24732 (-0.24873) + | > log_mle: -0.39206 (-0.35984) + | > loss_dur: 0.14474 (0.11112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.51995 (15.17634) + | > current_lr: 0.00006 + | > step_time: 2.00010 (2.62843) + | > loader_time: 0.00320 (0.03326) + + + --> STEP: 119/234 -- GLOBAL_STEP: 54875 + | > loss: -0.24496 (-0.24816) + | > log_mle: -0.39283 (-0.36146) + | > loss_dur: 0.14787 (0.11330) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.65565 (15.58202) + | > current_lr: 0.00006 + | > step_time: 4.80190 (2.61372) + | > loader_time: 0.00330 (0.03352) + + + --> STEP: 124/234 -- GLOBAL_STEP: 54880 + | > loss: -0.25741 (-0.24743) + | > log_mle: -0.41999 (-0.36240) + | > loss_dur: 0.16258 (0.11497) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.15500 (15.89341) + | > current_lr: 0.00006 + | > step_time: 2.94470 (2.61741) + | > loader_time: 0.00750 (0.03467) + + + --> STEP: 129/234 -- GLOBAL_STEP: 54885 + | > loss: -0.23350 (-0.24770) + | > log_mle: -0.41236 (-0.36501) + | > loss_dur: 0.17887 (0.11731) + | > amp_scaler: 2048.00000 (1055.75194) + | > grad_norm: 28.40652 (16.59075) + | > current_lr: 0.00006 + | > step_time: 1.80430 (2.59924) + | > loader_time: 0.00230 (0.03348) + + + --> STEP: 134/234 -- GLOBAL_STEP: 54890 + | > loss: -0.26758 (-0.24852) + | > log_mle: -0.46148 (-0.36824) + | > loss_dur: 0.19390 (0.11971) + | > amp_scaler: 2048.00000 (1092.77612) + | > grad_norm: 40.26461 (17.65846) + | > current_lr: 0.00006 + | > step_time: 0.89590 (2.58942) + | > loader_time: 0.00300 (0.03235) + + + --> STEP: 139/234 -- GLOBAL_STEP: 54895 + | > loss: -0.32599 (-0.24922) + | > log_mle: -0.52053 (-0.37123) + | > loss_dur: 0.19454 (0.12201) + | > amp_scaler: 2048.00000 (1127.13669) + | > grad_norm: 71.98627 (18.61657) + | > current_lr: 0.00006 + | > step_time: 4.70530 (2.59487) + | > loader_time: 0.09220 (0.03259) + + + --> STEP: 144/234 -- GLOBAL_STEP: 54900 + | > loss: -0.28639 (-0.24997) + | > log_mle: -0.48666 (-0.37444) + | > loss_dur: 0.20027 (0.12447) + | > amp_scaler: 2048.00000 (1159.11111) + | > grad_norm: 53.22670 (19.64909) + | > current_lr: 0.00006 + | > step_time: 3.31280 (2.67230) + | > loader_time: 0.18830 (0.03472) + + + --> STEP: 149/234 -- GLOBAL_STEP: 54905 + | > loss: -0.35334 (-0.25175) + | > log_mle: -0.54534 (-0.37821) + | > loss_dur: 0.19200 (0.12646) + | > amp_scaler: 2048.00000 (1188.93960) + | > grad_norm: 48.93218 (20.33003) + | > current_lr: 0.00006 + | > step_time: 6.49010 (2.77445) + | > loader_time: 0.30270 (0.03880) + + + --> STEP: 154/234 -- GLOBAL_STEP: 54910 + | > loss: -0.31106 (-0.25374) + | > log_mle: -0.50139 (-0.38250) + | > loss_dur: 0.19033 (0.12876) + | > amp_scaler: 2048.00000 (1216.83117) + | > grad_norm: 50.73185 (21.44841) + | > current_lr: 0.00006 + | > step_time: 2.49590 (2.78126) + | > loader_time: 0.00270 (0.03943) + + + --> STEP: 159/234 -- GLOBAL_STEP: 54915 + | > loss: -0.32143 (-0.25565) + | > log_mle: -0.52560 (-0.38670) + | > loss_dur: 0.20417 (0.13105) + | > amp_scaler: 2048.00000 (1242.96855) + | > grad_norm: 54.83058 (22.48149) + | > current_lr: 0.00006 + | > step_time: 1.02810 (2.80053) + | > loader_time: 0.00230 (0.03943) + + + --> STEP: 164/234 -- GLOBAL_STEP: 54920 + | > loss: -0.29711 (-0.25747) + | > log_mle: -0.50927 (-0.39057) + | > loss_dur: 0.21215 (0.13311) + | > amp_scaler: 2048.00000 (1267.51220) + | > grad_norm: 66.53363 (23.71130) + | > current_lr: 0.00006 + | > step_time: 3.99710 (2.80292) + | > loader_time: 0.10090 (0.03947) + + + --> STEP: 169/234 -- GLOBAL_STEP: 54925 + | > loss: -0.31444 (-0.25975) + | > log_mle: -0.52022 (-0.39494) + | > loss_dur: 0.20578 (0.13519) + | > amp_scaler: 2048.00000 (1290.60355) + | > grad_norm: 39.67480 (24.61198) + | > current_lr: 0.00006 + | > step_time: 4.09400 (2.78687) + | > loader_time: 0.00360 (0.03986) + + + --> STEP: 174/234 -- GLOBAL_STEP: 54930 + | > loss: -0.38597 (-0.26274) + | > log_mle: -0.59749 (-0.40009) + | > loss_dur: 0.21153 (0.13735) + | > amp_scaler: 2048.00000 (1312.36782) + | > grad_norm: 64.83307 (25.88427) + | > current_lr: 0.00006 + | > step_time: 1.01400 (2.78034) + | > loader_time: 0.08260 (0.03979) + + + --> STEP: 179/234 -- GLOBAL_STEP: 54935 + | > loss: -0.35786 (-0.26505) + | > log_mle: -0.59853 (-0.40489) + | > loss_dur: 0.24067 (0.13984) + | > amp_scaler: 2048.00000 (1332.91620) + | > grad_norm: 69.64851 (26.79687) + | > current_lr: 0.00006 + | > step_time: 3.40450 (2.76041) + | > loader_time: 0.00540 (0.03919) + + + --> STEP: 184/234 -- GLOBAL_STEP: 54940 + | > loss: -0.31751 (-0.26673) + | > log_mle: -0.53721 (-0.40881) + | > loss_dur: 0.21970 (0.14208) + | > amp_scaler: 2048.00000 (1352.34783) + | > grad_norm: 63.42212 (28.08597) + | > current_lr: 0.00006 + | > step_time: 1.49670 (2.80547) + | > loader_time: 0.00350 (0.03986) + + + --> STEP: 189/234 -- GLOBAL_STEP: 54945 + | > loss: -0.32574 (-0.26867) + | > log_mle: -0.54108 (-0.41290) + | > loss_dur: 0.21535 (0.14422) + | > amp_scaler: 2048.00000 (1370.75132) + | > grad_norm: 91.24008 (29.19313) + | > current_lr: 0.00006 + | > step_time: 4.31490 (2.81973) + | > loader_time: 0.09580 (0.03988) + + + --> STEP: 194/234 -- GLOBAL_STEP: 54950 + | > loss: -0.38059 (-0.27129) + | > log_mle: -0.59491 (-0.41720) + | > loss_dur: 0.21433 (0.14591) + | > amp_scaler: 2048.00000 (1388.20619) + | > grad_norm: 53.14694 (30.22473) + | > current_lr: 0.00006 + | > step_time: 1.39910 (2.82558) + | > loader_time: 0.00290 (0.03986) + + + --> STEP: 199/234 -- GLOBAL_STEP: 54955 + | > loss: -0.37326 (-0.27354) + | > log_mle: -0.59208 (-0.42127) + | > loss_dur: 0.21882 (0.14773) + | > amp_scaler: 2048.00000 (1404.78392) + | > grad_norm: 77.68171 (31.23805) + | > current_lr: 0.00006 + | > step_time: 4.20520 (2.85765) + | > loader_time: 0.00420 (0.03937) + + + --> STEP: 204/234 -- GLOBAL_STEP: 54960 + | > loss: -0.38954 (-0.27546) + | > log_mle: -0.63855 (-0.42525) + | > loss_dur: 0.24901 (0.14979) + | > amp_scaler: 2048.00000 (1420.54902) + | > grad_norm: 82.97365 (32.09789) + | > current_lr: 0.00006 + | > step_time: 3.81710 (2.84805) + | > loader_time: 0.08860 (0.03989) + + + --> STEP: 209/234 -- GLOBAL_STEP: 54965 + | > loss: -0.37433 (-0.27805) + | > log_mle: -0.59142 (-0.42957) + | > loss_dur: 0.21710 (0.15152) + | > amp_scaler: 2048.00000 (1435.55981) + | > grad_norm: 59.00327 (33.15789) + | > current_lr: 0.00006 + | > step_time: 14.39430 (2.92680) + | > loader_time: 0.10080 (0.03986) + + + --> STEP: 214/234 -- GLOBAL_STEP: 54970 + | > loss: -0.41044 (-0.28124) + | > log_mle: -0.62000 (-0.43470) + | > loss_dur: 0.20956 (0.15346) + | > amp_scaler: 2048.00000 (1449.86916) + | > grad_norm: 72.72463 (34.18336) + | > current_lr: 0.00006 + | > step_time: 4.70530 (3.02516) + | > loader_time: 0.20390 (0.04133) + + + --> STEP: 219/234 -- GLOBAL_STEP: 54975 + | > loss: -0.50457 (-0.28443) + | > log_mle: -0.73725 (-0.43989) + | > loss_dur: 0.23268 (0.15546) + | > amp_scaler: 2048.00000 (1463.52511) + | > grad_norm: 88.53715 (35.27192) + | > current_lr: 0.00006 + | > step_time: 0.91650 (3.01561) + | > loader_time: 0.00650 (0.04148) + + + --> STEP: 224/234 -- GLOBAL_STEP: 54980 + | > loss: -0.45246 (-0.28747) + | > log_mle: -0.68250 (-0.44474) + | > loss_dur: 0.23004 (0.15728) + | > amp_scaler: 2048.00000 (1476.57143) + | > grad_norm: 72.33694 (36.37647) + | > current_lr: 0.00006 + | > step_time: 0.22510 (2.96067) + | > loader_time: 0.00350 (0.04063) + + + --> STEP: 229/234 -- GLOBAL_STEP: 54985 + | > loss: -0.43928 (-0.29066) + | > log_mle: -0.72688 (-0.45019) + | > loss_dur: 0.28760 (0.15953) + | > amp_scaler: 2048.00000 (1489.04803) + | > grad_norm: 95.76285 (37.61983) + | > current_lr: 0.00006 + | > step_time: 0.24750 (2.90125) + | > loader_time: 0.00410 (0.03982) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.49951 (+0.27178) + | > avg_loss: -0.29905 (+0.02733) + | > avg_log_mle: -0.52270 (+0.01708) + | > avg_loss_dur: 0.22365 (+0.01026) + + + > EPOCH: 235/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 10:34:18)  + + --> STEP: 0/234 -- GLOBAL_STEP: 54990 + | > loss: -0.29993 (-0.29993) + | > log_mle: -0.44812 (-0.44812) + | > loss_dur: 0.14820 (0.14820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.38175 (24.38175) + | > current_lr: 0.00006 + | > step_time: 4.50400 (4.50403) + | > loader_time: 5.67740 (5.67741) + + + --> STEP: 5/234 -- GLOBAL_STEP: 54995 + | > loss: -0.27675 (-0.25918) + | > log_mle: -0.36430 (-0.36250) + | > loss_dur: 0.08756 (0.10333) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.17848 (19.84333) + | > current_lr: 0.00006 + | > step_time: 3.20350 (3.90163) + | > loader_time: 0.09170 (0.03939) + + + --> STEP: 10/234 -- GLOBAL_STEP: 55000 + | > loss: -0.27654 (-0.27080) + | > log_mle: -0.36184 (-0.36567) + | > loss_dur: 0.08530 (0.09486) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.72123 (17.86882) + | > current_lr: 0.00006 + | > step_time: 3.30550 (5.36132) + | > loader_time: 0.00330 (0.04962) + + + --> STEP: 15/234 -- GLOBAL_STEP: 55005 + | > loss: -0.27941 (-0.27537) + | > log_mle: -0.37007 (-0.36740) + | > loss_dur: 0.09066 (0.09202) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.84879 (16.76186) + | > current_lr: 0.00006 + | > step_time: 0.72160 (4.62622) + | > loader_time: 0.00190 (0.03529) + + + --> STEP: 20/234 -- GLOBAL_STEP: 55010 + | > loss: -0.30454 (-0.27896) + | > log_mle: -0.36996 (-0.36659) + | > loss_dur: 0.06542 (0.08763) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.37880 (15.31640) + | > current_lr: 0.00006 + | > step_time: 2.33500 (3.84781) + | > loader_time: 0.00210 (0.03095) + + + --> STEP: 25/234 -- GLOBAL_STEP: 55015 + | > loss: -0.26940 (-0.28000) + | > log_mle: -0.34959 (-0.36514) + | > loss_dur: 0.08018 (0.08513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.72965 (14.42120) + | > current_lr: 0.00006 + | > step_time: 1.08540 (3.31374) + | > loader_time: 0.00320 (0.02827) + + + --> STEP: 30/234 -- GLOBAL_STEP: 55020 + | > loss: -0.26337 (-0.28014) + | > log_mle: -0.34907 (-0.36389) + | > loss_dur: 0.08570 (0.08375) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.14565 (14.04346) + | > current_lr: 0.00006 + | > step_time: 3.99310 (3.14432) + | > loader_time: 0.09930 (0.02994) + + + --> STEP: 35/234 -- GLOBAL_STEP: 55025 + | > loss: -0.23700 (-0.27725) + | > log_mle: -0.34411 (-0.36186) + | > loss_dur: 0.10711 (0.08461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.42870 (13.83138) + | > current_lr: 0.00006 + | > step_time: 1.90480 (2.96415) + | > loader_time: 0.00200 (0.03332) + + + --> STEP: 40/234 -- GLOBAL_STEP: 55030 + | > loss: -0.23459 (-0.27418) + | > log_mle: -0.33877 (-0.35989) + | > loss_dur: 0.10419 (0.08570) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.98591 (13.49202) + | > current_lr: 0.00006 + | > step_time: 1.30710 (2.81649) + | > loader_time: 0.00190 (0.03136) + + + --> STEP: 45/234 -- GLOBAL_STEP: 55035 + | > loss: -0.25456 (-0.27197) + | > log_mle: -0.36419 (-0.35875) + | > loss_dur: 0.10963 (0.08678) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.93364 (13.17650) + | > current_lr: 0.00006 + | > step_time: 0.95830 (2.70008) + | > loader_time: 0.00200 (0.02822) + + + --> STEP: 50/234 -- GLOBAL_STEP: 55040 + | > loss: -0.25031 (-0.27014) + | > log_mle: -0.34294 (-0.35765) + | > loss_dur: 0.09264 (0.08751) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.37451 (12.70370) + | > current_lr: 0.00006 + | > step_time: 2.09620 (2.60394) + | > loader_time: 0.00210 (0.02731) + + + --> STEP: 55/234 -- GLOBAL_STEP: 55045 + | > loss: -0.25983 (-0.26799) + | > log_mle: -0.34948 (-0.35659) + | > loss_dur: 0.08965 (0.08860) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.55953 (12.51210) + | > current_lr: 0.00006 + | > step_time: 2.10420 (2.56965) + | > loader_time: 0.00370 (0.02812) + + + --> STEP: 60/234 -- GLOBAL_STEP: 55050 + | > loss: -0.22431 (-0.26548) + | > log_mle: -0.35737 (-0.35578) + | > loss_dur: 0.13307 (0.09029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.10543 (12.48381) + | > current_lr: 0.00006 + | > step_time: 3.90590 (2.55930) + | > loader_time: 0.00190 (0.02718) + + + --> STEP: 65/234 -- GLOBAL_STEP: 55055 + | > loss: -0.23721 (-0.26249) + | > log_mle: -0.34229 (-0.35500) + | > loss_dur: 0.10508 (0.09250) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.36456 (12.51230) + | > current_lr: 0.00006 + | > step_time: 0.80000 (2.50355) + | > loader_time: 0.00220 (0.02683) + + + --> STEP: 70/234 -- GLOBAL_STEP: 55060 + | > loss: -0.20589 (-0.25988) + | > log_mle: -0.33084 (-0.35371) + | > loss_dur: 0.12495 (0.09384) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.28938 (12.35928) + | > current_lr: 0.00006 + | > step_time: 1.49420 (2.48034) + | > loader_time: 0.00210 (0.02514) + + + --> STEP: 75/234 -- GLOBAL_STEP: 55065 + | > loss: -0.21759 (-0.25728) + | > log_mle: -0.34623 (-0.35309) + | > loss_dur: 0.12864 (0.09581) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08078 (12.56645) + | > current_lr: 0.00006 + | > step_time: 1.83990 (2.49598) + | > loader_time: 0.00200 (0.02664) + + + --> STEP: 80/234 -- GLOBAL_STEP: 55070 + | > loss: -0.23044 (-0.25556) + | > log_mle: -0.32999 (-0.35218) + | > loss_dur: 0.09956 (0.09662) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.59069 (12.50794) + | > current_lr: 0.00006 + | > step_time: 1.93870 (2.45713) + | > loader_time: 0.00230 (0.02510) + + + --> STEP: 85/234 -- GLOBAL_STEP: 55075 + | > loss: -0.23056 (-0.25376) + | > log_mle: -0.33963 (-0.35171) + | > loss_dur: 0.10906 (0.09795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.29543 (12.51403) + | > current_lr: 0.00006 + | > step_time: 2.50490 (2.45018) + | > loader_time: 0.00190 (0.02476) + + + --> STEP: 90/234 -- GLOBAL_STEP: 55080 + | > loss: -0.22602 (-0.25223) + | > log_mle: -0.36142 (-0.35217) + | > loss_dur: 0.13540 (0.09994) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.69751 (12.70728) + | > current_lr: 0.00006 + | > step_time: 1.79120 (2.42388) + | > loader_time: 0.00200 (0.02446) + + + --> STEP: 95/234 -- GLOBAL_STEP: 55085 + | > loss: -0.27661 (-0.25189) + | > log_mle: -0.44357 (-0.35440) + | > loss_dur: 0.16696 (0.10251) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.66618 (13.31704) + | > current_lr: 0.00006 + | > step_time: 1.55540 (2.40465) + | > loader_time: 0.00190 (0.02532) + + + --> STEP: 100/234 -- GLOBAL_STEP: 55090 + | > loss: -0.23927 (-0.25095) + | > log_mle: -0.37166 (-0.35508) + | > loss_dur: 0.13239 (0.10414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.61912 (13.54160) + | > current_lr: 0.00006 + | > step_time: 2.88700 (2.39828) + | > loader_time: 0.00470 (0.02592) + + + --> STEP: 105/234 -- GLOBAL_STEP: 55095 + | > loss: -0.22310 (-0.25041) + | > log_mle: -0.35039 (-0.35688) + | > loss_dur: 0.12728 (0.10647) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.58957 (14.05312) + | > current_lr: 0.00006 + | > step_time: 2.92380 (2.38355) + | > loader_time: 0.00510 (0.02558) + + + --> STEP: 110/234 -- GLOBAL_STEP: 55100 + | > loss: -0.22375 (-0.24925) + | > log_mle: -0.37290 (-0.35838) + | > loss_dur: 0.14915 (0.10912) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.88916 (14.44533) + | > current_lr: 0.00006 + | > step_time: 1.79760 (2.35298) + | > loader_time: 0.00170 (0.02455) + + + --> STEP: 115/234 -- GLOBAL_STEP: 55105 + | > loss: -0.22567 (-0.24911) + | > log_mle: -0.39751 (-0.36079) + | > loss_dur: 0.17184 (0.11167) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.81947 (15.11592) + | > current_lr: 0.00006 + | > step_time: 2.28850 (2.34456) + | > loader_time: 0.00240 (0.02431) + + + --> STEP: 120/234 -- GLOBAL_STEP: 55110 + | > loss: -0.28068 (-0.24882) + | > log_mle: -0.44395 (-0.36277) + | > loss_dur: 0.16327 (0.11395) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.24363 (15.76598) + | > current_lr: 0.00006 + | > step_time: 2.71530 (2.32946) + | > loader_time: 0.00220 (0.02477) + + + --> STEP: 125/234 -- GLOBAL_STEP: 55115 + | > loss: -0.24954 (-0.24792) + | > log_mle: -0.43115 (-0.36351) + | > loss_dur: 0.18161 (0.11559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.65330 (16.17163) + | > current_lr: 0.00006 + | > step_time: 2.37980 (2.37613) + | > loader_time: 0.00200 (0.02685) + + + --> STEP: 130/234 -- GLOBAL_STEP: 55120 + | > loss: -0.26782 (-0.24822) + | > log_mle: -0.44295 (-0.36626) + | > loss_dur: 0.17513 (0.11803) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.31267 (17.08461) + | > current_lr: 0.00006 + | > step_time: 2.40790 (2.38311) + | > loader_time: 0.00380 (0.02730) + + + --> STEP: 135/234 -- GLOBAL_STEP: 55125 + | > loss: -0.22183 (-0.24879) + | > log_mle: -0.36878 (-0.36887) + | > loss_dur: 0.14695 (0.12008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.54915 (17.81130) + | > current_lr: 0.00006 + | > step_time: 1.42160 (2.37282) + | > loader_time: 0.00400 (0.02829) + + + --> STEP: 140/234 -- GLOBAL_STEP: 55130 + | > loss: -0.21268 (-0.24940) + | > log_mle: -0.40156 (-0.37204) + | > loss_dur: 0.18888 (0.12264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.40245 (18.82087) + | > current_lr: 0.00006 + | > step_time: 3.20890 (2.38464) + | > loader_time: 0.08520 (0.02856) + + + --> STEP: 145/234 -- GLOBAL_STEP: 55135 + | > loss: -0.31502 (-0.25081) + | > log_mle: -0.50401 (-0.37597) + | > loss_dur: 0.18899 (0.12516) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.00287 (19.70563) + | > current_lr: 0.00006 + | > step_time: 4.00080 (2.40928) + | > loader_time: 0.09980 (0.02903) + + + --> STEP: 150/234 -- GLOBAL_STEP: 55140 + | > loss: -0.28096 (-0.25234) + | > log_mle: -0.49024 (-0.37972) + | > loss_dur: 0.20928 (0.12738) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.29856 (20.51034) + | > current_lr: 0.00006 + | > step_time: 2.59980 (2.44227) + | > loader_time: 0.09560 (0.02998) + + + --> STEP: 155/234 -- GLOBAL_STEP: 55145 + | > loss: -0.36885 (-0.25509) + | > log_mle: -0.55648 (-0.38455) + | > loss_dur: 0.18763 (0.12946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.09607 (21.76121) + | > current_lr: 0.00006 + | > step_time: 4.29480 (2.55455) + | > loader_time: 0.00310 (0.03109) + + + --> STEP: 160/234 -- GLOBAL_STEP: 55150 + | > loss: -0.32941 (-0.25691) + | > log_mle: -0.54785 (-0.38868) + | > loss_dur: 0.21844 (0.13178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.39907 (22.72942) + | > current_lr: 0.00006 + | > step_time: 4.09130 (2.62086) + | > loader_time: 0.00670 (0.03253) + + + --> STEP: 165/234 -- GLOBAL_STEP: 55155 + | > loss: -0.34038 (-0.25874) + | > log_mle: -0.54276 (-0.39248) + | > loss_dur: 0.20238 (0.13373) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.41087 (23.67654) + | > current_lr: 0.00006 + | > step_time: 4.80630 (2.72406) + | > loader_time: 0.09080 (0.03864) + + + --> STEP: 170/234 -- GLOBAL_STEP: 55160 + | > loss: -0.35811 (-0.26074) + | > log_mle: -0.58134 (-0.39659) + | > loss_dur: 0.22323 (0.13584) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.06704 (24.83801) + | > current_lr: 0.00006 + | > step_time: 8.41110 (2.77866) + | > loader_time: 0.49210 (0.04054) + + + --> STEP: 175/234 -- GLOBAL_STEP: 55165 + | > loss: -0.32460 (-0.26353) + | > log_mle: -0.56018 (-0.40161) + | > loss_dur: 0.23558 (0.13808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.14141 (25.76004) + | > current_lr: 0.00006 + | > step_time: 5.99230 (2.83813) + | > loader_time: 0.00240 (0.04225) + + + --> STEP: 180/234 -- GLOBAL_STEP: 55170 + | > loss: -0.35237 (-0.26595) + | > log_mle: -0.56840 (-0.40642) + | > loss_dur: 0.21602 (0.14047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.34535 (26.61125) + | > current_lr: 0.00006 + | > step_time: 8.91520 (2.89334) + | > loader_time: 0.09200 (0.04264) + + + --> STEP: 185/234 -- GLOBAL_STEP: 55175 + | > loss: -0.37609 (-0.26819) + | > log_mle: -0.60121 (-0.41083) + | > loss_dur: 0.22511 (0.14264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.84713 (27.51707) + | > current_lr: 0.00006 + | > step_time: 3.59580 (3.03677) + | > loader_time: 0.00410 (0.04469) + + + --> STEP: 190/234 -- GLOBAL_STEP: 55180 + | > loss: -0.37382 (-0.27060) + | > log_mle: -0.57412 (-0.41526) + | > loss_dur: 0.20030 (0.14466) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.96761 (28.51224) + | > current_lr: 0.00006 + | > step_time: 9.99620 (3.12110) + | > loader_time: 0.29860 (0.04563) + + + --> STEP: 195/234 -- GLOBAL_STEP: 55185 + | > loss: -0.37081 (-0.27354) + | > log_mle: -0.59518 (-0.41995) + | > loss_dur: 0.22437 (0.14641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.00507 (29.60805) + | > current_lr: 0.00006 + | > step_time: 1.90740 (3.14939) + | > loader_time: 0.00280 (0.04648) + + + --> STEP: 200/234 -- GLOBAL_STEP: 55190 + | > loss: -0.36595 (-0.27595) + | > log_mle: -0.60834 (-0.42433) + | > loss_dur: 0.24239 (0.14837) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.85123 (30.86236) + | > current_lr: 0.00006 + | > step_time: 5.40600 (3.21572) + | > loader_time: 1.08220 (0.05226) + + + --> STEP: 205/234 -- GLOBAL_STEP: 55195 + | > loss: -0.36988 (-0.27826) + | > log_mle: -0.59367 (-0.42855) + | > loss_dur: 0.22379 (0.15029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.53735 (31.99373) + | > current_lr: 0.00006 + | > step_time: 5.70770 (3.29536) + | > loader_time: 0.11230 (0.05495) + + + --> STEP: 210/234 -- GLOBAL_STEP: 55200 + | > loss: -0.43763 (-0.28152) + | > log_mle: -0.67773 (-0.43363) + | > loss_dur: 0.24010 (0.15211) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.58593 (33.28376) + | > current_lr: 0.00006 + | > step_time: 2.50240 (3.30965) + | > loader_time: 0.00300 (0.05514) + + + --> STEP: 215/234 -- GLOBAL_STEP: 55205 + | > loss: -0.39900 (-0.28485) + | > log_mle: -0.62895 (-0.43883) + | > loss_dur: 0.22995 (0.15398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.44336 (34.64924) + | > current_lr: 0.00006 + | > step_time: 8.10420 (3.37923) + | > loader_time: 0.09520 (0.05660) + + + --> STEP: 220/234 -- GLOBAL_STEP: 55210 + | > loss: -0.43348 (-0.28848) + | > log_mle: -0.67299 (-0.44441) + | > loss_dur: 0.23952 (0.15592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.70553 (36.12307) + | > current_lr: 0.00006 + | > step_time: 1.29340 (3.42926) + | > loader_time: 0.00400 (0.05671) + + + --> STEP: 225/234 -- GLOBAL_STEP: 55215 + | > loss: -0.47469 (-0.29170) + | > log_mle: -0.72048 (-0.44952) + | > loss_dur: 0.24579 (0.15781) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.54771 (37.25254) + | > current_lr: 0.00006 + | > step_time: 0.24220 (3.36518) + | > loader_time: 0.00450 (0.05589) + + + --> STEP: 230/234 -- GLOBAL_STEP: 55220 + | > loss: -0.46626 (-0.29460) + | > log_mle: -0.79277 (-0.45511) + | > loss_dur: 0.32650 (0.16051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.77493 (38.32327) + | > current_lr: 0.00006 + | > step_time: 0.25830 (3.29755) + | > loader_time: 0.00470 (0.05477) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32969 (-0.16982) + | > avg_loss: -0.32177 (-0.02272) + | > avg_log_mle: -0.53324 (-0.01054) + | > avg_loss_dur: 0.21146 (-0.01219) + + + > EPOCH: 236/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 10:48:05)  + + --> STEP: 1/234 -- GLOBAL_STEP: 55225 + | > loss: -0.27432 (-0.27432) + | > log_mle: -0.36115 (-0.36115) + | > loss_dur: 0.08683 (0.08683) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.39958 (18.39958) + | > current_lr: 0.00006 + | > step_time: 3.51310 (3.51312) + | > loader_time: 0.08440 (0.08435) + + + --> STEP: 6/234 -- GLOBAL_STEP: 55230 + | > loss: -0.29460 (-0.27078) + | > log_mle: -0.36196 (-0.36454) + | > loss_dur: 0.06736 (0.09376) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.64549 (18.25052) + | > current_lr: 0.00006 + | > step_time: 2.69460 (6.31992) + | > loader_time: 0.00330 (0.07897) + + + --> STEP: 11/234 -- GLOBAL_STEP: 55235 + | > loss: -0.30252 (-0.27285) + | > log_mle: -0.37552 (-0.36800) + | > loss_dur: 0.07300 (0.09516) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.17310 (17.66704) + | > current_lr: 0.00006 + | > step_time: 17.61840 (5.63059) + | > loader_time: 0.18180 (0.06051) + + + --> STEP: 16/234 -- GLOBAL_STEP: 55240 + | > loss: -0.30670 (-0.27835) + | > log_mle: -0.37514 (-0.36950) + | > loss_dur: 0.06844 (0.09115) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58005 (16.85429) + | > current_lr: 0.00006 + | > step_time: 1.28460 (4.17156) + | > loader_time: 0.00100 (0.05422) + + + --> STEP: 21/234 -- GLOBAL_STEP: 55245 + | > loss: -0.27097 (-0.28054) + | > log_mle: -0.34789 (-0.36722) + | > loss_dur: 0.07691 (0.08668) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.09716 (16.07132) + | > current_lr: 0.00006 + | > step_time: 4.60000 (4.26871) + | > loader_time: 0.00260 (0.05071) + + + --> STEP: 26/234 -- GLOBAL_STEP: 55250 + | > loss: -0.27195 (-0.28101) + | > log_mle: -0.35551 (-0.36614) + | > loss_dur: 0.08356 (0.08513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.20064 (15.14936) + | > current_lr: 0.00006 + | > step_time: 2.40140 (4.02192) + | > loader_time: 0.09420 (0.05893) + + + --> STEP: 31/234 -- GLOBAL_STEP: 55255 + | > loss: -0.23540 (-0.28002) + | > log_mle: -0.34631 (-0.36441) + | > loss_dur: 0.11091 (0.08439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.61996 (14.56604) + | > current_lr: 0.00006 + | > step_time: 1.59910 (4.19293) + | > loader_time: 0.00210 (0.05617) + + + --> STEP: 36/234 -- GLOBAL_STEP: 55260 + | > loss: -0.24699 (-0.27734) + | > log_mle: -0.34327 (-0.36248) + | > loss_dur: 0.09628 (0.08514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.31396 (14.28403) + | > current_lr: 0.00006 + | > step_time: 2.28670 (3.82799) + | > loader_time: 0.00160 (0.04867) + + + --> STEP: 41/234 -- GLOBAL_STEP: 55265 + | > loss: -0.27596 (-0.27454) + | > log_mle: -0.35444 (-0.36075) + | > loss_dur: 0.07848 (0.08622) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.67040 (14.04709) + | > current_lr: 0.00006 + | > step_time: 2.89840 (3.61216) + | > loader_time: 0.00200 (0.04300) + + + --> STEP: 46/234 -- GLOBAL_STEP: 55270 + | > loss: -0.24463 (-0.27110) + | > log_mle: -0.34630 (-0.35919) + | > loss_dur: 0.10167 (0.08808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.65281 (13.91540) + | > current_lr: 0.00006 + | > step_time: 1.09570 (3.40163) + | > loader_time: 0.00200 (0.03861) + + + --> STEP: 51/234 -- GLOBAL_STEP: 55275 + | > loss: -0.25241 (-0.26959) + | > log_mle: -0.33974 (-0.35788) + | > loss_dur: 0.08733 (0.08829) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.34802 (13.37299) + | > current_lr: 0.00006 + | > step_time: 0.98670 (3.23133) + | > loader_time: 0.00300 (0.03850) + + + --> STEP: 56/234 -- GLOBAL_STEP: 55280 + | > loss: -0.24123 (-0.26777) + | > log_mle: -0.34689 (-0.35693) + | > loss_dur: 0.10566 (0.08916) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.67884 (13.17047) + | > current_lr: 0.00006 + | > step_time: 1.26800 (3.07463) + | > loader_time: 0.00210 (0.03818) + + + --> STEP: 61/234 -- GLOBAL_STEP: 55285 + | > loss: -0.23917 (-0.26558) + | > log_mle: -0.34321 (-0.35606) + | > loss_dur: 0.10404 (0.09048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.11709 (13.12869) + | > current_lr: 0.00006 + | > step_time: 1.26060 (2.92714) + | > loader_time: 0.00240 (0.03663) + + + --> STEP: 66/234 -- GLOBAL_STEP: 55290 + | > loss: -0.25381 (-0.26338) + | > log_mle: -0.33990 (-0.35516) + | > loss_dur: 0.08609 (0.09178) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.81858 (13.11509) + | > current_lr: 0.00006 + | > step_time: 2.50870 (2.84022) + | > loader_time: 0.07750 (0.03655) + + + --> STEP: 71/234 -- GLOBAL_STEP: 55295 + | > loss: -0.21105 (-0.26028) + | > log_mle: -0.36279 (-0.35423) + | > loss_dur: 0.15174 (0.09395) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.37511 (13.05733) + | > current_lr: 0.00006 + | > step_time: 1.57070 (2.83404) + | > loader_time: 0.00240 (0.03665) + + + --> STEP: 76/234 -- GLOBAL_STEP: 55300 + | > loss: -0.22863 (-0.25732) + | > log_mle: -0.34508 (-0.35331) + | > loss_dur: 0.11645 (0.09599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.87462 (13.06639) + | > current_lr: 0.00006 + | > step_time: 2.49630 (2.77901) + | > loader_time: 0.08840 (0.03551) + + + --> STEP: 81/234 -- GLOBAL_STEP: 55305 + | > loss: -0.23297 (-0.25568) + | > log_mle: -0.35584 (-0.35255) + | > loss_dur: 0.12287 (0.09687) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.87419 (12.92424) + | > current_lr: 0.00006 + | > step_time: 1.49070 (2.73963) + | > loader_time: 0.00600 (0.03556) + + + --> STEP: 86/234 -- GLOBAL_STEP: 55310 + | > loss: -0.22866 (-0.25373) + | > log_mle: -0.34648 (-0.35196) + | > loss_dur: 0.11782 (0.09823) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.34985 (12.97322) + | > current_lr: 0.00006 + | > step_time: 3.00490 (2.72173) + | > loader_time: 0.08630 (0.03469) + + + --> STEP: 91/234 -- GLOBAL_STEP: 55315 + | > loss: -0.22409 (-0.25213) + | > log_mle: -0.35899 (-0.35242) + | > loss_dur: 0.13490 (0.10029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.92024 (13.29939) + | > current_lr: 0.00006 + | > step_time: 1.99240 (2.69209) + | > loader_time: 0.00330 (0.03391) + + + --> STEP: 96/234 -- GLOBAL_STEP: 55320 + | > loss: -0.22877 (-0.25188) + | > log_mle: -0.34781 (-0.35447) + | > loss_dur: 0.11904 (0.10259) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78093 (13.74784) + | > current_lr: 0.00006 + | > step_time: 1.79790 (2.65900) + | > loader_time: 0.09320 (0.03414) + + + --> STEP: 101/234 -- GLOBAL_STEP: 55325 + | > loss: -0.23459 (-0.25109) + | > log_mle: -0.39797 (-0.35571) + | > loss_dur: 0.16337 (0.10462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.88581 (14.09194) + | > current_lr: 0.00006 + | > step_time: 1.11000 (2.68382) + | > loader_time: 0.08390 (0.03526) + + + --> STEP: 106/234 -- GLOBAL_STEP: 55330 + | > loss: -0.20865 (-0.25020) + | > log_mle: -0.39313 (-0.35741) + | > loss_dur: 0.18449 (0.10721) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.85625 (14.59054) + | > current_lr: 0.00006 + | > step_time: 2.60310 (2.69719) + | > loader_time: 0.00350 (0.03710) + + + --> STEP: 111/234 -- GLOBAL_STEP: 55335 + | > loss: -0.25239 (-0.24940) + | > log_mle: -0.44875 (-0.35922) + | > loss_dur: 0.19636 (0.10982) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.40326 (15.20557) + | > current_lr: 0.00006 + | > step_time: 2.12100 (2.67117) + | > loader_time: 0.00370 (0.03706) + + + --> STEP: 116/234 -- GLOBAL_STEP: 55340 + | > loss: -0.23174 (-0.24879) + | > log_mle: -0.41424 (-0.36111) + | > loss_dur: 0.18250 (0.11232) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.60728 (15.75341) + | > current_lr: 0.00006 + | > step_time: 1.47590 (2.65330) + | > loader_time: 0.00250 (0.03715) + + + --> STEP: 121/234 -- GLOBAL_STEP: 55345 + | > loss: -0.19424 (-0.24822) + | > log_mle: -0.32914 (-0.36227) + | > loss_dur: 0.13490 (0.11405) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.43138 (15.99930) + | > current_lr: 0.00006 + | > step_time: 1.29150 (2.63280) + | > loader_time: 0.00270 (0.03644) + + + --> STEP: 126/234 -- GLOBAL_STEP: 55350 + | > loss: -0.27374 (-0.24799) + | > log_mle: -0.45778 (-0.36398) + | > loss_dur: 0.18404 (0.11599) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.12550 (16.54229) + | > current_lr: 0.00006 + | > step_time: 4.51700 (2.63230) + | > loader_time: 0.08210 (0.03710) + + + --> STEP: 131/234 -- GLOBAL_STEP: 55355 + | > loss: -0.32491 (-0.24880) + | > log_mle: -0.51389 (-0.36700) + | > loss_dur: 0.18898 (0.11820) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.81425 (17.27062) + | > current_lr: 0.00006 + | > step_time: 2.88970 (2.63297) + | > loader_time: 0.00350 (0.03580) + + + --> STEP: 136/234 -- GLOBAL_STEP: 55360 + | > loss: -0.34871 (-0.24976) + | > log_mle: -0.56125 (-0.37008) + | > loss_dur: 0.21254 (0.12032) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.75185 (18.03914) + | > current_lr: 0.00006 + | > step_time: 1.39120 (2.61617) + | > loader_time: 0.00260 (0.03524) + + + --> STEP: 141/234 -- GLOBAL_STEP: 55365 + | > loss: -0.28448 (-0.25015) + | > log_mle: -0.45860 (-0.37260) + | > loss_dur: 0.17412 (0.12245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.31271 (18.64928) + | > current_lr: 0.00006 + | > step_time: 3.59610 (2.61547) + | > loader_time: 0.18890 (0.03677) + + + --> STEP: 146/234 -- GLOBAL_STEP: 55370 + | > loss: -0.31381 (-0.25191) + | > log_mle: -0.51307 (-0.37704) + | > loss_dur: 0.19926 (0.12514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.80613 (19.77474) + | > current_lr: 0.00006 + | > step_time: 6.91010 (2.67946) + | > loader_time: 0.00690 (0.03809) + + + --> STEP: 151/234 -- GLOBAL_STEP: 55375 + | > loss: -0.29893 (-0.25355) + | > log_mle: -0.47700 (-0.38059) + | > loss_dur: 0.17807 (0.12704) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.04952 (20.69010) + | > current_lr: 0.00006 + | > step_time: 2.50180 (2.72125) + | > loader_time: 0.00360 (0.03890) + + + --> STEP: 156/234 -- GLOBAL_STEP: 55380 + | > loss: -0.32473 (-0.25637) + | > log_mle: -0.51814 (-0.38572) + | > loss_dur: 0.19341 (0.12934) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.08765 (21.99306) + | > current_lr: 0.00006 + | > step_time: 2.19800 (2.73557) + | > loader_time: 0.08440 (0.04067) + + + --> STEP: 161/234 -- GLOBAL_STEP: 55385 + | > loss: -0.35665 (-0.25819) + | > log_mle: -0.54192 (-0.38980) + | > loss_dur: 0.18527 (0.13162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.49928 (23.26857) + | > current_lr: 0.00006 + | > step_time: 4.90760 (2.73701) + | > loader_time: 0.08870 (0.04053) + + + --> STEP: 166/234 -- GLOBAL_STEP: 55390 + | > loss: -0.30525 (-0.25981) + | > log_mle: -0.48195 (-0.39336) + | > loss_dur: 0.17670 (0.13354) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.25570 (24.13321) + | > current_lr: 0.00006 + | > step_time: 2.69910 (2.76078) + | > loader_time: 0.00310 (0.04002) + + + --> STEP: 171/234 -- GLOBAL_STEP: 55395 + | > loss: -0.39582 (-0.26282) + | > log_mle: -0.59903 (-0.39858) + | > loss_dur: 0.20321 (0.13576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.72539 (25.32080) + | > current_lr: 0.00006 + | > step_time: 7.19850 (2.79370) + | > loader_time: 0.29930 (0.04286) + + + --> STEP: 176/234 -- GLOBAL_STEP: 55400 + | > loss: -0.36166 (-0.26581) + | > log_mle: -0.57026 (-0.40375) + | > loss_dur: 0.20860 (0.13794) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.72329 (26.41263) + | > current_lr: 0.00006 + | > step_time: 6.70590 (2.87794) + | > loader_time: 0.09640 (0.04721) + + + --> STEP: 181/234 -- GLOBAL_STEP: 55405 + | > loss: -0.29054 (-0.26805) + | > log_mle: -0.50046 (-0.40831) + | > loss_dur: 0.20992 (0.14026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.49202 (27.52886) + | > current_lr: 0.00006 + | > step_time: 4.90520 (2.93232) + | > loader_time: 0.08970 (0.04857) + + + --> STEP: 186/234 -- GLOBAL_STEP: 55410 + | > loss: -0.31016 (-0.27044) + | > log_mle: -0.54307 (-0.41299) + | > loss_dur: 0.23290 (0.14255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.62516 (28.78149) + | > current_lr: 0.00006 + | > step_time: 4.49150 (3.00567) + | > loader_time: 0.00300 (0.04838) + + + --> STEP: 191/234 -- GLOBAL_STEP: 55415 + | > loss: -0.36399 (-0.27310) + | > log_mle: -0.55956 (-0.41740) + | > loss_dur: 0.19557 (0.14431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.67023 (29.64045) + | > current_lr: 0.00006 + | > step_time: 8.29170 (3.08071) + | > loader_time: 0.00350 (0.04919) + + + --> STEP: 196/234 -- GLOBAL_STEP: 55420 + | > loss: -0.33183 (-0.27589) + | > log_mle: -0.55860 (-0.42198) + | > loss_dur: 0.22678 (0.14609) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.26873 (30.69489) + | > current_lr: 0.00006 + | > step_time: 4.79980 (3.12303) + | > loader_time: 0.00800 (0.05456) + + + --> STEP: 201/234 -- GLOBAL_STEP: 55425 + | > loss: -0.28165 (-0.27805) + | > log_mle: -0.50699 (-0.42602) + | > loss_dur: 0.22534 (0.14797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.01111 (31.72665) + | > current_lr: 0.00006 + | > step_time: 6.03390 (3.22368) + | > loader_time: 0.29750 (0.06150) + + + --> STEP: 206/234 -- GLOBAL_STEP: 55430 + | > loss: -0.41519 (-0.28086) + | > log_mle: -0.63106 (-0.43066) + | > loss_dur: 0.21587 (0.14979) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.76276 (32.61314) + | > current_lr: 0.00006 + | > step_time: 5.59480 (3.25109) + | > loader_time: 0.10320 (0.06193) + + + --> STEP: 211/234 -- GLOBAL_STEP: 55435 + | > loss: -0.46460 (-0.28425) + | > log_mle: -0.71025 (-0.43604) + | > loss_dur: 0.24565 (0.15179) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.24456 (33.90070) + | > current_lr: 0.00006 + | > step_time: 6.59620 (3.35835) + | > loader_time: 0.19900 (0.06286) + + + --> STEP: 216/234 -- GLOBAL_STEP: 55440 + | > loss: -0.44888 (-0.28744) + | > log_mle: -0.70002 (-0.44112) + | > loss_dur: 0.25114 (0.15367) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.54692 (35.20234) + | > current_lr: 0.00006 + | > step_time: 3.70670 (3.41818) + | > loader_time: 0.09380 (0.06368) + + + --> STEP: 221/234 -- GLOBAL_STEP: 55445 + | > loss: -0.38489 (-0.29065) + | > log_mle: -0.60208 (-0.44614) + | > loss_dur: 0.21719 (0.15549) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.18203 (36.47351) + | > current_lr: 0.00006 + | > step_time: 2.19310 (3.43126) + | > loader_time: 0.00370 (0.06314) + + + --> STEP: 226/234 -- GLOBAL_STEP: 55450 + | > loss: -0.45512 (-0.29424) + | > log_mle: -0.70613 (-0.45179) + | > loss_dur: 0.25101 (0.15755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.65153 (37.68284) + | > current_lr: 0.00006 + | > step_time: 0.22940 (3.37237) + | > loader_time: 0.00250 (0.06183) + + + --> STEP: 231/234 -- GLOBAL_STEP: 55455 + | > loss: -0.40138 (-0.29714) + | > log_mle: -0.78050 (-0.45780) + | > loss_dur: 0.37912 (0.16066) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.08974 (39.08489) + | > current_lr: 0.00006 + | > step_time: 0.27270 (3.30493) + | > loader_time: 0.00430 (0.06058) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22878 (-0.10091) + | > avg_loss: -0.31654 (+0.00523) + | > avg_log_mle: -0.53632 (-0.00308) + | > avg_loss_dur: 0.21978 (+0.00831) + + + > EPOCH: 237/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 11:02:14)  + + --> STEP: 2/234 -- GLOBAL_STEP: 55460 + | > loss: -0.29764 (-0.27813) + | > log_mle: -0.37606 (-0.36868) + | > loss_dur: 0.07842 (0.09055) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.76855 (22.45272) + | > current_lr: 0.00006 + | > step_time: 2.60430 (8.25269) + | > loader_time: 0.00590 (0.00622) + + + --> STEP: 7/234 -- GLOBAL_STEP: 55465 + | > loss: -0.29658 (-0.27065) + | > log_mle: -0.36833 (-0.36491) + | > loss_dur: 0.07175 (0.09426) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.67762 (20.18967) + | > current_lr: 0.00006 + | > step_time: 2.49420 (5.28688) + | > loader_time: 0.00120 (0.01642) + + + --> STEP: 12/234 -- GLOBAL_STEP: 55470 + | > loss: -0.25654 (-0.27225) + | > log_mle: -0.36267 (-0.36775) + | > loss_dur: 0.10613 (0.09550) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.41754 (19.47080) + | > current_lr: 0.00006 + | > step_time: 2.69730 (4.02256) + | > loader_time: 0.00670 (0.01821) + + + --> STEP: 17/234 -- GLOBAL_STEP: 55475 + | > loss: -0.29221 (-0.27923) + | > log_mle: -0.36179 (-0.36912) + | > loss_dur: 0.06958 (0.08988) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.49793 (17.30336) + | > current_lr: 0.00006 + | > step_time: 3.90000 (3.46503) + | > loader_time: 0.00110 (0.01890) + + + --> STEP: 22/234 -- GLOBAL_STEP: 55480 + | > loss: -0.28073 (-0.28019) + | > log_mle: -0.36766 (-0.36715) + | > loss_dur: 0.08692 (0.08696) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.21529 (16.19492) + | > current_lr: 0.00006 + | > step_time: 4.70130 (3.88254) + | > loader_time: 0.08840 (0.03182) + + + --> STEP: 27/234 -- GLOBAL_STEP: 55485 + | > loss: -0.28222 (-0.28175) + | > log_mle: -0.36152 (-0.36606) + | > loss_dur: 0.07930 (0.08431) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.74236 (15.26118) + | > current_lr: 0.00006 + | > step_time: 2.39360 (3.46409) + | > loader_time: 0.00210 (0.02647) + + + --> STEP: 32/234 -- GLOBAL_STEP: 55490 + | > loss: -0.28969 (-0.28116) + | > log_mle: -0.36190 (-0.36495) + | > loss_dur: 0.07221 (0.08380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.26995 (14.48338) + | > current_lr: 0.00006 + | > step_time: 2.69170 (3.46098) + | > loader_time: 0.00260 (0.02833) + + + --> STEP: 37/234 -- GLOBAL_STEP: 55495 + | > loss: -0.25648 (-0.27809) + | > log_mle: -0.34189 (-0.36270) + | > loss_dur: 0.08541 (0.08461) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.39799 (14.07959) + | > current_lr: 0.00006 + | > step_time: 3.90020 (3.53414) + | > loader_time: 0.09660 (0.03465) + + + --> STEP: 42/234 -- GLOBAL_STEP: 55500 + | > loss: -0.24308 (-0.27560) + | > log_mle: -0.33350 (-0.36105) + | > loss_dur: 0.09042 (0.08546) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.63923 (13.67061) + | > current_lr: 0.00006 + | > step_time: 1.34620 (3.50038) + | > loader_time: 0.00190 (0.03521) + + + --> STEP: 47/234 -- GLOBAL_STEP: 55505 + | > loss: -0.24670 (-0.27262) + | > log_mle: -0.34830 (-0.36014) + | > loss_dur: 0.10160 (0.08752) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.21410 (13.49212) + | > current_lr: 0.00006 + | > step_time: 2.71330 (3.32731) + | > loader_time: 0.09300 (0.03542) + + + --> STEP: 52/234 -- GLOBAL_STEP: 55510 + | > loss: -0.24214 (-0.27144) + | > log_mle: -0.34516 (-0.35899) + | > loss_dur: 0.10303 (0.08755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.77530 (13.13935) + | > current_lr: 0.00006 + | > step_time: 1.18740 (3.14359) + | > loader_time: 0.00170 (0.03387) + + + --> STEP: 57/234 -- GLOBAL_STEP: 55515 + | > loss: -0.22510 (-0.26947) + | > log_mle: -0.32936 (-0.35789) + | > loss_dur: 0.10427 (0.08842) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.52817 (13.03109) + | > current_lr: 0.00006 + | > step_time: 1.18980 (3.00117) + | > loader_time: 0.00210 (0.03108) + + + --> STEP: 62/234 -- GLOBAL_STEP: 55520 + | > loss: -0.19607 (-0.26678) + | > log_mle: -0.36383 (-0.35768) + | > loss_dur: 0.16776 (0.09090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.25069 (13.11475) + | > current_lr: 0.00006 + | > step_time: 1.81880 (2.90596) + | > loader_time: 0.00300 (0.03031) + + + --> STEP: 67/234 -- GLOBAL_STEP: 55525 + | > loss: -0.23830 (-0.26494) + | > log_mle: -0.35374 (-0.35664) + | > loss_dur: 0.11544 (0.09170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.45917 (12.85895) + | > current_lr: 0.00006 + | > step_time: 1.91570 (2.83264) + | > loader_time: 0.00300 (0.03098) + + + --> STEP: 72/234 -- GLOBAL_STEP: 55530 + | > loss: -0.23249 (-0.26206) + | > log_mle: -0.33840 (-0.35547) + | > loss_dur: 0.10591 (0.09341) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.77296 (12.90756) + | > current_lr: 0.00006 + | > step_time: 1.48830 (2.75972) + | > loader_time: 0.00240 (0.03020) + + + --> STEP: 77/234 -- GLOBAL_STEP: 55535 + | > loss: -0.23830 (-0.25934) + | > log_mle: -0.34079 (-0.35451) + | > loss_dur: 0.10249 (0.09517) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.24326 (13.04456) + | > current_lr: 0.00006 + | > step_time: 1.22290 (2.69447) + | > loader_time: 0.00240 (0.02945) + + + --> STEP: 82/234 -- GLOBAL_STEP: 55540 + | > loss: -0.22776 (-0.25749) + | > log_mle: -0.33760 (-0.35358) + | > loss_dur: 0.10985 (0.09608) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.32304 (12.98702) + | > current_lr: 0.00006 + | > step_time: 1.95940 (2.65633) + | > loader_time: 0.08710 (0.03107) + + + --> STEP: 87/234 -- GLOBAL_STEP: 55545 + | > loss: -0.21119 (-0.25539) + | > log_mle: -0.33902 (-0.35299) + | > loss_dur: 0.12784 (0.09760) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.81304 (13.00836) + | > current_lr: 0.00006 + | > step_time: 2.10220 (2.62349) + | > loader_time: 0.00270 (0.03039) + + + --> STEP: 92/234 -- GLOBAL_STEP: 55550 + | > loss: -0.24681 (-0.25399) + | > log_mle: -0.38412 (-0.35395) + | > loss_dur: 0.13731 (0.09995) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.73585 (13.23241) + | > current_lr: 0.00006 + | > step_time: 1.77690 (2.61476) + | > loader_time: 0.00250 (0.03178) + + + --> STEP: 97/234 -- GLOBAL_STEP: 55555 + | > loss: -0.23194 (-0.25350) + | > log_mle: -0.36667 (-0.35576) + | > loss_dur: 0.13473 (0.10226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.79923 (13.72452) + | > current_lr: 0.00006 + | > step_time: 0.90470 (2.56851) + | > loader_time: 0.08490 (0.03115) + + + --> STEP: 102/234 -- GLOBAL_STEP: 55560 + | > loss: -0.20555 (-0.25248) + | > log_mle: -0.35434 (-0.35682) + | > loss_dur: 0.14879 (0.10434) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.52564 (14.03226) + | > current_lr: 0.00006 + | > step_time: 1.51320 (2.53373) + | > loader_time: 0.00340 (0.03139) + + + --> STEP: 107/234 -- GLOBAL_STEP: 55565 + | > loss: -0.23453 (-0.25198) + | > log_mle: -0.39224 (-0.35895) + | > loss_dur: 0.15772 (0.10697) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.04268 (14.67874) + | > current_lr: 0.00006 + | > step_time: 2.83570 (2.50731) + | > loader_time: 0.10110 (0.03173) + + + --> STEP: 112/234 -- GLOBAL_STEP: 55570 + | > loss: -0.23051 (-0.25121) + | > log_mle: -0.40631 (-0.36079) + | > loss_dur: 0.17580 (0.10957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.55618 (15.43232) + | > current_lr: 0.00006 + | > step_time: 1.30260 (2.51886) + | > loader_time: 0.00370 (0.03205) + + + --> STEP: 117/234 -- GLOBAL_STEP: 55575 + | > loss: -0.24530 (-0.25061) + | > log_mle: -0.40105 (-0.36248) + | > loss_dur: 0.15575 (0.11187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.72428 (16.15368) + | > current_lr: 0.00006 + | > step_time: 5.91270 (2.53025) + | > loader_time: 0.09600 (0.03226) + + + --> STEP: 122/234 -- GLOBAL_STEP: 55580 + | > loss: -0.22391 (-0.24992) + | > log_mle: -0.37122 (-0.36338) + | > loss_dur: 0.14732 (0.11346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.59740 (16.39208) + | > current_lr: 0.00006 + | > step_time: 1.78810 (2.51663) + | > loader_time: 0.00190 (0.03336) + + + --> STEP: 127/234 -- GLOBAL_STEP: 55585 + | > loss: -0.25093 (-0.24962) + | > log_mle: -0.43617 (-0.36551) + | > loss_dur: 0.18524 (0.11589) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.60762 (16.88511) + | > current_lr: 0.00006 + | > step_time: 1.90140 (2.52387) + | > loader_time: 0.00440 (0.03371) + + + --> STEP: 132/234 -- GLOBAL_STEP: 55590 + | > loss: -0.25682 (-0.25028) + | > log_mle: -0.41279 (-0.36826) + | > loss_dur: 0.15597 (0.11798) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.48623 (17.48689) + | > current_lr: 0.00006 + | > step_time: 4.38460 (2.52887) + | > loader_time: 0.00600 (0.03261) + + + --> STEP: 137/234 -- GLOBAL_STEP: 55595 + | > loss: -0.23737 (-0.25095) + | > log_mle: -0.42987 (-0.37125) + | > loss_dur: 0.19250 (0.12030) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.97941 (18.25461) + | > current_lr: 0.00006 + | > step_time: 1.42430 (2.48921) + | > loader_time: 0.00290 (0.03275) + + + --> STEP: 142/234 -- GLOBAL_STEP: 55600 + | > loss: -0.24929 (-0.25131) + | > log_mle: -0.44201 (-0.37383) + | > loss_dur: 0.19272 (0.12252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.71138 (18.85356) + | > current_lr: 0.00006 + | > step_time: 1.90120 (2.46968) + | > loader_time: 0.00430 (0.03250) + + + --> STEP: 147/234 -- GLOBAL_STEP: 55605 + | > loss: -0.25900 (-0.25314) + | > log_mle: -0.44500 (-0.37818) + | > loss_dur: 0.18601 (0.12504) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.97920 (19.76052) + | > current_lr: 0.00006 + | > step_time: 3.09930 (2.46312) + | > loader_time: 0.00280 (0.03151) + + + --> STEP: 152/234 -- GLOBAL_STEP: 55610 + | > loss: -0.31338 (-0.25499) + | > log_mle: -0.52974 (-0.38219) + | > loss_dur: 0.21636 (0.12720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.14703 (20.74050) + | > current_lr: 0.00006 + | > step_time: 4.20940 (2.50642) + | > loader_time: 0.09210 (0.03181) + + + --> STEP: 157/234 -- GLOBAL_STEP: 55615 + | > loss: -0.28822 (-0.25740) + | > log_mle: -0.47614 (-0.38678) + | > loss_dur: 0.18792 (0.12938) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.02820 (21.92283) + | > current_lr: 0.00006 + | > step_time: 2.90130 (2.55586) + | > loader_time: 0.00400 (0.03339) + + + --> STEP: 162/234 -- GLOBAL_STEP: 55620 + | > loss: -0.31225 (-0.25957) + | > log_mle: -0.50257 (-0.39109) + | > loss_dur: 0.19032 (0.13153) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.26015 (22.95576) + | > current_lr: 0.00006 + | > step_time: 6.48780 (2.62060) + | > loader_time: 0.20440 (0.03372) + + + --> STEP: 167/234 -- GLOBAL_STEP: 55625 + | > loss: -0.40898 (-0.26169) + | > log_mle: -0.59870 (-0.39514) + | > loss_dur: 0.18972 (0.13345) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.88244 (23.93776) + | > current_lr: 0.00006 + | > step_time: 0.88940 (2.66254) + | > loader_time: 0.01070 (0.03741) + + + --> STEP: 172/234 -- GLOBAL_STEP: 55630 + | > loss: -0.38994 (-0.26451) + | > log_mle: -0.60406 (-0.40033) + | > loss_dur: 0.21412 (0.13582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 63.79420 (24.96304) + | > current_lr: 0.00006 + | > step_time: 1.60210 (2.66182) + | > loader_time: 0.00330 (0.03700) + + + --> STEP: 177/234 -- GLOBAL_STEP: 55635 + | > loss: -0.34844 (-0.26718) + | > log_mle: -0.55324 (-0.40520) + | > loss_dur: 0.20479 (0.13802) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.48352 (26.08310) + | > current_lr: 0.00006 + | > step_time: 4.80670 (2.68028) + | > loader_time: 0.08700 (0.03754) + + + --> STEP: 182/234 -- GLOBAL_STEP: 55640 + | > loss: -0.35447 (-0.26951) + | > log_mle: -0.59487 (-0.40989) + | > loss_dur: 0.24040 (0.14038) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.00557 (27.21968) + | > current_lr: 0.00006 + | > step_time: 3.19520 (2.73027) + | > loader_time: 0.00360 (0.03920) + + + --> STEP: 187/234 -- GLOBAL_STEP: 55645 + | > loss: -0.37117 (-0.27146) + | > log_mle: -0.58882 (-0.41421) + | > loss_dur: 0.21765 (0.14275) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.92059 (28.60785) + | > current_lr: 0.00006 + | > step_time: 4.91490 (2.82593) + | > loader_time: 0.18930 (0.04071) + + + --> STEP: 192/234 -- GLOBAL_STEP: 55650 + | > loss: -0.41773 (-0.27413) + | > log_mle: -0.62098 (-0.41873) + | > loss_dur: 0.20325 (0.14460) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.34012 (29.58295) + | > current_lr: 0.00006 + | > step_time: 2.91170 (2.85088) + | > loader_time: 0.08040 (0.04151) + + + --> STEP: 197/234 -- GLOBAL_STEP: 55655 + | > loss: -0.39508 (-0.27680) + | > log_mle: -0.58912 (-0.42316) + | > loss_dur: 0.19404 (0.14636) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.29334 (30.59839) + | > current_lr: 0.00006 + | > step_time: 5.99380 (2.89622) + | > loader_time: 0.30070 (0.04210) + + + --> STEP: 202/234 -- GLOBAL_STEP: 55660 + | > loss: -0.47329 (-0.27939) + | > log_mle: -0.69020 (-0.42773) + | > loss_dur: 0.21692 (0.14834) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 90.22581 (31.52366) + | > current_lr: 0.00006 + | > step_time: 4.29280 (2.91613) + | > loader_time: 0.00350 (0.04241) + + + --> STEP: 207/234 -- GLOBAL_STEP: 55665 + | > loss: -0.45153 (-0.28218) + | > log_mle: -0.68095 (-0.43240) + | > loss_dur: 0.22942 (0.15022) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.63548 (32.58723) + | > current_lr: 0.00006 + | > step_time: 3.58660 (2.96527) + | > loader_time: 0.20020 (0.04282) + + + --> STEP: 212/234 -- GLOBAL_STEP: 55670 + | > loss: -0.42376 (-0.28543) + | > log_mle: -0.65144 (-0.43761) + | > loss_dur: 0.22768 (0.15219) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.72516 (34.17633) + | > current_lr: 0.00006 + | > step_time: 3.79900 (2.97119) + | > loader_time: 0.08680 (0.04358) + + + --> STEP: 217/234 -- GLOBAL_STEP: 55675 + | > loss: -0.43593 (-0.28869) + | > log_mle: -0.67899 (-0.44277) + | > loss_dur: 0.24306 (0.15408) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.45058 (35.55326) + | > current_lr: 0.00006 + | > step_time: 8.38990 (3.06400) + | > loader_time: 0.10390 (0.04615) + + + --> STEP: 222/234 -- GLOBAL_STEP: 55680 + | > loss: -0.43686 (-0.29192) + | > log_mle: -0.69634 (-0.44787) + | > loss_dur: 0.25948 (0.15595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.13420 (36.66198) + | > current_lr: 0.00006 + | > step_time: 1.19630 (3.05804) + | > loader_time: 0.00350 (0.04554) + + + --> STEP: 227/234 -- GLOBAL_STEP: 55685 + | > loss: -0.41294 (-0.29546) + | > log_mle: -0.66509 (-0.45334) + | > loss_dur: 0.25215 (0.15788) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.32943 (37.68611) + | > current_lr: 0.00006 + | > step_time: 0.25080 (2.99659) + | > loader_time: 0.00360 (0.04496) + + + --> STEP: 232/234 -- GLOBAL_STEP: 55690 + | > loss: -0.38595 (-0.29835) + | > log_mle: -0.87956 (-0.46030) + | > loss_dur: 0.49361 (0.16196) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 162.76428 (39.39432) + | > current_lr: 0.00006 + | > step_time: 0.34030 (2.93806) + | > loader_time: 0.08720 (0.04443) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.25395 (+0.02516) + | > avg_loss: -0.28951 (+0.02703) + | > avg_log_mle: -0.51922 (+0.01709) + | > avg_loss_dur: 0.22971 (+0.00993) + + + > EPOCH: 238/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 11:14:59)  + + --> STEP: 3/234 -- GLOBAL_STEP: 55695 + | > loss: -0.20089 (-0.25695) + | > log_mle: -0.35995 (-0.36812) + | > loss_dur: 0.15906 (0.11117) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.36611 (22.42414) + | > current_lr: 0.00006 + | > step_time: 11.30420 (7.07347) + | > loader_time: 0.19690 (0.09767) + + + --> STEP: 8/234 -- GLOBAL_STEP: 55700 + | > loss: -0.28907 (-0.27384) + | > log_mle: -0.38033 (-0.36893) + | > loss_dur: 0.09126 (0.09509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.55081 (18.79449) + | > current_lr: 0.00006 + | > step_time: 1.77970 (4.12651) + | > loader_time: 0.10480 (0.06159) + + + --> STEP: 13/234 -- GLOBAL_STEP: 55705 + | > loss: -0.31918 (-0.27910) + | > log_mle: -0.38701 (-0.37021) + | > loss_dur: 0.06783 (0.09111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.95993 (17.77228) + | > current_lr: 0.00006 + | > step_time: 1.15340 (2.96103) + | > loader_time: 0.00170 (0.04462) + + + --> STEP: 18/234 -- GLOBAL_STEP: 55710 + | > loss: -0.27650 (-0.28351) + | > log_mle: -0.35907 (-0.37016) + | > loss_dur: 0.08257 (0.08665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.40909 (16.06382) + | > current_lr: 0.00006 + | > step_time: 0.89750 (2.50484) + | > loader_time: 0.00210 (0.03297) + + + --> STEP: 23/234 -- GLOBAL_STEP: 55715 + | > loss: -0.30648 (-0.28575) + | > log_mle: -0.37490 (-0.36937) + | > loss_dur: 0.06842 (0.08363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.76898 (15.11454) + | > current_lr: 0.00006 + | > step_time: 3.00090 (2.45440) + | > loader_time: 0.20680 (0.03516) + + + --> STEP: 28/234 -- GLOBAL_STEP: 55720 + | > loss: -0.33132 (-0.28663) + | > log_mle: -0.38625 (-0.36814) + | > loss_dur: 0.05493 (0.08150) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.20968 (14.68542) + | > current_lr: 0.00006 + | > step_time: 2.89920 (2.82361) + | > loader_time: 0.00290 (0.04216) + + + --> STEP: 33/234 -- GLOBAL_STEP: 55725 + | > loss: -0.27603 (-0.28327) + | > log_mle: -0.35203 (-0.36519) + | > loss_dur: 0.07601 (0.08192) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.10579 (14.50754) + | > current_lr: 0.00006 + | > step_time: 7.28570 (3.18898) + | > loader_time: 0.00830 (0.05090) + + + --> STEP: 38/234 -- GLOBAL_STEP: 55730 + | > loss: -0.27080 (-0.27918) + | > log_mle: -0.36129 (-0.36240) + | > loss_dur: 0.09048 (0.08322) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78312 (14.27360) + | > current_lr: 0.00006 + | > step_time: 2.79580 (3.03530) + | > loader_time: 0.00360 (0.04683) + + + --> STEP: 43/234 -- GLOBAL_STEP: 55735 + | > loss: -0.21986 (-0.27480) + | > log_mle: -0.34251 (-0.35975) + | > loss_dur: 0.12265 (0.08496) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.92881 (14.04464) + | > current_lr: 0.00006 + | > step_time: 2.60340 (2.87489) + | > loader_time: 0.00330 (0.04167) + + + --> STEP: 48/234 -- GLOBAL_STEP: 55740 + | > loss: -0.27306 (-0.27229) + | > log_mle: -0.34812 (-0.35836) + | > loss_dur: 0.07506 (0.08607) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.32996 (13.65284) + | > current_lr: 0.00006 + | > step_time: 1.60360 (2.78812) + | > loader_time: 0.08790 (0.04255) + + + --> STEP: 53/234 -- GLOBAL_STEP: 55745 + | > loss: -0.24063 (-0.26996) + | > log_mle: -0.34688 (-0.35700) + | > loss_dur: 0.10625 (0.08703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.77252 (13.24196) + | > current_lr: 0.00006 + | > step_time: 1.70880 (2.70077) + | > loader_time: 0.00400 (0.04159) + + + --> STEP: 58/234 -- GLOBAL_STEP: 55750 + | > loss: -0.25680 (-0.26814) + | > log_mle: -0.34638 (-0.35580) + | > loss_dur: 0.08958 (0.08765) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.13202 (13.00828) + | > current_lr: 0.00006 + | > step_time: 2.20150 (2.60713) + | > loader_time: 0.08400 (0.03969) + + + --> STEP: 63/234 -- GLOBAL_STEP: 55755 + | > loss: -0.22330 (-0.26517) + | > log_mle: -0.33711 (-0.35547) + | > loss_dur: 0.11380 (0.09031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.73043 (13.09415) + | > current_lr: 0.00006 + | > step_time: 3.54770 (2.57589) + | > loader_time: 0.10200 (0.04118) + + + --> STEP: 68/234 -- GLOBAL_STEP: 55760 + | > loss: -0.22052 (-0.26345) + | > log_mle: -0.33772 (-0.35459) + | > loss_dur: 0.11720 (0.09114) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.57464 (12.81695) + | > current_lr: 0.00006 + | > step_time: 1.16400 (2.48098) + | > loader_time: 0.00240 (0.03835) + + + --> STEP: 73/234 -- GLOBAL_STEP: 55765 + | > loss: -0.20519 (-0.26044) + | > log_mle: -0.34603 (-0.35373) + | > loss_dur: 0.14084 (0.09329) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.29394 (12.85717) + | > current_lr: 0.00006 + | > step_time: 2.12700 (2.45035) + | > loader_time: 0.08580 (0.03708) + + + --> STEP: 78/234 -- GLOBAL_STEP: 55770 + | > loss: -0.22002 (-0.25796) + | > log_mle: -0.33067 (-0.35279) + | > loss_dur: 0.11065 (0.09483) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.03548 (12.88507) + | > current_lr: 0.00006 + | > step_time: 1.80610 (2.41107) + | > loader_time: 0.00550 (0.03490) + + + --> STEP: 83/234 -- GLOBAL_STEP: 55775 + | > loss: -0.19830 (-0.25569) + | > log_mle: -0.34946 (-0.35226) + | > loss_dur: 0.15116 (0.09657) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.61641 (12.91973) + | > current_lr: 0.00006 + | > step_time: 2.56520 (2.41331) + | > loader_time: 0.00340 (0.03412) + + + --> STEP: 88/234 -- GLOBAL_STEP: 55780 + | > loss: -0.22677 (-0.25410) + | > log_mle: -0.38132 (-0.35219) + | > loss_dur: 0.15454 (0.09809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.22977 (13.05604) + | > current_lr: 0.00006 + | > step_time: 2.49110 (2.38092) + | > loader_time: 0.00270 (0.03330) + + + --> STEP: 93/234 -- GLOBAL_STEP: 55785 + | > loss: -0.23701 (-0.25304) + | > log_mle: -0.39199 (-0.35333) + | > loss_dur: 0.15498 (0.10029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.66040 (13.42398) + | > current_lr: 0.00006 + | > step_time: 0.96250 (2.40691) + | > loader_time: 0.00190 (0.03438) + + + --> STEP: 98/234 -- GLOBAL_STEP: 55790 + | > loss: -0.21701 (-0.25265) + | > log_mle: -0.33111 (-0.35460) + | > loss_dur: 0.11409 (0.10195) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.25525 (13.69346) + | > current_lr: 0.00006 + | > step_time: 2.01560 (2.38004) + | > loader_time: 0.08500 (0.03564) + + + --> STEP: 103/234 -- GLOBAL_STEP: 55795 + | > loss: -0.25918 (-0.25206) + | > log_mle: -0.42670 (-0.35657) + | > loss_dur: 0.16752 (0.10451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.16805 (14.34911) + | > current_lr: 0.00006 + | > step_time: 1.81630 (2.37729) + | > loader_time: 0.00310 (0.03477) + + + --> STEP: 108/234 -- GLOBAL_STEP: 55800 + | > loss: -0.23565 (-0.25148) + | > log_mle: -0.36654 (-0.35813) + | > loss_dur: 0.13088 (0.10665) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.72940 (14.95864) + | > current_lr: 0.00006 + | > step_time: 1.89300 (2.34019) + | > loader_time: 0.00290 (0.03398) + + + --> STEP: 113/234 -- GLOBAL_STEP: 55805 + | > loss: -0.23916 (-0.25058) + | > log_mle: -0.40885 (-0.36018) + | > loss_dur: 0.16969 (0.10961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.67010 (16.03601) + | > current_lr: 0.00006 + | > step_time: 1.11200 (2.35363) + | > loader_time: 0.00260 (0.03500) + + + --> STEP: 118/234 -- GLOBAL_STEP: 55810 + | > loss: -0.21792 (-0.24966) + | > log_mle: -0.38282 (-0.36160) + | > loss_dur: 0.16489 (0.11194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.07848 (16.42739) + | > current_lr: 0.00006 + | > step_time: 2.00150 (2.33442) + | > loader_time: 0.00300 (0.03364) + + + --> STEP: 123/234 -- GLOBAL_STEP: 55815 + | > loss: -0.20086 (-0.24876) + | > log_mle: -0.35161 (-0.36225) + | > loss_dur: 0.15075 (0.11349) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.11691 (16.60510) + | > current_lr: 0.00006 + | > step_time: 3.30920 (2.39860) + | > loader_time: 0.00640 (0.03538) + + + --> STEP: 128/234 -- GLOBAL_STEP: 55820 + | > loss: -0.26452 (-0.24923) + | > log_mle: -0.41096 (-0.36488) + | > loss_dur: 0.14644 (0.11565) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.82077 (17.18248) + | > current_lr: 0.00006 + | > step_time: 1.16650 (2.38899) + | > loader_time: 0.00260 (0.03411) + + + --> STEP: 133/234 -- GLOBAL_STEP: 55825 + | > loss: -0.26801 (-0.25005) + | > log_mle: -0.43969 (-0.36788) + | > loss_dur: 0.17168 (0.11784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.51433 (17.86062) + | > current_lr: 0.00006 + | > step_time: 1.59280 (2.37049) + | > loader_time: 0.00340 (0.03426) + + + --> STEP: 138/234 -- GLOBAL_STEP: 55830 + | > loss: -0.21951 (-0.25051) + | > log_mle: -0.38785 (-0.37057) + | > loss_dur: 0.16834 (0.12006) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.11500 (18.57191) + | > current_lr: 0.00006 + | > step_time: 3.00150 (2.38371) + | > loader_time: 0.00320 (0.03319) + + + --> STEP: 143/234 -- GLOBAL_STEP: 55835 + | > loss: -0.31104 (-0.25151) + | > log_mle: -0.53849 (-0.37414) + | > loss_dur: 0.22745 (0.12263) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.93407 (19.42224) + | > current_lr: 0.00006 + | > step_time: 4.11290 (2.39254) + | > loader_time: 0.08730 (0.03388) + + + --> STEP: 148/234 -- GLOBAL_STEP: 55840 + | > loss: -0.29945 (-0.25312) + | > log_mle: -0.44971 (-0.37781) + | > loss_dur: 0.15026 (0.12469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.34399 (20.22616) + | > current_lr: 0.00006 + | > step_time: 1.60430 (2.38650) + | > loader_time: 0.00470 (0.03393) + + + --> STEP: 153/234 -- GLOBAL_STEP: 55845 + | > loss: -0.38684 (-0.25570) + | > log_mle: -0.58514 (-0.38270) + | > loss_dur: 0.19830 (0.12700) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.49697 (21.16974) + | > current_lr: 0.00006 + | > step_time: 3.00810 (2.40092) + | > loader_time: 0.09830 (0.03417) + + + --> STEP: 158/234 -- GLOBAL_STEP: 55850 + | > loss: -0.30847 (-0.25754) + | > log_mle: -0.51284 (-0.38684) + | > loss_dur: 0.20438 (0.12930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.65097 (22.31430) + | > current_lr: 0.00006 + | > step_time: 3.89910 (2.41800) + | > loader_time: 0.00440 (0.03426) + + + --> STEP: 163/234 -- GLOBAL_STEP: 55855 + | > loss: -0.29048 (-0.25973) + | > log_mle: -0.48948 (-0.39107) + | > loss_dur: 0.19900 (0.13133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.03412 (23.21542) + | > current_lr: 0.00006 + | > step_time: 3.40390 (2.52416) + | > loader_time: 0.08000 (0.03620) + + + --> STEP: 168/234 -- GLOBAL_STEP: 55860 + | > loss: -0.32114 (-0.26209) + | > log_mle: -0.54149 (-0.39548) + | > loss_dur: 0.22034 (0.13339) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.41061 (24.42266) + | > current_lr: 0.00006 + | > step_time: 0.70580 (2.51443) + | > loader_time: 0.00340 (0.03521) + + + --> STEP: 173/234 -- GLOBAL_STEP: 55865 + | > loss: -0.34246 (-0.26494) + | > log_mle: -0.55016 (-0.40062) + | > loss_dur: 0.20770 (0.13568) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.33890 (25.46039) + | > current_lr: 0.00006 + | > step_time: 5.40060 (2.53086) + | > loader_time: 0.28540 (0.03696) + + + --> STEP: 178/234 -- GLOBAL_STEP: 55870 + | > loss: -0.38597 (-0.26763) + | > log_mle: -0.61287 (-0.40567) + | > loss_dur: 0.22690 (0.13804) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.24092 (26.75255) + | > current_lr: 0.00006 + | > step_time: 5.78440 (2.62710) + | > loader_time: 0.00340 (0.04145) + + + --> STEP: 183/234 -- GLOBAL_STEP: 55875 + | > loss: -0.40159 (-0.27015) + | > log_mle: -0.61286 (-0.41044) + | > loss_dur: 0.21127 (0.14029) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.51956 (27.70503) + | > current_lr: 0.00006 + | > step_time: 1.08840 (2.61984) + | > loader_time: 0.00250 (0.04189) + + + --> STEP: 188/234 -- GLOBAL_STEP: 55880 + | > loss: -0.40749 (-0.27264) + | > log_mle: -0.61494 (-0.41509) + | > loss_dur: 0.20746 (0.14245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.65033 (28.84229) + | > current_lr: 0.00006 + | > step_time: 3.10350 (2.61846) + | > loader_time: 0.00650 (0.04232) + + + --> STEP: 193/234 -- GLOBAL_STEP: 55885 + | > loss: -0.41053 (-0.27542) + | > log_mle: -0.61670 (-0.41960) + | > loss_dur: 0.20618 (0.14418) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.13853 (29.86695) + | > current_lr: 0.00006 + | > step_time: 5.08790 (2.69535) + | > loader_time: 0.17550 (0.04316) + + + --> STEP: 198/234 -- GLOBAL_STEP: 55890 + | > loss: -0.40383 (-0.27804) + | > log_mle: -0.61611 (-0.42395) + | > loss_dur: 0.21228 (0.14591) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.98624 (30.78800) + | > current_lr: 0.00006 + | > step_time: 4.70030 (2.72573) + | > loader_time: 0.08340 (0.04446) + + + --> STEP: 203/234 -- GLOBAL_STEP: 55895 + | > loss: -0.29879 (-0.28009) + | > log_mle: -0.51358 (-0.42793) + | > loss_dur: 0.21479 (0.14784) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.75397 (32.15042) + | > current_lr: 0.00006 + | > step_time: 5.59960 (2.79015) + | > loader_time: 0.01420 (0.04537) + + + --> STEP: 208/234 -- GLOBAL_STEP: 55900 + | > loss: -0.38060 (-0.28270) + | > log_mle: -0.61374 (-0.43256) + | > loss_dur: 0.23314 (0.14986) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.52051 (33.06840) + | > current_lr: 0.00006 + | > step_time: 2.51890 (2.82014) + | > loader_time: 0.08830 (0.04532) + + + --> STEP: 213/234 -- GLOBAL_STEP: 55905 + | > loss: -0.41622 (-0.28584) + | > log_mle: -0.65773 (-0.43771) + | > loss_dur: 0.24151 (0.15187) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.44457 (34.29465) + | > current_lr: 0.00006 + | > step_time: 3.98540 (2.89214) + | > loader_time: 0.00440 (0.05450) + + + --> STEP: 218/234 -- GLOBAL_STEP: 55910 + | > loss: -0.39203 (-0.28874) + | > log_mle: -0.62761 (-0.44245) + | > loss_dur: 0.23558 (0.15371) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.22218 (35.37982) + | > current_lr: 0.00006 + | > step_time: 4.59650 (2.98581) + | > loader_time: 0.00720 (0.05476) + + + --> STEP: 223/234 -- GLOBAL_STEP: 55915 + | > loss: -0.44435 (-0.29199) + | > log_mle: -0.67360 (-0.44752) + | > loss_dur: 0.22925 (0.15553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.81321 (36.51062) + | > current_lr: 0.00006 + | > step_time: 0.24700 (2.95245) + | > loader_time: 0.00330 (0.05403) + + + --> STEP: 228/234 -- GLOBAL_STEP: 55920 + | > loss: -0.39873 (-0.29502) + | > log_mle: -0.66802 (-0.45263) + | > loss_dur: 0.26929 (0.15762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 80.04965 (38.05687) + | > current_lr: 0.00006 + | > step_time: 0.25070 (2.89311) + | > loader_time: 0.00380 (0.05292) + + + --> STEP: 233/234 -- GLOBAL_STEP: 55925 + | > loss: 0.05739 (-0.29573) + | > log_mle: -0.63561 (-0.45916) + | > loss_dur: 0.69299 (0.16344) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 98.32773 (39.39119) + | > current_lr: 0.00006 + | > step_time: 0.18580 (2.83673) + | > loader_time: 0.00270 (0.05192) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.82567 (+0.57172) + | > avg_loss: -0.31660 (-0.02708) + | > avg_log_mle: -0.53288 (-0.01366) + | > avg_loss_dur: 0.21628 (-0.01343) + + + > EPOCH: 239/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 11:27:16)  + + --> STEP: 4/234 -- GLOBAL_STEP: 55930 + | > loss: -0.26938 (-0.26058) + | > log_mle: -0.36439 (-0.36591) + | > loss_dur: 0.09501 (0.10533) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.68543 (19.89105) + | > current_lr: 0.00006 + | > step_time: 1.30870 (5.40120) + | > loader_time: 0.00230 (0.97811) + + + --> STEP: 9/234 -- GLOBAL_STEP: 55935 + | > loss: -0.27402 (-0.27331) + | > log_mle: -0.37672 (-0.36972) + | > loss_dur: 0.10270 (0.09641) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.76831 (17.60121) + | > current_lr: 0.00006 + | > step_time: 6.91040 (4.32688) + | > loader_time: 0.39520 (0.49781) + + + --> STEP: 14/234 -- GLOBAL_STEP: 55940 + | > loss: -0.29767 (-0.28078) + | > log_mle: -0.37384 (-0.37185) + | > loss_dur: 0.07617 (0.09107) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.76773 (16.43999) + | > current_lr: 0.00006 + | > step_time: 3.79410 (4.08865) + | > loader_time: 0.00110 (0.34095) + + + --> STEP: 19/234 -- GLOBAL_STEP: 55945 + | > loss: -0.28955 (-0.28423) + | > log_mle: -0.36533 (-0.37074) + | > loss_dur: 0.07577 (0.08651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.43284 (16.04449) + | > current_lr: 0.00006 + | > step_time: 3.69400 (4.31753) + | > loader_time: 0.19440 (0.26707) + + + --> STEP: 24/234 -- GLOBAL_STEP: 55950 + | > loss: -0.29221 (-0.28569) + | > log_mle: -0.36087 (-0.36968) + | > loss_dur: 0.06867 (0.08399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.01274 (15.50297) + | > current_lr: 0.00006 + | > step_time: 3.50260 (4.27732) + | > loader_time: 0.19090 (0.23203) + + + --> STEP: 29/234 -- GLOBAL_STEP: 55955 + | > loss: -0.25476 (-0.28525) + | > log_mle: -0.34389 (-0.36748) + | > loss_dur: 0.08913 (0.08223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.28654 (15.25297) + | > current_lr: 0.00006 + | > step_time: 1.20500 (4.48169) + | > loader_time: 0.17810 (0.20494) + + + --> STEP: 34/234 -- GLOBAL_STEP: 55960 + | > loss: -0.26868 (-0.28191) + | > log_mle: -0.35195 (-0.36489) + | > loss_dur: 0.08327 (0.08298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.46412 (15.03316) + | > current_lr: 0.00006 + | > step_time: 4.01020 (4.17861) + | > loader_time: 0.09670 (0.18598) + + + --> STEP: 39/234 -- GLOBAL_STEP: 55965 + | > loss: -0.24936 (-0.27790) + | > log_mle: -0.34751 (-0.36248) + | > loss_dur: 0.09815 (0.08458) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.37272 (14.80969) + | > current_lr: 0.00006 + | > step_time: 4.59210 (4.28642) + | > loader_time: 0.00320 (0.17476) + + + --> STEP: 44/234 -- GLOBAL_STEP: 55970 + | > loss: -0.26412 (-0.27438) + | > log_mle: -0.33843 (-0.36008) + | > loss_dur: 0.07430 (0.08570) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.76395 (14.49655) + | > current_lr: 0.00006 + | > step_time: 0.69700 (4.07803) + | > loader_time: 0.00250 (0.16348) + + + --> STEP: 49/234 -- GLOBAL_STEP: 55975 + | > loss: -0.26430 (-0.27243) + | > log_mle: -0.34965 (-0.35917) + | > loss_dur: 0.08535 (0.08674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.94438 (14.09387) + | > current_lr: 0.00006 + | > step_time: 2.10250 (3.84375) + | > loader_time: 0.00190 (0.14721) + + + --> STEP: 54/234 -- GLOBAL_STEP: 55980 + | > loss: -0.26493 (-0.27019) + | > log_mle: -0.35398 (-0.35787) + | > loss_dur: 0.08905 (0.08767) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.39351 (13.81919) + | > current_lr: 0.00006 + | > step_time: 2.10770 (3.67854) + | > loader_time: 0.00230 (0.13380) + + + --> STEP: 59/234 -- GLOBAL_STEP: 55985 + | > loss: -0.24378 (-0.26804) + | > log_mle: -0.35065 (-0.35673) + | > loss_dur: 0.10688 (0.08870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78231 (13.58784) + | > current_lr: 0.00006 + | > step_time: 1.99560 (3.48522) + | > loader_time: 0.00410 (0.12353) + + + --> STEP: 64/234 -- GLOBAL_STEP: 55990 + | > loss: -0.24287 (-0.26493) + | > log_mle: -0.33809 (-0.35603) + | > loss_dur: 0.09522 (0.09111) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.97362 (13.62136) + | > current_lr: 0.00006 + | > step_time: 2.20880 (3.40121) + | > loader_time: 0.07860 (0.11745) + + + --> STEP: 69/234 -- GLOBAL_STEP: 55995 + | > loss: -0.22409 (-0.26248) + | > log_mle: -0.33061 (-0.35491) + | > loss_dur: 0.10652 (0.09242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.10923 (13.37476) + | > current_lr: 0.00006 + | > step_time: 1.20510 (3.31713) + | > loader_time: 0.00340 (0.11167) + + + --> STEP: 74/234 -- GLOBAL_STEP: 56000 + | > loss: -0.21260 (-0.25947) + | > log_mle: -0.33089 (-0.35398) + | > loss_dur: 0.11828 (0.09451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.33099 (13.49035) + | > current_lr: 0.00006 + | > step_time: 2.70560 (3.24980) + | > loader_time: 0.00230 (0.10666) + + + --> STEP: 79/234 -- GLOBAL_STEP: 56005 + | > loss: -0.22714 (-0.25720) + | > log_mle: -0.34711 (-0.35330) + | > loss_dur: 0.11998 (0.09610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.22503 (13.44774) + | > current_lr: 0.00006 + | > step_time: 1.69640 (3.14466) + | > loader_time: 0.00250 (0.10010) + + + --> STEP: 84/234 -- GLOBAL_STEP: 56010 + | > loss: -0.24329 (-0.25538) + | > log_mle: -0.34447 (-0.35281) + | > loss_dur: 0.10118 (0.09743) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.66379 (13.34671) + | > current_lr: 0.00006 + | > step_time: 2.69680 (3.10047) + | > loader_time: 0.00300 (0.09637) + + + --> STEP: 89/234 -- GLOBAL_STEP: 56015 + | > loss: -0.24415 (-0.25387) + | > log_mle: -0.36543 (-0.35295) + | > loss_dur: 0.12128 (0.09908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.24782 (13.55981) + | > current_lr: 0.00006 + | > step_time: 2.78710 (3.03220) + | > loader_time: 0.00240 (0.09208) + + + --> STEP: 94/234 -- GLOBAL_STEP: 56020 + | > loss: -0.25573 (-0.25312) + | > log_mle: -0.39465 (-0.35433) + | > loss_dur: 0.13892 (0.10121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.06252 (13.94987) + | > current_lr: 0.00006 + | > step_time: 1.80600 (2.97793) + | > loader_time: 0.00320 (0.08742) + + + --> STEP: 99/234 -- GLOBAL_STEP: 56025 + | > loss: -0.25990 (-0.25267) + | > log_mle: -0.42381 (-0.35579) + | > loss_dur: 0.16391 (0.10312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.27920 (14.34389) + | > current_lr: 0.00006 + | > step_time: 3.30350 (2.93945) + | > loader_time: 0.00260 (0.08395) + + + --> STEP: 104/234 -- GLOBAL_STEP: 56030 + | > loss: -0.28856 (-0.25234) + | > log_mle: -0.43868 (-0.35773) + | > loss_dur: 0.15012 (0.10539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.28351 (14.96497) + | > current_lr: 0.00006 + | > step_time: 1.71210 (2.91837) + | > loader_time: 0.00300 (0.08085) + + + --> STEP: 109/234 -- GLOBAL_STEP: 56035 + | > loss: -0.21821 (-0.25099) + | > log_mle: -0.40911 (-0.35894) + | > loss_dur: 0.19091 (0.10795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.90296 (15.25276) + | > current_lr: 0.00006 + | > step_time: 2.39870 (2.90363) + | > loader_time: 0.00340 (0.07819) + + + --> STEP: 114/234 -- GLOBAL_STEP: 56040 + | > loss: -0.24153 (-0.25059) + | > log_mle: -0.39254 (-0.36110) + | > loss_dur: 0.15101 (0.11051) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.90825 (15.75202) + | > current_lr: 0.00006 + | > step_time: 4.80880 (2.90879) + | > loader_time: 0.00300 (0.07490) + + + --> STEP: 119/234 -- GLOBAL_STEP: 56045 + | > loss: -0.23702 (-0.24967) + | > log_mle: -0.38669 (-0.36254) + | > loss_dur: 0.14967 (0.11288) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.30580 (16.30723) + | > current_lr: 0.00006 + | > step_time: 1.35600 (2.89750) + | > loader_time: 0.00150 (0.07419) + + + --> STEP: 124/234 -- GLOBAL_STEP: 56050 + | > loss: -0.26260 (-0.24881) + | > log_mle: -0.41385 (-0.36323) + | > loss_dur: 0.15126 (0.11442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.64584 (16.65671) + | > current_lr: 0.00006 + | > step_time: 0.90080 (2.83412) + | > loader_time: 0.07840 (0.07381) + + + --> STEP: 129/234 -- GLOBAL_STEP: 56055 + | > loss: -0.23321 (-0.24901) + | > log_mle: -0.40950 (-0.36573) + | > loss_dur: 0.17629 (0.11672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.08084 (17.12459) + | > current_lr: 0.00006 + | > step_time: 2.03810 (2.84236) + | > loader_time: 0.08540 (0.07245) + + + --> STEP: 134/234 -- GLOBAL_STEP: 56060 + | > loss: -0.26808 (-0.24996) + | > log_mle: -0.46036 (-0.36899) + | > loss_dur: 0.19228 (0.11903) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.72519 (18.11160) + | > current_lr: 0.00006 + | > step_time: 1.19470 (2.79663) + | > loader_time: 0.00740 (0.07053) + + + --> STEP: 139/234 -- GLOBAL_STEP: 56065 + | > loss: -0.33651 (-0.25080) + | > log_mle: -0.52953 (-0.37214) + | > loss_dur: 0.19302 (0.12134) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.44556 (18.86759) + | > current_lr: 0.00006 + | > step_time: 4.91150 (2.81256) + | > loader_time: 0.09280 (0.06945) + + + --> STEP: 144/234 -- GLOBAL_STEP: 56070 + | > loss: -0.29270 (-0.25155) + | > log_mle: -0.49962 (-0.37547) + | > loss_dur: 0.20692 (0.12392) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.46163 (19.81305) + | > current_lr: 0.00006 + | > step_time: 2.22740 (2.79503) + | > loader_time: 0.00580 (0.06776) + + + --> STEP: 149/234 -- GLOBAL_STEP: 56075 + | > loss: -0.35494 (-0.25347) + | > log_mle: -0.54994 (-0.37935) + | > loss_dur: 0.19500 (0.12588) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.48623 (20.67305) + | > current_lr: 0.00006 + | > step_time: 2.29530 (2.79298) + | > loader_time: 0.00670 (0.06630) + + + --> STEP: 154/234 -- GLOBAL_STEP: 56080 + | > loss: -0.31764 (-0.25585) + | > log_mle: -0.50517 (-0.38388) + | > loss_dur: 0.18753 (0.12804) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.34939 (21.74790) + | > current_lr: 0.00006 + | > step_time: 1.61180 (2.76199) + | > loader_time: 0.09340 (0.06483) + + + --> STEP: 159/234 -- GLOBAL_STEP: 56085 + | > loss: -0.33343 (-0.25786) + | > log_mle: -0.53247 (-0.38817) + | > loss_dur: 0.19904 (0.13031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.03058 (22.76684) + | > current_lr: 0.00006 + | > step_time: 2.69980 (2.79975) + | > loader_time: 0.00310 (0.06525) + + + --> STEP: 164/234 -- GLOBAL_STEP: 56090 + | > loss: -0.30399 (-0.25961) + | > log_mle: -0.51674 (-0.39205) + | > loss_dur: 0.21275 (0.13244) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.55357 (23.55060) + | > current_lr: 0.00006 + | > step_time: 3.40240 (2.91074) + | > loader_time: 0.00470 (0.06344) + + + --> STEP: 169/234 -- GLOBAL_STEP: 56095 + | > loss: -0.31498 (-0.26192) + | > log_mle: -0.52000 (-0.39642) + | > loss_dur: 0.20502 (0.13450) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.81460 (24.51851) + | > current_lr: 0.00006 + | > step_time: 0.71540 (2.99171) + | > loader_time: 0.00370 (0.06400) + + + --> STEP: 174/234 -- GLOBAL_STEP: 56100 + | > loss: -0.39430 (-0.26525) + | > log_mle: -0.60781 (-0.40201) + | > loss_dur: 0.21351 (0.13676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.58664 (25.76234) + | > current_lr: 0.00006 + | > step_time: 3.19910 (2.99245) + | > loader_time: 0.08950 (0.06328) + + + --> STEP: 179/234 -- GLOBAL_STEP: 56105 + | > loss: -0.37016 (-0.26783) + | > log_mle: -0.60816 (-0.40702) + | > loss_dur: 0.23800 (0.13919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.86039 (26.83551) + | > current_lr: 0.00006 + | > step_time: 7.20100 (3.05354) + | > loader_time: 0.00710 (0.06168) + + + --> STEP: 184/234 -- GLOBAL_STEP: 56110 + | > loss: -0.36026 (-0.27028) + | > log_mle: -0.57897 (-0.41158) + | > loss_dur: 0.21870 (0.14130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.91173 (27.80448) + | > current_lr: 0.00006 + | > step_time: 3.10070 (3.15376) + | > loader_time: 0.08930 (0.06316) + + + --> STEP: 189/234 -- GLOBAL_STEP: 56115 + | > loss: -0.34651 (-0.27287) + | > log_mle: -0.56594 (-0.41633) + | > loss_dur: 0.21942 (0.14346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.26577 (28.88247) + | > current_lr: 0.00006 + | > step_time: 7.76370 (3.22634) + | > loader_time: 0.11250 (0.06415) + + + --> STEP: 194/234 -- GLOBAL_STEP: 56120 + | > loss: -0.38046 (-0.27563) + | > log_mle: -0.59181 (-0.42083) + | > loss_dur: 0.21136 (0.14520) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.41645 (30.02037) + | > current_lr: 0.00006 + | > step_time: 6.89080 (3.31574) + | > loader_time: 0.10540 (0.06468) + + + --> STEP: 199/234 -- GLOBAL_STEP: 56125 + | > loss: -0.37272 (-0.27787) + | > log_mle: -0.59653 (-0.42492) + | > loss_dur: 0.22382 (0.14706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.68883 (31.04688) + | > current_lr: 0.00006 + | > step_time: 5.38900 (3.33340) + | > loader_time: 0.02220 (0.06529) + + + --> STEP: 204/234 -- GLOBAL_STEP: 56130 + | > loss: -0.41360 (-0.28017) + | > log_mle: -0.65077 (-0.42916) + | > loss_dur: 0.23717 (0.14899) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.07790 (31.89172) + | > current_lr: 0.00006 + | > step_time: 3.99820 (3.34495) + | > loader_time: 0.10500 (0.06523) + + + --> STEP: 209/234 -- GLOBAL_STEP: 56135 + | > loss: -0.37597 (-0.28295) + | > log_mle: -0.59685 (-0.43378) + | > loss_dur: 0.22088 (0.15083) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.88482 (32.89998) + | > current_lr: 0.00006 + | > step_time: 4.19030 (3.35687) + | > loader_time: 0.00570 (0.06541) + + + --> STEP: 214/234 -- GLOBAL_STEP: 56140 + | > loss: -0.41954 (-0.28650) + | > log_mle: -0.63496 (-0.43926) + | > loss_dur: 0.21542 (0.15276) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.59853 (33.96396) + | > current_lr: 0.00006 + | > step_time: 14.69660 (3.42320) + | > loader_time: 0.10030 (0.06525) + + + --> STEP: 219/234 -- GLOBAL_STEP: 56145 + | > loss: -0.51135 (-0.28988) + | > log_mle: -0.73803 (-0.44453) + | > loss_dur: 0.22668 (0.15465) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.82954 (35.10836) + | > current_lr: 0.00006 + | > step_time: 4.00880 (3.46615) + | > loader_time: 0.09040 (0.06551) + + + --> STEP: 224/234 -- GLOBAL_STEP: 56150 + | > loss: -0.45233 (-0.29299) + | > log_mle: -0.68471 (-0.44949) + | > loss_dur: 0.23238 (0.15650) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.84095 (36.25282) + | > current_lr: 0.00006 + | > step_time: 0.82580 (3.45640) + | > loader_time: 0.07330 (0.06450) + + + --> STEP: 229/234 -- GLOBAL_STEP: 56155 + | > loss: -0.43017 (-0.29616) + | > log_mle: -0.72274 (-0.45498) + | > loss_dur: 0.29258 (0.15882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 96.58128 (37.53093) + | > current_lr: 0.00006 + | > step_time: 0.25860 (3.39319) + | > loader_time: 0.00370 (0.06351) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.19296 (-0.63271) + | > avg_loss: -0.32015 (-0.00355) + | > avg_log_mle: -0.53619 (-0.00331) + | > avg_loss_dur: 0.21604 (-0.00024) + + + > EPOCH: 240/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 11:41:43)  + + --> STEP: 0/234 -- GLOBAL_STEP: 56160 + | > loss: -0.29043 (-0.29043) + | > log_mle: -0.44956 (-0.44956) + | > loss_dur: 0.15913 (0.15913) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.29951 (27.29951) + | > current_lr: 0.00006 + | > step_time: 8.80810 (8.80812) + | > loader_time: 11.47410 (11.47415) + + + --> STEP: 5/234 -- GLOBAL_STEP: 56165 + | > loss: -0.25738 (-0.25857) + | > log_mle: -0.37036 (-0.36880) + | > loss_dur: 0.11299 (0.11023) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.75236 (21.22191) + | > current_lr: 0.00006 + | > step_time: 5.39950 (5.38528) + | > loader_time: 0.09710 (0.27920) + + + --> STEP: 10/234 -- GLOBAL_STEP: 56170 + | > loss: -0.26297 (-0.27312) + | > log_mle: -0.36326 (-0.37108) + | > loss_dur: 0.10030 (0.09797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.66431 (19.25206) + | > current_lr: 0.00006 + | > step_time: 4.59970 (4.69422) + | > loader_time: 0.09230 (0.22662) + + + --> STEP: 15/234 -- GLOBAL_STEP: 56175 + | > loss: -0.30689 (-0.28109) + | > log_mle: -0.37761 (-0.37285) + | > loss_dur: 0.07072 (0.09176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.79690 (18.25121) + | > current_lr: 0.00006 + | > step_time: 4.50850 (4.89723) + | > loader_time: 0.09700 (0.24455) + + + --> STEP: 20/234 -- GLOBAL_STEP: 56180 + | > loss: -0.29950 (-0.28500) + | > log_mle: -0.37100 (-0.37144) + | > loss_dur: 0.07150 (0.08644) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.94579 (16.62579) + | > current_lr: 0.00006 + | > step_time: 2.01350 (4.53902) + | > loader_time: 0.08630 (0.19695) + + + --> STEP: 25/234 -- GLOBAL_STEP: 56185 + | > loss: -0.27315 (-0.28495) + | > log_mle: -0.35158 (-0.36954) + | > loss_dur: 0.07843 (0.08459) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.68558 (15.66130) + | > current_lr: 0.00006 + | > step_time: 4.59730 (4.60766) + | > loader_time: 0.09900 (0.18093) + + + --> STEP: 30/234 -- GLOBAL_STEP: 56190 + | > loss: -0.26559 (-0.28434) + | > log_mle: -0.34914 (-0.36796) + | > loss_dur: 0.08355 (0.08362) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.50678 (14.80887) + | > current_lr: 0.00006 + | > step_time: 1.20570 (4.27969) + | > loader_time: 0.00270 (0.15755) + + + --> STEP: 35/234 -- GLOBAL_STEP: 56195 + | > loss: -0.24764 (-0.28114) + | > log_mle: -0.34896 (-0.36583) + | > loss_dur: 0.10132 (0.08469) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.73434 (14.35241) + | > current_lr: 0.00006 + | > step_time: 3.09230 (4.11968) + | > loader_time: 0.20120 (0.14598) + + + --> STEP: 40/234 -- GLOBAL_STEP: 56200 + | > loss: -0.23762 (-0.27784) + | > log_mle: -0.33958 (-0.36356) + | > loss_dur: 0.10196 (0.08572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.14422 (13.97864) + | > current_lr: 0.00006 + | > step_time: 1.00430 (3.99030) + | > loader_time: 0.00210 (0.13740) + + + --> STEP: 45/234 -- GLOBAL_STEP: 56205 + | > loss: -0.25134 (-0.27572) + | > log_mle: -0.36444 (-0.36211) + | > loss_dur: 0.11310 (0.08640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.04340 (13.69188) + | > current_lr: 0.00006 + | > step_time: 1.51080 (3.71550) + | > loader_time: 0.00320 (0.12241) + + + --> STEP: 50/234 -- GLOBAL_STEP: 56210 + | > loss: -0.26216 (-0.27436) + | > log_mle: -0.34319 (-0.36095) + | > loss_dur: 0.08103 (0.08659) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.95083 (13.26281) + | > current_lr: 0.00006 + | > step_time: 1.88480 (3.47573) + | > loader_time: 0.00270 (0.11210) + + + --> STEP: 55/234 -- GLOBAL_STEP: 56215 + | > loss: -0.26570 (-0.27252) + | > log_mle: -0.35484 (-0.35998) + | > loss_dur: 0.08915 (0.08746) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.37005 (12.98903) + | > current_lr: 0.00006 + | > step_time: 1.80730 (3.29025) + | > loader_time: 0.00180 (0.10254) + + + --> STEP: 60/234 -- GLOBAL_STEP: 56220 + | > loss: -0.23865 (-0.27036) + | > log_mle: -0.36397 (-0.35923) + | > loss_dur: 0.12532 (0.08887) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.93445 (13.09029) + | > current_lr: 0.00006 + | > step_time: 1.06710 (3.15549) + | > loader_time: 0.00220 (0.09421) + + + --> STEP: 65/234 -- GLOBAL_STEP: 56225 + | > loss: -0.24858 (-0.26746) + | > log_mle: -0.34741 (-0.35840) + | > loss_dur: 0.09882 (0.09094) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.91862 (13.20137) + | > current_lr: 0.00006 + | > step_time: 1.31460 (3.00664) + | > loader_time: 0.07690 (0.08958) + + + --> STEP: 70/234 -- GLOBAL_STEP: 56230 + | > loss: -0.21806 (-0.26505) + | > log_mle: -0.33460 (-0.35717) + | > loss_dur: 0.11654 (0.09212) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.04599 (13.08155) + | > current_lr: 0.00006 + | > step_time: 1.19350 (2.92600) + | > loader_time: 0.00800 (0.08345) + + + --> STEP: 75/234 -- GLOBAL_STEP: 56235 + | > loss: -0.21853 (-0.26220) + | > log_mle: -0.35173 (-0.35666) + | > loss_dur: 0.13319 (0.09445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.93959 (13.12404) + | > current_lr: 0.00006 + | > step_time: 3.79860 (2.88790) + | > loader_time: 0.00340 (0.07810) + + + --> STEP: 80/234 -- GLOBAL_STEP: 56240 + | > loss: -0.23830 (-0.26051) + | > log_mle: -0.33520 (-0.35585) + | > loss_dur: 0.09690 (0.09535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.57482 (12.99431) + | > current_lr: 0.00006 + | > step_time: 1.29900 (2.82523) + | > loader_time: 0.08920 (0.07663) + + + --> STEP: 85/234 -- GLOBAL_STEP: 56245 + | > loss: -0.22729 (-0.25849) + | > log_mle: -0.33968 (-0.35526) + | > loss_dur: 0.11240 (0.09677) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.36865 (12.99811) + | > current_lr: 0.00006 + | > step_time: 1.89950 (2.79226) + | > loader_time: 0.08770 (0.07431) + + + --> STEP: 90/234 -- GLOBAL_STEP: 56250 + | > loss: -0.22137 (-0.25693) + | > log_mle: -0.36429 (-0.35564) + | > loss_dur: 0.14292 (0.09870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.34976 (13.16467) + | > current_lr: 0.00006 + | > step_time: 3.51500 (2.78619) + | > loader_time: 0.08270 (0.07314) + + + --> STEP: 95/234 -- GLOBAL_STEP: 56255 + | > loss: -0.28040 (-0.25659) + | > log_mle: -0.44507 (-0.35784) + | > loss_dur: 0.16467 (0.10125) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.23953 (13.72483) + | > current_lr: 0.00006 + | > step_time: 1.98820 (2.77624) + | > loader_time: 0.00170 (0.06948) + + + --> STEP: 100/234 -- GLOBAL_STEP: 56260 + | > loss: -0.25385 (-0.25565) + | > log_mle: -0.37608 (-0.35857) + | > loss_dur: 0.12223 (0.10292) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.78315 (13.93616) + | > current_lr: 0.00006 + | > step_time: 3.20150 (2.77717) + | > loader_time: 0.00500 (0.06617) + + + --> STEP: 105/234 -- GLOBAL_STEP: 56265 + | > loss: -0.23259 (-0.25531) + | > log_mle: -0.35569 (-0.36043) + | > loss_dur: 0.12310 (0.10512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.14709 (14.38778) + | > current_lr: 0.00006 + | > step_time: 1.99320 (2.75745) + | > loader_time: 0.08870 (0.06479) + + + --> STEP: 110/234 -- GLOBAL_STEP: 56270 + | > loss: -0.22601 (-0.25408) + | > log_mle: -0.37649 (-0.36185) + | > loss_dur: 0.15047 (0.10777) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.65576 (14.96257) + | > current_lr: 0.00006 + | > step_time: 2.41000 (2.75104) + | > loader_time: 0.00330 (0.06298) + + + --> STEP: 115/234 -- GLOBAL_STEP: 56275 + | > loss: -0.22502 (-0.25370) + | > log_mle: -0.39474 (-0.36416) + | > loss_dur: 0.16972 (0.11046) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.02131 (15.69411) + | > current_lr: 0.00006 + | > step_time: 1.51070 (2.69493) + | > loader_time: 0.08240 (0.06107) + + + --> STEP: 120/234 -- GLOBAL_STEP: 56280 + | > loss: -0.27685 (-0.25333) + | > log_mle: -0.44876 (-0.36609) + | > loss_dur: 0.17191 (0.11276) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.14162 (16.10641) + | > current_lr: 0.00006 + | > step_time: 1.68260 (2.68005) + | > loader_time: 0.00220 (0.05945) + + + --> STEP: 125/234 -- GLOBAL_STEP: 56285 + | > loss: -0.26536 (-0.25244) + | > log_mle: -0.43362 (-0.36685) + | > loss_dur: 0.16825 (0.11441) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.65198 (16.35555) + | > current_lr: 0.00006 + | > step_time: 1.50330 (2.65847) + | > loader_time: 0.10290 (0.05940) + + + --> STEP: 130/234 -- GLOBAL_STEP: 56290 + | > loss: -0.26982 (-0.25276) + | > log_mle: -0.44828 (-0.36952) + | > loss_dur: 0.17846 (0.11676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.96108 (17.04983) + | > current_lr: 0.00006 + | > step_time: 1.70320 (2.62620) + | > loader_time: 0.00310 (0.05725) + + + --> STEP: 135/234 -- GLOBAL_STEP: 56295 + | > loss: -0.22730 (-0.25342) + | > log_mle: -0.37430 (-0.37226) + | > loss_dur: 0.14700 (0.11883) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.49193 (17.82254) + | > current_lr: 0.00006 + | > step_time: 2.59420 (2.61493) + | > loader_time: 0.08770 (0.05587) + + + --> STEP: 140/234 -- GLOBAL_STEP: 56300 + | > loss: -0.22971 (-0.25403) + | > log_mle: -0.40345 (-0.37527) + | > loss_dur: 0.17374 (0.12125) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.14468 (19.06262) + | > current_lr: 0.00006 + | > step_time: 2.91000 (2.59248) + | > loader_time: 0.08930 (0.05520) + + + --> STEP: 145/234 -- GLOBAL_STEP: 56305 + | > loss: -0.31446 (-0.25540) + | > log_mle: -0.50535 (-0.37917) + | > loss_dur: 0.19088 (0.12377) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.37786 (19.92993) + | > current_lr: 0.00006 + | > step_time: 0.82870 (2.62201) + | > loader_time: 0.07670 (0.05461) + + + --> STEP: 150/234 -- GLOBAL_STEP: 56310 + | > loss: -0.28549 (-0.25692) + | > log_mle: -0.49289 (-0.38285) + | > loss_dur: 0.20740 (0.12593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.93857 (20.70950) + | > current_lr: 0.00006 + | > step_time: 1.99800 (2.60068) + | > loader_time: 0.00340 (0.05411) + + + --> STEP: 155/234 -- GLOBAL_STEP: 56315 + | > loss: -0.35699 (-0.25948) + | > log_mle: -0.55958 (-0.38766) + | > loss_dur: 0.20259 (0.12819) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.17167 (21.75696) + | > current_lr: 0.00006 + | > step_time: 2.49130 (2.67227) + | > loader_time: 0.00160 (0.05367) + + + --> STEP: 160/234 -- GLOBAL_STEP: 56320 + | > loss: -0.34541 (-0.26131) + | > log_mle: -0.55675 (-0.39183) + | > loss_dur: 0.21134 (0.13052) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.57201 (22.64498) + | > current_lr: 0.00006 + | > step_time: 15.29800 (2.78007) + | > loader_time: 0.08860 (0.05426) + + + --> STEP: 165/234 -- GLOBAL_STEP: 56325 + | > loss: -0.36323 (-0.26333) + | > log_mle: -0.55799 (-0.39581) + | > loss_dur: 0.19476 (0.13248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.62789 (23.58837) + | > current_lr: 0.00006 + | > step_time: 2.97850 (2.81749) + | > loader_time: 0.09610 (0.05506) + + + --> STEP: 170/234 -- GLOBAL_STEP: 56330 + | > loss: -0.33893 (-0.26545) + | > log_mle: -0.55961 (-0.39998) + | > loss_dur: 0.22068 (0.13453) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.96616 (25.14499) + | > current_lr: 0.00006 + | > step_time: 3.23480 (2.79239) + | > loader_time: 0.08490 (0.05511) + + + --> STEP: 175/234 -- GLOBAL_STEP: 56335 + | > loss: -0.32259 (-0.26740) + | > log_mle: -0.55451 (-0.40435) + | > loss_dur: 0.23192 (0.13695) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.58221 (26.04615) + | > current_lr: 0.00006 + | > step_time: 5.00350 (2.82076) + | > loader_time: 0.00420 (0.05579) + + + --> STEP: 180/234 -- GLOBAL_STEP: 56340 + | > loss: -0.31665 (-0.26952) + | > log_mle: -0.52837 (-0.40866) + | > loss_dur: 0.21172 (0.13914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.06714 (27.41595) + | > current_lr: 0.00006 + | > step_time: 9.89490 (2.92561) + | > loader_time: 0.11140 (0.05551) + + + --> STEP: 185/234 -- GLOBAL_STEP: 56345 + | > loss: -0.37159 (-0.27117) + | > log_mle: -0.59282 (-0.41256) + | > loss_dur: 0.22123 (0.14140) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.25863 (28.09190) + | > current_lr: 0.00006 + | > step_time: 6.10880 (3.00175) + | > loader_time: 0.19000 (0.05675) + + + --> STEP: 190/234 -- GLOBAL_STEP: 56350 + | > loss: -0.36269 (-0.27307) + | > log_mle: -0.56757 (-0.41653) + | > loss_dur: 0.20489 (0.14346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.71552 (28.85522) + | > current_lr: 0.00006 + | > step_time: 2.11010 (3.00121) + | > loader_time: 0.08590 (0.05678) + + + --> STEP: 195/234 -- GLOBAL_STEP: 56355 + | > loss: -0.36994 (-0.27573) + | > log_mle: -0.59401 (-0.42095) + | > loss_dur: 0.22407 (0.14522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.72501 (29.96628) + | > current_lr: 0.00006 + | > step_time: 5.30220 (3.08475) + | > loader_time: 0.00850 (0.05636) + + + --> STEP: 200/234 -- GLOBAL_STEP: 56360 + | > loss: -0.34383 (-0.27779) + | > log_mle: -0.59417 (-0.42499) + | > loss_dur: 0.25035 (0.14720) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.41265 (31.16162) + | > current_lr: 0.00006 + | > step_time: 5.09520 (3.16160) + | > loader_time: 0.09420 (0.05699) + + + --> STEP: 205/234 -- GLOBAL_STEP: 56365 + | > loss: -0.36711 (-0.27999) + | > log_mle: -0.58648 (-0.42903) + | > loss_dur: 0.21936 (0.14904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.17234 (31.83694) + | > current_lr: 0.00006 + | > step_time: 1.99440 (3.19425) + | > loader_time: 0.00340 (0.05655) + + + --> STEP: 210/234 -- GLOBAL_STEP: 56370 + | > loss: -0.42934 (-0.28289) + | > log_mle: -0.67129 (-0.43386) + | > loss_dur: 0.24196 (0.15098) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.49532 (32.87624) + | > current_lr: 0.00006 + | > step_time: 4.18430 (3.24854) + | > loader_time: 0.00400 (0.05624) + + + --> STEP: 215/234 -- GLOBAL_STEP: 56375 + | > loss: -0.39385 (-0.28620) + | > log_mle: -0.62839 (-0.43903) + | > loss_dur: 0.23454 (0.15283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.00029 (33.88196) + | > current_lr: 0.00006 + | > step_time: 5.70160 (3.36745) + | > loader_time: 0.09970 (0.06004) + + + --> STEP: 220/234 -- GLOBAL_STEP: 56380 + | > loss: -0.42992 (-0.28979) + | > log_mle: -0.67126 (-0.44460) + | > loss_dur: 0.24133 (0.15482) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 100.78285 (35.23588) + | > current_lr: 0.00006 + | > step_time: 2.19600 (3.33108) + | > loader_time: 0.00260 (0.05989) + + + --> STEP: 225/234 -- GLOBAL_STEP: 56385 + | > loss: -0.50136 (-0.29324) + | > log_mle: -0.74931 (-0.44998) + | > loss_dur: 0.24795 (0.15674) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.91612 (36.52135) + | > current_lr: 0.00006 + | > step_time: 1.90620 (3.29590) + | > loader_time: 0.00540 (0.05938) + + + --> STEP: 230/234 -- GLOBAL_STEP: 56390 + | > loss: -0.49561 (-0.29660) + | > log_mle: -0.80568 (-0.45592) + | > loss_dur: 0.31008 (0.15932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.41392 (37.75029) + | > current_lr: 0.00006 + | > step_time: 0.25530 (3.24201) + | > loader_time: 0.00410 (0.05819) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.29880 (+0.10584) + | > avg_loss: -0.31757 (+0.00258) + | > avg_log_mle: -0.53460 (+0.00159) + | > avg_loss_dur: 0.21703 (+0.00099) + + + > EPOCH: 241/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 11:55:35)  + + --> STEP: 1/234 -- GLOBAL_STEP: 56395 + | > loss: -0.27417 (-0.27417) + | > log_mle: -0.36585 (-0.36585) + | > loss_dur: 0.09168 (0.09168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.87049 (24.87049) + | > current_lr: 0.00006 + | > step_time: 13.09670 (13.09670) + | > loader_time: 8.79510 (8.79510) + + + --> STEP: 6/234 -- GLOBAL_STEP: 56400 + | > loss: -0.30719 (-0.27209) + | > log_mle: -0.36882 (-0.37076) + | > loss_dur: 0.06163 (0.09867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.14784 (19.21878) + | > current_lr: 0.00006 + | > step_time: 4.29750 (6.43097) + | > loader_time: 0.10780 (1.55076) + + + --> STEP: 11/234 -- GLOBAL_STEP: 56405 + | > loss: -0.30746 (-0.27951) + | > log_mle: -0.38205 (-0.37439) + | > loss_dur: 0.07459 (0.09488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.39281 (18.52262) + | > current_lr: 0.00006 + | > step_time: 1.49060 (4.22466) + | > loader_time: 0.00100 (0.85371) + + + --> STEP: 16/234 -- GLOBAL_STEP: 56410 + | > loss: -0.30347 (-0.28526) + | > log_mle: -0.37503 (-0.37486) + | > loss_dur: 0.07156 (0.08960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.85108 (17.94344) + | > current_lr: 0.00006 + | > step_time: 2.31610 (4.33681) + | > loader_time: 0.00170 (0.59905) + + + --> STEP: 21/234 -- GLOBAL_STEP: 56415 + | > loss: -0.28369 (-0.28590) + | > log_mle: -0.35168 (-0.37218) + | > loss_dur: 0.06799 (0.08628) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.37734 (16.38555) + | > current_lr: 0.00006 + | > step_time: 3.29340 (4.14248) + | > loader_time: 0.00200 (0.48427) + + + --> STEP: 26/234 -- GLOBAL_STEP: 56420 + | > loss: -0.28101 (-0.28647) + | > log_mle: -0.36178 (-0.37091) + | > loss_dur: 0.08076 (0.08444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.00874 (15.36778) + | > current_lr: 0.00006 + | > step_time: 3.30800 (3.89214) + | > loader_time: 0.00650 (0.39796) + + + --> STEP: 31/234 -- GLOBAL_STEP: 56425 + | > loss: -0.23514 (-0.28501) + | > log_mle: -0.34671 (-0.36891) + | > loss_dur: 0.11157 (0.08390) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.87904 (14.71849) + | > current_lr: 0.00006 + | > step_time: 4.90360 (3.88757) + | > loader_time: 0.00860 (0.33754) + + + --> STEP: 36/234 -- GLOBAL_STEP: 56430 + | > loss: -0.23518 (-0.28229) + | > log_mle: -0.34359 (-0.36664) + | > loss_dur: 0.10841 (0.08435) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.16438 (14.32848) + | > current_lr: 0.00006 + | > step_time: 2.78870 (3.82827) + | > loader_time: 0.09780 (0.30117) + + + --> STEP: 41/234 -- GLOBAL_STEP: 56435 + | > loss: -0.27915 (-0.27973) + | > log_mle: -0.35767 (-0.36485) + | > loss_dur: 0.07853 (0.08512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.18085 (13.76597) + | > current_lr: 0.00006 + | > step_time: 1.06060 (3.48403) + | > loader_time: 0.00370 (0.26553) + + + --> STEP: 46/234 -- GLOBAL_STEP: 56440 + | > loss: -0.24811 (-0.27632) + | > log_mle: -0.34964 (-0.36308) + | > loss_dur: 0.10154 (0.08676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.52972 (13.81270) + | > current_lr: 0.00006 + | > step_time: 1.62610 (3.22542) + | > loader_time: 0.00230 (0.23687) + + + --> STEP: 51/234 -- GLOBAL_STEP: 56445 + | > loss: -0.25525 (-0.27491) + | > log_mle: -0.34241 (-0.36169) + | > loss_dur: 0.08717 (0.08678) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.39485 (13.39791) + | > current_lr: 0.00006 + | > step_time: 0.97560 (3.02738) + | > loader_time: 0.00160 (0.21561) + + + --> STEP: 56/234 -- GLOBAL_STEP: 56450 + | > loss: -0.24922 (-0.27318) + | > log_mle: -0.35141 (-0.36085) + | > loss_dur: 0.10219 (0.08767) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.78255 (13.15268) + | > current_lr: 0.00006 + | > step_time: 1.30230 (2.89467) + | > loader_time: 0.00300 (0.19794) + + + --> STEP: 61/234 -- GLOBAL_STEP: 56455 + | > loss: -0.24336 (-0.27057) + | > log_mle: -0.34680 (-0.35987) + | > loss_dur: 0.10345 (0.08930) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.18837 (12.95093) + | > current_lr: 0.00006 + | > step_time: 1.32480 (2.78341) + | > loader_time: 0.00200 (0.18350) + + + --> STEP: 66/234 -- GLOBAL_STEP: 56460 + | > loss: -0.25041 (-0.26817) + | > log_mle: -0.34032 (-0.35890) + | > loss_dur: 0.08992 (0.09073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.65493 (12.86302) + | > current_lr: 0.00006 + | > step_time: 1.29850 (2.68820) + | > loader_time: 0.00330 (0.17411) + + + --> STEP: 71/234 -- GLOBAL_STEP: 56465 + | > loss: -0.22198 (-0.26533) + | > log_mle: -0.36637 (-0.35791) + | > loss_dur: 0.14440 (0.09258) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.60410 (12.99473) + | > current_lr: 0.00006 + | > step_time: 1.80080 (2.61599) + | > loader_time: 0.00260 (0.16206) + + + --> STEP: 76/234 -- GLOBAL_STEP: 56470 + | > loss: -0.23320 (-0.26259) + | > log_mle: -0.34978 (-0.35703) + | > loss_dur: 0.11658 (0.09444) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.54238 (13.02700) + | > current_lr: 0.00006 + | > step_time: 1.49130 (2.61216) + | > loader_time: 0.00230 (0.15381) + + + --> STEP: 81/234 -- GLOBAL_STEP: 56475 + | > loss: -0.23466 (-0.26080) + | > log_mle: -0.35515 (-0.35622) + | > loss_dur: 0.12049 (0.09543) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.05965 (12.96832) + | > current_lr: 0.00006 + | > step_time: 2.38060 (2.55789) + | > loader_time: 0.07600 (0.14535) + + + --> STEP: 86/234 -- GLOBAL_STEP: 56480 + | > loss: -0.23534 (-0.25909) + | > log_mle: -0.35399 (-0.35565) + | > loss_dur: 0.11865 (0.09656) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.92602 (12.99708) + | > current_lr: 0.00006 + | > step_time: 1.20030 (2.49773) + | > loader_time: 0.08480 (0.13805) + + + --> STEP: 91/234 -- GLOBAL_STEP: 56485 + | > loss: -0.22303 (-0.25765) + | > log_mle: -0.36383 (-0.35619) + | > loss_dur: 0.14080 (0.09855) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.04788 (13.19755) + | > current_lr: 0.00006 + | > step_time: 2.01370 (2.50363) + | > loader_time: 0.18470 (0.13563) + + + --> STEP: 96/234 -- GLOBAL_STEP: 56490 + | > loss: -0.22833 (-0.25730) + | > log_mle: -0.35126 (-0.35819) + | > loss_dur: 0.12293 (0.10089) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.37997 (13.67414) + | > current_lr: 0.00006 + | > step_time: 1.40110 (2.47572) + | > loader_time: 0.00230 (0.12973) + + + --> STEP: 101/234 -- GLOBAL_STEP: 56495 + | > loss: -0.24944 (-0.25667) + | > log_mle: -0.40326 (-0.35938) + | > loss_dur: 0.15383 (0.10272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.34456 (13.88752) + | > current_lr: 0.00006 + | > step_time: 1.69920 (2.45623) + | > loader_time: 0.00430 (0.12692) + + + --> STEP: 106/234 -- GLOBAL_STEP: 56500 + | > loss: -0.21017 (-0.25573) + | > log_mle: -0.39050 (-0.36098) + | > loss_dur: 0.18033 (0.10525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.74124 (14.41087) + | > current_lr: 0.00006 + | > step_time: 1.89780 (2.45262) + | > loader_time: 0.00310 (0.12373) + + + --> STEP: 111/234 -- GLOBAL_STEP: 56505 + | > loss: -0.26134 (-0.25478) + | > log_mle: -0.44983 (-0.36270) + | > loss_dur: 0.18849 (0.10792) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.73298 (15.04163) + | > current_lr: 0.00006 + | > step_time: 1.09790 (2.41500) + | > loader_time: 0.08570 (0.11983) + + + --> STEP: 116/234 -- GLOBAL_STEP: 56510 + | > loss: -0.23618 (-0.25416) + | > log_mle: -0.41799 (-0.36461) + | > loss_dur: 0.18181 (0.11045) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.01768 (15.56511) + | > current_lr: 0.00006 + | > step_time: 1.79370 (2.42207) + | > loader_time: 0.00230 (0.11556) + + + --> STEP: 121/234 -- GLOBAL_STEP: 56515 + | > loss: -0.20043 (-0.25347) + | > log_mle: -0.33245 (-0.36582) + | > loss_dur: 0.13202 (0.11235) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.63110 (15.92245) + | > current_lr: 0.00006 + | > step_time: 2.41400 (2.40570) + | > loader_time: 0.00350 (0.11094) + + + --> STEP: 126/234 -- GLOBAL_STEP: 56520 + | > loss: -0.27165 (-0.25312) + | > log_mle: -0.46265 (-0.36757) + | > loss_dur: 0.19100 (0.11445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.25006 (16.36647) + | > current_lr: 0.00006 + | > step_time: 2.52330 (2.38422) + | > loader_time: 0.18380 (0.10939) + + + --> STEP: 131/234 -- GLOBAL_STEP: 56525 + | > loss: -0.32887 (-0.25386) + | > log_mle: -0.51566 (-0.37054) + | > loss_dur: 0.18679 (0.11669) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.94534 (16.99174) + | > current_lr: 0.00006 + | > step_time: 3.49090 (2.37404) + | > loader_time: 0.00840 (0.10599) + + + --> STEP: 136/234 -- GLOBAL_STEP: 56530 + | > loss: -0.35010 (-0.25462) + | > log_mle: -0.55458 (-0.37335) + | > loss_dur: 0.20448 (0.11873) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.77602 (18.13709) + | > current_lr: 0.00006 + | > step_time: 2.09670 (2.37949) + | > loader_time: 0.00240 (0.10420) + + + --> STEP: 141/234 -- GLOBAL_STEP: 56535 + | > loss: -0.28640 (-0.25484) + | > log_mle: -0.45921 (-0.37574) + | > loss_dur: 0.17281 (0.12090) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.78583 (18.82194) + | > current_lr: 0.00006 + | > step_time: 2.50500 (2.41455) + | > loader_time: 0.00450 (0.10187) + + + --> STEP: 146/234 -- GLOBAL_STEP: 56540 + | > loss: -0.31906 (-0.25661) + | > log_mle: -0.50875 (-0.38007) + | > loss_dur: 0.18969 (0.12346) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.90821 (19.63742) + | > current_lr: 0.00006 + | > step_time: 4.09250 (2.44104) + | > loader_time: 0.00350 (0.10037) + + + --> STEP: 151/234 -- GLOBAL_STEP: 56545 + | > loss: -0.29642 (-0.25824) + | > log_mle: -0.47628 (-0.38358) + | > loss_dur: 0.17985 (0.12534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.08439 (20.45685) + | > current_lr: 0.00006 + | > step_time: 3.71240 (2.47076) + | > loader_time: 0.08380 (0.09835) + + + --> STEP: 156/234 -- GLOBAL_STEP: 56550 + | > loss: -0.32406 (-0.26097) + | > log_mle: -0.51696 (-0.38862) + | > loss_dur: 0.19290 (0.12766) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.80141 (21.78366) + | > current_lr: 0.00006 + | > step_time: 3.61150 (2.50890) + | > loader_time: 0.10160 (0.09727) + + + --> STEP: 161/234 -- GLOBAL_STEP: 56555 + | > loss: -0.36503 (-0.26305) + | > log_mle: -0.55238 (-0.39286) + | > loss_dur: 0.18736 (0.12982) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.85432 (22.75197) + | > current_lr: 0.00006 + | > step_time: 3.01310 (2.53295) + | > loader_time: 0.08800 (0.09603) + + + --> STEP: 166/234 -- GLOBAL_STEP: 56560 + | > loss: -0.30704 (-0.26460) + | > log_mle: -0.48366 (-0.39637) + | > loss_dur: 0.17662 (0.13177) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.30267 (23.67294) + | > current_lr: 0.00006 + | > step_time: 4.29350 (2.63676) + | > loader_time: 0.00230 (0.09500) + + + --> STEP: 171/234 -- GLOBAL_STEP: 56565 + | > loss: -0.38742 (-0.26758) + | > log_mle: -0.59134 (-0.40156) + | > loss_dur: 0.20392 (0.13398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.62282 (24.73533) + | > current_lr: 0.00006 + | > step_time: 0.67940 (2.66781) + | > loader_time: 0.00340 (0.09436) + + + --> STEP: 176/234 -- GLOBAL_STEP: 56570 + | > loss: -0.36754 (-0.27039) + | > log_mle: -0.57204 (-0.40669) + | > loss_dur: 0.20449 (0.13630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.09641 (25.80081) + | > current_lr: 0.00006 + | > step_time: 2.90770 (2.66074) + | > loader_time: 0.00440 (0.09216) + + + --> STEP: 181/234 -- GLOBAL_STEP: 56575 + | > loss: -0.28943 (-0.27257) + | > log_mle: -0.49686 (-0.41121) + | > loss_dur: 0.20742 (0.13864) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.42566 (26.81177) + | > current_lr: 0.00006 + | > step_time: 2.48540 (2.64005) + | > loader_time: 0.00350 (0.09020) + + + --> STEP: 186/234 -- GLOBAL_STEP: 56580 + | > loss: -0.31653 (-0.27489) + | > log_mle: -0.54328 (-0.41588) + | > loss_dur: 0.22674 (0.14100) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.15020 (27.79614) + | > current_lr: 0.00006 + | > step_time: 3.19640 (2.64230) + | > loader_time: 0.00440 (0.08829) + + + --> STEP: 191/234 -- GLOBAL_STEP: 56585 + | > loss: -0.36893 (-0.27745) + | > log_mle: -0.57022 (-0.42042) + | > loss_dur: 0.20130 (0.14298) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.74929 (28.82314) + | > current_lr: 0.00006 + | > step_time: 5.60220 (2.72607) + | > loader_time: 0.09410 (0.08792) + + + --> STEP: 196/234 -- GLOBAL_STEP: 56590 + | > loss: -0.33192 (-0.28009) + | > log_mle: -0.55342 (-0.42496) + | > loss_dur: 0.22150 (0.14487) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 77.15979 (30.04805) + | > current_lr: 0.00006 + | > step_time: 4.99300 (2.79475) + | > loader_time: 0.39070 (0.08869) + + + --> STEP: 201/234 -- GLOBAL_STEP: 56595 + | > loss: -0.28392 (-0.28224) + | > log_mle: -0.51267 (-0.42903) + | > loss_dur: 0.22875 (0.14679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.85734 (30.96731) + | > current_lr: 0.00006 + | > step_time: 4.10570 (2.78339) + | > loader_time: 0.18650 (0.08799) + + + --> STEP: 206/234 -- GLOBAL_STEP: 56600 + | > loss: -0.39594 (-0.28488) + | > log_mle: -0.61842 (-0.43356) + | > loss_dur: 0.22249 (0.14868) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 93.48489 (32.17132) + | > current_lr: 0.00006 + | > step_time: 3.30250 (2.82214) + | > loader_time: 0.00350 (0.08686) + + + --> STEP: 211/234 -- GLOBAL_STEP: 56605 + | > loss: -0.44244 (-0.28778) + | > log_mle: -0.68855 (-0.43851) + | > loss_dur: 0.24611 (0.15073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.93865 (33.39649) + | > current_lr: 0.00006 + | > step_time: 4.89520 (2.87137) + | > loader_time: 0.10950 (0.08760) + + + --> STEP: 216/234 -- GLOBAL_STEP: 56610 + | > loss: -0.43597 (-0.29089) + | > log_mle: -0.68171 (-0.44348) + | > loss_dur: 0.24574 (0.15259) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 95.76629 (34.42496) + | > current_lr: 0.00006 + | > step_time: 5.29630 (2.92438) + | > loader_time: 0.09770 (0.08746) + + + --> STEP: 221/234 -- GLOBAL_STEP: 56615 + | > loss: -0.37837 (-0.29391) + | > log_mle: -0.59443 (-0.44831) + | > loss_dur: 0.21605 (0.15440) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.17210 (35.70178) + | > current_lr: 0.00006 + | > step_time: 1.38220 (2.96545) + | > loader_time: 0.00240 (0.08636) + + + --> STEP: 226/234 -- GLOBAL_STEP: 56620 + | > loss: -0.44768 (-0.29748) + | > log_mle: -0.69567 (-0.45383) + | > loss_dur: 0.24799 (0.15634) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 115.47816 (36.77797) + | > current_lr: 0.00006 + | > step_time: 0.24380 (2.91503) + | > loader_time: 0.00310 (0.08451) + + + --> STEP: 231/234 -- GLOBAL_STEP: 56625 + | > loss: -0.40537 (-0.30032) + | > log_mle: -0.78244 (-0.45981) + | > loss_dur: 0.37707 (0.15949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 106.96919 (37.97657) + | > current_lr: 0.00006 + | > step_time: 0.27310 (2.85739) + | > loader_time: 0.00410 (0.08278) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.22699 (-0.07181) + | > avg_loss: -0.32185 (-0.00428) + | > avg_log_mle: -0.54265 (-0.00805) + | > avg_loss_dur: 0.22080 (+0.00377) + + + > EPOCH: 242/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 12:07:52)  + + --> STEP: 2/234 -- GLOBAL_STEP: 56630 + | > loss: -0.29975 (-0.28815) + | > log_mle: -0.38437 (-0.37410) + | > loss_dur: 0.08462 (0.08596) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.07619 (23.10988) + | > current_lr: 0.00006 + | > step_time: 5.30510 (4.40290) + | > loader_time: 0.00250 (0.00400) + + + --> STEP: 7/234 -- GLOBAL_STEP: 56635 + | > loss: -0.30783 (-0.27564) + | > log_mle: -0.37305 (-0.36951) + | > loss_dur: 0.06522 (0.09387) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.54104 (22.49152) + | > current_lr: 0.00006 + | > step_time: 1.51910 (5.85047) + | > loader_time: 0.08150 (0.95567) + + + --> STEP: 12/234 -- GLOBAL_STEP: 56640 + | > loss: -0.26989 (-0.27970) + | > log_mle: -0.36795 (-0.37241) + | > loss_dur: 0.09806 (0.09271) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.18867 (19.82263) + | > current_lr: 0.00006 + | > step_time: 8.58750 (4.83726) + | > loader_time: 0.11650 (0.57596) + + + --> STEP: 17/234 -- GLOBAL_STEP: 56645 + | > loss: -0.30845 (-0.28733) + | > log_mle: -0.36886 (-0.37459) + | > loss_dur: 0.06041 (0.08726) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.03526 (17.16045) + | > current_lr: 0.00006 + | > step_time: 0.58790 (3.94843) + | > loader_time: 0.00100 (0.41299) + + + --> STEP: 22/234 -- GLOBAL_STEP: 56650 + | > loss: -0.28534 (-0.28809) + | > log_mle: -0.37509 (-0.37323) + | > loss_dur: 0.08975 (0.08513) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01545 (15.68951) + | > current_lr: 0.00006 + | > step_time: 2.01320 (3.40921) + | > loader_time: 0.08770 (0.32672) + + + --> STEP: 27/234 -- GLOBAL_STEP: 56655 + | > loss: -0.28848 (-0.28888) + | > log_mle: -0.36482 (-0.37200) + | > loss_dur: 0.07635 (0.08312) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.01279 (14.84579) + | > current_lr: 0.00006 + | > step_time: 0.69800 (3.39307) + | > loader_time: 0.00250 (0.26665) + + + --> STEP: 32/234 -- GLOBAL_STEP: 56660 + | > loss: -0.28918 (-0.28801) + | > log_mle: -0.36704 (-0.37068) + | > loss_dur: 0.07786 (0.08267) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.05090 (14.33635) + | > current_lr: 0.00006 + | > step_time: 3.09900 (3.17954) + | > loader_time: 0.08810 (0.22808) + + + --> STEP: 37/234 -- GLOBAL_STEP: 56665 + | > loss: -0.26365 (-0.28503) + | > log_mle: -0.34392 (-0.36828) + | > loss_dur: 0.08027 (0.08325) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.41406 (13.91570) + | > current_lr: 0.00006 + | > step_time: 2.02330 (3.09388) + | > loader_time: 0.00310 (0.20260) + + + --> STEP: 42/234 -- GLOBAL_STEP: 56670 + | > loss: -0.24207 (-0.28197) + | > log_mle: -0.33684 (-0.36642) + | > loss_dur: 0.09476 (0.08445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.29671 (13.64370) + | > current_lr: 0.00006 + | > step_time: 1.52690 (2.93075) + | > loader_time: 0.00230 (0.17871) + + + --> STEP: 47/234 -- GLOBAL_STEP: 56675 + | > loss: -0.25616 (-0.27891) + | > log_mle: -0.35518 (-0.36528) + | > loss_dur: 0.09902 (0.08637) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.76551 (13.47442) + | > current_lr: 0.00006 + | > step_time: 2.72080 (2.88706) + | > loader_time: 0.00340 (0.16012) + + + --> STEP: 52/234 -- GLOBAL_STEP: 56680 + | > loss: -0.23872 (-0.27764) + | > log_mle: -0.34690 (-0.36390) + | > loss_dur: 0.10818 (0.08626) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.59198 (13.16796) + | > current_lr: 0.00006 + | > step_time: 1.69540 (2.73920) + | > loader_time: 0.00280 (0.14492) + + + --> STEP: 57/234 -- GLOBAL_STEP: 56685 + | > loss: -0.23394 (-0.27535) + | > log_mle: -0.33509 (-0.36281) + | > loss_dur: 0.10115 (0.08746) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.32376 (13.04038) + | > current_lr: 0.00006 + | > step_time: 2.20670 (2.69215) + | > loader_time: 0.00280 (0.13672) + + + --> STEP: 62/234 -- GLOBAL_STEP: 56690 + | > loss: -0.20977 (-0.27308) + | > log_mle: -0.37188 (-0.36255) + | > loss_dur: 0.16210 (0.08947) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.99964 (13.18703) + | > current_lr: 0.00006 + | > step_time: 2.21440 (2.67039) + | > loader_time: 0.00290 (0.12742) + + + --> STEP: 67/234 -- GLOBAL_STEP: 56695 + | > loss: -0.25166 (-0.27117) + | > log_mle: -0.36033 (-0.36148) + | > loss_dur: 0.10867 (0.09032) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.37431 (12.90019) + | > current_lr: 0.00006 + | > step_time: 1.46090 (2.63011) + | > loader_time: 0.00170 (0.11809) + + + --> STEP: 72/234 -- GLOBAL_STEP: 56700 + | > loss: -0.23348 (-0.26811) + | > log_mle: -0.34475 (-0.36028) + | > loss_dur: 0.11128 (0.09217) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.77694 (12.91471) + | > current_lr: 0.00006 + | > step_time: 3.04020 (2.57897) + | > loader_time: 0.00250 (0.11117) + + + --> STEP: 77/234 -- GLOBAL_STEP: 56705 + | > loss: -0.23838 (-0.26536) + | > log_mle: -0.34821 (-0.35934) + | > loss_dur: 0.10983 (0.09398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.23507 (13.02231) + | > current_lr: 0.00006 + | > step_time: 2.38390 (2.54935) + | > loader_time: 0.00210 (0.10411) + + + --> STEP: 82/234 -- GLOBAL_STEP: 56710 + | > loss: -0.22490 (-0.26313) + | > log_mle: -0.34160 (-0.35842) + | > loss_dur: 0.11670 (0.09528) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.53152 (12.94945) + | > current_lr: 0.00006 + | > step_time: 1.25710 (2.49087) + | > loader_time: 0.00130 (0.09901) + + + --> STEP: 87/234 -- GLOBAL_STEP: 56715 + | > loss: -0.22458 (-0.26098) + | > log_mle: -0.34442 (-0.35776) + | > loss_dur: 0.11985 (0.09678) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.52288 (13.06789) + | > current_lr: 0.00006 + | > step_time: 1.61210 (2.43392) + | > loader_time: 0.00250 (0.09552) + + + --> STEP: 92/234 -- GLOBAL_STEP: 56720 + | > loss: -0.25661 (-0.25968) + | > log_mle: -0.38667 (-0.35861) + | > loss_dur: 0.13006 (0.09893) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.59474 (13.29279) + | > current_lr: 0.00006 + | > step_time: 2.39410 (2.44822) + | > loader_time: 0.00190 (0.09146) + + + --> STEP: 97/234 -- GLOBAL_STEP: 56725 + | > loss: -0.22974 (-0.25908) + | > log_mle: -0.36723 (-0.36030) + | > loss_dur: 0.13749 (0.10121) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.75670 (13.79710) + | > current_lr: 0.00006 + | > step_time: 1.29780 (2.40917) + | > loader_time: 0.00480 (0.08780) + + + --> STEP: 102/234 -- GLOBAL_STEP: 56730 + | > loss: -0.21731 (-0.25795) + | > log_mle: -0.35779 (-0.36126) + | > loss_dur: 0.14049 (0.10331) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.20992 (14.12216) + | > current_lr: 0.00006 + | > step_time: 1.15390 (2.38105) + | > loader_time: 0.00230 (0.08441) + + + --> STEP: 107/234 -- GLOBAL_STEP: 56735 + | > loss: -0.24405 (-0.25758) + | > log_mle: -0.39653 (-0.36334) + | > loss_dur: 0.15248 (0.10576) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.73280 (14.47568) + | > current_lr: 0.00006 + | > step_time: 2.06560 (2.38161) + | > loader_time: 0.00470 (0.08215) + + + --> STEP: 112/234 -- GLOBAL_STEP: 56740 + | > loss: -0.23626 (-0.25669) + | > log_mle: -0.40819 (-0.36526) + | > loss_dur: 0.17192 (0.10856) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.86995 (15.08808) + | > current_lr: 0.00006 + | > step_time: 2.58450 (2.35543) + | > loader_time: 0.00230 (0.08021) + + + --> STEP: 117/234 -- GLOBAL_STEP: 56745 + | > loss: -0.24904 (-0.25598) + | > log_mle: -0.40415 (-0.36700) + | > loss_dur: 0.15511 (0.11101) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.83181 (15.66084) + | > current_lr: 0.00006 + | > step_time: 2.00500 (2.35922) + | > loader_time: 0.00210 (0.07846) + + + --> STEP: 122/234 -- GLOBAL_STEP: 56750 + | > loss: -0.21994 (-0.25512) + | > log_mle: -0.37463 (-0.36785) + | > loss_dur: 0.15470 (0.11272) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.38753 (15.99049) + | > current_lr: 0.00006 + | > step_time: 1.30500 (2.32410) + | > loader_time: 0.00310 (0.07536) + + + --> STEP: 127/234 -- GLOBAL_STEP: 56755 + | > loss: -0.26154 (-0.25501) + | > log_mle: -0.43808 (-0.36999) + | > loss_dur: 0.17654 (0.11499) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.38637 (16.60696) + | > current_lr: 0.00006 + | > step_time: 3.49900 (2.32136) + | > loader_time: 0.00280 (0.07252) + + + --> STEP: 132/234 -- GLOBAL_STEP: 56760 + | > loss: -0.26568 (-0.25576) + | > log_mle: -0.42091 (-0.37277) + | > loss_dur: 0.15524 (0.11701) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.58776 (17.22101) + | > current_lr: 0.00006 + | > step_time: 1.29980 (2.30080) + | > loader_time: 0.00250 (0.07050) + + + --> STEP: 137/234 -- GLOBAL_STEP: 56765 + | > loss: -0.24738 (-0.25664) + | > log_mle: -0.43689 (-0.37590) + | > loss_dur: 0.18951 (0.11926) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.80407 (18.02153) + | > current_lr: 0.00006 + | > step_time: 0.98840 (2.28537) + | > loader_time: 0.00210 (0.06807) + + + --> STEP: 142/234 -- GLOBAL_STEP: 56770 + | > loss: -0.25998 (-0.25713) + | > log_mle: -0.44777 (-0.37853) + | > loss_dur: 0.18780 (0.12140) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.05283 (18.69432) + | > current_lr: 0.00006 + | > step_time: 1.30640 (2.28607) + | > loader_time: 0.00250 (0.06766) + + + --> STEP: 147/234 -- GLOBAL_STEP: 56775 + | > loss: -0.26542 (-0.25888) + | > log_mle: -0.44976 (-0.38287) + | > loss_dur: 0.18433 (0.12398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.99415 (19.68448) + | > current_lr: 0.00006 + | > step_time: 7.99240 (2.32324) + | > loader_time: 0.11180 (0.06624) + + + --> STEP: 152/234 -- GLOBAL_STEP: 56780 + | > loss: -0.32453 (-0.26083) + | > log_mle: -0.53885 (-0.38688) + | > loss_dur: 0.21433 (0.12605) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.30867 (20.78607) + | > current_lr: 0.00006 + | > step_time: 2.62200 (2.31366) + | > loader_time: 0.07870 (0.06578) + + + --> STEP: 157/234 -- GLOBAL_STEP: 56785 + | > loss: -0.30073 (-0.26325) + | > log_mle: -0.48271 (-0.39151) + | > loss_dur: 0.18198 (0.12826) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.20642 (21.95712) + | > current_lr: 0.00006 + | > step_time: 4.28520 (2.32077) + | > loader_time: 0.00350 (0.06539) + + + --> STEP: 162/234 -- GLOBAL_STEP: 56790 + | > loss: -0.33148 (-0.26542) + | > log_mle: -0.51971 (-0.39596) + | > loss_dur: 0.18823 (0.13054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.91927 (23.07072) + | > current_lr: 0.00006 + | > step_time: 1.81400 (2.39127) + | > loader_time: 0.08310 (0.06625) + + + --> STEP: 167/234 -- GLOBAL_STEP: 56795 + | > loss: -0.39189 (-0.26706) + | > log_mle: -0.59476 (-0.39960) + | > loss_dur: 0.20287 (0.13253) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.89949 (24.53247) + | > current_lr: 0.00006 + | > step_time: 3.40210 (2.39467) + | > loader_time: 0.00390 (0.06609) + + + --> STEP: 172/234 -- GLOBAL_STEP: 56800 + | > loss: -0.38101 (-0.26949) + | > log_mle: -0.59319 (-0.40453) + | > loss_dur: 0.21218 (0.13503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.76167 (25.68120) + | > current_lr: 0.00006 + | > step_time: 1.60160 (2.38977) + | > loader_time: 0.00400 (0.06473) + + + --> STEP: 177/234 -- GLOBAL_STEP: 56805 + | > loss: -0.34755 (-0.27211) + | > log_mle: -0.55676 (-0.40935) + | > loss_dur: 0.20922 (0.13724) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.84962 (26.58621) + | > current_lr: 0.00006 + | > step_time: 3.20140 (2.37882) + | > loader_time: 0.00280 (0.06346) + + + --> STEP: 182/234 -- GLOBAL_STEP: 56810 + | > loss: -0.36061 (-0.27440) + | > log_mle: -0.59699 (-0.41401) + | > loss_dur: 0.23639 (0.13961) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.71493 (27.64226) + | > current_lr: 0.00006 + | > step_time: 4.30330 (2.42278) + | > loader_time: 0.00430 (0.06342) + + + --> STEP: 187/234 -- GLOBAL_STEP: 56815 + | > loss: -0.38764 (-0.27686) + | > log_mle: -0.60772 (-0.41875) + | > loss_dur: 0.22008 (0.14189) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.21712 (28.65371) + | > current_lr: 0.00006 + | > step_time: 12.59350 (2.51299) + | > loader_time: 0.00690 (0.06445) + + + --> STEP: 192/234 -- GLOBAL_STEP: 56820 + | > loss: -0.41548 (-0.27951) + | > log_mle: -0.62884 (-0.42333) + | > loss_dur: 0.21337 (0.14382) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 69.16300 (29.73009) + | > current_lr: 0.00006 + | > step_time: 13.39290 (2.60844) + | > loader_time: 0.20370 (0.06689) + + + --> STEP: 197/234 -- GLOBAL_STEP: 56825 + | > loss: -0.39902 (-0.28220) + | > log_mle: -0.59534 (-0.42782) + | > loss_dur: 0.19632 (0.14561) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.66937 (30.72163) + | > current_lr: 0.00006 + | > step_time: 3.28720 (2.61334) + | > loader_time: 0.00820 (0.06639) + + + --> STEP: 202/234 -- GLOBAL_STEP: 56830 + | > loss: -0.47842 (-0.28491) + | > log_mle: -0.69519 (-0.43246) + | > loss_dur: 0.21677 (0.14755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.01662 (31.67766) + | > current_lr: 0.00006 + | > step_time: 9.69520 (2.70994) + | > loader_time: 0.09470 (0.06615) + + + --> STEP: 207/234 -- GLOBAL_STEP: 56835 + | > loss: -0.45252 (-0.28760) + | > log_mle: -0.67624 (-0.43706) + | > loss_dur: 0.22372 (0.14946) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 81.70815 (32.68359) + | > current_lr: 0.00006 + | > step_time: 3.70690 (2.77889) + | > loader_time: 0.08900 (0.06932) + + + --> STEP: 212/234 -- GLOBAL_STEP: 56840 + | > loss: -0.42332 (-0.29094) + | > log_mle: -0.65844 (-0.44237) + | > loss_dur: 0.23512 (0.15144) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 91.36227 (33.92019) + | > current_lr: 0.00006 + | > step_time: 9.50210 (2.88373) + | > loader_time: 0.49190 (0.07142) + + + --> STEP: 217/234 -- GLOBAL_STEP: 56845 + | > loss: -0.45057 (-0.29436) + | > log_mle: -0.68661 (-0.44763) + | > loss_dur: 0.23604 (0.15327) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.62859 (35.35046) + | > current_lr: 0.00006 + | > step_time: 2.98250 (2.90881) + | > loader_time: 0.00230 (0.06989) + + + --> STEP: 222/234 -- GLOBAL_STEP: 56850 + | > loss: -0.44798 (-0.29774) + | > log_mle: -0.70843 (-0.45290) + | > loss_dur: 0.26046 (0.15516) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 82.34734 (36.59487) + | > current_lr: 0.00006 + | > step_time: 2.80590 (2.91675) + | > loader_time: 0.09700 (0.06921) + + + --> STEP: 227/234 -- GLOBAL_STEP: 56855 + | > loss: -0.41279 (-0.30131) + | > log_mle: -0.66175 (-0.45837) + | > loss_dur: 0.24896 (0.15706) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 102.20367 (38.09618) + | > current_lr: 0.00006 + | > step_time: 2.79780 (2.91013) + | > loader_time: 0.00430 (0.06816) + + + --> STEP: 232/234 -- GLOBAL_STEP: 56860 + | > loss: -0.43190 (-0.30413) + | > log_mle: -0.89016 (-0.46522) + | > loss_dur: 0.45827 (0.16109) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 107.80607 (39.46162) + | > current_lr: 0.00006 + | > step_time: 0.36930 (2.86655) + | > loader_time: 0.10140 (0.06786) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.62501 (+1.39803) + | > avg_loss: -0.30439 (+0.01746) + | > avg_log_mle: -0.52841 (+0.01424) + | > avg_loss_dur: 0.22402 (+0.00322) + + + > EPOCH: 243/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 12:20:14)  + + --> STEP: 3/234 -- GLOBAL_STEP: 56865 + | > loss: -0.19951 (-0.26156) + | > log_mle: -0.35819 (-0.36924) + | > loss_dur: 0.15868 (0.10768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.37025 (25.60665) + | > current_lr: 0.00006 + | > step_time: 1.90330 (3.36937) + | > loader_time: 0.00380 (0.06499) + + + --> STEP: 8/234 -- GLOBAL_STEP: 56870 + | > loss: -0.28856 (-0.27833) + | > log_mle: -0.38704 (-0.37271) + | > loss_dur: 0.09848 (0.09438) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.69362 (20.21550) + | > current_lr: 0.00006 + | > step_time: 4.73210 (4.26955) + | > loader_time: 0.10380 (0.08701) + + + --> STEP: 13/234 -- GLOBAL_STEP: 56875 + | > loss: -0.32795 (-0.28478) + | > log_mle: -0.39249 (-0.37475) + | > loss_dur: 0.06453 (0.08997) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.49864 (19.50705) + | > current_lr: 0.00006 + | > step_time: 5.99220 (4.93500) + | > loader_time: 0.00140 (0.11586) + + + --> STEP: 18/234 -- GLOBAL_STEP: 56880 + | > loss: -0.29103 (-0.28985) + | > log_mle: -0.36812 (-0.37524) + | > loss_dur: 0.07709 (0.08539) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.27542 (17.15589) + | > current_lr: 0.00006 + | > step_time: 2.40790 (4.74289) + | > loader_time: 0.09300 (0.10549) + + + --> STEP: 23/234 -- GLOBAL_STEP: 56885 + | > loss: -0.31498 (-0.29238) + | > log_mle: -0.38588 (-0.37484) + | > loss_dur: 0.07091 (0.08245) + | > amp_scaler: 4096.00000 (2404.17391) + | > grad_norm: 7.96719 (15.82508) + | > current_lr: 0.00006 + | > step_time: 1.50200 (4.30231) + | > loader_time: 0.00220 (0.08740) + + + --> STEP: 28/234 -- GLOBAL_STEP: 56890 + | > loss: -0.34307 (-0.29318) + | > log_mle: -0.39436 (-0.37384) + | > loss_dur: 0.05128 (0.08066) + | > amp_scaler: 4096.00000 (2706.28571) + | > grad_norm: 4.85897 (14.92479) + | > current_lr: 0.00006 + | > step_time: 2.89680 (4.40971) + | > loader_time: 0.08730 (0.07533) + + + --> STEP: 33/234 -- GLOBAL_STEP: 56895 + | > loss: -0.28344 (-0.28945) + | > log_mle: -0.35865 (-0.37099) + | > loss_dur: 0.07521 (0.08154) + | > amp_scaler: 4096.00000 (2916.84848) + | > grad_norm: 11.83776 (14.51001) + | > current_lr: 0.00006 + | > step_time: 3.50530 (4.43555) + | > loader_time: 0.00240 (0.06977) + + + --> STEP: 38/234 -- GLOBAL_STEP: 56900 + | > loss: -0.28137 (-0.28594) + | > log_mle: -0.36368 (-0.36845) + | > loss_dur: 0.08232 (0.08251) + | > amp_scaler: 4096.00000 (3072.00000) + | > grad_norm: 13.23866 (14.42021) + | > current_lr: 0.00006 + | > step_time: 0.91930 (4.20224) + | > loader_time: 0.00270 (0.06802) + + + --> STEP: 43/234 -- GLOBAL_STEP: 56905 + | > loss: -0.24155 (-0.28239) + | > log_mle: -0.35329 (-0.36614) + | > loss_dur: 0.11175 (0.08376) + | > amp_scaler: 4096.00000 (3191.06977) + | > grad_norm: 13.22378 (14.10283) + | > current_lr: 0.00006 + | > step_time: 1.34180 (3.87997) + | > loader_time: 0.00140 (0.06432) + + + --> STEP: 48/234 -- GLOBAL_STEP: 56910 + | > loss: -0.27713 (-0.28040) + | > log_mle: -0.35218 (-0.36503) + | > loss_dur: 0.07505 (0.08463) + | > amp_scaler: 4096.00000 (3285.33333) + | > grad_norm: 9.30453 (13.70541) + | > current_lr: 0.00006 + | > step_time: 1.91950 (3.65197) + | > loader_time: 0.00190 (0.05790) + + + --> STEP: 53/234 -- GLOBAL_STEP: 56915 + | > loss: -0.25753 (-0.27805) + | > log_mle: -0.35599 (-0.36375) + | > loss_dur: 0.09846 (0.08570) + | > amp_scaler: 4096.00000 (3361.81132) + | > grad_norm: 10.54374 (13.29875) + | > current_lr: 0.00006 + | > step_time: 3.90580 (3.47840) + | > loader_time: 0.18380 (0.05762) + + + --> STEP: 58/234 -- GLOBAL_STEP: 56920 + | > loss: -0.26376 (-0.27639) + | > log_mle: -0.35050 (-0.36263) + | > loss_dur: 0.08674 (0.08623) + | > amp_scaler: 4096.00000 (3425.10345) + | > grad_norm: 8.61832 (13.06705) + | > current_lr: 0.00006 + | > step_time: 1.60270 (3.34702) + | > loader_time: 0.00280 (0.05289) + + + --> STEP: 63/234 -- GLOBAL_STEP: 56925 + | > loss: -0.23656 (-0.27328) + | > log_mle: -0.34402 (-0.36222) + | > loss_dur: 0.10746 (0.08894) + | > amp_scaler: 4096.00000 (3478.34921) + | > grad_norm: 9.12152 (13.12762) + | > current_lr: 0.00006 + | > step_time: 1.30830 (3.25338) + | > loader_time: 0.00370 (0.05168) + + + --> STEP: 68/234 -- GLOBAL_STEP: 56930 + | > loss: -0.22392 (-0.27111) + | > log_mle: -0.34229 (-0.36120) + | > loss_dur: 0.11837 (0.09008) + | > amp_scaler: 4096.00000 (3523.76471) + | > grad_norm: 13.13205 (12.92595) + | > current_lr: 0.00006 + | > step_time: 1.39920 (3.16864) + | > loader_time: 0.00230 (0.04808) + + + --> STEP: 73/234 -- GLOBAL_STEP: 56935 + | > loss: -0.21077 (-0.26806) + | > log_mle: -0.34954 (-0.36014) + | > loss_dur: 0.13877 (0.09209) + | > amp_scaler: 4096.00000 (3562.95890) + | > grad_norm: 18.30377 (12.98812) + | > current_lr: 0.00006 + | > step_time: 1.09600 (3.06541) + | > loader_time: 0.00140 (0.04495) + + + --> STEP: 78/234 -- GLOBAL_STEP: 56940 + | > loss: -0.22966 (-0.26546) + | > log_mle: -0.33441 (-0.35903) + | > loss_dur: 0.10476 (0.09358) + | > amp_scaler: 4096.00000 (3597.12821) + | > grad_norm: 11.91609 (13.02240) + | > current_lr: 0.00006 + | > step_time: 1.38600 (3.03895) + | > loader_time: 0.00200 (0.04470) + + + --> STEP: 83/234 -- GLOBAL_STEP: 56945 + | > loss: -0.19981 (-0.26312) + | > log_mle: -0.34916 (-0.35827) + | > loss_dur: 0.14935 (0.09516) + | > amp_scaler: 4096.00000 (3627.18072) + | > grad_norm: 18.26217 (13.05910) + | > current_lr: 0.00006 + | > step_time: 1.69720 (2.96905) + | > loader_time: 0.08860 (0.04628) + + + --> STEP: 88/234 -- GLOBAL_STEP: 56950 + | > loss: -0.23667 (-0.26132) + | > log_mle: -0.38537 (-0.35808) + | > loss_dur: 0.14869 (0.09677) + | > amp_scaler: 4096.00000 (3653.81818) + | > grad_norm: 19.50907 (13.10342) + | > current_lr: 0.00006 + | > step_time: 2.15590 (2.95959) + | > loader_time: 0.00200 (0.04764) + + + --> STEP: 93/234 -- GLOBAL_STEP: 56955 + | > loss: -0.24554 (-0.26036) + | > log_mle: -0.39998 (-0.35919) + | > loss_dur: 0.15444 (0.09883) + | > amp_scaler: 4096.00000 (3677.59140) + | > grad_norm: 22.79219 (13.34569) + | > current_lr: 0.00006 + | > step_time: 2.40080 (2.92049) + | > loader_time: 0.08680 (0.04711) + + + --> STEP: 98/234 -- GLOBAL_STEP: 56960 + | > loss: -0.22635 (-0.25970) + | > log_mle: -0.33554 (-0.36031) + | > loss_dur: 0.10919 (0.10061) + | > amp_scaler: 4096.00000 (3698.93878) + | > grad_norm: 8.48248 (13.62948) + | > current_lr: 0.00006 + | > step_time: 1.70750 (2.90539) + | > loader_time: 0.00260 (0.04569) + + + --> STEP: 103/234 -- GLOBAL_STEP: 56965 + | > loss: -0.26559 (-0.25908) + | > log_mle: -0.42458 (-0.36215) + | > loss_dur: 0.15898 (0.10307) + | > amp_scaler: 4096.00000 (3718.21359) + | > grad_norm: 43.32854 (14.35849) + | > current_lr: 0.00006 + | > step_time: 1.69340 (2.84943) + | > loader_time: 0.00390 (0.04539) + + + --> STEP: 108/234 -- GLOBAL_STEP: 56970 + | > loss: -0.24130 (-0.25830) + | > log_mle: -0.37385 (-0.36354) + | > loss_dur: 0.13255 (0.10523) + | > amp_scaler: 4096.00000 (3735.70370) + | > grad_norm: 16.69020 (14.90000) + | > current_lr: 0.00006 + | > step_time: 1.69540 (2.80701) + | > loader_time: 0.00580 (0.04344) + + + --> STEP: 113/234 -- GLOBAL_STEP: 56975 + | > loss: -0.25759 (-0.25760) + | > log_mle: -0.41990 (-0.36580) + | > loss_dur: 0.16232 (0.10820) + | > amp_scaler: 4096.00000 (3751.64602) + | > grad_norm: 34.10199 (15.75406) + | > current_lr: 0.00006 + | > step_time: 3.21000 (2.78562) + | > loader_time: 0.00270 (0.04244) + + + --> STEP: 118/234 -- GLOBAL_STEP: 56980 + | > loss: -0.22591 (-0.25682) + | > log_mle: -0.38842 (-0.36734) + | > loss_dur: 0.16251 (0.11052) + | > amp_scaler: 4096.00000 (3766.23729) + | > grad_norm: 24.97304 (16.09917) + | > current_lr: 0.00006 + | > step_time: 2.60890 (2.79493) + | > loader_time: 0.19280 (0.04467) + + + --> STEP: 123/234 -- GLOBAL_STEP: 56985 + | > loss: -0.20362 (-0.25576) + | > log_mle: -0.35490 (-0.36787) + | > loss_dur: 0.15128 (0.11211) + | > amp_scaler: 4096.00000 (3779.64228) + | > grad_norm: 21.81876 (16.48566) + | > current_lr: 0.00006 + | > step_time: 3.59200 (2.79444) + | > loader_time: 0.00220 (0.04371) + + + --> STEP: 128/234 -- GLOBAL_STEP: 56990 + | > loss: -0.26453 (-0.25610) + | > log_mle: -0.41443 (-0.37051) + | > loss_dur: 0.14990 (0.11441) + | > amp_scaler: 4096.00000 (3792.00000) + | > grad_norm: 30.10415 (17.10370) + | > current_lr: 0.00006 + | > step_time: 1.79940 (2.79692) + | > loader_time: 0.00370 (0.04431) + + + --> STEP: 133/234 -- GLOBAL_STEP: 56995 + | > loss: -0.27171 (-0.25669) + | > log_mle: -0.44516 (-0.37345) + | > loss_dur: 0.17345 (0.11676) + | > amp_scaler: 4096.00000 (3803.42857) + | > grad_norm: 31.38040 (17.84675) + | > current_lr: 0.00006 + | > step_time: 3.90040 (2.78585) + | > loader_time: 0.00310 (0.04350) + + + --> STEP: 138/234 -- GLOBAL_STEP: 57000 + | > loss: -0.22694 (-0.25699) + | > log_mle: -0.39348 (-0.37601) + | > loss_dur: 0.16655 (0.11902) + | > amp_scaler: 4096.00000 (3814.02899) + | > grad_norm: 20.87645 (18.70964) + | > current_lr: 0.00006 + | > step_time: 2.39950 (2.78569) + | > loader_time: 0.00410 (0.04487) + + + --> STEP: 143/234 -- GLOBAL_STEP: 57005 + | > loss: -0.32250 (-0.25794) + | > log_mle: -0.54321 (-0.37940) + | > loss_dur: 0.22071 (0.12146) + | > amp_scaler: 4096.00000 (3823.88811) + | > grad_norm: 53.36522 (19.60787) + | > current_lr: 0.00006 + | > step_time: 1.40840 (2.75616) + | > loader_time: 0.08240 (0.04520) + + + --> STEP: 148/234 -- GLOBAL_STEP: 57010 + | > loss: -0.29823 (-0.25929) + | > log_mle: -0.45335 (-0.38299) + | > loss_dur: 0.15512 (0.12370) + | > amp_scaler: 4096.00000 (3833.08108) + | > grad_norm: 30.10523 (20.37642) + | > current_lr: 0.00006 + | > step_time: 2.50330 (2.77585) + | > loader_time: 0.00250 (0.04516) + + + --> STEP: 153/234 -- GLOBAL_STEP: 57015 + | > loss: -0.38814 (-0.26172) + | > log_mle: -0.58347 (-0.38784) + | > loss_dur: 0.19533 (0.12613) + | > amp_scaler: 4096.00000 (3841.67320) + | > grad_norm: 72.83945 (21.51488) + | > current_lr: 0.00006 + | > step_time: 1.79930 (2.81468) + | > loader_time: 0.00440 (0.04691) + + + --> STEP: 158/234 -- GLOBAL_STEP: 57020 + | > loss: -0.30710 (-0.26348) + | > log_mle: -0.51258 (-0.39191) + | > loss_dur: 0.20547 (0.12843) + | > amp_scaler: 4096.00000 (3849.72152) + | > grad_norm: 45.16351 (22.45424) + | > current_lr: 0.00006 + | > step_time: 1.89420 (2.78982) + | > loader_time: 0.00380 (0.04607) + + + --> STEP: 163/234 -- GLOBAL_STEP: 57025 + | > loss: -0.30516 (-0.26572) + | > log_mle: -0.49149 (-0.39616) + | > loss_dur: 0.18633 (0.13044) + | > amp_scaler: 4096.00000 (3857.27607) + | > grad_norm: 40.81303 (23.31347) + | > current_lr: 0.00006 + | > step_time: 2.01080 (2.78665) + | > loader_time: 0.09530 (0.04643) + + + --> STEP: 168/234 -- GLOBAL_STEP: 57030 + | > loss: -0.32568 (-0.26810) + | > log_mle: -0.54577 (-0.40061) + | > loss_dur: 0.22009 (0.13250) + | > amp_scaler: 4096.00000 (3864.38095) + | > grad_norm: 61.65432 (24.28393) + | > current_lr: 0.00006 + | > step_time: 2.49530 (2.76048) + | > loader_time: 0.00450 (0.04658) + + + --> STEP: 173/234 -- GLOBAL_STEP: 57035 + | > loss: -0.34726 (-0.27090) + | > log_mle: -0.55876 (-0.40571) + | > loss_dur: 0.21150 (0.13481) + | > amp_scaler: 4096.00000 (3871.07514) + | > grad_norm: 52.46377 (25.30278) + | > current_lr: 0.00006 + | > step_time: 5.38520 (2.76212) + | > loader_time: 0.20070 (0.04756) + + + --> STEP: 178/234 -- GLOBAL_STEP: 57040 + | > loss: -0.38744 (-0.27363) + | > log_mle: -0.61603 (-0.41079) + | > loss_dur: 0.22859 (0.13715) + | > amp_scaler: 4096.00000 (3877.39326) + | > grad_norm: 60.01018 (26.46828) + | > current_lr: 0.00006 + | > step_time: 1.40470 (2.75326) + | > loader_time: 0.00320 (0.04721) + + + --> STEP: 183/234 -- GLOBAL_STEP: 57045 + | > loss: -0.40403 (-0.27592) + | > log_mle: -0.61385 (-0.41539) + | > loss_dur: 0.20983 (0.13947) + | > amp_scaler: 4096.00000 (3883.36612) + | > grad_norm: 74.83591 (27.79969) + | > current_lr: 0.00006 + | > step_time: 6.29890 (2.79832) + | > loader_time: 0.09530 (0.04801) + + + --> STEP: 188/234 -- GLOBAL_STEP: 57050 + | > loss: -0.40611 (-0.27837) + | > log_mle: -0.62064 (-0.42006) + | > loss_dur: 0.21453 (0.14169) + | > amp_scaler: 4096.00000 (3889.02128) + | > grad_norm: 78.56390 (28.80004) + | > current_lr: 0.00006 + | > step_time: 2.90510 (2.80484) + | > loader_time: 0.00260 (0.04731) + + + --> STEP: 193/234 -- GLOBAL_STEP: 57055 + | > loss: -0.41413 (-0.28106) + | > log_mle: -0.62502 (-0.42456) + | > loss_dur: 0.21090 (0.14350) + | > amp_scaler: 4096.00000 (3894.38342) + | > grad_norm: 84.56470 (29.95321) + | > current_lr: 0.00006 + | > step_time: 5.47290 (2.84300) + | > loader_time: 0.13720 (0.04931) + + + --> STEP: 198/234 -- GLOBAL_STEP: 57060 + | > loss: -0.38289 (-0.28346) + | > log_mle: -0.61024 (-0.42880) + | > loss_dur: 0.22735 (0.14535) + | > amp_scaler: 4096.00000 (3899.47475) + | > grad_norm: 94.54659 (31.13142) + | > current_lr: 0.00006 + | > step_time: 6.29910 (2.88393) + | > loader_time: 0.09530 (0.04948) + + + --> STEP: 203/234 -- GLOBAL_STEP: 57065 + | > loss: -0.31978 (-0.28548) + | > log_mle: -0.53086 (-0.43265) + | > loss_dur: 0.21107 (0.14717) + | > amp_scaler: 4096.00000 (3904.31527) + | > grad_norm: 49.14803 (32.27742) + | > current_lr: 0.00006 + | > step_time: 14.30300 (2.98680) + | > loader_time: 0.09900 (0.05266) + + + --> STEP: 208/234 -- GLOBAL_STEP: 57070 + | > loss: -0.39510 (-0.28814) + | > log_mle: -0.62416 (-0.43744) + | > loss_dur: 0.22906 (0.14930) + | > amp_scaler: 4096.00000 (3908.92308) + | > grad_norm: 64.98367 (33.39433) + | > current_lr: 0.00006 + | > step_time: 8.90920 (3.07720) + | > loader_time: 0.19100 (0.05332) + + + --> STEP: 213/234 -- GLOBAL_STEP: 57075 + | > loss: -0.44472 (-0.29148) + | > log_mle: -0.68281 (-0.44278) + | > loss_dur: 0.23809 (0.15130) + | > amp_scaler: 4096.00000 (3913.31455) + | > grad_norm: 71.38054 (34.41233) + | > current_lr: 0.00006 + | > step_time: 4.09730 (3.11913) + | > loader_time: 0.10050 (0.05349) + + + --> STEP: 218/234 -- GLOBAL_STEP: 57080 + | > loss: -0.39632 (-0.29436) + | > log_mle: -0.63266 (-0.44751) + | > loss_dur: 0.23634 (0.15314) + | > amp_scaler: 2048.00000 (3889.32110) + | > grad_norm: 80.96987 (35.18094) + | > current_lr: 0.00006 + | > step_time: 5.81410 (3.18257) + | > loader_time: 0.20030 (0.05806) + + + --> STEP: 223/234 -- GLOBAL_STEP: 57085 + | > loss: -0.44728 (-0.29774) + | > log_mle: -0.68302 (-0.45275) + | > loss_dur: 0.23573 (0.15501) + | > amp_scaler: 2048.00000 (3848.03587) + | > grad_norm: 94.78067 (36.32358) + | > current_lr: 0.00006 + | > step_time: 1.10450 (3.15924) + | > loader_time: 0.00510 (0.05725) + + + --> STEP: 228/234 -- GLOBAL_STEP: 57090 + | > loss: -0.40880 (-0.30097) + | > log_mle: -0.67660 (-0.45803) + | > loss_dur: 0.26780 (0.15706) + | > amp_scaler: 2048.00000 (3808.56140) + | > grad_norm: 93.77094 (37.86677) + | > current_lr: 0.00006 + | > step_time: 0.24350 (3.11277) + | > loader_time: 0.00440 (0.05710) + + + --> STEP: 233/234 -- GLOBAL_STEP: 57095 + | > loss: 0.11134 (-0.30104) + | > log_mle: -0.63493 (-0.46431) + | > loss_dur: 0.74627 (0.16327) + | > amp_scaler: 2048.00000 (3770.78112) + | > grad_norm: 82.09307 (39.55847) + | > current_lr: 0.00006 + | > step_time: 0.18920 (3.05159) + | > loader_time: 0.00300 (0.05597) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.87453 (-0.75048) + | > avg_loss: -0.28328 (+0.02111) + | > avg_log_mle: -0.51661 (+0.01180) + | > avg_loss_dur: 0.23334 (+0.00931) + + + > EPOCH: 244/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 12:33:28)  + + --> STEP: 4/234 -- GLOBAL_STEP: 57100 + | > loss: -0.26107 (-0.26126) + | > log_mle: -0.36941 (-0.36881) + | > loss_dur: 0.10834 (0.10754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.56869 (23.12738) + | > current_lr: 0.00006 + | > step_time: 2.90000 (6.37874) + | > loader_time: 0.00500 (0.12116) + + + --> STEP: 9/234 -- GLOBAL_STEP: 57105 + | > loss: -0.26869 (-0.27715) + | > log_mle: -0.38171 (-0.37310) + | > loss_dur: 0.11302 (0.09595) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.04019 (20.30165) + | > current_lr: 0.00006 + | > step_time: 5.50740 (4.40176) + | > loader_time: 0.00360 (0.06474) + + + --> STEP: 14/234 -- GLOBAL_STEP: 57110 + | > loss: -0.30713 (-0.28475) + | > log_mle: -0.37670 (-0.37450) + | > loss_dur: 0.06957 (0.08975) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.67373 (18.85061) + | > current_lr: 0.00006 + | > step_time: 4.20990 (4.12429) + | > loader_time: 0.09140 (0.06742) + + + --> STEP: 19/234 -- GLOBAL_STEP: 57115 + | > loss: -0.30420 (-0.28945) + | > log_mle: -0.37274 (-0.37445) + | > loss_dur: 0.06854 (0.08500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.36794 (17.13880) + | > current_lr: 0.00006 + | > step_time: 3.89580 (4.38503) + | > loader_time: 0.10810 (0.06135) + + + --> STEP: 24/234 -- GLOBAL_STEP: 57120 + | > loss: -0.30325 (-0.29203) + | > log_mle: -0.36746 (-0.37407) + | > loss_dur: 0.06421 (0.08203) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.84772 (15.82812) + | > current_lr: 0.00006 + | > step_time: 2.59180 (4.52690) + | > loader_time: 0.00530 (0.06434) + + + --> STEP: 29/234 -- GLOBAL_STEP: 57125 + | > loss: -0.27745 (-0.29258) + | > log_mle: -0.35612 (-0.37312) + | > loss_dur: 0.07867 (0.08054) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.96830 (14.78084) + | > current_lr: 0.00006 + | > step_time: 3.29740 (4.50889) + | > loader_time: 0.10510 (0.07021) + + + --> STEP: 34/234 -- GLOBAL_STEP: 57130 + | > loss: -0.28028 (-0.29033) + | > log_mle: -0.36102 (-0.37138) + | > loss_dur: 0.08074 (0.08105) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.55009 (14.30842) + | > current_lr: 0.00006 + | > step_time: 2.87100 (4.44181) + | > loader_time: 0.00240 (0.06888) + + + --> STEP: 39/234 -- GLOBAL_STEP: 57135 + | > loss: -0.26460 (-0.28659) + | > log_mle: -0.35604 (-0.36906) + | > loss_dur: 0.09144 (0.08246) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.02720 (14.30656) + | > current_lr: 0.00006 + | > step_time: 1.06580 (3.99648) + | > loader_time: 0.00180 (0.06229) + + + --> STEP: 44/234 -- GLOBAL_STEP: 57140 + | > loss: -0.28555 (-0.28418) + | > log_mle: -0.35174 (-0.36709) + | > loss_dur: 0.06619 (0.08291) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.66790 (13.80763) + | > current_lr: 0.00006 + | > step_time: 1.39010 (3.82127) + | > loader_time: 0.00140 (0.06187) + + + --> STEP: 49/234 -- GLOBAL_STEP: 57145 + | > loss: -0.28459 (-0.28299) + | > log_mle: -0.36362 (-0.36647) + | > loss_dur: 0.07903 (0.08348) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.39770 (13.66565) + | > current_lr: 0.00006 + | > step_time: 0.84380 (3.57562) + | > loader_time: 0.00210 (0.05576) + + + --> STEP: 54/234 -- GLOBAL_STEP: 57150 + | > loss: -0.27172 (-0.28054) + | > log_mle: -0.35597 (-0.36507) + | > loss_dur: 0.08424 (0.08452) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.43848 (13.43807) + | > current_lr: 0.00006 + | > step_time: 1.10360 (3.35933) + | > loader_time: 0.00270 (0.05230) + + + --> STEP: 59/234 -- GLOBAL_STEP: 57155 + | > loss: -0.26590 (-0.27839) + | > log_mle: -0.35881 (-0.36395) + | > loss_dur: 0.09291 (0.08556) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.46470 (13.27679) + | > current_lr: 0.00006 + | > step_time: 1.70640 (3.23570) + | > loader_time: 0.00290 (0.04809) + + + --> STEP: 64/234 -- GLOBAL_STEP: 57160 + | > loss: -0.24639 (-0.27501) + | > log_mle: -0.34605 (-0.36334) + | > loss_dur: 0.09966 (0.08833) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.43517 (13.27456) + | > current_lr: 0.00006 + | > step_time: 2.81520 (3.21127) + | > loader_time: 0.10030 (0.04742) + + + --> STEP: 69/234 -- GLOBAL_STEP: 57165 + | > loss: -0.23852 (-0.27277) + | > log_mle: -0.33858 (-0.36214) + | > loss_dur: 0.10006 (0.08937) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.84446 (13.07904) + | > current_lr: 0.00006 + | > step_time: 1.50360 (3.12434) + | > loader_time: 0.09010 (0.04699) + + + --> STEP: 74/234 -- GLOBAL_STEP: 57170 + | > loss: -0.21886 (-0.26931) + | > log_mle: -0.33397 (-0.36107) + | > loss_dur: 0.11510 (0.09176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.57220 (13.12136) + | > current_lr: 0.00006 + | > step_time: 0.92270 (3.01918) + | > loader_time: 0.00250 (0.04740) + + + --> STEP: 79/234 -- GLOBAL_STEP: 57175 + | > loss: -0.23771 (-0.26723) + | > log_mle: -0.35031 (-0.36026) + | > loss_dur: 0.11260 (0.09303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.83450 (13.06209) + | > current_lr: 0.00006 + | > step_time: 2.18150 (2.95781) + | > loader_time: 0.00220 (0.04454) + + + --> STEP: 84/234 -- GLOBAL_STEP: 57180 + | > loss: -0.24268 (-0.26494) + | > log_mle: -0.34751 (-0.35942) + | > loss_dur: 0.10483 (0.09448) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.49387 (13.09256) + | > current_lr: 0.00006 + | > step_time: 2.80040 (2.89877) + | > loader_time: 0.00330 (0.04399) + + + --> STEP: 89/234 -- GLOBAL_STEP: 57185 + | > loss: -0.24727 (-0.26327) + | > log_mle: -0.37047 (-0.35944) + | > loss_dur: 0.12320 (0.09617) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05493 (13.26854) + | > current_lr: 0.00006 + | > step_time: 0.77520 (2.84576) + | > loader_time: 0.00210 (0.04281) + + + --> STEP: 94/234 -- GLOBAL_STEP: 57190 + | > loss: -0.26148 (-0.26199) + | > log_mle: -0.40008 (-0.36069) + | > loss_dur: 0.13860 (0.09870) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.67936 (13.66706) + | > current_lr: 0.00006 + | > step_time: 1.28340 (2.81123) + | > loader_time: 0.00170 (0.04244) + + + --> STEP: 99/234 -- GLOBAL_STEP: 57195 + | > loss: -0.24538 (-0.26106) + | > log_mle: -0.42609 (-0.36199) + | > loss_dur: 0.18071 (0.10094) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.47932 (14.16348) + | > current_lr: 0.00006 + | > step_time: 2.49920 (2.76031) + | > loader_time: 0.00380 (0.04126) + + + --> STEP: 104/234 -- GLOBAL_STEP: 57200 + | > loss: -0.29038 (-0.26067) + | > log_mle: -0.44449 (-0.36393) + | > loss_dur: 0.15412 (0.10326) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.46296 (14.69563) + | > current_lr: 0.00006 + | > step_time: 1.40440 (2.69782) + | > loader_time: 0.09930 (0.04034) + + + --> STEP: 109/234 -- GLOBAL_STEP: 57205 + | > loss: -0.21782 (-0.25945) + | > log_mle: -0.41411 (-0.36511) + | > loss_dur: 0.19629 (0.10566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.70078 (15.04906) + | > current_lr: 0.00006 + | > step_time: 1.58750 (2.68586) + | > loader_time: 0.00180 (0.04106) + + + --> STEP: 114/234 -- GLOBAL_STEP: 57210 + | > loss: -0.25195 (-0.25906) + | > log_mle: -0.39602 (-0.36721) + | > loss_dur: 0.14408 (0.10815) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.74086 (15.60966) + | > current_lr: 0.00006 + | > step_time: 2.69630 (2.66650) + | > loader_time: 0.10350 (0.04177) + + + --> STEP: 119/234 -- GLOBAL_STEP: 57215 + | > loss: -0.24698 (-0.25829) + | > log_mle: -0.39743 (-0.36877) + | > loss_dur: 0.15044 (0.11048) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.11575 (15.92118) + | > current_lr: 0.00006 + | > step_time: 2.89430 (2.63850) + | > loader_time: 0.00310 (0.04148) + + + --> STEP: 124/234 -- GLOBAL_STEP: 57220 + | > loss: -0.26623 (-0.25741) + | > log_mle: -0.42551 (-0.36957) + | > loss_dur: 0.15927 (0.11216) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.64591 (16.23454) + | > current_lr: 0.00006 + | > step_time: 5.20570 (2.63272) + | > loader_time: 0.08630 (0.04129) + + + --> STEP: 129/234 -- GLOBAL_STEP: 57225 + | > loss: -0.24283 (-0.25755) + | > log_mle: -0.41688 (-0.37214) + | > loss_dur: 0.17405 (0.11459) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.72918 (16.80956) + | > current_lr: 0.00006 + | > step_time: 2.18740 (2.61032) + | > loader_time: 0.02030 (0.04142) + + + --> STEP: 134/234 -- GLOBAL_STEP: 57230 + | > loss: -0.26540 (-0.25827) + | > log_mle: -0.45456 (-0.37523) + | > loss_dur: 0.18916 (0.11696) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.84278 (18.11831) + | > current_lr: 0.00006 + | > step_time: 3.20580 (2.59906) + | > loader_time: 0.09620 (0.04185) + + + --> STEP: 139/234 -- GLOBAL_STEP: 57235 + | > loss: -0.33080 (-0.25876) + | > log_mle: -0.53092 (-0.37815) + | > loss_dur: 0.20012 (0.11939) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.98965 (18.98843) + | > current_lr: 0.00006 + | > step_time: 3.00010 (2.62294) + | > loader_time: 0.08560 (0.04304) + + + --> STEP: 144/234 -- GLOBAL_STEP: 57240 + | > loss: -0.29455 (-0.25933) + | > log_mle: -0.49293 (-0.38123) + | > loss_dur: 0.19838 (0.12190) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.60276 (20.10065) + | > current_lr: 0.00006 + | > step_time: 1.30260 (2.60825) + | > loader_time: 0.08210 (0.04343) + + + --> STEP: 149/234 -- GLOBAL_STEP: 57245 + | > loss: -0.35757 (-0.26094) + | > log_mle: -0.55111 (-0.38494) + | > loss_dur: 0.19355 (0.12399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.41762 (20.87394) + | > current_lr: 0.00006 + | > step_time: 2.49230 (2.59592) + | > loader_time: 0.00360 (0.04268) + + + --> STEP: 154/234 -- GLOBAL_STEP: 57250 + | > loss: -0.31167 (-0.26292) + | > log_mle: -0.50619 (-0.38927) + | > loss_dur: 0.19452 (0.12635) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.46770 (21.92542) + | > current_lr: 0.00006 + | > step_time: 7.30170 (2.72068) + | > loader_time: 0.00370 (0.04394) + + + --> STEP: 159/234 -- GLOBAL_STEP: 57255 + | > loss: -0.32347 (-0.26446) + | > log_mle: -0.52083 (-0.39326) + | > loss_dur: 0.19735 (0.12879) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.92028 (23.10946) + | > current_lr: 0.00006 + | > step_time: 3.29880 (2.74527) + | > loader_time: 0.00330 (0.04446) + + + --> STEP: 164/234 -- GLOBAL_STEP: 57260 + | > loss: -0.31093 (-0.26632) + | > log_mle: -0.52063 (-0.39723) + | > loss_dur: 0.20969 (0.13091) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.40746 (23.79516) + | > current_lr: 0.00006 + | > step_time: 7.00940 (2.77453) + | > loader_time: 0.08530 (0.04539) + + + --> STEP: 169/234 -- GLOBAL_STEP: 57265 + | > loss: -0.31363 (-0.26861) + | > log_mle: -0.51930 (-0.40156) + | > loss_dur: 0.20567 (0.13295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.61059 (24.65360) + | > current_lr: 0.00006 + | > step_time: 2.60110 (2.83513) + | > loader_time: 0.00520 (0.04578) + + + --> STEP: 174/234 -- GLOBAL_STEP: 57270 + | > loss: -0.39309 (-0.27165) + | > log_mle: -0.60817 (-0.40690) + | > loss_dur: 0.21508 (0.13525) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.33892 (25.95706) + | > current_lr: 0.00006 + | > step_time: 6.29060 (2.88357) + | > loader_time: 0.01090 (0.04733) + + + --> STEP: 179/234 -- GLOBAL_STEP: 57275 + | > loss: -0.36697 (-0.27410) + | > log_mle: -0.60965 (-0.41179) + | > loss_dur: 0.24267 (0.13770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.48846 (26.87038) + | > current_lr: 0.00006 + | > step_time: 2.28190 (2.86292) + | > loader_time: 0.00240 (0.04695) + + + --> STEP: 184/234 -- GLOBAL_STEP: 57280 + | > loss: -0.34209 (-0.27584) + | > log_mle: -0.55792 (-0.41571) + | > loss_dur: 0.21583 (0.13987) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.74854 (27.87132) + | > current_lr: 0.00006 + | > step_time: 6.71240 (2.90193) + | > loader_time: 0.29050 (0.04783) + + + --> STEP: 189/234 -- GLOBAL_STEP: 57285 + | > loss: -0.33903 (-0.27741) + | > log_mle: -0.54998 (-0.41954) + | > loss_dur: 0.21095 (0.14213) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.53359 (28.64865) + | > current_lr: 0.00006 + | > step_time: 2.90530 (2.96163) + | > loader_time: 0.09900 (0.04719) + + + --> STEP: 194/234 -- GLOBAL_STEP: 57290 + | > loss: -0.37760 (-0.27976) + | > log_mle: -0.58382 (-0.42365) + | > loss_dur: 0.20623 (0.14389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.58212 (29.46794) + | > current_lr: 0.00006 + | > step_time: 5.80060 (3.00035) + | > loader_time: 0.09680 (0.04792) + + + --> STEP: 199/234 -- GLOBAL_STEP: 57295 + | > loss: -0.38866 (-0.28197) + | > log_mle: -0.60420 (-0.42774) + | > loss_dur: 0.21554 (0.14577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.92185 (30.11680) + | > current_lr: 0.00006 + | > step_time: 7.20110 (3.05168) + | > loader_time: 0.10440 (0.04922) + + + --> STEP: 204/234 -- GLOBAL_STEP: 57300 + | > loss: -0.38942 (-0.28393) + | > log_mle: -0.63845 (-0.43175) + | > loss_dur: 0.24903 (0.14782) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 114.61050 (31.22012) + | > current_lr: 0.00006 + | > step_time: 8.40290 (3.16563) + | > loader_time: 0.00290 (0.05004) + + + --> STEP: 209/234 -- GLOBAL_STEP: 57305 + | > loss: -0.37081 (-0.28650) + | > log_mle: -0.59381 (-0.43623) + | > loss_dur: 0.22300 (0.14973) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 123.61166 (32.45877) + | > current_lr: 0.00006 + | > step_time: 2.59180 (3.22096) + | > loader_time: 0.00420 (0.05036) + + + --> STEP: 214/234 -- GLOBAL_STEP: 57310 + | > loss: -0.42069 (-0.28976) + | > log_mle: -0.63160 (-0.44146) + | > loss_dur: 0.21091 (0.15170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.13015 (33.82349) + | > current_lr: 0.00006 + | > step_time: 10.99030 (3.31378) + | > loader_time: 0.00490 (0.05025) + + + --> STEP: 219/234 -- GLOBAL_STEP: 57315 + | > loss: -0.49793 (-0.29297) + | > log_mle: -0.73389 (-0.44659) + | > loss_dur: 0.23596 (0.15362) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.09489 (34.86654) + | > current_lr: 0.00006 + | > step_time: 5.00060 (3.38527) + | > loader_time: 0.09000 (0.05413) + + + --> STEP: 224/234 -- GLOBAL_STEP: 57320 + | > loss: -0.45978 (-0.29600) + | > log_mle: -0.69176 (-0.45149) + | > loss_dur: 0.23198 (0.15549) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.61206 (35.73163) + | > current_lr: 0.00006 + | > step_time: 0.34460 (3.34303) + | > loader_time: 0.00650 (0.05335) + + + --> STEP: 229/234 -- GLOBAL_STEP: 57325 + | > loss: -0.44499 (-0.29928) + | > log_mle: -0.73579 (-0.45710) + | > loss_dur: 0.29080 (0.15782) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.24445 (36.94117) + | > current_lr: 0.00006 + | > step_time: 0.24740 (3.27540) + | > loader_time: 0.00370 (0.05226) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.80182 (-0.07271) + | > avg_loss: -0.27556 (+0.00772) + | > avg_log_mle: -0.51750 (-0.00088) + | > avg_loss_dur: 0.24194 (+0.00860) + + + > EPOCH: 245/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 12:47:27)  + + --> STEP: 0/234 -- GLOBAL_STEP: 57330 + | > loss: -0.29492 (-0.29492) + | > log_mle: -0.45332 (-0.45332) + | > loss_dur: 0.15840 (0.15840) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.14918 (31.14918) + | > current_lr: 0.00006 + | > step_time: 2.00830 (2.00828) + | > loader_time: 11.59860 (11.59857) + + + --> STEP: 5/234 -- GLOBAL_STEP: 57335 + | > loss: -0.26895 (-0.26545) + | > log_mle: -0.37077 (-0.37094) + | > loss_dur: 0.10182 (0.10549) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.24605 (25.61989) + | > current_lr: 0.00006 + | > step_time: 7.31060 (6.76882) + | > loader_time: 0.00130 (0.05781) + + + --> STEP: 10/234 -- GLOBAL_STEP: 57340 + | > loss: -0.26837 (-0.27354) + | > log_mle: -0.36644 (-0.37269) + | > loss_dur: 0.09808 (0.09914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.98756 (22.52271) + | > current_lr: 0.00006 + | > step_time: 2.30340 (5.67708) + | > loader_time: 0.19940 (0.10762) + + + --> STEP: 15/234 -- GLOBAL_STEP: 57345 + | > loss: -0.31249 (-0.28462) + | > log_mle: -0.38464 (-0.37603) + | > loss_dur: 0.07215 (0.09141) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.26762 (19.73999) + | > current_lr: 0.00006 + | > step_time: 1.82930 (4.94278) + | > loader_time: 0.00100 (0.09157) + + + --> STEP: 20/234 -- GLOBAL_STEP: 57350 + | > loss: -0.31618 (-0.29062) + | > log_mle: -0.37688 (-0.37616) + | > loss_dur: 0.06070 (0.08553) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.95932 (17.27847) + | > current_lr: 0.00006 + | > step_time: 0.90200 (4.11018) + | > loader_time: 0.00290 (0.06919) + + + --> STEP: 25/234 -- GLOBAL_STEP: 57355 + | > loss: -0.28156 (-0.29162) + | > log_mle: -0.35446 (-0.37457) + | > loss_dur: 0.07290 (0.08295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.13830 (16.40415) + | > current_lr: 0.00006 + | > step_time: 3.31460 (3.92170) + | > loader_time: 0.09180 (0.06314) + + + --> STEP: 30/234 -- GLOBAL_STEP: 57360 + | > loss: -0.27556 (-0.29194) + | > log_mle: -0.35461 (-0.37337) + | > loss_dur: 0.07905 (0.08143) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.79411 (15.65335) + | > current_lr: 0.00006 + | > step_time: 8.20410 (3.88805) + | > loader_time: 0.09310 (0.06588) + + + --> STEP: 35/234 -- GLOBAL_STEP: 57365 + | > loss: -0.23885 (-0.28827) + | > log_mle: -0.34653 (-0.37069) + | > loss_dur: 0.10768 (0.08242) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.66352 (15.20385) + | > current_lr: 0.00006 + | > step_time: 1.61050 (3.66224) + | > loader_time: 0.08750 (0.06640) + + + --> STEP: 40/234 -- GLOBAL_STEP: 57370 + | > loss: -0.25296 (-0.28362) + | > log_mle: -0.34252 (-0.36781) + | > loss_dur: 0.08956 (0.08419) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.81371 (15.01062) + | > current_lr: 0.00006 + | > step_time: 1.47860 (3.64598) + | > loader_time: 0.00200 (0.06322) + + + --> STEP: 45/234 -- GLOBAL_STEP: 57375 + | > loss: -0.25160 (-0.28098) + | > log_mle: -0.36773 (-0.36610) + | > loss_dur: 0.11613 (0.08512) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.53979 (14.68820) + | > current_lr: 0.00006 + | > step_time: 4.46000 (3.54870) + | > loader_time: 0.08780 (0.05844) + + + --> STEP: 50/234 -- GLOBAL_STEP: 57380 + | > loss: -0.25299 (-0.27922) + | > log_mle: -0.34713 (-0.36488) + | > loss_dur: 0.09414 (0.08566) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 6.24310 (14.09493) + | > current_lr: 0.00006 + | > step_time: 3.50710 (3.35594) + | > loader_time: 0.00330 (0.05282) + + + --> STEP: 55/234 -- GLOBAL_STEP: 57385 + | > loss: -0.27021 (-0.27741) + | > log_mle: -0.35759 (-0.36384) + | > loss_dur: 0.08738 (0.08643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.73047 (13.61625) + | > current_lr: 0.00006 + | > step_time: 2.61230 (3.25452) + | > loader_time: 0.00350 (0.04832) + + + --> STEP: 60/234 -- GLOBAL_STEP: 57390 + | > loss: -0.24510 (-0.27495) + | > log_mle: -0.36788 (-0.36297) + | > loss_dur: 0.12278 (0.08802) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.43365 (13.37501) + | > current_lr: 0.00006 + | > step_time: 1.20010 (3.16151) + | > loader_time: 0.00340 (0.04454) + + + --> STEP: 65/234 -- GLOBAL_STEP: 57395 + | > loss: -0.25485 (-0.27215) + | > log_mle: -0.35120 (-0.36212) + | > loss_dur: 0.09635 (0.08996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.02291 (13.32947) + | > current_lr: 0.00006 + | > step_time: 1.37930 (3.06737) + | > loader_time: 0.00210 (0.04403) + + + --> STEP: 70/234 -- GLOBAL_STEP: 57400 + | > loss: -0.20732 (-0.26958) + | > log_mle: -0.33459 (-0.36085) + | > loss_dur: 0.12727 (0.09126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.67476 (13.20670) + | > current_lr: 0.00006 + | > step_time: 2.11730 (2.98821) + | > loader_time: 0.09570 (0.04345) + + + --> STEP: 75/234 -- GLOBAL_STEP: 57405 + | > loss: -0.23143 (-0.26702) + | > log_mle: -0.34972 (-0.36020) + | > loss_dur: 0.11829 (0.09318) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.92720 (13.32523) + | > current_lr: 0.00006 + | > step_time: 2.98950 (2.94186) + | > loader_time: 0.00350 (0.04080) + + + --> STEP: 80/234 -- GLOBAL_STEP: 57410 + | > loss: -0.23549 (-0.26522) + | > log_mle: -0.33855 (-0.35933) + | > loss_dur: 0.10305 (0.09412) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.27008 (13.19566) + | > current_lr: 0.00006 + | > step_time: 2.25940 (2.88267) + | > loader_time: 0.00320 (0.04191) + + + --> STEP: 85/234 -- GLOBAL_STEP: 57415 + | > loss: -0.23706 (-0.26316) + | > log_mle: -0.34291 (-0.35873) + | > loss_dur: 0.10585 (0.09557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.38062 (13.26377) + | > current_lr: 0.00006 + | > step_time: 4.19350 (2.86830) + | > loader_time: 0.00340 (0.04172) + + + --> STEP: 90/234 -- GLOBAL_STEP: 57420 + | > loss: -0.22284 (-0.26159) + | > log_mle: -0.36176 (-0.35901) + | > loss_dur: 0.13892 (0.09742) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.18142 (13.49453) + | > current_lr: 0.00006 + | > step_time: 0.80300 (2.86238) + | > loader_time: 0.00250 (0.03964) + + + --> STEP: 95/234 -- GLOBAL_STEP: 57425 + | > loss: -0.28969 (-0.26094) + | > log_mle: -0.44333 (-0.36101) + | > loss_dur: 0.15364 (0.10008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.11684 (14.00025) + | > current_lr: 0.00006 + | > step_time: 3.90570 (2.84022) + | > loader_time: 0.00200 (0.03934) + + + --> STEP: 100/234 -- GLOBAL_STEP: 57430 + | > loss: -0.24780 (-0.25997) + | > log_mle: -0.37537 (-0.36158) + | > loss_dur: 0.12756 (0.10162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.03919 (14.19088) + | > current_lr: 0.00006 + | > step_time: 2.31710 (2.79863) + | > loader_time: 0.08290 (0.04161) + + + --> STEP: 105/234 -- GLOBAL_STEP: 57435 + | > loss: -0.23052 (-0.25952) + | > log_mle: -0.35795 (-0.36333) + | > loss_dur: 0.12744 (0.10381) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.41733 (14.59638) + | > current_lr: 0.00006 + | > step_time: 1.81740 (2.78155) + | > loader_time: 0.00220 (0.04057) + + + --> STEP: 110/234 -- GLOBAL_STEP: 57440 + | > loss: -0.23093 (-0.25812) + | > log_mle: -0.37884 (-0.36466) + | > loss_dur: 0.14792 (0.10654) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.06392 (15.07662) + | > current_lr: 0.00006 + | > step_time: 1.20490 (2.77779) + | > loader_time: 0.00200 (0.03965) + + + --> STEP: 115/234 -- GLOBAL_STEP: 57445 + | > loss: -0.23495 (-0.25772) + | > log_mle: -0.40032 (-0.36691) + | > loss_dur: 0.16537 (0.10919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.85997 (15.61215) + | > current_lr: 0.00006 + | > step_time: 0.80770 (2.72521) + | > loader_time: 0.00220 (0.03883) + + + --> STEP: 120/234 -- GLOBAL_STEP: 57450 + | > loss: -0.27921 (-0.25709) + | > log_mle: -0.44619 (-0.36877) + | > loss_dur: 0.16698 (0.11169) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.57056 (16.16392) + | > current_lr: 0.00006 + | > step_time: 3.50090 (2.71442) + | > loader_time: 0.00730 (0.03887) + + + --> STEP: 125/234 -- GLOBAL_STEP: 57455 + | > loss: -0.25752 (-0.25605) + | > log_mle: -0.43403 (-0.36941) + | > loss_dur: 0.17651 (0.11336) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.91688 (16.39142) + | > current_lr: 0.00006 + | > step_time: 3.19380 (2.81203) + | > loader_time: 0.09060 (0.03902) + + + --> STEP: 130/234 -- GLOBAL_STEP: 57460 + | > loss: -0.26405 (-0.25608) + | > log_mle: -0.44238 (-0.37180) + | > loss_dur: 0.17833 (0.11572) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.08461 (17.19802) + | > current_lr: 0.00006 + | > step_time: 2.69880 (2.76780) + | > loader_time: 0.00860 (0.03836) + + + --> STEP: 135/234 -- GLOBAL_STEP: 57465 + | > loss: -0.22674 (-0.25638) + | > log_mle: -0.36946 (-0.37408) + | > loss_dur: 0.14272 (0.11770) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.38506 (18.19152) + | > current_lr: 0.00006 + | > step_time: 4.11130 (2.74815) + | > loader_time: 0.08780 (0.03831) + + + --> STEP: 140/234 -- GLOBAL_STEP: 57470 + | > loss: -0.21846 (-0.25676) + | > log_mle: -0.40509 (-0.37702) + | > loss_dur: 0.18663 (0.12026) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.11027 (18.79776) + | > current_lr: 0.00006 + | > step_time: 3.90800 (2.78856) + | > loader_time: 0.09540 (0.03912) + + + --> STEP: 145/234 -- GLOBAL_STEP: 57475 + | > loss: -0.31750 (-0.25798) + | > log_mle: -0.50708 (-0.38076) + | > loss_dur: 0.18957 (0.12277) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.12163 (19.46449) + | > current_lr: 0.00006 + | > step_time: 1.61310 (2.77175) + | > loader_time: 0.00380 (0.03842) + + + --> STEP: 150/234 -- GLOBAL_STEP: 57480 + | > loss: -0.29463 (-0.25949) + | > log_mle: -0.49669 (-0.38437) + | > loss_dur: 0.20206 (0.12488) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.99076 (20.15124) + | > current_lr: 0.00006 + | > step_time: 5.60390 (2.78065) + | > loader_time: 0.00780 (0.03904) + + + --> STEP: 155/234 -- GLOBAL_STEP: 57485 + | > loss: -0.34825 (-0.26177) + | > log_mle: -0.55461 (-0.38891) + | > loss_dur: 0.20637 (0.12714) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.86923 (21.63710) + | > current_lr: 0.00006 + | > step_time: 1.70660 (2.79813) + | > loader_time: 0.00320 (0.03905) + + + --> STEP: 160/234 -- GLOBAL_STEP: 57490 + | > loss: -0.34211 (-0.26339) + | > log_mle: -0.55494 (-0.39290) + | > loss_dur: 0.21283 (0.12951) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.50309 (22.60368) + | > current_lr: 0.00006 + | > step_time: 4.31440 (2.87268) + | > loader_time: 0.09500 (0.04155) + + + --> STEP: 165/234 -- GLOBAL_STEP: 57495 + | > loss: -0.35156 (-0.26539) + | > log_mle: -0.55277 (-0.39676) + | > loss_dur: 0.20122 (0.13137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.76186 (23.58803) + | > current_lr: 0.00006 + | > step_time: 6.59310 (2.92025) + | > loader_time: 0.11340 (0.04206) + + + --> STEP: 170/234 -- GLOBAL_STEP: 57500 + | > loss: -0.37259 (-0.26751) + | > log_mle: -0.58803 (-0.40097) + | > loss_dur: 0.21545 (0.13347) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.49230 (24.79087) + | > current_lr: 0.00006 + | > step_time: 2.09110 (2.92023) + | > loader_time: 0.00330 (0.04321) + + + --> STEP: 175/234 -- GLOBAL_STEP: 57505 + | > loss: -0.33274 (-0.27026) + | > log_mle: -0.56439 (-0.40605) + | > loss_dur: 0.23165 (0.13579) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.87982 (25.67211) + | > current_lr: 0.00006 + | > step_time: 3.89730 (2.97466) + | > loader_time: 0.00230 (0.04416) + + + --> STEP: 180/234 -- GLOBAL_STEP: 57510 + | > loss: -0.34459 (-0.27269) + | > log_mle: -0.56540 (-0.41088) + | > loss_dur: 0.22081 (0.13818) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.60653 (26.69435) + | > current_lr: 0.00006 + | > step_time: 4.59620 (3.01147) + | > loader_time: 0.00950 (0.04404) + + + --> STEP: 185/234 -- GLOBAL_STEP: 57515 + | > loss: -0.37533 (-0.27486) + | > log_mle: -0.60397 (-0.41531) + | > loss_dur: 0.22864 (0.14045) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 72.04486 (27.69045) + | > current_lr: 0.00006 + | > step_time: 6.99280 (3.04737) + | > loader_time: 0.99720 (0.04979) + + + --> STEP: 190/234 -- GLOBAL_STEP: 57520 + | > loss: -0.38175 (-0.27709) + | > log_mle: -0.58357 (-0.41966) + | > loss_dur: 0.20182 (0.14256) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.04631 (28.52737) + | > current_lr: 0.00006 + | > step_time: 2.89610 (3.05822) + | > loader_time: 0.09740 (0.05112) + + + --> STEP: 195/234 -- GLOBAL_STEP: 57525 + | > loss: -0.37506 (-0.27988) + | > log_mle: -0.59768 (-0.42428) + | > loss_dur: 0.22262 (0.14439) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.42508 (29.60016) + | > current_lr: 0.00006 + | > step_time: 4.89680 (3.09628) + | > loader_time: 0.28190 (0.05278) + + + --> STEP: 200/234 -- GLOBAL_STEP: 57530 + | > loss: -0.36128 (-0.28224) + | > log_mle: -0.60940 (-0.42857) + | > loss_dur: 0.24813 (0.14633) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.62063 (30.37237) + | > current_lr: 0.00006 + | > step_time: 3.79740 (3.15872) + | > loader_time: 0.00480 (0.05314) + + + --> STEP: 205/234 -- GLOBAL_STEP: 57535 + | > loss: -0.36678 (-0.28451) + | > log_mle: -0.58826 (-0.43275) + | > loss_dur: 0.22147 (0.14824) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.61662 (31.28510) + | > current_lr: 0.00006 + | > step_time: 2.91650 (3.16704) + | > loader_time: 0.00360 (0.05288) + + + --> STEP: 210/234 -- GLOBAL_STEP: 57540 + | > loss: -0.45154 (-0.28772) + | > log_mle: -0.68594 (-0.43785) + | > loss_dur: 0.23440 (0.15012) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 75.85616 (32.30933) + | > current_lr: 0.00006 + | > step_time: 8.19740 (3.20017) + | > loader_time: 0.09570 (0.05253) + + + --> STEP: 215/234 -- GLOBAL_STEP: 57545 + | > loss: -0.39862 (-0.29112) + | > log_mle: -0.62816 (-0.44317) + | > loss_dur: 0.22953 (0.15205) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 104.79569 (33.68176) + | > current_lr: 0.00006 + | > step_time: 7.60190 (3.26914) + | > loader_time: 0.49460 (0.05457) + + + --> STEP: 220/234 -- GLOBAL_STEP: 57550 + | > loss: -0.44869 (-0.29491) + | > log_mle: -0.68786 (-0.44890) + | > loss_dur: 0.23917 (0.15399) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.32317 (34.84230) + | > current_lr: 0.00006 + | > step_time: 2.30090 (3.27895) + | > loader_time: 0.08670 (0.05496) + + + --> STEP: 225/234 -- GLOBAL_STEP: 57555 + | > loss: -0.51169 (-0.29838) + | > log_mle: -0.74933 (-0.45422) + | > loss_dur: 0.23764 (0.15583) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 111.47677 (36.10667) + | > current_lr: 0.00006 + | > step_time: 0.23680 (3.23365) + | > loader_time: 0.00450 (0.05384) + + + --> STEP: 230/234 -- GLOBAL_STEP: 57560 + | > loss: -0.49714 (-0.30159) + | > log_mle: -0.80621 (-0.46003) + | > loss_dur: 0.30907 (0.15845) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 84.70875 (37.30954) + | > current_lr: 0.00006 + | > step_time: 0.25160 (3.16867) + | > loader_time: 0.00460 (0.05275) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.24876 (-0.55306) + | > avg_loss: -0.31145 (-0.03589) + | > avg_log_mle: -0.54019 (-0.02269) + | > avg_loss_dur: 0.22874 (-0.01320) + + + > EPOCH: 246/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 13:00:53)  + + --> STEP: 1/234 -- GLOBAL_STEP: 57565 + | > loss: -0.28200 (-0.28200) + | > log_mle: -0.36692 (-0.36692) + | > loss_dur: 0.08492 (0.08492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.14522 (26.14522) + | > current_lr: 0.00006 + | > step_time: 11.10620 (11.10616) + | > loader_time: 0.09910 (0.09911) + + + --> STEP: 6/234 -- GLOBAL_STEP: 57570 + | > loss: -0.29572 (-0.27513) + | > log_mle: -0.37151 (-0.37279) + | > loss_dur: 0.07578 (0.09766) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.50730 (22.10375) + | > current_lr: 0.00006 + | > step_time: 3.70430 (5.95227) + | > loader_time: 0.19440 (0.13011) + + + --> STEP: 11/234 -- GLOBAL_STEP: 57575 + | > loss: -0.32064 (-0.28578) + | > log_mle: -0.38671 (-0.37658) + | > loss_dur: 0.06607 (0.09081) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.96647 (19.79734) + | > current_lr: 0.00006 + | > step_time: 6.10440 (5.85743) + | > loader_time: 0.08330 (0.11054) + + + --> STEP: 16/234 -- GLOBAL_STEP: 57580 + | > loss: -0.32161 (-0.29332) + | > log_mle: -0.38688 (-0.37924) + | > loss_dur: 0.06527 (0.08592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.73455 (18.01926) + | > current_lr: 0.00006 + | > step_time: 2.90590 (4.92552) + | > loader_time: 0.08620 (0.08786) + + + --> STEP: 21/234 -- GLOBAL_STEP: 57585 + | > loss: -0.29242 (-0.29498) + | > log_mle: -0.35640 (-0.37713) + | > loss_dur: 0.06399 (0.08215) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.27104 (16.76137) + | > current_lr: 0.00006 + | > step_time: 5.30590 (4.72066) + | > loader_time: 0.00210 (0.07241) + + + --> STEP: 26/234 -- GLOBAL_STEP: 57590 + | > loss: -0.28138 (-0.29400) + | > log_mle: -0.36196 (-0.37529) + | > loss_dur: 0.08058 (0.08130) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.43538 (15.94975) + | > current_lr: 0.00006 + | > step_time: 3.30150 (4.73927) + | > loader_time: 0.07610 (0.08055) + + + --> STEP: 31/234 -- GLOBAL_STEP: 57595 + | > loss: -0.24848 (-0.29282) + | > log_mle: -0.35254 (-0.37364) + | > loss_dur: 0.10405 (0.08082) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.20827 (15.11117) + | > current_lr: 0.00006 + | > step_time: 6.80660 (4.50381) + | > loader_time: 0.09150 (0.07124) + + + --> STEP: 36/234 -- GLOBAL_STEP: 57600 + | > loss: -0.24893 (-0.28986) + | > log_mle: -0.35075 (-0.37154) + | > loss_dur: 0.10182 (0.08168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.28076 (14.66484) + | > current_lr: 0.00006 + | > step_time: 1.28570 (4.24985) + | > loader_time: 0.00130 (0.06486) + + + --> STEP: 41/234 -- GLOBAL_STEP: 57605 + | > loss: -0.29226 (-0.28764) + | > log_mle: -0.36198 (-0.36955) + | > loss_dur: 0.06972 (0.08191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.74060 (14.20308) + | > current_lr: 0.00006 + | > step_time: 2.01220 (3.90448) + | > loader_time: 0.00260 (0.05925) + + + --> STEP: 46/234 -- GLOBAL_STEP: 57610 + | > loss: -0.25515 (-0.28424) + | > log_mle: -0.35617 (-0.36798) + | > loss_dur: 0.10102 (0.08375) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.43342 (13.94051) + | > current_lr: 0.00006 + | > step_time: 1.80750 (3.74748) + | > loader_time: 0.07720 (0.05876) + + + --> STEP: 51/234 -- GLOBAL_STEP: 57615 + | > loss: -0.26460 (-0.28289) + | > log_mle: -0.35052 (-0.36678) + | > loss_dur: 0.08592 (0.08389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.03058 (13.42992) + | > current_lr: 0.00006 + | > step_time: 1.99100 (3.59524) + | > loader_time: 0.00280 (0.05332) + + + --> STEP: 56/234 -- GLOBAL_STEP: 57620 + | > loss: -0.24838 (-0.28084) + | > log_mle: -0.35295 (-0.36578) + | > loss_dur: 0.10458 (0.08494) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.36275 (13.28499) + | > current_lr: 0.00006 + | > step_time: 1.70810 (3.43796) + | > loader_time: 0.08350 (0.05325) + + + --> STEP: 61/234 -- GLOBAL_STEP: 57625 + | > loss: -0.25112 (-0.27841) + | > log_mle: -0.35217 (-0.36482) + | > loss_dur: 0.10105 (0.08640) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.40640 (13.10512) + | > current_lr: 0.00006 + | > step_time: 5.61380 (3.38801) + | > loader_time: 0.28950 (0.05682) + + + --> STEP: 66/234 -- GLOBAL_STEP: 57630 + | > loss: -0.25871 (-0.27568) + | > log_mle: -0.34550 (-0.36377) + | > loss_dur: 0.08679 (0.08809) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.08812 (13.03157) + | > current_lr: 0.00006 + | > step_time: 2.79760 (3.27016) + | > loader_time: 0.08360 (0.05512) + + + --> STEP: 71/234 -- GLOBAL_STEP: 57635 + | > loss: -0.23032 (-0.27275) + | > log_mle: -0.36968 (-0.36271) + | > loss_dur: 0.13936 (0.08996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.89666 (13.01233) + | > current_lr: 0.00006 + | > step_time: 1.51170 (3.15683) + | > loader_time: 0.00250 (0.05259) + + + --> STEP: 76/234 -- GLOBAL_STEP: 57640 + | > loss: -0.23337 (-0.26983) + | > log_mle: -0.35294 (-0.36171) + | > loss_dur: 0.11957 (0.09188) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.31488 (12.92439) + | > current_lr: 0.00006 + | > step_time: 2.36530 (3.11192) + | > loader_time: 0.00190 (0.05038) + + + --> STEP: 81/234 -- GLOBAL_STEP: 57645 + | > loss: -0.24095 (-0.26788) + | > log_mle: -0.35852 (-0.36089) + | > loss_dur: 0.11756 (0.09301) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.23879 (12.74682) + | > current_lr: 0.00006 + | > step_time: 2.18510 (3.02074) + | > loader_time: 0.10640 (0.04881) + + + --> STEP: 86/234 -- GLOBAL_STEP: 57650 + | > loss: -0.23955 (-0.26579) + | > log_mle: -0.35615 (-0.36025) + | > loss_dur: 0.11660 (0.09445) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.88263 (12.74016) + | > current_lr: 0.00006 + | > step_time: 2.39950 (2.98583) + | > loader_time: 0.00260 (0.04712) + + + --> STEP: 91/234 -- GLOBAL_STEP: 57655 + | > loss: -0.23694 (-0.26431) + | > log_mle: -0.36805 (-0.36070) + | > loss_dur: 0.13111 (0.09639) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.42849 (12.90082) + | > current_lr: 0.00006 + | > step_time: 3.20320 (2.93131) + | > loader_time: 0.00270 (0.04621) + + + --> STEP: 96/234 -- GLOBAL_STEP: 57660 + | > loss: -0.23456 (-0.26397) + | > log_mle: -0.35467 (-0.36271) + | > loss_dur: 0.12011 (0.09874) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.93069 (13.37913) + | > current_lr: 0.00006 + | > step_time: 5.91630 (2.96761) + | > loader_time: 0.08750 (0.04763) + + + --> STEP: 101/234 -- GLOBAL_STEP: 57665 + | > loss: -0.25580 (-0.26329) + | > log_mle: -0.40284 (-0.36384) + | > loss_dur: 0.14704 (0.10055) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.90440 (13.90096) + | > current_lr: 0.00006 + | > step_time: 2.28370 (2.91652) + | > loader_time: 0.00320 (0.04542) + + + --> STEP: 106/234 -- GLOBAL_STEP: 57670 + | > loss: -0.22466 (-0.26240) + | > log_mle: -0.39935 (-0.36543) + | > loss_dur: 0.17469 (0.10303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.63843 (14.51247) + | > current_lr: 0.00006 + | > step_time: 1.84860 (2.88213) + | > loader_time: 0.00200 (0.04516) + + + --> STEP: 111/234 -- GLOBAL_STEP: 57675 + | > loss: -0.26484 (-0.26137) + | > log_mle: -0.45126 (-0.36714) + | > loss_dur: 0.18642 (0.10577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.32109 (15.19526) + | > current_lr: 0.00006 + | > step_time: 1.07900 (2.83050) + | > loader_time: 0.00250 (0.04393) + + + --> STEP: 116/234 -- GLOBAL_STEP: 57680 + | > loss: -0.24029 (-0.26054) + | > log_mle: -0.41956 (-0.36898) + | > loss_dur: 0.17927 (0.10844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.01383 (15.69109) + | > current_lr: 0.00006 + | > step_time: 1.68390 (2.79530) + | > loader_time: 0.00220 (0.04293) + + + --> STEP: 121/234 -- GLOBAL_STEP: 57685 + | > loss: -0.20241 (-0.25981) + | > log_mle: -0.33624 (-0.37013) + | > loss_dur: 0.13383 (0.11033) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.19093 (15.90417) + | > current_lr: 0.00006 + | > step_time: 2.81400 (2.78324) + | > loader_time: 0.08890 (0.04274) + + + --> STEP: 126/234 -- GLOBAL_STEP: 57690 + | > loss: -0.28738 (-0.25936) + | > log_mle: -0.46671 (-0.37185) + | > loss_dur: 0.17933 (0.11248) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 41.23587 (16.33106) + | > current_lr: 0.00006 + | > step_time: 1.60850 (2.74888) + | > loader_time: 0.00250 (0.04190) + + + --> STEP: 131/234 -- GLOBAL_STEP: 57695 + | > loss: -0.33246 (-0.25999) + | > log_mle: -0.52324 (-0.37484) + | > loss_dur: 0.19078 (0.11486) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 38.43100 (16.97215) + | > current_lr: 0.00006 + | > step_time: 1.89050 (2.75553) + | > loader_time: 0.00310 (0.04232) + + + --> STEP: 136/234 -- GLOBAL_STEP: 57700 + | > loss: -0.36394 (-0.26094) + | > log_mle: -0.56704 (-0.37787) + | > loss_dur: 0.20311 (0.11693) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.21226 (17.74385) + | > current_lr: 0.00006 + | > step_time: 1.41510 (2.72355) + | > loader_time: 0.08820 (0.04211) + + + --> STEP: 141/234 -- GLOBAL_STEP: 57705 + | > loss: -0.29300 (-0.26137) + | > log_mle: -0.46238 (-0.38039) + | > loss_dur: 0.16938 (0.11902) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.57589 (18.37566) + | > current_lr: 0.00006 + | > step_time: 2.26360 (2.77716) + | > loader_time: 0.00240 (0.04146) + + + --> STEP: 146/234 -- GLOBAL_STEP: 57710 + | > loss: -0.31866 (-0.26304) + | > log_mle: -0.51452 (-0.38473) + | > loss_dur: 0.19586 (0.12168) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.80379 (19.47228) + | > current_lr: 0.00006 + | > step_time: 2.29400 (2.74346) + | > loader_time: 0.10740 (0.04089) + + + --> STEP: 151/234 -- GLOBAL_STEP: 57715 + | > loss: -0.31001 (-0.26449) + | > log_mle: -0.48233 (-0.38823) + | > loss_dur: 0.17232 (0.12374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.54317 (20.29398) + | > current_lr: 0.00006 + | > step_time: 2.30130 (2.71880) + | > loader_time: 0.10200 (0.04029) + + + --> STEP: 156/234 -- GLOBAL_STEP: 57720 + | > loss: -0.33495 (-0.26729) + | > log_mle: -0.52984 (-0.39338) + | > loss_dur: 0.19490 (0.12610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.54105 (21.55634) + | > current_lr: 0.00006 + | > step_time: 2.60250 (2.75930) + | > loader_time: 0.05850 (0.03950) + + + --> STEP: 161/234 -- GLOBAL_STEP: 57725 + | > loss: -0.37257 (-0.26920) + | > log_mle: -0.55316 (-0.39751) + | > loss_dur: 0.18059 (0.12831) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.13090 (22.57875) + | > current_lr: 0.00006 + | > step_time: 2.40420 (2.79355) + | > loader_time: 0.00270 (0.04013) + + + --> STEP: 166/234 -- GLOBAL_STEP: 57730 + | > loss: -0.30465 (-0.27066) + | > log_mle: -0.48603 (-0.40097) + | > loss_dur: 0.18138 (0.13031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.98962 (23.35587) + | > current_lr: 0.00006 + | > step_time: 3.10660 (2.77079) + | > loader_time: 0.00570 (0.04009) + + + --> STEP: 171/234 -- GLOBAL_STEP: 57735 + | > loss: -0.39445 (-0.27353) + | > log_mle: -0.59825 (-0.40616) + | > loss_dur: 0.20380 (0.13264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 74.26421 (24.46880) + | > current_lr: 0.00006 + | > step_time: 1.20460 (2.75609) + | > loader_time: 0.00420 (0.04004) + + + --> STEP: 176/234 -- GLOBAL_STEP: 57740 + | > loss: -0.37509 (-0.27637) + | > log_mle: -0.57827 (-0.41130) + | > loss_dur: 0.20318 (0.13493) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.84518 (25.44712) + | > current_lr: 0.00006 + | > step_time: 3.49790 (2.76413) + | > loader_time: 0.00730 (0.03953) + + + --> STEP: 181/234 -- GLOBAL_STEP: 57745 + | > loss: -0.30322 (-0.27858) + | > log_mle: -0.50462 (-0.41580) + | > loss_dur: 0.20140 (0.13722) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 66.55830 (26.82018) + | > current_lr: 0.00006 + | > step_time: 4.69330 (2.76738) + | > loader_time: 0.00600 (0.03859) + + + --> STEP: 186/234 -- GLOBAL_STEP: 57750 + | > loss: -0.32048 (-0.28090) + | > log_mle: -0.54988 (-0.42051) + | > loss_dur: 0.22940 (0.13960) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.43761 (28.21973) + | > current_lr: 0.00006 + | > step_time: 3.10660 (2.75583) + | > loader_time: 0.00380 (0.03769) + + + --> STEP: 191/234 -- GLOBAL_STEP: 57755 + | > loss: -0.36284 (-0.28347) + | > log_mle: -0.56488 (-0.42499) + | > loss_dur: 0.20204 (0.14152) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.50401 (29.30620) + | > current_lr: 0.00006 + | > step_time: 1.90170 (2.73961) + | > loader_time: 0.00330 (0.03814) + + + --> STEP: 196/234 -- GLOBAL_STEP: 57760 + | > loss: -0.33735 (-0.28626) + | > log_mle: -0.56347 (-0.42968) + | > loss_dur: 0.22612 (0.14342) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.94304 (30.29481) + | > current_lr: 0.00006 + | > step_time: 5.29010 (2.74523) + | > loader_time: 0.00520 (0.03882) + + + --> STEP: 201/234 -- GLOBAL_STEP: 57765 + | > loss: -0.28093 (-0.28802) + | > log_mle: -0.51359 (-0.43343) + | > loss_dur: 0.23266 (0.14541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.17254 (31.39249) + | > current_lr: 0.00006 + | > step_time: 3.21270 (2.74472) + | > loader_time: 0.09130 (0.03880) + + + --> STEP: 206/234 -- GLOBAL_STEP: 57770 + | > loss: -0.40830 (-0.29050) + | > log_mle: -0.62512 (-0.43786) + | > loss_dur: 0.21682 (0.14736) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.31696 (32.44265) + | > current_lr: 0.00006 + | > step_time: 3.58990 (2.73961) + | > loader_time: 0.11110 (0.03850) + + + --> STEP: 211/234 -- GLOBAL_STEP: 57775 + | > loss: -0.46586 (-0.29374) + | > log_mle: -0.71160 (-0.44312) + | > loss_dur: 0.24574 (0.14938) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.72333 (33.62618) + | > current_lr: 0.00006 + | > step_time: 2.60770 (2.76493) + | > loader_time: 0.00480 (0.04036) + + + --> STEP: 216/234 -- GLOBAL_STEP: 57780 + | > loss: -0.44261 (-0.29682) + | > log_mle: -0.68703 (-0.44808) + | > loss_dur: 0.24442 (0.15125) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 85.24723 (34.67332) + | > current_lr: 0.00006 + | > step_time: 5.69860 (2.76858) + | > loader_time: 0.00310 (0.04038) + + + --> STEP: 221/234 -- GLOBAL_STEP: 57785 + | > loss: -0.38632 (-0.30000) + | > log_mle: -0.60271 (-0.45314) + | > loss_dur: 0.21639 (0.15314) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.19063 (35.71018) + | > current_lr: 0.00006 + | > step_time: 3.70090 (2.84530) + | > loader_time: 0.10310 (0.04448) + + + --> STEP: 226/234 -- GLOBAL_STEP: 57790 + | > loss: -0.48020 (-0.30364) + | > log_mle: -0.72697 (-0.45885) + | > loss_dur: 0.24678 (0.15522) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 92.55173 (37.04286) + | > current_lr: 0.00006 + | > step_time: 0.23200 (2.79482) + | > loader_time: 0.00270 (0.04359) + + + --> STEP: 231/234 -- GLOBAL_STEP: 57795 + | > loss: -0.33104 (-0.30544) + | > log_mle: -0.70494 (-0.46385) + | > loss_dur: 0.37390 (0.15841) + | > amp_scaler: 1024.00000 (2025.83550) + | > grad_norm: 129.75284 (38.81955) + | > current_lr: 0.00006 + | > step_time: 0.26700 (2.73949) + | > loader_time: 0.00380 (0.04272) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00232 (-0.24644) + | > avg_loss: -0.26128 (+0.05018) + | > avg_log_mle: -0.50744 (+0.03275) + | > avg_loss_dur: 0.24617 (+0.01743) + + + > EPOCH: 247/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 13:12:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 57800 + | > loss: -0.30874 (-0.29359) + | > log_mle: -0.38870 (-0.37817) + | > loss_dur: 0.07997 (0.08458) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.62876 (15.11541) + | > current_lr: 0.00006 + | > step_time: 6.40340 (7.29935) + | > loader_time: 0.69370 (0.34958) + + + --> STEP: 7/234 -- GLOBAL_STEP: 57805 + | > loss: -0.30850 (-0.28177) + | > log_mle: -0.37304 (-0.37196) + | > loss_dur: 0.06454 (0.09019) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.18032 (17.48311) + | > current_lr: 0.00006 + | > step_time: 9.09500 (4.98592) + | > loader_time: 0.10410 (0.14246) + + + --> STEP: 12/234 -- GLOBAL_STEP: 57810 + | > loss: -0.27354 (-0.28699) + | > log_mle: -0.36981 (-0.37438) + | > loss_dur: 0.09627 (0.08740) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.16720 (18.11032) + | > current_lr: 0.00006 + | > step_time: 0.91060 (4.33174) + | > loader_time: 0.07780 (0.09806) + + + --> STEP: 17/234 -- GLOBAL_STEP: 57815 + | > loss: -0.31148 (-0.29478) + | > log_mle: -0.37524 (-0.37740) + | > loss_dur: 0.06376 (0.08263) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.78566 (16.40454) + | > current_lr: 0.00006 + | > step_time: 2.50090 (4.27090) + | > loader_time: 0.00190 (0.07999) + + + --> STEP: 22/234 -- GLOBAL_STEP: 57820 + | > loss: -0.28967 (-0.29518) + | > log_mle: -0.37597 (-0.37623) + | > loss_dur: 0.08630 (0.08105) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.15078 (15.40913) + | > current_lr: 0.00006 + | > step_time: 1.02270 (3.86155) + | > loader_time: 0.07430 (0.07417) + + + --> STEP: 27/234 -- GLOBAL_STEP: 57825 + | > loss: -0.28918 (-0.29523) + | > log_mle: -0.36784 (-0.37522) + | > loss_dur: 0.07866 (0.07999) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.54317 (14.47699) + | > current_lr: 0.00006 + | > step_time: 2.18270 (3.46998) + | > loader_time: 0.00230 (0.06081) + + + --> STEP: 32/234 -- GLOBAL_STEP: 57830 + | > loss: -0.29604 (-0.29492) + | > log_mle: -0.37090 (-0.37422) + | > loss_dur: 0.07487 (0.07931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.10408 (13.89265) + | > current_lr: 0.00006 + | > step_time: 1.78560 (3.30671) + | > loader_time: 0.00140 (0.05767) + + + --> STEP: 37/234 -- GLOBAL_STEP: 57835 + | > loss: -0.25758 (-0.29027) + | > log_mle: -0.34478 (-0.37146) + | > loss_dur: 0.08720 (0.08119) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.50165 (13.53173) + | > current_lr: 0.00006 + | > step_time: 1.59950 (3.07411) + | > loader_time: 0.00230 (0.05515) + + + --> STEP: 42/234 -- GLOBAL_STEP: 57840 + | > loss: -0.24744 (-0.28758) + | > log_mle: -0.34335 (-0.36965) + | > loss_dur: 0.09591 (0.08208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.76277 (13.18686) + | > current_lr: 0.00006 + | > step_time: 2.11150 (2.96225) + | > loader_time: 0.00330 (0.05086) + + + --> STEP: 47/234 -- GLOBAL_STEP: 57845 + | > loss: -0.26086 (-0.28453) + | > log_mle: -0.35940 (-0.36874) + | > loss_dur: 0.09854 (0.08421) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.17783 (12.95554) + | > current_lr: 0.00006 + | > step_time: 1.89720 (2.81938) + | > loader_time: 0.00170 (0.04568) + + + --> STEP: 52/234 -- GLOBAL_STEP: 57850 + | > loss: -0.24237 (-0.28291) + | > log_mle: -0.35005 (-0.36750) + | > loss_dur: 0.10768 (0.08459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.40667 (12.54047) + | > current_lr: 0.00006 + | > step_time: 2.01420 (2.74833) + | > loader_time: 0.00240 (0.04166) + + + --> STEP: 57/234 -- GLOBAL_STEP: 57855 + | > loss: -0.23927 (-0.28083) + | > log_mle: -0.33752 (-0.36639) + | > loss_dur: 0.09825 (0.08556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.92354 (12.48227) + | > current_lr: 0.00006 + | > step_time: 4.50260 (2.66834) + | > loader_time: 0.00190 (0.03820) + + + --> STEP: 62/234 -- GLOBAL_STEP: 57860 + | > loss: -0.21254 (-0.27816) + | > log_mle: -0.37199 (-0.36607) + | > loss_dur: 0.15945 (0.08791) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.31191 (12.60097) + | > current_lr: 0.00006 + | > step_time: 3.50660 (2.61030) + | > loader_time: 0.00320 (0.03935) + + + --> STEP: 67/234 -- GLOBAL_STEP: 57865 + | > loss: -0.25043 (-0.27616) + | > log_mle: -0.36457 (-0.36501) + | > loss_dur: 0.11414 (0.08885) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.36274 (12.45098) + | > current_lr: 0.00006 + | > step_time: 4.61770 (2.64140) + | > loader_time: 0.00610 (0.03931) + + + --> STEP: 72/234 -- GLOBAL_STEP: 57870 + | > loss: -0.25680 (-0.27332) + | > log_mle: -0.35032 (-0.36393) + | > loss_dur: 0.09352 (0.09061) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.77903 (12.42678) + | > current_lr: 0.00006 + | > step_time: 2.72310 (2.63464) + | > loader_time: 0.07590 (0.03896) + + + --> STEP: 77/234 -- GLOBAL_STEP: 57875 + | > loss: -0.25799 (-0.27080) + | > log_mle: -0.35548 (-0.36324) + | > loss_dur: 0.09749 (0.09244) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.60618 (12.52710) + | > current_lr: 0.00006 + | > step_time: 2.13330 (2.57294) + | > loader_time: 0.00130 (0.03659) + + + --> STEP: 82/234 -- GLOBAL_STEP: 57880 + | > loss: -0.23230 (-0.26883) + | > log_mle: -0.34768 (-0.36239) + | > loss_dur: 0.11538 (0.09356) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.52939 (12.45305) + | > current_lr: 0.00006 + | > step_time: 1.99550 (2.53982) + | > loader_time: 0.00250 (0.03668) + + + --> STEP: 87/234 -- GLOBAL_STEP: 57885 + | > loss: -0.22958 (-0.26669) + | > log_mle: -0.34904 (-0.36180) + | > loss_dur: 0.11946 (0.09512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.61318 (12.51956) + | > current_lr: 0.00006 + | > step_time: 2.69900 (2.51642) + | > loader_time: 0.00600 (0.03569) + + + --> STEP: 92/234 -- GLOBAL_STEP: 57890 + | > loss: -0.25675 (-0.26526) + | > log_mle: -0.39329 (-0.36274) + | > loss_dur: 0.13654 (0.09748) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.52310 (12.78988) + | > current_lr: 0.00006 + | > step_time: 4.49280 (2.50649) + | > loader_time: 0.09870 (0.03579) + + + --> STEP: 97/234 -- GLOBAL_STEP: 57895 + | > loss: -0.24218 (-0.26465) + | > log_mle: -0.37521 (-0.36454) + | > loss_dur: 0.13303 (0.09989) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.62234 (13.35055) + | > current_lr: 0.00006 + | > step_time: 1.93890 (2.49074) + | > loader_time: 0.00900 (0.03417) + + + --> STEP: 102/234 -- GLOBAL_STEP: 57900 + | > loss: -0.21423 (-0.26346) + | > log_mle: -0.35904 (-0.36550) + | > loss_dur: 0.14481 (0.10205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.39291 (13.76874) + | > current_lr: 0.00006 + | > step_time: 1.49670 (2.43640) + | > loader_time: 0.00260 (0.03260) + + + --> STEP: 107/234 -- GLOBAL_STEP: 57905 + | > loss: -0.24235 (-0.26281) + | > log_mle: -0.39911 (-0.36744) + | > loss_dur: 0.15677 (0.10463) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.58620 (14.53724) + | > current_lr: 0.00006 + | > step_time: 1.28410 (2.40059) + | > loader_time: 0.00190 (0.03211) + + + --> STEP: 112/234 -- GLOBAL_STEP: 57910 + | > loss: -0.23989 (-0.26171) + | > log_mle: -0.41210 (-0.36916) + | > loss_dur: 0.17221 (0.10745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.26321 (15.23303) + | > current_lr: 0.00006 + | > step_time: 1.91640 (2.35873) + | > loader_time: 0.09720 (0.03235) + + + --> STEP: 117/234 -- GLOBAL_STEP: 57915 + | > loss: -0.25122 (-0.26098) + | > log_mle: -0.40972 (-0.37089) + | > loss_dur: 0.15851 (0.10991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.62897 (15.66445) + | > current_lr: 0.00006 + | > step_time: 2.99690 (2.35146) + | > loader_time: 0.08960 (0.03260) + + + --> STEP: 122/234 -- GLOBAL_STEP: 57920 + | > loss: -0.22952 (-0.26006) + | > log_mle: -0.37853 (-0.37169) + | > loss_dur: 0.14901 (0.11163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.19552 (16.00499) + | > current_lr: 0.00006 + | > step_time: 1.70700 (2.34452) + | > loader_time: 0.07740 (0.03357) + + + --> STEP: 127/234 -- GLOBAL_STEP: 57925 + | > loss: -0.25928 (-0.25993) + | > log_mle: -0.44006 (-0.37380) + | > loss_dur: 0.18078 (0.11387) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.38096 (16.53468) + | > current_lr: 0.00006 + | > step_time: 3.70980 (2.40972) + | > loader_time: 0.00400 (0.03388) + + + --> STEP: 132/234 -- GLOBAL_STEP: 57930 + | > loss: -0.26446 (-0.26044) + | > log_mle: -0.42152 (-0.37646) + | > loss_dur: 0.15706 (0.11602) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.23183 (17.12849) + | > current_lr: 0.00006 + | > step_time: 1.71140 (2.38844) + | > loader_time: 0.08220 (0.03580) + + + --> STEP: 137/234 -- GLOBAL_STEP: 57935 + | > loss: -0.24967 (-0.26117) + | > log_mle: -0.44013 (-0.37958) + | > loss_dur: 0.19046 (0.11841) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.03502 (17.83829) + | > current_lr: 0.00006 + | > step_time: 1.49590 (2.43055) + | > loader_time: 0.00320 (0.03640) + + + --> STEP: 142/234 -- GLOBAL_STEP: 57940 + | > loss: -0.26701 (-0.26151) + | > log_mle: -0.45219 (-0.38212) + | > loss_dur: 0.18518 (0.12061) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.10981 (18.46067) + | > current_lr: 0.00006 + | > step_time: 2.81180 (2.42511) + | > loader_time: 0.00270 (0.03642) + + + --> STEP: 147/234 -- GLOBAL_STEP: 57945 + | > loss: -0.26850 (-0.26327) + | > log_mle: -0.45091 (-0.38638) + | > loss_dur: 0.18242 (0.12311) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.75969 (19.47317) + | > current_lr: 0.00006 + | > step_time: 2.00840 (2.41861) + | > loader_time: 0.08390 (0.03649) + + + --> STEP: 152/234 -- GLOBAL_STEP: 57950 + | > loss: -0.33575 (-0.26520) + | > log_mle: -0.54149 (-0.39039) + | > loss_dur: 0.20574 (0.12519) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.49183 (20.34553) + | > current_lr: 0.00006 + | > step_time: 1.20810 (2.42724) + | > loader_time: 0.09240 (0.03718) + + + --> STEP: 157/234 -- GLOBAL_STEP: 57955 + | > loss: -0.29541 (-0.26764) + | > log_mle: -0.48663 (-0.39506) + | > loss_dur: 0.19122 (0.12742) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.43243 (21.35326) + | > current_lr: 0.00006 + | > step_time: 2.49980 (2.41875) + | > loader_time: 0.07890 (0.03768) + + + --> STEP: 162/234 -- GLOBAL_STEP: 57960 + | > loss: -0.32804 (-0.26984) + | > log_mle: -0.51870 (-0.39938) + | > loss_dur: 0.19066 (0.12955) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.83141 (22.33253) + | > current_lr: 0.00006 + | > step_time: 1.50230 (2.40461) + | > loader_time: 0.00300 (0.03664) + + + --> STEP: 167/234 -- GLOBAL_STEP: 57965 + | > loss: -0.38618 (-0.27152) + | > log_mle: -0.59912 (-0.40326) + | > loss_dur: 0.21293 (0.13174) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.48031 (23.57697) + | > current_lr: 0.00006 + | > step_time: 4.89780 (2.42604) + | > loader_time: 0.08790 (0.03789) + + + --> STEP: 172/234 -- GLOBAL_STEP: 57970 + | > loss: -0.38898 (-0.27398) + | > log_mle: -0.60317 (-0.40813) + | > loss_dur: 0.21419 (0.13415) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.11787 (24.77437) + | > current_lr: 0.00006 + | > step_time: 1.80250 (2.50398) + | > loader_time: 0.00380 (0.03894) + + + --> STEP: 177/234 -- GLOBAL_STEP: 57975 + | > loss: -0.33343 (-0.27625) + | > log_mle: -0.54592 (-0.41271) + | > loss_dur: 0.21249 (0.13646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.25082 (25.89188) + | > current_lr: 0.00006 + | > step_time: 1.69960 (2.49924) + | > loader_time: 0.00440 (0.03793) + + + --> STEP: 182/234 -- GLOBAL_STEP: 57980 + | > loss: -0.35387 (-0.27846) + | > log_mle: -0.59640 (-0.41732) + | > loss_dur: 0.24254 (0.13886) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.96842 (26.90522) + | > current_lr: 0.00006 + | > step_time: 2.49360 (2.58988) + | > loader_time: 0.00770 (0.03918) + + + --> STEP: 187/234 -- GLOBAL_STEP: 57985 + | > loss: -0.36397 (-0.28070) + | > log_mle: -0.59008 (-0.42179) + | > loss_dur: 0.22611 (0.14109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.29703 (27.86035) + | > current_lr: 0.00006 + | > step_time: 4.19820 (2.59547) + | > loader_time: 0.00260 (0.04022) + + + --> STEP: 192/234 -- GLOBAL_STEP: 57990 + | > loss: -0.42067 (-0.28336) + | > log_mle: -0.62580 (-0.42623) + | > loss_dur: 0.20513 (0.14287) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.60315 (28.80206) + | > current_lr: 0.00006 + | > step_time: 6.38660 (2.68765) + | > loader_time: 0.30230 (0.04189) + + + --> STEP: 197/234 -- GLOBAL_STEP: 57995 + | > loss: -0.38921 (-0.28585) + | > log_mle: -0.59048 (-0.43056) + | > loss_dur: 0.20127 (0.14472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.63086 (29.90071) + | > current_lr: 0.00006 + | > step_time: 2.80970 (2.74634) + | > loader_time: 0.08570 (0.04185) + + + --> STEP: 202/234 -- GLOBAL_STEP: 58000 + | > loss: -0.47371 (-0.28829) + | > log_mle: -0.68934 (-0.43496) + | > loss_dur: 0.21563 (0.14667) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.00675 (30.95245) + | > current_lr: 0.00006 + | > step_time: 4.49640 (2.78321) + | > loader_time: 0.08640 (0.04282) + + + --> STEP: 207/234 -- GLOBAL_STEP: 58005 + | > loss: -0.44814 (-0.29073) + | > log_mle: -0.67932 (-0.43935) + | > loss_dur: 0.23117 (0.14862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.81752 (32.02044) + | > current_lr: 0.00006 + | > step_time: 8.59990 (2.87731) + | > loader_time: 1.70320 (0.05055) + + + --> STEP: 212/234 -- GLOBAL_STEP: 58010 + | > loss: -0.40495 (-0.29369) + | > log_mle: -0.64258 (-0.44432) + | > loss_dur: 0.23763 (0.15063) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 101.50482 (33.73075) + | > current_lr: 0.00006 + | > step_time: 4.51030 (2.89735) + | > loader_time: 0.08900 (0.05152) + + + --> STEP: 217/234 -- GLOBAL_STEP: 58015 + | > loss: -0.43702 (-0.29687) + | > log_mle: -0.67878 (-0.44937) + | > loss_dur: 0.24177 (0.15250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.43493 (34.79089) + | > current_lr: 0.00006 + | > step_time: 2.30800 (2.95082) + | > loader_time: 0.00710 (0.05262) + + + --> STEP: 222/234 -- GLOBAL_STEP: 58020 + | > loss: -0.42094 (-0.29997) + | > log_mle: -0.68442 (-0.45434) + | > loss_dur: 0.26348 (0.15438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.87559 (35.83064) + | > current_lr: 0.00006 + | > step_time: 5.26810 (2.96797) + | > loader_time: 0.01280 (0.05157) + + + --> STEP: 227/234 -- GLOBAL_STEP: 58025 + | > loss: -0.38091 (-0.30313) + | > log_mle: -0.64965 (-0.45951) + | > loss_dur: 0.26874 (0.15639) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.09737 (36.93903) + | > current_lr: 0.00006 + | > step_time: 2.60950 (2.95555) + | > loader_time: 0.00370 (0.08045) + + + --> STEP: 232/234 -- GLOBAL_STEP: 58030 + | > loss: -0.39995 (-0.30580) + | > log_mle: -0.87398 (-0.46623) + | > loss_dur: 0.47402 (0.16043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 132.73163 (38.22682) + | > current_lr: 0.00006 + | > step_time: 0.39390 (2.91586) + | > loader_time: 0.10910 (0.07995) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00254 (+0.00023) + | > avg_loss: -0.32979 (-0.06851) + | > avg_log_mle: -0.54494 (-0.03750) + | > avg_loss_dur: 0.21516 (-0.03101) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_58032.pth + + > EPOCH: 248/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 13:25:25)  + + --> STEP: 3/234 -- GLOBAL_STEP: 58035 + | > loss: -0.17985 (-0.25363) + | > log_mle: -0.36571 (-0.37612) + | > loss_dur: 0.18586 (0.12249) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.25789 (21.14066) + | > current_lr: 0.00006 + | > step_time: 6.29330 (4.66432) + | > loader_time: 0.00110 (2.16582) + + + --> STEP: 8/234 -- GLOBAL_STEP: 58040 + | > loss: -0.30594 (-0.27994) + | > log_mle: -0.39416 (-0.37885) + | > loss_dur: 0.08821 (0.09892) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.13705 (19.85958) + | > current_lr: 0.00006 + | > step_time: 5.19840 (5.44762) + | > loader_time: 0.19200 (0.87271) + + + --> STEP: 13/234 -- GLOBAL_STEP: 58045 + | > loss: -0.32431 (-0.28594) + | > log_mle: -0.39692 (-0.38082) + | > loss_dur: 0.07260 (0.09488) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42742 (18.57507) + | > current_lr: 0.00006 + | > step_time: 2.21190 (4.31595) + | > loader_time: 0.18360 (0.57314) + + + --> STEP: 18/234 -- GLOBAL_STEP: 58050 + | > loss: -0.28971 (-0.29217) + | > log_mle: -0.37039 (-0.38130) + | > loss_dur: 0.08068 (0.08913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.86260 (17.00722) + | > current_lr: 0.00006 + | > step_time: 4.69490 (3.84020) + | > loader_time: 0.00200 (0.41452) + + + --> STEP: 23/234 -- GLOBAL_STEP: 58055 + | > loss: -0.32439 (-0.29485) + | > log_mle: -0.39340 (-0.38080) + | > loss_dur: 0.06901 (0.08594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.92144 (15.75601) + | > current_lr: 0.00006 + | > step_time: 2.10660 (4.09730) + | > loader_time: 0.00170 (0.33709) + + + --> STEP: 28/234 -- GLOBAL_STEP: 58060 + | > loss: -0.34751 (-0.29658) + | > log_mle: -0.40126 (-0.37986) + | > loss_dur: 0.05376 (0.08328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.35474 (14.89746) + | > current_lr: 0.00006 + | > step_time: 1.00130 (4.21279) + | > loader_time: 0.00380 (0.29063) + + + --> STEP: 33/234 -- GLOBAL_STEP: 58065 + | > loss: -0.29288 (-0.29412) + | > log_mle: -0.36616 (-0.37731) + | > loss_dur: 0.07328 (0.08319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.57254 (14.47081) + | > current_lr: 0.00006 + | > step_time: 1.37730 (3.79153) + | > loader_time: 0.00150 (0.24687) + + + --> STEP: 38/234 -- GLOBAL_STEP: 58070 + | > loss: -0.26749 (-0.29089) + | > log_mle: -0.36590 (-0.37466) + | > loss_dur: 0.09842 (0.08377) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.46769 (14.24462) + | > current_lr: 0.00006 + | > step_time: 3.10570 (3.54158) + | > loader_time: 0.00350 (0.21687) + + + --> STEP: 43/234 -- GLOBAL_STEP: 58075 + | > loss: -0.25666 (-0.28747) + | > log_mle: -0.35989 (-0.37228) + | > loss_dur: 0.10323 (0.08480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.49963 (13.99636) + | > current_lr: 0.00006 + | > step_time: 1.23320 (3.24731) + | > loader_time: 0.00170 (0.19410) + + + --> STEP: 48/234 -- GLOBAL_STEP: 58080 + | > loss: -0.29066 (-0.28532) + | > log_mle: -0.36144 (-0.37095) + | > loss_dur: 0.07078 (0.08563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.34887 (13.69700) + | > current_lr: 0.00006 + | > step_time: 0.89600 (3.03418) + | > loader_time: 0.00180 (0.17409) + + + --> STEP: 53/234 -- GLOBAL_STEP: 58085 + | > loss: -0.24814 (-0.28266) + | > log_mle: -0.35303 (-0.36907) + | > loss_dur: 0.10489 (0.08641) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.55984 (13.62783) + | > current_lr: 0.00006 + | > step_time: 2.20160 (2.89412) + | > loader_time: 0.07650 (0.16087) + + + --> STEP: 58/234 -- GLOBAL_STEP: 58090 + | > loss: -0.26853 (-0.28050) + | > log_mle: -0.35747 (-0.36754) + | > loss_dur: 0.08894 (0.08704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.15848 (13.34821) + | > current_lr: 0.00006 + | > step_time: 4.50910 (2.83288) + | > loader_time: 0.00290 (0.15004) + + + --> STEP: 63/234 -- GLOBAL_STEP: 58095 + | > loss: -0.23958 (-0.27727) + | > log_mle: -0.34540 (-0.36679) + | > loss_dur: 0.10582 (0.08952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.32272 (13.40497) + | > current_lr: 0.00006 + | > step_time: 1.72510 (2.79085) + | > loader_time: 0.00180 (0.13979) + + + --> STEP: 68/234 -- GLOBAL_STEP: 58100 + | > loss: -0.22012 (-0.27490) + | > log_mle: -0.33810 (-0.36537) + | > loss_dur: 0.11798 (0.09046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.19395 (13.35131) + | > current_lr: 0.00006 + | > step_time: 1.85130 (2.72587) + | > loader_time: 0.00230 (0.13262) + + + --> STEP: 73/234 -- GLOBAL_STEP: 58105 + | > loss: -0.21382 (-0.27147) + | > log_mle: -0.35206 (-0.36408) + | > loss_dur: 0.13824 (0.09261) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.30854 (13.54146) + | > current_lr: 0.00006 + | > step_time: 1.28780 (2.64727) + | > loader_time: 0.00400 (0.12610) + + + --> STEP: 78/234 -- GLOBAL_STEP: 58110 + | > loss: -0.22451 (-0.26900) + | > log_mle: -0.33826 (-0.36298) + | > loss_dur: 0.11375 (0.09398) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.80737 (13.56132) + | > current_lr: 0.00006 + | > step_time: 2.15480 (2.64010) + | > loader_time: 0.00220 (0.12063) + + + --> STEP: 83/234 -- GLOBAL_STEP: 58115 + | > loss: -0.20995 (-0.26701) + | > log_mle: -0.35391 (-0.36230) + | > loss_dur: 0.14396 (0.09529) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.49710 (13.50165) + | > current_lr: 0.00006 + | > step_time: 2.20080 (2.58486) + | > loader_time: 0.00230 (0.11352) + + + --> STEP: 88/234 -- GLOBAL_STEP: 58120 + | > loss: -0.24129 (-0.26558) + | > log_mle: -0.38704 (-0.36213) + | > loss_dur: 0.14575 (0.09654) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.08347 (13.45873) + | > current_lr: 0.00006 + | > step_time: 1.51280 (2.54133) + | > loader_time: 0.08810 (0.11248) + + + --> STEP: 93/234 -- GLOBAL_STEP: 58125 + | > loss: -0.24747 (-0.26449) + | > log_mle: -0.40077 (-0.36309) + | > loss_dur: 0.15330 (0.09860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.03320 (13.66796) + | > current_lr: 0.00006 + | > step_time: 1.89290 (2.55205) + | > loader_time: 0.00220 (0.10749) + + + --> STEP: 98/234 -- GLOBAL_STEP: 58130 + | > loss: -0.23140 (-0.26369) + | > log_mle: -0.33853 (-0.36410) + | > loss_dur: 0.10713 (0.10040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.39299 (13.95669) + | > current_lr: 0.00006 + | > step_time: 1.59890 (2.50650) + | > loader_time: 0.07770 (0.10364) + + + --> STEP: 103/234 -- GLOBAL_STEP: 58135 + | > loss: -0.26769 (-0.26308) + | > log_mle: -0.43149 (-0.36598) + | > loss_dur: 0.16380 (0.10291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.47079 (14.43243) + | > current_lr: 0.00006 + | > step_time: 4.30540 (2.50242) + | > loader_time: 0.09430 (0.10044) + + + --> STEP: 108/234 -- GLOBAL_STEP: 58140 + | > loss: -0.24612 (-0.26229) + | > log_mle: -0.37300 (-0.36738) + | > loss_dur: 0.12688 (0.10509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.05379 (14.90067) + | > current_lr: 0.00006 + | > step_time: 2.79230 (2.52194) + | > loader_time: 0.00360 (0.09749) + + + --> STEP: 113/234 -- GLOBAL_STEP: 58145 + | > loss: -0.25520 (-0.26127) + | > log_mle: -0.41523 (-0.36920) + | > loss_dur: 0.16004 (0.10794) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.50085 (15.60140) + | > current_lr: 0.00006 + | > step_time: 9.10190 (2.71237) + | > loader_time: 0.00280 (0.09729) + + + --> STEP: 118/234 -- GLOBAL_STEP: 58150 + | > loss: -0.22215 (-0.26015) + | > log_mle: -0.38756 (-0.37052) + | > loss_dur: 0.16541 (0.11038) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.63874 (15.96342) + | > current_lr: 0.00006 + | > step_time: 3.60070 (2.69049) + | > loader_time: 0.09030 (0.09592) + + + --> STEP: 123/234 -- GLOBAL_STEP: 58155 + | > loss: -0.20500 (-0.25890) + | > log_mle: -0.35559 (-0.37088) + | > loss_dur: 0.15059 (0.11198) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.25711 (16.34997) + | > current_lr: 0.00006 + | > step_time: 1.46100 (2.70276) + | > loader_time: 0.00230 (0.09534) + + + --> STEP: 128/234 -- GLOBAL_STEP: 58160 + | > loss: -0.26664 (-0.25911) + | > log_mle: -0.41441 (-0.37326) + | > loss_dur: 0.14777 (0.11414) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.77862 (16.90096) + | > current_lr: 0.00006 + | > step_time: 1.69790 (2.64946) + | > loader_time: 0.00390 (0.09173) + + + --> STEP: 133/234 -- GLOBAL_STEP: 58165 + | > loss: -0.27179 (-0.25952) + | > log_mle: -0.44313 (-0.37602) + | > loss_dur: 0.17135 (0.11650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.87078 (17.62831) + | > current_lr: 0.00006 + | > step_time: 1.61400 (2.69345) + | > loader_time: 0.08070 (0.08973) + + + --> STEP: 138/234 -- GLOBAL_STEP: 58170 + | > loss: -0.22580 (-0.25984) + | > log_mle: -0.39219 (-0.37859) + | > loss_dur: 0.16638 (0.11875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.73834 (18.18782) + | > current_lr: 0.00006 + | > step_time: 6.50780 (2.71334) + | > loader_time: 0.08270 (0.08780) + + + --> STEP: 143/234 -- GLOBAL_STEP: 58175 + | > loss: -0.32057 (-0.26070) + | > log_mle: -0.54365 (-0.38208) + | > loss_dur: 0.22308 (0.12138) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.09290 (18.95360) + | > current_lr: 0.00006 + | > step_time: 1.20170 (2.73597) + | > loader_time: 0.08540 (0.08867) + + + --> STEP: 148/234 -- GLOBAL_STEP: 58180 + | > loss: -0.29901 (-0.26200) + | > log_mle: -0.44997 (-0.38547) + | > loss_dur: 0.15096 (0.12347) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.93546 (19.84478) + | > current_lr: 0.00006 + | > step_time: 2.52150 (2.74028) + | > loader_time: 0.08070 (0.08690) + + + --> STEP: 153/234 -- GLOBAL_STEP: 58185 + | > loss: -0.38660 (-0.26438) + | > log_mle: -0.57984 (-0.39011) + | > loss_dur: 0.19324 (0.12573) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.99507 (20.97500) + | > current_lr: 0.00006 + | > step_time: 2.59500 (2.70885) + | > loader_time: 0.00360 (0.08472) + + + --> STEP: 158/234 -- GLOBAL_STEP: 58190 + | > loss: -0.30405 (-0.26622) + | > log_mle: -0.51742 (-0.39419) + | > loss_dur: 0.21337 (0.12797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.73224 (21.80585) + | > current_lr: 0.00006 + | > step_time: 1.23080 (2.69087) + | > loader_time: 0.00250 (0.08315) + + + --> STEP: 163/234 -- GLOBAL_STEP: 58195 + | > loss: -0.29453 (-0.26817) + | > log_mle: -0.48785 (-0.39820) + | > loss_dur: 0.19333 (0.13003) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.02979 (22.65614) + | > current_lr: 0.00006 + | > step_time: 1.90420 (2.68573) + | > loader_time: 0.08300 (0.08217) + + + --> STEP: 168/234 -- GLOBAL_STEP: 58200 + | > loss: -0.33267 (-0.27051) + | > log_mle: -0.54627 (-0.40257) + | > loss_dur: 0.21360 (0.13207) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.70014 (23.54984) + | > current_lr: 0.00006 + | > step_time: 2.49990 (2.67195) + | > loader_time: 0.00410 (0.08089) + + + --> STEP: 173/234 -- GLOBAL_STEP: 58205 + | > loss: -0.34003 (-0.27296) + | > log_mle: -0.55210 (-0.40741) + | > loss_dur: 0.21208 (0.13445) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.86926 (24.80300) + | > current_lr: 0.00006 + | > step_time: 2.39720 (2.73578) + | > loader_time: 0.00370 (0.08377) + + + --> STEP: 178/234 -- GLOBAL_STEP: 58210 + | > loss: -0.36959 (-0.27545) + | > log_mle: -0.60772 (-0.41230) + | > loss_dur: 0.23813 (0.13685) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.70387 (25.79934) + | > current_lr: 0.00006 + | > step_time: 1.58910 (2.77741) + | > loader_time: 0.00180 (0.08468) + + + --> STEP: 183/234 -- GLOBAL_STEP: 58215 + | > loss: -0.39773 (-0.27764) + | > log_mle: -0.60876 (-0.41687) + | > loss_dur: 0.21102 (0.13923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.37267 (26.73410) + | > current_lr: 0.00006 + | > step_time: 4.80540 (2.85516) + | > loader_time: 0.29600 (0.08559) + + + --> STEP: 188/234 -- GLOBAL_STEP: 58220 + | > loss: -0.40606 (-0.28001) + | > log_mle: -0.62370 (-0.42141) + | > loss_dur: 0.21764 (0.14139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.69115 (27.60027) + | > current_lr: 0.00006 + | > step_time: 7.09840 (2.95695) + | > loader_time: 0.11200 (0.08967) + + + --> STEP: 193/234 -- GLOBAL_STEP: 58225 + | > loss: -0.40764 (-0.28244) + | > log_mle: -0.61964 (-0.42566) + | > loss_dur: 0.21199 (0.14323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.66289 (28.66813) + | > current_lr: 0.00006 + | > step_time: 8.59900 (3.06067) + | > loader_time: 0.00780 (0.08945) + + + --> STEP: 198/234 -- GLOBAL_STEP: 58230 + | > loss: -0.37973 (-0.28471) + | > log_mle: -0.60041 (-0.42975) + | > loss_dur: 0.22068 (0.14503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.81018 (29.63249) + | > current_lr: 0.00006 + | > step_time: 1.49290 (3.06624) + | > loader_time: 0.00510 (0.08915) + + + --> STEP: 203/234 -- GLOBAL_STEP: 58235 + | > loss: -0.31618 (-0.28660) + | > log_mle: -0.53750 (-0.43357) + | > loss_dur: 0.22132 (0.14697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.50167 (30.39984) + | > current_lr: 0.00006 + | > step_time: 3.18510 (3.09952) + | > loader_time: 0.00900 (0.08888) + + + --> STEP: 208/234 -- GLOBAL_STEP: 58240 + | > loss: -0.38541 (-0.28933) + | > log_mle: -0.62699 (-0.43845) + | > loss_dur: 0.24158 (0.14911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.59928 (31.28063) + | > current_lr: 0.00006 + | > step_time: 2.49440 (3.14283) + | > loader_time: 0.00620 (0.08829) + + + --> STEP: 213/234 -- GLOBAL_STEP: 58245 + | > loss: -0.44425 (-0.29270) + | > log_mle: -0.68315 (-0.44381) + | > loss_dur: 0.23890 (0.15111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.99522 (32.47577) + | > current_lr: 0.00006 + | > step_time: 3.70130 (3.19641) + | > loader_time: 0.19250 (0.08983) + + + --> STEP: 218/234 -- GLOBAL_STEP: 58250 + | > loss: -0.42130 (-0.29581) + | > log_mle: -0.65266 (-0.44873) + | > loss_dur: 0.23136 (0.15292) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.43110 (33.67871) + | > current_lr: 0.00006 + | > step_time: 6.80220 (3.22872) + | > loader_time: 0.08860 (0.09077) + + + --> STEP: 223/234 -- GLOBAL_STEP: 58255 + | > loss: -0.45491 (-0.29907) + | > log_mle: -0.68650 (-0.45388) + | > loss_dur: 0.23159 (0.15481) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.54217 (35.03228) + | > current_lr: 0.00006 + | > step_time: 0.23550 (3.20657) + | > loader_time: 0.00410 (0.08918) + + + --> STEP: 228/234 -- GLOBAL_STEP: 58260 + | > loss: -0.40399 (-0.30238) + | > log_mle: -0.67407 (-0.45930) + | > loss_dur: 0.27008 (0.15692) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 124.26751 (36.16466) + | > current_lr: 0.00006 + | > step_time: 0.24560 (3.14172) + | > loader_time: 0.00380 (0.08731) + + + --> STEP: 233/234 -- GLOBAL_STEP: 58265 + | > loss: -0.05487 (-0.30335) + | > log_mle: -0.64646 (-0.46566) + | > loss_dur: 0.59159 (0.16231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.54500 (37.70889) + | > current_lr: 0.00006 + | > step_time: 0.20180 (3.08017) + | > loader_time: 0.00270 (0.08562) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.30304 (+0.30049) + | > avg_loss: -0.29786 (+0.03192) + | > avg_log_mle: -0.53427 (+0.01067) + | > avg_loss_dur: 0.23641 (+0.02125) + + + > EPOCH: 249/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 13:38:45)  + + --> STEP: 4/234 -- GLOBAL_STEP: 58270 + | > loss: -0.27620 (-0.26465) + | > log_mle: -0.38125 (-0.37831) + | > loss_dur: 0.10505 (0.11367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.18376 (19.61855) + | > current_lr: 0.00006 + | > step_time: 2.89690 (2.32793) + | > loader_time: 0.09730 (0.04716) + + + --> STEP: 9/234 -- GLOBAL_STEP: 58275 + | > loss: -0.26867 (-0.28024) + | > log_mle: -0.38868 (-0.38166) + | > loss_dur: 0.12001 (0.10142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.63703 (17.90739) + | > current_lr: 0.00006 + | > step_time: 7.00310 (5.58896) + | > loader_time: 0.10110 (0.06550) + + + --> STEP: 14/234 -- GLOBAL_STEP: 58280 + | > loss: -0.30939 (-0.28997) + | > log_mle: -0.38593 (-0.38269) + | > loss_dur: 0.07653 (0.09272) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.15272 (17.81108) + | > current_lr: 0.00006 + | > step_time: 2.20320 (4.61947) + | > loader_time: 0.00150 (0.05594) + + + --> STEP: 19/234 -- GLOBAL_STEP: 58285 + | > loss: -0.31394 (-0.29594) + | > log_mle: -0.38287 (-0.38274) + | > loss_dur: 0.06893 (0.08679) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.56627 (16.50153) + | > current_lr: 0.00006 + | > step_time: 8.30530 (4.27654) + | > loader_time: 0.10580 (0.05157) + + + --> STEP: 24/234 -- GLOBAL_STEP: 58290 + | > loss: -0.30734 (-0.29852) + | > log_mle: -0.37017 (-0.38197) + | > loss_dur: 0.06283 (0.08345) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.35582 (15.32988) + | > current_lr: 0.00006 + | > step_time: 0.68210 (3.63383) + | > loader_time: 0.00390 (0.04142) + + + --> STEP: 29/234 -- GLOBAL_STEP: 58295 + | > loss: -0.28154 (-0.29851) + | > log_mle: -0.35870 (-0.38051) + | > loss_dur: 0.07716 (0.08200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.61922 (14.68415) + | > current_lr: 0.00006 + | > step_time: 1.88850 (3.43838) + | > loader_time: 0.00770 (0.03799) + + + --> STEP: 34/234 -- GLOBAL_STEP: 58300 + | > loss: -0.27975 (-0.29559) + | > log_mle: -0.36151 (-0.37789) + | > loss_dur: 0.08176 (0.08230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.71051 (14.29115) + | > current_lr: 0.00006 + | > step_time: 0.75020 (3.14639) + | > loader_time: 0.00180 (0.03808) + + + --> STEP: 39/234 -- GLOBAL_STEP: 58305 + | > loss: -0.26581 (-0.29126) + | > log_mle: -0.35724 (-0.37483) + | > loss_dur: 0.09143 (0.08357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.67555 (14.26653) + | > current_lr: 0.00006 + | > step_time: 0.88190 (2.88821) + | > loader_time: 0.00160 (0.03343) + + + --> STEP: 44/234 -- GLOBAL_STEP: 58310 + | > loss: -0.28698 (-0.28783) + | > log_mle: -0.35515 (-0.37226) + | > loss_dur: 0.06817 (0.08443) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.16614 (13.87269) + | > current_lr: 0.00006 + | > step_time: 1.18730 (2.70449) + | > loader_time: 0.00170 (0.02985) + + + --> STEP: 49/234 -- GLOBAL_STEP: 58315 + | > loss: -0.27878 (-0.28579) + | > log_mle: -0.36376 (-0.37127) + | > loss_dur: 0.08497 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.06776 (13.52157) + | > current_lr: 0.00006 + | > step_time: 3.78900 (2.64987) + | > loader_time: 0.09980 (0.02947) + + + --> STEP: 54/234 -- GLOBAL_STEP: 58320 + | > loss: -0.27076 (-0.28358) + | > log_mle: -0.36057 (-0.36977) + | > loss_dur: 0.08981 (0.08620) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.85417 (13.10645) + | > current_lr: 0.00006 + | > step_time: 1.44630 (2.52441) + | > loader_time: 0.00170 (0.02694) + + + --> STEP: 59/234 -- GLOBAL_STEP: 58325 + | > loss: -0.27171 (-0.28196) + | > log_mle: -0.36509 (-0.36878) + | > loss_dur: 0.09339 (0.08682) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.42598 (12.87084) + | > current_lr: 0.00006 + | > step_time: 2.56960 (2.47613) + | > loader_time: 0.00180 (0.02483) + + + --> STEP: 64/234 -- GLOBAL_STEP: 58330 + | > loss: -0.26035 (-0.27919) + | > log_mle: -0.34822 (-0.36804) + | > loss_dur: 0.08787 (0.08884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.25254 (12.86649) + | > current_lr: 0.00006 + | > step_time: 2.60940 (2.45178) + | > loader_time: 0.09980 (0.02600) + + + --> STEP: 69/234 -- GLOBAL_STEP: 58335 + | > loss: -0.24704 (-0.27703) + | > log_mle: -0.34100 (-0.36679) + | > loss_dur: 0.09396 (0.08976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.70194 (12.73629) + | > current_lr: 0.00006 + | > step_time: 2.73630 (2.42869) + | > loader_time: 0.00200 (0.02547) + + + --> STEP: 74/234 -- GLOBAL_STEP: 58340 + | > loss: -0.22736 (-0.27400) + | > log_mle: -0.34007 (-0.36576) + | > loss_dur: 0.11271 (0.09176) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.25360 (12.77452) + | > current_lr: 0.00006 + | > step_time: 2.58850 (2.39801) + | > loader_time: 0.00270 (0.02517) + + + --> STEP: 79/234 -- GLOBAL_STEP: 58345 + | > loss: -0.24265 (-0.27168) + | > log_mle: -0.35554 (-0.36487) + | > loss_dur: 0.11289 (0.09320) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.88161 (12.78486) + | > current_lr: 0.00006 + | > step_time: 1.83130 (2.37636) + | > loader_time: 0.00190 (0.02468) + + + --> STEP: 84/234 -- GLOBAL_STEP: 58350 + | > loss: -0.24600 (-0.26964) + | > log_mle: -0.35319 (-0.36412) + | > loss_dur: 0.10719 (0.09448) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.28927 (12.78230) + | > current_lr: 0.00006 + | > step_time: 2.00990 (2.33501) + | > loader_time: 0.00290 (0.02341) + + + --> STEP: 89/234 -- GLOBAL_STEP: 58355 + | > loss: -0.25694 (-0.26815) + | > log_mle: -0.37674 (-0.36416) + | > loss_dur: 0.11980 (0.09601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.82354 (12.80568) + | > current_lr: 0.00006 + | > step_time: 1.41890 (2.29906) + | > loader_time: 0.08740 (0.02317) + + + --> STEP: 94/234 -- GLOBAL_STEP: 58360 + | > loss: -0.26441 (-0.26710) + | > log_mle: -0.40215 (-0.36537) + | > loss_dur: 0.13774 (0.09827) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.06283 (13.13639) + | > current_lr: 0.00006 + | > step_time: 2.50680 (2.30028) + | > loader_time: 0.10430 (0.02393) + + + --> STEP: 99/234 -- GLOBAL_STEP: 58365 + | > loss: -0.26418 (-0.26657) + | > log_mle: -0.43447 (-0.36674) + | > loss_dur: 0.17029 (0.10017) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.30820 (13.54867) + | > current_lr: 0.00006 + | > step_time: 1.78570 (2.28884) + | > loader_time: 0.00230 (0.02363) + + + --> STEP: 104/234 -- GLOBAL_STEP: 58370 + | > loss: -0.30033 (-0.26620) + | > log_mle: -0.44900 (-0.36860) + | > loss_dur: 0.14868 (0.10241) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.52431 (14.06871) + | > current_lr: 0.00006 + | > step_time: 2.66020 (2.27743) + | > loader_time: 0.08530 (0.02412) + + + --> STEP: 109/234 -- GLOBAL_STEP: 58375 + | > loss: -0.22493 (-0.26471) + | > log_mle: -0.41506 (-0.36967) + | > loss_dur: 0.19014 (0.10496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.90642 (14.54574) + | > current_lr: 0.00006 + | > step_time: 2.49280 (2.28610) + | > loader_time: 0.00610 (0.02374) + + + --> STEP: 114/234 -- GLOBAL_STEP: 58380 + | > loss: -0.24843 (-0.26418) + | > log_mle: -0.39928 (-0.37154) + | > loss_dur: 0.15086 (0.10737) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.46552 (15.27352) + | > current_lr: 0.00006 + | > step_time: 1.39870 (2.28070) + | > loader_time: 0.00220 (0.02425) + + + --> STEP: 119/234 -- GLOBAL_STEP: 58385 + | > loss: -0.24932 (-0.26307) + | > log_mle: -0.39866 (-0.37282) + | > loss_dur: 0.14934 (0.10975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.43847 (15.89148) + | > current_lr: 0.00006 + | > step_time: 0.89610 (2.29589) + | > loader_time: 0.00300 (0.02558) + + + --> STEP: 124/234 -- GLOBAL_STEP: 58390 + | > loss: -0.26789 (-0.26188) + | > log_mle: -0.42376 (-0.37342) + | > loss_dur: 0.15587 (0.11154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.28770 (16.36716) + | > current_lr: 0.00006 + | > step_time: 1.70240 (2.27132) + | > loader_time: 0.00300 (0.02590) + + + --> STEP: 129/234 -- GLOBAL_STEP: 58395 + | > loss: -0.24210 (-0.26183) + | > log_mle: -0.41839 (-0.37580) + | > loss_dur: 0.17629 (0.11397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.96322 (16.84103) + | > current_lr: 0.00006 + | > step_time: 2.59840 (2.25576) + | > loader_time: 0.19080 (0.02768) + + + --> STEP: 134/234 -- GLOBAL_STEP: 58400 + | > loss: -0.29083 (-0.26292) + | > log_mle: -0.47264 (-0.37906) + | > loss_dur: 0.18181 (0.11614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.80675 (17.44595) + | > current_lr: 0.00006 + | > step_time: 1.52210 (2.23220) + | > loader_time: 0.00310 (0.02869) + + + --> STEP: 139/234 -- GLOBAL_STEP: 58405 + | > loss: -0.33658 (-0.26349) + | > log_mle: -0.52969 (-0.38188) + | > loss_dur: 0.19310 (0.11839) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.51668 (18.44422) + | > current_lr: 0.00006 + | > step_time: 1.20040 (2.22402) + | > loader_time: 0.00280 (0.02831) + + + --> STEP: 144/234 -- GLOBAL_STEP: 58410 + | > loss: -0.30782 (-0.26414) + | > log_mle: -0.50251 (-0.38507) + | > loss_dur: 0.19469 (0.12094) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.64757 (19.18819) + | > current_lr: 0.00006 + | > step_time: 2.70350 (2.27826) + | > loader_time: 0.00690 (0.02936) + + + --> STEP: 149/234 -- GLOBAL_STEP: 58415 + | > loss: -0.36982 (-0.26600) + | > log_mle: -0.56061 (-0.38897) + | > loss_dur: 0.19079 (0.12297) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.75570 (19.99121) + | > current_lr: 0.00006 + | > step_time: 1.48730 (2.26874) + | > loader_time: 0.00240 (0.02902) + + + --> STEP: 154/234 -- GLOBAL_STEP: 58420 + | > loss: -0.32088 (-0.26809) + | > log_mle: -0.51596 (-0.39342) + | > loss_dur: 0.19508 (0.12533) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.35504 (20.84272) + | > current_lr: 0.00006 + | > step_time: 3.13820 (2.34142) + | > loader_time: 0.09950 (0.03129) + + + --> STEP: 159/234 -- GLOBAL_STEP: 58425 + | > loss: -0.34748 (-0.27009) + | > log_mle: -0.53617 (-0.39765) + | > loss_dur: 0.18869 (0.12757) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.73110 (21.76725) + | > current_lr: 0.00006 + | > step_time: 1.41130 (2.33605) + | > loader_time: 0.07630 (0.03132) + + + --> STEP: 164/234 -- GLOBAL_STEP: 58430 + | > loss: -0.31276 (-0.27203) + | > log_mle: -0.52580 (-0.40174) + | > loss_dur: 0.21304 (0.12971) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.59850 (22.81649) + | > current_lr: 0.00006 + | > step_time: 4.70670 (2.34505) + | > loader_time: 0.09170 (0.03262) + + + --> STEP: 169/234 -- GLOBAL_STEP: 58435 + | > loss: -0.31488 (-0.27413) + | > log_mle: -0.52730 (-0.40600) + | > loss_dur: 0.21242 (0.13188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.67189 (23.87024) + | > current_lr: 0.00006 + | > step_time: 0.91350 (2.34952) + | > loader_time: 0.08220 (0.03344) + + + --> STEP: 174/234 -- GLOBAL_STEP: 58440 + | > loss: -0.39736 (-0.27710) + | > log_mle: -0.61133 (-0.41124) + | > loss_dur: 0.21397 (0.13414) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.35927 (25.01970) + | > current_lr: 0.00006 + | > step_time: 1.89810 (2.35045) + | > loader_time: 0.09770 (0.03365) + + + --> STEP: 179/234 -- GLOBAL_STEP: 58445 + | > loss: -0.37905 (-0.27950) + | > log_mle: -0.62010 (-0.41615) + | > loss_dur: 0.24104 (0.13665) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.33666 (26.06543) + | > current_lr: 0.00006 + | > step_time: 2.29350 (2.39744) + | > loader_time: 0.00470 (0.03447) + + + --> STEP: 184/234 -- GLOBAL_STEP: 58450 + | > loss: -0.35915 (-0.28142) + | > log_mle: -0.56628 (-0.42027) + | > loss_dur: 0.20714 (0.13886) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.17348 (27.22194) + | > current_lr: 0.00006 + | > step_time: 5.79570 (2.43991) + | > loader_time: 0.00250 (0.03555) + + + --> STEP: 189/234 -- GLOBAL_STEP: 58455 + | > loss: -0.35641 (-0.28353) + | > log_mle: -0.57021 (-0.42458) + | > loss_dur: 0.21380 (0.14106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.17319 (28.14953) + | > current_lr: 0.00006 + | > step_time: 7.99990 (2.48332) + | > loader_time: 0.10360 (0.03717) + + + --> STEP: 194/234 -- GLOBAL_STEP: 58460 + | > loss: -0.38572 (-0.28618) + | > log_mle: -0.60415 (-0.42894) + | > loss_dur: 0.21843 (0.14277) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.64585 (29.20696) + | > current_lr: 0.00006 + | > step_time: 5.68940 (2.52388) + | > loader_time: 0.10300 (0.03703) + + + --> STEP: 199/234 -- GLOBAL_STEP: 58465 + | > loss: -0.39871 (-0.28853) + | > log_mle: -0.61823 (-0.43316) + | > loss_dur: 0.21952 (0.14463) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.01381 (30.04274) + | > current_lr: 0.00006 + | > step_time: 2.99400 (2.55295) + | > loader_time: 0.00340 (0.03809) + + + --> STEP: 204/234 -- GLOBAL_STEP: 58470 + | > loss: -0.41009 (-0.29064) + | > log_mle: -0.65424 (-0.43736) + | > loss_dur: 0.24415 (0.14671) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.66792 (30.83842) + | > current_lr: 0.00006 + | > step_time: 3.50790 (2.61483) + | > loader_time: 0.09500 (0.03867) + + + --> STEP: 209/234 -- GLOBAL_STEP: 58475 + | > loss: -0.39873 (-0.29329) + | > log_mle: -0.61867 (-0.44196) + | > loss_dur: 0.21993 (0.14868) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.25201 (31.78699) + | > current_lr: 0.00006 + | > step_time: 3.89630 (2.70355) + | > loader_time: 0.00940 (0.03871) + + + --> STEP: 214/234 -- GLOBAL_STEP: 58480 + | > loss: -0.42773 (-0.29684) + | > log_mle: -0.63591 (-0.44748) + | > loss_dur: 0.20819 (0.15064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.09875 (33.04890) + | > current_lr: 0.00006 + | > step_time: 2.90500 (2.75640) + | > loader_time: 0.07100 (0.03950) + + + --> STEP: 219/234 -- GLOBAL_STEP: 58485 + | > loss: -0.51894 (-0.30016) + | > log_mle: -0.75068 (-0.45273) + | > loss_dur: 0.23174 (0.15257) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.88065 (34.29768) + | > current_lr: 0.00006 + | > step_time: 1.30010 (2.83095) + | > loader_time: 0.00290 (0.04176) + + + --> STEP: 224/234 -- GLOBAL_STEP: 58490 + | > loss: -0.45850 (-0.30316) + | > log_mle: -0.69212 (-0.45767) + | > loss_dur: 0.23362 (0.15451) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.58003 (35.45930) + | > current_lr: 0.00006 + | > step_time: 2.10130 (2.80528) + | > loader_time: 0.00320 (0.04129) + + + --> STEP: 229/234 -- GLOBAL_STEP: 58495 + | > loss: -0.44249 (-0.30644) + | > log_mle: -0.74390 (-0.46328) + | > loss_dur: 0.30141 (0.15684) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.72216 (36.66888) + | > current_lr: 0.00006 + | > step_time: 0.23810 (2.75507) + | > loader_time: 0.00280 (0.04085) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.48922 (+0.18618) + | > avg_loss: -0.30814 (-0.01028) + | > avg_log_mle: -0.52904 (+0.00523) + | > avg_loss_dur: 0.22090 (-0.01551) + + + > EPOCH: 250/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 13:50:24)  + + --> STEP: 0/234 -- GLOBAL_STEP: 58500 + | > loss: -0.31632 (-0.31632) + | > log_mle: -0.45747 (-0.45747) + | > loss_dur: 0.14116 (0.14116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.26940 (20.26940) + | > current_lr: 0.00006 + | > step_time: 5.60400 (5.60396) + | > loader_time: 18.54370 (18.54374) + + + --> STEP: 5/234 -- GLOBAL_STEP: 58505 + | > loss: -0.28587 (-0.27189) + | > log_mle: -0.37753 (-0.37588) + | > loss_dur: 0.09166 (0.10398) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.02669 (21.78430) + | > current_lr: 0.00006 + | > step_time: 3.70250 (3.18424) + | > loader_time: 0.00370 (0.05688) + + + --> STEP: 10/234 -- GLOBAL_STEP: 58510 + | > loss: -0.29213 (-0.28154) + | > log_mle: -0.37551 (-0.37875) + | > loss_dur: 0.08338 (0.09721) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.62782 (19.77333) + | > current_lr: 0.00006 + | > step_time: 8.47010 (3.97933) + | > loader_time: 0.18890 (0.05960) + + + --> STEP: 15/234 -- GLOBAL_STEP: 58515 + | > loss: -0.31293 (-0.29239) + | > log_mle: -0.38861 (-0.38199) + | > loss_dur: 0.07568 (0.08960) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.34414 (17.96035) + | > current_lr: 0.00006 + | > step_time: 0.90000 (4.64027) + | > loader_time: 0.07620 (0.07703) + + + --> STEP: 20/234 -- GLOBAL_STEP: 58520 + | > loss: -0.31376 (-0.29666) + | > log_mle: -0.38268 (-0.38178) + | > loss_dur: 0.06893 (0.08512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.93464 (16.17123) + | > current_lr: 0.00006 + | > step_time: 3.00460 (4.21460) + | > loader_time: 0.00320 (0.06791) + + + --> STEP: 25/234 -- GLOBAL_STEP: 58525 + | > loss: -0.28509 (-0.29792) + | > log_mle: -0.36015 (-0.38033) + | > loss_dur: 0.07505 (0.08241) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.38817 (15.43329) + | > current_lr: 0.00006 + | > step_time: 2.30150 (3.79955) + | > loader_time: 0.08140 (0.06169) + + + --> STEP: 30/234 -- GLOBAL_STEP: 58530 + | > loss: -0.27596 (-0.29770) + | > log_mle: -0.35726 (-0.37883) + | > loss_dur: 0.08130 (0.08112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.71495 (14.86929) + | > current_lr: 0.00006 + | > step_time: 2.32300 (3.45066) + | > loader_time: 0.00290 (0.05169) + + + --> STEP: 35/234 -- GLOBAL_STEP: 58535 + | > loss: -0.26215 (-0.29452) + | > log_mle: -0.35289 (-0.37629) + | > loss_dur: 0.09074 (0.08178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.35679 (14.40941) + | > current_lr: 0.00006 + | > step_time: 1.81260 (3.19297) + | > loader_time: 0.00190 (0.04697) + + + --> STEP: 40/234 -- GLOBAL_STEP: 58540 + | > loss: -0.24661 (-0.29080) + | > log_mle: -0.34973 (-0.37382) + | > loss_dur: 0.10312 (0.08302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.31834 (13.95031) + | > current_lr: 0.00006 + | > step_time: 1.95830 (3.00244) + | > loader_time: 0.09170 (0.04561) + + + --> STEP: 45/234 -- GLOBAL_STEP: 58545 + | > loss: -0.26746 (-0.28852) + | > log_mle: -0.37399 (-0.37217) + | > loss_dur: 0.10653 (0.08365) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.67315 (13.61423) + | > current_lr: 0.00006 + | > step_time: 4.29870 (2.87337) + | > loader_time: 0.10380 (0.04301) + + + --> STEP: 50/234 -- GLOBAL_STEP: 58550 + | > loss: -0.26913 (-0.28695) + | > log_mle: -0.35517 (-0.37108) + | > loss_dur: 0.08605 (0.08413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.91101 (13.09170) + | > current_lr: 0.00006 + | > step_time: 1.29530 (2.75749) + | > loader_time: 0.00250 (0.04100) + + + --> STEP: 55/234 -- GLOBAL_STEP: 58555 + | > loss: -0.28238 (-0.28530) + | > log_mle: -0.36507 (-0.37006) + | > loss_dur: 0.08269 (0.08476) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.52613 (12.87847) + | > current_lr: 0.00006 + | > step_time: 2.69630 (2.68854) + | > loader_time: 0.00170 (0.04041) + + + --> STEP: 60/234 -- GLOBAL_STEP: 58560 + | > loss: -0.24992 (-0.28312) + | > log_mle: -0.37122 (-0.36921) + | > loss_dur: 0.12131 (0.08609) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.33008 (12.71799) + | > current_lr: 0.00006 + | > step_time: 1.99090 (2.67926) + | > loader_time: 0.00210 (0.04061) + + + --> STEP: 65/234 -- GLOBAL_STEP: 58565 + | > loss: -0.25857 (-0.28054) + | > log_mle: -0.35668 (-0.36835) + | > loss_dur: 0.09811 (0.08781) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.75949 (12.59184) + | > current_lr: 0.00006 + | > step_time: 1.00500 (2.61176) + | > loader_time: 0.08260 (0.04007) + + + --> STEP: 70/234 -- GLOBAL_STEP: 58570 + | > loss: -0.22245 (-0.27774) + | > log_mle: -0.33989 (-0.36684) + | > loss_dur: 0.11744 (0.08911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.60297 (12.56255) + | > current_lr: 0.00006 + | > step_time: 1.27940 (2.55642) + | > loader_time: 0.00190 (0.03738) + + + --> STEP: 75/234 -- GLOBAL_STEP: 58575 + | > loss: -0.21721 (-0.27459) + | > log_mle: -0.35421 (-0.36601) + | > loss_dur: 0.13700 (0.09143) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.12465 (12.83165) + | > current_lr: 0.00006 + | > step_time: 1.08010 (2.50870) + | > loader_time: 0.00220 (0.03622) + + + --> STEP: 80/234 -- GLOBAL_STEP: 58580 + | > loss: -0.24066 (-0.27257) + | > log_mle: -0.34278 (-0.36499) + | > loss_dur: 0.10213 (0.09242) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.45175 (12.80262) + | > current_lr: 0.00006 + | > step_time: 1.18810 (2.47184) + | > loader_time: 0.00230 (0.03530) + + + --> STEP: 85/234 -- GLOBAL_STEP: 58585 + | > loss: -0.24331 (-0.27072) + | > log_mle: -0.34763 (-0.36440) + | > loss_dur: 0.10432 (0.09368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.91878 (12.74644) + | > current_lr: 0.00006 + | > step_time: 2.20780 (2.45807) + | > loader_time: 0.00200 (0.03340) + + + --> STEP: 90/234 -- GLOBAL_STEP: 58590 + | > loss: -0.23341 (-0.26929) + | > log_mle: -0.37183 (-0.36480) + | > loss_dur: 0.13842 (0.09551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.78868 (12.84525) + | > current_lr: 0.00006 + | > step_time: 1.39240 (2.44050) + | > loader_time: 0.10650 (0.03376) + + + --> STEP: 95/234 -- GLOBAL_STEP: 58595 + | > loss: -0.28951 (-0.26892) + | > log_mle: -0.45543 (-0.36698) + | > loss_dur: 0.16592 (0.09806) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.40428 (13.31171) + | > current_lr: 0.00006 + | > step_time: 1.75190 (2.44345) + | > loader_time: 0.00200 (0.03293) + + + --> STEP: 100/234 -- GLOBAL_STEP: 58600 + | > loss: -0.24942 (-0.26771) + | > log_mle: -0.38304 (-0.36757) + | > loss_dur: 0.13361 (0.09986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.05117 (13.55831) + | > current_lr: 0.00006 + | > step_time: 1.70470 (2.40942) + | > loader_time: 0.08660 (0.03659) + + + --> STEP: 105/234 -- GLOBAL_STEP: 58605 + | > loss: -0.23427 (-0.26708) + | > log_mle: -0.36179 (-0.36914) + | > loss_dur: 0.12753 (0.10206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.54460 (14.27223) + | > current_lr: 0.00006 + | > step_time: 1.40560 (2.37082) + | > loader_time: 0.00560 (0.03575) + + + --> STEP: 110/234 -- GLOBAL_STEP: 58610 + | > loss: -0.23187 (-0.26560) + | > log_mle: -0.38185 (-0.37037) + | > loss_dur: 0.14998 (0.10477) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.62558 (14.72910) + | > current_lr: 0.00006 + | > step_time: 1.89780 (2.37700) + | > loader_time: 0.08270 (0.03583) + + + --> STEP: 115/234 -- GLOBAL_STEP: 58615 + | > loss: -0.23891 (-0.26512) + | > log_mle: -0.40540 (-0.37261) + | > loss_dur: 0.16649 (0.10750) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.82878 (15.24541) + | > current_lr: 0.00006 + | > step_time: 1.40620 (2.36187) + | > loader_time: 0.07570 (0.03691) + + + --> STEP: 120/234 -- GLOBAL_STEP: 58620 + | > loss: -0.28528 (-0.26458) + | > log_mle: -0.45302 (-0.37449) + | > loss_dur: 0.16774 (0.10991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.02423 (15.61860) + | > current_lr: 0.00006 + | > step_time: 2.00320 (2.36517) + | > loader_time: 0.00210 (0.03693) + + + --> STEP: 125/234 -- GLOBAL_STEP: 58625 + | > loss: -0.26746 (-0.26337) + | > log_mle: -0.43908 (-0.37511) + | > loss_dur: 0.17161 (0.11173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.99529 (15.92076) + | > current_lr: 0.00006 + | > step_time: 3.79020 (2.35269) + | > loader_time: 0.00360 (0.03689) + + + --> STEP: 130/234 -- GLOBAL_STEP: 58630 + | > loss: -0.28195 (-0.26354) + | > log_mle: -0.45097 (-0.37757) + | > loss_dur: 0.16902 (0.11402) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.42935 (16.70390) + | > current_lr: 0.00006 + | > step_time: 3.35010 (2.38261) + | > loader_time: 0.00520 (0.03565) + + + --> STEP: 135/234 -- GLOBAL_STEP: 58635 + | > loss: -0.23037 (-0.26397) + | > log_mle: -0.37875 (-0.38016) + | > loss_dur: 0.14838 (0.11619) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.23754 (17.31451) + | > current_lr: 0.00006 + | > step_time: 1.84570 (2.37786) + | > loader_time: 0.00230 (0.03504) + + + --> STEP: 140/234 -- GLOBAL_STEP: 58640 + | > loss: -0.23228 (-0.26451) + | > log_mle: -0.41359 (-0.38332) + | > loss_dur: 0.18131 (0.11880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.92716 (18.22814) + | > current_lr: 0.00006 + | > step_time: 5.09070 (2.39610) + | > loader_time: 0.10590 (0.03531) + + + --> STEP: 145/234 -- GLOBAL_STEP: 58645 + | > loss: -0.33577 (-0.26596) + | > log_mle: -0.51911 (-0.38730) + | > loss_dur: 0.18334 (0.12134) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.77532 (19.17612) + | > current_lr: 0.00006 + | > step_time: 1.60350 (2.40944) + | > loader_time: 0.09130 (0.03616) + + + --> STEP: 150/234 -- GLOBAL_STEP: 58650 + | > loss: -0.30012 (-0.26745) + | > log_mle: -0.49937 (-0.39091) + | > loss_dur: 0.19924 (0.12346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.36975 (20.08513) + | > current_lr: 0.00006 + | > step_time: 3.42270 (2.40140) + | > loader_time: 0.08430 (0.03563) + + + --> STEP: 155/234 -- GLOBAL_STEP: 58655 + | > loss: -0.36215 (-0.26996) + | > log_mle: -0.57117 (-0.39576) + | > loss_dur: 0.20902 (0.12580) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.52181 (21.04502) + | > current_lr: 0.00006 + | > step_time: 0.97170 (2.38300) + | > loader_time: 0.00250 (0.03582) + + + --> STEP: 160/234 -- GLOBAL_STEP: 58660 + | > loss: -0.35953 (-0.27186) + | > log_mle: -0.56546 (-0.39987) + | > loss_dur: 0.20593 (0.12801) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.70905 (21.96452) + | > current_lr: 0.00006 + | > step_time: 0.88270 (2.37342) + | > loader_time: 0.00920 (0.03532) + + + --> STEP: 165/234 -- GLOBAL_STEP: 58665 + | > loss: -0.35119 (-0.27359) + | > log_mle: -0.56000 (-0.40380) + | > loss_dur: 0.20881 (0.13021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.26234 (22.89488) + | > current_lr: 0.00006 + | > step_time: 5.30520 (2.44440) + | > loader_time: 0.00310 (0.03669) + + + --> STEP: 170/234 -- GLOBAL_STEP: 58670 + | > loss: -0.38223 (-0.27599) + | > log_mle: -0.60113 (-0.40838) + | > loss_dur: 0.21889 (0.13239) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.08797 (23.85708) + | > current_lr: 0.00006 + | > step_time: 3.30670 (2.44442) + | > loader_time: 0.00300 (0.03661) + + + --> STEP: 175/234 -- GLOBAL_STEP: 58675 + | > loss: -0.34919 (-0.27889) + | > log_mle: -0.57967 (-0.41360) + | > loss_dur: 0.23048 (0.13471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.24433 (24.93749) + | > current_lr: 0.00006 + | > step_time: 3.12220 (2.43978) + | > loader_time: 0.08520 (0.03614) + + + --> STEP: 180/234 -- GLOBAL_STEP: 58680 + | > loss: -0.37194 (-0.28149) + | > log_mle: -0.57855 (-0.41853) + | > loss_dur: 0.20660 (0.13704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.65306 (26.01858) + | > current_lr: 0.00006 + | > step_time: 1.48810 (2.42866) + | > loader_time: 0.00370 (0.03620) + + + --> STEP: 185/234 -- GLOBAL_STEP: 58685 + | > loss: -0.37430 (-0.28348) + | > log_mle: -0.61272 (-0.42287) + | > loss_dur: 0.23842 (0.13940) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.82482 (26.99245) + | > current_lr: 0.00006 + | > step_time: 2.09260 (2.48037) + | > loader_time: 0.00510 (0.03673) + + + --> STEP: 190/234 -- GLOBAL_STEP: 58690 + | > loss: -0.38418 (-0.28572) + | > log_mle: -0.58536 (-0.42720) + | > loss_dur: 0.20118 (0.14148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.06343 (27.95998) + | > current_lr: 0.00006 + | > step_time: 1.79050 (2.48844) + | > loader_time: 0.00410 (0.03731) + + + --> STEP: 195/234 -- GLOBAL_STEP: 58695 + | > loss: -0.38267 (-0.28843) + | > log_mle: -0.60760 (-0.43180) + | > loss_dur: 0.22492 (0.14337) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.33126 (28.99072) + | > current_lr: 0.00006 + | > step_time: 4.39400 (2.50209) + | > loader_time: 0.00370 (0.03779) + + + --> STEP: 200/234 -- GLOBAL_STEP: 58700 + | > loss: -0.36623 (-0.29071) + | > log_mle: -0.61629 (-0.43605) + | > loss_dur: 0.25006 (0.14535) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.56243 (29.89203) + | > current_lr: 0.00006 + | > step_time: 1.89970 (2.56703) + | > loader_time: 0.00260 (0.03978) + + + --> STEP: 205/234 -- GLOBAL_STEP: 58705 + | > loss: -0.36676 (-0.29281) + | > log_mle: -0.59195 (-0.44007) + | > loss_dur: 0.22519 (0.14726) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.88348 (31.06677) + | > current_lr: 0.00006 + | > step_time: 5.49890 (2.65992) + | > loader_time: 0.01790 (0.04040) + + + --> STEP: 210/234 -- GLOBAL_STEP: 58710 + | > loss: -0.44687 (-0.29579) + | > log_mle: -0.67873 (-0.44499) + | > loss_dur: 0.23186 (0.14920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.56737 (32.11847) + | > current_lr: 0.00006 + | > step_time: 7.10830 (2.70901) + | > loader_time: 0.29310 (0.04095) + + + --> STEP: 215/234 -- GLOBAL_STEP: 58715 + | > loss: -0.39666 (-0.29909) + | > log_mle: -0.63333 (-0.45021) + | > loss_dur: 0.23667 (0.15112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.90525 (33.20720) + | > current_lr: 0.00006 + | > step_time: 6.11440 (2.76419) + | > loader_time: 0.09090 (0.04282) + + + --> STEP: 220/234 -- GLOBAL_STEP: 58720 + | > loss: -0.42297 (-0.30251) + | > log_mle: -0.67046 (-0.45567) + | > loss_dur: 0.24749 (0.15316) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 116.08160 (34.63357) + | > current_lr: 0.00006 + | > step_time: 3.39860 (2.78200) + | > loader_time: 0.09910 (0.04350) + + + --> STEP: 225/234 -- GLOBAL_STEP: 58725 + | > loss: -0.45172 (-0.30540) + | > log_mle: -0.69572 (-0.46046) + | > loss_dur: 0.24400 (0.15506) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.37731 (36.45988) + | > current_lr: 0.00006 + | > step_time: 0.48140 (2.76405) + | > loader_time: 0.07560 (0.04371) + + + --> STEP: 230/234 -- GLOBAL_STEP: 58730 + | > loss: -0.46977 (-0.30785) + | > log_mle: -0.79012 (-0.46559) + | > loss_dur: 0.32036 (0.15774) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.04119 (37.19888) + | > current_lr: 0.00006 + | > step_time: 0.27020 (2.70937) + | > loader_time: 0.00440 (0.04285) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.49609 (+0.00687) + | > avg_loss: -0.32573 (-0.01758) + | > avg_log_mle: -0.54521 (-0.01617) + | > avg_loss_dur: 0.21948 (-0.00142) + + + > EPOCH: 251/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 14:02:05)  + + --> STEP: 1/234 -- GLOBAL_STEP: 58735 + | > loss: -0.29564 (-0.29564) + | > log_mle: -0.37283 (-0.37283) + | > loss_dur: 0.07719 (0.07719) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.57528 (24.57528) + | > current_lr: 0.00006 + | > step_time: 6.39920 (6.39921) + | > loader_time: 0.39790 (0.39794) + + + --> STEP: 6/234 -- GLOBAL_STEP: 58740 + | > loss: -0.30748 (-0.28044) + | > log_mle: -0.37727 (-0.37663) + | > loss_dur: 0.06980 (0.09618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.42959 (21.42122) + | > current_lr: 0.00006 + | > step_time: 1.00420 (6.01945) + | > loader_time: 0.00090 (0.12948) + + + --> STEP: 11/234 -- GLOBAL_STEP: 58745 + | > loss: -0.32729 (-0.28626) + | > log_mle: -0.38701 (-0.38043) + | > loss_dur: 0.05972 (0.09417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.03157 (19.41358) + | > current_lr: 0.00006 + | > step_time: 1.32870 (4.90532) + | > loader_time: 0.00120 (0.13508) + + + --> STEP: 16/234 -- GLOBAL_STEP: 58750 + | > loss: -0.32276 (-0.29434) + | > log_mle: -0.39055 (-0.38267) + | > loss_dur: 0.06779 (0.08833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.89846 (17.93955) + | > current_lr: 0.00006 + | > step_time: 2.91550 (3.87710) + | > loader_time: 0.00150 (0.09856) + + + --> STEP: 21/234 -- GLOBAL_STEP: 58755 + | > loss: -0.29859 (-0.29749) + | > log_mle: -0.36752 (-0.38130) + | > loss_dur: 0.06893 (0.08380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.53434 (16.13269) + | > current_lr: 0.00006 + | > step_time: 1.58770 (3.73523) + | > loader_time: 0.00240 (0.08770) + + + --> STEP: 26/234 -- GLOBAL_STEP: 58760 + | > loss: -0.28033 (-0.29819) + | > log_mle: -0.36816 (-0.38030) + | > loss_dur: 0.08783 (0.08211) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.44320 (15.48387) + | > current_lr: 0.00006 + | > step_time: 1.28620 (3.35406) + | > loader_time: 0.00170 (0.07431) + + + --> STEP: 31/234 -- GLOBAL_STEP: 58765 + | > loss: -0.24228 (-0.29656) + | > log_mle: -0.34858 (-0.37813) + | > loss_dur: 0.10631 (0.08157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.20259 (14.73330) + | > current_lr: 0.00006 + | > step_time: 1.21230 (3.10047) + | > loader_time: 0.00250 (0.06279) + + + --> STEP: 36/234 -- GLOBAL_STEP: 58770 + | > loss: -0.25542 (-0.29341) + | > log_mle: -0.34547 (-0.37534) + | > loss_dur: 0.09004 (0.08194) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.93499 (14.41708) + | > current_lr: 0.00006 + | > step_time: 1.69420 (2.97145) + | > loader_time: 0.00140 (0.05678) + + + --> STEP: 41/234 -- GLOBAL_STEP: 58775 + | > loss: -0.28876 (-0.29085) + | > log_mle: -0.36399 (-0.37316) + | > loss_dur: 0.07523 (0.08231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.74334 (13.91600) + | > current_lr: 0.00006 + | > step_time: 1.66990 (2.86959) + | > loader_time: 0.00170 (0.05476) + + + --> STEP: 46/234 -- GLOBAL_STEP: 58780 + | > loss: -0.25850 (-0.28771) + | > log_mle: -0.35828 (-0.37156) + | > loss_dur: 0.09978 (0.08385) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.42652 (13.66013) + | > current_lr: 0.00006 + | > step_time: 1.41110 (2.76192) + | > loader_time: 0.00170 (0.05276) + + + --> STEP: 51/234 -- GLOBAL_STEP: 58785 + | > loss: -0.26554 (-0.28633) + | > log_mle: -0.35300 (-0.37029) + | > loss_dur: 0.08746 (0.08395) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.75329 (13.29078) + | > current_lr: 0.00006 + | > step_time: 1.24470 (2.66685) + | > loader_time: 0.00170 (0.04778) + + + --> STEP: 56/234 -- GLOBAL_STEP: 58790 + | > loss: -0.24863 (-0.28401) + | > log_mle: -0.35504 (-0.36914) + | > loss_dur: 0.10641 (0.08513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.43327 (13.15785) + | > current_lr: 0.00006 + | > step_time: 2.02480 (2.57983) + | > loader_time: 0.00230 (0.04369) + + + --> STEP: 61/234 -- GLOBAL_STEP: 58795 + | > loss: -0.25739 (-0.28178) + | > log_mle: -0.35501 (-0.36824) + | > loss_dur: 0.09762 (0.08646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.51127 (12.97260) + | > current_lr: 0.00006 + | > step_time: 1.53630 (2.51816) + | > loader_time: 0.00330 (0.04471) + + + --> STEP: 66/234 -- GLOBAL_STEP: 58800 + | > loss: -0.26097 (-0.27927) + | > log_mle: -0.34672 (-0.36719) + | > loss_dur: 0.08575 (0.08792) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.41049 (12.85588) + | > current_lr: 0.00006 + | > step_time: 1.20180 (2.43097) + | > loader_time: 0.00180 (0.04284) + + + --> STEP: 71/234 -- GLOBAL_STEP: 58805 + | > loss: -0.24345 (-0.27635) + | > log_mle: -0.37221 (-0.36615) + | > loss_dur: 0.12876 (0.08980) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.06835 (12.78893) + | > current_lr: 0.00006 + | > step_time: 2.24500 (2.37234) + | > loader_time: 0.00250 (0.04094) + + + --> STEP: 76/234 -- GLOBAL_STEP: 58810 + | > loss: -0.24571 (-0.27369) + | > log_mle: -0.35502 (-0.36518) + | > loss_dur: 0.10931 (0.09150) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.79341 (12.77806) + | > current_lr: 0.00006 + | > step_time: 1.90280 (2.34291) + | > loader_time: 0.00260 (0.03973) + + + --> STEP: 81/234 -- GLOBAL_STEP: 58815 + | > loss: -0.24824 (-0.27188) + | > log_mle: -0.36333 (-0.36438) + | > loss_dur: 0.11510 (0.09250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.76257 (12.79573) + | > current_lr: 0.00006 + | > step_time: 4.10090 (2.36882) + | > loader_time: 0.00220 (0.03970) + + + --> STEP: 86/234 -- GLOBAL_STEP: 58820 + | > loss: -0.24492 (-0.26986) + | > log_mle: -0.36265 (-0.36375) + | > loss_dur: 0.11774 (0.09389) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.58149 (13.10695) + | > current_lr: 0.00006 + | > step_time: 3.25870 (2.35623) + | > loader_time: 0.00860 (0.03864) + + + --> STEP: 91/234 -- GLOBAL_STEP: 58825 + | > loss: -0.23689 (-0.26846) + | > log_mle: -0.37250 (-0.36428) + | > loss_dur: 0.13561 (0.09582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.95533 (13.23286) + | > current_lr: 0.00006 + | > step_time: 1.99200 (2.32464) + | > loader_time: 0.00200 (0.03762) + + + --> STEP: 96/234 -- GLOBAL_STEP: 58830 + | > loss: -0.23850 (-0.26804) + | > log_mle: -0.35902 (-0.36633) + | > loss_dur: 0.12052 (0.09828) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.81775 (13.74236) + | > current_lr: 0.00006 + | > step_time: 1.78990 (2.34006) + | > loader_time: 0.00320 (0.03671) + + + --> STEP: 101/234 -- GLOBAL_STEP: 58835 + | > loss: -0.22970 (-0.26688) + | > log_mle: -0.39885 (-0.36730) + | > loss_dur: 0.16914 (0.10042) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.70546 (14.20721) + | > current_lr: 0.00006 + | > step_time: 2.49850 (2.33605) + | > loader_time: 0.00270 (0.03505) + + + --> STEP: 106/234 -- GLOBAL_STEP: 58840 + | > loss: -0.20904 (-0.26547) + | > log_mle: -0.38970 (-0.36828) + | > loss_dur: 0.18067 (0.10280) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.23179 (14.99756) + | > current_lr: 0.00006 + | > step_time: 2.40060 (2.32111) + | > loader_time: 0.00300 (0.03354) + + + --> STEP: 111/234 -- GLOBAL_STEP: 58845 + | > loss: -0.26172 (-0.26418) + | > log_mle: -0.45351 (-0.36977) + | > loss_dur: 0.19179 (0.10559) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.59800 (15.21124) + | > current_lr: 0.00006 + | > step_time: 4.79800 (2.35236) + | > loader_time: 0.00270 (0.03383) + + + --> STEP: 116/234 -- GLOBAL_STEP: 58850 + | > loss: -0.22594 (-0.26308) + | > log_mle: -0.41124 (-0.37123) + | > loss_dur: 0.18531 (0.10815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.36778 (15.83651) + | > current_lr: 0.00006 + | > step_time: 2.24610 (2.36523) + | > loader_time: 0.08240 (0.03394) + + + --> STEP: 121/234 -- GLOBAL_STEP: 58855 + | > loss: -0.20416 (-0.26207) + | > log_mle: -0.33583 (-0.37212) + | > loss_dur: 0.13167 (0.11004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.78266 (16.01547) + | > current_lr: 0.00006 + | > step_time: 1.60160 (2.34446) + | > loader_time: 0.07560 (0.03325) + + + --> STEP: 126/234 -- GLOBAL_STEP: 58860 + | > loss: -0.28301 (-0.26149) + | > log_mle: -0.46287 (-0.37360) + | > loss_dur: 0.17986 (0.11211) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.91080 (16.53559) + | > current_lr: 0.00006 + | > step_time: 2.61210 (2.33212) + | > loader_time: 0.08480 (0.03269) + + + --> STEP: 131/234 -- GLOBAL_STEP: 58865 + | > loss: -0.32503 (-0.26208) + | > log_mle: -0.51319 (-0.37644) + | > loss_dur: 0.18816 (0.11436) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.33238 (17.28056) + | > current_lr: 0.00006 + | > step_time: 2.21150 (2.34642) + | > loader_time: 0.00180 (0.03290) + + + --> STEP: 136/234 -- GLOBAL_STEP: 58870 + | > loss: -0.36374 (-0.26295) + | > log_mle: -0.56704 (-0.37944) + | > loss_dur: 0.20330 (0.11649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.73368 (17.86824) + | > current_lr: 0.00006 + | > step_time: 2.00440 (2.33555) + | > loader_time: 0.08920 (0.03259) + + + --> STEP: 141/234 -- GLOBAL_STEP: 58875 + | > loss: -0.29721 (-0.26314) + | > log_mle: -0.46564 (-0.38187) + | > loss_dur: 0.16843 (0.11873) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.56937 (18.67308) + | > current_lr: 0.00006 + | > step_time: 3.31310 (2.34005) + | > loader_time: 0.08640 (0.03276) + + + --> STEP: 146/234 -- GLOBAL_STEP: 58880 + | > loss: -0.32060 (-0.26470) + | > log_mle: -0.51751 (-0.38617) + | > loss_dur: 0.19692 (0.12147) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.32917 (19.63124) + | > current_lr: 0.00006 + | > step_time: 3.28730 (2.34450) + | > loader_time: 0.00190 (0.03180) + + + --> STEP: 151/234 -- GLOBAL_STEP: 58885 + | > loss: -0.29258 (-0.26619) + | > log_mle: -0.48038 (-0.38962) + | > loss_dur: 0.18780 (0.12343) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.26100 (20.46034) + | > current_lr: 0.00006 + | > step_time: 2.35110 (2.32992) + | > loader_time: 0.08940 (0.03141) + + + --> STEP: 156/234 -- GLOBAL_STEP: 58890 + | > loss: -0.34082 (-0.26888) + | > log_mle: -0.53041 (-0.39468) + | > loss_dur: 0.18959 (0.12580) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.75187 (21.57532) + | > current_lr: 0.00006 + | > step_time: 1.21650 (2.33891) + | > loader_time: 0.08950 (0.03226) + + + --> STEP: 161/234 -- GLOBAL_STEP: 58895 + | > loss: -0.35204 (-0.27074) + | > log_mle: -0.53516 (-0.39879) + | > loss_dur: 0.18312 (0.12805) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.01839 (22.65202) + | > current_lr: 0.00006 + | > step_time: 1.11880 (2.39089) + | > loader_time: 0.00250 (0.03305) + + + --> STEP: 166/234 -- GLOBAL_STEP: 58900 + | > loss: -0.27621 (-0.27084) + | > log_mle: -0.44932 (-0.40086) + | > loss_dur: 0.17311 (0.13002) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.50116 (24.75827) + | > current_lr: 0.00006 + | > step_time: 3.26390 (2.39520) + | > loader_time: 0.00400 (0.03398) + + + --> STEP: 171/234 -- GLOBAL_STEP: 58905 + | > loss: -0.37123 (-0.27275) + | > log_mle: -0.57605 (-0.40504) + | > loss_dur: 0.20482 (0.13229) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.65681 (25.38410) + | > current_lr: 0.00006 + | > step_time: 5.88810 (2.47601) + | > loader_time: 0.11300 (0.03600) + + + --> STEP: 176/234 -- GLOBAL_STEP: 58910 + | > loss: -0.35856 (-0.27506) + | > log_mle: -0.56484 (-0.40971) + | > loss_dur: 0.20628 (0.13465) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.61970 (26.02198) + | > current_lr: 0.00006 + | > step_time: 4.01370 (2.54501) + | > loader_time: 0.18300 (0.03771) + + + --> STEP: 181/234 -- GLOBAL_STEP: 58915 + | > loss: -0.28192 (-0.27687) + | > log_mle: -0.48934 (-0.41388) + | > loss_dur: 0.20742 (0.13701) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.89811 (26.78581) + | > current_lr: 0.00006 + | > step_time: 2.01290 (2.55922) + | > loader_time: 0.07580 (0.03816) + + + --> STEP: 186/234 -- GLOBAL_STEP: 58920 + | > loss: -0.30463 (-0.27886) + | > log_mle: -0.53663 (-0.41826) + | > loss_dur: 0.23200 (0.13940) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.64643 (27.70769) + | > current_lr: 0.00006 + | > step_time: 4.10160 (2.61845) + | > loader_time: 0.00330 (0.03878) + + + --> STEP: 191/234 -- GLOBAL_STEP: 58925 + | > loss: -0.36444 (-0.28134) + | > log_mle: -0.57016 (-0.42272) + | > loss_dur: 0.20572 (0.14137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.62025 (28.70125) + | > current_lr: 0.00006 + | > step_time: 4.99600 (2.66618) + | > loader_time: 0.09450 (0.04031) + + + --> STEP: 196/234 -- GLOBAL_STEP: 58930 + | > loss: -0.32452 (-0.28393) + | > log_mle: -0.55202 (-0.42722) + | > loss_dur: 0.22749 (0.14329) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.16547 (29.87690) + | > current_lr: 0.00006 + | > step_time: 11.89870 (2.75439) + | > loader_time: 0.08760 (0.04166) + + + --> STEP: 201/234 -- GLOBAL_STEP: 58935 + | > loss: -0.29290 (-0.28601) + | > log_mle: -0.52271 (-0.43130) + | > loss_dur: 0.22981 (0.14530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.92621 (30.65444) + | > current_lr: 0.00006 + | > step_time: 2.70780 (2.83627) + | > loader_time: 0.08420 (0.04285) + + + --> STEP: 206/234 -- GLOBAL_STEP: 58940 + | > loss: -0.40786 (-0.28883) + | > log_mle: -0.62905 (-0.43602) + | > loss_dur: 0.22119 (0.14719) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.16295 (31.80946) + | > current_lr: 0.00006 + | > step_time: 3.50360 (2.87709) + | > loader_time: 0.00380 (0.04372) + + + --> STEP: 211/234 -- GLOBAL_STEP: 58945 + | > loss: -0.46044 (-0.29220) + | > log_mle: -0.70523 (-0.44137) + | > loss_dur: 0.24478 (0.14917) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.02096 (32.86782) + | > current_lr: 0.00006 + | > step_time: 2.70870 (2.90701) + | > loader_time: 0.09980 (0.04367) + + + --> STEP: 216/234 -- GLOBAL_STEP: 58950 + | > loss: -0.44904 (-0.29545) + | > log_mle: -0.69773 (-0.44651) + | > loss_dur: 0.24869 (0.15105) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.56561 (33.88756) + | > current_lr: 0.00006 + | > step_time: 4.99960 (2.93592) + | > loader_time: 0.01020 (0.04327) + + + --> STEP: 221/234 -- GLOBAL_STEP: 58955 + | > loss: -0.39434 (-0.29875) + | > log_mle: -0.61075 (-0.45164) + | > loss_dur: 0.21641 (0.15289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.24446 (35.05302) + | > current_lr: 0.00006 + | > step_time: 2.59610 (3.00935) + | > loader_time: 0.00470 (0.04326) + + + --> STEP: 226/234 -- GLOBAL_STEP: 58960 + | > loss: -0.46498 (-0.30243) + | > log_mle: -0.71354 (-0.45735) + | > loss_dur: 0.24856 (0.15492) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.20709 (36.31793) + | > current_lr: 0.00006 + | > step_time: 1.90700 (2.97954) + | > loader_time: 0.39290 (0.04447) + + + --> STEP: 231/234 -- GLOBAL_STEP: 58965 + | > loss: -0.42074 (-0.30534) + | > log_mle: -0.79732 (-0.46344) + | > loss_dur: 0.37658 (0.15810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.62412 (37.50232) + | > current_lr: 0.00006 + | > step_time: 0.26620 (2.92695) + | > loader_time: 0.00350 (0.04421) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.78319 (+0.28710) + | > avg_loss: -0.31191 (+0.01382) + | > avg_log_mle: -0.54584 (-0.00063) + | > avg_loss_dur: 0.23393 (+0.01445) + + + > EPOCH: 252/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 14:14:41)  + + --> STEP: 2/234 -- GLOBAL_STEP: 58970 + | > loss: -0.31102 (-0.29911) + | > log_mle: -0.39239 (-0.38367) + | > loss_dur: 0.08136 (0.08457) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.91733 (23.18951) + | > current_lr: 0.00006 + | > step_time: 8.10460 (12.20670) + | > loader_time: 0.19200 (0.19829) + + + --> STEP: 7/234 -- GLOBAL_STEP: 58975 + | > loss: -0.31019 (-0.29023) + | > log_mle: -0.38194 (-0.38067) + | > loss_dur: 0.07175 (0.09044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.23743 (20.40862) + | > current_lr: 0.00006 + | > step_time: 4.70500 (5.99221) + | > loader_time: 0.09870 (0.11529) + + + --> STEP: 12/234 -- GLOBAL_STEP: 58980 + | > loss: -0.29684 (-0.29422) + | > log_mle: -0.38097 (-0.38400) + | > loss_dur: 0.08413 (0.08978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.56456 (18.65522) + | > current_lr: 0.00006 + | > step_time: 4.59960 (5.43587) + | > loader_time: 0.09210 (0.09044) + + + --> STEP: 17/234 -- GLOBAL_STEP: 58985 + | > loss: -0.31785 (-0.30085) + | > log_mle: -0.37978 (-0.38568) + | > loss_dur: 0.06193 (0.08483) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.90931 (17.66220) + | > current_lr: 0.00006 + | > step_time: 3.01310 (4.84432) + | > loader_time: 0.07710 (0.08141) + + + --> STEP: 22/234 -- GLOBAL_STEP: 58990 + | > loss: -0.29679 (-0.30185) + | > log_mle: -0.37918 (-0.38413) + | > loss_dur: 0.08240 (0.08228) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.91419 (16.65740) + | > current_lr: 0.00006 + | > step_time: 1.16140 (4.87388) + | > loader_time: 0.07410 (0.07109) + + + --> STEP: 27/234 -- GLOBAL_STEP: 58995 + | > loss: -0.30146 (-0.30256) + | > log_mle: -0.37612 (-0.38272) + | > loss_dur: 0.07465 (0.08017) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87832 (15.38781) + | > current_lr: 0.00006 + | > step_time: 1.40450 (4.19297) + | > loader_time: 0.00340 (0.06465) + + + --> STEP: 32/234 -- GLOBAL_STEP: 59000 + | > loss: -0.28081 (-0.30011) + | > log_mle: -0.37074 (-0.38107) + | > loss_dur: 0.08993 (0.08096) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.12466 (14.74353) + | > current_lr: 0.00006 + | > step_time: 1.39590 (3.81304) + | > loader_time: 0.00570 (0.05500) + + + --> STEP: 37/234 -- GLOBAL_STEP: 59005 + | > loss: -0.28050 (-0.29644) + | > log_mle: -0.35276 (-0.37849) + | > loss_dur: 0.07226 (0.08204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.73161 (14.35277) + | > current_lr: 0.00006 + | > step_time: 1.30860 (3.45664) + | > loader_time: 0.09060 (0.05242) + + + --> STEP: 42/234 -- GLOBAL_STEP: 59010 + | > loss: -0.26707 (-0.29365) + | > log_mle: -0.35013 (-0.37650) + | > loss_dur: 0.08305 (0.08285) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.33586 (13.97675) + | > current_lr: 0.00006 + | > step_time: 1.40000 (3.23744) + | > loader_time: 0.00240 (0.04657) + + + --> STEP: 47/234 -- GLOBAL_STEP: 59015 + | > loss: -0.27581 (-0.29122) + | > log_mle: -0.36430 (-0.37537) + | > loss_dur: 0.08850 (0.08415) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.09048 (13.77954) + | > current_lr: 0.00006 + | > step_time: 1.18810 (3.04540) + | > loader_time: 0.00170 (0.04185) + + + --> STEP: 52/234 -- GLOBAL_STEP: 59020 + | > loss: -0.24984 (-0.28933) + | > log_mle: -0.35498 (-0.37392) + | > loss_dur: 0.10514 (0.08459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.83966 (13.44052) + | > current_lr: 0.00006 + | > step_time: 1.19150 (2.89881) + | > loader_time: 0.00220 (0.03956) + + + --> STEP: 57/234 -- GLOBAL_STEP: 59025 + | > loss: -0.25104 (-0.28749) + | > log_mle: -0.34540 (-0.37271) + | > loss_dur: 0.09436 (0.08523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.98165 (13.19904) + | > current_lr: 0.00006 + | > step_time: 3.39400 (2.81700) + | > loader_time: 0.06290 (0.03886) + + + --> STEP: 62/234 -- GLOBAL_STEP: 59030 + | > loss: -0.21437 (-0.28504) + | > log_mle: -0.37466 (-0.37230) + | > loss_dur: 0.16029 (0.08726) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.32424 (13.31542) + | > current_lr: 0.00006 + | > step_time: 1.73150 (2.71749) + | > loader_time: 0.00230 (0.03595) + + + --> STEP: 67/234 -- GLOBAL_STEP: 59035 + | > loss: -0.26044 (-0.28330) + | > log_mle: -0.36585 (-0.37102) + | > loss_dur: 0.10541 (0.08772) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.77016 (13.12506) + | > current_lr: 0.00006 + | > step_time: 2.91970 (2.64644) + | > loader_time: 0.09780 (0.03488) + + + --> STEP: 72/234 -- GLOBAL_STEP: 59040 + | > loss: -0.25364 (-0.28015) + | > log_mle: -0.35464 (-0.36974) + | > loss_dur: 0.10100 (0.08959) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.54548 (13.03527) + | > current_lr: 0.00006 + | > step_time: 1.80850 (2.61652) + | > loader_time: 0.00310 (0.03268) + + + --> STEP: 77/234 -- GLOBAL_STEP: 59045 + | > loss: -0.25359 (-0.27729) + | > log_mle: -0.35741 (-0.36876) + | > loss_dur: 0.10382 (0.09147) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.84441 (12.99296) + | > current_lr: 0.00006 + | > step_time: 3.29030 (2.61395) + | > loader_time: 0.00490 (0.03079) + + + --> STEP: 82/234 -- GLOBAL_STEP: 59050 + | > loss: -0.23936 (-0.27529) + | > log_mle: -0.35131 (-0.36776) + | > loss_dur: 0.11194 (0.09248) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.59474 (12.94813) + | > current_lr: 0.00006 + | > step_time: 2.20730 (2.58992) + | > loader_time: 0.00240 (0.03138) + + + --> STEP: 87/234 -- GLOBAL_STEP: 59055 + | > loss: -0.23318 (-0.27318) + | > log_mle: -0.35178 (-0.36700) + | > loss_dur: 0.11859 (0.09381) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.70047 (13.04616) + | > current_lr: 0.00006 + | > step_time: 2.21200 (2.56316) + | > loader_time: 0.00360 (0.03152) + + + --> STEP: 92/234 -- GLOBAL_STEP: 59060 + | > loss: -0.26262 (-0.27185) + | > log_mle: -0.39233 (-0.36780) + | > loss_dur: 0.12970 (0.09595) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.81743 (13.33159) + | > current_lr: 0.00006 + | > step_time: 1.20010 (2.51868) + | > loader_time: 0.00240 (0.02997) + + + --> STEP: 97/234 -- GLOBAL_STEP: 59065 + | > loss: -0.25284 (-0.27131) + | > log_mle: -0.38033 (-0.36958) + | > loss_dur: 0.12748 (0.09827) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.27603 (13.76527) + | > current_lr: 0.00006 + | > step_time: 1.43020 (2.48729) + | > loader_time: 0.00340 (0.02946) + + + --> STEP: 102/234 -- GLOBAL_STEP: 59070 + | > loss: -0.21529 (-0.26984) + | > log_mle: -0.35868 (-0.37017) + | > loss_dur: 0.14339 (0.10033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.81202 (14.23787) + | > current_lr: 0.00006 + | > step_time: 2.39880 (2.46465) + | > loader_time: 0.08700 (0.02970) + + + --> STEP: 107/234 -- GLOBAL_STEP: 59075 + | > loss: -0.24343 (-0.26896) + | > log_mle: -0.40132 (-0.37196) + | > loss_dur: 0.15789 (0.10300) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.44244 (14.78088) + | > current_lr: 0.00006 + | > step_time: 2.89640 (2.46055) + | > loader_time: 0.00280 (0.02846) + + + --> STEP: 112/234 -- GLOBAL_STEP: 59080 + | > loss: -0.25114 (-0.26796) + | > log_mle: -0.41663 (-0.37369) + | > loss_dur: 0.16549 (0.10573) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.28306 (15.26044) + | > current_lr: 0.00006 + | > step_time: 3.27490 (2.46053) + | > loader_time: 0.00340 (0.02884) + + + --> STEP: 117/234 -- GLOBAL_STEP: 59085 + | > loss: -0.25007 (-0.26721) + | > log_mle: -0.41015 (-0.37532) + | > loss_dur: 0.16008 (0.10811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.15777 (15.74116) + | > current_lr: 0.00006 + | > step_time: 1.60980 (2.44703) + | > loader_time: 0.00210 (0.02857) + + + --> STEP: 122/234 -- GLOBAL_STEP: 59090 + | > loss: -0.23751 (-0.26631) + | > log_mle: -0.38405 (-0.37613) + | > loss_dur: 0.14654 (0.10982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.83896 (15.98201) + | > current_lr: 0.00006 + | > step_time: 1.10380 (2.42558) + | > loader_time: 0.07820 (0.02815) + + + --> STEP: 127/234 -- GLOBAL_STEP: 59095 + | > loss: -0.26328 (-0.26595) + | > log_mle: -0.44487 (-0.37816) + | > loss_dur: 0.18159 (0.11220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.94274 (16.77648) + | > current_lr: 0.00006 + | > step_time: 4.39650 (2.43860) + | > loader_time: 0.00790 (0.02862) + + + --> STEP: 132/234 -- GLOBAL_STEP: 59100 + | > loss: -0.26638 (-0.26651) + | > log_mle: -0.42554 (-0.38077) + | > loss_dur: 0.15916 (0.11426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.86331 (17.52695) + | > current_lr: 0.00006 + | > step_time: 2.79900 (2.49111) + | > loader_time: 0.09720 (0.03125) + + + --> STEP: 137/234 -- GLOBAL_STEP: 59105 + | > loss: -0.24732 (-0.26700) + | > log_mle: -0.44217 (-0.38372) + | > loss_dur: 0.19485 (0.11673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.78081 (18.28145) + | > current_lr: 0.00006 + | > step_time: 1.29510 (2.47437) + | > loader_time: 0.00310 (0.03164) + + + --> STEP: 142/234 -- GLOBAL_STEP: 59110 + | > loss: -0.27126 (-0.26729) + | > log_mle: -0.45377 (-0.38622) + | > loss_dur: 0.18252 (0.11892) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.92325 (18.87029) + | > current_lr: 0.00006 + | > step_time: 3.19650 (2.47959) + | > loader_time: 0.00260 (0.03113) + + + --> STEP: 147/234 -- GLOBAL_STEP: 59115 + | > loss: -0.27186 (-0.26897) + | > log_mle: -0.45797 (-0.39055) + | > loss_dur: 0.18611 (0.12157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.48050 (19.87519) + | > current_lr: 0.00006 + | > step_time: 5.20050 (2.52381) + | > loader_time: 0.08880 (0.03273) + + + --> STEP: 152/234 -- GLOBAL_STEP: 59120 + | > loss: -0.32222 (-0.27079) + | > log_mle: -0.53943 (-0.39454) + | > loss_dur: 0.21722 (0.12375) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.48767 (21.08899) + | > current_lr: 0.00006 + | > step_time: 5.50960 (2.59103) + | > loader_time: 0.09800 (0.03408) + + + --> STEP: 157/234 -- GLOBAL_STEP: 59125 + | > loss: -0.27270 (-0.27256) + | > log_mle: -0.46808 (-0.39861) + | > loss_dur: 0.19538 (0.12605) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.83737 (22.60494) + | > current_lr: 0.00006 + | > step_time: 1.42250 (2.59848) + | > loader_time: 0.09210 (0.03426) + + + --> STEP: 162/234 -- GLOBAL_STEP: 59130 + | > loss: -0.31588 (-0.27404) + | > log_mle: -0.50724 (-0.40243) + | > loss_dur: 0.19136 (0.12838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.96997 (23.50627) + | > current_lr: 0.00006 + | > step_time: 3.88910 (2.63305) + | > loader_time: 0.00290 (0.03544) + + + --> STEP: 167/234 -- GLOBAL_STEP: 59135 + | > loss: -0.38448 (-0.27575) + | > log_mle: -0.57824 (-0.40609) + | > loss_dur: 0.19376 (0.13034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.85651 (24.49973) + | > current_lr: 0.00006 + | > step_time: 1.06090 (2.62633) + | > loader_time: 0.00190 (0.03447) + + + --> STEP: 172/234 -- GLOBAL_STEP: 59140 + | > loss: -0.36918 (-0.27778) + | > log_mle: -0.58162 (-0.41043) + | > loss_dur: 0.21244 (0.13265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.48464 (25.29804) + | > current_lr: 0.00006 + | > step_time: 1.99920 (2.60547) + | > loader_time: 0.07750 (0.03446) + + + --> STEP: 177/234 -- GLOBAL_STEP: 59145 + | > loss: -0.33203 (-0.27977) + | > log_mle: -0.54585 (-0.41479) + | > loss_dur: 0.21382 (0.13502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.54584 (26.05455) + | > current_lr: 0.00006 + | > step_time: 4.70090 (2.66355) + | > loader_time: 0.59180 (0.03851) + + + --> STEP: 182/234 -- GLOBAL_STEP: 59150 + | > loss: -0.35352 (-0.28174) + | > log_mle: -0.59607 (-0.41928) + | > loss_dur: 0.24255 (0.13753) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.71559 (26.83974) + | > current_lr: 0.00006 + | > step_time: 3.98890 (2.68707) + | > loader_time: 0.00810 (0.03966) + + + --> STEP: 187/234 -- GLOBAL_STEP: 59155 + | > loss: -0.37650 (-0.28400) + | > log_mle: -0.60033 (-0.42383) + | > loss_dur: 0.22383 (0.13983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.87774 (27.64536) + | > current_lr: 0.00006 + | > step_time: 10.39840 (2.77631) + | > loader_time: 0.09640 (0.04066) + + + --> STEP: 192/234 -- GLOBAL_STEP: 59160 + | > loss: -0.42071 (-0.28664) + | > log_mle: -0.62911 (-0.42833) + | > loss_dur: 0.20839 (0.14169) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.68797 (28.58650) + | > current_lr: 0.00006 + | > step_time: 8.71030 (2.86142) + | > loader_time: 0.08410 (0.04102) + + + --> STEP: 197/234 -- GLOBAL_STEP: 59165 + | > loss: -0.39769 (-0.28914) + | > log_mle: -0.59529 (-0.43264) + | > loss_dur: 0.19760 (0.14350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.70217 (29.71504) + | > current_lr: 0.00006 + | > step_time: 7.69230 (2.90107) + | > loader_time: 0.80610 (0.04538) + + + --> STEP: 202/234 -- GLOBAL_STEP: 59170 + | > loss: -0.48879 (-0.29164) + | > log_mle: -0.70127 (-0.43713) + | > loss_dur: 0.21248 (0.14549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.97914 (30.78645) + | > current_lr: 0.00006 + | > step_time: 6.51650 (3.03152) + | > loader_time: 0.00320 (0.04567) + + + --> STEP: 207/234 -- GLOBAL_STEP: 59175 + | > loss: -0.45241 (-0.29424) + | > log_mle: -0.68215 (-0.44166) + | > loss_dur: 0.22974 (0.14742) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.39433 (32.01178) + | > current_lr: 0.00006 + | > step_time: 4.59160 (3.08202) + | > loader_time: 0.00780 (0.04871) + + + --> STEP: 212/234 -- GLOBAL_STEP: 59180 + | > loss: -0.43333 (-0.29740) + | > log_mle: -0.66602 (-0.44692) + | > loss_dur: 0.23269 (0.14951) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.74660 (33.18415) + | > current_lr: 0.00006 + | > step_time: 10.30330 (3.15785) + | > loader_time: 0.10720 (0.05006) + + + --> STEP: 217/234 -- GLOBAL_STEP: 59185 + | > loss: -0.44906 (-0.30074) + | > log_mle: -0.68825 (-0.45219) + | > loss_dur: 0.23919 (0.15144) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.66077 (34.47017) + | > current_lr: 0.00006 + | > step_time: 7.09170 (3.28741) + | > loader_time: 0.39800 (0.05406) + + + --> STEP: 222/234 -- GLOBAL_STEP: 59190 + | > loss: -0.43894 (-0.30407) + | > log_mle: -0.69809 (-0.45738) + | > loss_dur: 0.25915 (0.15330) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.68171 (35.77342) + | > current_lr: 0.00006 + | > step_time: 0.26400 (3.23299) + | > loader_time: 0.00540 (0.05292) + + + --> STEP: 227/234 -- GLOBAL_STEP: 59195 + | > loss: -0.42073 (-0.30761) + | > log_mle: -0.67235 (-0.46283) + | > loss_dur: 0.25163 (0.15523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.01874 (37.05067) + | > current_lr: 0.00006 + | > step_time: 0.23060 (3.16709) + | > loader_time: 0.00360 (0.05184) + + + --> STEP: 232/234 -- GLOBAL_STEP: 59200 + | > loss: -0.41531 (-0.31025) + | > log_mle: -0.88158 (-0.46948) + | > loss_dur: 0.46627 (0.15923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 127.20298 (38.59092) + | > current_lr: 0.00006 + | > step_time: 0.33430 (3.10473) + | > loader_time: 0.00650 (0.05082) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.65572 (-0.12747) + | > avg_loss: -0.28002 (+0.03188) + | > avg_log_mle: -0.52529 (+0.02055) + | > avg_loss_dur: 0.24527 (+0.01134) + + + > EPOCH: 253/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 14:27:53)  + + --> STEP: 3/234 -- GLOBAL_STEP: 59205 + | > loss: -0.23693 (-0.27243) + | > log_mle: -0.37356 (-0.38040) + | > loss_dur: 0.13663 (0.10796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.11609 (21.97540) + | > current_lr: 0.00006 + | > step_time: 6.29690 (5.19993) + | > loader_time: 0.00090 (0.00138) + + + --> STEP: 8/234 -- GLOBAL_STEP: 59210 + | > loss: -0.31949 (-0.29172) + | > log_mle: -0.39834 (-0.38324) + | > loss_dur: 0.07886 (0.09152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.70530 (18.19831) + | > current_lr: 0.00006 + | > step_time: 1.51320 (3.69146) + | > loader_time: 0.08300 (0.05858) + + + --> STEP: 13/234 -- GLOBAL_STEP: 59215 + | > loss: -0.33981 (-0.29830) + | > log_mle: -0.40491 (-0.38632) + | > loss_dur: 0.06511 (0.08803) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.46298 (17.81045) + | > current_lr: 0.00006 + | > step_time: 5.39840 (4.06219) + | > loader_time: 0.09540 (0.06629) + + + --> STEP: 18/234 -- GLOBAL_STEP: 59220 + | > loss: -0.29499 (-0.30202) + | > log_mle: -0.37274 (-0.38580) + | > loss_dur: 0.07775 (0.08378) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.29953 (16.93516) + | > current_lr: 0.00006 + | > step_time: 2.69320 (3.61559) + | > loader_time: 0.00080 (0.05940) + + + --> STEP: 23/234 -- GLOBAL_STEP: 59225 + | > loss: -0.33824 (-0.30475) + | > log_mle: -0.39842 (-0.38545) + | > loss_dur: 0.06017 (0.08070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.58586 (15.78609) + | > current_lr: 0.00006 + | > step_time: 1.71030 (3.55206) + | > loader_time: 0.07950 (0.06321) + + + --> STEP: 28/234 -- GLOBAL_STEP: 59230 + | > loss: -0.34873 (-0.30500) + | > log_mle: -0.40269 (-0.38413) + | > loss_dur: 0.05396 (0.07912) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.89542 (14.93686) + | > current_lr: 0.00006 + | > step_time: 1.29270 (3.21728) + | > loader_time: 0.00340 (0.05240) + + + --> STEP: 33/234 -- GLOBAL_STEP: 59235 + | > loss: -0.29792 (-0.30173) + | > log_mle: -0.36836 (-0.38137) + | > loss_dur: 0.07043 (0.07964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.66359 (14.33864) + | > current_lr: 0.00006 + | > step_time: 7.79940 (3.35173) + | > loader_time: 0.00610 (0.04790) + + + --> STEP: 38/234 -- GLOBAL_STEP: 59240 + | > loss: -0.26355 (-0.29650) + | > log_mle: -0.36815 (-0.37788) + | > loss_dur: 0.10460 (0.08137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.53400 (14.33192) + | > current_lr: 0.00006 + | > step_time: 2.83500 (3.35143) + | > loader_time: 0.00200 (0.05237) + + + --> STEP: 43/234 -- GLOBAL_STEP: 59245 + | > loss: -0.22721 (-0.29140) + | > log_mle: -0.35088 (-0.37467) + | > loss_dur: 0.12367 (0.08326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42977 (13.91932) + | > current_lr: 0.00006 + | > step_time: 1.15440 (3.11356) + | > loader_time: 0.00120 (0.04650) + + + --> STEP: 48/234 -- GLOBAL_STEP: 59250 + | > loss: -0.27846 (-0.28854) + | > log_mle: -0.35933 (-0.37277) + | > loss_dur: 0.08087 (0.08423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.22559 (13.57534) + | > current_lr: 0.00006 + | > step_time: 2.60830 (2.93907) + | > loader_time: 0.00170 (0.04364) + + + --> STEP: 53/234 -- GLOBAL_STEP: 59255 + | > loss: -0.25212 (-0.28564) + | > log_mle: -0.35987 (-0.37099) + | > loss_dur: 0.10775 (0.08535) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.65261 (13.23031) + | > current_lr: 0.00006 + | > step_time: 0.99330 (2.89014) + | > loader_time: 0.00160 (0.04511) + + + --> STEP: 58/234 -- GLOBAL_STEP: 59260 + | > loss: -0.28046 (-0.28383) + | > log_mle: -0.35978 (-0.36970) + | > loss_dur: 0.07932 (0.08587) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.86199 (12.97640) + | > current_lr: 0.00006 + | > step_time: 1.50430 (2.76877) + | > loader_time: 0.00310 (0.04416) + + + --> STEP: 63/234 -- GLOBAL_STEP: 59265 + | > loss: -0.24150 (-0.28085) + | > log_mle: -0.34654 (-0.36901) + | > loss_dur: 0.10505 (0.08816) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.97439 (13.22306) + | > current_lr: 0.00006 + | > step_time: 3.49100 (2.69493) + | > loader_time: 0.09760 (0.04491) + + + --> STEP: 68/234 -- GLOBAL_STEP: 59270 + | > loss: -0.23239 (-0.27879) + | > log_mle: -0.34468 (-0.36781) + | > loss_dur: 0.11228 (0.08901) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.84625 (13.13394) + | > current_lr: 0.00006 + | > step_time: 1.30660 (2.63187) + | > loader_time: 0.08950 (0.04305) + + + --> STEP: 73/234 -- GLOBAL_STEP: 59275 + | > loss: -0.22284 (-0.27602) + | > log_mle: -0.35736 (-0.36682) + | > loss_dur: 0.13452 (0.09080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.57909 (13.07123) + | > current_lr: 0.00006 + | > step_time: 1.19620 (2.54484) + | > loader_time: 0.00280 (0.04135) + + + --> STEP: 78/234 -- GLOBAL_STEP: 59280 + | > loss: -0.23930 (-0.27359) + | > log_mle: -0.34200 (-0.36577) + | > loss_dur: 0.10269 (0.09218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.47102 (13.01571) + | > current_lr: 0.00006 + | > step_time: 2.91720 (2.53204) + | > loader_time: 0.08490 (0.04180) + + + --> STEP: 83/234 -- GLOBAL_STEP: 59285 + | > loss: -0.22089 (-0.27158) + | > log_mle: -0.35892 (-0.36512) + | > loss_dur: 0.13803 (0.09354) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.03116 (12.95170) + | > current_lr: 0.00006 + | > step_time: 2.01330 (2.49151) + | > loader_time: 0.00640 (0.04036) + + + --> STEP: 88/234 -- GLOBAL_STEP: 59290 + | > loss: -0.24442 (-0.26997) + | > log_mle: -0.39403 (-0.36498) + | > loss_dur: 0.14961 (0.09502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.33171 (12.89703) + | > current_lr: 0.00006 + | > step_time: 1.67760 (2.48061) + | > loader_time: 0.00200 (0.04038) + + + --> STEP: 93/234 -- GLOBAL_STEP: 59295 + | > loss: -0.24773 (-0.26866) + | > log_mle: -0.40249 (-0.36600) + | > loss_dur: 0.15476 (0.09734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.57374 (13.12825) + | > current_lr: 0.00006 + | > step_time: 2.89350 (2.45998) + | > loader_time: 0.00260 (0.03836) + + + --> STEP: 98/234 -- GLOBAL_STEP: 59300 + | > loss: -0.23780 (-0.26801) + | > log_mle: -0.34200 (-0.36712) + | > loss_dur: 0.10420 (0.09911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.18849 (13.35352) + | > current_lr: 0.00006 + | > step_time: 2.89050 (2.43939) + | > loader_time: 0.00320 (0.03742) + + + --> STEP: 103/234 -- GLOBAL_STEP: 59305 + | > loss: -0.27772 (-0.26705) + | > log_mle: -0.43292 (-0.36891) + | > loss_dur: 0.15520 (0.10186) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.02663 (13.90577) + | > current_lr: 0.00006 + | > step_time: 2.10830 (2.42231) + | > loader_time: 0.08520 (0.03883) + + + --> STEP: 108/234 -- GLOBAL_STEP: 59310 + | > loss: -0.25227 (-0.26646) + | > log_mle: -0.37874 (-0.37032) + | > loss_dur: 0.12648 (0.10386) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.58648 (14.32944) + | > current_lr: 0.00006 + | > step_time: 2.28590 (2.40100) + | > loader_time: 0.00140 (0.03892) + + + --> STEP: 113/234 -- GLOBAL_STEP: 59315 + | > loss: -0.26151 (-0.26571) + | > log_mle: -0.42147 (-0.37256) + | > loss_dur: 0.15996 (0.10684) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.75156 (14.98900) + | > current_lr: 0.00006 + | > step_time: 1.68870 (2.36389) + | > loader_time: 0.00190 (0.03807) + + + --> STEP: 118/234 -- GLOBAL_STEP: 59320 + | > loss: -0.22888 (-0.26470) + | > log_mle: -0.39598 (-0.37403) + | > loss_dur: 0.16710 (0.10933) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.78691 (15.46457) + | > current_lr: 0.00006 + | > step_time: 1.40050 (2.35155) + | > loader_time: 0.07470 (0.03718) + + + --> STEP: 123/234 -- GLOBAL_STEP: 59325 + | > loss: -0.20971 (-0.26349) + | > log_mle: -0.36351 (-0.37461) + | > loss_dur: 0.15380 (0.11111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.48108 (15.66633) + | > current_lr: 0.00006 + | > step_time: 2.40190 (2.33893) + | > loader_time: 0.00470 (0.03653) + + + --> STEP: 128/234 -- GLOBAL_STEP: 59330 + | > loss: -0.28023 (-0.26386) + | > log_mle: -0.41896 (-0.37712) + | > loss_dur: 0.13873 (0.11326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.60231 (16.36216) + | > current_lr: 0.00006 + | > step_time: 4.51330 (2.36179) + | > loader_time: 0.18670 (0.03801) + + + --> STEP: 133/234 -- GLOBAL_STEP: 59335 + | > loss: -0.27752 (-0.26430) + | > log_mle: -0.45427 (-0.38006) + | > loss_dur: 0.17675 (0.11576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.24844 (17.05124) + | > current_lr: 0.00006 + | > step_time: 1.27280 (2.36010) + | > loader_time: 0.00260 (0.03808) + + + --> STEP: 138/234 -- GLOBAL_STEP: 59340 + | > loss: -0.23017 (-0.26467) + | > log_mle: -0.39990 (-0.38276) + | > loss_dur: 0.16972 (0.11809) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.58633 (17.68478) + | > current_lr: 0.00006 + | > step_time: 2.71300 (2.36401) + | > loader_time: 0.09410 (0.03867) + + + --> STEP: 143/234 -- GLOBAL_STEP: 59345 + | > loss: -0.33123 (-0.26586) + | > log_mle: -0.55539 (-0.38646) + | > loss_dur: 0.22416 (0.12060) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.92851 (18.54686) + | > current_lr: 0.00006 + | > step_time: 2.20140 (2.35775) + | > loader_time: 0.00370 (0.03931) + + + --> STEP: 148/234 -- GLOBAL_STEP: 59350 + | > loss: -0.31215 (-0.26743) + | > log_mle: -0.46372 (-0.39009) + | > loss_dur: 0.15157 (0.12266) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.15055 (19.43820) + | > current_lr: 0.00006 + | > step_time: 1.40720 (2.34171) + | > loader_time: 0.08240 (0.04026) + + + --> STEP: 153/234 -- GLOBAL_STEP: 59355 + | > loss: -0.39944 (-0.26987) + | > log_mle: -0.59717 (-0.39499) + | > loss_dur: 0.19774 (0.12512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.12321 (20.52729) + | > current_lr: 0.00006 + | > step_time: 3.30180 (2.35529) + | > loader_time: 0.09350 (0.04198) + + + --> STEP: 158/234 -- GLOBAL_STEP: 59360 + | > loss: -0.32597 (-0.27190) + | > log_mle: -0.53176 (-0.39933) + | > loss_dur: 0.20579 (0.12744) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.17513 (21.48544) + | > current_lr: 0.00006 + | > step_time: 1.49830 (2.36420) + | > loader_time: 0.09270 (0.04135) + + + --> STEP: 163/234 -- GLOBAL_STEP: 59365 + | > loss: -0.29520 (-0.27389) + | > log_mle: -0.48647 (-0.40330) + | > loss_dur: 0.19127 (0.12941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.46295 (22.57410) + | > current_lr: 0.00006 + | > step_time: 2.49650 (2.36725) + | > loader_time: 0.01040 (0.04022) + + + --> STEP: 168/234 -- GLOBAL_STEP: 59370 + | > loss: -0.33894 (-0.27615) + | > log_mle: -0.55272 (-0.40759) + | > loss_dur: 0.21378 (0.13145) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.24217 (23.65579) + | > current_lr: 0.00006 + | > step_time: 2.39140 (2.42370) + | > loader_time: 0.00380 (0.04016) + + + --> STEP: 173/234 -- GLOBAL_STEP: 59375 + | > loss: -0.35983 (-0.27902) + | > log_mle: -0.56460 (-0.41273) + | > loss_dur: 0.20477 (0.13371) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.23396 (24.59146) + | > current_lr: 0.00006 + | > step_time: 6.00460 (2.44560) + | > loader_time: 0.09870 (0.04129) + + + --> STEP: 178/234 -- GLOBAL_STEP: 59380 + | > loss: -0.40961 (-0.28188) + | > log_mle: -0.63295 (-0.41790) + | > loss_dur: 0.22334 (0.13602) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.40713 (25.66868) + | > current_lr: 0.00006 + | > step_time: 3.28710 (2.49201) + | > loader_time: 0.00190 (0.04197) + + + --> STEP: 183/234 -- GLOBAL_STEP: 59385 + | > loss: -0.40974 (-0.28416) + | > log_mle: -0.61726 (-0.42247) + | > loss_dur: 0.20752 (0.13831) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.74721 (26.83908) + | > current_lr: 0.00006 + | > step_time: 4.56620 (2.49166) + | > loader_time: 0.08680 (0.04222) + + + --> STEP: 188/234 -- GLOBAL_STEP: 59390 + | > loss: -0.42144 (-0.28673) + | > log_mle: -0.63528 (-0.42717) + | > loss_dur: 0.21384 (0.14044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.67812 (28.07005) + | > current_lr: 0.00006 + | > step_time: 2.91450 (2.48475) + | > loader_time: 0.08440 (0.04253) + + + --> STEP: 193/234 -- GLOBAL_STEP: 59395 + | > loss: -0.42735 (-0.28944) + | > log_mle: -0.63248 (-0.43166) + | > loss_dur: 0.20513 (0.14222) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.27151 (29.03909) + | > current_lr: 0.00006 + | > step_time: 6.80100 (2.57846) + | > loader_time: 0.08810 (0.04540) + + + --> STEP: 198/234 -- GLOBAL_STEP: 59400 + | > loss: -0.41043 (-0.29214) + | > log_mle: -0.62800 (-0.43608) + | > loss_dur: 0.21757 (0.14394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.83012 (29.84407) + | > current_lr: 0.00006 + | > step_time: 2.10630 (2.64027) + | > loader_time: 0.08490 (0.04569) + + + --> STEP: 203/234 -- GLOBAL_STEP: 59405 + | > loss: -0.32154 (-0.29417) + | > log_mle: -0.54282 (-0.44014) + | > loss_dur: 0.22128 (0.14597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.70569 (30.86366) + | > current_lr: 0.00006 + | > step_time: 4.49930 (2.66963) + | > loader_time: 0.00430 (0.04564) + + + --> STEP: 208/234 -- GLOBAL_STEP: 59410 + | > loss: -0.39721 (-0.29698) + | > log_mle: -0.62658 (-0.44498) + | > loss_dur: 0.22937 (0.14799) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.81786 (31.89409) + | > current_lr: 0.00006 + | > step_time: 7.40020 (2.78811) + | > loader_time: 0.20150 (0.04894) + + + --> STEP: 213/234 -- GLOBAL_STEP: 59415 + | > loss: -0.45888 (-0.30038) + | > log_mle: -0.69239 (-0.45037) + | > loss_dur: 0.23351 (0.15000) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.59845 (32.96701) + | > current_lr: 0.00006 + | > step_time: 6.79040 (2.85498) + | > loader_time: 0.00290 (0.05000) + + + --> STEP: 218/234 -- GLOBAL_STEP: 59420 + | > loss: -0.40523 (-0.30330) + | > log_mle: -0.63507 (-0.45515) + | > loss_dur: 0.22984 (0.15186) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.60940 (33.97652) + | > current_lr: 0.00006 + | > step_time: 4.59810 (2.92300) + | > loader_time: 0.00620 (0.05195) + + + --> STEP: 223/234 -- GLOBAL_STEP: 59425 + | > loss: -0.45814 (-0.30645) + | > log_mle: -0.69282 (-0.46027) + | > loss_dur: 0.23468 (0.15382) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.54507 (35.07898) + | > current_lr: 0.00006 + | > step_time: 2.02000 (2.92028) + | > loader_time: 0.00250 (0.05130) + + + --> STEP: 228/234 -- GLOBAL_STEP: 59430 + | > loss: -0.42376 (-0.30953) + | > log_mle: -0.68661 (-0.46550) + | > loss_dur: 0.26285 (0.15596) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 101.26012 (36.46212) + | > current_lr: 0.00006 + | > step_time: 0.25000 (2.86145) + | > loader_time: 0.00300 (0.05024) + + + --> STEP: 233/234 -- GLOBAL_STEP: 59435 + | > loss: 0.04917 (-0.31034) + | > log_mle: -0.64270 (-0.47214) + | > loss_dur: 0.69187 (0.16180) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 108.87881 (37.74629) + | > current_lr: 0.00006 + | > step_time: 0.18730 (2.80579) + | > loader_time: 0.00260 (0.04924) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.14772 (-0.50800) + | > avg_loss: -0.30032 (-0.02029) + | > avg_log_mle: -0.52848 (-0.00318) + | > avg_loss_dur: 0.22816 (-0.01711) + + + > EPOCH: 254/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 14:40:00)  + + --> STEP: 4/234 -- GLOBAL_STEP: 59440 + | > loss: -0.29068 (-0.27180) + | > log_mle: -0.38477 (-0.38319) + | > loss_dur: 0.09410 (0.11139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.09476 (19.58913) + | > current_lr: 0.00006 + | > step_time: 3.29480 (3.40377) + | > loader_time: 0.00300 (0.66806) + + + --> STEP: 9/234 -- GLOBAL_STEP: 59445 + | > loss: -0.28490 (-0.28883) + | > log_mle: -0.39750 (-0.38703) + | > loss_dur: 0.11260 (0.09820) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.58778 (18.43491) + | > current_lr: 0.00006 + | > step_time: 4.30900 (5.73775) + | > loader_time: 0.09860 (0.80605) + + + --> STEP: 14/234 -- GLOBAL_STEP: 59450 + | > loss: -0.29901 (-0.29829) + | > log_mle: -0.38935 (-0.38885) + | > loss_dur: 0.09033 (0.09057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.55068 (17.35675) + | > current_lr: 0.00006 + | > step_time: 1.20270 (4.25900) + | > loader_time: 0.00410 (0.51926) + + + --> STEP: 19/234 -- GLOBAL_STEP: 59455 + | > loss: -0.31520 (-0.30256) + | > log_mle: -0.38692 (-0.38850) + | > loss_dur: 0.07171 (0.08594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.96198 (16.18762) + | > current_lr: 0.00006 + | > step_time: 2.70900 (3.99510) + | > loader_time: 0.00180 (0.38832) + + + --> STEP: 24/234 -- GLOBAL_STEP: 59460 + | > loss: -0.31841 (-0.30473) + | > log_mle: -0.37880 (-0.38771) + | > loss_dur: 0.06040 (0.08298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.12157 (15.23614) + | > current_lr: 0.00006 + | > step_time: 1.47280 (3.45590) + | > loader_time: 0.01410 (0.31204) + + + --> STEP: 29/234 -- GLOBAL_STEP: 59465 + | > loss: -0.28610 (-0.30446) + | > log_mle: -0.37044 (-0.38602) + | > loss_dur: 0.08434 (0.08156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.21765 (14.56125) + | > current_lr: 0.00006 + | > step_time: 3.49900 (3.57107) + | > loader_time: 0.00550 (0.26211) + + + --> STEP: 34/234 -- GLOBAL_STEP: 59470 + | > loss: -0.28679 (-0.30154) + | > log_mle: -0.37026 (-0.38359) + | > loss_dur: 0.08348 (0.08205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.39674 (13.98389) + | > current_lr: 0.00006 + | > step_time: 3.09450 (3.55592) + | > loader_time: 0.00420 (0.22889) + + + --> STEP: 39/234 -- GLOBAL_STEP: 59475 + | > loss: -0.27334 (-0.29758) + | > log_mle: -0.36560 (-0.38077) + | > loss_dur: 0.09226 (0.08319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.49409 (14.15337) + | > current_lr: 0.00006 + | > step_time: 1.00740 (3.32781) + | > loader_time: 0.00170 (0.20448) + + + --> STEP: 44/234 -- GLOBAL_STEP: 59480 + | > loss: -0.28480 (-0.29445) + | > log_mle: -0.35799 (-0.37840) + | > loss_dur: 0.07319 (0.08395) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.86105 (13.74602) + | > current_lr: 0.00006 + | > step_time: 1.40060 (3.11349) + | > loader_time: 0.09890 (0.18368) + + + --> STEP: 49/234 -- GLOBAL_STEP: 59485 + | > loss: -0.28582 (-0.29284) + | > log_mle: -0.37097 (-0.37750) + | > loss_dur: 0.08516 (0.08466) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.14481 (13.39001) + | > current_lr: 0.00006 + | > step_time: 0.99750 (2.95189) + | > loader_time: 0.00200 (0.16516) + + + --> STEP: 54/234 -- GLOBAL_STEP: 59490 + | > loss: -0.28657 (-0.29055) + | > log_mle: -0.36701 (-0.37594) + | > loss_dur: 0.08044 (0.08539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.23333 (13.05296) + | > current_lr: 0.00006 + | > step_time: 1.60530 (2.80833) + | > loader_time: 0.00130 (0.15156) + + + --> STEP: 59/234 -- GLOBAL_STEP: 59495 + | > loss: -0.27143 (-0.28852) + | > log_mle: -0.36866 (-0.37484) + | > loss_dur: 0.09723 (0.08631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.28652 (12.81853) + | > current_lr: 0.00006 + | > step_time: 1.41370 (2.75568) + | > loader_time: 0.08450 (0.14340) + + + --> STEP: 64/234 -- GLOBAL_STEP: 59500 + | > loss: -0.26702 (-0.28531) + | > log_mle: -0.35358 (-0.37392) + | > loss_dur: 0.08657 (0.08862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.05375 (12.75570) + | > current_lr: 0.00006 + | > step_time: 1.40740 (2.65289) + | > loader_time: 0.08270 (0.13489) + + + --> STEP: 69/234 -- GLOBAL_STEP: 59505 + | > loss: -0.24683 (-0.28288) + | > log_mle: -0.34435 (-0.37237) + | > loss_dur: 0.09752 (0.08949) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.95192 (12.69267) + | > current_lr: 0.00006 + | > step_time: 1.72110 (2.56964) + | > loader_time: 0.00300 (0.12533) + + + --> STEP: 74/234 -- GLOBAL_STEP: 59510 + | > loss: -0.23031 (-0.27992) + | > log_mle: -0.34335 (-0.37122) + | > loss_dur: 0.11305 (0.09130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.15863 (12.75107) + | > current_lr: 0.00006 + | > step_time: 2.10940 (2.51727) + | > loader_time: 0.08580 (0.11917) + + + --> STEP: 79/234 -- GLOBAL_STEP: 59515 + | > loss: -0.25221 (-0.27750) + | > log_mle: -0.35972 (-0.37019) + | > loss_dur: 0.10751 (0.09269) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.10824 (12.70537) + | > current_lr: 0.00006 + | > step_time: 1.72070 (2.47030) + | > loader_time: 0.00140 (0.11379) + + + --> STEP: 84/234 -- GLOBAL_STEP: 59520 + | > loss: -0.25160 (-0.27514) + | > log_mle: -0.35334 (-0.36920) + | > loss_dur: 0.10174 (0.09406) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.41536 (12.80799) + | > current_lr: 0.00006 + | > step_time: 1.41150 (2.43481) + | > loader_time: 0.07980 (0.10899) + + + --> STEP: 89/234 -- GLOBAL_STEP: 59525 + | > loss: -0.25731 (-0.27345) + | > log_mle: -0.37847 (-0.36911) + | > loss_dur: 0.12116 (0.09567) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.72904 (12.84426) + | > current_lr: 0.00006 + | > step_time: 1.70430 (2.46768) + | > loader_time: 0.08740 (0.10613) + + + --> STEP: 94/234 -- GLOBAL_STEP: 59530 + | > loss: -0.27261 (-0.27234) + | > log_mle: -0.40571 (-0.37023) + | > loss_dur: 0.13310 (0.09789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.68481 (13.19990) + | > current_lr: 0.00006 + | > step_time: 1.43730 (2.42409) + | > loader_time: 0.00290 (0.10241) + + + --> STEP: 99/234 -- GLOBAL_STEP: 59535 + | > loss: -0.27790 (-0.27148) + | > log_mle: -0.43991 (-0.37154) + | > loss_dur: 0.16201 (0.10006) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.10704 (13.59750) + | > current_lr: 0.00006 + | > step_time: 2.81290 (2.37990) + | > loader_time: 0.08490 (0.09818) + + + --> STEP: 104/234 -- GLOBAL_STEP: 59540 + | > loss: -0.29813 (-0.27098) + | > log_mle: -0.45185 (-0.37334) + | > loss_dur: 0.15372 (0.10236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.47815 (14.14961) + | > current_lr: 0.00006 + | > step_time: 1.51120 (2.34162) + | > loader_time: 0.00260 (0.09517) + + + --> STEP: 109/234 -- GLOBAL_STEP: 59545 + | > loss: -0.22917 (-0.26946) + | > log_mle: -0.41947 (-0.37434) + | > loss_dur: 0.19030 (0.10487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.54590 (14.52637) + | > current_lr: 0.00006 + | > step_time: 2.69660 (2.33254) + | > loader_time: 0.00510 (0.09171) + + + --> STEP: 114/234 -- GLOBAL_STEP: 59550 + | > loss: -0.25315 (-0.26876) + | > log_mle: -0.40233 (-0.37626) + | > loss_dur: 0.14917 (0.10750) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.82253 (15.05973) + | > current_lr: 0.00006 + | > step_time: 2.20890 (2.32159) + | > loader_time: 0.08380 (0.08857) + + + --> STEP: 119/234 -- GLOBAL_STEP: 59555 + | > loss: -0.24711 (-0.26760) + | > log_mle: -0.40065 (-0.37758) + | > loss_dur: 0.15353 (0.10998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.10299 (15.54007) + | > current_lr: 0.00006 + | > step_time: 7.49390 (2.36877) + | > loader_time: 0.00290 (0.08723) + + + --> STEP: 124/234 -- GLOBAL_STEP: 59560 + | > loss: -0.26933 (-0.26650) + | > log_mle: -0.42338 (-0.37813) + | > loss_dur: 0.15405 (0.11163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.78085 (15.89457) + | > current_lr: 0.00006 + | > step_time: 1.00390 (2.40455) + | > loader_time: 0.00960 (0.08565) + + + --> STEP: 129/234 -- GLOBAL_STEP: 59565 + | > loss: -0.23720 (-0.26635) + | > log_mle: -0.41382 (-0.38028) + | > loss_dur: 0.17662 (0.11394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.24177 (16.87300) + | > current_lr: 0.00006 + | > step_time: 4.09790 (2.40937) + | > loader_time: 0.00490 (0.08450) + + + --> STEP: 134/234 -- GLOBAL_STEP: 59570 + | > loss: -0.28419 (-0.26703) + | > log_mle: -0.46775 (-0.38327) + | > loss_dur: 0.18356 (0.11623) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.16269 (17.71659) + | > current_lr: 0.00006 + | > step_time: 3.50340 (2.41823) + | > loader_time: 0.09700 (0.08464) + + + --> STEP: 139/234 -- GLOBAL_STEP: 59575 + | > loss: -0.33603 (-0.26755) + | > log_mle: -0.53371 (-0.38615) + | > loss_dur: 0.19767 (0.11860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.10219 (18.21360) + | > current_lr: 0.00006 + | > step_time: 1.49440 (2.39096) + | > loader_time: 0.00240 (0.08225) + + + --> STEP: 144/234 -- GLOBAL_STEP: 59580 + | > loss: -0.30930 (-0.26813) + | > log_mle: -0.50131 (-0.38921) + | > loss_dur: 0.19201 (0.12108) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.26736 (18.96849) + | > current_lr: 0.00006 + | > step_time: 4.39900 (2.39008) + | > loader_time: 0.09600 (0.08075) + + + --> STEP: 149/234 -- GLOBAL_STEP: 59585 + | > loss: -0.35380 (-0.26940) + | > log_mle: -0.54665 (-0.39252) + | > loss_dur: 0.19284 (0.12312) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.61156 (19.77186) + | > current_lr: 0.00006 + | > step_time: 5.11710 (2.42265) + | > loader_time: 0.09830 (0.08073) + + + --> STEP: 154/234 -- GLOBAL_STEP: 59590 + | > loss: -0.33013 (-0.27138) + | > log_mle: -0.51737 (-0.39671) + | > loss_dur: 0.18724 (0.12533) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.10436 (20.60031) + | > current_lr: 0.00006 + | > step_time: 5.10650 (2.48237) + | > loader_time: 0.08480 (0.08255) + + + --> STEP: 159/234 -- GLOBAL_STEP: 59595 + | > loss: -0.34447 (-0.27318) + | > log_mle: -0.53772 (-0.40076) + | > loss_dur: 0.19325 (0.12758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.50150 (21.64590) + | > current_lr: 0.00006 + | > step_time: 5.89990 (2.56518) + | > loader_time: 0.09760 (0.08125) + + + --> STEP: 164/234 -- GLOBAL_STEP: 59600 + | > loss: -0.31925 (-0.27503) + | > log_mle: -0.52503 (-0.40468) + | > loss_dur: 0.20578 (0.12965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.49378 (22.46672) + | > current_lr: 0.00006 + | > step_time: 4.27820 (2.64320) + | > loader_time: 0.09940 (0.08108) + + + --> STEP: 169/234 -- GLOBAL_STEP: 59605 + | > loss: -0.31795 (-0.27730) + | > log_mle: -0.53298 (-0.40901) + | > loss_dur: 0.21504 (0.13171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.77196 (23.36554) + | > current_lr: 0.00006 + | > step_time: 2.20260 (2.69821) + | > loader_time: 0.08640 (0.08038) + + + --> STEP: 174/234 -- GLOBAL_STEP: 59610 + | > loss: -0.41244 (-0.28050) + | > log_mle: -0.62474 (-0.41454) + | > loss_dur: 0.21231 (0.13404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.40589 (24.52380) + | > current_lr: 0.00006 + | > step_time: 6.71190 (2.81626) + | > loader_time: 0.00550 (0.08149) + + + --> STEP: 179/234 -- GLOBAL_STEP: 59615 + | > loss: -0.38617 (-0.28299) + | > log_mle: -0.62350 (-0.41946) + | > loss_dur: 0.23733 (0.13646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.47273 (25.58339) + | > current_lr: 0.00006 + | > step_time: 3.71480 (2.84783) + | > loader_time: 0.10300 (0.08084) + + + --> STEP: 184/234 -- GLOBAL_STEP: 59620 + | > loss: -0.35938 (-0.28495) + | > log_mle: -0.57658 (-0.42364) + | > loss_dur: 0.21720 (0.13869) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.04849 (26.72155) + | > current_lr: 0.00006 + | > step_time: 4.49540 (2.92789) + | > loader_time: 0.09130 (0.07977) + + + --> STEP: 189/234 -- GLOBAL_STEP: 59625 + | > loss: -0.36529 (-0.28729) + | > log_mle: -0.57549 (-0.42810) + | > loss_dur: 0.21020 (0.14082) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.65512 (27.82501) + | > current_lr: 0.00006 + | > step_time: 1.70600 (3.01385) + | > loader_time: 0.00260 (0.08034) + + + --> STEP: 194/234 -- GLOBAL_STEP: 59630 + | > loss: -0.39846 (-0.29007) + | > log_mle: -0.60499 (-0.43262) + | > loss_dur: 0.20653 (0.14255) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.23135 (28.77422) + | > current_lr: 0.00006 + | > step_time: 6.80480 (3.08256) + | > loader_time: 0.10060 (0.08034) + + + --> STEP: 199/234 -- GLOBAL_STEP: 59635 + | > loss: -0.39570 (-0.29247) + | > log_mle: -0.61664 (-0.43683) + | > loss_dur: 0.22094 (0.14437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.67019 (29.74596) + | > current_lr: 0.00006 + | > step_time: 5.28970 (3.10852) + | > loader_time: 0.30560 (0.08130) + + + --> STEP: 204/234 -- GLOBAL_STEP: 59640 + | > loss: -0.42366 (-0.29459) + | > log_mle: -0.66460 (-0.44101) + | > loss_dur: 0.24094 (0.14642) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.86732 (30.58823) + | > current_lr: 0.00006 + | > step_time: 10.61390 (3.21359) + | > loader_time: 0.09080 (0.08284) + + + --> STEP: 209/234 -- GLOBAL_STEP: 59645 + | > loss: -0.39417 (-0.29725) + | > log_mle: -0.61242 (-0.44559) + | > loss_dur: 0.21824 (0.14835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.81161 (31.51332) + | > current_lr: 0.00006 + | > step_time: 9.78990 (3.25767) + | > loader_time: 0.10360 (0.08269) + + + --> STEP: 214/234 -- GLOBAL_STEP: 59650 + | > loss: -0.44347 (-0.30074) + | > log_mle: -0.65216 (-0.45107) + | > loss_dur: 0.20869 (0.15033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.45410 (32.79882) + | > current_lr: 0.00006 + | > step_time: 4.70080 (3.31342) + | > loader_time: 0.00440 (0.08131) + + + --> STEP: 219/234 -- GLOBAL_STEP: 59655 + | > loss: -0.52355 (-0.30407) + | > log_mle: -0.75305 (-0.45637) + | > loss_dur: 0.22950 (0.15231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.42976 (33.83717) + | > current_lr: 0.00006 + | > step_time: 4.29880 (3.42211) + | > loader_time: 0.08700 (0.08213) + + + --> STEP: 224/234 -- GLOBAL_STEP: 59660 + | > loss: -0.46892 (-0.30725) + | > log_mle: -0.70553 (-0.46141) + | > loss_dur: 0.23661 (0.15415) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.93580 (34.79744) + | > current_lr: 0.00006 + | > step_time: 0.22740 (3.37066) + | > loader_time: 0.00530 (0.08114) + + + --> STEP: 229/234 -- GLOBAL_STEP: 59665 + | > loss: -0.46079 (-0.31056) + | > log_mle: -0.75647 (-0.46709) + | > loss_dur: 0.29568 (0.15653) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.47659 (36.01806) + | > current_lr: 0.00006 + | > step_time: 0.26850 (3.30251) + | > loader_time: 0.00360 (0.07943) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.33006 (+1.18234) + | > avg_loss: -0.29072 (+0.00960) + | > avg_log_mle: -0.53806 (-0.00958) + | > avg_loss_dur: 0.24734 (+0.01918) + + + > EPOCH: 255/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 14:53:54)  + + --> STEP: 0/234 -- GLOBAL_STEP: 59670 + | > loss: -0.29212 (-0.29212) + | > log_mle: -0.46649 (-0.46649) + | > loss_dur: 0.17437 (0.17437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.43865 (25.43865) + | > current_lr: 0.00006 + | > step_time: 3.09400 (3.09404) + | > loader_time: 7.71430 (7.71426) + + + --> STEP: 5/234 -- GLOBAL_STEP: 59675 + | > loss: -0.27534 (-0.28628) + | > log_mle: -0.38026 (-0.38119) + | > loss_dur: 0.10492 (0.09491) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.75835 (20.53724) + | > current_lr: 0.00006 + | > step_time: 2.59640 (4.24178) + | > loader_time: 0.00300 (1.98163) + + + --> STEP: 10/234 -- GLOBAL_STEP: 59680 + | > loss: -0.29055 (-0.29364) + | > log_mle: -0.37827 (-0.38454) + | > loss_dur: 0.08772 (0.09090) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.87690 (19.63599) + | > current_lr: 0.00006 + | > step_time: 5.49500 (4.74058) + | > loader_time: 0.19350 (1.06843) + + + --> STEP: 15/234 -- GLOBAL_STEP: 59685 + | > loss: -0.31395 (-0.29989) + | > log_mle: -0.39143 (-0.38700) + | > loss_dur: 0.07748 (0.08710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.05408 (17.99877) + | > current_lr: 0.00006 + | > step_time: 4.28930 (4.17974) + | > loader_time: 0.10700 (0.72016) + + + --> STEP: 20/234 -- GLOBAL_STEP: 59690 + | > loss: -0.32320 (-0.30425) + | > log_mle: -0.38638 (-0.38682) + | > loss_dur: 0.06318 (0.08256) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.48528 (16.52280) + | > current_lr: 0.00006 + | > step_time: 4.89980 (3.89428) + | > loader_time: 0.00160 (0.54581) + + + --> STEP: 25/234 -- GLOBAL_STEP: 59695 + | > loss: -0.31043 (-0.30617) + | > log_mle: -0.37420 (-0.38585) + | > loss_dur: 0.06376 (0.07969) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.33707 (15.36315) + | > current_lr: 0.00006 + | > step_time: 2.47490 (3.93508) + | > loader_time: 0.00140 (0.44828) + + + --> STEP: 30/234 -- GLOBAL_STEP: 59700 + | > loss: -0.29225 (-0.30659) + | > log_mle: -0.37206 (-0.38507) + | > loss_dur: 0.07982 (0.07849) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.53512 (14.47540) + | > current_lr: 0.00006 + | > step_time: 1.01610 (3.59095) + | > loader_time: 0.02000 (0.37931) + + + --> STEP: 35/234 -- GLOBAL_STEP: 59705 + | > loss: -0.25084 (-0.30357) + | > log_mle: -0.36424 (-0.38320) + | > loss_dur: 0.11340 (0.07963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.95159 (13.95776) + | > current_lr: 0.00006 + | > step_time: 1.00040 (3.49715) + | > loader_time: 0.00150 (0.32633) + + + --> STEP: 40/234 -- GLOBAL_STEP: 59710 + | > loss: -0.27387 (-0.30044) + | > log_mle: -0.36288 (-0.38100) + | > loss_dur: 0.08901 (0.08056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.99773 (13.60019) + | > current_lr: 0.00006 + | > step_time: 2.41590 (3.29482) + | > loader_time: 0.08860 (0.28999) + + + --> STEP: 45/234 -- GLOBAL_STEP: 59715 + | > loss: -0.27701 (-0.29866) + | > log_mle: -0.38456 (-0.37976) + | > loss_dur: 0.10755 (0.08110) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.99740 (13.45600) + | > current_lr: 0.00006 + | > step_time: 3.33210 (3.13804) + | > loader_time: 0.07770 (0.26308) + + + --> STEP: 50/234 -- GLOBAL_STEP: 59720 + | > loss: -0.27675 (-0.29702) + | > log_mle: -0.36204 (-0.37880) + | > loss_dur: 0.08529 (0.08178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.77219 (12.97750) + | > current_lr: 0.00006 + | > step_time: 2.09540 (2.97359) + | > loader_time: 0.00250 (0.23702) + + + --> STEP: 55/234 -- GLOBAL_STEP: 59725 + | > loss: -0.28956 (-0.29467) + | > log_mle: -0.37155 (-0.37760) + | > loss_dur: 0.08199 (0.08293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.81252 (12.84100) + | > current_lr: 0.00006 + | > step_time: 1.49680 (2.95744) + | > loader_time: 0.00400 (0.22080) + + + --> STEP: 60/234 -- GLOBAL_STEP: 59730 + | > loss: -0.24408 (-0.29215) + | > log_mle: -0.37446 (-0.37658) + | > loss_dur: 0.13038 (0.08443) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.10372 (12.81187) + | > current_lr: 0.00006 + | > step_time: 3.49170 (2.87081) + | > loader_time: 0.00220 (0.20518) + + + --> STEP: 65/234 -- GLOBAL_STEP: 59735 + | > loss: -0.26575 (-0.28915) + | > log_mle: -0.35869 (-0.37551) + | > loss_dur: 0.09294 (0.08636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.09948 (12.81335) + | > current_lr: 0.00006 + | > step_time: 1.21960 (2.76457) + | > loader_time: 0.08000 (0.19076) + + + --> STEP: 70/234 -- GLOBAL_STEP: 59740 + | > loss: -0.23215 (-0.28612) + | > log_mle: -0.34491 (-0.37380) + | > loss_dur: 0.11276 (0.08768) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.11598 (12.81001) + | > current_lr: 0.00006 + | > step_time: 1.27540 (2.66552) + | > loader_time: 0.00180 (0.17871) + + + --> STEP: 75/234 -- GLOBAL_STEP: 59745 + | > loss: -0.23536 (-0.28302) + | > log_mle: -0.36115 (-0.37293) + | > loss_dur: 0.12579 (0.08991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06080 (12.89514) + | > current_lr: 0.00006 + | > step_time: 4.60240 (2.62787) + | > loader_time: 0.00220 (0.16901) + + + --> STEP: 80/234 -- GLOBAL_STEP: 59750 + | > loss: -0.24841 (-0.28087) + | > log_mle: -0.34658 (-0.37184) + | > loss_dur: 0.09816 (0.09097) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.76054 (12.83077) + | > current_lr: 0.00006 + | > step_time: 7.49620 (2.66854) + | > loader_time: 0.09310 (0.16095) + + + --> STEP: 85/234 -- GLOBAL_STEP: 59755 + | > loss: -0.24188 (-0.27835) + | > log_mle: -0.35291 (-0.37101) + | > loss_dur: 0.11103 (0.09266) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.84044 (12.96053) + | > current_lr: 0.00006 + | > step_time: 1.94420 (2.60645) + | > loader_time: 0.06730 (0.15245) + + + --> STEP: 90/234 -- GLOBAL_STEP: 59760 + | > loss: -0.23472 (-0.27668) + | > log_mle: -0.37546 (-0.37121) + | > loss_dur: 0.14075 (0.09454) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.26432 (13.17193) + | > current_lr: 0.00006 + | > step_time: 0.99720 (2.57364) + | > loader_time: 0.00330 (0.14508) + + + --> STEP: 95/234 -- GLOBAL_STEP: 59765 + | > loss: -0.29275 (-0.27602) + | > log_mle: -0.45900 (-0.37322) + | > loss_dur: 0.16625 (0.09720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.89171 (13.67762) + | > current_lr: 0.00006 + | > step_time: 1.01280 (2.52898) + | > loader_time: 0.00150 (0.13929) + + + --> STEP: 100/234 -- GLOBAL_STEP: 59770 + | > loss: -0.25288 (-0.27469) + | > log_mle: -0.38384 (-0.37367) + | > loss_dur: 0.13096 (0.09898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.99695 (13.78840) + | > current_lr: 0.00006 + | > step_time: 0.95990 (2.52042) + | > loader_time: 0.00140 (0.13424) + + + --> STEP: 105/234 -- GLOBAL_STEP: 59775 + | > loss: -0.24029 (-0.27382) + | > log_mle: -0.36419 (-0.37513) + | > loss_dur: 0.12390 (0.10131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.89301 (14.19186) + | > current_lr: 0.00006 + | > step_time: 0.96560 (2.49484) + | > loader_time: 0.00210 (0.12883) + + + --> STEP: 110/234 -- GLOBAL_STEP: 59780 + | > loss: -0.23822 (-0.27192) + | > log_mle: -0.38525 (-0.37619) + | > loss_dur: 0.14703 (0.10426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.22905 (14.74831) + | > current_lr: 0.00006 + | > step_time: 3.70780 (2.49940) + | > loader_time: 0.08700 (0.12460) + + + --> STEP: 115/234 -- GLOBAL_STEP: 59785 + | > loss: -0.22935 (-0.27107) + | > log_mle: -0.39717 (-0.37802) + | > loss_dur: 0.16782 (0.10695) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.32927 (15.62614) + | > current_lr: 0.00006 + | > step_time: 1.60180 (2.52422) + | > loader_time: 0.00220 (0.12010) + + + --> STEP: 120/234 -- GLOBAL_STEP: 59790 + | > loss: -0.28023 (-0.27014) + | > log_mle: -0.44931 (-0.37945) + | > loss_dur: 0.16907 (0.10931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.37763 (16.12612) + | > current_lr: 0.00006 + | > step_time: 2.02130 (2.49904) + | > loader_time: 0.19480 (0.11681) + + + --> STEP: 125/234 -- GLOBAL_STEP: 59795 + | > loss: -0.26346 (-0.26885) + | > log_mle: -0.43552 (-0.37985) + | > loss_dur: 0.17206 (0.11100) + | > amp_scaler: 2048.00000 (1064.96000) + | > grad_norm: 29.53465 (16.29771) + | > current_lr: 0.00006 + | > step_time: 2.20490 (2.47889) + | > loader_time: 0.00180 (0.11350) + + + --> STEP: 130/234 -- GLOBAL_STEP: 59800 + | > loss: -0.27954 (-0.26876) + | > log_mle: -0.45309 (-0.38225) + | > loss_dur: 0.17355 (0.11350) + | > amp_scaler: 2048.00000 (1102.76923) + | > grad_norm: 33.09717 (16.87354) + | > current_lr: 0.00006 + | > step_time: 1.99750 (2.44972) + | > loader_time: 0.08820 (0.11120) + + + --> STEP: 135/234 -- GLOBAL_STEP: 59805 + | > loss: -0.23744 (-0.26920) + | > log_mle: -0.38099 (-0.38477) + | > loss_dur: 0.14355 (0.11557) + | > amp_scaler: 2048.00000 (1137.77778) + | > grad_norm: 19.60849 (17.51147) + | > current_lr: 0.00006 + | > step_time: 3.10320 (2.49457) + | > loader_time: 0.19370 (0.10935) + + + --> STEP: 140/234 -- GLOBAL_STEP: 59810 + | > loss: -0.22705 (-0.26973) + | > log_mle: -0.41594 (-0.38789) + | > loss_dur: 0.18890 (0.11816) + | > amp_scaler: 2048.00000 (1170.28571) + | > grad_norm: 24.28414 (18.48136) + | > current_lr: 0.00006 + | > step_time: 1.61930 (2.47936) + | > loader_time: 0.00250 (0.10600) + + + --> STEP: 145/234 -- GLOBAL_STEP: 59815 + | > loss: -0.31570 (-0.27068) + | > log_mle: -0.50015 (-0.39143) + | > loss_dur: 0.18445 (0.12075) + | > amp_scaler: 2048.00000 (1200.55172) + | > grad_norm: 129.80565 (20.10973) + | > current_lr: 0.00006 + | > step_time: 2.12970 (2.46687) + | > loader_time: 0.00180 (0.10246) + + + --> STEP: 150/234 -- GLOBAL_STEP: 59820 + | > loss: -0.29462 (-0.27157) + | > log_mle: -0.49441 (-0.39443) + | > loss_dur: 0.19979 (0.12285) + | > amp_scaler: 2048.00000 (1228.80000) + | > grad_norm: 32.67739 (20.71594) + | > current_lr: 0.00006 + | > step_time: 1.00880 (2.45398) + | > loader_time: 0.00240 (0.10020) + + + --> STEP: 155/234 -- GLOBAL_STEP: 59825 + | > loss: -0.34524 (-0.27360) + | > log_mle: -0.54755 (-0.39873) + | > loss_dur: 0.20231 (0.12513) + | > amp_scaler: 2048.00000 (1255.22581) + | > grad_norm: 74.65680 (21.76010) + | > current_lr: 0.00006 + | > step_time: 2.01640 (2.44716) + | > loader_time: 0.00300 (0.09760) + + + --> STEP: 160/234 -- GLOBAL_STEP: 59830 + | > loss: -0.33865 (-0.27501) + | > log_mle: -0.55456 (-0.40252) + | > loss_dur: 0.21592 (0.12751) + | > amp_scaler: 2048.00000 (1280.00000) + | > grad_norm: 70.58630 (22.71090) + | > current_lr: 0.00006 + | > step_time: 1.50970 (2.43799) + | > loader_time: 0.00250 (0.09516) + + + --> STEP: 165/234 -- GLOBAL_STEP: 59835 + | > loss: -0.36486 (-0.27664) + | > log_mle: -0.56124 (-0.40619) + | > loss_dur: 0.19638 (0.12956) + | > amp_scaler: 2048.00000 (1303.27273) + | > grad_norm: 48.42394 (23.37189) + | > current_lr: 0.00006 + | > step_time: 3.70080 (2.47205) + | > loader_time: 0.00300 (0.09290) + + + --> STEP: 170/234 -- GLOBAL_STEP: 59840 + | > loss: -0.38372 (-0.27890) + | > log_mle: -0.60174 (-0.41055) + | > loss_dur: 0.21802 (0.13166) + | > amp_scaler: 2048.00000 (1325.17647) + | > grad_norm: 57.77961 (24.21562) + | > current_lr: 0.00006 + | > step_time: 2.20400 (2.45938) + | > loader_time: 0.00290 (0.09120) + + + --> STEP: 175/234 -- GLOBAL_STEP: 59845 + | > loss: -0.34850 (-0.28175) + | > log_mle: -0.57890 (-0.41576) + | > loss_dur: 0.23040 (0.13401) + | > amp_scaler: 2048.00000 (1345.82857) + | > grad_norm: 57.54824 (25.26001) + | > current_lr: 0.00006 + | > step_time: 2.51380 (2.45151) + | > loader_time: 0.08760 (0.08960) + + + --> STEP: 180/234 -- GLOBAL_STEP: 59850 + | > loss: -0.37919 (-0.28429) + | > log_mle: -0.58259 (-0.42069) + | > loss_dur: 0.20340 (0.13640) + | > amp_scaler: 2048.00000 (1365.33333) + | > grad_norm: 67.55389 (26.14985) + | > current_lr: 0.00006 + | > step_time: 0.69320 (2.44016) + | > loader_time: 0.00330 (0.08812) + + + --> STEP: 185/234 -- GLOBAL_STEP: 59855 + | > loss: -0.39699 (-0.28653) + | > log_mle: -0.62415 (-0.42524) + | > loss_dur: 0.22716 (0.13870) + | > amp_scaler: 2048.00000 (1383.78378) + | > grad_norm: 77.61474 (27.24205) + | > current_lr: 0.00006 + | > step_time: 4.38880 (2.47204) + | > loader_time: 0.07890 (0.08733) + + + --> STEP: 190/234 -- GLOBAL_STEP: 59860 + | > loss: -0.38189 (-0.28880) + | > log_mle: -0.58977 (-0.42968) + | > loss_dur: 0.20788 (0.14088) + | > amp_scaler: 2048.00000 (1401.26316) + | > grad_norm: 69.14579 (28.22799) + | > current_lr: 0.00006 + | > step_time: 3.71020 (2.48500) + | > loader_time: 0.00470 (0.08769) + + + --> STEP: 195/234 -- GLOBAL_STEP: 59865 + | > loss: -0.39088 (-0.29182) + | > log_mle: -0.61721 (-0.43446) + | > loss_dur: 0.22634 (0.14264) + | > amp_scaler: 2048.00000 (1417.84615) + | > grad_norm: 57.16055 (29.24197) + | > current_lr: 0.00006 + | > step_time: 3.80090 (2.59261) + | > loader_time: 0.00410 (0.08749) + + + --> STEP: 200/234 -- GLOBAL_STEP: 59870 + | > loss: -0.36800 (-0.29418) + | > log_mle: -0.62040 (-0.43881) + | > loss_dur: 0.25240 (0.14463) + | > amp_scaler: 2048.00000 (1433.60000) + | > grad_norm: 65.55514 (30.19204) + | > current_lr: 0.00006 + | > step_time: 3.90050 (2.68725) + | > loader_time: 0.30950 (0.08789) + + + --> STEP: 205/234 -- GLOBAL_STEP: 59875 + | > loss: -0.38011 (-0.29645) + | > log_mle: -0.59899 (-0.44297) + | > loss_dur: 0.21888 (0.14652) + | > amp_scaler: 2048.00000 (1448.58537) + | > grad_norm: 74.87933 (31.12238) + | > current_lr: 0.00006 + | > step_time: 6.00410 (2.73247) + | > loader_time: 0.00590 (0.08767) + + + --> STEP: 210/234 -- GLOBAL_STEP: 59880 + | > loss: -0.42740 (-0.29931) + | > log_mle: -0.67093 (-0.44782) + | > loss_dur: 0.24354 (0.14851) + | > amp_scaler: 2048.00000 (1462.85714) + | > grad_norm: 101.84248 (32.29827) + | > current_lr: 0.00006 + | > step_time: 2.79510 (2.81214) + | > loader_time: 0.09590 (0.08740) + + + --> STEP: 215/234 -- GLOBAL_STEP: 59885 + | > loss: -0.39731 (-0.30243) + | > log_mle: -0.62888 (-0.45285) + | > loss_dur: 0.23157 (0.15042) + | > amp_scaler: 2048.00000 (1476.46512) + | > grad_norm: 74.24596 (33.68026) + | > current_lr: 0.00006 + | > step_time: 5.29780 (2.85006) + | > loader_time: 0.00330 (0.08864) + + + --> STEP: 220/234 -- GLOBAL_STEP: 59890 + | > loss: -0.43929 (-0.30586) + | > log_mle: -0.67511 (-0.45826) + | > loss_dur: 0.23582 (0.15240) + | > amp_scaler: 2048.00000 (1489.45455) + | > grad_norm: 96.16768 (34.95835) + | > current_lr: 0.00006 + | > step_time: 3.49870 (2.88620) + | > loader_time: 0.10000 (0.08842) + + + --> STEP: 225/234 -- GLOBAL_STEP: 59895 + | > loss: -0.50617 (-0.30908) + | > log_mle: -0.74896 (-0.46333) + | > loss_dur: 0.24278 (0.15425) + | > amp_scaler: 2048.00000 (1501.86667) + | > grad_norm: 83.10760 (36.04717) + | > current_lr: 0.00006 + | > step_time: 0.22910 (2.86800) + | > loader_time: 0.00250 (0.08693) + + + --> STEP: 230/234 -- GLOBAL_STEP: 59900 + | > loss: -0.48439 (-0.31187) + | > log_mle: -0.80928 (-0.46878) + | > loss_dur: 0.32490 (0.15691) + | > amp_scaler: 2048.00000 (1513.73913) + | > grad_norm: 75.31567 (37.06142) + | > current_lr: 0.00006 + | > step_time: 0.26900 (2.81127) + | > loader_time: 0.00340 (0.08511) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07768 (-1.25238) + | > avg_loss: -0.29498 (-0.00426) + | > avg_log_mle: -0.52572 (+0.01234) + | > avg_loss_dur: 0.23074 (-0.01660) + + + > EPOCH: 256/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 15:05:57)  + + --> STEP: 1/234 -- GLOBAL_STEP: 59905 + | > loss: -0.28919 (-0.28919) + | > log_mle: -0.37714 (-0.37714) + | > loss_dur: 0.08795 (0.08795) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.31086 (26.31086) + | > current_lr: 0.00006 + | > step_time: 10.80630 (10.80627) + | > loader_time: 0.00210 (0.00214) + + + --> STEP: 6/234 -- GLOBAL_STEP: 59910 + | > loss: -0.30662 (-0.28922) + | > log_mle: -0.38169 (-0.38338) + | > loss_dur: 0.07506 (0.09416) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.17195 (19.51772) + | > current_lr: 0.00006 + | > step_time: 10.00300 (7.68166) + | > loader_time: 0.09650 (0.01873) + + + --> STEP: 11/234 -- GLOBAL_STEP: 59915 + | > loss: -0.33562 (-0.29857) + | > log_mle: -0.39827 (-0.38780) + | > loss_dur: 0.06264 (0.08923) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.34552 (18.68573) + | > current_lr: 0.00006 + | > step_time: 2.10140 (5.53762) + | > loader_time: 0.00110 (0.02763) + + + --> STEP: 16/234 -- GLOBAL_STEP: 59920 + | > loss: -0.33215 (-0.30571) + | > log_mle: -0.39921 (-0.39014) + | > loss_dur: 0.06706 (0.08443) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.09813 (17.37260) + | > current_lr: 0.00006 + | > step_time: 1.26930 (4.77514) + | > loader_time: 0.00200 (0.01977) + + + --> STEP: 21/234 -- GLOBAL_STEP: 59925 + | > loss: -0.30588 (-0.30798) + | > log_mle: -0.37425 (-0.38845) + | > loss_dur: 0.06837 (0.08047) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.43258 (16.27081) + | > current_lr: 0.00006 + | > step_time: 1.27600 (3.96690) + | > loader_time: 0.00200 (0.01546) + + + --> STEP: 26/234 -- GLOBAL_STEP: 59930 + | > loss: -0.28966 (-0.30738) + | > log_mle: -0.37328 (-0.38745) + | > loss_dur: 0.08362 (0.08008) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.07650 (15.39685) + | > current_lr: 0.00006 + | > step_time: 2.39950 (3.53580) + | > loader_time: 0.00920 (0.01642) + + + --> STEP: 31/234 -- GLOBAL_STEP: 59935 + | > loss: -0.26763 (-0.30694) + | > log_mle: -0.36667 (-0.38637) + | > loss_dur: 0.09905 (0.07943) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.92988 (14.81861) + | > current_lr: 0.00006 + | > step_time: 1.48600 (3.60672) + | > loader_time: 0.01660 (0.02320) + + + --> STEP: 36/234 -- GLOBAL_STEP: 59940 + | > loss: -0.26791 (-0.30408) + | > log_mle: -0.36413 (-0.38433) + | > loss_dur: 0.09623 (0.08025) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.79023 (14.26365) + | > current_lr: 0.00006 + | > step_time: 5.19910 (3.87635) + | > loader_time: 0.29960 (0.03311) + + + --> STEP: 41/234 -- GLOBAL_STEP: 59945 + | > loss: -0.30630 (-0.30160) + | > log_mle: -0.37468 (-0.38242) + | > loss_dur: 0.06838 (0.08082) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.12420 (13.81680) + | > current_lr: 0.00006 + | > step_time: 1.42260 (3.64569) + | > loader_time: 0.18850 (0.03629) + + + --> STEP: 46/234 -- GLOBAL_STEP: 59950 + | > loss: -0.26757 (-0.29836) + | > log_mle: -0.37025 (-0.38084) + | > loss_dur: 0.10269 (0.08249) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.73084 (13.76753) + | > current_lr: 0.00006 + | > step_time: 2.03260 (3.46594) + | > loader_time: 0.00210 (0.03258) + + + --> STEP: 51/234 -- GLOBAL_STEP: 59955 + | > loss: -0.26900 (-0.29700) + | > log_mle: -0.36090 (-0.37964) + | > loss_dur: 0.09190 (0.08264) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.08486 (13.33251) + | > current_lr: 0.00006 + | > step_time: 1.80810 (3.29044) + | > loader_time: 0.00240 (0.03109) + + + --> STEP: 56/234 -- GLOBAL_STEP: 59960 + | > loss: -0.25584 (-0.29465) + | > log_mle: -0.36650 (-0.37863) + | > loss_dur: 0.11066 (0.08398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.07150 (13.07659) + | > current_lr: 0.00006 + | > step_time: 1.71410 (3.15457) + | > loader_time: 0.00170 (0.02852) + + + --> STEP: 61/234 -- GLOBAL_STEP: 59965 + | > loss: -0.26817 (-0.29246) + | > log_mle: -0.36200 (-0.37756) + | > loss_dur: 0.09383 (0.08510) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.80821 (12.90961) + | > current_lr: 0.00006 + | > step_time: 3.40310 (3.04741) + | > loader_time: 0.00340 (0.02766) + + + --> STEP: 66/234 -- GLOBAL_STEP: 59970 + | > loss: -0.27200 (-0.28965) + | > log_mle: -0.35599 (-0.37640) + | > loss_dur: 0.08400 (0.08675) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.53567 (12.79567) + | > current_lr: 0.00006 + | > step_time: 1.01250 (2.92442) + | > loader_time: 0.00210 (0.02798) + + + --> STEP: 71/234 -- GLOBAL_STEP: 59975 + | > loss: -0.25753 (-0.28671) + | > log_mle: -0.37838 (-0.37513) + | > loss_dur: 0.12084 (0.08842) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.67476 (12.81814) + | > current_lr: 0.00006 + | > step_time: 6.10670 (2.95754) + | > loader_time: 0.20850 (0.03038) + + + --> STEP: 76/234 -- GLOBAL_STEP: 59980 + | > loss: -0.25156 (-0.28392) + | > log_mle: -0.36164 (-0.37399) + | > loss_dur: 0.11007 (0.09006) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.83109 (12.87327) + | > current_lr: 0.00006 + | > step_time: 4.89370 (2.89258) + | > loader_time: 0.00250 (0.02857) + + + --> STEP: 81/234 -- GLOBAL_STEP: 59985 + | > loss: -0.25112 (-0.28193) + | > log_mle: -0.36779 (-0.37294) + | > loss_dur: 0.11667 (0.09102) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.48820 (12.80613) + | > current_lr: 0.00006 + | > step_time: 1.70310 (2.82565) + | > loader_time: 0.05700 (0.02773) + + + --> STEP: 86/234 -- GLOBAL_STEP: 59990 + | > loss: -0.25605 (-0.27966) + | > log_mle: -0.36830 (-0.37211) + | > loss_dur: 0.11225 (0.09245) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.59682 (12.88085) + | > current_lr: 0.00006 + | > step_time: 1.18730 (2.80228) + | > loader_time: 0.00780 (0.02897) + + + --> STEP: 91/234 -- GLOBAL_STEP: 59995 + | > loss: -0.24228 (-0.27782) + | > log_mle: -0.37717 (-0.37244) + | > loss_dur: 0.13489 (0.09462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.24647 (12.98830) + | > current_lr: 0.00006 + | > step_time: 1.29510 (2.77396) + | > loader_time: 0.00170 (0.02938) + + + --> STEP: 96/234 -- GLOBAL_STEP: 60000 + | > loss: -0.24380 (-0.27721) + | > log_mle: -0.36326 (-0.37424) + | > loss_dur: 0.11946 (0.09702) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.05406 (13.56547) + | > current_lr: 0.00006 + | > step_time: 2.12290 (2.75659) + | > loader_time: 0.58960 (0.03409) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_60000.pth + + --> STEP: 101/234 -- GLOBAL_STEP: 60005 + | > loss: -0.24505 (-0.27580) + | > log_mle: -0.40974 (-0.37512) + | > loss_dur: 0.16469 (0.09932) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.53018 (13.97255) + | > current_lr: 0.00006 + | > step_time: 1.08220 (2.69786) + | > loader_time: 0.00200 (0.03280) + + + --> STEP: 106/234 -- GLOBAL_STEP: 60010 + | > loss: -0.22312 (-0.27445) + | > log_mle: -0.40077 (-0.37633) + | > loss_dur: 0.17765 (0.10188) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.39387 (14.39065) + | > current_lr: 0.00006 + | > step_time: 1.16100 (2.66656) + | > loader_time: 0.00250 (0.03304) + + + --> STEP: 111/234 -- GLOBAL_STEP: 60015 + | > loss: -0.27474 (-0.27327) + | > log_mle: -0.45837 (-0.37782) + | > loss_dur: 0.18364 (0.10455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.77824 (14.80744) + | > current_lr: 0.00006 + | > step_time: 2.69080 (2.63730) + | > loader_time: 0.00160 (0.03238) + + + --> STEP: 116/234 -- GLOBAL_STEP: 60020 + | > loss: -0.23958 (-0.27232) + | > log_mle: -0.42505 (-0.37946) + | > loss_dur: 0.18547 (0.10715) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.38621 (15.27881) + | > current_lr: 0.00006 + | > step_time: 3.10340 (2.63149) + | > loader_time: 0.00240 (0.03247) + + + --> STEP: 121/234 -- GLOBAL_STEP: 60025 + | > loss: -0.21014 (-0.27133) + | > log_mle: -0.33915 (-0.38040) + | > loss_dur: 0.12901 (0.10908) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.46666 (15.72676) + | > current_lr: 0.00006 + | > step_time: 1.80520 (2.60152) + | > loader_time: 0.00200 (0.03197) + + + --> STEP: 126/234 -- GLOBAL_STEP: 60030 + | > loss: -0.28076 (-0.27058) + | > log_mle: -0.47030 (-0.38182) + | > loss_dur: 0.18954 (0.11124) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.97757 (16.41148) + | > current_lr: 0.00006 + | > step_time: 1.22240 (2.56194) + | > loader_time: 0.08280 (0.03230) + + + --> STEP: 131/234 -- GLOBAL_STEP: 60035 + | > loss: -0.32996 (-0.27089) + | > log_mle: -0.52334 (-0.38455) + | > loss_dur: 0.19338 (0.11366) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.07467 (17.09344) + | > current_lr: 0.00006 + | > step_time: 1.59670 (2.54397) + | > loader_time: 0.00220 (0.03186) + + + --> STEP: 136/234 -- GLOBAL_STEP: 60040 + | > loss: -0.36709 (-0.27166) + | > log_mle: -0.57084 (-0.38742) + | > loss_dur: 0.20375 (0.11577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.57257 (17.70949) + | > current_lr: 0.00006 + | > step_time: 3.50090 (2.54113) + | > loader_time: 0.00280 (0.03163) + + + --> STEP: 141/234 -- GLOBAL_STEP: 60045 + | > loss: -0.29032 (-0.27161) + | > log_mle: -0.46446 (-0.38969) + | > loss_dur: 0.17414 (0.11808) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.11652 (18.56606) + | > current_lr: 0.00006 + | > step_time: 4.61580 (2.54338) + | > loader_time: 0.09580 (0.03247) + + + --> STEP: 146/234 -- GLOBAL_STEP: 60050 + | > loss: -0.33066 (-0.27294) + | > log_mle: -0.52361 (-0.39390) + | > loss_dur: 0.19294 (0.12096) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.27676 (19.49372) + | > current_lr: 0.00006 + | > step_time: 4.31960 (2.58180) + | > loader_time: 0.08850 (0.03271) + + + --> STEP: 151/234 -- GLOBAL_STEP: 60055 + | > loss: -0.30737 (-0.27415) + | > log_mle: -0.48073 (-0.39711) + | > loss_dur: 0.17336 (0.12296) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.76203 (20.21246) + | > current_lr: 0.00006 + | > step_time: 1.60470 (2.56059) + | > loader_time: 0.08490 (0.03227) + + + --> STEP: 156/234 -- GLOBAL_STEP: 60060 + | > loss: -0.34153 (-0.27664) + | > log_mle: -0.53135 (-0.40198) + | > loss_dur: 0.18982 (0.12534) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.64886 (21.27670) + | > current_lr: 0.00006 + | > step_time: 4.69190 (2.58881) + | > loader_time: 0.00860 (0.03255) + + + --> STEP: 161/234 -- GLOBAL_STEP: 60065 + | > loss: -0.37548 (-0.27859) + | > log_mle: -0.55985 (-0.40617) + | > loss_dur: 0.18438 (0.12758) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.82718 (22.12891) + | > current_lr: 0.00006 + | > step_time: 1.50730 (2.60712) + | > loader_time: 0.09360 (0.03406) + + + --> STEP: 166/234 -- GLOBAL_STEP: 60070 + | > loss: -0.30882 (-0.28006) + | > log_mle: -0.49248 (-0.40963) + | > loss_dur: 0.18365 (0.12957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.48259 (22.98452) + | > current_lr: 0.00006 + | > step_time: 4.20170 (2.67494) + | > loader_time: 0.09550 (0.03548) + + + --> STEP: 171/234 -- GLOBAL_STEP: 60075 + | > loss: -0.40113 (-0.28281) + | > log_mle: -0.60590 (-0.41465) + | > loss_dur: 0.20477 (0.13185) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 54.36838 (24.24026) + | > current_lr: 0.00006 + | > step_time: 2.68980 (2.73114) + | > loader_time: 0.00470 (0.03802) + + + --> STEP: 176/234 -- GLOBAL_STEP: 60080 + | > loss: -0.36889 (-0.28523) + | > log_mle: -0.57634 (-0.41948) + | > loss_dur: 0.20746 (0.13425) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.97419 (25.44570) + | > current_lr: 0.00006 + | > step_time: 2.89330 (2.73130) + | > loader_time: 0.19660 (0.03867) + + + --> STEP: 181/234 -- GLOBAL_STEP: 60085 + | > loss: -0.30765 (-0.28720) + | > log_mle: -0.51433 (-0.42383) + | > loss_dur: 0.20668 (0.13663) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 32.51524 (26.48715) + | > current_lr: 0.00006 + | > step_time: 9.10090 (2.86574) + | > loader_time: 0.10450 (0.03972) + + + --> STEP: 186/234 -- GLOBAL_STEP: 60090 + | > loss: -0.31033 (-0.28932) + | > log_mle: -0.54697 (-0.42837) + | > loss_dur: 0.23664 (0.13905) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.89847 (27.43620) + | > current_lr: 0.00006 + | > step_time: 2.90240 (2.87049) + | > loader_time: 0.09950 (0.04124) + + + --> STEP: 191/234 -- GLOBAL_STEP: 60095 + | > loss: -0.36020 (-0.29165) + | > log_mle: -0.56595 (-0.43257) + | > loss_dur: 0.20574 (0.14092) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.72318 (28.37040) + | > current_lr: 0.00006 + | > step_time: 6.19410 (2.92214) + | > loader_time: 0.10370 (0.04224) + + + --> STEP: 196/234 -- GLOBAL_STEP: 60100 + | > loss: -0.34299 (-0.29421) + | > log_mle: -0.56674 (-0.43701) + | > loss_dur: 0.22375 (0.14280) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.65121 (29.14100) + | > current_lr: 0.00006 + | > step_time: 2.20070 (2.92105) + | > loader_time: 0.08730 (0.04246) + + + --> STEP: 201/234 -- GLOBAL_STEP: 60105 + | > loss: -0.29735 (-0.29619) + | > log_mle: -0.52259 (-0.44097) + | > loss_dur: 0.22524 (0.14478) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.66299 (29.91300) + | > current_lr: 0.00006 + | > step_time: 3.80000 (2.97023) + | > loader_time: 0.09090 (0.04242) + + + --> STEP: 206/234 -- GLOBAL_STEP: 60110 + | > loss: -0.41675 (-0.29900) + | > log_mle: -0.63714 (-0.44561) + | > loss_dur: 0.22039 (0.14661) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.06812 (30.82245) + | > current_lr: 0.00006 + | > step_time: 3.41080 (3.01280) + | > loader_time: 0.00310 (0.04238) + + + --> STEP: 211/234 -- GLOBAL_STEP: 60115 + | > loss: -0.47797 (-0.30222) + | > log_mle: -0.72349 (-0.45089) + | > loss_dur: 0.24553 (0.14867) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.16061 (31.94955) + | > current_lr: 0.00006 + | > step_time: 8.40040 (3.08214) + | > loader_time: 0.00690 (0.04277) + + + --> STEP: 216/234 -- GLOBAL_STEP: 60120 + | > loss: -0.45262 (-0.30527) + | > log_mle: -0.70639 (-0.45596) + | > loss_dur: 0.25377 (0.15069) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 94.23667 (33.16971) + | > current_lr: 0.00006 + | > step_time: 2.10520 (3.13317) + | > loader_time: 0.08730 (0.05186) + + + --> STEP: 221/234 -- GLOBAL_STEP: 60125 + | > loss: -0.40261 (-0.30867) + | > log_mle: -0.62129 (-0.46114) + | > loss_dur: 0.21868 (0.15247) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.71883 (34.23886) + | > current_lr: 0.00006 + | > step_time: 5.51090 (3.20898) + | > loader_time: 0.10680 (0.09953) + + + --> STEP: 226/234 -- GLOBAL_STEP: 60130 + | > loss: -0.47669 (-0.31240) + | > log_mle: -0.72113 (-0.46691) + | > loss_dur: 0.24444 (0.15451) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.89144 (35.69366) + | > current_lr: 0.00006 + | > step_time: 0.23850 (3.15949) + | > loader_time: 0.00360 (0.09860) + + + --> STEP: 231/234 -- GLOBAL_STEP: 60135 + | > loss: -0.40796 (-0.31542) + | > log_mle: -0.78272 (-0.47304) + | > loss_dur: 0.37476 (0.15762) + | > amp_scaler: 1024.00000 (2043.56710) + | > grad_norm: 0.00000 (36.50828) + | > current_lr: 0.00006 + | > step_time: 0.31240 (3.09712) + | > loader_time: 0.00480 (0.09655) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.94965 (+0.87196) + | > avg_loss: -0.29105 (+0.00393) + | > avg_log_mle: -0.50321 (+0.02251) + | > avg_loss_dur: 0.21216 (-0.01858) + + + > EPOCH: 257/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 15:19:27)  + + --> STEP: 2/234 -- GLOBAL_STEP: 60140 + | > loss: -0.29805 (-0.29922) + | > log_mle: -0.39497 (-0.38415) + | > loss_dur: 0.09693 (0.08493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.77376 (20.44280) + | > current_lr: 0.00006 + | > step_time: 7.19910 (7.70053) + | > loader_time: 0.00670 (0.04799) + + + --> STEP: 7/234 -- GLOBAL_STEP: 60145 + | > loss: -0.32171 (-0.29233) + | > log_mle: -0.38563 (-0.38369) + | > loss_dur: 0.06392 (0.09136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.91696 (17.48544) + | > current_lr: 0.00006 + | > step_time: 4.10560 (7.07637) + | > loader_time: 0.09040 (0.08143) + + + --> STEP: 12/234 -- GLOBAL_STEP: 60150 + | > loss: -0.30149 (-0.29855) + | > log_mle: -0.38375 (-0.38770) + | > loss_dur: 0.08226 (0.08915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.80064 (17.55740) + | > current_lr: 0.00006 + | > step_time: 4.52160 (5.60600) + | > loader_time: 0.18540 (0.07255) + + + --> STEP: 17/234 -- GLOBAL_STEP: 60155 + | > loss: -0.32059 (-0.30733) + | > log_mle: -0.38703 (-0.39038) + | > loss_dur: 0.06643 (0.08304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.51486 (16.14779) + | > current_lr: 0.00006 + | > step_time: 3.70770 (5.41743) + | > loader_time: 0.00330 (0.06884) + + + --> STEP: 22/234 -- GLOBAL_STEP: 60160 + | > loss: -0.30539 (-0.30812) + | > log_mle: -0.38814 (-0.38920) + | > loss_dur: 0.08275 (0.08107) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.79946 (15.42024) + | > current_lr: 0.00006 + | > step_time: 4.70180 (5.10374) + | > loader_time: 0.09490 (0.06256) + + + --> STEP: 27/234 -- GLOBAL_STEP: 60165 + | > loss: -0.30677 (-0.30900) + | > log_mle: -0.38217 (-0.38836) + | > loss_dur: 0.07540 (0.07936) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.66552 (14.75499) + | > current_lr: 0.00006 + | > step_time: 0.83720 (4.86134) + | > loader_time: 0.08290 (0.06774) + + + --> STEP: 32/234 -- GLOBAL_STEP: 60170 + | > loss: -0.30878 (-0.30820) + | > log_mle: -0.38341 (-0.38746) + | > loss_dur: 0.07463 (0.07926) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.22673 (14.02062) + | > current_lr: 0.00006 + | > step_time: 3.68560 (4.51645) + | > loader_time: 0.01140 (0.06042) + + + --> STEP: 37/234 -- GLOBAL_STEP: 60175 + | > loss: -0.27976 (-0.30486) + | > log_mle: -0.35799 (-0.38476) + | > loss_dur: 0.07823 (0.07990) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.34789 (13.87434) + | > current_lr: 0.00006 + | > step_time: 5.01500 (4.43393) + | > loader_time: 0.00280 (0.05875) + + + --> STEP: 42/234 -- GLOBAL_STEP: 60180 + | > loss: -0.27414 (-0.30214) + | > log_mle: -0.35440 (-0.38258) + | > loss_dur: 0.08026 (0.08045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.01383 (13.56034) + | > current_lr: 0.00006 + | > step_time: 1.11440 (4.04915) + | > loader_time: 0.00220 (0.05205) + + + --> STEP: 47/234 -- GLOBAL_STEP: 60185 + | > loss: -0.28301 (-0.29885) + | > log_mle: -0.37360 (-0.38114) + | > loss_dur: 0.09058 (0.08229) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.60371 (13.46150) + | > current_lr: 0.00006 + | > step_time: 1.74230 (3.84448) + | > loader_time: 0.00160 (0.04678) + + + --> STEP: 52/234 -- GLOBAL_STEP: 60190 + | > loss: -0.24838 (-0.29694) + | > log_mle: -0.36095 (-0.37965) + | > loss_dur: 0.11257 (0.08270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.75165 (13.05672) + | > current_lr: 0.00006 + | > step_time: 1.70550 (3.61155) + | > loader_time: 0.00240 (0.04248) + + + --> STEP: 57/234 -- GLOBAL_STEP: 60195 + | > loss: -0.25603 (-0.29495) + | > log_mle: -0.35044 (-0.37839) + | > loss_dur: 0.09442 (0.08343) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.80430 (12.94737) + | > current_lr: 0.00006 + | > step_time: 2.13230 (3.45799) + | > loader_time: 0.08450 (0.04197) + + + --> STEP: 62/234 -- GLOBAL_STEP: 60200 + | > loss: -0.23050 (-0.29244) + | > log_mle: -0.37598 (-0.37779) + | > loss_dur: 0.14547 (0.08535) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.51742 (13.08095) + | > current_lr: 0.00006 + | > step_time: 2.79660 (3.33650) + | > loader_time: 0.00190 (0.03881) + + + --> STEP: 67/234 -- GLOBAL_STEP: 60205 + | > loss: -0.26123 (-0.29039) + | > log_mle: -0.36776 (-0.37639) + | > loss_dur: 0.10653 (0.08600) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06968 (12.86740) + | > current_lr: 0.00006 + | > step_time: 1.81270 (3.19932) + | > loader_time: 0.08310 (0.03856) + + + --> STEP: 72/234 -- GLOBAL_STEP: 60210 + | > loss: -0.25991 (-0.28745) + | > log_mle: -0.35824 (-0.37497) + | > loss_dur: 0.09833 (0.08751) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.35254 (12.77673) + | > current_lr: 0.00006 + | > step_time: 2.90560 (3.11599) + | > loader_time: 0.09440 (0.03858) + + + --> STEP: 77/234 -- GLOBAL_STEP: 60215 + | > loss: -0.26620 (-0.28459) + | > log_mle: -0.35974 (-0.37377) + | > loss_dur: 0.09354 (0.08919) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.99872 (12.79380) + | > current_lr: 0.00006 + | > step_time: 3.49460 (3.12645) + | > loader_time: 0.00490 (0.03870) + + + --> STEP: 82/234 -- GLOBAL_STEP: 60220 + | > loss: -0.24519 (-0.28231) + | > log_mle: -0.35203 (-0.37263) + | > loss_dur: 0.10684 (0.09032) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.56107 (12.75541) + | > current_lr: 0.00006 + | > step_time: 3.57160 (3.06959) + | > loader_time: 0.09890 (0.03767) + + + --> STEP: 87/234 -- GLOBAL_STEP: 60225 + | > loss: -0.24237 (-0.27981) + | > log_mle: -0.35621 (-0.37181) + | > loss_dur: 0.11384 (0.09201) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.36531 (12.73666) + | > current_lr: 0.00006 + | > step_time: 1.30380 (2.99476) + | > loader_time: 0.00240 (0.03633) + + + --> STEP: 92/234 -- GLOBAL_STEP: 60230 + | > loss: -0.26258 (-0.27848) + | > log_mle: -0.39438 (-0.37252) + | > loss_dur: 0.13179 (0.09403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.06322 (13.02158) + | > current_lr: 0.00006 + | > step_time: 6.29340 (3.02340) + | > loader_time: 0.10550 (0.03748) + + + --> STEP: 97/234 -- GLOBAL_STEP: 60235 + | > loss: -0.24360 (-0.27737) + | > log_mle: -0.37760 (-0.37398) + | > loss_dur: 0.13400 (0.09661) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.56857 (13.50418) + | > current_lr: 0.00006 + | > step_time: 1.25960 (2.96069) + | > loader_time: 0.00190 (0.03568) + + + --> STEP: 102/234 -- GLOBAL_STEP: 60240 + | > loss: -0.22357 (-0.27544) + | > log_mle: -0.35899 (-0.37428) + | > loss_dur: 0.13542 (0.09885) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.01545 (13.88027) + | > current_lr: 0.00006 + | > step_time: 5.69520 (2.94865) + | > loader_time: 0.00770 (0.03508) + + + --> STEP: 107/234 -- GLOBAL_STEP: 60245 + | > loss: -0.24289 (-0.27428) + | > log_mle: -0.40156 (-0.37575) + | > loss_dur: 0.15867 (0.10147) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.86728 (14.31705) + | > current_lr: 0.00006 + | > step_time: 0.92850 (2.90823) + | > loader_time: 0.00270 (0.03541) + + + --> STEP: 112/234 -- GLOBAL_STEP: 60250 + | > loss: -0.25398 (-0.27299) + | > log_mle: -0.41675 (-0.37727) + | > loss_dur: 0.16278 (0.10429) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.66328 (14.76990) + | > current_lr: 0.00006 + | > step_time: 1.41170 (2.85152) + | > loader_time: 0.08510 (0.03613) + + + --> STEP: 117/234 -- GLOBAL_STEP: 60255 + | > loss: -0.26108 (-0.27204) + | > log_mle: -0.41201 (-0.37882) + | > loss_dur: 0.15092 (0.10678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.25272 (15.16560) + | > current_lr: 0.00006 + | > step_time: 0.97820 (2.81741) + | > loader_time: 0.00280 (0.03621) + + + --> STEP: 122/234 -- GLOBAL_STEP: 60260 + | > loss: -0.23470 (-0.27097) + | > log_mle: -0.38387 (-0.37949) + | > loss_dur: 0.14918 (0.10852) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.93774 (15.46030) + | > current_lr: 0.00006 + | > step_time: 1.90030 (2.77455) + | > loader_time: 0.08390 (0.03618) + + + --> STEP: 127/234 -- GLOBAL_STEP: 60265 + | > loss: -0.26553 (-0.27060) + | > log_mle: -0.44806 (-0.38146) + | > loss_dur: 0.18253 (0.11086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.33927 (16.03574) + | > current_lr: 0.00006 + | > step_time: 3.99940 (2.76035) + | > loader_time: 0.00260 (0.03551) + + + --> STEP: 132/234 -- GLOBAL_STEP: 60270 + | > loss: -0.27620 (-0.27102) + | > log_mle: -0.43035 (-0.38408) + | > loss_dur: 0.15415 (0.11306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.30376 (16.71342) + | > current_lr: 0.00006 + | > step_time: 2.55480 (2.75098) + | > loader_time: 0.06940 (0.03553) + + + --> STEP: 137/234 -- GLOBAL_STEP: 60275 + | > loss: -0.25457 (-0.27162) + | > log_mle: -0.43854 (-0.38707) + | > loss_dur: 0.18397 (0.11545) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.76293 (17.39430) + | > current_lr: 0.00006 + | > step_time: 6.08680 (2.75505) + | > loader_time: 0.10370 (0.03508) + + + --> STEP: 142/234 -- GLOBAL_STEP: 60280 + | > loss: -0.26227 (-0.27168) + | > log_mle: -0.45472 (-0.38946) + | > loss_dur: 0.19246 (0.11778) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.80519 (18.05677) + | > current_lr: 0.00006 + | > step_time: 2.20170 (2.73546) + | > loader_time: 0.00400 (0.03457) + + + --> STEP: 147/234 -- GLOBAL_STEP: 60285 + | > loss: -0.28361 (-0.27328) + | > log_mle: -0.45857 (-0.39368) + | > loss_dur: 0.17496 (0.12041) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.00189 (19.00807) + | > current_lr: 0.00006 + | > step_time: 3.99850 (2.74242) + | > loader_time: 0.00280 (0.03552) + + + --> STEP: 152/234 -- GLOBAL_STEP: 60290 + | > loss: -0.33677 (-0.27503) + | > log_mle: -0.55078 (-0.39767) + | > loss_dur: 0.21401 (0.12264) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.66420 (19.83981) + | > current_lr: 0.00006 + | > step_time: 1.50750 (2.77197) + | > loader_time: 0.08260 (0.03683) + + + --> STEP: 157/234 -- GLOBAL_STEP: 60295 + | > loss: -0.29778 (-0.27743) + | > log_mle: -0.49002 (-0.40235) + | > loss_dur: 0.19224 (0.12493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.84475 (21.00114) + | > current_lr: 0.00006 + | > step_time: 2.20270 (2.73858) + | > loader_time: 0.09500 (0.03686) + + + --> STEP: 162/234 -- GLOBAL_STEP: 60300 + | > loss: -0.33299 (-0.27929) + | > log_mle: -0.52018 (-0.40652) + | > loss_dur: 0.18719 (0.12723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.63721 (22.17699) + | > current_lr: 0.00006 + | > step_time: 2.50030 (2.73612) + | > loader_time: 0.00340 (0.03652) + + + --> STEP: 167/234 -- GLOBAL_STEP: 60305 + | > loss: -0.42478 (-0.28132) + | > log_mle: -0.62202 (-0.41059) + | > loss_dur: 0.19725 (0.12927) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.01836 (23.05230) + | > current_lr: 0.00006 + | > step_time: 2.19680 (2.73080) + | > loader_time: 0.00380 (0.03721) + + + --> STEP: 172/234 -- GLOBAL_STEP: 60310 + | > loss: -0.40070 (-0.28402) + | > log_mle: -0.61264 (-0.41566) + | > loss_dur: 0.21194 (0.13164) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.25111 (24.17455) + | > current_lr: 0.00006 + | > step_time: 1.50850 (2.70356) + | > loader_time: 0.00280 (0.03621) + + + --> STEP: 177/234 -- GLOBAL_STEP: 60315 + | > loss: -0.35083 (-0.28645) + | > log_mle: -0.56281 (-0.42040) + | > loss_dur: 0.21198 (0.13395) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.53707 (25.36685) + | > current_lr: 0.00006 + | > step_time: 4.30850 (2.75780) + | > loader_time: 0.08750 (0.03684) + + + --> STEP: 182/234 -- GLOBAL_STEP: 60320 + | > loss: -0.36420 (-0.28858) + | > log_mle: -0.60831 (-0.42511) + | > loss_dur: 0.24411 (0.13653) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.32532 (26.40491) + | > current_lr: 0.00006 + | > step_time: 2.50450 (2.74206) + | > loader_time: 0.08900 (0.03738) + + + --> STEP: 187/234 -- GLOBAL_STEP: 60325 + | > loss: -0.40815 (-0.29110) + | > log_mle: -0.61897 (-0.42981) + | > loss_dur: 0.21082 (0.13871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.24583 (27.35506) + | > current_lr: 0.00006 + | > step_time: 10.79490 (2.84253) + | > loader_time: 0.39540 (0.03953) + + + --> STEP: 192/234 -- GLOBAL_STEP: 60330 + | > loss: -0.43414 (-0.29400) + | > log_mle: -0.63633 (-0.43450) + | > loss_dur: 0.20219 (0.14050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.94811 (28.50923) + | > current_lr: 0.00006 + | > step_time: 1.30870 (2.86915) + | > loader_time: 0.00310 (0.04112) + + + --> STEP: 197/234 -- GLOBAL_STEP: 60335 + | > loss: -0.40952 (-0.29658) + | > log_mle: -0.60699 (-0.43888) + | > loss_dur: 0.19747 (0.14230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.84392 (29.70515) + | > current_lr: 0.00006 + | > step_time: 6.59840 (2.98307) + | > loader_time: 0.00410 (0.04315) + + + --> STEP: 202/234 -- GLOBAL_STEP: 60340 + | > loss: -0.49583 (-0.29903) + | > log_mle: -0.70542 (-0.44332) + | > loss_dur: 0.20958 (0.14429) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.29580 (30.90357) + | > current_lr: 0.00006 + | > step_time: 1.99040 (2.99037) + | > loader_time: 0.00570 (0.04360) + + + --> STEP: 207/234 -- GLOBAL_STEP: 60345 + | > loss: -0.41283 (-0.30113) + | > log_mle: -0.66258 (-0.44739) + | > loss_dur: 0.24975 (0.14627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 109.99020 (32.46332) + | > current_lr: 0.00006 + | > step_time: 6.70850 (3.02477) + | > loader_time: 0.09500 (0.04349) + + + --> STEP: 212/234 -- GLOBAL_STEP: 60350 + | > loss: -0.40686 (-0.30336) + | > log_mle: -0.63869 (-0.45181) + | > loss_dur: 0.23183 (0.14844) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.39486 (33.89033) + | > current_lr: 0.00006 + | > step_time: 2.90490 (3.09306) + | > loader_time: 0.07580 (0.04413) + + + --> STEP: 217/234 -- GLOBAL_STEP: 60355 + | > loss: -0.43357 (-0.30594) + | > log_mle: -0.66884 (-0.45631) + | > loss_dur: 0.23527 (0.15037) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.10445 (34.90987) + | > current_lr: 0.00006 + | > step_time: 7.09900 (3.16283) + | > loader_time: 0.00580 (0.04503) + + + --> STEP: 222/234 -- GLOBAL_STEP: 60360 + | > loss: -0.40777 (-0.30858) + | > log_mle: -0.68160 (-0.46093) + | > loss_dur: 0.27383 (0.15235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.25286 (35.86177) + | > current_lr: 0.00006 + | > step_time: 4.90280 (3.19477) + | > loader_time: 0.09650 (0.04488) + + + --> STEP: 227/234 -- GLOBAL_STEP: 60365 + | > loss: -0.38239 (-0.31138) + | > log_mle: -0.63378 (-0.46573) + | > loss_dur: 0.25139 (0.15435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.96335 (36.94999) + | > current_lr: 0.00006 + | > step_time: 1.81200 (3.15961) + | > loader_time: 0.00300 (0.05003) + + + --> STEP: 232/234 -- GLOBAL_STEP: 60370 + | > loss: -0.25120 (-0.31271) + | > log_mle: -0.73305 (-0.47117) + | > loss_dur: 0.48185 (0.15846) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 148.63667 (38.83470) + | > current_lr: 0.00006 + | > step_time: 0.36270 (3.10071) + | > loader_time: 0.02330 (0.04956) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55741 (-0.39224) + | > avg_loss: -0.29087 (+0.00018) + | > avg_log_mle: -0.51266 (-0.00945) + | > avg_loss_dur: 0.22179 (+0.00963) + + + > EPOCH: 258/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 15:32:39)  + + --> STEP: 3/234 -- GLOBAL_STEP: 60375 + | > loss: -0.24838 (-0.28661) + | > log_mle: -0.37629 (-0.38139) + | > loss_dur: 0.12791 (0.09478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.48077 (20.24413) + | > current_lr: 0.00006 + | > step_time: 3.10410 (5.96964) + | > loader_time: 0.00750 (0.03591) + + + --> STEP: 8/234 -- GLOBAL_STEP: 60380 + | > loss: -0.30457 (-0.29401) + | > log_mle: -0.39713 (-0.38312) + | > loss_dur: 0.09256 (0.08911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.40697 (20.14005) + | > current_lr: 0.00006 + | > step_time: 3.50730 (3.56360) + | > loader_time: 0.00620 (0.01507) + + + --> STEP: 13/234 -- GLOBAL_STEP: 60385 + | > loss: -0.34789 (-0.29936) + | > log_mle: -0.40668 (-0.38537) + | > loss_dur: 0.05879 (0.08601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.89641 (18.76235) + | > current_lr: 0.00006 + | > step_time: 2.60180 (3.64927) + | > loader_time: 0.00320 (0.09316) + + + --> STEP: 18/234 -- GLOBAL_STEP: 60390 + | > loss: -0.30320 (-0.30459) + | > log_mle: -0.37974 (-0.38670) + | > loss_dur: 0.07654 (0.08211) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.34049 (16.82387) + | > current_lr: 0.00006 + | > step_time: 5.59790 (4.14983) + | > loader_time: 0.10670 (0.07363) + + + --> STEP: 23/234 -- GLOBAL_STEP: 60395 + | > loss: -0.33848 (-0.30726) + | > log_mle: -0.40173 (-0.38703) + | > loss_dur: 0.06325 (0.07978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.64008 (15.64659) + | > current_lr: 0.00006 + | > step_time: 4.51150 (4.12207) + | > loader_time: 0.10060 (0.06642) + + + --> STEP: 28/234 -- GLOBAL_STEP: 60400 + | > loss: -0.36005 (-0.30894) + | > log_mle: -0.41240 (-0.38669) + | > loss_dur: 0.05235 (0.07775) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.00125 (14.90584) + | > current_lr: 0.00006 + | > step_time: 4.91770 (4.33276) + | > loader_time: 0.00200 (0.05502) + + + --> STEP: 33/234 -- GLOBAL_STEP: 60405 + | > loss: -0.30594 (-0.30701) + | > log_mle: -0.37984 (-0.38506) + | > loss_dur: 0.07390 (0.07805) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.81662 (14.31438) + | > current_lr: 0.00006 + | > step_time: 0.67670 (4.16051) + | > loader_time: 0.00230 (0.05838) + + + --> STEP: 38/234 -- GLOBAL_STEP: 60410 + | > loss: -0.29067 (-0.30412) + | > log_mle: -0.37747 (-0.38281) + | > loss_dur: 0.08680 (0.07869) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.91421 (13.89184) + | > current_lr: 0.00006 + | > step_time: 0.96050 (3.77313) + | > loader_time: 0.00230 (0.05295) + + + --> STEP: 43/234 -- GLOBAL_STEP: 60415 + | > loss: -0.26478 (-0.30086) + | > log_mle: -0.36748 (-0.38101) + | > loss_dur: 0.10270 (0.08015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.18543 (13.51264) + | > current_lr: 0.00006 + | > step_time: 1.58300 (3.51551) + | > loader_time: 0.00150 (0.04890) + + + --> STEP: 48/234 -- GLOBAL_STEP: 60420 + | > loss: -0.30639 (-0.29966) + | > log_mle: -0.37527 (-0.38016) + | > loss_dur: 0.06887 (0.08050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.93297 (13.20747) + | > current_lr: 0.00006 + | > step_time: 1.14740 (3.29071) + | > loader_time: 0.00240 (0.04453) + + + --> STEP: 53/234 -- GLOBAL_STEP: 60425 + | > loss: -0.28111 (-0.29744) + | > log_mle: -0.37241 (-0.37885) + | > loss_dur: 0.09130 (0.08141) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.82704 (12.90046) + | > current_lr: 0.00006 + | > step_time: 2.11100 (3.17429) + | > loader_time: 0.00210 (0.04057) + + + --> STEP: 58/234 -- GLOBAL_STEP: 60430 + | > loss: -0.29751 (-0.29557) + | > log_mle: -0.37214 (-0.37782) + | > loss_dur: 0.07463 (0.08224) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.52059 (12.64045) + | > current_lr: 0.00006 + | > step_time: 1.92490 (3.05002) + | > loader_time: 0.08190 (0.04034) + + + --> STEP: 63/234 -- GLOBAL_STEP: 60435 + | > loss: -0.24910 (-0.29240) + | > log_mle: -0.35496 (-0.37721) + | > loss_dur: 0.10586 (0.08481) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.86386 (12.73850) + | > current_lr: 0.00006 + | > step_time: 1.32730 (2.97563) + | > loader_time: 0.00180 (0.03871) + + + --> STEP: 68/234 -- GLOBAL_STEP: 60440 + | > loss: -0.23306 (-0.29024) + | > log_mle: -0.34892 (-0.37588) + | > loss_dur: 0.11586 (0.08564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.28315 (12.68127) + | > current_lr: 0.00006 + | > step_time: 3.89170 (2.96544) + | > loader_time: 0.10560 (0.03999) + + + --> STEP: 73/234 -- GLOBAL_STEP: 60445 + | > loss: -0.22624 (-0.28711) + | > log_mle: -0.36537 (-0.37469) + | > loss_dur: 0.13913 (0.08758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.70956 (12.82122) + | > current_lr: 0.00006 + | > step_time: 3.01610 (2.92778) + | > loader_time: 0.00230 (0.03746) + + + --> STEP: 78/234 -- GLOBAL_STEP: 60450 + | > loss: -0.24402 (-0.28460) + | > log_mle: -0.34977 (-0.37366) + | > loss_dur: 0.10575 (0.08906) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.72860 (12.82240) + | > current_lr: 0.00006 + | > step_time: 1.49470 (2.85715) + | > loader_time: 0.00170 (0.03734) + + + --> STEP: 83/234 -- GLOBAL_STEP: 60455 + | > loss: -0.21189 (-0.28234) + | > log_mle: -0.35750 (-0.37269) + | > loss_dur: 0.14560 (0.09034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.45107 (13.14493) + | > current_lr: 0.00006 + | > step_time: 1.66020 (2.79885) + | > loader_time: 0.00210 (0.03748) + + + --> STEP: 88/234 -- GLOBAL_STEP: 60460 + | > loss: -0.25425 (-0.28062) + | > log_mle: -0.39279 (-0.37225) + | > loss_dur: 0.13854 (0.09163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.22031 (13.50390) + | > current_lr: 0.00006 + | > step_time: 1.45710 (2.77170) + | > loader_time: 0.00240 (0.03549) + + + --> STEP: 93/234 -- GLOBAL_STEP: 60465 + | > loss: -0.24704 (-0.27902) + | > log_mle: -0.40514 (-0.37298) + | > loss_dur: 0.15810 (0.09396) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.52334 (13.90540) + | > current_lr: 0.00006 + | > step_time: 4.11410 (2.79274) + | > loader_time: 0.07080 (0.03629) + + + --> STEP: 98/234 -- GLOBAL_STEP: 60470 + | > loss: -0.23770 (-0.27792) + | > log_mle: -0.34553 (-0.37383) + | > loss_dur: 0.10783 (0.09591) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.26092 (14.28877) + | > current_lr: 0.00006 + | > step_time: 1.80850 (2.74931) + | > loader_time: 0.00220 (0.03462) + + + --> STEP: 103/234 -- GLOBAL_STEP: 60475 + | > loss: -0.27549 (-0.27671) + | > log_mle: -0.43757 (-0.37536) + | > loss_dur: 0.16209 (0.09865) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.10520 (14.72393) + | > current_lr: 0.00006 + | > step_time: 1.21000 (2.68852) + | > loader_time: 0.08690 (0.03387) + + + --> STEP: 108/234 -- GLOBAL_STEP: 60480 + | > loss: -0.24552 (-0.27550) + | > log_mle: -0.38133 (-0.37656) + | > loss_dur: 0.13582 (0.10106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.39025 (15.04982) + | > current_lr: 0.00006 + | > step_time: 1.49000 (2.63807) + | > loader_time: 0.00200 (0.03312) + + + --> STEP: 113/234 -- GLOBAL_STEP: 60485 + | > loss: -0.27061 (-0.27431) + | > log_mle: -0.42363 (-0.37847) + | > loss_dur: 0.15302 (0.10417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.59585 (15.69108) + | > current_lr: 0.00006 + | > step_time: 1.80600 (2.63115) + | > loader_time: 0.00350 (0.03493) + + + --> STEP: 118/234 -- GLOBAL_STEP: 60490 + | > loss: -0.23104 (-0.27295) + | > log_mle: -0.39602 (-0.37972) + | > loss_dur: 0.16498 (0.10677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.96681 (16.02411) + | > current_lr: 0.00006 + | > step_time: 1.70370 (2.62661) + | > loader_time: 0.00230 (0.03583) + + + --> STEP: 123/234 -- GLOBAL_STEP: 60495 + | > loss: -0.21690 (-0.27165) + | > log_mle: -0.36366 (-0.38012) + | > loss_dur: 0.14675 (0.10847) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.75160 (16.28455) + | > current_lr: 0.00006 + | > step_time: 2.20350 (2.59054) + | > loader_time: 0.00340 (0.03530) + + + --> STEP: 128/234 -- GLOBAL_STEP: 60500 + | > loss: -0.26257 (-0.27152) + | > log_mle: -0.41479 (-0.38237) + | > loss_dur: 0.15222 (0.11085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.44960 (17.22841) + | > current_lr: 0.00006 + | > step_time: 4.20990 (2.58887) + | > loader_time: 0.00670 (0.03622) + + + --> STEP: 133/234 -- GLOBAL_STEP: 60505 + | > loss: -0.27759 (-0.27171) + | > log_mle: -0.44889 (-0.38496) + | > loss_dur: 0.17129 (0.11326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.77498 (17.88122) + | > current_lr: 0.00006 + | > step_time: 1.68630 (2.56175) + | > loader_time: 0.00210 (0.03497) + + + --> STEP: 138/234 -- GLOBAL_STEP: 60510 + | > loss: -0.23564 (-0.27162) + | > log_mle: -0.39720 (-0.38730) + | > loss_dur: 0.16156 (0.11569) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.58301 (18.53856) + | > current_lr: 0.00006 + | > step_time: 1.79440 (2.54138) + | > loader_time: 0.09890 (0.03452) + + + --> STEP: 143/234 -- GLOBAL_STEP: 60515 + | > loss: -0.31931 (-0.27216) + | > log_mle: -0.55083 (-0.39065) + | > loss_dur: 0.23153 (0.11849) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.02997 (19.44689) + | > current_lr: 0.00006 + | > step_time: 1.60210 (2.53982) + | > loader_time: 0.00340 (0.03402) + + + --> STEP: 148/234 -- GLOBAL_STEP: 60520 + | > loss: -0.30252 (-0.27348) + | > log_mle: -0.45952 (-0.39412) + | > loss_dur: 0.15700 (0.12064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.73073 (20.02868) + | > current_lr: 0.00006 + | > step_time: 5.49460 (2.55201) + | > loader_time: 0.00860 (0.03302) + + + --> STEP: 153/234 -- GLOBAL_STEP: 60525 + | > loss: -0.39184 (-0.27582) + | > log_mle: -0.59492 (-0.39893) + | > loss_dur: 0.20308 (0.12311) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.25509 (20.87393) + | > current_lr: 0.00006 + | > step_time: 1.59080 (2.55120) + | > loader_time: 0.00190 (0.03282) + + + --> STEP: 158/234 -- GLOBAL_STEP: 60530 + | > loss: -0.30822 (-0.27754) + | > log_mle: -0.52315 (-0.40305) + | > loss_dur: 0.21493 (0.12550) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.83004 (21.79723) + | > current_lr: 0.00006 + | > step_time: 7.39740 (2.61405) + | > loader_time: 0.10000 (0.03441) + + + --> STEP: 163/234 -- GLOBAL_STEP: 60535 + | > loss: -0.30138 (-0.27940) + | > log_mle: -0.49317 (-0.40708) + | > loss_dur: 0.19179 (0.12768) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.90593 (22.89158) + | > current_lr: 0.00006 + | > step_time: 3.30520 (2.61613) + | > loader_time: 0.00800 (0.03456) + + + --> STEP: 168/234 -- GLOBAL_STEP: 60540 + | > loss: -0.33979 (-0.28170) + | > log_mle: -0.55181 (-0.41146) + | > loss_dur: 0.21202 (0.12976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.84718 (23.79822) + | > current_lr: 0.00006 + | > step_time: 3.79710 (2.65312) + | > loader_time: 0.00460 (0.03523) + + + --> STEP: 173/234 -- GLOBAL_STEP: 60545 + | > loss: -0.35287 (-0.28424) + | > log_mle: -0.56626 (-0.41637) + | > loss_dur: 0.21339 (0.13213) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.18893 (25.00265) + | > current_lr: 0.00006 + | > step_time: 2.20090 (2.71565) + | > loader_time: 0.08750 (0.03595) + + + --> STEP: 178/234 -- GLOBAL_STEP: 60550 + | > loss: -0.39368 (-0.28691) + | > log_mle: -0.62453 (-0.42142) + | > loss_dur: 0.23085 (0.13451) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.99223 (25.86304) + | > current_lr: 0.00006 + | > step_time: 2.19020 (2.70460) + | > loader_time: 0.00300 (0.03663) + + + --> STEP: 183/234 -- GLOBAL_STEP: 60555 + | > loss: -0.40728 (-0.28914) + | > log_mle: -0.62116 (-0.42602) + | > loss_dur: 0.21388 (0.13688) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.08325 (26.79624) + | > current_lr: 0.00006 + | > step_time: 2.49360 (2.70873) + | > loader_time: 0.00680 (0.03715) + + + --> STEP: 188/234 -- GLOBAL_STEP: 60560 + | > loss: -0.42583 (-0.29160) + | > log_mle: -0.63757 (-0.43066) + | > loss_dur: 0.21173 (0.13906) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.90211 (27.97443) + | > current_lr: 0.00006 + | > step_time: 8.20810 (2.72885) + | > loader_time: 0.09830 (0.03767) + + + --> STEP: 193/234 -- GLOBAL_STEP: 60565 + | > loss: -0.41207 (-0.29419) + | > log_mle: -0.63170 (-0.43511) + | > loss_dur: 0.21963 (0.14092) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.68948 (28.91225) + | > current_lr: 0.00006 + | > step_time: 2.90500 (2.83119) + | > loader_time: 0.00400 (0.03823) + + + --> STEP: 198/234 -- GLOBAL_STEP: 60570 + | > loss: -0.39811 (-0.29657) + | > log_mle: -0.62423 (-0.43940) + | > loss_dur: 0.22612 (0.14283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.48491 (29.78832) + | > current_lr: 0.00006 + | > step_time: 5.70070 (2.89200) + | > loader_time: 0.29620 (0.03930) + + + --> STEP: 203/234 -- GLOBAL_STEP: 60575 + | > loss: -0.34841 (-0.29883) + | > log_mle: -0.55479 (-0.44355) + | > loss_dur: 0.20638 (0.14472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.23559 (30.62830) + | > current_lr: 0.00006 + | > step_time: 6.10810 (2.92673) + | > loader_time: 0.00340 (0.04024) + + + --> STEP: 208/234 -- GLOBAL_STEP: 60580 + | > loss: -0.40752 (-0.30155) + | > log_mle: -0.64238 (-0.44842) + | > loss_dur: 0.23486 (0.14688) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.00688 (31.86762) + | > current_lr: 0.00006 + | > step_time: 4.51220 (2.99827) + | > loader_time: 0.08500 (0.04105) + + + --> STEP: 213/234 -- GLOBAL_STEP: 60585 + | > loss: -0.45241 (-0.30489) + | > log_mle: -0.69294 (-0.45384) + | > loss_dur: 0.24053 (0.14894) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.06458 (32.94247) + | > current_lr: 0.00006 + | > step_time: 2.81470 (3.08150) + | > loader_time: 0.08410 (0.04139) + + + --> STEP: 218/234 -- GLOBAL_STEP: 60590 + | > loss: -0.41564 (-0.30794) + | > log_mle: -0.65445 (-0.45879) + | > loss_dur: 0.23881 (0.15085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.14792 (33.89724) + | > current_lr: 0.00006 + | > step_time: 4.01720 (3.12599) + | > loader_time: 0.07940 (0.04132) + + + --> STEP: 223/234 -- GLOBAL_STEP: 60595 + | > loss: -0.45996 (-0.31126) + | > log_mle: -0.69794 (-0.46401) + | > loss_dur: 0.23799 (0.15275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.03928 (35.01008) + | > current_lr: 0.00006 + | > step_time: 1.69220 (3.09994) + | > loader_time: 0.00240 (0.04084) + + + --> STEP: 228/234 -- GLOBAL_STEP: 60600 + | > loss: -0.42720 (-0.31459) + | > log_mle: -0.68730 (-0.46941) + | > loss_dur: 0.26009 (0.15482) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.90353 (36.14966) + | > current_lr: 0.00006 + | > step_time: 0.24780 (3.04370) + | > loader_time: 0.00480 (0.04003) + + + --> STEP: 233/234 -- GLOBAL_STEP: 60605 + | > loss: 0.05945 (-0.31524) + | > log_mle: -0.66452 (-0.47614) + | > loss_dur: 0.72397 (0.16090) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.10255 (37.44481) + | > current_lr: 0.00006 + | > step_time: 0.20200 (2.98427) + | > loader_time: 0.00250 (0.03945) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.64832 (+0.09091) + | > avg_loss: -0.32559 (-0.03472) + | > avg_log_mle: -0.54118 (-0.02852) + | > avg_loss_dur: 0.21559 (-0.00620) + + + > EPOCH: 259/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 15:45:25)  + + --> STEP: 4/234 -- GLOBAL_STEP: 60610 + | > loss: -0.30301 (-0.28454) + | > log_mle: -0.38705 (-0.38766) + | > loss_dur: 0.08404 (0.10312) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.18704 (21.70903) + | > current_lr: 0.00006 + | > step_time: 5.89200 (8.82138) + | > loader_time: 0.10840 (0.64842) + + + --> STEP: 9/234 -- GLOBAL_STEP: 60615 + | > loss: -0.28866 (-0.29645) + | > log_mle: -0.39883 (-0.39039) + | > loss_dur: 0.11018 (0.09394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.95646 (19.80490) + | > current_lr: 0.00006 + | > step_time: 2.40840 (4.82948) + | > loader_time: 0.09150 (0.29915) + + + --> STEP: 14/234 -- GLOBAL_STEP: 60620 + | > loss: -0.31592 (-0.30454) + | > log_mle: -0.39583 (-0.39260) + | > loss_dur: 0.07992 (0.08806) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.70614 (18.06568) + | > current_lr: 0.00006 + | > step_time: 0.98980 (3.66602) + | > loader_time: 0.00160 (0.19800) + + + --> STEP: 19/234 -- GLOBAL_STEP: 60625 + | > loss: -0.32069 (-0.30986) + | > log_mle: -0.39424 (-0.39293) + | > loss_dur: 0.07356 (0.08307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.48321 (16.36559) + | > current_lr: 0.00006 + | > step_time: 3.80860 (3.52842) + | > loader_time: 0.00390 (0.15671) + + + --> STEP: 24/234 -- GLOBAL_STEP: 60630 + | > loss: -0.32045 (-0.31212) + | > log_mle: -0.38733 (-0.39245) + | > loss_dur: 0.06687 (0.08033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.74091 (15.37167) + | > current_lr: 0.00006 + | > step_time: 1.19500 (3.09857) + | > loader_time: 0.00370 (0.12450) + + + --> STEP: 29/234 -- GLOBAL_STEP: 60635 + | > loss: -0.30361 (-0.31288) + | > log_mle: -0.37973 (-0.39136) + | > loss_dur: 0.07612 (0.07848) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.05394 (14.60492) + | > current_lr: 0.00006 + | > step_time: 5.21420 (2.92689) + | > loader_time: 0.00270 (0.10643) + + + --> STEP: 34/234 -- GLOBAL_STEP: 60640 + | > loss: -0.30503 (-0.31090) + | > log_mle: -0.37620 (-0.38920) + | > loss_dur: 0.07117 (0.07830) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.08832 (14.14706) + | > current_lr: 0.00006 + | > step_time: 2.38930 (2.82321) + | > loader_time: 0.00230 (0.09593) + + + --> STEP: 39/234 -- GLOBAL_STEP: 60645 + | > loss: -0.27516 (-0.30641) + | > log_mle: -0.37012 (-0.38627) + | > loss_dur: 0.09496 (0.07986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.65802 (14.03430) + | > current_lr: 0.00006 + | > step_time: 0.78600 (3.05880) + | > loader_time: 0.00220 (0.08666) + + + --> STEP: 44/234 -- GLOBAL_STEP: 60650 + | > loss: -0.29303 (-0.30357) + | > log_mle: -0.36207 (-0.38386) + | > loss_dur: 0.06904 (0.08028) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29008 (13.63183) + | > current_lr: 0.00006 + | > step_time: 1.06560 (2.88556) + | > loader_time: 0.00120 (0.07893) + + + --> STEP: 49/234 -- GLOBAL_STEP: 60655 + | > loss: -0.29579 (-0.30218) + | > log_mle: -0.37404 (-0.38297) + | > loss_dur: 0.07825 (0.08079) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45221 (13.40760) + | > current_lr: 0.00006 + | > step_time: 1.90990 (2.78483) + | > loader_time: 0.00120 (0.07478) + + + --> STEP: 54/234 -- GLOBAL_STEP: 60660 + | > loss: -0.29209 (-0.29961) + | > log_mle: -0.37140 (-0.38131) + | > loss_dur: 0.07931 (0.08170) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.06000 (13.12883) + | > current_lr: 0.00006 + | > step_time: 1.03750 (2.64802) + | > loader_time: 0.00210 (0.06951) + + + --> STEP: 59/234 -- GLOBAL_STEP: 60665 + | > loss: -0.27055 (-0.29770) + | > log_mle: -0.36730 (-0.38006) + | > loss_dur: 0.09675 (0.08236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.78926 (13.02939) + | > current_lr: 0.00006 + | > step_time: 2.49200 (2.60523) + | > loader_time: 0.00290 (0.06675) + + + --> STEP: 64/234 -- GLOBAL_STEP: 60670 + | > loss: -0.26979 (-0.29445) + | > log_mle: -0.35883 (-0.37885) + | > loss_dur: 0.08904 (0.08439) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.26406 (13.20029) + | > current_lr: 0.00006 + | > step_time: 2.41210 (2.56911) + | > loader_time: 0.08810 (0.06425) + + + --> STEP: 69/234 -- GLOBAL_STEP: 60675 + | > loss: -0.26024 (-0.29158) + | > log_mle: -0.34938 (-0.37721) + | > loss_dur: 0.08914 (0.08563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.89646 (13.03700) + | > current_lr: 0.00006 + | > step_time: 2.30180 (2.56068) + | > loader_time: 0.00710 (0.06118) + + + --> STEP: 74/234 -- GLOBAL_STEP: 60680 + | > loss: -0.24383 (-0.28840) + | > log_mle: -0.34933 (-0.37603) + | > loss_dur: 0.10550 (0.08763) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.81730 (12.98519) + | > current_lr: 0.00006 + | > step_time: 2.90090 (2.53934) + | > loader_time: 0.08970 (0.05956) + + + --> STEP: 79/234 -- GLOBAL_STEP: 60685 + | > loss: -0.26137 (-0.28622) + | > log_mle: -0.36584 (-0.37505) + | > loss_dur: 0.10447 (0.08883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.34987 (12.90849) + | > current_lr: 0.00006 + | > step_time: 1.70230 (2.50242) + | > loader_time: 0.00350 (0.05692) + + + --> STEP: 84/234 -- GLOBAL_STEP: 60690 + | > loss: -0.25470 (-0.28359) + | > log_mle: -0.35984 (-0.37407) + | > loss_dur: 0.10514 (0.09048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.08700 (12.85941) + | > current_lr: 0.00006 + | > step_time: 1.40810 (2.50821) + | > loader_time: 0.08740 (0.05560) + + + --> STEP: 89/234 -- GLOBAL_STEP: 60695 + | > loss: -0.26076 (-0.28177) + | > log_mle: -0.38118 (-0.37393) + | > loss_dur: 0.12042 (0.09216) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.59409 (13.00597) + | > current_lr: 0.00006 + | > step_time: 2.29850 (2.47385) + | > loader_time: 0.08200 (0.05448) + + + --> STEP: 94/234 -- GLOBAL_STEP: 60700 + | > loss: -0.27195 (-0.28048) + | > log_mle: -0.40802 (-0.37497) + | > loss_dur: 0.13606 (0.09449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.54032 (13.36998) + | > current_lr: 0.00006 + | > step_time: 1.38210 (2.44640) + | > loader_time: 0.00240 (0.05172) + + + --> STEP: 99/234 -- GLOBAL_STEP: 60705 + | > loss: -0.27605 (-0.27950) + | > log_mle: -0.44297 (-0.37613) + | > loss_dur: 0.16692 (0.09662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.05833 (13.72956) + | > current_lr: 0.00006 + | > step_time: 2.00040 (2.42256) + | > loader_time: 0.18740 (0.05284) + + + --> STEP: 104/234 -- GLOBAL_STEP: 60710 + | > loss: -0.29073 (-0.27848) + | > log_mle: -0.44925 (-0.37772) + | > loss_dur: 0.15852 (0.09924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.91621 (14.23194) + | > current_lr: 0.00006 + | > step_time: 1.39950 (2.39955) + | > loader_time: 0.00270 (0.05124) + + + --> STEP: 109/234 -- GLOBAL_STEP: 60715 + | > loss: -0.22788 (-0.27672) + | > log_mle: -0.41842 (-0.37858) + | > loss_dur: 0.19054 (0.10187) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.53577 (14.58093) + | > current_lr: 0.00006 + | > step_time: 1.72370 (2.36576) + | > loader_time: 0.08300 (0.05120) + + + --> STEP: 114/234 -- GLOBAL_STEP: 60720 + | > loss: -0.26061 (-0.27595) + | > log_mle: -0.40529 (-0.38034) + | > loss_dur: 0.14468 (0.10438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.87016 (15.20366) + | > current_lr: 0.00006 + | > step_time: 1.50220 (2.34725) + | > loader_time: 0.00390 (0.04988) + + + --> STEP: 119/234 -- GLOBAL_STEP: 60725 + | > loss: -0.25204 (-0.27457) + | > log_mle: -0.40237 (-0.38153) + | > loss_dur: 0.15033 (0.10696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.32291 (15.56354) + | > current_lr: 0.00006 + | > step_time: 2.29510 (2.33189) + | > loader_time: 0.00420 (0.04866) + + + --> STEP: 124/234 -- GLOBAL_STEP: 60730 + | > loss: -0.28057 (-0.27335) + | > log_mle: -0.43161 (-0.38212) + | > loss_dur: 0.15104 (0.10877) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.43884 (15.92074) + | > current_lr: 0.00006 + | > step_time: 1.70900 (2.30868) + | > loader_time: 0.00250 (0.04686) + + + --> STEP: 129/234 -- GLOBAL_STEP: 60735 + | > loss: -0.24835 (-0.27306) + | > log_mle: -0.42210 (-0.38429) + | > loss_dur: 0.17375 (0.11123) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.29078 (16.63472) + | > current_lr: 0.00006 + | > step_time: 1.91140 (2.33232) + | > loader_time: 0.00260 (0.04576) + + + --> STEP: 134/234 -- GLOBAL_STEP: 60740 + | > loss: -0.28116 (-0.27382) + | > log_mle: -0.47728 (-0.38749) + | > loss_dur: 0.19612 (0.11367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.22209 (17.34575) + | > current_lr: 0.00006 + | > step_time: 3.31260 (2.35557) + | > loader_time: 0.08070 (0.04488) + + + --> STEP: 139/234 -- GLOBAL_STEP: 60745 + | > loss: -0.33715 (-0.27435) + | > log_mle: -0.52799 (-0.39035) + | > loss_dur: 0.19084 (0.11599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.35265 (18.27966) + | > current_lr: 0.00006 + | > step_time: 0.97120 (2.33549) + | > loader_time: 0.00230 (0.04463) + + + --> STEP: 144/234 -- GLOBAL_STEP: 60750 + | > loss: -0.30870 (-0.27454) + | > log_mle: -0.50747 (-0.39319) + | > loss_dur: 0.19876 (0.11865) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.18149 (19.47246) + | > current_lr: 0.00006 + | > step_time: 2.70710 (2.39487) + | > loader_time: 0.08580 (0.04511) + + + --> STEP: 149/234 -- GLOBAL_STEP: 60755 + | > loss: -0.37542 (-0.27613) + | > log_mle: -0.56410 (-0.39685) + | > loss_dur: 0.18868 (0.12072) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.43788 (20.09941) + | > current_lr: 0.00006 + | > step_time: 1.39790 (2.41632) + | > loader_time: 0.08560 (0.04758) + + + --> STEP: 154/234 -- GLOBAL_STEP: 60760 + | > loss: -0.32876 (-0.27817) + | > log_mle: -0.52056 (-0.40118) + | > loss_dur: 0.19180 (0.12301) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.78112 (21.03305) + | > current_lr: 0.00006 + | > step_time: 1.41200 (2.42706) + | > loader_time: 0.00520 (0.04731) + + + --> STEP: 159/234 -- GLOBAL_STEP: 60765 + | > loss: -0.34305 (-0.27993) + | > log_mle: -0.54281 (-0.40531) + | > loss_dur: 0.19976 (0.12538) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.23915 (21.79656) + | > current_lr: 0.00006 + | > step_time: 2.59760 (2.44188) + | > loader_time: 0.10910 (0.04768) + + + --> STEP: 164/234 -- GLOBAL_STEP: 60770 + | > loss: -0.32018 (-0.28167) + | > log_mle: -0.52310 (-0.40918) + | > loss_dur: 0.20291 (0.12751) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.07386 (22.83966) + | > current_lr: 0.00006 + | > step_time: 3.29630 (2.53690) + | > loader_time: 0.30370 (0.04991) + + + --> STEP: 169/234 -- GLOBAL_STEP: 60775 + | > loss: -0.30886 (-0.28368) + | > log_mle: -0.52258 (-0.41329) + | > loss_dur: 0.21372 (0.12961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.55846 (23.80399) + | > current_lr: 0.00006 + | > step_time: 3.59230 (2.60166) + | > loader_time: 0.00680 (0.05115) + + + --> STEP: 174/234 -- GLOBAL_STEP: 60780 + | > loss: -0.41627 (-0.28659) + | > log_mle: -0.62354 (-0.41857) + | > loss_dur: 0.20726 (0.13198) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.75715 (24.77657) + | > current_lr: 0.00006 + | > step_time: 2.69440 (2.57922) + | > loader_time: 0.08640 (0.05085) + + + --> STEP: 179/234 -- GLOBAL_STEP: 60785 + | > loss: -0.36530 (-0.28894) + | > log_mle: -0.60870 (-0.42341) + | > loss_dur: 0.24340 (0.13447) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.43544 (25.80949) + | > current_lr: 0.00006 + | > step_time: 1.96280 (2.56251) + | > loader_time: 0.00370 (0.04999) + + + --> STEP: 184/234 -- GLOBAL_STEP: 60790 + | > loss: -0.36271 (-0.29071) + | > log_mle: -0.57188 (-0.42734) + | > loss_dur: 0.20916 (0.13662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.98682 (26.98915) + | > current_lr: 0.00006 + | > step_time: 1.80930 (2.57171) + | > loader_time: 0.09490 (0.04971) + + + --> STEP: 189/234 -- GLOBAL_STEP: 60795 + | > loss: -0.36539 (-0.29276) + | > log_mle: -0.57825 (-0.43162) + | > loss_dur: 0.21286 (0.13885) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.40579 (27.88621) + | > current_lr: 0.00006 + | > step_time: 8.89010 (2.65762) + | > loader_time: 0.10860 (0.05049) + + + --> STEP: 194/234 -- GLOBAL_STEP: 60800 + | > loss: -0.39678 (-0.29539) + | > log_mle: -0.60405 (-0.43605) + | > loss_dur: 0.20727 (0.14066) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.13681 (28.71068) + | > current_lr: 0.00006 + | > step_time: 5.81240 (2.69904) + | > loader_time: 0.09010 (0.05513) + + + --> STEP: 199/234 -- GLOBAL_STEP: 60805 + | > loss: -0.39401 (-0.29761) + | > log_mle: -0.61769 (-0.44019) + | > loss_dur: 0.22368 (0.14259) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.52080 (29.53954) + | > current_lr: 0.00006 + | > step_time: 10.58500 (2.79351) + | > loader_time: 0.00680 (0.05521) + + + --> STEP: 204/234 -- GLOBAL_STEP: 60810 + | > loss: -0.40443 (-0.29950) + | > log_mle: -0.64809 (-0.44424) + | > loss_dur: 0.24366 (0.14473) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 105.67554 (30.42676) + | > current_lr: 0.00006 + | > step_time: 16.50120 (2.89423) + | > loader_time: 0.10070 (0.05581) + + + --> STEP: 209/234 -- GLOBAL_STEP: 60815 + | > loss: -0.39278 (-0.30194) + | > log_mle: -0.60858 (-0.44858) + | > loss_dur: 0.21580 (0.14664) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.45621 (31.50215) + | > current_lr: 0.00006 + | > step_time: 4.39970 (2.95647) + | > loader_time: 0.00420 (0.05927) + + + --> STEP: 214/234 -- GLOBAL_STEP: 60820 + | > loss: -0.43720 (-0.30518) + | > log_mle: -0.63972 (-0.45391) + | > loss_dur: 0.20252 (0.14873) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 93.24744 (32.75230) + | > current_lr: 0.00006 + | > step_time: 1.61070 (2.99538) + | > loader_time: 0.08680 (0.06240) + + + --> STEP: 219/234 -- GLOBAL_STEP: 60825 + | > loss: -0.51760 (-0.30834) + | > log_mle: -0.75128 (-0.45914) + | > loss_dur: 0.23369 (0.15080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.22176 (33.80056) + | > current_lr: 0.00006 + | > step_time: 5.28660 (3.08132) + | > loader_time: 0.01260 (0.06197) + + + --> STEP: 224/234 -- GLOBAL_STEP: 60830 + | > loss: -0.46444 (-0.31135) + | > log_mle: -0.70316 (-0.46412) + | > loss_dur: 0.23873 (0.15277) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.57867 (34.66589) + | > current_lr: 0.00006 + | > step_time: 3.50200 (3.08306) + | > loader_time: 0.00520 (0.06109) + + + --> STEP: 229/234 -- GLOBAL_STEP: 60835 + | > loss: -0.45032 (-0.31462) + | > log_mle: -0.74602 (-0.46975) + | > loss_dur: 0.29569 (0.15513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.28967 (35.89764) + | > current_lr: 0.00006 + | > step_time: 0.26220 (3.04148) + | > loader_time: 0.00320 (0.05984) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.79379 (+0.14547) + | > avg_loss: -0.27438 (+0.05121) + | > avg_log_mle: -0.51303 (+0.02814) + | > avg_loss_dur: 0.23866 (+0.02307) + + + > EPOCH: 260/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 15:58:24)  + + --> STEP: 0/234 -- GLOBAL_STEP: 60840 + | > loss: -0.28293 (-0.28293) + | > log_mle: -0.46035 (-0.46035) + | > loss_dur: 0.17742 (0.17742) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.26445 (42.26445) + | > current_lr: 0.00006 + | > step_time: 4.39510 (4.39511) + | > loader_time: 4.65360 (4.65362) + + + --> STEP: 5/234 -- GLOBAL_STEP: 60845 + | > loss: -0.30574 (-0.28789) + | > log_mle: -0.38880 (-0.38739) + | > loss_dur: 0.08306 (0.09950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.07173 (18.52038) + | > current_lr: 0.00006 + | > step_time: 2.59250 (1.51895) + | > loader_time: 0.00570 (0.01945) + + + --> STEP: 10/234 -- GLOBAL_STEP: 60850 + | > loss: -0.29924 (-0.29785) + | > log_mle: -0.38585 (-0.39039) + | > loss_dur: 0.08661 (0.09253) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.58225 (17.35073) + | > current_lr: 0.00006 + | > step_time: 4.29760 (2.66010) + | > loader_time: 0.00740 (1.23048) + + + --> STEP: 15/234 -- GLOBAL_STEP: 60855 + | > loss: -0.33741 (-0.30796) + | > log_mle: -0.40030 (-0.39364) + | > loss_dur: 0.06290 (0.08568) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.88869 (15.76319) + | > current_lr: 0.00006 + | > step_time: 2.60810 (3.52931) + | > loader_time: 0.09880 (0.83413) + + + --> STEP: 20/234 -- GLOBAL_STEP: 60860 + | > loss: -0.32710 (-0.31262) + | > log_mle: -0.39244 (-0.39345) + | > loss_dur: 0.06534 (0.08083) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.35634 (14.62447) + | > current_lr: 0.00006 + | > step_time: 3.69780 (3.70179) + | > loader_time: 0.00120 (0.63095) + + + --> STEP: 25/234 -- GLOBAL_STEP: 60865 + | > loss: -0.29969 (-0.31329) + | > log_mle: -0.37455 (-0.39212) + | > loss_dur: 0.07487 (0.07883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.15767 (13.94022) + | > current_lr: 0.00006 + | > step_time: 1.44500 (3.92467) + | > loader_time: 0.00110 (0.50948) + + + --> STEP: 30/234 -- GLOBAL_STEP: 60870 + | > loss: -0.30102 (-0.31366) + | > log_mle: -0.37763 (-0.39135) + | > loss_dur: 0.07660 (0.07769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.75860 (13.47188) + | > current_lr: 0.00006 + | > step_time: 3.00090 (3.54119) + | > loader_time: 0.00190 (0.42489) + + + --> STEP: 35/234 -- GLOBAL_STEP: 60875 + | > loss: -0.27669 (-0.31168) + | > log_mle: -0.36807 (-0.38939) + | > loss_dur: 0.09138 (0.07771) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.03632 (13.29430) + | > current_lr: 0.00006 + | > step_time: 1.72960 (3.41700) + | > loader_time: 0.00300 (0.37800) + + + --> STEP: 40/234 -- GLOBAL_STEP: 60880 + | > loss: -0.26728 (-0.30736) + | > log_mle: -0.36660 (-0.38660) + | > loss_dur: 0.09932 (0.07925) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.63476 (13.22929) + | > current_lr: 0.00006 + | > step_time: 1.15090 (3.19004) + | > loader_time: 0.00210 (0.33145) + + + --> STEP: 45/234 -- GLOBAL_STEP: 60885 + | > loss: -0.27575 (-0.30464) + | > log_mle: -0.38405 (-0.38475) + | > loss_dur: 0.10830 (0.08011) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.77138 (13.18775) + | > current_lr: 0.00006 + | > step_time: 1.70750 (3.03957) + | > loader_time: 0.09420 (0.29687) + + + --> STEP: 50/234 -- GLOBAL_STEP: 60890 + | > loss: -0.28053 (-0.30296) + | > log_mle: -0.36157 (-0.38352) + | > loss_dur: 0.08104 (0.08056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.72391 (12.76632) + | > current_lr: 0.00006 + | > step_time: 3.21430 (2.94069) + | > loader_time: 0.08340 (0.26905) + + + --> STEP: 55/234 -- GLOBAL_STEP: 60895 + | > loss: -0.29511 (-0.30085) + | > log_mle: -0.37404 (-0.38216) + | > loss_dur: 0.07893 (0.08131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.07838 (12.74212) + | > current_lr: 0.00006 + | > step_time: 2.21020 (2.82061) + | > loader_time: 0.00210 (0.24480) + + + --> STEP: 60/234 -- GLOBAL_STEP: 60900 + | > loss: -0.24562 (-0.29806) + | > log_mle: -0.37485 (-0.38078) + | > loss_dur: 0.12923 (0.08272) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.08978 (12.83806) + | > current_lr: 0.00006 + | > step_time: 2.06500 (2.74367) + | > loader_time: 0.00220 (0.22462) + + + --> STEP: 65/234 -- GLOBAL_STEP: 60905 + | > loss: -0.26476 (-0.29486) + | > log_mle: -0.36088 (-0.37951) + | > loss_dur: 0.09612 (0.08466) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.79372 (12.91801) + | > current_lr: 0.00006 + | > step_time: 1.37870 (2.65495) + | > loader_time: 0.00260 (0.20759) + + + --> STEP: 70/234 -- GLOBAL_STEP: 60910 + | > loss: -0.24034 (-0.29206) + | > log_mle: -0.34912 (-0.37788) + | > loss_dur: 0.10877 (0.08582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.21334 (12.80636) + | > current_lr: 0.00006 + | > step_time: 1.30800 (2.63607) + | > loader_time: 0.00260 (0.19538) + + + --> STEP: 75/234 -- GLOBAL_STEP: 60915 + | > loss: -0.24202 (-0.28917) + | > log_mle: -0.36342 (-0.37700) + | > loss_dur: 0.12140 (0.08784) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.95200 (12.74964) + | > current_lr: 0.00006 + | > step_time: 2.71590 (2.57901) + | > loader_time: 0.10100 (0.18386) + + + --> STEP: 80/234 -- GLOBAL_STEP: 60920 + | > loss: -0.24585 (-0.28697) + | > log_mle: -0.34841 (-0.37586) + | > loss_dur: 0.10256 (0.08889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.78154 (12.64232) + | > current_lr: 0.00006 + | > step_time: 2.08480 (2.53501) + | > loader_time: 0.00230 (0.17360) + + + --> STEP: 85/234 -- GLOBAL_STEP: 60925 + | > loss: -0.24551 (-0.28449) + | > log_mle: -0.35519 (-0.37497) + | > loss_dur: 0.10968 (0.09048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.80477 (12.70478) + | > current_lr: 0.00006 + | > step_time: 1.70470 (2.52705) + | > loader_time: 0.00220 (0.16470) + + + --> STEP: 90/234 -- GLOBAL_STEP: 60930 + | > loss: -0.25152 (-0.28275) + | > log_mle: -0.37782 (-0.37507) + | > loss_dur: 0.12630 (0.09232) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.04543 (12.87387) + | > current_lr: 0.00006 + | > step_time: 3.49690 (2.50856) + | > loader_time: 0.00250 (0.15571) + + + --> STEP: 95/234 -- GLOBAL_STEP: 60935 + | > loss: -0.29645 (-0.28185) + | > log_mle: -0.45794 (-0.37692) + | > loss_dur: 0.16149 (0.09508) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.90989 (13.35224) + | > current_lr: 0.00006 + | > step_time: 3.99680 (2.50612) + | > loader_time: 0.00610 (0.14942) + + + --> STEP: 100/234 -- GLOBAL_STEP: 60940 + | > loss: -0.26311 (-0.28054) + | > log_mle: -0.38691 (-0.37731) + | > loss_dur: 0.12380 (0.09677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.21520 (13.54224) + | > current_lr: 0.00006 + | > step_time: 2.20770 (2.47416) + | > loader_time: 0.00210 (0.14298) + + + --> STEP: 105/234 -- GLOBAL_STEP: 60945 + | > loss: -0.24463 (-0.27953) + | > log_mle: -0.36654 (-0.37872) + | > loss_dur: 0.12192 (0.09919) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.59340 (14.06522) + | > current_lr: 0.00006 + | > step_time: 2.48850 (2.46784) + | > loader_time: 0.20320 (0.13821) + + + --> STEP: 110/234 -- GLOBAL_STEP: 60950 + | > loss: -0.24072 (-0.27779) + | > log_mle: -0.38742 (-0.37971) + | > loss_dur: 0.14671 (0.10192) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.30635 (14.52752) + | > current_lr: 0.00006 + | > step_time: 1.30650 (2.45312) + | > loader_time: 0.07930 (0.13602) + + + --> STEP: 115/234 -- GLOBAL_STEP: 60955 + | > loss: -0.24485 (-0.27696) + | > log_mle: -0.40862 (-0.38167) + | > loss_dur: 0.16378 (0.10471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.05504 (15.05341) + | > current_lr: 0.00006 + | > step_time: 1.90520 (2.47025) + | > loader_time: 0.00250 (0.13332) + + + --> STEP: 120/234 -- GLOBAL_STEP: 60960 + | > loss: -0.28280 (-0.27608) + | > log_mle: -0.45178 (-0.38320) + | > loss_dur: 0.16898 (0.10712) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.49290 (15.53062) + | > current_lr: 0.00006 + | > step_time: 2.50660 (2.44830) + | > loader_time: 0.00250 (0.12944) + + + --> STEP: 125/234 -- GLOBAL_STEP: 60965 + | > loss: -0.26315 (-0.27456) + | > log_mle: -0.43495 (-0.38354) + | > loss_dur: 0.17180 (0.10898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.38764 (15.75448) + | > current_lr: 0.00006 + | > step_time: 3.39440 (2.44967) + | > loader_time: 0.00330 (0.12578) + + + --> STEP: 130/234 -- GLOBAL_STEP: 60970 + | > loss: -0.28457 (-0.27442) + | > log_mle: -0.45380 (-0.38584) + | > loss_dur: 0.16924 (0.11142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.91243 (16.34835) + | > current_lr: 0.00006 + | > step_time: 1.70960 (2.42238) + | > loader_time: 0.08720 (0.12295) + + + --> STEP: 135/234 -- GLOBAL_STEP: 60975 + | > loss: -0.24291 (-0.27483) + | > log_mle: -0.38307 (-0.38833) + | > loss_dur: 0.14016 (0.11350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.55016 (16.90879) + | > current_lr: 0.00006 + | > step_time: 3.29750 (2.45130) + | > loader_time: 0.09800 (0.12053) + + + --> STEP: 140/234 -- GLOBAL_STEP: 60980 + | > loss: -0.23108 (-0.27521) + | > log_mle: -0.41713 (-0.39136) + | > loss_dur: 0.18605 (0.11614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.70947 (17.59336) + | > current_lr: 0.00006 + | > step_time: 1.41450 (2.47912) + | > loader_time: 0.00350 (0.11771) + + + --> STEP: 145/234 -- GLOBAL_STEP: 60985 + | > loss: -0.33994 (-0.27639) + | > log_mle: -0.51617 (-0.39511) + | > loss_dur: 0.17623 (0.11872) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.20251 (18.37831) + | > current_lr: 0.00006 + | > step_time: 1.09390 (2.47904) + | > loader_time: 0.00260 (0.11504) + + + --> STEP: 150/234 -- GLOBAL_STEP: 60990 + | > loss: -0.30144 (-0.27768) + | > log_mle: -0.50320 (-0.39860) + | > loss_dur: 0.20176 (0.12092) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.67295 (19.23372) + | > current_lr: 0.00006 + | > step_time: 1.60620 (2.46540) + | > loader_time: 0.00380 (0.11307) + + + --> STEP: 155/234 -- GLOBAL_STEP: 60995 + | > loss: -0.36802 (-0.28012) + | > log_mle: -0.57177 (-0.40334) + | > loss_dur: 0.20375 (0.12323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.91068 (20.46821) + | > current_lr: 0.00006 + | > step_time: 9.29990 (2.55872) + | > loader_time: 0.10210 (0.11076) + + + --> STEP: 160/234 -- GLOBAL_STEP: 61000 + | > loss: -0.34668 (-0.28156) + | > log_mle: -0.55626 (-0.40707) + | > loss_dur: 0.20959 (0.12552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.26139 (21.52887) + | > current_lr: 0.00006 + | > step_time: 4.99460 (2.57917) + | > loader_time: 0.00220 (0.10744) + + + --> STEP: 165/234 -- GLOBAL_STEP: 61005 + | > loss: -0.35066 (-0.28300) + | > log_mle: -0.55095 (-0.41069) + | > loss_dur: 0.20029 (0.12769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.16827 (22.35679) + | > current_lr: 0.00006 + | > step_time: 1.78700 (2.56639) + | > loader_time: 0.00890 (0.10431) + + + --> STEP: 170/234 -- GLOBAL_STEP: 61010 + | > loss: -0.39586 (-0.28506) + | > log_mle: -0.60969 (-0.41494) + | > loss_dur: 0.21383 (0.12988) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.88865 (23.34585) + | > current_lr: 0.00006 + | > step_time: 10.29580 (2.62027) + | > loader_time: 0.40510 (0.10426) + + + --> STEP: 175/234 -- GLOBAL_STEP: 61015 + | > loss: -0.35516 (-0.28788) + | > log_mle: -0.58415 (-0.42010) + | > loss_dur: 0.22900 (0.13222) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.69294 (24.08855) + | > current_lr: 0.00006 + | > step_time: 2.40300 (2.68536) + | > loader_time: 0.09430 (0.10425) + + + --> STEP: 180/234 -- GLOBAL_STEP: 61020 + | > loss: -0.37641 (-0.29035) + | > log_mle: -0.58551 (-0.42492) + | > loss_dur: 0.20910 (0.13457) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.51293 (24.89190) + | > current_lr: 0.00006 + | > step_time: 3.40560 (2.70061) + | > loader_time: 0.09880 (0.10201) + + + --> STEP: 185/234 -- GLOBAL_STEP: 61025 + | > loss: -0.37993 (-0.29219) + | > log_mle: -0.60967 (-0.42912) + | > loss_dur: 0.22974 (0.13693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.81391 (26.19287) + | > current_lr: 0.00006 + | > step_time: 6.31380 (2.75424) + | > loader_time: 0.09390 (0.10083) + + + --> STEP: 190/234 -- GLOBAL_STEP: 61030 + | > loss: -0.38820 (-0.29433) + | > log_mle: -0.59074 (-0.43338) + | > loss_dur: 0.20254 (0.13905) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.60298 (27.25500) + | > current_lr: 0.00006 + | > step_time: 3.19110 (2.81081) + | > loader_time: 0.00330 (0.10011) + + + --> STEP: 195/234 -- GLOBAL_STEP: 61035 + | > loss: -0.39483 (-0.29717) + | > log_mle: -0.61815 (-0.43807) + | > loss_dur: 0.22332 (0.14090) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.69641 (28.13072) + | > current_lr: 0.00006 + | > step_time: 6.48750 (2.84560) + | > loader_time: 0.20450 (0.10057) + + + --> STEP: 200/234 -- GLOBAL_STEP: 61040 + | > loss: -0.36484 (-0.29928) + | > log_mle: -0.61142 (-0.44214) + | > loss_dur: 0.24659 (0.14286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.17135 (29.17551) + | > current_lr: 0.00006 + | > step_time: 5.50790 (2.90499) + | > loader_time: 0.08750 (0.10093) + + + --> STEP: 205/234 -- GLOBAL_STEP: 61045 + | > loss: -0.37047 (-0.30134) + | > log_mle: -0.60018 (-0.44616) + | > loss_dur: 0.22971 (0.14483) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.75543 (29.99784) + | > current_lr: 0.00006 + | > step_time: 5.01050 (2.99999) + | > loader_time: 0.48630 (0.10182) + + + --> STEP: 210/234 -- GLOBAL_STEP: 61050 + | > loss: -0.45126 (-0.30414) + | > log_mle: -0.68357 (-0.45096) + | > loss_dur: 0.23230 (0.14682) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.52393 (31.04864) + | > current_lr: 0.00006 + | > step_time: 6.78420 (3.02929) + | > loader_time: 0.10660 (0.10135) + + + --> STEP: 215/234 -- GLOBAL_STEP: 61055 + | > loss: -0.41275 (-0.30731) + | > log_mle: -0.64412 (-0.45614) + | > loss_dur: 0.23137 (0.14883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.48934 (32.12988) + | > current_lr: 0.00006 + | > step_time: 9.69030 (3.08125) + | > loader_time: 0.11200 (0.10410) + + + --> STEP: 220/234 -- GLOBAL_STEP: 61060 + | > loss: -0.46168 (-0.31085) + | > log_mle: -0.69688 (-0.46176) + | > loss_dur: 0.23520 (0.15092) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 96.64528 (33.52843) + | > current_lr: 0.00006 + | > step_time: 3.90110 (3.11078) + | > loader_time: 0.10260 (0.10306) + + + --> STEP: 225/234 -- GLOBAL_STEP: 61065 + | > loss: -0.48889 (-0.31389) + | > log_mle: -0.73872 (-0.46679) + | > loss_dur: 0.24983 (0.15290) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 132.19348 (35.09192) + | > current_lr: 0.00006 + | > step_time: 0.25310 (3.08975) + | > loader_time: 0.00310 (0.10163) + + + --> STEP: 230/234 -- GLOBAL_STEP: 61070 + | > loss: -0.49277 (-0.31673) + | > log_mle: -0.80584 (-0.47220) + | > loss_dur: 0.31308 (0.15547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.21827 (36.32166) + | > current_lr: 0.00006 + | > step_time: 0.24730 (3.02792) + | > loader_time: 0.00370 (0.09951) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.20079 (-0.59300) + | > avg_loss: -0.30702 (-0.03265) + | > avg_log_mle: -0.53855 (-0.02552) + | > avg_loss_dur: 0.23153 (-0.00713) + + + > EPOCH: 261/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 16:11:21)  + + --> STEP: 1/234 -- GLOBAL_STEP: 61075 + | > loss: -0.29188 (-0.29188) + | > log_mle: -0.38498 (-0.38498) + | > loss_dur: 0.09310 (0.09310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.76822 (18.76822) + | > current_lr: 0.00007 + | > step_time: 5.79700 (5.79696) + | > loader_time: 0.00160 (0.00157) + + + --> STEP: 6/234 -- GLOBAL_STEP: 61080 + | > loss: -0.31879 (-0.29841) + | > log_mle: -0.38845 (-0.39098) + | > loss_dur: 0.06966 (0.09257) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.63595 (19.17641) + | > current_lr: 0.00007 + | > step_time: 4.30450 (4.08808) + | > loader_time: 0.19320 (0.04780) + + + --> STEP: 11/234 -- GLOBAL_STEP: 61085 + | > loss: -0.34810 (-0.30926) + | > log_mle: -0.41032 (-0.39536) + | > loss_dur: 0.06222 (0.08610) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.11309 (17.50830) + | > current_lr: 0.00007 + | > step_time: 1.10990 (4.47586) + | > loader_time: 0.00270 (0.06404) + + + --> STEP: 16/234 -- GLOBAL_STEP: 61090 + | > loss: -0.32862 (-0.31496) + | > log_mle: -0.40248 (-0.39719) + | > loss_dur: 0.07386 (0.08223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.93698 (16.52146) + | > current_lr: 0.00007 + | > step_time: 5.29770 (4.29174) + | > loader_time: 0.09870 (0.05677) + + + --> STEP: 21/234 -- GLOBAL_STEP: 61095 + | > loss: -0.30755 (-0.31724) + | > log_mle: -0.38068 (-0.39558) + | > loss_dur: 0.07313 (0.07833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.60753 (15.39913) + | > current_lr: 0.00007 + | > step_time: 7.19630 (4.41729) + | > loader_time: 0.00500 (0.04823) + + + --> STEP: 26/234 -- GLOBAL_STEP: 61100 + | > loss: -0.29791 (-0.31727) + | > log_mle: -0.38216 (-0.39447) + | > loss_dur: 0.08426 (0.07720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.99795 (14.53136) + | > current_lr: 0.00007 + | > step_time: 3.20910 (4.16156) + | > loader_time: 0.07790 (0.04557) + + + --> STEP: 31/234 -- GLOBAL_STEP: 61105 + | > loss: -0.28148 (-0.31649) + | > log_mle: -0.37354 (-0.39334) + | > loss_dur: 0.09206 (0.07685) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.26705 (13.99578) + | > current_lr: 0.00007 + | > step_time: 1.33000 (3.75643) + | > loader_time: 0.00210 (0.04113) + + + --> STEP: 36/234 -- GLOBAL_STEP: 61110 + | > loss: -0.27561 (-0.31371) + | > log_mle: -0.36734 (-0.39127) + | > loss_dur: 0.09174 (0.07755) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.09474 (13.67931) + | > current_lr: 0.00007 + | > step_time: 1.70250 (3.41799) + | > loader_time: 0.08800 (0.04034) + + + --> STEP: 41/234 -- GLOBAL_STEP: 61115 + | > loss: -0.30935 (-0.31134) + | > log_mle: -0.37979 (-0.38910) + | > loss_dur: 0.07044 (0.07776) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.21481 (13.35507) + | > current_lr: 0.00007 + | > step_time: 5.21100 (3.58230) + | > loader_time: 0.08590 (0.05128) + + + --> STEP: 46/234 -- GLOBAL_STEP: 61120 + | > loss: -0.27547 (-0.30773) + | > log_mle: -0.37362 (-0.38709) + | > loss_dur: 0.09816 (0.07937) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.18860 (13.28114) + | > current_lr: 0.00007 + | > step_time: 0.87430 (3.38370) + | > loader_time: 0.07710 (0.04767) + + + --> STEP: 51/234 -- GLOBAL_STEP: 61125 + | > loss: -0.28241 (-0.30626) + | > log_mle: -0.36788 (-0.38582) + | > loss_dur: 0.08547 (0.07956) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.64357 (12.76557) + | > current_lr: 0.00007 + | > step_time: 1.89200 (3.20628) + | > loader_time: 0.00360 (0.04323) + + + --> STEP: 56/234 -- GLOBAL_STEP: 61130 + | > loss: -0.27498 (-0.30403) + | > log_mle: -0.37211 (-0.38465) + | > loss_dur: 0.09713 (0.08062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.23841 (12.52771) + | > current_lr: 0.00007 + | > step_time: 1.10740 (3.04805) + | > loader_time: 0.00810 (0.04111) + + + --> STEP: 61/234 -- GLOBAL_STEP: 61135 + | > loss: -0.27094 (-0.30117) + | > log_mle: -0.36465 (-0.38321) + | > loss_dur: 0.09371 (0.08204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.02343 (12.78809) + | > current_lr: 0.00007 + | > step_time: 2.21520 (2.95252) + | > loader_time: 0.08420 (0.04332) + + + --> STEP: 66/234 -- GLOBAL_STEP: 61140 + | > loss: -0.27666 (-0.29807) + | > log_mle: -0.35919 (-0.38180) + | > loss_dur: 0.08254 (0.08373) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47525 (12.75920) + | > current_lr: 0.00007 + | > step_time: 1.49860 (2.88747) + | > loader_time: 0.00340 (0.04178) + + + --> STEP: 71/234 -- GLOBAL_STEP: 61145 + | > loss: -0.25167 (-0.29482) + | > log_mle: -0.37578 (-0.38031) + | > loss_dur: 0.12411 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.31029 (12.94045) + | > current_lr: 0.00007 + | > step_time: 3.10990 (2.87461) + | > loader_time: 0.00250 (0.04143) + + + --> STEP: 76/234 -- GLOBAL_STEP: 61150 + | > loss: -0.25778 (-0.29183) + | > log_mle: -0.36649 (-0.37918) + | > loss_dur: 0.10871 (0.08734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.33011 (12.93390) + | > current_lr: 0.00007 + | > step_time: 1.30060 (2.84838) + | > loader_time: 0.00230 (0.04006) + + + --> STEP: 81/234 -- GLOBAL_STEP: 61155 + | > loss: -0.25401 (-0.28969) + | > log_mle: -0.37277 (-0.37806) + | > loss_dur: 0.11876 (0.08836) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.69435 (12.82462) + | > current_lr: 0.00007 + | > step_time: 1.38560 (2.80712) + | > loader_time: 0.00380 (0.03885) + + + --> STEP: 86/234 -- GLOBAL_STEP: 61160 + | > loss: -0.25825 (-0.28720) + | > log_mle: -0.37192 (-0.37713) + | > loss_dur: 0.11367 (0.08993) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.31674 (12.86152) + | > current_lr: 0.00007 + | > step_time: 2.10420 (2.77316) + | > loader_time: 0.00190 (0.03779) + + + --> STEP: 91/234 -- GLOBAL_STEP: 61165 + | > loss: -0.24949 (-0.28528) + | > log_mle: -0.37968 (-0.37732) + | > loss_dur: 0.13018 (0.09204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.32667 (12.96805) + | > current_lr: 0.00007 + | > step_time: 1.51150 (2.70688) + | > loader_time: 0.00370 (0.03772) + + + --> STEP: 96/234 -- GLOBAL_STEP: 61170 + | > loss: -0.24804 (-0.28444) + | > log_mle: -0.36444 (-0.37896) + | > loss_dur: 0.11640 (0.09453) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.94134 (13.51600) + | > current_lr: 0.00007 + | > step_time: 1.98720 (2.65851) + | > loader_time: 0.00340 (0.03674) + + + --> STEP: 101/234 -- GLOBAL_STEP: 61175 + | > loss: -0.24706 (-0.28285) + | > log_mle: -0.40632 (-0.37966) + | > loss_dur: 0.15926 (0.09681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.78737 (13.89464) + | > current_lr: 0.00007 + | > step_time: 1.04950 (2.62133) + | > loader_time: 0.00240 (0.03504) + + + --> STEP: 106/234 -- GLOBAL_STEP: 61180 + | > loss: -0.22795 (-0.28152) + | > log_mle: -0.41023 (-0.38096) + | > loss_dur: 0.18229 (0.09944) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.53922 (14.31450) + | > current_lr: 0.00007 + | > step_time: 1.20900 (2.57791) + | > loader_time: 0.00270 (0.03356) + + + --> STEP: 111/234 -- GLOBAL_STEP: 61185 + | > loss: -0.27143 (-0.28006) + | > log_mle: -0.46096 (-0.38242) + | > loss_dur: 0.18953 (0.10235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.98748 (14.84356) + | > current_lr: 0.00007 + | > step_time: 1.50750 (2.55308) + | > loader_time: 0.00330 (0.03714) + + + --> STEP: 116/234 -- GLOBAL_STEP: 61190 + | > loss: -0.24801 (-0.27886) + | > log_mle: -0.42641 (-0.38397) + | > loss_dur: 0.17840 (0.10511) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.01030 (15.35315) + | > current_lr: 0.00007 + | > step_time: 1.29620 (2.56314) + | > loader_time: 0.00410 (0.03691) + + + --> STEP: 121/234 -- GLOBAL_STEP: 61195 + | > loss: -0.21269 (-0.27773) + | > log_mle: -0.34419 (-0.38483) + | > loss_dur: 0.13149 (0.10710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.74074 (15.70260) + | > current_lr: 0.00007 + | > step_time: 2.70050 (2.52867) + | > loader_time: 0.00250 (0.03613) + + + --> STEP: 126/234 -- GLOBAL_STEP: 61200 + | > loss: -0.27935 (-0.27684) + | > log_mle: -0.46514 (-0.38606) + | > loss_dur: 0.18579 (0.10922) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.08723 (16.47548) + | > current_lr: 0.00007 + | > step_time: 1.77850 (2.50370) + | > loader_time: 0.00310 (0.03621) + + + --> STEP: 131/234 -- GLOBAL_STEP: 61205 + | > loss: -0.33151 (-0.27689) + | > log_mle: -0.52078 (-0.38861) + | > loss_dur: 0.18927 (0.11172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.03148 (17.15967) + | > current_lr: 0.00007 + | > step_time: 1.29700 (2.46928) + | > loader_time: 0.00260 (0.03498) + + + --> STEP: 136/234 -- GLOBAL_STEP: 61210 + | > loss: -0.36167 (-0.27745) + | > log_mle: -0.57561 (-0.39144) + | > loss_dur: 0.21394 (0.11399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.19013 (17.73433) + | > current_lr: 0.00007 + | > step_time: 1.40500 (2.43729) + | > loader_time: 0.00330 (0.03384) + + + --> STEP: 141/234 -- GLOBAL_STEP: 61215 + | > loss: -0.29087 (-0.27742) + | > log_mle: -0.47063 (-0.39374) + | > loss_dur: 0.17976 (0.11631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.58303 (18.31421) + | > current_lr: 0.00007 + | > step_time: 2.51910 (2.41901) + | > loader_time: 0.00290 (0.03391) + + + --> STEP: 146/234 -- GLOBAL_STEP: 61220 + | > loss: -0.32774 (-0.27887) + | > log_mle: -0.52257 (-0.39790) + | > loss_dur: 0.19483 (0.11903) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.22321 (19.45916) + | > current_lr: 0.00007 + | > step_time: 3.39170 (2.42315) + | > loader_time: 0.10200 (0.03416) + + + --> STEP: 151/234 -- GLOBAL_STEP: 61225 + | > loss: -0.31448 (-0.28008) + | > log_mle: -0.48852 (-0.40125) + | > loss_dur: 0.17404 (0.12117) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.14564 (20.22095) + | > current_lr: 0.00007 + | > step_time: 2.10210 (2.41142) + | > loader_time: 0.09090 (0.03481) + + + --> STEP: 156/234 -- GLOBAL_STEP: 61230 + | > loss: -0.33641 (-0.28228) + | > log_mle: -0.52984 (-0.40601) + | > loss_dur: 0.19343 (0.12373) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.96201 (21.75953) + | > current_lr: 0.00007 + | > step_time: 3.70520 (2.44569) + | > loader_time: 0.08430 (0.03536) + + + --> STEP: 161/234 -- GLOBAL_STEP: 61235 + | > loss: -0.36309 (-0.28397) + | > log_mle: -0.55580 (-0.41004) + | > loss_dur: 0.19270 (0.12607) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.54839 (22.53683) + | > current_lr: 0.00007 + | > step_time: 5.60010 (2.53050) + | > loader_time: 0.00340 (0.03802) + + + --> STEP: 166/234 -- GLOBAL_STEP: 61240 + | > loss: -0.32182 (-0.28535) + | > log_mle: -0.49813 (-0.41349) + | > loss_dur: 0.17632 (0.12814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.06785 (23.24199) + | > current_lr: 0.00007 + | > step_time: 1.79420 (2.54216) + | > loader_time: 0.00380 (0.03817) + + + --> STEP: 171/234 -- GLOBAL_STEP: 61245 + | > loss: -0.40519 (-0.28797) + | > log_mle: -0.60331 (-0.41839) + | > loss_dur: 0.19812 (0.13042) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.57238 (24.33057) + | > current_lr: 0.00007 + | > step_time: 3.39250 (2.53016) + | > loader_time: 0.00270 (0.03769) + + + --> STEP: 176/234 -- GLOBAL_STEP: 61250 + | > loss: -0.37537 (-0.29054) + | > log_mle: -0.58007 (-0.42330) + | > loss_dur: 0.20470 (0.13276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.62980 (25.16686) + | > current_lr: 0.00007 + | > step_time: 1.90390 (2.54123) + | > loader_time: 0.00400 (0.03732) + + + --> STEP: 181/234 -- GLOBAL_STEP: 61255 + | > loss: -0.31223 (-0.29259) + | > log_mle: -0.51457 (-0.42764) + | > loss_dur: 0.20234 (0.13505) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.94283 (26.19137) + | > current_lr: 0.00007 + | > step_time: 5.50730 (2.55938) + | > loader_time: 0.08730 (0.03688) + + + --> STEP: 186/234 -- GLOBAL_STEP: 61260 + | > loss: -0.32928 (-0.29483) + | > log_mle: -0.55788 (-0.43230) + | > loss_dur: 0.22859 (0.13747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.69020 (27.14156) + | > current_lr: 0.00007 + | > step_time: 3.10490 (2.61909) + | > loader_time: 0.09230 (0.03951) + + + --> STEP: 191/234 -- GLOBAL_STEP: 61265 + | > loss: -0.38172 (-0.29731) + | > log_mle: -0.58309 (-0.43672) + | > loss_dur: 0.20137 (0.13941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.51680 (28.14618) + | > current_lr: 0.00007 + | > step_time: 3.91060 (2.64539) + | > loader_time: 0.20070 (0.04058) + + + --> STEP: 196/234 -- GLOBAL_STEP: 61270 + | > loss: -0.36064 (-0.30007) + | > log_mle: -0.57859 (-0.44142) + | > loss_dur: 0.21795 (0.14135) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.40904 (29.22096) + | > current_lr: 0.00007 + | > step_time: 6.19800 (2.75053) + | > loader_time: 0.29310 (0.04256) + + + --> STEP: 201/234 -- GLOBAL_STEP: 61275 + | > loss: -0.30349 (-0.30221) + | > log_mle: -0.52404 (-0.44548) + | > loss_dur: 0.22055 (0.14327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.78791 (30.37541) + | > current_lr: 0.00007 + | > step_time: 2.92550 (2.75894) + | > loader_time: 0.09010 (0.04341) + + + --> STEP: 206/234 -- GLOBAL_STEP: 61280 + | > loss: -0.41293 (-0.30453) + | > log_mle: -0.63472 (-0.44984) + | > loss_dur: 0.22179 (0.14531) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.88508 (31.35713) + | > current_lr: 0.00007 + | > step_time: 3.40570 (2.81524) + | > loader_time: 0.00440 (0.04540) + + + --> STEP: 211/234 -- GLOBAL_STEP: 61285 + | > loss: -0.46513 (-0.30757) + | > log_mle: -0.71351 (-0.45498) + | > loss_dur: 0.24838 (0.14741) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.39497 (32.33710) + | > current_lr: 0.00007 + | > step_time: 6.70940 (2.85910) + | > loader_time: 0.09460 (0.04620) + + + --> STEP: 216/234 -- GLOBAL_STEP: 61290 + | > loss: -0.44072 (-0.31057) + | > log_mle: -0.69171 (-0.45995) + | > loss_dur: 0.25099 (0.14938) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.19030 (33.35608) + | > current_lr: 0.00007 + | > step_time: 5.19570 (2.92202) + | > loader_time: 0.01000 (0.04614) + + + --> STEP: 221/234 -- GLOBAL_STEP: 61295 + | > loss: -0.40349 (-0.31368) + | > log_mle: -0.61713 (-0.46489) + | > loss_dur: 0.21363 (0.15121) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.90173 (34.26982) + | > current_lr: 0.00007 + | > step_time: 6.50000 (2.97049) + | > loader_time: 0.00560 (0.04561) + + + --> STEP: 226/234 -- GLOBAL_STEP: 61300 + | > loss: -0.49243 (-0.31738) + | > log_mle: -0.73561 (-0.47066) + | > loss_dur: 0.24318 (0.15328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.07145 (35.34283) + | > current_lr: 0.00007 + | > step_time: 1.09520 (2.95381) + | > loader_time: 0.00500 (0.04509) + + + --> STEP: 231/234 -- GLOBAL_STEP: 61305 + | > loss: -0.41767 (-0.32020) + | > log_mle: -0.79295 (-0.47665) + | > loss_dur: 0.37527 (0.15646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 134.07687 (37.14267) + | > current_lr: 0.00007 + | > step_time: 0.27140 (2.89714) + | > loader_time: 0.00500 (0.04421) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.13021 (-0.07058) + | > avg_loss: -0.31444 (-0.00741) + | > avg_log_mle: -0.55527 (-0.01671) + | > avg_loss_dur: 0.24083 (+0.00930) + + + > EPOCH: 262/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 16:23:55)  + + --> STEP: 2/234 -- GLOBAL_STEP: 61310 + | > loss: -0.32711 (-0.32005) + | > log_mle: -0.40390 (-0.39578) + | > loss_dur: 0.07679 (0.07572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.71416 (22.46830) + | > current_lr: 0.00007 + | > step_time: 15.68890 (9.49678) + | > loader_time: 0.20110 (0.50030) + + + --> STEP: 7/234 -- GLOBAL_STEP: 61315 + | > loss: -0.31810 (-0.30717) + | > log_mle: -0.39115 (-0.39188) + | > loss_dur: 0.07304 (0.08472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.17311 (20.39493) + | > current_lr: 0.00007 + | > step_time: 2.70260 (3.78873) + | > loader_time: 0.00170 (0.15667) + + + --> STEP: 12/234 -- GLOBAL_STEP: 61320 + | > loss: -0.29340 (-0.31031) + | > log_mle: -0.38936 (-0.39561) + | > loss_dur: 0.09596 (0.08530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.97249 (18.19981) + | > current_lr: 0.00007 + | > step_time: 2.08830 (3.74262) + | > loader_time: 0.00150 (0.09300) + + + --> STEP: 17/234 -- GLOBAL_STEP: 61325 + | > loss: -0.32792 (-0.31699) + | > log_mle: -0.39266 (-0.39735) + | > loss_dur: 0.06474 (0.08036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.17723 (16.97635) + | > current_lr: 0.00007 + | > step_time: 11.20930 (4.38971) + | > loader_time: 0.08270 (0.09849) + + + --> STEP: 22/234 -- GLOBAL_STEP: 61330 + | > loss: -0.31477 (-0.31739) + | > log_mle: -0.39191 (-0.39550) + | > loss_dur: 0.07714 (0.07811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.60866 (16.56739) + | > current_lr: 0.00007 + | > step_time: 4.89710 (4.08416) + | > loader_time: 0.00310 (0.09299) + + + --> STEP: 27/234 -- GLOBAL_STEP: 61335 + | > loss: -0.31659 (-0.31733) + | > log_mle: -0.38554 (-0.39402) + | > loss_dur: 0.06896 (0.07669) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.30617 (15.76143) + | > current_lr: 0.00007 + | > step_time: 6.49710 (4.12769) + | > loader_time: 0.00170 (0.07624) + + + --> STEP: 32/234 -- GLOBAL_STEP: 61340 + | > loss: -0.30764 (-0.31607) + | > log_mle: -0.38778 (-0.39281) + | > loss_dur: 0.08013 (0.07674) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.26870 (15.01161) + | > current_lr: 0.00007 + | > step_time: 5.01260 (4.27476) + | > loader_time: 0.18950 (0.08265) + + + --> STEP: 37/234 -- GLOBAL_STEP: 61345 + | > loss: -0.29559 (-0.31347) + | > log_mle: -0.36388 (-0.39030) + | > loss_dur: 0.06829 (0.07683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.84362 (14.41787) + | > current_lr: 0.00007 + | > step_time: 4.41500 (4.06334) + | > loader_time: 0.00500 (0.07639) + + + --> STEP: 42/234 -- GLOBAL_STEP: 61350 + | > loss: -0.27818 (-0.31085) + | > log_mle: -0.36054 (-0.38830) + | > loss_dur: 0.08236 (0.07745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.74712 (13.91493) + | > current_lr: 0.00007 + | > step_time: 1.78560 (3.76917) + | > loader_time: 0.00340 (0.06759) + + + --> STEP: 47/234 -- GLOBAL_STEP: 61355 + | > loss: -0.29347 (-0.30775) + | > log_mle: -0.37931 (-0.38685) + | > loss_dur: 0.08584 (0.07910) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.62375 (13.88871) + | > current_lr: 0.00007 + | > step_time: 1.20580 (3.55695) + | > loader_time: 0.00300 (0.06263) + + + --> STEP: 52/234 -- GLOBAL_STEP: 61360 + | > loss: -0.25661 (-0.30574) + | > log_mle: -0.36618 (-0.38549) + | > loss_dur: 0.10956 (0.07974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.06553 (13.36978) + | > current_lr: 0.00007 + | > step_time: 1.29080 (3.32543) + | > loader_time: 0.00300 (0.05688) + + + --> STEP: 57/234 -- GLOBAL_STEP: 61365 + | > loss: -0.25222 (-0.30348) + | > log_mle: -0.35178 (-0.38420) + | > loss_dur: 0.09956 (0.08072) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.52449 (13.18359) + | > current_lr: 0.00007 + | > step_time: 1.39810 (3.19260) + | > loader_time: 0.00290 (0.05580) + + + --> STEP: 62/234 -- GLOBAL_STEP: 61370 + | > loss: -0.23018 (-0.30061) + | > log_mle: -0.38389 (-0.38354) + | > loss_dur: 0.15371 (0.08293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.84805 (13.18717) + | > current_lr: 0.00007 + | > step_time: 1.32910 (3.09939) + | > loader_time: 0.00180 (0.05159) + + + --> STEP: 67/234 -- GLOBAL_STEP: 61375 + | > loss: -0.26895 (-0.29839) + | > log_mle: -0.37362 (-0.38213) + | > loss_dur: 0.10468 (0.08374) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.23553 (12.90876) + | > current_lr: 0.00007 + | > step_time: 1.88840 (2.97094) + | > loader_time: 0.01420 (0.05043) + + + --> STEP: 72/234 -- GLOBAL_STEP: 61380 + | > loss: -0.27085 (-0.29535) + | > log_mle: -0.36300 (-0.38060) + | > loss_dur: 0.09215 (0.08525) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.04455 (13.07679) + | > current_lr: 0.00007 + | > step_time: 1.69440 (2.93293) + | > loader_time: 0.00230 (0.04837) + + + --> STEP: 77/234 -- GLOBAL_STEP: 61385 + | > loss: -0.25715 (-0.29220) + | > log_mle: -0.36295 (-0.37944) + | > loss_dur: 0.10581 (0.08725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.07829 (13.14043) + | > current_lr: 0.00007 + | > step_time: 1.48300 (2.85237) + | > loader_time: 0.00220 (0.04786) + + + --> STEP: 82/234 -- GLOBAL_STEP: 61390 + | > loss: -0.24747 (-0.28999) + | > log_mle: -0.35895 (-0.37831) + | > loss_dur: 0.11148 (0.08832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.40148 (13.01581) + | > current_lr: 0.00007 + | > step_time: 2.29700 (2.80940) + | > loader_time: 0.00340 (0.04706) + + + --> STEP: 87/234 -- GLOBAL_STEP: 61395 + | > loss: -0.24663 (-0.28766) + | > log_mle: -0.36099 (-0.37749) + | > loss_dur: 0.11436 (0.08983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.26883 (12.97893) + | > current_lr: 0.00007 + | > step_time: 1.30420 (2.75477) + | > loader_time: 0.00210 (0.04452) + + + --> STEP: 92/234 -- GLOBAL_STEP: 61400 + | > loss: -0.27293 (-0.28597) + | > log_mle: -0.40291 (-0.37815) + | > loss_dur: 0.12998 (0.09218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.17280 (13.14795) + | > current_lr: 0.00007 + | > step_time: 2.14410 (2.71007) + | > loader_time: 0.09350 (0.04415) + + + --> STEP: 97/234 -- GLOBAL_STEP: 61405 + | > loss: -0.25989 (-0.28500) + | > log_mle: -0.38487 (-0.37960) + | > loss_dur: 0.12498 (0.09460) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.98366 (13.66087) + | > current_lr: 0.00007 + | > step_time: 1.28710 (2.66562) + | > loader_time: 0.00530 (0.04207) + + + --> STEP: 102/234 -- GLOBAL_STEP: 61410 + | > loss: -0.22859 (-0.28333) + | > log_mle: -0.37003 (-0.38030) + | > loss_dur: 0.14144 (0.09697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.24415 (14.07465) + | > current_lr: 0.00007 + | > step_time: 2.30410 (2.64357) + | > loader_time: 0.00290 (0.04111) + + + --> STEP: 107/234 -- GLOBAL_STEP: 61415 + | > loss: -0.25166 (-0.28236) + | > log_mle: -0.41219 (-0.38211) + | > loss_dur: 0.16053 (0.09974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.68723 (14.67727) + | > current_lr: 0.00007 + | > step_time: 1.19800 (2.61238) + | > loader_time: 0.00270 (0.04106) + + + --> STEP: 112/234 -- GLOBAL_STEP: 61420 + | > loss: -0.25385 (-0.28116) + | > log_mle: -0.42361 (-0.38372) + | > loss_dur: 0.16977 (0.10257) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.74694 (15.27747) + | > current_lr: 0.00007 + | > step_time: 1.78240 (2.61348) + | > loader_time: 0.00190 (0.04162) + + + --> STEP: 117/234 -- GLOBAL_STEP: 61425 + | > loss: -0.26388 (-0.28008) + | > log_mle: -0.41623 (-0.38524) + | > loss_dur: 0.15235 (0.10516) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.85576 (15.68387) + | > current_lr: 0.00007 + | > step_time: 1.71130 (2.60281) + | > loader_time: 0.08610 (0.04399) + + + --> STEP: 122/234 -- GLOBAL_STEP: 61430 + | > loss: -0.24516 (-0.27874) + | > log_mle: -0.38862 (-0.38579) + | > loss_dur: 0.14346 (0.10706) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.47076 (16.05388) + | > current_lr: 0.00007 + | > step_time: 5.72110 (2.65213) + | > loader_time: 0.09360 (0.04396) + + + --> STEP: 127/234 -- GLOBAL_STEP: 61435 + | > loss: -0.26056 (-0.27812) + | > log_mle: -0.45086 (-0.38769) + | > loss_dur: 0.19029 (0.10957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.74754 (16.59449) + | > current_lr: 0.00007 + | > step_time: 1.31140 (2.62960) + | > loader_time: 0.07510 (0.04358) + + + --> STEP: 132/234 -- GLOBAL_STEP: 61440 + | > loss: -0.27166 (-0.27823) + | > log_mle: -0.43046 (-0.39017) + | > loss_dur: 0.15880 (0.11194) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.13281 (17.15007) + | > current_lr: 0.00007 + | > step_time: 2.51170 (2.61192) + | > loader_time: 0.08930 (0.04330) + + + --> STEP: 137/234 -- GLOBAL_STEP: 61445 + | > loss: -0.25750 (-0.27878) + | > log_mle: -0.44807 (-0.39308) + | > loss_dur: 0.19057 (0.11430) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.29277 (17.86367) + | > current_lr: 0.00007 + | > step_time: 1.95180 (2.59777) + | > loader_time: 0.00360 (0.04191) + + + --> STEP: 142/234 -- GLOBAL_STEP: 61450 + | > loss: -0.26600 (-0.27883) + | > log_mle: -0.45560 (-0.39544) + | > loss_dur: 0.18961 (0.11661) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.53220 (18.60919) + | > current_lr: 0.00007 + | > step_time: 1.79960 (2.56200) + | > loader_time: 0.00600 (0.04195) + + + --> STEP: 147/234 -- GLOBAL_STEP: 61455 + | > loss: -0.28006 (-0.28030) + | > log_mle: -0.46313 (-0.39966) + | > loss_dur: 0.18307 (0.11936) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.40790 (19.59097) + | > current_lr: 0.00007 + | > step_time: 2.00000 (2.56269) + | > loader_time: 0.08850 (0.04181) + + + --> STEP: 152/234 -- GLOBAL_STEP: 61460 + | > loss: -0.34496 (-0.28203) + | > log_mle: -0.55287 (-0.40354) + | > loss_dur: 0.20791 (0.12151) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.34764 (20.50367) + | > current_lr: 0.00007 + | > step_time: 4.32030 (2.57337) + | > loader_time: 0.08360 (0.04167) + + + --> STEP: 157/234 -- GLOBAL_STEP: 61465 + | > loss: -0.30919 (-0.28427) + | > log_mle: -0.49133 (-0.40801) + | > loss_dur: 0.18214 (0.12374) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.33060 (21.66919) + | > current_lr: 0.00007 + | > step_time: 3.09470 (2.56379) + | > loader_time: 0.00390 (0.04152) + + + --> STEP: 162/234 -- GLOBAL_STEP: 61470 + | > loss: -0.33239 (-0.28605) + | > log_mle: -0.52163 (-0.41212) + | > loss_dur: 0.18924 (0.12606) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.67122 (22.69447) + | > current_lr: 0.00007 + | > step_time: 2.80500 (2.54927) + | > loader_time: 0.09450 (0.04090) + + + --> STEP: 167/234 -- GLOBAL_STEP: 61475 + | > loss: -0.43344 (-0.28797) + | > log_mle: -0.62315 (-0.41597) + | > loss_dur: 0.18971 (0.12801) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.99196 (23.55223) + | > current_lr: 0.00007 + | > step_time: 1.70040 (2.52804) + | > loader_time: 0.00380 (0.04026) + + + --> STEP: 172/234 -- GLOBAL_STEP: 61480 + | > loss: -0.39654 (-0.29048) + | > log_mle: -0.61100 (-0.42090) + | > loss_dur: 0.21446 (0.13043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.34138 (24.53585) + | > current_lr: 0.00007 + | > step_time: 2.19570 (2.53280) + | > loader_time: 0.00390 (0.03923) + + + --> STEP: 177/234 -- GLOBAL_STEP: 61485 + | > loss: -0.35865 (-0.29294) + | > log_mle: -0.56994 (-0.42564) + | > loss_dur: 0.21130 (0.13270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.33713 (25.50198) + | > current_lr: 0.00007 + | > step_time: 4.09230 (2.53918) + | > loader_time: 0.29230 (0.04136) + + + --> STEP: 182/234 -- GLOBAL_STEP: 61490 + | > loss: -0.37962 (-0.29519) + | > log_mle: -0.61872 (-0.43036) + | > loss_dur: 0.23910 (0.13517) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.88751 (26.65596) + | > current_lr: 0.00007 + | > step_time: 5.50030 (2.61491) + | > loader_time: 0.10370 (0.04199) + + + --> STEP: 187/234 -- GLOBAL_STEP: 61495 + | > loss: -0.39411 (-0.29750) + | > log_mle: -0.61141 (-0.43483) + | > loss_dur: 0.21730 (0.13733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.48643 (27.88420) + | > current_lr: 0.00007 + | > step_time: 9.40200 (2.66601) + | > loader_time: 0.00780 (0.04241) + + + --> STEP: 192/234 -- GLOBAL_STEP: 61500 + | > loss: -0.42493 (-0.29989) + | > log_mle: -0.63210 (-0.43910) + | > loss_dur: 0.20717 (0.13921) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.67583 (29.04393) + | > current_lr: 0.00007 + | > step_time: 4.33390 (2.71651) + | > loader_time: 0.07820 (0.04233) + + + --> STEP: 197/234 -- GLOBAL_STEP: 61505 + | > loss: -0.40906 (-0.30232) + | > log_mle: -0.59978 (-0.44336) + | > loss_dur: 0.19072 (0.14104) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.06433 (29.78374) + | > current_lr: 0.00007 + | > step_time: 5.10640 (2.74275) + | > loader_time: 0.18850 (0.04410) + + + --> STEP: 202/234 -- GLOBAL_STEP: 61510 + | > loss: -0.47636 (-0.30460) + | > log_mle: -0.69779 (-0.44773) + | > loss_dur: 0.22143 (0.14313) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.19815 (30.65756) + | > current_lr: 0.00007 + | > step_time: 1.90450 (2.77241) + | > loader_time: 0.00450 (0.04543) + + + --> STEP: 207/234 -- GLOBAL_STEP: 61515 + | > loss: -0.37716 (-0.30557) + | > log_mle: -0.61181 (-0.45078) + | > loss_dur: 0.23465 (0.14520) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.16988 (31.93920) + | > current_lr: 0.00007 + | > step_time: 1.99520 (2.77537) + | > loader_time: 0.00590 (0.04540) + + + --> STEP: 212/234 -- GLOBAL_STEP: 61520 + | > loss: -0.38447 (-0.30700) + | > log_mle: -0.61899 (-0.45430) + | > loss_dur: 0.23451 (0.14730) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.06570 (32.57983) + | > current_lr: 0.00007 + | > step_time: 7.99380 (2.81711) + | > loader_time: 0.20370 (0.04619) + + + --> STEP: 217/234 -- GLOBAL_STEP: 61525 + | > loss: -0.42808 (-0.30919) + | > log_mle: -0.66858 (-0.45849) + | > loss_dur: 0.24050 (0.14930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.84689 (33.26630) + | > current_lr: 0.00007 + | > step_time: 3.50460 (2.88033) + | > loader_time: 0.10550 (0.04701) + + + --> STEP: 222/234 -- GLOBAL_STEP: 61530 + | > loss: -0.42234 (-0.31180) + | > log_mle: -0.68510 (-0.46308) + | > loss_dur: 0.26276 (0.15128) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.41388 (33.87705) + | > current_lr: 0.00007 + | > step_time: 2.40810 (2.88663) + | > loader_time: 0.00500 (0.04682) + + + --> STEP: 227/234 -- GLOBAL_STEP: 61535 + | > loss: -0.41706 (-0.31504) + | > log_mle: -0.67304 (-0.46834) + | > loss_dur: 0.25599 (0.15331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.46391 (34.77545) + | > current_lr: 0.00007 + | > step_time: 0.24230 (2.85584) + | > loader_time: 0.00390 (0.04619) + + + --> STEP: 232/234 -- GLOBAL_STEP: 61540 + | > loss: -0.40448 (-0.31757) + | > log_mle: -0.88285 (-0.47503) + | > loss_dur: 0.47837 (0.15746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 153.55215 (36.48296) + | > current_lr: 0.00007 + | > step_time: 0.38970 (2.80045) + | > loader_time: 0.05730 (0.04552) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.51120 (+0.38098) + | > avg_loss: -0.33374 (-0.01931) + | > avg_log_mle: -0.55156 (+0.00371) + | > avg_loss_dur: 0.21782 (-0.02302) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_61542.pth + + > EPOCH: 263/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 16:36:04)  + + --> STEP: 3/234 -- GLOBAL_STEP: 61545 + | > loss: -0.23351 (-0.27876) + | > log_mle: -0.37868 (-0.38808) + | > loss_dur: 0.14517 (0.10932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.75632 (24.22715) + | > current_lr: 0.00007 + | > step_time: 1.70040 (1.93047) + | > loader_time: 0.00160 (0.00179) + + + --> STEP: 8/234 -- GLOBAL_STEP: 61550 + | > loss: -0.33183 (-0.30284) + | > log_mle: -0.40952 (-0.39106) + | > loss_dur: 0.07769 (0.08822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.91723 (21.10092) + | > current_lr: 0.00007 + | > step_time: 11.21170 (3.92365) + | > loader_time: 1.58970 (0.20118) + + + --> STEP: 13/234 -- GLOBAL_STEP: 61555 + | > loss: -0.34282 (-0.31045) + | > log_mle: -0.41287 (-0.39496) + | > loss_dur: 0.07005 (0.08451) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.36807 (19.02541) + | > current_lr: 0.00007 + | > step_time: 2.51040 (4.46932) + | > loader_time: 0.08840 (0.19888) + + + --> STEP: 18/234 -- GLOBAL_STEP: 61560 + | > loss: -0.31490 (-0.31565) + | > log_mle: -0.38767 (-0.39549) + | > loss_dur: 0.07277 (0.07984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.39364 (17.46984) + | > current_lr: 0.00007 + | > step_time: 5.19760 (4.97724) + | > loader_time: 0.00180 (0.16635) + + + --> STEP: 23/234 -- GLOBAL_STEP: 61565 + | > loss: -0.34495 (-0.31794) + | > log_mle: -0.40979 (-0.39533) + | > loss_dur: 0.06484 (0.07738) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.89439 (16.21500) + | > current_lr: 0.00007 + | > step_time: 0.69380 (4.52509) + | > loader_time: 0.00370 (0.13544) + + + --> STEP: 28/234 -- GLOBAL_STEP: 61570 + | > loss: -0.36411 (-0.31876) + | > log_mle: -0.41397 (-0.39414) + | > loss_dur: 0.04986 (0.07538) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.24542 (15.41460) + | > current_lr: 0.00007 + | > step_time: 5.29420 (4.27462) + | > loader_time: 0.00330 (0.11517) + + + --> STEP: 33/234 -- GLOBAL_STEP: 61575 + | > loss: -0.31442 (-0.31587) + | > log_mle: -0.38159 (-0.39192) + | > loss_dur: 0.06716 (0.07605) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.63902 (14.99588) + | > current_lr: 0.00007 + | > step_time: 3.47340 (4.32670) + | > loader_time: 0.07870 (0.10830) + + + --> STEP: 38/234 -- GLOBAL_STEP: 61580 + | > loss: -0.29496 (-0.31203) + | > log_mle: -0.38054 (-0.38926) + | > loss_dur: 0.08557 (0.07724) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.56842 (14.75938) + | > current_lr: 0.00007 + | > step_time: 0.97930 (3.94388) + | > loader_time: 0.00210 (0.09429) + + + --> STEP: 43/234 -- GLOBAL_STEP: 61585 + | > loss: -0.26037 (-0.30850) + | > log_mle: -0.36950 (-0.38695) + | > loss_dur: 0.10913 (0.07845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.59580 (14.29479) + | > current_lr: 0.00007 + | > step_time: 1.28740 (3.91955) + | > loader_time: 0.00250 (0.08579) + + + --> STEP: 48/234 -- GLOBAL_STEP: 61590 + | > loss: -0.31829 (-0.30688) + | > log_mle: -0.38136 (-0.38584) + | > loss_dur: 0.06307 (0.07896) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.32714 (13.80103) + | > current_lr: 0.00007 + | > step_time: 1.80360 (3.72408) + | > loader_time: 0.00320 (0.08262) + + + --> STEP: 53/234 -- GLOBAL_STEP: 61595 + | > loss: -0.28250 (-0.30468) + | > log_mle: -0.37465 (-0.38442) + | > loss_dur: 0.09215 (0.07974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.34004 (13.33685) + | > current_lr: 0.00007 + | > step_time: 2.21410 (3.64406) + | > loader_time: 0.07680 (0.07972) + + + --> STEP: 58/234 -- GLOBAL_STEP: 61600 + | > loss: -0.29872 (-0.30296) + | > log_mle: -0.37571 (-0.38332) + | > loss_dur: 0.07698 (0.08036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.36256 (12.92876) + | > current_lr: 0.00007 + | > step_time: 2.02660 (3.49603) + | > loader_time: 0.00290 (0.07311) + + + --> STEP: 63/234 -- GLOBAL_STEP: 61605 + | > loss: -0.27311 (-0.29984) + | > log_mle: -0.36308 (-0.38262) + | > loss_dur: 0.08997 (0.08279) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.41436 (12.84706) + | > current_lr: 0.00007 + | > step_time: 1.11080 (3.34508) + | > loader_time: 0.00320 (0.06756) + + + --> STEP: 68/234 -- GLOBAL_STEP: 61610 + | > loss: -0.24498 (-0.29756) + | > log_mle: -0.35561 (-0.38126) + | > loss_dur: 0.11063 (0.08369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.74557 (12.69886) + | > current_lr: 0.00007 + | > step_time: 1.97770 (3.23797) + | > loader_time: 0.00210 (0.06274) + + + --> STEP: 73/234 -- GLOBAL_STEP: 61615 + | > loss: -0.24021 (-0.29462) + | > log_mle: -0.36700 (-0.38009) + | > loss_dur: 0.12680 (0.08546) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.08273 (12.82370) + | > current_lr: 0.00007 + | > step_time: 2.98510 (3.18268) + | > loader_time: 0.07830 (0.06088) + + + --> STEP: 78/234 -- GLOBAL_STEP: 61620 + | > loss: -0.24469 (-0.29172) + | > log_mle: -0.35224 (-0.37883) + | > loss_dur: 0.10755 (0.08711) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.67810 (12.86913) + | > current_lr: 0.00007 + | > step_time: 1.70660 (3.09276) + | > loader_time: 0.00180 (0.05820) + + + --> STEP: 83/234 -- GLOBAL_STEP: 61625 + | > loss: -0.22818 (-0.28919) + | > log_mle: -0.36473 (-0.37793) + | > loss_dur: 0.13655 (0.08874) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.22879 (12.83111) + | > current_lr: 0.00007 + | > step_time: 2.36510 (3.03720) + | > loader_time: 0.00720 (0.05490) + + + --> STEP: 88/234 -- GLOBAL_STEP: 61630 + | > loss: -0.24971 (-0.28725) + | > log_mle: -0.39738 (-0.37749) + | > loss_dur: 0.14767 (0.09024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.99623 (12.99080) + | > current_lr: 0.00007 + | > step_time: 1.00660 (2.94205) + | > loader_time: 0.00210 (0.05196) + + + --> STEP: 93/234 -- GLOBAL_STEP: 61635 + | > loss: -0.25568 (-0.28550) + | > log_mle: -0.41105 (-0.37820) + | > loss_dur: 0.15537 (0.09270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.47036 (13.37736) + | > current_lr: 0.00007 + | > step_time: 1.79090 (2.88606) + | > loader_time: 0.00210 (0.04937) + + + --> STEP: 98/234 -- GLOBAL_STEP: 61640 + | > loss: -0.24254 (-0.28434) + | > log_mle: -0.35016 (-0.37902) + | > loss_dur: 0.10761 (0.09468) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.01564 (13.60798) + | > current_lr: 0.00007 + | > step_time: 1.97650 (2.86100) + | > loader_time: 0.00270 (0.05066) + + + --> STEP: 103/234 -- GLOBAL_STEP: 61645 + | > loss: -0.28323 (-0.28318) + | > log_mle: -0.44137 (-0.38054) + | > loss_dur: 0.15814 (0.09736) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.08335 (14.14820) + | > current_lr: 0.00007 + | > step_time: 1.60400 (2.80836) + | > loader_time: 0.00310 (0.04837) + + + --> STEP: 108/234 -- GLOBAL_STEP: 61650 + | > loss: -0.24581 (-0.28210) + | > log_mle: -0.37779 (-0.38164) + | > loss_dur: 0.13198 (0.09954) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.59730 (14.77082) + | > current_lr: 0.00007 + | > step_time: 3.19570 (2.77459) + | > loader_time: 0.00320 (0.04789) + + + --> STEP: 113/234 -- GLOBAL_STEP: 61655 + | > loss: -0.26018 (-0.28066) + | > log_mle: -0.42011 (-0.38324) + | > loss_dur: 0.15993 (0.10258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.78427 (15.25157) + | > current_lr: 0.00007 + | > step_time: 2.59310 (2.76497) + | > loader_time: 0.10510 (0.04837) + + + --> STEP: 118/234 -- GLOBAL_STEP: 61660 + | > loss: -0.23500 (-0.27919) + | > log_mle: -0.39778 (-0.38433) + | > loss_dur: 0.16278 (0.10514) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.90568 (15.52215) + | > current_lr: 0.00007 + | > step_time: 0.91260 (2.72854) + | > loader_time: 0.08860 (0.04792) + + + --> STEP: 123/234 -- GLOBAL_STEP: 61665 + | > loss: -0.21817 (-0.27777) + | > log_mle: -0.36581 (-0.38460) + | > loss_dur: 0.14764 (0.10683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.13342 (15.71613) + | > current_lr: 0.00007 + | > step_time: 1.08990 (2.68532) + | > loader_time: 0.01090 (0.04618) + + + --> STEP: 128/234 -- GLOBAL_STEP: 61670 + | > loss: -0.27340 (-0.27775) + | > log_mle: -0.42676 (-0.38699) + | > loss_dur: 0.15336 (0.10924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.09098 (16.22741) + | > current_lr: 0.00007 + | > step_time: 1.79270 (2.68120) + | > loader_time: 0.00340 (0.04513) + + + --> STEP: 133/234 -- GLOBAL_STEP: 61675 + | > loss: -0.28937 (-0.27791) + | > log_mle: -0.45299 (-0.38958) + | > loss_dur: 0.16361 (0.11168) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.63971 (16.92881) + | > current_lr: 0.00007 + | > step_time: 3.19160 (2.68421) + | > loader_time: 0.00510 (0.04488) + + + --> STEP: 138/234 -- GLOBAL_STEP: 61680 + | > loss: -0.24080 (-0.27779) + | > log_mle: -0.39904 (-0.39185) + | > loss_dur: 0.15824 (0.11406) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.02885 (17.59031) + | > current_lr: 0.00007 + | > step_time: 1.18160 (2.64637) + | > loader_time: 0.00320 (0.04396) + + + --> STEP: 143/234 -- GLOBAL_STEP: 61685 + | > loss: -0.31547 (-0.27812) + | > log_mle: -0.54358 (-0.39500) + | > loss_dur: 0.22811 (0.11688) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.57276 (18.32132) + | > current_lr: 0.00007 + | > step_time: 2.91440 (2.63507) + | > loader_time: 0.08420 (0.04432) + + + --> STEP: 148/234 -- GLOBAL_STEP: 61690 + | > loss: -0.30633 (-0.27914) + | > log_mle: -0.46028 (-0.39825) + | > loss_dur: 0.15395 (0.11911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.84855 (19.00097) + | > current_lr: 0.00007 + | > step_time: 4.29480 (2.70883) + | > loader_time: 0.10130 (0.04564) + + + --> STEP: 153/234 -- GLOBAL_STEP: 61695 + | > loss: -0.39868 (-0.28136) + | > log_mle: -0.59347 (-0.40283) + | > loss_dur: 0.19479 (0.12148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.08939 (19.87652) + | > current_lr: 0.00007 + | > step_time: 1.10080 (2.71143) + | > loader_time: 0.08780 (0.04640) + + + --> STEP: 158/234 -- GLOBAL_STEP: 61700 + | > loss: -0.31398 (-0.28290) + | > log_mle: -0.52453 (-0.40673) + | > loss_dur: 0.21055 (0.12383) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.10497 (20.69337) + | > current_lr: 0.00007 + | > step_time: 3.10210 (2.75398) + | > loader_time: 0.00740 (0.04705) + + + --> STEP: 163/234 -- GLOBAL_STEP: 61705 + | > loss: -0.30869 (-0.28480) + | > log_mle: -0.49965 (-0.41079) + | > loss_dur: 0.19096 (0.12600) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.30131 (21.41473) + | > current_lr: 0.00007 + | > step_time: 2.17920 (2.72399) + | > loader_time: 0.01830 (0.04631) + + + --> STEP: 168/234 -- GLOBAL_STEP: 61710 + | > loss: -0.34933 (-0.28711) + | > log_mle: -0.55805 (-0.41516) + | > loss_dur: 0.20872 (0.12804) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.28196 (22.44140) + | > current_lr: 0.00007 + | > step_time: 2.58550 (2.74881) + | > loader_time: 0.00530 (0.04728) + + + --> STEP: 173/234 -- GLOBAL_STEP: 61715 + | > loss: -0.33499 (-0.28953) + | > log_mle: -0.55646 (-0.42003) + | > loss_dur: 0.22147 (0.13050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.03131 (23.85238) + | > current_lr: 0.00007 + | > step_time: 2.29490 (2.76371) + | > loader_time: 0.00210 (0.04712) + + + --> STEP: 178/234 -- GLOBAL_STEP: 61720 + | > loss: -0.39776 (-0.29203) + | > log_mle: -0.62462 (-0.42497) + | > loss_dur: 0.22686 (0.13295) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.13381 (24.79202) + | > current_lr: 0.00007 + | > step_time: 4.89700 (2.76823) + | > loader_time: 0.00880 (0.04689) + + + --> STEP: 183/234 -- GLOBAL_STEP: 61725 + | > loss: -0.41929 (-0.29408) + | > log_mle: -0.63022 (-0.42956) + | > loss_dur: 0.21093 (0.13548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.48746 (25.64715) + | > current_lr: 0.00007 + | > step_time: 1.50430 (2.81067) + | > loader_time: 0.00360 (0.04726) + + + --> STEP: 188/234 -- GLOBAL_STEP: 61730 + | > loss: -0.42724 (-0.29648) + | > log_mle: -0.64100 (-0.43420) + | > loss_dur: 0.21375 (0.13772) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.58730 (26.61874) + | > current_lr: 0.00007 + | > step_time: 12.20630 (2.90037) + | > loader_time: 1.50590 (0.05466) + + + --> STEP: 193/234 -- GLOBAL_STEP: 61735 + | > loss: -0.42398 (-0.29909) + | > log_mle: -0.63470 (-0.43863) + | > loss_dur: 0.21073 (0.13954) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.67456 (27.71153) + | > current_lr: 0.00007 + | > step_time: 2.20090 (2.92936) + | > loader_time: 0.00350 (0.05387) + + + --> STEP: 198/234 -- GLOBAL_STEP: 61740 + | > loss: -0.40231 (-0.30146) + | > log_mle: -0.62280 (-0.44284) + | > loss_dur: 0.22049 (0.14137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.72567 (28.73314) + | > current_lr: 0.00007 + | > step_time: 3.31340 (3.05240) + | > loader_time: 0.69010 (0.05800) + + + --> STEP: 203/234 -- GLOBAL_STEP: 61745 + | > loss: -0.33453 (-0.30362) + | > log_mle: -0.55478 (-0.44695) + | > loss_dur: 0.22025 (0.14333) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.46666 (29.43637) + | > current_lr: 0.00007 + | > step_time: 2.00260 (3.06437) + | > loader_time: 0.00400 (0.05815) + + + --> STEP: 208/234 -- GLOBAL_STEP: 61750 + | > loss: -0.39554 (-0.30636) + | > log_mle: -0.63349 (-0.45183) + | > loss_dur: 0.23795 (0.14547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.14597 (30.59452) + | > current_lr: 0.00007 + | > step_time: 17.50670 (3.18839) + | > loader_time: 0.19520 (0.06301) + + + --> STEP: 213/234 -- GLOBAL_STEP: 61755 + | > loss: -0.45757 (-0.30956) + | > log_mle: -0.68738 (-0.45707) + | > loss_dur: 0.22981 (0.14751) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.50457 (31.65655) + | > current_lr: 0.00007 + | > step_time: 14.10500 (3.27539) + | > loader_time: 0.08330 (0.06389) + + + --> STEP: 218/234 -- GLOBAL_STEP: 61760 + | > loss: -0.41361 (-0.31247) + | > log_mle: -0.65249 (-0.46194) + | > loss_dur: 0.23888 (0.14946) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.80999 (32.57554) + | > current_lr: 0.00007 + | > step_time: 3.69550 (3.35891) + | > loader_time: 0.00370 (0.06302) + + + --> STEP: 223/234 -- GLOBAL_STEP: 61765 + | > loss: -0.46160 (-0.31565) + | > log_mle: -0.69880 (-0.46708) + | > loss_dur: 0.23721 (0.15143) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.99533 (33.58250) + | > current_lr: 0.00007 + | > step_time: 0.23230 (3.31456) + | > loader_time: 0.00460 (0.06252) + + + --> STEP: 228/234 -- GLOBAL_STEP: 61770 + | > loss: -0.43221 (-0.31873) + | > log_mle: -0.69492 (-0.47231) + | > loss_dur: 0.26272 (0.15357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.19145 (34.61681) + | > current_lr: 0.00007 + | > step_time: 0.24300 (3.24706) + | > loader_time: 0.00450 (0.06124) + + + --> STEP: 233/234 -- GLOBAL_STEP: 61775 + | > loss: -0.02437 (-0.31978) + | > log_mle: -0.67728 (-0.47911) + | > loss_dur: 0.65290 (0.15933) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.47285 (35.95890) + | > current_lr: 0.00007 + | > step_time: 0.20980 (3.18306) + | > loader_time: 0.00330 (0.06006) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.72928 (+0.21808) + | > avg_loss: -0.29858 (+0.03516) + | > avg_log_mle: -0.55082 (+0.00073) + | > avg_loss_dur: 0.25224 (+0.03443) + + + > EPOCH: 264/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 16:49:57)  + + --> STEP: 4/234 -- GLOBAL_STEP: 61780 + | > loss: -0.29536 (-0.29278) + | > log_mle: -0.38956 (-0.39113) + | > loss_dur: 0.09420 (0.09836) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.46730 (22.70262) + | > current_lr: 0.00007 + | > step_time: 10.39530 (7.07611) + | > loader_time: 0.00170 (0.05063) + + + --> STEP: 9/234 -- GLOBAL_STEP: 61785 + | > loss: -0.30309 (-0.30363) + | > log_mle: -0.40054 (-0.39377) + | > loss_dur: 0.09745 (0.09015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.75097 (21.28019) + | > current_lr: 0.00007 + | > step_time: 2.20280 (5.18138) + | > loader_time: 0.00440 (0.06590) + + + --> STEP: 14/234 -- GLOBAL_STEP: 61790 + | > loss: -0.32850 (-0.31114) + | > log_mle: -0.40123 (-0.39637) + | > loss_dur: 0.07272 (0.08523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.16923 (19.17453) + | > current_lr: 0.00007 + | > step_time: 2.60670 (4.22420) + | > loader_time: 0.08180 (0.05409) + + + --> STEP: 19/234 -- GLOBAL_STEP: 61795 + | > loss: -0.33287 (-0.31569) + | > log_mle: -0.39810 (-0.39696) + | > loss_dur: 0.06522 (0.08128) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.53727 (17.56759) + | > current_lr: 0.00007 + | > step_time: 2.18680 (3.77430) + | > loader_time: 0.00150 (0.04496) + + + --> STEP: 24/234 -- GLOBAL_STEP: 61800 + | > loss: -0.33464 (-0.31825) + | > log_mle: -0.39283 (-0.39658) + | > loss_dur: 0.05819 (0.07833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.52951 (16.02571) + | > current_lr: 0.00007 + | > step_time: 2.69430 (3.47340) + | > loader_time: 0.00220 (0.03593) + + + --> STEP: 29/234 -- GLOBAL_STEP: 61805 + | > loss: -0.30948 (-0.31910) + | > log_mle: -0.38591 (-0.39557) + | > loss_dur: 0.07643 (0.07647) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.82359 (14.98137) + | > current_lr: 0.00007 + | > step_time: 2.91260 (3.60649) + | > loader_time: 0.00230 (0.04191) + + + --> STEP: 34/234 -- GLOBAL_STEP: 61810 + | > loss: -0.30721 (-0.31633) + | > log_mle: -0.38533 (-0.39390) + | > loss_dur: 0.07812 (0.07756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.71326 (14.35258) + | > current_lr: 0.00007 + | > step_time: 7.40490 (3.85023) + | > loader_time: 0.20630 (0.04745) + + + --> STEP: 39/234 -- GLOBAL_STEP: 61815 + | > loss: -0.28756 (-0.31291) + | > log_mle: -0.37799 (-0.39161) + | > loss_dur: 0.09043 (0.07870) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.28079 (14.19083) + | > current_lr: 0.00007 + | > step_time: 1.92910 (3.53692) + | > loader_time: 0.00350 (0.04166) + + + --> STEP: 44/234 -- GLOBAL_STEP: 61820 + | > loss: -0.30242 (-0.31060) + | > log_mle: -0.36979 (-0.38955) + | > loss_dur: 0.06736 (0.07894) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.57962 (13.67694) + | > current_lr: 0.00007 + | > step_time: 2.30510 (3.30569) + | > loader_time: 0.07960 (0.04083) + + + --> STEP: 49/234 -- GLOBAL_STEP: 61825 + | > loss: -0.30349 (-0.30938) + | > log_mle: -0.38101 (-0.38888) + | > loss_dur: 0.07752 (0.07950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74046 (13.53044) + | > current_lr: 0.00007 + | > step_time: 1.17490 (3.17666) + | > loader_time: 0.00210 (0.03881) + + + --> STEP: 54/234 -- GLOBAL_STEP: 61830 + | > loss: -0.29201 (-0.30700) + | > log_mle: -0.37658 (-0.38732) + | > loss_dur: 0.08458 (0.08033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.86091 (13.22617) + | > current_lr: 0.00007 + | > step_time: 2.20640 (3.05615) + | > loader_time: 0.00220 (0.03868) + + + --> STEP: 59/234 -- GLOBAL_STEP: 61835 + | > loss: -0.27977 (-0.30502) + | > log_mle: -0.37612 (-0.38617) + | > loss_dur: 0.09635 (0.08115) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.77149 (12.96930) + | > current_lr: 0.00007 + | > step_time: 3.00840 (3.08723) + | > loader_time: 0.08500 (0.03883) + + + --> STEP: 64/234 -- GLOBAL_STEP: 61840 + | > loss: -0.26778 (-0.30113) + | > log_mle: -0.36521 (-0.38509) + | > loss_dur: 0.09743 (0.08396) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.48129 (13.06553) + | > current_lr: 0.00007 + | > step_time: 1.92130 (3.02927) + | > loader_time: 0.07480 (0.04001) + + + --> STEP: 69/234 -- GLOBAL_STEP: 61845 + | > loss: -0.26765 (-0.29851) + | > log_mle: -0.35656 (-0.38344) + | > loss_dur: 0.08891 (0.08493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.62865 (12.88281) + | > current_lr: 0.00007 + | > step_time: 2.20130 (2.96738) + | > loader_time: 0.00200 (0.03848) + + + --> STEP: 74/234 -- GLOBAL_STEP: 61850 + | > loss: -0.24603 (-0.29508) + | > log_mle: -0.35556 (-0.38224) + | > loss_dur: 0.10953 (0.08716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.81022 (12.87061) + | > current_lr: 0.00007 + | > step_time: 2.19080 (2.88882) + | > loader_time: 0.00500 (0.03607) + + + --> STEP: 79/234 -- GLOBAL_STEP: 61855 + | > loss: -0.26025 (-0.29239) + | > log_mle: -0.36516 (-0.38095) + | > loss_dur: 0.10491 (0.08856) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.28905 (13.28201) + | > current_lr: 0.00007 + | > step_time: 1.91160 (2.86068) + | > loader_time: 0.00240 (0.03529) + + + --> STEP: 84/234 -- GLOBAL_STEP: 61860 + | > loss: -0.26273 (-0.28987) + | > log_mle: -0.36360 (-0.37991) + | > loss_dur: 0.10087 (0.09004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.93926 (13.33725) + | > current_lr: 0.00007 + | > step_time: 2.61910 (2.82748) + | > loader_time: 0.09620 (0.03549) + + + --> STEP: 89/234 -- GLOBAL_STEP: 61865 + | > loss: -0.27029 (-0.28789) + | > log_mle: -0.38815 (-0.37960) + | > loss_dur: 0.11786 (0.09172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.64065 (13.50864) + | > current_lr: 0.00007 + | > step_time: 0.97860 (2.76487) + | > loader_time: 0.00280 (0.03549) + + + --> STEP: 94/234 -- GLOBAL_STEP: 61870 + | > loss: -0.27704 (-0.28630) + | > log_mle: -0.41238 (-0.38055) + | > loss_dur: 0.13534 (0.09424) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.09402 (13.91968) + | > current_lr: 0.00007 + | > step_time: 1.00540 (2.70402) + | > loader_time: 0.00360 (0.03549) + + + --> STEP: 99/234 -- GLOBAL_STEP: 61875 + | > loss: -0.27794 (-0.28535) + | > log_mle: -0.44216 (-0.38163) + | > loss_dur: 0.16422 (0.09627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.12065 (14.30952) + | > current_lr: 0.00007 + | > step_time: 1.99390 (2.65814) + | > loader_time: 0.00240 (0.03590) + + + --> STEP: 104/234 -- GLOBAL_STEP: 61880 + | > loss: -0.30362 (-0.28420) + | > log_mle: -0.44582 (-0.38279) + | > loss_dur: 0.14219 (0.09859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.88068 (15.41723) + | > current_lr: 0.00007 + | > step_time: 1.18400 (2.60548) + | > loader_time: 0.00150 (0.03508) + + + --> STEP: 109/234 -- GLOBAL_STEP: 61885 + | > loss: -0.23736 (-0.28226) + | > log_mle: -0.42300 (-0.38344) + | > loss_dur: 0.18564 (0.10118) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.01866 (15.71390) + | > current_lr: 0.00007 + | > step_time: 1.19980 (2.58515) + | > loader_time: 0.00300 (0.03512) + + + --> STEP: 114/234 -- GLOBAL_STEP: 61890 + | > loss: -0.25173 (-0.28131) + | > log_mle: -0.39977 (-0.38500) + | > loss_dur: 0.14804 (0.10369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.32365 (16.51631) + | > current_lr: 0.00007 + | > step_time: 2.50770 (2.57992) + | > loader_time: 0.09400 (0.03527) + + + --> STEP: 119/234 -- GLOBAL_STEP: 61895 + | > loss: -0.26156 (-0.27965) + | > log_mle: -0.40456 (-0.38596) + | > loss_dur: 0.14300 (0.10631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.02300 (16.68497) + | > current_lr: 0.00007 + | > step_time: 2.70580 (2.57749) + | > loader_time: 0.00370 (0.03553) + + + --> STEP: 124/234 -- GLOBAL_STEP: 61900 + | > loss: -0.27950 (-0.27844) + | > log_mle: -0.43300 (-0.38640) + | > loss_dur: 0.15350 (0.10796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.33239 (16.85891) + | > current_lr: 0.00007 + | > step_time: 4.30020 (2.55919) + | > loader_time: 0.00430 (0.03491) + + + --> STEP: 129/234 -- GLOBAL_STEP: 61905 + | > loss: -0.25276 (-0.27814) + | > log_mle: -0.42748 (-0.38869) + | > loss_dur: 0.17473 (0.11054) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.27537 (17.23248) + | > current_lr: 0.00007 + | > step_time: 1.50300 (2.53691) + | > loader_time: 0.08220 (0.03499) + + + --> STEP: 134/234 -- GLOBAL_STEP: 61910 + | > loss: -0.29819 (-0.27885) + | > log_mle: -0.48009 (-0.39178) + | > loss_dur: 0.18190 (0.11293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.00977 (17.93254) + | > current_lr: 0.00007 + | > step_time: 5.39750 (2.57150) + | > loader_time: 0.10150 (0.03870) + + + --> STEP: 139/234 -- GLOBAL_STEP: 61915 + | > loss: -0.35309 (-0.27928) + | > log_mle: -0.54501 (-0.39471) + | > loss_dur: 0.19192 (0.11543) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.73951 (18.77866) + | > current_lr: 0.00007 + | > step_time: 1.80600 (2.54544) + | > loader_time: 0.00410 (0.03804) + + + --> STEP: 144/234 -- GLOBAL_STEP: 61920 + | > loss: -0.31840 (-0.27969) + | > log_mle: -0.51352 (-0.39783) + | > loss_dur: 0.19511 (0.11814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.85023 (19.66678) + | > current_lr: 0.00007 + | > step_time: 2.30910 (2.53120) + | > loader_time: 0.00370 (0.03738) + + + --> STEP: 149/234 -- GLOBAL_STEP: 61925 + | > loss: -0.38440 (-0.28115) + | > log_mle: -0.57252 (-0.40142) + | > loss_dur: 0.18811 (0.12027) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.39406 (20.65408) + | > current_lr: 0.00007 + | > step_time: 1.79230 (2.51212) + | > loader_time: 0.00220 (0.03734) + + + --> STEP: 154/234 -- GLOBAL_STEP: 61930 + | > loss: -0.34164 (-0.28319) + | > log_mle: -0.52588 (-0.40576) + | > loss_dur: 0.18425 (0.12257) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.36652 (21.58262) + | > current_lr: 0.00007 + | > step_time: 2.50470 (2.52460) + | > loader_time: 0.08760 (0.03814) + + + --> STEP: 159/234 -- GLOBAL_STEP: 61935 + | > loss: -0.35199 (-0.28505) + | > log_mle: -0.54484 (-0.40992) + | > loss_dur: 0.19285 (0.12487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.39029 (22.35343) + | > current_lr: 0.00007 + | > step_time: 3.40970 (2.54657) + | > loader_time: 0.00480 (0.03838) + + + --> STEP: 164/234 -- GLOBAL_STEP: 61940 + | > loss: -0.34112 (-0.28701) + | > log_mle: -0.53587 (-0.41388) + | > loss_dur: 0.19476 (0.12688) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.40013 (23.07547) + | > current_lr: 0.00007 + | > step_time: 10.30780 (2.63612) + | > loader_time: 0.08820 (0.04073) + + + --> STEP: 169/234 -- GLOBAL_STEP: 61945 + | > loss: -0.33312 (-0.28907) + | > log_mle: -0.53501 (-0.41806) + | > loss_dur: 0.20190 (0.12899) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.20425 (24.06689) + | > current_lr: 0.00007 + | > step_time: 3.39110 (2.69096) + | > loader_time: 0.00260 (0.04246) + + + --> STEP: 174/234 -- GLOBAL_STEP: 61950 + | > loss: -0.41284 (-0.29195) + | > log_mle: -0.62395 (-0.42337) + | > loss_dur: 0.21111 (0.13143) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.59163 (25.45991) + | > current_lr: 0.00007 + | > step_time: 2.09600 (2.75259) + | > loader_time: 0.10270 (0.04399) + + + --> STEP: 179/234 -- GLOBAL_STEP: 61955 + | > loss: -0.39360 (-0.29427) + | > log_mle: -0.62857 (-0.42819) + | > loss_dur: 0.23497 (0.13392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.86914 (26.59481) + | > current_lr: 0.00007 + | > step_time: 9.40770 (2.79239) + | > loader_time: 0.08950 (0.04501) + + + --> STEP: 184/234 -- GLOBAL_STEP: 61960 + | > loss: -0.36332 (-0.29611) + | > log_mle: -0.58077 (-0.43237) + | > loss_dur: 0.21744 (0.13625) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.43352 (27.54377) + | > current_lr: 0.00007 + | > step_time: 1.50300 (2.89326) + | > loader_time: 0.00410 (0.05307) + + + --> STEP: 189/234 -- GLOBAL_STEP: 61965 + | > loss: -0.37318 (-0.29845) + | > log_mle: -0.58757 (-0.43686) + | > loss_dur: 0.21439 (0.13841) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.28169 (28.36981) + | > current_lr: 0.00007 + | > step_time: 6.89890 (2.95737) + | > loader_time: 0.09410 (0.05488) + + + --> STEP: 194/234 -- GLOBAL_STEP: 61970 + | > loss: -0.41686 (-0.30131) + | > log_mle: -0.62127 (-0.44147) + | > loss_dur: 0.20441 (0.14016) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.16329 (29.33012) + | > current_lr: 0.00007 + | > step_time: 5.89600 (3.01109) + | > loader_time: 0.09840 (0.05453) + + + --> STEP: 199/234 -- GLOBAL_STEP: 61975 + | > loss: -0.40097 (-0.30352) + | > log_mle: -0.62040 (-0.44558) + | > loss_dur: 0.21943 (0.14205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.36800 (30.59318) + | > current_lr: 0.00007 + | > step_time: 3.10040 (3.02745) + | > loader_time: 0.00490 (0.05511) + + + --> STEP: 204/234 -- GLOBAL_STEP: 61980 + | > loss: -0.44044 (-0.30555) + | > log_mle: -0.67606 (-0.44966) + | > loss_dur: 0.23562 (0.14411) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.63970 (31.37388) + | > current_lr: 0.00007 + | > step_time: 9.59220 (3.10529) + | > loader_time: 0.00750 (0.05570) + + + --> STEP: 209/234 -- GLOBAL_STEP: 61985 + | > loss: -0.40242 (-0.30808) + | > log_mle: -0.61923 (-0.45409) + | > loss_dur: 0.21681 (0.14601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.67681 (32.32727) + | > current_lr: 0.00007 + | > step_time: 3.50110 (3.15406) + | > loader_time: 0.09030 (0.05674) + + + --> STEP: 214/234 -- GLOBAL_STEP: 61990 + | > loss: -0.44819 (-0.31155) + | > log_mle: -0.65753 (-0.45952) + | > loss_dur: 0.20935 (0.14797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.60287 (33.30066) + | > current_lr: 0.00007 + | > step_time: 6.70320 (3.22295) + | > loader_time: 0.19270 (0.05764) + + + --> STEP: 219/234 -- GLOBAL_STEP: 61995 + | > loss: -0.53659 (-0.31484) + | > log_mle: -0.76239 (-0.46480) + | > loss_dur: 0.22580 (0.14996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.23375 (34.63923) + | > current_lr: 0.00007 + | > step_time: 3.88830 (3.25832) + | > loader_time: 0.00380 (0.05726) + + + --> STEP: 224/234 -- GLOBAL_STEP: 62000 + | > loss: -0.47465 (-0.31796) + | > log_mle: -0.71365 (-0.46989) + | > loss_dur: 0.23900 (0.15193) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.36401 (35.82042) + | > current_lr: 0.00007 + | > step_time: 0.22840 (3.19783) + | > loader_time: 0.00360 (0.05643) + + + --> STEP: 229/234 -- GLOBAL_STEP: 62005 + | > loss: -0.46316 (-0.32124) + | > log_mle: -0.76634 (-0.47561) + | > loss_dur: 0.30318 (0.15437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.92976 (37.00727) + | > current_lr: 0.00007 + | > step_time: 0.25870 (3.13332) + | > loader_time: 0.00640 (0.05529) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02337 (-0.70591) + | > avg_loss: -0.31773 (-0.01915) + | > avg_log_mle: -0.55023 (+0.00059) + | > avg_loss_dur: 0.23250 (-0.01974) + + + > EPOCH: 265/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 17:03:17)  + + --> STEP: 0/234 -- GLOBAL_STEP: 62010 + | > loss: -0.31672 (-0.31672) + | > log_mle: -0.47614 (-0.47614) + | > loss_dur: 0.15942 (0.15942) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.36398 (29.36398) + | > current_lr: 0.00007 + | > step_time: 3.48890 (3.48890) + | > loader_time: 4.70510 (4.70508) + + + --> STEP: 5/234 -- GLOBAL_STEP: 62015 + | > loss: -0.30763 (-0.29649) + | > log_mle: -0.39161 (-0.39326) + | > loss_dur: 0.08399 (0.09677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.26930 (20.74657) + | > current_lr: 0.00007 + | > step_time: 2.59830 (7.41922) + | > loader_time: 0.09010 (0.40094) + + + --> STEP: 10/234 -- GLOBAL_STEP: 62020 + | > loss: -0.31249 (-0.30527) + | > log_mle: -0.39491 (-0.39526) + | > loss_dur: 0.08242 (0.08999) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.56723 (18.92137) + | > current_lr: 0.00007 + | > step_time: 3.00110 (5.48079) + | > loader_time: 0.08760 (0.22632) + + + --> STEP: 15/234 -- GLOBAL_STEP: 62025 + | > loss: -0.33004 (-0.31425) + | > log_mle: -0.40436 (-0.39838) + | > loss_dur: 0.07432 (0.08413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.57747 (17.75476) + | > current_lr: 0.00007 + | > step_time: 3.41160 (4.94168) + | > loader_time: 0.00280 (0.18289) + + + --> STEP: 20/234 -- GLOBAL_STEP: 62030 + | > loss: -0.33686 (-0.31898) + | > log_mle: -0.39918 (-0.39845) + | > loss_dur: 0.06233 (0.07947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.28476 (15.98390) + | > current_lr: 0.00007 + | > step_time: 0.98360 (4.00811) + | > loader_time: 0.00130 (0.14150) + + + --> STEP: 25/234 -- GLOBAL_STEP: 62035 + | > loss: -0.31978 (-0.32087) + | > log_mle: -0.38374 (-0.39784) + | > loss_dur: 0.06396 (0.07697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.80465 (15.05804) + | > current_lr: 0.00007 + | > step_time: 1.36890 (3.54938) + | > loader_time: 0.00150 (0.11357) + + + --> STEP: 30/234 -- GLOBAL_STEP: 62040 + | > loss: -0.29226 (-0.32056) + | > log_mle: -0.38118 (-0.39695) + | > loss_dur: 0.08892 (0.07640) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.31746 (14.43375) + | > current_lr: 0.00007 + | > step_time: 1.97800 (3.23471) + | > loader_time: 0.00110 (0.10054) + + + --> STEP: 35/234 -- GLOBAL_STEP: 62045 + | > loss: -0.27303 (-0.31760) + | > log_mle: -0.37548 (-0.39482) + | > loss_dur: 0.10245 (0.07721) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.90604 (14.11412) + | > current_lr: 0.00007 + | > step_time: 1.19320 (3.00643) + | > loader_time: 0.00190 (0.08663) + + + --> STEP: 40/234 -- GLOBAL_STEP: 62050 + | > loss: -0.28965 (-0.31408) + | > log_mle: -0.37451 (-0.39241) + | > loss_dur: 0.08486 (0.07833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.68832 (13.70236) + | > current_lr: 0.00007 + | > step_time: 2.69390 (2.84084) + | > loader_time: 0.00270 (0.07825) + + + --> STEP: 45/234 -- GLOBAL_STEP: 62055 + | > loss: -0.28576 (-0.31176) + | > log_mle: -0.38880 (-0.39071) + | > loss_dur: 0.10304 (0.07896) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.75214 (13.51887) + | > current_lr: 0.00007 + | > step_time: 1.87640 (2.76082) + | > loader_time: 0.00210 (0.07226) + + + --> STEP: 50/234 -- GLOBAL_STEP: 62060 + | > loss: -0.29561 (-0.31069) + | > log_mle: -0.37148 (-0.38968) + | > loss_dur: 0.07587 (0.07899) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.19169 (13.00507) + | > current_lr: 0.00007 + | > step_time: 2.38530 (2.67181) + | > loader_time: 0.00310 (0.06530) + + + --> STEP: 55/234 -- GLOBAL_STEP: 62065 + | > loss: -0.30819 (-0.30889) + | > log_mle: -0.38188 (-0.38855) + | > loss_dur: 0.07369 (0.07966) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06073 (12.81812) + | > current_lr: 0.00007 + | > step_time: 1.84400 (2.62028) + | > loader_time: 0.00270 (0.05969) + + + --> STEP: 60/234 -- GLOBAL_STEP: 62070 + | > loss: -0.24868 (-0.30606) + | > log_mle: -0.38154 (-0.38731) + | > loss_dur: 0.13286 (0.08126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.62547 (12.80233) + | > current_lr: 0.00007 + | > step_time: 5.21930 (2.59604) + | > loader_time: 0.08960 (0.05637) + + + --> STEP: 65/234 -- GLOBAL_STEP: 62075 + | > loss: -0.27981 (-0.30317) + | > log_mle: -0.36824 (-0.38615) + | > loss_dur: 0.08843 (0.08297) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.54302 (12.78259) + | > current_lr: 0.00007 + | > step_time: 1.77910 (2.58935) + | > loader_time: 0.00250 (0.05225) + + + --> STEP: 70/234 -- GLOBAL_STEP: 62080 + | > loss: -0.23626 (-0.30011) + | > log_mle: -0.35152 (-0.38428) + | > loss_dur: 0.11526 (0.08417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.50233 (12.72667) + | > current_lr: 0.00007 + | > step_time: 1.25620 (2.54677) + | > loader_time: 0.00210 (0.05004) + + + --> STEP: 75/234 -- GLOBAL_STEP: 62085 + | > loss: -0.23854 (-0.29677) + | > log_mle: -0.36601 (-0.38320) + | > loss_dur: 0.12747 (0.08642) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.96052 (12.82347) + | > current_lr: 0.00007 + | > step_time: 1.59360 (2.54727) + | > loader_time: 0.00750 (0.04796) + + + --> STEP: 80/234 -- GLOBAL_STEP: 62090 + | > loss: -0.25943 (-0.29443) + | > log_mle: -0.35338 (-0.38180) + | > loss_dur: 0.09395 (0.08738) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.34601 (12.78024) + | > current_lr: 0.00007 + | > step_time: 2.59990 (2.56895) + | > loader_time: 0.09990 (0.04635) + + + --> STEP: 85/234 -- GLOBAL_STEP: 62095 + | > loss: -0.25145 (-0.29194) + | > log_mle: -0.35875 (-0.38084) + | > loss_dur: 0.10730 (0.08890) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.93319 (12.79239) + | > current_lr: 0.00007 + | > step_time: 1.81970 (2.55684) + | > loader_time: 0.00200 (0.04464) + + + --> STEP: 90/234 -- GLOBAL_STEP: 62100 + | > loss: -0.24918 (-0.29004) + | > log_mle: -0.38143 (-0.38083) + | > loss_dur: 0.13226 (0.09079) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.46907 (12.86868) + | > current_lr: 0.00007 + | > step_time: 2.60000 (2.56099) + | > loader_time: 0.00530 (0.04239) + + + --> STEP: 95/234 -- GLOBAL_STEP: 62105 + | > loss: -0.30151 (-0.28906) + | > log_mle: -0.46150 (-0.38255) + | > loss_dur: 0.16000 (0.09349) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.07127 (13.50300) + | > current_lr: 0.00007 + | > step_time: 1.91190 (2.54845) + | > loader_time: 0.08770 (0.04336) + + + --> STEP: 100/234 -- GLOBAL_STEP: 62110 + | > loss: -0.25924 (-0.28742) + | > log_mle: -0.38835 (-0.38271) + | > loss_dur: 0.12911 (0.09528) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.09547 (13.93886) + | > current_lr: 0.00007 + | > step_time: 1.59670 (2.50065) + | > loader_time: 0.00310 (0.04229) + + + --> STEP: 105/234 -- GLOBAL_STEP: 62115 + | > loss: -0.24400 (-0.28628) + | > log_mle: -0.37107 (-0.38402) + | > loss_dur: 0.12707 (0.09773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.17235 (14.41282) + | > current_lr: 0.00007 + | > step_time: 0.99110 (2.47029) + | > loader_time: 0.00260 (0.04183) + + + --> STEP: 110/234 -- GLOBAL_STEP: 62120 + | > loss: -0.24432 (-0.28425) + | > log_mle: -0.38704 (-0.38479) + | > loss_dur: 0.14272 (0.10054) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.98640 (15.06595) + | > current_lr: 0.00007 + | > step_time: 1.60650 (2.45887) + | > loader_time: 0.09170 (0.04177) + + + --> STEP: 115/234 -- GLOBAL_STEP: 62125 + | > loss: -0.25039 (-0.28339) + | > log_mle: -0.41306 (-0.38666) + | > loss_dur: 0.16267 (0.10327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.25784 (15.43740) + | > current_lr: 0.00007 + | > step_time: 4.58850 (2.48023) + | > loader_time: 0.10590 (0.04176) + + + --> STEP: 120/234 -- GLOBAL_STEP: 62130 + | > loss: -0.29172 (-0.28258) + | > log_mle: -0.46228 (-0.38824) + | > loss_dur: 0.17056 (0.10566) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.21892 (15.78815) + | > current_lr: 0.00007 + | > step_time: 1.99250 (2.46361) + | > loader_time: 0.00780 (0.04222) + + + --> STEP: 125/234 -- GLOBAL_STEP: 62135 + | > loss: -0.26841 (-0.28099) + | > log_mle: -0.44279 (-0.38858) + | > loss_dur: 0.17438 (0.10759) + | > amp_scaler: 2048.00000 (1032.19200) + | > grad_norm: 32.94780 (16.15072) + | > current_lr: 0.00007 + | > step_time: 1.98850 (2.43622) + | > loader_time: 0.00400 (0.04206) + + + --> STEP: 130/234 -- GLOBAL_STEP: 62140 + | > loss: -0.27961 (-0.28085) + | > log_mle: -0.45626 (-0.39090) + | > loss_dur: 0.17665 (0.11004) + | > amp_scaler: 2048.00000 (1071.26154) + | > grad_norm: 42.28492 (16.93348) + | > current_lr: 0.00007 + | > step_time: 2.29810 (2.43492) + | > loader_time: 0.00340 (0.04187) + + + --> STEP: 135/234 -- GLOBAL_STEP: 62145 + | > loss: -0.24431 (-0.28112) + | > log_mle: -0.38717 (-0.39330) + | > loss_dur: 0.14286 (0.11218) + | > amp_scaler: 2048.00000 (1107.43704) + | > grad_norm: 18.93825 (17.74823) + | > current_lr: 0.00007 + | > step_time: 1.31030 (2.42390) + | > loader_time: 0.08810 (0.04110) + + + --> STEP: 140/234 -- GLOBAL_STEP: 62150 + | > loss: -0.24545 (-0.28169) + | > log_mle: -0.42526 (-0.39647) + | > loss_dur: 0.17981 (0.11478) + | > amp_scaler: 2048.00000 (1141.02857) + | > grad_norm: 25.26709 (18.44730) + | > current_lr: 0.00007 + | > step_time: 2.49980 (2.45745) + | > loader_time: 0.00550 (0.04185) + + + --> STEP: 145/234 -- GLOBAL_STEP: 62155 + | > loss: -0.34889 (-0.28273) + | > log_mle: -0.53157 (-0.40031) + | > loss_dur: 0.18268 (0.11757) + | > amp_scaler: 2048.00000 (1172.30345) + | > grad_norm: 36.84769 (19.19390) + | > current_lr: 0.00007 + | > step_time: 3.58200 (2.47682) + | > loader_time: 0.00230 (0.04117) + + + --> STEP: 150/234 -- GLOBAL_STEP: 62160 + | > loss: -0.31013 (-0.28407) + | > log_mle: -0.50646 (-0.40378) + | > loss_dur: 0.19633 (0.11971) + | > amp_scaler: 2048.00000 (1201.49333) + | > grad_norm: 47.39710 (19.90752) + | > current_lr: 0.00007 + | > step_time: 2.01500 (2.51924) + | > loader_time: 0.08810 (0.04224) + + + --> STEP: 155/234 -- GLOBAL_STEP: 62165 + | > loss: -0.36114 (-0.28628) + | > log_mle: -0.56289 (-0.40829) + | > loss_dur: 0.20175 (0.12201) + | > amp_scaler: 2048.00000 (1228.80000) + | > grad_norm: 89.30412 (21.17911) + | > current_lr: 0.00007 + | > step_time: 5.09400 (2.67156) + | > loader_time: 0.60170 (0.04787) + + + --> STEP: 160/234 -- GLOBAL_STEP: 62170 + | > loss: -0.35898 (-0.28752) + | > log_mle: -0.56699 (-0.41195) + | > loss_dur: 0.20801 (0.12443) + | > amp_scaler: 2048.00000 (1254.40000) + | > grad_norm: 46.71745 (22.21457) + | > current_lr: 0.00007 + | > step_time: 2.79210 (2.65114) + | > loader_time: 0.10180 (0.04811) + + + --> STEP: 165/234 -- GLOBAL_STEP: 62175 + | > loss: -0.37190 (-0.28910) + | > log_mle: -0.56792 (-0.41563) + | > loss_dur: 0.19603 (0.12652) + | > amp_scaler: 2048.00000 (1278.44848) + | > grad_norm: 51.09581 (22.86835) + | > current_lr: 0.00007 + | > step_time: 2.91270 (2.62533) + | > loader_time: 0.00300 (0.04675) + + + --> STEP: 170/234 -- GLOBAL_STEP: 62180 + | > loss: -0.38438 (-0.29116) + | > log_mle: -0.60841 (-0.41985) + | > loss_dur: 0.22403 (0.12869) + | > amp_scaler: 2048.00000 (1301.08235) + | > grad_norm: 64.29542 (23.91526) + | > current_lr: 0.00007 + | > step_time: 2.59460 (2.70283) + | > loader_time: 0.00440 (0.04706) + + + --> STEP: 175/234 -- GLOBAL_STEP: 62185 + | > loss: -0.36059 (-0.29403) + | > log_mle: -0.58309 (-0.42507) + | > loss_dur: 0.22250 (0.13104) + | > amp_scaler: 2048.00000 (1322.42286) + | > grad_norm: 65.96244 (24.70320) + | > current_lr: 0.00007 + | > step_time: 5.00790 (2.72212) + | > loader_time: 0.18370 (0.04788) + + + --> STEP: 180/234 -- GLOBAL_STEP: 62190 + | > loss: -0.37652 (-0.29641) + | > log_mle: -0.58683 (-0.42989) + | > loss_dur: 0.21031 (0.13348) + | > amp_scaler: 2048.00000 (1342.57778) + | > grad_norm: 68.04156 (25.69024) + | > current_lr: 0.00007 + | > step_time: 5.50120 (2.82377) + | > loader_time: 0.10360 (0.04873) + + + --> STEP: 185/234 -- GLOBAL_STEP: 62195 + | > loss: -0.38984 (-0.29836) + | > log_mle: -0.60916 (-0.43419) + | > loss_dur: 0.21931 (0.13583) + | > amp_scaler: 2048.00000 (1361.64324) + | > grad_norm: 101.20988 (26.96446) + | > current_lr: 0.00007 + | > step_time: 11.09590 (2.92425) + | > loader_time: 0.09900 (0.05108) + + + --> STEP: 190/234 -- GLOBAL_STEP: 62200 + | > loss: -0.38293 (-0.30053) + | > log_mle: -0.58978 (-0.43850) + | > loss_dur: 0.20686 (0.13797) + | > amp_scaler: 2048.00000 (1379.70526) + | > grad_norm: 63.72744 (27.97809) + | > current_lr: 0.00007 + | > step_time: 1.02540 (2.95041) + | > loader_time: 0.00270 (0.05127) + + + --> STEP: 195/234 -- GLOBAL_STEP: 62205 + | > loss: -0.39098 (-0.30320) + | > log_mle: -0.61228 (-0.44310) + | > loss_dur: 0.22131 (0.13990) + | > amp_scaler: 2048.00000 (1396.84103) + | > grad_norm: 61.51445 (29.13183) + | > current_lr: 0.00007 + | > step_time: 3.90570 (2.96775) + | > loader_time: 0.08720 (0.05134) + + + --> STEP: 200/234 -- GLOBAL_STEP: 62210 + | > loss: -0.38370 (-0.30557) + | > log_mle: -0.62629 (-0.44739) + | > loss_dur: 0.24259 (0.14183) + | > amp_scaler: 2048.00000 (1413.12000) + | > grad_norm: 60.05829 (30.03811) + | > current_lr: 0.00007 + | > step_time: 3.38360 (2.99744) + | > loader_time: 0.00630 (0.05110) + + + --> STEP: 205/234 -- GLOBAL_STEP: 62215 + | > loss: -0.38613 (-0.30778) + | > log_mle: -0.60677 (-0.45156) + | > loss_dur: 0.22063 (0.14378) + | > amp_scaler: 2048.00000 (1428.60488) + | > grad_norm: 78.59138 (30.95176) + | > current_lr: 0.00007 + | > step_time: 4.39430 (3.06594) + | > loader_time: 0.00360 (0.05127) + + + --> STEP: 210/234 -- GLOBAL_STEP: 62220 + | > loss: -0.45276 (-0.31070) + | > log_mle: -0.69022 (-0.45645) + | > loss_dur: 0.23746 (0.14575) + | > amp_scaler: 2048.00000 (1443.35238) + | > grad_norm: 70.38400 (32.01232) + | > current_lr: 0.00007 + | > step_time: 5.89960 (3.15294) + | > loader_time: 0.00220 (0.05061) + + + --> STEP: 215/234 -- GLOBAL_STEP: 62225 + | > loss: -0.41240 (-0.31383) + | > log_mle: -0.64078 (-0.46157) + | > loss_dur: 0.22839 (0.14773) + | > amp_scaler: 2048.00000 (1457.41395) + | > grad_norm: 70.07473 (33.00882) + | > current_lr: 0.00007 + | > step_time: 5.29940 (3.21180) + | > loader_time: 0.19680 (0.05209) + + + --> STEP: 220/234 -- GLOBAL_STEP: 62230 + | > loss: -0.45411 (-0.31744) + | > log_mle: -0.68892 (-0.46713) + | > loss_dur: 0.23481 (0.14969) + | > amp_scaler: 2048.00000 (1470.83636) + | > grad_norm: 88.19684 (34.20559) + | > current_lr: 0.00007 + | > step_time: 7.89150 (3.27462) + | > loader_time: 0.09650 (0.05462) + + + --> STEP: 225/234 -- GLOBAL_STEP: 62235 + | > loss: -0.51703 (-0.32062) + | > log_mle: -0.75617 (-0.47225) + | > loss_dur: 0.23913 (0.15164) + | > amp_scaler: 2048.00000 (1483.66222) + | > grad_norm: 126.59526 (35.44997) + | > current_lr: 0.00007 + | > step_time: 0.23470 (3.21613) + | > loader_time: 0.00340 (0.05380) + + + --> STEP: 230/234 -- GLOBAL_STEP: 62240 + | > loss: -0.51171 (-0.32364) + | > log_mle: -0.82329 (-0.47794) + | > loss_dur: 0.31158 (0.15430) + | > amp_scaler: 2048.00000 (1495.93043) + | > grad_norm: 86.51998 (36.50028) + | > current_lr: 0.00007 + | > step_time: 0.25300 (3.15153) + | > loader_time: 0.00370 (0.05272) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.57691 (+0.55354) + | > avg_loss: -0.31565 (+0.00208) + | > avg_log_mle: -0.53896 (+0.01127) + | > avg_loss_dur: 0.22331 (-0.00919) + + + > EPOCH: 266/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 17:16:25)  + + --> STEP: 1/234 -- GLOBAL_STEP: 62245 + | > loss: -0.30478 (-0.30478) + | > log_mle: -0.39071 (-0.39071) + | > loss_dur: 0.08593 (0.08593) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.68363 (18.68363) + | > current_lr: 0.00007 + | > step_time: 2.00460 (2.00458) + | > loader_time: 0.08670 (0.08669) + + + --> STEP: 6/234 -- GLOBAL_STEP: 62250 + | > loss: -0.33208 (-0.30620) + | > log_mle: -0.39411 (-0.39640) + | > loss_dur: 0.06203 (0.09020) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.30094 (18.69939) + | > current_lr: 0.00007 + | > step_time: 4.30710 (4.11914) + | > loader_time: 0.08810 (1.78035) + + + --> STEP: 11/234 -- GLOBAL_STEP: 62255 + | > loss: -0.35279 (-0.31608) + | > log_mle: -0.40931 (-0.39988) + | > loss_dur: 0.05652 (0.08380) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.34900 (17.87997) + | > current_lr: 0.00007 + | > step_time: 8.69710 (4.70089) + | > loader_time: 0.00760 (0.99063) + + + --> STEP: 16/234 -- GLOBAL_STEP: 62260 + | > loss: -0.34414 (-0.32068) + | > log_mle: -0.40833 (-0.40147) + | > loss_dur: 0.06419 (0.08079) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.34721 (16.71195) + | > current_lr: 0.00007 + | > step_time: 2.40300 (4.33111) + | > loader_time: 0.00290 (0.69413) + + + --> STEP: 21/234 -- GLOBAL_STEP: 62265 + | > loss: -0.32269 (-0.32243) + | > log_mle: -0.38664 (-0.39998) + | > loss_dur: 0.06395 (0.07755) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.16977 (15.62019) + | > current_lr: 0.00007 + | > step_time: 4.31170 (4.17620) + | > loader_time: 0.08590 (0.53817) + + + --> STEP: 26/234 -- GLOBAL_STEP: 62270 + | > loss: -0.31264 (-0.32336) + | > log_mle: -0.38689 (-0.39918) + | > loss_dur: 0.07424 (0.07582) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.50891 (14.88547) + | > current_lr: 0.00007 + | > step_time: 4.50930 (4.06644) + | > loader_time: 0.00430 (0.43898) + + + --> STEP: 31/234 -- GLOBAL_STEP: 62275 + | > loss: -0.27868 (-0.32252) + | > log_mle: -0.37663 (-0.39810) + | > loss_dur: 0.09794 (0.07557) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.17885 (14.14625) + | > current_lr: 0.00007 + | > step_time: 5.40800 (4.16564) + | > loader_time: 0.00360 (0.37171) + + + --> STEP: 36/234 -- GLOBAL_STEP: 62280 + | > loss: -0.28808 (-0.32004) + | > log_mle: -0.37637 (-0.39631) + | > loss_dur: 0.08829 (0.07627) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.39388 (13.65593) + | > current_lr: 0.00007 + | > step_time: 1.88530 (4.22892) + | > loader_time: 0.00180 (0.32792) + + + --> STEP: 41/234 -- GLOBAL_STEP: 62285 + | > loss: -0.32717 (-0.31763) + | > log_mle: -0.38912 (-0.39414) + | > loss_dur: 0.06195 (0.07651) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.29492 (13.45345) + | > current_lr: 0.00007 + | > step_time: 1.28410 (3.85240) + | > loader_time: 0.00200 (0.28814) + + + --> STEP: 46/234 -- GLOBAL_STEP: 62290 + | > loss: -0.29639 (-0.31440) + | > log_mle: -0.38097 (-0.39214) + | > loss_dur: 0.08458 (0.07775) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.85204 (13.37338) + | > current_lr: 0.00007 + | > step_time: 1.49430 (3.57822) + | > loader_time: 0.00200 (0.25704) + + + --> STEP: 51/234 -- GLOBAL_STEP: 62295 + | > loss: -0.29406 (-0.31343) + | > log_mle: -0.37199 (-0.39097) + | > loss_dur: 0.07794 (0.07754) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.58392 (12.90928) + | > current_lr: 0.00007 + | > step_time: 1.47690 (3.36356) + | > loader_time: 0.00220 (0.23203) + + + --> STEP: 56/234 -- GLOBAL_STEP: 62300 + | > loss: -0.28046 (-0.31098) + | > log_mle: -0.37968 (-0.38980) + | > loss_dur: 0.09922 (0.07882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.84550 (12.61464) + | > current_lr: 0.00007 + | > step_time: 2.00340 (3.21268) + | > loader_time: 0.00240 (0.21151) + + + --> STEP: 61/234 -- GLOBAL_STEP: 62305 + | > loss: -0.27757 (-0.30789) + | > log_mle: -0.37247 (-0.38847) + | > loss_dur: 0.09490 (0.08058) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.03270 (12.47011) + | > current_lr: 0.00007 + | > step_time: 1.01220 (3.05906) + | > loader_time: 0.08390 (0.19578) + + + --> STEP: 66/234 -- GLOBAL_STEP: 62310 + | > loss: -0.28107 (-0.30498) + | > log_mle: -0.36280 (-0.38711) + | > loss_dur: 0.08173 (0.08213) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.06677 (12.39181) + | > current_lr: 0.00007 + | > step_time: 1.23900 (2.94063) + | > loader_time: 0.02880 (0.18267) + + + --> STEP: 71/234 -- GLOBAL_STEP: 62315 + | > loss: -0.25742 (-0.30170) + | > log_mle: -0.38280 (-0.38559) + | > loss_dur: 0.12538 (0.08389) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.19588 (12.45384) + | > current_lr: 0.00007 + | > step_time: 3.90030 (2.92328) + | > loader_time: 0.00420 (0.17002) + + + --> STEP: 76/234 -- GLOBAL_STEP: 62320 + | > loss: -0.25628 (-0.29859) + | > log_mle: -0.36468 (-0.38419) + | > loss_dur: 0.10840 (0.08559) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 17.47377 (12.57818) + | > current_lr: 0.00007 + | > step_time: 3.02300 (2.85980) + | > loader_time: 0.00270 (0.15900) + + + --> STEP: 81/234 -- GLOBAL_STEP: 62325 + | > loss: -0.26552 (-0.29617) + | > log_mle: -0.37644 (-0.38290) + | > loss_dur: 0.11092 (0.08673) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.83513 (12.58308) + | > current_lr: 0.00007 + | > step_time: 1.70400 (2.81988) + | > loader_time: 0.00210 (0.15043) + + + --> STEP: 86/234 -- GLOBAL_STEP: 62330 + | > loss: -0.26029 (-0.29366) + | > log_mle: -0.37578 (-0.38191) + | > loss_dur: 0.11549 (0.08825) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.10894 (12.57967) + | > current_lr: 0.00007 + | > step_time: 3.90780 (2.79728) + | > loader_time: 0.09820 (0.14296) + + + --> STEP: 91/234 -- GLOBAL_STEP: 62335 + | > loss: -0.24674 (-0.29145) + | > log_mle: -0.38318 (-0.38195) + | > loss_dur: 0.13644 (0.09050) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.63267 (12.75907) + | > current_lr: 0.00007 + | > step_time: 2.40530 (2.77260) + | > loader_time: 0.09540 (0.13717) + + + --> STEP: 96/234 -- GLOBAL_STEP: 62340 + | > loss: -0.25256 (-0.29064) + | > log_mle: -0.36700 (-0.38357) + | > loss_dur: 0.11444 (0.09293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.26159 (13.24986) + | > current_lr: 0.00007 + | > step_time: 2.21270 (2.75659) + | > loader_time: 0.08350 (0.13182) + + + --> STEP: 101/234 -- GLOBAL_STEP: 62345 + | > loss: -0.25627 (-0.28910) + | > log_mle: -0.41481 (-0.38419) + | > loss_dur: 0.15854 (0.09509) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 28.38642 (13.80626) + | > current_lr: 0.00007 + | > step_time: 0.91650 (2.76154) + | > loader_time: 0.00320 (0.12824) + + + --> STEP: 106/234 -- GLOBAL_STEP: 62350 + | > loss: -0.24021 (-0.28781) + | > log_mle: -0.41465 (-0.38549) + | > loss_dur: 0.17444 (0.09769) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.13132 (14.17851) + | > current_lr: 0.00007 + | > step_time: 2.79260 (2.76357) + | > loader_time: 0.11650 (0.12426) + + + --> STEP: 111/234 -- GLOBAL_STEP: 62355 + | > loss: -0.28177 (-0.28646) + | > log_mle: -0.46782 (-0.38684) + | > loss_dur: 0.18605 (0.10038) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 33.42807 (14.73111) + | > current_lr: 0.00007 + | > step_time: 1.60420 (2.77708) + | > loader_time: 0.00310 (0.12122) + + + --> STEP: 116/234 -- GLOBAL_STEP: 62360 + | > loss: -0.24546 (-0.28509) + | > log_mle: -0.42996 (-0.38837) + | > loss_dur: 0.18451 (0.10328) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.23498 (15.17966) + | > current_lr: 0.00007 + | > step_time: 3.09390 (2.77312) + | > loader_time: 0.00260 (0.11688) + + + --> STEP: 121/234 -- GLOBAL_STEP: 62365 + | > loss: -0.22229 (-0.28396) + | > log_mle: -0.34766 (-0.38917) + | > loss_dur: 0.12537 (0.10521) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.02424 (15.45580) + | > current_lr: 0.00007 + | > step_time: 2.89570 (2.75196) + | > loader_time: 0.00220 (0.11441) + + + --> STEP: 126/234 -- GLOBAL_STEP: 62370 + | > loss: -0.29445 (-0.28309) + | > log_mle: -0.48177 (-0.39050) + | > loss_dur: 0.18731 (0.10741) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 29.03222 (15.90954) + | > current_lr: 0.00007 + | > step_time: 1.30900 (2.73512) + | > loader_time: 0.00250 (0.11142) + + + --> STEP: 131/234 -- GLOBAL_STEP: 62375 + | > loss: -0.34098 (-0.28317) + | > log_mle: -0.52312 (-0.39287) + | > loss_dur: 0.18214 (0.10970) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.45165 (16.78780) + | > current_lr: 0.00007 + | > step_time: 1.60240 (2.77047) + | > loader_time: 0.10190 (0.10964) + + + --> STEP: 136/234 -- GLOBAL_STEP: 62380 + | > loss: -0.36827 (-0.28361) + | > log_mle: -0.57877 (-0.39558) + | > loss_dur: 0.21050 (0.11197) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.03418 (17.32070) + | > current_lr: 0.00007 + | > step_time: 4.09060 (2.77452) + | > loader_time: 0.10740 (0.10851) + + + --> STEP: 141/234 -- GLOBAL_STEP: 62385 + | > loss: -0.30738 (-0.28349) + | > log_mle: -0.47292 (-0.39783) + | > loss_dur: 0.16554 (0.11433) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.20814 (18.00717) + | > current_lr: 0.00007 + | > step_time: 5.39670 (2.81798) + | > loader_time: 0.00220 (0.10670) + + + --> STEP: 146/234 -- GLOBAL_STEP: 62390 + | > loss: -0.32464 (-0.28470) + | > log_mle: -0.51370 (-0.40175) + | > loss_dur: 0.18906 (0.11705) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.64796 (19.41685) + | > current_lr: 0.00007 + | > step_time: 1.09600 (2.87270) + | > loader_time: 0.00320 (0.10446) + + + --> STEP: 151/234 -- GLOBAL_STEP: 62395 + | > loss: -0.30795 (-0.28558) + | > log_mle: -0.48345 (-0.40473) + | > loss_dur: 0.17550 (0.11915) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.39016 (20.09059) + | > current_lr: 0.00007 + | > step_time: 1.30020 (2.84784) + | > loader_time: 0.08810 (0.10357) + + + --> STEP: 156/234 -- GLOBAL_STEP: 62400 + | > loss: -0.33385 (-0.28775) + | > log_mle: -0.53278 (-0.40937) + | > loss_dur: 0.19894 (0.12162) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.57695 (21.23979) + | > current_lr: 0.00007 + | > step_time: 2.39900 (2.82884) + | > loader_time: 0.00620 (0.10088) + + + --> STEP: 161/234 -- GLOBAL_STEP: 62405 + | > loss: -0.37743 (-0.28937) + | > log_mle: -0.55742 (-0.41335) + | > loss_dur: 0.17998 (0.12398) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.21989 (22.02343) + | > current_lr: 0.00007 + | > step_time: 1.90760 (2.79382) + | > loader_time: 0.08320 (0.09893) + + + --> STEP: 166/234 -- GLOBAL_STEP: 62410 + | > loss: -0.32695 (-0.29076) + | > log_mle: -0.49894 (-0.41672) + | > loss_dur: 0.17198 (0.12596) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.88636 (22.68395) + | > current_lr: 0.00007 + | > step_time: 1.51300 (2.76755) + | > loader_time: 0.00410 (0.09731) + + + --> STEP: 171/234 -- GLOBAL_STEP: 62415 + | > loss: -0.40743 (-0.29341) + | > log_mle: -0.60782 (-0.42175) + | > loss_dur: 0.20039 (0.12835) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 78.49878 (23.87338) + | > current_lr: 0.00007 + | > step_time: 3.50320 (2.77907) + | > loader_time: 0.10610 (0.09610) + + + --> STEP: 176/234 -- GLOBAL_STEP: 62420 + | > loss: -0.39210 (-0.29604) + | > log_mle: -0.59358 (-0.42687) + | > loss_dur: 0.20149 (0.13082) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.00338 (24.89707) + | > current_lr: 0.00007 + | > step_time: 1.89010 (2.79726) + | > loader_time: 0.00710 (0.09619) + + + --> STEP: 181/234 -- GLOBAL_STEP: 62425 + | > loss: -0.31959 (-0.29814) + | > log_mle: -0.52028 (-0.43134) + | > loss_dur: 0.20069 (0.13320) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 43.52221 (25.93497) + | > current_lr: 0.00007 + | > step_time: 13.68800 (2.85764) + | > loader_time: 0.30250 (0.09722) + + + --> STEP: 186/234 -- GLOBAL_STEP: 62430 + | > loss: -0.32572 (-0.29992) + | > log_mle: -0.54933 (-0.43563) + | > loss_dur: 0.22361 (0.13571) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 86.01147 (27.54252) + | > current_lr: 0.00007 + | > step_time: 7.40100 (2.92394) + | > loader_time: 0.09890 (0.09724) + + + --> STEP: 191/234 -- GLOBAL_STEP: 62435 + | > loss: -0.36353 (-0.30186) + | > log_mle: -0.56699 (-0.43959) + | > loss_dur: 0.20346 (0.13773) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.73141 (28.41111) + | > current_lr: 0.00007 + | > step_time: 6.29910 (2.98922) + | > loader_time: 0.00300 (0.09534) + + + --> STEP: 196/234 -- GLOBAL_STEP: 62440 + | > loss: -0.35574 (-0.30439) + | > log_mle: -0.57169 (-0.44401) + | > loss_dur: 0.21595 (0.13963) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.57286 (29.28333) + | > current_lr: 0.00007 + | > step_time: 1.50180 (2.96611) + | > loader_time: 0.00270 (0.09427) + + + --> STEP: 201/234 -- GLOBAL_STEP: 62445 + | > loss: -0.30864 (-0.30622) + | > log_mle: -0.52869 (-0.44786) + | > loss_dur: 0.22005 (0.14164) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.99077 (30.02670) + | > current_lr: 0.00007 + | > step_time: 8.20330 (3.01427) + | > loader_time: 0.09270 (0.09570) + + + --> STEP: 206/234 -- GLOBAL_STEP: 62450 + | > loss: -0.42998 (-0.30889) + | > log_mle: -0.64811 (-0.45247) + | > loss_dur: 0.21813 (0.14359) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 83.46127 (30.91468) + | > current_lr: 0.00007 + | > step_time: 3.61160 (3.06543) + | > loader_time: 0.00340 (0.09431) + + + --> STEP: 211/234 -- GLOBAL_STEP: 62455 + | > loss: -0.48135 (-0.31201) + | > log_mle: -0.72801 (-0.45778) + | > loss_dur: 0.24666 (0.14577) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 87.31052 (32.10370) + | > current_lr: 0.00007 + | > step_time: 4.68850 (3.12119) + | > loader_time: 0.00710 (0.09303) + + + --> STEP: 216/234 -- GLOBAL_STEP: 62460 + | > loss: -0.46989 (-0.31512) + | > log_mle: -0.71146 (-0.46283) + | > loss_dur: 0.24156 (0.14771) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 97.18532 (33.49826) + | > current_lr: 0.00007 + | > step_time: 5.49580 (3.23408) + | > loader_time: 0.09290 (0.09189) + + + --> STEP: 221/234 -- GLOBAL_STEP: 62465 + | > loss: -0.41954 (-0.31847) + | > log_mle: -0.63011 (-0.46799) + | > loss_dur: 0.21057 (0.14952) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.54551 (34.56451) + | > current_lr: 0.00007 + | > step_time: 2.30550 (3.26184) + | > loader_time: 0.09250 (0.09114) + + + --> STEP: 226/234 -- GLOBAL_STEP: 62470 + | > loss: -0.48678 (-0.32189) + | > log_mle: -0.73147 (-0.47360) + | > loss_dur: 0.24469 (0.15170) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 116.52145 (35.98064) + | > current_lr: 0.00007 + | > step_time: 0.25380 (3.21154) + | > loader_time: 0.00740 (0.08963) + + + --> STEP: 231/234 -- GLOBAL_STEP: 62475 + | > loss: -0.41098 (-0.32470) + | > log_mle: -0.80195 (-0.47962) + | > loss_dur: 0.39097 (0.15492) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 113.00359 (37.25226) + | > current_lr: 0.00007 + | > step_time: 0.29080 (3.14773) + | > loader_time: 0.00380 (0.08781) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.55605 (-0.02086) + | > avg_loss: -0.32605 (-0.01040) + | > avg_log_mle: -0.54593 (-0.00698) + | > avg_loss_dur: 0.21989 (-0.00343) + + + > EPOCH: 267/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 17:29:49)  + + --> STEP: 2/234 -- GLOBAL_STEP: 62480 + | > loss: -0.33942 (-0.32174) + | > log_mle: -0.41363 (-0.40273) + | > loss_dur: 0.07421 (0.08099) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.13612 (18.54177) + | > current_lr: 0.00007 + | > step_time: 11.99750 (7.75169) + | > loader_time: 0.08270 (0.04266) + + + --> STEP: 7/234 -- GLOBAL_STEP: 62485 + | > loss: -0.32892 (-0.30874) + | > log_mle: -0.39777 (-0.39824) + | > loss_dur: 0.06885 (0.08950) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.04007 (17.66204) + | > current_lr: 0.00007 + | > step_time: 1.12260 (4.30530) + | > loader_time: 0.00100 (0.07007) + + + --> STEP: 12/234 -- GLOBAL_STEP: 62490 + | > loss: -0.32421 (-0.31468) + | > log_mle: -0.39688 (-0.40180) + | > loss_dur: 0.07268 (0.08711) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.16746 (16.80490) + | > current_lr: 0.00007 + | > step_time: 3.30710 (3.16724) + | > loader_time: 0.00130 (0.04742) + + + --> STEP: 17/234 -- GLOBAL_STEP: 62495 + | > loss: -0.34409 (-0.32385) + | > log_mle: -0.40095 (-0.40417) + | > loss_dur: 0.05686 (0.08031) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.16302 (15.35254) + | > current_lr: 0.00007 + | > step_time: 0.81300 (2.91992) + | > loader_time: 0.00240 (0.03921) + + + --> STEP: 22/234 -- GLOBAL_STEP: 62500 + | > loss: -0.32290 (-0.32359) + | > log_mle: -0.39876 (-0.40245) + | > loss_dur: 0.07586 (0.07886) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.73667 (14.84779) + | > current_lr: 0.00007 + | > step_time: 2.50010 (2.86205) + | > loader_time: 0.00240 (0.03562) + + + --> STEP: 27/234 -- GLOBAL_STEP: 62505 + | > loss: -0.30966 (-0.32416) + | > log_mle: -0.38480 (-0.40087) + | > loss_dur: 0.07514 (0.07670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.38455 (14.71832) + | > current_lr: 0.00007 + | > step_time: 4.59990 (3.48348) + | > loader_time: 0.10590 (0.03972) + + + --> STEP: 32/234 -- GLOBAL_STEP: 62510 + | > loss: -0.32007 (-0.32320) + | > log_mle: -0.39505 (-0.39929) + | > loss_dur: 0.07498 (0.07610) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.85127 (14.25996) + | > current_lr: 0.00007 + | > step_time: 4.29040 (3.53861) + | > loader_time: 0.00380 (0.03714) + + + --> STEP: 37/234 -- GLOBAL_STEP: 62515 + | > loss: -0.30066 (-0.32015) + | > log_mle: -0.36651 (-0.39645) + | > loss_dur: 0.06586 (0.07630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.90530 (13.84098) + | > current_lr: 0.00007 + | > step_time: 0.67350 (3.45001) + | > loader_time: 0.00220 (0.03494) + + + --> STEP: 42/234 -- GLOBAL_STEP: 62520 + | > loss: -0.28822 (-0.31749) + | > log_mle: -0.36643 (-0.39441) + | > loss_dur: 0.07821 (0.07692) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.46024 (13.37485) + | > current_lr: 0.00007 + | > step_time: 2.60480 (3.23386) + | > loader_time: 0.00250 (0.03101) + + + --> STEP: 47/234 -- GLOBAL_STEP: 62525 + | > loss: -0.29563 (-0.31416) + | > log_mle: -0.38184 (-0.39276) + | > loss_dur: 0.08621 (0.07861) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.09065 (13.24498) + | > current_lr: 0.00007 + | > step_time: 1.69200 (3.07416) + | > loader_time: 0.00180 (0.03000) + + + --> STEP: 52/234 -- GLOBAL_STEP: 62530 + | > loss: -0.27103 (-0.31240) + | > log_mle: -0.37262 (-0.39137) + | > loss_dur: 0.10159 (0.07898) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.64001 (12.82563) + | > current_lr: 0.00007 + | > step_time: 3.45330 (2.93360) + | > loader_time: 0.09430 (0.03070) + + + --> STEP: 57/234 -- GLOBAL_STEP: 62535 + | > loss: -0.25533 (-0.30985) + | > log_mle: -0.35609 (-0.38986) + | > loss_dur: 0.10076 (0.08001) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.47405 (12.67336) + | > current_lr: 0.00007 + | > step_time: 2.11010 (2.81962) + | > loader_time: 0.00330 (0.02984) + + + --> STEP: 62/234 -- GLOBAL_STEP: 62540 + | > loss: -0.22793 (-0.30664) + | > log_mle: -0.38478 (-0.38887) + | > loss_dur: 0.15684 (0.08223) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.40801 (12.97215) + | > current_lr: 0.00007 + | > step_time: 0.77960 (2.73237) + | > loader_time: 0.00180 (0.02950) + + + --> STEP: 67/234 -- GLOBAL_STEP: 62545 + | > loss: -0.26983 (-0.30417) + | > log_mle: -0.37470 (-0.38730) + | > loss_dur: 0.10487 (0.08313) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.12141 (12.81956) + | > current_lr: 0.00007 + | > step_time: 1.83250 (2.68116) + | > loader_time: 0.09280 (0.03109) + + + --> STEP: 72/234 -- GLOBAL_STEP: 62550 + | > loss: -0.27683 (-0.30105) + | > log_mle: -0.36695 (-0.38579) + | > loss_dur: 0.09012 (0.08474) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.02593 (12.71108) + | > current_lr: 0.00007 + | > step_time: 1.79340 (2.61965) + | > loader_time: 0.00190 (0.03030) + + + --> STEP: 77/234 -- GLOBAL_STEP: 62555 + | > loss: -0.26065 (-0.29774) + | > log_mle: -0.36461 (-0.38450) + | > loss_dur: 0.10397 (0.08676) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.86596 (12.74769) + | > current_lr: 0.00007 + | > step_time: 1.55190 (2.58936) + | > loader_time: 0.00180 (0.03188) + + + --> STEP: 82/234 -- GLOBAL_STEP: 62560 + | > loss: -0.25797 (-0.29521) + | > log_mle: -0.36190 (-0.38318) + | > loss_dur: 0.10393 (0.08797) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.18882 (12.64405) + | > current_lr: 0.00007 + | > step_time: 1.39400 (2.56465) + | > loader_time: 0.00250 (0.03444) + + + --> STEP: 87/234 -- GLOBAL_STEP: 62565 + | > loss: -0.24584 (-0.29272) + | > log_mle: -0.36060 (-0.38216) + | > loss_dur: 0.11477 (0.08944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.52888 (12.67761) + | > current_lr: 0.00007 + | > step_time: 2.47950 (2.54286) + | > loader_time: 0.00230 (0.03357) + + + --> STEP: 92/234 -- GLOBAL_STEP: 62570 + | > loss: -0.27200 (-0.29103) + | > log_mle: -0.40273 (-0.38274) + | > loss_dur: 0.13073 (0.09171) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.99299 (12.92132) + | > current_lr: 0.00007 + | > step_time: 2.20820 (2.55486) + | > loader_time: 0.00310 (0.03579) + + + --> STEP: 97/234 -- GLOBAL_STEP: 62575 + | > loss: -0.25991 (-0.28995) + | > log_mle: -0.38953 (-0.38408) + | > loss_dur: 0.12962 (0.09413) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 14.52756 (13.50060) + | > current_lr: 0.00007 + | > step_time: 1.69740 (2.51660) + | > loader_time: 0.00330 (0.03487) + + + --> STEP: 102/234 -- GLOBAL_STEP: 62580 + | > loss: -0.23933 (-0.28828) + | > log_mle: -0.37267 (-0.38462) + | > loss_dur: 0.13334 (0.09635) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.02336 (13.79401) + | > current_lr: 0.00007 + | > step_time: 1.98040 (2.50923) + | > loader_time: 0.00650 (0.03585) + + + --> STEP: 107/234 -- GLOBAL_STEP: 62585 + | > loss: -0.25291 (-0.28706) + | > log_mle: -0.40615 (-0.38603) + | > loss_dur: 0.15324 (0.09897) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.67164 (14.42862) + | > current_lr: 0.00007 + | > step_time: 1.40680 (2.49010) + | > loader_time: 0.00300 (0.03520) + + + --> STEP: 112/234 -- GLOBAL_STEP: 62590 + | > loss: -0.26175 (-0.28579) + | > log_mle: -0.42524 (-0.38743) + | > loss_dur: 0.16348 (0.10164) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.22362 (14.91244) + | > current_lr: 0.00007 + | > step_time: 3.70790 (2.48985) + | > loader_time: 0.00330 (0.03381) + + + --> STEP: 117/234 -- GLOBAL_STEP: 62595 + | > loss: -0.26113 (-0.28461) + | > log_mle: -0.42074 (-0.38893) + | > loss_dur: 0.15960 (0.10432) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.73765 (15.31821) + | > current_lr: 0.00007 + | > step_time: 3.11430 (2.47938) + | > loader_time: 0.00910 (0.03409) + + + --> STEP: 122/234 -- GLOBAL_STEP: 62600 + | > loss: -0.24501 (-0.28320) + | > log_mle: -0.39167 (-0.38950) + | > loss_dur: 0.14666 (0.10630) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.78685 (15.65563) + | > current_lr: 0.00007 + | > step_time: 1.90610 (2.45643) + | > loader_time: 0.07530 (0.03358) + + + --> STEP: 127/234 -- GLOBAL_STEP: 62605 + | > loss: -0.27144 (-0.28245) + | > log_mle: -0.44691 (-0.39127) + | > loss_dur: 0.17547 (0.10882) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.08809 (16.46893) + | > current_lr: 0.00007 + | > step_time: 3.29440 (2.47390) + | > loader_time: 0.00370 (0.03444) + + + --> STEP: 132/234 -- GLOBAL_STEP: 62610 + | > loss: -0.27941 (-0.28244) + | > log_mle: -0.43209 (-0.39350) + | > loss_dur: 0.15268 (0.11106) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.60157 (17.28415) + | > current_lr: 0.00007 + | > step_time: 2.69880 (2.46735) + | > loader_time: 0.01050 (0.03451) + + + --> STEP: 137/234 -- GLOBAL_STEP: 62615 + | > loss: -0.26034 (-0.28282) + | > log_mle: -0.44569 (-0.39620) + | > loss_dur: 0.18535 (0.11338) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.37706 (18.07099) + | > current_lr: 0.00007 + | > step_time: 2.50650 (2.45973) + | > loader_time: 0.08790 (0.03461) + + + --> STEP: 142/234 -- GLOBAL_STEP: 62620 + | > loss: -0.25897 (-0.28265) + | > log_mle: -0.45369 (-0.39840) + | > loss_dur: 0.19473 (0.11574) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.65036 (18.83165) + | > current_lr: 0.00007 + | > step_time: 5.69670 (2.51268) + | > loader_time: 0.00440 (0.03694) + + + --> STEP: 147/234 -- GLOBAL_STEP: 62625 + | > loss: -0.28535 (-0.28399) + | > log_mle: -0.46453 (-0.40243) + | > loss_dur: 0.17918 (0.11844) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 25.77704 (19.64215) + | > current_lr: 0.00007 + | > step_time: 7.80310 (2.54634) + | > loader_time: 0.19410 (0.03950) + + + --> STEP: 152/234 -- GLOBAL_STEP: 62630 + | > loss: -0.34293 (-0.28560) + | > log_mle: -0.55338 (-0.40623) + | > loss_dur: 0.21046 (0.12064) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.79353 (20.46193) + | > current_lr: 0.00007 + | > step_time: 3.72840 (2.56471) + | > loader_time: 0.07620 (0.04129) + + + --> STEP: 157/234 -- GLOBAL_STEP: 62635 + | > loss: -0.30497 (-0.28776) + | > log_mle: -0.49311 (-0.41069) + | > loss_dur: 0.18815 (0.12293) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 37.62573 (21.43900) + | > current_lr: 0.00007 + | > step_time: 3.89270 (2.55347) + | > loader_time: 0.00300 (0.04007) + + + --> STEP: 162/234 -- GLOBAL_STEP: 62640 + | > loss: -0.32916 (-0.28934) + | > log_mle: -0.51210 (-0.41452) + | > loss_dur: 0.18294 (0.12518) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 45.65093 (22.43107) + | > current_lr: 0.00007 + | > step_time: 4.49340 (2.61915) + | > loader_time: 0.00200 (0.04174) + + + --> STEP: 167/234 -- GLOBAL_STEP: 62645 + | > loss: -0.42576 (-0.29110) + | > log_mle: -0.61977 (-0.41829) + | > loss_dur: 0.19401 (0.12718) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 68.17188 (23.08249) + | > current_lr: 0.00007 + | > step_time: 0.93360 (2.61100) + | > loader_time: 0.00260 (0.04260) + + + --> STEP: 172/234 -- GLOBAL_STEP: 62650 + | > loss: -0.40100 (-0.29350) + | > log_mle: -0.61225 (-0.42315) + | > loss_dur: 0.21125 (0.12965) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.59277 (24.10485) + | > current_lr: 0.00007 + | > step_time: 2.69690 (2.61433) + | > loader_time: 0.10010 (0.04202) + + + --> STEP: 177/234 -- GLOBAL_STEP: 62655 + | > loss: -0.36265 (-0.29593) + | > log_mle: -0.57122 (-0.42787) + | > loss_dur: 0.20857 (0.13194) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 67.53984 (25.08001) + | > current_lr: 0.00007 + | > step_time: 3.90540 (2.65915) + | > loader_time: 0.00530 (0.04192) + + + --> STEP: 182/234 -- GLOBAL_STEP: 62660 + | > loss: -0.38629 (-0.29815) + | > log_mle: -0.62285 (-0.43257) + | > loss_dur: 0.23656 (0.13442) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.45100 (26.27644) + | > current_lr: 0.00007 + | > step_time: 6.39520 (2.74437) + | > loader_time: 0.00470 (0.04295) + + + --> STEP: 187/234 -- GLOBAL_STEP: 62665 + | > loss: -0.39786 (-0.30018) + | > log_mle: -0.61639 (-0.43697) + | > loss_dur: 0.21853 (0.13679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 59.17005 (27.68923) + | > current_lr: 0.00007 + | > step_time: 1.29850 (2.73403) + | > loader_time: 0.07760 (0.04268) + + + --> STEP: 192/234 -- GLOBAL_STEP: 62670 + | > loss: -0.43428 (-0.30282) + | > log_mle: -0.64411 (-0.44148) + | > loss_dur: 0.20983 (0.13866) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.05916 (28.63340) + | > current_lr: 0.00007 + | > step_time: 9.29610 (2.79045) + | > loader_time: 0.00220 (0.04373) + + + --> STEP: 197/234 -- GLOBAL_STEP: 62675 + | > loss: -0.42044 (-0.30551) + | > log_mle: -0.61668 (-0.44601) + | > loss_dur: 0.19625 (0.14050) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.86691 (29.53744) + | > current_lr: 0.00007 + | > step_time: 4.09930 (2.85112) + | > loader_time: 0.00690 (0.04367) + + + --> STEP: 202/234 -- GLOBAL_STEP: 62680 + | > loss: -0.49521 (-0.30804) + | > log_mle: -0.70967 (-0.45065) + | > loss_dur: 0.21447 (0.14261) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 99.59526 (30.60435) + | > current_lr: 0.00007 + | > step_time: 7.30190 (2.89553) + | > loader_time: 0.08960 (0.04448) + + + --> STEP: 207/234 -- GLOBAL_STEP: 62685 + | > loss: -0.46159 (-0.31054) + | > log_mle: -0.68912 (-0.45516) + | > loss_dur: 0.22753 (0.14462) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 105.36768 (31.71406) + | > current_lr: 0.00007 + | > step_time: 3.19410 (2.97240) + | > loader_time: 0.00830 (0.04490) + + + --> STEP: 212/234 -- GLOBAL_STEP: 62690 + | > loss: -0.44752 (-0.31360) + | > log_mle: -0.67908 (-0.46030) + | > loss_dur: 0.23156 (0.14670) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 73.46291 (33.00683) + | > current_lr: 0.00007 + | > step_time: 5.71260 (3.01374) + | > loader_time: 0.00470 (0.04478) + + + --> STEP: 217/234 -- GLOBAL_STEP: 62695 + | > loss: -0.46281 (-0.31678) + | > log_mle: -0.69872 (-0.46543) + | > loss_dur: 0.23592 (0.14865) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 89.00703 (34.17477) + | > current_lr: 0.00007 + | > step_time: 4.68860 (3.14094) + | > loader_time: 0.00670 (0.04526) + + + --> STEP: 222/234 -- GLOBAL_STEP: 62700 + | > loss: -0.45815 (-0.32003) + | > log_mle: -0.71755 (-0.47059) + | > loss_dur: 0.25940 (0.15056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.33738 (35.18736) + | > current_lr: 0.00007 + | > step_time: 1.40100 (3.11341) + | > loader_time: 0.09050 (0.04556) + + + --> STEP: 227/234 -- GLOBAL_STEP: 62705 + | > loss: -0.43091 (-0.32356) + | > log_mle: -0.68381 (-0.47611) + | > loss_dur: 0.25290 (0.15255) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.40947 (36.28735) + | > current_lr: 0.00007 + | > step_time: 0.24860 (3.06870) + | > loader_time: 0.00340 (0.04535) + + + --> STEP: 232/234 -- GLOBAL_STEP: 62710 + | > loss: -0.42575 (-0.32641) + | > log_mle: -0.90043 (-0.48309) + | > loss_dur: 0.47468 (0.15667) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 153.02834 (37.80415) + | > current_lr: 0.00007 + | > step_time: 0.38810 (3.00858) + | > loader_time: 0.00450 (0.04446) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.59832 (+0.04227) + | > avg_loss: -0.33642 (-0.01037) + | > avg_log_mle: -0.55781 (-0.01187) + | > avg_loss_dur: 0.22138 (+0.00150) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_62712.pth + + > EPOCH: 268/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 17:42:50)  + + --> STEP: 3/234 -- GLOBAL_STEP: 62715 + | > loss: -0.24627 (-0.28851) + | > log_mle: -0.38560 (-0.39590) + | > loss_dur: 0.13933 (0.10739) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.03622 (22.93404) + | > current_lr: 0.00007 + | > step_time: 3.89690 (4.73295) + | > loader_time: 0.00270 (0.00406) + + + --> STEP: 8/234 -- GLOBAL_STEP: 62720 + | > loss: -0.32856 (-0.30510) + | > log_mle: -0.41235 (-0.39805) + | > loss_dur: 0.08379 (0.09295) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.72813 (19.88640) + | > current_lr: 0.00007 + | > step_time: 9.30240 (5.39716) + | > loader_time: 0.10030 (0.01546) + + + --> STEP: 13/234 -- GLOBAL_STEP: 62725 + | > loss: -0.34952 (-0.31271) + | > log_mle: -0.42037 (-0.40163) + | > loss_dur: 0.07085 (0.08892) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.90980 (19.25192) + | > current_lr: 0.00007 + | > step_time: 4.10540 (5.14573) + | > loader_time: 0.19770 (0.04744) + + + --> STEP: 18/234 -- GLOBAL_STEP: 62730 + | > loss: -0.31847 (-0.31975) + | > log_mle: -0.39339 (-0.40264) + | > loss_dur: 0.07492 (0.08290) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.79040 (17.40340) + | > current_lr: 0.00007 + | > step_time: 4.79640 (4.74495) + | > loader_time: 0.00190 (0.05070) + + + --> STEP: 23/234 -- GLOBAL_STEP: 62735 + | > loss: -0.35197 (-0.32337) + | > log_mle: -0.41808 (-0.40293) + | > loss_dur: 0.06610 (0.07957) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.90468 (15.89012) + | > current_lr: 0.00007 + | > step_time: 3.68490 (4.25633) + | > loader_time: 0.00100 (0.04013) + + + --> STEP: 28/234 -- GLOBAL_STEP: 62740 + | > loss: -0.37191 (-0.32459) + | > log_mle: -0.42541 (-0.40228) + | > loss_dur: 0.05350 (0.07768) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.49300 (15.19435) + | > current_lr: 0.00007 + | > step_time: 3.50260 (4.27215) + | > loader_time: 0.10190 (0.04059) + + + --> STEP: 33/234 -- GLOBAL_STEP: 62745 + | > loss: -0.32638 (-0.32265) + | > log_mle: -0.39421 (-0.40031) + | > loss_dur: 0.06783 (0.07766) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.12588 (14.63610) + | > current_lr: 0.00007 + | > step_time: 2.70780 (4.38034) + | > loader_time: 0.09330 (0.04304) + + + --> STEP: 38/234 -- GLOBAL_STEP: 62750 + | > loss: -0.29198 (-0.31920) + | > log_mle: -0.38496 (-0.39760) + | > loss_dur: 0.09298 (0.07841) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.13731 (14.47552) + | > current_lr: 0.00007 + | > step_time: 1.61050 (4.09979) + | > loader_time: 0.07390 (0.04589) + + + --> STEP: 43/234 -- GLOBAL_STEP: 62755 + | > loss: -0.27294 (-0.31528) + | > log_mle: -0.37011 (-0.39481) + | > loss_dur: 0.09717 (0.07953) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 19.30660 (14.46559) + | > current_lr: 0.00007 + | > step_time: 1.89530 (3.85744) + | > loader_time: 0.19190 (0.04535) + + + --> STEP: 48/234 -- GLOBAL_STEP: 62760 + | > loss: -0.31970 (-0.31288) + | > log_mle: -0.38549 (-0.39306) + | > loss_dur: 0.06579 (0.08018) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.76383 (14.28962) + | > current_lr: 0.00007 + | > step_time: 2.71380 (3.63901) + | > loader_time: 0.00160 (0.04083) + + + --> STEP: 53/234 -- GLOBAL_STEP: 62765 + | > loss: -0.27359 (-0.31018) + | > log_mle: -0.37562 (-0.39122) + | > loss_dur: 0.10203 (0.08104) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.33926 (13.83619) + | > current_lr: 0.00007 + | > step_time: 1.42060 (3.43760) + | > loader_time: 0.00210 (0.03742) + + + --> STEP: 58/234 -- GLOBAL_STEP: 62770 + | > loss: -0.30600 (-0.30826) + | > log_mle: -0.37855 (-0.38964) + | > loss_dur: 0.07256 (0.08138) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.22693 (13.52244) + | > current_lr: 0.00007 + | > step_time: 1.20570 (3.27079) + | > loader_time: 0.00300 (0.03582) + + + --> STEP: 63/234 -- GLOBAL_STEP: 62775 + | > loss: -0.26927 (-0.30442) + | > log_mle: -0.36603 (-0.38836) + | > loss_dur: 0.09676 (0.08394) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.46692 (13.50032) + | > current_lr: 0.00007 + | > step_time: 3.90320 (3.15601) + | > loader_time: 0.00280 (0.03321) + + + --> STEP: 68/234 -- GLOBAL_STEP: 62780 + | > loss: -0.24301 (-0.30199) + | > log_mle: -0.35703 (-0.38664) + | > loss_dur: 0.11402 (0.08465) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.26729 (13.25764) + | > current_lr: 0.00007 + | > step_time: 2.50080 (3.05645) + | > loader_time: 0.00240 (0.03235) + + + --> STEP: 73/234 -- GLOBAL_STEP: 62785 + | > loss: -0.23723 (-0.29879) + | > log_mle: -0.36935 (-0.38526) + | > loss_dur: 0.13212 (0.08647) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.92894 (13.15250) + | > current_lr: 0.00007 + | > step_time: 2.10430 (2.95296) + | > loader_time: 0.08600 (0.03257) + + + --> STEP: 78/234 -- GLOBAL_STEP: 62790 + | > loss: -0.25016 (-0.29594) + | > log_mle: -0.35545 (-0.38381) + | > loss_dur: 0.10529 (0.08787) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.08228 (13.07169) + | > current_lr: 0.00007 + | > step_time: 2.60630 (2.92376) + | > loader_time: 0.00220 (0.03065) + + + --> STEP: 83/234 -- GLOBAL_STEP: 62795 + | > loss: -0.22197 (-0.29336) + | > log_mle: -0.36562 (-0.38272) + | > loss_dur: 0.14365 (0.08936) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.16884 (13.08679) + | > current_lr: 0.00007 + | > step_time: 1.69580 (2.84318) + | > loader_time: 0.00300 (0.03097) + + + --> STEP: 88/234 -- GLOBAL_STEP: 62800 + | > loss: -0.25740 (-0.29125) + | > log_mle: -0.39928 (-0.38210) + | > loss_dur: 0.14188 (0.09086) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.99098 (13.28760) + | > current_lr: 0.00007 + | > step_time: 1.81060 (2.77115) + | > loader_time: 0.08700 (0.03031) + + + --> STEP: 93/234 -- GLOBAL_STEP: 62805 + | > loss: -0.26234 (-0.28963) + | > log_mle: -0.41005 (-0.38266) + | > loss_dur: 0.14771 (0.09303) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 23.78422 (13.69250) + | > current_lr: 0.00007 + | > step_time: 3.70160 (2.76220) + | > loader_time: 0.09820 (0.03075) + + + --> STEP: 98/234 -- GLOBAL_STEP: 62810 + | > loss: -0.24188 (-0.28851) + | > log_mle: -0.35278 (-0.38342) + | > loss_dur: 0.11091 (0.09491) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.13017 (13.99391) + | > current_lr: 0.00007 + | > step_time: 2.78370 (2.81693) + | > loader_time: 0.00270 (0.03033) + + + --> STEP: 103/234 -- GLOBAL_STEP: 62815 + | > loss: -0.27762 (-0.28724) + | > log_mle: -0.44399 (-0.38487) + | > loss_dur: 0.16637 (0.09762) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.91510 (14.71696) + | > current_lr: 0.00007 + | > step_time: 2.10050 (2.74911) + | > loader_time: 0.00320 (0.02901) + + + --> STEP: 108/234 -- GLOBAL_STEP: 62820 + | > loss: -0.25736 (-0.28612) + | > log_mle: -0.38515 (-0.38587) + | > loss_dur: 0.12779 (0.09974) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.21032 (15.10452) + | > current_lr: 0.00007 + | > step_time: 1.60970 (2.70453) + | > loader_time: 0.08480 (0.02858) + + + --> STEP: 113/234 -- GLOBAL_STEP: 62825 + | > loss: -0.26595 (-0.28469) + | > log_mle: -0.42790 (-0.38752) + | > loss_dur: 0.16195 (0.10283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.24913 (15.75362) + | > current_lr: 0.00007 + | > step_time: 0.79170 (2.71444) + | > loader_time: 0.00310 (0.02927) + + + --> STEP: 118/234 -- GLOBAL_STEP: 62830 + | > loss: -0.24168 (-0.28328) + | > log_mle: -0.40225 (-0.38863) + | > loss_dur: 0.16057 (0.10535) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.85464 (16.06976) + | > current_lr: 0.00007 + | > step_time: 1.79300 (2.68136) + | > loader_time: 0.00310 (0.02884) + + + --> STEP: 123/234 -- GLOBAL_STEP: 62835 + | > loss: -0.21965 (-0.28185) + | > log_mle: -0.36805 (-0.38887) + | > loss_dur: 0.14840 (0.10703) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.69999 (16.23352) + | > current_lr: 0.00007 + | > step_time: 2.09690 (2.63661) + | > loader_time: 0.00210 (0.02783) + + + --> STEP: 128/234 -- GLOBAL_STEP: 62840 + | > loss: -0.28233 (-0.28176) + | > log_mle: -0.42421 (-0.39105) + | > loss_dur: 0.14188 (0.10929) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.53549 (16.92150) + | > current_lr: 0.00007 + | > step_time: 3.04920 (2.63164) + | > loader_time: 0.11130 (0.02773) + + + --> STEP: 133/234 -- GLOBAL_STEP: 62845 + | > loss: -0.29493 (-0.28202) + | > log_mle: -0.46105 (-0.39361) + | > loss_dur: 0.16612 (0.11159) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.06585 (17.37212) + | > current_lr: 0.00007 + | > step_time: 3.09320 (2.60586) + | > loader_time: 0.00350 (0.02809) + + + --> STEP: 138/234 -- GLOBAL_STEP: 62850 + | > loss: -0.23665 (-0.28200) + | > log_mle: -0.40697 (-0.39604) + | > loss_dur: 0.17032 (0.11404) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.45508 (17.92736) + | > current_lr: 0.00007 + | > step_time: 2.21240 (2.59829) + | > loader_time: 0.08720 (0.02786) + + + --> STEP: 143/234 -- GLOBAL_STEP: 62855 + | > loss: -0.33842 (-0.28276) + | > log_mle: -0.55907 (-0.39941) + | > loss_dur: 0.22065 (0.11666) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 70.90339 (18.84634) + | > current_lr: 0.00007 + | > step_time: 1.04250 (2.59190) + | > loader_time: 0.00210 (0.02713) + + + --> STEP: 148/234 -- GLOBAL_STEP: 62860 + | > loss: -0.30717 (-0.28397) + | > log_mle: -0.46632 (-0.40287) + | > loss_dur: 0.15915 (0.11890) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.03437 (19.72520) + | > current_lr: 0.00007 + | > step_time: 4.99690 (2.60423) + | > loader_time: 0.08660 (0.02741) + + + --> STEP: 153/234 -- GLOBAL_STEP: 62865 + | > loss: -0.39873 (-0.28605) + | > log_mle: -0.59516 (-0.40743) + | > loss_dur: 0.19643 (0.12137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 58.55167 (20.82000) + | > current_lr: 0.00007 + | > step_time: 2.40530 (2.62445) + | > loader_time: 0.09140 (0.02955) + + + --> STEP: 158/234 -- GLOBAL_STEP: 62870 + | > loss: -0.32062 (-0.28771) + | > log_mle: -0.52780 (-0.41139) + | > loss_dur: 0.20718 (0.12368) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 56.20546 (21.69135) + | > current_lr: 0.00007 + | > step_time: 1.28340 (2.61389) + | > loader_time: 0.00320 (0.02912) + + + --> STEP: 163/234 -- GLOBAL_STEP: 62875 + | > loss: -0.31660 (-0.28950) + | > log_mle: -0.50437 (-0.41542) + | > loss_dur: 0.18777 (0.12592) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 34.29526 (22.37061) + | > current_lr: 0.00007 + | > step_time: 1.19300 (2.61944) + | > loader_time: 0.00230 (0.03006) + + + --> STEP: 168/234 -- GLOBAL_STEP: 62880 + | > loss: -0.35462 (-0.29174) + | > log_mle: -0.56417 (-0.41975) + | > loss_dur: 0.20955 (0.12801) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.80946 (23.14414) + | > current_lr: 0.00007 + | > step_time: 2.91090 (2.63373) + | > loader_time: 0.09490 (0.03028) + + + --> STEP: 173/234 -- GLOBAL_STEP: 62885 + | > loss: -0.37455 (-0.29443) + | > log_mle: -0.58107 (-0.42485) + | > loss_dur: 0.20651 (0.13042) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 47.51699 (24.13751) + | > current_lr: 0.00007 + | > step_time: 4.19720 (2.67378) + | > loader_time: 0.00330 (0.03162) + + + --> STEP: 178/234 -- GLOBAL_STEP: 62890 + | > loss: -0.40025 (-0.29695) + | > log_mle: -0.62112 (-0.42978) + | > loss_dur: 0.22087 (0.13283) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 88.56953 (25.38038) + | > current_lr: 0.00007 + | > step_time: 2.91040 (2.67972) + | > loader_time: 0.00380 (0.03170) + + + --> STEP: 183/234 -- GLOBAL_STEP: 62895 + | > loss: -0.41086 (-0.29894) + | > log_mle: -0.62531 (-0.43425) + | > loss_dur: 0.21445 (0.13531) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 76.66163 (26.45535) + | > current_lr: 0.00007 + | > step_time: 2.10570 (2.66350) + | > loader_time: 0.00530 (0.03237) + + + --> STEP: 188/234 -- GLOBAL_STEP: 62900 + | > loss: -0.42846 (-0.30116) + | > log_mle: -0.64369 (-0.43875) + | > loss_dur: 0.21523 (0.13760) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 57.34204 (27.34019) + | > current_lr: 0.00007 + | > step_time: 1.70620 (2.66048) + | > loader_time: 0.00270 (0.03248) + + + --> STEP: 193/234 -- GLOBAL_STEP: 62905 + | > loss: -0.43341 (-0.30368) + | > log_mle: -0.64788 (-0.44317) + | > loss_dur: 0.21447 (0.13949) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.65525 (28.17480) + | > current_lr: 0.00007 + | > step_time: 7.88750 (2.72306) + | > loader_time: 0.00240 (0.03176) + + + --> STEP: 198/234 -- GLOBAL_STEP: 62910 + | > loss: -0.40594 (-0.30606) + | > log_mle: -0.63218 (-0.44742) + | > loss_dur: 0.22625 (0.14136) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.37627 (29.10823) + | > current_lr: 0.00007 + | > step_time: 3.30130 (2.78660) + | > loader_time: 0.09030 (0.03195) + + + --> STEP: 203/234 -- GLOBAL_STEP: 62915 + | > loss: -0.34072 (-0.30804) + | > log_mle: -0.55527 (-0.45141) + | > loss_dur: 0.21454 (0.14337) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 60.27961 (30.21718) + | > current_lr: 0.00007 + | > step_time: 4.50280 (2.80220) + | > loader_time: 0.00500 (0.03214) + + + --> STEP: 208/234 -- GLOBAL_STEP: 62920 + | > loss: -0.41483 (-0.31087) + | > log_mle: -0.64339 (-0.45630) + | > loss_dur: 0.22856 (0.14543) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 71.50627 (31.15779) + | > current_lr: 0.00007 + | > step_time: 4.20100 (2.85371) + | > loader_time: 0.00360 (0.03280) + + + --> STEP: 213/234 -- GLOBAL_STEP: 62925 + | > loss: -0.44859 (-0.31403) + | > log_mle: -0.68148 (-0.46148) + | > loss_dur: 0.23290 (0.14745) + | > amp_scaler: 1024.00000 (2043.19249) + | > grad_norm: 0.00000 (31.91120) + | > current_lr: 0.00007 + | > step_time: 11.89200 (2.95579) + | > loader_time: 0.10740 (0.03439) + + + --> STEP: 218/234 -- GLOBAL_STEP: 62930 + | > loss: -0.41532 (-0.31649) + | > log_mle: -0.64151 (-0.46588) + | > loss_dur: 0.22620 (0.14939) + | > amp_scaler: 1024.00000 (2019.81651) + | > grad_norm: 66.61617 (33.45462) + | > current_lr: 0.00007 + | > step_time: 3.59960 (3.01238) + | > loader_time: 0.00380 (0.03500) + + + --> STEP: 223/234 -- GLOBAL_STEP: 62935 + | > loss: -0.46082 (-0.31949) + | > log_mle: -0.69031 (-0.47083) + | > loss_dur: 0.22949 (0.15134) + | > amp_scaler: 1024.00000 (1997.48879) + | > grad_norm: 91.05201 (34.68877) + | > current_lr: 0.00007 + | > step_time: 5.00410 (3.02548) + | > loader_time: 0.09910 (0.03475) + + + --> STEP: 228/234 -- GLOBAL_STEP: 62940 + | > loss: -0.40508 (-0.32222) + | > log_mle: -0.67639 (-0.47571) + | > loss_dur: 0.27131 (0.15350) + | > amp_scaler: 1024.00000 (1976.14035) + | > grad_norm: 82.01068 (35.84511) + | > current_lr: 0.00007 + | > step_time: 0.27410 (3.00395) + | > loader_time: 0.00350 (0.03488) + + + --> STEP: 233/234 -- GLOBAL_STEP: 62945 + | > loss: 0.08371 (-0.32252) + | > log_mle: -0.66517 (-0.48224) + | > loss_dur: 0.74888 (0.15972) + | > amp_scaler: 1024.00000 (1955.70815) + | > grad_norm: 95.39383 (36.95957) + | > current_lr: 0.00007 + | > step_time: 0.18460 (2.94506) + | > loader_time: 0.00250 (0.03421) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.29355 (-0.30478) + | > avg_loss: -0.31089 (+0.02553) + | > avg_log_mle: -0.55052 (+0.00729) + | > avg_loss_dur: 0.23963 (+0.01825) + + + > EPOCH: 269/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 17:55:29)  + + --> STEP: 4/234 -- GLOBAL_STEP: 62950 + | > loss: -0.29646 (-0.29316) + | > log_mle: -0.39464 (-0.39498) + | > loss_dur: 0.09818 (0.10183) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.34227 (22.11917) + | > current_lr: 0.00007 + | > step_time: 9.79910 (6.52652) + | > loader_time: 0.00180 (0.60003) + + + --> STEP: 9/234 -- GLOBAL_STEP: 62955 + | > loss: -0.30288 (-0.30847) + | > log_mle: -0.40301 (-0.39917) + | > loss_dur: 0.10013 (0.09070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.02131 (20.43667) + | > current_lr: 0.00007 + | > step_time: 3.40690 (5.83543) + | > loader_time: 0.08160 (0.30901) + + + --> STEP: 14/234 -- GLOBAL_STEP: 62960 + | > loss: -0.32957 (-0.31574) + | > log_mle: -0.40667 (-0.40146) + | > loss_dur: 0.07710 (0.08572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.65257 (18.97851) + | > current_lr: 0.00007 + | > step_time: 3.69620 (4.85090) + | > loader_time: 0.00170 (0.23209) + + + --> STEP: 19/234 -- GLOBAL_STEP: 62965 + | > loss: -0.34419 (-0.32191) + | > log_mle: -0.40725 (-0.40251) + | > loss_dur: 0.06306 (0.08060) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.37022 (16.85884) + | > current_lr: 0.00007 + | > step_time: 2.19790 (4.71231) + | > loader_time: 0.00510 (0.18611) + + + --> STEP: 24/234 -- GLOBAL_STEP: 62970 + | > loss: -0.34169 (-0.32483) + | > log_mle: -0.40056 (-0.40276) + | > loss_dur: 0.05888 (0.07793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.05182 (15.44730) + | > current_lr: 0.00007 + | > step_time: 4.69800 (4.51722) + | > loader_time: 0.10930 (0.15231) + + + --> STEP: 29/234 -- GLOBAL_STEP: 62975 + | > loss: -0.31331 (-0.32537) + | > log_mle: -0.38911 (-0.40158) + | > loss_dur: 0.07580 (0.07621) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.93582 (14.79597) + | > current_lr: 0.00007 + | > step_time: 4.00610 (4.36586) + | > loader_time: 0.10570 (0.14565) + + + --> STEP: 34/234 -- GLOBAL_STEP: 62980 + | > loss: -0.30973 (-0.32322) + | > log_mle: -0.38834 (-0.39971) + | > loss_dur: 0.07862 (0.07650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.23317 (14.04573) + | > current_lr: 0.00007 + | > step_time: 4.50910 (4.24139) + | > loader_time: 0.28410 (0.14194) + + + --> STEP: 39/234 -- GLOBAL_STEP: 62985 + | > loss: -0.27565 (-0.31825) + | > log_mle: -0.37674 (-0.39672) + | > loss_dur: 0.10109 (0.07847) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.28826 (14.18805) + | > current_lr: 0.00007 + | > step_time: 4.90860 (4.12358) + | > loader_time: 0.00240 (0.13070) + + + --> STEP: 44/234 -- GLOBAL_STEP: 62990 + | > loss: -0.29764 (-0.31500) + | > log_mle: -0.37081 (-0.39428) + | > loss_dur: 0.07316 (0.07928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.10065 (13.77425) + | > current_lr: 0.00007 + | > step_time: 1.98360 (3.92529) + | > loader_time: 0.00280 (0.12901) + + + --> STEP: 49/234 -- GLOBAL_STEP: 62995 + | > loss: -0.30683 (-0.31325) + | > log_mle: -0.38686 (-0.39341) + | > loss_dur: 0.08003 (0.08015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.39207 (13.52114) + | > current_lr: 0.00007 + | > step_time: 1.52500 (3.72626) + | > loader_time: 0.00150 (0.11795) + + + --> STEP: 54/234 -- GLOBAL_STEP: 63000 + | > loss: -0.29868 (-0.31098) + | > log_mle: -0.37842 (-0.39174) + | > loss_dur: 0.07974 (0.08076) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.82428 (13.06247) + | > current_lr: 0.00007 + | > step_time: 1.19800 (3.52348) + | > loader_time: 0.10020 (0.10906) + + + --> STEP: 59/234 -- GLOBAL_STEP: 63005 + | > loss: -0.28972 (-0.30913) + | > log_mle: -0.37747 (-0.39025) + | > loss_dur: 0.08775 (0.08112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.55857 (12.83888) + | > current_lr: 0.00007 + | > step_time: 2.60810 (3.42110) + | > loader_time: 0.00960 (0.10011) + + + --> STEP: 64/234 -- GLOBAL_STEP: 63010 + | > loss: -0.27794 (-0.30560) + | > log_mle: -0.36621 (-0.38885) + | > loss_dur: 0.08826 (0.08324) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.04988 (12.96907) + | > current_lr: 0.00007 + | > step_time: 2.20830 (3.27845) + | > loader_time: 0.08510 (0.09381) + + + --> STEP: 69/234 -- GLOBAL_STEP: 63015 + | > loss: -0.27438 (-0.30308) + | > log_mle: -0.36027 (-0.38711) + | > loss_dur: 0.08589 (0.08402) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.52783 (12.79122) + | > current_lr: 0.00007 + | > step_time: 2.29070 (3.16834) + | > loader_time: 0.00210 (0.08974) + + + --> STEP: 74/234 -- GLOBAL_STEP: 63020 + | > loss: -0.25135 (-0.29990) + | > log_mle: -0.35794 (-0.38579) + | > loss_dur: 0.10659 (0.08590) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.65703 (12.80882) + | > current_lr: 0.00007 + | > step_time: 2.37070 (3.08505) + | > loader_time: 0.00260 (0.08388) + + + --> STEP: 79/234 -- GLOBAL_STEP: 63025 + | > loss: -0.26657 (-0.29732) + | > log_mle: -0.37081 (-0.38454) + | > loss_dur: 0.10424 (0.08722) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.97404 (12.72516) + | > current_lr: 0.00007 + | > step_time: 1.66940 (3.01358) + | > loader_time: 0.00210 (0.07872) + + + --> STEP: 84/234 -- GLOBAL_STEP: 63030 + | > loss: -0.27168 (-0.29493) + | > log_mle: -0.36608 (-0.38350) + | > loss_dur: 0.09440 (0.08856) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.64419 (12.64026) + | > current_lr: 0.00007 + | > step_time: 2.36560 (2.95344) + | > loader_time: 0.00180 (0.07517) + + + --> STEP: 89/234 -- GLOBAL_STEP: 63035 + | > loss: -0.26415 (-0.29280) + | > log_mle: -0.39189 (-0.38324) + | > loss_dur: 0.12774 (0.09044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.43429 (12.81010) + | > current_lr: 0.00007 + | > step_time: 1.79870 (2.89773) + | > loader_time: 0.00370 (0.07111) + + + --> STEP: 94/234 -- GLOBAL_STEP: 63040 + | > loss: -0.27790 (-0.29116) + | > log_mle: -0.41445 (-0.38409) + | > loss_dur: 0.13655 (0.09293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.68402 (13.22674) + | > current_lr: 0.00007 + | > step_time: 3.79650 (2.90202) + | > loader_time: 0.09770 (0.06849) + + + --> STEP: 99/234 -- GLOBAL_STEP: 63045 + | > loss: -0.28634 (-0.28994) + | > log_mle: -0.44018 (-0.38492) + | > loss_dur: 0.15384 (0.09498) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.86005 (13.67033) + | > current_lr: 0.00007 + | > step_time: 1.21670 (2.85659) + | > loader_time: 0.08270 (0.06681) + + + --> STEP: 104/234 -- GLOBAL_STEP: 63050 + | > loss: -0.30105 (-0.28871) + | > log_mle: -0.45628 (-0.38632) + | > loss_dur: 0.15523 (0.09762) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.31624 (14.11021) + | > current_lr: 0.00007 + | > step_time: 1.38840 (2.79966) + | > loader_time: 0.00270 (0.06542) + + + --> STEP: 109/234 -- GLOBAL_STEP: 63055 + | > loss: -0.23572 (-0.28689) + | > log_mle: -0.42119 (-0.38700) + | > loss_dur: 0.18547 (0.10011) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.66435 (14.62250) + | > current_lr: 0.00007 + | > step_time: 1.98100 (2.76112) + | > loader_time: 0.00140 (0.06254) + + + --> STEP: 114/234 -- GLOBAL_STEP: 63060 + | > loss: -0.26545 (-0.28612) + | > log_mle: -0.41010 (-0.38865) + | > loss_dur: 0.14465 (0.10253) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.48110 (15.12370) + | > current_lr: 0.00007 + | > step_time: 3.30470 (2.77267) + | > loader_time: 0.00270 (0.06295) + + + --> STEP: 119/234 -- GLOBAL_STEP: 63065 + | > loss: -0.25925 (-0.28453) + | > log_mle: -0.40738 (-0.38973) + | > loss_dur: 0.14814 (0.10519) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.13171 (15.52028) + | > current_lr: 0.00007 + | > step_time: 1.09780 (2.80168) + | > loader_time: 0.00300 (0.06452) + + + --> STEP: 124/234 -- GLOBAL_STEP: 63070 + | > loss: -0.27743 (-0.28307) + | > log_mle: -0.43575 (-0.39017) + | > loss_dur: 0.15832 (0.10710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.04430 (15.72541) + | > current_lr: 0.00007 + | > step_time: 2.39230 (2.75814) + | > loader_time: 0.00250 (0.06332) + + + --> STEP: 129/234 -- GLOBAL_STEP: 63075 + | > loss: -0.24247 (-0.28246) + | > log_mle: -0.42363 (-0.39224) + | > loss_dur: 0.18116 (0.10978) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.45020 (16.62251) + | > current_lr: 0.00007 + | > step_time: 3.40080 (2.73750) + | > loader_time: 0.00240 (0.06379) + + + --> STEP: 134/234 -- GLOBAL_STEP: 63080 + | > loss: -0.29896 (-0.28298) + | > log_mle: -0.48162 (-0.39518) + | > loss_dur: 0.18266 (0.11220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.67600 (17.28588) + | > current_lr: 0.00007 + | > step_time: 3.92180 (2.72525) + | > loader_time: 0.09580 (0.06349) + + + --> STEP: 139/234 -- GLOBAL_STEP: 63085 + | > loss: -0.34563 (-0.28335) + | > log_mle: -0.53795 (-0.39789) + | > loss_dur: 0.19232 (0.11455) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.11285 (18.02093) + | > current_lr: 0.00007 + | > step_time: 1.68860 (2.71268) + | > loader_time: 0.01430 (0.06142) + + + --> STEP: 144/234 -- GLOBAL_STEP: 63090 + | > loss: -0.31959 (-0.28371) + | > log_mle: -0.51217 (-0.40077) + | > loss_dur: 0.19258 (0.11705) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.70749 (18.73094) + | > current_lr: 0.00007 + | > step_time: 3.50970 (2.70897) + | > loader_time: 0.09820 (0.06188) + + + --> STEP: 149/234 -- GLOBAL_STEP: 63095 + | > loss: -0.37324 (-0.28504) + | > log_mle: -0.56412 (-0.40426) + | > loss_dur: 0.19087 (0.11922) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.54721 (19.57669) + | > current_lr: 0.00007 + | > step_time: 2.70090 (2.72475) + | > loader_time: 0.09840 (0.06179) + + + --> STEP: 154/234 -- GLOBAL_STEP: 63100 + | > loss: -0.34310 (-0.28690) + | > log_mle: -0.52709 (-0.40852) + | > loss_dur: 0.18400 (0.12161) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.32028 (20.47965) + | > current_lr: 0.00007 + | > step_time: 0.98630 (2.74658) + | > loader_time: 0.09920 (0.06359) + + + --> STEP: 159/234 -- GLOBAL_STEP: 63105 + | > loss: -0.34148 (-0.28853) + | > log_mle: -0.54062 (-0.41252) + | > loss_dur: 0.19914 (0.12399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.36959 (21.32981) + | > current_lr: 0.00007 + | > step_time: 2.50330 (2.73013) + | > loader_time: 0.00400 (0.06173) + + + --> STEP: 164/234 -- GLOBAL_STEP: 63110 + | > loss: -0.31710 (-0.29008) + | > log_mle: -0.53070 (-0.41624) + | > loss_dur: 0.21360 (0.12616) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.95877 (22.16909) + | > current_lr: 0.00007 + | > step_time: 6.69690 (2.81777) + | > loader_time: 0.11110 (0.06287) + + + --> STEP: 169/234 -- GLOBAL_STEP: 63115 + | > loss: -0.34752 (-0.29234) + | > log_mle: -0.54257 (-0.42052) + | > loss_dur: 0.19505 (0.12818) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.29400 (22.87735) + | > current_lr: 0.00007 + | > step_time: 0.60010 (2.89343) + | > loader_time: 0.00300 (0.06338) + + + --> STEP: 174/234 -- GLOBAL_STEP: 63120 + | > loss: -0.43202 (-0.29543) + | > log_mle: -0.63770 (-0.42595) + | > loss_dur: 0.20567 (0.13052) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.32986 (23.90466) + | > current_lr: 0.00007 + | > step_time: 1.70990 (2.89941) + | > loader_time: 0.08150 (0.06306) + + + --> STEP: 179/234 -- GLOBAL_STEP: 63125 + | > loss: -0.38079 (-0.29746) + | > log_mle: -0.62255 (-0.43056) + | > loss_dur: 0.24177 (0.13309) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.44786 (24.96996) + | > current_lr: 0.00007 + | > step_time: 2.51860 (2.93743) + | > loader_time: 0.08730 (0.06348) + + + --> STEP: 184/234 -- GLOBAL_STEP: 63130 + | > loss: -0.36182 (-0.29933) + | > log_mle: -0.58251 (-0.43470) + | > loss_dur: 0.22070 (0.13537) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.26842 (25.98662) + | > current_lr: 0.00007 + | > step_time: 5.10090 (2.97828) + | > loader_time: 0.00630 (0.06398) + + + --> STEP: 189/234 -- GLOBAL_STEP: 63135 + | > loss: -0.36211 (-0.30139) + | > log_mle: -0.57495 (-0.43900) + | > loss_dur: 0.21284 (0.13761) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.51823 (27.12313) + | > current_lr: 0.00007 + | > step_time: 5.70350 (3.06255) + | > loader_time: 0.09520 (0.06384) + + + --> STEP: 194/234 -- GLOBAL_STEP: 63140 + | > loss: -0.41189 (-0.30408) + | > log_mle: -0.62134 (-0.44353) + | > loss_dur: 0.20944 (0.13945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.96341 (27.88568) + | > current_lr: 0.00007 + | > step_time: 1.19750 (3.05877) + | > loader_time: 0.00410 (0.06524) + + + --> STEP: 199/234 -- GLOBAL_STEP: 63145 + | > loss: -0.41381 (-0.30635) + | > log_mle: -0.63214 (-0.44771) + | > loss_dur: 0.21833 (0.14136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.36084 (28.91166) + | > current_lr: 0.00007 + | > step_time: 2.29890 (3.05376) + | > loader_time: 0.09470 (0.06492) + + + --> STEP: 204/234 -- GLOBAL_STEP: 63150 + | > loss: -0.43606 (-0.30840) + | > log_mle: -0.67589 (-0.45187) + | > loss_dur: 0.23983 (0.14347) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.63252 (29.76212) + | > current_lr: 0.00007 + | > step_time: 6.48940 (3.06512) + | > loader_time: 0.00270 (0.06345) + + + --> STEP: 209/234 -- GLOBAL_STEP: 63155 + | > loss: -0.40908 (-0.31102) + | > log_mle: -0.62663 (-0.45651) + | > loss_dur: 0.21755 (0.14548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.10584 (30.80694) + | > current_lr: 0.00007 + | > step_time: 2.69910 (3.10615) + | > loader_time: 0.00430 (0.06247) + + + --> STEP: 214/234 -- GLOBAL_STEP: 63160 + | > loss: -0.44520 (-0.31428) + | > log_mle: -0.65747 (-0.46181) + | > loss_dur: 0.21227 (0.14753) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.40269 (31.98787) + | > current_lr: 0.00007 + | > step_time: 7.81060 (3.20805) + | > loader_time: 0.09560 (0.06418) + + + --> STEP: 219/234 -- GLOBAL_STEP: 63165 + | > loss: -0.54119 (-0.31766) + | > log_mle: -0.77249 (-0.46718) + | > loss_dur: 0.23130 (0.14952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.37663 (33.02278) + | > current_lr: 0.00007 + | > step_time: 2.19370 (3.26913) + | > loader_time: 0.00840 (0.06498) + + + --> STEP: 224/234 -- GLOBAL_STEP: 63170 + | > loss: -0.48532 (-0.32079) + | > log_mle: -0.72248 (-0.47229) + | > loss_dur: 0.23716 (0.15149) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.69666 (34.00900) + | > current_lr: 0.00007 + | > step_time: 0.22960 (3.22169) + | > loader_time: 0.00370 (0.06439) + + + --> STEP: 229/234 -- GLOBAL_STEP: 63175 + | > loss: -0.45179 (-0.32374) + | > log_mle: -0.74406 (-0.47762) + | > loss_dur: 0.29227 (0.15387) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 127.08067 (35.62288) + | > current_lr: 0.00007 + | > step_time: 0.24380 (3.15660) + | > loader_time: 0.00340 (0.06306) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.49404 (+0.20049) + | > avg_loss: -0.32903 (-0.01814) + | > avg_log_mle: -0.55465 (-0.00413) + | > avg_loss_dur: 0.22562 (-0.01401) + + + > EPOCH: 270/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 18:08:43)  + + --> STEP: 0/234 -- GLOBAL_STEP: 63180 + | > loss: -0.31334 (-0.31334) + | > log_mle: -0.48231 (-0.48231) + | > loss_dur: 0.16897 (0.16897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.13378 (26.13378) + | > current_lr: 0.00007 + | > step_time: 0.98300 (0.98298) + | > loader_time: 6.74450 (6.74454) + + + --> STEP: 5/234 -- GLOBAL_STEP: 63185 + | > loss: -0.30487 (-0.29433) + | > log_mle: -0.40217 (-0.39858) + | > loss_dur: 0.09729 (0.10426) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.41703 (20.00076) + | > current_lr: 0.00007 + | > step_time: 1.05890 (5.61644) + | > loader_time: 0.00120 (4.11898) + + + --> STEP: 10/234 -- GLOBAL_STEP: 63190 + | > loss: -0.31467 (-0.30834) + | > log_mle: -0.39951 (-0.40166) + | > loss_dur: 0.08484 (0.09333) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.81468 (18.33382) + | > current_lr: 0.00007 + | > step_time: 1.30520 (3.74985) + | > loader_time: 0.00160 (2.07666) + + + --> STEP: 15/234 -- GLOBAL_STEP: 63195 + | > loss: -0.33538 (-0.31766) + | > log_mle: -0.40897 (-0.40424) + | > loss_dur: 0.07359 (0.08658) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.54785 (17.19111) + | > current_lr: 0.00007 + | > step_time: 3.30160 (3.47792) + | > loader_time: 0.00310 (1.39081) + + + --> STEP: 20/234 -- GLOBAL_STEP: 63200 + | > loss: -0.34834 (-0.32397) + | > log_mle: -0.40972 (-0.40483) + | > loss_dur: 0.06138 (0.08086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63685 (15.79875) + | > current_lr: 0.00007 + | > step_time: 4.20940 (3.62774) + | > loader_time: 0.00150 (1.05395) + + + --> STEP: 25/234 -- GLOBAL_STEP: 63205 + | > loss: -0.32392 (-0.32592) + | > log_mle: -0.39141 (-0.40416) + | > loss_dur: 0.06750 (0.07823) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.02501 (14.98043) + | > current_lr: 0.00007 + | > step_time: 5.30590 (3.55078) + | > loader_time: 0.09370 (0.85470) + + + --> STEP: 30/234 -- GLOBAL_STEP: 63210 + | > loss: -0.31445 (-0.32680) + | > log_mle: -0.39244 (-0.40323) + | > loss_dur: 0.07799 (0.07644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.03741 (14.31286) + | > current_lr: 0.00007 + | > step_time: 3.59840 (3.75519) + | > loader_time: 0.00180 (0.72225) + + + --> STEP: 35/234 -- GLOBAL_STEP: 63215 + | > loss: -0.28061 (-0.32418) + | > log_mle: -0.37869 (-0.40127) + | > loss_dur: 0.09808 (0.07709) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.78321 (13.97387) + | > current_lr: 0.00007 + | > step_time: 3.09560 (3.87501) + | > loader_time: 0.10030 (0.63273) + + + --> STEP: 40/234 -- GLOBAL_STEP: 63220 + | > loss: -0.28628 (-0.32040) + | > log_mle: -0.37800 (-0.39866) + | > loss_dur: 0.09172 (0.07825) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.38040 (13.59918) + | > current_lr: 0.00007 + | > step_time: 1.11730 (3.60419) + | > loader_time: 0.00300 (0.55660) + + + --> STEP: 45/234 -- GLOBAL_STEP: 63225 + | > loss: -0.28083 (-0.31776) + | > log_mle: -0.39517 (-0.39678) + | > loss_dur: 0.11434 (0.07902) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.68192 (13.34429) + | > current_lr: 0.00007 + | > step_time: 2.61880 (3.39003) + | > loader_time: 0.00250 (0.49498) + + + --> STEP: 50/234 -- GLOBAL_STEP: 63230 + | > loss: -0.30361 (-0.31642) + | > log_mle: -0.37893 (-0.39560) + | > loss_dur: 0.07532 (0.07917) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.84273 (12.86173) + | > current_lr: 0.00007 + | > step_time: 1.83880 (3.23453) + | > loader_time: 0.00150 (0.44579) + + + --> STEP: 55/234 -- GLOBAL_STEP: 63235 + | > loss: -0.30775 (-0.31466) + | > log_mle: -0.38483 (-0.39434) + | > loss_dur: 0.07708 (0.07968) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.46486 (12.59101) + | > current_lr: 0.00007 + | > step_time: 3.02600 (3.11371) + | > loader_time: 0.08540 (0.40830) + + + --> STEP: 60/234 -- GLOBAL_STEP: 63240 + | > loss: -0.26939 (-0.31212) + | > log_mle: -0.38577 (-0.39310) + | > loss_dur: 0.11638 (0.08098) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.06462 (12.45406) + | > current_lr: 0.00007 + | > step_time: 2.51260 (3.05103) + | > loader_time: 0.10080 (0.37763) + + + --> STEP: 65/234 -- GLOBAL_STEP: 63245 + | > loss: -0.27798 (-0.30873) + | > log_mle: -0.37263 (-0.39169) + | > loss_dur: 0.09465 (0.08296) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.36899 (12.35724) + | > current_lr: 0.00007 + | > step_time: 0.60420 (2.94518) + | > loader_time: 0.00220 (0.35018) + + + --> STEP: 70/234 -- GLOBAL_STEP: 63250 + | > loss: -0.24329 (-0.30555) + | > log_mle: -0.35860 (-0.38985) + | > loss_dur: 0.11531 (0.08429) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.68199 (12.27950) + | > current_lr: 0.00007 + | > step_time: 1.20000 (2.90530) + | > loader_time: 0.00690 (0.32540) + + + --> STEP: 75/234 -- GLOBAL_STEP: 63255 + | > loss: -0.25382 (-0.30253) + | > log_mle: -0.36794 (-0.38866) + | > loss_dur: 0.11412 (0.08614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.38788 (12.40593) + | > current_lr: 0.00007 + | > step_time: 1.70710 (2.86646) + | > loader_time: 0.00450 (0.30634) + + + --> STEP: 80/234 -- GLOBAL_STEP: 63260 + | > loss: -0.27005 (-0.30038) + | > log_mle: -0.36002 (-0.38728) + | > loss_dur: 0.08997 (0.08690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.36170 (12.35925) + | > current_lr: 0.00007 + | > step_time: 1.29040 (2.81098) + | > loader_time: 0.00270 (0.28827) + + + --> STEP: 85/234 -- GLOBAL_STEP: 63265 + | > loss: -0.26061 (-0.29776) + | > log_mle: -0.36152 (-0.38622) + | > loss_dur: 0.10090 (0.08846) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.22248 (12.39132) + | > current_lr: 0.00007 + | > step_time: 1.58320 (2.74773) + | > loader_time: 0.00250 (0.27151) + + + --> STEP: 90/234 -- GLOBAL_STEP: 63270 + | > loss: -0.25437 (-0.29565) + | > log_mle: -0.38470 (-0.38610) + | > loss_dur: 0.13033 (0.09045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.30187 (12.60483) + | > current_lr: 0.00007 + | > step_time: 1.70770 (2.66923) + | > loader_time: 0.07610 (0.25743) + + + --> STEP: 95/234 -- GLOBAL_STEP: 63275 + | > loss: -0.30789 (-0.29477) + | > log_mle: -0.46561 (-0.38779) + | > loss_dur: 0.15772 (0.09302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.78909 (13.02269) + | > current_lr: 0.00007 + | > step_time: 2.49920 (2.67491) + | > loader_time: 0.00370 (0.24590) + + + --> STEP: 100/234 -- GLOBAL_STEP: 63280 + | > loss: -0.27138 (-0.29324) + | > log_mle: -0.39275 (-0.38796) + | > loss_dur: 0.12138 (0.09472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.81495 (13.29192) + | > current_lr: 0.00007 + | > step_time: 5.01240 (2.66840) + | > loader_time: 0.19550 (0.23729) + + + --> STEP: 105/234 -- GLOBAL_STEP: 63285 + | > loss: -0.25203 (-0.29188) + | > log_mle: -0.37366 (-0.38921) + | > loss_dur: 0.12163 (0.09733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.54153 (13.84497) + | > current_lr: 0.00007 + | > step_time: 1.82060 (2.62680) + | > loader_time: 0.00280 (0.22611) + + + --> STEP: 110/234 -- GLOBAL_STEP: 63290 + | > loss: -0.25637 (-0.28998) + | > log_mle: -0.39563 (-0.39007) + | > loss_dur: 0.13926 (0.10009) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.43662 (14.26634) + | > current_lr: 0.00007 + | > step_time: 2.90210 (2.59820) + | > loader_time: 0.00290 (0.21676) + + + --> STEP: 115/234 -- GLOBAL_STEP: 63295 + | > loss: -0.24332 (-0.28897) + | > log_mle: -0.41250 (-0.39183) + | > loss_dur: 0.16919 (0.10286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.96749 (15.00300) + | > current_lr: 0.00007 + | > step_time: 2.35810 (2.58973) + | > loader_time: 0.08520 (0.20898) + + + --> STEP: 120/234 -- GLOBAL_STEP: 63300 + | > loss: -0.30311 (-0.28791) + | > log_mle: -0.46409 (-0.39330) + | > loss_dur: 0.16098 (0.10539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.15230 (15.51560) + | > current_lr: 0.00007 + | > step_time: 1.49210 (2.56541) + | > loader_time: 0.00210 (0.20118) + + + --> STEP: 125/234 -- GLOBAL_STEP: 63305 + | > loss: -0.27363 (-0.28647) + | > log_mle: -0.44479 (-0.39357) + | > loss_dur: 0.17117 (0.10710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.36723 (15.91108) + | > current_lr: 0.00007 + | > step_time: 3.51220 (2.56494) + | > loader_time: 0.00330 (0.19323) + + + --> STEP: 130/234 -- GLOBAL_STEP: 63310 + | > loss: -0.28496 (-0.28620) + | > log_mle: -0.46330 (-0.39583) + | > loss_dur: 0.17834 (0.10963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.23319 (16.53227) + | > current_lr: 0.00007 + | > step_time: 2.36200 (2.54304) + | > loader_time: 0.09300 (0.18726) + + + --> STEP: 135/234 -- GLOBAL_STEP: 63315 + | > loss: -0.25411 (-0.28623) + | > log_mle: -0.38929 (-0.39806) + | > loss_dur: 0.13518 (0.11183) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.95882 (17.22042) + | > current_lr: 0.00007 + | > step_time: 2.00500 (2.54686) + | > loader_time: 0.08530 (0.18104) + + + --> STEP: 140/234 -- GLOBAL_STEP: 63320 + | > loss: -0.23942 (-0.28635) + | > log_mle: -0.41787 (-0.40086) + | > loss_dur: 0.17845 (0.11451) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.68463 (18.14654) + | > current_lr: 0.00007 + | > step_time: 1.31020 (2.52172) + | > loader_time: 0.08560 (0.17651) + + + --> STEP: 145/234 -- GLOBAL_STEP: 63325 + | > loss: -0.33515 (-0.28713) + | > log_mle: -0.50999 (-0.40421) + | > loss_dur: 0.17484 (0.11708) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.58367 (19.03312) + | > current_lr: 0.00007 + | > step_time: 2.69280 (2.54236) + | > loader_time: 0.00760 (0.17062) + + + --> STEP: 150/234 -- GLOBAL_STEP: 63330 + | > loss: -0.30515 (-0.28809) + | > log_mle: -0.50551 (-0.40745) + | > loss_dur: 0.20036 (0.11936) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.29663 (19.56174) + | > current_lr: 0.00007 + | > step_time: 3.20890 (2.53433) + | > loader_time: 0.00310 (0.16607) + + + --> STEP: 155/234 -- GLOBAL_STEP: 63335 + | > loss: -0.36288 (-0.29020) + | > log_mle: -0.56923 (-0.41194) + | > loss_dur: 0.20635 (0.12174) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.63833 (20.59449) + | > current_lr: 0.00007 + | > step_time: 2.29430 (2.51587) + | > loader_time: 0.00390 (0.16175) + + + --> STEP: 160/234 -- GLOBAL_STEP: 63340 + | > loss: -0.36614 (-0.29165) + | > log_mle: -0.57207 (-0.41574) + | > loss_dur: 0.20594 (0.12409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.51318 (21.60005) + | > current_lr: 0.00007 + | > step_time: 5.10630 (2.53936) + | > loader_time: 0.09230 (0.15784) + + + --> STEP: 165/234 -- GLOBAL_STEP: 63345 + | > loss: -0.38082 (-0.29343) + | > log_mle: -0.57682 (-0.41957) + | > loss_dur: 0.19600 (0.12614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.86738 (22.55470) + | > current_lr: 0.00007 + | > step_time: 2.69800 (2.54363) + | > loader_time: 0.10050 (0.15427) + + + --> STEP: 170/234 -- GLOBAL_STEP: 63350 + | > loss: -0.39154 (-0.29557) + | > log_mle: -0.61300 (-0.42393) + | > loss_dur: 0.22145 (0.12836) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.31071 (23.67908) + | > current_lr: 0.00007 + | > step_time: 1.81610 (2.57299) + | > loader_time: 0.08640 (0.15254) + + + --> STEP: 175/234 -- GLOBAL_STEP: 63355 + | > loss: -0.35108 (-0.29842) + | > log_mle: -0.58212 (-0.42918) + | > loss_dur: 0.23104 (0.13075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.97530 (24.68722) + | > current_lr: 0.00007 + | > step_time: 1.39060 (2.58801) + | > loader_time: 0.00380 (0.14978) + + + --> STEP: 180/234 -- GLOBAL_STEP: 63360 + | > loss: -0.38574 (-0.30079) + | > log_mle: -0.59390 (-0.43396) + | > loss_dur: 0.20816 (0.13317) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.55326 (25.70842) + | > current_lr: 0.00007 + | > step_time: 4.29120 (2.63068) + | > loader_time: 0.31140 (0.14842) + + + --> STEP: 185/234 -- GLOBAL_STEP: 63365 + | > loss: -0.41603 (-0.30295) + | > log_mle: -0.63624 (-0.43844) + | > loss_dur: 0.22021 (0.13549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.36226 (26.77905) + | > current_lr: 0.00007 + | > step_time: 5.60470 (2.68463) + | > loader_time: 0.00630 (0.14597) + + + --> STEP: 190/234 -- GLOBAL_STEP: 63370 + | > loss: -0.38317 (-0.30494) + | > log_mle: -0.59020 (-0.44250) + | > loss_dur: 0.20704 (0.13756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.72523 (27.89820) + | > current_lr: 0.00007 + | > step_time: 3.30080 (2.74345) + | > loader_time: 0.08320 (0.14412) + + + --> STEP: 195/234 -- GLOBAL_STEP: 63375 + | > loss: -0.38682 (-0.30749) + | > log_mle: -0.61597 (-0.44699) + | > loss_dur: 0.22915 (0.13950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.19136 (28.81611) + | > current_lr: 0.00007 + | > step_time: 5.19790 (2.82442) + | > loader_time: 0.09400 (0.14284) + + + --> STEP: 200/234 -- GLOBAL_STEP: 63380 + | > loss: -0.37016 (-0.30973) + | > log_mle: -0.61411 (-0.45120) + | > loss_dur: 0.24394 (0.14147) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.12337 (29.91203) + | > current_lr: 0.00007 + | > step_time: 1.28990 (2.79926) + | > loader_time: 0.00260 (0.13976) + + + --> STEP: 205/234 -- GLOBAL_STEP: 63385 + | > loss: -0.38371 (-0.31162) + | > log_mle: -0.60281 (-0.45500) + | > loss_dur: 0.21910 (0.14338) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.51512 (30.81990) + | > current_lr: 0.00007 + | > step_time: 1.68470 (2.81972) + | > loader_time: 0.39630 (0.14062) + + + --> STEP: 210/234 -- GLOBAL_STEP: 63390 + | > loss: -0.46027 (-0.31437) + | > log_mle: -0.69230 (-0.45977) + | > loss_dur: 0.23203 (0.14539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.39622 (31.70089) + | > current_lr: 0.00007 + | > step_time: 7.99990 (2.86362) + | > loader_time: 0.29220 (0.13958) + + + --> STEP: 215/234 -- GLOBAL_STEP: 63395 + | > loss: -0.40689 (-0.31739) + | > log_mle: -0.64363 (-0.46482) + | > loss_dur: 0.23674 (0.14743) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.59495 (32.60386) + | > current_lr: 0.00007 + | > step_time: 3.80930 (2.95694) + | > loader_time: 0.00500 (0.13972) + + + --> STEP: 220/234 -- GLOBAL_STEP: 63400 + | > loss: -0.46906 (-0.32098) + | > log_mle: -0.70333 (-0.47038) + | > loss_dur: 0.23427 (0.14940) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.22045 (33.58241) + | > current_lr: 0.00007 + | > step_time: 5.39240 (2.98647) + | > loader_time: 0.00490 (0.13833) + + + --> STEP: 225/234 -- GLOBAL_STEP: 63405 + | > loss: -0.52383 (-0.32426) + | > log_mle: -0.76917 (-0.47560) + | > loss_dur: 0.24534 (0.15133) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.14246 (34.74080) + | > current_lr: 0.00007 + | > step_time: 2.81310 (2.98728) + | > loader_time: 0.08840 (0.13605) + + + --> STEP: 230/234 -- GLOBAL_STEP: 63410 + | > loss: -0.51319 (-0.32724) + | > log_mle: -0.82140 (-0.48132) + | > loss_dur: 0.30820 (0.15407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.69082 (36.01318) + | > current_lr: 0.00007 + | > step_time: 0.25140 (2.93676) + | > loader_time: 0.00360 (0.13317) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.77763 (+0.28359) + | > avg_loss: -0.34287 (-0.01385) + | > avg_log_mle: -0.56616 (-0.01151) + | > avg_loss_dur: 0.22329 (-0.00233) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_63414.pth + + > EPOCH: 271/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 18:21:24)  + + --> STEP: 1/234 -- GLOBAL_STEP: 63415 + | > loss: -0.29970 (-0.29970) + | > log_mle: -0.39000 (-0.39000) + | > loss_dur: 0.09030 (0.09030) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.98368 (28.98368) + | > current_lr: 0.00007 + | > step_time: 1.90930 (1.90930) + | > loader_time: 0.00170 (0.00174) + + + --> STEP: 6/234 -- GLOBAL_STEP: 63420 + | > loss: -0.33783 (-0.30631) + | > log_mle: -0.39781 (-0.39771) + | > loss_dur: 0.05997 (0.09140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.46175 (21.38168) + | > current_lr: 0.00007 + | > step_time: 2.49540 (2.40167) + | > loader_time: 0.00600 (1.63228) + + + --> STEP: 11/234 -- GLOBAL_STEP: 63425 + | > loss: -0.35400 (-0.31570) + | > log_mle: -0.41490 (-0.40308) + | > loss_dur: 0.06090 (0.08738) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.59600 (18.48119) + | > current_lr: 0.00007 + | > step_time: 8.90630 (4.27559) + | > loader_time: 0.19010 (0.93447) + + + --> STEP: 16/234 -- GLOBAL_STEP: 63430 + | > loss: -0.35890 (-0.32416) + | > log_mle: -0.41915 (-0.40652) + | > loss_dur: 0.06025 (0.08236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.45252 (16.67012) + | > current_lr: 0.00007 + | > step_time: 5.21480 (4.44752) + | > loader_time: 0.08810 (0.66572) + + + --> STEP: 21/234 -- GLOBAL_STEP: 63435 + | > loss: -0.32137 (-0.32703) + | > log_mle: -0.39396 (-0.40575) + | > loss_dur: 0.07258 (0.07873) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45524 (15.45079) + | > current_lr: 0.00007 + | > step_time: 0.59000 (3.98547) + | > loader_time: 0.00120 (0.52014) + + + --> STEP: 26/234 -- GLOBAL_STEP: 63440 + | > loss: -0.31754 (-0.32784) + | > log_mle: -0.39223 (-0.40472) + | > loss_dur: 0.07468 (0.07687) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.37798 (14.82616) + | > current_lr: 0.00007 + | > step_time: 7.38750 (3.83898) + | > loader_time: 0.39690 (0.44158) + + + --> STEP: 31/234 -- GLOBAL_STEP: 63445 + | > loss: -0.28855 (-0.32703) + | > log_mle: -0.38218 (-0.40353) + | > loss_dur: 0.09364 (0.07650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.84420 (14.17365) + | > current_lr: 0.00007 + | > step_time: 2.90510 (3.77475) + | > loader_time: 0.00390 (0.37411) + + + --> STEP: 36/234 -- GLOBAL_STEP: 63450 + | > loss: -0.29967 (-0.32436) + | > log_mle: -0.37879 (-0.40122) + | > loss_dur: 0.07911 (0.07686) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.31464 (13.89284) + | > current_lr: 0.00007 + | > step_time: 1.80200 (3.88112) + | > loader_time: 0.00180 (0.33264) + + + --> STEP: 41/234 -- GLOBAL_STEP: 63455 + | > loss: -0.32842 (-0.32186) + | > log_mle: -0.39206 (-0.39910) + | > loss_dur: 0.06364 (0.07724) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.13516 (13.70367) + | > current_lr: 0.00007 + | > step_time: 2.91410 (3.70593) + | > loader_time: 0.08590 (0.29659) + + + --> STEP: 46/234 -- GLOBAL_STEP: 63460 + | > loss: -0.28638 (-0.31865) + | > log_mle: -0.38258 (-0.39703) + | > loss_dur: 0.09619 (0.07838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.91075 (13.55097) + | > current_lr: 0.00007 + | > step_time: 1.10220 (3.52493) + | > loader_time: 0.00200 (0.26456) + + + --> STEP: 51/234 -- GLOBAL_STEP: 63465 + | > loss: -0.30093 (-0.31761) + | > log_mle: -0.37809 (-0.39594) + | > loss_dur: 0.07716 (0.07832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.78863 (13.08282) + | > current_lr: 0.00007 + | > step_time: 1.15500 (3.29319) + | > loader_time: 0.00190 (0.23879) + + + --> STEP: 56/234 -- GLOBAL_STEP: 63470 + | > loss: -0.29394 (-0.31545) + | > log_mle: -0.38291 (-0.39479) + | > loss_dur: 0.08897 (0.07935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.17412 (12.85737) + | > current_lr: 0.00007 + | > step_time: 3.00920 (3.16024) + | > loader_time: 0.00400 (0.21943) + + + --> STEP: 61/234 -- GLOBAL_STEP: 63475 + | > loss: -0.28424 (-0.31310) + | > log_mle: -0.37377 (-0.39334) + | > loss_dur: 0.08953 (0.08024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.72940 (12.74990) + | > current_lr: 0.00007 + | > step_time: 1.79840 (3.05184) + | > loader_time: 0.08560 (0.20431) + + + --> STEP: 66/234 -- GLOBAL_STEP: 63480 + | > loss: -0.28742 (-0.31004) + | > log_mle: -0.36496 (-0.39186) + | > loss_dur: 0.07755 (0.08182) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.46642 (12.70224) + | > current_lr: 0.00007 + | > step_time: 1.66100 (2.98725) + | > loader_time: 0.00180 (0.18903) + + + --> STEP: 71/234 -- GLOBAL_STEP: 63485 + | > loss: -0.25874 (-0.30653) + | > log_mle: -0.38967 (-0.39035) + | > loss_dur: 0.13093 (0.08382) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.24206 (12.69576) + | > current_lr: 0.00007 + | > step_time: 2.99360 (2.92257) + | > loader_time: 0.20270 (0.17984) + + + --> STEP: 76/234 -- GLOBAL_STEP: 63490 + | > loss: -0.25790 (-0.30315) + | > log_mle: -0.36984 (-0.38885) + | > loss_dur: 0.11194 (0.08570) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.03854 (12.79616) + | > current_lr: 0.00007 + | > step_time: 1.48320 (2.84052) + | > loader_time: 0.00190 (0.16933) + + + --> STEP: 81/234 -- GLOBAL_STEP: 63495 + | > loss: -0.25990 (-0.30046) + | > log_mle: -0.37426 (-0.38736) + | > loss_dur: 0.11436 (0.08689) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.34404 (12.76355) + | > current_lr: 0.00007 + | > step_time: 2.11640 (2.76321) + | > loader_time: 0.00330 (0.16008) + + + --> STEP: 86/234 -- GLOBAL_STEP: 63500 + | > loss: -0.25276 (-0.29752) + | > log_mle: -0.37410 (-0.38603) + | > loss_dur: 0.12134 (0.08851) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.92535 (12.85234) + | > current_lr: 0.00007 + | > step_time: 1.39810 (2.70412) + | > loader_time: 0.00190 (0.15206) + + + --> STEP: 91/234 -- GLOBAL_STEP: 63505 + | > loss: -0.26226 (-0.29545) + | > log_mle: -0.38360 (-0.38589) + | > loss_dur: 0.12134 (0.09044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.39699 (12.92554) + | > current_lr: 0.00007 + | > step_time: 1.40810 (2.68444) + | > loader_time: 0.00220 (0.14675) + + + --> STEP: 96/234 -- GLOBAL_STEP: 63510 + | > loss: -0.24851 (-0.29432) + | > log_mle: -0.36997 (-0.38731) + | > loss_dur: 0.12146 (0.09299) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.33991 (13.33846) + | > current_lr: 0.00007 + | > step_time: 1.31500 (2.62182) + | > loader_time: 0.00200 (0.14099) + + + --> STEP: 101/234 -- GLOBAL_STEP: 63515 + | > loss: -0.25318 (-0.29269) + | > log_mle: -0.41853 (-0.38795) + | > loss_dur: 0.16535 (0.09526) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.66423 (13.77043) + | > current_lr: 0.00007 + | > step_time: 1.36330 (2.56289) + | > loader_time: 0.00290 (0.13500) + + + --> STEP: 106/234 -- GLOBAL_STEP: 63520 + | > loss: -0.23859 (-0.29130) + | > log_mle: -0.41417 (-0.38917) + | > loss_dur: 0.17558 (0.09788) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.44471 (14.23796) + | > current_lr: 0.00007 + | > step_time: 1.69350 (2.52849) + | > loader_time: 0.00410 (0.13049) + + + --> STEP: 111/234 -- GLOBAL_STEP: 63525 + | > loss: -0.27855 (-0.28969) + | > log_mle: -0.46335 (-0.39033) + | > loss_dur: 0.18480 (0.10064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.02645 (14.91901) + | > current_lr: 0.00007 + | > step_time: 1.29910 (2.48401) + | > loader_time: 0.00260 (0.12552) + + + --> STEP: 116/234 -- GLOBAL_STEP: 63530 + | > loss: -0.24719 (-0.28831) + | > log_mle: -0.42689 (-0.39162) + | > loss_dur: 0.17970 (0.10331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.02016 (15.45103) + | > current_lr: 0.00007 + | > step_time: 1.69090 (2.46148) + | > loader_time: 0.00300 (0.12184) + + + --> STEP: 121/234 -- GLOBAL_STEP: 63535 + | > loss: -0.22535 (-0.28698) + | > log_mle: -0.35015 (-0.39231) + | > loss_dur: 0.12480 (0.10534) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.71323 (15.66403) + | > current_lr: 0.00007 + | > step_time: 2.50120 (2.45482) + | > loader_time: 0.00250 (0.11771) + + + --> STEP: 126/234 -- GLOBAL_STEP: 63540 + | > loss: -0.29350 (-0.28601) + | > log_mle: -0.48210 (-0.39359) + | > loss_dur: 0.18861 (0.10758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.76328 (16.08002) + | > current_lr: 0.00007 + | > step_time: 3.21580 (2.47278) + | > loader_time: 0.28750 (0.11681) + + + --> STEP: 131/234 -- GLOBAL_STEP: 63545 + | > loss: -0.34538 (-0.28614) + | > log_mle: -0.53280 (-0.39621) + | > loss_dur: 0.18742 (0.11007) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.22603 (16.67635) + | > current_lr: 0.00007 + | > step_time: 0.90400 (2.50128) + | > loader_time: 0.00540 (0.11379) + + + --> STEP: 136/234 -- GLOBAL_STEP: 63550 + | > loss: -0.37701 (-0.28646) + | > log_mle: -0.57808 (-0.39892) + | > loss_dur: 0.20107 (0.11245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.46196 (17.49648) + | > current_lr: 0.00007 + | > step_time: 2.01850 (2.50892) + | > loader_time: 0.11580 (0.11169) + + + --> STEP: 141/234 -- GLOBAL_STEP: 63555 + | > loss: -0.30461 (-0.28621) + | > log_mle: -0.47306 (-0.40110) + | > loss_dur: 0.16845 (0.11489) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.94643 (18.30461) + | > current_lr: 0.00007 + | > step_time: 0.79680 (2.50621) + | > loader_time: 0.00240 (0.10788) + + + --> STEP: 146/234 -- GLOBAL_STEP: 63560 + | > loss: -0.33013 (-0.28760) + | > log_mle: -0.53205 (-0.40526) + | > loss_dur: 0.20191 (0.11767) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.83580 (19.11777) + | > current_lr: 0.00007 + | > step_time: 3.79290 (2.54081) + | > loader_time: 0.19930 (0.10691) + + + --> STEP: 151/234 -- GLOBAL_STEP: 63565 + | > loss: -0.30798 (-0.28864) + | > log_mle: -0.48900 (-0.40843) + | > loss_dur: 0.18102 (0.11980) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.93759 (19.94381) + | > current_lr: 0.00007 + | > step_time: 1.29050 (2.56665) + | > loader_time: 0.00430 (0.10603) + + + --> STEP: 156/234 -- GLOBAL_STEP: 63570 + | > loss: -0.35215 (-0.29094) + | > log_mle: -0.53873 (-0.41318) + | > loss_dur: 0.18657 (0.12223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.25938 (21.25551) + | > current_lr: 0.00007 + | > step_time: 4.19700 (2.61581) + | > loader_time: 0.20760 (0.10534) + + + --> STEP: 161/234 -- GLOBAL_STEP: 63575 + | > loss: -0.38145 (-0.29244) + | > log_mle: -0.56294 (-0.41696) + | > loss_dur: 0.18149 (0.12452) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.29654 (22.41972) + | > current_lr: 0.00007 + | > step_time: 3.70000 (2.60900) + | > loader_time: 0.08170 (0.10267) + + + --> STEP: 166/234 -- GLOBAL_STEP: 63580 + | > loss: -0.32833 (-0.29369) + | > log_mle: -0.50286 (-0.42025) + | > loss_dur: 0.17453 (0.12656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.96203 (23.03813) + | > current_lr: 0.00007 + | > step_time: 1.89820 (2.58167) + | > loader_time: 0.07770 (0.10059) + + + --> STEP: 171/234 -- GLOBAL_STEP: 63585 + | > loss: -0.40088 (-0.29639) + | > log_mle: -0.60793 (-0.42530) + | > loss_dur: 0.20705 (0.12891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.04176 (24.07109) + | > current_lr: 0.00007 + | > step_time: 2.20820 (2.56422) + | > loader_time: 0.00340 (0.09871) + + + --> STEP: 176/234 -- GLOBAL_STEP: 63590 + | > loss: -0.37723 (-0.29865) + | > log_mle: -0.58362 (-0.43000) + | > loss_dur: 0.20640 (0.13135) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.91141 (25.37974) + | > current_lr: 0.00007 + | > step_time: 2.89550 (2.54864) + | > loader_time: 0.00570 (0.09647) + + + --> STEP: 181/234 -- GLOBAL_STEP: 63595 + | > loss: -0.31297 (-0.30053) + | > log_mle: -0.52047 (-0.43431) + | > loss_dur: 0.20750 (0.13378) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.99267 (26.29471) + | > current_lr: 0.00007 + | > step_time: 2.50470 (2.53877) + | > loader_time: 0.00330 (0.09490) + + + --> STEP: 186/234 -- GLOBAL_STEP: 63600 + | > loss: -0.32712 (-0.30267) + | > log_mle: -0.56038 (-0.43891) + | > loss_dur: 0.23326 (0.13624) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.10276 (27.39855) + | > current_lr: 0.00007 + | > step_time: 3.11620 (2.68998) + | > loader_time: 0.08920 (0.09659) + + + --> STEP: 191/234 -- GLOBAL_STEP: 63605 + | > loss: -0.37513 (-0.30489) + | > log_mle: -0.58612 (-0.44318) + | > loss_dur: 0.21099 (0.13829) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.69111 (28.30368) + | > current_lr: 0.00007 + | > step_time: 1.59850 (2.66613) + | > loader_time: 0.00620 (0.09478) + + + --> STEP: 196/234 -- GLOBAL_STEP: 63610 + | > loss: -0.35884 (-0.30736) + | > log_mle: -0.58086 (-0.44764) + | > loss_dur: 0.22203 (0.14029) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.46771 (29.34209) + | > current_lr: 0.00007 + | > step_time: 4.09760 (2.71051) + | > loader_time: 0.10610 (0.09536) + + + --> STEP: 201/234 -- GLOBAL_STEP: 63615 + | > loss: -0.30529 (-0.30904) + | > log_mle: -0.52882 (-0.45131) + | > loss_dur: 0.22353 (0.14226) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.73172 (30.65574) + | > current_lr: 0.00007 + | > step_time: 3.90080 (2.78787) + | > loader_time: 0.00240 (0.09453) + + + --> STEP: 206/234 -- GLOBAL_STEP: 63620 + | > loss: -0.42550 (-0.31162) + | > log_mle: -0.65098 (-0.45578) + | > loss_dur: 0.22548 (0.14417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.10156 (31.50625) + | > current_lr: 0.00007 + | > step_time: 15.51170 (2.88980) + | > loader_time: 0.00460 (0.09317) + + + --> STEP: 211/234 -- GLOBAL_STEP: 63625 + | > loss: -0.48685 (-0.31475) + | > log_mle: -0.73128 (-0.46106) + | > loss_dur: 0.24443 (0.14631) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.82663 (32.62148) + | > current_lr: 0.00007 + | > step_time: 5.89820 (2.94508) + | > loader_time: 0.09890 (0.09373) + + + --> STEP: 216/234 -- GLOBAL_STEP: 63630 + | > loss: -0.48570 (-0.31805) + | > log_mle: -0.72166 (-0.46626) + | > loss_dur: 0.23596 (0.14821) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.99039 (33.57452) + | > current_lr: 0.00007 + | > step_time: 9.21170 (3.02980) + | > loader_time: 0.00270 (0.09615) + + + --> STEP: 221/234 -- GLOBAL_STEP: 63635 + | > loss: -0.41083 (-0.32132) + | > log_mle: -0.62209 (-0.47136) + | > loss_dur: 0.21126 (0.15005) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.12591 (34.92180) + | > current_lr: 0.00007 + | > step_time: 4.62040 (3.06220) + | > loader_time: 0.00750 (0.09508) + + + --> STEP: 226/234 -- GLOBAL_STEP: 63640 + | > loss: -0.48236 (-0.32471) + | > log_mle: -0.72252 (-0.47688) + | > loss_dur: 0.24015 (0.15217) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.07192 (36.14943) + | > current_lr: 0.00007 + | > step_time: 0.23920 (3.02024) + | > loader_time: 0.00330 (0.09335) + + + --> STEP: 231/234 -- GLOBAL_STEP: 63645 + | > loss: -0.42307 (-0.32710) + | > log_mle: -0.79160 (-0.48248) + | > loss_dur: 0.36854 (0.15539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.29882 (37.60027) + | > current_lr: 0.00007 + | > step_time: 0.27850 (2.96059) + | > loader_time: 0.00400 (0.09142) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.20570 (-0.57193) + | > avg_loss: -0.33324 (+0.00963) + | > avg_log_mle: -0.55525 (+0.01091) + | > avg_loss_dur: 0.22201 (-0.00128) + + + > EPOCH: 272/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 18:34:12)  + + --> STEP: 2/234 -- GLOBAL_STEP: 63650 + | > loss: -0.33861 (-0.31985) + | > log_mle: -0.41620 (-0.40292) + | > loss_dur: 0.07759 (0.08307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.80649 (22.92896) + | > current_lr: 0.00007 + | > step_time: 11.60020 (7.29443) + | > loader_time: 0.19250 (0.64745) + + + --> STEP: 7/234 -- GLOBAL_STEP: 63655 + | > loss: -0.32909 (-0.30347) + | > log_mle: -0.40189 (-0.39754) + | > loss_dur: 0.07280 (0.09407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.27321 (22.73589) + | > current_lr: 0.00007 + | > step_time: 3.30220 (6.23415) + | > loader_time: 0.00410 (0.23907) + + + --> STEP: 12/234 -- GLOBAL_STEP: 63660 + | > loss: -0.31563 (-0.31301) + | > log_mle: -0.40059 (-0.40268) + | > loss_dur: 0.08497 (0.08967) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.20054 (19.56485) + | > current_lr: 0.00007 + | > step_time: 2.11430 (5.21344) + | > loader_time: 0.08160 (0.17989) + + + --> STEP: 17/234 -- GLOBAL_STEP: 63665 + | > loss: -0.35317 (-0.32454) + | > log_mle: -0.41185 (-0.40706) + | > loss_dur: 0.05868 (0.08251) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.77425 (16.95691) + | > current_lr: 0.00007 + | > step_time: 2.20040 (4.65819) + | > loader_time: 0.08350 (0.13798) + + + --> STEP: 22/234 -- GLOBAL_STEP: 63670 + | > loss: -0.32472 (-0.32617) + | > log_mle: -0.40051 (-0.40582) + | > loss_dur: 0.07579 (0.07965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.47247 (15.96689) + | > current_lr: 0.00007 + | > step_time: 2.98600 (3.91623) + | > loader_time: 0.00110 (0.10697) + + + --> STEP: 27/234 -- GLOBAL_STEP: 63675 + | > loss: -0.31404 (-0.32624) + | > log_mle: -0.38954 (-0.40421) + | > loss_dur: 0.07550 (0.07798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.50324 (15.53029) + | > current_lr: 0.00007 + | > step_time: 2.09110 (3.69739) + | > loader_time: 0.00190 (0.09840) + + + --> STEP: 32/234 -- GLOBAL_STEP: 63680 + | > loss: -0.32297 (-0.32612) + | > log_mle: -0.39858 (-0.40361) + | > loss_dur: 0.07561 (0.07749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.03103 (14.53472) + | > current_lr: 0.00007 + | > step_time: 1.89490 (3.42305) + | > loader_time: 0.00300 (0.08637) + + + --> STEP: 37/234 -- GLOBAL_STEP: 63685 + | > loss: -0.30838 (-0.32369) + | > log_mle: -0.37558 (-0.40115) + | > loss_dur: 0.06719 (0.07746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.51203 (14.14164) + | > current_lr: 0.00007 + | > step_time: 0.90910 (3.12569) + | > loader_time: 0.00320 (0.07707) + + + --> STEP: 42/234 -- GLOBAL_STEP: 63690 + | > loss: -0.29852 (-0.32169) + | > log_mle: -0.36919 (-0.39907) + | > loss_dur: 0.07067 (0.07738) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.20998 (13.61602) + | > current_lr: 0.00007 + | > step_time: 3.50380 (2.99388) + | > loader_time: 0.00370 (0.07018) + + + --> STEP: 47/234 -- GLOBAL_STEP: 63695 + | > loss: -0.29911 (-0.31891) + | > log_mle: -0.38723 (-0.39745) + | > loss_dur: 0.08812 (0.07854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.51834 (13.36957) + | > current_lr: 0.00007 + | > step_time: 2.60010 (2.90047) + | > loader_time: 0.00210 (0.06479) + + + --> STEP: 52/234 -- GLOBAL_STEP: 63700 + | > loss: -0.27863 (-0.31729) + | > log_mle: -0.37567 (-0.39605) + | > loss_dur: 0.09704 (0.07876) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.73604 (13.10135) + | > current_lr: 0.00007 + | > step_time: 2.01060 (2.77388) + | > loader_time: 0.00230 (0.06425) + + + --> STEP: 57/234 -- GLOBAL_STEP: 63705 + | > loss: -0.27281 (-0.31544) + | > log_mle: -0.36413 (-0.39467) + | > loss_dur: 0.09132 (0.07924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.36342 (12.93109) + | > current_lr: 0.00007 + | > step_time: 1.10870 (2.69922) + | > loader_time: 0.00170 (0.06042) + + + --> STEP: 62/234 -- GLOBAL_STEP: 63710 + | > loss: -0.24718 (-0.31263) + | > log_mle: -0.39080 (-0.39374) + | > loss_dur: 0.14362 (0.08111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.92409 (12.97864) + | > current_lr: 0.00007 + | > step_time: 1.89790 (2.63228) + | > loader_time: 0.00360 (0.05721) + + + --> STEP: 67/234 -- GLOBAL_STEP: 63715 + | > loss: -0.26906 (-0.30998) + | > log_mle: -0.37808 (-0.39213) + | > loss_dur: 0.10902 (0.08215) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.17923 (12.76404) + | > current_lr: 0.00007 + | > step_time: 6.03260 (2.62699) + | > loader_time: 0.08870 (0.05444) + + + --> STEP: 72/234 -- GLOBAL_STEP: 63720 + | > loss: -0.26985 (-0.30656) + | > log_mle: -0.36866 (-0.39038) + | > loss_dur: 0.09881 (0.08382) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.82842 (12.77648) + | > current_lr: 0.00007 + | > step_time: 2.64050 (2.55707) + | > loader_time: 0.08260 (0.05318) + + + --> STEP: 77/234 -- GLOBAL_STEP: 63725 + | > loss: -0.26612 (-0.30321) + | > log_mle: -0.36810 (-0.38890) + | > loss_dur: 0.10198 (0.08569) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.88809 (12.77133) + | > current_lr: 0.00007 + | > step_time: 1.47660 (2.51754) + | > loader_time: 0.00170 (0.05123) + + + --> STEP: 82/234 -- GLOBAL_STEP: 63730 + | > loss: -0.24921 (-0.30068) + | > log_mle: -0.36330 (-0.38753) + | > loss_dur: 0.11410 (0.08685) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.17269 (12.72386) + | > current_lr: 0.00007 + | > step_time: 1.39650 (2.48123) + | > loader_time: 0.00260 (0.04908) + + + --> STEP: 87/234 -- GLOBAL_STEP: 63735 + | > loss: -0.24990 (-0.29791) + | > log_mle: -0.36453 (-0.38646) + | > loss_dur: 0.11463 (0.08855) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.54344 (12.73527) + | > current_lr: 0.00007 + | > step_time: 1.01190 (2.41888) + | > loader_time: 0.00300 (0.04643) + + + --> STEP: 92/234 -- GLOBAL_STEP: 63740 + | > loss: -0.26838 (-0.29598) + | > log_mle: -0.40564 (-0.38685) + | > loss_dur: 0.13726 (0.09087) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.60296 (12.99221) + | > current_lr: 0.00007 + | > step_time: 3.60540 (2.44488) + | > loader_time: 0.18230 (0.04777) + + + --> STEP: 97/234 -- GLOBAL_STEP: 63745 + | > loss: -0.26114 (-0.29490) + | > log_mle: -0.38918 (-0.38818) + | > loss_dur: 0.12803 (0.09328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.83570 (13.35149) + | > current_lr: 0.00007 + | > step_time: 1.82140 (2.39474) + | > loader_time: 0.00290 (0.04544) + + + --> STEP: 102/234 -- GLOBAL_STEP: 63750 + | > loss: -0.24319 (-0.29336) + | > log_mle: -0.37472 (-0.38865) + | > loss_dur: 0.13153 (0.09528) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.62731 (13.69589) + | > current_lr: 0.00007 + | > step_time: 1.80100 (2.36414) + | > loader_time: 0.00870 (0.04342) + + + --> STEP: 107/234 -- GLOBAL_STEP: 63755 + | > loss: -0.26089 (-0.29221) + | > log_mle: -0.41434 (-0.39019) + | > loss_dur: 0.15345 (0.09799) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.54873 (14.10399) + | > current_lr: 0.00007 + | > step_time: 2.80650 (2.33399) + | > loader_time: 0.00410 (0.04152) + + + --> STEP: 112/234 -- GLOBAL_STEP: 63760 + | > loss: -0.26131 (-0.29072) + | > log_mle: -0.42735 (-0.39160) + | > loss_dur: 0.16604 (0.10088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.38306 (14.53642) + | > current_lr: 0.00007 + | > step_time: 1.48610 (2.31905) + | > loader_time: 0.00200 (0.04062) + + + --> STEP: 117/234 -- GLOBAL_STEP: 63765 + | > loss: -0.27039 (-0.28959) + | > log_mle: -0.42186 (-0.39298) + | > loss_dur: 0.15147 (0.10339) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.04167 (15.09197) + | > current_lr: 0.00007 + | > step_time: 1.83960 (2.29170) + | > loader_time: 0.00150 (0.04032) + + + --> STEP: 122/234 -- GLOBAL_STEP: 63770 + | > loss: -0.24376 (-0.28811) + | > log_mle: -0.38932 (-0.39343) + | > loss_dur: 0.14556 (0.10532) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.07456 (15.46260) + | > current_lr: 0.00007 + | > step_time: 2.09740 (2.26156) + | > loader_time: 0.00420 (0.03940) + + + --> STEP: 127/234 -- GLOBAL_STEP: 63775 + | > loss: -0.27097 (-0.28724) + | > log_mle: -0.44876 (-0.39510) + | > loss_dur: 0.17779 (0.10787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.59219 (16.19133) + | > current_lr: 0.00007 + | > step_time: 2.39140 (2.24803) + | > loader_time: 0.00650 (0.03993) + + + --> STEP: 132/234 -- GLOBAL_STEP: 63780 + | > loss: -0.27665 (-0.28705) + | > log_mle: -0.43318 (-0.39717) + | > loss_dur: 0.15652 (0.11012) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.66698 (16.76258) + | > current_lr: 0.00007 + | > step_time: 6.70630 (2.29014) + | > loader_time: 0.00300 (0.04062) + + + --> STEP: 137/234 -- GLOBAL_STEP: 63785 + | > loss: -0.26324 (-0.28735) + | > log_mle: -0.44982 (-0.39986) + | > loss_dur: 0.18658 (0.11250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.48891 (17.42381) + | > current_lr: 0.00007 + | > step_time: 1.11120 (2.29551) + | > loader_time: 0.08140 (0.04119) + + + --> STEP: 142/234 -- GLOBAL_STEP: 63790 + | > loss: -0.28083 (-0.28736) + | > log_mle: -0.46411 (-0.40216) + | > loss_dur: 0.18328 (0.11480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.62904 (18.00793) + | > current_lr: 0.00007 + | > step_time: 1.80780 (2.29589) + | > loader_time: 0.08580 (0.04237) + + + --> STEP: 147/234 -- GLOBAL_STEP: 63795 + | > loss: -0.29019 (-0.28871) + | > log_mle: -0.46588 (-0.40614) + | > loss_dur: 0.17569 (0.11743) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.93182 (19.15497) + | > current_lr: 0.00007 + | > step_time: 4.00930 (2.33636) + | > loader_time: 0.00440 (0.04296) + + + --> STEP: 152/234 -- GLOBAL_STEP: 63800 + | > loss: -0.32307 (-0.28989) + | > log_mle: -0.53394 (-0.40960) + | > loss_dur: 0.21087 (0.11971) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.06767 (20.64244) + | > current_lr: 0.00007 + | > step_time: 1.81200 (2.36624) + | > loader_time: 0.00250 (0.04344) + + + --> STEP: 157/234 -- GLOBAL_STEP: 63805 + | > loss: -0.31677 (-0.29192) + | > log_mle: -0.49625 (-0.41393) + | > loss_dur: 0.17948 (0.12201) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.95838 (21.79783) + | > current_lr: 0.00007 + | > step_time: 2.89550 (2.41969) + | > loader_time: 0.10090 (0.04451) + + + --> STEP: 162/234 -- GLOBAL_STEP: 63810 + | > loss: -0.33164 (-0.29362) + | > log_mle: -0.52279 (-0.41803) + | > loss_dur: 0.19115 (0.12441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.66086 (22.53078) + | > current_lr: 0.00007 + | > step_time: 5.31580 (2.46848) + | > loader_time: 0.18820 (0.04606) + + + --> STEP: 167/234 -- GLOBAL_STEP: 63815 + | > loss: -0.43822 (-0.29547) + | > log_mle: -0.63184 (-0.42197) + | > loss_dur: 0.19362 (0.12650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.33247 (23.30968) + | > current_lr: 0.00007 + | > step_time: 3.99540 (2.49282) + | > loader_time: 0.00300 (0.04769) + + + --> STEP: 172/234 -- GLOBAL_STEP: 63820 + | > loss: -0.39364 (-0.29779) + | > log_mle: -0.61255 (-0.42676) + | > loss_dur: 0.21891 (0.12897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.51926 (24.52869) + | > current_lr: 0.00007 + | > step_time: 4.00870 (2.56170) + | > loader_time: 0.08290 (0.04742) + + + --> STEP: 177/234 -- GLOBAL_STEP: 63825 + | > loss: -0.35074 (-0.30000) + | > log_mle: -0.56338 (-0.43133) + | > loss_dur: 0.21263 (0.13133) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.44336 (25.39579) + | > current_lr: 0.00007 + | > step_time: 4.49860 (2.61529) + | > loader_time: 0.08890 (0.04760) + + + --> STEP: 182/234 -- GLOBAL_STEP: 63830 + | > loss: -0.37387 (-0.30200) + | > log_mle: -0.61412 (-0.43589) + | > loss_dur: 0.24025 (0.13389) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.00034 (26.34108) + | > current_lr: 0.00007 + | > step_time: 7.02790 (2.75406) + | > loader_time: 0.38840 (0.05121) + + + --> STEP: 187/234 -- GLOBAL_STEP: 63835 + | > loss: -0.38049 (-0.30405) + | > log_mle: -0.59642 (-0.44020) + | > loss_dur: 0.21593 (0.13615) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.72357 (27.45792) + | > current_lr: 0.00007 + | > step_time: 2.19910 (2.78353) + | > loader_time: 0.00300 (0.05100) + + + --> STEP: 192/234 -- GLOBAL_STEP: 63840 + | > loss: -0.33177 (-0.30409) + | > log_mle: -0.54888 (-0.44221) + | > loss_dur: 0.21711 (0.13811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.75154 (29.28825) + | > current_lr: 0.00007 + | > step_time: 2.40060 (2.83349) + | > loader_time: 0.00240 (0.05117) + + + --> STEP: 197/234 -- GLOBAL_STEP: 63845 + | > loss: -0.34449 (-0.30460) + | > log_mle: -0.54257 (-0.44464) + | > loss_dur: 0.19808 (0.14004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.88752 (29.76819) + | > current_lr: 0.00007 + | > step_time: 5.69550 (2.92488) + | > loader_time: 0.00240 (0.05191) + + + --> STEP: 202/234 -- GLOBAL_STEP: 63850 + | > loss: -0.44976 (-0.30565) + | > log_mle: -0.66417 (-0.44789) + | > loss_dur: 0.21441 (0.14223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.73418 (30.04892) + | > current_lr: 0.00007 + | > step_time: 4.28950 (2.92391) + | > loader_time: 0.00210 (0.05118) + + + --> STEP: 207/234 -- GLOBAL_STEP: 63855 + | > loss: -0.42699 (-0.30719) + | > log_mle: -0.65741 (-0.45148) + | > loss_dur: 0.23041 (0.14430) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.56978 (30.79805) + | > current_lr: 0.00007 + | > step_time: 3.58820 (2.98086) + | > loader_time: 0.00360 (0.05223) + + + --> STEP: 212/234 -- GLOBAL_STEP: 63860 + | > loss: -0.40921 (-0.30959) + | > log_mle: -0.64699 (-0.45602) + | > loss_dur: 0.23778 (0.14642) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.17976 (31.67500) + | > current_lr: 0.00007 + | > step_time: 3.80480 (3.02611) + | > loader_time: 0.00830 (0.05285) + + + --> STEP: 217/234 -- GLOBAL_STEP: 63865 + | > loss: -0.43295 (-0.31213) + | > log_mle: -0.67226 (-0.46057) + | > loss_dur: 0.23931 (0.14845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.68228 (32.50003) + | > current_lr: 0.00007 + | > step_time: 6.00490 (3.04266) + | > loader_time: 0.00230 (0.05326) + + + --> STEP: 222/234 -- GLOBAL_STEP: 63870 + | > loss: -0.43366 (-0.31487) + | > log_mle: -0.69672 (-0.46532) + | > loss_dur: 0.26306 (0.15045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.51292 (33.18544) + | > current_lr: 0.00007 + | > step_time: 2.83360 (3.04461) + | > loader_time: 0.07970 (0.05373) + + + --> STEP: 227/234 -- GLOBAL_STEP: 63875 + | > loss: -0.40810 (-0.31805) + | > log_mle: -0.66571 (-0.47065) + | > loss_dur: 0.25760 (0.15260) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.14510 (34.08115) + | > current_lr: 0.00007 + | > step_time: 0.25010 (2.98607) + | > loader_time: 0.00300 (0.05263) + + + --> STEP: 232/234 -- GLOBAL_STEP: 63880 + | > loss: -0.43001 (-0.32099) + | > log_mle: -0.90813 (-0.47772) + | > loss_dur: 0.47813 (0.15673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 114.19946 (35.32589) + | > current_lr: 0.00007 + | > step_time: 0.33430 (2.92767) + | > loader_time: 0.08420 (0.05194) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.44509 (+0.23939) + | > avg_loss: -0.32272 (+0.01052) + | > avg_log_mle: -0.54498 (+0.01027) + | > avg_loss_dur: 0.22226 (+0.00026) + + + > EPOCH: 273/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 18:46:39)  + + --> STEP: 3/234 -- GLOBAL_STEP: 63885 + | > loss: -0.24412 (-0.28752) + | > log_mle: -0.38468 (-0.39659) + | > loss_dur: 0.14056 (0.10907) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.94366 (22.09805) + | > current_lr: 0.00007 + | > step_time: 4.50680 (3.17557) + | > loader_time: 2.00020 (3.59865) + + + --> STEP: 8/234 -- GLOBAL_STEP: 63890 + | > loss: -0.34591 (-0.31068) + | > log_mle: -0.42313 (-0.40186) + | > loss_dur: 0.07721 (0.09119) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.28968 (18.29026) + | > current_lr: 0.00007 + | > step_time: 2.79790 (3.24134) + | > loader_time: 0.00100 (1.35073) + + + --> STEP: 13/234 -- GLOBAL_STEP: 63895 + | > loss: -0.36243 (-0.32066) + | > log_mle: -0.42792 (-0.40605) + | > loss_dur: 0.06549 (0.08538) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.08337 (17.88514) + | > current_lr: 0.00007 + | > step_time: 5.00600 (4.36021) + | > loader_time: 0.08170 (0.85269) + + + --> STEP: 18/234 -- GLOBAL_STEP: 63900 + | > loss: -0.32716 (-0.32730) + | > log_mle: -0.39941 (-0.40774) + | > loss_dur: 0.07225 (0.08044) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.50723 (16.39371) + | > current_lr: 0.00007 + | > step_time: 2.41250 (4.17210) + | > loader_time: 0.00130 (0.63718) + + + --> STEP: 23/234 -- GLOBAL_STEP: 63905 + | > loss: -0.35592 (-0.33045) + | > log_mle: -0.42095 (-0.40781) + | > loss_dur: 0.06504 (0.07737) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.55204 (15.47408) + | > current_lr: 0.00007 + | > step_time: 2.89400 (4.33967) + | > loader_time: 0.00250 (0.51160) + + + --> STEP: 28/234 -- GLOBAL_STEP: 63910 + | > loss: -0.38143 (-0.33071) + | > log_mle: -0.43178 (-0.40668) + | > loss_dur: 0.05035 (0.07597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.66764 (14.60065) + | > current_lr: 0.00007 + | > step_time: 3.14660 (4.42432) + | > loader_time: 0.00100 (0.43439) + + + --> STEP: 33/234 -- GLOBAL_STEP: 63915 + | > loss: -0.33296 (-0.32903) + | > log_mle: -0.39661 (-0.40497) + | > loss_dur: 0.06365 (0.07594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.91071 (13.95614) + | > current_lr: 0.00007 + | > step_time: 2.40960 (3.95791) + | > loader_time: 0.00210 (0.37136) + + + --> STEP: 38/234 -- GLOBAL_STEP: 63920 + | > loss: -0.30221 (-0.32559) + | > log_mle: -0.39041 (-0.40254) + | > loss_dur: 0.08820 (0.07696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.40069 (13.58593) + | > current_lr: 0.00007 + | > step_time: 4.39950 (3.80942) + | > loader_time: 0.00260 (0.32964) + + + --> STEP: 43/234 -- GLOBAL_STEP: 63925 + | > loss: -0.29195 (-0.32246) + | > log_mle: -0.38686 (-0.40045) + | > loss_dur: 0.09491 (0.07800) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.72216 (13.25327) + | > current_lr: 0.00007 + | > step_time: 3.46580 (3.67567) + | > loader_time: 0.19470 (0.30337) + + + --> STEP: 48/234 -- GLOBAL_STEP: 63930 + | > loss: -0.32348 (-0.31978) + | > log_mle: -0.38857 (-0.39877) + | > loss_dur: 0.06508 (0.07899) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.32910 (12.95419) + | > current_lr: 0.00007 + | > step_time: 0.71400 (3.42942) + | > loader_time: 0.00270 (0.27220) + + + --> STEP: 53/234 -- GLOBAL_STEP: 63935 + | > loss: -0.28310 (-0.31721) + | > log_mle: -0.38311 (-0.39695) + | > loss_dur: 0.10001 (0.07974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.89047 (12.56739) + | > current_lr: 0.00007 + | > step_time: 1.30660 (3.22947) + | > loader_time: 0.00160 (0.24670) + + + --> STEP: 58/234 -- GLOBAL_STEP: 63940 + | > loss: -0.31071 (-0.31558) + | > log_mle: -0.38436 (-0.39545) + | > loss_dur: 0.07365 (0.07987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87989 (12.30253) + | > current_lr: 0.00007 + | > step_time: 1.32870 (3.06522) + | > loader_time: 0.00250 (0.22564) + + + --> STEP: 63/234 -- GLOBAL_STEP: 63945 + | > loss: -0.27087 (-0.31188) + | > log_mle: -0.36719 (-0.39411) + | > loss_dur: 0.09632 (0.08223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.35352 (12.70764) + | > current_lr: 0.00007 + | > step_time: 1.71260 (2.94696) + | > loader_time: 0.07990 (0.20912) + + + --> STEP: 68/234 -- GLOBAL_STEP: 63950 + | > loss: -0.24751 (-0.30902) + | > log_mle: -0.35884 (-0.39217) + | > loss_dur: 0.11133 (0.08315) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.73278 (12.74107) + | > current_lr: 0.00007 + | > step_time: 1.48900 (2.82529) + | > loader_time: 0.00180 (0.19511) + + + --> STEP: 73/234 -- GLOBAL_STEP: 63955 + | > loss: -0.24461 (-0.30544) + | > log_mle: -0.36933 (-0.39054) + | > loss_dur: 0.12472 (0.08510) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.18240 (12.85529) + | > current_lr: 0.00007 + | > step_time: 2.40040 (2.73259) + | > loader_time: 0.00260 (0.18302) + + + --> STEP: 78/234 -- GLOBAL_STEP: 63960 + | > loss: -0.25867 (-0.30247) + | > log_mle: -0.36044 (-0.38891) + | > loss_dur: 0.10177 (0.08644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.27308 (12.91324) + | > current_lr: 0.00007 + | > step_time: 1.41870 (2.68814) + | > loader_time: 0.08670 (0.17354) + + + --> STEP: 83/234 -- GLOBAL_STEP: 63965 + | > loss: -0.23326 (-0.29996) + | > log_mle: -0.37055 (-0.38772) + | > loss_dur: 0.13729 (0.08777) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.81980 (12.93053) + | > current_lr: 0.00007 + | > step_time: 1.78840 (2.64133) + | > loader_time: 0.00340 (0.16323) + + + --> STEP: 88/234 -- GLOBAL_STEP: 63970 + | > loss: -0.26523 (-0.29773) + | > log_mle: -0.40579 (-0.38704) + | > loss_dur: 0.14055 (0.08932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.41799 (12.96892) + | > current_lr: 0.00007 + | > step_time: 1.09360 (2.58878) + | > loader_time: 0.00280 (0.15410) + + + --> STEP: 93/234 -- GLOBAL_STEP: 63975 + | > loss: -0.27143 (-0.29609) + | > log_mle: -0.41577 (-0.38763) + | > loss_dur: 0.14434 (0.09154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.17861 (13.20284) + | > current_lr: 0.00007 + | > step_time: 1.71090 (2.56037) + | > loader_time: 0.00290 (0.14767) + + + --> STEP: 98/234 -- GLOBAL_STEP: 63980 + | > loss: -0.25133 (-0.29506) + | > log_mle: -0.35628 (-0.38836) + | > loss_dur: 0.10495 (0.09330) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74760 (13.34827) + | > current_lr: 0.00007 + | > step_time: 2.16710 (2.53611) + | > loader_time: 0.00190 (0.14024) + + + --> STEP: 103/234 -- GLOBAL_STEP: 63985 + | > loss: -0.28702 (-0.29388) + | > log_mle: -0.44691 (-0.38976) + | > loss_dur: 0.15989 (0.09587) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.73855 (13.83618) + | > current_lr: 0.00007 + | > step_time: 3.59790 (2.52346) + | > loader_time: 0.00240 (0.13440) + + + --> STEP: 108/234 -- GLOBAL_STEP: 63990 + | > loss: -0.24620 (-0.29228) + | > log_mle: -0.38245 (-0.39047) + | > loss_dur: 0.13625 (0.09819) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.53384 (14.52567) + | > current_lr: 0.00007 + | > step_time: 1.60180 (2.50751) + | > loader_time: 0.00270 (0.12842) + + + --> STEP: 113/234 -- GLOBAL_STEP: 63995 + | > loss: -0.27324 (-0.29076) + | > log_mle: -0.42873 (-0.39194) + | > loss_dur: 0.15549 (0.10118) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.07201 (15.10230) + | > current_lr: 0.00007 + | > step_time: 1.70510 (2.50639) + | > loader_time: 0.08790 (0.12368) + + + --> STEP: 118/234 -- GLOBAL_STEP: 64000 + | > loss: -0.23829 (-0.28889) + | > log_mle: -0.40222 (-0.39282) + | > loss_dur: 0.16394 (0.10393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.20277 (15.51094) + | > current_lr: 0.00007 + | > step_time: 1.61210 (2.52911) + | > loader_time: 0.07730 (0.12076) + + + --> STEP: 123/234 -- GLOBAL_STEP: 64005 + | > loss: -0.22321 (-0.28720) + | > log_mle: -0.37081 (-0.39286) + | > loss_dur: 0.14760 (0.10566) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.24520 (15.77874) + | > current_lr: 0.00007 + | > step_time: 1.50260 (2.51673) + | > loader_time: 0.00260 (0.11807) + + + --> STEP: 128/234 -- GLOBAL_STEP: 64010 + | > loss: -0.27207 (-0.28694) + | > log_mle: -0.42204 (-0.39493) + | > loss_dur: 0.14997 (0.10799) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.07831 (16.53325) + | > current_lr: 0.00007 + | > step_time: 2.38580 (2.48323) + | > loader_time: 0.00280 (0.11356) + + + --> STEP: 133/234 -- GLOBAL_STEP: 64015 + | > loss: -0.29589 (-0.28714) + | > log_mle: -0.46053 (-0.39740) + | > loss_dur: 0.16464 (0.11025) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.36791 (17.12454) + | > current_lr: 0.00007 + | > step_time: 2.28820 (2.46672) + | > loader_time: 0.00510 (0.11013) + + + --> STEP: 138/234 -- GLOBAL_STEP: 64020 + | > loss: -0.24414 (-0.28708) + | > log_mle: -0.40443 (-0.39971) + | > loss_dur: 0.16029 (0.11263) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.57657 (17.69480) + | > current_lr: 0.00007 + | > step_time: 1.32530 (2.45253) + | > loader_time: 0.00170 (0.10698) + + + --> STEP: 143/234 -- GLOBAL_STEP: 64025 + | > loss: -0.32748 (-0.28743) + | > log_mle: -0.54840 (-0.40274) + | > loss_dur: 0.22091 (0.11531) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.14816 (18.52906) + | > current_lr: 0.00007 + | > step_time: 4.41180 (2.47738) + | > loader_time: 0.00420 (0.10534) + + + --> STEP: 148/234 -- GLOBAL_STEP: 64030 + | > loss: -0.29957 (-0.28818) + | > log_mle: -0.46458 (-0.40584) + | > loss_dur: 0.16502 (0.11766) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.45019 (19.23053) + | > current_lr: 0.00007 + | > step_time: 1.01270 (2.49645) + | > loader_time: 0.07760 (0.10294) + + + --> STEP: 153/234 -- GLOBAL_STEP: 64035 + | > loss: -0.40245 (-0.29021) + | > log_mle: -0.59768 (-0.41033) + | > loss_dur: 0.19524 (0.12013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.19263 (20.24583) + | > current_lr: 0.00007 + | > step_time: 4.00610 (2.51050) + | > loader_time: 0.09710 (0.10199) + + + --> STEP: 158/234 -- GLOBAL_STEP: 64040 + | > loss: -0.32145 (-0.29172) + | > log_mle: -0.53226 (-0.41430) + | > loss_dur: 0.21080 (0.12258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.84633 (21.06055) + | > current_lr: 0.00007 + | > step_time: 2.40840 (2.50475) + | > loader_time: 0.00550 (0.10053) + + + --> STEP: 163/234 -- GLOBAL_STEP: 64045 + | > loss: -0.30979 (-0.29361) + | > log_mle: -0.50582 (-0.41837) + | > loss_dur: 0.19602 (0.12475) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.13214 (21.90486) + | > current_lr: 0.00007 + | > step_time: 3.09700 (2.50956) + | > loader_time: 0.08540 (0.09869) + + + --> STEP: 168/234 -- GLOBAL_STEP: 64050 + | > loss: -0.34758 (-0.29569) + | > log_mle: -0.55906 (-0.42261) + | > loss_dur: 0.21148 (0.12692) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.23602 (23.08591) + | > current_lr: 0.00007 + | > step_time: 2.90140 (2.60993) + | > loader_time: 0.00300 (0.09819) + + + --> STEP: 173/234 -- GLOBAL_STEP: 64055 + | > loss: -0.35701 (-0.29820) + | > log_mle: -0.56828 (-0.42761) + | > loss_dur: 0.21127 (0.12941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.37275 (24.18298) + | > current_lr: 0.00007 + | > step_time: 1.90120 (2.60172) + | > loader_time: 0.00290 (0.09656) + + + --> STEP: 178/234 -- GLOBAL_STEP: 64060 + | > loss: -0.40091 (-0.30039) + | > log_mle: -0.62741 (-0.43227) + | > loss_dur: 0.22650 (0.13189) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.10122 (25.22857) + | > current_lr: 0.00007 + | > step_time: 1.10250 (2.59043) + | > loader_time: 0.00340 (0.09441) + + + --> STEP: 183/234 -- GLOBAL_STEP: 64065 + | > loss: -0.43292 (-0.30253) + | > log_mle: -0.64049 (-0.43681) + | > loss_dur: 0.20757 (0.13429) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.89225 (26.14800) + | > current_lr: 0.00007 + | > step_time: 3.68910 (2.59291) + | > loader_time: 0.00300 (0.09331) + + + --> STEP: 188/234 -- GLOBAL_STEP: 64070 + | > loss: -0.42872 (-0.30487) + | > log_mle: -0.64536 (-0.44139) + | > loss_dur: 0.21665 (0.13653) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.66589 (27.19398) + | > current_lr: 0.00007 + | > step_time: 3.47460 (2.70143) + | > loader_time: 0.01310 (0.09254) + + + --> STEP: 193/234 -- GLOBAL_STEP: 64075 + | > loss: -0.42613 (-0.30709) + | > log_mle: -0.64035 (-0.44558) + | > loss_dur: 0.21422 (0.13848) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.47126 (28.32856) + | > current_lr: 0.00007 + | > step_time: 4.60570 (2.72250) + | > loader_time: 0.08940 (0.09122) + + + --> STEP: 198/234 -- GLOBAL_STEP: 64080 + | > loss: -0.41735 (-0.30937) + | > log_mle: -0.63347 (-0.44972) + | > loss_dur: 0.21612 (0.14035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.00593 (29.18110) + | > current_lr: 0.00007 + | > step_time: 1.71500 (2.75944) + | > loader_time: 0.29860 (0.09300) + + + --> STEP: 203/234 -- GLOBAL_STEP: 64085 + | > loss: -0.33934 (-0.31124) + | > log_mle: -0.54659 (-0.45360) + | > loss_dur: 0.20726 (0.14236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.36945 (29.88282) + | > current_lr: 0.00007 + | > step_time: 3.40330 (2.76290) + | > loader_time: 0.00390 (0.09163) + + + --> STEP: 208/234 -- GLOBAL_STEP: 64090 + | > loss: -0.40924 (-0.31389) + | > log_mle: -0.64290 (-0.45832) + | > loss_dur: 0.23365 (0.14444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.33543 (30.72028) + | > current_lr: 0.00007 + | > step_time: 1.48570 (2.82544) + | > loader_time: 0.00550 (0.09556) + + + --> STEP: 213/234 -- GLOBAL_STEP: 64095 + | > loss: -0.46202 (-0.31712) + | > log_mle: -0.70039 (-0.46360) + | > loss_dur: 0.23837 (0.14648) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.93661 (31.65819) + | > current_lr: 0.00007 + | > step_time: 13.69920 (2.92202) + | > loader_time: 2.09190 (0.10548) + + + --> STEP: 218/234 -- GLOBAL_STEP: 64100 + | > loss: -0.42087 (-0.31992) + | > log_mle: -0.66014 (-0.46835) + | > loss_dur: 0.23927 (0.14843) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.55288 (32.83792) + | > current_lr: 0.00007 + | > step_time: 4.09740 (2.99311) + | > loader_time: 0.00490 (0.10484) + + + --> STEP: 223/234 -- GLOBAL_STEP: 64105 + | > loss: -0.47065 (-0.32294) + | > log_mle: -0.70454 (-0.47340) + | > loss_dur: 0.23389 (0.15047) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.43835 (34.23283) + | > current_lr: 0.00007 + | > step_time: 0.22740 (2.94972) + | > loader_time: 0.00320 (0.10257) + + + --> STEP: 228/234 -- GLOBAL_STEP: 64110 + | > loss: -0.41648 (-0.32584) + | > log_mle: -0.68701 (-0.47851) + | > loss_dur: 0.27053 (0.15267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.07564 (35.63337) + | > current_lr: 0.00007 + | > step_time: 0.23580 (2.89025) + | > loader_time: 0.00340 (0.10039) + + + --> STEP: 233/234 -- GLOBAL_STEP: 64115 + | > loss: 0.02427 (-0.32612) + | > log_mle: -0.64539 (-0.48469) + | > loss_dur: 0.66966 (0.15857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 123.50503 (37.25169) + | > current_lr: 0.00007 + | > step_time: 0.19110 (2.83374) + | > loader_time: 0.00250 (0.09832) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.57649 (+0.13140) + | > avg_loss: -0.28062 (+0.04209) + | > avg_log_mle: -0.52478 (+0.02020) + | > avg_loss_dur: 0.24416 (+0.02190) + + + > EPOCH: 274/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 18:58:56)  + + --> STEP: 4/234 -- GLOBAL_STEP: 64120 + | > loss: -0.31143 (-0.29930) + | > log_mle: -0.39876 (-0.39620) + | > loss_dur: 0.08733 (0.09690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.93611 (23.07343) + | > current_lr: 0.00007 + | > step_time: 3.19210 (3.30076) + | > loader_time: 0.00590 (0.02621) + + + --> STEP: 9/234 -- GLOBAL_STEP: 64125 + | > loss: -0.29627 (-0.31319) + | > log_mle: -0.41295 (-0.40367) + | > loss_dur: 0.11668 (0.09048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.74653 (17.86419) + | > current_lr: 0.00007 + | > step_time: 4.89220 (3.65741) + | > loader_time: 0.00110 (0.04403) + + + --> STEP: 14/234 -- GLOBAL_STEP: 64130 + | > loss: -0.34074 (-0.32173) + | > log_mle: -0.41258 (-0.40666) + | > loss_dur: 0.07184 (0.08493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.15024 (16.87548) + | > current_lr: 0.00007 + | > step_time: 3.80640 (3.55119) + | > loader_time: 0.00350 (0.02983) + + + --> STEP: 19/234 -- GLOBAL_STEP: 64135 + | > loss: -0.34887 (-0.32826) + | > log_mle: -0.41372 (-0.40807) + | > loss_dur: 0.06484 (0.07981) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.89508 (15.40779) + | > current_lr: 0.00007 + | > step_time: 2.60050 (3.53776) + | > loader_time: 0.00120 (0.05204) + + + --> STEP: 24/234 -- GLOBAL_STEP: 64140 + | > loss: -0.34311 (-0.33103) + | > log_mle: -0.40182 (-0.40773) + | > loss_dur: 0.05871 (0.07669) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.09429 (14.61725) + | > current_lr: 0.00007 + | > step_time: 3.09360 (3.66368) + | > loader_time: 0.00380 (0.04188) + + + --> STEP: 29/234 -- GLOBAL_STEP: 64145 + | > loss: -0.32572 (-0.33172) + | > log_mle: -0.40004 (-0.40702) + | > loss_dur: 0.07431 (0.07530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.78703 (13.91870) + | > current_lr: 0.00007 + | > step_time: 1.46910 (3.46249) + | > loader_time: 0.00180 (0.04099) + + + --> STEP: 34/234 -- GLOBAL_STEP: 64150 + | > loss: -0.31839 (-0.32939) + | > log_mle: -0.39587 (-0.40511) + | > loss_dur: 0.07748 (0.07572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.98438 (13.56960) + | > current_lr: 0.00007 + | > step_time: 1.96840 (3.14326) + | > loader_time: 0.00160 (0.03795) + + + --> STEP: 39/234 -- GLOBAL_STEP: 64155 + | > loss: -0.29616 (-0.32549) + | > log_mle: -0.38349 (-0.40219) + | > loss_dur: 0.08733 (0.07670) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.39389 (13.39341) + | > current_lr: 0.00007 + | > step_time: 1.18100 (2.93449) + | > loader_time: 0.00230 (0.03577) + + + --> STEP: 44/234 -- GLOBAL_STEP: 64160 + | > loss: -0.30264 (-0.32220) + | > log_mle: -0.37044 (-0.39950) + | > loss_dur: 0.06780 (0.07730) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.72760 (13.03545) + | > current_lr: 0.00007 + | > step_time: 1.55360 (2.78487) + | > loader_time: 0.00160 (0.03202) + + + --> STEP: 49/234 -- GLOBAL_STEP: 64165 + | > loss: -0.31793 (-0.32023) + | > log_mle: -0.39325 (-0.39830) + | > loss_dur: 0.07531 (0.07807) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.90256 (12.79653) + | > current_lr: 0.00007 + | > step_time: 2.93090 (2.65372) + | > loader_time: 0.00400 (0.02897) + + + --> STEP: 54/234 -- GLOBAL_STEP: 64170 + | > loss: -0.30453 (-0.31749) + | > log_mle: -0.38280 (-0.39642) + | > loss_dur: 0.07827 (0.07893) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42863 (12.62465) + | > current_lr: 0.00007 + | > step_time: 1.15750 (2.54578) + | > loader_time: 0.00170 (0.02981) + + + --> STEP: 59/234 -- GLOBAL_STEP: 64175 + | > loss: -0.29780 (-0.31572) + | > log_mle: -0.38295 (-0.39501) + | > loss_dur: 0.08515 (0.07929) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.90889 (12.31863) + | > current_lr: 0.00007 + | > step_time: 1.89330 (2.50820) + | > loader_time: 0.00190 (0.02875) + + + --> STEP: 64/234 -- GLOBAL_STEP: 64180 + | > loss: -0.28442 (-0.31161) + | > log_mle: -0.36919 (-0.39337) + | > loss_dur: 0.08477 (0.08176) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.82381 (12.44000) + | > current_lr: 0.00007 + | > step_time: 0.93980 (2.47774) + | > loader_time: 0.00260 (0.02938) + + + --> STEP: 69/234 -- GLOBAL_STEP: 64185 + | > loss: -0.27680 (-0.30896) + | > log_mle: -0.36341 (-0.39140) + | > loss_dur: 0.08661 (0.08245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.87337 (12.27347) + | > current_lr: 0.00007 + | > step_time: 3.10150 (2.41975) + | > loader_time: 0.00390 (0.02746) + + + --> STEP: 74/234 -- GLOBAL_STEP: 64190 + | > loss: -0.23897 (-0.30500) + | > log_mle: -0.35414 (-0.38960) + | > loss_dur: 0.11517 (0.08459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.12761 (12.44985) + | > current_lr: 0.00007 + | > step_time: 2.09680 (2.40739) + | > loader_time: 0.00340 (0.02703) + + + --> STEP: 79/234 -- GLOBAL_STEP: 64195 + | > loss: -0.25069 (-0.30159) + | > log_mle: -0.36826 (-0.38797) + | > loss_dur: 0.11757 (0.08637) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.14329 (12.54095) + | > current_lr: 0.00007 + | > step_time: 1.67790 (2.35743) + | > loader_time: 0.00180 (0.02646) + + + --> STEP: 84/234 -- GLOBAL_STEP: 64200 + | > loss: -0.27108 (-0.29902) + | > log_mle: -0.36874 (-0.38680) + | > loss_dur: 0.09766 (0.08777) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.48446 (12.35376) + | > current_lr: 0.00007 + | > step_time: 2.41490 (2.34834) + | > loader_time: 0.09320 (0.02714) + + + --> STEP: 89/234 -- GLOBAL_STEP: 64205 + | > loss: -0.27728 (-0.29713) + | > log_mle: -0.38954 (-0.38643) + | > loss_dur: 0.11226 (0.08930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.06054 (12.37620) + | > current_lr: 0.00007 + | > step_time: 2.99570 (2.37008) + | > loader_time: 0.09880 (0.02895) + + + --> STEP: 94/234 -- GLOBAL_STEP: 64210 + | > loss: -0.28903 (-0.29554) + | > log_mle: -0.41910 (-0.38731) + | > loss_dur: 0.13007 (0.09176) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.80386 (12.60145) + | > current_lr: 0.00007 + | > step_time: 2.91340 (2.35345) + | > loader_time: 0.07940 (0.02837) + + + --> STEP: 99/234 -- GLOBAL_STEP: 64215 + | > loss: -0.27794 (-0.29426) + | > log_mle: -0.44486 (-0.38822) + | > loss_dur: 0.16692 (0.09396) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.44556 (13.35020) + | > current_lr: 0.00007 + | > step_time: 1.27710 (2.35764) + | > loader_time: 0.00150 (0.02783) + + + --> STEP: 104/234 -- GLOBAL_STEP: 64220 + | > loss: -0.31143 (-0.29322) + | > log_mle: -0.45632 (-0.38956) + | > loss_dur: 0.14489 (0.09634) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.75019 (13.86435) + | > current_lr: 0.00007 + | > step_time: 0.98870 (2.34326) + | > loader_time: 0.00220 (0.02920) + + + --> STEP: 109/234 -- GLOBAL_STEP: 64225 + | > loss: -0.24476 (-0.29120) + | > log_mle: -0.42871 (-0.39024) + | > loss_dur: 0.18394 (0.09904) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.27674 (14.15516) + | > current_lr: 0.00007 + | > step_time: 1.20560 (2.31040) + | > loader_time: 0.00280 (0.02950) + + + --> STEP: 114/234 -- GLOBAL_STEP: 64230 + | > loss: -0.26663 (-0.29022) + | > log_mle: -0.41057 (-0.39190) + | > loss_dur: 0.14394 (0.10168) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.49172 (14.58548) + | > current_lr: 0.00007 + | > step_time: 2.98470 (2.30919) + | > loader_time: 0.00240 (0.02838) + + + --> STEP: 119/234 -- GLOBAL_STEP: 64235 + | > loss: -0.25973 (-0.28881) + | > log_mle: -0.41235 (-0.39300) + | > loss_dur: 0.15262 (0.10420) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.85808 (14.90603) + | > current_lr: 0.00007 + | > step_time: 2.10360 (2.29814) + | > loader_time: 0.08710 (0.02936) + + + --> STEP: 124/234 -- GLOBAL_STEP: 64240 + | > loss: -0.28286 (-0.28742) + | > log_mle: -0.44053 (-0.39345) + | > loss_dur: 0.15767 (0.10604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.99515 (15.25402) + | > current_lr: 0.00007 + | > step_time: 4.21110 (2.30011) + | > loader_time: 0.00450 (0.03035) + + + --> STEP: 129/234 -- GLOBAL_STEP: 64245 + | > loss: -0.26493 (-0.28702) + | > log_mle: -0.42944 (-0.39557) + | > loss_dur: 0.16451 (0.10855) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.40307 (15.90687) + | > current_lr: 0.00007 + | > step_time: 1.49770 (2.29932) + | > loader_time: 0.00380 (0.02934) + + + --> STEP: 134/234 -- GLOBAL_STEP: 64250 + | > loss: -0.30605 (-0.28771) + | > log_mle: -0.48627 (-0.39862) + | > loss_dur: 0.18022 (0.11091) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.87223 (16.51436) + | > current_lr: 0.00007 + | > step_time: 2.19600 (2.28381) + | > loader_time: 0.00300 (0.02961) + + + --> STEP: 139/234 -- GLOBAL_STEP: 64255 + | > loss: -0.34724 (-0.28809) + | > log_mle: -0.54394 (-0.40137) + | > loss_dur: 0.19670 (0.11328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.51501 (17.36712) + | > current_lr: 0.00007 + | > step_time: 4.20720 (2.28943) + | > loader_time: 0.09270 (0.02930) + + + --> STEP: 144/234 -- GLOBAL_STEP: 64260 + | > loss: -0.32961 (-0.28848) + | > log_mle: -0.52216 (-0.40444) + | > loss_dur: 0.19256 (0.11596) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.42543 (18.07730) + | > current_lr: 0.00007 + | > step_time: 6.80560 (2.32662) + | > loader_time: 0.18340 (0.03032) + + + --> STEP: 149/234 -- GLOBAL_STEP: 64265 + | > loss: -0.38242 (-0.28986) + | > log_mle: -0.57286 (-0.40798) + | > loss_dur: 0.19044 (0.11812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.49824 (18.89161) + | > current_lr: 0.00007 + | > step_time: 3.40290 (2.38621) + | > loader_time: 0.07410 (0.03125) + + + --> STEP: 154/234 -- GLOBAL_STEP: 64270 + | > loss: -0.35191 (-0.29167) + | > log_mle: -0.53268 (-0.41225) + | > loss_dur: 0.18077 (0.12058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.84021 (19.64697) + | > current_lr: 0.00007 + | > step_time: 1.19440 (2.36847) + | > loader_time: 0.08850 (0.03265) + + + --> STEP: 159/234 -- GLOBAL_STEP: 64275 + | > loss: -0.35991 (-0.29343) + | > log_mle: -0.55080 (-0.41635) + | > loss_dur: 0.19089 (0.12291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.52452 (20.51679) + | > current_lr: 0.00007 + | > step_time: 3.01430 (2.38032) + | > loader_time: 0.08620 (0.03344) + + + --> STEP: 164/234 -- GLOBAL_STEP: 64280 + | > loss: -0.33406 (-0.29514) + | > log_mle: -0.54206 (-0.42038) + | > loss_dur: 0.20800 (0.12524) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.74781 (21.33120) + | > current_lr: 0.00007 + | > step_time: 1.25990 (2.43195) + | > loader_time: 0.00330 (0.03522) + + + --> STEP: 169/234 -- GLOBAL_STEP: 64285 + | > loss: -0.34191 (-0.29738) + | > log_mle: -0.54694 (-0.42479) + | > loss_dur: 0.20503 (0.12741) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.55411 (22.32378) + | > current_lr: 0.00007 + | > step_time: 1.79260 (2.40850) + | > loader_time: 0.00510 (0.03477) + + + --> STEP: 174/234 -- GLOBAL_STEP: 64290 + | > loss: -0.42456 (-0.30049) + | > log_mle: -0.63154 (-0.43028) + | > loss_dur: 0.20698 (0.12979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.28734 (23.71936) + | > current_lr: 0.00007 + | > step_time: 1.31640 (2.47915) + | > loader_time: 0.08610 (0.03746) + + + --> STEP: 179/234 -- GLOBAL_STEP: 64295 + | > loss: -0.39324 (-0.30270) + | > log_mle: -0.63221 (-0.43504) + | > loss_dur: 0.23897 (0.13234) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.40913 (24.84089) + | > current_lr: 0.00007 + | > step_time: 2.30670 (2.46184) + | > loader_time: 0.00280 (0.03741) + + + --> STEP: 184/234 -- GLOBAL_STEP: 64300 + | > loss: -0.36712 (-0.30461) + | > log_mle: -0.59111 (-0.43930) + | > loss_dur: 0.22399 (0.13470) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.41975 (25.61293) + | > current_lr: 0.00007 + | > step_time: 1.50410 (2.44809) + | > loader_time: 0.00240 (0.03789) + + + --> STEP: 189/234 -- GLOBAL_STEP: 64305 + | > loss: -0.37985 (-0.30671) + | > log_mle: -0.59382 (-0.44372) + | > loss_dur: 0.21396 (0.13701) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.37779 (26.93074) + | > current_lr: 0.00007 + | > step_time: 5.80590 (2.49302) + | > loader_time: 0.09010 (0.03897) + + + --> STEP: 194/234 -- GLOBAL_STEP: 64310 + | > loss: -0.40974 (-0.30950) + | > log_mle: -0.62829 (-0.44828) + | > loss_dur: 0.21854 (0.13878) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.10924 (27.74287) + | > current_lr: 0.00007 + | > step_time: 3.48780 (2.55248) + | > loader_time: 0.00370 (0.03951) + + + --> STEP: 199/234 -- GLOBAL_STEP: 64315 + | > loss: -0.42992 (-0.31196) + | > log_mle: -0.64364 (-0.45267) + | > loss_dur: 0.21372 (0.14071) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.72464 (28.65283) + | > current_lr: 0.00007 + | > step_time: 3.59290 (2.59943) + | > loader_time: 0.00640 (0.04039) + + + --> STEP: 204/234 -- GLOBAL_STEP: 64320 + | > loss: -0.41815 (-0.31358) + | > log_mle: -0.65161 (-0.45640) + | > loss_dur: 0.23346 (0.14282) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.23546 (30.10049) + | > current_lr: 0.00007 + | > step_time: 6.70070 (2.66768) + | > loader_time: 0.18890 (0.04179) + + + --> STEP: 209/234 -- GLOBAL_STEP: 64325 + | > loss: -0.39897 (-0.31573) + | > log_mle: -0.61217 (-0.46053) + | > loss_dur: 0.21320 (0.14480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.70293 (30.79074) + | > current_lr: 0.00007 + | > step_time: 2.30770 (2.76563) + | > loader_time: 0.08960 (0.04298) + + + --> STEP: 214/234 -- GLOBAL_STEP: 64330 + | > loss: -0.42748 (-0.31883) + | > log_mle: -0.63993 (-0.46563) + | > loss_dur: 0.21244 (0.14681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.89946 (31.93893) + | > current_lr: 0.00007 + | > step_time: 2.59040 (2.80142) + | > loader_time: 0.10220 (0.04294) + + + --> STEP: 219/234 -- GLOBAL_STEP: 64335 + | > loss: -0.50886 (-0.32176) + | > log_mle: -0.75338 (-0.47065) + | > loss_dur: 0.24452 (0.14889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 96.23008 (33.17468) + | > current_lr: 0.00007 + | > step_time: 11.11010 (2.87875) + | > loader_time: 0.09640 (0.04422) + + + --> STEP: 224/234 -- GLOBAL_STEP: 64340 + | > loss: -0.47895 (-0.32452) + | > log_mle: -0.70918 (-0.47538) + | > loss_dur: 0.23023 (0.15086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.27479 (34.07960) + | > current_lr: 0.00007 + | > step_time: 0.23950 (2.84414) + | > loader_time: 0.00310 (0.04332) + + + --> STEP: 229/234 -- GLOBAL_STEP: 64345 + | > loss: -0.43925 (-0.32753) + | > log_mle: -0.73673 (-0.48069) + | > loss_dur: 0.29749 (0.15316) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 121.07582 (35.49773) + | > current_lr: 0.00007 + | > step_time: 0.24980 (2.78727) + | > loader_time: 0.00290 (0.04245) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12774 (-0.44875) + | > avg_loss: -0.32086 (-0.04023) + | > avg_log_mle: -0.55641 (-0.03162) + | > avg_loss_dur: 0.23555 (-0.00861) + + + > EPOCH: 275/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 19:10:57)  + + --> STEP: 0/234 -- GLOBAL_STEP: 64350 + | > loss: -0.31161 (-0.31161) + | > log_mle: -0.48454 (-0.48454) + | > loss_dur: 0.17293 (0.17293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.07710 (30.07710) + | > current_lr: 0.00007 + | > step_time: 10.90790 (10.90788) + | > loader_time: 19.65480 (19.65478) + + + --> STEP: 5/234 -- GLOBAL_STEP: 64355 + | > loss: -0.31228 (-0.30554) + | > log_mle: -0.40419 (-0.40154) + | > loss_dur: 0.09191 (0.09599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.18645 (18.61236) + | > current_lr: 0.00007 + | > step_time: 3.69830 (2.96326) + | > loader_time: 0.89320 (0.23684) + + + --> STEP: 10/234 -- GLOBAL_STEP: 64360 + | > loss: -0.31654 (-0.31779) + | > log_mle: -0.40603 (-0.40661) + | > loss_dur: 0.08949 (0.08882) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.03955 (18.79965) + | > current_lr: 0.00007 + | > step_time: 2.51790 (3.78430) + | > loader_time: 0.00120 (0.13017) + + + --> STEP: 15/234 -- GLOBAL_STEP: 64365 + | > loss: -0.34359 (-0.32527) + | > log_mle: -0.41428 (-0.40898) + | > loss_dur: 0.07069 (0.08372) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.85131 (18.00126) + | > current_lr: 0.00007 + | > step_time: 2.17130 (2.95358) + | > loader_time: 0.08630 (0.09744) + + + --> STEP: 20/234 -- GLOBAL_STEP: 64370 + | > loss: -0.34167 (-0.33072) + | > log_mle: -0.41132 (-0.40959) + | > loss_dur: 0.06965 (0.07887) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.19736 (16.41259) + | > current_lr: 0.00007 + | > step_time: 0.96920 (2.50570) + | > loader_time: 0.00110 (0.07342) + + + --> STEP: 25/234 -- GLOBAL_STEP: 64375 + | > loss: -0.32452 (-0.33196) + | > log_mle: -0.39542 (-0.40845) + | > loss_dur: 0.07090 (0.07649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.02022 (15.47792) + | > current_lr: 0.00007 + | > step_time: 1.90820 (2.38543) + | > loader_time: 0.00180 (0.06545) + + + --> STEP: 30/234 -- GLOBAL_STEP: 64380 + | > loss: -0.32577 (-0.33337) + | > log_mle: -0.40096 (-0.40805) + | > loss_dur: 0.07519 (0.07468) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.05512 (14.66167) + | > current_lr: 0.00007 + | > step_time: 1.33520 (2.34847) + | > loader_time: 0.00190 (0.05810) + + + --> STEP: 35/234 -- GLOBAL_STEP: 64385 + | > loss: -0.28192 (-0.33004) + | > log_mle: -0.37893 (-0.40587) + | > loss_dur: 0.09701 (0.07583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.47628 (14.55425) + | > current_lr: 0.00007 + | > step_time: 1.24960 (2.21046) + | > loader_time: 0.00180 (0.05258) + + + --> STEP: 40/234 -- GLOBAL_STEP: 64390 + | > loss: -0.28741 (-0.32599) + | > log_mle: -0.37959 (-0.40308) + | > loss_dur: 0.09219 (0.07710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.55587 (14.32560) + | > current_lr: 0.00007 + | > step_time: 1.46650 (2.09888) + | > loader_time: 0.00180 (0.04627) + + + --> STEP: 45/234 -- GLOBAL_STEP: 64395 + | > loss: -0.29390 (-0.32319) + | > log_mle: -0.39216 (-0.40095) + | > loss_dur: 0.09826 (0.07776) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.28111 (14.01321) + | > current_lr: 0.00007 + | > step_time: 2.70290 (2.05842) + | > loader_time: 0.08790 (0.04327) + + + --> STEP: 50/234 -- GLOBAL_STEP: 64400 + | > loss: -0.29890 (-0.32148) + | > log_mle: -0.37791 (-0.39945) + | > loss_dur: 0.07901 (0.07796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.91569 (13.67367) + | > current_lr: 0.00007 + | > step_time: 1.30690 (1.99414) + | > loader_time: 0.00250 (0.04092) + + + --> STEP: 55/234 -- GLOBAL_STEP: 64405 + | > loss: -0.31005 (-0.31869) + | > log_mle: -0.38712 (-0.39749) + | > loss_dur: 0.07707 (0.07880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.44777 (13.55365) + | > current_lr: 0.00007 + | > step_time: 1.22630 (1.96971) + | > loader_time: 0.00170 (0.03915) + + + --> STEP: 60/234 -- GLOBAL_STEP: 64410 + | > loss: -0.25193 (-0.31583) + | > log_mle: -0.38282 (-0.39603) + | > loss_dur: 0.13089 (0.08021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.85457 (13.46482) + | > current_lr: 0.00007 + | > step_time: 3.60260 (1.99950) + | > loader_time: 0.00280 (0.03900) + + + --> STEP: 65/234 -- GLOBAL_STEP: 64415 + | > loss: -0.27872 (-0.31177) + | > log_mle: -0.37040 (-0.39410) + | > loss_dur: 0.09168 (0.08233) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.30390 (13.63709) + | > current_lr: 0.00007 + | > step_time: 1.59940 (2.01807) + | > loader_time: 0.00280 (0.04150) + + + --> STEP: 70/234 -- GLOBAL_STEP: 64420 + | > loss: -0.24577 (-0.30857) + | > log_mle: -0.35776 (-0.39206) + | > loss_dur: 0.11200 (0.08350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.69262 (13.37565) + | > current_lr: 0.00007 + | > step_time: 1.69130 (2.06688) + | > loader_time: 0.00200 (0.04118) + + + --> STEP: 75/234 -- GLOBAL_STEP: 64425 + | > loss: -0.23867 (-0.30493) + | > log_mle: -0.36556 (-0.39065) + | > loss_dur: 0.12689 (0.08572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.13615 (13.43752) + | > current_lr: 0.00007 + | > step_time: 1.90880 (2.02415) + | > loader_time: 0.00230 (0.04065) + + + --> STEP: 80/234 -- GLOBAL_STEP: 64430 + | > loss: -0.27192 (-0.30234) + | > log_mle: -0.36049 (-0.38907) + | > loss_dur: 0.08856 (0.08673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.29494 (13.22742) + | > current_lr: 0.00007 + | > step_time: 1.92630 (2.04804) + | > loader_time: 0.00230 (0.04163) + + + --> STEP: 85/234 -- GLOBAL_STEP: 64435 + | > loss: -0.25915 (-0.29994) + | > log_mle: -0.36370 (-0.38796) + | > loss_dur: 0.10455 (0.08802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.41863 (13.13092) + | > current_lr: 0.00007 + | > step_time: 1.18400 (2.01731) + | > loader_time: 0.00180 (0.04018) + + + --> STEP: 90/234 -- GLOBAL_STEP: 64440 + | > loss: -0.25648 (-0.29778) + | > log_mle: -0.38322 (-0.38773) + | > loss_dur: 0.12674 (0.08995) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.11008 (13.22378) + | > current_lr: 0.00007 + | > step_time: 2.79580 (2.06095) + | > loader_time: 0.00320 (0.03993) + + + --> STEP: 95/234 -- GLOBAL_STEP: 64445 + | > loss: -0.31974 (-0.29680) + | > log_mle: -0.46632 (-0.38933) + | > loss_dur: 0.14658 (0.09252) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.00765 (13.72989) + | > current_lr: 0.00007 + | > step_time: 1.80040 (2.07041) + | > loader_time: 0.00420 (0.03890) + + + --> STEP: 100/234 -- GLOBAL_STEP: 64450 + | > loss: -0.26653 (-0.29512) + | > log_mle: -0.39425 (-0.38954) + | > loss_dur: 0.12772 (0.09442) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.94908 (13.86184) + | > current_lr: 0.00007 + | > step_time: 1.71860 (2.08211) + | > loader_time: 0.00210 (0.03887) + + + --> STEP: 105/234 -- GLOBAL_STEP: 64455 + | > loss: -0.24953 (-0.29407) + | > log_mle: -0.37571 (-0.39084) + | > loss_dur: 0.12618 (0.09677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.65344 (14.21850) + | > current_lr: 0.00007 + | > step_time: 3.49870 (2.10858) + | > loader_time: 0.00350 (0.03809) + + + --> STEP: 110/234 -- GLOBAL_STEP: 64460 + | > loss: -0.24684 (-0.29214) + | > log_mle: -0.39544 (-0.39164) + | > loss_dur: 0.14860 (0.09951) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.09496 (14.58442) + | > current_lr: 0.00007 + | > step_time: 1.61700 (2.09901) + | > loader_time: 0.00310 (0.03731) + + + --> STEP: 115/234 -- GLOBAL_STEP: 64465 + | > loss: -0.24879 (-0.29114) + | > log_mle: -0.41749 (-0.39349) + | > loss_dur: 0.16870 (0.10235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.18914 (15.15231) + | > current_lr: 0.00007 + | > step_time: 4.29550 (2.10512) + | > loader_time: 0.09470 (0.03811) + + + --> STEP: 120/234 -- GLOBAL_STEP: 64470 + | > loss: -0.29272 (-0.28987) + | > log_mle: -0.45981 (-0.39488) + | > loss_dur: 0.16709 (0.10501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.72857 (15.61847) + | > current_lr: 0.00007 + | > step_time: 3.89090 (2.11582) + | > loader_time: 0.00260 (0.03874) + + + --> STEP: 125/234 -- GLOBAL_STEP: 64475 + | > loss: -0.26429 (-0.28814) + | > log_mle: -0.44348 (-0.39507) + | > loss_dur: 0.17920 (0.10693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.67018 (15.95434) + | > current_lr: 0.00007 + | > step_time: 1.31120 (2.13878) + | > loader_time: 0.28820 (0.04252) + + + --> STEP: 130/234 -- GLOBAL_STEP: 64480 + | > loss: -0.29062 (-0.28802) + | > log_mle: -0.46650 (-0.39734) + | > loss_dur: 0.17588 (0.10932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.03657 (16.44246) + | > current_lr: 0.00007 + | > step_time: 3.70640 (2.14508) + | > loader_time: 0.00210 (0.04098) + + + --> STEP: 135/234 -- GLOBAL_STEP: 64485 + | > loss: -0.26039 (-0.28848) + | > log_mle: -0.39467 (-0.39981) + | > loss_dur: 0.13429 (0.11133) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.79397 (17.01925) + | > current_lr: 0.00007 + | > step_time: 1.99620 (2.13432) + | > loader_time: 0.00270 (0.04091) + + + --> STEP: 140/234 -- GLOBAL_STEP: 64490 + | > loss: -0.24735 (-0.28896) + | > log_mle: -0.42631 (-0.40292) + | > loss_dur: 0.17896 (0.11396) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.32094 (17.77616) + | > current_lr: 0.00007 + | > step_time: 2.02910 (2.16065) + | > loader_time: 0.00210 (0.04198) + + + --> STEP: 145/234 -- GLOBAL_STEP: 64495 + | > loss: -0.35966 (-0.29016) + | > log_mle: -0.53423 (-0.40676) + | > loss_dur: 0.17457 (0.11660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.02730 (18.61219) + | > current_lr: 0.00007 + | > step_time: 5.20520 (2.17740) + | > loader_time: 0.09210 (0.04338) + + + --> STEP: 150/234 -- GLOBAL_STEP: 64500 + | > loss: -0.29841 (-0.29119) + | > log_mle: -0.50041 (-0.41012) + | > loss_dur: 0.20200 (0.11893) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.87517 (19.58007) + | > current_lr: 0.00007 + | > step_time: 5.20390 (2.23154) + | > loader_time: 0.28700 (0.04459) + + + --> STEP: 155/234 -- GLOBAL_STEP: 64505 + | > loss: -0.37351 (-0.29304) + | > log_mle: -0.57256 (-0.41444) + | > loss_dur: 0.19905 (0.12140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.82437 (20.53922) + | > current_lr: 0.00007 + | > step_time: 13.90430 (2.34431) + | > loader_time: 0.09970 (0.04561) + + + --> STEP: 160/234 -- GLOBAL_STEP: 64510 + | > loss: -0.37351 (-0.29466) + | > log_mle: -0.58013 (-0.41838) + | > loss_dur: 0.20662 (0.12372) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.25900 (21.24816) + | > current_lr: 0.00007 + | > step_time: 10.30800 (2.41314) + | > loader_time: 0.00300 (0.04597) + + + --> STEP: 165/234 -- GLOBAL_STEP: 64515 + | > loss: -0.37654 (-0.29649) + | > log_mle: -0.58230 (-0.42228) + | > loss_dur: 0.20576 (0.12579) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.77813 (22.04322) + | > current_lr: 0.00007 + | > step_time: 3.82290 (2.48434) + | > loader_time: 0.00330 (0.04581) + + + --> STEP: 170/234 -- GLOBAL_STEP: 64520 + | > loss: -0.40494 (-0.29868) + | > log_mle: -0.61604 (-0.42665) + | > loss_dur: 0.21110 (0.12797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.15624 (23.11778) + | > current_lr: 0.00007 + | > step_time: 2.79660 (2.50319) + | > loader_time: 0.00170 (0.04557) + + + --> STEP: 175/234 -- GLOBAL_STEP: 64525 + | > loss: -0.36276 (-0.30152) + | > log_mle: -0.59522 (-0.43181) + | > loss_dur: 0.23245 (0.13029) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.55693 (24.03602) + | > current_lr: 0.00007 + | > step_time: 7.40540 (2.55280) + | > loader_time: 0.08280 (0.04637) + + + --> STEP: 180/234 -- GLOBAL_STEP: 64530 + | > loss: -0.37867 (-0.30384) + | > log_mle: -0.59070 (-0.43659) + | > loss_dur: 0.21203 (0.13275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.21999 (25.05181) + | > current_lr: 0.00007 + | > step_time: 5.00380 (2.59351) + | > loader_time: 0.00470 (0.04728) + + + --> STEP: 185/234 -- GLOBAL_STEP: 64535 + | > loss: -0.41162 (-0.30584) + | > log_mle: -0.63052 (-0.44089) + | > loss_dur: 0.21890 (0.13504) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.64637 (25.95922) + | > current_lr: 0.00007 + | > step_time: 3.31020 (2.64605) + | > loader_time: 0.08890 (0.04761) + + + --> STEP: 190/234 -- GLOBAL_STEP: 64540 + | > loss: -0.38863 (-0.30777) + | > log_mle: -0.59173 (-0.44499) + | > loss_dur: 0.20310 (0.13721) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.73732 (26.91885) + | > current_lr: 0.00007 + | > step_time: 8.08720 (2.74680) + | > loader_time: 0.30120 (0.04860) + + + --> STEP: 195/234 -- GLOBAL_STEP: 64545 + | > loss: -0.39381 (-0.31034) + | > log_mle: -0.62188 (-0.44952) + | > loss_dur: 0.22807 (0.13918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.94085 (27.77211) + | > current_lr: 0.00007 + | > step_time: 9.20220 (2.83371) + | > loader_time: 0.09390 (0.04839) + + + --> STEP: 200/234 -- GLOBAL_STEP: 64550 + | > loss: -0.38381 (-0.31265) + | > log_mle: -0.62908 (-0.45378) + | > loss_dur: 0.24527 (0.14113) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.57971 (28.56352) + | > current_lr: 0.00007 + | > step_time: 6.80430 (2.88784) + | > loader_time: 0.09520 (0.05174) + + + --> STEP: 205/234 -- GLOBAL_STEP: 64555 + | > loss: -0.39476 (-0.31475) + | > log_mle: -0.61480 (-0.45790) + | > loss_dur: 0.22003 (0.14315) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.52856 (29.30572) + | > current_lr: 0.00007 + | > step_time: 1.59120 (2.89239) + | > loader_time: 0.00300 (0.05105) + + + --> STEP: 210/234 -- GLOBAL_STEP: 64560 + | > loss: -0.45728 (-0.31773) + | > log_mle: -0.69304 (-0.46287) + | > loss_dur: 0.23576 (0.14513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 123.93644 (30.61691) + | > current_lr: 0.00007 + | > step_time: 3.59530 (2.92918) + | > loader_time: 0.00240 (0.05262) + + + --> STEP: 215/234 -- GLOBAL_STEP: 64565 + | > loss: -0.40368 (-0.32079) + | > log_mle: -0.63390 (-0.46796) + | > loss_dur: 0.23022 (0.14717) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 96.26999 (31.89633) + | > current_lr: 0.00007 + | > step_time: 3.89660 (3.02008) + | > loader_time: 0.18680 (0.05558) + + + --> STEP: 220/234 -- GLOBAL_STEP: 64570 + | > loss: -0.44526 (-0.32399) + | > log_mle: -0.68854 (-0.47328) + | > loss_dur: 0.24328 (0.14929) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.29955 (33.19869) + | > current_lr: 0.00007 + | > step_time: 4.52210 (3.03064) + | > loader_time: 0.00770 (0.05557) + + + --> STEP: 225/234 -- GLOBAL_STEP: 64575 + | > loss: -0.52450 (-0.32708) + | > log_mle: -0.77174 (-0.47836) + | > loss_dur: 0.24724 (0.15128) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.83018 (34.18303) + | > current_lr: 0.00007 + | > step_time: 0.76980 (3.02187) + | > loader_time: 0.00570 (0.05581) + + + --> STEP: 230/234 -- GLOBAL_STEP: 64580 + | > loss: -0.49438 (-0.33003) + | > log_mle: -0.79466 (-0.48389) + | > loss_dur: 0.30028 (0.15386) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 107.20602 (35.77919) + | > current_lr: 0.00007 + | > step_time: 0.25360 (2.96151) + | > loader_time: 0.00590 (0.05469) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12808 (+0.00034) + | > avg_loss: -0.32813 (-0.00727) + | > avg_log_mle: -0.54848 (+0.00793) + | > avg_loss_dur: 0.22035 (-0.01520) + + + > EPOCH: 276/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 19:23:47)  + + --> STEP: 1/234 -- GLOBAL_STEP: 64585 + | > loss: -0.31024 (-0.31024) + | > log_mle: -0.39780 (-0.39780) + | > loss_dur: 0.08756 (0.08756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.20181 (19.20181) + | > current_lr: 0.00007 + | > step_time: 8.91220 (8.91216) + | > loader_time: 0.00160 (0.00156) + + + --> STEP: 6/234 -- GLOBAL_STEP: 64590 + | > loss: -0.34437 (-0.31152) + | > log_mle: -0.40600 (-0.40414) + | > loss_dur: 0.06164 (0.09262) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.98091 (17.78865) + | > current_lr: 0.00007 + | > step_time: 2.80250 (5.76739) + | > loader_time: 0.19310 (0.06493) + + + --> STEP: 11/234 -- GLOBAL_STEP: 64595 + | > loss: -0.36239 (-0.32249) + | > log_mle: -0.41970 (-0.40845) + | > loss_dur: 0.05732 (0.08595) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.08794 (18.35692) + | > current_lr: 0.00007 + | > step_time: 4.30520 (5.52675) + | > loader_time: 0.39440 (0.08041) + + + --> STEP: 16/234 -- GLOBAL_STEP: 64600 + | > loss: -0.35789 (-0.32980) + | > log_mle: -0.42186 (-0.41141) + | > loss_dur: 0.06397 (0.08161) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.50743 (16.61432) + | > current_lr: 0.00007 + | > step_time: 3.31000 (5.13834) + | > loader_time: 0.08130 (0.08563) + + + --> STEP: 21/234 -- GLOBAL_STEP: 64605 + | > loss: -0.33489 (-0.33303) + | > log_mle: -0.39854 (-0.41083) + | > loss_dur: 0.06364 (0.07781) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.74518 (15.14103) + | > current_lr: 0.00007 + | > step_time: 2.49780 (4.38136) + | > loader_time: 0.00140 (0.06568) + + + --> STEP: 26/234 -- GLOBAL_STEP: 64610 + | > loss: -0.31398 (-0.33257) + | > log_mle: -0.39459 (-0.40971) + | > loss_dur: 0.08061 (0.07714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.06296 (14.36249) + | > current_lr: 0.00007 + | > step_time: 2.79270 (3.94824) + | > loader_time: 0.00110 (0.05343) + + + --> STEP: 31/234 -- GLOBAL_STEP: 64615 + | > loss: -0.29296 (-0.33256) + | > log_mle: -0.38627 (-0.40899) + | > loss_dur: 0.09331 (0.07643) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.02543 (13.80069) + | > current_lr: 0.00007 + | > step_time: 1.22240 (3.58877) + | > loader_time: 0.00210 (0.04508) + + + --> STEP: 36/234 -- GLOBAL_STEP: 64620 + | > loss: -0.29345 (-0.33009) + | > log_mle: -0.38586 (-0.40687) + | > loss_dur: 0.09241 (0.07678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61954 (13.40312) + | > current_lr: 0.00007 + | > step_time: 2.40050 (3.58234) + | > loader_time: 0.00260 (0.05416) + + + --> STEP: 41/234 -- GLOBAL_STEP: 64625 + | > loss: -0.33797 (-0.32795) + | > log_mle: -0.40435 (-0.40484) + | > loss_dur: 0.06638 (0.07689) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.99373 (13.01446) + | > current_lr: 0.00007 + | > step_time: 2.39940 (3.41414) + | > loader_time: 0.00220 (0.04992) + + + --> STEP: 46/234 -- GLOBAL_STEP: 64630 + | > loss: -0.28754 (-0.32429) + | > log_mle: -0.38261 (-0.40244) + | > loss_dur: 0.09507 (0.07815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.42814 (12.88751) + | > current_lr: 0.00007 + | > step_time: 1.79410 (3.24645) + | > loader_time: 0.00410 (0.04669) + + + --> STEP: 51/234 -- GLOBAL_STEP: 64635 + | > loss: -0.30470 (-0.32307) + | > log_mle: -0.37929 (-0.40094) + | > loss_dur: 0.07459 (0.07786) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.39775 (12.41584) + | > current_lr: 0.00007 + | > step_time: 1.70520 (3.13409) + | > loader_time: 0.00290 (0.04625) + + + --> STEP: 56/234 -- GLOBAL_STEP: 64640 + | > loss: -0.29073 (-0.32068) + | > log_mle: -0.38310 (-0.39937) + | > loss_dur: 0.09237 (0.07869) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.21951 (12.28605) + | > current_lr: 0.00007 + | > step_time: 1.76310 (3.03981) + | > loader_time: 0.02540 (0.04795) + + + --> STEP: 61/234 -- GLOBAL_STEP: 64645 + | > loss: -0.28344 (-0.31796) + | > log_mle: -0.37528 (-0.39785) + | > loss_dur: 0.09184 (0.07989) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.81350 (12.20051) + | > current_lr: 0.00007 + | > step_time: 2.60690 (2.92968) + | > loader_time: 0.00210 (0.04419) + + + --> STEP: 66/234 -- GLOBAL_STEP: 64650 + | > loss: -0.28813 (-0.31460) + | > log_mle: -0.36790 (-0.39612) + | > loss_dur: 0.07978 (0.08152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.35987 (12.12858) + | > current_lr: 0.00007 + | > step_time: 1.63460 (2.82002) + | > loader_time: 0.00270 (0.04110) + + + --> STEP: 71/234 -- GLOBAL_STEP: 64655 + | > loss: -0.26771 (-0.31108) + | > log_mle: -0.39136 (-0.39447) + | > loss_dur: 0.12365 (0.08339) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.51339 (12.18977) + | > current_lr: 0.00007 + | > step_time: 3.10330 (2.74464) + | > loader_time: 0.09130 (0.04082) + + + --> STEP: 76/234 -- GLOBAL_STEP: 64660 + | > loss: -0.27004 (-0.30791) + | > log_mle: -0.37571 (-0.39296) + | > loss_dur: 0.10567 (0.08505) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.33954 (12.23239) + | > current_lr: 0.00007 + | > step_time: 1.45110 (2.70440) + | > loader_time: 0.00440 (0.03835) + + + --> STEP: 81/234 -- GLOBAL_STEP: 64665 + | > loss: -0.26829 (-0.30561) + | > log_mle: -0.38142 (-0.39160) + | > loss_dur: 0.11312 (0.08598) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.49690 (12.15638) + | > current_lr: 0.00007 + | > step_time: 1.50730 (2.67522) + | > loader_time: 0.00230 (0.03725) + + + --> STEP: 86/234 -- GLOBAL_STEP: 64670 + | > loss: -0.26879 (-0.30301) + | > log_mle: -0.38181 (-0.39047) + | > loss_dur: 0.11301 (0.08747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.29714 (12.27895) + | > current_lr: 0.00007 + | > step_time: 1.10260 (2.62692) + | > loader_time: 0.00210 (0.03724) + + + --> STEP: 91/234 -- GLOBAL_STEP: 64675 + | > loss: -0.26403 (-0.30087) + | > log_mle: -0.39066 (-0.39047) + | > loss_dur: 0.12663 (0.08960) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.59620 (12.48870) + | > current_lr: 0.00007 + | > step_time: 1.50300 (2.63552) + | > loader_time: 0.00290 (0.03640) + + + --> STEP: 96/234 -- GLOBAL_STEP: 64680 + | > loss: -0.25596 (-0.29968) + | > log_mle: -0.37481 (-0.39187) + | > loss_dur: 0.11885 (0.09219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.12937 (13.28005) + | > current_lr: 0.00007 + | > step_time: 1.18580 (2.61375) + | > loader_time: 0.00290 (0.03739) + + + --> STEP: 101/234 -- GLOBAL_STEP: 64685 + | > loss: -0.25639 (-0.29802) + | > log_mle: -0.42210 (-0.39244) + | > loss_dur: 0.16571 (0.09442) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.90648 (13.89445) + | > current_lr: 0.00007 + | > step_time: 2.70450 (2.59693) + | > loader_time: 0.09260 (0.03739) + + + --> STEP: 106/234 -- GLOBAL_STEP: 64690 + | > loss: -0.22809 (-0.29633) + | > log_mle: -0.41572 (-0.39347) + | > loss_dur: 0.18764 (0.09715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.19215 (14.44320) + | > current_lr: 0.00007 + | > step_time: 3.20470 (2.59423) + | > loader_time: 0.00570 (0.03580) + + + --> STEP: 111/234 -- GLOBAL_STEP: 64695 + | > loss: -0.28463 (-0.29493) + | > log_mle: -0.47575 (-0.39484) + | > loss_dur: 0.19112 (0.09992) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.47060 (14.87327) + | > current_lr: 0.00007 + | > step_time: 2.48550 (2.58202) + | > loader_time: 0.00850 (0.03524) + + + --> STEP: 116/234 -- GLOBAL_STEP: 64700 + | > loss: -0.25739 (-0.29358) + | > log_mle: -0.43650 (-0.39626) + | > loss_dur: 0.17911 (0.10268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.08719 (15.46293) + | > current_lr: 0.00007 + | > step_time: 3.09910 (2.58390) + | > loader_time: 0.00250 (0.03597) + + + --> STEP: 121/234 -- GLOBAL_STEP: 64705 + | > loss: -0.22700 (-0.29237) + | > log_mle: -0.35366 (-0.39698) + | > loss_dur: 0.12666 (0.10461) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.32890 (15.85411) + | > current_lr: 0.00007 + | > step_time: 1.59850 (2.52936) + | > loader_time: 0.00270 (0.03601) + + + --> STEP: 126/234 -- GLOBAL_STEP: 64710 + | > loss: -0.30211 (-0.29141) + | > log_mle: -0.48612 (-0.39824) + | > loss_dur: 0.18402 (0.10683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.58115 (16.33328) + | > current_lr: 0.00007 + | > step_time: 1.71650 (2.54173) + | > loader_time: 0.07940 (0.03679) + + + --> STEP: 131/234 -- GLOBAL_STEP: 64715 + | > loss: -0.34184 (-0.29142) + | > log_mle: -0.53236 (-0.40073) + | > loss_dur: 0.19051 (0.10930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.15771 (17.03380) + | > current_lr: 0.00007 + | > step_time: 3.10110 (2.53182) + | > loader_time: 0.00290 (0.03550) + + + --> STEP: 136/234 -- GLOBAL_STEP: 64720 + | > loss: -0.36140 (-0.29172) + | > log_mle: -0.57376 (-0.40334) + | > loss_dur: 0.21235 (0.11162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.79266 (17.78447) + | > current_lr: 0.00007 + | > step_time: 4.20280 (2.59105) + | > loader_time: 0.00410 (0.03852) + + + --> STEP: 141/234 -- GLOBAL_STEP: 64725 + | > loss: -0.31614 (-0.29156) + | > log_mle: -0.47845 (-0.40546) + | > loss_dur: 0.16231 (0.11390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.46911 (18.44906) + | > current_lr: 0.00007 + | > step_time: 2.09910 (2.62355) + | > loader_time: 0.00420 (0.04070) + + + --> STEP: 146/234 -- GLOBAL_STEP: 64730 + | > loss: -0.33256 (-0.29270) + | > log_mle: -0.52957 (-0.40942) + | > loss_dur: 0.19701 (0.11672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.02068 (19.21328) + | > current_lr: 0.00007 + | > step_time: 1.41380 (2.62203) + | > loader_time: 0.00550 (0.04009) + + + --> STEP: 151/234 -- GLOBAL_STEP: 64735 + | > loss: -0.32555 (-0.29386) + | > log_mle: -0.49832 (-0.41261) + | > loss_dur: 0.17277 (0.11875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.54208 (19.81017) + | > current_lr: 0.00007 + | > step_time: 4.62930 (2.64278) + | > loader_time: 0.18720 (0.04174) + + + --> STEP: 156/234 -- GLOBAL_STEP: 64740 + | > loss: -0.35502 (-0.29634) + | > log_mle: -0.54642 (-0.41749) + | > loss_dur: 0.19140 (0.12115) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.64285 (20.87421) + | > current_lr: 0.00007 + | > step_time: 4.65640 (2.65878) + | > loader_time: 0.00570 (0.04061) + + + --> STEP: 161/234 -- GLOBAL_STEP: 64745 + | > loss: -0.37316 (-0.29776) + | > log_mle: -0.56068 (-0.42125) + | > loss_dur: 0.18753 (0.12349) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.26437 (21.80840) + | > current_lr: 0.00007 + | > step_time: 0.80440 (2.63830) + | > loader_time: 0.08730 (0.04119) + + + --> STEP: 166/234 -- GLOBAL_STEP: 64750 + | > loss: -0.32095 (-0.29892) + | > log_mle: -0.49840 (-0.42435) + | > loss_dur: 0.17744 (0.12543) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.37005 (22.45929) + | > current_lr: 0.00007 + | > step_time: 2.80960 (2.65640) + | > loader_time: 0.00550 (0.04111) + + + --> STEP: 171/234 -- GLOBAL_STEP: 64755 + | > loss: -0.39479 (-0.30132) + | > log_mle: -0.61093 (-0.42924) + | > loss_dur: 0.21614 (0.12792) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.37868 (23.57530) + | > current_lr: 0.00007 + | > step_time: 2.29860 (2.68685) + | > loader_time: 0.00230 (0.04285) + + + --> STEP: 176/234 -- GLOBAL_STEP: 64760 + | > loss: -0.37586 (-0.30382) + | > log_mle: -0.59291 (-0.43419) + | > loss_dur: 0.21705 (0.13036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.41402 (24.40468) + | > current_lr: 0.00007 + | > step_time: 2.78790 (2.74459) + | > loader_time: 0.00200 (0.04387) + + + --> STEP: 181/234 -- GLOBAL_STEP: 64765 + | > loss: -0.32337 (-0.30574) + | > log_mle: -0.52744 (-0.43859) + | > loss_dur: 0.20407 (0.13285) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.14150 (25.36160) + | > current_lr: 0.00007 + | > step_time: 3.99210 (2.77372) + | > loader_time: 0.00540 (0.04334) + + + --> STEP: 186/234 -- GLOBAL_STEP: 64770 + | > loss: -0.33172 (-0.30790) + | > log_mle: -0.56220 (-0.44321) + | > loss_dur: 0.23049 (0.13531) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.45619 (26.64972) + | > current_lr: 0.00007 + | > step_time: 5.80100 (2.86068) + | > loader_time: 0.19120 (0.04571) + + + --> STEP: 191/234 -- GLOBAL_STEP: 64775 + | > loss: -0.38795 (-0.31016) + | > log_mle: -0.58886 (-0.44752) + | > loss_dur: 0.20091 (0.13736) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.63431 (27.74682) + | > current_lr: 0.00007 + | > step_time: 3.29840 (2.91036) + | > loader_time: 0.00740 (0.04604) + + + --> STEP: 196/234 -- GLOBAL_STEP: 64780 + | > loss: -0.36400 (-0.31283) + | > log_mle: -0.58451 (-0.45218) + | > loss_dur: 0.22051 (0.13935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.70395 (28.74333) + | > current_lr: 0.00007 + | > step_time: 5.80600 (2.98355) + | > loader_time: 0.08250 (0.04684) + + + --> STEP: 201/234 -- GLOBAL_STEP: 64785 + | > loss: -0.32018 (-0.31489) + | > log_mle: -0.54255 (-0.45626) + | > loss_dur: 0.22237 (0.14137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.91167 (29.65479) + | > current_lr: 0.00007 + | > step_time: 5.20160 (3.05714) + | > loader_time: 0.09470 (0.04859) + + + --> STEP: 206/234 -- GLOBAL_STEP: 64790 + | > loss: -0.41716 (-0.31741) + | > log_mle: -0.64382 (-0.46078) + | > loss_dur: 0.22665 (0.14336) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.25405 (30.84819) + | > current_lr: 0.00007 + | > step_time: 4.10690 (3.10380) + | > loader_time: 0.00830 (0.05100) + + + --> STEP: 211/234 -- GLOBAL_STEP: 64795 + | > loss: -0.48965 (-0.32048) + | > log_mle: -0.73128 (-0.46595) + | > loss_dur: 0.24163 (0.14547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.59262 (31.98776) + | > current_lr: 0.00007 + | > step_time: 5.40360 (3.14734) + | > loader_time: 0.00500 (0.05266) + + + --> STEP: 216/234 -- GLOBAL_STEP: 64800 + | > loss: -0.46400 (-0.32351) + | > log_mle: -0.70892 (-0.47096) + | > loss_dur: 0.24492 (0.14745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 99.50092 (33.28175) + | > current_lr: 0.00007 + | > step_time: 6.91320 (3.20188) + | > loader_time: 0.18740 (0.05364) + + + --> STEP: 221/234 -- GLOBAL_STEP: 64805 + | > loss: -0.41155 (-0.32672) + | > log_mle: -0.62522 (-0.47602) + | > loss_dur: 0.21366 (0.14930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.85261 (34.20456) + | > current_lr: 0.00007 + | > step_time: 4.48640 (3.23700) + | > loader_time: 0.00280 (0.05300) + + + --> STEP: 226/234 -- GLOBAL_STEP: 64810 + | > loss: -0.49085 (-0.33032) + | > log_mle: -0.74435 (-0.48177) + | > loss_dur: 0.25350 (0.15145) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.56944 (35.27104) + | > current_lr: 0.00007 + | > step_time: 0.23830 (3.20986) + | > loader_time: 0.00260 (0.06416) + + + --> STEP: 231/234 -- GLOBAL_STEP: 64815 + | > loss: -0.43001 (-0.33322) + | > log_mle: -0.80885 (-0.48786) + | > loss_dur: 0.37884 (0.15464) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 117.00584 (36.55203) + | > current_lr: 0.00007 + | > step_time: 0.27100 (3.14585) + | > loader_time: 0.00310 (0.06284) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.99858 (+0.87049) + | > avg_loss: -0.32168 (+0.00645) + | > avg_log_mle: -0.55310 (-0.00463) + | > avg_loss_dur: 0.23142 (+0.01108) + + + > EPOCH: 277/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 19:37:18)  + + --> STEP: 2/234 -- GLOBAL_STEP: 64820 + | > loss: -0.34028 (-0.32004) + | > log_mle: -0.42031 (-0.41085) + | > loss_dur: 0.08004 (0.09082) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.83335 (18.26524) + | > current_lr: 0.00007 + | > step_time: 3.40500 (4.80725) + | > loader_time: 0.09780 (2.89664) + + + --> STEP: 7/234 -- GLOBAL_STEP: 64825 + | > loss: -0.33821 (-0.31662) + | > log_mle: -0.40725 (-0.40690) + | > loss_dur: 0.06904 (0.09028) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.77719 (17.92170) + | > current_lr: 0.00007 + | > step_time: 5.40700 (5.27597) + | > loader_time: 0.08530 (0.85373) + + + --> STEP: 12/234 -- GLOBAL_STEP: 64830 + | > loss: -0.31233 (-0.32238) + | > log_mle: -0.40354 (-0.41091) + | > loss_dur: 0.09121 (0.08853) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.10652 (18.31495) + | > current_lr: 0.00007 + | > step_time: 4.81280 (4.80974) + | > loader_time: 0.08560 (0.51309) + + + --> STEP: 17/234 -- GLOBAL_STEP: 64835 + | > loss: -0.35718 (-0.33218) + | > log_mle: -0.41755 (-0.41409) + | > loss_dur: 0.06037 (0.08192) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.40306 (16.92534) + | > current_lr: 0.00007 + | > step_time: 3.10230 (4.80115) + | > loader_time: 0.00190 (0.38029) + + + --> STEP: 22/234 -- GLOBAL_STEP: 64840 + | > loss: -0.33035 (-0.33399) + | > log_mle: -0.40857 (-0.41318) + | > loss_dur: 0.07823 (0.07918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.14677 (15.69366) + | > current_lr: 0.00007 + | > step_time: 2.29020 (4.71625) + | > loader_time: 0.00100 (0.31484) + + + --> STEP: 27/234 -- GLOBAL_STEP: 64845 + | > loss: -0.32502 (-0.33377) + | > log_mle: -0.39771 (-0.41161) + | > loss_dur: 0.07268 (0.07785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.33310 (14.96875) + | > current_lr: 0.00007 + | > step_time: 2.68840 (4.42440) + | > loader_time: 0.00160 (0.26397) + + + --> STEP: 32/234 -- GLOBAL_STEP: 64850 + | > loss: -0.32434 (-0.33386) + | > log_mle: -0.39793 (-0.41060) + | > loss_dur: 0.07360 (0.07674) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74510 (14.22857) + | > current_lr: 0.00007 + | > step_time: 1.60260 (3.97319) + | > loader_time: 0.00300 (0.22558) + + + --> STEP: 37/234 -- GLOBAL_STEP: 64855 + | > loss: -0.31273 (-0.33128) + | > log_mle: -0.38133 (-0.40792) + | > loss_dur: 0.06860 (0.07665) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.57465 (13.74982) + | > current_lr: 0.00007 + | > step_time: 1.29510 (3.60062) + | > loader_time: 0.00180 (0.19540) + + + --> STEP: 42/234 -- GLOBAL_STEP: 64860 + | > loss: -0.28719 (-0.32811) + | > log_mle: -0.36592 (-0.40532) + | > loss_dur: 0.07873 (0.07720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.59209 (13.50665) + | > current_lr: 0.00007 + | > step_time: 1.60030 (3.41484) + | > loader_time: 0.08680 (0.17833) + + + --> STEP: 47/234 -- GLOBAL_STEP: 64865 + | > loss: -0.30738 (-0.32477) + | > log_mle: -0.39102 (-0.40314) + | > loss_dur: 0.08364 (0.07837) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.67045 (13.35593) + | > current_lr: 0.00007 + | > step_time: 1.12210 (3.22040) + | > loader_time: 0.00270 (0.15959) + + + --> STEP: 52/234 -- GLOBAL_STEP: 64870 + | > loss: -0.27335 (-0.32236) + | > log_mle: -0.37518 (-0.40120) + | > loss_dur: 0.10183 (0.07884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.22612 (12.88670) + | > current_lr: 0.00007 + | > step_time: 1.01900 (3.03192) + | > loader_time: 0.00190 (0.14441) + + + --> STEP: 57/234 -- GLOBAL_STEP: 64875 + | > loss: -0.27677 (-0.32031) + | > log_mle: -0.36638 (-0.39948) + | > loss_dur: 0.08961 (0.07917) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.35131 (12.61488) + | > current_lr: 0.00007 + | > step_time: 1.37820 (2.89781) + | > loader_time: 0.00150 (0.13352) + + + --> STEP: 62/234 -- GLOBAL_STEP: 64880 + | > loss: -0.22650 (-0.31674) + | > log_mle: -0.38607 (-0.39823) + | > loss_dur: 0.15958 (0.08149) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.29558 (13.06073) + | > current_lr: 0.00007 + | > step_time: 1.26950 (2.79566) + | > loader_time: 0.00180 (0.12429) + + + --> STEP: 67/234 -- GLOBAL_STEP: 64885 + | > loss: -0.27614 (-0.31400) + | > log_mle: -0.37736 (-0.39618) + | > loss_dur: 0.10122 (0.08219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.24946 (12.77734) + | > current_lr: 0.00007 + | > step_time: 2.10810 (2.76426) + | > loader_time: 0.00200 (0.11655) + + + --> STEP: 72/234 -- GLOBAL_STEP: 64890 + | > loss: -0.28007 (-0.31061) + | > log_mle: -0.37327 (-0.39420) + | > loss_dur: 0.09320 (0.08358) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.17382 (12.69608) + | > current_lr: 0.00007 + | > step_time: 2.40960 (2.70283) + | > loader_time: 0.08430 (0.10987) + + + --> STEP: 77/234 -- GLOBAL_STEP: 64895 + | > loss: -0.27408 (-0.30716) + | > log_mle: -0.37229 (-0.39265) + | > loss_dur: 0.09821 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.13310 (12.53472) + | > current_lr: 0.00007 + | > step_time: 1.28610 (2.65826) + | > loader_time: 0.01210 (0.10405) + + + --> STEP: 82/234 -- GLOBAL_STEP: 64900 + | > loss: -0.26259 (-0.30485) + | > log_mle: -0.36827 (-0.39127) + | > loss_dur: 0.10568 (0.08642) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.66197 (12.46737) + | > current_lr: 0.00007 + | > step_time: 1.50310 (2.58594) + | > loader_time: 0.00220 (0.09894) + + + --> STEP: 87/234 -- GLOBAL_STEP: 64905 + | > loss: -0.24592 (-0.30218) + | > log_mle: -0.36707 (-0.39021) + | > loss_dur: 0.12116 (0.08802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.96210 (12.54942) + | > current_lr: 0.00007 + | > step_time: 3.20680 (2.54516) + | > loader_time: 0.10270 (0.09569) + + + --> STEP: 92/234 -- GLOBAL_STEP: 64910 + | > loss: -0.27859 (-0.30022) + | > log_mle: -0.41103 (-0.39062) + | > loss_dur: 0.13244 (0.09040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.88400 (12.83640) + | > current_lr: 0.00007 + | > step_time: 4.49360 (2.58949) + | > loader_time: 0.10650 (0.09280) + + + --> STEP: 97/234 -- GLOBAL_STEP: 64915 + | > loss: -0.26647 (-0.29909) + | > log_mle: -0.39190 (-0.39188) + | > loss_dur: 0.12543 (0.09279) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.64523 (13.27403) + | > current_lr: 0.00007 + | > step_time: 1.50610 (2.61078) + | > loader_time: 0.00220 (0.08999) + + + --> STEP: 102/234 -- GLOBAL_STEP: 64920 + | > loss: -0.24105 (-0.29730) + | > log_mle: -0.37807 (-0.39221) + | > loss_dur: 0.13703 (0.09491) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.85080 (13.65896) + | > current_lr: 0.00007 + | > step_time: 4.61970 (2.62439) + | > loader_time: 0.19450 (0.08838) + + + --> STEP: 107/234 -- GLOBAL_STEP: 64925 + | > loss: -0.26478 (-0.29607) + | > log_mle: -0.41841 (-0.39370) + | > loss_dur: 0.15363 (0.09763) + | > amp_scaler: 2048.00000 (1033.57009) + | > grad_norm: 20.42446 (14.13369) + | > current_lr: 0.00007 + | > step_time: 2.60580 (2.59791) + | > loader_time: 0.00170 (0.08592) + + + --> STEP: 112/234 -- GLOBAL_STEP: 64930 + | > loss: -0.25726 (-0.29461) + | > log_mle: -0.43003 (-0.39511) + | > loss_dur: 0.17277 (0.10050) + | > amp_scaler: 2048.00000 (1078.85714) + | > grad_norm: 29.70737 (14.74904) + | > current_lr: 0.00007 + | > step_time: 5.40160 (2.61873) + | > loader_time: 0.00200 (0.08369) + + + --> STEP: 117/234 -- GLOBAL_STEP: 64935 + | > loss: -0.27213 (-0.29343) + | > log_mle: -0.42781 (-0.39651) + | > loss_dur: 0.15568 (0.10308) + | > amp_scaler: 2048.00000 (1120.27350) + | > grad_norm: 24.66798 (15.17916) + | > current_lr: 0.00007 + | > step_time: 3.09740 (2.61881) + | > loader_time: 0.00180 (0.08023) + + + --> STEP: 122/234 -- GLOBAL_STEP: 64940 + | > loss: -0.25081 (-0.29203) + | > log_mle: -0.39540 (-0.39703) + | > loss_dur: 0.14460 (0.10500) + | > amp_scaler: 2048.00000 (1158.29508) + | > grad_norm: 21.57801 (15.40766) + | > current_lr: 0.00007 + | > step_time: 1.69310 (2.56527) + | > loader_time: 0.00360 (0.07787) + + + --> STEP: 127/234 -- GLOBAL_STEP: 64945 + | > loss: -0.27402 (-0.29125) + | > log_mle: -0.45841 (-0.39882) + | > loss_dur: 0.18438 (0.10757) + | > amp_scaler: 2048.00000 (1193.32283) + | > grad_norm: 32.71115 (15.95957) + | > current_lr: 0.00007 + | > step_time: 1.83000 (2.56732) + | > loader_time: 0.00200 (0.07558) + + + --> STEP: 132/234 -- GLOBAL_STEP: 64950 + | > loss: -0.28661 (-0.29145) + | > log_mle: -0.43970 (-0.40127) + | > loss_dur: 0.15309 (0.10982) + | > amp_scaler: 2048.00000 (1225.69697) + | > grad_norm: 34.02124 (16.62542) + | > current_lr: 0.00007 + | > step_time: 2.47590 (2.57664) + | > loader_time: 0.00260 (0.07389) + + + --> STEP: 137/234 -- GLOBAL_STEP: 64955 + | > loss: -0.26025 (-0.29180) + | > log_mle: -0.45442 (-0.40406) + | > loss_dur: 0.19417 (0.11225) + | > amp_scaler: 2048.00000 (1255.70803) + | > grad_norm: 34.08144 (17.33999) + | > current_lr: 0.00007 + | > step_time: 2.50930 (2.56031) + | > loader_time: 0.00380 (0.07130) + + + --> STEP: 142/234 -- GLOBAL_STEP: 64960 + | > loss: -0.28290 (-0.29173) + | > log_mle: -0.46715 (-0.40629) + | > loss_dur: 0.18425 (0.11456) + | > amp_scaler: 2048.00000 (1283.60563) + | > grad_norm: 26.10304 (17.93309) + | > current_lr: 0.00007 + | > step_time: 2.72800 (2.57121) + | > loader_time: 0.00560 (0.07136) + + + --> STEP: 147/234 -- GLOBAL_STEP: 64965 + | > loss: -0.28701 (-0.29295) + | > log_mle: -0.46872 (-0.41027) + | > loss_dur: 0.18170 (0.11732) + | > amp_scaler: 2048.00000 (1309.60544) + | > grad_norm: 29.63053 (18.89232) + | > current_lr: 0.00007 + | > step_time: 1.80210 (2.58895) + | > loader_time: 0.00280 (0.07030) + + + --> STEP: 152/234 -- GLOBAL_STEP: 64970 + | > loss: -0.34493 (-0.29434) + | > log_mle: -0.55283 (-0.41390) + | > loss_dur: 0.20790 (0.11955) + | > amp_scaler: 2048.00000 (1333.89474) + | > grad_norm: 48.62298 (19.70443) + | > current_lr: 0.00007 + | > step_time: 1.61100 (2.57633) + | > loader_time: 0.00440 (0.06870) + + + --> STEP: 157/234 -- GLOBAL_STEP: 64975 + | > loss: -0.31148 (-0.29650) + | > log_mle: -0.49884 (-0.41836) + | > loss_dur: 0.18736 (0.12186) + | > amp_scaler: 2048.00000 (1356.63694) + | > grad_norm: 36.86400 (20.69879) + | > current_lr: 0.00007 + | > step_time: 2.74190 (2.56377) + | > loader_time: 0.09690 (0.06847) + + + --> STEP: 162/234 -- GLOBAL_STEP: 64980 + | > loss: -0.33706 (-0.29816) + | > log_mle: -0.52920 (-0.42254) + | > loss_dur: 0.19214 (0.12438) + | > amp_scaler: 2048.00000 (1377.97531) + | > grad_norm: 44.92497 (21.63267) + | > current_lr: 0.00007 + | > step_time: 4.18440 (2.56354) + | > loader_time: 0.00200 (0.06698) + + + --> STEP: 167/234 -- GLOBAL_STEP: 64985 + | > loss: -0.44392 (-0.30008) + | > log_mle: -0.63169 (-0.42646) + | > loss_dur: 0.18777 (0.12637) + | > amp_scaler: 2048.00000 (1398.03593) + | > grad_norm: 72.06534 (22.56226) + | > current_lr: 0.00007 + | > step_time: 9.20710 (2.62879) + | > loader_time: 0.20260 (0.06848) + + + --> STEP: 172/234 -- GLOBAL_STEP: 64990 + | > loss: -0.40163 (-0.30258) + | > log_mle: -0.61337 (-0.43136) + | > loss_dur: 0.21174 (0.12879) + | > amp_scaler: 2048.00000 (1416.93023) + | > grad_norm: 88.73736 (23.66843) + | > current_lr: 0.00007 + | > step_time: 6.09190 (2.71054) + | > loader_time: 0.00370 (0.06833) + + + --> STEP: 177/234 -- GLOBAL_STEP: 64995 + | > loss: -0.36476 (-0.30475) + | > log_mle: -0.57408 (-0.43592) + | > loss_dur: 0.20933 (0.13118) + | > amp_scaler: 2048.00000 (1434.75706) + | > grad_norm: 73.59921 (24.91166) + | > current_lr: 0.00007 + | > step_time: 2.91140 (2.75896) + | > loader_time: 0.10250 (0.06963) + + + --> STEP: 182/234 -- GLOBAL_STEP: 65000 + | > loss: -0.36867 (-0.30671) + | > log_mle: -0.60860 (-0.44042) + | > loss_dur: 0.23993 (0.13371) + | > amp_scaler: 2048.00000 (1451.60440) + | > grad_norm: 75.35036 (25.98886) + | > current_lr: 0.00007 + | > step_time: 0.91310 (2.72922) + | > loader_time: 0.08230 (0.06915) + + + --> STEP: 187/234 -- GLOBAL_STEP: 65005 + | > loss: -0.41276 (-0.30904) + | > log_mle: -0.62462 (-0.44499) + | > loss_dur: 0.21186 (0.13595) + | > amp_scaler: 2048.00000 (1467.55080) + | > grad_norm: 56.03326 (26.88566) + | > current_lr: 0.00007 + | > step_time: 2.29200 (2.74027) + | > loader_time: 0.00290 (0.06792) + + + --> STEP: 192/234 -- GLOBAL_STEP: 65010 + | > loss: -0.44172 (-0.31153) + | > log_mle: -0.65218 (-0.44940) + | > loss_dur: 0.21046 (0.13787) + | > amp_scaler: 2048.00000 (1482.66667) + | > grad_norm: 60.03221 (27.78545) + | > current_lr: 0.00007 + | > step_time: 2.30730 (2.78200) + | > loader_time: 0.09500 (0.06799) + + + --> STEP: 197/234 -- GLOBAL_STEP: 65015 + | > loss: -0.41564 (-0.31395) + | > log_mle: -0.60919 (-0.45369) + | > loss_dur: 0.19355 (0.13974) + | > amp_scaler: 2048.00000 (1497.01523) + | > grad_norm: 66.39342 (28.72678) + | > current_lr: 0.00007 + | > step_time: 9.61380 (2.86573) + | > loader_time: 0.08480 (0.06823) + + + --> STEP: 202/234 -- GLOBAL_STEP: 65020 + | > loss: -0.49351 (-0.31623) + | > log_mle: -0.70933 (-0.45801) + | > loss_dur: 0.21582 (0.14178) + | > amp_scaler: 2048.00000 (1510.65347) + | > grad_norm: 89.18040 (29.70128) + | > current_lr: 0.00007 + | > step_time: 4.19550 (2.95171) + | > loader_time: 0.09920 (0.06864) + + + --> STEP: 207/234 -- GLOBAL_STEP: 65025 + | > loss: -0.46943 (-0.31860) + | > log_mle: -0.70136 (-0.46239) + | > loss_dur: 0.23193 (0.14380) + | > amp_scaler: 2048.00000 (1523.63285) + | > grad_norm: 75.34403 (30.73334) + | > current_lr: 0.00007 + | > step_time: 2.01600 (2.99981) + | > loader_time: 0.08380 (0.06871) + + + --> STEP: 212/234 -- GLOBAL_STEP: 65030 + | > loss: -0.44447 (-0.32162) + | > log_mle: -0.67459 (-0.46753) + | > loss_dur: 0.23012 (0.14591) + | > amp_scaler: 2048.00000 (1536.00000) + | > grad_norm: 82.43590 (31.79653) + | > current_lr: 0.00007 + | > step_time: 5.69690 (3.03287) + | > loader_time: 0.09330 (0.06883) + + + --> STEP: 217/234 -- GLOBAL_STEP: 65035 + | > loss: -0.46443 (-0.32478) + | > log_mle: -0.70225 (-0.47263) + | > loss_dur: 0.23782 (0.14785) + | > amp_scaler: 2048.00000 (1547.79724) + | > grad_norm: 80.84481 (32.88948) + | > current_lr: 0.00007 + | > step_time: 6.73050 (3.09108) + | > loader_time: 0.09580 (0.06821) + + + --> STEP: 222/234 -- GLOBAL_STEP: 65040 + | > loss: -0.45120 (-0.32780) + | > log_mle: -0.71981 (-0.47771) + | > loss_dur: 0.26861 (0.14991) + | > amp_scaler: 2048.00000 (1559.06306) + | > grad_norm: 76.06660 (34.01156) + | > current_lr: 0.00007 + | > step_time: 1.89250 (3.10203) + | > loader_time: 0.00300 (0.06764) + + + --> STEP: 227/234 -- GLOBAL_STEP: 65045 + | > loss: -0.43544 (-0.33119) + | > log_mle: -0.68385 (-0.48317) + | > loss_dur: 0.24842 (0.15198) + | > amp_scaler: 2048.00000 (1569.83260) + | > grad_norm: 90.01108 (35.09683) + | > current_lr: 0.00007 + | > step_time: 0.47100 (3.06896) + | > loader_time: 0.00300 (0.06700) + + + --> STEP: 232/234 -- GLOBAL_STEP: 65050 + | > loss: -0.44205 (-0.33412) + | > log_mle: -0.91233 (-0.49014) + | > loss_dur: 0.47028 (0.15602) + | > amp_scaler: 1024.00000 (1575.72414) + | > grad_norm: 0.00000 (35.89728) + | > current_lr: 0.00007 + | > step_time: 0.31570 (3.00874) + | > loader_time: 0.11300 (0.06610) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.45849 (-0.54009) + | > avg_loss: -0.34120 (-0.01952) + | > avg_log_mle: -0.56320 (-0.01010) + | > avg_loss_dur: 0.22200 (-0.00942) + + + > EPOCH: 278/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 19:50:08)  + + --> STEP: 3/234 -- GLOBAL_STEP: 65055 + | > loss: -0.26240 (-0.30093) + | > log_mle: -0.39761 (-0.40643) + | > loss_dur: 0.13521 (0.10550) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.36917 (20.49499) + | > current_lr: 0.00007 + | > step_time: 5.69390 (6.40357) + | > loader_time: 0.00410 (1.76079) + + + --> STEP: 8/234 -- GLOBAL_STEP: 65060 + | > loss: -0.34641 (-0.31855) + | > log_mle: -0.42670 (-0.41039) + | > loss_dur: 0.08029 (0.09184) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.53869 (18.07162) + | > current_lr: 0.00007 + | > step_time: 4.00430 (4.36486) + | > loader_time: 0.00510 (0.66231) + + + --> STEP: 13/234 -- GLOBAL_STEP: 65065 + | > loss: -0.36360 (-0.32639) + | > log_mle: -0.43174 (-0.41348) + | > loss_dur: 0.06814 (0.08709) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.08538 (16.97608) + | > current_lr: 0.00007 + | > step_time: 11.20600 (5.24857) + | > loader_time: 0.08540 (0.45911) + + + --> STEP: 18/234 -- GLOBAL_STEP: 65070 + | > loss: -0.33217 (-0.33344) + | > log_mle: -0.40610 (-0.41458) + | > loss_dur: 0.07393 (0.08114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.37372 (16.24137) + | > current_lr: 0.00007 + | > step_time: 0.68650 (4.42324) + | > loader_time: 0.00100 (0.33251) + + + --> STEP: 23/234 -- GLOBAL_STEP: 65075 + | > loss: -0.35426 (-0.33587) + | > log_mle: -0.42717 (-0.41459) + | > loss_dur: 0.07291 (0.07872) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.04615 (15.41678) + | > current_lr: 0.00007 + | > step_time: 1.05530 (4.06048) + | > loader_time: 0.00100 (0.29941) + + + --> STEP: 28/234 -- GLOBAL_STEP: 65080 + | > loss: -0.38347 (-0.33621) + | > log_mle: -0.43735 (-0.41344) + | > loss_dur: 0.05388 (0.07723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.76474 (14.63015) + | > current_lr: 0.00007 + | > step_time: 1.07890 (3.60965) + | > loader_time: 0.00110 (0.24618) + + + --> STEP: 33/234 -- GLOBAL_STEP: 65085 + | > loss: -0.33073 (-0.33403) + | > log_mle: -0.40329 (-0.41150) + | > loss_dur: 0.07256 (0.07747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.99396 (13.95827) + | > current_lr: 0.00007 + | > step_time: 1.78970 (3.26427) + | > loader_time: 0.00160 (0.20917) + + + --> STEP: 38/234 -- GLOBAL_STEP: 65090 + | > loss: -0.30495 (-0.33090) + | > log_mle: -0.39402 (-0.40877) + | > loss_dur: 0.08907 (0.07787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.73251 (13.64337) + | > current_lr: 0.00007 + | > step_time: 1.59930 (3.01566) + | > loader_time: 0.00180 (0.18194) + + + --> STEP: 43/234 -- GLOBAL_STEP: 65095 + | > loss: -0.29404 (-0.32780) + | > log_mle: -0.39162 (-0.40652) + | > loss_dur: 0.09758 (0.07872) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.24946 (13.23223) + | > current_lr: 0.00007 + | > step_time: 3.40360 (2.93956) + | > loader_time: 0.00190 (0.16320) + + + --> STEP: 48/234 -- GLOBAL_STEP: 65100 + | > loss: -0.33499 (-0.32615) + | > log_mle: -0.40012 (-0.40502) + | > loss_dur: 0.06513 (0.07887) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.56056 (13.00362) + | > current_lr: 0.00007 + | > step_time: 2.00440 (2.82046) + | > loader_time: 0.00210 (0.14813) + + + --> STEP: 53/234 -- GLOBAL_STEP: 65105 + | > loss: -0.29529 (-0.32387) + | > log_mle: -0.38904 (-0.40327) + | > loss_dur: 0.09375 (0.07940) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.98947 (12.66607) + | > current_lr: 0.00007 + | > step_time: 1.08450 (2.68853) + | > loader_time: 0.00330 (0.13441) + + + --> STEP: 58/234 -- GLOBAL_STEP: 65110 + | > loss: -0.31640 (-0.32200) + | > log_mle: -0.38821 (-0.40175) + | > loss_dur: 0.07181 (0.07975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.59803 (12.47583) + | > current_lr: 0.00007 + | > step_time: 1.10660 (2.58184) + | > loader_time: 0.08820 (0.12598) + + + --> STEP: 63/234 -- GLOBAL_STEP: 65115 + | > loss: -0.27604 (-0.31803) + | > log_mle: -0.37136 (-0.40025) + | > loss_dur: 0.09533 (0.08222) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.80054 (12.69578) + | > current_lr: 0.00007 + | > step_time: 2.40640 (2.53424) + | > loader_time: 0.09480 (0.12197) + + + --> STEP: 68/234 -- GLOBAL_STEP: 65120 + | > loss: -0.26480 (-0.31554) + | > log_mle: -0.36730 (-0.39844) + | > loss_dur: 0.10249 (0.08290) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.74742 (12.57823) + | > current_lr: 0.00007 + | > step_time: 2.00300 (2.45432) + | > loader_time: 0.00270 (0.11318) + + + --> STEP: 73/234 -- GLOBAL_STEP: 65125 + | > loss: -0.25103 (-0.31224) + | > log_mle: -0.37596 (-0.39689) + | > loss_dur: 0.12492 (0.08465) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.29425 (12.58372) + | > current_lr: 0.00007 + | > step_time: 2.52700 (2.40458) + | > loader_time: 0.00260 (0.10558) + + + --> STEP: 78/234 -- GLOBAL_STEP: 65130 + | > loss: -0.25520 (-0.30886) + | > log_mle: -0.36269 (-0.39512) + | > loss_dur: 0.10749 (0.08627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.89997 (12.59092) + | > current_lr: 0.00007 + | > step_time: 1.50770 (2.40912) + | > loader_time: 0.00180 (0.10151) + + + --> STEP: 83/234 -- GLOBAL_STEP: 65135 + | > loss: -0.22783 (-0.30607) + | > log_mle: -0.37236 (-0.39378) + | > loss_dur: 0.14453 (0.08772) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.96486 (12.63436) + | > current_lr: 0.00007 + | > step_time: 0.85630 (2.40993) + | > loader_time: 0.00210 (0.09779) + + + --> STEP: 88/234 -- GLOBAL_STEP: 65140 + | > loss: -0.27106 (-0.30410) + | > log_mle: -0.40790 (-0.39296) + | > loss_dur: 0.13683 (0.08886) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.77879 (12.67213) + | > current_lr: 0.00007 + | > step_time: 2.68900 (2.40723) + | > loader_time: 0.00210 (0.09445) + + + --> STEP: 93/234 -- GLOBAL_STEP: 65145 + | > loss: -0.27334 (-0.30236) + | > log_mle: -0.42003 (-0.39342) + | > loss_dur: 0.14669 (0.09107) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.42281 (12.81246) + | > current_lr: 0.00007 + | > step_time: 2.39050 (2.39645) + | > loader_time: 0.00220 (0.09199) + + + --> STEP: 98/234 -- GLOBAL_STEP: 65150 + | > loss: -0.24991 (-0.30090) + | > log_mle: -0.35822 (-0.39392) + | > loss_dur: 0.10831 (0.09301) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.03478 (13.09095) + | > current_lr: 0.00007 + | > step_time: 3.00000 (2.36825) + | > loader_time: 0.00180 (0.08908) + + + --> STEP: 103/234 -- GLOBAL_STEP: 65155 + | > loss: -0.28368 (-0.29922) + | > log_mle: -0.44466 (-0.39501) + | > loss_dur: 0.16099 (0.09579) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.99247 (13.58002) + | > current_lr: 0.00007 + | > step_time: 2.41580 (2.35068) + | > loader_time: 0.08350 (0.08655) + + + --> STEP: 108/234 -- GLOBAL_STEP: 65160 + | > loss: -0.26360 (-0.29775) + | > log_mle: -0.39417 (-0.39586) + | > loss_dur: 0.13056 (0.09812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.42107 (13.85255) + | > current_lr: 0.00007 + | > step_time: 1.52810 (2.32353) + | > loader_time: 0.00210 (0.08428) + + + --> STEP: 113/234 -- GLOBAL_STEP: 65165 + | > loss: -0.28341 (-0.29640) + | > log_mle: -0.43625 (-0.39754) + | > loss_dur: 0.15285 (0.10114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.41828 (14.55430) + | > current_lr: 0.00007 + | > step_time: 3.92540 (2.31771) + | > loader_time: 0.09650 (0.08148) + + + --> STEP: 118/234 -- GLOBAL_STEP: 65170 + | > loss: -0.25681 (-0.29489) + | > log_mle: -0.41060 (-0.39856) + | > loss_dur: 0.15379 (0.10367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.64621 (14.99844) + | > current_lr: 0.00007 + | > step_time: 2.40310 (2.33150) + | > loader_time: 0.08820 (0.08043) + + + --> STEP: 123/234 -- GLOBAL_STEP: 65175 + | > loss: -0.23396 (-0.29327) + | > log_mle: -0.37714 (-0.39868) + | > loss_dur: 0.14318 (0.10540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.12704 (15.27555) + | > current_lr: 0.00007 + | > step_time: 0.96440 (2.31123) + | > loader_time: 0.00210 (0.07732) + + + --> STEP: 128/234 -- GLOBAL_STEP: 65180 + | > loss: -0.28068 (-0.29290) + | > log_mle: -0.42692 (-0.40074) + | > loss_dur: 0.14625 (0.10784) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.64303 (15.99923) + | > current_lr: 0.00007 + | > step_time: 5.31230 (2.32167) + | > loader_time: 0.00340 (0.07442) + + + --> STEP: 133/234 -- GLOBAL_STEP: 65185 + | > loss: -0.29075 (-0.29312) + | > log_mle: -0.46052 (-0.40316) + | > loss_dur: 0.16977 (0.11004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.51734 (16.56412) + | > current_lr: 0.00007 + | > step_time: 2.60850 (2.32083) + | > loader_time: 0.00340 (0.07301) + + + --> STEP: 138/234 -- GLOBAL_STEP: 65190 + | > loss: -0.24667 (-0.29291) + | > log_mle: -0.40915 (-0.40534) + | > loss_dur: 0.16248 (0.11243) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.02355 (17.09878) + | > current_lr: 0.00007 + | > step_time: 3.30410 (2.31981) + | > loader_time: 0.10350 (0.07178) + + + --> STEP: 143/234 -- GLOBAL_STEP: 65195 + | > loss: -0.35064 (-0.29326) + | > log_mle: -0.56559 (-0.40838) + | > loss_dur: 0.21496 (0.11512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.18114 (18.01328) + | > current_lr: 0.00007 + | > step_time: 2.10300 (2.32048) + | > loader_time: 0.00330 (0.07080) + + + --> STEP: 148/234 -- GLOBAL_STEP: 65200 + | > loss: -0.31486 (-0.29414) + | > log_mle: -0.47125 (-0.41154) + | > loss_dur: 0.15638 (0.11740) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.53762 (18.70860) + | > current_lr: 0.00007 + | > step_time: 3.19450 (2.34828) + | > loader_time: 0.00280 (0.07090) + + + --> STEP: 153/234 -- GLOBAL_STEP: 65205 + | > loss: -0.40813 (-0.29623) + | > log_mle: -0.59930 (-0.41602) + | > loss_dur: 0.19117 (0.11980) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.29197 (19.85230) + | > current_lr: 0.00007 + | > step_time: 2.19910 (2.39589) + | > loader_time: 0.07580 (0.07233) + + + --> STEP: 158/234 -- GLOBAL_STEP: 65210 + | > loss: -0.31951 (-0.29768) + | > log_mle: -0.53291 (-0.41989) + | > loss_dur: 0.21340 (0.12221) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.38102 (20.79715) + | > current_lr: 0.00007 + | > step_time: 1.79170 (2.37889) + | > loader_time: 0.00220 (0.07117) + + + --> STEP: 163/234 -- GLOBAL_STEP: 65215 + | > loss: -0.32122 (-0.29947) + | > log_mle: -0.50708 (-0.42374) + | > loss_dur: 0.18586 (0.12428) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.91644 (21.76719) + | > current_lr: 0.00007 + | > step_time: 2.99000 (2.39837) + | > loader_time: 0.00250 (0.06918) + + + --> STEP: 168/234 -- GLOBAL_STEP: 65220 + | > loss: -0.35004 (-0.30156) + | > log_mle: -0.56516 (-0.42797) + | > loss_dur: 0.21512 (0.12641) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.80667 (22.62215) + | > current_lr: 0.00007 + | > step_time: 2.48620 (2.39022) + | > loader_time: 0.00430 (0.06882) + + + --> STEP: 173/234 -- GLOBAL_STEP: 65225 + | > loss: -0.36806 (-0.30402) + | > log_mle: -0.57012 (-0.43276) + | > loss_dur: 0.20206 (0.12873) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.16976 (23.65636) + | > current_lr: 0.00007 + | > step_time: 3.50670 (2.38580) + | > loader_time: 0.00350 (0.06742) + + + --> STEP: 178/234 -- GLOBAL_STEP: 65230 + | > loss: -0.40289 (-0.30631) + | > log_mle: -0.63463 (-0.43750) + | > loss_dur: 0.23175 (0.13119) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.08736 (24.52575) + | > current_lr: 0.00007 + | > step_time: 2.20050 (2.39403) + | > loader_time: 0.00850 (0.06569) + + + --> STEP: 183/234 -- GLOBAL_STEP: 65235 + | > loss: -0.42112 (-0.30827) + | > log_mle: -0.63496 (-0.44190) + | > loss_dur: 0.21384 (0.13363) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.88730 (25.43153) + | > current_lr: 0.00007 + | > step_time: 8.90210 (2.44301) + | > loader_time: 0.08620 (0.06632) + + + --> STEP: 188/234 -- GLOBAL_STEP: 65240 + | > loss: -0.43167 (-0.31041) + | > log_mle: -0.64782 (-0.44630) + | > loss_dur: 0.21615 (0.13590) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.22411 (26.30707) + | > current_lr: 0.00007 + | > step_time: 6.80280 (2.45900) + | > loader_time: 0.10040 (0.06601) + + + --> STEP: 193/234 -- GLOBAL_STEP: 65245 + | > loss: -0.43443 (-0.31280) + | > log_mle: -0.64941 (-0.45067) + | > loss_dur: 0.21499 (0.13787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.75408 (27.35860) + | > current_lr: 0.00007 + | > step_time: 4.18820 (2.47814) + | > loader_time: 0.00350 (0.06443) + + + --> STEP: 198/234 -- GLOBAL_STEP: 65250 + | > loss: -0.42534 (-0.31512) + | > log_mle: -0.65173 (-0.45498) + | > loss_dur: 0.22638 (0.13986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.93969 (28.53439) + | > current_lr: 0.00007 + | > step_time: 5.01820 (2.59628) + | > loader_time: 0.01080 (0.06404) + + + --> STEP: 203/234 -- GLOBAL_STEP: 65255 + | > loss: -0.34858 (-0.31687) + | > log_mle: -0.55549 (-0.45880) + | > loss_dur: 0.20691 (0.14193) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.91105 (29.68812) + | > current_lr: 0.00007 + | > step_time: 1.40660 (2.58882) + | > loader_time: 0.00270 (0.06291) + + + --> STEP: 208/234 -- GLOBAL_STEP: 65260 + | > loss: -0.41728 (-0.31958) + | > log_mle: -0.65529 (-0.46367) + | > loss_dur: 0.23800 (0.14409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.37004 (30.69453) + | > current_lr: 0.00007 + | > step_time: 7.01010 (2.62475) + | > loader_time: 0.09950 (0.06303) + + + --> STEP: 213/234 -- GLOBAL_STEP: 65265 + | > loss: -0.46787 (-0.32295) + | > log_mle: -0.70583 (-0.46909) + | > loss_dur: 0.23796 (0.14614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.29302 (31.96393) + | > current_lr: 0.00007 + | > step_time: 5.89040 (2.67293) + | > loader_time: 0.09100 (0.06339) + + + --> STEP: 218/234 -- GLOBAL_STEP: 65270 + | > loss: -0.42336 (-0.32598) + | > log_mle: -0.65802 (-0.47407) + | > loss_dur: 0.23466 (0.14810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.04173 (32.97734) + | > current_lr: 0.00007 + | > step_time: 7.79590 (2.74474) + | > loader_time: 0.09860 (0.06330) + + + --> STEP: 223/234 -- GLOBAL_STEP: 65275 + | > loss: -0.48018 (-0.32930) + | > log_mle: -0.71782 (-0.47940) + | > loss_dur: 0.23764 (0.15010) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.94514 (34.00223) + | > current_lr: 0.00007 + | > step_time: 0.22980 (2.77987) + | > loader_time: 0.00320 (0.06200) + + + --> STEP: 228/234 -- GLOBAL_STEP: 65280 + | > loss: -0.44818 (-0.33252) + | > log_mle: -0.71624 (-0.48483) + | > loss_dur: 0.26807 (0.15230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.09228 (35.34430) + | > current_lr: 0.00007 + | > step_time: 0.24450 (2.72406) + | > loader_time: 0.00390 (0.06071) + + + --> STEP: 233/234 -- GLOBAL_STEP: 65285 + | > loss: -0.03409 (-0.33324) + | > log_mle: -0.66721 (-0.49121) + | > loss_dur: 0.63312 (0.15797) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 123.78469 (37.29229) + | > current_lr: 0.00007 + | > step_time: 0.19470 (2.67114) + | > loader_time: 0.00330 (0.05980) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.34828 (+0.88979) + | > avg_loss: -0.32996 (+0.01124) + | > avg_log_mle: -0.54828 (+0.01492) + | > avg_loss_dur: 0.21832 (-0.00369) + + + > EPOCH: 279/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 20:01:30)  + + --> STEP: 4/234 -- GLOBAL_STEP: 65290 + | > loss: -0.32827 (-0.30417) + | > log_mle: -0.40583 (-0.40562) + | > loss_dur: 0.07756 (0.10145) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.56386 (16.58483) + | > current_lr: 0.00007 + | > step_time: 0.55770 (6.15551) + | > loader_time: 0.00120 (0.20072) + + + --> STEP: 9/234 -- GLOBAL_STEP: 65295 + | > loss: -0.29548 (-0.32030) + | > log_mle: -0.41468 (-0.40993) + | > loss_dur: 0.11920 (0.08963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.41385 (16.76772) + | > current_lr: 0.00007 + | > step_time: 0.70800 (3.14015) + | > loader_time: 0.00540 (0.09054) + + + --> STEP: 14/234 -- GLOBAL_STEP: 65300 + | > loss: -0.34098 (-0.32781) + | > log_mle: -0.41436 (-0.41222) + | > loss_dur: 0.07338 (0.08441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.88368 (16.17873) + | > current_lr: 0.00007 + | > step_time: 0.79450 (3.18322) + | > loader_time: 0.00170 (0.07065) + + + --> STEP: 19/234 -- GLOBAL_STEP: 65305 + | > loss: -0.35534 (-0.33403) + | > log_mle: -0.41968 (-0.41381) + | > loss_dur: 0.06434 (0.07979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.61942 (14.65584) + | > current_lr: 0.00007 + | > step_time: 2.49950 (3.11341) + | > loader_time: 0.00100 (0.06225) + + + --> STEP: 24/234 -- GLOBAL_STEP: 65310 + | > loss: -0.33740 (-0.33605) + | > log_mle: -0.40460 (-0.41353) + | > loss_dur: 0.06720 (0.07747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.35964 (14.26526) + | > current_lr: 0.00007 + | > step_time: 2.85610 (3.05569) + | > loader_time: 0.00200 (0.05348) + + + --> STEP: 29/234 -- GLOBAL_STEP: 65315 + | > loss: -0.33050 (-0.33666) + | > log_mle: -0.40251 (-0.41222) + | > loss_dur: 0.07202 (0.07556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.41757 (13.92286) + | > current_lr: 0.00007 + | > step_time: 1.47640 (2.75867) + | > loader_time: 0.00170 (0.04746) + + + --> STEP: 34/234 -- GLOBAL_STEP: 65320 + | > loss: -0.33643 (-0.33485) + | > log_mle: -0.40059 (-0.41029) + | > loss_dur: 0.06416 (0.07544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.32079 (13.32300) + | > current_lr: 0.00007 + | > step_time: 1.42790 (2.66956) + | > loader_time: 0.00210 (0.04080) + + + --> STEP: 39/234 -- GLOBAL_STEP: 65325 + | > loss: -0.28373 (-0.33053) + | > log_mle: -0.38838 (-0.40733) + | > loss_dur: 0.10465 (0.07680) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.99307 (13.07075) + | > current_lr: 0.00007 + | > step_time: 1.11460 (2.45824) + | > loader_time: 0.00160 (0.03581) + + + --> STEP: 44/234 -- GLOBAL_STEP: 65330 + | > loss: -0.31804 (-0.32807) + | > log_mle: -0.38256 (-0.40511) + | > loss_dur: 0.06452 (0.07704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.79521 (12.70082) + | > current_lr: 0.00007 + | > step_time: 1.79820 (2.40533) + | > loader_time: 0.08360 (0.03574) + + + --> STEP: 49/234 -- GLOBAL_STEP: 65335 + | > loss: -0.32599 (-0.32656) + | > log_mle: -0.39847 (-0.40409) + | > loss_dur: 0.07248 (0.07753) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.63912 (12.52037) + | > current_lr: 0.00007 + | > step_time: 2.40090 (2.34919) + | > loader_time: 0.00990 (0.03616) + + + --> STEP: 54/234 -- GLOBAL_STEP: 65340 + | > loss: -0.31217 (-0.32405) + | > log_mle: -0.38979 (-0.40234) + | > loss_dur: 0.07762 (0.07829) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.16585 (12.28733) + | > current_lr: 0.00007 + | > step_time: 1.23300 (2.25110) + | > loader_time: 0.00170 (0.03299) + + + --> STEP: 59/234 -- GLOBAL_STEP: 65345 + | > loss: -0.30211 (-0.32221) + | > log_mle: -0.38722 (-0.40083) + | > loss_dur: 0.08511 (0.07862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.17894 (12.14555) + | > current_lr: 0.00007 + | > step_time: 1.54390 (2.22558) + | > loader_time: 0.00200 (0.03039) + + + --> STEP: 64/234 -- GLOBAL_STEP: 65350 + | > loss: -0.29170 (-0.31861) + | > log_mle: -0.37641 (-0.39934) + | > loss_dur: 0.08471 (0.08074) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.20512 (12.22011) + | > current_lr: 0.00007 + | > step_time: 1.23160 (2.18277) + | > loader_time: 0.00230 (0.02947) + + + --> STEP: 69/234 -- GLOBAL_STEP: 65355 + | > loss: -0.29102 (-0.31590) + | > log_mle: -0.37334 (-0.39757) + | > loss_dur: 0.08232 (0.08167) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.28761 (12.08857) + | > current_lr: 0.00007 + | > step_time: 1.80980 (2.19415) + | > loader_time: 0.00760 (0.02764) + + + --> STEP: 74/234 -- GLOBAL_STEP: 65360 + | > loss: -0.26093 (-0.31258) + | > log_mle: -0.36379 (-0.39608) + | > loss_dur: 0.10286 (0.08351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.54788 (12.18601) + | > current_lr: 0.00007 + | > step_time: 1.81060 (2.14719) + | > loader_time: 0.10090 (0.02732) + + + --> STEP: 79/234 -- GLOBAL_STEP: 65365 + | > loss: -0.26690 (-0.30977) + | > log_mle: -0.37698 (-0.39474) + | > loss_dur: 0.11008 (0.08497) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.92032 (12.10642) + | > current_lr: 0.00007 + | > step_time: 1.58630 (2.10853) + | > loader_time: 0.00180 (0.02675) + + + --> STEP: 84/234 -- GLOBAL_STEP: 65370 + | > loss: -0.28084 (-0.30716) + | > log_mle: -0.37775 (-0.39356) + | > loss_dur: 0.09691 (0.08641) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.48985 (12.08009) + | > current_lr: 0.00007 + | > step_time: 1.71260 (2.13434) + | > loader_time: 0.00390 (0.03073) + + + --> STEP: 89/234 -- GLOBAL_STEP: 65375 + | > loss: -0.28839 (-0.30520) + | > log_mle: -0.39957 (-0.39321) + | > loss_dur: 0.11118 (0.08802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.37485 (12.15359) + | > current_lr: 0.00007 + | > step_time: 3.20280 (2.15583) + | > loader_time: 0.09800 (0.03024) + + + --> STEP: 94/234 -- GLOBAL_STEP: 65380 + | > loss: -0.29091 (-0.30354) + | > log_mle: -0.42299 (-0.39397) + | > loss_dur: 0.13208 (0.09043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.95304 (12.62395) + | > current_lr: 0.00007 + | > step_time: 1.55690 (2.14516) + | > loader_time: 0.00230 (0.03056) + + + --> STEP: 99/234 -- GLOBAL_STEP: 65385 + | > loss: -0.29631 (-0.30228) + | > log_mle: -0.45546 (-0.39492) + | > loss_dur: 0.15915 (0.09264) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.54296 (12.98574) + | > current_lr: 0.00007 + | > step_time: 1.48540 (2.14487) + | > loader_time: 0.00400 (0.03092) + + + --> STEP: 104/234 -- GLOBAL_STEP: 65390 + | > loss: -0.31953 (-0.30133) + | > log_mle: -0.46201 (-0.39624) + | > loss_dur: 0.14248 (0.09491) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.05000 (13.71972) + | > current_lr: 0.00007 + | > step_time: 1.87600 (2.13984) + | > loader_time: 0.09710 (0.03118) + + + --> STEP: 109/234 -- GLOBAL_STEP: 65395 + | > loss: -0.24454 (-0.29918) + | > log_mle: -0.43228 (-0.39662) + | > loss_dur: 0.18773 (0.09744) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.18245 (14.33002) + | > current_lr: 0.00007 + | > step_time: 1.19820 (2.11121) + | > loader_time: 0.00240 (0.02986) + + + --> STEP: 114/234 -- GLOBAL_STEP: 65400 + | > loss: -0.26508 (-0.29778) + | > log_mle: -0.41369 (-0.39792) + | > loss_dur: 0.14861 (0.10014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.01275 (15.21660) + | > current_lr: 0.00007 + | > step_time: 2.16560 (2.13522) + | > loader_time: 0.00210 (0.03021) + + + --> STEP: 119/234 -- GLOBAL_STEP: 65405 + | > loss: -0.27172 (-0.29610) + | > log_mle: -0.41321 (-0.39884) + | > loss_dur: 0.14149 (0.10273) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.57018 (15.54149) + | > current_lr: 0.00007 + | > step_time: 2.20040 (2.12785) + | > loader_time: 0.08430 (0.03124) + + + --> STEP: 124/234 -- GLOBAL_STEP: 65410 + | > loss: -0.28760 (-0.29449) + | > log_mle: -0.44229 (-0.39913) + | > loss_dur: 0.15469 (0.10464) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.50988 (15.73676) + | > current_lr: 0.00007 + | > step_time: 2.99840 (2.15521) + | > loader_time: 0.00340 (0.03078) + + + --> STEP: 129/234 -- GLOBAL_STEP: 65415 + | > loss: -0.27157 (-0.29390) + | > log_mle: -0.43627 (-0.40114) + | > loss_dur: 0.16470 (0.10723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.40564 (16.34350) + | > current_lr: 0.00007 + | > step_time: 2.99750 (2.16152) + | > loader_time: 0.00200 (0.02971) + + + --> STEP: 134/234 -- GLOBAL_STEP: 65420 + | > loss: -0.29649 (-0.29438) + | > log_mle: -0.47418 (-0.40394) + | > loss_dur: 0.17769 (0.10957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.37531 (17.29014) + | > current_lr: 0.00007 + | > step_time: 3.60730 (2.20930) + | > loader_time: 0.29250 (0.03280) + + + --> STEP: 139/234 -- GLOBAL_STEP: 65425 + | > loss: -0.34794 (-0.29427) + | > log_mle: -0.54205 (-0.40635) + | > loss_dur: 0.19410 (0.11208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.83465 (18.46683) + | > current_lr: 0.00007 + | > step_time: 1.93210 (2.22336) + | > loader_time: 0.00210 (0.03264) + + + --> STEP: 144/234 -- GLOBAL_STEP: 65430 + | > loss: -0.32699 (-0.29445) + | > log_mle: -0.51838 (-0.40922) + | > loss_dur: 0.19138 (0.11477) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.84377 (19.11134) + | > current_lr: 0.00007 + | > step_time: 1.79870 (2.20891) + | > loader_time: 0.00320 (0.03161) + + + --> STEP: 149/234 -- GLOBAL_STEP: 65435 + | > loss: -0.39077 (-0.29589) + | > log_mle: -0.57708 (-0.41278) + | > loss_dur: 0.18631 (0.11689) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.35280 (19.87789) + | > current_lr: 0.00007 + | > step_time: 3.64910 (2.21291) + | > loader_time: 0.00370 (0.03196) + + + --> STEP: 154/234 -- GLOBAL_STEP: 65440 + | > loss: -0.34034 (-0.29768) + | > log_mle: -0.52863 (-0.41697) + | > loss_dur: 0.18829 (0.11928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.44662 (20.71086) + | > current_lr: 0.00007 + | > step_time: 2.39780 (2.19892) + | > loader_time: 0.09910 (0.03166) + + + --> STEP: 159/234 -- GLOBAL_STEP: 65445 + | > loss: -0.35987 (-0.29928) + | > log_mle: -0.55412 (-0.42094) + | > loss_dur: 0.19425 (0.12165) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.05547 (21.54710) + | > current_lr: 0.00007 + | > step_time: 2.40450 (2.20243) + | > loader_time: 0.08850 (0.03130) + + + --> STEP: 164/234 -- GLOBAL_STEP: 65450 + | > loss: -0.33899 (-0.30093) + | > log_mle: -0.54437 (-0.42480) + | > loss_dur: 0.20538 (0.12387) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.10207 (22.47141) + | > current_lr: 0.00007 + | > step_time: 2.10200 (2.23289) + | > loader_time: 0.08160 (0.03380) + + + --> STEP: 169/234 -- GLOBAL_STEP: 65455 + | > loss: -0.33789 (-0.30295) + | > log_mle: -0.53954 (-0.42894) + | > loss_dur: 0.20165 (0.12599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.55080 (23.47988) + | > current_lr: 0.00007 + | > step_time: 4.30780 (2.25121) + | > loader_time: 0.09070 (0.03343) + + + --> STEP: 174/234 -- GLOBAL_STEP: 65460 + | > loss: -0.43582 (-0.30585) + | > log_mle: -0.63758 (-0.43422) + | > loss_dur: 0.20176 (0.12837) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.86047 (24.57652) + | > current_lr: 0.00007 + | > step_time: 3.30390 (2.26548) + | > loader_time: 0.00320 (0.03415) + + + --> STEP: 179/234 -- GLOBAL_STEP: 65465 + | > loss: -0.40004 (-0.30807) + | > log_mle: -0.63541 (-0.43893) + | > loss_dur: 0.23537 (0.13086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.32719 (25.94849) + | > current_lr: 0.00007 + | > step_time: 3.70330 (2.29050) + | > loader_time: 0.09260 (0.03436) + + + --> STEP: 184/234 -- GLOBAL_STEP: 65470 + | > loss: -0.37992 (-0.30985) + | > log_mle: -0.59509 (-0.44307) + | > loss_dur: 0.21517 (0.13322) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.21359 (27.02271) + | > current_lr: 0.00007 + | > step_time: 4.51050 (2.34576) + | > loader_time: 0.09170 (0.03559) + + + --> STEP: 189/234 -- GLOBAL_STEP: 65475 + | > loss: -0.37277 (-0.31194) + | > log_mle: -0.59236 (-0.44744) + | > loss_dur: 0.21959 (0.13551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.03372 (27.81236) + | > current_lr: 0.00007 + | > step_time: 6.19380 (2.43613) + | > loader_time: 0.20550 (0.03772) + + + --> STEP: 194/234 -- GLOBAL_STEP: 65480 + | > loss: -0.42004 (-0.31464) + | > log_mle: -0.62731 (-0.45195) + | > loss_dur: 0.20728 (0.13732) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.80253 (28.75597) + | > current_lr: 0.00007 + | > step_time: 4.99930 (2.45405) + | > loader_time: 0.08970 (0.03733) + + + --> STEP: 199/234 -- GLOBAL_STEP: 65485 + | > loss: -0.42107 (-0.31679) + | > log_mle: -0.64040 (-0.45615) + | > loss_dur: 0.21933 (0.13935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.79398 (29.82795) + | > current_lr: 0.00007 + | > step_time: 3.58890 (2.49642) + | > loader_time: 0.09680 (0.03785) + + + --> STEP: 204/234 -- GLOBAL_STEP: 65490 + | > loss: -0.40839 (-0.31843) + | > log_mle: -0.64552 (-0.45991) + | > loss_dur: 0.23713 (0.14148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.78677 (30.88323) + | > current_lr: 0.00007 + | > step_time: 5.30820 (2.57065) + | > loader_time: 0.07950 (0.03916) + + + --> STEP: 209/234 -- GLOBAL_STEP: 65495 + | > loss: -0.38791 (-0.32045) + | > log_mle: -0.60716 (-0.46396) + | > loss_dur: 0.21925 (0.14351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.26433 (31.97120) + | > current_lr: 0.00007 + | > step_time: 5.99860 (2.63596) + | > loader_time: 0.20390 (0.04064) + + + --> STEP: 214/234 -- GLOBAL_STEP: 65500 + | > loss: -0.44301 (-0.32321) + | > log_mle: -0.65582 (-0.46885) + | > loss_dur: 0.21282 (0.14564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.64756 (33.38918) + | > current_lr: 0.00007 + | > step_time: 3.30800 (2.70699) + | > loader_time: 0.08750 (0.04111) + + + --> STEP: 219/234 -- GLOBAL_STEP: 65505 + | > loss: -0.53131 (-0.32611) + | > log_mle: -0.75621 (-0.47374) + | > loss_dur: 0.22490 (0.14763) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.20514 (34.59279) + | > current_lr: 0.00007 + | > step_time: 11.00300 (2.79864) + | > loader_time: 0.00430 (0.04199) + + + --> STEP: 224/234 -- GLOBAL_STEP: 65510 + | > loss: -0.46183 (-0.32871) + | > log_mle: -0.69823 (-0.47840) + | > loss_dur: 0.23640 (0.14969) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.15314 (35.29074) + | > current_lr: 0.00007 + | > step_time: 0.22850 (2.77523) + | > loader_time: 0.00410 (0.04295) + + + --> STEP: 229/234 -- GLOBAL_STEP: 65515 + | > loss: -0.43842 (-0.33150) + | > log_mle: -0.73591 (-0.48356) + | > loss_dur: 0.29749 (0.15206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 122.07568 (36.65162) + | > current_lr: 0.00007 + | > step_time: 0.24460 (2.71983) + | > loader_time: 0.00270 (0.04210) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12902 (-1.21925) + | > avg_loss: -0.31219 (+0.01777) + | > avg_log_mle: -0.54551 (+0.00277) + | > avg_loss_dur: 0.23331 (+0.01500) + + + > EPOCH: 280/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 20:13:09)  + + --> STEP: 0/234 -- GLOBAL_STEP: 65520 + | > loss: -0.31411 (-0.31411) + | > log_mle: -0.48125 (-0.48125) + | > loss_dur: 0.16714 (0.16714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.33890 (30.33890) + | > current_lr: 0.00007 + | > step_time: 8.81330 (8.81333) + | > loader_time: 20.75880 (20.75878) + + + --> STEP: 5/234 -- GLOBAL_STEP: 65525 + | > loss: -0.30904 (-0.31246) + | > log_mle: -0.40550 (-0.40476) + | > loss_dur: 0.09646 (0.09230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.52924 (26.01819) + | > current_lr: 0.00007 + | > step_time: 1.23990 (3.79008) + | > loader_time: 0.00330 (0.06167) + + + --> STEP: 10/234 -- GLOBAL_STEP: 65530 + | > loss: -0.32221 (-0.32272) + | > log_mle: -0.41200 (-0.40955) + | > loss_dur: 0.08979 (0.08683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.95733 (21.95980) + | > current_lr: 0.00007 + | > step_time: 1.89460 (2.46578) + | > loader_time: 0.00120 (0.03152) + + + --> STEP: 15/234 -- GLOBAL_STEP: 65535 + | > loss: -0.36054 (-0.33272) + | > log_mle: -0.42273 (-0.41341) + | > loss_dur: 0.06219 (0.08070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.61351 (19.12464) + | > current_lr: 0.00007 + | > step_time: 0.85480 (2.53572) + | > loader_time: 0.00200 (0.04893) + + + --> STEP: 20/234 -- GLOBAL_STEP: 65540 + | > loss: -0.35641 (-0.33772) + | > log_mle: -0.42271 (-0.41509) + | > loss_dur: 0.06630 (0.07737) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.80268 (17.23339) + | > current_lr: 0.00007 + | > step_time: 3.52370 (2.68147) + | > loader_time: 0.08310 (0.04124) + + + --> STEP: 25/234 -- GLOBAL_STEP: 65545 + | > loss: -0.31905 (-0.33792) + | > log_mle: -0.39627 (-0.41388) + | > loss_dur: 0.07722 (0.07597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.10909 (16.29528) + | > current_lr: 0.00007 + | > step_time: 2.99630 (3.07750) + | > loader_time: 0.11830 (0.03836) + + + --> STEP: 30/234 -- GLOBAL_STEP: 65550 + | > loss: -0.32097 (-0.33787) + | > log_mle: -0.40252 (-0.41304) + | > loss_dur: 0.08155 (0.07517) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.25349 (15.60906) + | > current_lr: 0.00007 + | > step_time: 7.28650 (3.03156) + | > loader_time: 0.00640 (0.03489) + + + --> STEP: 35/234 -- GLOBAL_STEP: 65555 + | > loss: -0.29257 (-0.33482) + | > log_mle: -0.38532 (-0.41084) + | > loss_dur: 0.09275 (0.07602) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.13184 (15.02460) + | > current_lr: 0.00007 + | > step_time: 1.09520 (3.22223) + | > loader_time: 0.00360 (0.04406) + + + --> STEP: 40/234 -- GLOBAL_STEP: 65560 + | > loss: -0.30274 (-0.33067) + | > log_mle: -0.39098 (-0.40812) + | > loss_dur: 0.08824 (0.07745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29567 (14.58067) + | > current_lr: 0.00007 + | > step_time: 2.16830 (3.11916) + | > loader_time: 0.00150 (0.04272) + + + --> STEP: 45/234 -- GLOBAL_STEP: 65565 + | > loss: -0.27116 (-0.32752) + | > log_mle: -0.38896 (-0.40573) + | > loss_dur: 0.11780 (0.07822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.42621 (14.62841) + | > current_lr: 0.00007 + | > step_time: 1.56570 (2.91870) + | > loader_time: 0.00150 (0.04000) + + + --> STEP: 50/234 -- GLOBAL_STEP: 65570 + | > loss: -0.30066 (-0.32606) + | > log_mle: -0.38031 (-0.40421) + | > loss_dur: 0.07965 (0.07815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.55902 (14.26924) + | > current_lr: 0.00007 + | > step_time: 4.49730 (2.80209) + | > loader_time: 0.10240 (0.03828) + + + --> STEP: 55/234 -- GLOBAL_STEP: 65575 + | > loss: -0.31396 (-0.32358) + | > log_mle: -0.38836 (-0.40240) + | > loss_dur: 0.07440 (0.07882) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.51979 (13.95290) + | > current_lr: 0.00007 + | > step_time: 2.53220 (2.75081) + | > loader_time: 0.00220 (0.03500) + + + --> STEP: 60/234 -- GLOBAL_STEP: 65580 + | > loss: -0.26919 (-0.32080) + | > log_mle: -0.38807 (-0.40090) + | > loss_dur: 0.11888 (0.08010) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.78763 (13.77078) + | > current_lr: 0.00007 + | > step_time: 1.39880 (2.69417) + | > loader_time: 0.07480 (0.03665) + + + --> STEP: 65/234 -- GLOBAL_STEP: 65585 + | > loss: -0.28340 (-0.31728) + | > log_mle: -0.37577 (-0.39917) + | > loss_dur: 0.09237 (0.08189) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.91180 (13.77267) + | > current_lr: 0.00007 + | > step_time: 0.81130 (2.58697) + | > loader_time: 0.00350 (0.03815) + + + --> STEP: 70/234 -- GLOBAL_STEP: 65590 + | > loss: -0.25375 (-0.31416) + | > log_mle: -0.36214 (-0.39721) + | > loss_dur: 0.10840 (0.08305) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.30178 (13.56038) + | > current_lr: 0.00007 + | > step_time: 1.60130 (2.52909) + | > loader_time: 0.00240 (0.03684) + + + --> STEP: 75/234 -- GLOBAL_STEP: 65595 + | > loss: -0.25330 (-0.31099) + | > log_mle: -0.37795 (-0.39602) + | > loss_dur: 0.12465 (0.08503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.56805 (13.40639) + | > current_lr: 0.00007 + | > step_time: 1.49510 (2.47075) + | > loader_time: 0.00230 (0.03591) + + + --> STEP: 80/234 -- GLOBAL_STEP: 65600 + | > loss: -0.27197 (-0.30833) + | > log_mle: -0.36452 (-0.39449) + | > loss_dur: 0.09255 (0.08616) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.96334 (13.43394) + | > current_lr: 0.00007 + | > step_time: 1.36180 (2.41443) + | > loader_time: 0.00240 (0.03381) + + + --> STEP: 85/234 -- GLOBAL_STEP: 65605 + | > loss: -0.26663 (-0.30578) + | > log_mle: -0.37062 (-0.39342) + | > loss_dur: 0.10399 (0.08764) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.04404 (13.37323) + | > current_lr: 0.00007 + | > step_time: 3.00010 (2.43816) + | > loader_time: 0.09090 (0.03776) + + + --> STEP: 90/234 -- GLOBAL_STEP: 65610 + | > loss: -0.26454 (-0.30373) + | > log_mle: -0.38872 (-0.39325) + | > loss_dur: 0.12418 (0.08953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.94258 (13.42577) + | > current_lr: 0.00007 + | > step_time: 1.98900 (2.39692) + | > loader_time: 0.00200 (0.03794) + + + --> STEP: 95/234 -- GLOBAL_STEP: 65615 + | > loss: -0.30979 (-0.30264) + | > log_mle: -0.46728 (-0.39483) + | > loss_dur: 0.15749 (0.09219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.69318 (13.87032) + | > current_lr: 0.00007 + | > step_time: 1.39550 (2.36543) + | > loader_time: 0.08980 (0.03707) + + + --> STEP: 100/234 -- GLOBAL_STEP: 65620 + | > loss: -0.27083 (-0.30094) + | > log_mle: -0.39685 (-0.39486) + | > loss_dur: 0.12602 (0.09392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.93356 (14.11872) + | > current_lr: 0.00007 + | > step_time: 2.48590 (2.37506) + | > loader_time: 0.00540 (0.03832) + + + --> STEP: 105/234 -- GLOBAL_STEP: 65625 + | > loss: -0.25055 (-0.29943) + | > log_mle: -0.37541 (-0.39577) + | > loss_dur: 0.12486 (0.09634) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.39463 (14.68684) + | > current_lr: 0.00007 + | > step_time: 1.17110 (2.34260) + | > loader_time: 0.00190 (0.03817) + + + --> STEP: 110/234 -- GLOBAL_STEP: 65630 + | > loss: -0.24577 (-0.29730) + | > log_mle: -0.39171 (-0.39628) + | > loss_dur: 0.14594 (0.09898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.75696 (15.32010) + | > current_lr: 0.00007 + | > step_time: 2.79600 (2.38257) + | > loader_time: 0.00460 (0.03813) + + + --> STEP: 115/234 -- GLOBAL_STEP: 65635 + | > loss: -0.26519 (-0.29624) + | > log_mle: -0.41898 (-0.39792) + | > loss_dur: 0.15378 (0.10168) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.63942 (15.91201) + | > current_lr: 0.00007 + | > step_time: 2.39730 (2.36991) + | > loader_time: 0.10750 (0.03837) + + + --> STEP: 120/234 -- GLOBAL_STEP: 65640 + | > loss: -0.29953 (-0.29500) + | > log_mle: -0.46437 (-0.39923) + | > loss_dur: 0.16483 (0.10424) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.65552 (16.36803) + | > current_lr: 0.00007 + | > step_time: 1.79870 (2.33540) + | > loader_time: 0.00260 (0.03833) + + + --> STEP: 125/234 -- GLOBAL_STEP: 65645 + | > loss: -0.26850 (-0.29348) + | > log_mle: -0.44441 (-0.39937) + | > loss_dur: 0.17591 (0.10589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.23299 (16.78027) + | > current_lr: 0.00007 + | > step_time: 1.56110 (2.34267) + | > loader_time: 0.00260 (0.03974) + + + --> STEP: 130/234 -- GLOBAL_STEP: 65650 + | > loss: -0.27098 (-0.29293) + | > log_mle: -0.44958 (-0.40133) + | > loss_dur: 0.17860 (0.10841) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.80008 (17.43169) + | > current_lr: 0.00007 + | > step_time: 2.39980 (2.32276) + | > loader_time: 0.00310 (0.03968) + + + --> STEP: 135/234 -- GLOBAL_STEP: 65655 + | > loss: -0.25573 (-0.29288) + | > log_mle: -0.39325 (-0.40346) + | > loss_dur: 0.13753 (0.11058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.94907 (17.95531) + | > current_lr: 0.00007 + | > step_time: 2.18410 (2.33936) + | > loader_time: 0.00260 (0.04046) + + + --> STEP: 140/234 -- GLOBAL_STEP: 65660 + | > loss: -0.23858 (-0.29302) + | > log_mle: -0.42662 (-0.40628) + | > loss_dur: 0.18805 (0.11327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.96841 (18.59814) + | > current_lr: 0.00007 + | > step_time: 1.80670 (2.32307) + | > loader_time: 0.08410 (0.04022) + + + --> STEP: 145/234 -- GLOBAL_STEP: 65665 + | > loss: -0.35293 (-0.29405) + | > log_mle: -0.53343 (-0.40999) + | > loss_dur: 0.18050 (0.11594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.67893 (19.37579) + | > current_lr: 0.00007 + | > step_time: 1.30780 (2.33745) + | > loader_time: 0.09220 (0.03957) + + + --> STEP: 150/234 -- GLOBAL_STEP: 65670 + | > loss: -0.31678 (-0.29530) + | > log_mle: -0.51298 (-0.41349) + | > loss_dur: 0.19619 (0.11818) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.41601 (20.18561) + | > current_lr: 0.00007 + | > step_time: 1.39380 (2.34489) + | > loader_time: 0.00440 (0.03963) + + + --> STEP: 155/234 -- GLOBAL_STEP: 65675 + | > loss: -0.37592 (-0.29736) + | > log_mle: -0.57454 (-0.41792) + | > loss_dur: 0.19862 (0.12056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.23736 (21.46836) + | > current_lr: 0.00007 + | > step_time: 2.28590 (2.33432) + | > loader_time: 0.00220 (0.03905) + + + --> STEP: 160/234 -- GLOBAL_STEP: 65680 + | > loss: -0.35742 (-0.29858) + | > log_mle: -0.57062 (-0.42163) + | > loss_dur: 0.21320 (0.12306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.42493 (22.56924) + | > current_lr: 0.00007 + | > step_time: 1.49350 (2.36155) + | > loader_time: 0.00310 (0.04081) + + + --> STEP: 165/234 -- GLOBAL_STEP: 65685 + | > loss: -0.33285 (-0.29951) + | > log_mle: -0.52917 (-0.42461) + | > loss_dur: 0.19633 (0.12510) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.32372 (24.67328) + | > current_lr: 0.00007 + | > step_time: 0.69790 (2.35060) + | > loader_time: 0.00370 (0.04022) + + + --> STEP: 170/234 -- GLOBAL_STEP: 65690 + | > loss: -0.36159 (-0.30068) + | > log_mle: -0.58342 (-0.42803) + | > loss_dur: 0.22183 (0.12735) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.28749 (25.56613) + | > current_lr: 0.00007 + | > step_time: 3.50660 (2.36284) + | > loader_time: 0.00440 (0.04125) + + + --> STEP: 175/234 -- GLOBAL_STEP: 65695 + | > loss: -0.35353 (-0.30295) + | > log_mle: -0.57948 (-0.43270) + | > loss_dur: 0.22595 (0.12976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.92278 (26.12967) + | > current_lr: 0.00007 + | > step_time: 0.79950 (2.46667) + | > loader_time: 0.00250 (0.04129) + + + --> STEP: 180/234 -- GLOBAL_STEP: 65700 + | > loss: -0.36705 (-0.30481) + | > log_mle: -0.58136 (-0.43704) + | > loss_dur: 0.21431 (0.13223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.82733 (26.98432) + | > current_lr: 0.00007 + | > step_time: 2.99850 (2.46859) + | > loader_time: 0.00320 (0.04071) + + + --> STEP: 185/234 -- GLOBAL_STEP: 65705 + | > loss: -0.39296 (-0.30652) + | > log_mle: -0.61573 (-0.44103) + | > loss_dur: 0.22277 (0.13451) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.44801 (27.72699) + | > current_lr: 0.00007 + | > step_time: 2.49520 (2.46982) + | > loader_time: 0.20000 (0.04133) + + + --> STEP: 190/234 -- GLOBAL_STEP: 65710 + | > loss: -0.38881 (-0.30840) + | > log_mle: -0.58950 (-0.44509) + | > loss_dur: 0.20069 (0.13669) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.80699 (28.52178) + | > current_lr: 0.00007 + | > step_time: 2.10190 (2.51595) + | > loader_time: 0.09670 (0.04233) + + + --> STEP: 195/234 -- GLOBAL_STEP: 65715 + | > loss: -0.39374 (-0.31102) + | > log_mle: -0.62242 (-0.44963) + | > loss_dur: 0.22869 (0.13861) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.01991 (29.46668) + | > current_lr: 0.00007 + | > step_time: 3.09390 (2.57954) + | > loader_time: 0.00730 (0.04334) + + + --> STEP: 200/234 -- GLOBAL_STEP: 65720 + | > loss: -0.39604 (-0.31317) + | > log_mle: -0.63003 (-0.45381) + | > loss_dur: 0.23399 (0.14064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.20271 (30.31109) + | > current_lr: 0.00007 + | > step_time: 3.49270 (2.62762) + | > loader_time: 0.10290 (0.04457) + + + --> STEP: 205/234 -- GLOBAL_STEP: 65725 + | > loss: -0.39030 (-0.31519) + | > log_mle: -0.60959 (-0.45774) + | > loss_dur: 0.21929 (0.14255) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.30014 (31.30973) + | > current_lr: 0.00007 + | > step_time: 4.80050 (2.66746) + | > loader_time: 0.20870 (0.04552) + + + --> STEP: 210/234 -- GLOBAL_STEP: 65730 + | > loss: -0.45106 (-0.31806) + | > log_mle: -0.69524 (-0.46264) + | > loss_dur: 0.24418 (0.14458) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.38741 (32.25300) + | > current_lr: 0.00007 + | > step_time: 4.50540 (2.75451) + | > loader_time: 0.29680 (0.04910) + + + --> STEP: 215/234 -- GLOBAL_STEP: 65735 + | > loss: -0.42209 (-0.32124) + | > log_mle: -0.65032 (-0.46775) + | > loss_dur: 0.22823 (0.14651) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.97453 (33.11442) + | > current_lr: 0.00007 + | > step_time: 5.79200 (2.86540) + | > loader_time: 0.00570 (0.05022) + + + --> STEP: 220/234 -- GLOBAL_STEP: 65740 + | > loss: -0.44944 (-0.32464) + | > log_mle: -0.69655 (-0.47324) + | > loss_dur: 0.24711 (0.14860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.07186 (34.11953) + | > current_lr: 0.00007 + | > step_time: 1.12430 (2.91085) + | > loader_time: 0.18540 (0.05219) + + + --> STEP: 225/234 -- GLOBAL_STEP: 65745 + | > loss: -0.53460 (-0.32791) + | > log_mle: -0.77759 (-0.47848) + | > loss_dur: 0.24299 (0.15057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.02947 (34.98384) + | > current_lr: 0.00007 + | > step_time: 0.23410 (2.85491) + | > loader_time: 0.00420 (0.05111) + + + --> STEP: 230/234 -- GLOBAL_STEP: 65750 + | > loss: -0.52434 (-0.33111) + | > log_mle: -0.83112 (-0.48433) + | > loss_dur: 0.30678 (0.15322) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.93363 (36.21068) + | > current_lr: 0.00007 + | > step_time: 0.25070 (2.79816) + | > loader_time: 0.00400 (0.05008) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.17747 (+0.04845) + | > avg_loss: -0.34137 (-0.02918) + | > avg_log_mle: -0.55909 (-0.01359) + | > avg_loss_dur: 0.21772 (-0.01559) + + + > EPOCH: 281/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 20:25:23)  + + --> STEP: 1/234 -- GLOBAL_STEP: 65755 + | > loss: -0.31707 (-0.31707) + | > log_mle: -0.40357 (-0.40357) + | > loss_dur: 0.08650 (0.08650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.88847 (24.88847) + | > current_lr: 0.00007 + | > step_time: 17.10580 (17.10580) + | > loader_time: 2.69670 (2.69675) + + + --> STEP: 6/234 -- GLOBAL_STEP: 65760 + | > loss: -0.34819 (-0.31473) + | > log_mle: -0.41249 (-0.41044) + | > loss_dur: 0.06430 (0.09571) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.47639 (17.61281) + | > current_lr: 0.00007 + | > step_time: 3.09970 (7.34956) + | > loader_time: 0.08410 (0.49460) + + + --> STEP: 11/234 -- GLOBAL_STEP: 65765 + | > loss: -0.35934 (-0.32573) + | > log_mle: -0.42490 (-0.41499) + | > loss_dur: 0.06556 (0.08927) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.74768 (17.35422) + | > current_lr: 0.00007 + | > step_time: 6.10110 (5.82165) + | > loader_time: 0.00130 (0.30634) + + + --> STEP: 16/234 -- GLOBAL_STEP: 65770 + | > loss: -0.36461 (-0.33462) + | > log_mle: -0.42730 (-0.41717) + | > loss_dur: 0.06269 (0.08255) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.49604 (16.97857) + | > current_lr: 0.00007 + | > step_time: 0.69700 (5.17596) + | > loader_time: 0.00170 (0.22358) + + + --> STEP: 21/234 -- GLOBAL_STEP: 65775 + | > loss: -0.32662 (-0.33769) + | > log_mle: -0.40212 (-0.41646) + | > loss_dur: 0.07551 (0.07877) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.60307 (15.77822) + | > current_lr: 0.00007 + | > step_time: 2.21100 (4.62971) + | > loader_time: 1.18490 (0.23074) + + + --> STEP: 26/234 -- GLOBAL_STEP: 65780 + | > loss: -0.32685 (-0.33825) + | > log_mle: -0.40073 (-0.41499) + | > loss_dur: 0.07388 (0.07675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.99416 (14.68972) + | > current_lr: 0.00007 + | > step_time: 3.79040 (4.52672) + | > loader_time: 0.10230 (0.19854) + + + --> STEP: 31/234 -- GLOBAL_STEP: 65785 + | > loss: -0.29917 (-0.33858) + | > log_mle: -0.39666 (-0.41446) + | > loss_dur: 0.09749 (0.07588) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.77455 (13.87638) + | > current_lr: 0.00007 + | > step_time: 3.00460 (4.37388) + | > loader_time: 0.09420 (0.16993) + + + --> STEP: 36/234 -- GLOBAL_STEP: 65790 + | > loss: -0.29851 (-0.33621) + | > log_mle: -0.38731 (-0.41217) + | > loss_dur: 0.08880 (0.07596) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.45867 (13.64581) + | > current_lr: 0.00007 + | > step_time: 9.50470 (4.53634) + | > loader_time: 0.10260 (0.15707) + + + --> STEP: 41/234 -- GLOBAL_STEP: 65795 + | > loss: -0.34753 (-0.33385) + | > log_mle: -0.40621 (-0.40996) + | > loss_dur: 0.05868 (0.07611) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.42384 (13.25924) + | > current_lr: 0.00007 + | > step_time: 1.20250 (4.14577) + | > loader_time: 0.00190 (0.13813) + + + --> STEP: 46/234 -- GLOBAL_STEP: 65800 + | > loss: -0.29804 (-0.33005) + | > log_mle: -0.38772 (-0.40734) + | > loss_dur: 0.08969 (0.07728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.28839 (13.08869) + | > current_lr: 0.00007 + | > step_time: 0.90810 (3.88599) + | > loader_time: 0.00460 (0.12341) + + + --> STEP: 51/234 -- GLOBAL_STEP: 65805 + | > loss: -0.30430 (-0.32878) + | > log_mle: -0.38542 (-0.40580) + | > loss_dur: 0.08112 (0.07702) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.99026 (12.68849) + | > current_lr: 0.00007 + | > step_time: 1.59670 (3.65189) + | > loader_time: 0.00220 (0.11159) + + + --> STEP: 56/234 -- GLOBAL_STEP: 65810 + | > loss: -0.30146 (-0.32635) + | > log_mle: -0.38990 (-0.40417) + | > loss_dur: 0.08844 (0.07782) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.30945 (12.55080) + | > current_lr: 0.00007 + | > step_time: 2.79970 (3.52027) + | > loader_time: 0.00530 (0.10369) + + + --> STEP: 61/234 -- GLOBAL_STEP: 65815 + | > loss: -0.28803 (-0.32348) + | > log_mle: -0.37966 (-0.40249) + | > loss_dur: 0.09164 (0.07901) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.19468 (12.46113) + | > current_lr: 0.00007 + | > step_time: 2.09960 (3.39509) + | > loader_time: 0.08280 (0.09669) + + + --> STEP: 66/234 -- GLOBAL_STEP: 65820 + | > loss: -0.29766 (-0.32030) + | > log_mle: -0.37494 (-0.40078) + | > loss_dur: 0.07727 (0.08048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.65889 (12.27451) + | > current_lr: 0.00007 + | > step_time: 2.89810 (3.28196) + | > loader_time: 0.00260 (0.08959) + + + --> STEP: 71/234 -- GLOBAL_STEP: 65825 + | > loss: -0.26982 (-0.31716) + | > log_mle: -0.39221 (-0.39913) + | > loss_dur: 0.12239 (0.08197) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.05524 (12.30645) + | > current_lr: 0.00007 + | > step_time: 1.20420 (3.21739) + | > loader_time: 0.09680 (0.08577) + + + --> STEP: 76/234 -- GLOBAL_STEP: 65830 + | > loss: -0.27239 (-0.31379) + | > log_mle: -0.37869 (-0.39761) + | > loss_dur: 0.10630 (0.08383) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.68989 (12.24888) + | > current_lr: 0.00007 + | > step_time: 3.61600 (3.19898) + | > loader_time: 0.19470 (0.08417) + + + --> STEP: 81/234 -- GLOBAL_STEP: 65835 + | > loss: -0.26953 (-0.31113) + | > log_mle: -0.38552 (-0.39614) + | > loss_dur: 0.11599 (0.08501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.28217 (12.14303) + | > current_lr: 0.00007 + | > step_time: 1.17980 (3.12234) + | > loader_time: 0.00220 (0.08140) + + + --> STEP: 86/234 -- GLOBAL_STEP: 65840 + | > loss: -0.27402 (-0.30849) + | > log_mle: -0.38607 (-0.39498) + | > loss_dur: 0.11205 (0.08649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.51042 (12.23577) + | > current_lr: 0.00007 + | > step_time: 2.90180 (3.07465) + | > loader_time: 0.00340 (0.07685) + + + --> STEP: 91/234 -- GLOBAL_STEP: 65845 + | > loss: -0.26674 (-0.30621) + | > log_mle: -0.39326 (-0.39479) + | > loss_dur: 0.12652 (0.08859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.18412 (12.47074) + | > current_lr: 0.00007 + | > step_time: 1.99320 (3.02590) + | > loader_time: 0.00180 (0.07277) + + + --> STEP: 96/234 -- GLOBAL_STEP: 65850 + | > loss: -0.26179 (-0.30496) + | > log_mle: -0.37812 (-0.39607) + | > loss_dur: 0.11634 (0.09111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.52643 (12.95091) + | > current_lr: 0.00007 + | > step_time: 1.56220 (2.94692) + | > loader_time: 0.00230 (0.06912) + + + --> STEP: 101/234 -- GLOBAL_STEP: 65855 + | > loss: -0.26391 (-0.30326) + | > log_mle: -0.41775 (-0.39643) + | > loss_dur: 0.15384 (0.09317) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.42319 (13.31290) + | > current_lr: 0.00007 + | > step_time: 2.59920 (2.90513) + | > loader_time: 0.00280 (0.06759) + + + --> STEP: 106/234 -- GLOBAL_STEP: 65860 + | > loss: -0.23475 (-0.30156) + | > log_mle: -0.41476 (-0.39735) + | > loss_dur: 0.18001 (0.09579) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.23517 (13.82593) + | > current_lr: 0.00007 + | > step_time: 2.40730 (2.86627) + | > loader_time: 0.00240 (0.06452) + + + --> STEP: 111/234 -- GLOBAL_STEP: 65865 + | > loss: -0.28540 (-0.29993) + | > log_mle: -0.47444 (-0.39847) + | > loss_dur: 0.18904 (0.09854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.65184 (14.27834) + | > current_lr: 0.00007 + | > step_time: 2.80260 (2.83877) + | > loader_time: 0.09490 (0.06422) + + + --> STEP: 116/234 -- GLOBAL_STEP: 65870 + | > loss: -0.25566 (-0.29835) + | > log_mle: -0.43816 (-0.39977) + | > loss_dur: 0.18251 (0.10141) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.37452 (14.65362) + | > current_lr: 0.00007 + | > step_time: 5.29180 (2.87830) + | > loader_time: 0.00710 (0.06491) + + + --> STEP: 121/234 -- GLOBAL_STEP: 65875 + | > loss: -0.22953 (-0.29697) + | > log_mle: -0.35689 (-0.40043) + | > loss_dur: 0.12736 (0.10346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42273 (14.84735) + | > current_lr: 0.00007 + | > step_time: 4.53880 (2.87961) + | > loader_time: 0.10840 (0.06556) + + + --> STEP: 126/234 -- GLOBAL_STEP: 65880 + | > loss: -0.30145 (-0.29584) + | > log_mle: -0.48642 (-0.40162) + | > loss_dur: 0.18497 (0.10578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.87752 (15.31346) + | > current_lr: 0.00007 + | > step_time: 0.92170 (2.81770) + | > loader_time: 0.08110 (0.06495) + + + --> STEP: 131/234 -- GLOBAL_STEP: 65885 + | > loss: -0.35184 (-0.29584) + | > log_mle: -0.53769 (-0.40410) + | > loss_dur: 0.18585 (0.10826) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.94024 (16.00931) + | > current_lr: 0.00007 + | > step_time: 1.21200 (2.79215) + | > loader_time: 0.00310 (0.06452) + + + --> STEP: 136/234 -- GLOBAL_STEP: 65890 + | > loss: -0.37409 (-0.29632) + | > log_mle: -0.58371 (-0.40678) + | > loss_dur: 0.20962 (0.11046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.38704 (16.68140) + | > current_lr: 0.00007 + | > step_time: 1.39600 (2.77185) + | > loader_time: 0.00200 (0.06290) + + + --> STEP: 141/234 -- GLOBAL_STEP: 65895 + | > loss: -0.32205 (-0.29632) + | > log_mle: -0.48743 (-0.40903) + | > loss_dur: 0.16538 (0.11271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.06398 (17.37520) + | > current_lr: 0.00007 + | > step_time: 2.71410 (2.75930) + | > loader_time: 0.08820 (0.06189) + + + --> STEP: 146/234 -- GLOBAL_STEP: 65900 + | > loss: -0.33975 (-0.29763) + | > log_mle: -0.53609 (-0.41306) + | > loss_dur: 0.19633 (0.11543) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.42738 (18.37148) + | > current_lr: 0.00007 + | > step_time: 2.61310 (2.75785) + | > loader_time: 0.08430 (0.06162) + + + --> STEP: 151/234 -- GLOBAL_STEP: 65905 + | > loss: -0.32962 (-0.29856) + | > log_mle: -0.49657 (-0.41611) + | > loss_dur: 0.16695 (0.11756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.63188 (19.19463) + | > current_lr: 0.00007 + | > step_time: 1.10200 (2.75880) + | > loader_time: 0.00260 (0.05969) + + + --> STEP: 156/234 -- GLOBAL_STEP: 65910 + | > loss: -0.35034 (-0.30090) + | > log_mle: -0.54800 (-0.42096) + | > loss_dur: 0.19766 (0.12006) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.93090 (20.17643) + | > current_lr: 0.00007 + | > step_time: 1.71120 (2.74079) + | > loader_time: 0.08160 (0.05956) + + + --> STEP: 161/234 -- GLOBAL_STEP: 65915 + | > loss: -0.38918 (-0.30255) + | > log_mle: -0.57277 (-0.42495) + | > loss_dur: 0.18360 (0.12240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.10513 (21.25166) + | > current_lr: 0.00007 + | > step_time: 2.71110 (2.74333) + | > loader_time: 0.00300 (0.05943) + + + --> STEP: 166/234 -- GLOBAL_STEP: 65920 + | > loss: -0.33443 (-0.30376) + | > log_mle: -0.50583 (-0.42817) + | > loss_dur: 0.17140 (0.12441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.51029 (22.01492) + | > current_lr: 0.00007 + | > step_time: 9.20130 (2.79441) + | > loader_time: 0.19610 (0.05941) + + + --> STEP: 171/234 -- GLOBAL_STEP: 65925 + | > loss: -0.41101 (-0.30609) + | > log_mle: -0.61096 (-0.43290) + | > loss_dur: 0.19996 (0.12681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.44076 (23.15827) + | > current_lr: 0.00007 + | > step_time: 1.91410 (2.80123) + | > loader_time: 0.00440 (0.05896) + + + --> STEP: 176/234 -- GLOBAL_STEP: 65930 + | > loss: -0.37736 (-0.30838) + | > log_mle: -0.58800 (-0.43760) + | > loss_dur: 0.21064 (0.12922) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.23077 (23.92720) + | > current_lr: 0.00007 + | > step_time: 2.27560 (2.76876) + | > loader_time: 0.00230 (0.05776) + + + --> STEP: 181/234 -- GLOBAL_STEP: 65935 + | > loss: -0.32405 (-0.31029) + | > log_mle: -0.52455 (-0.44189) + | > loss_dur: 0.20050 (0.13160) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.81078 (24.68613) + | > current_lr: 0.00007 + | > step_time: 4.69480 (2.78614) + | > loader_time: 0.00220 (0.05727) + + + --> STEP: 186/234 -- GLOBAL_STEP: 65940 + | > loss: -0.33394 (-0.31243) + | > log_mle: -0.55939 (-0.44640) + | > loss_dur: 0.22544 (0.13397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.17020 (25.83652) + | > current_lr: 0.00007 + | > step_time: 2.80560 (2.78764) + | > loader_time: 0.00280 (0.05720) + + + --> STEP: 191/234 -- GLOBAL_STEP: 65945 + | > loss: -0.38148 (-0.31466) + | > log_mle: -0.58625 (-0.45063) + | > loss_dur: 0.20477 (0.13596) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.09710 (26.81215) + | > current_lr: 0.00007 + | > step_time: 1.89760 (2.77000) + | > loader_time: 0.08630 (0.05627) + + + --> STEP: 196/234 -- GLOBAL_STEP: 65950 + | > loss: -0.35611 (-0.31707) + | > log_mle: -0.57607 (-0.45503) + | > loss_dur: 0.21996 (0.13796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.05782 (27.81521) + | > current_lr: 0.00007 + | > step_time: 5.19450 (2.80876) + | > loader_time: 0.20860 (0.05787) + + + --> STEP: 201/234 -- GLOBAL_STEP: 65955 + | > loss: -0.31280 (-0.31899) + | > log_mle: -0.53602 (-0.45895) + | > loss_dur: 0.22323 (0.13996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.21943 (28.80763) + | > current_lr: 0.00007 + | > step_time: 6.39610 (2.83604) + | > loader_time: 0.10750 (0.05838) + + + --> STEP: 206/234 -- GLOBAL_STEP: 65960 + | > loss: -0.43796 (-0.32172) + | > log_mle: -0.65929 (-0.46362) + | > loss_dur: 0.22134 (0.14190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.12349 (29.49686) + | > current_lr: 0.00007 + | > step_time: 2.80960 (2.89143) + | > loader_time: 0.00470 (0.05992) + + + --> STEP: 211/234 -- GLOBAL_STEP: 65965 + | > loss: -0.46916 (-0.32471) + | > log_mle: -0.71561 (-0.46875) + | > loss_dur: 0.24645 (0.14404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.51035 (30.75289) + | > current_lr: 0.00007 + | > step_time: 8.21090 (2.99031) + | > loader_time: 0.08530 (0.06129) + + + --> STEP: 216/234 -- GLOBAL_STEP: 65970 + | > loss: -0.46669 (-0.32742) + | > log_mle: -0.71194 (-0.47350) + | > loss_dur: 0.24525 (0.14607) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.67499 (31.79601) + | > current_lr: 0.00007 + | > step_time: 3.50590 (2.99437) + | > loader_time: 0.08320 (0.06338) + + + --> STEP: 221/234 -- GLOBAL_STEP: 65975 + | > loss: -0.42347 (-0.33045) + | > log_mle: -0.63165 (-0.47847) + | > loss_dur: 0.20818 (0.14802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.80240 (32.64280) + | > current_lr: 0.00007 + | > step_time: 3.89530 (3.05370) + | > loader_time: 0.09970 (0.06334) + + + --> STEP: 226/234 -- GLOBAL_STEP: 65980 + | > loss: -0.47856 (-0.33397) + | > log_mle: -0.73140 (-0.48413) + | > loss_dur: 0.25284 (0.15016) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.51398 (33.67992) + | > current_lr: 0.00007 + | > step_time: 0.24080 (3.00290) + | > loader_time: 0.00840 (0.06273) + + + --> STEP: 231/234 -- GLOBAL_STEP: 65985 + | > loss: -0.37860 (-0.33581) + | > log_mle: -0.76101 (-0.48931) + | > loss_dur: 0.38241 (0.15350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.58611 (35.02718) + | > current_lr: 0.00007 + | > step_time: 0.27840 (2.94335) + | > loader_time: 0.00800 (0.06149) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04851 (-0.12896) + | > avg_loss: -0.31361 (+0.02776) + | > avg_log_mle: -0.54106 (+0.01803) + | > avg_loss_dur: 0.22745 (+0.00973) + + + > EPOCH: 282/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 20:37:53)  + + --> STEP: 2/234 -- GLOBAL_STEP: 65990 + | > loss: -0.33209 (-0.32144) + | > log_mle: -0.42232 (-0.41201) + | > loss_dur: 0.09023 (0.09057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.28317 (17.26442) + | > current_lr: 0.00007 + | > step_time: 4.00910 (5.75486) + | > loader_time: 0.09980 (7.74385) + + + --> STEP: 7/234 -- GLOBAL_STEP: 65995 + | > loss: -0.33221 (-0.31335) + | > log_mle: -0.40726 (-0.40744) + | > loss_dur: 0.07505 (0.09409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.40822 (18.34173) + | > current_lr: 0.00007 + | > step_time: 2.01370 (4.04220) + | > loader_time: 0.10320 (2.22795) + + + --> STEP: 12/234 -- GLOBAL_STEP: 66000 + | > loss: -0.33367 (-0.32300) + | > log_mle: -0.40929 (-0.41172) + | > loss_dur: 0.07561 (0.08873) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.02137 (16.70616) + | > current_lr: 0.00007 + | > step_time: 2.10010 (3.52523) + | > loader_time: 0.00120 (1.31548) + + + --> STEP: 17/234 -- GLOBAL_STEP: 66005 + | > loss: -0.36531 (-0.33423) + | > log_mle: -0.41866 (-0.41519) + | > loss_dur: 0.05335 (0.08096) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.10342 (15.42527) + | > current_lr: 0.00007 + | > step_time: 4.61130 (3.26995) + | > loader_time: 0.29820 (0.95152) + + + --> STEP: 22/234 -- GLOBAL_STEP: 66010 + | > loss: -0.32294 (-0.33556) + | > log_mle: -0.40600 (-0.41431) + | > loss_dur: 0.08306 (0.07875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.91933 (14.65853) + | > current_lr: 0.00007 + | > step_time: 2.79790 (3.40447) + | > loader_time: 0.00160 (0.74454) + + + --> STEP: 27/234 -- GLOBAL_STEP: 66015 + | > loss: -0.33282 (-0.33603) + | > log_mle: -0.40199 (-0.41314) + | > loss_dur: 0.06917 (0.07711) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.46618 (13.84670) + | > current_lr: 0.00007 + | > step_time: 9.40720 (3.80432) + | > loader_time: 0.09380 (0.62086) + + + --> STEP: 32/234 -- GLOBAL_STEP: 66020 + | > loss: -0.32569 (-0.33629) + | > log_mle: -0.40141 (-0.41262) + | > loss_dur: 0.07572 (0.07634) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.42585 (13.20362) + | > current_lr: 0.00007 + | > step_time: 6.30520 (3.91066) + | > loader_time: 0.00290 (0.53299) + + + --> STEP: 37/234 -- GLOBAL_STEP: 66025 + | > loss: -0.31605 (-0.33342) + | > log_mle: -0.38639 (-0.41004) + | > loss_dur: 0.07034 (0.07662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.53719 (12.91710) + | > current_lr: 0.00007 + | > step_time: 2.20710 (3.66294) + | > loader_time: 0.08160 (0.46563) + + + --> STEP: 42/234 -- GLOBAL_STEP: 66030 + | > loss: -0.29523 (-0.33043) + | > log_mle: -0.37506 (-0.40767) + | > loss_dur: 0.07984 (0.07723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.94907 (12.83304) + | > current_lr: 0.00007 + | > step_time: 2.34590 (3.45839) + | > loader_time: 0.28710 (0.41736) + + + --> STEP: 47/234 -- GLOBAL_STEP: 66035 + | > loss: -0.30804 (-0.32713) + | > log_mle: -0.39436 (-0.40564) + | > loss_dur: 0.08632 (0.07852) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.69266 (12.90886) + | > current_lr: 0.00007 + | > step_time: 1.18980 (3.27395) + | > loader_time: 0.00340 (0.37737) + + + --> STEP: 52/234 -- GLOBAL_STEP: 66040 + | > loss: -0.27847 (-0.32550) + | > log_mle: -0.38170 (-0.40409) + | > loss_dur: 0.10323 (0.07859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.31623 (12.46481) + | > current_lr: 0.00007 + | > step_time: 2.50900 (3.18365) + | > loader_time: 0.00210 (0.34321) + + + --> STEP: 57/234 -- GLOBAL_STEP: 66045 + | > loss: -0.28480 (-0.32348) + | > log_mle: -0.37251 (-0.40262) + | > loss_dur: 0.08772 (0.07913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.27435 (12.17698) + | > current_lr: 0.00007 + | > step_time: 1.09470 (3.02437) + | > loader_time: 0.00170 (0.31330) + + + --> STEP: 62/234 -- GLOBAL_STEP: 66050 + | > loss: -0.24260 (-0.32032) + | > log_mle: -0.39606 (-0.40156) + | > loss_dur: 0.15346 (0.08125) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.45548 (12.26586) + | > current_lr: 0.00007 + | > step_time: 5.99780 (2.99017) + | > loader_time: 0.09700 (0.28979) + + + --> STEP: 67/234 -- GLOBAL_STEP: 66055 + | > loss: -0.28750 (-0.31800) + | > log_mle: -0.38690 (-0.39981) + | > loss_dur: 0.09941 (0.08181) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.05792 (12.15970) + | > current_lr: 0.00007 + | > step_time: 1.07270 (2.88414) + | > loader_time: 0.00230 (0.26945) + + + --> STEP: 72/234 -- GLOBAL_STEP: 66060 + | > loss: -0.28768 (-0.31481) + | > log_mle: -0.38259 (-0.39822) + | > loss_dur: 0.09490 (0.08341) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.71640 (12.12420) + | > current_lr: 0.00007 + | > step_time: 2.30260 (2.85754) + | > loader_time: 0.00230 (0.25236) + + + --> STEP: 77/234 -- GLOBAL_STEP: 66065 + | > loss: -0.27405 (-0.31158) + | > log_mle: -0.37939 (-0.39682) + | > loss_dur: 0.10535 (0.08524) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.08432 (12.19603) + | > current_lr: 0.00007 + | > step_time: 2.80520 (2.80474) + | > loader_time: 0.08470 (0.23719) + + + --> STEP: 82/234 -- GLOBAL_STEP: 66070 + | > loss: -0.26403 (-0.30935) + | > log_mle: -0.37085 (-0.39547) + | > loss_dur: 0.10681 (0.08612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.19695 (12.11754) + | > current_lr: 0.00007 + | > step_time: 1.69420 (2.76683) + | > loader_time: 0.00380 (0.22389) + + + --> STEP: 87/234 -- GLOBAL_STEP: 66075 + | > loss: -0.25618 (-0.30688) + | > log_mle: -0.37125 (-0.39440) + | > loss_dur: 0.11507 (0.08752) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.78123 (12.18794) + | > current_lr: 0.00007 + | > step_time: 1.11440 (2.69665) + | > loader_time: 0.08460 (0.21310) + + + --> STEP: 92/234 -- GLOBAL_STEP: 66080 + | > loss: -0.28320 (-0.30500) + | > log_mle: -0.41187 (-0.39468) + | > loss_dur: 0.12868 (0.08968) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.86719 (12.40334) + | > current_lr: 0.00007 + | > step_time: 1.09170 (2.63243) + | > loader_time: 0.00220 (0.20363) + + + --> STEP: 97/234 -- GLOBAL_STEP: 66085 + | > loss: -0.26152 (-0.30364) + | > log_mle: -0.39143 (-0.39575) + | > loss_dur: 0.12991 (0.09212) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.23058 (12.78350) + | > current_lr: 0.00007 + | > step_time: 2.02290 (2.59591) + | > loader_time: 0.00370 (0.19331) + + + --> STEP: 102/234 -- GLOBAL_STEP: 66090 + | > loss: -0.23841 (-0.30180) + | > log_mle: -0.37875 (-0.39606) + | > loss_dur: 0.14034 (0.09425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.57681 (13.22105) + | > current_lr: 0.00007 + | > step_time: 3.21970 (2.58526) + | > loader_time: 0.00700 (0.18668) + + + --> STEP: 107/234 -- GLOBAL_STEP: 66095 + | > loss: -0.26836 (-0.30041) + | > log_mle: -0.41624 (-0.39732) + | > loss_dur: 0.14787 (0.09691) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.38744 (13.75199) + | > current_lr: 0.00007 + | > step_time: 5.08790 (2.57909) + | > loader_time: 0.10190 (0.17901) + + + --> STEP: 112/234 -- GLOBAL_STEP: 66100 + | > loss: -0.26357 (-0.29873) + | > log_mle: -0.42998 (-0.39857) + | > loss_dur: 0.16640 (0.09984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.47094 (14.45124) + | > current_lr: 0.00007 + | > step_time: 1.69810 (2.53085) + | > loader_time: 0.09870 (0.17423) + + + --> STEP: 117/234 -- GLOBAL_STEP: 66105 + | > loss: -0.26714 (-0.29711) + | > log_mle: -0.43074 (-0.39980) + | > loss_dur: 0.16360 (0.10268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.92880 (14.83131) + | > current_lr: 0.00007 + | > step_time: 1.48110 (2.50355) + | > loader_time: 0.00320 (0.16771) + + + --> STEP: 122/234 -- GLOBAL_STEP: 66110 + | > loss: -0.24551 (-0.29573) + | > log_mle: -0.39769 (-0.40025) + | > loss_dur: 0.15218 (0.10453) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.93291 (15.01897) + | > current_lr: 0.00007 + | > step_time: 2.56060 (2.48947) + | > loader_time: 0.00290 (0.16161) + + + --> STEP: 127/234 -- GLOBAL_STEP: 66115 + | > loss: -0.27874 (-0.29504) + | > log_mle: -0.45538 (-0.40200) + | > loss_dur: 0.17663 (0.10696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.45012 (15.64507) + | > current_lr: 0.00007 + | > step_time: 2.90850 (2.47570) + | > loader_time: 0.07530 (0.15725) + + + --> STEP: 132/234 -- GLOBAL_STEP: 66120 + | > loss: -0.28866 (-0.29509) + | > log_mle: -0.44048 (-0.40423) + | > loss_dur: 0.15182 (0.10913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.27864 (16.18564) + | > current_lr: 0.00007 + | > step_time: 1.22450 (2.47380) + | > loader_time: 0.08290 (0.15264) + + + --> STEP: 137/234 -- GLOBAL_STEP: 66125 + | > loss: -0.26484 (-0.29531) + | > log_mle: -0.45022 (-0.40692) + | > loss_dur: 0.18538 (0.11162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.63240 (17.03297) + | > current_lr: 0.00007 + | > step_time: 2.30200 (2.45310) + | > loader_time: 0.00460 (0.14718) + + + --> STEP: 142/234 -- GLOBAL_STEP: 66130 + | > loss: -0.27996 (-0.29504) + | > log_mle: -0.46375 (-0.40898) + | > loss_dur: 0.18379 (0.11394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.00543 (17.73656) + | > current_lr: 0.00007 + | > step_time: 1.81630 (2.44560) + | > loader_time: 0.07690 (0.14388) + + + --> STEP: 147/234 -- GLOBAL_STEP: 66135 + | > loss: -0.28942 (-0.29613) + | > log_mle: -0.47114 (-0.41292) + | > loss_dur: 0.18171 (0.11678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.09266 (18.36699) + | > current_lr: 0.00007 + | > step_time: 3.70010 (2.49977) + | > loader_time: 0.00200 (0.14116) + + + --> STEP: 152/234 -- GLOBAL_STEP: 66140 + | > loss: -0.34975 (-0.29750) + | > log_mle: -0.54958 (-0.41656) + | > loss_dur: 0.19983 (0.11906) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.13445 (19.29260) + | > current_lr: 0.00007 + | > step_time: 8.20650 (2.60061) + | > loader_time: 0.19020 (0.14082) + + + --> STEP: 157/234 -- GLOBAL_STEP: 66145 + | > loss: -0.31639 (-0.29961) + | > log_mle: -0.50273 (-0.42095) + | > loss_dur: 0.18635 (0.12135) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.25240 (20.20261) + | > current_lr: 0.00007 + | > step_time: 0.99080 (2.65987) + | > loader_time: 0.00380 (0.13701) + + + --> STEP: 162/234 -- GLOBAL_STEP: 66150 + | > loss: -0.34276 (-0.30131) + | > log_mle: -0.52887 (-0.42496) + | > loss_dur: 0.18610 (0.12365) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.64601 (21.33026) + | > current_lr: 0.00007 + | > step_time: 1.91230 (2.64696) + | > loader_time: 0.00330 (0.13340) + + + --> STEP: 167/234 -- GLOBAL_STEP: 66155 + | > loss: -0.43122 (-0.30300) + | > log_mle: -0.63254 (-0.42876) + | > loss_dur: 0.20132 (0.12576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.68712 (22.19750) + | > current_lr: 0.00007 + | > step_time: 4.00270 (2.63793) + | > loader_time: 0.08520 (0.13046) + + + --> STEP: 172/234 -- GLOBAL_STEP: 66160 + | > loss: -0.40505 (-0.30533) + | > log_mle: -0.61653 (-0.43360) + | > loss_dur: 0.21148 (0.12827) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.55414 (23.08689) + | > current_lr: 0.00007 + | > step_time: 2.10300 (2.61177) + | > loader_time: 0.00530 (0.12762) + + + --> STEP: 177/234 -- GLOBAL_STEP: 66165 + | > loss: -0.34230 (-0.30731) + | > log_mle: -0.55176 (-0.43794) + | > loss_dur: 0.20945 (0.13064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.17688 (24.49267) + | > current_lr: 0.00007 + | > step_time: 3.30270 (2.65557) + | > loader_time: 0.08870 (0.12635) + + + --> STEP: 182/234 -- GLOBAL_STEP: 66170 + | > loss: -0.38310 (-0.30904) + | > log_mle: -0.61930 (-0.44223) + | > loss_dur: 0.23620 (0.13318) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.66246 (25.48213) + | > current_lr: 0.00007 + | > step_time: 1.89610 (2.67824) + | > loader_time: 0.00730 (0.12406) + + + --> STEP: 187/234 -- GLOBAL_STEP: 66175 + | > loss: -0.40039 (-0.31110) + | > log_mle: -0.61616 (-0.44663) + | > loss_dur: 0.21577 (0.13553) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.92252 (26.37744) + | > current_lr: 0.00007 + | > step_time: 4.71240 (2.68006) + | > loader_time: 0.08280 (0.12333) + + + --> STEP: 192/234 -- GLOBAL_STEP: 66180 + | > loss: -0.43410 (-0.31344) + | > log_mle: -0.63944 (-0.45087) + | > loss_dur: 0.20534 (0.13743) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.46984 (27.20485) + | > current_lr: 0.00007 + | > step_time: 4.80000 (2.72483) + | > loader_time: 0.00430 (0.12230) + + + --> STEP: 197/234 -- GLOBAL_STEP: 66185 + | > loss: -0.41515 (-0.31571) + | > log_mle: -0.60951 (-0.45501) + | > loss_dur: 0.19436 (0.13930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.11499 (27.95911) + | > current_lr: 0.00007 + | > step_time: 1.51900 (2.76191) + | > loader_time: 0.08860 (0.12105) + + + --> STEP: 202/234 -- GLOBAL_STEP: 66190 + | > loss: -0.49495 (-0.31797) + | > log_mle: -0.70890 (-0.45934) + | > loss_dur: 0.21395 (0.14136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.91100 (28.88815) + | > current_lr: 0.00007 + | > step_time: 3.29920 (2.76048) + | > loader_time: 0.09330 (0.11941) + + + --> STEP: 207/234 -- GLOBAL_STEP: 66195 + | > loss: -0.44675 (-0.32006) + | > log_mle: -0.67866 (-0.46349) + | > loss_dur: 0.23191 (0.14342) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.78647 (30.01886) + | > current_lr: 0.00007 + | > step_time: 13.69370 (2.83878) + | > loader_time: 0.11220 (0.11799) + + + --> STEP: 212/234 -- GLOBAL_STEP: 66200 + | > loss: -0.43814 (-0.32289) + | > log_mle: -0.67300 (-0.46843) + | > loss_dur: 0.23486 (0.14554) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.09392 (30.97438) + | > current_lr: 0.00007 + | > step_time: 4.69510 (2.91727) + | > loader_time: 0.00310 (0.11798) + + + --> STEP: 217/234 -- GLOBAL_STEP: 66205 + | > loss: -0.43546 (-0.32553) + | > log_mle: -0.68409 (-0.47328) + | > loss_dur: 0.24863 (0.14775) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 111.74123 (32.10257) + | > current_lr: 0.00007 + | > step_time: 3.50610 (2.96296) + | > loader_time: 0.00400 (0.11757) + + + --> STEP: 222/234 -- GLOBAL_STEP: 66210 + | > loss: -0.44819 (-0.32831) + | > log_mle: -0.71184 (-0.47804) + | > loss_dur: 0.26366 (0.14973) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.19472 (33.14853) + | > current_lr: 0.00007 + | > step_time: 2.00760 (2.98637) + | > loader_time: 0.00340 (0.11546) + + + --> STEP: 227/234 -- GLOBAL_STEP: 66215 + | > loss: -0.43176 (-0.33170) + | > log_mle: -0.68652 (-0.48343) + | > loss_dur: 0.25476 (0.15172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.86734 (34.07553) + | > current_lr: 0.00007 + | > step_time: 0.27770 (2.92852) + | > loader_time: 0.00300 (0.11299) + + + --> STEP: 232/234 -- GLOBAL_STEP: 66220 + | > loss: -0.40100 (-0.33433) + | > log_mle: -0.89602 (-0.49023) + | > loss_dur: 0.49502 (0.15589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 129.03651 (35.39964) + | > current_lr: 0.00007 + | > step_time: 0.33420 (2.87154) + | > loader_time: 0.00400 (0.11065) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04740 (-0.00111) + | > avg_loss: -0.33866 (-0.02505) + | > avg_log_mle: -0.55968 (-0.01861) + | > avg_loss_dur: 0.22101 (-0.00643) + + + > EPOCH: 283/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 20:50:25)  + + --> STEP: 3/234 -- GLOBAL_STEP: 66225 + | > loss: -0.28312 (-0.30673) + | > log_mle: -0.40271 (-0.41017) + | > loss_dur: 0.11959 (0.10344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.78679 (21.80210) + | > current_lr: 0.00007 + | > step_time: 4.20200 (5.86874) + | > loader_time: 0.09320 (0.09472) + + + --> STEP: 8/234 -- GLOBAL_STEP: 66230 + | > loss: -0.34633 (-0.32449) + | > log_mle: -0.43097 (-0.41311) + | > loss_dur: 0.08464 (0.08862) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.75095 (18.51463) + | > current_lr: 0.00007 + | > step_time: 2.49740 (4.39070) + | > loader_time: 0.08980 (0.05847) + + + --> STEP: 13/234 -- GLOBAL_STEP: 66235 + | > loss: -0.36460 (-0.33320) + | > log_mle: -0.43704 (-0.41727) + | > loss_dur: 0.07244 (0.08407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.01820 (16.69723) + | > current_lr: 0.00007 + | > step_time: 6.59590 (4.10035) + | > loader_time: 0.00440 (0.03718) + + + --> STEP: 18/234 -- GLOBAL_STEP: 66240 + | > loss: -0.34882 (-0.34026) + | > log_mle: -0.41230 (-0.41864) + | > loss_dur: 0.06348 (0.07838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.52442 (15.27856) + | > current_lr: 0.00007 + | > step_time: 5.10090 (4.52861) + | > loader_time: 0.00190 (0.04957) + + + --> STEP: 23/234 -- GLOBAL_STEP: 66245 + | > loss: -0.36749 (-0.34287) + | > log_mle: -0.43399 (-0.41905) + | > loss_dur: 0.06650 (0.07618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.75344 (14.33020) + | > current_lr: 0.00007 + | > step_time: 1.69640 (4.20587) + | > loader_time: 0.00170 (0.04733) + + + --> STEP: 28/234 -- GLOBAL_STEP: 66250 + | > loss: -0.39291 (-0.34430) + | > log_mle: -0.44266 (-0.41834) + | > loss_dur: 0.04975 (0.07403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.45851 (13.71801) + | > current_lr: 0.00007 + | > step_time: 1.59160 (4.10375) + | > loader_time: 0.00340 (0.04358) + + + --> STEP: 33/234 -- GLOBAL_STEP: 66255 + | > loss: -0.34169 (-0.34289) + | > log_mle: -0.40459 (-0.41640) + | > loss_dur: 0.06290 (0.07350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.11046 (13.56591) + | > current_lr: 0.00007 + | > step_time: 1.06170 (3.79983) + | > loader_time: 0.00200 (0.04022) + + + --> STEP: 38/234 -- GLOBAL_STEP: 66260 + | > loss: -0.30961 (-0.33839) + | > log_mle: -0.39705 (-0.41301) + | > loss_dur: 0.08744 (0.07463) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47331 (13.45324) + | > current_lr: 0.00007 + | > step_time: 1.44240 (3.51389) + | > loader_time: 0.00380 (0.03745) + + + --> STEP: 43/234 -- GLOBAL_STEP: 66265 + | > loss: -0.28824 (-0.33434) + | > log_mle: -0.38232 (-0.40998) + | > loss_dur: 0.09408 (0.07565) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63974 (13.30790) + | > current_lr: 0.00007 + | > step_time: 1.39550 (3.31763) + | > loader_time: 0.00210 (0.03545) + + + --> STEP: 48/234 -- GLOBAL_STEP: 66270 + | > loss: -0.33607 (-0.33114) + | > log_mle: -0.39919 (-0.40769) + | > loss_dur: 0.06313 (0.07655) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.64841 (13.20225) + | > current_lr: 0.00007 + | > step_time: 1.50390 (3.13779) + | > loader_time: 0.00180 (0.03202) + + + --> STEP: 53/234 -- GLOBAL_STEP: 66275 + | > loss: -0.29710 (-0.32842) + | > log_mle: -0.39290 (-0.40573) + | > loss_dur: 0.09580 (0.07731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.58565 (12.81479) + | > current_lr: 0.00007 + | > step_time: 3.31080 (3.03599) + | > loader_time: 0.00440 (0.03106) + + + --> STEP: 58/234 -- GLOBAL_STEP: 66280 + | > loss: -0.32396 (-0.32653) + | > log_mle: -0.39111 (-0.40407) + | > loss_dur: 0.06716 (0.07754) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.27264 (12.46318) + | > current_lr: 0.00007 + | > step_time: 1.70390 (2.91216) + | > loader_time: 0.00200 (0.03005) + + + --> STEP: 63/234 -- GLOBAL_STEP: 66285 + | > loss: -0.27897 (-0.32289) + | > log_mle: -0.37591 (-0.40267) + | > loss_dur: 0.09694 (0.07979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.17515 (12.39238) + | > current_lr: 0.00007 + | > step_time: 1.69440 (2.85275) + | > loader_time: 0.00480 (0.03216) + + + --> STEP: 68/234 -- GLOBAL_STEP: 66290 + | > loss: -0.26481 (-0.32039) + | > log_mle: -0.37053 (-0.40089) + | > loss_dur: 0.10571 (0.08050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.84929 (12.18768) + | > current_lr: 0.00007 + | > step_time: 1.18340 (2.78424) + | > loader_time: 0.00230 (0.03249) + + + --> STEP: 73/234 -- GLOBAL_STEP: 66295 + | > loss: -0.25737 (-0.31713) + | > log_mle: -0.37808 (-0.39931) + | > loss_dur: 0.12071 (0.08218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.35854 (12.27258) + | > current_lr: 0.00007 + | > step_time: 1.99330 (2.74124) + | > loader_time: 0.09270 (0.03395) + + + --> STEP: 78/234 -- GLOBAL_STEP: 66300 + | > loss: -0.27187 (-0.31421) + | > log_mle: -0.36941 (-0.39781) + | > loss_dur: 0.09754 (0.08360) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.32029 (12.22746) + | > current_lr: 0.00007 + | > step_time: 3.30520 (2.69540) + | > loader_time: 0.00270 (0.03299) + + + --> STEP: 83/234 -- GLOBAL_STEP: 66305 + | > loss: -0.24283 (-0.31150) + | > log_mle: -0.37963 (-0.39661) + | > loss_dur: 0.13680 (0.08512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.22235 (12.19945) + | > current_lr: 0.00007 + | > step_time: 3.19860 (2.67997) + | > loader_time: 0.00150 (0.03307) + + + --> STEP: 88/234 -- GLOBAL_STEP: 66310 + | > loss: -0.26729 (-0.30913) + | > log_mle: -0.41016 (-0.39586) + | > loss_dur: 0.14287 (0.08673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.54128 (12.36656) + | > current_lr: 0.00007 + | > step_time: 0.99670 (2.65272) + | > loader_time: 0.00240 (0.03344) + + + --> STEP: 93/234 -- GLOBAL_STEP: 66315 + | > loss: -0.27366 (-0.30714) + | > log_mle: -0.42121 (-0.39624) + | > loss_dur: 0.14755 (0.08910) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.27241 (12.70553) + | > current_lr: 0.00007 + | > step_time: 1.40220 (2.62858) + | > loader_time: 0.00240 (0.03184) + + + --> STEP: 98/234 -- GLOBAL_STEP: 66320 + | > loss: -0.26129 (-0.30580) + | > log_mle: -0.36285 (-0.39675) + | > loss_dur: 0.10157 (0.09095) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.73156 (12.95583) + | > current_lr: 0.00007 + | > step_time: 2.00500 (2.59101) + | > loader_time: 0.07600 (0.03474) + + + --> STEP: 103/234 -- GLOBAL_STEP: 66325 + | > loss: -0.28600 (-0.30394) + | > log_mle: -0.44852 (-0.39782) + | > loss_dur: 0.16252 (0.09388) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.51394 (13.45662) + | > current_lr: 0.00007 + | > step_time: 1.49950 (2.60332) + | > loader_time: 0.00220 (0.03590) + + + --> STEP: 108/234 -- GLOBAL_STEP: 66330 + | > loss: -0.26661 (-0.30243) + | > log_mle: -0.39631 (-0.39853) + | > loss_dur: 0.12970 (0.09611) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.86735 (13.88231) + | > current_lr: 0.00007 + | > step_time: 1.60170 (2.59336) + | > loader_time: 0.00270 (0.03695) + + + --> STEP: 113/234 -- GLOBAL_STEP: 66335 + | > loss: -0.27779 (-0.30083) + | > log_mle: -0.43467 (-0.40004) + | > loss_dur: 0.15688 (0.09921) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.51910 (14.61172) + | > current_lr: 0.00007 + | > step_time: 1.39410 (2.56657) + | > loader_time: 0.00320 (0.03787) + + + --> STEP: 118/234 -- GLOBAL_STEP: 66340 + | > loss: -0.25909 (-0.29927) + | > log_mle: -0.40954 (-0.40096) + | > loss_dur: 0.15045 (0.10170) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.41610 (14.99378) + | > current_lr: 0.00007 + | > step_time: 1.71120 (2.56549) + | > loader_time: 0.08640 (0.03870) + + + --> STEP: 123/234 -- GLOBAL_STEP: 66345 + | > loss: -0.23370 (-0.29760) + | > log_mle: -0.37839 (-0.40106) + | > loss_dur: 0.14470 (0.10346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.44186 (15.16670) + | > current_lr: 0.00007 + | > step_time: 1.99770 (2.55113) + | > loader_time: 0.09500 (0.03859) + + + --> STEP: 128/234 -- GLOBAL_STEP: 66350 + | > loss: -0.29286 (-0.29726) + | > log_mle: -0.43564 (-0.40310) + | > loss_dur: 0.14278 (0.10583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.58727 (15.99094) + | > current_lr: 0.00007 + | > step_time: 2.69890 (2.53563) + | > loader_time: 0.08450 (0.03847) + + + --> STEP: 133/234 -- GLOBAL_STEP: 66355 + | > loss: -0.29683 (-0.29692) + | > log_mle: -0.46349 (-0.40526) + | > loss_dur: 0.16666 (0.10834) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.67442 (17.31892) + | > current_lr: 0.00007 + | > step_time: 2.10450 (2.54309) + | > loader_time: 0.00210 (0.03842) + + + --> STEP: 138/234 -- GLOBAL_STEP: 66360 + | > loss: -0.24350 (-0.29651) + | > log_mle: -0.40992 (-0.40734) + | > loss_dur: 0.16642 (0.11083) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.81707 (17.82655) + | > current_lr: 0.00007 + | > step_time: 2.89300 (2.52845) + | > loader_time: 0.00360 (0.03907) + + + --> STEP: 143/234 -- GLOBAL_STEP: 66365 + | > loss: -0.31727 (-0.29676) + | > log_mle: -0.54557 (-0.41025) + | > loss_dur: 0.22830 (0.11349) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.21306 (18.67030) + | > current_lr: 0.00007 + | > step_time: 4.71650 (2.54519) + | > loader_time: 0.08620 (0.03914) + + + --> STEP: 148/234 -- GLOBAL_STEP: 66370 + | > loss: -0.30819 (-0.29734) + | > log_mle: -0.46598 (-0.41308) + | > loss_dur: 0.15779 (0.11574) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.71342 (19.42789) + | > current_lr: 0.00007 + | > step_time: 3.83440 (2.54654) + | > loader_time: 0.08650 (0.03905) + + + --> STEP: 153/234 -- GLOBAL_STEP: 66375 + | > loss: -0.40845 (-0.29931) + | > log_mle: -0.60353 (-0.41748) + | > loss_dur: 0.19508 (0.11818) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.78638 (20.23530) + | > current_lr: 0.00007 + | > step_time: 1.70430 (2.52832) + | > loader_time: 0.10020 (0.03908) + + + --> STEP: 158/234 -- GLOBAL_STEP: 66380 + | > loss: -0.32531 (-0.30068) + | > log_mle: -0.53726 (-0.42130) + | > loss_dur: 0.21196 (0.12062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.63364 (21.26982) + | > current_lr: 0.00007 + | > step_time: 0.91230 (2.50035) + | > loader_time: 0.00300 (0.03902) + + + --> STEP: 163/234 -- GLOBAL_STEP: 66385 + | > loss: -0.31044 (-0.30209) + | > log_mle: -0.50123 (-0.42492) + | > loss_dur: 0.19078 (0.12283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.39215 (22.47444) + | > current_lr: 0.00007 + | > step_time: 10.70220 (2.61079) + | > loader_time: 0.10020 (0.04014) + + + --> STEP: 168/234 -- GLOBAL_STEP: 66390 + | > loss: -0.35150 (-0.30375) + | > log_mle: -0.56296 (-0.42889) + | > loss_dur: 0.21147 (0.12514) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.14561 (23.37265) + | > current_lr: 0.00007 + | > step_time: 0.99520 (2.61519) + | > loader_time: 0.00280 (0.03956) + + + --> STEP: 173/234 -- GLOBAL_STEP: 66395 + | > loss: -0.33714 (-0.30578) + | > log_mle: -0.54940 (-0.43337) + | > loss_dur: 0.21226 (0.12759) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.32491 (25.06611) + | > current_lr: 0.00007 + | > step_time: 4.81160 (2.66059) + | > loader_time: 0.00890 (0.04050) + + + --> STEP: 178/234 -- GLOBAL_STEP: 66400 + | > loss: -0.38113 (-0.30738) + | > log_mle: -0.60528 (-0.43752) + | > loss_dur: 0.22415 (0.13014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 131.82629 (26.74970) + | > current_lr: 0.00007 + | > step_time: 7.10850 (2.66867) + | > loader_time: 0.00470 (0.04086) + + + --> STEP: 183/234 -- GLOBAL_STEP: 66405 + | > loss: -0.39720 (-0.30834) + | > log_mle: -0.61415 (-0.44100) + | > loss_dur: 0.21694 (0.13265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.44087 (27.40469) + | > current_lr: 0.00007 + | > step_time: 6.91260 (2.76586) + | > loader_time: 0.29490 (0.04347) + + + --> STEP: 188/234 -- GLOBAL_STEP: 66410 + | > loss: -0.40355 (-0.30990) + | > log_mle: -0.62455 (-0.44499) + | > loss_dur: 0.22100 (0.13509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.10085 (28.01377) + | > current_lr: 0.00007 + | > step_time: 2.20580 (2.74030) + | > loader_time: 0.08960 (0.04529) + + + --> STEP: 193/234 -- GLOBAL_STEP: 66415 + | > loss: -0.41593 (-0.31203) + | > log_mle: -0.63119 (-0.44903) + | > loss_dur: 0.21526 (0.13700) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.62824 (28.49327) + | > current_lr: 0.00007 + | > step_time: 1.91030 (2.85344) + | > loader_time: 0.09530 (0.04673) + + + --> STEP: 198/234 -- GLOBAL_STEP: 66420 + | > loss: -0.41047 (-0.31395) + | > log_mle: -0.62896 (-0.45288) + | > loss_dur: 0.21849 (0.13893) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.02568 (29.25227) + | > current_lr: 0.00007 + | > step_time: 2.79480 (2.92728) + | > loader_time: 0.00510 (0.04662) + + + --> STEP: 203/234 -- GLOBAL_STEP: 66425 + | > loss: -0.33859 (-0.31563) + | > log_mle: -0.55274 (-0.45665) + | > loss_dur: 0.21415 (0.14102) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.07371 (30.04436) + | > current_lr: 0.00007 + | > step_time: 3.80080 (2.92759) + | > loader_time: 0.07850 (0.04781) + + + --> STEP: 208/234 -- GLOBAL_STEP: 66430 + | > loss: -0.40991 (-0.31801) + | > log_mle: -0.64210 (-0.46130) + | > loss_dur: 0.23219 (0.14329) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.51440 (30.97867) + | > current_lr: 0.00007 + | > step_time: 1.60220 (2.90258) + | > loader_time: 0.00470 (0.04713) + + + --> STEP: 213/234 -- GLOBAL_STEP: 66435 + | > loss: -0.46715 (-0.32104) + | > log_mle: -0.70247 (-0.46647) + | > loss_dur: 0.23532 (0.14543) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.39480 (31.91541) + | > current_lr: 0.00007 + | > step_time: 3.19300 (2.89916) + | > loader_time: 0.00320 (0.04748) + + + --> STEP: 218/234 -- GLOBAL_STEP: 66440 + | > loss: -0.42250 (-0.32360) + | > log_mle: -0.65123 (-0.47104) + | > loss_dur: 0.22874 (0.14743) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.66408 (33.10254) + | > current_lr: 0.00007 + | > step_time: 2.39380 (2.94687) + | > loader_time: 0.00340 (0.04778) + + + --> STEP: 223/234 -- GLOBAL_STEP: 66445 + | > loss: -0.45691 (-0.32662) + | > log_mle: -0.68928 (-0.47601) + | > loss_dur: 0.23237 (0.14939) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.27067 (34.05028) + | > current_lr: 0.00007 + | > step_time: 1.50040 (2.91578) + | > loader_time: 0.00390 (0.04679) + + + --> STEP: 228/234 -- GLOBAL_STEP: 66450 + | > loss: -0.43737 (-0.32973) + | > log_mle: -0.70566 (-0.48129) + | > loss_dur: 0.26829 (0.15156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.93306 (35.13705) + | > current_lr: 0.00007 + | > step_time: 0.34210 (2.88357) + | > loader_time: 0.00300 (0.04658) + + + --> STEP: 233/234 -- GLOBAL_STEP: 66455 + | > loss: 0.03824 (-0.33033) + | > log_mle: -0.66485 (-0.48792) + | > loss_dur: 0.70309 (0.15759) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 84.59805 (36.86421) + | > current_lr: 0.00007 + | > step_time: 0.21220 (2.82775) + | > loader_time: 0.00270 (0.04568) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27429 (+0.22689) + | > avg_loss: -0.35184 (-0.01318) + | > avg_log_mle: -0.57069 (-0.01102) + | > avg_loss_dur: 0.21885 (-0.00216) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_66456.pth + + > EPOCH: 284/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 21:02:43)  + + --> STEP: 4/234 -- GLOBAL_STEP: 66460 + | > loss: -0.31864 (-0.30493) + | > log_mle: -0.40889 (-0.41020) + | > loss_dur: 0.09025 (0.10527) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.10295 (20.73698) + | > current_lr: 0.00007 + | > step_time: 1.10970 (6.04926) + | > loader_time: 0.08630 (0.17782) + + + --> STEP: 9/234 -- GLOBAL_STEP: 66465 + | > loss: -0.32022 (-0.32052) + | > log_mle: -0.42265 (-0.41465) + | > loss_dur: 0.10243 (0.09413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.33178 (20.43499) + | > current_lr: 0.00007 + | > step_time: 2.09900 (4.14099) + | > loader_time: 0.00270 (0.10216) + + + --> STEP: 14/234 -- GLOBAL_STEP: 66470 + | > loss: -0.35100 (-0.33179) + | > log_mle: -0.42281 (-0.41759) + | > loss_dur: 0.07181 (0.08580) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.96020 (18.58994) + | > current_lr: 0.00007 + | > step_time: 1.01490 (3.72086) + | > loader_time: 0.07800 (0.07189) + + + --> STEP: 19/234 -- GLOBAL_STEP: 66475 + | > loss: -0.35631 (-0.33838) + | > log_mle: -0.42597 (-0.41911) + | > loss_dur: 0.06966 (0.08073) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.16399 (16.92470) + | > current_lr: 0.00007 + | > step_time: 1.10820 (2.99452) + | > loader_time: 0.00130 (0.05339) + + + --> STEP: 24/234 -- GLOBAL_STEP: 66480 + | > loss: -0.34879 (-0.34116) + | > log_mle: -0.41570 (-0.41925) + | > loss_dur: 0.06691 (0.07810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.20773 (15.50566) + | > current_lr: 0.00007 + | > step_time: 2.09720 (2.81243) + | > loader_time: 0.00240 (0.04633) + + + --> STEP: 29/234 -- GLOBAL_STEP: 66485 + | > loss: -0.34368 (-0.34254) + | > log_mle: -0.41083 (-0.41832) + | > loss_dur: 0.06715 (0.07578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.32133 (14.65263) + | > current_lr: 0.00007 + | > step_time: 1.69150 (2.94167) + | > loader_time: 0.00220 (0.04506) + + + --> STEP: 34/234 -- GLOBAL_STEP: 66490 + | > loss: -0.33052 (-0.33958) + | > log_mle: -0.40140 (-0.41561) + | > loss_dur: 0.07089 (0.07603) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.98221 (14.19872) + | > current_lr: 0.00007 + | > step_time: 0.99360 (3.27424) + | > loader_time: 0.00230 (0.04717) + + + --> STEP: 39/234 -- GLOBAL_STEP: 66495 + | > loss: -0.30680 (-0.33527) + | > log_mle: -0.39169 (-0.41232) + | > loss_dur: 0.08489 (0.07705) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.29682 (14.13621) + | > current_lr: 0.00007 + | > step_time: 2.89080 (3.13106) + | > loader_time: 0.00950 (0.04827) + + + --> STEP: 44/234 -- GLOBAL_STEP: 66500 + | > loss: -0.30832 (-0.33230) + | > log_mle: -0.37849 (-0.40946) + | > loss_dur: 0.07017 (0.07716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.62955 (13.69847) + | > current_lr: 0.00007 + | > step_time: 2.21760 (3.07246) + | > loader_time: 0.00130 (0.04736) + + + --> STEP: 49/234 -- GLOBAL_STEP: 66505 + | > loss: -0.32572 (-0.33025) + | > log_mle: -0.40133 (-0.40799) + | > loss_dur: 0.07560 (0.07774) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.01230 (13.40139) + | > current_lr: 0.00007 + | > step_time: 1.78870 (2.95136) + | > loader_time: 0.00480 (0.04456) + + + --> STEP: 54/234 -- GLOBAL_STEP: 66510 + | > loss: -0.31358 (-0.32768) + | > log_mle: -0.38767 (-0.40592) + | > loss_dur: 0.07409 (0.07825) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.46912 (13.19127) + | > current_lr: 0.00007 + | > step_time: 1.70200 (2.88986) + | > loader_time: 0.08340 (0.04518) + + + --> STEP: 59/234 -- GLOBAL_STEP: 66515 + | > loss: -0.29377 (-0.32566) + | > log_mle: -0.39003 (-0.40442) + | > loss_dur: 0.09626 (0.07877) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.57655 (12.93606) + | > current_lr: 0.00007 + | > step_time: 1.28220 (2.80023) + | > loader_time: 0.00170 (0.04151) + + + --> STEP: 64/234 -- GLOBAL_STEP: 66520 + | > loss: -0.28633 (-0.32158) + | > log_mle: -0.37813 (-0.40274) + | > loss_dur: 0.09180 (0.08116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.16979 (13.00403) + | > current_lr: 0.00007 + | > step_time: 2.10240 (2.75469) + | > loader_time: 0.00230 (0.03845) + + + --> STEP: 69/234 -- GLOBAL_STEP: 66525 + | > loss: -0.28821 (-0.31866) + | > log_mle: -0.37324 (-0.40071) + | > loss_dur: 0.08503 (0.08204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.74071 (12.89199) + | > current_lr: 0.00007 + | > step_time: 1.49900 (2.72024) + | > loader_time: 0.00560 (0.03597) + + + --> STEP: 74/234 -- GLOBAL_STEP: 66530 + | > loss: -0.25908 (-0.31509) + | > log_mle: -0.36887 (-0.39899) + | > loss_dur: 0.10979 (0.08390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.52217 (13.07365) + | > current_lr: 0.00007 + | > step_time: 2.60870 (2.65384) + | > loader_time: 0.00310 (0.03482) + + + --> STEP: 79/234 -- GLOBAL_STEP: 66535 + | > loss: -0.27568 (-0.31236) + | > log_mle: -0.37852 (-0.39762) + | > loss_dur: 0.10284 (0.08526) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.83798 (12.95066) + | > current_lr: 0.00007 + | > step_time: 1.63740 (2.59784) + | > loader_time: 0.00230 (0.03278) + + + --> STEP: 84/234 -- GLOBAL_STEP: 66540 + | > loss: -0.27989 (-0.30978) + | > log_mle: -0.37936 (-0.39636) + | > loss_dur: 0.09948 (0.08658) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.58783 (12.83508) + | > current_lr: 0.00007 + | > step_time: 4.04210 (2.64008) + | > loader_time: 0.00260 (0.03208) + + + --> STEP: 89/234 -- GLOBAL_STEP: 66545 + | > loss: -0.28141 (-0.30757) + | > log_mle: -0.39579 (-0.39578) + | > loss_dur: 0.11438 (0.08820) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.85946 (13.03540) + | > current_lr: 0.00007 + | > step_time: 3.79650 (2.62589) + | > loader_time: 0.00270 (0.03045) + + + --> STEP: 94/234 -- GLOBAL_STEP: 66550 + | > loss: -0.28706 (-0.30563) + | > log_mle: -0.42144 (-0.39630) + | > loss_dur: 0.13439 (0.09067) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.97752 (13.32782) + | > current_lr: 0.00007 + | > step_time: 1.81350 (2.58502) + | > loader_time: 0.00190 (0.02982) + + + --> STEP: 99/234 -- GLOBAL_STEP: 66555 + | > loss: -0.29263 (-0.30433) + | > log_mle: -0.45299 (-0.39705) + | > loss_dur: 0.16036 (0.09272) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.18646 (13.69995) + | > current_lr: 0.00007 + | > step_time: 2.00960 (2.54519) + | > loader_time: 0.00150 (0.02921) + + + --> STEP: 104/234 -- GLOBAL_STEP: 66560 + | > loss: -0.31423 (-0.30314) + | > log_mle: -0.46664 (-0.39835) + | > loss_dur: 0.15241 (0.09521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.21596 (14.15171) + | > current_lr: 0.00007 + | > step_time: 2.21480 (2.52297) + | > loader_time: 0.08590 (0.02942) + + + --> STEP: 109/234 -- GLOBAL_STEP: 66565 + | > loss: -0.24474 (-0.30104) + | > log_mle: -0.43239 (-0.39880) + | > loss_dur: 0.18765 (0.09775) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.21476 (14.70533) + | > current_lr: 0.00007 + | > step_time: 1.84700 (2.48377) + | > loader_time: 0.19850 (0.03151) + + + --> STEP: 114/234 -- GLOBAL_STEP: 66570 + | > loss: -0.26966 (-0.29949) + | > log_mle: -0.41466 (-0.39997) + | > loss_dur: 0.14499 (0.10047) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.06589 (15.30957) + | > current_lr: 0.00007 + | > step_time: 2.71030 (2.46886) + | > loader_time: 0.00270 (0.03108) + + + --> STEP: 119/234 -- GLOBAL_STEP: 66575 + | > loss: -0.27470 (-0.29794) + | > log_mle: -0.41835 (-0.40094) + | > loss_dur: 0.14365 (0.10300) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.11393 (15.49723) + | > current_lr: 0.00007 + | > step_time: 1.60840 (2.45508) + | > loader_time: 0.08390 (0.03059) + + + --> STEP: 124/234 -- GLOBAL_STEP: 66580 + | > loss: -0.29325 (-0.29649) + | > log_mle: -0.44508 (-0.40122) + | > loss_dur: 0.15183 (0.10474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.78462 (15.78779) + | > current_lr: 0.00007 + | > step_time: 1.81960 (2.45314) + | > loader_time: 0.08440 (0.03077) + + + --> STEP: 129/234 -- GLOBAL_STEP: 66585 + | > loss: -0.26128 (-0.29608) + | > log_mle: -0.43538 (-0.40328) + | > loss_dur: 0.17411 (0.10720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.51813 (16.28004) + | > current_lr: 0.00007 + | > step_time: 1.34730 (2.42182) + | > loader_time: 0.00220 (0.02966) + + + --> STEP: 134/234 -- GLOBAL_STEP: 66590 + | > loss: -0.31479 (-0.29662) + | > log_mle: -0.49414 (-0.40625) + | > loss_dur: 0.17935 (0.10963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.92685 (16.83924) + | > current_lr: 0.00007 + | > step_time: 1.99150 (2.39792) + | > loader_time: 0.10000 (0.03001) + + + --> STEP: 139/234 -- GLOBAL_STEP: 66595 + | > loss: -0.36585 (-0.29699) + | > log_mle: -0.55633 (-0.40899) + | > loss_dur: 0.19048 (0.11200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.34938 (17.46386) + | > current_lr: 0.00007 + | > step_time: 2.00870 (2.39881) + | > loader_time: 0.08090 (0.02960) + + + --> STEP: 144/234 -- GLOBAL_STEP: 66600 + | > loss: -0.33188 (-0.29712) + | > log_mle: -0.52201 (-0.41181) + | > loss_dur: 0.19014 (0.11469) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.87127 (18.42407) + | > current_lr: 0.00007 + | > step_time: 1.91350 (2.39691) + | > loader_time: 0.08330 (0.03038) + + + --> STEP: 149/234 -- GLOBAL_STEP: 66605 + | > loss: -0.39787 (-0.29858) + | > log_mle: -0.58353 (-0.41539) + | > loss_dur: 0.18566 (0.11681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.41403 (19.18486) + | > current_lr: 0.00007 + | > step_time: 2.90110 (2.41308) + | > loader_time: 0.08910 (0.03064) + + + --> STEP: 154/234 -- GLOBAL_STEP: 66610 + | > loss: -0.34508 (-0.30034) + | > log_mle: -0.53523 (-0.41963) + | > loss_dur: 0.19015 (0.11929) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.22530 (20.02819) + | > current_lr: 0.00007 + | > step_time: 6.00000 (2.47525) + | > loader_time: 0.00670 (0.03034) + + + --> STEP: 159/234 -- GLOBAL_STEP: 66615 + | > loss: -0.35243 (-0.30194) + | > log_mle: -0.55587 (-0.42367) + | > loss_dur: 0.20344 (0.12173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.47071 (20.76752) + | > current_lr: 0.00007 + | > step_time: 2.10010 (2.45811) + | > loader_time: 0.10090 (0.03087) + + + --> STEP: 164/234 -- GLOBAL_STEP: 66620 + | > loss: -0.34130 (-0.30356) + | > log_mle: -0.54586 (-0.42746) + | > loss_dur: 0.20456 (0.12390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.17124 (21.87919) + | > current_lr: 0.00007 + | > step_time: 8.69280 (2.56180) + | > loader_time: 0.39980 (0.03468) + + + --> STEP: 169/234 -- GLOBAL_STEP: 66625 + | > loss: -0.31987 (-0.30514) + | > log_mle: -0.52324 (-0.43117) + | > loss_dur: 0.20336 (0.12603) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.49980 (23.50401) + | > current_lr: 0.00007 + | > step_time: 2.08320 (2.56451) + | > loader_time: 0.00250 (0.03499) + + + --> STEP: 174/234 -- GLOBAL_STEP: 66630 + | > loss: -0.41016 (-0.30752) + | > log_mle: -0.62149 (-0.43603) + | > loss_dur: 0.21134 (0.12851) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.04131 (24.57790) + | > current_lr: 0.00007 + | > step_time: 7.00660 (2.63906) + | > loader_time: 0.09260 (0.03626) + + + --> STEP: 179/234 -- GLOBAL_STEP: 66635 + | > loss: -0.36841 (-0.30908) + | > log_mle: -0.60899 (-0.44016) + | > loss_dur: 0.24059 (0.13108) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.29237 (25.60805) + | > current_lr: 0.00007 + | > step_time: 5.76090 (2.72204) + | > loader_time: 0.11880 (0.03929) + + + --> STEP: 184/234 -- GLOBAL_STEP: 66640 + | > loss: -0.36372 (-0.31043) + | > log_mle: -0.57977 (-0.44381) + | > loss_dur: 0.21606 (0.13338) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.79930 (26.20345) + | > current_lr: 0.00007 + | > step_time: 1.99370 (2.70938) + | > loader_time: 0.00320 (0.03921) + + + --> STEP: 189/234 -- GLOBAL_STEP: 66645 + | > loss: -0.36697 (-0.31231) + | > log_mle: -0.57738 (-0.44796) + | > loss_dur: 0.21042 (0.13564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.81529 (26.92594) + | > current_lr: 0.00007 + | > step_time: 3.10560 (2.72283) + | > loader_time: 0.08640 (0.04068) + + + --> STEP: 194/234 -- GLOBAL_STEP: 66650 + | > loss: -0.40672 (-0.31474) + | > log_mle: -0.61318 (-0.45221) + | > loss_dur: 0.20646 (0.13747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.92213 (27.81664) + | > current_lr: 0.00007 + | > step_time: 2.60170 (2.78060) + | > loader_time: 0.00430 (0.04073) + + + --> STEP: 199/234 -- GLOBAL_STEP: 66655 + | > loss: -0.40473 (-0.31669) + | > log_mle: -0.62737 (-0.45619) + | > loss_dur: 0.22264 (0.13950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.61114 (28.52544) + | > current_lr: 0.00007 + | > step_time: 2.99460 (2.79961) + | > loader_time: 0.09170 (0.04170) + + + --> STEP: 204/234 -- GLOBAL_STEP: 66660 + | > loss: -0.44339 (-0.31849) + | > log_mle: -0.68177 (-0.46014) + | > loss_dur: 0.23839 (0.14166) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.11415 (29.25347) + | > current_lr: 0.00007 + | > step_time: 4.19290 (2.81974) + | > loader_time: 0.00300 (0.04121) + + + --> STEP: 209/234 -- GLOBAL_STEP: 66665 + | > loss: -0.41505 (-0.32081) + | > log_mle: -0.62467 (-0.46446) + | > loss_dur: 0.20962 (0.14365) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.65702 (30.01506) + | > current_lr: 0.00007 + | > step_time: 6.09680 (2.87302) + | > loader_time: 0.19470 (0.08570) + + + --> STEP: 214/234 -- GLOBAL_STEP: 66670 + | > loss: -0.44794 (-0.32406) + | > log_mle: -0.65531 (-0.46971) + | > loss_dur: 0.20737 (0.14565) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.20374 (30.91054) + | > current_lr: 0.00007 + | > step_time: 1.10920 (2.94930) + | > loader_time: 20.59710 (0.18146) + + + --> STEP: 219/234 -- GLOBAL_STEP: 66675 + | > loss: -0.53917 (-0.32712) + | > log_mle: -0.76890 (-0.47482) + | > loss_dur: 0.22973 (0.14771) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.03350 (31.85976) + | > current_lr: 0.00007 + | > step_time: 2.70090 (2.96776) + | > loader_time: 0.00290 (0.17779) + + + --> STEP: 224/234 -- GLOBAL_STEP: 66680 + | > loss: -0.48429 (-0.33015) + | > log_mle: -0.71894 (-0.47979) + | > loss_dur: 0.23464 (0.14964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.84743 (32.66165) + | > current_lr: 0.00007 + | > step_time: 1.49170 (2.94712) + | > loader_time: 0.00290 (0.18095) + + + --> STEP: 229/234 -- GLOBAL_STEP: 66685 + | > loss: -0.46648 (-0.33334) + | > log_mle: -0.76589 (-0.48544) + | > loss_dur: 0.29941 (0.15210) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 109.24900 (33.82116) + | > current_lr: 0.00007 + | > step_time: 0.24980 (2.88898) + | > loader_time: 0.00450 (0.17745) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67270 (+0.39840) + | > avg_loss: -0.33246 (+0.01938) + | > avg_log_mle: -0.55782 (+0.01287) + | > avg_loss_dur: 0.22536 (+0.00651) + + + > EPOCH: 285/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 21:15:43)  + + --> STEP: 0/234 -- GLOBAL_STEP: 66690 + | > loss: -0.29899 (-0.29899) + | > log_mle: -0.49099 (-0.49099) + | > loss_dur: 0.19200 (0.19200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.05326 (30.05326) + | > current_lr: 0.00007 + | > step_time: 7.51220 (7.51225) + | > loader_time: 13.50450 (13.50447) + + + --> STEP: 5/234 -- GLOBAL_STEP: 66695 + | > loss: -0.31402 (-0.31115) + | > log_mle: -0.41255 (-0.41255) + | > loss_dur: 0.09852 (0.10140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.29670 (18.00776) + | > current_lr: 0.00007 + | > step_time: 4.40160 (6.52318) + | > loader_time: 0.00190 (0.01969) + + + --> STEP: 10/234 -- GLOBAL_STEP: 66700 + | > loss: -0.33534 (-0.32329) + | > log_mle: -0.41582 (-0.41633) + | > loss_dur: 0.08049 (0.09304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.90035 (17.72873) + | > current_lr: 0.00007 + | > step_time: 0.96100 (4.37029) + | > loader_time: 0.00260 (0.02733) + + + --> STEP: 15/234 -- GLOBAL_STEP: 66705 + | > loss: -0.35621 (-0.33381) + | > log_mle: -0.42515 (-0.41917) + | > loss_dur: 0.06894 (0.08535) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45539 (17.24459) + | > current_lr: 0.00007 + | > step_time: 3.70330 (3.78118) + | > loader_time: 0.00120 (0.02926) + + + --> STEP: 20/234 -- GLOBAL_STEP: 66710 + | > loss: -0.36289 (-0.34043) + | > log_mle: -0.42621 (-0.42029) + | > loss_dur: 0.06333 (0.07986) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.04892 (15.94281) + | > current_lr: 0.00007 + | > step_time: 3.60750 (3.48648) + | > loader_time: 0.09280 (0.03588) + + + --> STEP: 25/234 -- GLOBAL_STEP: 66715 + | > loss: -0.34108 (-0.34192) + | > log_mle: -0.40464 (-0.41915) + | > loss_dur: 0.06356 (0.07723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.99545 (15.07055) + | > current_lr: 0.00007 + | > step_time: 4.00660 (3.43231) + | > loader_time: 0.00190 (0.02902) + + + --> STEP: 30/234 -- GLOBAL_STEP: 66720 + | > loss: -0.33708 (-0.34241) + | > log_mle: -0.41434 (-0.41831) + | > loss_dur: 0.07726 (0.07589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.65699 (14.22484) + | > current_lr: 0.00007 + | > step_time: 1.88880 (3.53332) + | > loader_time: 0.00280 (0.03041) + + + --> STEP: 35/234 -- GLOBAL_STEP: 66725 + | > loss: -0.29612 (-0.33820) + | > log_mle: -0.38646 (-0.41491) + | > loss_dur: 0.09034 (0.07671) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.97577 (13.90510) + | > current_lr: 0.00007 + | > step_time: 5.98990 (3.38612) + | > loader_time: 0.00210 (0.03157) + + + --> STEP: 40/234 -- GLOBAL_STEP: 66730 + | > loss: -0.30348 (-0.33371) + | > log_mle: -0.38578 (-0.41130) + | > loss_dur: 0.08230 (0.07758) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.33922 (13.70000) + | > current_lr: 0.00007 + | > step_time: 1.67440 (3.31839) + | > loader_time: 0.00210 (0.03471) + + + --> STEP: 45/234 -- GLOBAL_STEP: 66735 + | > loss: -0.29340 (-0.33085) + | > log_mle: -0.39595 (-0.40903) + | > loss_dur: 0.10255 (0.07817) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.25240 (13.38657) + | > current_lr: 0.00007 + | > step_time: 1.71560 (3.11151) + | > loader_time: 0.00410 (0.03310) + + + --> STEP: 50/234 -- GLOBAL_STEP: 66740 + | > loss: -0.31221 (-0.32967) + | > log_mle: -0.38775 (-0.40779) + | > loss_dur: 0.07554 (0.07812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.66038 (12.83139) + | > current_lr: 0.00007 + | > step_time: 1.40600 (2.98645) + | > loader_time: 0.00250 (0.03342) + + + --> STEP: 55/234 -- GLOBAL_STEP: 66745 + | > loss: -0.32479 (-0.32780) + | > log_mle: -0.39548 (-0.40622) + | > loss_dur: 0.07069 (0.07842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.67557 (12.51181) + | > current_lr: 0.00007 + | > step_time: 1.96930 (2.87304) + | > loader_time: 0.00300 (0.03341) + + + --> STEP: 60/234 -- GLOBAL_STEP: 66750 + | > loss: -0.28046 (-0.32543) + | > log_mle: -0.39176 (-0.40476) + | > loss_dur: 0.11130 (0.07933) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06168 (12.43021) + | > current_lr: 0.00007 + | > step_time: 1.27370 (2.74099) + | > loader_time: 0.00180 (0.03079) + + + --> STEP: 65/234 -- GLOBAL_STEP: 66755 + | > loss: -0.28484 (-0.32173) + | > log_mle: -0.37755 (-0.40289) + | > loss_dur: 0.09271 (0.08117) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.35120 (12.49481) + | > current_lr: 0.00007 + | > step_time: 1.74310 (2.73536) + | > loader_time: 0.06860 (0.03124) + + + --> STEP: 70/234 -- GLOBAL_STEP: 66760 + | > loss: -0.24865 (-0.31867) + | > log_mle: -0.36249 (-0.40084) + | > loss_dur: 0.11384 (0.08217) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.00873 (12.28176) + | > current_lr: 0.00007 + | > step_time: 1.80560 (2.65140) + | > loader_time: 0.08600 (0.03154) + + + --> STEP: 75/234 -- GLOBAL_STEP: 66765 + | > loss: -0.26365 (-0.31534) + | > log_mle: -0.37796 (-0.39935) + | > loss_dur: 0.11431 (0.08401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.74651 (12.36521) + | > current_lr: 0.00007 + | > step_time: 2.61430 (2.66694) + | > loader_time: 0.09250 (0.03200) + + + --> STEP: 80/234 -- GLOBAL_STEP: 66770 + | > loss: -0.27998 (-0.31304) + | > log_mle: -0.37027 (-0.39787) + | > loss_dur: 0.09028 (0.08483) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.10422 (12.24427) + | > current_lr: 0.00007 + | > step_time: 6.21380 (2.67664) + | > loader_time: 0.00200 (0.03129) + + + --> STEP: 85/234 -- GLOBAL_STEP: 66775 + | > loss: -0.26953 (-0.31041) + | > log_mle: -0.37148 (-0.39673) + | > loss_dur: 0.10195 (0.08632) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.68032 (12.30467) + | > current_lr: 0.00007 + | > step_time: 1.48160 (2.60952) + | > loader_time: 0.00270 (0.03157) + + + --> STEP: 90/234 -- GLOBAL_STEP: 66780 + | > loss: -0.26388 (-0.30840) + | > log_mle: -0.39027 (-0.39651) + | > loss_dur: 0.12640 (0.08811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.82540 (12.47033) + | > current_lr: 0.00007 + | > step_time: 2.46230 (2.59504) + | > loader_time: 0.00470 (0.03191) + + + --> STEP: 95/234 -- GLOBAL_STEP: 66785 + | > loss: -0.32218 (-0.30739) + | > log_mle: -0.47043 (-0.39795) + | > loss_dur: 0.14825 (0.09056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.35524 (13.11846) + | > current_lr: 0.00007 + | > step_time: 2.69890 (2.57136) + | > loader_time: 0.00380 (0.03126) + + + --> STEP: 100/234 -- GLOBAL_STEP: 66790 + | > loss: -0.27272 (-0.30582) + | > log_mle: -0.39637 (-0.39801) + | > loss_dur: 0.12364 (0.09218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.76614 (13.27839) + | > current_lr: 0.00007 + | > step_time: 0.90020 (2.51736) + | > loader_time: 0.00360 (0.02981) + + + --> STEP: 105/234 -- GLOBAL_STEP: 66795 + | > loss: -0.25200 (-0.30424) + | > log_mle: -0.37841 (-0.39896) + | > loss_dur: 0.12641 (0.09472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.72040 (13.76482) + | > current_lr: 0.00007 + | > step_time: 1.30030 (2.50705) + | > loader_time: 0.08880 (0.03259) + + + --> STEP: 110/234 -- GLOBAL_STEP: 66800 + | > loss: -0.25439 (-0.30204) + | > log_mle: -0.39718 (-0.39938) + | > loss_dur: 0.14279 (0.09733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.17086 (14.31285) + | > current_lr: 0.00007 + | > step_time: 1.48520 (2.45760) + | > loader_time: 0.00160 (0.03122) + + + --> STEP: 115/234 -- GLOBAL_STEP: 66805 + | > loss: -0.25777 (-0.30084) + | > log_mle: -0.42206 (-0.40098) + | > loss_dur: 0.16429 (0.10014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.20212 (14.71880) + | > current_lr: 0.00007 + | > step_time: 4.39740 (2.48763) + | > loader_time: 0.00290 (0.03229) + + + --> STEP: 120/234 -- GLOBAL_STEP: 66810 + | > loss: -0.31164 (-0.29981) + | > log_mle: -0.46839 (-0.40232) + | > loss_dur: 0.15674 (0.10251) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.43931 (15.10300) + | > current_lr: 0.00007 + | > step_time: 2.29010 (2.47388) + | > loader_time: 0.00290 (0.03110) + + + --> STEP: 125/234 -- GLOBAL_STEP: 66815 + | > loss: -0.28409 (-0.29809) + | > log_mle: -0.45119 (-0.40247) + | > loss_dur: 0.16710 (0.10438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.32787 (15.34249) + | > current_lr: 0.00007 + | > step_time: 3.67170 (2.49586) + | > loader_time: 0.00270 (0.03196) + + + --> STEP: 130/234 -- GLOBAL_STEP: 66820 + | > loss: -0.29580 (-0.29775) + | > log_mle: -0.47208 (-0.40465) + | > loss_dur: 0.17628 (0.10690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.10574 (15.86882) + | > current_lr: 0.00007 + | > step_time: 2.00380 (2.47884) + | > loader_time: 0.00510 (0.03144) + + + --> STEP: 135/234 -- GLOBAL_STEP: 66825 + | > loss: -0.25987 (-0.29804) + | > log_mle: -0.39462 (-0.40695) + | > loss_dur: 0.13475 (0.10891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.69371 (16.73271) + | > current_lr: 0.00007 + | > step_time: 1.11700 (2.45567) + | > loader_time: 0.08400 (0.03166) + + + --> STEP: 140/234 -- GLOBAL_STEP: 66830 + | > loss: -0.23103 (-0.29804) + | > log_mle: -0.41386 (-0.40959) + | > loss_dur: 0.18282 (0.11155) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.36152 (17.86349) + | > current_lr: 0.00007 + | > step_time: 1.27870 (2.43502) + | > loader_time: 0.00180 (0.03062) + + + --> STEP: 145/234 -- GLOBAL_STEP: 66835 + | > loss: -0.34615 (-0.29853) + | > log_mle: -0.52265 (-0.41274) + | > loss_dur: 0.17650 (0.11421) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.24933 (18.53821) + | > current_lr: 0.00007 + | > step_time: 2.20330 (2.42710) + | > loader_time: 0.08360 (0.03202) + + + --> STEP: 150/234 -- GLOBAL_STEP: 66840 + | > loss: -0.32392 (-0.29941) + | > log_mle: -0.51386 (-0.41591) + | > loss_dur: 0.18994 (0.11650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.40733 (19.03130) + | > current_lr: 0.00007 + | > step_time: 1.61750 (2.40566) + | > loader_time: 0.00200 (0.03215) + + + --> STEP: 155/234 -- GLOBAL_STEP: 66845 + | > loss: -0.39181 (-0.30152) + | > log_mle: -0.58596 (-0.42044) + | > loss_dur: 0.19415 (0.11892) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.60535 (19.81475) + | > current_lr: 0.00007 + | > step_time: 2.90540 (2.40347) + | > loader_time: 0.08430 (0.03232) + + + --> STEP: 160/234 -- GLOBAL_STEP: 66850 + | > loss: -0.36750 (-0.30293) + | > log_mle: -0.58148 (-0.42429) + | > loss_dur: 0.21399 (0.12136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.76225 (20.53829) + | > current_lr: 0.00007 + | > step_time: 1.28100 (2.40332) + | > loader_time: 0.00300 (0.03188) + + + --> STEP: 165/234 -- GLOBAL_STEP: 66855 + | > loss: -0.39312 (-0.30456) + | > log_mle: -0.58322 (-0.42807) + | > loss_dur: 0.19010 (0.12351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.10566 (21.34297) + | > current_lr: 0.00007 + | > step_time: 3.31710 (2.39113) + | > loader_time: 0.00440 (0.03102) + + + --> STEP: 170/234 -- GLOBAL_STEP: 66860 + | > loss: -0.41027 (-0.30669) + | > log_mle: -0.63242 (-0.43255) + | > loss_dur: 0.22216 (0.12586) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.00989 (22.19788) + | > current_lr: 0.00007 + | > step_time: 1.30410 (2.37492) + | > loader_time: 0.00260 (0.03022) + + + --> STEP: 175/234 -- GLOBAL_STEP: 66865 + | > loss: -0.37693 (-0.30961) + | > log_mle: -0.59776 (-0.43783) + | > loss_dur: 0.22084 (0.12822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.94112 (23.48479) + | > current_lr: 0.00007 + | > step_time: 1.30060 (2.37670) + | > loader_time: 0.08580 (0.03101) + + + --> STEP: 180/234 -- GLOBAL_STEP: 66870 + | > loss: -0.38874 (-0.31201) + | > log_mle: -0.60184 (-0.44267) + | > loss_dur: 0.21310 (0.13066) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.99203 (24.63419) + | > current_lr: 0.00007 + | > step_time: 6.29510 (2.42349) + | > loader_time: 0.00410 (0.03171) + + + --> STEP: 185/234 -- GLOBAL_STEP: 66875 + | > loss: -0.40421 (-0.31394) + | > log_mle: -0.62511 (-0.44698) + | > loss_dur: 0.22090 (0.13304) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.33445 (25.88173) + | > current_lr: 0.00007 + | > step_time: 2.50340 (2.46459) + | > loader_time: 0.08810 (0.03341) + + + --> STEP: 190/234 -- GLOBAL_STEP: 66880 + | > loss: -0.41048 (-0.31602) + | > log_mle: -0.61284 (-0.45128) + | > loss_dur: 0.20236 (0.13526) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.67835 (26.73985) + | > current_lr: 0.00007 + | > step_time: 3.81390 (2.50131) + | > loader_time: 0.09260 (0.03309) + + + --> STEP: 195/234 -- GLOBAL_STEP: 66885 + | > loss: -0.41946 (-0.31882) + | > log_mle: -0.64090 (-0.45594) + | > loss_dur: 0.22144 (0.13712) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.38987 (27.82530) + | > current_lr: 0.00007 + | > step_time: 0.66750 (2.55011) + | > loader_time: 0.00300 (0.03429) + + + --> STEP: 200/234 -- GLOBAL_STEP: 66890 + | > loss: -0.40280 (-0.32114) + | > log_mle: -0.64242 (-0.46025) + | > loss_dur: 0.23963 (0.13911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.29791 (28.60507) + | > current_lr: 0.00007 + | > step_time: 5.39580 (2.54149) + | > loader_time: 0.10690 (0.03445) + + + --> STEP: 205/234 -- GLOBAL_STEP: 66895 + | > loss: -0.39841 (-0.32325) + | > log_mle: -0.62189 (-0.46434) + | > loss_dur: 0.22347 (0.14108) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.51228 (29.38934) + | > current_lr: 0.00007 + | > step_time: 4.29660 (2.54594) + | > loader_time: 0.00560 (0.03412) + + + --> STEP: 210/234 -- GLOBAL_STEP: 66900 + | > loss: -0.46174 (-0.32610) + | > log_mle: -0.69520 (-0.46920) + | > loss_dur: 0.23346 (0.14309) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 106.57295 (30.56425) + | > current_lr: 0.00007 + | > step_time: 10.50180 (2.62644) + | > loader_time: 0.28340 (0.03638) + + + --> STEP: 215/234 -- GLOBAL_STEP: 66905 + | > loss: -0.42335 (-0.32912) + | > log_mle: -0.65324 (-0.47423) + | > loss_dur: 0.22990 (0.14511) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.04308 (31.72960) + | > current_lr: 0.00007 + | > step_time: 6.83000 (2.65983) + | > loader_time: 0.19570 (0.03815) + + + --> STEP: 220/234 -- GLOBAL_STEP: 66910 + | > loss: -0.46823 (-0.33268) + | > log_mle: -0.71306 (-0.47982) + | > loss_dur: 0.24484 (0.14714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.61455 (32.70074) + | > current_lr: 0.00007 + | > step_time: 1.21450 (2.69645) + | > loader_time: 0.00220 (0.04091) + + + --> STEP: 225/234 -- GLOBAL_STEP: 66915 + | > loss: -0.54305 (-0.33605) + | > log_mle: -0.78071 (-0.48513) + | > loss_dur: 0.23766 (0.14909) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.97191 (33.69174) + | > current_lr: 0.00007 + | > step_time: 0.22740 (2.65228) + | > loader_time: 0.00370 (0.04009) + + + --> STEP: 230/234 -- GLOBAL_STEP: 66920 + | > loss: -0.51065 (-0.33896) + | > log_mle: -0.82139 (-0.49073) + | > loss_dur: 0.31075 (0.15177) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.26743 (35.10422) + | > current_lr: 0.00007 + | > step_time: 0.25620 (2.60006) + | > loader_time: 0.00370 (0.03928) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.97448 (+0.30178) + | > avg_loss: -0.33362 (-0.00116) + | > avg_log_mle: -0.55485 (+0.00297) + | > avg_loss_dur: 0.22123 (-0.00413) + + + > EPOCH: 286/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 21:26:57)  + + --> STEP: 1/234 -- GLOBAL_STEP: 66925 + | > loss: -0.31641 (-0.31641) + | > log_mle: -0.40697 (-0.40697) + | > loss_dur: 0.09056 (0.09056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.48430 (24.48430) + | > current_lr: 0.00007 + | > step_time: 5.89520 (5.89523) + | > loader_time: 11.28890 (11.28891) + + + --> STEP: 6/234 -- GLOBAL_STEP: 66930 + | > loss: -0.34997 (-0.32149) + | > log_mle: -0.41633 (-0.41330) + | > loss_dur: 0.06636 (0.09181) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.37721 (19.48100) + | > current_lr: 0.00007 + | > step_time: 3.61270 (3.38732) + | > loader_time: 0.00380 (1.96310) + + + --> STEP: 11/234 -- GLOBAL_STEP: 66935 + | > loss: -0.37246 (-0.33191) + | > log_mle: -0.42704 (-0.41729) + | > loss_dur: 0.05458 (0.08538) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.90573 (19.34966) + | > current_lr: 0.00007 + | > step_time: 4.31970 (4.07729) + | > loader_time: 0.00500 (1.08803) + + + --> STEP: 16/234 -- GLOBAL_STEP: 66940 + | > loss: -0.37324 (-0.34091) + | > log_mle: -0.43388 (-0.42052) + | > loss_dur: 0.06064 (0.07961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.78001 (18.07361) + | > current_lr: 0.00007 + | > step_time: 1.21410 (4.58081) + | > loader_time: 0.07820 (0.76919) + + + --> STEP: 21/234 -- GLOBAL_STEP: 66945 + | > loss: -0.34555 (-0.34504) + | > log_mle: -0.40809 (-0.42052) + | > loss_dur: 0.06254 (0.07547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.85316 (16.67060) + | > current_lr: 0.00007 + | > step_time: 1.19190 (3.82337) + | > loader_time: 0.00940 (0.59027) + + + --> STEP: 26/234 -- GLOBAL_STEP: 66950 + | > loss: -0.32751 (-0.34528) + | > log_mle: -0.40465 (-0.41958) + | > loss_dur: 0.07715 (0.07430) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.21219 (15.70439) + | > current_lr: 0.00007 + | > step_time: 1.98900 (3.48333) + | > loader_time: 0.00160 (0.47726) + + + --> STEP: 31/234 -- GLOBAL_STEP: 66955 + | > loss: -0.28908 (-0.34432) + | > log_mle: -0.39186 (-0.41845) + | > loss_dur: 0.10278 (0.07413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.45221 (14.94348) + | > current_lr: 0.00007 + | > step_time: 2.09450 (3.32868) + | > loader_time: 0.00360 (0.40076) + + + --> STEP: 36/234 -- GLOBAL_STEP: 66960 + | > loss: -0.31185 (-0.34081) + | > log_mle: -0.38936 (-0.41550) + | > loss_dur: 0.07751 (0.07468) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.63772 (14.44606) + | > current_lr: 0.00007 + | > step_time: 2.86670 (3.10906) + | > loader_time: 0.00500 (0.34548) + + + --> STEP: 41/234 -- GLOBAL_STEP: 66965 + | > loss: -0.33849 (-0.33710) + | > log_mle: -0.40427 (-0.41263) + | > loss_dur: 0.06577 (0.07552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.31494 (14.06881) + | > current_lr: 0.00007 + | > step_time: 2.39450 (2.99570) + | > loader_time: 0.00670 (0.30574) + + + --> STEP: 46/234 -- GLOBAL_STEP: 66970 + | > loss: -0.29662 (-0.33298) + | > log_mle: -0.39017 (-0.41000) + | > loss_dur: 0.09355 (0.07702) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.47569 (14.01122) + | > current_lr: 0.00007 + | > step_time: 2.31120 (2.90965) + | > loader_time: 0.08720 (0.27810) + + + --> STEP: 51/234 -- GLOBAL_STEP: 66975 + | > loss: -0.31307 (-0.33212) + | > log_mle: -0.38889 (-0.40884) + | > loss_dur: 0.07582 (0.07672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.20063 (13.46419) + | > current_lr: 0.00007 + | > step_time: 1.80440 (2.80213) + | > loader_time: 0.00330 (0.25106) + + + --> STEP: 56/234 -- GLOBAL_STEP: 66980 + | > loss: -0.29780 (-0.32997) + | > log_mle: -0.39042 (-0.40730) + | > loss_dur: 0.09262 (0.07733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42722 (13.32995) + | > current_lr: 0.00007 + | > step_time: 3.90650 (2.81111) + | > loader_time: 0.00410 (0.23354) + + + --> STEP: 61/234 -- GLOBAL_STEP: 66985 + | > loss: -0.29033 (-0.32721) + | > log_mle: -0.37954 (-0.40563) + | > loss_dur: 0.08921 (0.07842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.46495 (13.22400) + | > current_lr: 0.00007 + | > step_time: 2.08530 (2.73163) + | > loader_time: 0.00210 (0.21748) + + + --> STEP: 66/234 -- GLOBAL_STEP: 66990 + | > loss: -0.30224 (-0.32416) + | > log_mle: -0.38006 (-0.40393) + | > loss_dur: 0.07782 (0.07977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.99516 (13.14097) + | > current_lr: 0.00007 + | > step_time: 1.77470 (2.65951) + | > loader_time: 0.00460 (0.20250) + + + --> STEP: 71/234 -- GLOBAL_STEP: 66995 + | > loss: -0.26712 (-0.32095) + | > log_mle: -0.39421 (-0.40221) + | > loss_dur: 0.12709 (0.08126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.17892 (13.16160) + | > current_lr: 0.00007 + | > step_time: 2.01070 (2.60765) + | > loader_time: 0.00170 (0.19098) + + + --> STEP: 76/234 -- GLOBAL_STEP: 67000 + | > loss: -0.27646 (-0.31765) + | > log_mle: -0.38157 (-0.40066) + | > loss_dur: 0.10511 (0.08301) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.68708 (13.16780) + | > current_lr: 0.00007 + | > step_time: 1.99340 (2.55840) + | > loader_time: 0.00170 (0.18074) + + + --> STEP: 81/234 -- GLOBAL_STEP: 67005 + | > loss: -0.27425 (-0.31529) + | > log_mle: -0.38950 (-0.39933) + | > loss_dur: 0.11525 (0.08404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.00743 (13.02393) + | > current_lr: 0.00007 + | > step_time: 2.50710 (2.52084) + | > loader_time: 0.00230 (0.16972) + + + --> STEP: 86/234 -- GLOBAL_STEP: 67010 + | > loss: -0.27251 (-0.31256) + | > log_mle: -0.38386 (-0.39806) + | > loss_dur: 0.11135 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.80646 (13.08975) + | > current_lr: 0.00007 + | > step_time: 2.50350 (2.52004) + | > loader_time: 0.01160 (0.16327) + + + --> STEP: 91/234 -- GLOBAL_STEP: 67015 + | > loss: -0.26760 (-0.31025) + | > log_mle: -0.39466 (-0.39786) + | > loss_dur: 0.12706 (0.08761) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.76584 (13.14971) + | > current_lr: 0.00007 + | > step_time: 2.92280 (2.49584) + | > loader_time: 0.00240 (0.15442) + + + --> STEP: 96/234 -- GLOBAL_STEP: 67020 + | > loss: -0.26103 (-0.30877) + | > log_mle: -0.38063 (-0.39913) + | > loss_dur: 0.11960 (0.09037) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.33121 (13.44470) + | > current_lr: 0.00007 + | > step_time: 2.47670 (2.46220) + | > loader_time: 0.10020 (0.14753) + + + --> STEP: 101/234 -- GLOBAL_STEP: 67025 + | > loss: -0.27249 (-0.30713) + | > log_mle: -0.42308 (-0.39960) + | > loss_dur: 0.15059 (0.09247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.04053 (13.75227) + | > current_lr: 0.00007 + | > step_time: 2.21660 (2.43858) + | > loader_time: 0.08350 (0.14119) + + + --> STEP: 106/234 -- GLOBAL_STEP: 67030 + | > loss: -0.25238 (-0.30550) + | > log_mle: -0.42285 (-0.40064) + | > loss_dur: 0.17047 (0.09514) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.08421 (14.37087) + | > current_lr: 0.00007 + | > step_time: 5.90990 (2.48119) + | > loader_time: 0.09640 (0.13705) + + + --> STEP: 111/234 -- GLOBAL_STEP: 67035 + | > loss: -0.29786 (-0.30393) + | > log_mle: -0.48048 (-0.40186) + | > loss_dur: 0.18262 (0.09793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.12616 (14.75573) + | > current_lr: 0.00007 + | > step_time: 1.18710 (2.48568) + | > loader_time: 0.00150 (0.13268) + + + --> STEP: 116/234 -- GLOBAL_STEP: 67040 + | > loss: -0.26226 (-0.30256) + | > log_mle: -0.44421 (-0.40323) + | > loss_dur: 0.18195 (0.10067) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.00654 (15.25261) + | > current_lr: 0.00007 + | > step_time: 2.10830 (2.46251) + | > loader_time: 0.08790 (0.12932) + + + --> STEP: 121/234 -- GLOBAL_STEP: 67045 + | > loss: -0.23622 (-0.30136) + | > log_mle: -0.36140 (-0.40397) + | > loss_dur: 0.12518 (0.10261) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.53891 (15.46859) + | > current_lr: 0.00007 + | > step_time: 1.50370 (2.45897) + | > loader_time: 0.00290 (0.12553) + + + --> STEP: 126/234 -- GLOBAL_STEP: 67050 + | > loss: -0.30617 (-0.30015) + | > log_mle: -0.48629 (-0.40510) + | > loss_dur: 0.18011 (0.10494) + | > amp_scaler: 2048.00000 (1032.12698) + | > grad_norm: 36.43951 (15.90075) + | > current_lr: 0.00007 + | > step_time: 7.20490 (2.49788) + | > loader_time: 0.08830 (0.12135) + + + --> STEP: 131/234 -- GLOBAL_STEP: 67055 + | > loss: -0.36042 (-0.30029) + | > log_mle: -0.53808 (-0.40755) + | > loss_dur: 0.17766 (0.10726) + | > amp_scaler: 2048.00000 (1070.90076) + | > grad_norm: 52.04274 (16.56771) + | > current_lr: 0.00007 + | > step_time: 1.90990 (2.53229) + | > loader_time: 0.00290 (0.11830) + + + --> STEP: 136/234 -- GLOBAL_STEP: 67060 + | > loss: -0.38926 (-0.30057) + | > log_mle: -0.59163 (-0.41015) + | > loss_dur: 0.20237 (0.10958) + | > amp_scaler: 2048.00000 (1106.82353) + | > grad_norm: 52.61403 (17.37785) + | > current_lr: 0.00007 + | > step_time: 2.81080 (2.53571) + | > loader_time: 0.09060 (0.11604) + + + --> STEP: 141/234 -- GLOBAL_STEP: 67065 + | > loss: -0.31076 (-0.30019) + | > log_mle: -0.48415 (-0.41217) + | > loss_dur: 0.17340 (0.11198) + | > amp_scaler: 2048.00000 (1140.19858) + | > grad_norm: 34.17455 (18.12872) + | > current_lr: 0.00007 + | > step_time: 8.21490 (2.63751) + | > loader_time: 0.09540 (0.11343) + + + --> STEP: 146/234 -- GLOBAL_STEP: 67070 + | > loss: -0.33273 (-0.30117) + | > log_mle: -0.52833 (-0.41590) + | > loss_dur: 0.19559 (0.11473) + | > amp_scaler: 2048.00000 (1171.28767) + | > grad_norm: 43.15808 (19.06583) + | > current_lr: 0.00007 + | > step_time: 5.51570 (2.67317) + | > loader_time: 0.08300 (0.11291) + + + --> STEP: 151/234 -- GLOBAL_STEP: 67075 + | > loss: -0.31305 (-0.30187) + | > log_mle: -0.49101 (-0.41884) + | > loss_dur: 0.17796 (0.11697) + | > amp_scaler: 2048.00000 (1200.31788) + | > grad_norm: 40.76767 (19.69703) + | > current_lr: 0.00007 + | > step_time: 4.89800 (2.70465) + | > loader_time: 0.00320 (0.11008) + + + --> STEP: 156/234 -- GLOBAL_STEP: 67080 + | > loss: -0.35819 (-0.30400) + | > log_mle: -0.54556 (-0.42342) + | > loss_dur: 0.18737 (0.11942) + | > amp_scaler: 2048.00000 (1227.48718) + | > grad_norm: 35.13896 (20.62310) + | > current_lr: 0.00007 + | > step_time: 4.38940 (2.80005) + | > loader_time: 0.19940 (0.10903) + + + --> STEP: 161/234 -- GLOBAL_STEP: 67085 + | > loss: -0.39418 (-0.30548) + | > log_mle: -0.57881 (-0.42738) + | > loss_dur: 0.18463 (0.12190) + | > amp_scaler: 2048.00000 (1252.96894) + | > grad_norm: 41.73786 (21.37831) + | > current_lr: 0.00007 + | > step_time: 3.51040 (2.88053) + | > loader_time: 0.00340 (0.10683) + + + --> STEP: 166/234 -- GLOBAL_STEP: 67090 + | > loss: -0.32485 (-0.30656) + | > log_mle: -0.50400 (-0.43056) + | > loss_dur: 0.17915 (0.12400) + | > amp_scaler: 2048.00000 (1276.91566) + | > grad_norm: 45.52895 (22.31362) + | > current_lr: 0.00007 + | > step_time: 2.90690 (2.92753) + | > loader_time: 0.08690 (0.10470) + + + --> STEP: 171/234 -- GLOBAL_STEP: 67095 + | > loss: -0.43433 (-0.30905) + | > log_mle: -0.62874 (-0.43555) + | > loss_dur: 0.19441 (0.12650) + | > amp_scaler: 2048.00000 (1299.46199) + | > grad_norm: 48.16463 (23.15295) + | > current_lr: 0.00007 + | > step_time: 5.03280 (2.94916) + | > loader_time: 0.00770 (0.10348) + + + --> STEP: 176/234 -- GLOBAL_STEP: 67100 + | > loss: -0.39912 (-0.31148) + | > log_mle: -0.59752 (-0.44044) + | > loss_dur: 0.19840 (0.12895) + | > amp_scaler: 2048.00000 (1320.72727) + | > grad_norm: 50.56240 (24.10970) + | > current_lr: 0.00007 + | > step_time: 5.18670 (2.95272) + | > loader_time: 0.10160 (0.10255) + + + --> STEP: 181/234 -- GLOBAL_STEP: 67105 + | > loss: -0.32413 (-0.31331) + | > log_mle: -0.52799 (-0.44475) + | > loss_dur: 0.20386 (0.13144) + | > amp_scaler: 2048.00000 (1340.81768) + | > grad_norm: 39.91616 (25.05919) + | > current_lr: 0.00007 + | > step_time: 4.61380 (2.98012) + | > loader_time: 0.09160 (0.10188) + + + --> STEP: 186/234 -- GLOBAL_STEP: 67110 + | > loss: -0.34565 (-0.31542) + | > log_mle: -0.57543 (-0.44933) + | > loss_dur: 0.22977 (0.13391) + | > amp_scaler: 2048.00000 (1359.82796) + | > grad_norm: 39.36883 (25.87558) + | > current_lr: 0.00007 + | > step_time: 2.30510 (2.99723) + | > loader_time: 0.07770 (0.09970) + + + --> STEP: 191/234 -- GLOBAL_STEP: 67115 + | > loss: -0.38328 (-0.31784) + | > log_mle: -0.58544 (-0.45370) + | > loss_dur: 0.20216 (0.13586) + | > amp_scaler: 2048.00000 (1377.84293) + | > grad_norm: 91.85901 (26.92640) + | > current_lr: 0.00007 + | > step_time: 5.68850 (3.01982) + | > loader_time: 0.10980 (0.09918) + + + --> STEP: 196/234 -- GLOBAL_STEP: 67120 + | > loss: -0.35940 (-0.32029) + | > log_mle: -0.57770 (-0.45808) + | > loss_dur: 0.21830 (0.13779) + | > amp_scaler: 2048.00000 (1394.93878) + | > grad_norm: 73.81196 (28.05401) + | > current_lr: 0.00007 + | > step_time: 4.40380 (3.07129) + | > loader_time: 0.00260 (0.09824) + + + --> STEP: 201/234 -- GLOBAL_STEP: 67125 + | > loss: -0.31582 (-0.32212) + | > log_mle: -0.54242 (-0.46193) + | > loss_dur: 0.22661 (0.13982) + | > amp_scaler: 2048.00000 (1411.18408) + | > grad_norm: 49.86268 (28.90806) + | > current_lr: 0.00007 + | > step_time: 7.59930 (3.14768) + | > loader_time: 0.19250 (0.09868) + + + --> STEP: 206/234 -- GLOBAL_STEP: 67130 + | > loss: -0.43731 (-0.32463) + | > log_mle: -0.65765 (-0.46650) + | > loss_dur: 0.22034 (0.14187) + | > amp_scaler: 2048.00000 (1426.64078) + | > grad_norm: 63.52673 (29.76890) + | > current_lr: 0.00007 + | > step_time: 1.99420 (3.17521) + | > loader_time: 0.01120 (0.09919) + + + --> STEP: 211/234 -- GLOBAL_STEP: 67135 + | > loss: -0.48893 (-0.32780) + | > log_mle: -0.73664 (-0.47181) + | > loss_dur: 0.24771 (0.14401) + | > amp_scaler: 2048.00000 (1441.36493) + | > grad_norm: 79.57903 (30.66415) + | > current_lr: 0.00007 + | > step_time: 10.11340 (3.26779) + | > loader_time: 0.08690 (0.09865) + + + --> STEP: 216/234 -- GLOBAL_STEP: 67140 + | > loss: -0.45433 (-0.33022) + | > log_mle: -0.69785 (-0.47623) + | > loss_dur: 0.24352 (0.14601) + | > amp_scaler: 2048.00000 (1455.40741) + | > grad_norm: 81.98530 (31.74107) + | > current_lr: 0.00007 + | > step_time: 11.10630 (3.36389) + | > loader_time: 0.08400 (0.09767) + + + --> STEP: 221/234 -- GLOBAL_STEP: 67145 + | > loss: -0.39739 (-0.33276) + | > log_mle: -0.61291 (-0.48077) + | > loss_dur: 0.21551 (0.14801) + | > amp_scaler: 2048.00000 (1468.81448) + | > grad_norm: 74.93198 (32.87347) + | > current_lr: 0.00007 + | > step_time: 1.01070 (3.38231) + | > loader_time: 0.00310 (0.09556) + + + --> STEP: 226/234 -- GLOBAL_STEP: 67150 + | > loss: -0.48589 (-0.33591) + | > log_mle: -0.73125 (-0.48604) + | > loss_dur: 0.24536 (0.15013) + | > amp_scaler: 1024.00000 (1458.97345) + | > grad_norm: 84.59627 (33.93085) + | > current_lr: 0.00007 + | > step_time: 0.24750 (3.31769) + | > loader_time: 0.00320 (0.09352) + + + --> STEP: 231/234 -- GLOBAL_STEP: 67155 + | > loss: -0.41931 (-0.33827) + | > log_mle: -0.79096 (-0.49158) + | > loss_dur: 0.37165 (0.15331) + | > amp_scaler: 1024.00000 (1449.55844) + | > grad_norm: 67.81502 (35.22316) + | > current_lr: 0.00007 + | > step_time: 0.27150 (3.25137) + | > loader_time: 0.00360 (0.09157) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.34958 (-0.62490) + | > avg_loss: -0.33051 (+0.00311) + | > avg_log_mle: -0.55733 (-0.00248) + | > avg_loss_dur: 0.22682 (+0.00559) + + + > EPOCH: 287/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 21:40:46)  + + --> STEP: 2/234 -- GLOBAL_STEP: 67160 + | > loss: -0.34589 (-0.33964) + | > log_mle: -0.42916 (-0.41960) + | > loss_dur: 0.08327 (0.07996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.72309 (22.03736) + | > current_lr: 0.00007 + | > step_time: 2.40600 (4.20285) + | > loader_time: 0.08270 (0.04412) + + + --> STEP: 7/234 -- GLOBAL_STEP: 67165 + | > loss: -0.34239 (-0.32487) + | > log_mle: -0.41363 (-0.41444) + | > loss_dur: 0.07124 (0.08957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.43228 (20.57850) + | > current_lr: 0.00007 + | > step_time: 17.29790 (5.53752) + | > loader_time: 0.51080 (0.11436) + + + --> STEP: 12/234 -- GLOBAL_STEP: 67170 + | > loss: -0.33410 (-0.33112) + | > log_mle: -0.41386 (-0.41794) + | > loss_dur: 0.07976 (0.08682) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.55170 (19.88137) + | > current_lr: 0.00007 + | > step_time: 6.00260 (4.57927) + | > loader_time: 0.00190 (0.08177) + + + --> STEP: 17/234 -- GLOBAL_STEP: 67175 + | > loss: -0.37771 (-0.34166) + | > log_mle: -0.42927 (-0.42147) + | > loss_dur: 0.05155 (0.07981) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.79182 (17.78558) + | > current_lr: 0.00007 + | > step_time: 4.19380 (4.54418) + | > loader_time: 0.19680 (0.07518) + + + --> STEP: 22/234 -- GLOBAL_STEP: 67180 + | > loss: -0.33329 (-0.34444) + | > log_mle: -0.41303 (-0.42085) + | > loss_dur: 0.07974 (0.07641) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.98057 (16.46903) + | > current_lr: 0.00007 + | > step_time: 1.09140 (4.37973) + | > loader_time: 0.00110 (0.06743) + + + --> STEP: 27/234 -- GLOBAL_STEP: 67185 + | > loss: -0.33300 (-0.34470) + | > log_mle: -0.40477 (-0.41949) + | > loss_dur: 0.07177 (0.07480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.93123 (15.64117) + | > current_lr: 0.00007 + | > step_time: 2.10710 (4.12450) + | > loader_time: 0.00750 (0.06210) + + + --> STEP: 32/234 -- GLOBAL_STEP: 67190 + | > loss: -0.33189 (-0.34405) + | > log_mle: -0.40425 (-0.41815) + | > loss_dur: 0.07236 (0.07410) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.23931 (14.87948) + | > current_lr: 0.00007 + | > step_time: 12.50310 (4.40533) + | > loader_time: 0.10890 (0.05877) + + + --> STEP: 37/234 -- GLOBAL_STEP: 67195 + | > loss: -0.31329 (-0.33968) + | > log_mle: -0.38280 (-0.41442) + | > loss_dur: 0.06951 (0.07474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.01034 (14.45667) + | > current_lr: 0.00007 + | > step_time: 0.59020 (4.06812) + | > loader_time: 0.00170 (0.05540) + + + --> STEP: 42/234 -- GLOBAL_STEP: 67200 + | > loss: -0.31458 (-0.33631) + | > log_mle: -0.38207 (-0.41175) + | > loss_dur: 0.06749 (0.07544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.03766 (13.86047) + | > current_lr: 0.00007 + | > step_time: 0.73400 (3.72107) + | > loader_time: 0.00130 (0.04911) + + + --> STEP: 47/234 -- GLOBAL_STEP: 67205 + | > loss: -0.31077 (-0.33266) + | > log_mle: -0.39546 (-0.40954) + | > loss_dur: 0.08469 (0.07689) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.29472 (13.50357) + | > current_lr: 0.00007 + | > step_time: 4.50480 (3.54415) + | > loader_time: 0.00170 (0.04590) + + + --> STEP: 52/234 -- GLOBAL_STEP: 67210 + | > loss: -0.29125 (-0.33151) + | > log_mle: -0.38305 (-0.40819) + | > loss_dur: 0.09180 (0.07667) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.74238 (12.99455) + | > current_lr: 0.00007 + | > step_time: 1.49580 (3.35901) + | > loader_time: 0.00200 (0.04347) + + + --> STEP: 57/234 -- GLOBAL_STEP: 67215 + | > loss: -0.29522 (-0.32937) + | > log_mle: -0.37700 (-0.40655) + | > loss_dur: 0.08177 (0.07718) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.15788 (12.94385) + | > current_lr: 0.00007 + | > step_time: 0.77160 (3.17971) + | > loader_time: 0.00120 (0.04122) + + + --> STEP: 62/234 -- GLOBAL_STEP: 67220 + | > loss: -0.24269 (-0.32584) + | > log_mle: -0.39181 (-0.40514) + | > loss_dur: 0.14911 (0.07930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.21151 (13.23394) + | > current_lr: 0.00007 + | > step_time: 2.21370 (3.13578) + | > loader_time: 0.00200 (0.04211) + + + --> STEP: 67/234 -- GLOBAL_STEP: 67225 + | > loss: -0.29130 (-0.32350) + | > log_mle: -0.38778 (-0.40339) + | > loss_dur: 0.09648 (0.07989) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.92508 (12.99241) + | > current_lr: 0.00007 + | > step_time: 1.39030 (3.01325) + | > loader_time: 0.00250 (0.03912) + + + --> STEP: 72/234 -- GLOBAL_STEP: 67230 + | > loss: -0.29161 (-0.32033) + | > log_mle: -0.38344 (-0.40179) + | > loss_dur: 0.09183 (0.08145) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.58489 (12.81815) + | > current_lr: 0.00007 + | > step_time: 1.39840 (2.90930) + | > loader_time: 0.00240 (0.03880) + + + --> STEP: 77/234 -- GLOBAL_STEP: 67235 + | > loss: -0.28708 (-0.31680) + | > log_mle: -0.38087 (-0.40023) + | > loss_dur: 0.09378 (0.08343) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.66939 (12.82634) + | > current_lr: 0.00007 + | > step_time: 3.00580 (2.86469) + | > loader_time: 0.00160 (0.03770) + + + --> STEP: 82/234 -- GLOBAL_STEP: 67240 + | > loss: -0.26574 (-0.31446) + | > log_mle: -0.37219 (-0.39886) + | > loss_dur: 0.10645 (0.08440) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.57086 (12.69349) + | > current_lr: 0.00007 + | > step_time: 1.30700 (2.79349) + | > loader_time: 0.07550 (0.03644) + + + --> STEP: 87/234 -- GLOBAL_STEP: 67245 + | > loss: -0.25874 (-0.31173) + | > log_mle: -0.37524 (-0.39771) + | > loss_dur: 0.11650 (0.08599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.42460 (12.73014) + | > current_lr: 0.00007 + | > step_time: 1.90830 (2.77325) + | > loader_time: 0.08540 (0.03549) + + + --> STEP: 92/234 -- GLOBAL_STEP: 67250 + | > loss: -0.28880 (-0.30980) + | > log_mle: -0.41406 (-0.39800) + | > loss_dur: 0.12527 (0.08820) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.04904 (12.95233) + | > current_lr: 0.00007 + | > step_time: 4.39680 (2.76512) + | > loader_time: 0.00570 (0.03465) + + + --> STEP: 97/234 -- GLOBAL_STEP: 67255 + | > loss: -0.27727 (-0.30879) + | > log_mle: -0.39969 (-0.39915) + | > loss_dur: 0.12241 (0.09036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.86924 (13.30852) + | > current_lr: 0.00007 + | > step_time: 1.95000 (2.75656) + | > loader_time: 0.00190 (0.03556) + + + --> STEP: 102/234 -- GLOBAL_STEP: 67260 + | > loss: -0.23659 (-0.30673) + | > log_mle: -0.38199 (-0.39942) + | > loss_dur: 0.14540 (0.09269) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.54502 (13.69131) + | > current_lr: 0.00007 + | > step_time: 1.61300 (2.71345) + | > loader_time: 0.08070 (0.03471) + + + --> STEP: 107/234 -- GLOBAL_STEP: 67265 + | > loss: -0.26015 (-0.30518) + | > log_mle: -0.42061 (-0.40067) + | > loss_dur: 0.16046 (0.09549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.71074 (14.22307) + | > current_lr: 0.00007 + | > step_time: 1.90650 (2.66673) + | > loader_time: 0.00480 (0.03320) + + + --> STEP: 112/234 -- GLOBAL_STEP: 67270 + | > loss: -0.27615 (-0.30381) + | > log_mle: -0.43848 (-0.40208) + | > loss_dur: 0.16233 (0.09826) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.85025 (14.51821) + | > current_lr: 0.00007 + | > step_time: 1.30110 (2.61281) + | > loader_time: 0.00520 (0.03317) + + + --> STEP: 117/234 -- GLOBAL_STEP: 67275 + | > loss: -0.28028 (-0.30234) + | > log_mle: -0.43438 (-0.40337) + | > loss_dur: 0.15410 (0.10103) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.60519 (14.96202) + | > current_lr: 0.00007 + | > step_time: 1.30740 (2.59198) + | > loader_time: 0.00290 (0.03268) + + + --> STEP: 122/234 -- GLOBAL_STEP: 67280 + | > loss: -0.25026 (-0.30073) + | > log_mle: -0.39665 (-0.40371) + | > loss_dur: 0.14639 (0.10298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.19799 (15.34594) + | > current_lr: 0.00007 + | > step_time: 1.29560 (2.55127) + | > loader_time: 0.00380 (0.03361) + + + --> STEP: 127/234 -- GLOBAL_STEP: 67285 + | > loss: -0.28754 (-0.29987) + | > log_mle: -0.46395 (-0.40539) + | > loss_dur: 0.17640 (0.10551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.80711 (15.84576) + | > current_lr: 0.00007 + | > step_time: 1.57580 (2.50824) + | > loader_time: 0.00200 (0.03306) + + + --> STEP: 132/234 -- GLOBAL_STEP: 67290 + | > loss: -0.28664 (-0.30007) + | > log_mle: -0.44315 (-0.40773) + | > loss_dur: 0.15651 (0.10766) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.33824 (16.41039) + | > current_lr: 0.00007 + | > step_time: 1.80010 (2.49790) + | > loader_time: 0.00280 (0.03332) + + + --> STEP: 137/234 -- GLOBAL_STEP: 67295 + | > loss: -0.26507 (-0.30020) + | > log_mle: -0.45137 (-0.41033) + | > loss_dur: 0.18630 (0.11013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.19344 (17.48609) + | > current_lr: 0.00007 + | > step_time: 2.80430 (2.48861) + | > loader_time: 0.08130 (0.03424) + + + --> STEP: 142/234 -- GLOBAL_STEP: 67300 + | > loss: -0.28353 (-0.30001) + | > log_mle: -0.46686 (-0.41248) + | > loss_dur: 0.18334 (0.11247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.87157 (18.15894) + | > current_lr: 0.00007 + | > step_time: 1.39180 (2.47661) + | > loader_time: 0.00770 (0.03503) + + + --> STEP: 147/234 -- GLOBAL_STEP: 67305 + | > loss: -0.29116 (-0.30118) + | > log_mle: -0.46979 (-0.41642) + | > loss_dur: 0.17863 (0.11523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.29108 (18.97137) + | > current_lr: 0.00007 + | > step_time: 2.70760 (2.48241) + | > loader_time: 0.08060 (0.03563) + + + --> STEP: 152/234 -- GLOBAL_STEP: 67310 + | > loss: -0.35620 (-0.30260) + | > log_mle: -0.55669 (-0.42009) + | > loss_dur: 0.20049 (0.11749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.92307 (19.69081) + | > current_lr: 0.00007 + | > step_time: 1.60100 (2.47128) + | > loader_time: 0.00610 (0.03470) + + + --> STEP: 157/234 -- GLOBAL_STEP: 67315 + | > loss: -0.31846 (-0.30468) + | > log_mle: -0.50174 (-0.42442) + | > loss_dur: 0.18328 (0.11975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.51998 (20.67440) + | > current_lr: 0.00007 + | > step_time: 5.40540 (2.50081) + | > loader_time: 0.19430 (0.03664) + + + --> STEP: 162/234 -- GLOBAL_STEP: 67320 + | > loss: -0.34704 (-0.30630) + | > log_mle: -0.53399 (-0.42842) + | > loss_dur: 0.18695 (0.12212) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.45279 (21.91219) + | > current_lr: 0.00007 + | > step_time: 3.01070 (2.52787) + | > loader_time: 0.09190 (0.03617) + + + --> STEP: 167/234 -- GLOBAL_STEP: 67325 + | > loss: -0.44725 (-0.30806) + | > log_mle: -0.63871 (-0.43222) + | > loss_dur: 0.19146 (0.12416) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.77126 (22.59304) + | > current_lr: 0.00007 + | > step_time: 3.72120 (2.53979) + | > loader_time: 0.07890 (0.03578) + + + --> STEP: 172/234 -- GLOBAL_STEP: 67330 + | > loss: -0.38772 (-0.31026) + | > log_mle: -0.60204 (-0.43698) + | > loss_dur: 0.21433 (0.12672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 101.75248 (23.73092) + | > current_lr: 0.00007 + | > step_time: 5.50030 (2.55031) + | > loader_time: 0.08990 (0.03632) + + + --> STEP: 177/234 -- GLOBAL_STEP: 67335 + | > loss: -0.35387 (-0.31198) + | > log_mle: -0.55776 (-0.44106) + | > loss_dur: 0.20390 (0.12908) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.52665 (24.59330) + | > current_lr: 0.00007 + | > step_time: 1.71600 (2.54395) + | > loader_time: 0.08930 (0.03587) + + + --> STEP: 182/234 -- GLOBAL_STEP: 67340 + | > loss: -0.34781 (-0.31336) + | > log_mle: -0.59460 (-0.44503) + | > loss_dur: 0.24678 (0.13167) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.04846 (25.45021) + | > current_lr: 0.00007 + | > step_time: 2.49740 (2.52935) + | > loader_time: 0.00350 (0.03543) + + + --> STEP: 187/234 -- GLOBAL_STEP: 67345 + | > loss: -0.38662 (-0.31491) + | > log_mle: -0.60691 (-0.44895) + | > loss_dur: 0.22029 (0.13404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.21965 (26.51230) + | > current_lr: 0.00007 + | > step_time: 4.00100 (2.55342) + | > loader_time: 0.28740 (0.03711) + + + --> STEP: 192/234 -- GLOBAL_STEP: 67350 + | > loss: -0.42384 (-0.31687) + | > log_mle: -0.63902 (-0.45298) + | > loss_dur: 0.21518 (0.13610) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.52721 (27.11574) + | > current_lr: 0.00007 + | > step_time: 4.61080 (2.63345) + | > loader_time: 0.08250 (0.03917) + + + --> STEP: 197/234 -- GLOBAL_STEP: 67355 + | > loss: -0.41367 (-0.31902) + | > log_mle: -0.60662 (-0.45705) + | > loss_dur: 0.19295 (0.13803) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.15415 (27.88268) + | > current_lr: 0.00007 + | > step_time: 2.00040 (2.65546) + | > loader_time: 0.00320 (0.03873) + + + --> STEP: 202/234 -- GLOBAL_STEP: 67360 + | > loss: -0.49750 (-0.32116) + | > log_mle: -0.71835 (-0.46130) + | > loss_dur: 0.22085 (0.14014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.69451 (28.74503) + | > current_lr: 0.00007 + | > step_time: 3.59350 (2.74067) + | > loader_time: 0.00320 (0.03881) + + + --> STEP: 207/234 -- GLOBAL_STEP: 67365 + | > loss: -0.46610 (-0.32349) + | > log_mle: -0.70530 (-0.46568) + | > loss_dur: 0.23920 (0.14219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.20853 (29.69942) + | > current_lr: 0.00007 + | > step_time: 4.38840 (2.82621) + | > loader_time: 0.40260 (0.04793) + + + --> STEP: 212/234 -- GLOBAL_STEP: 67370 + | > loss: -0.45575 (-0.32647) + | > log_mle: -0.69150 (-0.47086) + | > loss_dur: 0.23575 (0.14439) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.00250 (30.85583) + | > current_lr: 0.00007 + | > step_time: 2.70350 (2.88699) + | > loader_time: 0.00340 (0.04965) + + + --> STEP: 217/234 -- GLOBAL_STEP: 67375 + | > loss: -0.48607 (-0.32973) + | > log_mle: -0.71976 (-0.47613) + | > loss_dur: 0.23369 (0.14640) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.47560 (31.92924) + | > current_lr: 0.00007 + | > step_time: 3.59790 (2.94032) + | > loader_time: 0.00290 (0.05030) + + + --> STEP: 222/234 -- GLOBAL_STEP: 67380 + | > loss: -0.46809 (-0.33294) + | > log_mle: -0.73345 (-0.48136) + | > loss_dur: 0.26536 (0.14843) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.83907 (33.13883) + | > current_lr: 0.00007 + | > step_time: 2.99950 (2.94982) + | > loader_time: 0.09800 (0.05085) + + + --> STEP: 227/234 -- GLOBAL_STEP: 67385 + | > loss: -0.44972 (-0.33634) + | > log_mle: -0.70307 (-0.48687) + | > loss_dur: 0.25335 (0.15053) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.91177 (34.58184) + | > current_lr: 0.00007 + | > step_time: 1.78800 (2.92444) + | > loader_time: 0.00310 (0.05054) + + + --> STEP: 232/234 -- GLOBAL_STEP: 67390 + | > loss: -0.46082 (-0.33922) + | > log_mle: -0.93956 (-0.49400) + | > loss_dur: 0.47874 (0.15478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.84754 (36.21985) + | > current_lr: 0.00007 + | > step_time: 0.33950 (2.86865) + | > loader_time: 0.11130 (0.05034) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.96917 (+0.61959) + | > avg_loss: -0.33752 (-0.00701) + | > avg_log_mle: -0.56633 (-0.00901) + | > avg_loss_dur: 0.22881 (+0.00199) + + + > EPOCH: 288/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 21:53:11)  + + --> STEP: 3/234 -- GLOBAL_STEP: 67395 + | > loss: -0.26492 (-0.31632) + | > log_mle: -0.40559 (-0.41589) + | > loss_dur: 0.14067 (0.09957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.82053 (19.56746) + | > current_lr: 0.00007 + | > step_time: 1.50090 (3.56363) + | > loader_time: 0.00140 (0.00224) + + + --> STEP: 8/234 -- GLOBAL_STEP: 67400 + | > loss: -0.35286 (-0.33322) + | > log_mle: -0.43421 (-0.41824) + | > loss_dur: 0.08134 (0.08502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.24492 (18.33155) + | > current_lr: 0.00007 + | > step_time: 1.80620 (4.51290) + | > loader_time: 0.09060 (0.08455) + + + --> STEP: 13/234 -- GLOBAL_STEP: 67405 + | > loss: -0.37346 (-0.33657) + | > log_mle: -0.44127 (-0.42066) + | > loss_dur: 0.06781 (0.08408) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.69551 (17.53631) + | > current_lr: 0.00007 + | > step_time: 9.12540 (4.52437) + | > loader_time: 0.78380 (0.13357) + + + --> STEP: 18/234 -- GLOBAL_STEP: 67410 + | > loss: -0.34085 (-0.34305) + | > log_mle: -0.40975 (-0.42190) + | > loss_dur: 0.06890 (0.07885) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.78722 (16.04634) + | > current_lr: 0.00007 + | > step_time: 0.88610 (3.67205) + | > loader_time: 0.00160 (0.10692) + + + --> STEP: 23/234 -- GLOBAL_STEP: 67415 + | > loss: -0.37709 (-0.34641) + | > log_mle: -0.43853 (-0.42266) + | > loss_dur: 0.06144 (0.07626) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.62918 (14.83001) + | > current_lr: 0.00007 + | > step_time: 3.00000 (3.52677) + | > loader_time: 0.00170 (0.08830) + + + --> STEP: 28/234 -- GLOBAL_STEP: 67420 + | > loss: -0.40039 (-0.34760) + | > log_mle: -0.44851 (-0.42209) + | > loss_dur: 0.04811 (0.07449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.69297 (13.74200) + | > current_lr: 0.00007 + | > step_time: 15.60830 (3.96504) + | > loader_time: 0.19670 (0.08365) + + + --> STEP: 33/234 -- GLOBAL_STEP: 67425 + | > loss: -0.33783 (-0.34631) + | > log_mle: -0.40674 (-0.42022) + | > loss_dur: 0.06891 (0.07391) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.48500 (13.04843) + | > current_lr: 0.00007 + | > step_time: 6.29880 (3.97110) + | > loader_time: 0.10580 (0.08016) + + + --> STEP: 38/234 -- GLOBAL_STEP: 67430 + | > loss: -0.31264 (-0.34252) + | > log_mle: -0.40093 (-0.41692) + | > loss_dur: 0.08829 (0.07440) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.59238 (12.67982) + | > current_lr: 0.00007 + | > step_time: 3.90500 (3.75663) + | > loader_time: 0.10480 (0.07942) + + + --> STEP: 43/234 -- GLOBAL_STEP: 67435 + | > loss: -0.31217 (-0.33985) + | > log_mle: -0.39521 (-0.41454) + | > loss_dur: 0.08303 (0.07469) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.54803 (12.22667) + | > current_lr: 0.00007 + | > step_time: 1.60430 (3.55507) + | > loader_time: 0.08140 (0.07666) + + + --> STEP: 48/234 -- GLOBAL_STEP: 67440 + | > loss: -0.34204 (-0.33788) + | > log_mle: -0.40985 (-0.41305) + | > loss_dur: 0.06781 (0.07517) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.52516 (12.03419) + | > current_lr: 0.00007 + | > step_time: 1.38500 (3.38713) + | > loader_time: 0.00110 (0.07074) + + + --> STEP: 53/234 -- GLOBAL_STEP: 67445 + | > loss: -0.30404 (-0.33543) + | > log_mle: -0.39460 (-0.41128) + | > loss_dur: 0.09056 (0.07585) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.49459 (11.94400) + | > current_lr: 0.00007 + | > step_time: 1.99970 (3.20539) + | > loader_time: 0.00500 (0.06751) + + + --> STEP: 58/234 -- GLOBAL_STEP: 67450 + | > loss: -0.32374 (-0.33324) + | > log_mle: -0.39527 (-0.40940) + | > loss_dur: 0.07153 (0.07616) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.04054 (11.97390) + | > current_lr: 0.00007 + | > step_time: 3.49480 (3.13410) + | > loader_time: 0.00460 (0.06193) + + + --> STEP: 63/234 -- GLOBAL_STEP: 67455 + | > loss: -0.28674 (-0.32907) + | > log_mle: -0.37593 (-0.40757) + | > loss_dur: 0.08919 (0.07850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.30009 (12.25902) + | > current_lr: 0.00007 + | > step_time: 1.17910 (3.07847) + | > loader_time: 0.00180 (0.06002) + + + --> STEP: 68/234 -- GLOBAL_STEP: 67460 + | > loss: -0.27353 (-0.32645) + | > log_mle: -0.37523 (-0.40578) + | > loss_dur: 0.10170 (0.07933) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.44228 (12.03563) + | > current_lr: 0.00007 + | > step_time: 2.40920 (2.99038) + | > loader_time: 0.08590 (0.05702) + + + --> STEP: 73/234 -- GLOBAL_STEP: 67465 + | > loss: -0.25376 (-0.32296) + | > log_mle: -0.38270 (-0.40415) + | > loss_dur: 0.12894 (0.08119) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.39153 (12.05789) + | > current_lr: 0.00007 + | > step_time: 5.19580 (3.02065) + | > loader_time: 0.00400 (0.06022) + + + --> STEP: 78/234 -- GLOBAL_STEP: 67470 + | > loss: -0.27433 (-0.31981) + | > log_mle: -0.37251 (-0.40257) + | > loss_dur: 0.09818 (0.08277) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.38415 (11.97579) + | > current_lr: 0.00007 + | > step_time: 2.01860 (2.93120) + | > loader_time: 0.07410 (0.05842) + + + --> STEP: 83/234 -- GLOBAL_STEP: 67475 + | > loss: -0.24497 (-0.31712) + | > log_mle: -0.38078 (-0.40126) + | > loss_dur: 0.13582 (0.08413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.41895 (12.02683) + | > current_lr: 0.00007 + | > step_time: 0.99890 (2.84941) + | > loader_time: 0.00380 (0.05634) + + + --> STEP: 88/234 -- GLOBAL_STEP: 67480 + | > loss: -0.27104 (-0.31462) + | > log_mle: -0.41017 (-0.40032) + | > loss_dur: 0.13913 (0.08570) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.10461 (12.19677) + | > current_lr: 0.00007 + | > step_time: 1.99940 (2.81479) + | > loader_time: 0.00160 (0.05418) + + + --> STEP: 93/234 -- GLOBAL_STEP: 67485 + | > loss: -0.27245 (-0.31255) + | > log_mle: -0.42250 (-0.40053) + | > loss_dur: 0.15004 (0.08798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.13559 (12.51485) + | > current_lr: 0.00007 + | > step_time: 1.69550 (2.76336) + | > loader_time: 0.00230 (0.05229) + + + --> STEP: 98/234 -- GLOBAL_STEP: 67490 + | > loss: -0.26333 (-0.31122) + | > log_mle: -0.36600 (-0.40099) + | > loss_dur: 0.10266 (0.08977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.30509 (12.66836) + | > current_lr: 0.00007 + | > step_time: 1.70620 (2.70514) + | > loader_time: 0.00200 (0.04984) + + + --> STEP: 103/234 -- GLOBAL_STEP: 67495 + | > loss: -0.28770 (-0.30946) + | > log_mle: -0.44570 (-0.40209) + | > loss_dur: 0.15800 (0.09263) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.62321 (13.20954) + | > current_lr: 0.00007 + | > step_time: 3.19110 (2.71820) + | > loader_time: 0.00210 (0.05018) + + + --> STEP: 108/234 -- GLOBAL_STEP: 67500 + | > loss: -0.26559 (-0.30756) + | > log_mle: -0.39532 (-0.40262) + | > loss_dur: 0.12973 (0.09507) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.55141 (13.68832) + | > current_lr: 0.00007 + | > step_time: 2.30440 (2.67923) + | > loader_time: 0.07800 (0.04945) + + + --> STEP: 113/234 -- GLOBAL_STEP: 67505 + | > loss: -0.28168 (-0.30586) + | > log_mle: -0.43846 (-0.40411) + | > loss_dur: 0.15678 (0.09825) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.84135 (14.22641) + | > current_lr: 0.00007 + | > step_time: 4.10230 (2.66282) + | > loader_time: 0.10440 (0.05060) + + + --> STEP: 118/234 -- GLOBAL_STEP: 67510 + | > loss: -0.26097 (-0.30433) + | > log_mle: -0.41291 (-0.40502) + | > loss_dur: 0.15194 (0.10069) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.17582 (14.46832) + | > current_lr: 0.00007 + | > step_time: 2.59970 (2.65074) + | > loader_time: 0.19700 (0.05265) + + + --> STEP: 123/234 -- GLOBAL_STEP: 67515 + | > loss: -0.23193 (-0.30246) + | > log_mle: -0.38005 (-0.40505) + | > loss_dur: 0.14812 (0.10258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.93624 (14.60993) + | > current_lr: 0.00007 + | > step_time: 2.19180 (2.64705) + | > loader_time: 0.00200 (0.05195) + + + --> STEP: 128/234 -- GLOBAL_STEP: 67520 + | > loss: -0.29627 (-0.30215) + | > log_mle: -0.43863 (-0.40710) + | > loss_dur: 0.14236 (0.10496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.45627 (15.15481) + | > current_lr: 0.00007 + | > step_time: 3.29350 (2.61872) + | > loader_time: 0.11760 (0.05222) + + + --> STEP: 133/234 -- GLOBAL_STEP: 67525 + | > loss: -0.30357 (-0.30222) + | > log_mle: -0.47083 (-0.40958) + | > loss_dur: 0.16727 (0.10735) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.33406 (15.98632) + | > current_lr: 0.00007 + | > step_time: 2.39310 (2.61719) + | > loader_time: 0.00390 (0.05176) + + + --> STEP: 138/234 -- GLOBAL_STEP: 67530 + | > loss: -0.25015 (-0.30201) + | > log_mle: -0.41294 (-0.41175) + | > loss_dur: 0.16279 (0.10975) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.14417 (16.68027) + | > current_lr: 0.00007 + | > step_time: 3.19910 (2.60950) + | > loader_time: 0.09290 (0.05257) + + + --> STEP: 143/234 -- GLOBAL_STEP: 67535 + | > loss: -0.34288 (-0.30247) + | > log_mle: -0.57330 (-0.41496) + | > loss_dur: 0.23042 (0.11249) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.90107 (17.55800) + | > current_lr: 0.00007 + | > step_time: 1.90300 (2.57842) + | > loader_time: 0.00170 (0.05191) + + + --> STEP: 148/234 -- GLOBAL_STEP: 67540 + | > loss: -0.32346 (-0.30353) + | > log_mle: -0.47683 (-0.41824) + | > loss_dur: 0.15337 (0.11471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.82767 (18.27726) + | > current_lr: 0.00007 + | > step_time: 3.01410 (2.57116) + | > loader_time: 0.09240 (0.05264) + + + --> STEP: 153/234 -- GLOBAL_STEP: 67545 + | > loss: -0.42447 (-0.30572) + | > log_mle: -0.61598 (-0.42284) + | > loss_dur: 0.19151 (0.11712) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.27353 (19.16866) + | > current_lr: 0.00007 + | > step_time: 2.79150 (2.58969) + | > loader_time: 0.09850 (0.05289) + + + --> STEP: 158/234 -- GLOBAL_STEP: 67550 + | > loss: -0.33552 (-0.30724) + | > log_mle: -0.54395 (-0.42672) + | > loss_dur: 0.20843 (0.11948) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.25790 (20.10805) + | > current_lr: 0.00007 + | > step_time: 7.69920 (2.66611) + | > loader_time: 0.08650 (0.05297) + + + --> STEP: 163/234 -- GLOBAL_STEP: 67555 + | > loss: -0.32590 (-0.30899) + | > log_mle: -0.51383 (-0.43070) + | > loss_dur: 0.18793 (0.12170) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.14058 (20.98102) + | > current_lr: 0.00007 + | > step_time: 2.38720 (2.64071) + | > loader_time: 0.00470 (0.05243) + + + --> STEP: 168/234 -- GLOBAL_STEP: 67560 + | > loss: -0.35647 (-0.31103) + | > log_mle: -0.56891 (-0.43492) + | > loss_dur: 0.21244 (0.12389) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.43045 (22.02335) + | > current_lr: 0.00007 + | > step_time: 1.12030 (2.71483) + | > loader_time: 0.01680 (0.05447) + + + --> STEP: 173/234 -- GLOBAL_STEP: 67565 + | > loss: -0.37115 (-0.31332) + | > log_mle: -0.58615 (-0.43977) + | > loss_dur: 0.21499 (0.12645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.23225 (23.00482) + | > current_lr: 0.00007 + | > step_time: 5.29270 (2.79873) + | > loader_time: 0.00230 (0.05412) + + + --> STEP: 178/234 -- GLOBAL_STEP: 67570 + | > loss: -0.40802 (-0.31565) + | > log_mle: -0.63501 (-0.44456) + | > loss_dur: 0.22699 (0.12891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.90308 (24.00929) + | > current_lr: 0.00007 + | > step_time: 5.19720 (2.83932) + | > loader_time: 0.08910 (0.05470) + + + --> STEP: 183/234 -- GLOBAL_STEP: 67575 + | > loss: -0.43418 (-0.31757) + | > log_mle: -0.64109 (-0.44897) + | > loss_dur: 0.20692 (0.13139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.19372 (24.98172) + | > current_lr: 0.00007 + | > step_time: 1.49500 (2.81107) + | > loader_time: 0.00360 (0.05418) + + + --> STEP: 188/234 -- GLOBAL_STEP: 67580 + | > loss: -0.41349 (-0.31901) + | > log_mle: -0.62700 (-0.45267) + | > loss_dur: 0.21352 (0.13366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.07427 (26.20924) + | > current_lr: 0.00007 + | > step_time: 2.00840 (2.81506) + | > loader_time: 0.08480 (0.05383) + + + --> STEP: 193/234 -- GLOBAL_STEP: 67585 + | > loss: -0.41758 (-0.32075) + | > log_mle: -0.63239 (-0.45639) + | > loss_dur: 0.21481 (0.13564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.17539 (26.73254) + | > current_lr: 0.00007 + | > step_time: 2.72710 (2.83658) + | > loader_time: 0.07400 (0.05474) + + + --> STEP: 198/234 -- GLOBAL_STEP: 67590 + | > loss: -0.40633 (-0.32238) + | > log_mle: -0.62711 (-0.46017) + | > loss_dur: 0.22078 (0.13779) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.86468 (27.53960) + | > current_lr: 0.00007 + | > step_time: 2.71050 (2.83990) + | > loader_time: 0.00330 (0.05439) + + + --> STEP: 203/234 -- GLOBAL_STEP: 67595 + | > loss: -0.34711 (-0.32402) + | > log_mle: -0.55493 (-0.46380) + | > loss_dur: 0.20782 (0.13979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.37833 (28.30343) + | > current_lr: 0.00007 + | > step_time: 2.82030 (2.84646) + | > loader_time: 0.07690 (0.05523) + + + --> STEP: 208/234 -- GLOBAL_STEP: 67600 + | > loss: -0.41696 (-0.32642) + | > log_mle: -0.64718 (-0.46833) + | > loss_dur: 0.23022 (0.14191) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.54590 (28.94456) + | > current_lr: 0.00007 + | > step_time: 3.30590 (2.85500) + | > loader_time: 0.08430 (0.05482) + + + --> STEP: 213/234 -- GLOBAL_STEP: 67605 + | > loss: -0.45764 (-0.32936) + | > log_mle: -0.69299 (-0.47335) + | > loss_dur: 0.23534 (0.14399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.21687 (29.91758) + | > current_lr: 0.00007 + | > step_time: 7.89920 (2.97713) + | > loader_time: 0.09710 (0.05592) + + + --> STEP: 218/234 -- GLOBAL_STEP: 67610 + | > loss: -0.42641 (-0.33209) + | > log_mle: -0.66431 (-0.47813) + | > loss_dur: 0.23790 (0.14604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.70957 (30.69702) + | > current_lr: 0.00007 + | > step_time: 2.60610 (3.02262) + | > loader_time: 0.08770 (0.05598) + + + --> STEP: 223/234 -- GLOBAL_STEP: 67615 + | > loss: -0.48126 (-0.33530) + | > log_mle: -0.70895 (-0.48332) + | > loss_dur: 0.22769 (0.14802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.99023 (31.65967) + | > current_lr: 0.00007 + | > step_time: 0.75500 (2.99375) + | > loader_time: 0.00630 (0.05561) + + + --> STEP: 228/234 -- GLOBAL_STEP: 67620 + | > loss: -0.44202 (-0.33846) + | > log_mle: -0.70285 (-0.48864) + | > loss_dur: 0.26084 (0.15018) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.20040 (32.87176) + | > current_lr: 0.00007 + | > step_time: 0.23690 (2.93314) + | > loader_time: 0.00270 (0.05445) + + + --> STEP: 233/234 -- GLOBAL_STEP: 67625 + | > loss: -0.03180 (-0.33937) + | > log_mle: -0.68576 (-0.49537) + | > loss_dur: 0.65396 (0.15600) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.30196 (34.21380) + | > current_lr: 0.00007 + | > step_time: 0.19530 (2.87579) + | > loader_time: 0.00280 (0.05336) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04716 (-0.92200) + | > avg_loss: -0.34390 (-0.00638) + | > avg_log_mle: -0.56019 (+0.00614) + | > avg_loss_dur: 0.21629 (-0.01252) + + + > EPOCH: 289/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 22:05:38)  + + --> STEP: 4/234 -- GLOBAL_STEP: 67630 + | > loss: -0.32019 (-0.31174) + | > log_mle: -0.41321 (-0.41463) + | > loss_dur: 0.09303 (0.10288) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.94191 (20.93148) + | > current_lr: 0.00007 + | > step_time: 4.31390 (8.30380) + | > loader_time: 0.10360 (0.07622) + + + --> STEP: 9/234 -- GLOBAL_STEP: 67635 + | > loss: -0.32102 (-0.32649) + | > log_mle: -0.42665 (-0.41925) + | > loss_dur: 0.10563 (0.09276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.58581 (18.93576) + | > current_lr: 0.00007 + | > step_time: 0.49700 (4.21207) + | > loader_time: 0.00100 (0.04458) + + + --> STEP: 14/234 -- GLOBAL_STEP: 67640 + | > loss: -0.35734 (-0.33654) + | > log_mle: -0.42395 (-0.42190) + | > loss_dur: 0.06661 (0.08537) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.85458 (17.83221) + | > current_lr: 0.00007 + | > step_time: 1.47990 (3.06768) + | > loader_time: 0.00130 (0.02914) + + + --> STEP: 19/234 -- GLOBAL_STEP: 67645 + | > loss: -0.36141 (-0.34411) + | > log_mle: -0.43087 (-0.42346) + | > loss_dur: 0.06946 (0.07935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.55735 (16.63349) + | > current_lr: 0.00007 + | > step_time: 1.28080 (2.55975) + | > loader_time: 0.00110 (0.02627) + + + --> STEP: 24/234 -- GLOBAL_STEP: 67650 + | > loss: -0.35176 (-0.34544) + | > log_mle: -0.41731 (-0.42298) + | > loss_dur: 0.06555 (0.07755) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.78537 (15.95116) + | > current_lr: 0.00007 + | > step_time: 1.27000 (2.27796) + | > loader_time: 0.00170 (0.02163) + + + --> STEP: 29/234 -- GLOBAL_STEP: 67655 + | > loss: -0.34521 (-0.34632) + | > log_mle: -0.40728 (-0.42155) + | > loss_dur: 0.06207 (0.07523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.02068 (15.00726) + | > current_lr: 0.00007 + | > step_time: 3.59540 (2.22000) + | > loader_time: 0.00210 (0.02106) + + + --> STEP: 34/234 -- GLOBAL_STEP: 67660 + | > loss: -0.31198 (-0.34255) + | > log_mle: -0.39025 (-0.41797) + | > loss_dur: 0.07827 (0.07542) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.22978 (14.61116) + | > current_lr: 0.00007 + | > step_time: 1.17940 (2.47738) + | > loader_time: 0.00150 (0.02675) + + + --> STEP: 39/234 -- GLOBAL_STEP: 67665 + | > loss: -0.29833 (-0.33710) + | > log_mle: -0.38379 (-0.41349) + | > loss_dur: 0.08546 (0.07639) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.84965 (14.57279) + | > current_lr: 0.00007 + | > step_time: 1.22950 (2.36217) + | > loader_time: 0.00190 (0.02545) + + + --> STEP: 44/234 -- GLOBAL_STEP: 67670 + | > loss: -0.31355 (-0.33390) + | > log_mle: -0.38148 (-0.41063) + | > loss_dur: 0.06793 (0.07673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.46813 (13.97156) + | > current_lr: 0.00007 + | > step_time: 1.71250 (2.22577) + | > loader_time: 0.00140 (0.02277) + + + --> STEP: 49/234 -- GLOBAL_STEP: 67675 + | > loss: -0.32586 (-0.33184) + | > log_mle: -0.40030 (-0.40918) + | > loss_dur: 0.07445 (0.07734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.52361 (13.51432) + | > current_lr: 0.00007 + | > step_time: 1.21630 (2.12531) + | > loader_time: 0.00180 (0.02063) + + + --> STEP: 54/234 -- GLOBAL_STEP: 67680 + | > loss: -0.31459 (-0.32945) + | > log_mle: -0.39001 (-0.40723) + | > loss_dur: 0.07542 (0.07778) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47501 (13.00555) + | > current_lr: 0.00007 + | > step_time: 1.62650 (2.08354) + | > loader_time: 0.00220 (0.02065) + + + --> STEP: 59/234 -- GLOBAL_STEP: 67685 + | > loss: -0.29802 (-0.32715) + | > log_mle: -0.39008 (-0.40559) + | > loss_dur: 0.09206 (0.07843) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.37134 (12.71224) + | > current_lr: 0.00007 + | > step_time: 1.52720 (2.06928) + | > loader_time: 0.00170 (0.02181) + + + --> STEP: 64/234 -- GLOBAL_STEP: 67690 + | > loss: -0.30666 (-0.32361) + | > log_mle: -0.38573 (-0.40408) + | > loss_dur: 0.07907 (0.08047) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.66712 (12.64986) + | > current_lr: 0.00007 + | > step_time: 1.40070 (2.04693) + | > loader_time: 0.00200 (0.02434) + + + --> STEP: 69/234 -- GLOBAL_STEP: 67695 + | > loss: -0.29305 (-0.32116) + | > log_mle: -0.37560 (-0.40250) + | > loss_dur: 0.08254 (0.08134) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.64141 (12.49490) + | > current_lr: 0.00007 + | > step_time: 0.81300 (1.99605) + | > loader_time: 0.00300 (0.02278) + + + --> STEP: 74/234 -- GLOBAL_STEP: 67700 + | > loss: -0.27131 (-0.31785) + | > log_mle: -0.37026 (-0.40092) + | > loss_dur: 0.09895 (0.08307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.65717 (12.50084) + | > current_lr: 0.00007 + | > step_time: 1.11490 (1.99048) + | > loader_time: 0.00180 (0.02139) + + + --> STEP: 79/234 -- GLOBAL_STEP: 67705 + | > loss: -0.29117 (-0.31556) + | > log_mle: -0.38633 (-0.39976) + | > loss_dur: 0.09517 (0.08421) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.46306 (12.38787) + | > current_lr: 0.00007 + | > step_time: 1.29480 (1.99088) + | > loader_time: 0.10500 (0.02384) + + + --> STEP: 84/234 -- GLOBAL_STEP: 67710 + | > loss: -0.28749 (-0.31309) + | > log_mle: -0.38132 (-0.39856) + | > loss_dur: 0.09384 (0.08547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.06662 (12.37891) + | > current_lr: 0.00007 + | > step_time: 1.99360 (1.99383) + | > loader_time: 0.00410 (0.02379) + + + --> STEP: 89/234 -- GLOBAL_STEP: 67715 + | > loss: -0.28579 (-0.31107) + | > log_mle: -0.40247 (-0.39815) + | > loss_dur: 0.11669 (0.08707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.69658 (12.40094) + | > current_lr: 0.00007 + | > step_time: 1.13350 (2.04355) + | > loader_time: 0.00210 (0.02564) + + + --> STEP: 94/234 -- GLOBAL_STEP: 67720 + | > loss: -0.29037 (-0.30934) + | > log_mle: -0.42517 (-0.39881) + | > loss_dur: 0.13480 (0.08947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.55889 (12.75146) + | > current_lr: 0.00007 + | > step_time: 3.41570 (2.05863) + | > loader_time: 0.08850 (0.02533) + + + --> STEP: 99/234 -- GLOBAL_STEP: 67725 + | > loss: -0.28608 (-0.30805) + | > log_mle: -0.45360 (-0.39964) + | > loss_dur: 0.16752 (0.09159) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.33765 (13.36206) + | > current_lr: 0.00007 + | > step_time: 2.23110 (2.09406) + | > loader_time: 0.00420 (0.02504) + + + --> STEP: 104/234 -- GLOBAL_STEP: 67730 + | > loss: -0.31976 (-0.30686) + | > log_mle: -0.46734 (-0.40098) + | > loss_dur: 0.14758 (0.09412) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.57106 (13.97924) + | > current_lr: 0.00007 + | > step_time: 1.08430 (2.09304) + | > loader_time: 0.00330 (0.02397) + + + --> STEP: 109/234 -- GLOBAL_STEP: 67735 + | > loss: -0.25688 (-0.30474) + | > log_mle: -0.44289 (-0.40156) + | > loss_dur: 0.18600 (0.09682) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.24734 (14.32431) + | > current_lr: 0.00007 + | > step_time: 1.80110 (2.08786) + | > loader_time: 0.00130 (0.02392) + + + --> STEP: 114/234 -- GLOBAL_STEP: 67740 + | > loss: -0.26794 (-0.30362) + | > log_mle: -0.41898 (-0.40312) + | > loss_dur: 0.15104 (0.09950) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.32242 (14.88325) + | > current_lr: 0.00007 + | > step_time: 3.10210 (2.14364) + | > loader_time: 0.09670 (0.02553) + + + --> STEP: 119/234 -- GLOBAL_STEP: 67745 + | > loss: -0.27689 (-0.30228) + | > log_mle: -0.42176 (-0.40417) + | > loss_dur: 0.14487 (0.10189) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.60117 (15.20933) + | > current_lr: 0.00007 + | > step_time: 4.30920 (2.15033) + | > loader_time: 0.11000 (0.02624) + + + --> STEP: 124/234 -- GLOBAL_STEP: 67750 + | > loss: -0.28821 (-0.30080) + | > log_mle: -0.44447 (-0.40448) + | > loss_dur: 0.15626 (0.10368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.88480 (15.52944) + | > current_lr: 0.00007 + | > step_time: 3.79650 (2.16679) + | > loader_time: 0.00540 (0.02666) + + + --> STEP: 129/234 -- GLOBAL_STEP: 67755 + | > loss: -0.26111 (-0.30024) + | > log_mle: -0.44194 (-0.40661) + | > loss_dur: 0.18083 (0.10637) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.22734 (16.02882) + | > current_lr: 0.00007 + | > step_time: 2.50630 (2.16728) + | > loader_time: 0.09910 (0.02844) + + + --> STEP: 134/234 -- GLOBAL_STEP: 67760 + | > loss: -0.30752 (-0.30071) + | > log_mle: -0.49702 (-0.40961) + | > loss_dur: 0.18949 (0.10890) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.27085 (16.75969) + | > current_lr: 0.00007 + | > step_time: 2.20210 (2.16468) + | > loader_time: 0.00650 (0.02897) + + + --> STEP: 139/234 -- GLOBAL_STEP: 67765 + | > loss: -0.36196 (-0.30088) + | > log_mle: -0.54908 (-0.41219) + | > loss_dur: 0.18712 (0.11130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.71724 (17.53077) + | > current_lr: 0.00007 + | > step_time: 2.20250 (2.16883) + | > loader_time: 0.07620 (0.02965) + + + --> STEP: 144/234 -- GLOBAL_STEP: 67770 + | > loss: -0.34151 (-0.30132) + | > log_mle: -0.53133 (-0.41515) + | > loss_dur: 0.18982 (0.11383) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.70367 (18.29301) + | > current_lr: 0.00007 + | > step_time: 1.30600 (2.15174) + | > loader_time: 0.00270 (0.02933) + + + --> STEP: 149/234 -- GLOBAL_STEP: 67775 + | > loss: -0.39907 (-0.30284) + | > log_mle: -0.58381 (-0.41876) + | > loss_dur: 0.18474 (0.11592) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.83442 (19.09778) + | > current_lr: 0.00007 + | > step_time: 3.41370 (2.16610) + | > loader_time: 0.09160 (0.02961) + + + --> STEP: 154/234 -- GLOBAL_STEP: 67780 + | > loss: -0.36780 (-0.30471) + | > log_mle: -0.54350 (-0.42295) + | > loss_dur: 0.17570 (0.11824) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.36323 (19.99697) + | > current_lr: 0.00007 + | > step_time: 2.20410 (2.20041) + | > loader_time: 0.19440 (0.03064) + + + --> STEP: 159/234 -- GLOBAL_STEP: 67785 + | > loss: -0.35746 (-0.30607) + | > log_mle: -0.55249 (-0.42666) + | > loss_dur: 0.19503 (0.12059) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.71733 (21.20635) + | > current_lr: 0.00007 + | > step_time: 1.60110 (2.25685) + | > loader_time: 0.08750 (0.03385) + + + --> STEP: 164/234 -- GLOBAL_STEP: 67790 + | > loss: -0.32941 (-0.30728) + | > log_mle: -0.53077 (-0.43014) + | > loss_dur: 0.20136 (0.12286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.41376 (22.16662) + | > current_lr: 0.00007 + | > step_time: 5.89870 (2.30810) + | > loader_time: 0.00830 (0.03716) + + + --> STEP: 169/234 -- GLOBAL_STEP: 67795 + | > loss: -0.34380 (-0.30892) + | > log_mle: -0.54086 (-0.43397) + | > loss_dur: 0.19706 (0.12505) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.12114 (22.82635) + | > current_lr: 0.00007 + | > step_time: 2.10150 (2.35472) + | > loader_time: 0.00400 (0.03664) + + + --> STEP: 174/234 -- GLOBAL_STEP: 67800 + | > loss: -0.41637 (-0.31145) + | > log_mle: -0.62397 (-0.43894) + | > loss_dur: 0.20760 (0.12749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.89093 (23.70785) + | > current_lr: 0.00007 + | > step_time: 5.29700 (2.39225) + | > loader_time: 0.19710 (0.03774) + + + --> STEP: 179/234 -- GLOBAL_STEP: 67805 + | > loss: -0.38667 (-0.31340) + | > log_mle: -0.62302 (-0.44354) + | > loss_dur: 0.23634 (0.13014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.33636 (24.47610) + | > current_lr: 0.00007 + | > step_time: 3.50690 (2.42212) + | > loader_time: 0.00480 (0.03738) + + + --> STEP: 184/234 -- GLOBAL_STEP: 67810 + | > loss: -0.36539 (-0.31484) + | > log_mle: -0.58375 (-0.44738) + | > loss_dur: 0.21836 (0.13254) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.16907 (25.51745) + | > current_lr: 0.00007 + | > step_time: 4.00160 (2.49490) + | > loader_time: 0.10560 (0.04032) + + + --> STEP: 189/234 -- GLOBAL_STEP: 67815 + | > loss: -0.38187 (-0.31677) + | > log_mle: -0.59510 (-0.45165) + | > loss_dur: 0.21323 (0.13487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.33382 (26.13242) + | > current_lr: 0.00007 + | > step_time: 3.41530 (2.57652) + | > loader_time: 0.09530 (0.04143) + + + --> STEP: 194/234 -- GLOBAL_STEP: 67820 + | > loss: -0.43118 (-0.31945) + | > log_mle: -0.63612 (-0.45614) + | > loss_dur: 0.20495 (0.13669) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.34348 (26.98560) + | > current_lr: 0.00007 + | > step_time: 2.79180 (2.57035) + | > loader_time: 0.00240 (0.04177) + + + --> STEP: 199/234 -- GLOBAL_STEP: 67825 + | > loss: -0.42633 (-0.32167) + | > log_mle: -0.64130 (-0.46035) + | > loss_dur: 0.21498 (0.13868) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.65187 (27.80668) + | > current_lr: 0.00007 + | > step_time: 6.00440 (2.60725) + | > loader_time: 0.08390 (0.04166) + + + --> STEP: 204/234 -- GLOBAL_STEP: 67830 + | > loss: -0.43728 (-0.32368) + | > log_mle: -0.67425 (-0.46447) + | > loss_dur: 0.23697 (0.14078) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 106.80108 (28.94674) + | > current_lr: 0.00007 + | > step_time: 3.79410 (2.68999) + | > loader_time: 0.00740 (0.04446) + + + --> STEP: 209/234 -- GLOBAL_STEP: 67835 + | > loss: -0.42973 (-0.32616) + | > log_mle: -0.63555 (-0.46884) + | > loss_dur: 0.20582 (0.14268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.54551 (30.33203) + | > current_lr: 0.00007 + | > step_time: 3.09610 (2.79890) + | > loader_time: 0.08600 (0.04573) + + + --> STEP: 214/234 -- GLOBAL_STEP: 67840 + | > loss: -0.46077 (-0.32955) + | > log_mle: -0.66690 (-0.47428) + | > loss_dur: 0.20612 (0.14473) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.65525 (31.37283) + | > current_lr: 0.00007 + | > step_time: 8.41360 (2.84775) + | > loader_time: 0.00540 (0.04599) + + + --> STEP: 219/234 -- GLOBAL_STEP: 67845 + | > loss: -0.55578 (-0.33291) + | > log_mle: -0.78808 (-0.47969) + | > loss_dur: 0.23231 (0.14678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.34761 (32.39939) + | > current_lr: 0.00007 + | > step_time: 11.00980 (2.91518) + | > loader_time: 0.09720 (0.04670) + + + --> STEP: 224/234 -- GLOBAL_STEP: 67850 + | > loss: -0.47689 (-0.33556) + | > log_mle: -0.70808 (-0.48435) + | > loss_dur: 0.23118 (0.14879) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.09093 (33.39251) + | > current_lr: 0.00007 + | > step_time: 0.23760 (2.87618) + | > loader_time: 0.00600 (0.04608) + + + --> STEP: 229/234 -- GLOBAL_STEP: 67855 + | > loss: -0.46581 (-0.33867) + | > log_mle: -0.75697 (-0.48987) + | > loss_dur: 0.29116 (0.15120) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 147.92323 (34.70958) + | > current_lr: 0.00007 + | > step_time: 0.25960 (2.81877) + | > loader_time: 0.00380 (0.04514) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.12049 (+0.07332) + | > avg_loss: -0.35205 (-0.00815) + | > avg_log_mle: -0.56873 (-0.00854) + | > avg_loss_dur: 0.21668 (+0.00039) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_67860.pth + + > EPOCH: 290/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 22:17:39)  + + --> STEP: 0/234 -- GLOBAL_STEP: 67860 + | > loss: -0.34672 (-0.34672) + | > log_mle: -0.49739 (-0.49739) + | > loss_dur: 0.15067 (0.15067) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.44435 (23.44435) + | > current_lr: 0.00007 + | > step_time: 2.19560 (2.19563) + | > loader_time: 10.11270 (10.11265) + + + --> STEP: 5/234 -- GLOBAL_STEP: 67865 + | > loss: -0.32058 (-0.31705) + | > log_mle: -0.41728 (-0.41704) + | > loss_dur: 0.09670 (0.09999) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.96587 (20.21902) + | > current_lr: 0.00007 + | > step_time: 2.40990 (1.58125) + | > loader_time: 0.00170 (2.57798) + + + --> STEP: 10/234 -- GLOBAL_STEP: 67870 + | > loss: -0.33925 (-0.33088) + | > log_mle: -0.42007 (-0.42077) + | > loss_dur: 0.08082 (0.08989) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.00299 (20.08829) + | > current_lr: 0.00007 + | > step_time: 6.60460 (3.32101) + | > loader_time: 0.00160 (1.37775) + + + --> STEP: 15/234 -- GLOBAL_STEP: 67875 + | > loss: -0.36640 (-0.34113) + | > log_mle: -0.42948 (-0.42384) + | > loss_dur: 0.06307 (0.08271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.17392 (19.03623) + | > current_lr: 0.00007 + | > step_time: 6.01010 (4.04816) + | > loader_time: 0.00430 (0.95105) + + + --> STEP: 20/234 -- GLOBAL_STEP: 67880 + | > loss: -0.36010 (-0.34668) + | > log_mle: -0.42724 (-0.42490) + | > loss_dur: 0.06713 (0.07822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.12241 (17.18315) + | > current_lr: 0.00007 + | > step_time: 0.98260 (3.77096) + | > loader_time: 0.00100 (0.72390) + + + --> STEP: 25/234 -- GLOBAL_STEP: 67885 + | > loss: -0.34273 (-0.34746) + | > log_mle: -0.41558 (-0.42403) + | > loss_dur: 0.07285 (0.07656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.90705 (15.79558) + | > current_lr: 0.00007 + | > step_time: 1.11350 (3.54085) + | > loader_time: 0.00410 (0.58350) + + + --> STEP: 30/234 -- GLOBAL_STEP: 67890 + | > loss: -0.34964 (-0.34907) + | > log_mle: -0.41719 (-0.42348) + | > loss_dur: 0.06755 (0.07441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.05109 (14.69590) + | > current_lr: 0.00007 + | > step_time: 2.48900 (3.22597) + | > loader_time: 0.00620 (0.49212) + + + --> STEP: 35/234 -- GLOBAL_STEP: 67895 + | > loss: -0.29747 (-0.34545) + | > log_mle: -0.39296 (-0.42055) + | > loss_dur: 0.09550 (0.07510) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.56544 (14.21017) + | > current_lr: 0.00007 + | > step_time: 2.69270 (3.27638) + | > loader_time: 0.00160 (0.42990) + + + --> STEP: 40/234 -- GLOBAL_STEP: 67900 + | > loss: -0.30905 (-0.34111) + | > log_mle: -0.39249 (-0.41696) + | > loss_dur: 0.08344 (0.07585) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.88018 (13.99174) + | > current_lr: 0.00007 + | > step_time: 0.96310 (3.10318) + | > loader_time: 0.00170 (0.37654) + + + --> STEP: 45/234 -- GLOBAL_STEP: 67905 + | > loss: -0.30469 (-0.33827) + | > log_mle: -0.40144 (-0.41451) + | > loss_dur: 0.09675 (0.07624) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.40868 (13.87608) + | > current_lr: 0.00007 + | > step_time: 1.71210 (2.87666) + | > loader_time: 0.07610 (0.33660) + + + --> STEP: 50/234 -- GLOBAL_STEP: 67910 + | > loss: -0.32647 (-0.33714) + | > log_mle: -0.39589 (-0.41318) + | > loss_dur: 0.06942 (0.07604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.35307 (13.45818) + | > current_lr: 0.00007 + | > step_time: 1.32070 (2.71822) + | > loader_time: 0.00130 (0.30314) + + + --> STEP: 55/234 -- GLOBAL_STEP: 67915 + | > loss: -0.32190 (-0.33468) + | > log_mle: -0.39800 (-0.41147) + | > loss_dur: 0.07610 (0.07678) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.10737 (13.09816) + | > current_lr: 0.00007 + | > step_time: 7.17430 (2.70481) + | > loader_time: 0.00170 (0.27743) + + + --> STEP: 60/234 -- GLOBAL_STEP: 67920 + | > loss: -0.27513 (-0.33186) + | > log_mle: -0.39573 (-0.40976) + | > loss_dur: 0.12060 (0.07790) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.34241 (12.93515) + | > current_lr: 0.00007 + | > step_time: 0.90550 (2.63287) + | > loader_time: 0.07790 (0.25575) + + + --> STEP: 65/234 -- GLOBAL_STEP: 67925 + | > loss: -0.29677 (-0.32841) + | > log_mle: -0.38625 (-0.40802) + | > loss_dur: 0.08948 (0.07961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.90709 (12.91209) + | > current_lr: 0.00007 + | > step_time: 2.09570 (2.57333) + | > loader_time: 0.00310 (0.23741) + + + --> STEP: 70/234 -- GLOBAL_STEP: 67930 + | > loss: -0.25606 (-0.32526) + | > log_mle: -0.36751 (-0.40592) + | > loss_dur: 0.11145 (0.08066) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.21128 (12.86425) + | > current_lr: 0.00007 + | > step_time: 1.01000 (2.48108) + | > loader_time: 0.00200 (0.22199) + + + --> STEP: 75/234 -- GLOBAL_STEP: 67935 + | > loss: -0.26677 (-0.32159) + | > log_mle: -0.38535 (-0.40444) + | > loss_dur: 0.11858 (0.08285) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.27833 (13.10058) + | > current_lr: 0.00007 + | > step_time: 4.18870 (2.47813) + | > loader_time: 0.09850 (0.20976) + + + --> STEP: 80/234 -- GLOBAL_STEP: 67940 + | > loss: -0.28569 (-0.31915) + | > log_mle: -0.37259 (-0.40279) + | > loss_dur: 0.08690 (0.08364) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27362 (13.02711) + | > current_lr: 0.00007 + | > step_time: 1.52650 (2.43439) + | > loader_time: 0.00210 (0.19799) + + + --> STEP: 85/234 -- GLOBAL_STEP: 67945 + | > loss: -0.28237 (-0.31677) + | > log_mle: -0.37688 (-0.40152) + | > loss_dur: 0.09452 (0.08475) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.42239 (12.89820) + | > current_lr: 0.00007 + | > step_time: 1.80060 (2.39018) + | > loader_time: 0.00620 (0.18764) + + + --> STEP: 90/234 -- GLOBAL_STEP: 67950 + | > loss: -0.26322 (-0.31445) + | > log_mle: -0.38906 (-0.40116) + | > loss_dur: 0.12584 (0.08672) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.16860 (13.06753) + | > current_lr: 0.00007 + | > step_time: 2.49510 (2.42054) + | > loader_time: 0.00210 (0.17841) + + + --> STEP: 95/234 -- GLOBAL_STEP: 67955 + | > loss: -0.29944 (-0.31300) + | > log_mle: -0.46709 (-0.40244) + | > loss_dur: 0.16765 (0.08945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.64193 (13.61252) + | > current_lr: 0.00007 + | > step_time: 1.90350 (2.37941) + | > loader_time: 0.08570 (0.17089) + + + --> STEP: 100/234 -- GLOBAL_STEP: 67960 + | > loss: -0.27206 (-0.31095) + | > log_mle: -0.39974 (-0.40233) + | > loss_dur: 0.12767 (0.09137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.64167 (13.81523) + | > current_lr: 0.00007 + | > step_time: 2.48960 (2.36427) + | > loader_time: 0.00240 (0.16420) + + + --> STEP: 105/234 -- GLOBAL_STEP: 67965 + | > loss: -0.25744 (-0.30925) + | > log_mle: -0.38349 (-0.40336) + | > loss_dur: 0.12606 (0.09411) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.74191 (14.12908) + | > current_lr: 0.00007 + | > step_time: 2.19930 (2.33450) + | > loader_time: 0.08430 (0.15816) + + + --> STEP: 110/234 -- GLOBAL_STEP: 67970 + | > loss: -0.26454 (-0.30722) + | > log_mle: -0.40472 (-0.40404) + | > loss_dur: 0.14017 (0.09682) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.54820 (14.46028) + | > current_lr: 0.00007 + | > step_time: 2.30140 (2.35301) + | > loader_time: 0.00310 (0.15541) + + + --> STEP: 115/234 -- GLOBAL_STEP: 67975 + | > loss: -0.26681 (-0.30602) + | > log_mle: -0.42884 (-0.40575) + | > loss_dur: 0.16203 (0.09973) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.17540 (14.78381) + | > current_lr: 0.00007 + | > step_time: 1.79900 (2.32667) + | > loader_time: 0.00370 (0.15023) + + + --> STEP: 120/234 -- GLOBAL_STEP: 67980 + | > loss: -0.31281 (-0.30497) + | > log_mle: -0.47402 (-0.40714) + | > loss_dur: 0.16121 (0.10217) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.95968 (15.16814) + | > current_lr: 0.00007 + | > step_time: 1.81080 (2.30046) + | > loader_time: 0.00770 (0.14549) + + + --> STEP: 125/234 -- GLOBAL_STEP: 67985 + | > loss: -0.29522 (-0.30339) + | > log_mle: -0.45926 (-0.40731) + | > loss_dur: 0.16404 (0.10392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.23054 (15.41633) + | > current_lr: 0.00007 + | > step_time: 2.70790 (2.29744) + | > loader_time: 0.08580 (0.14115) + + + --> STEP: 130/234 -- GLOBAL_STEP: 67990 + | > loss: -0.29696 (-0.30291) + | > log_mle: -0.47392 (-0.40940) + | > loss_dur: 0.17696 (0.10648) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.15701 (16.12010) + | > current_lr: 0.00007 + | > step_time: 2.40320 (2.27841) + | > loader_time: 0.00300 (0.13735) + + + --> STEP: 135/234 -- GLOBAL_STEP: 67995 + | > loss: -0.26187 (-0.30307) + | > log_mle: -0.39847 (-0.41171) + | > loss_dur: 0.13660 (0.10864) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.61300 (16.83955) + | > current_lr: 0.00007 + | > step_time: 6.21420 (2.33993) + | > loader_time: 0.20760 (0.13396) + + + --> STEP: 140/234 -- GLOBAL_STEP: 68000 + | > loss: -0.24890 (-0.30307) + | > log_mle: -0.42918 (-0.41449) + | > loss_dur: 0.18028 (0.11142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.71301 (17.70474) + | > current_lr: 0.00007 + | > step_time: 2.39940 (2.31818) + | > loader_time: 0.00310 (0.13029) + + + --> STEP: 145/234 -- GLOBAL_STEP: 68005 + | > loss: -0.35329 (-0.30392) + | > log_mle: -0.53560 (-0.41799) + | > loss_dur: 0.18231 (0.11407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.49688 (18.66151) + | > current_lr: 0.00007 + | > step_time: 0.81280 (2.32793) + | > loader_time: 0.00300 (0.12716) + + + --> STEP: 150/234 -- GLOBAL_STEP: 68010 + | > loss: -0.33007 (-0.30508) + | > log_mle: -0.53080 (-0.42144) + | > loss_dur: 0.20074 (0.11636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.56347 (19.30359) + | > current_lr: 0.00007 + | > step_time: 3.38820 (2.35366) + | > loader_time: 0.00330 (0.12306) + + + --> STEP: 155/234 -- GLOBAL_STEP: 68015 + | > loss: -0.39446 (-0.30728) + | > log_mle: -0.58857 (-0.42603) + | > loss_dur: 0.19411 (0.11875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.05513 (20.30263) + | > current_lr: 0.00007 + | > step_time: 2.90920 (2.38305) + | > loader_time: 0.00350 (0.12169) + + + --> STEP: 160/234 -- GLOBAL_STEP: 68020 + | > loss: -0.37140 (-0.30865) + | > log_mle: -0.58324 (-0.42987) + | > loss_dur: 0.21184 (0.12122) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.63764 (21.09998) + | > current_lr: 0.00007 + | > step_time: 6.21510 (2.40875) + | > loader_time: 0.10340 (0.11863) + + + --> STEP: 165/234 -- GLOBAL_STEP: 68025 + | > loss: -0.38234 (-0.31001) + | > log_mle: -0.58366 (-0.43347) + | > loss_dur: 0.20132 (0.12346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.98156 (21.86511) + | > current_lr: 0.00007 + | > step_time: 2.80340 (2.40473) + | > loader_time: 0.00240 (0.11562) + + + --> STEP: 170/234 -- GLOBAL_STEP: 68030 + | > loss: -0.38902 (-0.31184) + | > log_mle: -0.60828 (-0.43753) + | > loss_dur: 0.21925 (0.12570) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.98294 (22.90388) + | > current_lr: 0.00007 + | > step_time: 7.80880 (2.47415) + | > loader_time: 0.00330 (0.11286) + + + --> STEP: 175/234 -- GLOBAL_STEP: 68035 + | > loss: -0.38249 (-0.31459) + | > log_mle: -0.59800 (-0.44258) + | > loss_dur: 0.21550 (0.12799) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.38747 (23.74636) + | > current_lr: 0.00007 + | > step_time: 2.21910 (2.51456) + | > loader_time: 0.00270 (0.11069) + + + --> STEP: 180/234 -- GLOBAL_STEP: 68040 + | > loss: -0.39691 (-0.31690) + | > log_mle: -0.60400 (-0.44727) + | > loss_dur: 0.20709 (0.13036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.05373 (24.54065) + | > current_lr: 0.00007 + | > step_time: 1.55230 (2.53769) + | > loader_time: 0.00250 (0.10873) + + + --> STEP: 185/234 -- GLOBAL_STEP: 68045 + | > loss: -0.42238 (-0.31896) + | > log_mle: -0.63929 (-0.45166) + | > loss_dur: 0.21690 (0.13270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.00507 (25.33682) + | > current_lr: 0.00007 + | > step_time: 3.10780 (2.53573) + | > loader_time: 0.09630 (0.10821) + + + --> STEP: 190/234 -- GLOBAL_STEP: 68050 + | > loss: -0.39304 (-0.32075) + | > log_mle: -0.59425 (-0.45554) + | > loss_dur: 0.20121 (0.13479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.59821 (26.28831) + | > current_lr: 0.00007 + | > step_time: 3.59880 (2.63378) + | > loader_time: 0.00310 (0.10794) + + + --> STEP: 195/234 -- GLOBAL_STEP: 68055 + | > loss: -0.40275 (-0.32311) + | > log_mle: -0.62213 (-0.45984) + | > loss_dur: 0.21938 (0.13674) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.23524 (27.02424) + | > current_lr: 0.00007 + | > step_time: 1.31350 (2.66955) + | > loader_time: 0.00350 (0.10719) + + + --> STEP: 200/234 -- GLOBAL_STEP: 68060 + | > loss: -0.38705 (-0.32498) + | > log_mle: -0.63315 (-0.46384) + | > loss_dur: 0.24609 (0.13887) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.05325 (27.87980) + | > current_lr: 0.00007 + | > step_time: 4.80100 (2.66133) + | > loader_time: 0.00420 (0.10502) + + + --> STEP: 205/234 -- GLOBAL_STEP: 68065 + | > loss: -0.38221 (-0.32678) + | > log_mle: -0.60530 (-0.46766) + | > loss_dur: 0.22309 (0.14088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.59814 (28.78969) + | > current_lr: 0.00007 + | > step_time: 2.29500 (2.66965) + | > loader_time: 0.00370 (0.10299) + + + --> STEP: 210/234 -- GLOBAL_STEP: 68070 + | > loss: -0.47312 (-0.32949) + | > log_mle: -0.70577 (-0.47241) + | > loss_dur: 0.23265 (0.14292) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.23479 (29.74385) + | > current_lr: 0.00007 + | > step_time: 5.49100 (2.74268) + | > loader_time: 0.11340 (0.10392) + + + --> STEP: 215/234 -- GLOBAL_STEP: 68075 + | > loss: -0.41124 (-0.33253) + | > log_mle: -0.64805 (-0.47749) + | > loss_dur: 0.23681 (0.14496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.00935 (30.84108) + | > current_lr: 0.00007 + | > step_time: 5.49480 (2.85290) + | > loader_time: 0.10930 (0.10428) + + + --> STEP: 220/234 -- GLOBAL_STEP: 68080 + | > loss: -0.42566 (-0.33539) + | > log_mle: -0.66912 (-0.48243) + | > loss_dur: 0.24346 (0.14704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.11180 (32.65160) + | > current_lr: 0.00007 + | > step_time: 1.02210 (2.83536) + | > loader_time: 0.07670 (0.10283) + + + --> STEP: 225/234 -- GLOBAL_STEP: 68085 + | > loss: -0.51541 (-0.33792) + | > log_mle: -0.75951 (-0.48695) + | > loss_dur: 0.24410 (0.14903) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.82994 (33.46339) + | > current_lr: 0.00007 + | > step_time: 0.23180 (2.78860) + | > loader_time: 0.00320 (0.10095) + + + --> STEP: 230/234 -- GLOBAL_STEP: 68090 + | > loss: -0.51341 (-0.34060) + | > log_mle: -0.82632 (-0.49240) + | > loss_dur: 0.31291 (0.15180) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.47287 (34.31820) + | > current_lr: 0.00007 + | > step_time: 0.25810 (2.73339) + | > loader_time: 0.00430 (0.09883) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.00286 (+0.88238) + | > avg_loss: -0.33268 (+0.01937) + | > avg_log_mle: -0.55564 (+0.01309) + | > avg_loss_dur: 0.22296 (+0.00629) + + + > EPOCH: 291/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 22:29:28)  + + --> STEP: 1/234 -- GLOBAL_STEP: 68095 + | > loss: -0.34311 (-0.34311) + | > log_mle: -0.41204 (-0.41204) + | > loss_dur: 0.06892 (0.06892) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.41140 (23.41140) + | > current_lr: 0.00007 + | > step_time: 11.09840 (11.09844) + | > loader_time: 0.00120 (0.00123) + + + --> STEP: 6/234 -- GLOBAL_STEP: 68100 + | > loss: -0.35275 (-0.32513) + | > log_mle: -0.41948 (-0.41810) + | > loss_dur: 0.06673 (0.09297) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.18435 (17.46810) + | > current_lr: 0.00007 + | > step_time: 0.99860 (4.51760) + | > loader_time: 0.00110 (0.01699) + + + --> STEP: 11/234 -- GLOBAL_STEP: 68105 + | > loss: -0.37664 (-0.33487) + | > log_mle: -0.43337 (-0.42267) + | > loss_dur: 0.05673 (0.08780) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.19852 (16.27288) + | > current_lr: 0.00007 + | > step_time: 0.80590 (3.41036) + | > loader_time: 0.00130 (0.02434) + + + --> STEP: 16/234 -- GLOBAL_STEP: 68110 + | > loss: -0.38348 (-0.34404) + | > log_mle: -0.44045 (-0.42570) + | > loss_dur: 0.05698 (0.08166) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.51431 (15.32529) + | > current_lr: 0.00007 + | > step_time: 1.88110 (2.81452) + | > loader_time: 0.00170 (0.02247) + + + --> STEP: 21/234 -- GLOBAL_STEP: 68115 + | > loss: -0.34897 (-0.34918) + | > log_mle: -0.41429 (-0.42583) + | > loss_dur: 0.06532 (0.07664) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.46513 (14.50851) + | > current_lr: 0.00007 + | > step_time: 1.59570 (2.96679) + | > loader_time: 0.00250 (0.03197) + + + --> STEP: 26/234 -- GLOBAL_STEP: 68120 + | > loss: -0.33154 (-0.34961) + | > log_mle: -0.41072 (-0.42504) + | > loss_dur: 0.07918 (0.07544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.31376 (14.08182) + | > current_lr: 0.00007 + | > step_time: 1.81690 (3.02024) + | > loader_time: 0.07530 (0.04020) + + + --> STEP: 31/234 -- GLOBAL_STEP: 68125 + | > loss: -0.31365 (-0.34973) + | > log_mle: -0.40177 (-0.42400) + | > loss_dur: 0.08812 (0.07427) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.41334 (13.46754) + | > current_lr: 0.00007 + | > step_time: 1.40190 (2.74862) + | > loader_time: 0.00170 (0.03687) + + + --> STEP: 36/234 -- GLOBAL_STEP: 68130 + | > loss: -0.29238 (-0.34531) + | > log_mle: -0.38062 (-0.42027) + | > loss_dur: 0.08824 (0.07496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.96263 (13.06719) + | > current_lr: 0.00007 + | > step_time: 3.40230 (2.74730) + | > loader_time: 0.00220 (0.03446) + + + --> STEP: 41/234 -- GLOBAL_STEP: 68135 + | > loss: -0.34020 (-0.34065) + | > log_mle: -0.40142 (-0.41613) + | > loss_dur: 0.06122 (0.07549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.62399 (12.90500) + | > current_lr: 0.00007 + | > step_time: 2.47330 (2.59717) + | > loader_time: 0.00170 (0.03487) + + + --> STEP: 46/234 -- GLOBAL_STEP: 68140 + | > loss: -0.30273 (-0.33611) + | > log_mle: -0.39211 (-0.41284) + | > loss_dur: 0.08938 (0.07673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.77838 (12.76944) + | > current_lr: 0.00007 + | > step_time: 1.31490 (2.50173) + | > loader_time: 0.08610 (0.03491) + + + --> STEP: 51/234 -- GLOBAL_STEP: 68145 + | > loss: -0.31130 (-0.33463) + | > log_mle: -0.38970 (-0.41131) + | > loss_dur: 0.07841 (0.07668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.36888 (12.26872) + | > current_lr: 0.00007 + | > step_time: 1.24340 (2.40278) + | > loader_time: 0.00170 (0.03172) + + + --> STEP: 56/234 -- GLOBAL_STEP: 68150 + | > loss: -0.29275 (-0.33209) + | > log_mle: -0.38895 (-0.40949) + | > loss_dur: 0.09620 (0.07740) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.99506 (12.17850) + | > current_lr: 0.00007 + | > step_time: 0.56550 (2.30341) + | > loader_time: 0.00230 (0.03053) + + + --> STEP: 61/234 -- GLOBAL_STEP: 68155 + | > loss: -0.28210 (-0.32913) + | > log_mle: -0.37843 (-0.40756) + | > loss_dur: 0.09633 (0.07843) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.47863 (12.32081) + | > current_lr: 0.00007 + | > step_time: 1.66610 (2.27826) + | > loader_time: 0.00180 (0.02955) + + + --> STEP: 66/234 -- GLOBAL_STEP: 68160 + | > loss: -0.30451 (-0.32602) + | > log_mle: -0.37917 (-0.40579) + | > loss_dur: 0.07466 (0.07977) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.59375 (12.40154) + | > current_lr: 0.00007 + | > step_time: 1.37320 (2.21685) + | > loader_time: 0.00130 (0.02746) + + + --> STEP: 71/234 -- GLOBAL_STEP: 68165 + | > loss: -0.27420 (-0.32255) + | > log_mle: -0.39491 (-0.40387) + | > loss_dur: 0.12072 (0.08132) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.61271 (12.45613) + | > current_lr: 0.00007 + | > step_time: 0.99690 (2.16767) + | > loader_time: 0.09360 (0.02818) + + + --> STEP: 76/234 -- GLOBAL_STEP: 68170 + | > loss: -0.27960 (-0.31941) + | > log_mle: -0.38417 (-0.40229) + | > loss_dur: 0.10457 (0.08287) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.34785 (12.47196) + | > current_lr: 0.00007 + | > step_time: 2.30290 (2.13654) + | > loader_time: 0.00220 (0.02758) + + + --> STEP: 81/234 -- GLOBAL_STEP: 68175 + | > loss: -0.28528 (-0.31722) + | > log_mle: -0.39190 (-0.40100) + | > loss_dur: 0.10662 (0.08379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.85869 (12.30644) + | > current_lr: 0.00007 + | > step_time: 1.21250 (2.11002) + | > loader_time: 0.08730 (0.02902) + + + --> STEP: 86/234 -- GLOBAL_STEP: 68180 + | > loss: -0.28731 (-0.31498) + | > log_mle: -0.39178 (-0.39995) + | > loss_dur: 0.10448 (0.08497) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.01211 (12.24341) + | > current_lr: 0.00007 + | > step_time: 2.10360 (2.10018) + | > loader_time: 0.00240 (0.02946) + + + --> STEP: 91/234 -- GLOBAL_STEP: 68185 + | > loss: -0.27150 (-0.31274) + | > log_mle: -0.39734 (-0.39989) + | > loss_dur: 0.12584 (0.08715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.79670 (12.23898) + | > current_lr: 0.00007 + | > step_time: 3.10700 (2.12453) + | > loader_time: 0.08950 (0.03207) + + + --> STEP: 96/234 -- GLOBAL_STEP: 68190 + | > loss: -0.25685 (-0.31153) + | > log_mle: -0.37954 (-0.40126) + | > loss_dur: 0.12269 (0.08973) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.28154 (12.69342) + | > current_lr: 0.00007 + | > step_time: 1.51730 (2.12643) + | > loader_time: 0.00200 (0.03322) + + + --> STEP: 101/234 -- GLOBAL_STEP: 68195 + | > loss: -0.26792 (-0.30967) + | > log_mle: -0.42935 (-0.40166) + | > loss_dur: 0.16143 (0.09200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.21369 (13.18522) + | > current_lr: 0.00007 + | > step_time: 1.90710 (2.12597) + | > loader_time: 0.08770 (0.03340) + + + --> STEP: 106/234 -- GLOBAL_STEP: 68200 + | > loss: -0.24633 (-0.30813) + | > log_mle: -0.42366 (-0.40276) + | > loss_dur: 0.17733 (0.09463) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.81685 (13.61577) + | > current_lr: 0.00007 + | > step_time: 2.29490 (2.13817) + | > loader_time: 0.00240 (0.03358) + + + --> STEP: 111/234 -- GLOBAL_STEP: 68205 + | > loss: -0.30572 (-0.30672) + | > log_mle: -0.48194 (-0.40403) + | > loss_dur: 0.17621 (0.09731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.43931 (14.01131) + | > current_lr: 0.00007 + | > step_time: 3.19210 (2.27428) + | > loader_time: 0.01210 (0.03398) + + + --> STEP: 116/234 -- GLOBAL_STEP: 68210 + | > loss: -0.27134 (-0.30554) + | > log_mle: -0.44529 (-0.40537) + | > loss_dur: 0.17396 (0.09983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.59800 (14.55671) + | > current_lr: 0.00007 + | > step_time: 2.51430 (2.27064) + | > loader_time: 0.00170 (0.03559) + + + --> STEP: 121/234 -- GLOBAL_STEP: 68215 + | > loss: -0.23117 (-0.30418) + | > log_mle: -0.36270 (-0.40602) + | > loss_dur: 0.13153 (0.10184) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.27421 (14.83358) + | > current_lr: 0.00007 + | > step_time: 2.20440 (2.31900) + | > loader_time: 0.00280 (0.03430) + + + --> STEP: 126/234 -- GLOBAL_STEP: 68220 + | > loss: -0.30572 (-0.30302) + | > log_mle: -0.48745 (-0.40713) + | > loss_dur: 0.18173 (0.10410) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.28837 (15.30607) + | > current_lr: 0.00007 + | > step_time: 3.68720 (2.30120) + | > loader_time: 0.00210 (0.03455) + + + --> STEP: 131/234 -- GLOBAL_STEP: 68225 + | > loss: -0.35673 (-0.30297) + | > log_mle: -0.54345 (-0.40961) + | > loss_dur: 0.18673 (0.10664) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.96237 (15.97260) + | > current_lr: 0.00007 + | > step_time: 4.70800 (2.30904) + | > loader_time: 0.19360 (0.03524) + + + --> STEP: 136/234 -- GLOBAL_STEP: 68230 + | > loss: -0.39053 (-0.30321) + | > log_mle: -0.59277 (-0.41222) + | > loss_dur: 0.20224 (0.10900) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.94927 (16.59099) + | > current_lr: 0.00007 + | > step_time: 1.49170 (2.29024) + | > loader_time: 0.00200 (0.03478) + + + --> STEP: 141/234 -- GLOBAL_STEP: 68235 + | > loss: -0.31926 (-0.30300) + | > log_mle: -0.48582 (-0.41427) + | > loss_dur: 0.16656 (0.11127) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.89676 (17.16526) + | > current_lr: 0.00007 + | > step_time: 1.36100 (2.33514) + | > loader_time: 0.00250 (0.03502) + + + --> STEP: 146/234 -- GLOBAL_STEP: 68240 + | > loss: -0.33934 (-0.30405) + | > log_mle: -0.53955 (-0.41821) + | > loss_dur: 0.20021 (0.11417) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.89759 (17.93381) + | > current_lr: 0.00007 + | > step_time: 0.90740 (2.34161) + | > loader_time: 0.00210 (0.03510) + + + --> STEP: 151/234 -- GLOBAL_STEP: 68245 + | > loss: -0.31293 (-0.30477) + | > log_mle: -0.48994 (-0.42122) + | > loss_dur: 0.17701 (0.11645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.02082 (18.64384) + | > current_lr: 0.00007 + | > step_time: 2.61060 (2.32258) + | > loader_time: 0.07930 (0.03611) + + + --> STEP: 156/234 -- GLOBAL_STEP: 68250 + | > loss: -0.37064 (-0.30708) + | > log_mle: -0.55090 (-0.42597) + | > loss_dur: 0.18027 (0.11889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.78125 (19.59684) + | > current_lr: 0.00007 + | > step_time: 10.10790 (2.43661) + | > loader_time: 0.29500 (0.03830) + + + --> STEP: 161/234 -- GLOBAL_STEP: 68255 + | > loss: -0.38445 (-0.30855) + | > log_mle: -0.57307 (-0.42988) + | > loss_dur: 0.18862 (0.12133) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.50962 (20.66839) + | > current_lr: 0.00007 + | > step_time: 3.59510 (2.44918) + | > loader_time: 0.00360 (0.03765) + + + --> STEP: 166/234 -- GLOBAL_STEP: 68260 + | > loss: -0.33710 (-0.30970) + | > log_mle: -0.51153 (-0.43309) + | > loss_dur: 0.17443 (0.12339) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.25990 (21.55866) + | > current_lr: 0.00007 + | > step_time: 3.29800 (2.46704) + | > loader_time: 0.00250 (0.03778) + + + --> STEP: 171/234 -- GLOBAL_STEP: 68265 + | > loss: -0.42361 (-0.31225) + | > log_mle: -0.63225 (-0.43812) + | > loss_dur: 0.20864 (0.12586) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.95305 (22.78271) + | > current_lr: 0.00007 + | > step_time: 3.79430 (2.53402) + | > loader_time: 0.10930 (0.03975) + + + --> STEP: 176/234 -- GLOBAL_STEP: 68270 + | > loss: -0.39740 (-0.31485) + | > log_mle: -0.59606 (-0.44298) + | > loss_dur: 0.19866 (0.12813) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.51594 (24.04728) + | > current_lr: 0.00007 + | > step_time: 1.00180 (2.54724) + | > loader_time: 0.00340 (0.03917) + + + --> STEP: 181/234 -- GLOBAL_STEP: 68275 + | > loss: -0.32726 (-0.31670) + | > log_mle: -0.52675 (-0.44726) + | > loss_dur: 0.19948 (0.13056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.79242 (25.40145) + | > current_lr: 0.00007 + | > step_time: 2.28610 (2.54848) + | > loader_time: 0.00340 (0.03819) + + + --> STEP: 186/234 -- GLOBAL_STEP: 68280 + | > loss: -0.31826 (-0.31821) + | > log_mle: -0.54410 (-0.45127) + | > loss_dur: 0.22583 (0.13306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.16935 (26.78308) + | > current_lr: 0.00007 + | > step_time: 5.49090 (2.59451) + | > loader_time: 0.00300 (0.03824) + + + --> STEP: 191/234 -- GLOBAL_STEP: 68285 + | > loss: -0.38518 (-0.32000) + | > log_mle: -0.58618 (-0.45512) + | > loss_dur: 0.20099 (0.13513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.64895 (27.50421) + | > current_lr: 0.00007 + | > step_time: 2.19200 (2.63397) + | > loader_time: 0.00260 (0.03827) + + + --> STEP: 196/234 -- GLOBAL_STEP: 68290 + | > loss: -0.35776 (-0.32215) + | > log_mle: -0.57356 (-0.45938) + | > loss_dur: 0.21580 (0.13723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.67932 (28.29377) + | > current_lr: 0.00007 + | > step_time: 6.28610 (2.69023) + | > loader_time: 0.10640 (0.04243) + + + --> STEP: 201/234 -- GLOBAL_STEP: 68295 + | > loss: -0.31976 (-0.32376) + | > log_mle: -0.53608 (-0.46302) + | > loss_dur: 0.21632 (0.13926) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.83559 (29.07085) + | > current_lr: 0.00007 + | > step_time: 9.38340 (2.76673) + | > loader_time: 0.11250 (0.04294) + + + --> STEP: 206/234 -- GLOBAL_STEP: 68300 + | > loss: -0.43851 (-0.32626) + | > log_mle: -0.65975 (-0.46755) + | > loss_dur: 0.22124 (0.14129) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.67913 (29.66523) + | > current_lr: 0.00007 + | > step_time: 3.60900 (2.83754) + | > loader_time: 0.10330 (0.04435) + + + --> STEP: 211/234 -- GLOBAL_STEP: 68305 + | > loss: -0.48644 (-0.32910) + | > log_mle: -0.73423 (-0.47262) + | > loss_dur: 0.24778 (0.14352) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.30177 (30.72246) + | > current_lr: 0.00007 + | > step_time: 10.71040 (2.90927) + | > loader_time: 0.09060 (0.04554) + + + --> STEP: 216/234 -- GLOBAL_STEP: 68310 + | > loss: -0.46867 (-0.33196) + | > log_mle: -0.71215 (-0.47744) + | > loss_dur: 0.24349 (0.14548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 89.37681 (31.97132) + | > current_lr: 0.00007 + | > step_time: 4.30740 (3.01139) + | > loader_time: 0.00710 (0.04540) + + + --> STEP: 221/234 -- GLOBAL_STEP: 68315 + | > loss: -0.42235 (-0.33485) + | > log_mle: -0.63532 (-0.48230) + | > loss_dur: 0.21297 (0.14745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.14107 (33.01737) + | > current_lr: 0.00007 + | > step_time: 1.66370 (2.99114) + | > loader_time: 0.01540 (0.04492) + + + --> STEP: 226/234 -- GLOBAL_STEP: 68320 + | > loss: -0.47356 (-0.33832) + | > log_mle: -0.72499 (-0.48791) + | > loss_dur: 0.25143 (0.14959) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 156.59567 (34.32132) + | > current_lr: 0.00007 + | > step_time: 0.23370 (2.93281) + | > loader_time: 0.00320 (0.04429) + + + --> STEP: 231/234 -- GLOBAL_STEP: 68325 + | > loss: -0.42753 (-0.34048) + | > log_mle: -0.80492 (-0.49339) + | > loss_dur: 0.37739 (0.15291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.61166 (35.51514) + | > current_lr: 0.00007 + | > step_time: 0.27800 (2.87473) + | > loader_time: 0.00300 (0.04341) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.87939 (-0.12347) + | > avg_loss: -0.33680 (-0.00412) + | > avg_log_mle: -0.56422 (-0.00858) + | > avg_loss_dur: 0.22742 (+0.00446) + + + > EPOCH: 292/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 22:41:42)  + + --> STEP: 2/234 -- GLOBAL_STEP: 68330 + | > loss: -0.34102 (-0.33121) + | > log_mle: -0.43254 (-0.42274) + | > loss_dur: 0.09151 (0.09153) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.40274 (18.51825) + | > current_lr: 0.00007 + | > step_time: 9.29800 (6.44459) + | > loader_time: 0.00230 (0.00161) + + + --> STEP: 7/234 -- GLOBAL_STEP: 68335 + | > loss: -0.35306 (-0.33039) + | > log_mle: -0.42176 (-0.41876) + | > loss_dur: 0.06871 (0.08837) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.40094 (18.00212) + | > current_lr: 0.00007 + | > step_time: 13.29920 (7.24059) + | > loader_time: 0.19190 (0.05739) + + + --> STEP: 12/234 -- GLOBAL_STEP: 68340 + | > loss: -0.34039 (-0.33710) + | > log_mle: -0.41915 (-0.42307) + | > loss_dur: 0.07875 (0.08597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.28669 (17.58319) + | > current_lr: 0.00007 + | > step_time: 1.59460 (4.80848) + | > loader_time: 0.00110 (0.05557) + + + --> STEP: 17/234 -- GLOBAL_STEP: 68345 + | > loss: -0.37752 (-0.34668) + | > log_mle: -0.43576 (-0.42668) + | > loss_dur: 0.05824 (0.08000) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.64651 (16.16874) + | > current_lr: 0.00007 + | > step_time: 2.99620 (4.00442) + | > loader_time: 0.00140 (0.04011) + + + --> STEP: 22/234 -- GLOBAL_STEP: 68350 + | > loss: -0.34534 (-0.34895) + | > log_mle: -0.42323 (-0.42585) + | > loss_dur: 0.07788 (0.07690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.36358 (15.55591) + | > current_lr: 0.00007 + | > step_time: 1.49830 (3.67609) + | > loader_time: 0.00610 (0.04924) + + + --> STEP: 27/234 -- GLOBAL_STEP: 68355 + | > loss: -0.33828 (-0.34960) + | > log_mle: -0.40941 (-0.42483) + | > loss_dur: 0.07112 (0.07523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.83337 (14.83989) + | > current_lr: 0.00007 + | > step_time: 1.90180 (3.51474) + | > loader_time: 0.06160 (0.04920) + + + --> STEP: 32/234 -- GLOBAL_STEP: 68360 + | > loss: -0.33924 (-0.34984) + | > log_mle: -0.40917 (-0.42399) + | > loss_dur: 0.06993 (0.07415) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.31023 (14.23555) + | > current_lr: 0.00007 + | > step_time: 4.30310 (3.76382) + | > loader_time: 0.00530 (0.05387) + + + --> STEP: 37/234 -- GLOBAL_STEP: 68365 + | > loss: -0.31743 (-0.34602) + | > log_mle: -0.38996 (-0.42038) + | > loss_dur: 0.07253 (0.07435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.93937 (13.73417) + | > current_lr: 0.00007 + | > step_time: 2.09860 (3.52026) + | > loader_time: 0.00220 (0.04922) + + + --> STEP: 42/234 -- GLOBAL_STEP: 68370 + | > loss: -0.30659 (-0.34258) + | > log_mle: -0.38566 (-0.41757) + | > loss_dur: 0.07908 (0.07499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.86401 (13.20234) + | > current_lr: 0.00007 + | > step_time: 1.98740 (3.28791) + | > loader_time: 0.00140 (0.04381) + + + --> STEP: 47/234 -- GLOBAL_STEP: 68375 + | > loss: -0.31933 (-0.33923) + | > log_mle: -0.40168 (-0.41542) + | > loss_dur: 0.08235 (0.07619) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.25031 (12.94000) + | > current_lr: 0.00007 + | > step_time: 1.21780 (3.06485) + | > loader_time: 0.00190 (0.03936) + + + --> STEP: 52/234 -- GLOBAL_STEP: 68380 + | > loss: -0.29096 (-0.33804) + | > log_mle: -0.39401 (-0.41418) + | > loss_dur: 0.10305 (0.07614) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.26570 (12.46470) + | > current_lr: 0.00007 + | > step_time: 1.09290 (2.88138) + | > loader_time: 0.00200 (0.03578) + + + --> STEP: 57/234 -- GLOBAL_STEP: 68385 + | > loss: -0.29885 (-0.33593) + | > log_mle: -0.37969 (-0.41248) + | > loss_dur: 0.08084 (0.07655) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.21206 (12.26901) + | > current_lr: 0.00007 + | > step_time: 1.10310 (2.75891) + | > loader_time: 0.00190 (0.03287) + + + --> STEP: 62/234 -- GLOBAL_STEP: 68390 + | > loss: -0.27051 (-0.33297) + | > log_mle: -0.40119 (-0.41118) + | > loss_dur: 0.13068 (0.07821) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.42809 (12.29314) + | > current_lr: 0.00007 + | > step_time: 1.04960 (2.64044) + | > loader_time: 0.00180 (0.03038) + + + --> STEP: 67/234 -- GLOBAL_STEP: 68395 + | > loss: -0.29899 (-0.33081) + | > log_mle: -0.39442 (-0.40945) + | > loss_dur: 0.09542 (0.07864) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.66566 (12.14502) + | > current_lr: 0.00007 + | > step_time: 1.37390 (2.56341) + | > loader_time: 0.00200 (0.02825) + + + --> STEP: 72/234 -- GLOBAL_STEP: 68400 + | > loss: -0.29334 (-0.32744) + | > log_mle: -0.38427 (-0.40758) + | > loss_dur: 0.09093 (0.08014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.08505 (12.40875) + | > current_lr: 0.00007 + | > step_time: 1.66000 (2.49807) + | > loader_time: 0.00220 (0.02693) + + + --> STEP: 77/234 -- GLOBAL_STEP: 68405 + | > loss: -0.29448 (-0.32425) + | > log_mle: -0.38690 (-0.40603) + | > loss_dur: 0.09242 (0.08178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.63634 (12.58037) + | > current_lr: 0.00007 + | > step_time: 1.66330 (2.47025) + | > loader_time: 0.00180 (0.02531) + + + --> STEP: 82/234 -- GLOBAL_STEP: 68410 + | > loss: -0.27705 (-0.32174) + | > log_mle: -0.37788 (-0.40465) + | > loss_dur: 0.10084 (0.08291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29902 (12.52383) + | > current_lr: 0.00007 + | > step_time: 2.21210 (2.44632) + | > loader_time: 0.00250 (0.02607) + + + --> STEP: 87/234 -- GLOBAL_STEP: 68415 + | > loss: -0.27274 (-0.31911) + | > log_mle: -0.37961 (-0.40339) + | > loss_dur: 0.10686 (0.08428) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.46224 (12.73367) + | > current_lr: 0.00007 + | > step_time: 2.38330 (2.43419) + | > loader_time: 0.00230 (0.02568) + + + --> STEP: 92/234 -- GLOBAL_STEP: 68420 + | > loss: -0.28554 (-0.31689) + | > log_mle: -0.41882 (-0.40350) + | > loss_dur: 0.13328 (0.08661) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.09636 (13.07003) + | > current_lr: 0.00007 + | > step_time: 3.00620 (2.42468) + | > loader_time: 0.00640 (0.02540) + + + --> STEP: 97/234 -- GLOBAL_STEP: 68425 + | > loss: -0.26331 (-0.31525) + | > log_mle: -0.39889 (-0.40445) + | > loss_dur: 0.13558 (0.08920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.42379 (13.40201) + | > current_lr: 0.00007 + | > step_time: 1.90590 (2.42855) + | > loader_time: 0.00230 (0.02531) + + + --> STEP: 102/234 -- GLOBAL_STEP: 68430 + | > loss: -0.24279 (-0.31304) + | > log_mle: -0.38241 (-0.40458) + | > loss_dur: 0.13962 (0.09154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.60805 (13.71047) + | > current_lr: 0.00007 + | > step_time: 2.50210 (2.40634) + | > loader_time: 0.09460 (0.02590) + + + --> STEP: 107/234 -- GLOBAL_STEP: 68435 + | > loss: -0.27389 (-0.31127) + | > log_mle: -0.42201 (-0.40567) + | > loss_dur: 0.14812 (0.09441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.48862 (14.27996) + | > current_lr: 0.00007 + | > step_time: 1.80540 (2.37810) + | > loader_time: 0.00760 (0.02556) + + + --> STEP: 112/234 -- GLOBAL_STEP: 68440 + | > loss: -0.27546 (-0.30962) + | > log_mle: -0.43854 (-0.40684) + | > loss_dur: 0.16308 (0.09722) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.39056 (14.70477) + | > current_lr: 0.00007 + | > step_time: 1.38870 (2.36465) + | > loader_time: 0.00210 (0.02525) + + + --> STEP: 117/234 -- GLOBAL_STEP: 68445 + | > loss: -0.27974 (-0.30829) + | > log_mle: -0.43476 (-0.40800) + | > loss_dur: 0.15502 (0.09970) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.24532 (15.11613) + | > current_lr: 0.00007 + | > step_time: 1.99600 (2.34815) + | > loader_time: 0.00390 (0.02506) + + + --> STEP: 122/234 -- GLOBAL_STEP: 68450 + | > loss: -0.25083 (-0.30664) + | > log_mle: -0.39646 (-0.40827) + | > loss_dur: 0.14563 (0.10162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.14291 (15.41802) + | > current_lr: 0.00007 + | > step_time: 1.30100 (2.33710) + | > loader_time: 0.00240 (0.02482) + + + --> STEP: 127/234 -- GLOBAL_STEP: 68455 + | > loss: -0.28854 (-0.30545) + | > log_mle: -0.46243 (-0.40967) + | > loss_dur: 0.17390 (0.10423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.30295 (16.07508) + | > current_lr: 0.00007 + | > step_time: 1.98530 (2.33238) + | > loader_time: 0.00220 (0.02395) + + + --> STEP: 132/234 -- GLOBAL_STEP: 68460 + | > loss: -0.28717 (-0.30532) + | > log_mle: -0.44218 (-0.41181) + | > loss_dur: 0.15501 (0.10649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.23330 (16.47472) + | > current_lr: 0.00007 + | > step_time: 1.80280 (2.33181) + | > loader_time: 0.09200 (0.02594) + + + --> STEP: 137/234 -- GLOBAL_STEP: 68465 + | > loss: -0.27149 (-0.30516) + | > log_mle: -0.45773 (-0.41431) + | > loss_dur: 0.18624 (0.10915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.04269 (17.12381) + | > current_lr: 0.00007 + | > step_time: 2.32510 (2.34388) + | > loader_time: 0.00360 (0.02705) + + + --> STEP: 142/234 -- GLOBAL_STEP: 68470 + | > loss: -0.28484 (-0.30496) + | > log_mle: -0.47092 (-0.41636) + | > loss_dur: 0.18607 (0.11140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.30963 (17.62219) + | > current_lr: 0.00007 + | > step_time: 1.30680 (2.31982) + | > loader_time: 0.08560 (0.02868) + + + --> STEP: 147/234 -- GLOBAL_STEP: 68475 + | > loss: -0.29347 (-0.30598) + | > log_mle: -0.47374 (-0.42021) + | > loss_dur: 0.18027 (0.11423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.14613 (18.49251) + | > current_lr: 0.00007 + | > step_time: 4.89580 (2.34650) + | > loader_time: 0.00290 (0.02892) + + + --> STEP: 152/234 -- GLOBAL_STEP: 68480 + | > loss: -0.37314 (-0.30743) + | > log_mle: -0.56576 (-0.42385) + | > loss_dur: 0.19262 (0.11642) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.58209 (19.17472) + | > current_lr: 0.00007 + | > step_time: 1.09650 (2.35035) + | > loader_time: 0.00300 (0.02938) + + + --> STEP: 157/234 -- GLOBAL_STEP: 68485 + | > loss: -0.32330 (-0.30949) + | > log_mle: -0.50939 (-0.42819) + | > loss_dur: 0.18609 (0.11870) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.04388 (20.23353) + | > current_lr: 0.00007 + | > step_time: 3.39160 (2.34372) + | > loader_time: 0.09570 (0.03014) + + + --> STEP: 162/234 -- GLOBAL_STEP: 68490 + | > loss: -0.35604 (-0.31135) + | > log_mle: -0.54288 (-0.43241) + | > loss_dur: 0.18685 (0.12106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.71560 (21.14716) + | > current_lr: 0.00007 + | > step_time: 5.44180 (2.37599) + | > loader_time: 0.09160 (0.03109) + + + --> STEP: 167/234 -- GLOBAL_STEP: 68495 + | > loss: -0.41124 (-0.31248) + | > log_mle: -0.61996 (-0.43580) + | > loss_dur: 0.20872 (0.12331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 104.44959 (23.09375) + | > current_lr: 0.00007 + | > step_time: 3.40440 (2.37175) + | > loader_time: 0.09490 (0.03181) + + + --> STEP: 172/234 -- GLOBAL_STEP: 68500 + | > loss: -0.41667 (-0.31455) + | > log_mle: -0.62860 (-0.44024) + | > loss_dur: 0.21192 (0.12569) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.84777 (24.27432) + | > current_lr: 0.00007 + | > step_time: 6.88780 (2.42090) + | > loader_time: 0.00860 (0.03257) + + + --> STEP: 177/234 -- GLOBAL_STEP: 68505 + | > loss: -0.35639 (-0.31621) + | > log_mle: -0.55654 (-0.44435) + | > loss_dur: 0.20015 (0.12814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.56854 (25.18965) + | > current_lr: 0.00007 + | > step_time: 3.77930 (2.55374) + | > loader_time: 0.02100 (0.03453) + + + --> STEP: 182/234 -- GLOBAL_STEP: 68510 + | > loss: -0.38108 (-0.31784) + | > log_mle: -0.61844 (-0.44860) + | > loss_dur: 0.23736 (0.13077) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.07956 (26.00690) + | > current_lr: 0.00007 + | > step_time: 1.68990 (2.56176) + | > loader_time: 0.10280 (0.03565) + + + --> STEP: 187/234 -- GLOBAL_STEP: 68515 + | > loss: -0.41182 (-0.31973) + | > log_mle: -0.62447 (-0.45290) + | > loss_dur: 0.21266 (0.13316) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.81007 (26.89710) + | > current_lr: 0.00007 + | > step_time: 3.29200 (2.57969) + | > loader_time: 0.08790 (0.03575) + + + --> STEP: 192/234 -- GLOBAL_STEP: 68520 + | > loss: -0.42879 (-0.32170) + | > log_mle: -0.64081 (-0.45691) + | > loss_dur: 0.21202 (0.13521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.17525 (28.13020) + | > current_lr: 0.00007 + | > step_time: 2.39610 (2.57523) + | > loader_time: 0.00620 (0.03631) + + + --> STEP: 197/234 -- GLOBAL_STEP: 68525 + | > loss: -0.41600 (-0.32369) + | > log_mle: -0.61168 (-0.46084) + | > loss_dur: 0.19567 (0.13715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.25126 (29.08418) + | > current_lr: 0.00007 + | > step_time: 3.59500 (2.63650) + | > loader_time: 0.08050 (0.03782) + + + --> STEP: 202/234 -- GLOBAL_STEP: 68530 + | > loss: -0.48856 (-0.32539) + | > log_mle: -0.69463 (-0.46472) + | > loss_dur: 0.20606 (0.13933) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.50626 (30.17291) + | > current_lr: 0.00007 + | > step_time: 3.49410 (2.64500) + | > loader_time: 0.00470 (0.03824) + + + --> STEP: 207/234 -- GLOBAL_STEP: 68535 + | > loss: -0.45657 (-0.32744) + | > log_mle: -0.69318 (-0.46886) + | > loss_dur: 0.23661 (0.14142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.11359 (30.99313) + | > current_lr: 0.00007 + | > step_time: 3.51620 (2.66149) + | > loader_time: 0.09850 (0.03911) + + + --> STEP: 212/234 -- GLOBAL_STEP: 68540 + | > loss: -0.41431 (-0.33001) + | > log_mle: -0.64669 (-0.47366) + | > loss_dur: 0.23238 (0.14365) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 112.18098 (32.01214) + | > current_lr: 0.00007 + | > step_time: 5.11350 (2.71334) + | > loader_time: 0.08610 (0.03999) + + + --> STEP: 217/234 -- GLOBAL_STEP: 68545 + | > loss: -0.46001 (-0.33264) + | > log_mle: -0.69270 (-0.47827) + | > loss_dur: 0.23269 (0.14562) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.99363 (32.85777) + | > current_lr: 0.00007 + | > step_time: 3.90950 (2.74339) + | > loader_time: 0.00400 (0.03919) + + + --> STEP: 222/234 -- GLOBAL_STEP: 68550 + | > loss: -0.45554 (-0.33546) + | > log_mle: -0.71959 (-0.48315) + | > loss_dur: 0.26405 (0.14770) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.82280 (33.55157) + | > current_lr: 0.00007 + | > step_time: 4.31820 (2.82317) + | > loader_time: 0.09560 (0.03963) + + + --> STEP: 227/234 -- GLOBAL_STEP: 68555 + | > loss: -0.41998 (-0.33866) + | > log_mle: -0.67226 (-0.48842) + | > loss_dur: 0.25228 (0.14976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 103.73393 (34.55489) + | > current_lr: 0.00007 + | > step_time: 0.25290 (2.80073) + | > loader_time: 0.00350 (0.03930) + + + --> STEP: 232/234 -- GLOBAL_STEP: 68560 + | > loss: -0.42190 (-0.34101) + | > log_mle: -0.89076 (-0.49494) + | > loss_dur: 0.46886 (0.15393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 100.82812 (35.76067) + | > current_lr: 0.00007 + | > step_time: 0.34580 (2.74633) + | > loader_time: 0.01230 (0.03857) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00180 (-0.87759) + | > avg_loss: -0.32976 (+0.00704) + | > avg_log_mle: -0.55859 (+0.00563) + | > avg_loss_dur: 0.22883 (+0.00141) + + + > EPOCH: 293/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 22:53:32)  + + --> STEP: 3/234 -- GLOBAL_STEP: 68565 + | > loss: -0.25858 (-0.30936) + | > log_mle: -0.41155 (-0.42021) + | > loss_dur: 0.15297 (0.11086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.99090 (17.06698) + | > current_lr: 0.00007 + | > step_time: 3.67900 (8.68861) + | > loader_time: 0.00100 (2.76667) + + + --> STEP: 8/234 -- GLOBAL_STEP: 68570 + | > loss: -0.36027 (-0.33084) + | > log_mle: -0.44117 (-0.42390) + | > loss_dur: 0.08089 (0.09306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.32464 (15.88116) + | > current_lr: 0.00007 + | > step_time: 2.99440 (5.59658) + | > loader_time: 0.00160 (1.07606) + + + --> STEP: 13/234 -- GLOBAL_STEP: 68575 + | > loss: -0.37106 (-0.33664) + | > log_mle: -0.44457 (-0.42585) + | > loss_dur: 0.07351 (0.08920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.10515 (17.43940) + | > current_lr: 0.00007 + | > step_time: 4.19850 (4.58105) + | > loader_time: 0.00090 (0.66303) + + + --> STEP: 18/234 -- GLOBAL_STEP: 68580 + | > loss: -0.34906 (-0.34442) + | > log_mle: -0.41333 (-0.42664) + | > loss_dur: 0.06427 (0.08222) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.87905 (16.20523) + | > current_lr: 0.00007 + | > step_time: 3.01190 (4.27725) + | > loader_time: 0.19990 (0.49066) + + + --> STEP: 23/234 -- GLOBAL_STEP: 68585 + | > loss: -0.37525 (-0.34776) + | > log_mle: -0.43879 (-0.42655) + | > loss_dur: 0.06353 (0.07880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.96097 (15.49889) + | > current_lr: 0.00007 + | > step_time: 1.91330 (3.75904) + | > loader_time: 0.09270 (0.38827) + + + --> STEP: 28/234 -- GLOBAL_STEP: 68590 + | > loss: -0.41013 (-0.35070) + | > log_mle: -0.45533 (-0.42592) + | > loss_dur: 0.04520 (0.07522) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.81521 (14.50189) + | > current_lr: 0.00007 + | > step_time: 3.20230 (3.49122) + | > loader_time: 0.00620 (0.31943) + + + --> STEP: 33/234 -- GLOBAL_STEP: 68595 + | > loss: -0.35229 (-0.34939) + | > log_mle: -0.41026 (-0.42377) + | > loss_dur: 0.05796 (0.07438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67529 (13.81439) + | > current_lr: 0.00007 + | > step_time: 1.98890 (3.20707) + | > loader_time: 0.00170 (0.27133) + + + --> STEP: 38/234 -- GLOBAL_STEP: 68600 + | > loss: -0.31874 (-0.34564) + | > log_mle: -0.40470 (-0.42042) + | > loss_dur: 0.08596 (0.07478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.99000 (13.35028) + | > current_lr: 0.00007 + | > step_time: 1.97440 (2.96816) + | > loader_time: 0.00250 (0.23812) + + + --> STEP: 43/234 -- GLOBAL_STEP: 68605 + | > loss: -0.29741 (-0.34167) + | > log_mle: -0.39456 (-0.41743) + | > loss_dur: 0.09716 (0.07576) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.00035 (13.41472) + | > current_lr: 0.00007 + | > step_time: 1.48040 (2.87819) + | > loader_time: 0.00120 (0.21292) + + + --> STEP: 48/234 -- GLOBAL_STEP: 68610 + | > loss: -0.35552 (-0.33980) + | > log_mle: -0.41278 (-0.41572) + | > loss_dur: 0.05726 (0.07592) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.41381 (13.21597) + | > current_lr: 0.00007 + | > step_time: 1.69240 (2.78509) + | > loader_time: 0.00170 (0.19462) + + + --> STEP: 53/234 -- GLOBAL_STEP: 68615 + | > loss: -0.31215 (-0.33769) + | > log_mle: -0.40054 (-0.41407) + | > loss_dur: 0.08839 (0.07638) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.89636 (12.82173) + | > current_lr: 0.00007 + | > step_time: 2.33530 (2.68130) + | > loader_time: 0.00250 (0.17809) + + + --> STEP: 58/234 -- GLOBAL_STEP: 68620 + | > loss: -0.33020 (-0.33595) + | > log_mle: -0.40207 (-0.41249) + | > loss_dur: 0.07188 (0.07654) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.95639 (12.43933) + | > current_lr: 0.00007 + | > step_time: 0.66940 (2.56815) + | > loader_time: 0.00160 (0.16603) + + + --> STEP: 63/234 -- GLOBAL_STEP: 68625 + | > loss: -0.28117 (-0.33193) + | > log_mle: -0.37918 (-0.41080) + | > loss_dur: 0.09801 (0.07887) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.57238 (12.41870) + | > current_lr: 0.00007 + | > step_time: 1.90970 (2.51018) + | > loader_time: 0.08480 (0.15567) + + + --> STEP: 68/234 -- GLOBAL_STEP: 68630 + | > loss: -0.28017 (-0.32959) + | > log_mle: -0.37801 (-0.40909) + | > loss_dur: 0.09784 (0.07949) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.32164 (12.13490) + | > current_lr: 0.00007 + | > step_time: 2.33370 (2.46605) + | > loader_time: 0.00740 (0.14546) + + + --> STEP: 73/234 -- GLOBAL_STEP: 68635 + | > loss: -0.26268 (-0.32615) + | > log_mle: -0.38320 (-0.40732) + | > loss_dur: 0.12052 (0.08117) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.98919 (12.11125) + | > current_lr: 0.00007 + | > step_time: 1.59680 (2.45124) + | > loader_time: 0.00180 (0.13562) + + + --> STEP: 78/234 -- GLOBAL_STEP: 68640 + | > loss: -0.27909 (-0.32288) + | > log_mle: -0.37626 (-0.40564) + | > loss_dur: 0.09717 (0.08276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.48084 (12.10887) + | > current_lr: 0.00007 + | > step_time: 1.43110 (2.40995) + | > loader_time: 0.00210 (0.12713) + + + --> STEP: 83/234 -- GLOBAL_STEP: 68645 + | > loss: -0.25444 (-0.32027) + | > log_mle: -0.38485 (-0.40439) + | > loss_dur: 0.13040 (0.08412) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.53248 (12.11497) + | > current_lr: 0.00007 + | > step_time: 2.60020 (2.39819) + | > loader_time: 0.08810 (0.12065) + + + --> STEP: 88/234 -- GLOBAL_STEP: 68650 + | > loss: -0.28158 (-0.31782) + | > log_mle: -0.41684 (-0.40357) + | > loss_dur: 0.13526 (0.08575) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.96591 (12.10416) + | > current_lr: 0.00007 + | > step_time: 1.88930 (2.38231) + | > loader_time: 0.00190 (0.11393) + + + --> STEP: 93/234 -- GLOBAL_STEP: 68655 + | > loss: -0.27768 (-0.31586) + | > log_mle: -0.42839 (-0.40390) + | > loss_dur: 0.15071 (0.08804) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.65226 (12.30295) + | > current_lr: 0.00007 + | > step_time: 1.19830 (2.37357) + | > loader_time: 0.00200 (0.10968) + + + --> STEP: 98/234 -- GLOBAL_STEP: 68660 + | > loss: -0.26126 (-0.31443) + | > log_mle: -0.36930 (-0.40423) + | > loss_dur: 0.10805 (0.08980) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.20111 (12.54599) + | > current_lr: 0.00007 + | > step_time: 2.89980 (2.33989) + | > loader_time: 0.00250 (0.10588) + + + --> STEP: 103/234 -- GLOBAL_STEP: 68665 + | > loss: -0.30852 (-0.31287) + | > log_mle: -0.45469 (-0.40531) + | > loss_dur: 0.14617 (0.09244) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.58898 (13.10073) + | > current_lr: 0.00007 + | > step_time: 1.79480 (2.35827) + | > loader_time: 0.00210 (0.10269) + + + --> STEP: 108/234 -- GLOBAL_STEP: 68670 + | > loss: -0.26808 (-0.31108) + | > log_mle: -0.40099 (-0.40594) + | > loss_dur: 0.13291 (0.09486) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.39653 (13.57938) + | > current_lr: 0.00007 + | > step_time: 1.91570 (2.31833) + | > loader_time: 0.00180 (0.09894) + + + --> STEP: 113/234 -- GLOBAL_STEP: 68675 + | > loss: -0.28357 (-0.30964) + | > log_mle: -0.43966 (-0.40748) + | > loss_dur: 0.15610 (0.09783) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.44043 (14.19000) + | > current_lr: 0.00007 + | > step_time: 2.40650 (2.30505) + | > loader_time: 0.08650 (0.09616) + + + --> STEP: 118/234 -- GLOBAL_STEP: 68680 + | > loss: -0.24615 (-0.30761) + | > log_mle: -0.40808 (-0.40800) + | > loss_dur: 0.16193 (0.10040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.08594 (14.66279) + | > current_lr: 0.00007 + | > step_time: 2.42790 (2.29738) + | > loader_time: 0.00240 (0.09357) + + + --> STEP: 123/234 -- GLOBAL_STEP: 68685 + | > loss: -0.23049 (-0.30559) + | > log_mle: -0.38072 (-0.40788) + | > loss_dur: 0.15023 (0.10229) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.42291 (14.72589) + | > current_lr: 0.00007 + | > step_time: 2.90180 (2.30305) + | > loader_time: 0.07370 (0.09111) + + + --> STEP: 128/234 -- GLOBAL_STEP: 68690 + | > loss: -0.29565 (-0.30511) + | > log_mle: -0.43774 (-0.40973) + | > loss_dur: 0.14209 (0.10462) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.04072 (15.22707) + | > current_lr: 0.00007 + | > step_time: 2.00150 (2.30763) + | > loader_time: 0.08770 (0.08958) + + + --> STEP: 133/234 -- GLOBAL_STEP: 68695 + | > loss: -0.30665 (-0.30524) + | > log_mle: -0.47080 (-0.41203) + | > loss_dur: 0.16415 (0.10679) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.66969 (15.71842) + | > current_lr: 0.00007 + | > step_time: 3.31090 (2.32697) + | > loader_time: 0.19580 (0.08932) + + + --> STEP: 138/234 -- GLOBAL_STEP: 68700 + | > loss: -0.25658 (-0.30483) + | > log_mle: -0.41786 (-0.41415) + | > loss_dur: 0.16128 (0.10932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.05667 (16.31048) + | > current_lr: 0.00007 + | > step_time: 7.59780 (2.36902) + | > loader_time: 0.00490 (0.08714) + + + --> STEP: 143/234 -- GLOBAL_STEP: 68705 + | > loss: -0.35753 (-0.30528) + | > log_mle: -0.57313 (-0.41736) + | > loss_dur: 0.21560 (0.11208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.46393 (17.02627) + | > current_lr: 0.00007 + | > step_time: 1.45220 (2.36013) + | > loader_time: 0.00220 (0.08494) + + + --> STEP: 148/234 -- GLOBAL_STEP: 68710 + | > loss: -0.32528 (-0.30620) + | > log_mle: -0.47901 (-0.42064) + | > loss_dur: 0.15373 (0.11444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.33620 (17.77789) + | > current_lr: 0.00007 + | > step_time: 2.80870 (2.39936) + | > loader_time: 0.00200 (0.08346) + + + --> STEP: 153/234 -- GLOBAL_STEP: 68715 + | > loss: -0.42020 (-0.30836) + | > log_mle: -0.61891 (-0.42527) + | > loss_dur: 0.19871 (0.11691) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.43291 (18.87411) + | > current_lr: 0.00007 + | > step_time: 1.79480 (2.45435) + | > loader_time: 0.10460 (0.08277) + + + --> STEP: 158/234 -- GLOBAL_STEP: 68720 + | > loss: -0.33883 (-0.30970) + | > log_mle: -0.54313 (-0.42907) + | > loss_dur: 0.20429 (0.11937) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.78373 (20.00399) + | > current_lr: 0.00007 + | > step_time: 1.91310 (2.46340) + | > loader_time: 0.18440 (0.08255) + + + --> STEP: 163/234 -- GLOBAL_STEP: 68725 + | > loss: -0.31854 (-0.31137) + | > log_mle: -0.51206 (-0.43294) + | > loss_dur: 0.19352 (0.12157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.88868 (20.86542) + | > current_lr: 0.00007 + | > step_time: 2.51610 (2.47633) + | > loader_time: 0.08420 (0.08116) + + + --> STEP: 168/234 -- GLOBAL_STEP: 68730 + | > loss: -0.35202 (-0.31325) + | > log_mle: -0.57042 (-0.43698) + | > loss_dur: 0.21839 (0.12373) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.55655 (21.99071) + | > current_lr: 0.00007 + | > step_time: 1.60390 (2.45873) + | > loader_time: 0.00260 (0.07981) + + + --> STEP: 173/234 -- GLOBAL_STEP: 68735 + | > loss: -0.38016 (-0.31560) + | > log_mle: -0.58837 (-0.44184) + | > loss_dur: 0.20822 (0.12624) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.51558 (22.83531) + | > current_lr: 0.00007 + | > step_time: 2.21690 (2.45650) + | > loader_time: 0.09230 (0.07977) + + + --> STEP: 178/234 -- GLOBAL_STEP: 68740 + | > loss: -0.39580 (-0.31770) + | > log_mle: -0.62746 (-0.44648) + | > loss_dur: 0.23166 (0.12878) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.02636 (24.13019) + | > current_lr: 0.00007 + | > step_time: 7.20150 (2.50258) + | > loader_time: 0.20030 (0.07918) + + + --> STEP: 183/234 -- GLOBAL_STEP: 68745 + | > loss: -0.42309 (-0.31937) + | > log_mle: -0.63132 (-0.45068) + | > loss_dur: 0.20823 (0.13131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.21766 (25.01343) + | > current_lr: 0.00007 + | > step_time: 3.59480 (2.58067) + | > loader_time: 0.00300 (0.07764) + + + --> STEP: 188/234 -- GLOBAL_STEP: 68750 + | > loss: -0.43562 (-0.32134) + | > log_mle: -0.65264 (-0.45502) + | > loss_dur: 0.21702 (0.13368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.72416 (25.79111) + | > current_lr: 0.00007 + | > step_time: 6.79340 (2.62742) + | > loader_time: 0.00340 (0.07659) + + + --> STEP: 193/234 -- GLOBAL_STEP: 68755 + | > loss: -0.45341 (-0.32387) + | > log_mle: -0.66118 (-0.45939) + | > loss_dur: 0.20777 (0.13552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.78762 (26.52319) + | > current_lr: 0.00007 + | > step_time: 4.80340 (2.66860) + | > loader_time: 0.09270 (0.07819) + + + --> STEP: 198/234 -- GLOBAL_STEP: 68760 + | > loss: -0.40215 (-0.32584) + | > log_mle: -0.62699 (-0.46333) + | > loss_dur: 0.22483 (0.13749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.89554 (27.68192) + | > current_lr: 0.00007 + | > step_time: 2.60340 (2.75520) + | > loader_time: 0.09300 (0.08733) + + + --> STEP: 203/234 -- GLOBAL_STEP: 68765 + | > loss: -0.35421 (-0.32720) + | > log_mle: -0.55920 (-0.46684) + | > loss_dur: 0.20499 (0.13963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.21650 (28.68065) + | > current_lr: 0.00007 + | > step_time: 4.80780 (2.78940) + | > loader_time: 0.09560 (0.08660) + + + --> STEP: 208/234 -- GLOBAL_STEP: 68770 + | > loss: -0.41793 (-0.32970) + | > log_mle: -0.65496 (-0.47142) + | > loss_dur: 0.23703 (0.14172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.46104 (29.52748) + | > current_lr: 0.00007 + | > step_time: 3.39000 (2.84982) + | > loader_time: 0.00290 (0.08676) + + + --> STEP: 213/234 -- GLOBAL_STEP: 68775 + | > loss: -0.47256 (-0.33279) + | > log_mle: -0.70475 (-0.47656) + | > loss_dur: 0.23219 (0.14377) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.74112 (30.67632) + | > current_lr: 0.00007 + | > step_time: 7.70770 (2.90131) + | > loader_time: 0.09520 (0.08656) + + + --> STEP: 218/234 -- GLOBAL_STEP: 68780 + | > loss: -0.44353 (-0.33534) + | > log_mle: -0.66678 (-0.48115) + | > loss_dur: 0.22326 (0.14581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.70101 (32.07174) + | > current_lr: 0.00007 + | > step_time: 4.20550 (2.97713) + | > loader_time: 0.29510 (0.08994) + + + --> STEP: 223/234 -- GLOBAL_STEP: 68785 + | > loss: -0.49050 (-0.33839) + | > log_mle: -0.71739 (-0.48624) + | > loss_dur: 0.22689 (0.14785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.61200 (33.16005) + | > current_lr: 0.00007 + | > step_time: 2.00800 (2.96161) + | > loader_time: 0.20870 (0.08892) + + + --> STEP: 228/234 -- GLOBAL_STEP: 68790 + | > loss: -0.43985 (-0.34139) + | > log_mle: -0.70586 (-0.49144) + | > loss_dur: 0.26600 (0.15005) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.13906 (34.43447) + | > current_lr: 0.00007 + | > step_time: 0.24440 (2.90963) + | > loader_time: 0.00530 (0.08705) + + + --> STEP: 233/234 -- GLOBAL_STEP: 68795 + | > loss: -0.00945 (-0.34223) + | > log_mle: -0.69534 (-0.49827) + | > loss_dur: 0.68589 (0.15604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.44563 (35.70592) + | > current_lr: 0.00007 + | > step_time: 0.19370 (2.85305) + | > loader_time: 0.00250 (0.08540) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.17147 (+1.16967) + | > avg_loss: -0.36244 (-0.03267) + | > avg_log_mle: -0.58035 (-0.02175) + | > avg_loss_dur: 0.21791 (-0.01092) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_68796.pth + + > EPOCH: 294/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 23:05:52)  + + --> STEP: 4/234 -- GLOBAL_STEP: 68800 + | > loss: -0.33072 (-0.31486) + | > log_mle: -0.41810 (-0.41640) + | > loss_dur: 0.08739 (0.10155) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.31112 (25.31577) + | > current_lr: 0.00007 + | > step_time: 0.77610 (5.53841) + | > loader_time: 0.00150 (0.02763) + + + --> STEP: 9/234 -- GLOBAL_STEP: 68805 + | > loss: -0.30631 (-0.33030) + | > log_mle: -0.42922 (-0.42262) + | > loss_dur: 0.12290 (0.09232) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.31920 (20.52082) + | > current_lr: 0.00007 + | > step_time: 2.18360 (3.01190) + | > loader_time: 0.00140 (0.01340) + + + --> STEP: 14/234 -- GLOBAL_STEP: 68810 + | > loss: -0.35307 (-0.34133) + | > log_mle: -0.43065 (-0.42624) + | > loss_dur: 0.07758 (0.08491) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.38800 (17.84383) + | > current_lr: 0.00007 + | > step_time: 2.50060 (3.20024) + | > loader_time: 0.00270 (0.01620) + + + --> STEP: 19/234 -- GLOBAL_STEP: 68815 + | > loss: -0.37036 (-0.34890) + | > log_mle: -0.43437 (-0.42815) + | > loss_dur: 0.06401 (0.07925) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.45965 (15.91108) + | > current_lr: 0.00007 + | > step_time: 2.10270 (2.70655) + | > loader_time: 0.00130 (0.02105) + + + --> STEP: 24/234 -- GLOBAL_STEP: 68820 + | > loss: -0.36909 (-0.35212) + | > log_mle: -0.42518 (-0.42810) + | > loss_dur: 0.05610 (0.07599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.05428 (14.96981) + | > current_lr: 0.00007 + | > step_time: 3.20770 (2.73434) + | > loader_time: 0.00260 (0.01719) + + + --> STEP: 29/234 -- GLOBAL_STEP: 68825 + | > loss: -0.35580 (-0.35440) + | > log_mle: -0.41832 (-0.42774) + | > loss_dur: 0.06253 (0.07334) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.38505 (14.21081) + | > current_lr: 0.00007 + | > step_time: 6.10150 (2.95932) + | > loader_time: 0.00650 (0.02125) + + + --> STEP: 34/234 -- GLOBAL_STEP: 68830 + | > loss: -0.34927 (-0.35257) + | > log_mle: -0.41226 (-0.42565) + | > loss_dur: 0.06299 (0.07308) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.13727 (13.64450) + | > current_lr: 0.00007 + | > step_time: 4.90550 (3.17785) + | > loader_time: 0.08790 (0.03164) + + + --> STEP: 39/234 -- GLOBAL_STEP: 68835 + | > loss: -0.32070 (-0.34895) + | > log_mle: -0.39795 (-0.42252) + | > loss_dur: 0.07724 (0.07357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.87435 (13.29338) + | > current_lr: 0.00007 + | > step_time: 1.59440 (3.37447) + | > loader_time: 0.00280 (0.03712) + + + --> STEP: 44/234 -- GLOBAL_STEP: 68840 + | > loss: -0.32322 (-0.34601) + | > log_mle: -0.39331 (-0.41998) + | > loss_dur: 0.07009 (0.07397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74818 (12.85940) + | > current_lr: 0.00007 + | > step_time: 1.04800 (3.13272) + | > loader_time: 0.00130 (0.03315) + + + --> STEP: 49/234 -- GLOBAL_STEP: 68845 + | > loss: -0.34278 (-0.34425) + | > log_mle: -0.41460 (-0.41886) + | > loss_dur: 0.07182 (0.07461) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.95363 (12.75387) + | > current_lr: 0.00007 + | > step_time: 1.92360 (2.98642) + | > loader_time: 0.00230 (0.02995) + + + --> STEP: 54/234 -- GLOBAL_STEP: 68850 + | > loss: -0.32177 (-0.34150) + | > log_mle: -0.40042 (-0.41701) + | > loss_dur: 0.07865 (0.07551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.41195 (12.51608) + | > current_lr: 0.00007 + | > step_time: 1.18410 (2.86228) + | > loader_time: 0.00220 (0.02790) + + + --> STEP: 59/234 -- GLOBAL_STEP: 68855 + | > loss: -0.30589 (-0.33978) + | > log_mle: -0.39607 (-0.41549) + | > loss_dur: 0.09018 (0.07571) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.67235 (12.22422) + | > current_lr: 0.00007 + | > step_time: 1.40550 (2.77008) + | > loader_time: 0.07800 (0.02846) + + + --> STEP: 64/234 -- GLOBAL_STEP: 68860 + | > loss: -0.31527 (-0.33613) + | > log_mle: -0.39465 (-0.41382) + | > loss_dur: 0.07938 (0.07769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.61794 (12.28580) + | > current_lr: 0.00007 + | > step_time: 3.40250 (2.73309) + | > loader_time: 0.08440 (0.03040) + + + --> STEP: 69/234 -- GLOBAL_STEP: 68865 + | > loss: -0.29575 (-0.33334) + | > log_mle: -0.38010 (-0.41184) + | > loss_dur: 0.08435 (0.07850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.56455 (12.26084) + | > current_lr: 0.00007 + | > step_time: 2.60490 (2.69754) + | > loader_time: 0.00180 (0.03087) + + + --> STEP: 74/234 -- GLOBAL_STEP: 68870 + | > loss: -0.26880 (-0.32931) + | > log_mle: -0.37089 (-0.40995) + | > loss_dur: 0.10209 (0.08064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.65561 (12.49259) + | > current_lr: 0.00007 + | > step_time: 2.51270 (2.63266) + | > loader_time: 0.08380 (0.03115) + + + --> STEP: 79/234 -- GLOBAL_STEP: 68875 + | > loss: -0.29172 (-0.32638) + | > log_mle: -0.39019 (-0.40841) + | > loss_dur: 0.09846 (0.08203) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.03109 (12.49398) + | > current_lr: 0.00007 + | > step_time: 2.90880 (2.68034) + | > loader_time: 0.00490 (0.03074) + + + --> STEP: 84/234 -- GLOBAL_STEP: 68880 + | > loss: -0.28686 (-0.32361) + | > log_mle: -0.38280 (-0.40693) + | > loss_dur: 0.09595 (0.08332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.47493 (12.61891) + | > current_lr: 0.00007 + | > step_time: 1.48260 (2.62392) + | > loader_time: 0.00400 (0.02910) + + + --> STEP: 89/234 -- GLOBAL_STEP: 68885 + | > loss: -0.29024 (-0.32134) + | > log_mle: -0.40591 (-0.40625) + | > loss_dur: 0.11567 (0.08491) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.16638 (12.72920) + | > current_lr: 0.00007 + | > step_time: 1.08720 (2.60593) + | > loader_time: 0.00210 (0.03065) + + + --> STEP: 94/234 -- GLOBAL_STEP: 68890 + | > loss: -0.29567 (-0.31924) + | > log_mle: -0.42674 (-0.40661) + | > loss_dur: 0.13108 (0.08737) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.93280 (13.19291) + | > current_lr: 0.00007 + | > step_time: 3.99210 (2.55671) + | > loader_time: 0.10870 (0.03181) + + + --> STEP: 99/234 -- GLOBAL_STEP: 68895 + | > loss: -0.29748 (-0.31767) + | > log_mle: -0.45821 (-0.40723) + | > loss_dur: 0.16072 (0.08956) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.60540 (13.47332) + | > current_lr: 0.00007 + | > step_time: 1.78960 (2.53598) + | > loader_time: 0.00190 (0.03130) + + + --> STEP: 104/234 -- GLOBAL_STEP: 68900 + | > loss: -0.32140 (-0.31599) + | > log_mle: -0.46640 (-0.40814) + | > loss_dur: 0.14500 (0.09215) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.55334 (13.86685) + | > current_lr: 0.00007 + | > step_time: 1.39200 (2.49844) + | > loader_time: 0.00180 (0.02989) + + + --> STEP: 109/234 -- GLOBAL_STEP: 68905 + | > loss: -0.25441 (-0.31346) + | > log_mle: -0.43910 (-0.40837) + | > loss_dur: 0.18469 (0.09492) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.51020 (14.19069) + | > current_lr: 0.00007 + | > step_time: 3.45720 (2.47806) + | > loader_time: 0.00320 (0.02945) + + + --> STEP: 114/234 -- GLOBAL_STEP: 68910 + | > loss: -0.28292 (-0.31207) + | > log_mle: -0.42173 (-0.40961) + | > loss_dur: 0.13881 (0.09754) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.74535 (14.68056) + | > current_lr: 0.00007 + | > step_time: 2.49000 (2.44247) + | > loader_time: 0.00280 (0.02830) + + + --> STEP: 119/234 -- GLOBAL_STEP: 68915 + | > loss: -0.27311 (-0.31037) + | > log_mle: -0.41957 (-0.41036) + | > loss_dur: 0.14647 (0.09999) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.26950 (15.00037) + | > current_lr: 0.00007 + | > step_time: 2.56290 (2.42844) + | > loader_time: 0.09820 (0.03049) + + + --> STEP: 124/234 -- GLOBAL_STEP: 68920 + | > loss: -0.28875 (-0.30860) + | > log_mle: -0.44553 (-0.41050) + | > loss_dur: 0.15678 (0.10190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.65785 (15.31432) + | > current_lr: 0.00007 + | > step_time: 1.60960 (2.39466) + | > loader_time: 0.00270 (0.02936) + + + --> STEP: 129/234 -- GLOBAL_STEP: 68925 + | > loss: -0.27110 (-0.30793) + | > log_mle: -0.44142 (-0.41239) + | > loss_dur: 0.17032 (0.10445) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.85399 (15.77616) + | > current_lr: 0.00007 + | > step_time: 1.09680 (2.42821) + | > loader_time: 0.00270 (0.03130) + + + --> STEP: 134/234 -- GLOBAL_STEP: 68930 + | > loss: -0.30329 (-0.30818) + | > log_mle: -0.49165 (-0.41515) + | > loss_dur: 0.18837 (0.10697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.77495 (16.35881) + | > current_lr: 0.00007 + | > step_time: 2.51910 (2.41856) + | > loader_time: 0.00240 (0.03155) + + + --> STEP: 139/234 -- GLOBAL_STEP: 68935 + | > loss: -0.36045 (-0.30804) + | > log_mle: -0.54987 (-0.41760) + | > loss_dur: 0.18942 (0.10955) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.74463 (17.15745) + | > current_lr: 0.00007 + | > step_time: 1.99090 (2.42009) + | > loader_time: 0.00240 (0.03055) + + + --> STEP: 144/234 -- GLOBAL_STEP: 68940 + | > loss: -0.33091 (-0.30796) + | > log_mle: -0.52725 (-0.42028) + | > loss_dur: 0.19634 (0.11232) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.57243 (17.95304) + | > current_lr: 0.00007 + | > step_time: 4.70340 (2.43137) + | > loader_time: 0.00300 (0.03205) + + + --> STEP: 149/234 -- GLOBAL_STEP: 68945 + | > loss: -0.38480 (-0.30897) + | > log_mle: -0.57825 (-0.42352) + | > loss_dur: 0.19345 (0.11455) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.16975 (18.81043) + | > current_lr: 0.00007 + | > step_time: 3.66740 (2.46357) + | > loader_time: 0.00620 (0.03437) + + + --> STEP: 154/234 -- GLOBAL_STEP: 68950 + | > loss: -0.34360 (-0.31038) + | > log_mle: -0.52572 (-0.42735) + | > loss_dur: 0.18212 (0.11697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.08995 (19.48251) + | > current_lr: 0.00007 + | > step_time: 1.32260 (2.45272) + | > loader_time: 0.00220 (0.03342) + + + --> STEP: 159/234 -- GLOBAL_STEP: 68955 + | > loss: -0.35941 (-0.31155) + | > log_mle: -0.54249 (-0.43090) + | > loss_dur: 0.18308 (0.11935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.77901 (20.27671) + | > current_lr: 0.00007 + | > step_time: 2.11820 (2.46487) + | > loader_time: 0.08240 (0.03470) + + + --> STEP: 164/234 -- GLOBAL_STEP: 68960 + | > loss: -0.30715 (-0.31165) + | > log_mle: -0.51076 (-0.43337) + | > loss_dur: 0.20361 (0.12172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.35561 (21.16275) + | > current_lr: 0.00007 + | > step_time: 1.79820 (2.44753) + | > loader_time: 0.07570 (0.03479) + + + --> STEP: 169/234 -- GLOBAL_STEP: 68965 + | > loss: -0.32446 (-0.31278) + | > log_mle: -0.53645 (-0.43679) + | > loss_dur: 0.21198 (0.12402) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.24087 (21.71132) + | > current_lr: 0.00007 + | > step_time: 4.90140 (2.49172) + | > loader_time: 0.00320 (0.03430) + + + --> STEP: 174/234 -- GLOBAL_STEP: 68970 + | > loss: -0.41840 (-0.31508) + | > log_mle: -0.62546 (-0.44157) + | > loss_dur: 0.20706 (0.12649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.43136 (22.33671) + | > current_lr: 0.00007 + | > step_time: 5.49020 (2.49100) + | > loader_time: 0.06620 (0.03471) + + + --> STEP: 179/234 -- GLOBAL_STEP: 68975 + | > loss: -0.38671 (-0.31684) + | > log_mle: -0.62394 (-0.44598) + | > loss_dur: 0.23724 (0.12913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.61404 (23.01864) + | > current_lr: 0.00007 + | > step_time: 5.70250 (2.52993) + | > loader_time: 0.00620 (0.03478) + + + --> STEP: 184/234 -- GLOBAL_STEP: 68980 + | > loss: -0.38609 (-0.31839) + | > log_mle: -0.59579 (-0.44995) + | > loss_dur: 0.20970 (0.13157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.22869 (23.91529) + | > current_lr: 0.00007 + | > step_time: 2.10830 (2.56729) + | > loader_time: 0.09440 (0.03748) + + + --> STEP: 189/234 -- GLOBAL_STEP: 68985 + | > loss: -0.38051 (-0.32035) + | > log_mle: -0.59562 (-0.45433) + | > loss_dur: 0.21511 (0.13398) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.71179 (25.05717) + | > current_lr: 0.00007 + | > step_time: 2.39890 (2.64700) + | > loader_time: 0.10480 (0.03760) + + + --> STEP: 194/234 -- GLOBAL_STEP: 68990 + | > loss: -0.42598 (-0.32288) + | > log_mle: -0.63455 (-0.45882) + | > loss_dur: 0.20856 (0.13594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.88271 (25.95803) + | > current_lr: 0.00007 + | > step_time: 9.79870 (2.75359) + | > loader_time: 0.00370 (0.03766) + + + --> STEP: 199/234 -- GLOBAL_STEP: 68995 + | > loss: -0.42964 (-0.32514) + | > log_mle: -0.64546 (-0.46303) + | > loss_dur: 0.21582 (0.13789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.04983 (26.95189) + | > current_lr: 0.00007 + | > step_time: 8.28830 (2.79948) + | > loader_time: 0.30220 (0.04002) + + + --> STEP: 204/234 -- GLOBAL_STEP: 69000 + | > loss: -0.45111 (-0.32721) + | > log_mle: -0.69006 (-0.46724) + | > loss_dur: 0.23895 (0.14004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.75687 (27.88201) + | > current_lr: 0.00007 + | > step_time: 1.60510 (2.79612) + | > loader_time: 0.09430 (0.04090) + + + --> STEP: 209/234 -- GLOBAL_STEP: 69005 + | > loss: -0.42965 (-0.32954) + | > log_mle: -0.63906 (-0.47163) + | > loss_dur: 0.20942 (0.14208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.10246 (28.95798) + | > current_lr: 0.00007 + | > step_time: 5.40170 (2.82632) + | > loader_time: 0.39790 (0.04232) + + + --> STEP: 214/234 -- GLOBAL_STEP: 69010 + | > loss: -0.47038 (-0.33288) + | > log_mle: -0.67536 (-0.47701) + | > loss_dur: 0.20498 (0.14413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.27876 (29.97292) + | > current_lr: 0.00007 + | > step_time: 3.60240 (2.90195) + | > loader_time: 0.09790 (0.04312) + + + --> STEP: 219/234 -- GLOBAL_STEP: 69015 + | > loss: -0.54221 (-0.33596) + | > log_mle: -0.78062 (-0.48226) + | > loss_dur: 0.23841 (0.14629) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.62864 (30.98426) + | > current_lr: 0.00007 + | > step_time: 7.50750 (3.00550) + | > loader_time: 0.00470 (0.04309) + + + --> STEP: 224/234 -- GLOBAL_STEP: 69020 + | > loss: -0.48901 (-0.33889) + | > log_mle: -0.71753 (-0.48714) + | > loss_dur: 0.22852 (0.14825) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.93470 (31.81943) + | > current_lr: 0.00007 + | > step_time: 3.10480 (2.99780) + | > loader_time: 0.00370 (0.04301) + + + --> STEP: 229/234 -- GLOBAL_STEP: 69025 + | > loss: -0.48275 (-0.34204) + | > log_mle: -0.77426 (-0.49268) + | > loss_dur: 0.29150 (0.15064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.15722 (32.92699) + | > current_lr: 0.00007 + | > step_time: 1.09150 (2.96424) + | > loader_time: 0.00270 (0.04729) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.67614 (-0.49534) + | > avg_loss: -0.34031 (+0.02213) + | > avg_log_mle: -0.56199 (+0.01836) + | > avg_loss_dur: 0.22168 (+0.00377) + + + > EPOCH: 295/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 23:18:37)  + + --> STEP: 0/234 -- GLOBAL_STEP: 69030 + | > loss: -0.33037 (-0.33037) + | > log_mle: -0.50599 (-0.50599) + | > loss_dur: 0.17562 (0.17562) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.36000 (31.36000) + | > current_lr: 0.00007 + | > step_time: 11.11700 (11.11700) + | > loader_time: 17.24750 (17.24746) + + + --> STEP: 5/234 -- GLOBAL_STEP: 69035 + | > loss: -0.31222 (-0.31897) + | > log_mle: -0.42034 (-0.41892) + | > loss_dur: 0.10812 (0.09995) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.79482 (20.53831) + | > current_lr: 0.00007 + | > step_time: 1.09860 (4.31729) + | > loader_time: 0.00120 (0.02397) + + + --> STEP: 10/234 -- GLOBAL_STEP: 69040 + | > loss: -0.34684 (-0.33472) + | > log_mle: -0.42580 (-0.42400) + | > loss_dur: 0.07896 (0.08928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.14133 (18.80491) + | > current_lr: 0.00007 + | > step_time: 3.82550 (2.91652) + | > loader_time: 0.10050 (0.02287) + + + --> STEP: 15/234 -- GLOBAL_STEP: 69045 + | > loss: -0.36556 (-0.34614) + | > log_mle: -0.43644 (-0.42845) + | > loss_dur: 0.07088 (0.08231) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.32884 (16.96480) + | > current_lr: 0.00007 + | > step_time: 4.80490 (2.81180) + | > loader_time: 0.10210 (0.02901) + + + --> STEP: 20/234 -- GLOBAL_STEP: 69050 + | > loss: -0.38378 (-0.35435) + | > log_mle: -0.43744 (-0.43028) + | > loss_dur: 0.05366 (0.07593) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.88927 (15.20658) + | > current_lr: 0.00007 + | > step_time: 2.10610 (3.36584) + | > loader_time: 0.08690 (0.04415) + + + --> STEP: 25/234 -- GLOBAL_STEP: 69055 + | > loss: -0.36357 (-0.35490) + | > log_mle: -0.41850 (-0.42878) + | > loss_dur: 0.05493 (0.07387) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.56652 (14.86053) + | > current_lr: 0.00007 + | > step_time: 1.98320 (3.09848) + | > loader_time: 0.00120 (0.03569) + + + --> STEP: 30/234 -- GLOBAL_STEP: 69060 + | > loss: -0.35039 (-0.35525) + | > log_mle: -0.42100 (-0.42838) + | > loss_dur: 0.07061 (0.07314) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.09617 (14.25530) + | > current_lr: 0.00007 + | > step_time: 1.36700 (3.13810) + | > loader_time: 0.00200 (0.03912) + + + --> STEP: 35/234 -- GLOBAL_STEP: 69065 + | > loss: -0.31387 (-0.35227) + | > log_mle: -0.39975 (-0.42570) + | > loss_dur: 0.08589 (0.07343) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.10442 (13.62631) + | > current_lr: 0.00007 + | > step_time: 1.37440 (2.93298) + | > loader_time: 0.00120 (0.03635) + + + --> STEP: 40/234 -- GLOBAL_STEP: 69070 + | > loss: -0.31254 (-0.34825) + | > log_mle: -0.39995 (-0.42250) + | > loss_dur: 0.08740 (0.07425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.84292 (13.10553) + | > current_lr: 0.00007 + | > step_time: 1.69580 (2.78026) + | > loader_time: 0.05980 (0.03352) + + + --> STEP: 45/234 -- GLOBAL_STEP: 69075 + | > loss: -0.31186 (-0.34557) + | > log_mle: -0.40536 (-0.42030) + | > loss_dur: 0.09350 (0.07473) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.09318 (12.75499) + | > current_lr: 0.00007 + | > step_time: 1.89440 (2.69957) + | > loader_time: 0.00190 (0.03180) + + + --> STEP: 50/234 -- GLOBAL_STEP: 69080 + | > loss: -0.33407 (-0.34466) + | > log_mle: -0.40181 (-0.41912) + | > loss_dur: 0.06775 (0.07446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.09092 (12.32013) + | > current_lr: 0.00007 + | > step_time: 1.20460 (2.62235) + | > loader_time: 0.07450 (0.03376) + + + --> STEP: 55/234 -- GLOBAL_STEP: 69085 + | > loss: -0.34617 (-0.34257) + | > log_mle: -0.41269 (-0.41778) + | > loss_dur: 0.06653 (0.07521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.17904 (11.94914) + | > current_lr: 0.00007 + | > step_time: 1.20650 (2.56650) + | > loader_time: 0.07970 (0.03834) + + + --> STEP: 60/234 -- GLOBAL_STEP: 69090 + | > loss: -0.29270 (-0.34034) + | > log_mle: -0.40364 (-0.41644) + | > loss_dur: 0.11094 (0.07610) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.85949 (11.84023) + | > current_lr: 0.00007 + | > step_time: 1.69580 (2.47225) + | > loader_time: 0.00200 (0.03662) + + + --> STEP: 65/234 -- GLOBAL_STEP: 69095 + | > loss: -0.30810 (-0.33693) + | > log_mle: -0.39056 (-0.41473) + | > loss_dur: 0.08247 (0.07780) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.76763 (11.88413) + | > current_lr: 0.00007 + | > step_time: 1.16080 (2.41083) + | > loader_time: 0.00180 (0.03398) + + + --> STEP: 70/234 -- GLOBAL_STEP: 69100 + | > loss: -0.27138 (-0.33383) + | > log_mle: -0.37354 (-0.41259) + | > loss_dur: 0.10216 (0.07875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.76478 (11.92991) + | > current_lr: 0.00007 + | > step_time: 1.20120 (2.33851) + | > loader_time: 0.00200 (0.03171) + + + --> STEP: 75/234 -- GLOBAL_STEP: 69105 + | > loss: -0.27662 (-0.33029) + | > log_mle: -0.38797 (-0.41104) + | > loss_dur: 0.11135 (0.08075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.94819 (12.17363) + | > current_lr: 0.00007 + | > step_time: 2.60460 (2.34124) + | > loader_time: 0.00230 (0.02980) + + + --> STEP: 80/234 -- GLOBAL_STEP: 69110 + | > loss: -0.29759 (-0.32764) + | > log_mle: -0.38119 (-0.40927) + | > loss_dur: 0.08360 (0.08162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.26355 (12.30279) + | > current_lr: 0.00007 + | > step_time: 2.40610 (2.33485) + | > loader_time: 0.08720 (0.03036) + + + --> STEP: 85/234 -- GLOBAL_STEP: 69115 + | > loss: -0.28445 (-0.32451) + | > log_mle: -0.38142 (-0.40779) + | > loss_dur: 0.09697 (0.08328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.80330 (12.48116) + | > current_lr: 0.00007 + | > step_time: 2.20560 (2.34218) + | > loader_time: 0.00290 (0.03099) + + + --> STEP: 90/234 -- GLOBAL_STEP: 69120 + | > loss: -0.27281 (-0.32231) + | > log_mle: -0.39833 (-0.40740) + | > loss_dur: 0.12552 (0.08509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.57273 (12.64077) + | > current_lr: 0.00007 + | > step_time: 2.22910 (2.34022) + | > loader_time: 0.07980 (0.03295) + + + --> STEP: 95/234 -- GLOBAL_STEP: 69125 + | > loss: -0.32669 (-0.32094) + | > log_mle: -0.47863 (-0.40867) + | > loss_dur: 0.15194 (0.08773) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.26888 (12.99990) + | > current_lr: 0.00007 + | > step_time: 3.00690 (2.32513) + | > loader_time: 0.00220 (0.03222) + + + --> STEP: 100/234 -- GLOBAL_STEP: 69130 + | > loss: -0.28157 (-0.31904) + | > log_mle: -0.40262 (-0.40850) + | > loss_dur: 0.12106 (0.08946) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.36322 (13.31209) + | > current_lr: 0.00007 + | > step_time: 2.99690 (2.29864) + | > loader_time: 0.09400 (0.03168) + + + --> STEP: 105/234 -- GLOBAL_STEP: 69135 + | > loss: -0.26970 (-0.31727) + | > log_mle: -0.38594 (-0.40926) + | > loss_dur: 0.11625 (0.09199) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.33450 (14.00553) + | > current_lr: 0.00007 + | > step_time: 1.39450 (2.27593) + | > loader_time: 0.00320 (0.03192) + + + --> STEP: 110/234 -- GLOBAL_STEP: 69140 + | > loss: -0.26017 (-0.31500) + | > log_mle: -0.40592 (-0.40976) + | > loss_dur: 0.14575 (0.09476) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.34105 (14.37971) + | > current_lr: 0.00007 + | > step_time: 1.60400 (2.26880) + | > loader_time: 0.09090 (0.03219) + + + --> STEP: 115/234 -- GLOBAL_STEP: 69145 + | > loss: -0.27702 (-0.31376) + | > log_mle: -0.42919 (-0.41126) + | > loss_dur: 0.15217 (0.09750) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.67669 (14.81740) + | > current_lr: 0.00007 + | > step_time: 2.49020 (2.29790) + | > loader_time: 0.00220 (0.03327) + + + --> STEP: 120/234 -- GLOBAL_STEP: 69150 + | > loss: -0.30398 (-0.31238) + | > log_mle: -0.47185 (-0.41240) + | > loss_dur: 0.16787 (0.10001) + | > amp_scaler: 2048.00000 (1066.66667) + | > grad_norm: 26.88648 (15.30386) + | > current_lr: 0.00007 + | > step_time: 1.09890 (2.28636) + | > loader_time: 0.00410 (0.03369) + + + --> STEP: 125/234 -- GLOBAL_STEP: 69155 + | > loss: -0.28557 (-0.31046) + | > log_mle: -0.45177 (-0.41230) + | > loss_dur: 0.16620 (0.10184) + | > amp_scaler: 2048.00000 (1105.92000) + | > grad_norm: 23.69977 (15.43075) + | > current_lr: 0.00007 + | > step_time: 2.26430 (2.27527) + | > loader_time: 0.00190 (0.03309) + + + --> STEP: 130/234 -- GLOBAL_STEP: 69160 + | > loss: -0.29766 (-0.30970) + | > log_mle: -0.46834 (-0.41407) + | > loss_dur: 0.17068 (0.10436) + | > amp_scaler: 2048.00000 (1142.15385) + | > grad_norm: 25.01925 (15.91982) + | > current_lr: 0.00007 + | > step_time: 1.59050 (2.26841) + | > loader_time: 0.00740 (0.03198) + + + --> STEP: 135/234 -- GLOBAL_STEP: 69165 + | > loss: -0.26278 (-0.30946) + | > log_mle: -0.39999 (-0.41604) + | > loss_dur: 0.13721 (0.10658) + | > amp_scaler: 2048.00000 (1175.70370) + | > grad_norm: 20.87560 (16.46174) + | > current_lr: 0.00007 + | > step_time: 1.17690 (2.26951) + | > loader_time: 0.00180 (0.03284) + + + --> STEP: 140/234 -- GLOBAL_STEP: 69170 + | > loss: -0.25143 (-0.30929) + | > log_mle: -0.42926 (-0.41862) + | > loss_dur: 0.17783 (0.10932) + | > amp_scaler: 2048.00000 (1206.85714) + | > grad_norm: 29.82840 (17.07125) + | > current_lr: 0.00007 + | > step_time: 2.91080 (2.26722) + | > loader_time: 0.00260 (0.03301) + + + --> STEP: 145/234 -- GLOBAL_STEP: 69175 + | > loss: -0.35159 (-0.31006) + | > log_mle: -0.53143 (-0.42200) + | > loss_dur: 0.17983 (0.11194) + | > amp_scaler: 2048.00000 (1235.86207) + | > grad_norm: 41.73064 (17.73934) + | > current_lr: 0.00007 + | > step_time: 6.00790 (2.30645) + | > loader_time: 0.08950 (0.03375) + + + --> STEP: 150/234 -- GLOBAL_STEP: 69180 + | > loss: -0.31465 (-0.31069) + | > log_mle: -0.51712 (-0.42511) + | > loss_dur: 0.20247 (0.11442) + | > amp_scaler: 2048.00000 (1262.93333) + | > grad_norm: 37.89471 (18.39390) + | > current_lr: 0.00007 + | > step_time: 5.12060 (2.34641) + | > loader_time: 0.28780 (0.03716) + + + --> STEP: 155/234 -- GLOBAL_STEP: 69185 + | > loss: -0.38339 (-0.31258) + | > log_mle: -0.58029 (-0.42943) + | > loss_dur: 0.19691 (0.11685) + | > amp_scaler: 2048.00000 (1288.25806) + | > grad_norm: 63.04937 (19.25145) + | > current_lr: 0.00007 + | > step_time: 5.69120 (2.38551) + | > loader_time: 0.11080 (0.03957) + + + --> STEP: 160/234 -- GLOBAL_STEP: 69190 + | > loss: -0.38028 (-0.31377) + | > log_mle: -0.58959 (-0.43311) + | > loss_dur: 0.20931 (0.11934) + | > amp_scaler: 2048.00000 (1312.00000) + | > grad_norm: 38.46312 (19.95434) + | > current_lr: 0.00007 + | > step_time: 2.29330 (2.45222) + | > loader_time: 0.00470 (0.04153) + + + --> STEP: 165/234 -- GLOBAL_STEP: 69195 + | > loss: -0.39731 (-0.31519) + | > log_mle: -0.59204 (-0.43674) + | > loss_dur: 0.19473 (0.12155) + | > amp_scaler: 2048.00000 (1334.30303) + | > grad_norm: 54.20234 (20.71709) + | > current_lr: 0.00007 + | > step_time: 1.50490 (2.46013) + | > loader_time: 0.00240 (0.04228) + + + --> STEP: 170/234 -- GLOBAL_STEP: 69200 + | > loss: -0.41133 (-0.31725) + | > log_mle: -0.62680 (-0.44100) + | > loss_dur: 0.21547 (0.12375) + | > amp_scaler: 2048.00000 (1355.29412) + | > grad_norm: 63.39460 (21.59169) + | > current_lr: 0.00007 + | > step_time: 3.89280 (2.49490) + | > loader_time: 0.00590 (0.04279) + + + --> STEP: 175/234 -- GLOBAL_STEP: 69205 + | > loss: -0.36459 (-0.31983) + | > log_mle: -0.59771 (-0.44606) + | > loss_dur: 0.23312 (0.12622) + | > amp_scaler: 2048.00000 (1375.08571) + | > grad_norm: 66.39371 (22.64631) + | > current_lr: 0.00007 + | > step_time: 1.29240 (2.58826) + | > loader_time: 0.00360 (0.04268) + + + --> STEP: 180/234 -- GLOBAL_STEP: 69210 + | > loss: -0.40138 (-0.32199) + | > log_mle: -0.60511 (-0.45067) + | > loss_dur: 0.20373 (0.12868) + | > amp_scaler: 2048.00000 (1393.77778) + | > grad_norm: 70.60366 (23.77258) + | > current_lr: 0.00007 + | > step_time: 1.49650 (2.62414) + | > loader_time: 0.00320 (0.04365) + + + --> STEP: 185/234 -- GLOBAL_STEP: 69215 + | > loss: -0.43300 (-0.32395) + | > log_mle: -0.64562 (-0.45502) + | > loss_dur: 0.21261 (0.13107) + | > amp_scaler: 2048.00000 (1411.45946) + | > grad_norm: 66.36342 (24.74582) + | > current_lr: 0.00007 + | > step_time: 10.21010 (2.65716) + | > loader_time: 0.19040 (0.04398) + + + --> STEP: 190/234 -- GLOBAL_STEP: 69220 + | > loss: -0.39961 (-0.32601) + | > log_mle: -0.59952 (-0.45924) + | > loss_dur: 0.19991 (0.13323) + | > amp_scaler: 2048.00000 (1428.21053) + | > grad_norm: 75.93807 (25.86859) + | > current_lr: 0.00007 + | > step_time: 9.40370 (2.68201) + | > loader_time: 0.69600 (0.04704) + + + --> STEP: 195/234 -- GLOBAL_STEP: 69225 + | > loss: -0.40153 (-0.32831) + | > log_mle: -0.62800 (-0.46355) + | > loss_dur: 0.22647 (0.13524) + | > amp_scaler: 2048.00000 (1444.10256) + | > grad_norm: 53.93678 (26.73685) + | > current_lr: 0.00007 + | > step_time: 5.70180 (2.73465) + | > loader_time: 0.09360 (0.04907) + + + --> STEP: 200/234 -- GLOBAL_STEP: 69230 + | > loss: -0.40346 (-0.33026) + | > log_mle: -0.63991 (-0.46760) + | > loss_dur: 0.23645 (0.13734) + | > amp_scaler: 2048.00000 (1459.20000) + | > grad_norm: 53.45389 (27.62897) + | > current_lr: 0.00007 + | > step_time: 3.91530 (2.80187) + | > loader_time: 0.09410 (0.05028) + + + --> STEP: 205/234 -- GLOBAL_STEP: 69235 + | > loss: -0.40264 (-0.33215) + | > log_mle: -0.62410 (-0.47157) + | > loss_dur: 0.22146 (0.13942) + | > amp_scaler: 2048.00000 (1473.56098) + | > grad_norm: 49.46373 (28.33020) + | > current_lr: 0.00007 + | > step_time: 7.19270 (2.89291) + | > loader_time: 0.00360 (0.05095) + + + --> STEP: 210/234 -- GLOBAL_STEP: 69240 + | > loss: -0.48155 (-0.33497) + | > log_mle: -0.71144 (-0.47642) + | > loss_dur: 0.22989 (0.14145) + | > amp_scaler: 2048.00000 (1487.23810) + | > grad_norm: 64.55531 (29.22725) + | > current_lr: 0.00007 + | > step_time: 2.69030 (2.91255) + | > loader_time: 0.00470 (0.04986) + + + --> STEP: 215/234 -- GLOBAL_STEP: 69245 + | > loss: -0.42622 (-0.33792) + | > log_mle: -0.66157 (-0.48146) + | > loss_dur: 0.23535 (0.14354) + | > amp_scaler: 2048.00000 (1500.27907) + | > grad_norm: 60.47989 (30.19238) + | > current_lr: 0.00007 + | > step_time: 7.57940 (2.98243) + | > loader_time: 0.21460 (0.05159) + + + --> STEP: 220/234 -- GLOBAL_STEP: 69250 + | > loss: -0.46950 (-0.34135) + | > log_mle: -0.71472 (-0.48699) + | > loss_dur: 0.24522 (0.14565) + | > amp_scaler: 2048.00000 (1512.72727) + | > grad_norm: 82.68489 (31.34740) + | > current_lr: 0.00007 + | > step_time: 4.31470 (3.02164) + | > loader_time: 0.18700 (0.05296) + + + --> STEP: 225/234 -- GLOBAL_STEP: 69255 + | > loss: -0.55147 (-0.34453) + | > log_mle: -0.79046 (-0.49220) + | > loss_dur: 0.23899 (0.14767) + | > amp_scaler: 2048.00000 (1524.62222) + | > grad_norm: 79.59850 (32.39869) + | > current_lr: 0.00007 + | > step_time: 0.24000 (3.00034) + | > loader_time: 0.00320 (0.05196) + + + --> STEP: 230/234 -- GLOBAL_STEP: 69260 + | > loss: -0.54161 (-0.34763) + | > log_mle: -0.84510 (-0.49796) + | > loss_dur: 0.30349 (0.15033) + | > amp_scaler: 2048.00000 (1536.00000) + | > grad_norm: 83.66848 (33.34889) + | > current_lr: 0.00007 + | > step_time: 0.26220 (2.94051) + | > loader_time: 0.00290 (0.05090) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.07499 (-0.60115) + | > avg_loss: -0.34353 (-0.00322) + | > avg_log_mle: -0.56404 (-0.00205) + | > avg_loss_dur: 0.22052 (-0.00117) + + + > EPOCH: 296/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 23:31:22)  + + --> STEP: 1/234 -- GLOBAL_STEP: 69265 + | > loss: -0.32075 (-0.32075) + | > log_mle: -0.41422 (-0.41422) + | > loss_dur: 0.09347 (0.09347) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.24997 (26.24997) + | > current_lr: 0.00007 + | > step_time: 0.78690 (0.78687) + | > loader_time: 0.00250 (0.00249) + + + --> STEP: 6/234 -- GLOBAL_STEP: 69270 + | > loss: -0.35015 (-0.32456) + | > log_mle: -0.42474 (-0.42056) + | > loss_dur: 0.07459 (0.09600) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.78625 (19.90357) + | > current_lr: 0.00007 + | > step_time: 10.50800 (4.11744) + | > loader_time: 0.09190 (1.41371) + + + --> STEP: 11/234 -- GLOBAL_STEP: 69275 + | > loss: -0.38089 (-0.33823) + | > log_mle: -0.44019 (-0.42601) + | > loss_dur: 0.05930 (0.08778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.88087 (19.15021) + | > current_lr: 0.00007 + | > step_time: 1.40340 (3.35607) + | > loader_time: 0.09220 (0.78745) + + + --> STEP: 16/234 -- GLOBAL_STEP: 69280 + | > loss: -0.38959 (-0.34843) + | > log_mle: -0.44436 (-0.42940) + | > loss_dur: 0.05477 (0.08097) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.99910 (17.92295) + | > current_lr: 0.00007 + | > step_time: 0.93400 (3.52339) + | > loader_time: 0.07980 (0.56951) + + + --> STEP: 21/234 -- GLOBAL_STEP: 69285 + | > loss: -0.34190 (-0.35232) + | > log_mle: -0.41306 (-0.42948) + | > loss_dur: 0.07116 (0.07716) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.09458 (16.57622) + | > current_lr: 0.00007 + | > step_time: 2.30500 (2.96262) + | > loader_time: 0.00290 (0.43429) + + + --> STEP: 26/234 -- GLOBAL_STEP: 69290 + | > loss: -0.35353 (-0.35330) + | > log_mle: -0.41884 (-0.42867) + | > loss_dur: 0.06531 (0.07537) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.61196 (15.45583) + | > current_lr: 0.00007 + | > step_time: 2.24220 (2.71610) + | > loader_time: 0.09590 (0.35479) + + + --> STEP: 31/234 -- GLOBAL_STEP: 69295 + | > loss: -0.32672 (-0.35384) + | > log_mle: -0.40580 (-0.42798) + | > loss_dur: 0.07908 (0.07414) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.79951 (15.06778) + | > current_lr: 0.00007 + | > step_time: 2.80980 (2.62833) + | > loader_time: 0.09120 (0.30094) + + + --> STEP: 36/234 -- GLOBAL_STEP: 69300 + | > loss: -0.31104 (-0.35101) + | > log_mle: -0.39846 (-0.42522) + | > loss_dur: 0.08743 (0.07421) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.41240 (14.72899) + | > current_lr: 0.00007 + | > step_time: 1.89540 (2.75492) + | > loader_time: 0.00230 (0.27590) + + + --> STEP: 41/234 -- GLOBAL_STEP: 69305 + | > loss: -0.35564 (-0.34816) + | > log_mle: -0.41705 (-0.42262) + | > loss_dur: 0.06141 (0.07446) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.47569 (14.23211) + | > current_lr: 0.00007 + | > step_time: 2.99610 (2.79295) + | > loader_time: 0.00260 (0.25127) + + + --> STEP: 46/234 -- GLOBAL_STEP: 69310 + | > loss: -0.31661 (-0.34491) + | > log_mle: -0.40383 (-0.42000) + | > loss_dur: 0.08722 (0.07508) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.42900 (13.72083) + | > current_lr: 0.00007 + | > step_time: 1.20320 (2.62658) + | > loader_time: 0.00210 (0.22595) + + + --> STEP: 51/234 -- GLOBAL_STEP: 69315 + | > loss: -0.32002 (-0.34384) + | > log_mle: -0.39960 (-0.41879) + | > loss_dur: 0.07958 (0.07495) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.92715 (13.10247) + | > current_lr: 0.00007 + | > step_time: 1.00370 (2.51612) + | > loader_time: 0.00210 (0.20957) + + + --> STEP: 56/234 -- GLOBAL_STEP: 69320 + | > loss: -0.31539 (-0.34161) + | > log_mle: -0.40195 (-0.41702) + | > loss_dur: 0.08656 (0.07541) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.47832 (13.07161) + | > current_lr: 0.00007 + | > step_time: 2.00500 (2.44365) + | > loader_time: 0.05880 (0.19547) + + + --> STEP: 61/234 -- GLOBAL_STEP: 69325 + | > loss: -0.29742 (-0.33878) + | > log_mle: -0.38832 (-0.41515) + | > loss_dur: 0.09090 (0.07637) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.99017 (12.97285) + | > current_lr: 0.00007 + | > step_time: 1.16920 (2.35435) + | > loader_time: 0.00190 (0.18105) + + + --> STEP: 66/234 -- GLOBAL_STEP: 69330 + | > loss: -0.31447 (-0.33567) + | > log_mle: -0.38607 (-0.41313) + | > loss_dur: 0.07159 (0.07747) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.23203 (12.78866) + | > current_lr: 0.00007 + | > step_time: 4.16200 (2.33134) + | > loader_time: 0.12720 (0.17398) + + + --> STEP: 71/234 -- GLOBAL_STEP: 69335 + | > loss: -0.27808 (-0.33184) + | > log_mle: -0.39352 (-0.41088) + | > loss_dur: 0.11544 (0.07904) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.38954 (12.78336) + | > current_lr: 0.00007 + | > step_time: 2.09360 (2.31204) + | > loader_time: 0.00330 (0.16311) + + + --> STEP: 76/234 -- GLOBAL_STEP: 69340 + | > loss: -0.27388 (-0.32808) + | > log_mle: -0.37966 (-0.40881) + | > loss_dur: 0.10577 (0.08073) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.35766 (12.88188) + | > current_lr: 0.00007 + | > step_time: 1.02260 (2.26545) + | > loader_time: 0.07390 (0.15565) + + + --> STEP: 81/234 -- GLOBAL_STEP: 69345 + | > loss: -0.27625 (-0.32521) + | > log_mle: -0.38730 (-0.40695) + | > loss_dur: 0.11105 (0.08174) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.40197 (12.85815) + | > current_lr: 0.00007 + | > step_time: 2.49180 (2.27122) + | > loader_time: 0.00290 (0.14626) + + + --> STEP: 86/234 -- GLOBAL_STEP: 69350 + | > loss: -0.28440 (-0.32235) + | > log_mle: -0.38669 (-0.40543) + | > loss_dur: 0.10230 (0.08308) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.73739 (12.69949) + | > current_lr: 0.00007 + | > step_time: 1.00120 (2.23698) + | > loader_time: 0.08500 (0.13887) + + + --> STEP: 91/234 -- GLOBAL_STEP: 69355 + | > loss: -0.27086 (-0.31991) + | > log_mle: -0.39838 (-0.40505) + | > loss_dur: 0.12752 (0.08514) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.55507 (12.58825) + | > current_lr: 0.00007 + | > step_time: 0.99890 (2.22273) + | > loader_time: 0.00160 (0.13415) + + + --> STEP: 96/234 -- GLOBAL_STEP: 69360 + | > loss: -0.27185 (-0.31870) + | > log_mle: -0.38472 (-0.40613) + | > loss_dur: 0.11287 (0.08744) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.71942 (12.85213) + | > current_lr: 0.00007 + | > step_time: 1.69180 (2.21222) + | > loader_time: 0.00220 (0.12812) + + + --> STEP: 101/234 -- GLOBAL_STEP: 69365 + | > loss: -0.28284 (-0.31679) + | > log_mle: -0.43053 (-0.40647) + | > loss_dur: 0.14769 (0.08969) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.88175 (13.12686) + | > current_lr: 0.00007 + | > step_time: 1.98500 (2.29471) + | > loader_time: 0.00270 (0.12471) + + + --> STEP: 106/234 -- GLOBAL_STEP: 69370 + | > loss: -0.26751 (-0.31521) + | > log_mle: -0.42674 (-0.40734) + | > loss_dur: 0.15923 (0.09214) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 21.83071 (13.51133) + | > current_lr: 0.00007 + | > step_time: 0.90540 (2.26952) + | > loader_time: 0.00200 (0.11974) + + + --> STEP: 111/234 -- GLOBAL_STEP: 69375 + | > loss: -0.30645 (-0.31349) + | > log_mle: -0.48466 (-0.40843) + | > loss_dur: 0.17821 (0.09494) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 27.13746 (13.90914) + | > current_lr: 0.00007 + | > step_time: 2.81060 (2.29175) + | > loader_time: 0.00330 (0.11526) + + + --> STEP: 116/234 -- GLOBAL_STEP: 69380 + | > loss: -0.26437 (-0.31185) + | > log_mle: -0.44862 (-0.40963) + | > loss_dur: 0.18425 (0.09778) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.98643 (14.39543) + | > current_lr: 0.00007 + | > step_time: 2.20920 (2.26280) + | > loader_time: 0.00500 (0.11187) + + + --> STEP: 121/234 -- GLOBAL_STEP: 69385 + | > loss: -0.23253 (-0.31029) + | > log_mle: -0.36602 (-0.41025) + | > loss_dur: 0.13349 (0.09996) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.97017 (14.56416) + | > current_lr: 0.00007 + | > step_time: 2.30640 (2.27270) + | > loader_time: 0.09970 (0.10889) + + + --> STEP: 126/234 -- GLOBAL_STEP: 69390 + | > loss: -0.31372 (-0.30921) + | > log_mle: -0.49335 (-0.41139) + | > loss_dur: 0.17964 (0.10218) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.79821 (14.98487) + | > current_lr: 0.00007 + | > step_time: 1.99400 (2.30389) + | > loader_time: 0.00310 (0.10539) + + + --> STEP: 131/234 -- GLOBAL_STEP: 69395 + | > loss: -0.37250 (-0.30929) + | > log_mle: -0.54552 (-0.41384) + | > loss_dur: 0.17303 (0.10455) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 51.79768 (15.73005) + | > current_lr: 0.00007 + | > step_time: 1.66210 (2.29280) + | > loader_time: 0.09660 (0.10422) + + + --> STEP: 136/234 -- GLOBAL_STEP: 69400 + | > loss: -0.38288 (-0.30937) + | > log_mle: -0.58636 (-0.41622) + | > loss_dur: 0.20347 (0.10685) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 62.61084 (16.69151) + | > current_lr: 0.00007 + | > step_time: 2.22670 (2.29556) + | > loader_time: 0.07610 (0.10165) + + + --> STEP: 141/234 -- GLOBAL_STEP: 69405 + | > loss: -0.32188 (-0.30884) + | > log_mle: -0.48894 (-0.41818) + | > loss_dur: 0.16706 (0.10935) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 30.20610 (17.26071) + | > current_lr: 0.00007 + | > step_time: 1.68090 (2.29752) + | > loader_time: 0.00270 (0.09934) + + + --> STEP: 146/234 -- GLOBAL_STEP: 69410 + | > loss: -0.34171 (-0.30985) + | > log_mle: -0.54085 (-0.42205) + | > loss_dur: 0.19915 (0.11220) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 52.37147 (18.12891) + | > current_lr: 0.00007 + | > step_time: 7.28530 (2.35583) + | > loader_time: 0.10660 (0.10072) + + + --> STEP: 151/234 -- GLOBAL_STEP: 69415 + | > loss: -0.32850 (-0.31068) + | > log_mle: -0.49874 (-0.42505) + | > loss_dur: 0.17024 (0.11437) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.38672 (18.88990) + | > current_lr: 0.00007 + | > step_time: 1.20030 (2.35528) + | > loader_time: 0.08410 (0.09915) + + + --> STEP: 156/234 -- GLOBAL_STEP: 69420 + | > loss: -0.36376 (-0.31289) + | > log_mle: -0.55450 (-0.42974) + | > loss_dur: 0.19074 (0.11685) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 36.20240 (19.76521) + | > current_lr: 0.00007 + | > step_time: 3.21080 (2.37208) + | > loader_time: 0.08410 (0.09917) + + + --> STEP: 161/234 -- GLOBAL_STEP: 69425 + | > loss: -0.39631 (-0.31433) + | > log_mle: -0.57505 (-0.43352) + | > loss_dur: 0.17874 (0.11919) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 46.18655 (20.52272) + | > current_lr: 0.00007 + | > step_time: 2.59660 (2.39252) + | > loader_time: 0.09670 (0.09902) + + + --> STEP: 166/234 -- GLOBAL_STEP: 69430 + | > loss: -0.33921 (-0.31533) + | > log_mle: -0.51680 (-0.43671) + | > loss_dur: 0.17759 (0.12137) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.00534 (21.04209) + | > current_lr: 0.00007 + | > step_time: 1.40470 (2.37623) + | > loader_time: 0.00610 (0.09678) + + + --> STEP: 171/234 -- GLOBAL_STEP: 69435 + | > loss: -0.43440 (-0.31791) + | > log_mle: -0.63220 (-0.44165) + | > loss_dur: 0.19780 (0.12374) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 64.55489 (22.37003) + | > current_lr: 0.00007 + | > step_time: 4.30800 (2.40569) + | > loader_time: 0.09270 (0.09613) + + + --> STEP: 176/234 -- GLOBAL_STEP: 69440 + | > loss: -0.40721 (-0.32044) + | > log_mle: -0.60754 (-0.44665) + | > loss_dur: 0.20033 (0.12621) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 55.11659 (23.36761) + | > current_lr: 0.00007 + | > step_time: 4.70730 (2.46391) + | > loader_time: 0.00240 (0.09563) + + + --> STEP: 181/234 -- GLOBAL_STEP: 69445 + | > loss: -0.32187 (-0.32225) + | > log_mle: -0.52967 (-0.45096) + | > loss_dur: 0.20780 (0.12871) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 49.05380 (24.35752) + | > current_lr: 0.00007 + | > step_time: 5.51300 (2.56659) + | > loader_time: 0.08840 (0.09565) + + + --> STEP: 186/234 -- GLOBAL_STEP: 69450 + | > loss: -0.34500 (-0.32410) + | > log_mle: -0.57281 (-0.45536) + | > loss_dur: 0.22781 (0.13126) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.69084 (25.61877) + | > current_lr: 0.00007 + | > step_time: 2.70790 (2.60833) + | > loader_time: 0.08480 (0.09578) + + + --> STEP: 191/234 -- GLOBAL_STEP: 69455 + | > loss: -0.38203 (-0.32609) + | > log_mle: -0.58311 (-0.45948) + | > loss_dur: 0.20108 (0.13340) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 53.30624 (26.62582) + | > current_lr: 0.00007 + | > step_time: 3.71040 (2.64846) + | > loader_time: 0.09500 (0.09585) + + + --> STEP: 196/234 -- GLOBAL_STEP: 69460 + | > loss: -0.35907 (-0.32844) + | > log_mle: -0.58661 (-0.46390) + | > loss_dur: 0.22753 (0.13546) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 50.39748 (27.34134) + | > current_lr: 0.00007 + | > step_time: 3.79820 (2.68043) + | > loader_time: 0.18740 (0.09547) + + + --> STEP: 201/234 -- GLOBAL_STEP: 69465 + | > loss: -0.31150 (-0.32973) + | > log_mle: -0.52744 (-0.46726) + | > loss_dur: 0.21594 (0.13753) + | > amp_scaler: 1024.00000 (2027.62189) + | > grad_norm: 50.34268 (28.40620) + | > current_lr: 0.00007 + | > step_time: 2.78990 (2.69961) + | > loader_time: 0.00880 (0.09542) + + + --> STEP: 206/234 -- GLOBAL_STEP: 69470 + | > loss: -0.43485 (-0.33188) + | > log_mle: -0.65192 (-0.47140) + | > loss_dur: 0.21707 (0.13952) + | > amp_scaler: 1024.00000 (2003.26214) + | > grad_norm: 56.70234 (29.02475) + | > current_lr: 0.00007 + | > step_time: 1.40070 (2.71476) + | > loader_time: 0.08130 (0.09452) + + + --> STEP: 211/234 -- GLOBAL_STEP: 69475 + | > loss: -0.46929 (-0.33472) + | > log_mle: -0.72617 (-0.47647) + | > loss_dur: 0.25688 (0.14175) + | > amp_scaler: 1024.00000 (1980.05687) + | > grad_norm: 82.69044 (29.94798) + | > current_lr: 0.00007 + | > step_time: 7.29530 (2.76980) + | > loader_time: 0.00960 (0.09339) + + + --> STEP: 216/234 -- GLOBAL_STEP: 69480 + | > loss: -0.49291 (-0.33763) + | > log_mle: -0.73529 (-0.48143) + | > loss_dur: 0.24238 (0.14380) + | > amp_scaler: 1024.00000 (1957.92593) + | > grad_norm: 67.19964 (30.95338) + | > current_lr: 0.00007 + | > step_time: 3.39420 (2.85206) + | > loader_time: 0.47720 (0.09578) + + + --> STEP: 221/234 -- GLOBAL_STEP: 69485 + | > loss: -0.43412 (-0.34084) + | > log_mle: -0.63814 (-0.48655) + | > loss_dur: 0.20402 (0.14571) + | > amp_scaler: 1024.00000 (1936.79638) + | > grad_norm: 67.13587 (31.89122) + | > current_lr: 0.00007 + | > step_time: 2.01040 (2.85677) + | > loader_time: 0.09030 (0.09449) + + + --> STEP: 226/234 -- GLOBAL_STEP: 69490 + | > loss: -0.49697 (-0.34434) + | > log_mle: -0.74015 (-0.49218) + | > loss_dur: 0.24317 (0.14784) + | > amp_scaler: 1024.00000 (1916.60177) + | > grad_norm: 84.21975 (32.98393) + | > current_lr: 0.00007 + | > step_time: 0.23090 (2.80185) + | > loader_time: 0.00280 (0.09281) + + + --> STEP: 231/234 -- GLOBAL_STEP: 69495 + | > loss: -0.45898 (-0.34714) + | > log_mle: -0.83268 (-0.49828) + | > loss_dur: 0.37370 (0.15114) + | > amp_scaler: 1024.00000 (1897.28139) + | > grad_norm: 80.88204 (34.26911) + | > current_lr: 0.00007 + | > step_time: 0.26670 (2.74681) + | > loader_time: 0.00360 (0.09088) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.60149 (+0.52650) + | > avg_loss: -0.34576 (-0.00223) + | > avg_log_mle: -0.56970 (-0.00566) + | > avg_loss_dur: 0.22395 (+0.00343) + + + > EPOCH: 297/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 23:43:15)  + + --> STEP: 2/234 -- GLOBAL_STEP: 69500 + | > loss: -0.35451 (-0.34083) + | > log_mle: -0.43733 (-0.42612) + | > loss_dur: 0.08281 (0.08529) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.37846 (16.44825) + | > current_lr: 0.00007 + | > step_time: 7.89730 (7.30023) + | > loader_time: 0.49840 (0.24972) + + + --> STEP: 7/234 -- GLOBAL_STEP: 69505 + | > loss: -0.36039 (-0.33331) + | > log_mle: -0.42578 (-0.42318) + | > loss_dur: 0.06539 (0.08987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.81739 (17.86142) + | > current_lr: 0.00007 + | > step_time: 1.61020 (3.94581) + | > loader_time: 0.00200 (0.07302) + + + --> STEP: 12/234 -- GLOBAL_STEP: 69510 + | > loss: -0.34163 (-0.34281) + | > log_mle: -0.42451 (-0.42780) + | > loss_dur: 0.08288 (0.08499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.54480 (17.41399) + | > current_lr: 0.00007 + | > step_time: 1.18450 (2.73941) + | > loader_time: 0.00090 (0.05650) + + + --> STEP: 17/234 -- GLOBAL_STEP: 69515 + | > loss: -0.38999 (-0.35435) + | > log_mle: -0.44162 (-0.43180) + | > loss_dur: 0.05163 (0.07745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.22161 (16.28252) + | > current_lr: 0.00007 + | > step_time: 2.30390 (2.55045) + | > loader_time: 0.00340 (0.04532) + + + --> STEP: 22/234 -- GLOBAL_STEP: 69520 + | > loss: -0.34282 (-0.35607) + | > log_mle: -0.42538 (-0.43095) + | > loss_dur: 0.08256 (0.07487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.93578 (15.58181) + | > current_lr: 0.00007 + | > step_time: 1.45220 (2.48943) + | > loader_time: 0.00180 (0.04415) + + + --> STEP: 27/234 -- GLOBAL_STEP: 69525 + | > loss: -0.33885 (-0.35647) + | > log_mle: -0.41440 (-0.42989) + | > loss_dur: 0.07556 (0.07342) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.86425 (14.83568) + | > current_lr: 0.00007 + | > step_time: 6.09310 (2.58678) + | > loader_time: 0.10350 (0.04717) + + + --> STEP: 32/234 -- GLOBAL_STEP: 69530 + | > loss: -0.35267 (-0.35641) + | > log_mle: -0.41833 (-0.42919) + | > loss_dur: 0.06566 (0.07278) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.88430 (14.05632) + | > current_lr: 0.00007 + | > step_time: 6.10370 (2.84251) + | > loader_time: 0.00220 (0.04869) + + + --> STEP: 37/234 -- GLOBAL_STEP: 69535 + | > loss: -0.32818 (-0.35330) + | > log_mle: -0.39102 (-0.42585) + | > loss_dur: 0.06284 (0.07255) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.11001 (14.08272) + | > current_lr: 0.00007 + | > step_time: 1.91090 (2.99786) + | > loader_time: 0.00180 (0.04918) + + + --> STEP: 42/234 -- GLOBAL_STEP: 69540 + | > loss: -0.31478 (-0.34960) + | > log_mle: -0.38856 (-0.42281) + | > loss_dur: 0.07378 (0.07321) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.17960 (13.96226) + | > current_lr: 0.00007 + | > step_time: 2.67180 (2.83789) + | > loader_time: 0.00150 (0.04529) + + + --> STEP: 47/234 -- GLOBAL_STEP: 69545 + | > loss: -0.32125 (-0.34601) + | > log_mle: -0.40342 (-0.42050) + | > loss_dur: 0.08217 (0.07449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.53688 (13.72205) + | > current_lr: 0.00007 + | > step_time: 1.29140 (2.76194) + | > loader_time: 0.00330 (0.04298) + + + --> STEP: 52/234 -- GLOBAL_STEP: 69550 + | > loss: -0.29222 (-0.34372) + | > log_mle: -0.38815 (-0.41848) + | > loss_dur: 0.09592 (0.07475) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.51448 (13.55010) + | > current_lr: 0.00007 + | > step_time: 0.76280 (2.61934) + | > loader_time: 0.00180 (0.03907) + + + --> STEP: 57/234 -- GLOBAL_STEP: 69555 + | > loss: -0.29408 (-0.34076) + | > log_mle: -0.38236 (-0.41649) + | > loss_dur: 0.08827 (0.07572) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61481 (13.35853) + | > current_lr: 0.00007 + | > step_time: 2.28990 (2.56799) + | > loader_time: 0.00230 (0.03754) + + + --> STEP: 62/234 -- GLOBAL_STEP: 69560 + | > loss: -0.25674 (-0.33697) + | > log_mle: -0.39671 (-0.41482) + | > loss_dur: 0.13998 (0.07785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.81372 (13.40469) + | > current_lr: 0.00007 + | > step_time: 1.09490 (2.48767) + | > loader_time: 0.00180 (0.03601) + + + --> STEP: 67/234 -- GLOBAL_STEP: 69565 + | > loss: -0.28506 (-0.33437) + | > log_mle: -0.39218 (-0.41289) + | > loss_dur: 0.10713 (0.07853) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.35864 (13.12456) + | > current_lr: 0.00007 + | > step_time: 1.21740 (2.43527) + | > loader_time: 0.09570 (0.03613) + + + --> STEP: 72/234 -- GLOBAL_STEP: 69570 + | > loss: -0.29304 (-0.33074) + | > log_mle: -0.38731 (-0.41098) + | > loss_dur: 0.09427 (0.08024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.41467 (13.02918) + | > current_lr: 0.00007 + | > step_time: 3.70150 (2.41681) + | > loader_time: 0.17140 (0.03755) + + + --> STEP: 77/234 -- GLOBAL_STEP: 69575 + | > loss: -0.28342 (-0.32715) + | > log_mle: -0.38255 (-0.40920) + | > loss_dur: 0.09913 (0.08205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.22963 (13.01050) + | > current_lr: 0.00007 + | > step_time: 2.79750 (2.44984) + | > loader_time: 0.00230 (0.03761) + + + --> STEP: 82/234 -- GLOBAL_STEP: 69580 + | > loss: -0.28363 (-0.32465) + | > log_mle: -0.38114 (-0.40772) + | > loss_dur: 0.09751 (0.08307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.37788 (12.85940) + | > current_lr: 0.00007 + | > step_time: 1.59500 (2.41030) + | > loader_time: 0.00140 (0.03636) + + + --> STEP: 87/234 -- GLOBAL_STEP: 69585 + | > loss: -0.28373 (-0.32207) + | > log_mle: -0.38383 (-0.40645) + | > loss_dur: 0.10010 (0.08438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.08130 (12.94261) + | > current_lr: 0.00007 + | > step_time: 1.61430 (2.40140) + | > loader_time: 0.08290 (0.03645) + + + --> STEP: 92/234 -- GLOBAL_STEP: 69590 + | > loss: -0.28765 (-0.31985) + | > log_mle: -0.41775 (-0.40647) + | > loss_dur: 0.13010 (0.08661) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.32698 (13.28022) + | > current_lr: 0.00007 + | > step_time: 0.93590 (2.36717) + | > loader_time: 0.00240 (0.03458) + + + --> STEP: 97/234 -- GLOBAL_STEP: 69595 + | > loss: -0.28580 (-0.31846) + | > log_mle: -0.40286 (-0.40723) + | > loss_dur: 0.11705 (0.08876) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.27917 (13.87278) + | > current_lr: 0.00007 + | > step_time: 1.41260 (2.40507) + | > loader_time: 0.00270 (0.03484) + + + --> STEP: 102/234 -- GLOBAL_STEP: 69600 + | > loss: -0.25186 (-0.31615) + | > log_mle: -0.38680 (-0.40729) + | > loss_dur: 0.13494 (0.09114) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.21544 (14.08340) + | > current_lr: 0.00007 + | > step_time: 1.89950 (2.38863) + | > loader_time: 0.08940 (0.03498) + + + --> STEP: 107/234 -- GLOBAL_STEP: 69605 + | > loss: -0.27044 (-0.31445) + | > log_mle: -0.42144 (-0.40836) + | > loss_dur: 0.15100 (0.09390) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.59442 (14.47017) + | > current_lr: 0.00007 + | > step_time: 1.80500 (2.38373) + | > loader_time: 0.00310 (0.03348) + + + --> STEP: 112/234 -- GLOBAL_STEP: 69610 + | > loss: -0.26563 (-0.31263) + | > log_mle: -0.43311 (-0.40931) + | > loss_dur: 0.16749 (0.09668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.75228 (14.95626) + | > current_lr: 0.00007 + | > step_time: 1.21620 (2.36064) + | > loader_time: 0.07030 (0.03439) + + + --> STEP: 117/234 -- GLOBAL_STEP: 69615 + | > loss: -0.28040 (-0.31107) + | > log_mle: -0.43395 (-0.41024) + | > loss_dur: 0.15356 (0.09916) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.34982 (15.29870) + | > current_lr: 0.00007 + | > step_time: 6.51050 (2.43253) + | > loader_time: 0.10280 (0.03617) + + + --> STEP: 122/234 -- GLOBAL_STEP: 69620 + | > loss: -0.25120 (-0.30951) + | > log_mle: -0.40377 (-0.41055) + | > loss_dur: 0.15258 (0.10104) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.53854 (15.30804) + | > current_lr: 0.00007 + | > step_time: 2.00120 (2.41800) + | > loader_time: 0.00320 (0.03639) + + + --> STEP: 127/234 -- GLOBAL_STEP: 69625 + | > loss: -0.29599 (-0.30871) + | > log_mle: -0.46593 (-0.41207) + | > loss_dur: 0.16994 (0.10337) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.29347 (15.75287) + | > current_lr: 0.00007 + | > step_time: 0.72190 (2.40101) + | > loader_time: 0.07500 (0.03686) + + + --> STEP: 132/234 -- GLOBAL_STEP: 69630 + | > loss: -0.29091 (-0.30857) + | > log_mle: -0.44731 (-0.41419) + | > loss_dur: 0.15640 (0.10561) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.66191 (16.42254) + | > current_lr: 0.00007 + | > step_time: 1.30140 (2.38104) + | > loader_time: 0.08360 (0.03621) + + + --> STEP: 137/234 -- GLOBAL_STEP: 69635 + | > loss: -0.28166 (-0.30874) + | > log_mle: -0.46353 (-0.41680) + | > loss_dur: 0.18187 (0.10806) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.37107 (17.11734) + | > current_lr: 0.00007 + | > step_time: 2.50230 (2.40020) + | > loader_time: 0.00300 (0.03702) + + + --> STEP: 142/234 -- GLOBAL_STEP: 69640 + | > loss: -0.29367 (-0.30857) + | > log_mle: -0.47216 (-0.41891) + | > loss_dur: 0.17848 (0.11034) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.97769 (17.70951) + | > current_lr: 0.00007 + | > step_time: 5.20650 (2.45265) + | > loader_time: 0.05860 (0.03809) + + + --> STEP: 147/234 -- GLOBAL_STEP: 69645 + | > loss: -0.29316 (-0.30966) + | > log_mle: -0.46765 (-0.42273) + | > loss_dur: 0.17449 (0.11307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.31640 (18.53318) + | > current_lr: 0.00007 + | > step_time: 4.80910 (2.53297) + | > loader_time: 0.08800 (0.03946) + + + --> STEP: 152/234 -- GLOBAL_STEP: 69650 + | > loss: -0.36046 (-0.31093) + | > log_mle: -0.56386 (-0.42630) + | > loss_dur: 0.20340 (0.11537) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.57265 (19.21418) + | > current_lr: 0.00007 + | > step_time: 2.59940 (2.59782) + | > loader_time: 0.08910 (0.04074) + + + --> STEP: 157/234 -- GLOBAL_STEP: 69655 + | > loss: -0.31627 (-0.31289) + | > log_mle: -0.50399 (-0.43063) + | > loss_dur: 0.18772 (0.11774) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.25431 (20.13037) + | > current_lr: 0.00007 + | > step_time: 1.19350 (2.61252) + | > loader_time: 0.00240 (0.04181) + + + --> STEP: 162/234 -- GLOBAL_STEP: 69660 + | > loss: -0.33575 (-0.31429) + | > log_mle: -0.53457 (-0.43459) + | > loss_dur: 0.19882 (0.12029) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.57337 (20.98249) + | > current_lr: 0.00007 + | > step_time: 2.81330 (2.59367) + | > loader_time: 0.08400 (0.04268) + + + --> STEP: 167/234 -- GLOBAL_STEP: 69665 + | > loss: -0.44926 (-0.31596) + | > log_mle: -0.63486 (-0.43828) + | > loss_dur: 0.18560 (0.12232) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.63488 (21.84515) + | > current_lr: 0.00007 + | > step_time: 1.69210 (2.63391) + | > loader_time: 0.01440 (0.04202) + + + --> STEP: 172/234 -- GLOBAL_STEP: 69670 + | > loss: -0.41632 (-0.31813) + | > log_mle: -0.62842 (-0.44306) + | > loss_dur: 0.21210 (0.12493) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.84760 (22.89330) + | > current_lr: 0.00007 + | > step_time: 2.00210 (2.62019) + | > loader_time: 0.00330 (0.04256) + + + --> STEP: 177/234 -- GLOBAL_STEP: 69675 + | > loss: -0.37259 (-0.32003) + | > log_mle: -0.57426 (-0.44731) + | > loss_dur: 0.20166 (0.12728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.87807 (23.93694) + | > current_lr: 0.00007 + | > step_time: 1.29540 (2.67724) + | > loader_time: 0.00250 (0.04290) + + + --> STEP: 182/234 -- GLOBAL_STEP: 69680 + | > loss: -0.38167 (-0.32182) + | > log_mle: -0.62445 (-0.45167) + | > loss_dur: 0.24278 (0.12984) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.38626 (24.74362) + | > current_lr: 0.00007 + | > step_time: 4.10660 (2.73465) + | > loader_time: 0.10020 (0.04328) + + + --> STEP: 187/234 -- GLOBAL_STEP: 69685 + | > loss: -0.42116 (-0.32388) + | > log_mle: -0.63355 (-0.45609) + | > loss_dur: 0.21239 (0.13221) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.27774 (25.61263) + | > current_lr: 0.00007 + | > step_time: 2.70630 (2.72640) + | > loader_time: 0.08670 (0.04358) + + + --> STEP: 192/234 -- GLOBAL_STEP: 69690 + | > loss: -0.44706 (-0.32624) + | > log_mle: -0.65570 (-0.46042) + | > loss_dur: 0.20863 (0.13418) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.45601 (26.33942) + | > current_lr: 0.00007 + | > step_time: 3.60020 (2.76942) + | > loader_time: 0.09840 (0.04567) + + + --> STEP: 197/234 -- GLOBAL_STEP: 69695 + | > loss: -0.43772 (-0.32857) + | > log_mle: -0.62804 (-0.46468) + | > loss_dur: 0.19032 (0.13611) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.35091 (27.22967) + | > current_lr: 0.00007 + | > step_time: 4.50170 (2.81439) + | > loader_time: 0.18470 (0.04555) + + + --> STEP: 202/234 -- GLOBAL_STEP: 69700 + | > loss: -0.52499 (-0.33077) + | > log_mle: -0.72932 (-0.46898) + | > loss_dur: 0.20433 (0.13822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.06468 (28.11444) + | > current_lr: 0.00007 + | > step_time: 7.00530 (2.82495) + | > loader_time: 0.18360 (0.04583) + + + --> STEP: 207/234 -- GLOBAL_STEP: 69705 + | > loss: -0.45761 (-0.33285) + | > log_mle: -0.69625 (-0.47320) + | > loss_dur: 0.23864 (0.14035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.79286 (28.99669) + | > current_lr: 0.00007 + | > step_time: 2.99390 (2.91177) + | > loader_time: 0.18900 (0.04794) + + + --> STEP: 212/234 -- GLOBAL_STEP: 69710 + | > loss: -0.45158 (-0.33557) + | > log_mle: -0.68092 (-0.47802) + | > loss_dur: 0.22935 (0.14245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.84103 (30.00755) + | > current_lr: 0.00007 + | > step_time: 8.53950 (2.98484) + | > loader_time: 0.39180 (0.05046) + + + --> STEP: 217/234 -- GLOBAL_STEP: 69715 + | > loss: -0.46850 (-0.33847) + | > log_mle: -0.70368 (-0.48293) + | > loss_dur: 0.23518 (0.14447) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.65884 (31.08106) + | > current_lr: 0.00007 + | > step_time: 5.79240 (3.04130) + | > loader_time: 0.00280 (0.05517) + + + --> STEP: 222/234 -- GLOBAL_STEP: 69720 + | > loss: -0.46750 (-0.34150) + | > log_mle: -0.73370 (-0.48794) + | > loss_dur: 0.26620 (0.14644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.70859 (31.99879) + | > current_lr: 0.00007 + | > step_time: 0.90310 (3.08680) + | > loader_time: 0.00420 (0.05530) + + + --> STEP: 227/234 -- GLOBAL_STEP: 69725 + | > loss: -0.45085 (-0.34484) + | > log_mle: -0.70105 (-0.49338) + | > loss_dur: 0.25020 (0.14854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.69953 (32.93582) + | > current_lr: 0.00007 + | > step_time: 0.24770 (3.02386) + | > loader_time: 0.00380 (0.05416) + + + --> STEP: 232/234 -- GLOBAL_STEP: 69730 + | > loss: -0.45841 (-0.34759) + | > log_mle: -0.93198 (-0.50041) + | > loss_dur: 0.47358 (0.15282) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 119.60297 (34.19545) + | > current_lr: 0.00007 + | > step_time: 0.33990 (2.96464) + | > loader_time: 0.01310 (0.05311) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.57973 (-0.02176) + | > avg_loss: -0.34668 (-0.00092) + | > avg_log_mle: -0.56488 (+0.00483) + | > avg_loss_dur: 0.21820 (-0.00575) + + + > EPOCH: 298/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-29 23:55:58)  + + --> STEP: 3/234 -- GLOBAL_STEP: 69735 + | > loss: -0.26474 (-0.31436) + | > log_mle: -0.40450 (-0.42058) + | > loss_dur: 0.13976 (0.10621) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.38490 (22.95180) + | > current_lr: 0.00007 + | > step_time: 1.71810 (5.20721) + | > loader_time: 0.00140 (0.10203) + + + --> STEP: 8/234 -- GLOBAL_STEP: 69740 + | > loss: -0.36559 (-0.33603) + | > log_mle: -0.44529 (-0.42561) + | > loss_dur: 0.07970 (0.08958) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.01749 (19.64260) + | > current_lr: 0.00007 + | > step_time: 3.30460 (3.01712) + | > loader_time: 0.00490 (0.05794) + + + --> STEP: 13/234 -- GLOBAL_STEP: 69745 + | > loss: -0.39154 (-0.34423) + | > log_mle: -0.45171 (-0.42952) + | > loss_dur: 0.06017 (0.08529) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.43404 (18.83886) + | > current_lr: 0.00007 + | > step_time: 1.15710 (2.41154) + | > loader_time: 0.00100 (0.03619) + + + --> STEP: 18/234 -- GLOBAL_STEP: 69750 + | > loss: -0.34207 (-0.35248) + | > log_mle: -0.41770 (-0.43082) + | > loss_dur: 0.07563 (0.07835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.72992 (17.02915) + | > current_lr: 0.00007 + | > step_time: 0.89910 (2.29578) + | > loader_time: 0.08230 (0.03601) + + + --> STEP: 23/234 -- GLOBAL_STEP: 69755 + | > loss: -0.38553 (-0.35548) + | > log_mle: -0.44900 (-0.43169) + | > loss_dur: 0.06347 (0.07621) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.13338 (15.52909) + | > current_lr: 0.00007 + | > step_time: 1.18080 (2.09031) + | > loader_time: 0.00180 (0.02860) + + + --> STEP: 28/234 -- GLOBAL_STEP: 69760 + | > loss: -0.41343 (-0.35760) + | > log_mle: -0.45962 (-0.43105) + | > loss_dur: 0.04619 (0.07346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.16915 (14.93534) + | > current_lr: 0.00007 + | > step_time: 4.90500 (2.11284) + | > loader_time: 0.08000 (0.02663) + + + --> STEP: 33/234 -- GLOBAL_STEP: 69765 + | > loss: -0.35052 (-0.35528) + | > log_mle: -0.41158 (-0.42879) + | > loss_dur: 0.06105 (0.07351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.55709 (14.55245) + | > current_lr: 0.00007 + | > step_time: 4.83350 (2.17171) + | > loader_time: 0.09460 (0.02572) + + + --> STEP: 38/234 -- GLOBAL_STEP: 69770 + | > loss: -0.32896 (-0.35069) + | > log_mle: -0.40725 (-0.42471) + | > loss_dur: 0.07829 (0.07402) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.75190 (14.37213) + | > current_lr: 0.00007 + | > step_time: 1.12940 (2.08872) + | > loader_time: 0.00180 (0.02257) + + + --> STEP: 43/234 -- GLOBAL_STEP: 69775 + | > loss: -0.30973 (-0.34697) + | > log_mle: -0.39700 (-0.42166) + | > loss_dur: 0.08726 (0.07469) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.40559 (13.94550) + | > current_lr: 0.00007 + | > step_time: 1.60840 (2.03819) + | > loader_time: 0.00230 (0.02016) + + + --> STEP: 48/234 -- GLOBAL_STEP: 69780 + | > loss: -0.35183 (-0.34461) + | > log_mle: -0.41506 (-0.41993) + | > loss_dur: 0.06323 (0.07533) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.70866 (13.58948) + | > current_lr: 0.00007 + | > step_time: 1.65330 (1.99426) + | > loader_time: 0.00180 (0.01866) + + + --> STEP: 53/234 -- GLOBAL_STEP: 69785 + | > loss: -0.31294 (-0.34209) + | > log_mle: -0.40268 (-0.41818) + | > loss_dur: 0.08974 (0.07610) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.55022 (13.07993) + | > current_lr: 0.00007 + | > step_time: 1.74530 (1.96953) + | > loader_time: 0.00180 (0.01718) + + + --> STEP: 58/234 -- GLOBAL_STEP: 69790 + | > loss: -0.33335 (-0.34031) + | > log_mle: -0.40344 (-0.41658) + | > loss_dur: 0.07009 (0.07627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.00748 (12.75687) + | > current_lr: 0.00007 + | > step_time: 2.81650 (1.95646) + | > loader_time: 0.00240 (0.01746) + + + --> STEP: 63/234 -- GLOBAL_STEP: 69795 + | > loss: -0.29868 (-0.33658) + | > log_mle: -0.38511 (-0.41481) + | > loss_dur: 0.08642 (0.07824) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.18425 (12.85927) + | > current_lr: 0.00007 + | > step_time: 1.30280 (1.91744) + | > loader_time: 0.00180 (0.01624) + + + --> STEP: 68/234 -- GLOBAL_STEP: 69800 + | > loss: -0.28027 (-0.33409) + | > log_mle: -0.38180 (-0.41303) + | > loss_dur: 0.10153 (0.07894) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.44473 (12.61204) + | > current_lr: 0.00007 + | > step_time: 1.46250 (1.88897) + | > loader_time: 0.00170 (0.01519) + + + --> STEP: 73/234 -- GLOBAL_STEP: 69805 + | > loss: -0.26398 (-0.33063) + | > log_mle: -0.38750 (-0.41120) + | > loss_dur: 0.12352 (0.08057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.26218 (12.61251) + | > current_lr: 0.00007 + | > step_time: 2.88320 (1.89000) + | > loader_time: 0.08620 (0.01670) + + + --> STEP: 78/234 -- GLOBAL_STEP: 69810 + | > loss: -0.28058 (-0.32748) + | > log_mle: -0.37770 (-0.40932) + | > loss_dur: 0.09712 (0.08184) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.41012 (12.52152) + | > current_lr: 0.00007 + | > step_time: 1.62050 (1.91793) + | > loader_time: 0.08160 (0.01684) + + + --> STEP: 83/234 -- GLOBAL_STEP: 69815 + | > loss: -0.25977 (-0.32486) + | > log_mle: -0.38420 (-0.40792) + | > loss_dur: 0.12443 (0.08306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.23480 (12.50784) + | > current_lr: 0.00007 + | > step_time: 2.47750 (1.92600) + | > loader_time: 0.00300 (0.01698) + + + --> STEP: 88/234 -- GLOBAL_STEP: 69820 + | > loss: -0.29064 (-0.32266) + | > log_mle: -0.42050 (-0.40713) + | > loss_dur: 0.12986 (0.08447) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.00696 (12.57188) + | > current_lr: 0.00007 + | > step_time: 3.59830 (1.92587) + | > loader_time: 0.00550 (0.01828) + + + --> STEP: 93/234 -- GLOBAL_STEP: 69825 + | > loss: -0.28896 (-0.32060) + | > log_mle: -0.42968 (-0.40720) + | > loss_dur: 0.14072 (0.08660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.34251 (13.04083) + | > current_lr: 0.00007 + | > step_time: 2.40590 (1.95932) + | > loader_time: 0.08540 (0.01836) + + + --> STEP: 98/234 -- GLOBAL_STEP: 69830 + | > loss: -0.27231 (-0.31902) + | > log_mle: -0.37227 (-0.40745) + | > loss_dur: 0.09996 (0.08843) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.88732 (13.56158) + | > current_lr: 0.00007 + | > step_time: 0.96790 (1.96146) + | > loader_time: 0.00160 (0.01928) + + + --> STEP: 103/234 -- GLOBAL_STEP: 69835 + | > loss: -0.29763 (-0.31724) + | > log_mle: -0.45563 (-0.40848) + | > loss_dur: 0.15800 (0.09124) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.80689 (14.02332) + | > current_lr: 0.00007 + | > step_time: 3.41530 (1.97810) + | > loader_time: 0.20250 (0.02216) + + + --> STEP: 108/234 -- GLOBAL_STEP: 69840 + | > loss: -0.28410 (-0.31559) + | > log_mle: -0.40378 (-0.40912) + | > loss_dur: 0.11968 (0.09352) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.53484 (14.64605) + | > current_lr: 0.00007 + | > step_time: 2.81380 (2.03130) + | > loader_time: 0.09090 (0.02364) + + + --> STEP: 113/234 -- GLOBAL_STEP: 69845 + | > loss: -0.29089 (-0.31412) + | > log_mle: -0.44388 (-0.41069) + | > loss_dur: 0.15299 (0.09657) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.47889 (15.29421) + | > current_lr: 0.00007 + | > step_time: 2.60590 (2.04160) + | > loader_time: 0.00260 (0.02414) + + + --> STEP: 118/234 -- GLOBAL_STEP: 69850 + | > loss: -0.26429 (-0.31227) + | > log_mle: -0.41733 (-0.41145) + | > loss_dur: 0.15304 (0.09918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.48826 (15.74322) + | > current_lr: 0.00007 + | > step_time: 2.99870 (2.05334) + | > loader_time: 0.09300 (0.02402) + + + --> STEP: 123/234 -- GLOBAL_STEP: 69855 + | > loss: -0.24668 (-0.31056) + | > log_mle: -0.38550 (-0.41140) + | > loss_dur: 0.13882 (0.10083) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.78926 (15.86994) + | > current_lr: 0.00007 + | > step_time: 2.49760 (2.06300) + | > loader_time: 0.00200 (0.02399) + + + --> STEP: 128/234 -- GLOBAL_STEP: 69860 + | > loss: -0.29998 (-0.30994) + | > log_mle: -0.43962 (-0.41325) + | > loss_dur: 0.13965 (0.10332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.33990 (16.50824) + | > current_lr: 0.00007 + | > step_time: 3.40000 (2.07244) + | > loader_time: 0.00240 (0.02510) + + + --> STEP: 133/234 -- GLOBAL_STEP: 69865 + | > loss: -0.29095 (-0.30977) + | > log_mle: -0.46440 (-0.41541) + | > loss_dur: 0.17345 (0.10563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.72082 (17.18962) + | > current_lr: 0.00007 + | > step_time: 2.10080 (2.09054) + | > loader_time: 0.00320 (0.02494) + + + --> STEP: 138/234 -- GLOBAL_STEP: 69870 + | > loss: -0.25760 (-0.30930) + | > log_mle: -0.41395 (-0.41741) + | > loss_dur: 0.15636 (0.10811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.78605 (17.88393) + | > current_lr: 0.00007 + | > step_time: 2.78350 (2.10379) + | > loader_time: 0.10740 (0.02490) + + + --> STEP: 143/234 -- GLOBAL_STEP: 69875 + | > loss: -0.34964 (-0.30952) + | > log_mle: -0.56483 (-0.42025) + | > loss_dur: 0.21518 (0.11073) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.18232 (18.82045) + | > current_lr: 0.00007 + | > step_time: 4.69320 (2.14225) + | > loader_time: 0.00500 (0.02544) + + + --> STEP: 148/234 -- GLOBAL_STEP: 69880 + | > loss: -0.32155 (-0.31017) + | > log_mle: -0.47867 (-0.42327) + | > loss_dur: 0.15712 (0.11310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.98936 (19.49995) + | > current_lr: 0.00007 + | > step_time: 1.99940 (2.19178) + | > loader_time: 0.06570 (0.02631) + + + --> STEP: 153/234 -- GLOBAL_STEP: 69885 + | > loss: -0.42579 (-0.31203) + | > log_mle: -0.61015 (-0.42764) + | > loss_dur: 0.18435 (0.11561) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.09099 (20.38983) + | > current_lr: 0.00007 + | > step_time: 3.30690 (2.25876) + | > loader_time: 0.18130 (0.02917) + + + --> STEP: 158/234 -- GLOBAL_STEP: 69890 + | > loss: -0.31213 (-0.31313) + | > log_mle: -0.52091 (-0.43115) + | > loss_dur: 0.20879 (0.11802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.35932 (21.29609) + | > current_lr: 0.00007 + | > step_time: 1.11760 (2.28921) + | > loader_time: 0.00400 (0.03008) + + + --> STEP: 163/234 -- GLOBAL_STEP: 69895 + | > loss: -0.32688 (-0.31460) + | > log_mle: -0.50946 (-0.43484) + | > loss_dur: 0.18258 (0.12024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.56510 (21.89959) + | > current_lr: 0.00007 + | > step_time: 1.29820 (2.27433) + | > loader_time: 0.00550 (0.02993) + + + --> STEP: 168/234 -- GLOBAL_STEP: 69900 + | > loss: -0.36007 (-0.31627) + | > log_mle: -0.57464 (-0.43874) + | > loss_dur: 0.21457 (0.12247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.67791 (22.70554) + | > current_lr: 0.00007 + | > step_time: 7.39480 (2.34348) + | > loader_time: 0.00320 (0.03083) + + + --> STEP: 173/234 -- GLOBAL_STEP: 69905 + | > loss: -0.38391 (-0.31836) + | > log_mle: -0.58739 (-0.44338) + | > loss_dur: 0.20348 (0.12502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.55824 (23.45692) + | > current_lr: 0.00007 + | > step_time: 6.91520 (2.39325) + | > loader_time: 0.09280 (0.03156) + + + --> STEP: 178/234 -- GLOBAL_STEP: 69910 + | > loss: -0.41488 (-0.32062) + | > log_mle: -0.62966 (-0.44803) + | > loss_dur: 0.21478 (0.12741) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.19514 (24.43304) + | > current_lr: 0.00007 + | > step_time: 2.59520 (2.38412) + | > loader_time: 0.00320 (0.03229) + + + --> STEP: 183/234 -- GLOBAL_STEP: 69915 + | > loss: -0.42949 (-0.32241) + | > log_mle: -0.63945 (-0.45234) + | > loss_dur: 0.20995 (0.12993) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 93.80021 (25.64419) + | > current_lr: 0.00007 + | > step_time: 3.24300 (2.42419) + | > loader_time: 0.08430 (0.03347) + + + --> STEP: 188/234 -- GLOBAL_STEP: 69920 + | > loss: -0.44432 (-0.32435) + | > log_mle: -0.65497 (-0.45675) + | > loss_dur: 0.21064 (0.13240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.77281 (26.54531) + | > current_lr: 0.00007 + | > step_time: 7.50660 (2.47542) + | > loader_time: 0.18490 (0.03470) + + + --> STEP: 193/234 -- GLOBAL_STEP: 69925 + | > loss: -0.44422 (-0.32689) + | > log_mle: -0.65760 (-0.46116) + | > loss_dur: 0.21337 (0.13427) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.53280 (27.30898) + | > current_lr: 0.00007 + | > step_time: 3.40310 (2.52214) + | > loader_time: 0.18920 (0.03630) + + + --> STEP: 198/234 -- GLOBAL_STEP: 69930 + | > loss: -0.39211 (-0.32884) + | > log_mle: -0.61592 (-0.46512) + | > loss_dur: 0.22381 (0.13628) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.75864 (28.46208) + | > current_lr: 0.00007 + | > step_time: 4.60470 (2.55308) + | > loader_time: 0.08820 (0.03678) + + + --> STEP: 203/234 -- GLOBAL_STEP: 69935 + | > loss: -0.35569 (-0.33028) + | > log_mle: -0.56860 (-0.46875) + | > loss_dur: 0.21291 (0.13847) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.22184 (29.42212) + | > current_lr: 0.00007 + | > step_time: 4.49440 (2.59418) + | > loader_time: 0.00290 (0.03774) + + + --> STEP: 208/234 -- GLOBAL_STEP: 69940 + | > loss: -0.39562 (-0.33237) + | > log_mle: -0.64575 (-0.47317) + | > loss_dur: 0.25013 (0.14080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.55237 (30.40816) + | > current_lr: 0.00007 + | > step_time: 4.60530 (2.67427) + | > loader_time: 0.09660 (0.03904) + + + --> STEP: 213/234 -- GLOBAL_STEP: 69945 + | > loss: -0.47479 (-0.33537) + | > log_mle: -0.70602 (-0.47831) + | > loss_dur: 0.23123 (0.14294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.75443 (31.18261) + | > current_lr: 0.00007 + | > step_time: 4.29000 (2.77997) + | > loader_time: 0.00320 (0.03947) + + + --> STEP: 218/234 -- GLOBAL_STEP: 69950 + | > loss: -0.40260 (-0.33763) + | > log_mle: -0.62857 (-0.48262) + | > loss_dur: 0.22597 (0.14499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 114.63260 (33.03441) + | > current_lr: 0.00007 + | > step_time: 5.09610 (2.81753) + | > loader_time: 0.00430 (0.03915) + + + --> STEP: 223/234 -- GLOBAL_STEP: 69955 + | > loss: -0.45941 (-0.34011) + | > log_mle: -0.70041 (-0.48717) + | > loss_dur: 0.24099 (0.14706) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.89729 (33.83680) + | > current_lr: 0.00007 + | > step_time: 3.71270 (2.83864) + | > loader_time: 0.00410 (0.03837) + + + --> STEP: 228/234 -- GLOBAL_STEP: 69960 + | > loss: -0.43489 (-0.34288) + | > log_mle: -0.69985 (-0.49218) + | > loss_dur: 0.26495 (0.14930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.94335 (34.81831) + | > current_lr: 0.00007 + | > step_time: 2.11040 (2.84393) + | > loader_time: 0.01060 (0.03847) + + + --> STEP: 233/234 -- GLOBAL_STEP: 69965 + | > loss: -0.02127 (-0.34346) + | > log_mle: -0.68688 (-0.49876) + | > loss_dur: 0.66561 (0.15530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.25629 (35.80308) + | > current_lr: 0.00007 + | > step_time: 0.19310 (2.78886) + | > loader_time: 0.00260 (0.03791) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.73812 (+0.15839) + | > avg_loss: -0.33126 (+0.01542) + | > avg_log_mle: -0.56571 (-0.00083) + | > avg_loss_dur: 0.23444 (+0.01625) + + + > EPOCH: 299/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 00:08:07)  + + --> STEP: 4/234 -- GLOBAL_STEP: 69970 + | > loss: -0.34067 (-0.32860) + | > log_mle: -0.42480 (-0.42344) + | > loss_dur: 0.08413 (0.09484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.52016 (17.92435) + | > current_lr: 0.00007 + | > step_time: 6.69840 (5.87914) + | > loader_time: 0.09730 (0.29634) + + + --> STEP: 9/234 -- GLOBAL_STEP: 69975 + | > loss: -0.32466 (-0.33652) + | > log_mle: -0.43607 (-0.42810) + | > loss_dur: 0.11141 (0.09159) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.46107 (17.77881) + | > current_lr: 0.00007 + | > step_time: 9.49180 (5.80233) + | > loader_time: 0.00100 (0.30021) + + + --> STEP: 14/234 -- GLOBAL_STEP: 69980 + | > loss: -0.35640 (-0.34577) + | > log_mle: -0.42895 (-0.43031) + | > loss_dur: 0.07255 (0.08454) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.96113 (17.31225) + | > current_lr: 0.00007 + | > step_time: 1.71840 (4.22393) + | > loader_time: 0.00110 (0.19362) + + + --> STEP: 19/234 -- GLOBAL_STEP: 69985 + | > loss: -0.38191 (-0.35370) + | > log_mle: -0.43748 (-0.43182) + | > loss_dur: 0.05557 (0.07812) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.56016 (16.02982) + | > current_lr: 0.00007 + | > step_time: 1.91600 (3.71711) + | > loader_time: 0.09320 (0.15759) + + + --> STEP: 24/234 -- GLOBAL_STEP: 69990 + | > loss: -0.36165 (-0.35615) + | > log_mle: -0.42495 (-0.43171) + | > loss_dur: 0.06330 (0.07556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.61178 (15.28059) + | > current_lr: 0.00007 + | > step_time: 8.29750 (3.71013) + | > loader_time: 0.19980 (0.13713) + + + --> STEP: 29/234 -- GLOBAL_STEP: 69995 + | > loss: -0.35748 (-0.35766) + | > log_mle: -0.42422 (-0.43117) + | > loss_dur: 0.06674 (0.07350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.89151 (14.34868) + | > current_lr: 0.00007 + | > step_time: 5.29840 (4.09780) + | > loader_time: 0.00250 (0.12037) + + + --> STEP: 34/234 -- GLOBAL_STEP: 70000 + | > loss: -0.34083 (-0.35592) + | > log_mle: -0.41376 (-0.42927) + | > loss_dur: 0.07293 (0.07335) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.55345 (13.66369) + | > current_lr: 0.00007 + | > step_time: 1.46550 (3.73292) + | > loader_time: 0.00130 (0.10518) + + + > CHECKPOINT : /root/TTS/run-April-27-2022_08+17AM-c410bc58/checkpoint_70000.pth + + --> STEP: 39/234 -- GLOBAL_STEP: 70005 + | > loss: -0.30855 (-0.35187) + | > log_mle: -0.39752 (-0.42599) + | > loss_dur: 0.08896 (0.07413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.71515 (13.46144) + | > current_lr: 0.00007 + | > step_time: 1.46660 (3.40471) + | > loader_time: 0.00370 (0.09235) + + + --> STEP: 44/234 -- GLOBAL_STEP: 70010 + | > loss: -0.34827 (-0.34964) + | > log_mle: -0.40402 (-0.42359) + | > loss_dur: 0.05575 (0.07395) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.75268 (13.04541) + | > current_lr: 0.00007 + | > step_time: 0.97240 (3.17172) + | > loader_time: 0.00130 (0.08402) + + + --> STEP: 49/234 -- GLOBAL_STEP: 70015 + | > loss: -0.34219 (-0.34751) + | > log_mle: -0.41642 (-0.42218) + | > loss_dur: 0.07423 (0.07467) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.86646 (12.75534) + | > current_lr: 0.00007 + | > step_time: 1.30070 (2.97013) + | > loader_time: 0.07540 (0.07824) + + + --> STEP: 54/234 -- GLOBAL_STEP: 70020 + | > loss: -0.32984 (-0.34519) + | > log_mle: -0.40339 (-0.42034) + | > loss_dur: 0.07356 (0.07515) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47767 (12.40471) + | > current_lr: 0.00007 + | > step_time: 1.60230 (2.87800) + | > loader_time: 0.00380 (0.07301) + + + --> STEP: 59/234 -- GLOBAL_STEP: 70025 + | > loss: -0.31533 (-0.34360) + | > log_mle: -0.40020 (-0.41882) + | > loss_dur: 0.08487 (0.07522) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.54588 (12.06879) + | > current_lr: 0.00007 + | > step_time: 1.40080 (2.74932) + | > loader_time: 0.08120 (0.07090) + + + --> STEP: 64/234 -- GLOBAL_STEP: 70030 + | > loss: -0.31706 (-0.33971) + | > log_mle: -0.39411 (-0.41702) + | > loss_dur: 0.07704 (0.07731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.25475 (12.11320) + | > current_lr: 0.00007 + | > step_time: 2.47050 (2.71167) + | > loader_time: 0.08700 (0.06704) + + + --> STEP: 69/234 -- GLOBAL_STEP: 70035 + | > loss: -0.30939 (-0.33675) + | > log_mle: -0.38628 (-0.41490) + | > loss_dur: 0.07689 (0.07815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.60709 (12.06534) + | > current_lr: 0.00007 + | > step_time: 1.05370 (2.60315) + | > loader_time: 0.00180 (0.06232) + + + --> STEP: 74/234 -- GLOBAL_STEP: 70040 + | > loss: -0.27009 (-0.33252) + | > log_mle: -0.37926 (-0.41297) + | > loss_dur: 0.10917 (0.08045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.97455 (12.08823) + | > current_lr: 0.00007 + | > step_time: 1.66810 (2.52547) + | > loader_time: 0.00190 (0.05940) + + + --> STEP: 79/234 -- GLOBAL_STEP: 70045 + | > loss: -0.29487 (-0.32961) + | > log_mle: -0.39414 (-0.41137) + | > loss_dur: 0.09927 (0.08176) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.72393 (12.01169) + | > current_lr: 0.00007 + | > step_time: 1.29670 (2.48888) + | > loader_time: 0.00270 (0.05801) + + + --> STEP: 84/234 -- GLOBAL_STEP: 70050 + | > loss: -0.29421 (-0.32674) + | > log_mle: -0.38821 (-0.40995) + | > loss_dur: 0.09400 (0.08321) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.60831 (11.92931) + | > current_lr: 0.00007 + | > step_time: 3.41610 (2.47920) + | > loader_time: 0.00250 (0.05669) + + + --> STEP: 89/234 -- GLOBAL_STEP: 70055 + | > loss: -0.28697 (-0.32421) + | > log_mle: -0.40741 (-0.40924) + | > loss_dur: 0.12044 (0.08503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.27435 (12.00816) + | > current_lr: 0.00007 + | > step_time: 1.56020 (2.44627) + | > loader_time: 0.00210 (0.05462) + + + --> STEP: 94/234 -- GLOBAL_STEP: 70060 + | > loss: -0.29186 (-0.32194) + | > log_mle: -0.42368 (-0.40941) + | > loss_dur: 0.13181 (0.08747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.42763 (12.47058) + | > current_lr: 0.00007 + | > step_time: 0.68870 (2.40002) + | > loader_time: 0.00200 (0.05354) + + + --> STEP: 99/234 -- GLOBAL_STEP: 70065 + | > loss: -0.29904 (-0.32036) + | > log_mle: -0.45779 (-0.40982) + | > loss_dur: 0.15876 (0.08947) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.43982 (12.84629) + | > current_lr: 0.00007 + | > step_time: 3.50220 (2.41832) + | > loader_time: 0.09100 (0.05529) + + + --> STEP: 104/234 -- GLOBAL_STEP: 70070 + | > loss: -0.32625 (-0.31885) + | > log_mle: -0.47148 (-0.41086) + | > loss_dur: 0.14523 (0.09201) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.60830 (13.12177) + | > current_lr: 0.00007 + | > step_time: 2.89460 (2.40786) + | > loader_time: 0.00300 (0.05353) + + + --> STEP: 109/234 -- GLOBAL_STEP: 70075 + | > loss: -0.25510 (-0.31640) + | > log_mle: -0.43777 (-0.41106) + | > loss_dur: 0.18267 (0.09466) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.59084 (13.55725) + | > current_lr: 0.00007 + | > step_time: 1.70720 (2.49597) + | > loader_time: 0.10410 (0.05551) + + + --> STEP: 114/234 -- GLOBAL_STEP: 70080 + | > loss: -0.28160 (-0.31491) + | > log_mle: -0.42389 (-0.41218) + | > loss_dur: 0.14229 (0.09727) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.31545 (14.12270) + | > current_lr: 0.00007 + | > step_time: 1.39380 (2.47946) + | > loader_time: 0.00270 (0.05386) + + + --> STEP: 119/234 -- GLOBAL_STEP: 70085 + | > loss: -0.27265 (-0.31296) + | > log_mle: -0.42114 (-0.41279) + | > loss_dur: 0.14849 (0.09982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.97094 (14.62086) + | > current_lr: 0.00007 + | > step_time: 1.21370 (2.44070) + | > loader_time: 0.00190 (0.05257) + + + --> STEP: 124/234 -- GLOBAL_STEP: 70090 + | > loss: -0.29885 (-0.31125) + | > log_mle: -0.44529 (-0.41276) + | > loss_dur: 0.14644 (0.10152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.93226 (14.90098) + | > current_lr: 0.00007 + | > step_time: 10.29730 (2.48264) + | > loader_time: 0.10820 (0.05146) + + + --> STEP: 129/234 -- GLOBAL_STEP: 70095 + | > loss: -0.27214 (-0.31031) + | > log_mle: -0.44021 (-0.41433) + | > loss_dur: 0.16807 (0.10403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.96743 (15.59278) + | > current_lr: 0.00007 + | > step_time: 1.41840 (2.46506) + | > loader_time: 0.09670 (0.05101) + + + --> STEP: 134/234 -- GLOBAL_STEP: 70100 + | > loss: -0.30563 (-0.31011) + | > log_mle: -0.48947 (-0.41662) + | > loss_dur: 0.18385 (0.10650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.38588 (16.56843) + | > current_lr: 0.00007 + | > step_time: 3.20890 (2.49707) + | > loader_time: 0.01070 (0.05070) + + + --> STEP: 139/234 -- GLOBAL_STEP: 70105 + | > loss: -0.36397 (-0.31004) + | > log_mle: -0.55351 (-0.41894) + | > loss_dur: 0.18954 (0.10890) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.70687 (17.21813) + | > current_lr: 0.00007 + | > step_time: 2.10860 (2.46071) + | > loader_time: 0.01630 (0.04964) + + + --> STEP: 144/234 -- GLOBAL_STEP: 70110 + | > loss: -0.33288 (-0.30987) + | > log_mle: -0.52966 (-0.42160) + | > loss_dur: 0.19678 (0.11173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.38336 (18.01149) + | > current_lr: 0.00007 + | > step_time: 1.59840 (2.43928) + | > loader_time: 0.08700 (0.04925) + + + --> STEP: 149/234 -- GLOBAL_STEP: 70115 + | > loss: -0.38599 (-0.31082) + | > log_mle: -0.58118 (-0.42489) + | > loss_dur: 0.19519 (0.11407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.69503 (18.76319) + | > current_lr: 0.00007 + | > step_time: 1.60770 (2.43594) + | > loader_time: 0.00290 (0.04774) + + + --> STEP: 154/234 -- GLOBAL_STEP: 70120 + | > loss: -0.34719 (-0.31217) + | > log_mle: -0.53483 (-0.42876) + | > loss_dur: 0.18764 (0.11658) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.89670 (19.77441) + | > current_lr: 0.00007 + | > step_time: 0.88910 (2.42090) + | > loader_time: 0.00250 (0.04812) + + + --> STEP: 159/234 -- GLOBAL_STEP: 70125 + | > loss: -0.35718 (-0.31336) + | > log_mle: -0.55274 (-0.43240) + | > loss_dur: 0.19557 (0.11905) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.96825 (20.84192) + | > current_lr: 0.00007 + | > step_time: 2.49810 (2.41215) + | > loader_time: 0.01050 (0.04737) + + + --> STEP: 164/234 -- GLOBAL_STEP: 70130 + | > loss: -0.33840 (-0.31455) + | > log_mle: -0.54540 (-0.43596) + | > loss_dur: 0.20700 (0.12140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.81144 (21.49127) + | > current_lr: 0.00007 + | > step_time: 4.10370 (2.44361) + | > loader_time: 0.09140 (0.04817) + + + --> STEP: 169/234 -- GLOBAL_STEP: 70135 + | > loss: -0.33833 (-0.31611) + | > log_mle: -0.54792 (-0.43973) + | > loss_dur: 0.20959 (0.12362) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.07961 (22.46905) + | > current_lr: 0.00007 + | > step_time: 1.09990 (2.41316) + | > loader_time: 0.00340 (0.04728) + + + --> STEP: 174/234 -- GLOBAL_STEP: 70140 + | > loss: -0.40552 (-0.31835) + | > log_mle: -0.61844 (-0.44450) + | > loss_dur: 0.21293 (0.12615) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.76548 (23.62694) + | > current_lr: 0.00007 + | > step_time: 5.09830 (2.43177) + | > loader_time: 0.08930 (0.04697) + + + --> STEP: 179/234 -- GLOBAL_STEP: 70145 + | > loss: -0.39826 (-0.32023) + | > log_mle: -0.63294 (-0.44903) + | > loss_dur: 0.23468 (0.12880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.71198 (24.45251) + | > current_lr: 0.00007 + | > step_time: 2.11830 (2.51537) + | > loader_time: 0.00250 (0.04846) + + + --> STEP: 184/234 -- GLOBAL_STEP: 70150 + | > loss: -0.37887 (-0.32183) + | > log_mle: -0.59253 (-0.45294) + | > loss_dur: 0.21366 (0.13111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.43596 (25.24937) + | > current_lr: 0.00007 + | > step_time: 4.88750 (2.54376) + | > loader_time: 0.09700 (0.05022) + + + --> STEP: 189/234 -- GLOBAL_STEP: 70155 + | > loss: -0.39190 (-0.32378) + | > log_mle: -0.59810 (-0.45720) + | > loss_dur: 0.20619 (0.13342) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.88802 (26.01340) + | > current_lr: 0.00007 + | > step_time: 1.51240 (2.56814) + | > loader_time: 0.08310 (0.05033) + + + --> STEP: 194/234 -- GLOBAL_STEP: 70160 + | > loss: -0.42093 (-0.32628) + | > log_mle: -0.62090 (-0.46153) + | > loss_dur: 0.19997 (0.13525) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.17590 (26.77326) + | > current_lr: 0.00007 + | > step_time: 1.70290 (2.59994) + | > loader_time: 0.08790 (0.05010) + + + --> STEP: 199/234 -- GLOBAL_STEP: 70165 + | > loss: -0.40896 (-0.32819) + | > log_mle: -0.63301 (-0.46550) + | > loss_dur: 0.22405 (0.13731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.08138 (27.70217) + | > current_lr: 0.00007 + | > step_time: 4.28600 (2.63557) + | > loader_time: 0.00520 (0.05033) + + + --> STEP: 204/234 -- GLOBAL_STEP: 70170 + | > loss: -0.41850 (-0.32942) + | > log_mle: -0.65076 (-0.46896) + | > loss_dur: 0.23225 (0.13953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.88167 (28.49842) + | > current_lr: 0.00007 + | > step_time: 5.10380 (2.71657) + | > loader_time: 0.10640 (0.05055) + + + --> STEP: 209/234 -- GLOBAL_STEP: 70175 + | > loss: -0.40699 (-0.33142) + | > log_mle: -0.61858 (-0.47302) + | > loss_dur: 0.21159 (0.14161) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.53360 (29.08766) + | > current_lr: 0.00007 + | > step_time: 2.61240 (2.76873) + | > loader_time: 0.08910 (0.05130) + + + --> STEP: 214/234 -- GLOBAL_STEP: 70180 + | > loss: -0.44290 (-0.33428) + | > log_mle: -0.65197 (-0.47792) + | > loss_dur: 0.20907 (0.14364) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.53609 (30.10846) + | > current_lr: 0.00007 + | > step_time: 5.20080 (2.84855) + | > loader_time: 0.00400 (0.05191) + + + --> STEP: 219/234 -- GLOBAL_STEP: 70185 + | > loss: -0.52367 (-0.33705) + | > log_mle: -0.75865 (-0.48283) + | > loss_dur: 0.23498 (0.14578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.47892 (30.91450) + | > current_lr: 0.00007 + | > step_time: 2.40810 (2.84982) + | > loader_time: 0.08390 (0.05267) + + + --> STEP: 224/234 -- GLOBAL_STEP: 70190 + | > loss: -0.47581 (-0.33957) + | > log_mle: -0.71031 (-0.48740) + | > loss_dur: 0.23450 (0.14783) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.47802 (31.84814) + | > current_lr: 0.00007 + | > step_time: 1.49320 (2.82993) + | > loader_time: 0.00320 (0.05201) + + + --> STEP: 229/234 -- GLOBAL_STEP: 70195 + | > loss: -0.44960 (-0.34217) + | > log_mle: -0.74310 (-0.49248) + | > loss_dur: 0.29350 (0.15031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 128.43156 (33.10748) + | > current_lr: 0.00007 + | > step_time: 0.25120 (2.78195) + | > loader_time: 0.00360 (0.05131) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02615 (-0.71196) + | > avg_loss: -0.33627 (-0.00501) + | > avg_log_mle: -0.56416 (+0.00155) + | > avg_loss_dur: 0.22789 (-0.00655) + + + > EPOCH: 300/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 00:20:01)  + + --> STEP: 0/234 -- GLOBAL_STEP: 70200 + | > loss: -0.35447 (-0.35447) + | > log_mle: -0.50841 (-0.50841) + | > loss_dur: 0.15394 (0.15394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.08161 (23.08161) + | > current_lr: 0.00007 + | > step_time: 1.28210 (1.28206) + | > loader_time: 7.07300 (7.07303) + + + --> STEP: 5/234 -- GLOBAL_STEP: 70205 + | > loss: -0.33058 (-0.32830) + | > log_mle: -0.42067 (-0.42161) + | > loss_dur: 0.09009 (0.09331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.86777 (19.91189) + | > current_lr: 0.00007 + | > step_time: 9.50680 (3.12046) + | > loader_time: 0.00130 (0.27790) + + + --> STEP: 10/234 -- GLOBAL_STEP: 70210 + | > loss: -0.34474 (-0.33952) + | > log_mle: -0.42809 (-0.42706) + | > loss_dur: 0.08334 (0.08753) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.66760 (17.01159) + | > current_lr: 0.00007 + | > step_time: 6.19670 (3.94952) + | > loader_time: 0.00240 (0.65945) + + + --> STEP: 15/234 -- GLOBAL_STEP: 70215 + | > loss: -0.37329 (-0.34974) + | > log_mle: -0.43801 (-0.43037) + | > loss_dur: 0.06472 (0.08064) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.58242 (15.96187) + | > current_lr: 0.00007 + | > step_time: 3.21540 (3.93874) + | > loader_time: 0.00150 (0.44837) + + + --> STEP: 20/234 -- GLOBAL_STEP: 70220 + | > loss: -0.38808 (-0.35706) + | > log_mle: -0.44405 (-0.43266) + | > loss_dur: 0.05597 (0.07560) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56328 (14.34467) + | > current_lr: 0.00007 + | > step_time: 3.00790 (4.01304) + | > loader_time: 0.00440 (0.33683) + + + --> STEP: 25/234 -- GLOBAL_STEP: 70225 + | > loss: -0.36134 (-0.35875) + | > log_mle: -0.42575 (-0.43247) + | > loss_dur: 0.06441 (0.07372) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.33061 (13.50061) + | > current_lr: 0.00007 + | > step_time: 2.20650 (4.25826) + | > loader_time: 0.08820 (0.28435) + + + --> STEP: 30/234 -- GLOBAL_STEP: 70230 + | > loss: -0.35276 (-0.36033) + | > log_mle: -0.42341 (-0.43207) + | > loss_dur: 0.07065 (0.07174) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.92927 (13.12470) + | > current_lr: 0.00007 + | > step_time: 7.49460 (4.16874) + | > loader_time: 0.11430 (0.24960) + + + --> STEP: 35/234 -- GLOBAL_STEP: 70235 + | > loss: -0.30880 (-0.35606) + | > log_mle: -0.39468 (-0.42876) + | > loss_dur: 0.08588 (0.07270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.54663 (13.06465) + | > current_lr: 0.00007 + | > step_time: 3.09590 (3.88116) + | > loader_time: 0.00170 (0.21660) + + + --> STEP: 40/234 -- GLOBAL_STEP: 70240 + | > loss: -0.31198 (-0.35166) + | > log_mle: -0.39669 (-0.42520) + | > loss_dur: 0.08472 (0.07355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.27168 (12.78418) + | > current_lr: 0.00007 + | > step_time: 2.00150 (3.80561) + | > loader_time: 0.08300 (0.19647) + + + --> STEP: 45/234 -- GLOBAL_STEP: 70245 + | > loss: -0.31794 (-0.34913) + | > log_mle: -0.40607 (-0.42287) + | > loss_dur: 0.08813 (0.07374) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.08630 (12.46997) + | > current_lr: 0.00007 + | > step_time: 1.06020 (3.52325) + | > loader_time: 0.00210 (0.17658) + + + --> STEP: 50/234 -- GLOBAL_STEP: 70250 + | > loss: -0.32983 (-0.34792) + | > log_mle: -0.40402 (-0.42171) + | > loss_dur: 0.07419 (0.07379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.42503 (12.13628) + | > current_lr: 0.00007 + | > step_time: 1.40960 (3.34123) + | > loader_time: 0.00310 (0.16292) + + + --> STEP: 55/234 -- GLOBAL_STEP: 70255 + | > loss: -0.34283 (-0.34574) + | > log_mle: -0.41158 (-0.42014) + | > loss_dur: 0.06875 (0.07439) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.39689 (11.91530) + | > current_lr: 0.00007 + | > step_time: 3.78470 (3.25487) + | > loader_time: 0.01550 (0.14853) + + + --> STEP: 60/234 -- GLOBAL_STEP: 70260 + | > loss: -0.29211 (-0.34306) + | > log_mle: -0.40520 (-0.41862) + | > loss_dur: 0.11309 (0.07556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.95992 (11.72233) + | > current_lr: 0.00007 + | > step_time: 2.59600 (3.16136) + | > loader_time: 0.00270 (0.13632) + + + --> STEP: 65/234 -- GLOBAL_STEP: 70265 + | > loss: -0.31107 (-0.33971) + | > log_mle: -0.39276 (-0.41678) + | > loss_dur: 0.08169 (0.07707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.82588 (11.64389) + | > current_lr: 0.00007 + | > step_time: 2.66780 (3.09193) + | > loader_time: 0.00200 (0.12720) + + + --> STEP: 70/234 -- GLOBAL_STEP: 70270 + | > loss: -0.26878 (-0.33650) + | > log_mle: -0.37554 (-0.41466) + | > loss_dur: 0.10676 (0.07816) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.42841 (11.50742) + | > current_lr: 0.00007 + | > step_time: 3.29700 (3.02845) + | > loader_time: 0.00180 (0.12062) + + + --> STEP: 75/234 -- GLOBAL_STEP: 70275 + | > loss: -0.26614 (-0.33258) + | > log_mle: -0.38946 (-0.41304) + | > loss_dur: 0.12332 (0.08046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.74423 (11.60450) + | > current_lr: 0.00007 + | > step_time: 1.49580 (2.95447) + | > loader_time: 0.00310 (0.11276) + + + --> STEP: 80/234 -- GLOBAL_STEP: 70280 + | > loss: -0.29667 (-0.33015) + | > log_mle: -0.38484 (-0.41140) + | > loss_dur: 0.08817 (0.08126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.85184 (11.56198) + | > current_lr: 0.00007 + | > step_time: 2.79360 (2.88593) + | > loader_time: 0.00350 (0.10801) + + + --> STEP: 85/234 -- GLOBAL_STEP: 70285 + | > loss: -0.28494 (-0.32728) + | > log_mle: -0.38428 (-0.41001) + | > loss_dur: 0.09934 (0.08273) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29805 (11.53607) + | > current_lr: 0.00007 + | > step_time: 1.46430 (2.81795) + | > loader_time: 0.00840 (0.10186) + + + --> STEP: 90/234 -- GLOBAL_STEP: 70290 + | > loss: -0.26261 (-0.32488) + | > log_mle: -0.39879 (-0.40958) + | > loss_dur: 0.13618 (0.08470) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.77655 (11.64382) + | > current_lr: 0.00007 + | > step_time: 1.84040 (2.75826) + | > loader_time: 0.00190 (0.09634) + + + --> STEP: 95/234 -- GLOBAL_STEP: 70295 + | > loss: -0.32144 (-0.32341) + | > log_mle: -0.47829 (-0.41083) + | > loss_dur: 0.15685 (0.08741) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.85416 (12.18828) + | > current_lr: 0.00007 + | > step_time: 3.01990 (2.72181) + | > loader_time: 0.08320 (0.09308) + + + --> STEP: 100/234 -- GLOBAL_STEP: 70300 + | > loss: -0.28267 (-0.32138) + | > log_mle: -0.40633 (-0.41053) + | > loss_dur: 0.12366 (0.08914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.90172 (12.35109) + | > current_lr: 0.00007 + | > step_time: 2.20890 (2.66941) + | > loader_time: 0.00300 (0.08854) + + + --> STEP: 105/234 -- GLOBAL_STEP: 70305 + | > loss: -0.27057 (-0.31965) + | > log_mle: -0.38813 (-0.41138) + | > loss_dur: 0.11756 (0.09173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.27729 (12.79434) + | > current_lr: 0.00007 + | > step_time: 1.18430 (2.64604) + | > loader_time: 0.00260 (0.08536) + + + --> STEP: 110/234 -- GLOBAL_STEP: 70310 + | > loss: -0.27423 (-0.31728) + | > log_mle: -0.40566 (-0.41176) + | > loss_dur: 0.13143 (0.09447) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.52651 (13.31778) + | > current_lr: 0.00007 + | > step_time: 2.99760 (2.63163) + | > loader_time: 0.09610 (0.08256) + + + --> STEP: 115/234 -- GLOBAL_STEP: 70315 + | > loss: -0.27133 (-0.31587) + | > log_mle: -0.42801 (-0.41304) + | > loss_dur: 0.15668 (0.09717) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.32040 (14.03620) + | > current_lr: 0.00007 + | > step_time: 1.48910 (2.63250) + | > loader_time: 0.00500 (0.07997) + + + --> STEP: 120/234 -- GLOBAL_STEP: 70320 + | > loss: -0.31674 (-0.31438) + | > log_mle: -0.47899 (-0.41421) + | > loss_dur: 0.16225 (0.09983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.39363 (14.32443) + | > current_lr: 0.00007 + | > step_time: 2.10320 (2.61772) + | > loader_time: 0.09540 (0.07754) + + + --> STEP: 125/234 -- GLOBAL_STEP: 70325 + | > loss: -0.28747 (-0.31238) + | > log_mle: -0.45591 (-0.41418) + | > loss_dur: 0.16844 (0.10181) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.32119 (14.62025) + | > current_lr: 0.00007 + | > step_time: 4.01300 (2.65420) + | > loader_time: 0.09240 (0.07661) + + + --> STEP: 130/234 -- GLOBAL_STEP: 70330 + | > loss: -0.31068 (-0.31181) + | > log_mle: -0.47875 (-0.41618) + | > loss_dur: 0.16807 (0.10437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.27910 (15.21487) + | > current_lr: 0.00007 + | > step_time: 1.89710 (2.65668) + | > loader_time: 0.00270 (0.07380) + + + --> STEP: 135/234 -- GLOBAL_STEP: 70335 + | > loss: -0.26966 (-0.31168) + | > log_mle: -0.40489 (-0.41827) + | > loss_dur: 0.13523 (0.10659) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.51402 (15.90360) + | > current_lr: 0.00007 + | > step_time: 2.69550 (2.63630) + | > loader_time: 0.00250 (0.07118) + + + --> STEP: 140/234 -- GLOBAL_STEP: 70340 + | > loss: -0.26056 (-0.31169) + | > log_mle: -0.43844 (-0.42103) + | > loss_dur: 0.17789 (0.10934) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.09395 (16.70205) + | > current_lr: 0.00007 + | > step_time: 1.76160 (2.62778) + | > loader_time: 0.00210 (0.06932) + + + --> STEP: 145/234 -- GLOBAL_STEP: 70345 + | > loss: -0.35851 (-0.31247) + | > log_mle: -0.54296 (-0.42450) + | > loss_dur: 0.18444 (0.11204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.08652 (17.59830) + | > current_lr: 0.00007 + | > step_time: 7.90430 (2.67385) + | > loader_time: 0.00680 (0.06827) + + + --> STEP: 150/234 -- GLOBAL_STEP: 70350 + | > loss: -0.30505 (-0.31327) + | > log_mle: -0.51321 (-0.42770) + | > loss_dur: 0.20816 (0.11443) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.84244 (18.63005) + | > current_lr: 0.00007 + | > step_time: 1.81120 (2.69623) + | > loader_time: 0.00270 (0.06681) + + + --> STEP: 155/234 -- GLOBAL_STEP: 70355 + | > loss: -0.40042 (-0.31533) + | > log_mle: -0.60232 (-0.43210) + | > loss_dur: 0.20190 (0.11676) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.70060 (19.64600) + | > current_lr: 0.00007 + | > step_time: 4.30800 (2.69816) + | > loader_time: 0.09320 (0.06534) + + + --> STEP: 160/234 -- GLOBAL_STEP: 70360 + | > loss: -0.35106 (-0.31615) + | > log_mle: -0.56530 (-0.43539) + | > loss_dur: 0.21424 (0.11924) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.42721 (21.13380) + | > current_lr: 0.00007 + | > step_time: 3.29510 (2.70443) + | > loader_time: 0.00360 (0.06396) + + + --> STEP: 165/234 -- GLOBAL_STEP: 70365 + | > loss: -0.36946 (-0.31710) + | > log_mle: -0.57403 (-0.43862) + | > loss_dur: 0.20457 (0.12152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.51131 (22.03857) + | > current_lr: 0.00007 + | > step_time: 1.80190 (2.69835) + | > loader_time: 0.00420 (0.06219) + + + --> STEP: 170/234 -- GLOBAL_STEP: 70370 + | > loss: -0.40519 (-0.31898) + | > log_mle: -0.62785 (-0.44275) + | > loss_dur: 0.22266 (0.12378) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.42091 (22.67482) + | > current_lr: 0.00007 + | > step_time: 3.99250 (2.70138) + | > loader_time: 0.00600 (0.06137) + + + --> STEP: 175/234 -- GLOBAL_STEP: 70375 + | > loss: -0.37235 (-0.32128) + | > log_mle: -0.59928 (-0.44757) + | > loss_dur: 0.22693 (0.12629) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.06942 (23.79920) + | > current_lr: 0.00007 + | > step_time: 2.30880 (2.70131) + | > loader_time: 0.00620 (0.05981) + + + --> STEP: 180/234 -- GLOBAL_STEP: 70380 + | > loss: -0.39060 (-0.32306) + | > log_mle: -0.60246 (-0.45195) + | > loss_dur: 0.21186 (0.12889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.26658 (24.79098) + | > current_lr: 0.00007 + | > step_time: 5.70690 (2.73017) + | > loader_time: 0.00510 (0.05923) + + + --> STEP: 185/234 -- GLOBAL_STEP: 70385 + | > loss: -0.39391 (-0.32460) + | > log_mle: -0.62286 (-0.45590) + | > loss_dur: 0.22895 (0.13130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.72827 (25.76599) + | > current_lr: 0.00007 + | > step_time: 1.79950 (2.84943) + | > loader_time: 0.00560 (0.06192) + + + --> STEP: 190/234 -- GLOBAL_STEP: 70390 + | > loss: -0.39266 (-0.32628) + | > log_mle: -0.59858 (-0.45977) + | > loss_dur: 0.20592 (0.13349) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.49725 (26.61953) + | > current_lr: 0.00007 + | > step_time: 6.60670 (2.89583) + | > loader_time: 0.00290 (0.06192) + + + --> STEP: 195/234 -- GLOBAL_STEP: 70395 + | > loss: -0.40527 (-0.32858) + | > log_mle: -0.63261 (-0.46405) + | > loss_dur: 0.22733 (0.13547) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.91015 (27.27573) + | > current_lr: 0.00007 + | > step_time: 5.30170 (2.94923) + | > loader_time: 0.10350 (0.06203) + + + --> STEP: 200/234 -- GLOBAL_STEP: 70400 + | > loss: -0.39995 (-0.33054) + | > log_mle: -0.64039 (-0.46808) + | > loss_dur: 0.24044 (0.13754) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.21009 (28.07834) + | > current_lr: 0.00007 + | > step_time: 4.40450 (3.00152) + | > loader_time: 0.08380 (0.06287) + + + --> STEP: 205/234 -- GLOBAL_STEP: 70405 + | > loss: -0.39314 (-0.33230) + | > log_mle: -0.61986 (-0.47191) + | > loss_dur: 0.22672 (0.13961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.03245 (28.88553) + | > current_lr: 0.00007 + | > step_time: 4.10790 (3.03619) + | > loader_time: 0.09590 (0.06262) + + + --> STEP: 210/234 -- GLOBAL_STEP: 70410 + | > loss: -0.47647 (-0.33511) + | > log_mle: -0.71284 (-0.47676) + | > loss_dur: 0.23637 (0.14165) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.98531 (29.90898) + | > current_lr: 0.00007 + | > step_time: 5.38380 (3.04384) + | > loader_time: 0.10860 (0.06241) + + + --> STEP: 215/234 -- GLOBAL_STEP: 70415 + | > loss: -0.42966 (-0.33812) + | > log_mle: -0.65978 (-0.48184) + | > loss_dur: 0.23012 (0.14371) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.03214 (31.09952) + | > current_lr: 0.00007 + | > step_time: 4.30400 (3.12772) + | > loader_time: 0.09170 (0.06474) + + + --> STEP: 220/234 -- GLOBAL_STEP: 70420 + | > loss: -0.46820 (-0.34143) + | > log_mle: -0.70924 (-0.48725) + | > loss_dur: 0.24104 (0.14582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.77978 (32.14056) + | > current_lr: 0.00007 + | > step_time: 1.39180 (3.20206) + | > loader_time: 0.00540 (0.06604) + + + --> STEP: 225/234 -- GLOBAL_STEP: 70425 + | > loss: -0.51646 (-0.34447) + | > log_mle: -0.76139 (-0.49233) + | > loss_dur: 0.24493 (0.14787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 120.05350 (33.23028) + | > current_lr: 0.00007 + | > step_time: 0.23600 (3.14278) + | > loader_time: 0.00280 (0.06465) + + + --> STEP: 230/234 -- GLOBAL_STEP: 70430 + | > loss: -0.48245 (-0.34650) + | > log_mle: -0.79904 (-0.49718) + | > loss_dur: 0.31659 (0.15068) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 111.06816 (34.37430) + | > current_lr: 0.00007 + | > step_time: 0.25900 (3.07980) + | > loader_time: 0.00500 (0.06334) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.15793 (+0.13178) + | > avg_loss: -0.33901 (-0.00274) + | > avg_log_mle: -0.57172 (-0.00757) + | > avg_loss_dur: 0.23272 (+0.00483) + + + > EPOCH: 301/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 00:32:58)  + + --> STEP: 1/234 -- GLOBAL_STEP: 70435 + | > loss: -0.31430 (-0.31430) + | > log_mle: -0.41709 (-0.41709) + | > loss_dur: 0.10278 (0.10278) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.70165 (22.70165) + | > current_lr: 0.00008 + | > step_time: 5.31590 (5.31586) + | > loader_time: 13.10110 (13.10109) + + + --> STEP: 6/234 -- GLOBAL_STEP: 70440 + | > loss: -0.36419 (-0.32383) + | > log_mle: -0.42749 (-0.42233) + | > loss_dur: 0.06330 (0.09850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.47954 (19.56485) + | > current_lr: 0.00008 + | > step_time: 0.59340 (6.05302) + | > loader_time: 0.00110 (2.23536) + + + --> STEP: 11/234 -- GLOBAL_STEP: 70445 + | > loss: -0.37782 (-0.33896) + | > log_mle: -0.44365 (-0.42927) + | > loss_dur: 0.06583 (0.09030) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.86333 (17.44549) + | > current_lr: 0.00008 + | > step_time: 2.31440 (4.28690) + | > loader_time: 0.00140 (1.22026) + + + --> STEP: 16/234 -- GLOBAL_STEP: 70450 + | > loss: -0.39362 (-0.35038) + | > log_mle: -0.45124 (-0.43302) + | > loss_dur: 0.05762 (0.08264) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.40763 (15.76246) + | > current_lr: 0.00008 + | > step_time: 8.11160 (4.38539) + | > loader_time: 0.19280 (0.87003) + + + --> STEP: 21/234 -- GLOBAL_STEP: 70455 + | > loss: -0.34842 (-0.35477) + | > log_mle: -0.42209 (-0.43379) + | > loss_dur: 0.07367 (0.07903) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.03351 (14.39198) + | > current_lr: 0.00008 + | > step_time: 2.99800 (4.10842) + | > loader_time: 0.00190 (0.67585) + + + --> STEP: 26/234 -- GLOBAL_STEP: 70460 + | > loss: -0.35419 (-0.35734) + | > log_mle: -0.42367 (-0.43379) + | > loss_dur: 0.06947 (0.07645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67567 (13.62726) + | > current_lr: 0.00008 + | > step_time: 3.60460 (3.69541) + | > loader_time: 0.09320 (0.55312) + + + --> STEP: 31/234 -- GLOBAL_STEP: 70465 + | > loss: -0.32739 (-0.35824) + | > log_mle: -0.41511 (-0.43356) + | > loss_dur: 0.08772 (0.07531) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.80130 (12.91433) + | > current_lr: 0.00008 + | > step_time: 1.62470 (3.93266) + | > loader_time: 0.08850 (0.46704) + + + --> STEP: 36/234 -- GLOBAL_STEP: 70470 + | > loss: -0.32671 (-0.35640) + | > log_mle: -0.40859 (-0.43096) + | > loss_dur: 0.08188 (0.07456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.43747 (12.58745) + | > current_lr: 0.00008 + | > step_time: 1.90210 (3.57759) + | > loader_time: 0.08450 (0.40479) + + + --> STEP: 41/234 -- GLOBAL_STEP: 70475 + | > loss: -0.36091 (-0.35411) + | > log_mle: -0.42201 (-0.42848) + | > loss_dur: 0.06110 (0.07437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.48411 (12.44872) + | > current_lr: 0.00008 + | > step_time: 1.13160 (3.77272) + | > loader_time: 0.00210 (0.36474) + + + --> STEP: 46/234 -- GLOBAL_STEP: 70480 + | > loss: -0.32489 (-0.35098) + | > log_mle: -0.40811 (-0.42607) + | > loss_dur: 0.08322 (0.07510) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.81803 (12.30900) + | > current_lr: 0.00008 + | > step_time: 2.90620 (3.52390) + | > loader_time: 0.00400 (0.32882) + + + --> STEP: 51/234 -- GLOBAL_STEP: 70485 + | > loss: -0.32621 (-0.35014) + | > log_mle: -0.41100 (-0.42508) + | > loss_dur: 0.08478 (0.07494) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.11662 (11.83637) + | > current_lr: 0.00008 + | > step_time: 3.60030 (3.45693) + | > loader_time: 0.00370 (0.30022) + + + --> STEP: 56/234 -- GLOBAL_STEP: 70490 + | > loss: -0.32024 (-0.34790) + | > log_mle: -0.40988 (-0.42371) + | > loss_dur: 0.08963 (0.07581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.68875 (11.61635) + | > current_lr: 0.00008 + | > step_time: 1.19340 (3.29999) + | > loader_time: 0.00390 (0.27370) + + + --> STEP: 61/234 -- GLOBAL_STEP: 70495 + | > loss: -0.31356 (-0.34547) + | > log_mle: -0.39631 (-0.42201) + | > loss_dur: 0.08275 (0.07654) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.79307 (11.61680) + | > current_lr: 0.00008 + | > step_time: 1.80870 (3.18524) + | > loader_time: 0.00270 (0.25447) + + + --> STEP: 66/234 -- GLOBAL_STEP: 70500 + | > loss: -0.32054 (-0.34233) + | > log_mle: -0.39390 (-0.41999) + | > loss_dur: 0.07336 (0.07766) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67561 (11.67562) + | > current_lr: 0.00008 + | > step_time: 3.11780 (3.11207) + | > loader_time: 0.08300 (0.23675) + + + --> STEP: 71/234 -- GLOBAL_STEP: 70505 + | > loss: -0.27970 (-0.33860) + | > log_mle: -0.40617 (-0.41806) + | > loss_dur: 0.12647 (0.07946) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.87347 (11.65652) + | > current_lr: 0.00008 + | > step_time: 1.81140 (3.02691) + | > loader_time: 0.00370 (0.22144) + + + --> STEP: 76/234 -- GLOBAL_STEP: 70510 + | > loss: -0.28214 (-0.33507) + | > log_mle: -0.39410 (-0.41633) + | > loss_dur: 0.11195 (0.08126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.34951 (11.63372) + | > current_lr: 0.00008 + | > step_time: 2.39830 (2.92602) + | > loader_time: 0.00300 (0.20701) + + + --> STEP: 81/234 -- GLOBAL_STEP: 70515 + | > loss: -0.29725 (-0.33263) + | > log_mle: -0.39895 (-0.41479) + | > loss_dur: 0.10170 (0.08215) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.29710 (11.59165) + | > current_lr: 0.00008 + | > step_time: 1.72500 (2.85424) + | > loader_time: 0.08860 (0.19642) + + + --> STEP: 86/234 -- GLOBAL_STEP: 70520 + | > loss: -0.29180 (-0.32977) + | > log_mle: -0.39495 (-0.41335) + | > loss_dur: 0.10315 (0.08358) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.21925 (11.66929) + | > current_lr: 0.00008 + | > step_time: 0.91110 (2.79582) + | > loader_time: 0.00250 (0.18705) + + + --> STEP: 91/234 -- GLOBAL_STEP: 70525 + | > loss: -0.28382 (-0.32733) + | > log_mle: -0.40389 (-0.41293) + | > loss_dur: 0.12006 (0.08560) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.05030 (11.84955) + | > current_lr: 0.00008 + | > step_time: 3.81720 (2.75573) + | > loader_time: 0.08570 (0.18040) + + + --> STEP: 96/234 -- GLOBAL_STEP: 70530 + | > loss: -0.28261 (-0.32579) + | > log_mle: -0.38999 (-0.41393) + | > loss_dur: 0.10738 (0.08815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.67033 (12.18975) + | > current_lr: 0.00008 + | > step_time: 1.92890 (2.79491) + | > loader_time: 0.00190 (0.17215) + + + --> STEP: 101/234 -- GLOBAL_STEP: 70535 + | > loss: -0.27976 (-0.32382) + | > log_mle: -0.43274 (-0.41416) + | > loss_dur: 0.15298 (0.09033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.85244 (12.62106) + | > current_lr: 0.00008 + | > step_time: 2.41140 (2.74225) + | > loader_time: 0.08390 (0.16624) + + + --> STEP: 106/234 -- GLOBAL_STEP: 70540 + | > loss: -0.25412 (-0.32187) + | > log_mle: -0.42706 (-0.41482) + | > loss_dur: 0.17294 (0.09295) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.86032 (13.21949) + | > current_lr: 0.00008 + | > step_time: 1.28470 (2.68360) + | > loader_time: 0.00190 (0.15993) + + + --> STEP: 111/234 -- GLOBAL_STEP: 70545 + | > loss: -0.31491 (-0.32002) + | > log_mle: -0.48578 (-0.41565) + | > loss_dur: 0.17088 (0.09563) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.45758 (13.85735) + | > current_lr: 0.00008 + | > step_time: 2.19180 (2.63448) + | > loader_time: 0.00240 (0.15361) + + + --> STEP: 116/234 -- GLOBAL_STEP: 70550 + | > loss: -0.26629 (-0.31814) + | > log_mle: -0.44639 (-0.41663) + | > loss_dur: 0.18010 (0.09849) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.40830 (14.33915) + | > current_lr: 0.00008 + | > step_time: 1.24120 (2.59055) + | > loader_time: 0.00210 (0.14773) + + + --> STEP: 121/234 -- GLOBAL_STEP: 70555 + | > loss: -0.25001 (-0.31665) + | > log_mle: -0.36834 (-0.41705) + | > loss_dur: 0.11832 (0.10039) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.94944 (14.63631) + | > current_lr: 0.00008 + | > step_time: 3.59550 (2.58190) + | > loader_time: 0.00500 (0.14257) + + + --> STEP: 126/234 -- GLOBAL_STEP: 70560 + | > loss: -0.30897 (-0.31528) + | > log_mle: -0.49340 (-0.41796) + | > loss_dur: 0.18443 (0.10268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.13519 (15.10791) + | > current_lr: 0.00008 + | > step_time: 2.39400 (2.62478) + | > loader_time: 0.00680 (0.13851) + + + --> STEP: 131/234 -- GLOBAL_STEP: 70565 + | > loss: -0.34473 (-0.31496) + | > log_mle: -0.53968 (-0.42012) + | > loss_dur: 0.19495 (0.10516) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.11615 (15.77305) + | > current_lr: 0.00008 + | > step_time: 1.70380 (2.67201) + | > loader_time: 0.00290 (0.13531) + + + --> STEP: 136/234 -- GLOBAL_STEP: 70570 + | > loss: -0.38651 (-0.31486) + | > log_mle: -0.58906 (-0.42240) + | > loss_dur: 0.20256 (0.10754) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.80159 (16.34576) + | > current_lr: 0.00008 + | > step_time: 1.47600 (2.66300) + | > loader_time: 0.00260 (0.13163) + + + --> STEP: 141/234 -- GLOBAL_STEP: 70575 + | > loss: -0.32575 (-0.31432) + | > log_mle: -0.48915 (-0.42423) + | > loss_dur: 0.16340 (0.10991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.65133 (16.89260) + | > current_lr: 0.00008 + | > step_time: 1.40050 (2.63162) + | > loader_time: 0.07750 (0.12879) + + + --> STEP: 146/234 -- GLOBAL_STEP: 70580 + | > loss: -0.35315 (-0.31511) + | > log_mle: -0.54880 (-0.42802) + | > loss_dur: 0.19565 (0.11291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.74762 (17.56560) + | > current_lr: 0.00008 + | > step_time: 1.30360 (2.60756) + | > loader_time: 0.00210 (0.12507) + + + --> STEP: 151/234 -- GLOBAL_STEP: 70585 + | > loss: -0.33418 (-0.31583) + | > log_mle: -0.50864 (-0.43095) + | > loss_dur: 0.17446 (0.11512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.98008 (18.38768) + | > current_lr: 0.00008 + | > step_time: 2.40820 (2.60409) + | > loader_time: 0.08260 (0.12288) + + + --> STEP: 156/234 -- GLOBAL_STEP: 70590 + | > loss: -0.36946 (-0.31779) + | > log_mle: -0.55331 (-0.43535) + | > loss_dur: 0.18385 (0.11756) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.83813 (19.86213) + | > current_lr: 0.00008 + | > step_time: 1.01380 (2.60230) + | > loader_time: 0.08300 (0.12178) + + + --> STEP: 161/234 -- GLOBAL_STEP: 70595 + | > loss: -0.40028 (-0.31913) + | > log_mle: -0.58068 (-0.43913) + | > loss_dur: 0.18039 (0.12000) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.11362 (20.69638) + | > current_lr: 0.00008 + | > step_time: 2.40360 (2.59312) + | > loader_time: 0.09700 (0.11914) + + + --> STEP: 166/234 -- GLOBAL_STEP: 70600 + | > loss: -0.33455 (-0.32014) + | > log_mle: -0.50643 (-0.44215) + | > loss_dur: 0.17188 (0.12201) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.52479 (21.42002) + | > current_lr: 0.00008 + | > step_time: 4.60240 (2.60649) + | > loader_time: 0.00360 (0.11620) + + + --> STEP: 171/234 -- GLOBAL_STEP: 70605 + | > loss: -0.42039 (-0.32238) + | > log_mle: -0.62868 (-0.44694) + | > loss_dur: 0.20829 (0.12456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.37823 (22.46789) + | > current_lr: 0.00008 + | > step_time: 2.89790 (2.64538) + | > loader_time: 0.00830 (0.11409) + + + --> STEP: 176/234 -- GLOBAL_STEP: 70610 + | > loss: -0.39688 (-0.32461) + | > log_mle: -0.59816 (-0.45162) + | > loss_dur: 0.20128 (0.12701) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.13702 (23.49243) + | > current_lr: 0.00008 + | > step_time: 4.49000 (2.71052) + | > loader_time: 0.01200 (0.11252) + + + --> STEP: 181/234 -- GLOBAL_STEP: 70615 + | > loss: -0.33217 (-0.32641) + | > log_mle: -0.52976 (-0.45576) + | > loss_dur: 0.19759 (0.12935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.90934 (24.32975) + | > current_lr: 0.00008 + | > step_time: 6.09320 (2.75615) + | > loader_time: 0.01100 (0.11150) + + + --> STEP: 186/234 -- GLOBAL_STEP: 70620 + | > loss: -0.35089 (-0.32824) + | > log_mle: -0.57584 (-0.46013) + | > loss_dur: 0.22495 (0.13189) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.70089 (25.18566) + | > current_lr: 0.00008 + | > step_time: 2.90000 (2.82237) + | > loader_time: 0.09200 (0.11063) + + + --> STEP: 191/234 -- GLOBAL_STEP: 70625 + | > loss: -0.40747 (-0.33037) + | > log_mle: -0.60262 (-0.46430) + | > loss_dur: 0.19515 (0.13393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.39658 (25.92961) + | > current_lr: 0.00008 + | > step_time: 7.50890 (2.90288) + | > loader_time: 0.18530 (0.10927) + + + --> STEP: 196/234 -- GLOBAL_STEP: 70630 + | > loss: -0.37627 (-0.33278) + | > log_mle: -0.59327 (-0.46874) + | > loss_dur: 0.21700 (0.13596) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.07560 (26.71029) + | > current_lr: 0.00008 + | > step_time: 2.80490 (2.97746) + | > loader_time: 0.00390 (0.10943) + + + --> STEP: 201/234 -- GLOBAL_STEP: 70635 + | > loss: -0.31167 (-0.33444) + | > log_mle: -0.54653 (-0.47252) + | > loss_dur: 0.23486 (0.13807) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.31900 (27.54293) + | > current_lr: 0.00008 + | > step_time: 3.40480 (3.05168) + | > loader_time: 0.09970 (0.10973) + + + --> STEP: 206/234 -- GLOBAL_STEP: 70640 + | > loss: -0.44487 (-0.33691) + | > log_mle: -0.66541 (-0.47700) + | > loss_dur: 0.22054 (0.14009) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.67652 (28.39129) + | > current_lr: 0.00008 + | > step_time: 11.19570 (3.10812) + | > loader_time: 0.08930 (0.10808) + + + --> STEP: 211/234 -- GLOBAL_STEP: 70645 + | > loss: -0.46009 (-0.33944) + | > log_mle: -0.70926 (-0.48184) + | > loss_dur: 0.24917 (0.14239) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 112.69118 (29.85789) + | > current_lr: 0.00008 + | > step_time: 3.90180 (3.13306) + | > loader_time: 0.09310 (0.10691) + + + --> STEP: 216/234 -- GLOBAL_STEP: 70650 + | > loss: -0.48106 (-0.34219) + | > log_mle: -0.73048 (-0.48664) + | > loss_dur: 0.24941 (0.14445) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.22563 (31.01243) + | > current_lr: 0.00008 + | > step_time: 4.89720 (3.18368) + | > loader_time: 0.08570 (0.10619) + + + --> STEP: 221/234 -- GLOBAL_STEP: 70655 + | > loss: -0.40721 (-0.34457) + | > log_mle: -0.61772 (-0.49102) + | > loss_dur: 0.21051 (0.14645) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.63451 (32.04388) + | > current_lr: 0.00008 + | > step_time: 5.79340 (3.26864) + | > loader_time: 0.00310 (0.10473) + + + --> STEP: 226/234 -- GLOBAL_STEP: 70660 + | > loss: -0.47308 (-0.34693) + | > log_mle: -0.71760 (-0.49585) + | > loss_dur: 0.24452 (0.14892) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.78468 (32.96823) + | > current_lr: 0.00008 + | > step_time: 0.24690 (3.20705) + | > loader_time: 0.00480 (0.10286) + + + --> STEP: 231/234 -- GLOBAL_STEP: 70665 + | > loss: -0.44948 (-0.34919) + | > log_mle: -0.82025 (-0.50141) + | > loss_dur: 0.37078 (0.15223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.95784 (33.81160) + | > current_lr: 0.00008 + | > step_time: 0.27220 (3.14321) + | > loader_time: 0.00410 (0.10071) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00253 (-0.15541) + | > avg_loss: -0.35343 (-0.01442) + | > avg_log_mle: -0.57315 (-0.00143) + | > avg_loss_dur: 0.21972 (-0.01299) + + + > EPOCH: 302/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 00:46:22)  + + --> STEP: 2/234 -- GLOBAL_STEP: 70670 + | > loss: -0.35852 (-0.34782) + | > log_mle: -0.44412 (-0.43239) + | > loss_dur: 0.08560 (0.08456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.13642 (16.46241) + | > current_lr: 0.00008 + | > step_time: 8.49660 (8.05157) + | > loader_time: 0.19950 (0.19477) + + + --> STEP: 7/234 -- GLOBAL_STEP: 70675 + | > loss: -0.36308 (-0.33613) + | > log_mle: -0.43336 (-0.42825) + | > loss_dur: 0.07028 (0.09213) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.92991 (17.13984) + | > current_lr: 0.00008 + | > step_time: 5.40750 (4.83289) + | > loader_time: 0.00190 (0.09361) + + + --> STEP: 12/234 -- GLOBAL_STEP: 70680 + | > loss: -0.34960 (-0.34475) + | > log_mle: -0.43029 (-0.43310) + | > loss_dur: 0.08069 (0.08835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.61954 (16.80338) + | > current_lr: 0.00008 + | > step_time: 5.20460 (4.92931) + | > loader_time: 0.28330 (0.10240) + + + --> STEP: 17/234 -- GLOBAL_STEP: 70685 + | > loss: -0.39021 (-0.35658) + | > log_mle: -0.44656 (-0.43684) + | > loss_dur: 0.05635 (0.08027) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.43893 (15.86687) + | > current_lr: 0.00008 + | > step_time: 3.05020 (4.57370) + | > loader_time: 0.00580 (0.07955) + + + --> STEP: 22/234 -- GLOBAL_STEP: 70690 + | > loss: -0.35323 (-0.35919) + | > log_mle: -0.42881 (-0.43622) + | > loss_dur: 0.07558 (0.07702) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.45800 (14.68924) + | > current_lr: 0.00008 + | > step_time: 1.42100 (3.89699) + | > loader_time: 0.00130 (0.06604) + + + --> STEP: 27/234 -- GLOBAL_STEP: 70695 + | > loss: -0.35882 (-0.36082) + | > log_mle: -0.42418 (-0.43568) + | > loss_dur: 0.06536 (0.07486) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.58542 (14.02604) + | > current_lr: 0.00008 + | > step_time: 5.19730 (3.58613) + | > loader_time: 0.00720 (0.05453) + + + --> STEP: 32/234 -- GLOBAL_STEP: 70700 + | > loss: -0.36382 (-0.36127) + | > log_mle: -0.42683 (-0.43528) + | > loss_dur: 0.06301 (0.07401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.70131 (13.38733) + | > current_lr: 0.00008 + | > step_time: 6.20140 (3.93935) + | > loader_time: 0.09760 (0.06057) + + + --> STEP: 37/234 -- GLOBAL_STEP: 70705 + | > loss: -0.33970 (-0.35793) + | > log_mle: -0.40202 (-0.43192) + | > loss_dur: 0.06232 (0.07399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.23561 (12.98112) + | > current_lr: 0.00008 + | > step_time: 2.61290 (3.89443) + | > loader_time: 0.09540 (0.05937) + + + --> STEP: 42/234 -- GLOBAL_STEP: 70710 + | > loss: -0.32475 (-0.35521) + | > log_mle: -0.39749 (-0.42926) + | > loss_dur: 0.07274 (0.07406) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.76071 (12.59054) + | > current_lr: 0.00008 + | > step_time: 1.13940 (3.58890) + | > loader_time: 0.00260 (0.05258) + + + --> STEP: 47/234 -- GLOBAL_STEP: 70715 + | > loss: -0.33530 (-0.35214) + | > log_mle: -0.41636 (-0.42737) + | > loss_dur: 0.08106 (0.07523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.46068 (12.44999) + | > current_lr: 0.00008 + | > step_time: 1.96150 (3.38571) + | > loader_time: 0.00230 (0.04726) + + + --> STEP: 52/234 -- GLOBAL_STEP: 70720 + | > loss: -0.31996 (-0.35115) + | > log_mle: -0.40399 (-0.42617) + | > loss_dur: 0.08403 (0.07502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.01049 (11.97593) + | > current_lr: 0.00008 + | > step_time: 1.22110 (3.22985) + | > loader_time: 0.00160 (0.04629) + + + --> STEP: 57/234 -- GLOBAL_STEP: 70725 + | > loss: -0.31945 (-0.34904) + | > log_mle: -0.39435 (-0.42455) + | > loss_dur: 0.07490 (0.07551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.13100 (11.81662) + | > current_lr: 0.00008 + | > step_time: 2.19310 (3.11370) + | > loader_time: 0.00280 (0.04387) + + + --> STEP: 62/234 -- GLOBAL_STEP: 70730 + | > loss: -0.27156 (-0.34566) + | > log_mle: -0.40569 (-0.42289) + | > loss_dur: 0.13413 (0.07723) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.18043 (12.06798) + | > current_lr: 0.00008 + | > step_time: 2.09910 (3.03697) + | > loader_time: 0.00440 (0.04190) + + + --> STEP: 67/234 -- GLOBAL_STEP: 70735 + | > loss: -0.30036 (-0.34320) + | > log_mle: -0.40017 (-0.42080) + | > loss_dur: 0.09981 (0.07759) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.68212 (11.91477) + | > current_lr: 0.00008 + | > step_time: 3.30460 (2.97721) + | > loader_time: 0.00270 (0.04019) + + + --> STEP: 72/234 -- GLOBAL_STEP: 70740 + | > loss: -0.29178 (-0.33925) + | > log_mle: -0.38999 (-0.41853) + | > loss_dur: 0.09821 (0.07928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.73681 (11.92363) + | > current_lr: 0.00008 + | > step_time: 2.24780 (2.90683) + | > loader_time: 0.08330 (0.03880) + + + --> STEP: 77/234 -- GLOBAL_STEP: 70745 + | > loss: -0.29264 (-0.33544) + | > log_mle: -0.38877 (-0.41654) + | > loss_dur: 0.09613 (0.08109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63541 (11.97999) + | > current_lr: 0.00008 + | > step_time: 2.60320 (2.81877) + | > loader_time: 0.00280 (0.03863) + + + --> STEP: 82/234 -- GLOBAL_STEP: 70750 + | > loss: -0.28199 (-0.33263) + | > log_mle: -0.38274 (-0.41471) + | > loss_dur: 0.10075 (0.08208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.38645 (11.97002) + | > current_lr: 0.00008 + | > step_time: 1.80590 (2.74936) + | > loader_time: 0.00250 (0.03644) + + + --> STEP: 87/234 -- GLOBAL_STEP: 70755 + | > loss: -0.27995 (-0.32965) + | > log_mle: -0.38615 (-0.41315) + | > loss_dur: 0.10619 (0.08351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.46480 (12.13517) + | > current_lr: 0.00008 + | > step_time: 1.99700 (2.69799) + | > loader_time: 0.00450 (0.03453) + + + --> STEP: 92/234 -- GLOBAL_STEP: 70760 + | > loss: -0.28139 (-0.32677) + | > log_mle: -0.41570 (-0.41272) + | > loss_dur: 0.13431 (0.08595) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.35243 (12.56132) + | > current_lr: 0.00008 + | > step_time: 1.31610 (2.65128) + | > loader_time: 0.00180 (0.03373) + + + --> STEP: 97/234 -- GLOBAL_STEP: 70765 + | > loss: -0.28757 (-0.32501) + | > log_mle: -0.40405 (-0.41326) + | > loss_dur: 0.11648 (0.08825) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.19381 (12.98700) + | > current_lr: 0.00008 + | > step_time: 2.70460 (2.62666) + | > loader_time: 0.00290 (0.03215) + + + --> STEP: 102/234 -- GLOBAL_STEP: 70770 + | > loss: -0.25494 (-0.32242) + | > log_mle: -0.38894 (-0.41303) + | > loss_dur: 0.13400 (0.09060) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.52840 (13.31864) + | > current_lr: 0.00008 + | > step_time: 2.35380 (2.61325) + | > loader_time: 0.00190 (0.03248) + + + --> STEP: 107/234 -- GLOBAL_STEP: 70775 + | > loss: -0.26259 (-0.32038) + | > log_mle: -0.41908 (-0.41383) + | > loss_dur: 0.15649 (0.09345) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.25177 (13.84938) + | > current_lr: 0.00008 + | > step_time: 1.99890 (2.61072) + | > loader_time: 0.00250 (0.03273) + + + --> STEP: 112/234 -- GLOBAL_STEP: 70780 + | > loss: -0.26956 (-0.31831) + | > log_mle: -0.43508 (-0.41463) + | > loss_dur: 0.16552 (0.09632) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.87524 (14.48172) + | > current_lr: 0.00008 + | > step_time: 0.99830 (2.56625) + | > loader_time: 0.00250 (0.03217) + + + --> STEP: 117/234 -- GLOBAL_STEP: 70785 + | > loss: -0.28385 (-0.31663) + | > log_mle: -0.43612 (-0.41554) + | > loss_dur: 0.15227 (0.09891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.28963 (14.83505) + | > current_lr: 0.00008 + | > step_time: 1.80320 (2.52868) + | > loader_time: 0.18560 (0.03456) + + + --> STEP: 122/234 -- GLOBAL_STEP: 70790 + | > loss: -0.26232 (-0.31482) + | > log_mle: -0.40226 (-0.41561) + | > loss_dur: 0.13994 (0.10079) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.59233 (15.24789) + | > current_lr: 0.00008 + | > step_time: 1.57120 (2.49116) + | > loader_time: 0.00270 (0.03455) + + + --> STEP: 127/234 -- GLOBAL_STEP: 70795 + | > loss: -0.29069 (-0.31365) + | > log_mle: -0.46114 (-0.41696) + | > loss_dur: 0.17045 (0.10331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.14167 (15.77380) + | > current_lr: 0.00008 + | > step_time: 1.60820 (2.48129) + | > loader_time: 0.08790 (0.03482) + + + --> STEP: 132/234 -- GLOBAL_STEP: 70800 + | > loss: -0.29670 (-0.31339) + | > log_mle: -0.44578 (-0.41890) + | > loss_dur: 0.14907 (0.10551) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.14348 (16.40129) + | > current_lr: 0.00008 + | > step_time: 1.18030 (2.46721) + | > loader_time: 0.00330 (0.03425) + + + --> STEP: 137/234 -- GLOBAL_STEP: 70805 + | > loss: -0.26948 (-0.31320) + | > log_mle: -0.45827 (-0.42122) + | > loss_dur: 0.18879 (0.10802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.05772 (17.05278) + | > current_lr: 0.00008 + | > step_time: 1.30410 (2.45982) + | > loader_time: 0.07880 (0.03498) + + + --> STEP: 142/234 -- GLOBAL_STEP: 70810 + | > loss: -0.29513 (-0.31282) + | > log_mle: -0.47544 (-0.42308) + | > loss_dur: 0.18031 (0.11027) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.73564 (17.45828) + | > current_lr: 0.00008 + | > step_time: 1.48540 (2.44437) + | > loader_time: 0.00260 (0.03446) + + + --> STEP: 147/234 -- GLOBAL_STEP: 70815 + | > loss: -0.30997 (-0.31387) + | > log_mle: -0.47738 (-0.42681) + | > loss_dur: 0.16741 (0.11294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.04682 (18.10209) + | > current_lr: 0.00008 + | > step_time: 2.89980 (2.43610) + | > loader_time: 0.08610 (0.03514) + + + --> STEP: 152/234 -- GLOBAL_STEP: 70820 + | > loss: -0.37223 (-0.31517) + | > log_mle: -0.56792 (-0.43037) + | > loss_dur: 0.19569 (0.11520) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.05941 (18.67004) + | > current_lr: 0.00008 + | > step_time: 3.89390 (2.43758) + | > loader_time: 0.00250 (0.03622) + + + --> STEP: 157/234 -- GLOBAL_STEP: 70825 + | > loss: -0.32350 (-0.31720) + | > log_mle: -0.50921 (-0.43469) + | > loss_dur: 0.18571 (0.11749) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.00511 (19.57185) + | > current_lr: 0.00008 + | > step_time: 3.88910 (2.49502) + | > loader_time: 0.00970 (0.03646) + + + --> STEP: 162/234 -- GLOBAL_STEP: 70830 + | > loss: -0.35823 (-0.31883) + | > log_mle: -0.54239 (-0.43876) + | > loss_dur: 0.18415 (0.11992) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.74426 (20.51445) + | > current_lr: 0.00008 + | > step_time: 8.19610 (2.58356) + | > loader_time: 0.01050 (0.04068) + + + --> STEP: 167/234 -- GLOBAL_STEP: 70835 + | > loss: -0.44631 (-0.32038) + | > log_mle: -0.63733 (-0.44242) + | > loss_dur: 0.19102 (0.12204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.10094 (21.48733) + | > current_lr: 0.00008 + | > step_time: 1.68690 (2.60089) + | > loader_time: 0.00390 (0.04233) + + + --> STEP: 172/234 -- GLOBAL_STEP: 70840 + | > loss: -0.43538 (-0.32266) + | > log_mle: -0.63927 (-0.44724) + | > loss_dur: 0.20389 (0.12459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.27465 (22.29764) + | > current_lr: 0.00008 + | > step_time: 2.69840 (2.63511) + | > loader_time: 0.08790 (0.04253) + + + --> STEP: 177/234 -- GLOBAL_STEP: 70845 + | > loss: -0.37480 (-0.32469) + | > log_mle: -0.58229 (-0.45178) + | > loss_dur: 0.20749 (0.12710) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.20786 (23.19183) + | > current_lr: 0.00008 + | > step_time: 2.90850 (2.64377) + | > loader_time: 0.39430 (0.04461) + + + --> STEP: 182/234 -- GLOBAL_STEP: 70850 + | > loss: -0.37339 (-0.32639) + | > log_mle: -0.62378 (-0.45618) + | > loss_dur: 0.25039 (0.12979) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.04437 (24.14329) + | > current_lr: 0.00008 + | > step_time: 3.40540 (2.69642) + | > loader_time: 0.00520 (0.08418) + + + --> STEP: 187/234 -- GLOBAL_STEP: 70855 + | > loss: -0.39840 (-0.32826) + | > log_mle: -0.61457 (-0.46042) + | > loss_dur: 0.21617 (0.13216) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.58276 (25.25010) + | > current_lr: 0.00008 + | > step_time: 5.30200 (2.77610) + | > loader_time: 0.10410 (0.08414) + + + --> STEP: 192/234 -- GLOBAL_STEP: 70860 + | > loss: -0.45007 (-0.33054) + | > log_mle: -0.66718 (-0.46472) + | > loss_dur: 0.21711 (0.13419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.65173 (26.07104) + | > current_lr: 0.00008 + | > step_time: 9.70870 (2.91852) + | > loader_time: 0.10450 (0.08769) + + + --> STEP: 197/234 -- GLOBAL_STEP: 70865 + | > loss: -0.44019 (-0.33281) + | > log_mle: -0.63045 (-0.46887) + | > loss_dur: 0.19026 (0.13606) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.70017 (26.75347) + | > current_lr: 0.00008 + | > step_time: 1.69270 (2.90844) + | > loader_time: 0.00590 (0.08741) + + + --> STEP: 202/234 -- GLOBAL_STEP: 70870 + | > loss: -0.51827 (-0.33499) + | > log_mle: -0.72565 (-0.47315) + | > loss_dur: 0.20738 (0.13816) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.78918 (27.67320) + | > current_lr: 0.00008 + | > step_time: 1.38150 (2.89627) + | > loader_time: 0.00270 (0.08573) + + + --> STEP: 207/234 -- GLOBAL_STEP: 70875 + | > loss: -0.47935 (-0.33717) + | > log_mle: -0.70861 (-0.47737) + | > loss_dur: 0.22926 (0.14019) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.96795 (28.43998) + | > current_lr: 0.00008 + | > step_time: 7.21180 (2.92005) + | > loader_time: 0.09480 (0.08496) + + + --> STEP: 212/234 -- GLOBAL_STEP: 70880 + | > loss: -0.44548 (-0.33974) + | > log_mle: -0.67082 (-0.48209) + | > loss_dur: 0.22534 (0.14235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.57078 (29.53806) + | > current_lr: 0.00008 + | > step_time: 7.29810 (2.97240) + | > loader_time: 0.00370 (0.08399) + + + --> STEP: 217/234 -- GLOBAL_STEP: 70885 + | > loss: -0.47309 (-0.34261) + | > log_mle: -0.71039 (-0.48697) + | > loss_dur: 0.23729 (0.14437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.85561 (30.42010) + | > current_lr: 0.00008 + | > step_time: 7.90460 (3.02425) + | > loader_time: 0.09760 (0.08379) + + + --> STEP: 222/234 -- GLOBAL_STEP: 70890 + | > loss: -0.45071 (-0.34524) + | > log_mle: -0.71435 (-0.49164) + | > loss_dur: 0.26363 (0.14640) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.52967 (31.39817) + | > current_lr: 0.00008 + | > step_time: 0.24460 (3.01116) + | > loader_time: 0.00300 (0.08237) + + + --> STEP: 227/234 -- GLOBAL_STEP: 70895 + | > loss: -0.44026 (-0.34820) + | > log_mle: -0.68794 (-0.49668) + | > loss_dur: 0.24768 (0.14847) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.14701 (32.32629) + | > current_lr: 0.00008 + | > step_time: 0.24370 (2.95000) + | > loader_time: 0.00340 (0.08064) + + + --> STEP: 232/234 -- GLOBAL_STEP: 70900 + | > loss: -0.44860 (-0.35073) + | > log_mle: -0.91641 (-0.50335) + | > loss_dur: 0.46782 (0.15262) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.41765 (33.65950) + | > current_lr: 0.00008 + | > step_time: 0.32120 (2.89216) + | > loader_time: 0.00400 (0.07899) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.20403 (+0.20151) + | > avg_loss: -0.34081 (+0.01262) + | > avg_log_mle: -0.57227 (+0.00089) + | > avg_loss_dur: 0.23146 (+0.01174) + + + > EPOCH: 303/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 00:58:57)  + + --> STEP: 3/234 -- GLOBAL_STEP: 70905 + | > loss: -0.27089 (-0.31212) + | > log_mle: -0.41269 (-0.42492) + | > loss_dur: 0.14180 (0.11280) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.16781 (20.46131) + | > current_lr: 0.00008 + | > step_time: 3.19680 (6.06690) + | > loader_time: 0.00380 (0.13363) + + + --> STEP: 8/234 -- GLOBAL_STEP: 70910 + | > loss: -0.38511 (-0.33934) + | > log_mle: -0.45062 (-0.42980) + | > loss_dur: 0.06551 (0.09046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.79565 (19.09146) + | > current_lr: 0.00008 + | > step_time: 1.29060 (3.96856) + | > loader_time: 0.00280 (0.10105) + + + --> STEP: 13/234 -- GLOBAL_STEP: 70915 + | > loss: -0.38905 (-0.34910) + | > log_mle: -0.45399 (-0.43451) + | > loss_dur: 0.06494 (0.08541) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.81798 (16.98531) + | > current_lr: 0.00008 + | > step_time: 1.99030 (3.31104) + | > loader_time: 0.00100 (0.07547) + + + --> STEP: 18/234 -- GLOBAL_STEP: 70920 + | > loss: -0.36734 (-0.35792) + | > log_mle: -0.43092 (-0.43715) + | > loss_dur: 0.06359 (0.07923) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.11147 (15.45129) + | > current_lr: 0.00008 + | > step_time: 4.09650 (3.24753) + | > loader_time: 0.10360 (0.06076) + + + --> STEP: 23/234 -- GLOBAL_STEP: 70925 + | > loss: -0.39625 (-0.36225) + | > log_mle: -0.45706 (-0.43796) + | > loss_dur: 0.06081 (0.07571) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.57402 (14.79556) + | > current_lr: 0.00008 + | > step_time: 2.89190 (3.53748) + | > loader_time: 0.00630 (0.05581) + + + --> STEP: 28/234 -- GLOBAL_STEP: 70930 + | > loss: -0.42155 (-0.36345) + | > log_mle: -0.46683 (-0.43688) + | > loss_dur: 0.04528 (0.07342) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.30507 (14.32745) + | > current_lr: 0.00008 + | > step_time: 6.19740 (3.86289) + | > loader_time: 0.00300 (0.05289) + + + --> STEP: 33/234 -- GLOBAL_STEP: 70935 + | > loss: -0.33659 (-0.36028) + | > log_mle: -0.40661 (-0.43343) + | > loss_dur: 0.07003 (0.07316) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.70949 (14.11041) + | > current_lr: 0.00008 + | > step_time: 4.58820 (3.75269) + | > loader_time: 0.10760 (0.05093) + + + --> STEP: 38/234 -- GLOBAL_STEP: 70940 + | > loss: -0.32436 (-0.35574) + | > log_mle: -0.40587 (-0.42911) + | > loss_dur: 0.08151 (0.07337) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67062 (13.81771) + | > current_lr: 0.00008 + | > step_time: 1.16780 (3.53871) + | > loader_time: 0.00400 (0.04692) + + + --> STEP: 43/234 -- GLOBAL_STEP: 70945 + | > loss: -0.31270 (-0.35143) + | > log_mle: -0.40222 (-0.42536) + | > loss_dur: 0.08952 (0.07393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.16492 (13.77519) + | > current_lr: 0.00008 + | > step_time: 1.50480 (3.29197) + | > loader_time: 0.00150 (0.04563) + + + --> STEP: 48/234 -- GLOBAL_STEP: 70950 + | > loss: -0.35904 (-0.34931) + | > log_mle: -0.42104 (-0.42339) + | > loss_dur: 0.06200 (0.07408) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.09583 (13.36114) + | > current_lr: 0.00008 + | > step_time: 1.12190 (3.11079) + | > loader_time: 0.00170 (0.04111) + + + --> STEP: 53/234 -- GLOBAL_STEP: 70955 + | > loss: -0.32799 (-0.34752) + | > log_mle: -0.40667 (-0.42190) + | > loss_dur: 0.07868 (0.07438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.92741 (12.75878) + | > current_lr: 0.00008 + | > step_time: 1.68410 (2.95511) + | > loader_time: 0.00210 (0.03905) + + + --> STEP: 58/234 -- GLOBAL_STEP: 70960 + | > loss: -0.34342 (-0.34592) + | > log_mle: -0.40783 (-0.42038) + | > loss_dur: 0.06442 (0.07446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.61647 (12.40571) + | > current_lr: 0.00008 + | > step_time: 1.38050 (2.85463) + | > loader_time: 0.00210 (0.03761) + + + --> STEP: 63/234 -- GLOBAL_STEP: 70965 + | > loss: -0.29587 (-0.34196) + | > log_mle: -0.38380 (-0.41857) + | > loss_dur: 0.08793 (0.07661) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.00656 (12.63037) + | > current_lr: 0.00008 + | > step_time: 1.48960 (2.76336) + | > loader_time: 0.00840 (0.03490) + + + --> STEP: 68/234 -- GLOBAL_STEP: 70970 + | > loss: -0.28869 (-0.33951) + | > log_mle: -0.38481 (-0.41667) + | > loss_dur: 0.09611 (0.07715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.00103 (12.41821) + | > current_lr: 0.00008 + | > step_time: 1.98320 (2.67876) + | > loader_time: 0.00230 (0.03518) + + + --> STEP: 73/234 -- GLOBAL_STEP: 70975 + | > loss: -0.26830 (-0.33590) + | > log_mle: -0.39273 (-0.41488) + | > loss_dur: 0.12443 (0.07898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.54159 (12.30863) + | > current_lr: 0.00008 + | > step_time: 1.31650 (2.63778) + | > loader_time: 0.00240 (0.03666) + + + --> STEP: 78/234 -- GLOBAL_STEP: 70980 + | > loss: -0.28178 (-0.33268) + | > log_mle: -0.37869 (-0.41302) + | > loss_dur: 0.09691 (0.08035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.00083 (12.23067) + | > current_lr: 0.00008 + | > step_time: 1.16610 (2.56825) + | > loader_time: 0.00190 (0.03448) + + + --> STEP: 83/234 -- GLOBAL_STEP: 70985 + | > loss: -0.26052 (-0.32995) + | > log_mle: -0.38916 (-0.41166) + | > loss_dur: 0.12863 (0.08171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.13177 (12.14587) + | > current_lr: 0.00008 + | > step_time: 2.64930 (2.55823) + | > loader_time: 0.00460 (0.03364) + + + --> STEP: 88/234 -- GLOBAL_STEP: 70990 + | > loss: -0.29307 (-0.32765) + | > log_mle: -0.42095 (-0.41073) + | > loss_dur: 0.12787 (0.08309) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.43994 (12.15899) + | > current_lr: 0.00008 + | > step_time: 2.50550 (2.53263) + | > loader_time: 0.00650 (0.03309) + + + --> STEP: 93/234 -- GLOBAL_STEP: 70995 + | > loss: -0.29010 (-0.32540) + | > log_mle: -0.43343 (-0.41089) + | > loss_dur: 0.14333 (0.08549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.97031 (12.28795) + | > current_lr: 0.00008 + | > step_time: 3.00880 (2.50746) + | > loader_time: 0.00260 (0.03421) + + + --> STEP: 98/234 -- GLOBAL_STEP: 71000 + | > loss: -0.26476 (-0.32346) + | > log_mle: -0.37376 (-0.41108) + | > loss_dur: 0.10900 (0.08762) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.52505 (12.39056) + | > current_lr: 0.00008 + | > step_time: 1.90450 (2.45509) + | > loader_time: 0.00700 (0.03351) + + + --> STEP: 103/234 -- GLOBAL_STEP: 71005 + | > loss: -0.30711 (-0.32154) + | > log_mle: -0.45900 (-0.41196) + | > loss_dur: 0.15189 (0.09042) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.14705 (12.79385) + | > current_lr: 0.00008 + | > step_time: 1.48360 (2.40662) + | > loader_time: 0.00190 (0.03299) + + + --> STEP: 108/234 -- GLOBAL_STEP: 71010 + | > loss: -0.27248 (-0.31951) + | > log_mle: -0.40299 (-0.41243) + | > loss_dur: 0.13051 (0.09292) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.77110 (13.04878) + | > current_lr: 0.00008 + | > step_time: 1.60370 (2.38602) + | > loader_time: 0.08440 (0.03235) + + + --> STEP: 113/234 -- GLOBAL_STEP: 71015 + | > loss: -0.28822 (-0.31771) + | > log_mle: -0.44917 (-0.41383) + | > loss_dur: 0.16095 (0.09612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.74436 (13.49602) + | > current_lr: 0.00008 + | > step_time: 2.91160 (2.41355) + | > loader_time: 0.00510 (0.03200) + + + --> STEP: 118/234 -- GLOBAL_STEP: 71020 + | > loss: -0.25952 (-0.31580) + | > log_mle: -0.42026 (-0.41461) + | > loss_dur: 0.16074 (0.09881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.09577 (13.82587) + | > current_lr: 0.00008 + | > step_time: 4.00040 (2.44367) + | > loader_time: 0.09360 (0.03386) + + + --> STEP: 123/234 -- GLOBAL_STEP: 71025 + | > loss: -0.23982 (-0.31391) + | > log_mle: -0.38642 (-0.41453) + | > loss_dur: 0.14660 (0.10062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.79379 (14.01616) + | > current_lr: 0.00008 + | > step_time: 2.58640 (2.47413) + | > loader_time: 0.00250 (0.03546) + + + --> STEP: 128/234 -- GLOBAL_STEP: 71030 + | > loss: -0.29537 (-0.31327) + | > log_mle: -0.44077 (-0.41637) + | > loss_dur: 0.14540 (0.10309) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.96541 (14.77734) + | > current_lr: 0.00008 + | > step_time: 0.90970 (2.44954) + | > loader_time: 0.08360 (0.03603) + + + --> STEP: 133/234 -- GLOBAL_STEP: 71035 + | > loss: -0.31074 (-0.31322) + | > log_mle: -0.47436 (-0.41870) + | > loss_dur: 0.16362 (0.10548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.67518 (15.49525) + | > current_lr: 0.00008 + | > step_time: 5.68690 (2.50623) + | > loader_time: 0.09490 (0.03756) + + + --> STEP: 138/234 -- GLOBAL_STEP: 71040 + | > loss: -0.26546 (-0.31264) + | > log_mle: -0.41932 (-0.42062) + | > loss_dur: 0.15386 (0.10798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.81001 (16.09276) + | > current_lr: 0.00008 + | > step_time: 1.69160 (2.53195) + | > loader_time: 0.00320 (0.03759) + + + --> STEP: 143/234 -- GLOBAL_STEP: 71045 + | > loss: -0.36791 (-0.31303) + | > log_mle: -0.57741 (-0.42355) + | > loss_dur: 0.20950 (0.11053) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.59121 (16.80575) + | > current_lr: 0.00008 + | > step_time: 4.00220 (2.54844) + | > loader_time: 0.09390 (0.03892) + + + --> STEP: 148/234 -- GLOBAL_STEP: 71050 + | > loss: -0.33590 (-0.31388) + | > log_mle: -0.48196 (-0.42667) + | > loss_dur: 0.14606 (0.11279) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.55133 (17.39345) + | > current_lr: 0.00008 + | > step_time: 1.69210 (2.52105) + | > loader_time: 0.00310 (0.03877) + + + --> STEP: 153/234 -- GLOBAL_STEP: 71055 + | > loss: -0.43487 (-0.31573) + | > log_mle: -0.62404 (-0.43114) + | > loss_dur: 0.18917 (0.11541) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.97831 (18.33479) + | > current_lr: 0.00008 + | > step_time: 4.10140 (2.56488) + | > loader_time: 0.00610 (0.04016) + + + --> STEP: 158/234 -- GLOBAL_STEP: 71060 + | > loss: -0.33870 (-0.31700) + | > log_mle: -0.55115 (-0.43490) + | > loss_dur: 0.21245 (0.11790) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.15702 (19.30644) + | > current_lr: 0.00008 + | > step_time: 2.30540 (2.64637) + | > loader_time: 0.00450 (0.04274) + + + --> STEP: 163/234 -- GLOBAL_STEP: 71065 + | > loss: -0.32996 (-0.31860) + | > log_mle: -0.51886 (-0.43876) + | > loss_dur: 0.18890 (0.12017) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.21603 (20.06865) + | > current_lr: 0.00008 + | > step_time: 4.50820 (2.72422) + | > loader_time: 0.10660 (0.04328) + + + --> STEP: 168/234 -- GLOBAL_STEP: 71070 + | > loss: -0.36453 (-0.32047) + | > log_mle: -0.57119 (-0.44289) + | > loss_dur: 0.20667 (0.12242) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.06799 (20.85431) + | > current_lr: 0.00008 + | > step_time: 1.60840 (2.70626) + | > loader_time: 0.28530 (0.04441) + + + --> STEP: 173/234 -- GLOBAL_STEP: 71075 + | > loss: -0.38490 (-0.32264) + | > log_mle: -0.58826 (-0.44764) + | > loss_dur: 0.20337 (0.12500) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.09681 (21.72307) + | > current_lr: 0.00008 + | > step_time: 4.80770 (2.75409) + | > loader_time: 0.09660 (0.04540) + + + --> STEP: 178/234 -- GLOBAL_STEP: 71080 + | > loss: -0.42838 (-0.32498) + | > log_mle: -0.65034 (-0.45245) + | > loss_dur: 0.22196 (0.12747) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.81146 (22.73569) + | > current_lr: 0.00008 + | > step_time: 5.71320 (2.83690) + | > loader_time: 0.09000 (0.04691) + + + --> STEP: 183/234 -- GLOBAL_STEP: 71085 + | > loss: -0.44028 (-0.32684) + | > log_mle: -0.65396 (-0.45690) + | > loss_dur: 0.21368 (0.13006) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.01792 (23.50767) + | > current_lr: 0.00008 + | > step_time: 5.49630 (2.92222) + | > loader_time: 0.50740 (0.05107) + + + --> STEP: 188/234 -- GLOBAL_STEP: 71090 + | > loss: -0.44863 (-0.32904) + | > log_mle: -0.66641 (-0.46132) + | > loss_dur: 0.21779 (0.13228) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.18996 (24.44642) + | > current_lr: 0.00008 + | > step_time: 3.35480 (2.96487) + | > loader_time: 0.00520 (0.05041) + + + --> STEP: 193/234 -- GLOBAL_STEP: 71095 + | > loss: -0.44821 (-0.33142) + | > log_mle: -0.66103 (-0.46565) + | > loss_dur: 0.21281 (0.13423) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.57009 (25.46840) + | > current_lr: 0.00008 + | > step_time: 3.81010 (2.96021) + | > loader_time: 0.09190 (0.05010) + + + --> STEP: 198/234 -- GLOBAL_STEP: 71100 + | > loss: -0.42620 (-0.33362) + | > log_mle: -0.65188 (-0.46979) + | > loss_dur: 0.22567 (0.13617) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.23746 (26.40929) + | > current_lr: 0.00008 + | > step_time: 3.61870 (2.99877) + | > loader_time: 0.19330 (0.05181) + + + --> STEP: 203/234 -- GLOBAL_STEP: 71105 + | > loss: -0.36664 (-0.33541) + | > log_mle: -0.57187 (-0.47370) + | > loss_dur: 0.20524 (0.13829) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.27701 (27.34649) + | > current_lr: 0.00008 + | > step_time: 3.48690 (3.04071) + | > loader_time: 0.00600 (0.05388) + + + --> STEP: 208/234 -- GLOBAL_STEP: 71110 + | > loss: -0.43453 (-0.33816) + | > log_mle: -0.67016 (-0.47863) + | > loss_dur: 0.23563 (0.14047) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.14383 (28.31775) + | > current_lr: 0.00008 + | > step_time: 6.31480 (3.09175) + | > loader_time: 0.08430 (0.05583) + + + --> STEP: 213/234 -- GLOBAL_STEP: 71115 + | > loss: -0.47731 (-0.34116) + | > log_mle: -0.71457 (-0.48382) + | > loss_dur: 0.23726 (0.14267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.39745 (29.53659) + | > current_lr: 0.00008 + | > step_time: 6.19750 (3.16573) + | > loader_time: 0.00350 (0.05641) + + + --> STEP: 218/234 -- GLOBAL_STEP: 71120 + | > loss: -0.43499 (-0.34383) + | > log_mle: -0.66942 (-0.48855) + | > loss_dur: 0.23443 (0.14472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.79177 (30.53254) + | > current_lr: 0.00008 + | > step_time: 3.89630 (3.22158) + | > loader_time: 0.09710 (0.05644) + + + --> STEP: 223/234 -- GLOBAL_STEP: 71125 + | > loss: -0.48810 (-0.34693) + | > log_mle: -0.71743 (-0.49368) + | > loss_dur: 0.22933 (0.14675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.06044 (31.44475) + | > current_lr: 0.00008 + | > step_time: 4.79700 (3.24120) + | > loader_time: 0.19440 (0.05695) + + + --> STEP: 228/234 -- GLOBAL_STEP: 71130 + | > loss: -0.47048 (-0.35018) + | > log_mle: -0.73065 (-0.49914) + | > loss_dur: 0.26017 (0.14896) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.29859 (32.25183) + | > current_lr: 0.00008 + | > step_time: 0.25610 (3.18341) + | > loader_time: 0.00760 (0.05658) + + + --> STEP: 233/234 -- GLOBAL_STEP: 71135 + | > loss: -0.04066 (-0.35115) + | > log_mle: -0.71251 (-0.50602) + | > loss_dur: 0.67186 (0.15487) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.14830 (33.67183) + | > current_lr: 0.00008 + | > step_time: 0.19880 (3.12082) + | > loader_time: 0.00300 (0.05550) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.38019 (+0.17616) + | > avg_loss: -0.35742 (-0.01661) + | > avg_log_mle: -0.58052 (-0.00825) + | > avg_loss_dur: 0.22310 (-0.00836) + + + > EPOCH: 304/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 01:12:21)  + + --> STEP: 4/234 -- GLOBAL_STEP: 71140 + | > loss: -0.35784 (-0.32798) + | > log_mle: -0.42719 (-0.42689) + | > loss_dur: 0.06935 (0.09891) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.60699 (19.50579) + | > current_lr: 0.00008 + | > step_time: 6.71320 (5.37733) + | > loader_time: 0.18750 (0.07303) + + + --> STEP: 9/234 -- GLOBAL_STEP: 71145 + | > loss: -0.34353 (-0.34052) + | > log_mle: -0.43739 (-0.43082) + | > loss_dur: 0.09386 (0.09030) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.75852 (19.02020) + | > current_lr: 0.00008 + | > step_time: 3.98690 (5.17653) + | > loader_time: 0.00120 (0.08818) + + + --> STEP: 14/234 -- GLOBAL_STEP: 71150 + | > loss: -0.37237 (-0.35154) + | > log_mle: -0.43843 (-0.43462) + | > loss_dur: 0.06605 (0.08308) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.17654 (17.21004) + | > current_lr: 0.00008 + | > step_time: 4.51610 (4.64360) + | > loader_time: 0.08180 (0.07690) + + + --> STEP: 19/234 -- GLOBAL_STEP: 71155 + | > loss: -0.39090 (-0.36027) + | > log_mle: -0.44947 (-0.43746) + | > loss_dur: 0.05856 (0.07718) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.81108 (15.43352) + | > current_lr: 0.00008 + | > step_time: 2.80070 (4.17844) + | > loader_time: 0.09260 (0.06213) + + + --> STEP: 24/234 -- GLOBAL_STEP: 71160 + | > loss: -0.37516 (-0.36437) + | > log_mle: -0.43183 (-0.43804) + | > loss_dur: 0.05667 (0.07367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.21737 (14.16274) + | > current_lr: 0.00008 + | > step_time: 3.20170 (3.85437) + | > loader_time: 0.00270 (0.05655) + + + --> STEP: 29/234 -- GLOBAL_STEP: 71165 + | > loss: -0.36316 (-0.36616) + | > log_mle: -0.42516 (-0.43755) + | > loss_dur: 0.06201 (0.07139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.91098 (13.61293) + | > current_lr: 0.00008 + | > step_time: 1.10640 (3.45812) + | > loader_time: 0.00140 (0.05007) + + + --> STEP: 34/234 -- GLOBAL_STEP: 71170 + | > loss: -0.34833 (-0.36318) + | > log_mle: -0.41885 (-0.43477) + | > loss_dur: 0.07053 (0.07158) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.04526 (13.11699) + | > current_lr: 0.00008 + | > step_time: 2.05560 (3.18960) + | > loader_time: 0.00390 (0.04558) + + + --> STEP: 39/234 -- GLOBAL_STEP: 71175 + | > loss: -0.31990 (-0.35917) + | > log_mle: -0.40226 (-0.43140) + | > loss_dur: 0.08236 (0.07223) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.56876 (12.80856) + | > current_lr: 0.00008 + | > step_time: 1.97240 (2.99838) + | > loader_time: 0.00260 (0.04676) + + + --> STEP: 44/234 -- GLOBAL_STEP: 71180 + | > loss: -0.33372 (-0.35616) + | > log_mle: -0.39993 (-0.42849) + | > loss_dur: 0.06620 (0.07233) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.08880 (12.66445) + | > current_lr: 0.00008 + | > step_time: 1.50280 (2.85787) + | > loader_time: 0.08680 (0.04530) + + + --> STEP: 49/234 -- GLOBAL_STEP: 71185 + | > loss: -0.35325 (-0.35440) + | > log_mle: -0.42159 (-0.42717) + | > loss_dur: 0.06834 (0.07277) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.01211 (12.62217) + | > current_lr: 0.00008 + | > step_time: 1.28670 (2.75007) + | > loader_time: 0.00210 (0.04424) + + + --> STEP: 54/234 -- GLOBAL_STEP: 71190 + | > loss: -0.32905 (-0.35189) + | > log_mle: -0.40740 (-0.42532) + | > loss_dur: 0.07835 (0.07344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29428 (12.36610) + | > current_lr: 0.00008 + | > step_time: 0.93350 (2.64690) + | > loader_time: 0.00190 (0.04194) + + + --> STEP: 59/234 -- GLOBAL_STEP: 71195 + | > loss: -0.31965 (-0.34986) + | > log_mle: -0.40378 (-0.42365) + | > loss_dur: 0.08413 (0.07379) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.46479 (12.05107) + | > current_lr: 0.00008 + | > step_time: 1.89420 (2.59347) + | > loader_time: 0.00260 (0.04147) + + + --> STEP: 64/234 -- GLOBAL_STEP: 71200 + | > loss: -0.32168 (-0.34593) + | > log_mle: -0.39716 (-0.42166) + | > loss_dur: 0.07548 (0.07573) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.93987 (11.98632) + | > current_lr: 0.00008 + | > step_time: 2.60560 (2.54425) + | > loader_time: 0.00250 (0.04105) + + + --> STEP: 69/234 -- GLOBAL_STEP: 71205 + | > loss: -0.31155 (-0.34304) + | > log_mle: -0.38986 (-0.41956) + | > loss_dur: 0.07831 (0.07652) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.25626 (11.86756) + | > current_lr: 0.00008 + | > step_time: 3.30980 (2.54426) + | > loader_time: 0.07450 (0.04055) + + + --> STEP: 74/234 -- GLOBAL_STEP: 71210 + | > loss: -0.27967 (-0.33879) + | > log_mle: -0.38263 (-0.41759) + | > loss_dur: 0.10296 (0.07880) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63421 (11.95157) + | > current_lr: 0.00008 + | > step_time: 2.60650 (2.51950) + | > loader_time: 0.09240 (0.04034) + + + --> STEP: 79/234 -- GLOBAL_STEP: 71215 + | > loss: -0.29426 (-0.33569) + | > log_mle: -0.39433 (-0.41576) + | > loss_dur: 0.10007 (0.08007) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.46959 (12.11604) + | > current_lr: 0.00008 + | > step_time: 2.21760 (2.47347) + | > loader_time: 0.05870 (0.03972) + + + --> STEP: 84/234 -- GLOBAL_STEP: 71220 + | > loss: -0.29727 (-0.33264) + | > log_mle: -0.38864 (-0.41404) + | > loss_dur: 0.09137 (0.08140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.92465 (12.15823) + | > current_lr: 0.00008 + | > step_time: 2.60600 (2.45332) + | > loader_time: 0.00180 (0.03758) + + + --> STEP: 89/234 -- GLOBAL_STEP: 71225 + | > loss: -0.29344 (-0.33000) + | > log_mle: -0.40897 (-0.41312) + | > loss_dur: 0.11553 (0.08313) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.93541 (12.15335) + | > current_lr: 0.00008 + | > step_time: 2.59600 (2.44891) + | > loader_time: 0.00240 (0.03893) + + + --> STEP: 94/234 -- GLOBAL_STEP: 71230 + | > loss: -0.30197 (-0.32772) + | > log_mle: -0.43155 (-0.41336) + | > loss_dur: 0.12958 (0.08565) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.37421 (12.37429) + | > current_lr: 0.00008 + | > step_time: 3.50400 (2.43343) + | > loader_time: 0.00200 (0.03699) + + + --> STEP: 99/234 -- GLOBAL_STEP: 71235 + | > loss: -0.29509 (-0.32589) + | > log_mle: -0.46055 (-0.41384) + | > loss_dur: 0.16546 (0.08795) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.45578 (12.60747) + | > current_lr: 0.00008 + | > step_time: 0.95480 (2.38126) + | > loader_time: 0.00280 (0.03622) + + + --> STEP: 104/234 -- GLOBAL_STEP: 71240 + | > loss: -0.32847 (-0.32421) + | > log_mle: -0.47350 (-0.41478) + | > loss_dur: 0.14504 (0.09058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.55324 (13.01040) + | > current_lr: 0.00008 + | > step_time: 1.30320 (2.38508) + | > loader_time: 0.09750 (0.03907) + + + --> STEP: 109/234 -- GLOBAL_STEP: 71245 + | > loss: -0.26362 (-0.32179) + | > log_mle: -0.44524 (-0.41502) + | > loss_dur: 0.18162 (0.09323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.20012 (13.37108) + | > current_lr: 0.00008 + | > step_time: 1.60190 (2.37113) + | > loader_time: 0.00360 (0.03994) + + + --> STEP: 114/234 -- GLOBAL_STEP: 71250 + | > loss: -0.28154 (-0.31992) + | > log_mle: -0.42044 (-0.41596) + | > loss_dur: 0.13890 (0.09604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.01376 (14.15346) + | > current_lr: 0.00008 + | > step_time: 1.50340 (2.34525) + | > loader_time: 0.00340 (0.03906) + + + --> STEP: 119/234 -- GLOBAL_STEP: 71255 + | > loss: -0.28331 (-0.31794) + | > log_mle: -0.42650 (-0.41658) + | > loss_dur: 0.14319 (0.09864) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.75432 (14.50303) + | > current_lr: 0.00008 + | > step_time: 1.50090 (2.39124) + | > loader_time: 0.00290 (0.03927) + + + --> STEP: 124/234 -- GLOBAL_STEP: 71260 + | > loss: -0.31369 (-0.31624) + | > log_mle: -0.45599 (-0.41670) + | > loss_dur: 0.14230 (0.10046) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.04652 (14.60615) + | > current_lr: 0.00008 + | > step_time: 1.51560 (2.36536) + | > loader_time: 0.07810 (0.03842) + + + --> STEP: 129/234 -- GLOBAL_STEP: 71265 + | > loss: -0.28769 (-0.31532) + | > log_mle: -0.44475 (-0.41839) + | > loss_dur: 0.15705 (0.10306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.34658 (15.33256) + | > current_lr: 0.00008 + | > step_time: 1.58460 (2.33887) + | > loader_time: 0.00530 (0.03768) + + + --> STEP: 134/234 -- GLOBAL_STEP: 71270 + | > loss: -0.30080 (-0.31533) + | > log_mle: -0.49310 (-0.42099) + | > loss_dur: 0.19230 (0.10566) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.99750 (15.91051) + | > current_lr: 0.00008 + | > step_time: 1.80420 (2.32172) + | > loader_time: 0.00400 (0.03701) + + + --> STEP: 139/234 -- GLOBAL_STEP: 71275 + | > loss: -0.38002 (-0.31528) + | > log_mle: -0.56340 (-0.42339) + | > loss_dur: 0.18338 (0.10811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.67502 (16.57426) + | > current_lr: 0.00008 + | > step_time: 2.70670 (2.31840) + | > loader_time: 0.08470 (0.03776) + + + --> STEP: 144/234 -- GLOBAL_STEP: 71280 + | > loss: -0.33890 (-0.31526) + | > log_mle: -0.53214 (-0.42615) + | > loss_dur: 0.19324 (0.11090) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.69509 (17.38560) + | > current_lr: 0.00008 + | > step_time: 1.60560 (2.33806) + | > loader_time: 0.00390 (0.03788) + + + --> STEP: 149/234 -- GLOBAL_STEP: 71285 + | > loss: -0.39657 (-0.31630) + | > log_mle: -0.58456 (-0.42955) + | > loss_dur: 0.18799 (0.11325) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.53379 (18.35502) + | > current_lr: 0.00008 + | > step_time: 1.59860 (2.31940) + | > loader_time: 0.00390 (0.03851) + + + --> STEP: 154/234 -- GLOBAL_STEP: 71290 + | > loss: -0.34976 (-0.31757) + | > log_mle: -0.53681 (-0.43330) + | > loss_dur: 0.18705 (0.11573) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.07130 (19.60658) + | > current_lr: 0.00008 + | > step_time: 1.70900 (2.33242) + | > loader_time: 0.08480 (0.03959) + + + --> STEP: 159/234 -- GLOBAL_STEP: 71295 + | > loss: -0.36329 (-0.31862) + | > log_mle: -0.55838 (-0.43683) + | > loss_dur: 0.19509 (0.11822) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.42129 (20.36344) + | > current_lr: 0.00008 + | > step_time: 6.51230 (2.45181) + | > loader_time: 0.08650 (0.04123) + + + --> STEP: 164/234 -- GLOBAL_STEP: 71300 + | > loss: -0.32075 (-0.31921) + | > log_mle: -0.52286 (-0.43970) + | > loss_dur: 0.20211 (0.12049) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.59836 (22.09863) + | > current_lr: 0.00008 + | > step_time: 2.21150 (2.45059) + | > loader_time: 0.00330 (0.04109) + + + --> STEP: 169/234 -- GLOBAL_STEP: 71305 + | > loss: -0.33747 (-0.32036) + | > log_mle: -0.54154 (-0.44311) + | > loss_dur: 0.20407 (0.12275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.00590 (22.56205) + | > current_lr: 0.00008 + | > step_time: 3.99430 (2.44201) + | > loader_time: 0.00750 (0.04047) + + + --> STEP: 174/234 -- GLOBAL_STEP: 71310 + | > loss: -0.40719 (-0.32245) + | > log_mle: -0.62012 (-0.44777) + | > loss_dur: 0.21293 (0.12533) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.03761 (23.30971) + | > current_lr: 0.00008 + | > step_time: 1.50840 (2.42202) + | > loader_time: 0.08630 (0.04081) + + + --> STEP: 179/234 -- GLOBAL_STEP: 71315 + | > loss: -0.38876 (-0.32409) + | > log_mle: -0.62688 (-0.45199) + | > loss_dur: 0.23812 (0.12790) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.68550 (24.23740) + | > current_lr: 0.00008 + | > step_time: 3.68070 (2.45142) + | > loader_time: 0.10110 (0.04356) + + + --> STEP: 184/234 -- GLOBAL_STEP: 71320 + | > loss: -0.38588 (-0.32562) + | > log_mle: -0.59323 (-0.45574) + | > loss_dur: 0.20735 (0.13012) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.93029 (24.98681) + | > current_lr: 0.00008 + | > step_time: 8.71390 (2.54967) + | > loader_time: 0.28880 (0.04544) + + + --> STEP: 189/234 -- GLOBAL_STEP: 71325 + | > loss: -0.39221 (-0.32745) + | > log_mle: -0.59878 (-0.45993) + | > loss_dur: 0.20657 (0.13248) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.50866 (25.82965) + | > current_lr: 0.00008 + | > step_time: 6.71440 (2.57653) + | > loader_time: 0.09350 (0.04614) + + + --> STEP: 194/234 -- GLOBAL_STEP: 71330 + | > loss: -0.43244 (-0.33000) + | > log_mle: -0.63581 (-0.46433) + | > loss_dur: 0.20337 (0.13433) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.79729 (26.51621) + | > current_lr: 0.00008 + | > step_time: 3.20070 (2.66016) + | > loader_time: 0.08680 (0.05046) + + + --> STEP: 199/234 -- GLOBAL_STEP: 71335 + | > loss: -0.42881 (-0.33210) + | > log_mle: -0.64580 (-0.46844) + | > loss_dur: 0.21699 (0.13634) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.38151 (27.35093) + | > current_lr: 0.00008 + | > step_time: 12.10650 (2.73063) + | > loader_time: 0.18940 (0.05117) + + + --> STEP: 204/234 -- GLOBAL_STEP: 71340 + | > loss: -0.42799 (-0.33369) + | > log_mle: -0.67635 (-0.47225) + | > loss_dur: 0.24836 (0.13856) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.96526 (28.44092) + | > current_lr: 0.00008 + | > step_time: 9.99590 (2.88731) + | > loader_time: 0.00210 (0.05128) + + + --> STEP: 209/234 -- GLOBAL_STEP: 71345 + | > loss: -0.41418 (-0.33577) + | > log_mle: -0.63115 (-0.47643) + | > loss_dur: 0.21697 (0.14066) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.07110 (29.26711) + | > current_lr: 0.00008 + | > step_time: 5.10700 (2.90442) + | > loader_time: 0.06460 (0.05173) + + + --> STEP: 214/234 -- GLOBAL_STEP: 71350 + | > loss: -0.46015 (-0.33875) + | > log_mle: -0.66870 (-0.48150) + | > loss_dur: 0.20855 (0.14275) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.85703 (30.01526) + | > current_lr: 0.00008 + | > step_time: 1.89760 (2.91148) + | > loader_time: 0.00270 (0.05144) + + + --> STEP: 219/234 -- GLOBAL_STEP: 71355 + | > loss: -0.55472 (-0.34175) + | > log_mle: -0.78043 (-0.48664) + | > loss_dur: 0.22571 (0.14489) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.89378 (30.76463) + | > current_lr: 0.00008 + | > step_time: 2.29820 (2.99159) + | > loader_time: 0.00340 (0.05252) + + + --> STEP: 224/234 -- GLOBAL_STEP: 71360 + | > loss: -0.49541 (-0.34458) + | > log_mle: -0.72746 (-0.49149) + | > loss_dur: 0.23205 (0.14691) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.87694 (31.63407) + | > current_lr: 0.00008 + | > step_time: 0.22930 (2.94078) + | > loader_time: 0.00560 (0.05181) + + + --> STEP: 229/234 -- GLOBAL_STEP: 71365 + | > loss: -0.49249 (-0.34775) + | > log_mle: -0.77930 (-0.49707) + | > loss_dur: 0.28681 (0.14932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.03556 (32.69702) + | > current_lr: 0.00008 + | > step_time: 0.24500 (2.88185) + | > loader_time: 0.00420 (0.05076) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.00389 (+0.62370) + | > avg_loss: -0.34815 (+0.00927) + | > avg_log_mle: -0.56502 (+0.01550) + | > avg_loss_dur: 0.21687 (-0.00623) + + + > EPOCH: 305/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 01:24:37)  + + --> STEP: 0/234 -- GLOBAL_STEP: 71370 + | > loss: -0.35090 (-0.35090) + | > log_mle: -0.51525 (-0.51525) + | > loss_dur: 0.16435 (0.16435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.57847 (23.57847) + | > current_lr: 0.00008 + | > step_time: 4.79880 (4.79875) + | > loader_time: 4.66690 (4.66688) + + + --> STEP: 5/234 -- GLOBAL_STEP: 71375 + | > loss: -0.31602 (-0.32290) + | > log_mle: -0.42089 (-0.42702) + | > loss_dur: 0.10487 (0.10412) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.83446 (18.30945) + | > current_lr: 0.00008 + | > step_time: 3.61050 (5.76047) + | > loader_time: 0.00130 (0.36240) + + + --> STEP: 10/234 -- GLOBAL_STEP: 71380 + | > loss: -0.34554 (-0.33866) + | > log_mle: -0.43093 (-0.43093) + | > loss_dur: 0.08539 (0.09227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.18673 (16.51516) + | > current_lr: 0.00008 + | > step_time: 4.98130 (4.83147) + | > loader_time: 0.00100 (0.19926) + + + --> STEP: 15/234 -- GLOBAL_STEP: 71385 + | > loss: -0.38169 (-0.35187) + | > log_mle: -0.44302 (-0.43471) + | > loss_dur: 0.06134 (0.08284) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.86488 (16.01872) + | > current_lr: 0.00008 + | > step_time: 9.60640 (4.92739) + | > loader_time: 0.08400 (0.14617) + + + --> STEP: 20/234 -- GLOBAL_STEP: 71390 + | > loss: -0.39148 (-0.35940) + | > log_mle: -0.45000 (-0.43753) + | > loss_dur: 0.05852 (0.07813) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.09970 (15.01298) + | > current_lr: 0.00008 + | > step_time: 2.50070 (4.74645) + | > loader_time: 0.00320 (0.12906) + + + --> STEP: 25/234 -- GLOBAL_STEP: 71395 + | > loss: -0.37154 (-0.36163) + | > log_mle: -0.43268 (-0.43696) + | > loss_dur: 0.06114 (0.07532) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.09637 (14.13649) + | > current_lr: 0.00008 + | > step_time: 2.90480 (4.46581) + | > loader_time: 0.00470 (0.10735) + + + --> STEP: 30/234 -- GLOBAL_STEP: 71400 + | > loss: -0.35676 (-0.36312) + | > log_mle: -0.42683 (-0.43645) + | > loss_dur: 0.07007 (0.07333) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.86381 (13.62056) + | > current_lr: 0.00008 + | > step_time: 4.69770 (4.48145) + | > loader_time: 0.08720 (0.09616) + + + --> STEP: 35/234 -- GLOBAL_STEP: 71405 + | > loss: -0.32367 (-0.36049) + | > log_mle: -0.40737 (-0.43382) + | > loss_dur: 0.08370 (0.07332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.38822 (13.29846) + | > current_lr: 0.00008 + | > step_time: 2.60170 (4.36401) + | > loader_time: 0.09630 (0.09033) + + + --> STEP: 40/234 -- GLOBAL_STEP: 71410 + | > loss: -0.32056 (-0.35713) + | > log_mle: -0.40563 (-0.43080) + | > loss_dur: 0.08507 (0.07367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.41986 (12.82576) + | > current_lr: 0.00008 + | > step_time: 1.33790 (4.18357) + | > loader_time: 0.00230 (0.08419) + + + --> STEP: 45/234 -- GLOBAL_STEP: 71415 + | > loss: -0.32653 (-0.35513) + | > log_mle: -0.41822 (-0.42869) + | > loss_dur: 0.09169 (0.07356) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.70158 (12.49589) + | > current_lr: 0.00008 + | > step_time: 1.29820 (3.85704) + | > loader_time: 0.00310 (0.07857) + + + --> STEP: 50/234 -- GLOBAL_STEP: 71420 + | > loss: -0.34501 (-0.35428) + | > log_mle: -0.41191 (-0.42770) + | > loss_dur: 0.06689 (0.07342) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.12721 (12.06385) + | > current_lr: 0.00008 + | > step_time: 1.60490 (3.63115) + | > loader_time: 0.00140 (0.07090) + + + --> STEP: 55/234 -- GLOBAL_STEP: 71425 + | > loss: -0.35237 (-0.35255) + | > log_mle: -0.41975 (-0.42640) + | > loss_dur: 0.06737 (0.07384) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.03648 (11.88633) + | > current_lr: 0.00008 + | > step_time: 3.19230 (3.48065) + | > loader_time: 0.08900 (0.06631) + + + --> STEP: 60/234 -- GLOBAL_STEP: 71430 + | > loss: -0.30775 (-0.35042) + | > log_mle: -0.41333 (-0.42502) + | > loss_dur: 0.10557 (0.07460) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.73991 (11.72151) + | > current_lr: 0.00008 + | > step_time: 2.08700 (3.36998) + | > loader_time: 0.00640 (0.06107) + + + --> STEP: 65/234 -- GLOBAL_STEP: 71435 + | > loss: -0.31807 (-0.34700) + | > log_mle: -0.39984 (-0.42307) + | > loss_dur: 0.08177 (0.07607) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.56338 (11.92154) + | > current_lr: 0.00008 + | > step_time: 0.67970 (3.21793) + | > loader_time: 0.00210 (0.05655) + + + --> STEP: 70/234 -- GLOBAL_STEP: 71440 + | > loss: -0.27492 (-0.34342) + | > log_mle: -0.37932 (-0.42071) + | > loss_dur: 0.10440 (0.07729) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.75255 (11.82795) + | > current_lr: 0.00008 + | > step_time: 4.59790 (3.16191) + | > loader_time: 0.08820 (0.05391) + + + --> STEP: 75/234 -- GLOBAL_STEP: 71445 + | > loss: -0.28418 (-0.33966) + | > log_mle: -0.39197 (-0.41896) + | > loss_dur: 0.10779 (0.07930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.38199 (11.80998) + | > current_lr: 0.00008 + | > step_time: 4.21600 (3.13935) + | > loader_time: 0.09400 (0.05284) + + + --> STEP: 80/234 -- GLOBAL_STEP: 71450 + | > loss: -0.30071 (-0.33684) + | > log_mle: -0.38765 (-0.41705) + | > loss_dur: 0.08694 (0.08021) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56978 (11.79379) + | > current_lr: 0.00008 + | > step_time: 1.70050 (3.03163) + | > loader_time: 0.00300 (0.04973) + + + --> STEP: 85/234 -- GLOBAL_STEP: 71455 + | > loss: -0.28454 (-0.33361) + | > log_mle: -0.38480 (-0.41535) + | > loss_dur: 0.10026 (0.08174) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.59097 (11.99828) + | > current_lr: 0.00008 + | > step_time: 1.23170 (2.95006) + | > loader_time: 0.00200 (0.04793) + + + --> STEP: 90/234 -- GLOBAL_STEP: 71460 + | > loss: -0.27559 (-0.33108) + | > log_mle: -0.39918 (-0.41463) + | > loss_dur: 0.12359 (0.08355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.09835 (12.26021) + | > current_lr: 0.00008 + | > step_time: 3.68730 (2.90970) + | > loader_time: 0.00490 (0.04729) + + + --> STEP: 95/234 -- GLOBAL_STEP: 71465 + | > loss: -0.31245 (-0.32919) + | > log_mle: -0.47413 (-0.41552) + | > loss_dur: 0.16167 (0.08633) + | > amp_scaler: 2048.00000 (1067.11579) + | > grad_norm: 68.03329 (13.16242) + | > current_lr: 0.00008 + | > step_time: 1.50620 (2.92627) + | > loader_time: 0.00590 (0.04699) + + + --> STEP: 100/234 -- GLOBAL_STEP: 71470 + | > loss: -0.28948 (-0.32675) + | > log_mle: -0.40450 (-0.41492) + | > loss_dur: 0.11502 (0.08817) + | > amp_scaler: 2048.00000 (1116.16000) + | > grad_norm: 17.90989 (13.40354) + | > current_lr: 0.00008 + | > step_time: 1.11950 (2.88650) + | > loader_time: 0.00370 (0.04675) + + + --> STEP: 105/234 -- GLOBAL_STEP: 71475 + | > loss: -0.27034 (-0.32475) + | > log_mle: -0.38785 (-0.41538) + | > loss_dur: 0.11751 (0.09063) + | > amp_scaler: 2048.00000 (1160.53333) + | > grad_norm: 10.81923 (13.75152) + | > current_lr: 0.00008 + | > step_time: 2.34880 (2.83394) + | > loader_time: 0.00910 (0.04470) + + + --> STEP: 110/234 -- GLOBAL_STEP: 71480 + | > loss: -0.26757 (-0.32211) + | > log_mle: -0.40310 (-0.41547) + | > loss_dur: 0.13552 (0.09336) + | > amp_scaler: 2048.00000 (1200.87273) + | > grad_norm: 22.12814 (14.16244) + | > current_lr: 0.00008 + | > step_time: 2.30840 (2.78604) + | > loader_time: 0.08340 (0.04350) + + + --> STEP: 115/234 -- GLOBAL_STEP: 71485 + | > loss: -0.26978 (-0.32007) + | > log_mle: -0.42454 (-0.41649) + | > loss_dur: 0.15476 (0.09642) + | > amp_scaler: 2048.00000 (1237.70435) + | > grad_norm: 22.59186 (14.67078) + | > current_lr: 0.00008 + | > step_time: 2.71200 (2.76758) + | > loader_time: 0.08860 (0.04408) + + + --> STEP: 120/234 -- GLOBAL_STEP: 71490 + | > loss: -0.32317 (-0.31817) + | > log_mle: -0.47368 (-0.41727) + | > loss_dur: 0.15051 (0.09910) + | > amp_scaler: 2048.00000 (1271.46667) + | > grad_norm: 23.78449 (15.12110) + | > current_lr: 0.00008 + | > step_time: 2.72850 (2.81215) + | > loader_time: 0.00250 (0.04465) + + + --> STEP: 125/234 -- GLOBAL_STEP: 71495 + | > loss: -0.27211 (-0.31604) + | > log_mle: -0.44679 (-0.41702) + | > loss_dur: 0.17469 (0.10099) + | > amp_scaler: 2048.00000 (1302.52800) + | > grad_norm: 58.97546 (15.50008) + | > current_lr: 0.00008 + | > step_time: 1.08520 (2.75302) + | > loader_time: 0.00340 (0.04428) + + + --> STEP: 130/234 -- GLOBAL_STEP: 71500 + | > loss: -0.29822 (-0.31515) + | > log_mle: -0.47379 (-0.41865) + | > loss_dur: 0.17557 (0.10350) + | > amp_scaler: 2048.00000 (1331.20000) + | > grad_norm: 26.81234 (16.00733) + | > current_lr: 0.00008 + | > step_time: 3.50500 (2.73793) + | > loader_time: 0.08840 (0.04404) + + + --> STEP: 135/234 -- GLOBAL_STEP: 71505 + | > loss: -0.26956 (-0.31491) + | > log_mle: -0.40386 (-0.42061) + | > loss_dur: 0.13430 (0.10570) + | > amp_scaler: 2048.00000 (1357.74815) + | > grad_norm: 14.08461 (16.35517) + | > current_lr: 0.00008 + | > step_time: 1.30530 (2.69954) + | > loader_time: 0.07600 (0.04369) + + + --> STEP: 140/234 -- GLOBAL_STEP: 71510 + | > loss: -0.25859 (-0.31463) + | > log_mle: -0.43500 (-0.42317) + | > loss_dur: 0.17641 (0.10854) + | > amp_scaler: 2048.00000 (1382.40000) + | > grad_norm: 32.57656 (17.10866) + | > current_lr: 0.00008 + | > step_time: 2.70560 (2.68535) + | > loader_time: 0.00320 (0.04409) + + + --> STEP: 145/234 -- GLOBAL_STEP: 71515 + | > loss: -0.34891 (-0.31479) + | > log_mle: -0.53098 (-0.42620) + | > loss_dur: 0.18206 (0.11141) + | > amp_scaler: 2048.00000 (1405.35172) + | > grad_norm: 43.08881 (18.42575) + | > current_lr: 0.00008 + | > step_time: 6.50270 (2.71363) + | > loader_time: 0.19560 (0.04579) + + + --> STEP: 150/234 -- GLOBAL_STEP: 71520 + | > loss: -0.31977 (-0.31537) + | > log_mle: -0.52294 (-0.42925) + | > loss_dur: 0.20317 (0.11388) + | > amp_scaler: 2048.00000 (1426.77333) + | > grad_norm: 43.32891 (19.09535) + | > current_lr: 0.00008 + | > step_time: 5.48910 (2.77588) + | > loader_time: 0.00440 (0.04691) + + + --> STEP: 155/234 -- GLOBAL_STEP: 71525 + | > loss: -0.39567 (-0.31716) + | > log_mle: -0.58687 (-0.43353) + | > loss_dur: 0.19120 (0.11636) + | > amp_scaler: 2048.00000 (1446.81290) + | > grad_norm: 51.40746 (19.86372) + | > current_lr: 0.00008 + | > step_time: 2.19190 (2.78893) + | > loader_time: 0.00270 (0.04730) + + + --> STEP: 160/234 -- GLOBAL_STEP: 71530 + | > loss: -0.37344 (-0.31828) + | > log_mle: -0.58838 (-0.43718) + | > loss_dur: 0.21493 (0.11890) + | > amp_scaler: 2048.00000 (1465.60000) + | > grad_norm: 40.65128 (20.54004) + | > current_lr: 0.00008 + | > step_time: 2.11200 (2.76881) + | > loader_time: 0.00260 (0.04641) + + + --> STEP: 165/234 -- GLOBAL_STEP: 71535 + | > loss: -0.39530 (-0.31962) + | > log_mle: -0.58745 (-0.44070) + | > loss_dur: 0.19215 (0.12108) + | > amp_scaler: 2048.00000 (1483.24848) + | > grad_norm: 59.16601 (21.27787) + | > current_lr: 0.00008 + | > step_time: 1.40250 (2.75579) + | > loader_time: 0.00390 (0.04618) + + + --> STEP: 170/234 -- GLOBAL_STEP: 71540 + | > loss: -0.40933 (-0.32140) + | > log_mle: -0.63620 (-0.44488) + | > loss_dur: 0.22687 (0.12348) + | > amp_scaler: 2048.00000 (1499.85882) + | > grad_norm: 59.08116 (22.17686) + | > current_lr: 0.00008 + | > step_time: 4.20770 (2.76129) + | > loader_time: 0.08290 (0.04685) + + + --> STEP: 175/234 -- GLOBAL_STEP: 71545 + | > loss: -0.37454 (-0.32383) + | > log_mle: -0.60563 (-0.44986) + | > loss_dur: 0.23108 (0.12604) + | > amp_scaler: 2048.00000 (1515.52000) + | > grad_norm: 49.76561 (23.17136) + | > current_lr: 0.00008 + | > step_time: 5.70800 (2.81047) + | > loader_time: 0.00440 (0.04780) + + + --> STEP: 180/234 -- GLOBAL_STEP: 71550 + | > loss: -0.38566 (-0.32579) + | > log_mle: -0.59942 (-0.45434) + | > loss_dur: 0.21376 (0.12854) + | > amp_scaler: 2048.00000 (1530.31111) + | > grad_norm: 75.03530 (24.34508) + | > current_lr: 0.00008 + | > step_time: 2.61330 (2.82748) + | > loader_time: 0.00670 (0.04764) + + + --> STEP: 185/234 -- GLOBAL_STEP: 71555 + | > loss: -0.42292 (-0.32758) + | > log_mle: -0.63760 (-0.45849) + | > loss_dur: 0.21467 (0.13090) + | > amp_scaler: 2048.00000 (1544.30270) + | > grad_norm: 80.08799 (25.43725) + | > current_lr: 0.00008 + | > step_time: 3.72530 (2.84309) + | > loader_time: 0.08900 (0.04746) + + + --> STEP: 190/234 -- GLOBAL_STEP: 71560 + | > loss: -0.40939 (-0.32928) + | > log_mle: -0.61165 (-0.46248) + | > loss_dur: 0.20227 (0.13319) + | > amp_scaler: 1024.00000 (1536.00000) + | > grad_norm: 83.96033 (26.42622) + | > current_lr: 0.00008 + | > step_time: 1.77830 (2.83555) + | > loader_time: 0.00320 (0.04725) + + + --> STEP: 195/234 -- GLOBAL_STEP: 71565 + | > loss: -0.41766 (-0.33163) + | > log_mle: -0.63425 (-0.46682) + | > loss_dur: 0.21658 (0.13519) + | > amp_scaler: 1024.00000 (1522.87179) + | > grad_norm: 51.44033 (27.40209) + | > current_lr: 0.00008 + | > step_time: 3.61440 (2.87362) + | > loader_time: 0.08470 (0.04837) + + + --> STEP: 200/234 -- GLOBAL_STEP: 71570 + | > loss: -0.38820 (-0.33347) + | > log_mle: -0.63471 (-0.47075) + | > loss_dur: 0.24651 (0.13728) + | > amp_scaler: 1024.00000 (1510.40000) + | > grad_norm: 62.98547 (28.32442) + | > current_lr: 0.00008 + | > step_time: 5.19350 (2.91484) + | > loader_time: 0.00320 (0.04855) + + + --> STEP: 205/234 -- GLOBAL_STEP: 71575 + | > loss: -0.38240 (-0.33473) + | > log_mle: -0.60576 (-0.47418) + | > loss_dur: 0.22336 (0.13945) + | > amp_scaler: 1024.00000 (1498.53659) + | > grad_norm: 53.20074 (29.46541) + | > current_lr: 0.00008 + | > step_time: 5.00920 (2.92385) + | > loader_time: 0.08870 (0.04891) + + + --> STEP: 210/234 -- GLOBAL_STEP: 71580 + | > loss: -0.47459 (-0.33731) + | > log_mle: -0.69840 (-0.47875) + | > loss_dur: 0.22381 (0.14144) + | > amp_scaler: 1024.00000 (1487.23810) + | > grad_norm: 64.60831 (30.04723) + | > current_lr: 0.00008 + | > step_time: 3.60390 (2.93757) + | > loader_time: 0.00340 (0.04871) + + + --> STEP: 215/234 -- GLOBAL_STEP: 71585 + | > loss: -0.42312 (-0.34012) + | > log_mle: -0.65442 (-0.48363) + | > loss_dur: 0.23130 (0.14350) + | > amp_scaler: 1024.00000 (1476.46512) + | > grad_norm: 67.59111 (30.91079) + | > current_lr: 0.00008 + | > step_time: 6.89490 (3.00740) + | > loader_time: 0.40200 (0.05683) + + + --> STEP: 220/234 -- GLOBAL_STEP: 71590 + | > loss: -0.48110 (-0.34334) + | > log_mle: -0.71778 (-0.48899) + | > loss_dur: 0.23669 (0.14565) + | > amp_scaler: 1024.00000 (1466.18182) + | > grad_norm: 69.83229 (31.86527) + | > current_lr: 0.00008 + | > step_time: 2.60350 (3.09813) + | > loader_time: 0.00650 (0.05689) + + + --> STEP: 225/234 -- GLOBAL_STEP: 71595 + | > loss: -0.53466 (-0.34638) + | > log_mle: -0.78074 (-0.49404) + | > loss_dur: 0.24607 (0.14766) + | > amp_scaler: 1024.00000 (1456.35556) + | > grad_norm: 126.92146 (33.14651) + | > current_lr: 0.00008 + | > step_time: 0.24020 (3.04340) + | > loader_time: 0.00330 (0.05570) + + + --> STEP: 230/234 -- GLOBAL_STEP: 71600 + | > loss: -0.52801 (-0.34935) + | > log_mle: -0.83483 (-0.49971) + | > loss_dur: 0.30681 (0.15036) + | > amp_scaler: 1024.00000 (1446.95652) + | > grad_norm: 92.89307 (34.43980) + | > current_lr: 0.00008 + | > step_time: 0.26070 (2.98259) + | > loader_time: 0.00300 (0.05456) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.27356 (-0.73033) + | > avg_loss: -0.35200 (-0.00385) + | > avg_log_mle: -0.56747 (-0.00245) + | > avg_loss_dur: 0.21547 (-0.00140) + + + > EPOCH: 306/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 01:37:10)  + + --> STEP: 1/234 -- GLOBAL_STEP: 71605 + | > loss: -0.32721 (-0.32721) + | > log_mle: -0.42290 (-0.42290) + | > loss_dur: 0.09569 (0.09569) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.94677 (18.94677) + | > current_lr: 0.00008 + | > step_time: 0.81770 (0.81774) + | > loader_time: 0.07860 (0.07864) + + + --> STEP: 6/234 -- GLOBAL_STEP: 71610 + | > loss: -0.36328 (-0.33220) + | > log_mle: -0.42745 (-0.42855) + | > loss_dur: 0.06418 (0.09635) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.12167 (19.66859) + | > current_lr: 0.00008 + | > step_time: 8.29680 (4.46985) + | > loader_time: 0.10700 (0.06441) + + + --> STEP: 11/234 -- GLOBAL_STEP: 71615 + | > loss: -0.39930 (-0.34917) + | > log_mle: -0.45327 (-0.43518) + | > loss_dur: 0.05397 (0.08601) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.79069 (17.86202) + | > current_lr: 0.00008 + | > step_time: 3.39950 (3.82762) + | > loader_time: 0.00360 (0.03688) + + + --> STEP: 16/234 -- GLOBAL_STEP: 71620 + | > loss: -0.40402 (-0.35857) + | > log_mle: -0.45802 (-0.43900) + | > loss_dur: 0.05400 (0.08043) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.96391 (16.55288) + | > current_lr: 0.00008 + | > step_time: 5.89700 (4.03140) + | > loader_time: 0.09680 (0.03775) + + + --> STEP: 21/234 -- GLOBAL_STEP: 71625 + | > loss: -0.35858 (-0.36346) + | > log_mle: -0.42975 (-0.43982) + | > loss_dur: 0.07117 (0.07636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.57254 (15.25424) + | > current_lr: 0.00008 + | > step_time: 9.90120 (4.38562) + | > loader_time: 0.19710 (0.04335) + + + --> STEP: 26/234 -- GLOBAL_STEP: 71630 + | > loss: -0.35480 (-0.36542) + | > log_mle: -0.42467 (-0.43958) + | > loss_dur: 0.06987 (0.07416) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.80157 (14.33485) + | > current_lr: 0.00008 + | > step_time: 7.50160 (4.21591) + | > loader_time: 0.00350 (0.04192) + + + --> STEP: 31/234 -- GLOBAL_STEP: 71635 + | > loss: -0.34177 (-0.36622) + | > log_mle: -0.42051 (-0.43905) + | > loss_dur: 0.07875 (0.07283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87686 (13.59271) + | > current_lr: 0.00008 + | > step_time: 2.51920 (4.26662) + | > loader_time: 0.19380 (0.05028) + + + --> STEP: 36/234 -- GLOBAL_STEP: 71640 + | > loss: -0.33772 (-0.36374) + | > log_mle: -0.41161 (-0.43627) + | > loss_dur: 0.07388 (0.07254) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.46515 (13.12645) + | > current_lr: 0.00008 + | > step_time: 1.79310 (4.14392) + | > loader_time: 0.00190 (0.04870) + + + --> STEP: 41/234 -- GLOBAL_STEP: 71645 + | > loss: -0.36237 (-0.36029) + | > log_mle: -0.42357 (-0.43320) + | > loss_dur: 0.06119 (0.07292) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56455 (12.94572) + | > current_lr: 0.00008 + | > step_time: 1.99790 (3.81182) + | > loader_time: 0.00380 (0.04686) + + + --> STEP: 46/234 -- GLOBAL_STEP: 71650 + | > loss: -0.31912 (-0.35616) + | > log_mle: -0.40559 (-0.43021) + | > loss_dur: 0.08647 (0.07404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.24253 (12.97441) + | > current_lr: 0.00008 + | > step_time: 1.14740 (3.56161) + | > loader_time: 0.00250 (0.04204) + + + --> STEP: 51/234 -- GLOBAL_STEP: 71655 + | > loss: -0.32727 (-0.35478) + | > log_mle: -0.40458 (-0.42856) + | > loss_dur: 0.07731 (0.07378) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.83533 (12.60968) + | > current_lr: 0.00008 + | > step_time: 1.90020 (3.34120) + | > loader_time: 0.00230 (0.03814) + + + --> STEP: 56/234 -- GLOBAL_STEP: 71660 + | > loss: -0.32398 (-0.35227) + | > log_mle: -0.40919 (-0.42668) + | > loss_dur: 0.08521 (0.07441) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.10747 (12.49687) + | > current_lr: 0.00008 + | > step_time: 2.19760 (3.24147) + | > loader_time: 0.00230 (0.03514) + + + --> STEP: 61/234 -- GLOBAL_STEP: 71665 + | > loss: -0.31688 (-0.34933) + | > log_mle: -0.39667 (-0.42465) + | > loss_dur: 0.07979 (0.07532) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.68031 (12.39760) + | > current_lr: 0.00008 + | > step_time: 3.21080 (3.13168) + | > loader_time: 0.09590 (0.03406) + + + --> STEP: 66/234 -- GLOBAL_STEP: 71670 + | > loss: -0.31640 (-0.34576) + | > log_mle: -0.38911 (-0.42232) + | > loss_dur: 0.07271 (0.07655) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.06574 (12.30622) + | > current_lr: 0.00008 + | > step_time: 1.49210 (3.04300) + | > loader_time: 0.00390 (0.03417) + + + --> STEP: 71/234 -- GLOBAL_STEP: 71675 + | > loss: -0.28150 (-0.34182) + | > log_mle: -0.40364 (-0.42010) + | > loss_dur: 0.12214 (0.07828) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.35961 (12.26522) + | > current_lr: 0.00008 + | > step_time: 2.27920 (2.98721) + | > loader_time: 0.00390 (0.03193) + + + --> STEP: 76/234 -- GLOBAL_STEP: 71680 + | > loss: -0.29280 (-0.33825) + | > log_mle: -0.39504 (-0.41816) + | > loss_dur: 0.10224 (0.07991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.96349 (12.18489) + | > current_lr: 0.00008 + | > step_time: 2.69680 (2.90982) + | > loader_time: 0.00220 (0.03000) + + + --> STEP: 81/234 -- GLOBAL_STEP: 71685 + | > loss: -0.28970 (-0.33544) + | > log_mle: -0.39739 (-0.41642) + | > loss_dur: 0.10769 (0.08098) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.67819 (12.05503) + | > current_lr: 0.00008 + | > step_time: 1.77840 (2.84830) + | > loader_time: 0.00230 (0.02931) + + + --> STEP: 86/234 -- GLOBAL_STEP: 71690 + | > loss: -0.29580 (-0.33279) + | > log_mle: -0.39402 (-0.41484) + | > loss_dur: 0.09822 (0.08206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.28200 (11.99876) + | > current_lr: 0.00008 + | > step_time: 2.39660 (2.80537) + | > loader_time: 0.00230 (0.02887) + + + --> STEP: 91/234 -- GLOBAL_STEP: 71695 + | > loss: -0.27783 (-0.32994) + | > log_mle: -0.39981 (-0.41420) + | > loss_dur: 0.12199 (0.08425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.54796 (12.29511) + | > current_lr: 0.00008 + | > step_time: 2.04800 (2.77710) + | > loader_time: 0.00250 (0.02747) + + + --> STEP: 96/234 -- GLOBAL_STEP: 71700 + | > loss: -0.28358 (-0.32804) + | > log_mle: -0.38889 (-0.41485) + | > loss_dur: 0.10531 (0.08681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.11637 (12.68180) + | > current_lr: 0.00008 + | > step_time: 1.88420 (2.72797) + | > loader_time: 0.00280 (0.02787) + + + --> STEP: 101/234 -- GLOBAL_STEP: 71705 + | > loss: -0.27569 (-0.32586) + | > log_mle: -0.43441 (-0.41499) + | > loss_dur: 0.15872 (0.08913) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.93302 (12.85111) + | > current_lr: 0.00008 + | > step_time: 2.99880 (2.69847) + | > loader_time: 0.00310 (0.02747) + + + --> STEP: 106/234 -- GLOBAL_STEP: 71710 + | > loss: -0.26214 (-0.32388) + | > log_mle: -0.42879 (-0.41567) + | > loss_dur: 0.16664 (0.09179) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.33757 (13.16019) + | > current_lr: 0.00008 + | > step_time: 4.10400 (2.68251) + | > loader_time: 0.08950 (0.02792) + + + --> STEP: 111/234 -- GLOBAL_STEP: 71715 + | > loss: -0.30952 (-0.32183) + | > log_mle: -0.48320 (-0.41645) + | > loss_dur: 0.17368 (0.09461) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.87657 (13.81334) + | > current_lr: 0.00008 + | > step_time: 7.30030 (2.78235) + | > loader_time: 0.00680 (0.03026) + + + --> STEP: 116/234 -- GLOBAL_STEP: 71720 + | > loss: -0.26160 (-0.31987) + | > log_mle: -0.44705 (-0.41737) + | > loss_dur: 0.18545 (0.09750) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.97499 (14.35324) + | > current_lr: 0.00008 + | > step_time: 1.10180 (2.75720) + | > loader_time: 0.09900 (0.03064) + + + --> STEP: 121/234 -- GLOBAL_STEP: 71725 + | > loss: -0.24881 (-0.31843) + | > log_mle: -0.36991 (-0.41784) + | > loss_dur: 0.12109 (0.09941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.38362 (14.57279) + | > current_lr: 0.00008 + | > step_time: 1.00990 (2.74607) + | > loader_time: 0.00190 (0.03024) + + + --> STEP: 126/234 -- GLOBAL_STEP: 71730 + | > loss: -0.30125 (-0.31699) + | > log_mle: -0.49144 (-0.41876) + | > loss_dur: 0.19019 (0.10177) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.55530 (15.01425) + | > current_lr: 0.00008 + | > step_time: 1.71160 (2.71564) + | > loader_time: 0.08950 (0.03052) + + + --> STEP: 131/234 -- GLOBAL_STEP: 71735 + | > loss: -0.36515 (-0.31653) + | > log_mle: -0.54369 (-0.42088) + | > loss_dur: 0.17854 (0.10435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.46279 (15.58651) + | > current_lr: 0.00008 + | > step_time: 6.21370 (2.78239) + | > loader_time: 0.18320 (0.03213) + + + --> STEP: 136/234 -- GLOBAL_STEP: 71740 + | > loss: -0.39663 (-0.31668) + | > log_mle: -0.60325 (-0.42330) + | > loss_dur: 0.20662 (0.10662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.53651 (16.21413) + | > current_lr: 0.00008 + | > step_time: 4.61190 (2.89196) + | > loader_time: 0.09600 (0.03314) + + + --> STEP: 141/234 -- GLOBAL_STEP: 71745 + | > loss: -0.32066 (-0.31601) + | > log_mle: -0.48591 (-0.42509) + | > loss_dur: 0.16524 (0.10908) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.94311 (16.92090) + | > current_lr: 0.00008 + | > step_time: 2.69510 (2.87647) + | > loader_time: 0.00200 (0.03273) + + + --> STEP: 146/234 -- GLOBAL_STEP: 71750 + | > loss: -0.35032 (-0.31673) + | > log_mle: -0.54410 (-0.42869) + | > loss_dur: 0.19378 (0.11196) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.62772 (17.69732) + | > current_lr: 0.00008 + | > step_time: 2.99110 (2.88396) + | > loader_time: 0.00480 (0.03172) + + + --> STEP: 151/234 -- GLOBAL_STEP: 71755 + | > loss: -0.33107 (-0.31743) + | > log_mle: -0.50714 (-0.43160) + | > loss_dur: 0.17607 (0.11418) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.06459 (18.30202) + | > current_lr: 0.00008 + | > step_time: 0.82240 (2.87062) + | > loader_time: 0.09890 (0.03202) + + + --> STEP: 156/234 -- GLOBAL_STEP: 71760 + | > loss: -0.36572 (-0.31956) + | > log_mle: -0.55910 (-0.43623) + | > loss_dur: 0.19338 (0.11667) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.92233 (19.15748) + | > current_lr: 0.00008 + | > step_time: 2.48820 (2.86054) + | > loader_time: 0.10600 (0.03178) + + + --> STEP: 161/234 -- GLOBAL_STEP: 71765 + | > loss: -0.37542 (-0.32075) + | > log_mle: -0.56824 (-0.43993) + | > loss_dur: 0.19282 (0.11918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.04882 (20.07885) + | > current_lr: 0.00008 + | > step_time: 14.92030 (2.95950) + | > loader_time: 0.98910 (0.03762) + + + --> STEP: 166/234 -- GLOBAL_STEP: 71770 + | > loss: -0.32947 (-0.32140) + | > log_mle: -0.50650 (-0.44274) + | > loss_dur: 0.17703 (0.12134) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.86718 (21.38796) + | > current_lr: 0.00008 + | > step_time: 3.79410 (2.98789) + | > loader_time: 0.00430 (0.03792) + + + --> STEP: 171/234 -- GLOBAL_STEP: 71775 + | > loss: -0.43013 (-0.32340) + | > log_mle: -0.62830 (-0.44720) + | > loss_dur: 0.19816 (0.12380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.06279 (22.26553) + | > current_lr: 0.00008 + | > step_time: 0.90490 (3.06019) + | > loader_time: 0.00180 (0.04105) + + + --> STEP: 176/234 -- GLOBAL_STEP: 71780 + | > loss: -0.38847 (-0.32557) + | > log_mle: -0.60010 (-0.45179) + | > loss_dur: 0.21163 (0.12622) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.19675 (23.21346) + | > current_lr: 0.00008 + | > step_time: 3.09760 (3.04250) + | > loader_time: 0.00250 (0.04048) + + + --> STEP: 181/234 -- GLOBAL_STEP: 71785 + | > loss: -0.29994 (-0.32649) + | > log_mle: -0.50680 (-0.45521) + | > loss_dur: 0.20686 (0.12871) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.26608 (24.53692) + | > current_lr: 0.00008 + | > step_time: 4.90170 (3.06278) + | > loader_time: 0.09410 (0.04211) + + + --> STEP: 186/234 -- GLOBAL_STEP: 71790 + | > loss: -0.31069 (-0.32765) + | > log_mle: -0.53849 (-0.45901) + | > loss_dur: 0.22781 (0.13136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 122.30267 (25.66184) + | > current_lr: 0.00008 + | > step_time: 3.50860 (3.08113) + | > loader_time: 0.09650 (0.04212) + + + --> STEP: 191/234 -- GLOBAL_STEP: 71795 + | > loss: -0.35209 (-0.32841) + | > log_mle: -0.55397 (-0.46176) + | > loss_dur: 0.20188 (0.13336) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.00339 (26.35530) + | > current_lr: 0.00008 + | > step_time: 5.20200 (3.13546) + | > loader_time: 0.00490 (0.04220) + + + --> STEP: 196/234 -- GLOBAL_STEP: 71800 + | > loss: -0.35173 (-0.33000) + | > log_mle: -0.56732 (-0.46530) + | > loss_dur: 0.21559 (0.13530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.02118 (26.69935) + | > current_lr: 0.00008 + | > step_time: 4.47270 (3.20099) + | > loader_time: 0.12770 (0.04733) + + + --> STEP: 201/234 -- GLOBAL_STEP: 71805 + | > loss: -0.29366 (-0.33105) + | > log_mle: -0.51609 (-0.46851) + | > loss_dur: 0.22243 (0.13746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.60238 (27.20798) + | > current_lr: 0.00008 + | > step_time: 5.50170 (3.23487) + | > loader_time: 0.00660 (0.05311) + + + --> STEP: 206/234 -- GLOBAL_STEP: 71810 + | > loss: -0.41808 (-0.33295) + | > log_mle: -0.64090 (-0.47244) + | > loss_dur: 0.22282 (0.13949) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.23061 (27.79186) + | > current_lr: 0.00008 + | > step_time: 5.59060 (3.29506) + | > loader_time: 0.10090 (0.05332) + + + --> STEP: 211/234 -- GLOBAL_STEP: 71815 + | > loss: -0.46942 (-0.33535) + | > log_mle: -0.72035 (-0.47712) + | > loss_dur: 0.25093 (0.14177) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.48007 (29.09757) + | > current_lr: 0.00008 + | > step_time: 4.39730 (3.34738) + | > loader_time: 0.40360 (0.05624) + + + --> STEP: 216/234 -- GLOBAL_STEP: 71820 + | > loss: -0.45928 (-0.33786) + | > log_mle: -0.70722 (-0.48167) + | > loss_dur: 0.24794 (0.14381) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 107.87884 (30.44425) + | > current_lr: 0.00008 + | > step_time: 12.89310 (3.44870) + | > loader_time: 0.19200 (0.05931) + + + --> STEP: 221/234 -- GLOBAL_STEP: 71825 + | > loss: -0.41694 (-0.34073) + | > log_mle: -0.63225 (-0.48651) + | > loss_dur: 0.21532 (0.14578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.98933 (31.30318) + | > current_lr: 0.00008 + | > step_time: 0.24610 (3.46191) + | > loader_time: 0.00360 (0.05925) + + + --> STEP: 226/234 -- GLOBAL_STEP: 71830 + | > loss: -0.50045 (-0.34401) + | > log_mle: -0.74324 (-0.49193) + | > loss_dur: 0.24279 (0.14792) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.80302 (32.38981) + | > current_lr: 0.00008 + | > step_time: 0.24300 (3.39060) + | > loader_time: 0.00380 (0.05803) + + + --> STEP: 231/234 -- GLOBAL_STEP: 71835 + | > loss: -0.45553 (-0.34676) + | > log_mle: -0.82417 (-0.49793) + | > loss_dur: 0.36864 (0.15117) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.43301 (33.27403) + | > current_lr: 0.00008 + | > step_time: 0.27580 (3.32286) + | > loader_time: 0.00350 (0.05687) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.15411 (+0.88055) + | > avg_loss: -0.33753 (+0.01447) + | > avg_log_mle: -0.56848 (-0.00101) + | > avg_loss_dur: 0.23095 (+0.01548) + + + > EPOCH: 307/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 01:51:20)  + + --> STEP: 2/234 -- GLOBAL_STEP: 71840 + | > loss: -0.36929 (-0.34220) + | > log_mle: -0.44877 (-0.43664) + | > loss_dur: 0.07948 (0.09444) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.86826 (19.04452) + | > current_lr: 0.00008 + | > step_time: 4.38730 (3.48442) + | > loader_time: 0.00110 (0.00109) + + + --> STEP: 7/234 -- GLOBAL_STEP: 71845 + | > loss: -0.37209 (-0.33414) + | > log_mle: -0.43556 (-0.42956) + | > loss_dur: 0.06348 (0.09542) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.99149 (19.65703) + | > current_lr: 0.00008 + | > step_time: 0.68300 (3.90875) + | > loader_time: 0.00280 (0.05542) + + + --> STEP: 12/234 -- GLOBAL_STEP: 71850 + | > loss: -0.35072 (-0.34871) + | > log_mle: -0.43176 (-0.43587) + | > loss_dur: 0.08104 (0.08716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.10234 (18.99308) + | > current_lr: 0.00008 + | > step_time: 1.68790 (2.76068) + | > loader_time: 0.00130 (0.03297) + + + --> STEP: 17/234 -- GLOBAL_STEP: 71855 + | > loss: -0.39425 (-0.36025) + | > log_mle: -0.45129 (-0.43991) + | > loss_dur: 0.05704 (0.07966) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.15967 (17.53401) + | > current_lr: 0.00008 + | > step_time: 1.01580 (2.36053) + | > loader_time: 0.00130 (0.02393) + + + --> STEP: 22/234 -- GLOBAL_STEP: 71860 + | > loss: -0.35900 (-0.36382) + | > log_mle: -0.43522 (-0.44004) + | > loss_dur: 0.07622 (0.07622) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.26061 (16.09390) + | > current_lr: 0.00008 + | > step_time: 1.04480 (2.09489) + | > loader_time: 0.00190 (0.01886) + + + --> STEP: 27/234 -- GLOBAL_STEP: 71865 + | > loss: -0.36053 (-0.36608) + | > log_mle: -0.42564 (-0.43985) + | > loss_dur: 0.06511 (0.07377) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.19526 (15.05199) + | > current_lr: 0.00008 + | > step_time: 1.05200 (1.93708) + | > loader_time: 0.19480 (0.02583) + + + --> STEP: 32/234 -- GLOBAL_STEP: 71870 + | > loss: -0.36541 (-0.36691) + | > log_mle: -0.43016 (-0.43970) + | > loss_dur: 0.06475 (0.07279) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.54175 (14.36185) + | > current_lr: 0.00008 + | > step_time: 2.80840 (1.98571) + | > loader_time: 0.09000 (0.02487) + + + --> STEP: 37/234 -- GLOBAL_STEP: 71875 + | > loss: -0.34811 (-0.36377) + | > log_mle: -0.41159 (-0.43629) + | > loss_dur: 0.06348 (0.07252) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.16007 (14.00251) + | > current_lr: 0.00008 + | > step_time: 2.80090 (2.19094) + | > loader_time: 0.09360 (0.03462) + + + --> STEP: 42/234 -- GLOBAL_STEP: 71880 + | > loss: -0.32981 (-0.36092) + | > log_mle: -0.40202 (-0.43363) + | > loss_dur: 0.07221 (0.07270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.75060 (13.38113) + | > current_lr: 0.00008 + | > step_time: 1.10400 (2.22322) + | > loader_time: 0.09250 (0.03552) + + + --> STEP: 47/234 -- GLOBAL_STEP: 71885 + | > loss: -0.34489 (-0.35801) + | > log_mle: -0.42109 (-0.43170) + | > loss_dur: 0.07620 (0.07368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.78573 (13.05779) + | > current_lr: 0.00008 + | > step_time: 1.24670 (2.13432) + | > loader_time: 0.00210 (0.03195) + + + --> STEP: 52/234 -- GLOBAL_STEP: 71890 + | > loss: -0.31440 (-0.35665) + | > log_mle: -0.40500 (-0.43029) + | > loss_dur: 0.09061 (0.07364) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.39180 (12.71253) + | > current_lr: 0.00008 + | > step_time: 2.79680 (2.10632) + | > loader_time: 0.09480 (0.03444) + + + --> STEP: 57/234 -- GLOBAL_STEP: 71895 + | > loss: -0.31594 (-0.35471) + | > log_mle: -0.39646 (-0.42863) + | > loss_dur: 0.08052 (0.07391) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.28784 (12.47752) + | > current_lr: 0.00008 + | > step_time: 2.07860 (2.12171) + | > loader_time: 0.00170 (0.03168) + + + --> STEP: 62/234 -- GLOBAL_STEP: 71900 + | > loss: -0.27669 (-0.35152) + | > log_mle: -0.40527 (-0.42712) + | > loss_dur: 0.12857 (0.07560) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.46903 (12.60761) + | > current_lr: 0.00008 + | > step_time: 4.09550 (2.15502) + | > loader_time: 0.00220 (0.03060) + + + --> STEP: 67/234 -- GLOBAL_STEP: 71905 + | > loss: -0.30736 (-0.34866) + | > log_mle: -0.40198 (-0.42489) + | > loss_dur: 0.09462 (0.07623) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.68813 (12.42894) + | > current_lr: 0.00008 + | > step_time: 2.24870 (2.13648) + | > loader_time: 0.00300 (0.02978) + + + --> STEP: 72/234 -- GLOBAL_STEP: 71910 + | > loss: -0.30462 (-0.34493) + | > log_mle: -0.39491 (-0.42258) + | > loss_dur: 0.09030 (0.07765) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.67091 (12.34159) + | > current_lr: 0.00008 + | > step_time: 2.70330 (2.12475) + | > loader_time: 0.00130 (0.02786) + + + --> STEP: 77/234 -- GLOBAL_STEP: 71915 + | > loss: -0.29348 (-0.34099) + | > log_mle: -0.39058 (-0.42048) + | > loss_dur: 0.09710 (0.07949) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.07786 (12.37527) + | > current_lr: 0.00008 + | > step_time: 2.32500 (2.14002) + | > loader_time: 0.00730 (0.02631) + + + --> STEP: 82/234 -- GLOBAL_STEP: 71920 + | > loss: -0.28027 (-0.33811) + | > log_mle: -0.38657 (-0.41867) + | > loss_dur: 0.10630 (0.08056) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29778 (12.29163) + | > current_lr: 0.00008 + | > step_time: 1.41350 (2.12609) + | > loader_time: 0.08570 (0.02589) + + + --> STEP: 87/234 -- GLOBAL_STEP: 71925 + | > loss: -0.29298 (-0.33532) + | > log_mle: -0.38991 (-0.41717) + | > loss_dur: 0.09693 (0.08185) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.51549 (12.25055) + | > current_lr: 0.00008 + | > step_time: 1.08640 (2.16729) + | > loader_time: 0.00330 (0.02661) + + + --> STEP: 92/234 -- GLOBAL_STEP: 71930 + | > loss: -0.30537 (-0.33299) + | > log_mle: -0.42771 (-0.41703) + | > loss_dur: 0.12234 (0.08404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.68368 (12.30525) + | > current_lr: 0.00008 + | > step_time: 1.92970 (2.17111) + | > loader_time: 0.00210 (0.02647) + + + --> STEP: 97/234 -- GLOBAL_STEP: 71935 + | > loss: -0.28487 (-0.33142) + | > log_mle: -0.41050 (-0.41774) + | > loss_dur: 0.12564 (0.08632) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.79407 (12.64169) + | > current_lr: 0.00008 + | > step_time: 2.40170 (2.18585) + | > loader_time: 0.00170 (0.02528) + + + --> STEP: 102/234 -- GLOBAL_STEP: 71940 + | > loss: -0.25799 (-0.32882) + | > log_mle: -0.39517 (-0.41761) + | > loss_dur: 0.13717 (0.08879) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.80421 (13.01387) + | > current_lr: 0.00008 + | > step_time: 1.19830 (2.18930) + | > loader_time: 0.00290 (0.02500) + + + --> STEP: 107/234 -- GLOBAL_STEP: 71945 + | > loss: -0.26385 (-0.32685) + | > log_mle: -0.42636 (-0.41850) + | > loss_dur: 0.16251 (0.09165) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.12540 (13.58790) + | > current_lr: 0.00008 + | > step_time: 1.80940 (2.17476) + | > loader_time: 0.00330 (0.02562) + + + --> STEP: 112/234 -- GLOBAL_STEP: 71950 + | > loss: -0.28369 (-0.32496) + | > log_mle: -0.44459 (-0.41947) + | > loss_dur: 0.16090 (0.09450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.71035 (14.03972) + | > current_lr: 0.00008 + | > step_time: 0.79920 (2.18204) + | > loader_time: 0.00320 (0.02634) + + + --> STEP: 117/234 -- GLOBAL_STEP: 71955 + | > loss: -0.28690 (-0.32328) + | > log_mle: -0.44240 (-0.42045) + | > loss_dur: 0.15550 (0.09717) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.60039 (14.39329) + | > current_lr: 0.00008 + | > step_time: 3.99680 (2.22130) + | > loader_time: 0.09090 (0.02611) + + + --> STEP: 122/234 -- GLOBAL_STEP: 71960 + | > loss: -0.26351 (-0.32141) + | > log_mle: -0.40396 (-0.42048) + | > loss_dur: 0.14045 (0.09907) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.21074 (14.71566) + | > current_lr: 0.00008 + | > step_time: 3.51440 (2.28515) + | > loader_time: 0.08200 (0.02823) + + + --> STEP: 127/234 -- GLOBAL_STEP: 71965 + | > loss: -0.29138 (-0.32007) + | > log_mle: -0.47049 (-0.42178) + | > loss_dur: 0.17912 (0.10171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.62833 (15.25704) + | > current_lr: 0.00008 + | > step_time: 1.30380 (2.24558) + | > loader_time: 0.00270 (0.02800) + + + --> STEP: 132/234 -- GLOBAL_STEP: 71970 + | > loss: -0.30177 (-0.31987) + | > log_mle: -0.45337 (-0.42384) + | > loss_dur: 0.15160 (0.10397) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.54584 (15.77606) + | > current_lr: 0.00008 + | > step_time: 5.60210 (2.26589) + | > loader_time: 0.08990 (0.02835) + + + --> STEP: 137/234 -- GLOBAL_STEP: 71975 + | > loss: -0.27998 (-0.31984) + | > log_mle: -0.46830 (-0.42630) + | > loss_dur: 0.18832 (0.10646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.60356 (16.38646) + | > current_lr: 0.00008 + | > step_time: 2.20590 (2.26158) + | > loader_time: 0.00380 (0.02859) + + + --> STEP: 142/234 -- GLOBAL_STEP: 71980 + | > loss: -0.30006 (-0.31936) + | > log_mle: -0.48018 (-0.42820) + | > loss_dur: 0.18012 (0.10884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.28857 (16.93082) + | > current_lr: 0.00008 + | > step_time: 5.09890 (2.30379) + | > loader_time: 0.10900 (0.02959) + + + --> STEP: 147/234 -- GLOBAL_STEP: 71985 + | > loss: -0.30191 (-0.32020) + | > log_mle: -0.47760 (-0.43184) + | > loss_dur: 0.17569 (0.11164) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.76828 (18.09812) + | > current_lr: 0.00008 + | > step_time: 3.60840 (2.37046) + | > loader_time: 0.00170 (0.03045) + + + --> STEP: 152/234 -- GLOBAL_STEP: 71990 + | > loss: -0.35554 (-0.32085) + | > log_mle: -0.55331 (-0.43479) + | > loss_dur: 0.19777 (0.11393) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.95765 (19.21695) + | > current_lr: 0.00008 + | > step_time: 5.40410 (2.41686) + | > loader_time: 0.09730 (0.03259) + + + --> STEP: 157/234 -- GLOBAL_STEP: 71995 + | > loss: -0.32939 (-0.32238) + | > log_mle: -0.50988 (-0.43882) + | > loss_dur: 0.18049 (0.11644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.01297 (19.85115) + | > current_lr: 0.00008 + | > step_time: 1.19100 (2.48183) + | > loader_time: 0.01120 (0.03403) + + + --> STEP: 162/234 -- GLOBAL_STEP: 72000 + | > loss: -0.35684 (-0.32374) + | > log_mle: -0.53737 (-0.44254) + | > loss_dur: 0.18053 (0.11881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.41133 (20.67726) + | > current_lr: 0.00008 + | > step_time: 3.18490 (2.48532) + | > loader_time: 0.01000 (0.03421) + + + --> STEP: 167/234 -- GLOBAL_STEP: 72005 + | > loss: -0.44152 (-0.32491) + | > log_mle: -0.62980 (-0.44590) + | > loss_dur: 0.18827 (0.12098) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.85189 (21.51122) + | > current_lr: 0.00008 + | > step_time: 7.10060 (2.53203) + | > loader_time: 0.09170 (0.03436) + + + --> STEP: 172/234 -- GLOBAL_STEP: 72010 + | > loss: -0.42045 (-0.32688) + | > log_mle: -0.63408 (-0.45045) + | > loss_dur: 0.21363 (0.12357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.18921 (22.43421) + | > current_lr: 0.00008 + | > step_time: 2.89400 (2.59569) + | > loader_time: 0.00720 (0.03672) + + + --> STEP: 177/234 -- GLOBAL_STEP: 72015 + | > loss: -0.38107 (-0.32880) + | > log_mle: -0.58100 (-0.45479) + | > loss_dur: 0.19993 (0.12599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.57342 (23.10761) + | > current_lr: 0.00008 + | > step_time: 4.19070 (2.62175) + | > loader_time: 0.00560 (0.03583) + + + --> STEP: 182/234 -- GLOBAL_STEP: 72020 + | > loss: -0.39033 (-0.33037) + | > log_mle: -0.62753 (-0.45897) + | > loss_dur: 0.23719 (0.12860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.49697 (24.05708) + | > current_lr: 0.00008 + | > step_time: 2.81280 (2.64214) + | > loader_time: 0.08820 (0.03692) + + + --> STEP: 187/234 -- GLOBAL_STEP: 72025 + | > loss: -0.41852 (-0.33209) + | > log_mle: -0.62960 (-0.46307) + | > loss_dur: 0.21108 (0.13099) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.18690 (25.26313) + | > current_lr: 0.00008 + | > step_time: 4.70450 (2.79399) + | > loader_time: 0.08300 (0.03804) + + + --> STEP: 192/234 -- GLOBAL_STEP: 72030 + | > loss: -0.45251 (-0.33431) + | > log_mle: -0.65942 (-0.46715) + | > loss_dur: 0.20691 (0.13284) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.90624 (26.17484) + | > current_lr: 0.00008 + | > step_time: 1.61340 (2.78959) + | > loader_time: 0.08830 (0.03813) + + + --> STEP: 197/234 -- GLOBAL_STEP: 72035 + | > loss: -0.41738 (-0.33618) + | > log_mle: -0.61433 (-0.47103) + | > loss_dur: 0.19695 (0.13486) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.92392 (27.03308) + | > current_lr: 0.00008 + | > step_time: 2.60150 (2.80897) + | > loader_time: 0.00210 (0.03856) + + + --> STEP: 202/234 -- GLOBAL_STEP: 72040 + | > loss: -0.50076 (-0.33794) + | > log_mle: -0.72544 (-0.47510) + | > loss_dur: 0.22469 (0.13716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.63596 (27.61834) + | > current_lr: 0.00008 + | > step_time: 2.09330 (2.80264) + | > loader_time: 0.00470 (0.03863) + + + --> STEP: 207/234 -- GLOBAL_STEP: 72045 + | > loss: -0.45068 (-0.33998) + | > log_mle: -0.69842 (-0.47929) + | > loss_dur: 0.24774 (0.13931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.75871 (28.38520) + | > current_lr: 0.00008 + | > step_time: 3.90790 (2.81193) + | > loader_time: 0.08940 (0.03951) + + + --> STEP: 212/234 -- GLOBAL_STEP: 72050 + | > loss: -0.45369 (-0.34262) + | > log_mle: -0.68324 (-0.48419) + | > loss_dur: 0.22955 (0.14156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.02113 (29.33493) + | > current_lr: 0.00008 + | > step_time: 8.59630 (2.88143) + | > loader_time: 0.11230 (0.04087) + + + --> STEP: 217/234 -- GLOBAL_STEP: 72055 + | > loss: -0.48297 (-0.34550) + | > log_mle: -0.71990 (-0.48916) + | > loss_dur: 0.23692 (0.14366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.05709 (30.28480) + | > current_lr: 0.00008 + | > step_time: 5.40600 (2.96195) + | > loader_time: 0.19020 (0.04216) + + + --> STEP: 222/234 -- GLOBAL_STEP: 72060 + | > loss: -0.46725 (-0.34835) + | > log_mle: -0.73547 (-0.49413) + | > loss_dur: 0.26822 (0.14578) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.41162 (31.41980) + | > current_lr: 0.00008 + | > step_time: 0.26080 (2.94308) + | > loader_time: 0.00360 (0.04210) + + + --> STEP: 227/234 -- GLOBAL_STEP: 72065 + | > loss: -0.43907 (-0.35162) + | > log_mle: -0.70238 (-0.49955) + | > loss_dur: 0.26331 (0.14793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.67297 (32.49100) + | > current_lr: 0.00008 + | > step_time: 0.28140 (2.88408) + | > loader_time: 0.00410 (0.04126) + + + --> STEP: 232/234 -- GLOBAL_STEP: 72070 + | > loss: -0.42366 (-0.35413) + | > log_mle: -0.92260 (-0.50647) + | > loss_dur: 0.49894 (0.15234) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.80547 (34.22105) + | > current_lr: 0.00008 + | > step_time: 0.35110 (2.82839) + | > loader_time: 0.00670 (0.04048) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.92287 (-0.23124) + | > avg_loss: -0.35024 (-0.01271) + | > avg_log_mle: -0.57012 (-0.00164) + | > avg_loss_dur: 0.21988 (-0.01107) + + + > EPOCH: 308/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 02:03:43)  + + --> STEP: 3/234 -- GLOBAL_STEP: 72075 + | > loss: -0.26156 (-0.32395) + | > log_mle: -0.42103 (-0.43313) + | > loss_dur: 0.15947 (0.10918) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.46516 (16.52099) + | > current_lr: 0.00008 + | > step_time: 1.70460 (9.47500) + | > loader_time: 0.00150 (2.83159) + + + --> STEP: 8/234 -- GLOBAL_STEP: 72080 + | > loss: -0.36859 (-0.34629) + | > log_mle: -0.45852 (-0.43735) + | > loss_dur: 0.08994 (0.09106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.02060 (15.36340) + | > current_lr: 0.00008 + | > step_time: 2.73770 (4.63182) + | > loader_time: 0.00300 (1.07425) + + + --> STEP: 13/234 -- GLOBAL_STEP: 72085 + | > loss: -0.39551 (-0.35370) + | > log_mle: -0.46155 (-0.44005) + | > loss_dur: 0.06604 (0.08635) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.67720 (17.11577) + | > current_lr: 0.00008 + | > step_time: 2.20690 (3.77785) + | > loader_time: 0.00130 (0.66932) + + + --> STEP: 18/234 -- GLOBAL_STEP: 72090 + | > loss: -0.37012 (-0.36211) + | > log_mle: -0.43392 (-0.44197) + | > loss_dur: 0.06381 (0.07987) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.60983 (15.49917) + | > current_lr: 0.00008 + | > step_time: 1.99740 (3.36240) + | > loader_time: 0.08960 (0.49447) + + + --> STEP: 23/234 -- GLOBAL_STEP: 72095 + | > loss: -0.40853 (-0.36711) + | > log_mle: -0.46368 (-0.44337) + | > loss_dur: 0.05515 (0.07627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.35540 (14.59818) + | > current_lr: 0.00008 + | > step_time: 4.59560 (3.25258) + | > loader_time: 0.00110 (0.39907) + + + --> STEP: 28/234 -- GLOBAL_STEP: 72100 + | > loss: -0.43316 (-0.36969) + | > log_mle: -0.48022 (-0.44342) + | > loss_dur: 0.04706 (0.07373) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.49477 (13.83405) + | > current_lr: 0.00008 + | > step_time: 3.81970 (3.60451) + | > loader_time: 0.10520 (0.34581) + + + --> STEP: 33/234 -- GLOBAL_STEP: 72105 + | > loss: -0.35441 (-0.36789) + | > log_mle: -0.42498 (-0.44103) + | > loss_dur: 0.07057 (0.07315) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63485 (13.57008) + | > current_lr: 0.00008 + | > step_time: 2.50330 (3.24510) + | > loader_time: 0.00170 (0.29376) + + + --> STEP: 38/234 -- GLOBAL_STEP: 72110 + | > loss: -0.33939 (-0.36363) + | > log_mle: -0.42362 (-0.43764) + | > loss_dur: 0.08423 (0.07401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.38621 (13.35067) + | > current_lr: 0.00008 + | > step_time: 7.59180 (3.48847) + | > loader_time: 0.11120 (0.26628) + + + --> STEP: 43/234 -- GLOBAL_STEP: 72115 + | > loss: -0.31124 (-0.35904) + | > log_mle: -0.40664 (-0.43405) + | > loss_dur: 0.09540 (0.07501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.22948 (13.32394) + | > current_lr: 0.00008 + | > step_time: 1.81520 (3.28504) + | > loader_time: 0.00170 (0.23560) + + + --> STEP: 48/234 -- GLOBAL_STEP: 72120 + | > loss: -0.36966 (-0.35652) + | > log_mle: -0.42825 (-0.43196) + | > loss_dur: 0.05858 (0.07544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.53025 (13.05352) + | > current_lr: 0.00008 + | > step_time: 1.08030 (3.18192) + | > loader_time: 0.00220 (0.21129) + + + --> STEP: 53/234 -- GLOBAL_STEP: 72125 + | > loss: -0.32782 (-0.35415) + | > log_mle: -0.41286 (-0.42999) + | > loss_dur: 0.08505 (0.07584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.81798 (12.68500) + | > current_lr: 0.00008 + | > step_time: 1.51030 (3.02673) + | > loader_time: 0.00770 (0.19164) + + + --> STEP: 58/234 -- GLOBAL_STEP: 72130 + | > loss: -0.34452 (-0.35241) + | > log_mle: -0.41703 (-0.42833) + | > loss_dur: 0.07251 (0.07593) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.00720 (12.43507) + | > current_lr: 0.00008 + | > step_time: 1.31810 (2.92666) + | > loader_time: 0.06690 (0.17910) + + + --> STEP: 63/234 -- GLOBAL_STEP: 72135 + | > loss: -0.31186 (-0.34858) + | > log_mle: -0.39385 (-0.42643) + | > loss_dur: 0.08199 (0.07785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.42876 (12.34961) + | > current_lr: 0.00008 + | > step_time: 1.63380 (2.84842) + | > loader_time: 0.00200 (0.16522) + + + --> STEP: 68/234 -- GLOBAL_STEP: 72140 + | > loss: -0.28682 (-0.34598) + | > log_mle: -0.39070 (-0.42439) + | > loss_dur: 0.10389 (0.07841) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47989 (12.10414) + | > current_lr: 0.00008 + | > step_time: 2.18800 (2.81095) + | > loader_time: 0.00170 (0.15459) + + + --> STEP: 73/234 -- GLOBAL_STEP: 72145 + | > loss: -0.27393 (-0.34241) + | > log_mle: -0.39223 (-0.42238) + | > loss_dur: 0.11830 (0.07996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.35737 (12.01300) + | > current_lr: 0.00008 + | > step_time: 0.90370 (2.73459) + | > loader_time: 0.00140 (0.14420) + + + --> STEP: 78/234 -- GLOBAL_STEP: 72150 + | > loss: -0.28491 (-0.33889) + | > log_mle: -0.38162 (-0.42018) + | > loss_dur: 0.09671 (0.08130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.93712 (11.94771) + | > current_lr: 0.00008 + | > step_time: 1.51400 (2.71478) + | > loader_time: 0.00220 (0.13729) + + + --> STEP: 83/234 -- GLOBAL_STEP: 72155 + | > loss: -0.26784 (-0.33611) + | > log_mle: -0.39212 (-0.41858) + | > loss_dur: 0.12429 (0.08246) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.97188 (11.91372) + | > current_lr: 0.00008 + | > step_time: 2.88770 (2.67096) + | > loader_time: 0.10060 (0.13235) + + + --> STEP: 88/234 -- GLOBAL_STEP: 72160 + | > loss: -0.29322 (-0.33377) + | > log_mle: -0.42457 (-0.41749) + | > loss_dur: 0.13135 (0.08372) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.02507 (11.90661) + | > current_lr: 0.00008 + | > step_time: 3.09130 (2.69205) + | > loader_time: 0.00180 (0.12594) + + + --> STEP: 93/234 -- GLOBAL_STEP: 72165 + | > loss: -0.29497 (-0.33157) + | > log_mle: -0.43868 (-0.41754) + | > loss_dur: 0.14371 (0.08597) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.93885 (12.04701) + | > current_lr: 0.00008 + | > step_time: 1.61050 (2.66470) + | > loader_time: 0.08800 (0.12131) + + + --> STEP: 98/234 -- GLOBAL_STEP: 72170 + | > loss: -0.27525 (-0.32980) + | > log_mle: -0.37715 (-0.41759) + | > loss_dur: 0.10190 (0.08779) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.25729 (12.27132) + | > current_lr: 0.00008 + | > step_time: 2.39670 (2.63891) + | > loader_time: 0.00250 (0.11729) + + + --> STEP: 103/234 -- GLOBAL_STEP: 72175 + | > loss: -0.30141 (-0.32792) + | > log_mle: -0.46338 (-0.41839) + | > loss_dur: 0.16198 (0.09047) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.17194 (12.66281) + | > current_lr: 0.00008 + | > step_time: 4.08330 (2.61536) + | > loader_time: 0.00810 (0.11342) + + + --> STEP: 108/234 -- GLOBAL_STEP: 72180 + | > loss: -0.28235 (-0.32606) + | > log_mle: -0.40733 (-0.41881) + | > loss_dur: 0.12498 (0.09276) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.18022 (13.04452) + | > current_lr: 0.00008 + | > step_time: 1.30110 (2.59256) + | > loader_time: 0.00250 (0.10967) + + + --> STEP: 113/234 -- GLOBAL_STEP: 72185 + | > loss: -0.30137 (-0.32445) + | > log_mle: -0.44951 (-0.42002) + | > loss_dur: 0.14814 (0.09557) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.34867 (13.81489) + | > current_lr: 0.00008 + | > step_time: 2.08880 (2.58412) + | > loader_time: 0.00310 (0.10573) + + + --> STEP: 118/234 -- GLOBAL_STEP: 72190 + | > loss: -0.26918 (-0.32238) + | > log_mle: -0.42503 (-0.42065) + | > loss_dur: 0.15585 (0.09827) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.21936 (14.26648) + | > current_lr: 0.00008 + | > step_time: 3.29200 (2.57814) + | > loader_time: 0.00480 (0.10294) + + + --> STEP: 123/234 -- GLOBAL_STEP: 72195 + | > loss: -0.24507 (-0.32031) + | > log_mle: -0.38982 (-0.42041) + | > loss_dur: 0.14475 (0.10010) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.06556 (14.61270) + | > current_lr: 0.00008 + | > step_time: 1.81510 (2.54885) + | > loader_time: 0.08530 (0.10028) + + + --> STEP: 128/234 -- GLOBAL_STEP: 72200 + | > loss: -0.30959 (-0.31974) + | > log_mle: -0.44588 (-0.42222) + | > loss_dur: 0.13630 (0.10248) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.94400 (15.11422) + | > current_lr: 0.00008 + | > step_time: 2.09090 (2.56711) + | > loader_time: 0.00240 (0.09797) + + + --> STEP: 133/234 -- GLOBAL_STEP: 72205 + | > loss: -0.31570 (-0.31968) + | > log_mle: -0.48033 (-0.42452) + | > loss_dur: 0.16462 (0.10484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.33191 (15.78971) + | > current_lr: 0.00008 + | > step_time: 5.50180 (2.59770) + | > loader_time: 0.18980 (0.09652) + + + --> STEP: 138/234 -- GLOBAL_STEP: 72210 + | > loss: -0.26591 (-0.31918) + | > log_mle: -0.42240 (-0.42648) + | > loss_dur: 0.15649 (0.10729) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.08114 (16.52053) + | > current_lr: 0.00008 + | > step_time: 3.80100 (2.63637) + | > loader_time: 0.00350 (0.09730) + + + --> STEP: 143/234 -- GLOBAL_STEP: 72215 + | > loss: -0.35285 (-0.31911) + | > log_mle: -0.57822 (-0.42934) + | > loss_dur: 0.22537 (0.11023) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.09012 (17.28533) + | > current_lr: 0.00008 + | > step_time: 3.38690 (2.68443) + | > loader_time: 0.10480 (0.09794) + + + --> STEP: 148/234 -- GLOBAL_STEP: 72220 + | > loss: -0.32142 (-0.31954) + | > log_mle: -0.48417 (-0.43214) + | > loss_dur: 0.16275 (0.11259) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.25506 (18.05965) + | > current_lr: 0.00008 + | > step_time: 6.29460 (2.78900) + | > loader_time: 0.19610 (0.09801) + + + --> STEP: 153/234 -- GLOBAL_STEP: 72225 + | > loss: -0.43195 (-0.32116) + | > log_mle: -0.61788 (-0.43619) + | > loss_dur: 0.18593 (0.11503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.90775 (19.16443) + | > current_lr: 0.00008 + | > step_time: 4.81760 (2.83763) + | > loader_time: 0.00290 (0.09788) + + + --> STEP: 158/234 -- GLOBAL_STEP: 72230 + | > loss: -0.34324 (-0.32223) + | > log_mle: -0.54745 (-0.43974) + | > loss_dur: 0.20421 (0.11752) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.82445 (19.97734) + | > current_lr: 0.00008 + | > step_time: 2.08880 (2.83826) + | > loader_time: 0.00330 (0.09553) + + + --> STEP: 163/234 -- GLOBAL_STEP: 72235 + | > loss: -0.32404 (-0.32346) + | > log_mle: -0.51058 (-0.44328) + | > loss_dur: 0.18654 (0.11982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.31562 (20.90632) + | > current_lr: 0.00008 + | > step_time: 1.61540 (2.96238) + | > loader_time: 0.00760 (0.09387) + + + --> STEP: 168/234 -- GLOBAL_STEP: 72240 + | > loss: -0.34834 (-0.32488) + | > log_mle: -0.56451 (-0.44698) + | > loss_dur: 0.21617 (0.12210) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 95.46493 (22.05523) + | > current_lr: 0.00008 + | > step_time: 4.00170 (2.95156) + | > loader_time: 0.00780 (0.09127) + + + --> STEP: 173/234 -- GLOBAL_STEP: 72245 + | > loss: -0.37538 (-0.32670) + | > log_mle: -0.58088 (-0.45127) + | > loss_dur: 0.20550 (0.12456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.91335 (23.15873) + | > current_lr: 0.00008 + | > step_time: 2.71300 (2.97328) + | > loader_time: 0.00260 (0.08920) + + + --> STEP: 178/234 -- GLOBAL_STEP: 72250 + | > loss: -0.41413 (-0.32878) + | > log_mle: -0.63927 (-0.45584) + | > loss_dur: 0.22514 (0.12706) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.36104 (23.84184) + | > current_lr: 0.00008 + | > step_time: 3.79360 (2.96736) + | > loader_time: 0.10410 (0.08783) + + + --> STEP: 183/234 -- GLOBAL_STEP: 72255 + | > loss: -0.44085 (-0.33054) + | > log_mle: -0.65083 (-0.46016) + | > loss_dur: 0.20997 (0.12962) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.73990 (24.44520) + | > current_lr: 0.00008 + | > step_time: 3.11350 (2.96289) + | > loader_time: 0.00190 (0.08659) + + + --> STEP: 188/234 -- GLOBAL_STEP: 72260 + | > loss: -0.44177 (-0.33246) + | > log_mle: -0.66103 (-0.46444) + | > loss_dur: 0.21926 (0.13198) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.00899 (25.50316) + | > current_lr: 0.00008 + | > step_time: 2.60740 (2.94372) + | > loader_time: 0.00340 (0.08517) + + + --> STEP: 193/234 -- GLOBAL_STEP: 72265 + | > loss: -0.44703 (-0.33476) + | > log_mle: -0.65736 (-0.46863) + | > loss_dur: 0.21032 (0.13388) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.85684 (26.37111) + | > current_lr: 0.00008 + | > step_time: 1.80110 (2.96441) + | > loader_time: 0.00440 (0.08552) + + + --> STEP: 198/234 -- GLOBAL_STEP: 72270 + | > loss: -0.43372 (-0.33682) + | > log_mle: -0.65320 (-0.47275) + | > loss_dur: 0.21949 (0.13593) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.61349 (27.13011) + | > current_lr: 0.00008 + | > step_time: 7.79260 (3.01306) + | > loader_time: 0.19440 (0.08530) + + + --> STEP: 203/234 -- GLOBAL_STEP: 72275 + | > loss: -0.37459 (-0.33863) + | > log_mle: -0.57804 (-0.47664) + | > loss_dur: 0.20345 (0.13801) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.58787 (27.88338) + | > current_lr: 0.00008 + | > step_time: 2.19780 (3.01940) + | > loader_time: 0.00380 (0.08544) + + + --> STEP: 208/234 -- GLOBAL_STEP: 72280 + | > loss: -0.44353 (-0.34133) + | > log_mle: -0.67541 (-0.48145) + | > loss_dur: 0.23188 (0.14012) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.18179 (28.66237) + | > current_lr: 0.00008 + | > step_time: 10.10460 (3.14682) + | > loader_time: 0.10160 (0.08722) + + + --> STEP: 213/234 -- GLOBAL_STEP: 72285 + | > loss: -0.46124 (-0.34395) + | > log_mle: -0.70380 (-0.48632) + | > loss_dur: 0.24256 (0.14237) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.18000 (29.68819) + | > current_lr: 0.00008 + | > step_time: 4.30580 (3.14061) + | > loader_time: 0.00510 (0.08658) + + + --> STEP: 218/234 -- GLOBAL_STEP: 72290 + | > loss: -0.44286 (-0.34665) + | > log_mle: -0.67432 (-0.49097) + | > loss_dur: 0.23146 (0.14433) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.11810 (30.44180) + | > current_lr: 0.00008 + | > step_time: 6.39430 (3.23595) + | > loader_time: 0.00650 (0.08553) + + + --> STEP: 223/234 -- GLOBAL_STEP: 72295 + | > loss: -0.47681 (-0.34960) + | > log_mle: -0.70810 (-0.49601) + | > loss_dur: 0.23130 (0.14640) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 127.33850 (31.73746) + | > current_lr: 0.00008 + | > step_time: 0.22240 (3.20695) + | > loader_time: 0.00310 (0.08408) + + + --> STEP: 228/234 -- GLOBAL_STEP: 72300 + | > loss: -0.43776 (-0.35246) + | > log_mle: -0.70441 (-0.50106) + | > loss_dur: 0.26665 (0.14861) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 118.48302 (33.07743) + | > current_lr: 0.00008 + | > step_time: 0.24070 (3.14172) + | > loader_time: 0.00510 (0.08232) + + + --> STEP: 233/234 -- GLOBAL_STEP: 72305 + | > loss: 0.04095 (-0.35256) + | > log_mle: -0.68046 (-0.50732) + | > loss_dur: 0.72141 (0.15476) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.39642 (34.83596) + | > current_lr: 0.00008 + | > step_time: 0.18500 (3.07980) + | > loader_time: 0.00250 (0.08068) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.35027 (-0.57260) + | > avg_loss: -0.33888 (+0.01136) + | > avg_log_mle: -0.55743 (+0.01269) + | > avg_loss_dur: 0.21855 (-0.00133) + + + > EPOCH: 309/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 02:17:08)  + + --> STEP: 4/234 -- GLOBAL_STEP: 72310 + | > loss: -0.34406 (-0.33211) + | > log_mle: -0.43030 (-0.43276) + | > loss_dur: 0.08625 (0.10065) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.51662 (19.10930) + | > current_lr: 0.00008 + | > step_time: 5.18100 (6.07342) + | > loader_time: 0.10820 (0.15148) + + + --> STEP: 9/234 -- GLOBAL_STEP: 72315 + | > loss: -0.33361 (-0.34549) + | > log_mle: -0.44515 (-0.43820) + | > loss_dur: 0.11153 (0.09271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.43866 (16.18710) + | > current_lr: 0.00008 + | > step_time: 3.79940 (4.00001) + | > loader_time: 0.00210 (0.09855) + + + --> STEP: 14/234 -- GLOBAL_STEP: 72320 + | > loss: -0.37758 (-0.35774) + | > log_mle: -0.44212 (-0.44166) + | > loss_dur: 0.06454 (0.08392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.07792 (15.14711) + | > current_lr: 0.00008 + | > step_time: 6.30380 (3.63572) + | > loader_time: 0.10240 (0.08488) + + + --> STEP: 19/234 -- GLOBAL_STEP: 72325 + | > loss: -0.39025 (-0.36659) + | > log_mle: -0.45528 (-0.44393) + | > loss_dur: 0.06503 (0.07734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.77629 (14.86293) + | > current_lr: 0.00008 + | > step_time: 2.61680 (3.78916) + | > loader_time: 0.08890 (0.07248) + + + --> STEP: 24/234 -- GLOBAL_STEP: 72330 + | > loss: -0.37663 (-0.36893) + | > log_mle: -0.43973 (-0.44417) + | > loss_dur: 0.06310 (0.07524) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.92474 (14.09946) + | > current_lr: 0.00008 + | > step_time: 5.79620 (3.54838) + | > loader_time: 0.09270 (0.06455) + + + --> STEP: 29/234 -- GLOBAL_STEP: 72335 + | > loss: -0.36705 (-0.37111) + | > log_mle: -0.43281 (-0.44405) + | > loss_dur: 0.06576 (0.07295) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.84637 (13.50955) + | > current_lr: 0.00008 + | > step_time: 1.14430 (3.36563) + | > loader_time: 0.00190 (0.05983) + + + --> STEP: 34/234 -- GLOBAL_STEP: 72340 + | > loss: -0.35410 (-0.36861) + | > log_mle: -0.41983 (-0.44090) + | > loss_dur: 0.06573 (0.07229) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.55026 (13.40218) + | > current_lr: 0.00008 + | > step_time: 1.80970 (3.06312) + | > loader_time: 0.08470 (0.05613) + + + --> STEP: 39/234 -- GLOBAL_STEP: 72345 + | > loss: -0.32049 (-0.36395) + | > log_mle: -0.40322 (-0.43692) + | > loss_dur: 0.08272 (0.07297) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.27547 (13.71381) + | > current_lr: 0.00008 + | > step_time: 0.88570 (2.91382) + | > loader_time: 0.00180 (0.04926) + + + --> STEP: 44/234 -- GLOBAL_STEP: 72350 + | > loss: -0.33564 (-0.36012) + | > log_mle: -0.40217 (-0.43334) + | > loss_dur: 0.06653 (0.07323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.54632 (13.51730) + | > current_lr: 0.00008 + | > step_time: 1.27160 (2.72157) + | > loader_time: 0.00160 (0.04387) + + + --> STEP: 49/234 -- GLOBAL_STEP: 72355 + | > loss: -0.35210 (-0.35791) + | > log_mle: -0.42349 (-0.43173) + | > loss_dur: 0.07139 (0.07382) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.04528 (13.18071) + | > current_lr: 0.00008 + | > step_time: 1.20940 (2.68469) + | > loader_time: 0.19060 (0.04690) + + + --> STEP: 54/234 -- GLOBAL_STEP: 72360 + | > loss: -0.33268 (-0.35501) + | > log_mle: -0.40807 (-0.42955) + | > loss_dur: 0.07539 (0.07454) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.63524 (12.81003) + | > current_lr: 0.00008 + | > step_time: 2.80110 (2.65303) + | > loader_time: 0.10330 (0.04630) + + + --> STEP: 59/234 -- GLOBAL_STEP: 72365 + | > loss: -0.31449 (-0.35279) + | > log_mle: -0.40693 (-0.42773) + | > loss_dur: 0.09244 (0.07495) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.39231 (12.60323) + | > current_lr: 0.00008 + | > step_time: 1.15190 (2.59523) + | > loader_time: 0.00180 (0.04393) + + + --> STEP: 64/234 -- GLOBAL_STEP: 72370 + | > loss: -0.32141 (-0.34881) + | > log_mle: -0.40118 (-0.42567) + | > loss_dur: 0.07978 (0.07686) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.66252 (12.57807) + | > current_lr: 0.00008 + | > step_time: 1.93810 (2.56021) + | > loader_time: 0.00230 (0.04203) + + + --> STEP: 69/234 -- GLOBAL_STEP: 72375 + | > loss: -0.31728 (-0.34612) + | > log_mle: -0.39293 (-0.42343) + | > loss_dur: 0.07565 (0.07731) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.14682 (12.48789) + | > current_lr: 0.00008 + | > step_time: 2.25120 (2.53863) + | > loader_time: 0.08350 (0.04033) + + + --> STEP: 74/234 -- GLOBAL_STEP: 72380 + | > loss: -0.29604 (-0.34243) + | > log_mle: -0.38764 (-0.42148) + | > loss_dur: 0.09160 (0.07905) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.03170 (12.51655) + | > current_lr: 0.00008 + | > step_time: 2.14090 (2.50265) + | > loader_time: 0.00180 (0.03776) + + + --> STEP: 79/234 -- GLOBAL_STEP: 72385 + | > loss: -0.30881 (-0.33933) + | > log_mle: -0.40170 (-0.41971) + | > loss_dur: 0.09289 (0.08038) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.45226 (12.52988) + | > current_lr: 0.00008 + | > step_time: 1.89150 (2.48735) + | > loader_time: 0.00430 (0.03667) + + + --> STEP: 84/234 -- GLOBAL_STEP: 72390 + | > loss: -0.30431 (-0.33655) + | > log_mle: -0.39333 (-0.41809) + | > loss_dur: 0.08902 (0.08154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67226 (12.54827) + | > current_lr: 0.00008 + | > step_time: 2.68220 (2.52858) + | > loader_time: 0.00270 (0.03684) + + + --> STEP: 89/234 -- GLOBAL_STEP: 72395 + | > loss: -0.30586 (-0.33439) + | > log_mle: -0.41135 (-0.41725) + | > loss_dur: 0.10549 (0.08286) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.55559 (12.70693) + | > current_lr: 0.00008 + | > step_time: 1.29730 (2.53710) + | > loader_time: 0.00280 (0.03692) + + + --> STEP: 94/234 -- GLOBAL_STEP: 72400 + | > loss: -0.30894 (-0.33224) + | > log_mle: -0.43605 (-0.41748) + | > loss_dur: 0.12711 (0.08524) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.37268 (13.22220) + | > current_lr: 0.00008 + | > step_time: 1.98650 (2.49337) + | > loader_time: 0.00260 (0.03759) + + + --> STEP: 99/234 -- GLOBAL_STEP: 72405 + | > loss: -0.30012 (-0.33046) + | > log_mle: -0.46208 (-0.41786) + | > loss_dur: 0.16196 (0.08740) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.98825 (13.54597) + | > current_lr: 0.00008 + | > step_time: 1.46110 (2.46705) + | > loader_time: 0.03130 (0.03699) + + + --> STEP: 104/234 -- GLOBAL_STEP: 72410 + | > loss: -0.32343 (-0.32872) + | > log_mle: -0.47350 (-0.41869) + | > loss_dur: 0.15007 (0.08996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.35366 (13.98830) + | > current_lr: 0.00008 + | > step_time: 1.89840 (2.44278) + | > loader_time: 0.00260 (0.03692) + + + --> STEP: 109/234 -- GLOBAL_STEP: 72415 + | > loss: -0.27686 (-0.32611) + | > log_mle: -0.44802 (-0.41872) + | > loss_dur: 0.17116 (0.09261) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.05212 (14.47647) + | > current_lr: 0.00008 + | > step_time: 4.73890 (2.50760) + | > loader_time: 0.00460 (0.03871) + + + --> STEP: 114/234 -- GLOBAL_STEP: 72420 + | > loss: -0.28872 (-0.32438) + | > log_mle: -0.42973 (-0.41978) + | > loss_dur: 0.14101 (0.09540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.91324 (14.90548) + | > current_lr: 0.00008 + | > step_time: 1.21390 (2.45533) + | > loader_time: 0.00280 (0.03714) + + + --> STEP: 119/234 -- GLOBAL_STEP: 72425 + | > loss: -0.27898 (-0.32247) + | > log_mle: -0.42733 (-0.42037) + | > loss_dur: 0.14836 (0.09791) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.19908 (15.21190) + | > current_lr: 0.00008 + | > step_time: 2.10140 (2.42303) + | > loader_time: 0.07520 (0.03693) + + + --> STEP: 124/234 -- GLOBAL_STEP: 72430 + | > loss: -0.29875 (-0.32054) + | > log_mle: -0.45342 (-0.42037) + | > loss_dur: 0.15467 (0.09983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.04197 (15.44035) + | > current_lr: 0.00008 + | > step_time: 1.64100 (2.45941) + | > loader_time: 0.09640 (0.03785) + + + --> STEP: 129/234 -- GLOBAL_STEP: 72435 + | > loss: -0.28522 (-0.31980) + | > log_mle: -0.44994 (-0.42215) + | > loss_dur: 0.16472 (0.10235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.85080 (15.88469) + | > current_lr: 0.00008 + | > step_time: 13.00600 (2.56178) + | > loader_time: 0.19760 (0.04028) + + + --> STEP: 134/234 -- GLOBAL_STEP: 72440 + | > loss: -0.32136 (-0.32002) + | > log_mle: -0.50459 (-0.42488) + | > loss_dur: 0.18323 (0.10485) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.36800 (16.49072) + | > current_lr: 0.00008 + | > step_time: 1.72710 (2.54023) + | > loader_time: 0.07800 (0.04059) + + + --> STEP: 139/234 -- GLOBAL_STEP: 72445 + | > loss: -0.37778 (-0.32009) + | > log_mle: -0.56025 (-0.42731) + | > loss_dur: 0.18247 (0.10721) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.88126 (17.41661) + | > current_lr: 0.00008 + | > step_time: 2.39710 (2.53671) + | > loader_time: 0.00260 (0.03987) + + + --> STEP: 144/234 -- GLOBAL_STEP: 72450 + | > loss: -0.32782 (-0.31979) + | > log_mle: -0.53290 (-0.42983) + | > loss_dur: 0.20508 (0.11004) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.14165 (18.22327) + | > current_lr: 0.00008 + | > step_time: 2.50480 (2.55305) + | > loader_time: 0.00290 (0.03939) + + + --> STEP: 149/234 -- GLOBAL_STEP: 72455 + | > loss: -0.40191 (-0.32081) + | > log_mle: -0.59148 (-0.43316) + | > loss_dur: 0.18957 (0.11235) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.42360 (18.86319) + | > current_lr: 0.00008 + | > step_time: 3.59480 (2.54949) + | > loader_time: 0.10370 (0.03885) + + + --> STEP: 154/234 -- GLOBAL_STEP: 72460 + | > loss: -0.36105 (-0.32224) + | > log_mle: -0.54551 (-0.43708) + | > loss_dur: 0.18446 (0.11484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.17811 (19.83738) + | > current_lr: 0.00008 + | > step_time: 4.09920 (2.58509) + | > loader_time: 0.00320 (0.03829) + + + --> STEP: 159/234 -- GLOBAL_STEP: 72465 + | > loss: -0.37637 (-0.32340) + | > log_mle: -0.56220 (-0.44069) + | > loss_dur: 0.18583 (0.11729) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.67401 (20.63583) + | > current_lr: 0.00008 + | > step_time: 2.19300 (2.61060) + | > loader_time: 0.00350 (0.04090) + + + --> STEP: 164/234 -- GLOBAL_STEP: 72470 + | > loss: -0.34476 (-0.32464) + | > log_mle: -0.55025 (-0.44419) + | > loss_dur: 0.20550 (0.11955) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.20545 (21.50578) + | > current_lr: 0.00008 + | > step_time: 2.59490 (2.61778) + | > loader_time: 0.00280 (0.04036) + + + --> STEP: 169/234 -- GLOBAL_STEP: 72475 + | > loss: -0.36087 (-0.32640) + | > log_mle: -0.55897 (-0.44807) + | > loss_dur: 0.19810 (0.12167) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.08809 (22.46258) + | > current_lr: 0.00008 + | > step_time: 6.70930 (2.71082) + | > loader_time: 0.00510 (0.04093) + + + --> STEP: 174/234 -- GLOBAL_STEP: 72480 + | > loss: -0.44684 (-0.32912) + | > log_mle: -0.65396 (-0.45324) + | > loss_dur: 0.20712 (0.12412) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.00493 (23.33070) + | > current_lr: 0.00008 + | > step_time: 1.99210 (2.74618) + | > loader_time: 0.00420 (0.04319) + + + --> STEP: 179/234 -- GLOBAL_STEP: 72485 + | > loss: -0.39826 (-0.33099) + | > log_mle: -0.63982 (-0.45774) + | > loss_dur: 0.24156 (0.12675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.62427 (24.32941) + | > current_lr: 0.00008 + | > step_time: 5.41130 (2.76393) + | > loader_time: 0.00480 (0.04258) + + + --> STEP: 184/234 -- GLOBAL_STEP: 72490 + | > loss: -0.39785 (-0.33259) + | > log_mle: -0.60169 (-0.46170) + | > loss_dur: 0.20384 (0.12911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.81142 (25.11200) + | > current_lr: 0.00008 + | > step_time: 6.51300 (2.85082) + | > loader_time: 0.20720 (0.04484) + + + --> STEP: 189/234 -- GLOBAL_STEP: 72495 + | > loss: -0.38525 (-0.33443) + | > log_mle: -0.60202 (-0.46598) + | > loss_dur: 0.21678 (0.13155) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.33260 (25.89947) + | > current_lr: 0.00008 + | > step_time: 3.89870 (2.87177) + | > loader_time: 0.00330 (0.04565) + + + --> STEP: 194/234 -- GLOBAL_STEP: 72500 + | > loss: -0.42165 (-0.33665) + | > log_mle: -0.62833 (-0.47009) + | > loss_dur: 0.20668 (0.13344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.86057 (26.53710) + | > current_lr: 0.00008 + | > step_time: 4.61570 (2.93657) + | > loader_time: 0.19180 (0.04854) + + + --> STEP: 199/234 -- GLOBAL_STEP: 72505 + | > loss: -0.42659 (-0.33866) + | > log_mle: -0.64616 (-0.47414) + | > loss_dur: 0.21956 (0.13548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 86.23597 (27.41376) + | > current_lr: 0.00008 + | > step_time: 1.90250 (2.94518) + | > loader_time: 0.00370 (0.04835) + + + --> STEP: 204/234 -- GLOBAL_STEP: 72510 + | > loss: -0.45111 (-0.34038) + | > log_mle: -0.68718 (-0.47807) + | > loss_dur: 0.23607 (0.13769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.27877 (28.25801) + | > current_lr: 0.00008 + | > step_time: 6.22170 (2.99620) + | > loader_time: 0.08270 (0.04952) + + + --> STEP: 209/234 -- GLOBAL_STEP: 72515 + | > loss: -0.42229 (-0.34284) + | > log_mle: -0.64697 (-0.48260) + | > loss_dur: 0.22468 (0.13976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.92436 (29.04203) + | > current_lr: 0.00008 + | > step_time: 3.38790 (3.02167) + | > loader_time: 0.00890 (0.04980) + + + --> STEP: 214/234 -- GLOBAL_STEP: 72520 + | > loss: -0.46520 (-0.34609) + | > log_mle: -0.67604 (-0.48795) + | > loss_dur: 0.21084 (0.14186) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.20590 (30.13285) + | > current_lr: 0.00008 + | > step_time: 8.40550 (3.07306) + | > loader_time: 0.28930 (0.05109) + + + --> STEP: 219/234 -- GLOBAL_STEP: 72525 + | > loss: -0.55239 (-0.34930) + | > log_mle: -0.78630 (-0.49329) + | > loss_dur: 0.23392 (0.14399) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 109.11549 (31.20185) + | > current_lr: 0.00008 + | > step_time: 2.89790 (3.09011) + | > loader_time: 0.19460 (0.05135) + + + --> STEP: 224/234 -- GLOBAL_STEP: 72530 + | > loss: -0.51076 (-0.35215) + | > log_mle: -0.73420 (-0.49814) + | > loss_dur: 0.22344 (0.14599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.43304 (32.13665) + | > current_lr: 0.00008 + | > step_time: 1.51130 (3.07802) + | > loader_time: 0.00280 (0.05175) + + + --> STEP: 229/234 -- GLOBAL_STEP: 72535 + | > loss: -0.46708 (-0.35465) + | > log_mle: -0.75963 (-0.50322) + | > loss_dur: 0.29256 (0.14857) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.96873 (33.58234) + | > current_lr: 0.00008 + | > step_time: 0.24960 (3.02804) + | > loader_time: 0.00270 (0.05104) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.79581 (+0.44554) + | > avg_loss: -0.34603 (-0.00715) + | > avg_log_mle: -0.57194 (-0.01451) + | > avg_loss_dur: 0.22592 (+0.00736) + + + > EPOCH: 310/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 02:30:02)  + + --> STEP: 0/234 -- GLOBAL_STEP: 72540 + | > loss: -0.36124 (-0.36124) + | > log_mle: -0.52136 (-0.52136) + | > loss_dur: 0.16013 (0.16013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.11039 (24.11039) + | > current_lr: 0.00008 + | > step_time: 26.99780 (26.99784) + | > loader_time: 3.77790 (3.77786) + + + --> STEP: 5/234 -- GLOBAL_STEP: 72545 + | > loss: -0.33757 (-0.32886) + | > log_mle: -0.43201 (-0.43426) + | > loss_dur: 0.09445 (0.10540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.07189 (22.26206) + | > current_lr: 0.00008 + | > step_time: 3.10530 (4.32236) + | > loader_time: 0.00490 (0.02246) + + + --> STEP: 10/234 -- GLOBAL_STEP: 72550 + | > loss: -0.36556 (-0.34902) + | > log_mle: -0.43877 (-0.43962) + | > loss_dur: 0.07322 (0.09060) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.73507 (19.76344) + | > current_lr: 0.00008 + | > step_time: 1.80680 (3.01584) + | > loader_time: 0.00350 (0.04865) + + + --> STEP: 15/234 -- GLOBAL_STEP: 72555 + | > loss: -0.38255 (-0.36141) + | > log_mle: -0.44891 (-0.44368) + | > loss_dur: 0.06636 (0.08227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.85626 (17.27803) + | > current_lr: 0.00008 + | > step_time: 6.21590 (4.42648) + | > loader_time: 0.00500 (0.08483) + + + --> STEP: 20/234 -- GLOBAL_STEP: 72560 + | > loss: -0.39546 (-0.37006) + | > log_mle: -0.45166 (-0.44561) + | > loss_dur: 0.05620 (0.07555) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.88291 (15.84042) + | > current_lr: 0.00008 + | > step_time: 1.40880 (3.64092) + | > loader_time: 0.07460 (0.06772) + + + --> STEP: 25/234 -- GLOBAL_STEP: 72565 + | > loss: -0.37883 (-0.37209) + | > log_mle: -0.44242 (-0.44528) + | > loss_dur: 0.06360 (0.07319) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.11029 (15.18349) + | > current_lr: 0.00008 + | > step_time: 1.56930 (3.72576) + | > loader_time: 0.00110 (0.07274) + + + --> STEP: 30/234 -- GLOBAL_STEP: 72570 + | > loss: -0.35587 (-0.37185) + | > log_mle: -0.42486 (-0.44402) + | > loss_dur: 0.06900 (0.07217) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.54207 (14.71855) + | > current_lr: 0.00008 + | > step_time: 0.98840 (3.29485) + | > loader_time: 0.00130 (0.06381) + + + --> STEP: 35/234 -- GLOBAL_STEP: 72575 + | > loss: -0.32177 (-0.36846) + | > log_mle: -0.40832 (-0.44088) + | > loss_dur: 0.08655 (0.07243) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.24809 (13.97207) + | > current_lr: 0.00008 + | > step_time: 1.38850 (3.16021) + | > loader_time: 0.00230 (0.05761) + + + --> STEP: 40/234 -- GLOBAL_STEP: 72580 + | > loss: -0.31982 (-0.36440) + | > log_mle: -0.40507 (-0.43745) + | > loss_dur: 0.08525 (0.07305) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.97357 (13.45651) + | > current_lr: 0.00008 + | > step_time: 1.67620 (3.00432) + | > loader_time: 0.00120 (0.05063) + + + --> STEP: 45/234 -- GLOBAL_STEP: 72585 + | > loss: -0.32999 (-0.36184) + | > log_mle: -0.42060 (-0.43509) + | > loss_dur: 0.09061 (0.07325) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.21694 (13.10170) + | > current_lr: 0.00008 + | > step_time: 1.38800 (2.93609) + | > loader_time: 0.00170 (0.04526) + + + --> STEP: 50/234 -- GLOBAL_STEP: 72590 + | > loss: -0.34784 (-0.36120) + | > log_mle: -0.41547 (-0.43407) + | > loss_dur: 0.06762 (0.07287) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.05200 (12.55642) + | > current_lr: 0.00008 + | > step_time: 1.79850 (2.86306) + | > loader_time: 0.00200 (0.04308) + + + --> STEP: 55/234 -- GLOBAL_STEP: 72595 + | > loss: -0.35033 (-0.35892) + | > log_mle: -0.42290 (-0.43249) + | > loss_dur: 0.07257 (0.07356) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.61258 (12.21210) + | > current_lr: 0.00008 + | > step_time: 3.10530 (2.77745) + | > loader_time: 0.00240 (0.04090) + + + --> STEP: 60/234 -- GLOBAL_STEP: 72600 + | > loss: -0.30739 (-0.35632) + | > log_mle: -0.41372 (-0.43084) + | > loss_dur: 0.10634 (0.07452) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.69724 (12.08642) + | > current_lr: 0.00008 + | > step_time: 2.16600 (2.78422) + | > loader_time: 0.08760 (0.04318) + + + --> STEP: 65/234 -- GLOBAL_STEP: 72605 + | > loss: -0.31953 (-0.35287) + | > log_mle: -0.40111 (-0.42873) + | > loss_dur: 0.08158 (0.07586) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67206 (11.99724) + | > current_lr: 0.00008 + | > step_time: 1.41660 (2.70388) + | > loader_time: 0.00180 (0.04141) + + + --> STEP: 70/234 -- GLOBAL_STEP: 72610 + | > loss: -0.27802 (-0.34957) + | > log_mle: -0.38203 (-0.42625) + | > loss_dur: 0.10401 (0.07668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.96525 (11.97524) + | > current_lr: 0.00008 + | > step_time: 1.79210 (2.65260) + | > loader_time: 0.00260 (0.04114) + + + --> STEP: 75/234 -- GLOBAL_STEP: 72615 + | > loss: -0.28722 (-0.34589) + | > log_mle: -0.39550 (-0.42442) + | > loss_dur: 0.10829 (0.07853) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.45218 (12.02634) + | > current_lr: 0.00008 + | > step_time: 4.47890 (2.67226) + | > loader_time: 0.09750 (0.04294) + + + --> STEP: 80/234 -- GLOBAL_STEP: 72620 + | > loss: -0.31073 (-0.34305) + | > log_mle: -0.39227 (-0.42258) + | > loss_dur: 0.08155 (0.07953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.03748 (11.87750) + | > current_lr: 0.00008 + | > step_time: 2.68910 (2.64794) + | > loader_time: 0.00360 (0.04236) + + + --> STEP: 85/234 -- GLOBAL_STEP: 72625 + | > loss: -0.29522 (-0.33996) + | > log_mle: -0.38916 (-0.42094) + | > loss_dur: 0.09394 (0.08097) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.33363 (11.82845) + | > current_lr: 0.00008 + | > step_time: 1.38440 (2.65074) + | > loader_time: 0.00330 (0.04136) + + + --> STEP: 90/234 -- GLOBAL_STEP: 72630 + | > loss: -0.27786 (-0.33721) + | > log_mle: -0.40568 (-0.42017) + | > loss_dur: 0.12782 (0.08296) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.14773 (11.91613) + | > current_lr: 0.00008 + | > step_time: 2.70590 (2.61815) + | > loader_time: 0.00610 (0.04030) + + + --> STEP: 95/234 -- GLOBAL_STEP: 72635 + | > loss: -0.31527 (-0.33522) + | > log_mle: -0.47321 (-0.42096) + | > loss_dur: 0.15793 (0.08574) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.49739 (12.39602) + | > current_lr: 0.00008 + | > step_time: 2.40900 (2.59712) + | > loader_time: 0.09160 (0.03927) + + + --> STEP: 100/234 -- GLOBAL_STEP: 72640 + | > loss: -0.28540 (-0.33276) + | > log_mle: -0.41122 (-0.42043) + | > loss_dur: 0.12583 (0.08767) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.96697 (12.59797) + | > current_lr: 0.00008 + | > step_time: 2.09890 (2.58743) + | > loader_time: 0.08570 (0.04074) + + + --> STEP: 105/234 -- GLOBAL_STEP: 72645 + | > loss: -0.27656 (-0.33082) + | > log_mle: -0.39586 (-0.42104) + | > loss_dur: 0.11930 (0.09022) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.02932 (13.08824) + | > current_lr: 0.00008 + | > step_time: 2.70310 (2.56906) + | > loader_time: 0.00270 (0.04152) + + + --> STEP: 110/234 -- GLOBAL_STEP: 72650 + | > loss: -0.26927 (-0.32809) + | > log_mle: -0.40774 (-0.42114) + | > loss_dur: 0.13848 (0.09305) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.44567 (13.50321) + | > current_lr: 0.00008 + | > step_time: 2.19300 (2.56047) + | > loader_time: 0.00240 (0.04142) + + + --> STEP: 115/234 -- GLOBAL_STEP: 72655 + | > loss: -0.27609 (-0.32648) + | > log_mle: -0.43468 (-0.42228) + | > loss_dur: 0.15860 (0.09581) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.63954 (14.08287) + | > current_lr: 0.00008 + | > step_time: 1.51520 (2.55258) + | > loader_time: 0.09540 (0.04308) + + + --> STEP: 120/234 -- GLOBAL_STEP: 72660 + | > loss: -0.32441 (-0.32487) + | > log_mle: -0.47951 (-0.42328) + | > loss_dur: 0.15510 (0.09840) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.95464 (14.44538) + | > current_lr: 0.00008 + | > step_time: 4.20180 (2.55366) + | > loader_time: 0.09440 (0.04216) + + + --> STEP: 125/234 -- GLOBAL_STEP: 72665 + | > loss: -0.30430 (-0.32287) + | > log_mle: -0.46344 (-0.42311) + | > loss_dur: 0.15914 (0.10024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.06620 (14.62771) + | > current_lr: 0.00008 + | > step_time: 5.88540 (2.58350) + | > loader_time: 0.09920 (0.04512) + + + --> STEP: 130/234 -- GLOBAL_STEP: 72670 + | > loss: -0.31180 (-0.32208) + | > log_mle: -0.48225 (-0.42495) + | > loss_dur: 0.17045 (0.10287) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.88374 (15.17532) + | > current_lr: 0.00008 + | > step_time: 2.20310 (2.58627) + | > loader_time: 0.09070 (0.04419) + + + --> STEP: 135/234 -- GLOBAL_STEP: 72675 + | > loss: -0.27152 (-0.32183) + | > log_mle: -0.40371 (-0.42692) + | > loss_dur: 0.13220 (0.10509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.61543 (15.71667) + | > current_lr: 0.00008 + | > step_time: 2.20760 (2.56611) + | > loader_time: 0.00200 (0.04462) + + + --> STEP: 140/234 -- GLOBAL_STEP: 72680 + | > loss: -0.25699 (-0.32123) + | > log_mle: -0.43594 (-0.42915) + | > loss_dur: 0.17895 (0.10792) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.48679 (16.56583) + | > current_lr: 0.00008 + | > step_time: 1.30750 (2.56107) + | > loader_time: 0.00230 (0.04369) + + + --> STEP: 145/234 -- GLOBAL_STEP: 72685 + | > loss: -0.36338 (-0.32171) + | > log_mle: -0.54804 (-0.43246) + | > loss_dur: 0.18465 (0.11074) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.09771 (17.32975) + | > current_lr: 0.00008 + | > step_time: 0.85480 (2.55405) + | > loader_time: 0.00180 (0.04346) + + + --> STEP: 150/234 -- GLOBAL_STEP: 72690 + | > loss: -0.33197 (-0.32267) + | > log_mle: -0.52761 (-0.43569) + | > loss_dur: 0.19564 (0.11302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.52440 (17.91524) + | > current_lr: 0.00008 + | > step_time: 1.21110 (2.54501) + | > loader_time: 0.07640 (0.04447) + + + --> STEP: 155/234 -- GLOBAL_STEP: 72695 + | > loss: -0.39377 (-0.32434) + | > log_mle: -0.59377 (-0.43982) + | > loss_dur: 0.19999 (0.11548) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.16040 (19.00402) + | > current_lr: 0.00008 + | > step_time: 2.40720 (2.55089) + | > loader_time: 0.00340 (0.04467) + + + --> STEP: 160/234 -- GLOBAL_STEP: 72700 + | > loss: -0.38617 (-0.32558) + | > log_mle: -0.59551 (-0.44346) + | > loss_dur: 0.20933 (0.11788) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.90144 (19.63707) + | > current_lr: 0.00008 + | > step_time: 4.19820 (2.60616) + | > loader_time: 0.00420 (0.04456) + + + --> STEP: 165/234 -- GLOBAL_STEP: 72705 + | > loss: -0.38603 (-0.32679) + | > log_mle: -0.59124 (-0.44691) + | > loss_dur: 0.20521 (0.12012) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.24026 (20.45782) + | > current_lr: 0.00008 + | > step_time: 2.09260 (2.63259) + | > loader_time: 0.00250 (0.04505) + + + --> STEP: 170/234 -- GLOBAL_STEP: 72710 + | > loss: -0.40591 (-0.32855) + | > log_mle: -0.63481 (-0.45098) + | > loss_dur: 0.22890 (0.12243) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.54537 (21.41635) + | > current_lr: 0.00008 + | > step_time: 3.41250 (2.65099) + | > loader_time: 0.00350 (0.04558) + + + --> STEP: 175/234 -- GLOBAL_STEP: 72715 + | > loss: -0.38259 (-0.33105) + | > log_mle: -0.60536 (-0.45583) + | > loss_dur: 0.22277 (0.12479) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.81556 (22.51527) + | > current_lr: 0.00008 + | > step_time: 2.63810 (2.71646) + | > loader_time: 0.00430 (0.04602) + + + --> STEP: 180/234 -- GLOBAL_STEP: 72720 + | > loss: -0.39478 (-0.33299) + | > log_mle: -0.60664 (-0.46031) + | > loss_dur: 0.21186 (0.12732) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.94481 (23.43566) + | > current_lr: 0.00008 + | > step_time: 2.07330 (2.79414) + | > loader_time: 0.00390 (0.04586) + + + --> STEP: 185/234 -- GLOBAL_STEP: 72725 + | > loss: -0.41779 (-0.33464) + | > log_mle: -0.63944 (-0.46441) + | > loss_dur: 0.22165 (0.12976) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.30273 (24.25621) + | > current_lr: 0.00008 + | > step_time: 4.99430 (2.79980) + | > loader_time: 0.00690 (0.04568) + + + --> STEP: 190/234 -- GLOBAL_STEP: 72730 + | > loss: -0.41357 (-0.33647) + | > log_mle: -0.61893 (-0.46845) + | > loss_dur: 0.20535 (0.13198) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.16657 (24.89028) + | > current_lr: 0.00008 + | > step_time: 9.59510 (2.90398) + | > loader_time: 0.10960 (0.04558) + + + --> STEP: 195/234 -- GLOBAL_STEP: 72735 + | > loss: -0.42991 (-0.33913) + | > log_mle: -0.65325 (-0.47305) + | > loss_dur: 0.22334 (0.13392) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.84743 (25.59564) + | > current_lr: 0.00008 + | > step_time: 5.00770 (2.91155) + | > loader_time: 0.00470 (0.04546) + + + --> STEP: 200/234 -- GLOBAL_STEP: 72740 + | > loss: -0.38772 (-0.34089) + | > log_mle: -0.63188 (-0.47701) + | > loss_dur: 0.24416 (0.13612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 83.14863 (26.80116) + | > current_lr: 0.00008 + | > step_time: 5.79920 (2.96876) + | > loader_time: 0.29070 (0.04771) + + + --> STEP: 205/234 -- GLOBAL_STEP: 72745 + | > loss: -0.40632 (-0.34250) + | > log_mle: -0.62867 (-0.48076) + | > loss_dur: 0.22236 (0.13826) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.32893 (27.49300) + | > current_lr: 0.00008 + | > step_time: 4.91750 (2.98003) + | > loader_time: 0.09240 (0.04824) + + + --> STEP: 210/234 -- GLOBAL_STEP: 72750 + | > loss: -0.49166 (-0.34515) + | > log_mle: -0.72241 (-0.48554) + | > loss_dur: 0.23076 (0.14039) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.82518 (28.56318) + | > current_lr: 0.00008 + | > step_time: 4.69100 (3.05517) + | > loader_time: 0.61160 (0.05326) + + + --> STEP: 215/234 -- GLOBAL_STEP: 72755 + | > loss: -0.43379 (-0.34798) + | > log_mle: -0.65942 (-0.49051) + | > loss_dur: 0.22563 (0.14253) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.74194 (29.92130) + | > current_lr: 0.00008 + | > step_time: 7.09430 (3.12191) + | > loader_time: 0.28790 (0.05664) + + + --> STEP: 220/234 -- GLOBAL_STEP: 72760 + | > loss: -0.45301 (-0.35089) + | > log_mle: -0.69383 (-0.49558) + | > loss_dur: 0.24083 (0.14470) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 144.45995 (31.44858) + | > current_lr: 0.00008 + | > step_time: 1.51120 (3.15741) + | > loader_time: 0.00400 (0.05803) + + + --> STEP: 225/234 -- GLOBAL_STEP: 72765 + | > loss: -0.53678 (-0.35369) + | > log_mle: -0.77387 (-0.50037) + | > loss_dur: 0.23709 (0.14668) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.41914 (32.49479) + | > current_lr: 0.00008 + | > step_time: 0.22390 (3.09820) + | > loader_time: 0.00290 (0.05682) + + + --> STEP: 230/234 -- GLOBAL_STEP: 72770 + | > loss: -0.51522 (-0.35611) + | > log_mle: -0.82251 (-0.50556) + | > loss_dur: 0.30729 (0.14946) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.03010 (33.55558) + | > current_lr: 0.00008 + | > step_time: 0.24950 (3.03612) + | > loader_time: 0.00400 (0.05567) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.06579 (-0.73001) + | > avg_loss: -0.30103 (+0.04499) + | > avg_log_mle: -0.55499 (+0.01695) + | > avg_loss_dur: 0.25396 (+0.02804) + + + > EPOCH: 311/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 02:43:10)  + + --> STEP: 1/234 -- GLOBAL_STEP: 72775 + | > loss: -0.32328 (-0.32328) + | > log_mle: -0.42809 (-0.42809) + | > loss_dur: 0.10480 (0.10480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.80199 (15.80199) + | > current_lr: 0.00008 + | > step_time: 8.59970 (8.59973) + | > loader_time: 0.19980 (0.19979) + + + --> STEP: 6/234 -- GLOBAL_STEP: 72780 + | > loss: -0.37190 (-0.33215) + | > log_mle: -0.43600 (-0.43421) + | > loss_dur: 0.06410 (0.10206) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.38216 (17.27964) + | > current_lr: 0.00008 + | > step_time: 4.09810 (6.28404) + | > loader_time: 0.00600 (0.05228) + + + --> STEP: 11/234 -- GLOBAL_STEP: 72785 + | > loss: -0.40149 (-0.35085) + | > log_mle: -0.45950 (-0.44091) + | > loss_dur: 0.05800 (0.09006) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.44775 (16.78305) + | > current_lr: 0.00008 + | > step_time: 4.10690 (4.68184) + | > loader_time: 0.00740 (0.03983) + + + --> STEP: 16/234 -- GLOBAL_STEP: 72790 + | > loss: -0.40114 (-0.36040) + | > log_mle: -0.45971 (-0.44357) + | > loss_dur: 0.05857 (0.08317) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.63258 (16.23598) + | > current_lr: 0.00008 + | > step_time: 3.70660 (4.42240) + | > loader_time: 0.09130 (0.05265) + + + --> STEP: 21/234 -- GLOBAL_STEP: 72795 + | > loss: -0.36500 (-0.36556) + | > log_mle: -0.43118 (-0.44394) + | > loss_dur: 0.06618 (0.07838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.79974 (14.82197) + | > current_lr: 0.00008 + | > step_time: 1.09890 (3.88266) + | > loader_time: 0.00110 (0.04522) + + + --> STEP: 26/234 -- GLOBAL_STEP: 72800 + | > loss: -0.35960 (-0.36786) + | > log_mle: -0.43268 (-0.44392) + | > loss_dur: 0.07308 (0.07605) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.94933 (14.15543) + | > current_lr: 0.00008 + | > step_time: 1.00730 (3.60234) + | > loader_time: 0.00240 (0.04103) + + + --> STEP: 31/234 -- GLOBAL_STEP: 72805 + | > loss: -0.34264 (-0.36846) + | > log_mle: -0.42404 (-0.44305) + | > loss_dur: 0.08140 (0.07458) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.03297 (13.61865) + | > current_lr: 0.00008 + | > step_time: 1.28110 (3.26551) + | > loader_time: 0.00160 (0.03742) + + + --> STEP: 36/234 -- GLOBAL_STEP: 72810 + | > loss: -0.33574 (-0.36553) + | > log_mle: -0.41238 (-0.43957) + | > loss_dur: 0.07664 (0.07404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.32091 (13.78288) + | > current_lr: 0.00008 + | > step_time: 3.21390 (3.12250) + | > loader_time: 0.09000 (0.04000) + + + --> STEP: 41/234 -- GLOBAL_STEP: 72815 + | > loss: -0.36794 (-0.36210) + | > log_mle: -0.42611 (-0.43620) + | > loss_dur: 0.05817 (0.07409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.83532 (13.45920) + | > current_lr: 0.00008 + | > step_time: 1.01640 (2.92029) + | > loader_time: 0.00190 (0.03540) + + + --> STEP: 46/234 -- GLOBAL_STEP: 72820 + | > loss: -0.32446 (-0.35812) + | > log_mle: -0.41013 (-0.43302) + | > loss_dur: 0.08567 (0.07490) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.90527 (13.26818) + | > current_lr: 0.00008 + | > step_time: 1.70290 (2.79656) + | > loader_time: 0.00300 (0.03342) + + + --> STEP: 51/234 -- GLOBAL_STEP: 72825 + | > loss: -0.34877 (-0.35782) + | > log_mle: -0.41998 (-0.43207) + | > loss_dur: 0.07121 (0.07425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.85856 (12.73502) + | > current_lr: 0.00008 + | > step_time: 2.39580 (2.72422) + | > loader_time: 0.00650 (0.03212) + + + --> STEP: 56/234 -- GLOBAL_STEP: 72830 + | > loss: -0.33045 (-0.35532) + | > log_mle: -0.41529 (-0.43040) + | > loss_dur: 0.08484 (0.07509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.37272 (12.33248) + | > current_lr: 0.00008 + | > step_time: 1.50190 (2.68081) + | > loader_time: 0.07870 (0.03409) + + + --> STEP: 61/234 -- GLOBAL_STEP: 72835 + | > loss: -0.31919 (-0.35279) + | > log_mle: -0.40461 (-0.42870) + | > loss_dur: 0.08541 (0.07592) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.34837 (12.08499) + | > current_lr: 0.00008 + | > step_time: 1.92840 (2.71590) + | > loader_time: 0.00200 (0.03612) + + + --> STEP: 66/234 -- GLOBAL_STEP: 72840 + | > loss: -0.32054 (-0.34917) + | > log_mle: -0.39612 (-0.42642) + | > loss_dur: 0.07558 (0.07725) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.85018 (12.14113) + | > current_lr: 0.00008 + | > step_time: 2.00870 (2.70073) + | > loader_time: 0.08640 (0.03743) + + + --> STEP: 71/234 -- GLOBAL_STEP: 72845 + | > loss: -0.28801 (-0.34507) + | > log_mle: -0.40632 (-0.42389) + | > loss_dur: 0.11830 (0.07883) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.49894 (12.31548) + | > current_lr: 0.00008 + | > step_time: 1.79750 (2.65001) + | > loader_time: 0.00240 (0.03635) + + + --> STEP: 76/234 -- GLOBAL_STEP: 72850 + | > loss: -0.29574 (-0.34136) + | > log_mle: -0.38988 (-0.42160) + | > loss_dur: 0.09414 (0.08024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.20135 (12.60209) + | > current_lr: 0.00008 + | > step_time: 1.29000 (2.59296) + | > loader_time: 0.00140 (0.03522) + + + --> STEP: 81/234 -- GLOBAL_STEP: 72855 + | > loss: -0.29738 (-0.33859) + | > log_mle: -0.39897 (-0.41975) + | > loss_dur: 0.10159 (0.08116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.10244 (12.56732) + | > current_lr: 0.00008 + | > step_time: 0.99860 (2.59242) + | > loader_time: 0.00430 (0.03564) + + + --> STEP: 86/234 -- GLOBAL_STEP: 72860 + | > loss: -0.29581 (-0.33568) + | > log_mle: -0.40161 (-0.41809) + | > loss_dur: 0.10579 (0.08241) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67455 (12.55498) + | > current_lr: 0.00008 + | > step_time: 1.59040 (2.57259) + | > loader_time: 0.00230 (0.03586) + + + --> STEP: 91/234 -- GLOBAL_STEP: 72865 + | > loss: -0.28611 (-0.33327) + | > log_mle: -0.40205 (-0.41746) + | > loss_dur: 0.11594 (0.08419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.54676 (12.73969) + | > current_lr: 0.00008 + | > step_time: 1.79670 (2.56561) + | > loader_time: 0.00320 (0.03597) + + + --> STEP: 96/234 -- GLOBAL_STEP: 72870 + | > loss: -0.27915 (-0.33128) + | > log_mle: -0.38936 (-0.41803) + | > loss_dur: 0.11021 (0.08675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.15249 (13.16592) + | > current_lr: 0.00008 + | > step_time: 2.70000 (2.55314) + | > loader_time: 0.00330 (0.03517) + + + --> STEP: 101/234 -- GLOBAL_STEP: 72875 + | > loss: -0.28505 (-0.32886) + | > log_mle: -0.42513 (-0.41784) + | > loss_dur: 0.14008 (0.08897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.63742 (13.51274) + | > current_lr: 0.00008 + | > step_time: 1.65290 (2.52021) + | > loader_time: 0.00200 (0.03356) + + + --> STEP: 106/234 -- GLOBAL_STEP: 72880 + | > loss: -0.25511 (-0.32649) + | > log_mle: -0.42436 (-0.41814) + | > loss_dur: 0.16924 (0.09165) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.84725 (13.96699) + | > current_lr: 0.00008 + | > step_time: 1.82320 (2.51885) + | > loader_time: 0.08000 (0.03283) + + + --> STEP: 111/234 -- GLOBAL_STEP: 72885 + | > loss: -0.30295 (-0.32438) + | > log_mle: -0.48127 (-0.41875) + | > loss_dur: 0.17833 (0.09437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.74326 (14.35843) + | > current_lr: 0.00008 + | > step_time: 2.49920 (2.49356) + | > loader_time: 0.08500 (0.03381) + + + --> STEP: 116/234 -- GLOBAL_STEP: 72890 + | > loss: -0.24532 (-0.32213) + | > log_mle: -0.43911 (-0.41942) + | > loss_dur: 0.19378 (0.09729) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.53070 (15.07422) + | > current_lr: 0.00008 + | > step_time: 7.11320 (2.58831) + | > loader_time: 0.09830 (0.03487) + + + --> STEP: 121/234 -- GLOBAL_STEP: 72895 + | > loss: -0.24892 (-0.32020) + | > log_mle: -0.37081 (-0.41965) + | > loss_dur: 0.12189 (0.09945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.38693 (15.34960) + | > current_lr: 0.00008 + | > step_time: 2.13250 (2.57761) + | > loader_time: 0.00210 (0.03421) + + + --> STEP: 126/234 -- GLOBAL_STEP: 72900 + | > loss: -0.31616 (-0.31863) + | > log_mle: -0.49008 (-0.42043) + | > loss_dur: 0.17392 (0.10181) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.74094 (15.81575) + | > current_lr: 0.00008 + | > step_time: 1.30840 (2.53926) + | > loader_time: 0.08830 (0.03364) + + + --> STEP: 131/234 -- GLOBAL_STEP: 72905 + | > loss: -0.35529 (-0.31807) + | > log_mle: -0.53704 (-0.42232) + | > loss_dur: 0.18175 (0.10425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.06668 (16.45323) + | > current_lr: 0.00008 + | > step_time: 5.39780 (2.56757) + | > loader_time: 0.08620 (0.03374) + + + --> STEP: 136/234 -- GLOBAL_STEP: 72910 + | > loss: -0.38110 (-0.31757) + | > log_mle: -0.58085 (-0.42420) + | > loss_dur: 0.19975 (0.10663) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 74.85634 (17.28623) + | > current_lr: 0.00008 + | > step_time: 3.39080 (2.59293) + | > loader_time: 0.00220 (0.03389) + + + --> STEP: 141/234 -- GLOBAL_STEP: 72915 + | > loss: -0.32085 (-0.31658) + | > log_mle: -0.48771 (-0.42568) + | > loss_dur: 0.16686 (0.10911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.05764 (17.83773) + | > current_lr: 0.00008 + | > step_time: 1.49260 (2.64415) + | > loader_time: 0.00280 (0.03351) + + + --> STEP: 146/234 -- GLOBAL_STEP: 72920 + | > loss: -0.34319 (-0.31730) + | > log_mle: -0.53652 (-0.42918) + | > loss_dur: 0.19334 (0.11188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.77534 (18.57536) + | > current_lr: 0.00008 + | > step_time: 1.68790 (2.61317) + | > loader_time: 0.00170 (0.03352) + + + --> STEP: 151/234 -- GLOBAL_STEP: 72925 + | > loss: -0.32282 (-0.31777) + | > log_mle: -0.49719 (-0.43188) + | > loss_dur: 0.17436 (0.11411) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.17770 (19.32830) + | > current_lr: 0.00008 + | > step_time: 3.11190 (2.63245) + | > loader_time: 0.09260 (0.03375) + + + --> STEP: 156/234 -- GLOBAL_STEP: 72930 + | > loss: -0.35798 (-0.31936) + | > log_mle: -0.55149 (-0.43599) + | > loss_dur: 0.19352 (0.11663) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.56598 (20.66872) + | > current_lr: 0.00008 + | > step_time: 1.45930 (2.61811) + | > loader_time: 0.00230 (0.03386) + + + --> STEP: 161/234 -- GLOBAL_STEP: 72935 + | > loss: -0.39478 (-0.32047) + | > log_mle: -0.57655 (-0.43957) + | > loss_dur: 0.18177 (0.11910) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.04997 (21.39371) + | > current_lr: 0.00008 + | > step_time: 2.68820 (2.61115) + | > loader_time: 0.00750 (0.03406) + + + --> STEP: 166/234 -- GLOBAL_STEP: 72940 + | > loss: -0.33355 (-0.32126) + | > log_mle: -0.50786 (-0.44243) + | > loss_dur: 0.17431 (0.12118) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.71727 (21.98096) + | > current_lr: 0.00008 + | > step_time: 2.00880 (2.61806) + | > loader_time: 0.08680 (0.03482) + + + --> STEP: 171/234 -- GLOBAL_STEP: 72945 + | > loss: -0.38671 (-0.32337) + | > log_mle: -0.60749 (-0.44702) + | > loss_dur: 0.22078 (0.12366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.98483 (22.77501) + | > current_lr: 0.00008 + | > step_time: 1.80180 (2.61527) + | > loader_time: 0.00320 (0.03494) + + + --> STEP: 176/234 -- GLOBAL_STEP: 72950 + | > loss: -0.36329 (-0.32483) + | > log_mle: -0.57128 (-0.45088) + | > loss_dur: 0.20799 (0.12604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.65981 (24.03228) + | > current_lr: 0.00008 + | > step_time: 6.20260 (2.68534) + | > loader_time: 0.08720 (0.03673) + + + --> STEP: 181/234 -- GLOBAL_STEP: 72955 + | > loss: -0.31633 (-0.32597) + | > log_mle: -0.52594 (-0.45457) + | > loss_dur: 0.20961 (0.12860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.47292 (24.35950) + | > current_lr: 0.00008 + | > step_time: 8.20640 (2.73435) + | > loader_time: 0.39080 (0.03844) + + + --> STEP: 186/234 -- GLOBAL_STEP: 72960 + | > loss: -0.33186 (-0.32738) + | > log_mle: -0.56255 (-0.45854) + | > loss_dur: 0.23069 (0.13116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.69862 (25.15363) + | > current_lr: 0.00008 + | > step_time: 2.88400 (2.77524) + | > loader_time: 0.00280 (0.03961) + + + --> STEP: 191/234 -- GLOBAL_STEP: 72965 + | > loss: -0.39307 (-0.32924) + | > log_mle: -0.59593 (-0.46253) + | > loss_dur: 0.20286 (0.13329) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.33021 (25.72085) + | > current_lr: 0.00008 + | > step_time: 6.10930 (2.80745) + | > loader_time: 0.09840 (0.03960) + + + --> STEP: 196/234 -- GLOBAL_STEP: 72970 + | > loss: -0.35713 (-0.33134) + | > log_mle: -0.58246 (-0.46674) + | > loss_dur: 0.22532 (0.13539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.57325 (26.58393) + | > current_lr: 0.00008 + | > step_time: 2.99740 (2.81970) + | > loader_time: 0.00240 (0.04044) + + + --> STEP: 201/234 -- GLOBAL_STEP: 72975 + | > loss: -0.31109 (-0.33290) + | > log_mle: -0.54176 (-0.47042) + | > loss_dur: 0.23067 (0.13752) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.84425 (27.14946) + | > current_lr: 0.00008 + | > step_time: 7.49240 (2.88828) + | > loader_time: 0.00410 (0.04146) + + + --> STEP: 206/234 -- GLOBAL_STEP: 72980 + | > loss: -0.43293 (-0.33527) + | > log_mle: -0.65414 (-0.47476) + | > loss_dur: 0.22122 (0.13948) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.91006 (28.03409) + | > current_lr: 0.00008 + | > step_time: 7.31180 (2.94824) + | > loader_time: 0.00480 (0.04204) + + + --> STEP: 211/234 -- GLOBAL_STEP: 72985 + | > loss: -0.46952 (-0.33776) + | > log_mle: -0.71858 (-0.47948) + | > loss_dur: 0.24906 (0.14171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.47980 (29.15799) + | > current_lr: 0.00008 + | > step_time: 5.10930 (2.99735) + | > loader_time: 0.09820 (0.04434) + + + --> STEP: 216/234 -- GLOBAL_STEP: 72990 + | > loss: -0.47666 (-0.34038) + | > log_mle: -0.71500 (-0.48411) + | > loss_dur: 0.23833 (0.14373) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.94094 (30.11383) + | > current_lr: 0.00008 + | > step_time: 3.90380 (3.06416) + | > loader_time: 0.10390 (0.04514) + + + --> STEP: 221/234 -- GLOBAL_STEP: 72995 + | > loss: -0.41987 (-0.34315) + | > log_mle: -0.63538 (-0.48882) + | > loss_dur: 0.21551 (0.14567) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.13855 (30.91143) + | > current_lr: 0.00008 + | > step_time: 1.70270 (3.07808) + | > loader_time: 0.09080 (0.04580) + + + --> STEP: 226/234 -- GLOBAL_STEP: 73000 + | > loss: -0.49969 (-0.34654) + | > log_mle: -0.75153 (-0.49439) + | > loss_dur: 0.25184 (0.14785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.02740 (31.79875) + | > current_lr: 0.00008 + | > step_time: 0.96040 (3.03610) + | > loader_time: 0.08510 (0.04582) + + + --> STEP: 231/234 -- GLOBAL_STEP: 73005 + | > loss: -0.42940 (-0.34883) + | > log_mle: -0.81534 (-0.50015) + | > loss_dur: 0.38595 (0.15132) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.19102 (32.96687) + | > current_lr: 0.00008 + | > step_time: 0.29530 (2.97654) + | > loader_time: 0.00420 (0.04492) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05147 (-0.01432) + | > avg_loss: -0.35224 (-0.05120) + | > avg_log_mle: -0.56910 (-0.01411) + | > avg_loss_dur: 0.21686 (-0.03709) + + + > EPOCH: 312/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 02:55:47)  + + --> STEP: 2/234 -- GLOBAL_STEP: 73010 + | > loss: -0.37600 (-0.36503) + | > log_mle: -0.45206 (-0.44230) + | > loss_dur: 0.07606 (0.07727) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.14975 (20.18771) + | > current_lr: 0.00008 + | > step_time: 4.70100 (4.20327) + | > loader_time: 0.08280 (0.08519) + + + --> STEP: 7/234 -- GLOBAL_STEP: 73015 + | > loss: -0.36944 (-0.34668) + | > log_mle: -0.43536 (-0.43459) + | > loss_dur: 0.06591 (0.08790) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.38423 (21.15925) + | > current_lr: 0.00008 + | > step_time: 6.30620 (5.27366) + | > loader_time: 0.29230 (0.16395) + + + --> STEP: 12/234 -- GLOBAL_STEP: 73020 + | > loss: -0.35574 (-0.35307) + | > log_mle: -0.43779 (-0.44015) + | > loss_dur: 0.08205 (0.08708) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.77964 (18.92147) + | > current_lr: 0.00008 + | > step_time: 1.27400 (4.35669) + | > loader_time: 0.00150 (0.09663) + + + --> STEP: 17/234 -- GLOBAL_STEP: 73025 + | > loss: -0.39597 (-0.36491) + | > log_mle: -0.45248 (-0.44482) + | > loss_dur: 0.05650 (0.07991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.77713 (17.05103) + | > current_lr: 0.00008 + | > step_time: 1.05280 (3.42118) + | > loader_time: 0.00100 (0.07811) + + + --> STEP: 22/234 -- GLOBAL_STEP: 73030 + | > loss: -0.35955 (-0.36816) + | > log_mle: -0.43315 (-0.44415) + | > loss_dur: 0.07360 (0.07599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.59283 (16.40379) + | > current_lr: 0.00008 + | > step_time: 5.39510 (3.48665) + | > loader_time: 0.00110 (0.07716) + + + --> STEP: 27/234 -- GLOBAL_STEP: 73035 + | > loss: -0.35960 (-0.36957) + | > log_mle: -0.42493 (-0.44301) + | > loss_dur: 0.06533 (0.07344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.49811 (15.46029) + | > current_lr: 0.00008 + | > step_time: 7.60460 (3.56127) + | > loader_time: 0.29440 (0.08095) + + + --> STEP: 32/234 -- GLOBAL_STEP: 73040 + | > loss: -0.36157 (-0.37010) + | > log_mle: -0.42865 (-0.44230) + | > loss_dur: 0.06708 (0.07220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.81142 (14.77275) + | > current_lr: 0.00008 + | > step_time: 4.09920 (3.37951) + | > loader_time: 0.00380 (0.07137) + + + --> STEP: 37/234 -- GLOBAL_STEP: 73045 + | > loss: -0.35167 (-0.36726) + | > log_mle: -0.40966 (-0.43881) + | > loss_dur: 0.05798 (0.07156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.20568 (14.37818) + | > current_lr: 0.00008 + | > step_time: 3.59840 (3.49897) + | > loader_time: 0.00150 (0.06946) + + + --> STEP: 42/234 -- GLOBAL_STEP: 73050 + | > loss: -0.32538 (-0.36378) + | > log_mle: -0.40314 (-0.43568) + | > loss_dur: 0.07776 (0.07190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.89179 (13.95083) + | > current_lr: 0.00008 + | > step_time: 2.39800 (3.26466) + | > loader_time: 0.00210 (0.06318) + + + --> STEP: 47/234 -- GLOBAL_STEP: 73055 + | > loss: -0.34342 (-0.36048) + | > log_mle: -0.42437 (-0.43330) + | > loss_dur: 0.08095 (0.07283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.79984 (13.80284) + | > current_lr: 0.00008 + | > step_time: 1.12030 (3.09942) + | > loader_time: 0.00220 (0.05864) + + + --> STEP: 52/234 -- GLOBAL_STEP: 73060 + | > loss: -0.30949 (-0.35923) + | > log_mle: -0.40606 (-0.43204) + | > loss_dur: 0.09657 (0.07281) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.29077 (13.27535) + | > current_lr: 0.00008 + | > step_time: 5.61600 (3.09407) + | > loader_time: 0.09850 (0.05656) + + + --> STEP: 57/234 -- GLOBAL_STEP: 73065 + | > loss: -0.31648 (-0.35727) + | > log_mle: -0.39938 (-0.43058) + | > loss_dur: 0.08290 (0.07331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.06697 (12.80728) + | > current_lr: 0.00008 + | > step_time: 1.27180 (2.97484) + | > loader_time: 0.00220 (0.05188) + + + --> STEP: 62/234 -- GLOBAL_STEP: 73070 + | > loss: -0.27524 (-0.35412) + | > log_mle: -0.40532 (-0.42895) + | > loss_dur: 0.13008 (0.07482) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.81669 (12.63929) + | > current_lr: 0.00008 + | > step_time: 1.20450 (2.89426) + | > loader_time: 0.08010 (0.05032) + + + --> STEP: 67/234 -- GLOBAL_STEP: 73075 + | > loss: -0.30913 (-0.35136) + | > log_mle: -0.40247 (-0.42662) + | > loss_dur: 0.09333 (0.07527) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.67070 (12.46070) + | > current_lr: 0.00008 + | > step_time: 1.89330 (2.87556) + | > loader_time: 0.00330 (0.05091) + + + --> STEP: 72/234 -- GLOBAL_STEP: 73080 + | > loss: -0.31497 (-0.34752) + | > log_mle: -0.39955 (-0.42434) + | > loss_dur: 0.08459 (0.07683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.94810 (12.32779) + | > current_lr: 0.00008 + | > step_time: 1.68180 (2.85484) + | > loader_time: 0.00140 (0.04950) + + + --> STEP: 77/234 -- GLOBAL_STEP: 73085 + | > loss: -0.29390 (-0.34342) + | > log_mle: -0.39238 (-0.42226) + | > loss_dur: 0.09848 (0.07884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.25992 (12.29242) + | > current_lr: 0.00008 + | > step_time: 2.58460 (2.83569) + | > loader_time: 0.00440 (0.05380) + + + --> STEP: 82/234 -- GLOBAL_STEP: 73090 + | > loss: -0.28937 (-0.34062) + | > log_mle: -0.38927 (-0.42056) + | > loss_dur: 0.09990 (0.07994) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.35453 (12.15198) + | > current_lr: 0.00008 + | > step_time: 4.10470 (2.83039) + | > loader_time: 0.19550 (0.05612) + + + --> STEP: 87/234 -- GLOBAL_STEP: 73095 + | > loss: -0.29020 (-0.33780) + | > log_mle: -0.39218 (-0.41907) + | > loss_dur: 0.10198 (0.08127) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.23066 (12.15147) + | > current_lr: 0.00008 + | > step_time: 1.39990 (2.76907) + | > loader_time: 0.00170 (0.05302) + + + --> STEP: 92/234 -- GLOBAL_STEP: 73100 + | > loss: -0.30848 (-0.33541) + | > log_mle: -0.42756 (-0.41887) + | > loss_dur: 0.11908 (0.08346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.02476 (12.27956) + | > current_lr: 0.00008 + | > step_time: 1.31370 (2.74162) + | > loader_time: 0.07610 (0.05385) + + + --> STEP: 97/234 -- GLOBAL_STEP: 73105 + | > loss: -0.28464 (-0.33365) + | > log_mle: -0.41256 (-0.41959) + | > loss_dur: 0.12792 (0.08594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.00436 (12.54079) + | > current_lr: 0.00008 + | > step_time: 2.70650 (2.71020) + | > loader_time: 0.00420 (0.05217) + + + --> STEP: 102/234 -- GLOBAL_STEP: 73110 + | > loss: -0.26182 (-0.33136) + | > log_mle: -0.39658 (-0.41963) + | > loss_dur: 0.13476 (0.08827) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.05351 (12.71482) + | > current_lr: 0.00008 + | > step_time: 1.99530 (2.71149) + | > loader_time: 0.00370 (0.04979) + + + --> STEP: 107/234 -- GLOBAL_STEP: 73115 + | > loss: -0.27415 (-0.32937) + | > log_mle: -0.42918 (-0.42053) + | > loss_dur: 0.15503 (0.09115) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.92996 (13.08455) + | > current_lr: 0.00008 + | > step_time: 2.10310 (2.70369) + | > loader_time: 0.00220 (0.05003) + + + --> STEP: 112/234 -- GLOBAL_STEP: 73120 + | > loss: -0.28350 (-0.32742) + | > log_mle: -0.44798 (-0.42145) + | > loss_dur: 0.16448 (0.09403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.91010 (13.55040) + | > current_lr: 0.00008 + | > step_time: 3.60330 (2.70053) + | > loader_time: 0.00360 (0.05101) + + + --> STEP: 117/234 -- GLOBAL_STEP: 73125 + | > loss: -0.28314 (-0.32553) + | > log_mle: -0.44380 (-0.42236) + | > loss_dur: 0.16066 (0.09683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.37185 (13.88411) + | > current_lr: 0.00008 + | > step_time: 3.68040 (2.73909) + | > loader_time: 0.08970 (0.05167) + + + --> STEP: 122/234 -- GLOBAL_STEP: 73130 + | > loss: -0.27078 (-0.32374) + | > log_mle: -0.40928 (-0.42248) + | > loss_dur: 0.13850 (0.09875) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.12566 (14.08127) + | > current_lr: 0.00008 + | > step_time: 4.90450 (2.81527) + | > loader_time: 0.10470 (0.05283) + + + --> STEP: 127/234 -- GLOBAL_STEP: 73135 + | > loss: -0.29448 (-0.32238) + | > log_mle: -0.47470 (-0.42374) + | > loss_dur: 0.18023 (0.10136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.53661 (14.78807) + | > current_lr: 0.00008 + | > step_time: 2.60540 (2.75641) + | > loader_time: 0.09290 (0.05221) + + + --> STEP: 132/234 -- GLOBAL_STEP: 73140 + | > loss: -0.29807 (-0.32198) + | > log_mle: -0.45075 (-0.42561) + | > loss_dur: 0.15269 (0.10363) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.59811 (15.48415) + | > current_lr: 0.00008 + | > step_time: 2.49340 (2.80645) + | > loader_time: 0.00220 (0.05167) + + + --> STEP: 137/234 -- GLOBAL_STEP: 73145 + | > loss: -0.28387 (-0.32171) + | > log_mle: -0.46475 (-0.42791) + | > loss_dur: 0.18088 (0.10620) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.77744 (16.25865) + | > current_lr: 0.00008 + | > step_time: 6.51610 (2.87951) + | > loader_time: 0.00610 (0.05122) + + + --> STEP: 142/234 -- GLOBAL_STEP: 73150 + | > loss: -0.31032 (-0.32119) + | > log_mle: -0.47850 (-0.42970) + | > loss_dur: 0.16818 (0.10851) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.16626 (16.97151) + | > current_lr: 0.00008 + | > step_time: 1.99220 (2.86095) + | > loader_time: 0.00300 (0.04953) + + + --> STEP: 147/234 -- GLOBAL_STEP: 73155 + | > loss: -0.29425 (-0.32204) + | > log_mle: -0.47871 (-0.43341) + | > loss_dur: 0.18445 (0.11137) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.00605 (17.78606) + | > current_lr: 0.00008 + | > step_time: 1.19330 (2.87116) + | > loader_time: 0.00160 (0.04912) + + + --> STEP: 152/234 -- GLOBAL_STEP: 73160 + | > loss: -0.36274 (-0.32306) + | > log_mle: -0.56424 (-0.43676) + | > loss_dur: 0.20150 (0.11370) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.58058 (18.59835) + | > current_lr: 0.00008 + | > step_time: 3.90210 (2.85384) + | > loader_time: 0.00210 (0.04825) + + + --> STEP: 157/234 -- GLOBAL_STEP: 73165 + | > loss: -0.31665 (-0.32467) + | > log_mle: -0.50714 (-0.44067) + | > loss_dur: 0.19049 (0.11599) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.78542 (19.50354) + | > current_lr: 0.00008 + | > step_time: 3.62330 (2.86567) + | > loader_time: 0.09440 (0.04842) + + + --> STEP: 162/234 -- GLOBAL_STEP: 73170 + | > loss: -0.35004 (-0.32588) + | > log_mle: -0.53399 (-0.44420) + | > loss_dur: 0.18394 (0.11833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.78399 (20.68404) + | > current_lr: 0.00008 + | > step_time: 2.89890 (2.87853) + | > loader_time: 0.09910 (0.04987) + + + --> STEP: 167/234 -- GLOBAL_STEP: 73175 + | > loss: -0.44044 (-0.32706) + | > log_mle: -0.62606 (-0.44758) + | > loss_dur: 0.18562 (0.12052) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.86798 (21.48692) + | > current_lr: 0.00008 + | > step_time: 2.99160 (2.87820) + | > loader_time: 0.00490 (0.04897) + + + --> STEP: 172/234 -- GLOBAL_STEP: 73180 + | > loss: -0.42106 (-0.32896) + | > log_mle: -0.62573 (-0.45206) + | > loss_dur: 0.20467 (0.12310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.80474 (22.42789) + | > current_lr: 0.00008 + | > step_time: 2.51850 (2.87087) + | > loader_time: 0.09740 (0.05037) + + + --> STEP: 177/234 -- GLOBAL_STEP: 73185 + | > loss: -0.37736 (-0.33075) + | > log_mle: -0.58291 (-0.45629) + | > loss_dur: 0.20556 (0.12555) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.16279 (23.29773) + | > current_lr: 0.00008 + | > step_time: 4.90460 (2.91183) + | > loader_time: 0.09060 (0.05163) + + + --> STEP: 182/234 -- GLOBAL_STEP: 73190 + | > loss: -0.38368 (-0.33236) + | > log_mle: -0.63397 (-0.46059) + | > loss_dur: 0.25029 (0.12823) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.82227 (24.08764) + | > current_lr: 0.00008 + | > step_time: 2.07850 (2.96762) + | > loader_time: 0.00840 (0.05249) + + + --> STEP: 187/234 -- GLOBAL_STEP: 73195 + | > loss: -0.40916 (-0.33424) + | > log_mle: -0.62454 (-0.46477) + | > loss_dur: 0.21538 (0.13053) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.11855 (25.17387) + | > current_lr: 0.00008 + | > step_time: 8.02150 (3.02214) + | > loader_time: 0.09390 (0.05469) + + + --> STEP: 192/234 -- GLOBAL_STEP: 73200 + | > loss: -0.45916 (-0.33644) + | > log_mle: -0.65829 (-0.46888) + | > loss_dur: 0.19913 (0.13244) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.32028 (26.38438) + | > current_lr: 0.00008 + | > step_time: 3.09760 (3.07865) + | > loader_time: 0.00250 (0.05447) + + + --> STEP: 197/234 -- GLOBAL_STEP: 73205 + | > loss: -0.43753 (-0.33865) + | > log_mle: -0.63326 (-0.47298) + | > loss_dur: 0.19573 (0.13432) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.12786 (27.21405) + | > current_lr: 0.00008 + | > step_time: 3.80010 (3.16896) + | > loader_time: 0.09440 (0.05522) + + + --> STEP: 202/234 -- GLOBAL_STEP: 73210 + | > loss: -0.51650 (-0.34066) + | > log_mle: -0.73330 (-0.47720) + | > loss_dur: 0.21681 (0.13655) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.96906 (27.98688) + | > current_lr: 0.00008 + | > step_time: 8.80180 (3.23164) + | > loader_time: 0.29810 (0.05543) + + + --> STEP: 207/234 -- GLOBAL_STEP: 73215 + | > loss: -0.46965 (-0.34283) + | > log_mle: -0.70417 (-0.48142) + | > loss_dur: 0.23453 (0.13860) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.79887 (28.87105) + | > current_lr: 0.00008 + | > step_time: 5.29120 (3.28204) + | > loader_time: 0.00730 (0.05661) + + + --> STEP: 212/234 -- GLOBAL_STEP: 73220 + | > loss: -0.46306 (-0.34553) + | > log_mle: -0.68603 (-0.48633) + | > loss_dur: 0.22297 (0.14080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.32343 (29.69471) + | > current_lr: 0.00008 + | > step_time: 8.30840 (3.35325) + | > loader_time: 0.18460 (0.05706) + + + --> STEP: 217/234 -- GLOBAL_STEP: 73225 + | > loss: -0.48745 (-0.34829) + | > log_mle: -0.72529 (-0.49119) + | > loss_dur: 0.23784 (0.14289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.78471 (30.46880) + | > current_lr: 0.00008 + | > step_time: 2.40150 (3.36040) + | > loader_time: 0.00440 (0.05623) + + + --> STEP: 222/234 -- GLOBAL_STEP: 73230 + | > loss: -0.47393 (-0.35111) + | > log_mle: -0.74230 (-0.49612) + | > loss_dur: 0.26837 (0.14501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.82405 (31.45627) + | > current_lr: 0.00008 + | > step_time: 4.39640 (3.37969) + | > loader_time: 0.10370 (0.05639) + + + --> STEP: 227/234 -- GLOBAL_STEP: 73235 + | > loss: -0.39771 (-0.35357) + | > log_mle: -0.64901 (-0.50071) + | > loss_dur: 0.25130 (0.14714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.97288 (33.15900) + | > current_lr: 0.00008 + | > step_time: 0.26490 (3.32249) + | > loader_time: 0.00410 (0.05558) + + + --> STEP: 232/234 -- GLOBAL_STEP: 73240 + | > loss: -0.36120 (-0.35461) + | > log_mle: -0.84071 (-0.50594) + | > loss_dur: 0.47951 (0.15133) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.16895 (34.21396) + | > current_lr: 0.00008 + | > step_time: 0.34180 (3.25706) + | > loader_time: 0.10240 (0.05492) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.73361 (+0.68214) + | > avg_loss: -0.33541 (+0.01683) + | > avg_log_mle: -0.55908 (+0.01002) + | > avg_loss_dur: 0.22368 (+0.00681) + + + > EPOCH: 313/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 03:09:53)  + + --> STEP: 3/234 -- GLOBAL_STEP: 73245 + | > loss: -0.28786 (-0.33443) + | > log_mle: -0.43245 (-0.43654) + | > loss_dur: 0.14459 (0.10211) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.21447 (16.92903) + | > current_lr: 0.00008 + | > step_time: 11.29480 (8.06614) + | > loader_time: 0.00390 (0.03408) + + + --> STEP: 8/234 -- GLOBAL_STEP: 73250 + | > loss: -0.38394 (-0.34808) + | > log_mle: -0.45980 (-0.43704) + | > loss_dur: 0.07586 (0.08897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.34312 (15.94898) + | > current_lr: 0.00008 + | > step_time: 6.59950 (5.96432) + | > loader_time: 0.09370 (0.03600) + + + --> STEP: 13/234 -- GLOBAL_STEP: 73255 + | > loss: -0.40003 (-0.35626) + | > log_mle: -0.46806 (-0.44161) + | > loss_dur: 0.06803 (0.08536) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.49366 (15.55551) + | > current_lr: 0.00008 + | > step_time: 1.18720 (4.61468) + | > loader_time: 0.00270 (0.04438) + + + --> STEP: 18/234 -- GLOBAL_STEP: 73260 + | > loss: -0.37867 (-0.36569) + | > log_mle: -0.43690 (-0.44411) + | > loss_dur: 0.05823 (0.07842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.61423 (14.44158) + | > current_lr: 0.00008 + | > step_time: 2.90960 (4.02652) + | > loader_time: 0.09540 (0.03774) + + + --> STEP: 23/234 -- GLOBAL_STEP: 73265 + | > loss: -0.40988 (-0.37030) + | > log_mle: -0.46304 (-0.44496) + | > loss_dur: 0.05316 (0.07466) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.94734 (13.65538) + | > current_lr: 0.00008 + | > step_time: 6.52330 (4.17767) + | > loader_time: 0.09770 (0.03463) + + + --> STEP: 28/234 -- GLOBAL_STEP: 73270 + | > loss: -0.42991 (-0.37239) + | > log_mle: -0.47885 (-0.44480) + | > loss_dur: 0.04895 (0.07241) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.28861 (13.07396) + | > current_lr: 0.00008 + | > step_time: 3.70510 (4.03831) + | > loader_time: 0.09070 (0.04185) + + + --> STEP: 33/234 -- GLOBAL_STEP: 73275 + | > loss: -0.36343 (-0.37066) + | > log_mle: -0.42842 (-0.44262) + | > loss_dur: 0.06499 (0.07196) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.99071 (12.44941) + | > current_lr: 0.00008 + | > step_time: 5.52230 (4.15152) + | > loader_time: 0.10200 (0.04471) + + + --> STEP: 38/234 -- GLOBAL_STEP: 73280 + | > loss: -0.35379 (-0.36737) + | > log_mle: -0.42929 (-0.43955) + | > loss_dur: 0.07551 (0.07218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.64363 (12.02387) + | > current_lr: 0.00008 + | > step_time: 0.81540 (3.99513) + | > loader_time: 0.00300 (0.05721) + + + --> STEP: 43/234 -- GLOBAL_STEP: 73285 + | > loss: -0.32932 (-0.36412) + | > log_mle: -0.41408 (-0.43653) + | > loss_dur: 0.08477 (0.07240) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.44045 (11.80287) + | > current_lr: 0.00008 + | > step_time: 1.61660 (3.70668) + | > loader_time: 0.00120 (0.05263) + + + --> STEP: 48/234 -- GLOBAL_STEP: 73290 + | > loss: -0.36850 (-0.36207) + | > log_mle: -0.43502 (-0.43478) + | > loss_dur: 0.06652 (0.07271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.44629 (11.70178) + | > current_lr: 0.00008 + | > step_time: 1.90680 (3.53769) + | > loader_time: 0.00150 (0.04918) + + + --> STEP: 53/234 -- GLOBAL_STEP: 73295 + | > loss: -0.34416 (-0.36008) + | > log_mle: -0.42373 (-0.43316) + | > loss_dur: 0.07956 (0.07307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.60655 (11.73788) + | > current_lr: 0.00008 + | > step_time: 1.38310 (3.34427) + | > loader_time: 0.00130 (0.04482) + + + --> STEP: 58/234 -- GLOBAL_STEP: 73300 + | > loss: -0.35633 (-0.35827) + | > log_mle: -0.42217 (-0.43168) + | > loss_dur: 0.06584 (0.07341) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.45572 (11.46091) + | > current_lr: 0.00008 + | > step_time: 2.80250 (3.20205) + | > loader_time: 0.00240 (0.04127) + + + --> STEP: 63/234 -- GLOBAL_STEP: 73305 + | > loss: -0.30058 (-0.35409) + | > log_mle: -0.39039 (-0.42952) + | > loss_dur: 0.08981 (0.07543) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.30054 (11.39784) + | > current_lr: 0.00008 + | > step_time: 1.59370 (3.10723) + | > loader_time: 0.00270 (0.03959) + + + --> STEP: 68/234 -- GLOBAL_STEP: 73310 + | > loss: -0.29265 (-0.35118) + | > log_mle: -0.39146 (-0.42729) + | > loss_dur: 0.09881 (0.07610) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67673 (11.20004) + | > current_lr: 0.00008 + | > step_time: 2.11980 (3.10882) + | > loader_time: 0.18310 (0.04331) + + + --> STEP: 73/234 -- GLOBAL_STEP: 73315 + | > loss: -0.27428 (-0.34737) + | > log_mle: -0.39538 (-0.42516) + | > loss_dur: 0.12110 (0.07779) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.52866 (11.06281) + | > current_lr: 0.00008 + | > step_time: 2.01250 (3.07122) + | > loader_time: 0.00530 (0.04416) + + + --> STEP: 78/234 -- GLOBAL_STEP: 73320 + | > loss: -0.28896 (-0.34376) + | > log_mle: -0.38701 (-0.42296) + | > loss_dur: 0.09805 (0.07920) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.26056 (11.00163) + | > current_lr: 0.00008 + | > step_time: 1.61190 (3.00447) + | > loader_time: 0.00360 (0.04474) + + + --> STEP: 83/234 -- GLOBAL_STEP: 73325 + | > loss: -0.26073 (-0.34064) + | > log_mle: -0.39254 (-0.42127) + | > loss_dur: 0.13180 (0.08063) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.17303 (11.03167) + | > current_lr: 0.00008 + | > step_time: 1.40570 (2.95626) + | > loader_time: 0.00240 (0.04433) + + + --> STEP: 88/234 -- GLOBAL_STEP: 73330 + | > loss: -0.29977 (-0.33827) + | > log_mle: -0.42731 (-0.42022) + | > loss_dur: 0.12754 (0.08195) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.92733 (10.99911) + | > current_lr: 0.00008 + | > step_time: 3.51230 (2.94756) + | > loader_time: 0.07750 (0.04408) + + + --> STEP: 93/234 -- GLOBAL_STEP: 73335 + | > loss: -0.30367 (-0.33587) + | > log_mle: -0.44143 (-0.42018) + | > loss_dur: 0.13776 (0.08431) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.80321 (11.15740) + | > current_lr: 0.00008 + | > step_time: 1.64250 (2.91390) + | > loader_time: 0.00200 (0.04277) + + + --> STEP: 98/234 -- GLOBAL_STEP: 73340 + | > loss: -0.28328 (-0.33392) + | > log_mle: -0.38158 (-0.42018) + | > loss_dur: 0.09829 (0.08626) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.55141 (11.37840) + | > current_lr: 0.00008 + | > step_time: 1.30250 (2.85664) + | > loader_time: 0.05120 (0.04300) + + + --> STEP: 103/234 -- GLOBAL_STEP: 73345 + | > loss: -0.30388 (-0.33177) + | > log_mle: -0.45976 (-0.42082) + | > loss_dur: 0.15588 (0.08905) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.96211 (11.84735) + | > current_lr: 0.00008 + | > step_time: 1.39820 (2.80723) + | > loader_time: 0.00260 (0.04189) + + + --> STEP: 108/234 -- GLOBAL_STEP: 73350 + | > loss: -0.27961 (-0.32985) + | > log_mle: -0.40815 (-0.42116) + | > loss_dur: 0.12855 (0.09130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.45634 (12.25410) + | > current_lr: 0.00008 + | > step_time: 2.70650 (2.82529) + | > loader_time: 0.00300 (0.04260) + + + --> STEP: 113/234 -- GLOBAL_STEP: 73355 + | > loss: -0.29899 (-0.32792) + | > log_mle: -0.45194 (-0.42234) + | > loss_dur: 0.15295 (0.09442) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.11021 (12.83618) + | > current_lr: 0.00008 + | > step_time: 1.68820 (2.80623) + | > loader_time: 0.00200 (0.04163) + + + --> STEP: 118/234 -- GLOBAL_STEP: 73360 + | > loss: -0.27484 (-0.32608) + | > log_mle: -0.42721 (-0.42302) + | > loss_dur: 0.15237 (0.09694) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.89422 (13.13455) + | > current_lr: 0.00008 + | > step_time: 2.27770 (2.78889) + | > loader_time: 0.00330 (0.04141) + + + --> STEP: 123/234 -- GLOBAL_STEP: 73365 + | > loss: -0.24714 (-0.32388) + | > log_mle: -0.39289 (-0.42281) + | > loss_dur: 0.14575 (0.09893) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.40409 (13.32138) + | > current_lr: 0.00008 + | > step_time: 3.09690 (2.79188) + | > loader_time: 0.00560 (0.04137) + + + --> STEP: 128/234 -- GLOBAL_STEP: 73370 + | > loss: -0.30311 (-0.32313) + | > log_mle: -0.44563 (-0.42455) + | > loss_dur: 0.14252 (0.10142) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.13085 (13.87260) + | > current_lr: 0.00008 + | > step_time: 1.21910 (2.77118) + | > loader_time: 0.08110 (0.04051) + + + --> STEP: 133/234 -- GLOBAL_STEP: 73375 + | > loss: -0.32426 (-0.32294) + | > log_mle: -0.48622 (-0.42683) + | > loss_dur: 0.16196 (0.10388) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 25.07233 (14.48766) + | > current_lr: 0.00008 + | > step_time: 4.89880 (2.88262) + | > loader_time: 0.08120 (0.04117) + + + --> STEP: 138/234 -- GLOBAL_STEP: 73380 + | > loss: -0.26910 (-0.32229) + | > log_mle: -0.42653 (-0.42875) + | > loss_dur: 0.15743 (0.10646) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.66869 (15.27659) + | > current_lr: 0.00008 + | > step_time: 3.20230 (2.87525) + | > loader_time: 0.09170 (0.04100) + + + --> STEP: 143/234 -- GLOBAL_STEP: 73385 + | > loss: -0.34093 (-0.32222) + | > log_mle: -0.56793 (-0.43154) + | > loss_dur: 0.22699 (0.10932) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.65079 (16.20302) + | > current_lr: 0.00008 + | > step_time: 1.59920 (2.86675) + | > loader_time: 0.00270 (0.04035) + + + --> STEP: 148/234 -- GLOBAL_STEP: 73390 + | > loss: -0.32603 (-0.32281) + | > log_mle: -0.48788 (-0.43456) + | > loss_dur: 0.16184 (0.11175) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.38333 (16.74241) + | > current_lr: 0.00008 + | > step_time: 2.28850 (2.82934) + | > loader_time: 0.00810 (0.03971) + + + --> STEP: 153/234 -- GLOBAL_STEP: 73395 + | > loss: -0.40541 (-0.32421) + | > log_mle: -0.59056 (-0.43859) + | > loss_dur: 0.18514 (0.11438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.44333 (17.96306) + | > current_lr: 0.00008 + | > step_time: 4.69950 (2.82713) + | > loader_time: 0.00340 (0.04085) + + + --> STEP: 158/234 -- GLOBAL_STEP: 73400 + | > loss: -0.33432 (-0.32507) + | > log_mle: -0.54227 (-0.44195) + | > loss_dur: 0.20796 (0.11687) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.01391 (18.62939) + | > current_lr: 0.00008 + | > step_time: 1.29390 (2.82573) + | > loader_time: 0.03040 (0.04155) + + + --> STEP: 163/234 -- GLOBAL_STEP: 73405 + | > loss: -0.32944 (-0.32629) + | > log_mle: -0.51485 (-0.44550) + | > loss_dur: 0.18541 (0.11921) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.05220 (19.38918) + | > current_lr: 0.00008 + | > step_time: 5.28600 (2.88565) + | > loader_time: 0.10900 (0.04278) + + + --> STEP: 168/234 -- GLOBAL_STEP: 73410 + | > loss: -0.36343 (-0.32778) + | > log_mle: -0.57578 (-0.44935) + | > loss_dur: 0.21235 (0.12157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.57255 (20.17459) + | > current_lr: 0.00008 + | > step_time: 1.39680 (2.86212) + | > loader_time: 0.00190 (0.04270) + + + --> STEP: 173/234 -- GLOBAL_STEP: 73415 + | > loss: -0.37300 (-0.32976) + | > log_mle: -0.58512 (-0.45395) + | > loss_dur: 0.21212 (0.12418) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.53645 (20.88602) + | > current_lr: 0.00008 + | > step_time: 2.00250 (2.86851) + | > loader_time: 0.07000 (0.04198) + + + --> STEP: 178/234 -- GLOBAL_STEP: 73420 + | > loss: -0.42454 (-0.33171) + | > log_mle: -0.65136 (-0.45835) + | > loss_dur: 0.22681 (0.12664) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.27344 (22.22868) + | > current_lr: 0.00008 + | > step_time: 1.19420 (2.87970) + | > loader_time: 0.00240 (0.04144) + + + --> STEP: 183/234 -- GLOBAL_STEP: 73425 + | > loss: -0.43900 (-0.33338) + | > log_mle: -0.64341 (-0.46253) + | > loss_dur: 0.20442 (0.12914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.33477 (23.09233) + | > current_lr: 0.00008 + | > step_time: 5.20030 (2.89922) + | > loader_time: 0.09370 (0.04094) + + + --> STEP: 188/234 -- GLOBAL_STEP: 73430 + | > loss: -0.44449 (-0.33522) + | > log_mle: -0.65718 (-0.46678) + | > loss_dur: 0.21269 (0.13156) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.06780 (23.81307) + | > current_lr: 0.00008 + | > step_time: 1.59930 (2.87750) + | > loader_time: 0.00470 (0.04093) + + + --> STEP: 193/234 -- GLOBAL_STEP: 73435 + | > loss: -0.45941 (-0.33755) + | > log_mle: -0.66779 (-0.47106) + | > loss_dur: 0.20838 (0.13351) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.71615 (24.44053) + | > current_lr: 0.00008 + | > step_time: 4.81100 (2.89164) + | > loader_time: 0.00310 (0.04042) + + + --> STEP: 198/234 -- GLOBAL_STEP: 73440 + | > loss: -0.43058 (-0.33963) + | > log_mle: -0.65364 (-0.47515) + | > loss_dur: 0.22305 (0.13552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.77648 (25.14198) + | > current_lr: 0.00008 + | > step_time: 5.69990 (2.96204) + | > loader_time: 0.10340 (0.04085) + + + --> STEP: 203/234 -- GLOBAL_STEP: 73445 + | > loss: -0.36285 (-0.34137) + | > log_mle: -0.57765 (-0.47901) + | > loss_dur: 0.21480 (0.13764) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.56201 (25.99528) + | > current_lr: 0.00008 + | > step_time: 2.77340 (3.03760) + | > loader_time: 0.08820 (0.04631) + + + --> STEP: 208/234 -- GLOBAL_STEP: 73450 + | > loss: -0.44144 (-0.34399) + | > log_mle: -0.67435 (-0.48384) + | > loss_dur: 0.23290 (0.13985) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.47064 (26.82828) + | > current_lr: 0.00008 + | > step_time: 11.38180 (3.17226) + | > loader_time: 0.00290 (0.04929) + + + --> STEP: 213/234 -- GLOBAL_STEP: 73455 + | > loss: -0.48434 (-0.34690) + | > log_mle: -0.72037 (-0.48894) + | > loss_dur: 0.23602 (0.14204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.51566 (27.84922) + | > current_lr: 0.00008 + | > step_time: 8.89610 (3.25935) + | > loader_time: 0.00710 (0.05047) + + + --> STEP: 218/234 -- GLOBAL_STEP: 73460 + | > loss: -0.44699 (-0.34960) + | > log_mle: -0.68100 (-0.49369) + | > loss_dur: 0.23401 (0.14409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.57761 (28.82717) + | > current_lr: 0.00008 + | > step_time: 2.29710 (3.29047) + | > loader_time: 0.00260 (0.05032) + + + --> STEP: 223/234 -- GLOBAL_STEP: 73465 + | > loss: -0.49485 (-0.35259) + | > log_mle: -0.72597 (-0.49877) + | > loss_dur: 0.23112 (0.14618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.06785 (29.89731) + | > current_lr: 0.00008 + | > step_time: 0.94470 (3.25821) + | > loader_time: 0.06890 (0.05035) + + + --> STEP: 228/234 -- GLOBAL_STEP: 73470 + | > loss: -0.45466 (-0.35569) + | > log_mle: -0.72230 (-0.50410) + | > loss_dur: 0.26764 (0.14841) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.30922 (30.75788) + | > current_lr: 0.00008 + | > step_time: 0.24040 (3.19209) + | > loader_time: 0.00350 (0.04931) + + + --> STEP: 233/234 -- GLOBAL_STEP: 73475 + | > loss: -0.04679 (-0.35663) + | > log_mle: -0.69662 (-0.51074) + | > loss_dur: 0.64984 (0.15411) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.70483 (32.10075) + | > current_lr: 0.00008 + | > step_time: 0.18590 (3.12914) + | > loader_time: 0.00250 (0.04858) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.08081 (-0.65280) + | > avg_loss: -0.34642 (-0.01101) + | > avg_log_mle: -0.57310 (-0.01402) + | > avg_loss_dur: 0.22668 (+0.00301) + + + > EPOCH: 314/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 03:23:18)  + + --> STEP: 4/234 -- GLOBAL_STEP: 73480 + | > loss: -0.36076 (-0.34491) + | > log_mle: -0.43352 (-0.43798) + | > loss_dur: 0.07276 (0.09307) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.04461 (19.40863) + | > current_lr: 0.00008 + | > step_time: 3.09820 (3.92210) + | > loader_time: 0.29430 (1.12477) + + + --> STEP: 9/234 -- GLOBAL_STEP: 73485 + | > loss: -0.34967 (-0.35509) + | > log_mle: -0.44914 (-0.44299) + | > loss_dur: 0.09947 (0.08790) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.25266 (17.40624) + | > current_lr: 0.00008 + | > step_time: 7.30120 (5.38395) + | > loader_time: 0.10660 (0.51372) + + + --> STEP: 14/234 -- GLOBAL_STEP: 73490 + | > loss: -0.38743 (-0.36616) + | > log_mle: -0.45083 (-0.44679) + | > loss_dur: 0.06340 (0.08063) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.30351 (17.47894) + | > current_lr: 0.00008 + | > step_time: 3.00000 (4.60460) + | > loader_time: 0.00740 (0.34972) + + + --> STEP: 19/234 -- GLOBAL_STEP: 73495 + | > loss: -0.39423 (-0.37287) + | > log_mle: -0.45623 (-0.44867) + | > loss_dur: 0.06200 (0.07580) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.08879 (16.35484) + | > current_lr: 0.00008 + | > step_time: 2.91090 (4.55717) + | > loader_time: 0.00150 (0.27321) + + + --> STEP: 24/234 -- GLOBAL_STEP: 73500 + | > loss: -0.38313 (-0.37455) + | > log_mle: -0.44406 (-0.44846) + | > loss_dur: 0.06093 (0.07391) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.18113 (15.38228) + | > current_lr: 0.00008 + | > step_time: 1.26210 (4.01396) + | > loader_time: 0.00220 (0.21688) + + + --> STEP: 29/234 -- GLOBAL_STEP: 73505 + | > loss: -0.36987 (-0.37610) + | > log_mle: -0.43607 (-0.44797) + | > loss_dur: 0.06620 (0.07187) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.90661 (14.41672) + | > current_lr: 0.00008 + | > step_time: 2.25630 (3.66614) + | > loader_time: 0.00180 (0.17988) + + + --> STEP: 34/234 -- GLOBAL_STEP: 73510 + | > loss: -0.35461 (-0.37380) + | > log_mle: -0.42526 (-0.44560) + | > loss_dur: 0.07065 (0.07180) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.51668 (13.85240) + | > current_lr: 0.00008 + | > step_time: 3.71350 (3.44372) + | > loader_time: 0.09340 (0.16142) + + + --> STEP: 39/234 -- GLOBAL_STEP: 73515 + | > loss: -0.34181 (-0.36978) + | > log_mle: -0.41203 (-0.44208) + | > loss_dur: 0.07022 (0.07230) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.72143 (13.53664) + | > current_lr: 0.00008 + | > step_time: 1.83890 (3.35444) + | > loader_time: 0.00560 (0.14295) + + + --> STEP: 44/234 -- GLOBAL_STEP: 73520 + | > loss: -0.34515 (-0.36664) + | > log_mle: -0.41125 (-0.43906) + | > loss_dur: 0.06610 (0.07242) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.37502 (13.02746) + | > current_lr: 0.00008 + | > step_time: 4.20550 (3.27445) + | > loader_time: 0.10660 (0.13111) + + + --> STEP: 49/234 -- GLOBAL_STEP: 73525 + | > loss: -0.36432 (-0.36499) + | > log_mle: -0.42953 (-0.43779) + | > loss_dur: 0.06522 (0.07280) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.62878 (12.58471) + | > current_lr: 0.00008 + | > step_time: 0.98180 (3.12755) + | > loader_time: 0.00150 (0.11794) + + + --> STEP: 54/234 -- GLOBAL_STEP: 73530 + | > loss: -0.34964 (-0.36286) + | > log_mle: -0.41754 (-0.43596) + | > loss_dur: 0.06790 (0.07310) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.59963 (12.32828) + | > current_lr: 0.00008 + | > step_time: 3.23360 (3.08363) + | > loader_time: 0.00180 (0.10745) + + + --> STEP: 59/234 -- GLOBAL_STEP: 73535 + | > loss: -0.33726 (-0.36143) + | > log_mle: -0.41608 (-0.43461) + | > loss_dur: 0.07883 (0.07318) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.41872 (12.01968) + | > current_lr: 0.00008 + | > step_time: 2.22510 (3.04326) + | > loader_time: 0.00180 (0.10021) + + + --> STEP: 64/234 -- GLOBAL_STEP: 73540 + | > loss: -0.32675 (-0.35769) + | > log_mle: -0.40714 (-0.43257) + | > loss_dur: 0.08039 (0.07488) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.05756 (11.88330) + | > current_lr: 0.00008 + | > step_time: 1.50950 (3.00193) + | > loader_time: 0.00450 (0.09402) + + + --> STEP: 69/234 -- GLOBAL_STEP: 73545 + | > loss: -0.32396 (-0.35470) + | > log_mle: -0.39810 (-0.43010) + | > loss_dur: 0.07414 (0.07540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.64225 (11.85608) + | > current_lr: 0.00008 + | > step_time: 1.31800 (2.90786) + | > loader_time: 0.10140 (0.08889) + + + --> STEP: 74/234 -- GLOBAL_STEP: 73550 + | > loss: -0.29273 (-0.35050) + | > log_mle: -0.38869 (-0.42786) + | > loss_dur: 0.09596 (0.07735) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.89586 (11.87997) + | > current_lr: 0.00008 + | > step_time: 1.28480 (2.87008) + | > loader_time: 0.00210 (0.08520) + + + --> STEP: 79/234 -- GLOBAL_STEP: 73555 + | > loss: -0.31526 (-0.34721) + | > log_mle: -0.40544 (-0.42598) + | > loss_dur: 0.09018 (0.07876) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.72129 (11.75476) + | > current_lr: 0.00008 + | > step_time: 1.29970 (2.86526) + | > loader_time: 0.00240 (0.08347) + + + --> STEP: 84/234 -- GLOBAL_STEP: 73560 + | > loss: -0.30622 (-0.34421) + | > log_mle: -0.39674 (-0.42420) + | > loss_dur: 0.09053 (0.07999) + | > amp_scaler: 2048.00000 (1072.76190) + | > grad_norm: 8.39418 (11.66782) + | > current_lr: 0.00008 + | > step_time: 1.63230 (2.81168) + | > loader_time: 0.00200 (0.07955) + + + --> STEP: 89/234 -- GLOBAL_STEP: 73565 + | > loss: -0.30200 (-0.34146) + | > log_mle: -0.41316 (-0.42300) + | > loss_dur: 0.11116 (0.08154) + | > amp_scaler: 2048.00000 (1127.55056) + | > grad_norm: 20.82291 (12.09658) + | > current_lr: 0.00008 + | > step_time: 2.01710 (2.76546) + | > loader_time: 0.09130 (0.07715) + + + --> STEP: 94/234 -- GLOBAL_STEP: 73570 + | > loss: -0.30124 (-0.33887) + | > log_mle: -0.42623 (-0.42281) + | > loss_dur: 0.12499 (0.08394) + | > amp_scaler: 2048.00000 (1176.51064) + | > grad_norm: 26.48359 (12.68865) + | > current_lr: 0.00008 + | > step_time: 1.87630 (2.78822) + | > loader_time: 0.00230 (0.07329) + + + --> STEP: 99/234 -- GLOBAL_STEP: 73575 + | > loss: -0.30359 (-0.33666) + | > log_mle: -0.46743 (-0.42299) + | > loss_dur: 0.16384 (0.08633) + | > amp_scaler: 2048.00000 (1220.52525) + | > grad_norm: 28.64696 (13.10781) + | > current_lr: 0.00008 + | > step_time: 2.53890 (2.72568) + | > loader_time: 0.00210 (0.07058) + + + --> STEP: 104/234 -- GLOBAL_STEP: 73580 + | > loss: -0.32493 (-0.33474) + | > log_mle: -0.47539 (-0.42369) + | > loss_dur: 0.15047 (0.08895) + | > amp_scaler: 2048.00000 (1260.30769) + | > grad_norm: 31.17957 (13.51420) + | > current_lr: 0.00008 + | > step_time: 2.19580 (2.73331) + | > loader_time: 0.00240 (0.06965) + + + --> STEP: 109/234 -- GLOBAL_STEP: 73585 + | > loss: -0.27124 (-0.33195) + | > log_mle: -0.44350 (-0.42353) + | > loss_dur: 0.17226 (0.09158) + | > amp_scaler: 2048.00000 (1296.44037) + | > grad_norm: 36.29523 (13.99242) + | > current_lr: 0.00008 + | > step_time: 1.00890 (2.70265) + | > loader_time: 0.00220 (0.06823) + + + --> STEP: 114/234 -- GLOBAL_STEP: 73590 + | > loss: -0.29233 (-0.33027) + | > log_mle: -0.42489 (-0.42433) + | > loss_dur: 0.13256 (0.09406) + | > amp_scaler: 2048.00000 (1329.40351) + | > grad_norm: 21.68866 (14.49625) + | > current_lr: 0.00008 + | > step_time: 1.30020 (2.66120) + | > loader_time: 0.00410 (0.06539) + + + --> STEP: 119/234 -- GLOBAL_STEP: 73595 + | > loss: -0.29032 (-0.32813) + | > log_mle: -0.43093 (-0.42484) + | > loss_dur: 0.14061 (0.09671) + | > amp_scaler: 2048.00000 (1359.59664) + | > grad_norm: 18.29635 (14.73945) + | > current_lr: 0.00008 + | > step_time: 2.50360 (2.63547) + | > loader_time: 0.08830 (0.06423) + + + --> STEP: 124/234 -- GLOBAL_STEP: 73600 + | > loss: -0.30736 (-0.32611) + | > log_mle: -0.45536 (-0.42474) + | > loss_dur: 0.14800 (0.09862) + | > amp_scaler: 2048.00000 (1387.35484) + | > grad_norm: 19.66582 (14.83779) + | > current_lr: 0.00008 + | > step_time: 6.49900 (2.66069) + | > loader_time: 0.09870 (0.06388) + + + --> STEP: 129/234 -- GLOBAL_STEP: 73605 + | > loss: -0.27984 (-0.32508) + | > log_mle: -0.44866 (-0.42628) + | > loss_dur: 0.16882 (0.10120) + | > amp_scaler: 2048.00000 (1412.96124) + | > grad_norm: 25.76934 (15.38967) + | > current_lr: 0.00008 + | > step_time: 2.98830 (2.66837) + | > loader_time: 0.00300 (0.06270) + + + --> STEP: 134/234 -- GLOBAL_STEP: 73610 + | > loss: -0.31063 (-0.32474) + | > log_mle: -0.49330 (-0.42850) + | > loss_dur: 0.18267 (0.10376) + | > amp_scaler: 2048.00000 (1436.65672) + | > grad_norm: 30.50309 (16.22692) + | > current_lr: 0.00008 + | > step_time: 1.40370 (2.69127) + | > loader_time: 0.08140 (0.06301) + + + --> STEP: 139/234 -- GLOBAL_STEP: 73615 + | > loss: -0.36128 (-0.32420) + | > log_mle: -0.55098 (-0.43039) + | > loss_dur: 0.18970 (0.10619) + | > amp_scaler: 2048.00000 (1458.64748) + | > grad_norm: 47.04651 (17.09720) + | > current_lr: 0.00008 + | > step_time: 1.31530 (2.67015) + | > loader_time: 0.08450 (0.06325) + + + --> STEP: 144/234 -- GLOBAL_STEP: 73620 + | > loss: -0.33822 (-0.32383) + | > log_mle: -0.53517 (-0.43285) + | > loss_dur: 0.19695 (0.10902) + | > amp_scaler: 2048.00000 (1479.11111) + | > grad_norm: 31.38337 (17.73896) + | > current_lr: 0.00008 + | > step_time: 3.80480 (2.69618) + | > loader_time: 0.09390 (0.06449) + + + --> STEP: 149/234 -- GLOBAL_STEP: 73625 + | > loss: -0.40158 (-0.32461) + | > log_mle: -0.58883 (-0.43595) + | > loss_dur: 0.18726 (0.11134) + | > amp_scaler: 2048.00000 (1498.20134) + | > grad_norm: 51.32043 (18.60183) + | > current_lr: 0.00008 + | > step_time: 1.41290 (2.73815) + | > loader_time: 0.08940 (0.06418) + + + --> STEP: 154/234 -- GLOBAL_STEP: 73630 + | > loss: -0.36003 (-0.32606) + | > log_mle: -0.55039 (-0.43986) + | > loss_dur: 0.19036 (0.11380) + | > amp_scaler: 2048.00000 (1516.05195) + | > grad_norm: 34.43119 (19.25988) + | > current_lr: 0.00008 + | > step_time: 5.51360 (2.84943) + | > loader_time: 0.48490 (0.06851) + + + --> STEP: 159/234 -- GLOBAL_STEP: 73635 + | > loss: -0.38184 (-0.32738) + | > log_mle: -0.56524 (-0.44357) + | > loss_dur: 0.18339 (0.11619) + | > amp_scaler: 2048.00000 (1532.77987) + | > grad_norm: 39.23301 (19.94301) + | > current_lr: 0.00008 + | > step_time: 1.40270 (2.84612) + | > loader_time: 0.00380 (0.07078) + + + --> STEP: 164/234 -- GLOBAL_STEP: 73640 + | > loss: -0.34055 (-0.32847) + | > log_mle: -0.53829 (-0.44697) + | > loss_dur: 0.19774 (0.11851) + | > amp_scaler: 2048.00000 (1548.48780) + | > grad_norm: 73.75333 (20.81949) + | > current_lr: 0.00008 + | > step_time: 4.91290 (2.89958) + | > loader_time: 0.08640 (0.07103) + + + --> STEP: 169/234 -- GLOBAL_STEP: 73645 + | > loss: -0.34275 (-0.32975) + | > log_mle: -0.54571 (-0.45058) + | > loss_dur: 0.20296 (0.12083) + | > amp_scaler: 2048.00000 (1563.26627) + | > grad_norm: 56.07784 (21.84264) + | > current_lr: 0.00008 + | > step_time: 2.28130 (2.92062) + | > loader_time: 0.00330 (0.07019) + + + --> STEP: 174/234 -- GLOBAL_STEP: 73650 + | > loss: -0.44256 (-0.33207) + | > log_mle: -0.64419 (-0.45542) + | > loss_dur: 0.20163 (0.12335) + | > amp_scaler: 2048.00000 (1577.19540) + | > grad_norm: 56.17088 (22.76144) + | > current_lr: 0.00008 + | > step_time: 1.61730 (2.90036) + | > loader_time: 0.19320 (0.06960) + + + --> STEP: 179/234 -- GLOBAL_STEP: 73655 + | > loss: -0.41648 (-0.33395) + | > log_mle: -0.65130 (-0.45991) + | > loss_dur: 0.23482 (0.12596) + | > amp_scaler: 2048.00000 (1590.34637) + | > grad_norm: 42.98640 (23.38753) + | > current_lr: 0.00008 + | > step_time: 3.89900 (3.02549) + | > loader_time: 0.18910 (0.07046) + + + --> STEP: 184/234 -- GLOBAL_STEP: 73660 + | > loss: -0.39029 (-0.33550) + | > log_mle: -0.60506 (-0.46384) + | > loss_dur: 0.21476 (0.12834) + | > amp_scaler: 2048.00000 (1602.78261) + | > grad_norm: 43.29564 (24.10852) + | > current_lr: 0.00008 + | > step_time: 5.70850 (3.07049) + | > loader_time: 0.18240 (0.07109) + + + --> STEP: 189/234 -- GLOBAL_STEP: 73665 + | > loss: -0.39026 (-0.33732) + | > log_mle: -0.59739 (-0.46790) + | > loss_dur: 0.20713 (0.13057) + | > amp_scaler: 2048.00000 (1614.56085) + | > grad_norm: 52.18969 (25.20041) + | > current_lr: 0.00008 + | > step_time: 4.86520 (3.08325) + | > loader_time: 0.11390 (0.06999) + + + --> STEP: 194/234 -- GLOBAL_STEP: 73670 + | > loss: -0.42644 (-0.33962) + | > log_mle: -0.63508 (-0.47222) + | > loss_dur: 0.20863 (0.13261) + | > amp_scaler: 2048.00000 (1625.73196) + | > grad_norm: 52.26713 (25.80396) + | > current_lr: 0.00008 + | > step_time: 3.21050 (3.10337) + | > loader_time: 0.08850 (0.06964) + + + --> STEP: 199/234 -- GLOBAL_STEP: 73675 + | > loss: -0.43363 (-0.34151) + | > log_mle: -0.64646 (-0.47624) + | > loss_dur: 0.21283 (0.13472) + | > amp_scaler: 2048.00000 (1636.34171) + | > grad_norm: 51.02195 (26.42237) + | > current_lr: 0.00008 + | > step_time: 2.27660 (3.09403) + | > loader_time: 0.11640 (0.06901) + + + --> STEP: 204/234 -- GLOBAL_STEP: 73680 + | > loss: -0.45723 (-0.34331) + | > log_mle: -0.69643 (-0.48022) + | > loss_dur: 0.23920 (0.13690) + | > amp_scaler: 2048.00000 (1646.43137) + | > grad_norm: 58.04054 (26.93724) + | > current_lr: 0.00008 + | > step_time: 5.71340 (3.14078) + | > loader_time: 0.09720 (0.06829) + + + --> STEP: 209/234 -- GLOBAL_STEP: 73685 + | > loss: -0.43173 (-0.34572) + | > log_mle: -0.65110 (-0.48468) + | > loss_dur: 0.21937 (0.13896) + | > amp_scaler: 2048.00000 (1656.03828) + | > grad_norm: 56.53981 (27.58332) + | > current_lr: 0.00008 + | > step_time: 7.81090 (3.23281) + | > loader_time: 0.09490 (0.06988) + + + --> STEP: 214/234 -- GLOBAL_STEP: 73690 + | > loss: -0.48818 (-0.34911) + | > log_mle: -0.68972 (-0.49013) + | > loss_dur: 0.20155 (0.14102) + | > amp_scaler: 2048.00000 (1665.19626) + | > grad_norm: 62.31773 (28.53065) + | > current_lr: 0.00008 + | > step_time: 6.40610 (3.24943) + | > loader_time: 0.00390 (0.07010) + + + --> STEP: 219/234 -- GLOBAL_STEP: 73695 + | > loss: -0.56671 (-0.35226) + | > log_mle: -0.79802 (-0.49543) + | > loss_dur: 0.23131 (0.14317) + | > amp_scaler: 2048.00000 (1673.93607) + | > grad_norm: 81.50710 (29.78872) + | > current_lr: 0.00008 + | > step_time: 2.80080 (3.29362) + | > loader_time: 0.00400 (0.06995) + + + --> STEP: 224/234 -- GLOBAL_STEP: 73700 + | > loss: -0.50244 (-0.35513) + | > log_mle: -0.73129 (-0.50038) + | > loss_dur: 0.22885 (0.14524) + | > amp_scaler: 2048.00000 (1682.28571) + | > grad_norm: 91.77081 (30.96999) + | > current_lr: 0.00008 + | > step_time: 0.21800 (3.24490) + | > loader_time: 0.00370 (0.06848) + + + --> STEP: 229/234 -- GLOBAL_STEP: 73705 + | > loss: -0.50381 (-0.35824) + | > log_mle: -0.79508 (-0.50597) + | > loss_dur: 0.29127 (0.14773) + | > amp_scaler: 2048.00000 (1690.27074) + | > grad_norm: 71.57069 (32.17478) + | > current_lr: 0.00008 + | > step_time: 0.24270 (3.17916) + | > loader_time: 0.00410 (0.06706) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.97519 (+0.89438) + | > avg_loss: -0.34538 (+0.00104) + | > avg_log_mle: -0.57155 (+0.00155) + | > avg_loss_dur: 0.22617 (-0.00051) + + + > EPOCH: 315/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 03:36:49)  + + --> STEP: 0/234 -- GLOBAL_STEP: 73710 + | > loss: -0.36847 (-0.36847) + | > log_mle: -0.52294 (-0.52294) + | > loss_dur: 0.15447 (0.15447) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 31.54991 (31.54991) + | > current_lr: 0.00008 + | > step_time: 5.20260 (5.20262) + | > loader_time: 22.86980 (22.86979) + + + --> STEP: 5/234 -- GLOBAL_STEP: 73715 + | > loss: -0.33700 (-0.34263) + | > log_mle: -0.43433 (-0.43853) + | > loss_dur: 0.09734 (0.09590) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.21569 (18.20362) + | > current_lr: 0.00008 + | > step_time: 7.09430 (5.30216) + | > loader_time: 0.09850 (0.05727) + + + --> STEP: 10/234 -- GLOBAL_STEP: 73720 + | > loss: -0.35428 (-0.35438) + | > log_mle: -0.43818 (-0.44314) + | > loss_dur: 0.08390 (0.08876) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.63196 (17.20132) + | > current_lr: 0.00008 + | > step_time: 2.40960 (3.24371) + | > loader_time: 0.00210 (0.02956) + + + --> STEP: 15/234 -- GLOBAL_STEP: 73725 + | > loss: -0.39767 (-0.36741) + | > log_mle: -0.45764 (-0.44717) + | > loss_dur: 0.05996 (0.07976) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.79571 (16.00953) + | > current_lr: 0.00008 + | > step_time: 1.60570 (3.67797) + | > loader_time: 0.09450 (0.05210) + + + --> STEP: 20/234 -- GLOBAL_STEP: 73730 + | > loss: -0.40383 (-0.37447) + | > log_mle: -0.45931 (-0.44949) + | > loss_dur: 0.05549 (0.07503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.93704 (14.38717) + | > current_lr: 0.00008 + | > step_time: 2.41280 (3.85606) + | > loader_time: 0.08860 (0.04852) + + + --> STEP: 25/234 -- GLOBAL_STEP: 73735 + | > loss: -0.38891 (-0.37606) + | > log_mle: -0.44850 (-0.44927) + | > loss_dur: 0.05958 (0.07321) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.39230 (13.65353) + | > current_lr: 0.00008 + | > step_time: 1.33420 (4.22337) + | > loader_time: 0.00170 (0.04702) + + + --> STEP: 30/234 -- GLOBAL_STEP: 73740 + | > loss: -0.37065 (-0.37697) + | > log_mle: -0.43671 (-0.44854) + | > loss_dur: 0.06606 (0.07157) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 10.62967 (13.25264) + | > current_lr: 0.00008 + | > step_time: 1.74110 (3.82546) + | > loader_time: 0.00250 (0.04196) + + + --> STEP: 35/234 -- GLOBAL_STEP: 73745 + | > loss: -0.33116 (-0.37383) + | > log_mle: -0.41355 (-0.44560) + | > loss_dur: 0.08240 (0.07176) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 20.03557 (13.10819) + | > current_lr: 0.00008 + | > step_time: 1.10300 (3.48850) + | > loader_time: 0.07670 (0.04051) + + + --> STEP: 40/234 -- GLOBAL_STEP: 73750 + | > loss: -0.32655 (-0.36975) + | > log_mle: -0.41298 (-0.44227) + | > loss_dur: 0.08644 (0.07252) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 7.35540 (12.74058) + | > current_lr: 0.00008 + | > step_time: 1.41390 (3.36770) + | > loader_time: 0.08280 (0.04215) + + + --> STEP: 45/234 -- GLOBAL_STEP: 73755 + | > loss: -0.33882 (-0.36759) + | > log_mle: -0.42423 (-0.43985) + | > loss_dur: 0.08541 (0.07226) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.95020 (12.38935) + | > current_lr: 0.00008 + | > step_time: 2.30750 (3.22746) + | > loader_time: 0.07590 (0.04140) + + + --> STEP: 50/234 -- GLOBAL_STEP: 73760 + | > loss: -0.34982 (-0.36666) + | > log_mle: -0.42093 (-0.43866) + | > loss_dur: 0.07111 (0.07200) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.31590 (12.09118) + | > current_lr: 0.00008 + | > step_time: 1.81120 (3.06909) + | > loader_time: 0.08710 (0.04431) + + + --> STEP: 55/234 -- GLOBAL_STEP: 73765 + | > loss: -0.36638 (-0.36406) + | > log_mle: -0.42564 (-0.43675) + | > loss_dur: 0.05926 (0.07269) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.58494 (12.19790) + | > current_lr: 0.00008 + | > step_time: 1.29250 (2.89568) + | > loader_time: 0.00180 (0.04046) + + + --> STEP: 60/234 -- GLOBAL_STEP: 73770 + | > loss: -0.31449 (-0.36147) + | > log_mle: -0.41747 (-0.43509) + | > loss_dur: 0.10298 (0.07363) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.51511 (12.18823) + | > current_lr: 0.00008 + | > step_time: 1.93460 (2.84516) + | > loader_time: 0.00280 (0.03737) + + + --> STEP: 65/234 -- GLOBAL_STEP: 73775 + | > loss: -0.31839 (-0.35771) + | > log_mle: -0.40135 (-0.43274) + | > loss_dur: 0.08296 (0.07503) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 8.67272 (12.39738) + | > current_lr: 0.00008 + | > step_time: 4.60570 (2.80998) + | > loader_time: 0.00220 (0.03468) + + + --> STEP: 70/234 -- GLOBAL_STEP: 73780 + | > loss: -0.28352 (-0.35419) + | > log_mle: -0.38491 (-0.43017) + | > loss_dur: 0.10139 (0.07597) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 12.47701 (12.18796) + | > current_lr: 0.00008 + | > step_time: 2.40030 (2.76750) + | > loader_time: 0.00200 (0.03509) + + + --> STEP: 75/234 -- GLOBAL_STEP: 73785 + | > loss: -0.29061 (-0.35010) + | > log_mle: -0.39811 (-0.42814) + | > loss_dur: 0.10750 (0.07804) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.86732 (12.18002) + | > current_lr: 0.00008 + | > step_time: 1.22250 (2.70695) + | > loader_time: 0.08180 (0.03396) + + + --> STEP: 80/234 -- GLOBAL_STEP: 73790 + | > loss: -0.31187 (-0.34707) + | > log_mle: -0.39471 (-0.42622) + | > loss_dur: 0.08285 (0.07914) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 9.11131 (12.09410) + | > current_lr: 0.00008 + | > step_time: 1.72070 (2.64944) + | > loader_time: 0.07750 (0.03395) + + + --> STEP: 85/234 -- GLOBAL_STEP: 73795 + | > loss: -0.29604 (-0.34393) + | > log_mle: -0.39039 (-0.42449) + | > loss_dur: 0.09435 (0.08056) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 15.17448 (12.11294) + | > current_lr: 0.00008 + | > step_time: 1.60630 (2.61799) + | > loader_time: 0.00240 (0.03209) + + + --> STEP: 90/234 -- GLOBAL_STEP: 73800 + | > loss: -0.28527 (-0.34129) + | > log_mle: -0.40921 (-0.42370) + | > loss_dur: 0.12394 (0.08241) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.02539 (12.27296) + | > current_lr: 0.00008 + | > step_time: 1.99260 (2.57354) + | > loader_time: 0.00310 (0.03046) + + + --> STEP: 95/234 -- GLOBAL_STEP: 73805 + | > loss: -0.33744 (-0.33962) + | > log_mle: -0.48794 (-0.42463) + | > loss_dur: 0.15050 (0.08500) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 39.73625 (12.91845) + | > current_lr: 0.00008 + | > step_time: 2.50780 (2.56898) + | > loader_time: 0.00210 (0.03198) + + + --> STEP: 100/234 -- GLOBAL_STEP: 73810 + | > loss: -0.29966 (-0.33735) + | > log_mle: -0.41680 (-0.42412) + | > loss_dur: 0.11715 (0.08677) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 13.41201 (13.17187) + | > current_lr: 0.00008 + | > step_time: 2.20990 (2.52119) + | > loader_time: 0.00300 (0.03049) + + + --> STEP: 105/234 -- GLOBAL_STEP: 73815 + | > loss: -0.28402 (-0.33563) + | > log_mle: -0.39825 (-0.42470) + | > loss_dur: 0.11423 (0.08906) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 11.89116 (13.80355) + | > current_lr: 0.00008 + | > step_time: 2.21230 (2.50790) + | > loader_time: 0.07420 (0.03149) + + + --> STEP: 110/234 -- GLOBAL_STEP: 73820 + | > loss: -0.27417 (-0.33307) + | > log_mle: -0.41254 (-0.42487) + | > loss_dur: 0.13836 (0.09181) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 18.66618 (14.21445) + | > current_lr: 0.00008 + | > step_time: 1.97890 (2.49721) + | > loader_time: 0.00200 (0.03027) + + + --> STEP: 115/234 -- GLOBAL_STEP: 73825 + | > loss: -0.27926 (-0.33129) + | > log_mle: -0.43467 (-0.42599) + | > loss_dur: 0.15541 (0.09470) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 16.00812 (14.85993) + | > current_lr: 0.00008 + | > step_time: 1.90650 (2.47221) + | > loader_time: 0.00430 (0.03058) + + + --> STEP: 120/234 -- GLOBAL_STEP: 73830 + | > loss: -0.32257 (-0.32963) + | > log_mle: -0.48312 (-0.42691) + | > loss_dur: 0.16055 (0.09728) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.87059 (15.17887) + | > current_lr: 0.00008 + | > step_time: 3.89890 (2.49063) + | > loader_time: 0.10960 (0.03099) + + + --> STEP: 125/234 -- GLOBAL_STEP: 73835 + | > loss: -0.30204 (-0.32741) + | > log_mle: -0.46598 (-0.42665) + | > loss_dur: 0.16393 (0.09924) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.40246 (15.34971) + | > current_lr: 0.00008 + | > step_time: 1.30620 (2.50289) + | > loader_time: 0.08810 (0.03272) + + + --> STEP: 130/234 -- GLOBAL_STEP: 73840 + | > loss: -0.32184 (-0.32658) + | > log_mle: -0.48256 (-0.42831) + | > loss_dur: 0.16072 (0.10173) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 24.81131 (15.98372) + | > current_lr: 0.00008 + | > step_time: 0.85740 (2.45905) + | > loader_time: 0.00210 (0.03224) + + + --> STEP: 135/234 -- GLOBAL_STEP: 73845 + | > loss: -0.27331 (-0.32613) + | > log_mle: -0.40557 (-0.43015) + | > loss_dur: 0.13225 (0.10402) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 22.21381 (16.58931) + | > current_lr: 0.00008 + | > step_time: 2.89070 (2.45821) + | > loader_time: 0.10020 (0.03254) + + + --> STEP: 140/234 -- GLOBAL_STEP: 73850 + | > loss: -0.26927 (-0.32575) + | > log_mle: -0.43917 (-0.43254) + | > loss_dur: 0.16990 (0.10679) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 26.14900 (17.37176) + | > current_lr: 0.00008 + | > step_time: 6.90210 (2.51828) + | > loader_time: 0.00210 (0.03292) + + + --> STEP: 145/234 -- GLOBAL_STEP: 73855 + | > loss: -0.37775 (-0.32634) + | > log_mle: -0.54832 (-0.43579) + | > loss_dur: 0.17057 (0.10944) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 40.02299 (18.25924) + | > current_lr: 0.00008 + | > step_time: 3.29970 (2.61692) + | > loader_time: 0.00740 (0.03524) + + + --> STEP: 150/234 -- GLOBAL_STEP: 73860 + | > loss: -0.32496 (-0.32697) + | > log_mle: -0.52721 (-0.43888) + | > loss_dur: 0.20225 (0.11191) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.53016 (18.96070) + | > current_lr: 0.00008 + | > step_time: 1.89550 (2.65488) + | > loader_time: 0.00260 (0.03674) + + + --> STEP: 155/234 -- GLOBAL_STEP: 73865 + | > loss: -0.40548 (-0.32893) + | > log_mle: -0.59869 (-0.44316) + | > loss_dur: 0.19320 (0.11424) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 65.10464 (19.89594) + | > current_lr: 0.00008 + | > step_time: 4.29910 (2.64153) + | > loader_time: 0.00680 (0.03629) + + + --> STEP: 160/234 -- GLOBAL_STEP: 73870 + | > loss: -0.38745 (-0.32998) + | > log_mle: -0.59252 (-0.44669) + | > loss_dur: 0.20507 (0.11672) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 44.87564 (20.72070) + | > current_lr: 0.00008 + | > step_time: 3.03210 (2.62850) + | > loader_time: 0.08410 (0.03628) + + + --> STEP: 165/234 -- GLOBAL_STEP: 73875 + | > loss: -0.40149 (-0.33111) + | > log_mle: -0.59622 (-0.45010) + | > loss_dur: 0.19473 (0.11899) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.32197 (21.50697) + | > current_lr: 0.00008 + | > step_time: 2.09390 (2.63773) + | > loader_time: 0.00830 (0.03715) + + + --> STEP: 170/234 -- GLOBAL_STEP: 73880 + | > loss: -0.41931 (-0.33290) + | > log_mle: -0.64256 (-0.45423) + | > loss_dur: 0.22325 (0.12133) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 48.93592 (22.27837) + | > current_lr: 0.00008 + | > step_time: 1.41820 (2.66248) + | > loader_time: 0.00360 (0.03679) + + + --> STEP: 175/234 -- GLOBAL_STEP: 73885 + | > loss: -0.40195 (-0.33533) + | > log_mle: -0.61712 (-0.45925) + | > loss_dur: 0.21517 (0.12392) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 42.17414 (23.14157) + | > current_lr: 0.00008 + | > step_time: 3.10060 (2.65789) + | > loader_time: 0.00450 (0.03677) + + + --> STEP: 180/234 -- GLOBAL_STEP: 73890 + | > loss: -0.34831 (-0.33676) + | > log_mle: -0.55054 (-0.46319) + | > loss_dur: 0.20223 (0.12643) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 61.88853 (24.85733) + | > current_lr: 0.00008 + | > step_time: 4.20180 (2.67961) + | > loader_time: 0.08510 (0.03793) + + + --> STEP: 185/234 -- GLOBAL_STEP: 73895 + | > loss: -0.36993 (-0.33692) + | > log_mle: -0.58759 (-0.46592) + | > loss_dur: 0.21766 (0.12900) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 35.72610 (25.50418) + | > current_lr: 0.00008 + | > step_time: 7.90690 (2.79864) + | > loader_time: 0.10580 (0.04115) + + + --> STEP: 190/234 -- GLOBAL_STEP: 73900 + | > loss: -0.36542 (-0.33748) + | > log_mle: -0.57770 (-0.46903) + | > loss_dur: 0.21228 (0.13156) + | > amp_scaler: 2048.00000 (2048.00000) + | > grad_norm: 79.45359 (26.19722) + | > current_lr: 0.00008 + | > step_time: 3.18900 (2.79287) + | > loader_time: 0.28710 (0.04311) + + + --> STEP: 195/234 -- GLOBAL_STEP: 73905 + | > loss: -0.37855 (-0.33889) + | > log_mle: -0.60542 (-0.47252) + | > loss_dur: 0.22686 (0.13364) + | > amp_scaler: 1024.00000 (2042.74872) + | > grad_norm: 0.00000 (26.95200) + | > current_lr: 0.00008 + | > step_time: 1.60150 (2.83048) + | > loader_time: 0.00310 (0.04347) + + + --> STEP: 200/234 -- GLOBAL_STEP: 73910 + | > loss: -0.35871 (-0.33968) + | > log_mle: -0.59726 (-0.47550) + | > loss_dur: 0.23856 (0.13582) + | > amp_scaler: 1024.00000 (2017.28000) + | > grad_norm: 58.74253 (28.27488) + | > current_lr: 0.00008 + | > step_time: 8.09560 (2.89021) + | > loader_time: 0.19800 (0.04387) + + + --> STEP: 205/234 -- GLOBAL_STEP: 73915 + | > loss: -0.38392 (-0.34087) + | > log_mle: -0.60635 (-0.47883) + | > loss_dur: 0.22244 (0.13796) + | > amp_scaler: 1024.00000 (1993.05366) + | > grad_norm: 53.56310 (28.65137) + | > current_lr: 0.00008 + | > step_time: 3.80620 (2.94276) + | > loader_time: 0.08350 (0.04746) + + + --> STEP: 210/234 -- GLOBAL_STEP: 73920 + | > loss: -0.45180 (-0.34301) + | > log_mle: -0.69286 (-0.48322) + | > loss_dur: 0.24106 (0.14021) + | > amp_scaler: 1024.00000 (1969.98095) + | > grad_norm: 67.59234 (29.24393) + | > current_lr: 0.00008 + | > step_time: 3.00080 (3.04974) + | > loader_time: 0.00340 (0.04922) + + + --> STEP: 215/234 -- GLOBAL_STEP: 73925 + | > loss: -0.42727 (-0.34573) + | > log_mle: -0.65051 (-0.48806) + | > loss_dur: 0.22325 (0.14233) + | > amp_scaler: 1024.00000 (1947.98140) + | > grad_norm: 58.71003 (30.04782) + | > current_lr: 0.00008 + | > step_time: 11.12660 (3.08958) + | > loader_time: 0.21230 (0.04961) + + + --> STEP: 220/234 -- GLOBAL_STEP: 73930 + | > loss: -0.46386 (-0.34879) + | > log_mle: -0.70500 (-0.49324) + | > loss_dur: 0.24115 (0.14445) + | > amp_scaler: 1024.00000 (1926.98182) + | > grad_norm: 66.78088 (31.15362) + | > current_lr: 0.00008 + | > step_time: 2.09230 (3.09377) + | > loader_time: 0.00270 (0.04952) + + + --> STEP: 225/234 -- GLOBAL_STEP: 73935 + | > loss: -0.52066 (-0.35158) + | > log_mle: -0.77464 (-0.49818) + | > loss_dur: 0.25398 (0.14660) + | > amp_scaler: 1024.00000 (1906.91556) + | > grad_norm: 91.34446 (32.04459) + | > current_lr: 0.00008 + | > step_time: 0.22980 (3.04666) + | > loader_time: 0.00370 (0.04850) + + + --> STEP: 230/234 -- GLOBAL_STEP: 73940 + | > loss: -0.53291 (-0.35433) + | > log_mle: -0.84241 (-0.50373) + | > loss_dur: 0.30950 (0.14940) + | > amp_scaler: 1024.00000 (1887.72174) + | > grad_norm: 90.38590 (33.00454) + | > current_lr: 0.00008 + | > step_time: 0.25590 (2.98579) + | > loader_time: 0.00460 (0.04754) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.09489 (-0.88030) + | > avg_loss: -0.33955 (+0.00583) + | > avg_log_mle: -0.56119 (+0.01036) + | > avg_loss_dur: 0.22164 (-0.00453) + + + > EPOCH: 316/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 03:49:43)  + + --> STEP: 1/234 -- GLOBAL_STEP: 73945 + | > loss: -0.34196 (-0.34196) + | > log_mle: -0.43277 (-0.43277) + | > loss_dur: 0.09081 (0.09081) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.42978 (20.42978) + | > current_lr: 0.00008 + | > step_time: 2.99530 (2.99535) + | > loader_time: 3.10530 (3.10530) + + + --> STEP: 6/234 -- GLOBAL_STEP: 73950 + | > loss: -0.37053 (-0.33974) + | > log_mle: -0.44181 (-0.43854) + | > loss_dur: 0.07128 (0.09881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.77554 (17.15478) + | > current_lr: 0.00008 + | > step_time: 3.99530 (5.13379) + | > loader_time: 0.00160 (0.56459) + + + --> STEP: 11/234 -- GLOBAL_STEP: 73955 + | > loss: -0.40638 (-0.35637) + | > log_mle: -0.46338 (-0.44551) + | > loss_dur: 0.05700 (0.08915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.93457 (16.41011) + | > current_lr: 0.00008 + | > step_time: 1.59680 (4.97270) + | > loader_time: 0.00110 (0.31721) + + + --> STEP: 16/234 -- GLOBAL_STEP: 73960 + | > loss: -0.40492 (-0.36686) + | > log_mle: -0.46566 (-0.44980) + | > loss_dur: 0.06074 (0.08293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.50294 (15.06745) + | > current_lr: 0.00008 + | > step_time: 2.40170 (4.38970) + | > loader_time: 0.09980 (0.26013) + + + --> STEP: 21/234 -- GLOBAL_STEP: 73965 + | > loss: -0.37219 (-0.37330) + | > log_mle: -0.43499 (-0.45012) + | > loss_dur: 0.06280 (0.07681) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.20457 (14.57385) + | > current_lr: 0.00008 + | > step_time: 1.90350 (4.17334) + | > loader_time: 0.00180 (0.19878) + + + --> STEP: 26/234 -- GLOBAL_STEP: 73970 + | > loss: -0.36955 (-0.37553) + | > log_mle: -0.43790 (-0.45002) + | > loss_dur: 0.06835 (0.07449) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.45765 (13.94269) + | > current_lr: 0.00008 + | > step_time: 1.39850 (4.10180) + | > loader_time: 0.00420 (0.17224) + + + --> STEP: 31/234 -- GLOBAL_STEP: 73975 + | > loss: -0.34510 (-0.37603) + | > log_mle: -0.43111 (-0.44921) + | > loss_dur: 0.08601 (0.07317) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.67639 (13.23063) + | > current_lr: 0.00008 + | > step_time: 8.50750 (4.56659) + | > loader_time: 0.10100 (0.15393) + + + --> STEP: 36/234 -- GLOBAL_STEP: 73980 + | > loss: -0.35163 (-0.37298) + | > log_mle: -0.42574 (-0.44621) + | > loss_dur: 0.07411 (0.07323) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.85136 (12.98448) + | > current_lr: 0.00008 + | > step_time: 1.30900 (4.29317) + | > loader_time: 0.00470 (0.13520) + + + --> STEP: 41/234 -- GLOBAL_STEP: 73985 + | > loss: -0.37289 (-0.37066) + | > log_mle: -0.43724 (-0.44361) + | > loss_dur: 0.06435 (0.07295) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.85408 (12.46904) + | > current_lr: 0.00008 + | > step_time: 1.98950 (3.96596) + | > loader_time: 0.09450 (0.12125) + + + --> STEP: 46/234 -- GLOBAL_STEP: 73990 + | > loss: -0.33485 (-0.36728) + | > log_mle: -0.41806 (-0.44074) + | > loss_dur: 0.08321 (0.07345) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.77214 (12.43782) + | > current_lr: 0.00008 + | > step_time: 1.16560 (3.67582) + | > loader_time: 0.00140 (0.10825) + + + --> STEP: 51/234 -- GLOBAL_STEP: 73995 + | > loss: -0.34390 (-0.36624) + | > log_mle: -0.41858 (-0.43936) + | > loss_dur: 0.07468 (0.07312) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.37341 (12.01495) + | > current_lr: 0.00008 + | > step_time: 3.17300 (3.55410) + | > loader_time: 0.00280 (0.10148) + + + --> STEP: 56/234 -- GLOBAL_STEP: 74000 + | > loss: -0.34610 (-0.36400) + | > log_mle: -0.42277 (-0.43760) + | > loss_dur: 0.07667 (0.07360) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.08215 (11.81377) + | > current_lr: 0.00008 + | > step_time: 2.70250 (3.36764) + | > loader_time: 0.08760 (0.09570) + + + --> STEP: 61/234 -- GLOBAL_STEP: 74005 + | > loss: -0.33760 (-0.36149) + | > log_mle: -0.41091 (-0.43568) + | > loss_dur: 0.07331 (0.07419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.63515 (11.73184) + | > current_lr: 0.00008 + | > step_time: 2.67380 (3.32883) + | > loader_time: 0.00210 (0.09116) + + + --> STEP: 66/234 -- GLOBAL_STEP: 74010 + | > loss: -0.33688 (-0.35804) + | > log_mle: -0.40439 (-0.43343) + | > loss_dur: 0.06751 (0.07539) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.28026 (11.64718) + | > current_lr: 0.00008 + | > step_time: 2.52020 (3.26791) + | > loader_time: 0.07670 (0.08565) + + + --> STEP: 71/234 -- GLOBAL_STEP: 74015 + | > loss: -0.30579 (-0.35437) + | > log_mle: -0.41710 (-0.43121) + | > loss_dur: 0.11131 (0.07684) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.93958 (11.55117) + | > current_lr: 0.00008 + | > step_time: 2.88330 (3.20404) + | > loader_time: 0.00360 (0.07983) + + + --> STEP: 76/234 -- GLOBAL_STEP: 74020 + | > loss: -0.30495 (-0.35074) + | > log_mle: -0.40221 (-0.42914) + | > loss_dur: 0.09726 (0.07840) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.48543 (11.53819) + | > current_lr: 0.00008 + | > step_time: 1.22700 (3.11540) + | > loader_time: 0.00180 (0.07477) + + + --> STEP: 81/234 -- GLOBAL_STEP: 74025 + | > loss: -0.30461 (-0.34801) + | > log_mle: -0.40428 (-0.42731) + | > loss_dur: 0.09967 (0.07931) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.75895 (11.54993) + | > current_lr: 0.00008 + | > step_time: 1.36630 (3.02374) + | > loader_time: 0.00410 (0.07031) + + + --> STEP: 86/234 -- GLOBAL_STEP: 74030 + | > loss: -0.30566 (-0.34486) + | > log_mle: -0.40632 (-0.42552) + | > loss_dur: 0.10066 (0.08065) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.26299 (11.60705) + | > current_lr: 0.00008 + | > step_time: 1.30100 (2.96644) + | > loader_time: 0.00220 (0.06645) + + + --> STEP: 91/234 -- GLOBAL_STEP: 74035 + | > loss: -0.29706 (-0.34210) + | > log_mle: -0.40912 (-0.42478) + | > loss_dur: 0.11206 (0.08268) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.30910 (11.67390) + | > current_lr: 0.00008 + | > step_time: 2.39320 (2.92578) + | > loader_time: 0.00370 (0.06479) + + + --> STEP: 96/234 -- GLOBAL_STEP: 74040 + | > loss: -0.28427 (-0.34019) + | > log_mle: -0.39706 (-0.42537) + | > loss_dur: 0.11279 (0.08518) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.28499 (12.15646) + | > current_lr: 0.00008 + | > step_time: 2.20690 (2.86820) + | > loader_time: 0.00220 (0.06159) + + + --> STEP: 101/234 -- GLOBAL_STEP: 74045 + | > loss: -0.29106 (-0.33816) + | > log_mle: -0.43899 (-0.42532) + | > loss_dur: 0.14792 (0.08716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.91988 (12.36114) + | > current_lr: 0.00008 + | > step_time: 3.41120 (2.86857) + | > loader_time: 0.00320 (0.06268) + + + --> STEP: 106/234 -- GLOBAL_STEP: 74050 + | > loss: -0.25690 (-0.33606) + | > log_mle: -0.43352 (-0.42574) + | > loss_dur: 0.17662 (0.08969) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.86812 (13.00932) + | > current_lr: 0.00008 + | > step_time: 2.81740 (2.84715) + | > loader_time: 0.19130 (0.06345) + + + --> STEP: 111/234 -- GLOBAL_STEP: 74055 + | > loss: -0.31533 (-0.33387) + | > log_mle: -0.49172 (-0.42632) + | > loss_dur: 0.17639 (0.09244) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.07206 (13.42553) + | > current_lr: 0.00008 + | > step_time: 5.71380 (2.85047) + | > loader_time: 0.09090 (0.06244) + + + --> STEP: 116/234 -- GLOBAL_STEP: 74060 + | > loss: -0.27773 (-0.33192) + | > log_mle: -0.45616 (-0.42711) + | > loss_dur: 0.17843 (0.09519) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.86759 (13.75001) + | > current_lr: 0.00008 + | > step_time: 4.51190 (2.82580) + | > loader_time: 0.07810 (0.06054) + + + --> STEP: 121/234 -- GLOBAL_STEP: 74065 + | > loss: -0.24225 (-0.32999) + | > log_mle: -0.37324 (-0.42730) + | > loss_dur: 0.13098 (0.09732) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.72951 (14.10374) + | > current_lr: 0.00008 + | > step_time: 1.69070 (2.79152) + | > loader_time: 0.00500 (0.05899) + + + --> STEP: 126/234 -- GLOBAL_STEP: 74070 + | > loss: -0.30979 (-0.32827) + | > log_mle: -0.49274 (-0.42796) + | > loss_dur: 0.18295 (0.09969) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.59818 (14.56978) + | > current_lr: 0.00008 + | > step_time: 3.91160 (2.82769) + | > loader_time: 0.00780 (0.05888) + + + --> STEP: 131/234 -- GLOBAL_STEP: 74075 + | > loss: -0.35535 (-0.32758) + | > log_mle: -0.54460 (-0.42991) + | > loss_dur: 0.18925 (0.10233) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.17184 (15.33854) + | > current_lr: 0.00008 + | > step_time: 1.14520 (2.80985) + | > loader_time: 0.00220 (0.05748) + + + --> STEP: 136/234 -- GLOBAL_STEP: 74080 + | > loss: -0.40669 (-0.32745) + | > log_mle: -0.60818 (-0.43221) + | > loss_dur: 0.20148 (0.10476) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.39814 (15.87222) + | > current_lr: 0.00008 + | > step_time: 1.09100 (2.76363) + | > loader_time: 0.00530 (0.05551) + + + --> STEP: 141/234 -- GLOBAL_STEP: 74085 + | > loss: -0.32561 (-0.32680) + | > log_mle: -0.49824 (-0.43399) + | > loss_dur: 0.17263 (0.10719) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.35836 (16.53334) + | > current_lr: 0.00008 + | > step_time: 2.99990 (2.74268) + | > loader_time: 0.00760 (0.05369) + + + --> STEP: 146/234 -- GLOBAL_STEP: 74090 + | > loss: -0.35693 (-0.32756) + | > log_mle: -0.55440 (-0.43771) + | > loss_dur: 0.19746 (0.11014) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.33843 (17.42071) + | > current_lr: 0.00008 + | > step_time: 12.60130 (2.81098) + | > loader_time: 0.00430 (0.05327) + + + --> STEP: 151/234 -- GLOBAL_STEP: 74095 + | > loss: -0.32023 (-0.32798) + | > log_mle: -0.50184 (-0.44044) + | > loss_dur: 0.18161 (0.11247) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.63508 (18.25203) + | > current_lr: 0.00008 + | > step_time: 1.22980 (2.83991) + | > loader_time: 0.19390 (0.05417) + + + --> STEP: 156/234 -- GLOBAL_STEP: 74100 + | > loss: -0.38037 (-0.32991) + | > log_mle: -0.56307 (-0.44485) + | > loss_dur: 0.18271 (0.11494) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.65664 (19.35900) + | > current_lr: 0.00008 + | > step_time: 4.39600 (2.85913) + | > loader_time: 0.01270 (0.05438) + + + --> STEP: 161/234 -- GLOBAL_STEP: 74105 + | > loss: -0.40145 (-0.33116) + | > log_mle: -0.58472 (-0.44850) + | > loss_dur: 0.18327 (0.11734) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.13169 (20.08444) + | > current_lr: 0.00008 + | > step_time: 6.39960 (2.93195) + | > loader_time: 0.20430 (0.05582) + + + --> STEP: 166/234 -- GLOBAL_STEP: 74110 + | > loss: -0.34133 (-0.33202) + | > log_mle: -0.51226 (-0.45142) + | > loss_dur: 0.17094 (0.11941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.88318 (21.00862) + | > current_lr: 0.00008 + | > step_time: 3.59750 (2.95410) + | > loader_time: 0.08560 (0.05700) + + + --> STEP: 171/234 -- GLOBAL_STEP: 74115 + | > loss: -0.42501 (-0.33408) + | > log_mle: -0.62822 (-0.45591) + | > loss_dur: 0.20321 (0.12183) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.80129 (22.20564) + | > current_lr: 0.00008 + | > step_time: 2.71400 (3.01763) + | > loader_time: 0.19420 (0.05760) + + + --> STEP: 176/234 -- GLOBAL_STEP: 74120 + | > loss: -0.40284 (-0.33612) + | > log_mle: -0.60567 (-0.46047) + | > loss_dur: 0.20282 (0.12435) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.90373 (22.98045) + | > current_lr: 0.00008 + | > step_time: 2.49820 (3.01367) + | > loader_time: 0.11530 (0.05718) + + + --> STEP: 181/234 -- GLOBAL_STEP: 74125 + | > loss: -0.33925 (-0.33750) + | > log_mle: -0.53324 (-0.46434) + | > loss_dur: 0.19400 (0.12684) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.45414 (24.27384) + | > current_lr: 0.00008 + | > step_time: 2.60190 (3.02555) + | > loader_time: 0.08420 (0.05659) + + + --> STEP: 186/234 -- GLOBAL_STEP: 74130 + | > loss: -0.33920 (-0.33919) + | > log_mle: -0.57328 (-0.46857) + | > loss_dur: 0.23408 (0.12938) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.83355 (25.13121) + | > current_lr: 0.00008 + | > step_time: 5.61070 (3.05620) + | > loader_time: 0.08830 (0.05875) + + + --> STEP: 191/234 -- GLOBAL_STEP: 74135 + | > loss: -0.38468 (-0.34099) + | > log_mle: -0.58841 (-0.47249) + | > loss_dur: 0.20374 (0.13150) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.43775 (26.03329) + | > current_lr: 0.00008 + | > step_time: 10.31040 (3.09865) + | > loader_time: 0.09420 (0.05823) + + + --> STEP: 196/234 -- GLOBAL_STEP: 74140 + | > loss: -0.37053 (-0.34315) + | > log_mle: -0.58966 (-0.47673) + | > loss_dur: 0.21913 (0.13358) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.76039 (26.91403) + | > current_lr: 0.00008 + | > step_time: 4.61050 (3.20184) + | > loader_time: 0.08980 (0.06370) + + + --> STEP: 201/234 -- GLOBAL_STEP: 74145 + | > loss: -0.33935 (-0.34468) + | > log_mle: -0.55080 (-0.48035) + | > loss_dur: 0.21145 (0.13566) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.14354 (27.73682) + | > current_lr: 0.00008 + | > step_time: 4.80430 (3.24565) + | > loader_time: 0.09480 (0.06457) + + + --> STEP: 206/234 -- GLOBAL_STEP: 74150 + | > loss: -0.45908 (-0.34698) + | > log_mle: -0.67336 (-0.48467) + | > loss_dur: 0.21427 (0.13769) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.88400 (28.43135) + | > current_lr: 0.00008 + | > step_time: 4.01050 (3.30033) + | > loader_time: 0.08650 (0.06539) + + + --> STEP: 211/234 -- GLOBAL_STEP: 74155 + | > loss: -0.49357 (-0.34973) + | > log_mle: -0.74255 (-0.48962) + | > loss_dur: 0.24897 (0.13990) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.18769 (29.34486) + | > current_lr: 0.00008 + | > step_time: 7.79680 (3.33017) + | > loader_time: 0.00490 (0.06517) + + + --> STEP: 216/234 -- GLOBAL_STEP: 74160 + | > loss: -0.45799 (-0.35232) + | > log_mle: -0.71988 (-0.49434) + | > loss_dur: 0.26189 (0.14202) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 145.76294 (30.59100) + | > current_lr: 0.00008 + | > step_time: 2.69910 (3.36181) + | > loader_time: 0.00240 (0.06515) + + + --> STEP: 221/234 -- GLOBAL_STEP: 74165 + | > loss: -0.42594 (-0.35486) + | > log_mle: -0.63736 (-0.49886) + | > loss_dur: 0.21142 (0.14400) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.20382 (31.63509) + | > current_lr: 0.00008 + | > step_time: 8.80110 (3.41007) + | > loader_time: 0.01140 (0.06423) + + + --> STEP: 226/234 -- GLOBAL_STEP: 74170 + | > loss: -0.50296 (-0.35790) + | > log_mle: -0.74229 (-0.50416) + | > loss_dur: 0.23933 (0.14626) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 115.22280 (32.70046) + | > current_lr: 0.00008 + | > step_time: 1.02160 (3.37990) + | > loader_time: 0.00390 (0.06507) + + + --> STEP: 231/234 -- GLOBAL_STEP: 74175 + | > loss: -0.44481 (-0.36002) + | > log_mle: -0.80981 (-0.50976) + | > loss_dur: 0.36500 (0.14974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 80.86194 (33.92819) + | > current_lr: 0.00008 + | > step_time: 0.26380 (3.31214) + | > loader_time: 0.00460 (0.06375) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.23063 (+0.13574) + | > avg_loss: -0.33839 (+0.00116) + | > avg_log_mle: -0.56433 (-0.00314) + | > avg_loss_dur: 0.22594 (+0.00430) + + + > EPOCH: 317/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 04:03:53)  + + --> STEP: 2/234 -- GLOBAL_STEP: 74180 + | > loss: -0.37505 (-0.36718) + | > log_mle: -0.45442 (-0.44389) + | > loss_dur: 0.07937 (0.07671) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.50566 (18.52225) + | > current_lr: 0.00008 + | > step_time: 1.71140 (5.75194) + | > loader_time: 0.40550 (0.20351) + + + --> STEP: 7/234 -- GLOBAL_STEP: 74185 + | > loss: -0.37162 (-0.34736) + | > log_mle: -0.44088 (-0.43672) + | > loss_dur: 0.06926 (0.08936) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.62915 (21.04216) + | > current_lr: 0.00008 + | > step_time: 0.89520 (3.53677) + | > loader_time: 0.00210 (0.06111) + + + --> STEP: 12/234 -- GLOBAL_STEP: 74190 + | > loss: -0.36161 (-0.35523) + | > log_mle: -0.44346 (-0.44316) + | > loss_dur: 0.08185 (0.08793) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.83201 (18.01397) + | > current_lr: 0.00008 + | > step_time: 3.40670 (2.92007) + | > loader_time: 0.00200 (0.05248) + + + --> STEP: 17/234 -- GLOBAL_STEP: 74195 + | > loss: -0.41086 (-0.37052) + | > log_mle: -0.46274 (-0.44966) + | > loss_dur: 0.05188 (0.07914) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.09702 (15.52672) + | > current_lr: 0.00008 + | > step_time: 1.25650 (3.24794) + | > loader_time: 0.00180 (0.04240) + + + --> STEP: 22/234 -- GLOBAL_STEP: 74200 + | > loss: -0.36647 (-0.37370) + | > log_mle: -0.44140 (-0.44904) + | > loss_dur: 0.07492 (0.07534) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.85784 (14.64393) + | > current_lr: 0.00008 + | > step_time: 2.10090 (2.76362) + | > loader_time: 0.00240 (0.03316) + + + --> STEP: 27/234 -- GLOBAL_STEP: 74205 + | > loss: -0.36133 (-0.37598) + | > log_mle: -0.43337 (-0.44889) + | > loss_dur: 0.07203 (0.07291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.96933 (14.01253) + | > current_lr: 0.00008 + | > step_time: 3.39720 (2.95448) + | > loader_time: 0.00910 (0.04343) + + + --> STEP: 32/234 -- GLOBAL_STEP: 74210 + | > loss: -0.37079 (-0.37628) + | > log_mle: -0.43578 (-0.44831) + | > loss_dur: 0.06499 (0.07203) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.37998 (13.53470) + | > current_lr: 0.00008 + | > step_time: 3.40310 (2.99964) + | > loader_time: 0.09950 (0.04013) + + + --> STEP: 37/234 -- GLOBAL_STEP: 74215 + | > loss: -0.35250 (-0.37263) + | > log_mle: -0.41841 (-0.44467) + | > loss_dur: 0.06591 (0.07204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.74524 (13.24116) + | > current_lr: 0.00008 + | > step_time: 6.10400 (3.13287) + | > loader_time: 0.10400 (0.04987) + + + --> STEP: 42/234 -- GLOBAL_STEP: 74220 + | > loss: -0.33061 (-0.36909) + | > log_mle: -0.40597 (-0.44159) + | > loss_dur: 0.07536 (0.07250) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.67729 (13.04740) + | > current_lr: 0.00008 + | > step_time: 0.42590 (2.91725) + | > loader_time: 0.00150 (0.04420) + + + --> STEP: 47/234 -- GLOBAL_STEP: 74225 + | > loss: -0.34962 (-0.36586) + | > log_mle: -0.42718 (-0.43897) + | > loss_dur: 0.07756 (0.07311) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.47541 (12.76546) + | > current_lr: 0.00008 + | > step_time: 1.71200 (2.77107) + | > loader_time: 0.08470 (0.04478) + + + --> STEP: 52/234 -- GLOBAL_STEP: 74230 + | > loss: -0.30973 (-0.36418) + | > log_mle: -0.40389 (-0.43720) + | > loss_dur: 0.09417 (0.07302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.48339 (12.34004) + | > current_lr: 0.00008 + | > step_time: 1.39610 (2.64010) + | > loader_time: 0.09440 (0.04252) + + + --> STEP: 57/234 -- GLOBAL_STEP: 74235 + | > loss: -0.31203 (-0.36173) + | > log_mle: -0.39911 (-0.43529) + | > loss_dur: 0.08708 (0.07355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.16323 (12.07106) + | > current_lr: 0.00008 + | > step_time: 1.08480 (2.53826) + | > loader_time: 0.00210 (0.03903) + + + --> STEP: 62/234 -- GLOBAL_STEP: 74240 + | > loss: -0.27610 (-0.35840) + | > log_mle: -0.41385 (-0.43361) + | > loss_dur: 0.13775 (0.07521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.91125 (12.04051) + | > current_lr: 0.00008 + | > step_time: 1.23210 (2.47342) + | > loader_time: 0.00210 (0.03920) + + + --> STEP: 67/234 -- GLOBAL_STEP: 74245 + | > loss: -0.31097 (-0.35573) + | > log_mle: -0.40919 (-0.43137) + | > loss_dur: 0.09822 (0.07564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.82754 (11.84816) + | > current_lr: 0.00008 + | > step_time: 2.42820 (2.42586) + | > loader_time: 0.00230 (0.03896) + + + --> STEP: 72/234 -- GLOBAL_STEP: 74250 + | > loss: -0.31376 (-0.35233) + | > log_mle: -0.40575 (-0.42918) + | > loss_dur: 0.09199 (0.07686) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.19776 (11.79729) + | > current_lr: 0.00008 + | > step_time: 2.37540 (2.38218) + | > loader_time: 0.08480 (0.03756) + + + --> STEP: 77/234 -- GLOBAL_STEP: 74255 + | > loss: -0.30652 (-0.34855) + | > log_mle: -0.39772 (-0.42710) + | > loss_dur: 0.09120 (0.07855) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.25011 (11.91545) + | > current_lr: 0.00008 + | > step_time: 0.99790 (2.35500) + | > loader_time: 0.00230 (0.03535) + + + --> STEP: 82/234 -- GLOBAL_STEP: 74260 + | > loss: -0.29119 (-0.34576) + | > log_mle: -0.38983 (-0.42529) + | > loss_dur: 0.09864 (0.07952) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.88998 (11.88864) + | > current_lr: 0.00008 + | > step_time: 1.00540 (2.31649) + | > loader_time: 0.08010 (0.03765) + + + --> STEP: 87/234 -- GLOBAL_STEP: 74265 + | > loss: -0.28354 (-0.34279) + | > log_mle: -0.39316 (-0.42367) + | > loss_dur: 0.10962 (0.08088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.47030 (11.92345) + | > current_lr: 0.00008 + | > step_time: 3.20050 (2.31555) + | > loader_time: 0.00290 (0.03767) + + + --> STEP: 92/234 -- GLOBAL_STEP: 74270 + | > loss: -0.30969 (-0.34021) + | > log_mle: -0.42618 (-0.42335) + | > loss_dur: 0.11649 (0.08314) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.91438 (12.25632) + | > current_lr: 0.00008 + | > step_time: 1.29510 (2.31157) + | > loader_time: 0.00210 (0.03773) + + + --> STEP: 97/234 -- GLOBAL_STEP: 74275 + | > loss: -0.29525 (-0.33835) + | > log_mle: -0.41377 (-0.42387) + | > loss_dur: 0.11852 (0.08552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.57368 (12.59070) + | > current_lr: 0.00008 + | > step_time: 1.59350 (2.27549) + | > loader_time: 0.00460 (0.03598) + + + --> STEP: 102/234 -- GLOBAL_STEP: 74280 + | > loss: -0.26072 (-0.33587) + | > log_mle: -0.39811 (-0.42376) + | > loss_dur: 0.13739 (0.08789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.87861 (12.82777) + | > current_lr: 0.00008 + | > step_time: 2.39990 (2.29503) + | > loader_time: 0.08640 (0.03624) + + + --> STEP: 107/234 -- GLOBAL_STEP: 74285 + | > loss: -0.28892 (-0.33420) + | > log_mle: -0.43364 (-0.42465) + | > loss_dur: 0.14472 (0.09045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.45526 (13.22954) + | > current_lr: 0.00008 + | > step_time: 1.11380 (2.30096) + | > loader_time: 0.07550 (0.03615) + + + --> STEP: 112/234 -- GLOBAL_STEP: 74290 + | > loss: -0.28602 (-0.33208) + | > log_mle: -0.44606 (-0.42536) + | > loss_dur: 0.16004 (0.09328) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.00725 (13.65506) + | > current_lr: 0.00008 + | > step_time: 1.44180 (2.28378) + | > loader_time: 0.00240 (0.03613) + + + --> STEP: 117/234 -- GLOBAL_STEP: 74295 + | > loss: -0.29121 (-0.32999) + | > log_mle: -0.44290 (-0.42608) + | > loss_dur: 0.15169 (0.09609) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.84387 (13.98163) + | > current_lr: 0.00008 + | > step_time: 1.90120 (2.28777) + | > loader_time: 0.08440 (0.03673) + + + --> STEP: 122/234 -- GLOBAL_STEP: 74300 + | > loss: -0.27172 (-0.32801) + | > log_mle: -0.41376 (-0.42612) + | > loss_dur: 0.14204 (0.09810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.65520 (14.17193) + | > current_lr: 0.00008 + | > step_time: 2.45670 (2.32993) + | > loader_time: 0.10130 (0.03823) + + + --> STEP: 127/234 -- GLOBAL_STEP: 74305 + | > loss: -0.29138 (-0.32664) + | > log_mle: -0.47503 (-0.42734) + | > loss_dur: 0.18365 (0.10070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.68843 (14.84203) + | > current_lr: 0.00008 + | > step_time: 1.91340 (2.30046) + | > loader_time: 0.08730 (0.03751) + + + --> STEP: 132/234 -- GLOBAL_STEP: 74310 + | > loss: -0.30680 (-0.32629) + | > log_mle: -0.45594 (-0.42918) + | > loss_dur: 0.14913 (0.10289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.49257 (15.40550) + | > current_lr: 0.00008 + | > step_time: 1.59170 (2.28742) + | > loader_time: 0.00750 (0.03625) + + + --> STEP: 137/234 -- GLOBAL_STEP: 74315 + | > loss: -0.28493 (-0.32594) + | > log_mle: -0.46962 (-0.43149) + | > loss_dur: 0.18469 (0.10554) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.16086 (16.00828) + | > current_lr: 0.00008 + | > step_time: 2.29050 (2.28434) + | > loader_time: 0.00430 (0.03696) + + + --> STEP: 142/234 -- GLOBAL_STEP: 74320 + | > loss: -0.30850 (-0.32549) + | > log_mle: -0.48120 (-0.43336) + | > loss_dur: 0.17270 (0.10787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.94591 (16.58330) + | > current_lr: 0.00008 + | > step_time: 2.18990 (2.28554) + | > loader_time: 0.10470 (0.03847) + + + --> STEP: 147/234 -- GLOBAL_STEP: 74325 + | > loss: -0.31441 (-0.32650) + | > log_mle: -0.48912 (-0.43715) + | > loss_dur: 0.17471 (0.11065) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.18034 (17.37527) + | > current_lr: 0.00008 + | > step_time: 1.50490 (2.29694) + | > loader_time: 0.00360 (0.04060) + + + --> STEP: 152/234 -- GLOBAL_STEP: 74330 + | > loss: -0.37593 (-0.32758) + | > log_mle: -0.56845 (-0.44052) + | > loss_dur: 0.19252 (0.11293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.95837 (18.24101) + | > current_lr: 0.00008 + | > step_time: 5.31320 (2.32078) + | > loader_time: 0.18680 (0.04442) + + + --> STEP: 157/234 -- GLOBAL_STEP: 74335 + | > loss: -0.33257 (-0.32942) + | > log_mle: -0.51501 (-0.44467) + | > loss_dur: 0.18244 (0.11525) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.85020 (19.09647) + | > current_lr: 0.00008 + | > step_time: 3.10480 (2.33238) + | > loader_time: 0.00310 (0.04416) + + + --> STEP: 162/234 -- GLOBAL_STEP: 74340 + | > loss: -0.36525 (-0.33106) + | > log_mle: -0.55372 (-0.44865) + | > loss_dur: 0.18847 (0.11760) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.01084 (19.95851) + | > current_lr: 0.00008 + | > step_time: 2.40870 (2.35909) + | > loader_time: 0.00370 (0.04405) + + + --> STEP: 167/234 -- GLOBAL_STEP: 74345 + | > loss: -0.45681 (-0.33270) + | > log_mle: -0.64146 (-0.45230) + | > loss_dur: 0.18465 (0.11959) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.41084 (20.77153) + | > current_lr: 0.00008 + | > step_time: 1.88620 (2.39657) + | > loader_time: 0.00320 (0.04338) + + + --> STEP: 172/234 -- GLOBAL_STEP: 74350 + | > loss: -0.43504 (-0.33481) + | > log_mle: -0.64397 (-0.45696) + | > loss_dur: 0.20893 (0.12215) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.47873 (21.75915) + | > current_lr: 0.00008 + | > step_time: 0.90500 (2.39844) + | > loader_time: 0.00370 (0.04327) + + + --> STEP: 177/234 -- GLOBAL_STEP: 74355 + | > loss: -0.38783 (-0.33664) + | > log_mle: -0.58734 (-0.46123) + | > loss_dur: 0.19951 (0.12459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.44056 (22.64159) + | > current_lr: 0.00008 + | > step_time: 11.61650 (2.45400) + | > loader_time: 0.08350 (0.04320) + + + --> STEP: 182/234 -- GLOBAL_STEP: 74360 + | > loss: -0.39690 (-0.33827) + | > log_mle: -0.63797 (-0.46555) + | > loss_dur: 0.24107 (0.12728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.29053 (23.44565) + | > current_lr: 0.00008 + | > step_time: 2.79630 (2.49745) + | > loader_time: 0.00360 (0.04369) + + + --> STEP: 187/234 -- GLOBAL_STEP: 74365 + | > loss: -0.43480 (-0.34033) + | > log_mle: -0.64168 (-0.46996) + | > loss_dur: 0.20688 (0.12963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.79285 (24.21551) + | > current_lr: 0.00008 + | > step_time: 1.20430 (2.53663) + | > loader_time: 0.00320 (0.04407) + + + --> STEP: 192/234 -- GLOBAL_STEP: 74370 + | > loss: -0.45688 (-0.34261) + | > log_mle: -0.66604 (-0.47424) + | > loss_dur: 0.20916 (0.13163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.90589 (25.28502) + | > current_lr: 0.00008 + | > step_time: 3.20640 (2.55830) + | > loader_time: 0.08860 (0.04441) + + + --> STEP: 197/234 -- GLOBAL_STEP: 74375 + | > loss: -0.43479 (-0.34464) + | > log_mle: -0.62692 (-0.47833) + | > loss_dur: 0.19213 (0.13369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.03709 (26.43031) + | > current_lr: 0.00008 + | > step_time: 13.69970 (2.66805) + | > loader_time: 0.00860 (0.04529) + + + --> STEP: 202/234 -- GLOBAL_STEP: 74380 + | > loss: -0.51874 (-0.34641) + | > log_mle: -0.73412 (-0.48238) + | > loss_dur: 0.21538 (0.13598) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.60379 (27.31128) + | > current_lr: 0.00008 + | > step_time: 6.00890 (2.74862) + | > loader_time: 0.09420 (0.04607) + + + --> STEP: 207/234 -- GLOBAL_STEP: 74385 + | > loss: -0.48896 (-0.34847) + | > log_mle: -0.72273 (-0.48661) + | > loss_dur: 0.23377 (0.13814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.57869 (28.11555) + | > current_lr: 0.00008 + | > step_time: 3.20770 (2.77494) + | > loader_time: 0.08820 (0.04589) + + + --> STEP: 212/234 -- GLOBAL_STEP: 74390 + | > loss: -0.45238 (-0.35083) + | > log_mle: -0.68206 (-0.49116) + | > loss_dur: 0.22968 (0.14033) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 98.44941 (29.21093) + | > current_lr: 0.00008 + | > step_time: 10.70760 (2.86848) + | > loader_time: 0.19600 (0.04676) + + + --> STEP: 217/234 -- GLOBAL_STEP: 74395 + | > loss: -0.46590 (-0.35321) + | > log_mle: -0.70335 (-0.49566) + | > loss_dur: 0.23745 (0.14245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.35180 (29.99573) + | > current_lr: 0.00008 + | > step_time: 6.39820 (2.97018) + | > loader_time: 0.00770 (0.04799) + + + --> STEP: 222/234 -- GLOBAL_STEP: 74400 + | > loss: -0.46824 (-0.35602) + | > log_mle: -0.72700 (-0.50048) + | > loss_dur: 0.25876 (0.14446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 102.82005 (30.75401) + | > current_lr: 0.00008 + | > step_time: 2.19530 (2.99905) + | > loader_time: 0.00730 (0.04839) + + + --> STEP: 227/234 -- GLOBAL_STEP: 74405 + | > loss: -0.44577 (-0.35896) + | > log_mle: -0.70086 (-0.50554) + | > loss_dur: 0.25509 (0.14657) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.62302 (31.79762) + | > current_lr: 0.00008 + | > step_time: 0.24890 (2.94578) + | > loader_time: 0.00520 (0.04741) + + + --> STEP: 232/234 -- GLOBAL_STEP: 74410 + | > loss: -0.45331 (-0.36130) + | > log_mle: -0.93065 (-0.51222) + | > loss_dur: 0.47735 (0.15092) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.42532 (33.06633) + | > current_lr: 0.00008 + | > step_time: 0.43850 (2.88877) + | > loader_time: 0.06070 (0.04673) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.47467 (+0.24404) + | > avg_loss: -0.34988 (-0.01149) + | > avg_log_mle: -0.56968 (-0.00534) + | > avg_loss_dur: 0.21980 (-0.00614) + + + > EPOCH: 318/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 04:16:22)  + + --> STEP: 3/234 -- GLOBAL_STEP: 74415 + | > loss: -0.28319 (-0.34183) + | > log_mle: -0.43230 (-0.44198) + | > loss_dur: 0.14911 (0.10015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.43110 (16.74879) + | > current_lr: 0.00008 + | > step_time: 0.50340 (10.63485) + | > loader_time: 0.00140 (0.00596) + + + --> STEP: 8/234 -- GLOBAL_STEP: 74420 + | > loss: -0.38477 (-0.36013) + | > log_mle: -0.46825 (-0.44467) + | > loss_dur: 0.08348 (0.08454) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.31385 (15.33866) + | > current_lr: 0.00008 + | > step_time: 4.00840 (5.53650) + | > loader_time: 0.18580 (0.02748) + + + --> STEP: 13/234 -- GLOBAL_STEP: 74425 + | > loss: -0.41370 (-0.36956) + | > log_mle: -0.47634 (-0.45010) + | > loss_dur: 0.06263 (0.08055) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.88808 (14.32152) + | > current_lr: 0.00008 + | > step_time: 1.41170 (4.16050) + | > loader_time: 0.00220 (0.01798) + + + --> STEP: 18/234 -- GLOBAL_STEP: 74430 + | > loss: -0.37979 (-0.37782) + | > log_mle: -0.44405 (-0.45266) + | > loss_dur: 0.06427 (0.07485) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.34803 (13.79349) + | > current_lr: 0.00008 + | > step_time: 1.54860 (3.34267) + | > loader_time: 0.00190 (0.01342) + + + --> STEP: 23/234 -- GLOBAL_STEP: 74435 + | > loss: -0.42269 (-0.38112) + | > log_mle: -0.47414 (-0.45300) + | > loss_dur: 0.05145 (0.07188) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87189 (13.25645) + | > current_lr: 0.00008 + | > step_time: 3.30200 (3.01228) + | > loader_time: 0.09590 (0.01820) + + + --> STEP: 28/234 -- GLOBAL_STEP: 74440 + | > loss: -0.43410 (-0.38324) + | > log_mle: -0.48302 (-0.45279) + | > loss_dur: 0.04891 (0.06955) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.21117 (12.73804) + | > current_lr: 0.00008 + | > step_time: 1.18350 (2.67602) + | > loader_time: 0.00180 (0.01536) + + + --> STEP: 33/234 -- GLOBAL_STEP: 74445 + | > loss: -0.36745 (-0.38048) + | > log_mle: -0.43080 (-0.45015) + | > loss_dur: 0.06335 (0.06967) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.03740 (12.47120) + | > current_lr: 0.00008 + | > step_time: 6.09460 (2.94659) + | > loader_time: 0.00540 (0.02827) + + + --> STEP: 38/234 -- GLOBAL_STEP: 74450 + | > loss: -0.35654 (-0.37667) + | > log_mle: -0.43242 (-0.44666) + | > loss_dur: 0.07587 (0.06998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.12272 (12.15901) + | > current_lr: 0.00008 + | > step_time: 1.58580 (2.83099) + | > loader_time: 0.00140 (0.02487) + + + --> STEP: 43/234 -- GLOBAL_STEP: 74455 + | > loss: -0.33672 (-0.37273) + | > log_mle: -0.41609 (-0.44325) + | > loss_dur: 0.07937 (0.07051) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.66594 (12.06188) + | > current_lr: 0.00008 + | > step_time: 2.80780 (2.81811) + | > loader_time: 0.08990 (0.02600) + + + --> STEP: 48/234 -- GLOBAL_STEP: 74460 + | > loss: -0.36939 (-0.36942) + | > log_mle: -0.43092 (-0.44053) + | > loss_dur: 0.06153 (0.07111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.47790 (12.26898) + | > current_lr: 0.00008 + | > step_time: 1.28590 (2.74118) + | > loader_time: 0.00210 (0.02722) + + + --> STEP: 53/234 -- GLOBAL_STEP: 74465 + | > loss: -0.33598 (-0.36669) + | > log_mle: -0.41436 (-0.43813) + | > loss_dur: 0.07837 (0.07143) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.31470 (12.20614) + | > current_lr: 0.00008 + | > step_time: 1.29220 (2.69588) + | > loader_time: 0.00300 (0.03058) + + + --> STEP: 58/234 -- GLOBAL_STEP: 74470 + | > loss: -0.35252 (-0.36462) + | > log_mle: -0.41928 (-0.43610) + | > loss_dur: 0.06676 (0.07148) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.48266 (12.12165) + | > current_lr: 0.00008 + | > step_time: 2.79360 (2.69439) + | > loader_time: 0.19860 (0.03483) + + + --> STEP: 63/234 -- GLOBAL_STEP: 74475 + | > loss: -0.30875 (-0.36011) + | > log_mle: -0.39644 (-0.43367) + | > loss_dur: 0.08769 (0.07356) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.82331 (12.51996) + | > current_lr: 0.00008 + | > step_time: 2.78710 (2.60870) + | > loader_time: 0.00190 (0.03491) + + + --> STEP: 68/234 -- GLOBAL_STEP: 74480 + | > loss: -0.29314 (-0.35665) + | > log_mle: -0.39010 (-0.43118) + | > loss_dur: 0.09696 (0.07453) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.27913 (12.47945) + | > current_lr: 0.00008 + | > step_time: 2.48790 (2.57953) + | > loader_time: 0.00210 (0.03500) + + + --> STEP: 73/234 -- GLOBAL_STEP: 74485 + | > loss: -0.28626 (-0.35281) + | > log_mle: -0.39636 (-0.42887) + | > loss_dur: 0.11010 (0.07606) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.53988 (12.39785) + | > current_lr: 0.00008 + | > step_time: 2.50380 (2.54433) + | > loader_time: 0.08500 (0.03623) + + + --> STEP: 78/234 -- GLOBAL_STEP: 74490 + | > loss: -0.29084 (-0.34877) + | > log_mle: -0.38863 (-0.42654) + | > loss_dur: 0.09779 (0.07777) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.46454 (12.27836) + | > current_lr: 0.00008 + | > step_time: 0.89070 (2.54771) + | > loader_time: 0.00290 (0.03657) + + + --> STEP: 83/234 -- GLOBAL_STEP: 74495 + | > loss: -0.26823 (-0.34558) + | > log_mle: -0.39346 (-0.42479) + | > loss_dur: 0.12523 (0.07922) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.58407 (12.15089) + | > current_lr: 0.00008 + | > step_time: 1.09710 (2.48548) + | > loader_time: 0.00210 (0.03652) + + + --> STEP: 88/234 -- GLOBAL_STEP: 74500 + | > loss: -0.30299 (-0.34299) + | > log_mle: -0.42608 (-0.42354) + | > loss_dur: 0.12309 (0.08055) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.45498 (12.08126) + | > current_lr: 0.00008 + | > step_time: 1.78480 (2.47603) + | > loader_time: 0.00240 (0.03468) + + + --> STEP: 93/234 -- GLOBAL_STEP: 74505 + | > loss: -0.30166 (-0.34045) + | > log_mle: -0.43871 (-0.42326) + | > loss_dur: 0.13705 (0.08281) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.80207 (12.41201) + | > current_lr: 0.00008 + | > step_time: 1.87220 (2.47707) + | > loader_time: 0.00240 (0.03391) + + + --> STEP: 98/234 -- GLOBAL_STEP: 74510 + | > loss: -0.28308 (-0.33851) + | > log_mle: -0.38269 (-0.42321) + | > loss_dur: 0.09960 (0.08471) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.43965 (12.62913) + | > current_lr: 0.00008 + | > step_time: 3.21400 (2.47007) + | > loader_time: 0.00310 (0.03235) + + + --> STEP: 103/234 -- GLOBAL_STEP: 74515 + | > loss: -0.30051 (-0.33634) + | > log_mle: -0.46350 (-0.42393) + | > loss_dur: 0.16299 (0.08759) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.02905 (13.00224) + | > current_lr: 0.00008 + | > step_time: 1.31430 (2.46797) + | > loader_time: 0.00250 (0.03090) + + + --> STEP: 108/234 -- GLOBAL_STEP: 74520 + | > loss: -0.28402 (-0.33418) + | > log_mle: -0.41039 (-0.42409) + | > loss_dur: 0.12637 (0.08991) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.56749 (13.39427) + | > current_lr: 0.00008 + | > step_time: 2.02040 (2.51057) + | > loader_time: 0.09800 (0.03136) + + + --> STEP: 113/234 -- GLOBAL_STEP: 74525 + | > loss: -0.30206 (-0.33217) + | > log_mle: -0.44920 (-0.42511) + | > loss_dur: 0.14714 (0.09294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.81749 (13.83384) + | > current_lr: 0.00008 + | > step_time: 2.01220 (2.50827) + | > loader_time: 0.00660 (0.03177) + + + --> STEP: 118/234 -- GLOBAL_STEP: 74530 + | > loss: -0.27121 (-0.33006) + | > log_mle: -0.42573 (-0.42564) + | > loss_dur: 0.15452 (0.09558) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.29088 (14.25106) + | > current_lr: 0.00008 + | > step_time: 2.11550 (2.50544) + | > loader_time: 0.07680 (0.03196) + + + --> STEP: 123/234 -- GLOBAL_STEP: 74535 + | > loss: -0.24807 (-0.32788) + | > log_mle: -0.39008 (-0.42534) + | > loss_dur: 0.14201 (0.09746) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.88279 (14.52370) + | > current_lr: 0.00008 + | > step_time: 2.51090 (2.52166) + | > loader_time: 0.00260 (0.03233) + + + --> STEP: 128/234 -- GLOBAL_STEP: 74540 + | > loss: -0.30992 (-0.32688) + | > log_mle: -0.44352 (-0.42697) + | > loss_dur: 0.13360 (0.10009) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.70681 (15.06030) + | > current_lr: 0.00008 + | > step_time: 1.46510 (2.51616) + | > loader_time: 0.00260 (0.03241) + + + --> STEP: 133/234 -- GLOBAL_STEP: 74545 + | > loss: -0.31496 (-0.32651) + | > log_mle: -0.48091 (-0.42907) + | > loss_dur: 0.16595 (0.10256) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.67729 (15.66041) + | > current_lr: 0.00008 + | > step_time: 2.31130 (2.52631) + | > loader_time: 0.09640 (0.03202) + + + --> STEP: 138/234 -- GLOBAL_STEP: 74550 + | > loss: -0.26678 (-0.32569) + | > log_mle: -0.42340 (-0.43084) + | > loss_dur: 0.15663 (0.10515) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.11933 (16.21281) + | > current_lr: 0.00008 + | > step_time: 3.69780 (2.51430) + | > loader_time: 0.10230 (0.03223) + + + --> STEP: 143/234 -- GLOBAL_STEP: 74555 + | > loss: -0.37130 (-0.32573) + | > log_mle: -0.57563 (-0.43361) + | > loss_dur: 0.20433 (0.10788) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.37072 (17.21984) + | > current_lr: 0.00008 + | > step_time: 2.58350 (2.54924) + | > loader_time: 0.00250 (0.03318) + + + --> STEP: 148/234 -- GLOBAL_STEP: 74560 + | > loss: -0.31728 (-0.32616) + | > log_mle: -0.48426 (-0.43646) + | > loss_dur: 0.16698 (0.11031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.75458 (17.87563) + | > current_lr: 0.00008 + | > step_time: 2.48500 (2.53885) + | > loader_time: 0.00230 (0.03270) + + + --> STEP: 153/234 -- GLOBAL_STEP: 74565 + | > loss: -0.42061 (-0.32753) + | > log_mle: -0.61632 (-0.44049) + | > loss_dur: 0.19571 (0.11296) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.97045 (18.78020) + | > current_lr: 0.00008 + | > step_time: 2.12860 (2.53958) + | > loader_time: 0.08190 (0.03347) + + + --> STEP: 158/234 -- GLOBAL_STEP: 74570 + | > loss: -0.34563 (-0.32850) + | > log_mle: -0.54789 (-0.44400) + | > loss_dur: 0.20226 (0.11550) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.84616 (19.49193) + | > current_lr: 0.00008 + | > step_time: 2.61150 (2.59573) + | > loader_time: 0.08730 (0.03621) + + + --> STEP: 163/234 -- GLOBAL_STEP: 74575 + | > loss: -0.32009 (-0.32957) + | > log_mle: -0.51240 (-0.44734) + | > loss_dur: 0.19231 (0.11777) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.02104 (20.25829) + | > current_lr: 0.00008 + | > step_time: 2.71020 (2.65245) + | > loader_time: 0.08960 (0.03739) + + + --> STEP: 168/234 -- GLOBAL_STEP: 74580 + | > loss: -0.37104 (-0.33114) + | > log_mle: -0.57130 (-0.45110) + | > loss_dur: 0.20025 (0.11996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.56402 (20.95877) + | > current_lr: 0.00008 + | > step_time: 1.30880 (2.66043) + | > loader_time: 0.08880 (0.03848) + + + --> STEP: 173/234 -- GLOBAL_STEP: 74585 + | > loss: -0.37749 (-0.33319) + | > log_mle: -0.58671 (-0.45575) + | > loss_dur: 0.20922 (0.12256) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.03859 (21.70395) + | > current_lr: 0.00008 + | > step_time: 1.96960 (2.68339) + | > loader_time: 0.00340 (0.03960) + + + --> STEP: 178/234 -- GLOBAL_STEP: 74590 + | > loss: -0.42893 (-0.33524) + | > log_mle: -0.64926 (-0.46025) + | > loss_dur: 0.22033 (0.12501) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.11605 (22.90026) + | > current_lr: 0.00008 + | > step_time: 4.20650 (2.69508) + | > loader_time: 0.00380 (0.03860) + + + --> STEP: 183/234 -- GLOBAL_STEP: 74595 + | > loss: -0.42915 (-0.33682) + | > log_mle: -0.63608 (-0.46439) + | > loss_dur: 0.20692 (0.12757) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.75134 (24.00484) + | > current_lr: 0.00008 + | > step_time: 3.41790 (2.71670) + | > loader_time: 0.07900 (0.03957) + + + --> STEP: 188/234 -- GLOBAL_STEP: 74600 + | > loss: -0.44487 (-0.33865) + | > log_mle: -0.66619 (-0.46864) + | > loss_dur: 0.22131 (0.12998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.34875 (24.86933) + | > current_lr: 0.00008 + | > step_time: 1.71380 (2.72168) + | > loader_time: 0.08820 (0.04008) + + + --> STEP: 193/234 -- GLOBAL_STEP: 74605 + | > loss: -0.45915 (-0.34095) + | > log_mle: -0.67193 (-0.47291) + | > loss_dur: 0.21278 (0.13196) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.82332 (25.56668) + | > current_lr: 0.00008 + | > step_time: 3.30390 (2.75536) + | > loader_time: 0.28180 (0.04110) + + + --> STEP: 198/234 -- GLOBAL_STEP: 74610 + | > loss: -0.43805 (-0.34295) + | > log_mle: -0.65851 (-0.47696) + | > loss_dur: 0.22046 (0.13401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.57194 (26.41298) + | > current_lr: 0.00008 + | > step_time: 3.99700 (2.79204) + | > loader_time: 0.08550 (0.04494) + + + --> STEP: 203/234 -- GLOBAL_STEP: 74615 + | > loss: -0.34981 (-0.34442) + | > log_mle: -0.55677 (-0.48068) + | > loss_dur: 0.20696 (0.13626) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.42014 (27.08492) + | > current_lr: 0.00008 + | > step_time: 3.20720 (2.83812) + | > loader_time: 0.00540 (0.04614) + + + --> STEP: 208/234 -- GLOBAL_STEP: 74620 + | > loss: -0.42280 (-0.34649) + | > log_mle: -0.65956 (-0.48499) + | > loss_dur: 0.23676 (0.13850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.73055 (27.98767) + | > current_lr: 0.00008 + | > step_time: 2.59480 (2.88769) + | > loader_time: 0.00410 (0.04689) + + + --> STEP: 213/234 -- GLOBAL_STEP: 74625 + | > loss: -0.48453 (-0.34942) + | > log_mle: -0.71247 (-0.48999) + | > loss_dur: 0.22794 (0.14057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 78.15025 (28.84788) + | > current_lr: 0.00008 + | > step_time: 2.91610 (2.98284) + | > loader_time: 0.08810 (0.04817) + + + --> STEP: 218/234 -- GLOBAL_STEP: 74630 + | > loss: -0.44502 (-0.35205) + | > log_mle: -0.67048 (-0.49465) + | > loss_dur: 0.22546 (0.14260) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.94411 (29.72560) + | > current_lr: 0.00008 + | > step_time: 3.10670 (3.03784) + | > loader_time: 0.00660 (0.04800) + + + --> STEP: 223/234 -- GLOBAL_STEP: 74635 + | > loss: -0.49120 (-0.35500) + | > log_mle: -0.72561 (-0.49974) + | > loss_dur: 0.23441 (0.14474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.13178 (30.56987) + | > current_lr: 0.00008 + | > step_time: 6.10180 (3.05535) + | > loader_time: 0.09990 (0.04751) + + + --> STEP: 228/234 -- GLOBAL_STEP: 74640 + | > loss: -0.46674 (-0.35808) + | > log_mle: -0.72527 (-0.50506) + | > loss_dur: 0.25852 (0.14698) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.16422 (31.52285) + | > current_lr: 0.00008 + | > step_time: 0.25490 (3.06688) + | > loader_time: 0.00300 (0.04734) + + + --> STEP: 233/234 -- GLOBAL_STEP: 74645 + | > loss: -0.05012 (-0.35899) + | > log_mle: -0.71580 (-0.51190) + | > loss_dur: 0.66567 (0.15291) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.75230 (32.91233) + | > current_lr: 0.00008 + | > step_time: 0.19390 (3.00672) + | > loader_time: 0.00700 (0.04644) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.05077 (-0.42390) + | > avg_loss: -0.32251 (+0.02737) + | > avg_log_mle: -0.56713 (+0.00255) + | > avg_loss_dur: 0.24461 (+0.02482) + + + > EPOCH: 319/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 04:29:06)  + + --> STEP: 4/234 -- GLOBAL_STEP: 74650 + | > loss: -0.35038 (-0.34832) + | > log_mle: -0.44141 (-0.44344) + | > loss_dur: 0.09102 (0.09512) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.43986 (19.60210) + | > current_lr: 0.00008 + | > step_time: 5.22050 (9.00062) + | > loader_time: 0.10220 (0.03107) + + + --> STEP: 9/234 -- GLOBAL_STEP: 74655 + | > loss: -0.34237 (-0.35843) + | > log_mle: -0.45315 (-0.44674) + | > loss_dur: 0.11078 (0.08831) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.25788 (18.77318) + | > current_lr: 0.00008 + | > step_time: 4.39510 (5.40885) + | > loader_time: 0.00110 (0.03326) + + + --> STEP: 14/234 -- GLOBAL_STEP: 74660 + | > loss: -0.39764 (-0.37122) + | > log_mle: -0.45867 (-0.45170) + | > loss_dur: 0.06103 (0.08048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.49747 (17.00287) + | > current_lr: 0.00008 + | > step_time: 1.50180 (4.10646) + | > loader_time: 0.00170 (0.04158) + + + --> STEP: 19/234 -- GLOBAL_STEP: 74665 + | > loss: -0.41079 (-0.38037) + | > log_mle: -0.46858 (-0.45469) + | > loss_dur: 0.05779 (0.07432) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.50573 (15.27909) + | > current_lr: 0.00008 + | > step_time: 2.50980 (3.87982) + | > loader_time: 0.19340 (0.04616) + + + --> STEP: 24/234 -- GLOBAL_STEP: 74670 + | > loss: -0.38714 (-0.38221) + | > log_mle: -0.44341 (-0.45402) + | > loss_dur: 0.05627 (0.07180) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.04798 (14.74612) + | > current_lr: 0.00008 + | > step_time: 11.49360 (4.36016) + | > loader_time: 0.00460 (0.07696) + + + --> STEP: 29/234 -- GLOBAL_STEP: 74675 + | > loss: -0.37302 (-0.38193) + | > log_mle: -0.43909 (-0.45255) + | > loss_dur: 0.06607 (0.07062) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.77831 (13.99506) + | > current_lr: 0.00008 + | > step_time: 5.09620 (4.15432) + | > loader_time: 0.00160 (0.07668) + + + --> STEP: 34/234 -- GLOBAL_STEP: 74680 + | > loss: -0.35676 (-0.37888) + | > log_mle: -0.42381 (-0.44942) + | > loss_dur: 0.06705 (0.07055) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.54434 (13.31116) + | > current_lr: 0.00008 + | > step_time: 12.01540 (4.20808) + | > loader_time: 0.19490 (0.07441) + + + --> STEP: 39/234 -- GLOBAL_STEP: 74685 + | > loss: -0.33374 (-0.37519) + | > log_mle: -0.41691 (-0.44607) + | > loss_dur: 0.08317 (0.07088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.09071 (12.85626) + | > current_lr: 0.00008 + | > step_time: 2.00060 (3.91976) + | > loader_time: 0.00240 (0.07404) + + + --> STEP: 44/234 -- GLOBAL_STEP: 74690 + | > loss: -0.35071 (-0.37200) + | > log_mle: -0.41231 (-0.44271) + | > loss_dur: 0.06160 (0.07070) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.94767 (12.34620) + | > current_lr: 0.00008 + | > step_time: 2.10440 (3.69047) + | > loader_time: 0.00150 (0.06601) + + + --> STEP: 49/234 -- GLOBAL_STEP: 74695 + | > loss: -0.36134 (-0.36946) + | > log_mle: -0.42748 (-0.44082) + | > loss_dur: 0.06614 (0.07136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.82696 (11.95715) + | > current_lr: 0.00008 + | > step_time: 2.27200 (3.51095) + | > loader_time: 0.00130 (0.05958) + + + --> STEP: 54/234 -- GLOBAL_STEP: 74700 + | > loss: -0.33149 (-0.36661) + | > log_mle: -0.40950 (-0.43846) + | > loss_dur: 0.07801 (0.07186) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.33665 (11.79311) + | > current_lr: 0.00008 + | > step_time: 2.49390 (3.38437) + | > loader_time: 0.00550 (0.06270) + + + --> STEP: 59/234 -- GLOBAL_STEP: 74705 + | > loss: -0.33409 (-0.36456) + | > log_mle: -0.41478 (-0.43658) + | > loss_dur: 0.08069 (0.07202) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.07547 (11.73709) + | > current_lr: 0.00008 + | > step_time: 2.40690 (3.30759) + | > loader_time: 0.08370 (0.06197) + + + --> STEP: 64/234 -- GLOBAL_STEP: 74710 + | > loss: -0.33210 (-0.35995) + | > log_mle: -0.40610 (-0.43401) + | > loss_dur: 0.07400 (0.07407) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.43950 (11.99586) + | > current_lr: 0.00008 + | > step_time: 4.30450 (3.29009) + | > loader_time: 0.00320 (0.05874) + + + --> STEP: 69/234 -- GLOBAL_STEP: 74715 + | > loss: -0.31726 (-0.35673) + | > log_mle: -0.39864 (-0.43156) + | > loss_dur: 0.08138 (0.07483) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.12449 (11.87861) + | > current_lr: 0.00008 + | > step_time: 3.30610 (3.29537) + | > loader_time: 0.00250 (0.05997) + + + --> STEP: 74/234 -- GLOBAL_STEP: 74720 + | > loss: -0.29179 (-0.35248) + | > log_mle: -0.39051 (-0.42924) + | > loss_dur: 0.09872 (0.07677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.17807 (12.03399) + | > current_lr: 0.00008 + | > step_time: 4.09100 (3.23883) + | > loader_time: 0.00920 (0.05994) + + + --> STEP: 79/234 -- GLOBAL_STEP: 74725 + | > loss: -0.30235 (-0.34886) + | > log_mle: -0.40425 (-0.42731) + | > loss_dur: 0.10190 (0.07845) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.41434 (12.00437) + | > current_lr: 0.00008 + | > step_time: 2.60020 (3.16815) + | > loader_time: 0.09350 (0.06182) + + + --> STEP: 84/234 -- GLOBAL_STEP: 74730 + | > loss: -0.30661 (-0.34589) + | > log_mle: -0.40060 (-0.42559) + | > loss_dur: 0.09398 (0.07971) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.20002 (11.85934) + | > current_lr: 0.00008 + | > step_time: 1.27570 (3.19512) + | > loader_time: 0.00180 (0.06145) + + + --> STEP: 89/234 -- GLOBAL_STEP: 74735 + | > loss: -0.30346 (-0.34343) + | > log_mle: -0.41830 (-0.42468) + | > loss_dur: 0.11484 (0.08125) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.10903 (11.76344) + | > current_lr: 0.00008 + | > step_time: 1.70030 (3.14153) + | > loader_time: 0.00320 (0.05905) + + + --> STEP: 94/234 -- GLOBAL_STEP: 74740 + | > loss: -0.30923 (-0.34112) + | > log_mle: -0.43803 (-0.42474) + | > loss_dur: 0.12880 (0.08362) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.53338 (12.11892) + | > current_lr: 0.00008 + | > step_time: 1.81640 (3.07496) + | > loader_time: 0.00270 (0.05787) + + + --> STEP: 99/234 -- GLOBAL_STEP: 74745 + | > loss: -0.30995 (-0.33910) + | > log_mle: -0.47113 (-0.42500) + | > loss_dur: 0.16118 (0.08590) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.47556 (12.43337) + | > current_lr: 0.00008 + | > step_time: 3.69470 (3.02945) + | > loader_time: 0.00630 (0.05600) + + + --> STEP: 104/234 -- GLOBAL_STEP: 74750 + | > loss: -0.33282 (-0.33724) + | > log_mle: -0.48097 (-0.42582) + | > loss_dur: 0.14815 (0.08859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.72697 (12.90079) + | > current_lr: 0.00008 + | > step_time: 2.50720 (2.99431) + | > loader_time: 0.00350 (0.05434) + + + --> STEP: 109/234 -- GLOBAL_STEP: 74755 + | > loss: -0.27657 (-0.33457) + | > log_mle: -0.45268 (-0.42587) + | > loss_dur: 0.17611 (0.09130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.57134 (13.31707) + | > current_lr: 0.00008 + | > step_time: 5.32670 (2.99206) + | > loader_time: 0.19080 (0.05442) + + + --> STEP: 114/234 -- GLOBAL_STEP: 74760 + | > loss: -0.29836 (-0.33300) + | > log_mle: -0.43588 (-0.42701) + | > loss_dur: 0.13752 (0.09401) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.68561 (13.75887) + | > current_lr: 0.00008 + | > step_time: 1.82340 (2.97306) + | > loader_time: 0.00270 (0.05451) + + + --> STEP: 119/234 -- GLOBAL_STEP: 74765 + | > loss: -0.29677 (-0.33106) + | > log_mle: -0.43514 (-0.42764) + | > loss_dur: 0.13838 (0.09658) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.71639 (14.09969) + | > current_lr: 0.00008 + | > step_time: 1.42140 (2.92884) + | > loader_time: 0.07400 (0.05499) + + + --> STEP: 124/234 -- GLOBAL_STEP: 74770 + | > loss: -0.31167 (-0.32899) + | > log_mle: -0.45759 (-0.42760) + | > loss_dur: 0.14593 (0.09861) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.66176 (14.51746) + | > current_lr: 0.00008 + | > step_time: 0.90410 (2.88152) + | > loader_time: 0.00210 (0.05288) + + + --> STEP: 129/234 -- GLOBAL_STEP: 74775 + | > loss: -0.28343 (-0.32796) + | > log_mle: -0.44387 (-0.42911) + | > loss_dur: 0.16044 (0.10115) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.67613 (15.33540) + | > current_lr: 0.00008 + | > step_time: 1.29060 (2.85344) + | > loader_time: 0.00250 (0.05233) + + + --> STEP: 134/234 -- GLOBAL_STEP: 74780 + | > loss: -0.32511 (-0.32807) + | > log_mle: -0.50616 (-0.43164) + | > loss_dur: 0.18104 (0.10357) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.87011 (15.87723) + | > current_lr: 0.00008 + | > step_time: 1.17930 (2.82874) + | > loader_time: 0.00210 (0.05265) + + + --> STEP: 139/234 -- GLOBAL_STEP: 74785 + | > loss: -0.38810 (-0.32776) + | > log_mle: -0.56823 (-0.43394) + | > loss_dur: 0.18014 (0.10618) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.43694 (16.54478) + | > current_lr: 0.00008 + | > step_time: 3.40410 (2.79224) + | > loader_time: 0.00260 (0.05144) + + + --> STEP: 144/234 -- GLOBAL_STEP: 74790 + | > loss: -0.34862 (-0.32759) + | > log_mle: -0.54337 (-0.43659) + | > loss_dur: 0.19474 (0.10900) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.69147 (17.40240) + | > current_lr: 0.00008 + | > step_time: 3.49180 (2.79312) + | > loader_time: 0.10160 (0.05163) + + + --> STEP: 149/234 -- GLOBAL_STEP: 74795 + | > loss: -0.40552 (-0.32859) + | > log_mle: -0.59241 (-0.43990) + | > loss_dur: 0.18689 (0.11131) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.04629 (18.16710) + | > current_lr: 0.00008 + | > step_time: 2.50860 (2.77482) + | > loader_time: 0.08310 (0.05160) + + + --> STEP: 154/234 -- GLOBAL_STEP: 74800 + | > loss: -0.36220 (-0.32966) + | > log_mle: -0.55115 (-0.44375) + | > loss_dur: 0.18895 (0.11409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.58588 (19.16668) + | > current_lr: 0.00008 + | > step_time: 3.29460 (2.78613) + | > loader_time: 0.09940 (0.05175) + + + --> STEP: 159/234 -- GLOBAL_STEP: 74805 + | > loss: -0.38172 (-0.33092) + | > log_mle: -0.57492 (-0.44748) + | > loss_dur: 0.19320 (0.11656) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.10100 (20.03181) + | > current_lr: 0.00008 + | > step_time: 3.19030 (2.78212) + | > loader_time: 0.00400 (0.05087) + + + --> STEP: 164/234 -- GLOBAL_STEP: 74810 + | > loss: -0.36232 (-0.33224) + | > log_mle: -0.56384 (-0.45102) + | > loss_dur: 0.20153 (0.11878) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.60408 (20.79978) + | > current_lr: 0.00008 + | > step_time: 1.19180 (2.86741) + | > loader_time: 0.01010 (0.05057) + + + --> STEP: 169/234 -- GLOBAL_STEP: 74815 + | > loss: -0.35783 (-0.33398) + | > log_mle: -0.56499 (-0.45495) + | > loss_dur: 0.20716 (0.12096) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.04680 (21.63533) + | > current_lr: 0.00008 + | > step_time: 2.31120 (2.85014) + | > loader_time: 0.48650 (0.05350) + + + --> STEP: 174/234 -- GLOBAL_STEP: 74820 + | > loss: -0.45070 (-0.33664) + | > log_mle: -0.65807 (-0.46012) + | > loss_dur: 0.20737 (0.12348) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.04239 (22.47940) + | > current_lr: 0.00008 + | > step_time: 3.63400 (2.84088) + | > loader_time: 0.19800 (0.05478) + + + --> STEP: 179/234 -- GLOBAL_STEP: 74825 + | > loss: -0.42107 (-0.33854) + | > log_mle: -0.65758 (-0.46473) + | > loss_dur: 0.23652 (0.12619) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.78585 (23.33291) + | > current_lr: 0.00008 + | > step_time: 3.50680 (2.84684) + | > loader_time: 0.00260 (0.05429) + + + --> STEP: 184/234 -- GLOBAL_STEP: 74830 + | > loss: -0.40774 (-0.34011) + | > log_mle: -0.61573 (-0.46865) + | > loss_dur: 0.20799 (0.12854) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.57387 (24.37475) + | > current_lr: 0.00008 + | > step_time: 4.81270 (2.87614) + | > loader_time: 0.08880 (0.05632) + + + --> STEP: 189/234 -- GLOBAL_STEP: 74835 + | > loss: -0.39120 (-0.34197) + | > log_mle: -0.60929 (-0.47294) + | > loss_dur: 0.21809 (0.13097) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 88.31489 (25.34702) + | > current_lr: 0.00008 + | > step_time: 2.68970 (2.94804) + | > loader_time: 0.00500 (0.05705) + + + --> STEP: 194/234 -- GLOBAL_STEP: 74840 + | > loss: -0.42772 (-0.34404) + | > log_mle: -0.63295 (-0.47689) + | > loss_dur: 0.20523 (0.13285) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.92208 (26.11902) + | > current_lr: 0.00008 + | > step_time: 5.39810 (2.99424) + | > loader_time: 0.20310 (0.05765) + + + --> STEP: 199/234 -- GLOBAL_STEP: 74845 + | > loss: -0.41914 (-0.34581) + | > log_mle: -0.64031 (-0.48079) + | > loss_dur: 0.22117 (0.13498) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.10701 (26.73969) + | > current_lr: 0.00008 + | > step_time: 1.49110 (3.03495) + | > loader_time: 0.00280 (0.05734) + + + --> STEP: 204/234 -- GLOBAL_STEP: 74850 + | > loss: -0.44187 (-0.34743) + | > log_mle: -0.67676 (-0.48458) + | > loss_dur: 0.23489 (0.13715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.26237 (27.33221) + | > current_lr: 0.00008 + | > step_time: 5.29930 (3.11503) + | > loader_time: 0.00460 (0.05824) + + + --> STEP: 209/234 -- GLOBAL_STEP: 74855 + | > loss: -0.42152 (-0.34971) + | > log_mle: -0.64278 (-0.48890) + | > loss_dur: 0.22126 (0.13919) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.67468 (28.07869) + | > current_lr: 0.00008 + | > step_time: 1.79720 (3.10312) + | > loader_time: 0.00430 (0.05733) + + + --> STEP: 214/234 -- GLOBAL_STEP: 74860 + | > loss: -0.48552 (-0.35302) + | > log_mle: -0.68925 (-0.49427) + | > loss_dur: 0.20373 (0.14125) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.03209 (29.16457) + | > current_lr: 0.00008 + | > step_time: 6.21560 (3.22747) + | > loader_time: 0.19740 (0.05888) + + + --> STEP: 219/234 -- GLOBAL_STEP: 74865 + | > loss: -0.56023 (-0.35597) + | > log_mle: -0.78996 (-0.49952) + | > loss_dur: 0.22973 (0.14355) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 93.37509 (30.17599) + | > current_lr: 0.00008 + | > step_time: 4.02460 (3.24796) + | > loader_time: 0.07960 (0.05836) + + + --> STEP: 224/234 -- GLOBAL_STEP: 74870 + | > loss: -0.50682 (-0.35877) + | > log_mle: -0.74260 (-0.50433) + | > loss_dur: 0.23579 (0.14556) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.83707 (31.20467) + | > current_lr: 0.00008 + | > step_time: 0.24400 (3.20734) + | > loader_time: 0.00470 (0.05751) + + + --> STEP: 229/234 -- GLOBAL_STEP: 74875 + | > loss: -0.46954 (-0.36160) + | > log_mle: -0.76483 (-0.50970) + | > loss_dur: 0.29529 (0.14811) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.04716 (32.56754) + | > current_lr: 0.00008 + | > step_time: 0.24840 (3.14266) + | > loader_time: 0.00440 (0.05634) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00366 (-0.04711) + | > avg_loss: -0.34944 (-0.02692) + | > avg_log_mle: -0.56667 (+0.00045) + | > avg_loss_dur: 0.21724 (-0.02738) + + + > EPOCH: 320/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 04:42:14)  + + --> STEP: 0/234 -- GLOBAL_STEP: 74880 + | > loss: -0.35651 (-0.35651) + | > log_mle: -0.52327 (-0.52327) + | > loss_dur: 0.16677 (0.16677) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.90527 (32.90527) + | > current_lr: 0.00008 + | > step_time: 6.40610 (6.40615) + | > loader_time: 20.93600 (20.93596) + + + --> STEP: 5/234 -- GLOBAL_STEP: 74885 + | > loss: -0.34594 (-0.34763) + | > log_mle: -0.43942 (-0.43973) + | > loss_dur: 0.09348 (0.09209) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.34842 (22.79338) + | > current_lr: 0.00008 + | > step_time: 9.99750 (7.14878) + | > loader_time: 0.00770 (0.06002) + + + --> STEP: 10/234 -- GLOBAL_STEP: 74890 + | > loss: -0.37087 (-0.35920) + | > log_mle: -0.45253 (-0.44603) + | > loss_dur: 0.08166 (0.08683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.37914 (19.95402) + | > current_lr: 0.00008 + | > step_time: 4.49130 (4.98348) + | > loader_time: 0.10310 (0.05288) + + + --> STEP: 15/234 -- GLOBAL_STEP: 74895 + | > loss: -0.40764 (-0.37203) + | > log_mle: -0.46563 (-0.45160) + | > loss_dur: 0.05799 (0.07957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.73228 (18.15551) + | > current_lr: 0.00008 + | > step_time: 1.02730 (4.00113) + | > loader_time: 0.00120 (0.05334) + + + --> STEP: 20/234 -- GLOBAL_STEP: 74900 + | > loss: -0.40357 (-0.37968) + | > log_mle: -0.45939 (-0.45406) + | > loss_dur: 0.05583 (0.07438) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.23472 (16.52511) + | > current_lr: 0.00008 + | > step_time: 3.10160 (3.73106) + | > loader_time: 0.00410 (0.04890) + + + --> STEP: 25/234 -- GLOBAL_STEP: 74905 + | > loss: -0.38366 (-0.38076) + | > log_mle: -0.44561 (-0.45295) + | > loss_dur: 0.06195 (0.07219) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.53314 (15.39247) + | > current_lr: 0.00008 + | > step_time: 1.39480 (3.32717) + | > loader_time: 0.00110 (0.03943) + + + --> STEP: 30/234 -- GLOBAL_STEP: 74910 + | > loss: -0.37359 (-0.38171) + | > log_mle: -0.44075 (-0.45222) + | > loss_dur: 0.06716 (0.07050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.49639 (14.77391) + | > current_lr: 0.00008 + | > step_time: 2.90130 (3.08364) + | > loader_time: 0.08530 (0.03597) + + + --> STEP: 35/234 -- GLOBAL_STEP: 74915 + | > loss: -0.32970 (-0.37795) + | > log_mle: -0.41449 (-0.44880) + | > loss_dur: 0.08479 (0.07085) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.19322 (14.20029) + | > current_lr: 0.00008 + | > step_time: 1.43590 (2.87557) + | > loader_time: 0.00380 (0.03600) + + + --> STEP: 40/234 -- GLOBAL_STEP: 74920 + | > loss: -0.33138 (-0.37319) + | > log_mle: -0.41640 (-0.44492) + | > loss_dur: 0.08502 (0.07172) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.20037 (13.81989) + | > current_lr: 0.00008 + | > step_time: 1.80820 (2.73034) + | > loader_time: 0.00310 (0.03406) + + + --> STEP: 45/234 -- GLOBAL_STEP: 74925 + | > loss: -0.33023 (-0.36964) + | > log_mle: -0.42271 (-0.44179) + | > loss_dur: 0.09249 (0.07215) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.20115 (13.37664) + | > current_lr: 0.00008 + | > step_time: 1.14480 (2.64620) + | > loader_time: 0.00310 (0.03264) + + + --> STEP: 50/234 -- GLOBAL_STEP: 74930 + | > loss: -0.35721 (-0.36825) + | > log_mle: -0.42214 (-0.44014) + | > loss_dur: 0.06493 (0.07189) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27286 (12.77131) + | > current_lr: 0.00008 + | > step_time: 2.60860 (2.55577) + | > loader_time: 0.00280 (0.02960) + + + --> STEP: 55/234 -- GLOBAL_STEP: 74935 + | > loss: -0.35332 (-0.36552) + | > log_mle: -0.42326 (-0.43797) + | > loss_dur: 0.06995 (0.07245) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.67973 (12.46754) + | > current_lr: 0.00008 + | > step_time: 2.74080 (2.51605) + | > loader_time: 0.00270 (0.02710) + + + --> STEP: 60/234 -- GLOBAL_STEP: 74940 + | > loss: -0.30826 (-0.36262) + | > log_mle: -0.40932 (-0.43587) + | > loss_dur: 0.10106 (0.07325) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.13726 (12.54273) + | > current_lr: 0.00008 + | > step_time: 2.30800 (2.52547) + | > loader_time: 0.00190 (0.02515) + + + --> STEP: 65/234 -- GLOBAL_STEP: 74945 + | > loss: -0.32405 (-0.35882) + | > log_mle: -0.40649 (-0.43344) + | > loss_dur: 0.08244 (0.07462) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.17087 (12.71400) + | > current_lr: 0.00008 + | > step_time: 2.36260 (2.47449) + | > loader_time: 0.00290 (0.02497) + + + --> STEP: 70/234 -- GLOBAL_STEP: 74950 + | > loss: -0.28490 (-0.35544) + | > log_mle: -0.38316 (-0.43088) + | > loss_dur: 0.09825 (0.07544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.20812 (12.61341) + | > current_lr: 0.00008 + | > step_time: 1.11600 (2.40698) + | > loader_time: 0.08740 (0.02577) + + + --> STEP: 75/234 -- GLOBAL_STEP: 74955 + | > loss: -0.29940 (-0.35167) + | > log_mle: -0.40012 (-0.42897) + | > loss_dur: 0.10072 (0.07730) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.91584 (12.55745) + | > current_lr: 0.00008 + | > step_time: 1.29160 (2.34803) + | > loader_time: 0.00290 (0.02544) + + + --> STEP: 80/234 -- GLOBAL_STEP: 74960 + | > loss: -0.31750 (-0.34896) + | > log_mle: -0.39744 (-0.42717) + | > loss_dur: 0.07994 (0.07821) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.31002 (12.43231) + | > current_lr: 0.00008 + | > step_time: 1.80220 (2.30220) + | > loader_time: 0.00230 (0.02402) + + + --> STEP: 85/234 -- GLOBAL_STEP: 74965 + | > loss: -0.29676 (-0.34594) + | > log_mle: -0.39458 (-0.42558) + | > loss_dur: 0.09782 (0.07964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.31228 (12.32967) + | > current_lr: 0.00008 + | > step_time: 3.07920 (2.31328) + | > loader_time: 0.00200 (0.02386) + + + --> STEP: 90/234 -- GLOBAL_STEP: 74970 + | > loss: -0.29045 (-0.34337) + | > log_mle: -0.41072 (-0.42493) + | > loss_dur: 0.12026 (0.08155) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.47483 (12.33225) + | > current_lr: 0.00008 + | > step_time: 1.94460 (2.29309) + | > loader_time: 0.00210 (0.02279) + + + --> STEP: 95/234 -- GLOBAL_STEP: 74975 + | > loss: -0.32679 (-0.34151) + | > log_mle: -0.48066 (-0.42570) + | > loss_dur: 0.15387 (0.08419) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.72569 (12.67837) + | > current_lr: 0.00008 + | > step_time: 2.06710 (2.26356) + | > loader_time: 0.00230 (0.02176) + + + --> STEP: 100/234 -- GLOBAL_STEP: 74980 + | > loss: -0.29730 (-0.33923) + | > log_mle: -0.41640 (-0.42517) + | > loss_dur: 0.11909 (0.08594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.34026 (12.78199) + | > current_lr: 0.00008 + | > step_time: 2.26120 (2.25377) + | > loader_time: 0.08410 (0.02164) + + + --> STEP: 105/234 -- GLOBAL_STEP: 74985 + | > loss: -0.28346 (-0.33728) + | > log_mle: -0.39827 (-0.42567) + | > loss_dur: 0.11480 (0.08839) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.29508 (13.23702) + | > current_lr: 0.00008 + | > step_time: 2.92640 (2.26128) + | > loader_time: 0.00240 (0.02073) + + + --> STEP: 110/234 -- GLOBAL_STEP: 74990 + | > loss: -0.27270 (-0.33454) + | > log_mle: -0.41004 (-0.42564) + | > loss_dur: 0.13734 (0.09109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.26342 (13.97172) + | > current_lr: 0.00008 + | > step_time: 2.89730 (2.27067) + | > loader_time: 0.00480 (0.02232) + + + --> STEP: 115/234 -- GLOBAL_STEP: 74995 + | > loss: -0.26971 (-0.33236) + | > log_mle: -0.42870 (-0.42630) + | > loss_dur: 0.15899 (0.09394) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.25264 (15.28846) + | > current_lr: 0.00008 + | > step_time: 1.90520 (2.24779) + | > loader_time: 0.08970 (0.02357) + + + --> STEP: 120/234 -- GLOBAL_STEP: 75000 + | > loss: -0.32767 (-0.33060) + | > log_mle: -0.48279 (-0.42710) + | > loss_dur: 0.15512 (0.09650) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.56442 (15.68390) + | > current_lr: 0.00008 + | > step_time: 1.91750 (2.22701) + | > loader_time: 0.00330 (0.02272) + + + --> STEP: 125/234 -- GLOBAL_STEP: 75005 + | > loss: -0.29956 (-0.32821) + | > log_mle: -0.46117 (-0.42678) + | > loss_dur: 0.16161 (0.09856) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.16008 (16.06796) + | > current_lr: 0.00008 + | > step_time: 1.11290 (2.19769) + | > loader_time: 0.00250 (0.02267) + + + --> STEP: 130/234 -- GLOBAL_STEP: 75010 + | > loss: -0.31582 (-0.32739) + | > log_mle: -0.48329 (-0.42848) + | > loss_dur: 0.16747 (0.10109) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.26989 (16.40607) + | > current_lr: 0.00008 + | > step_time: 2.61240 (2.18721) + | > loader_time: 0.08840 (0.02392) + + + --> STEP: 135/234 -- GLOBAL_STEP: 75015 + | > loss: -0.27371 (-0.32666) + | > log_mle: -0.40685 (-0.43016) + | > loss_dur: 0.13314 (0.10350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.07067 (17.02740) + | > current_lr: 0.00008 + | > step_time: 2.80220 (2.20181) + | > loader_time: 0.08800 (0.02444) + + + --> STEP: 140/234 -- GLOBAL_STEP: 75020 + | > loss: -0.26097 (-0.32590) + | > log_mle: -0.43682 (-0.43223) + | > loss_dur: 0.17584 (0.10633) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.31573 (18.01153) + | > current_lr: 0.00008 + | > step_time: 3.90110 (2.25314) + | > loader_time: 0.09480 (0.02781) + + + --> STEP: 145/234 -- GLOBAL_STEP: 75025 + | > loss: -0.36459 (-0.32602) + | > log_mle: -0.53774 (-0.43512) + | > loss_dur: 0.17315 (0.10909) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.31187 (19.02095) + | > current_lr: 0.00008 + | > step_time: 1.19690 (2.26102) + | > loader_time: 0.07640 (0.03059) + + + --> STEP: 150/234 -- GLOBAL_STEP: 75030 + | > loss: -0.32940 (-0.32631) + | > log_mle: -0.52073 (-0.43783) + | > loss_dur: 0.19133 (0.11152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.17641 (19.86419) + | > current_lr: 0.00008 + | > step_time: 2.38050 (2.27947) + | > loader_time: 0.00270 (0.03035) + + + --> STEP: 155/234 -- GLOBAL_STEP: 75035 + | > loss: -0.38076 (-0.32761) + | > log_mle: -0.58146 (-0.44159) + | > loss_dur: 0.20070 (0.11398) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.63737 (20.80575) + | > current_lr: 0.00008 + | > step_time: 5.78520 (2.32316) + | > loader_time: 0.00420 (0.02993) + + + --> STEP: 160/234 -- GLOBAL_STEP: 75040 + | > loss: -0.38927 (-0.32847) + | > log_mle: -0.59280 (-0.44500) + | > loss_dur: 0.20353 (0.11653) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.37255 (21.36093) + | > current_lr: 0.00008 + | > step_time: 1.90340 (2.33477) + | > loader_time: 0.09100 (0.03014) + + + --> STEP: 165/234 -- GLOBAL_STEP: 75045 + | > loss: -0.40122 (-0.32947) + | > log_mle: -0.59297 (-0.44825) + | > loss_dur: 0.19175 (0.11878) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.78296 (22.34141) + | > current_lr: 0.00008 + | > step_time: 2.68050 (2.36737) + | > loader_time: 0.00210 (0.02935) + + + --> STEP: 170/234 -- GLOBAL_STEP: 75050 + | > loss: -0.40563 (-0.33113) + | > log_mle: -0.62902 (-0.45219) + | > loss_dur: 0.22339 (0.12106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 61.49543 (23.19700) + | > current_lr: 0.00008 + | > step_time: 2.69370 (2.34820) + | > loader_time: 0.09910 (0.02963) + + + --> STEP: 175/234 -- GLOBAL_STEP: 75055 + | > loss: -0.38049 (-0.33331) + | > log_mle: -0.60215 (-0.45687) + | > loss_dur: 0.22166 (0.12356) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.34810 (24.28489) + | > current_lr: 0.00008 + | > step_time: 2.90140 (2.39725) + | > loader_time: 0.08680 (0.03382) + + + --> STEP: 180/234 -- GLOBAL_STEP: 75060 + | > loss: -0.38708 (-0.33508) + | > log_mle: -0.59844 (-0.46123) + | > loss_dur: 0.21136 (0.12615) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.82083 (25.20880) + | > current_lr: 0.00008 + | > step_time: 5.00420 (2.42513) + | > loader_time: 0.00330 (0.03451) + + + --> STEP: 185/234 -- GLOBAL_STEP: 75065 + | > loss: -0.41285 (-0.33659) + | > log_mle: -0.63198 (-0.46519) + | > loss_dur: 0.21913 (0.12859) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.66500 (26.14318) + | > current_lr: 0.00008 + | > step_time: 8.09730 (2.56286) + | > loader_time: 0.50660 (0.03744) + + + --> STEP: 190/234 -- GLOBAL_STEP: 75070 + | > loss: -0.41698 (-0.33839) + | > log_mle: -0.62181 (-0.46933) + | > loss_dur: 0.20483 (0.13094) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.26906 (26.64880) + | > current_lr: 0.00008 + | > step_time: 2.89870 (2.64802) + | > loader_time: 0.10080 (0.03851) + + + --> STEP: 195/234 -- GLOBAL_STEP: 75075 + | > loss: -0.40985 (-0.34079) + | > log_mle: -0.63751 (-0.47373) + | > loss_dur: 0.22766 (0.13294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 92.84274 (27.83640) + | > current_lr: 0.00008 + | > step_time: 3.80930 (2.65654) + | > loader_time: 0.11270 (0.03915) + + + --> STEP: 200/234 -- GLOBAL_STEP: 75080 + | > loss: -0.40862 (-0.34277) + | > log_mle: -0.65239 (-0.47787) + | > loss_dur: 0.24377 (0.13509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.60341 (28.50154) + | > current_lr: 0.00008 + | > step_time: 1.99980 (2.65416) + | > loader_time: 0.00390 (0.04007) + + + --> STEP: 205/234 -- GLOBAL_STEP: 75085 + | > loss: -0.41356 (-0.34460) + | > log_mle: -0.63652 (-0.48178) + | > loss_dur: 0.22297 (0.13718) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.57288 (29.04084) + | > current_lr: 0.00008 + | > step_time: 4.70120 (2.72743) + | > loader_time: 0.01040 (0.04020) + + + --> STEP: 210/234 -- GLOBAL_STEP: 75090 + | > loss: -0.47421 (-0.34703) + | > log_mle: -0.70653 (-0.48638) + | > loss_dur: 0.23232 (0.13935) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.26717 (30.02213) + | > current_lr: 0.00008 + | > step_time: 3.19080 (2.83717) + | > loader_time: 0.00430 (0.04113) + + + --> STEP: 215/234 -- GLOBAL_STEP: 75095 + | > loss: -0.44448 (-0.34995) + | > log_mle: -0.67006 (-0.49134) + | > loss_dur: 0.22558 (0.14139) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.80695 (30.75254) + | > current_lr: 0.00008 + | > step_time: 6.40730 (2.91619) + | > loader_time: 0.20350 (0.04169) + + + --> STEP: 220/234 -- GLOBAL_STEP: 75100 + | > loss: -0.49034 (-0.35325) + | > log_mle: -0.71886 (-0.49672) + | > loss_dur: 0.22852 (0.14347) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.12032 (31.57299) + | > current_lr: 0.00008 + | > step_time: 7.70750 (2.97408) + | > loader_time: 0.00440 (0.04252) + + + --> STEP: 225/234 -- GLOBAL_STEP: 75105 + | > loss: -0.55835 (-0.35651) + | > log_mle: -0.80402 (-0.50200) + | > loss_dur: 0.24567 (0.14549) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.31035 (32.30309) + | > current_lr: 0.00008 + | > step_time: 0.24530 (2.94011) + | > loader_time: 0.00260 (0.04168) + + + --> STEP: 230/234 -- GLOBAL_STEP: 75110 + | > loss: -0.53457 (-0.35933) + | > log_mle: -0.84546 (-0.50763) + | > loss_dur: 0.31090 (0.14829) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.90635 (33.56582) + | > current_lr: 0.00008 + | > step_time: 0.25960 (2.88153) + | > loader_time: 0.00460 (0.04087) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.78229 (+0.77863) + | > avg_loss: -0.32870 (+0.02073) + | > avg_log_mle: -0.56506 (+0.00162) + | > avg_loss_dur: 0.23635 (+0.01912) + + + > EPOCH: 321/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 04:54:40)  + + --> STEP: 1/234 -- GLOBAL_STEP: 75115 + | > loss: -0.36237 (-0.36237) + | > log_mle: -0.43910 (-0.43910) + | > loss_dur: 0.07673 (0.07673) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.22146 (23.22146) + | > current_lr: 0.00008 + | > step_time: 7.48610 (7.48613) + | > loader_time: 0.00660 (0.00658) + + + --> STEP: 6/234 -- GLOBAL_STEP: 75120 + | > loss: -0.37885 (-0.35157) + | > log_mle: -0.44315 (-0.44211) + | > loss_dur: 0.06430 (0.09054) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.33912 (20.03110) + | > current_lr: 0.00008 + | > step_time: 6.41190 (5.86861) + | > loader_time: 0.08920 (0.03283) + + + --> STEP: 11/234 -- GLOBAL_STEP: 75125 + | > loss: -0.40934 (-0.36577) + | > log_mle: -0.46777 (-0.44981) + | > loss_dur: 0.05843 (0.08404) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.97735 (18.48432) + | > current_lr: 0.00008 + | > step_time: 2.65650 (5.77360) + | > loader_time: 0.00170 (0.02787) + + + --> STEP: 16/234 -- GLOBAL_STEP: 75130 + | > loss: -0.41504 (-0.37704) + | > log_mle: -0.47388 (-0.45457) + | > loss_dur: 0.05885 (0.07753) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.64547 (17.07513) + | > current_lr: 0.00008 + | > step_time: 1.10210 (4.39908) + | > loader_time: 0.00150 (0.03073) + + + --> STEP: 21/234 -- GLOBAL_STEP: 75135 + | > loss: -0.38283 (-0.38148) + | > log_mle: -0.44277 (-0.45510) + | > loss_dur: 0.05994 (0.07362) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.86422 (15.60639) + | > current_lr: 0.00008 + | > step_time: 3.90290 (3.98071) + | > loader_time: 0.00560 (0.02846) + + + --> STEP: 26/234 -- GLOBAL_STEP: 75140 + | > loss: -0.38331 (-0.38285) + | > log_mle: -0.45071 (-0.45489) + | > loss_dur: 0.06740 (0.07204) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.13182 (14.38212) + | > current_lr: 0.00008 + | > step_time: 3.25190 (3.94424) + | > loader_time: 0.09890 (0.03069) + + + --> STEP: 31/234 -- GLOBAL_STEP: 75145 + | > loss: -0.36003 (-0.38305) + | > log_mle: -0.43649 (-0.45385) + | > loss_dur: 0.07646 (0.07080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.92189 (13.72354) + | > current_lr: 0.00008 + | > step_time: 1.66310 (3.58597) + | > loader_time: 0.00190 (0.02609) + + + --> STEP: 36/234 -- GLOBAL_STEP: 75150 + | > loss: -0.33886 (-0.37918) + | > log_mle: -0.41552 (-0.45011) + | > loss_dur: 0.07666 (0.07093) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.13750 (13.78365) + | > current_lr: 0.00008 + | > step_time: 2.09060 (3.55171) + | > loader_time: 0.00150 (0.03063) + + + --> STEP: 41/234 -- GLOBAL_STEP: 75155 + | > loss: -0.36226 (-0.37514) + | > log_mle: -0.42572 (-0.44644) + | > loss_dur: 0.06346 (0.07130) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.94346 (13.61409) + | > current_lr: 0.00008 + | > step_time: 2.48960 (3.40906) + | > loader_time: 0.00200 (0.03310) + + + --> STEP: 46/234 -- GLOBAL_STEP: 75160 + | > loss: -0.33653 (-0.37077) + | > log_mle: -0.41998 (-0.44326) + | > loss_dur: 0.08345 (0.07249) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.27902 (13.18484) + | > current_lr: 0.00008 + | > step_time: 1.89140 (3.34465) + | > loader_time: 0.00190 (0.03387) + + + --> STEP: 51/234 -- GLOBAL_STEP: 75165 + | > loss: -0.35759 (-0.36968) + | > log_mle: -0.42307 (-0.44183) + | > loss_dur: 0.06548 (0.07214) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.08512 (12.54232) + | > current_lr: 0.00008 + | > step_time: 2.10630 (3.24003) + | > loader_time: 0.00220 (0.03076) + + + --> STEP: 56/234 -- GLOBAL_STEP: 75170 + | > loss: -0.34380 (-0.36710) + | > log_mle: -0.42197 (-0.43969) + | > loss_dur: 0.07818 (0.07259) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.57358 (12.24529) + | > current_lr: 0.00008 + | > step_time: 1.40890 (3.06664) + | > loader_time: 0.00200 (0.02823) + + + --> STEP: 61/234 -- GLOBAL_STEP: 75175 + | > loss: -0.32404 (-0.36365) + | > log_mle: -0.40717 (-0.43733) + | > loss_dur: 0.08313 (0.07368) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.23949 (12.08943) + | > current_lr: 0.00008 + | > step_time: 2.60150 (2.97303) + | > loader_time: 0.00280 (0.02901) + + + --> STEP: 66/234 -- GLOBAL_STEP: 75180 + | > loss: -0.32873 (-0.36002) + | > log_mle: -0.40198 (-0.43487) + | > loss_dur: 0.07325 (0.07485) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.24233 (11.96027) + | > current_lr: 0.00008 + | > step_time: 1.12340 (2.87456) + | > loader_time: 0.00240 (0.02704) + + + --> STEP: 71/234 -- GLOBAL_STEP: 75185 + | > loss: -0.30670 (-0.35622) + | > log_mle: -0.41252 (-0.43246) + | > loss_dur: 0.10582 (0.07624) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.44209 (11.95889) + | > current_lr: 0.00008 + | > step_time: 2.29020 (2.79924) + | > loader_time: 0.00210 (0.02881) + + + --> STEP: 76/234 -- GLOBAL_STEP: 75190 + | > loss: -0.30359 (-0.35263) + | > log_mle: -0.40573 (-0.43052) + | > loss_dur: 0.10213 (0.07789) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.08592 (11.80901) + | > current_lr: 0.00008 + | > step_time: 1.80470 (2.75569) + | > loader_time: 0.08620 (0.02942) + + + --> STEP: 81/234 -- GLOBAL_STEP: 75195 + | > loss: -0.31361 (-0.35008) + | > log_mle: -0.41351 (-0.42885) + | > loss_dur: 0.09990 (0.07877) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.36405 (11.65125) + | > current_lr: 0.00008 + | > step_time: 1.69510 (2.74338) + | > loader_time: 0.00240 (0.02778) + + + --> STEP: 86/234 -- GLOBAL_STEP: 75200 + | > loss: -0.31292 (-0.34709) + | > log_mle: -0.40992 (-0.42721) + | > loss_dur: 0.09700 (0.08012) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.52369 (11.66434) + | > current_lr: 0.00008 + | > step_time: 2.38770 (2.68037) + | > loader_time: 0.00230 (0.02733) + + + --> STEP: 91/234 -- GLOBAL_STEP: 75205 + | > loss: -0.29907 (-0.34442) + | > log_mle: -0.41333 (-0.42655) + | > loss_dur: 0.11427 (0.08213) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.72016 (11.80134) + | > current_lr: 0.00008 + | > step_time: 2.89380 (2.67510) + | > loader_time: 0.00490 (0.02792) + + + --> STEP: 96/234 -- GLOBAL_STEP: 75210 + | > loss: -0.29684 (-0.34259) + | > log_mle: -0.40063 (-0.42719) + | > loss_dur: 0.10378 (0.08460) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.19243 (12.31422) + | > current_lr: 0.00008 + | > step_time: 1.29810 (2.62078) + | > loader_time: 0.00220 (0.02663) + + + --> STEP: 101/234 -- GLOBAL_STEP: 75215 + | > loss: -0.28949 (-0.34020) + | > log_mle: -0.44003 (-0.42703) + | > loss_dur: 0.15054 (0.08683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.99173 (12.65210) + | > current_lr: 0.00008 + | > step_time: 1.49140 (2.58920) + | > loader_time: 0.00230 (0.02629) + + + --> STEP: 106/234 -- GLOBAL_STEP: 75220 + | > loss: -0.27166 (-0.33800) + | > log_mle: -0.43419 (-0.42744) + | > loss_dur: 0.16254 (0.08945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.00072 (13.05238) + | > current_lr: 0.00008 + | > step_time: 2.00910 (2.56730) + | > loader_time: 0.00370 (0.02679) + + + --> STEP: 111/234 -- GLOBAL_STEP: 75225 + | > loss: -0.31832 (-0.33576) + | > log_mle: -0.49235 (-0.42803) + | > loss_dur: 0.17403 (0.09227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.43096 (13.59677) + | > current_lr: 0.00008 + | > step_time: 3.18730 (2.56042) + | > loader_time: 0.00790 (0.02750) + + + --> STEP: 116/234 -- GLOBAL_STEP: 75230 + | > loss: -0.28543 (-0.33366) + | > log_mle: -0.45541 (-0.42879) + | > loss_dur: 0.16998 (0.09513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.46712 (14.04259) + | > current_lr: 0.00008 + | > step_time: 1.79220 (2.54644) + | > loader_time: 0.00240 (0.02889) + + + --> STEP: 121/234 -- GLOBAL_STEP: 75235 + | > loss: -0.24713 (-0.33170) + | > log_mle: -0.37558 (-0.42890) + | > loss_dur: 0.12845 (0.09720) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.36309 (14.38018) + | > current_lr: 0.00008 + | > step_time: 2.09840 (2.54281) + | > loader_time: 0.00710 (0.02924) + + + --> STEP: 126/234 -- GLOBAL_STEP: 75240 + | > loss: -0.32267 (-0.32983) + | > log_mle: -0.49876 (-0.42955) + | > loss_dur: 0.17609 (0.09972) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 26.71656 (14.64164) + | > current_lr: 0.00008 + | > step_time: 1.38190 (2.51941) + | > loader_time: 0.00250 (0.03031) + + + --> STEP: 131/234 -- GLOBAL_STEP: 75245 + | > loss: -0.38412 (-0.32938) + | > log_mle: -0.55696 (-0.43156) + | > loss_dur: 0.17283 (0.10218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.94167 (15.20897) + | > current_lr: 0.00008 + | > step_time: 2.50380 (2.52166) + | > loader_time: 0.08430 (0.03053) + + + --> STEP: 136/234 -- GLOBAL_STEP: 75250 + | > loss: -0.39736 (-0.32934) + | > log_mle: -0.59806 (-0.43373) + | > loss_dur: 0.20070 (0.10439) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.96076 (15.76784) + | > current_lr: 0.00008 + | > step_time: 1.59810 (2.49012) + | > loader_time: 0.00330 (0.03089) + + + --> STEP: 141/234 -- GLOBAL_STEP: 75255 + | > loss: -0.33338 (-0.32864) + | > log_mle: -0.49886 (-0.43554) + | > loss_dur: 0.16547 (0.10690) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.11033 (16.29455) + | > current_lr: 0.00008 + | > step_time: 1.40170 (2.48414) + | > loader_time: 0.09860 (0.03234) + + + --> STEP: 146/234 -- GLOBAL_STEP: 75260 + | > loss: -0.35441 (-0.32939) + | > log_mle: -0.55027 (-0.43922) + | > loss_dur: 0.19585 (0.10983) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.89835 (17.25317) + | > current_lr: 0.00008 + | > step_time: 2.40540 (2.47709) + | > loader_time: 0.08660 (0.03193) + + + --> STEP: 151/234 -- GLOBAL_STEP: 75265 + | > loss: -0.34117 (-0.32990) + | > log_mle: -0.51433 (-0.44200) + | > loss_dur: 0.17316 (0.11210) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.56363 (17.96206) + | > current_lr: 0.00008 + | > step_time: 2.90410 (2.49323) + | > loader_time: 0.00400 (0.03270) + + + --> STEP: 156/234 -- GLOBAL_STEP: 75270 + | > loss: -0.37293 (-0.33184) + | > log_mle: -0.56563 (-0.44656) + | > loss_dur: 0.19270 (0.11472) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.57109 (19.14790) + | > current_lr: 0.00008 + | > step_time: 2.70350 (2.49946) + | > loader_time: 0.09220 (0.03290) + + + --> STEP: 161/234 -- GLOBAL_STEP: 75275 + | > loss: -0.40772 (-0.33300) + | > log_mle: -0.58413 (-0.45017) + | > loss_dur: 0.17641 (0.11716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.28804 (20.13367) + | > current_lr: 0.00008 + | > step_time: 1.30800 (2.49156) + | > loader_time: 0.00320 (0.03253) + + + --> STEP: 166/234 -- GLOBAL_STEP: 75280 + | > loss: -0.34304 (-0.33358) + | > log_mle: -0.51425 (-0.45299) + | > loss_dur: 0.17121 (0.11941) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.42614 (20.82777) + | > current_lr: 0.00008 + | > step_time: 2.11140 (2.49002) + | > loader_time: 0.00290 (0.03398) + + + --> STEP: 171/234 -- GLOBAL_STEP: 75285 + | > loss: -0.42688 (-0.33559) + | > log_mle: -0.63071 (-0.45749) + | > loss_dur: 0.20383 (0.12190) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.50021 (21.87555) + | > current_lr: 0.00008 + | > step_time: 2.00190 (2.49078) + | > loader_time: 0.00360 (0.03361) + + + --> STEP: 176/234 -- GLOBAL_STEP: 75290 + | > loss: -0.39403 (-0.33748) + | > log_mle: -0.59727 (-0.46194) + | > loss_dur: 0.20324 (0.12446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.89382 (22.63699) + | > current_lr: 0.00008 + | > step_time: 3.00700 (2.48931) + | > loader_time: 0.09470 (0.03329) + + + --> STEP: 181/234 -- GLOBAL_STEP: 75295 + | > loss: -0.33361 (-0.33883) + | > log_mle: -0.53481 (-0.46579) + | > loss_dur: 0.20120 (0.12696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.17441 (23.44698) + | > current_lr: 0.00008 + | > step_time: 3.21210 (2.54489) + | > loader_time: 0.20200 (0.03518) + + + --> STEP: 186/234 -- GLOBAL_STEP: 75300 + | > loss: -0.34665 (-0.34044) + | > log_mle: -0.57468 (-0.46998) + | > loss_dur: 0.22803 (0.12953) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.76111 (24.64001) + | > current_lr: 0.00008 + | > step_time: 1.90410 (2.54368) + | > loader_time: 0.07790 (0.03474) + + + --> STEP: 191/234 -- GLOBAL_STEP: 75305 + | > loss: -0.39525 (-0.34212) + | > log_mle: -0.60010 (-0.47378) + | > loss_dur: 0.20485 (0.13166) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.12832 (25.31997) + | > current_lr: 0.00008 + | > step_time: 4.52350 (2.55513) + | > loader_time: 0.09150 (0.03548) + + + --> STEP: 196/234 -- GLOBAL_STEP: 75310 + | > loss: -0.37526 (-0.34425) + | > log_mle: -0.60320 (-0.47805) + | > loss_dur: 0.22794 (0.13380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.21809 (26.07872) + | > current_lr: 0.00008 + | > step_time: 4.58830 (2.61072) + | > loader_time: 0.00900 (0.03705) + + + --> STEP: 201/234 -- GLOBAL_STEP: 75315 + | > loss: -0.31545 (-0.34553) + | > log_mle: -0.53609 (-0.48147) + | > loss_dur: 0.22064 (0.13594) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.07537 (27.27880) + | > current_lr: 0.00008 + | > step_time: 3.30170 (2.64066) + | > loader_time: 0.08850 (0.03809) + + + --> STEP: 206/234 -- GLOBAL_STEP: 75320 + | > loss: -0.45057 (-0.34777) + | > log_mle: -0.66792 (-0.48573) + | > loss_dur: 0.21734 (0.13796) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.96962 (28.15313) + | > current_lr: 0.00008 + | > step_time: 2.70470 (2.76487) + | > loader_time: 0.10370 (0.03918) + + + --> STEP: 211/234 -- GLOBAL_STEP: 75325 + | > loss: -0.51145 (-0.35055) + | > log_mle: -0.75139 (-0.49069) + | > loss_dur: 0.23994 (0.14015) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.18044 (29.03518) + | > current_lr: 0.00008 + | > step_time: 4.70690 (2.77770) + | > loader_time: 0.00650 (0.04142) + + + --> STEP: 216/234 -- GLOBAL_STEP: 75330 + | > loss: -0.46919 (-0.35264) + | > log_mle: -0.71547 (-0.49491) + | > loss_dur: 0.24628 (0.14227) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.04246 (30.05097) + | > current_lr: 0.00008 + | > step_time: 7.00760 (2.86243) + | > loader_time: 0.10250 (0.04425) + + + --> STEP: 221/234 -- GLOBAL_STEP: 75335 + | > loss: -0.42379 (-0.35514) + | > log_mle: -0.63627 (-0.49942) + | > loss_dur: 0.21248 (0.14428) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.29578 (30.60780) + | > current_lr: 0.00008 + | > step_time: 0.79980 (2.85699) + | > loader_time: 0.08370 (0.04531) + + + --> STEP: 226/234 -- GLOBAL_STEP: 75340 + | > loss: -0.50295 (-0.35823) + | > log_mle: -0.74500 (-0.50467) + | > loss_dur: 0.24205 (0.14644) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.78988 (31.56875) + | > current_lr: 0.00008 + | > step_time: 0.24290 (2.80210) + | > loader_time: 0.00820 (0.04445) + + + --> STEP: 231/234 -- GLOBAL_STEP: 75345 + | > loss: -0.45410 (-0.36050) + | > log_mle: -0.82504 (-0.51032) + | > loss_dur: 0.37093 (0.14982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 75.24796 (32.35365) + | > current_lr: 0.00008 + | > step_time: 0.27660 (2.74707) + | > loader_time: 0.00610 (0.04361) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.29577 (-0.48652) + | > avg_loss: -0.36640 (-0.03769) + | > avg_log_mle: -0.58848 (-0.02342) + | > avg_loss_dur: 0.22208 (-0.01427) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_75348.pth + + > EPOCH: 322/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 05:06:17)  + + --> STEP: 2/234 -- GLOBAL_STEP: 75350 + | > loss: -0.38062 (-0.36494) + | > log_mle: -0.46028 (-0.45015) + | > loss_dur: 0.07966 (0.08521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.15695 (15.43655) + | > current_lr: 0.00008 + | > step_time: 4.01470 (6.31230) + | > loader_time: 0.09240 (0.09520) + + + --> STEP: 7/234 -- GLOBAL_STEP: 75355 + | > loss: -0.38841 (-0.35445) + | > log_mle: -0.44932 (-0.44294) + | > loss_dur: 0.06091 (0.08849) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.88675 (19.01290) + | > current_lr: 0.00008 + | > step_time: 0.98260 (5.69101) + | > loader_time: 0.00250 (0.12857) + + + --> STEP: 12/234 -- GLOBAL_STEP: 75360 + | > loss: -0.36789 (-0.36417) + | > log_mle: -0.44844 (-0.44895) + | > loss_dur: 0.08055 (0.08478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.48738 (18.60229) + | > current_lr: 0.00008 + | > step_time: 1.14360 (3.92370) + | > loader_time: 0.00240 (0.09862) + + + --> STEP: 17/234 -- GLOBAL_STEP: 75365 + | > loss: -0.40904 (-0.37727) + | > log_mle: -0.46675 (-0.45518) + | > loss_dur: 0.05771 (0.07791) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.77676 (16.24328) + | > current_lr: 0.00008 + | > step_time: 1.29720 (3.23340) + | > loader_time: 0.00370 (0.07032) + + + --> STEP: 22/234 -- GLOBAL_STEP: 75370 + | > loss: -0.36886 (-0.38060) + | > log_mle: -0.44820 (-0.45516) + | > loss_dur: 0.07933 (0.07456) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.83273 (15.12536) + | > current_lr: 0.00008 + | > step_time: 2.99980 (2.88848) + | > loader_time: 0.00180 (0.05498) + + + --> STEP: 27/234 -- GLOBAL_STEP: 75375 + | > loss: -0.37337 (-0.38297) + | > log_mle: -0.43560 (-0.45467) + | > loss_dur: 0.06223 (0.07170) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.74273 (14.35556) + | > current_lr: 0.00008 + | > step_time: 1.49220 (2.71175) + | > loader_time: 0.00210 (0.05233) + + + --> STEP: 32/234 -- GLOBAL_STEP: 75380 + | > loss: -0.37511 (-0.38283) + | > log_mle: -0.43686 (-0.45357) + | > loss_dur: 0.06175 (0.07075) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.27117 (13.77685) + | > current_lr: 0.00008 + | > step_time: 2.28480 (2.55615) + | > loader_time: 0.00190 (0.04450) + + + --> STEP: 37/234 -- GLOBAL_STEP: 75385 + | > loss: -0.35600 (-0.37893) + | > log_mle: -0.42027 (-0.44961) + | > loss_dur: 0.06427 (0.07068) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.74495 (13.51915) + | > current_lr: 0.00008 + | > step_time: 1.73240 (2.47584) + | > loader_time: 0.00200 (0.03875) + + + --> STEP: 42/234 -- GLOBAL_STEP: 75390 + | > loss: -0.34106 (-0.37576) + | > log_mle: -0.41128 (-0.44665) + | > loss_dur: 0.07022 (0.07089) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.15610 (13.00256) + | > current_lr: 0.00008 + | > step_time: 1.96630 (2.43984) + | > loader_time: 0.00220 (0.03444) + + + --> STEP: 47/234 -- GLOBAL_STEP: 75395 + | > loss: -0.35560 (-0.37271) + | > log_mle: -0.43489 (-0.44450) + | > loss_dur: 0.07928 (0.07179) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.53912 (12.67984) + | > current_lr: 0.00008 + | > step_time: 1.85340 (2.42434) + | > loader_time: 0.00120 (0.03287) + + + --> STEP: 52/234 -- GLOBAL_STEP: 75400 + | > loss: -0.30673 (-0.37120) + | > log_mle: -0.40421 (-0.44282) + | > loss_dur: 0.09748 (0.07162) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.04991 (12.46362) + | > current_lr: 0.00008 + | > step_time: 1.85480 (2.39541) + | > loader_time: 0.09190 (0.03330) + + + --> STEP: 57/234 -- GLOBAL_STEP: 75405 + | > loss: -0.31952 (-0.36821) + | > log_mle: -0.40668 (-0.44058) + | > loss_dur: 0.08716 (0.07237) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.51781 (12.27732) + | > current_lr: 0.00008 + | > step_time: 1.38390 (2.37521) + | > loader_time: 0.00270 (0.03067) + + + --> STEP: 62/234 -- GLOBAL_STEP: 75410 + | > loss: -0.26316 (-0.36416) + | > log_mle: -0.41017 (-0.43849) + | > loss_dur: 0.14702 (0.07434) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.21933 (12.23730) + | > current_lr: 0.00008 + | > step_time: 3.03970 (2.39894) + | > loader_time: 0.08780 (0.03135) + + + --> STEP: 67/234 -- GLOBAL_STEP: 75415 + | > loss: -0.32464 (-0.36127) + | > log_mle: -0.41674 (-0.43630) + | > loss_dur: 0.09210 (0.07502) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.60011 (11.93521) + | > current_lr: 0.00008 + | > step_time: 2.07140 (2.39022) + | > loader_time: 0.00260 (0.03052) + + + --> STEP: 72/234 -- GLOBAL_STEP: 75420 + | > loss: -0.32986 (-0.35750) + | > log_mle: -0.41114 (-0.43397) + | > loss_dur: 0.08128 (0.07648) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.81290 (11.78910) + | > current_lr: 0.00008 + | > step_time: 1.70750 (2.37314) + | > loader_time: 0.00200 (0.03097) + + + --> STEP: 77/234 -- GLOBAL_STEP: 75425 + | > loss: -0.30470 (-0.35350) + | > log_mle: -0.39864 (-0.43186) + | > loss_dur: 0.09394 (0.07835) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.88083 (11.86556) + | > current_lr: 0.00008 + | > step_time: 1.40600 (2.35432) + | > loader_time: 0.00320 (0.03268) + + + --> STEP: 82/234 -- GLOBAL_STEP: 75430 + | > loss: -0.29934 (-0.35081) + | > log_mle: -0.39689 (-0.43009) + | > loss_dur: 0.09755 (0.07928) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.59839 (11.71688) + | > current_lr: 0.00008 + | > step_time: 3.03350 (2.39639) + | > loader_time: 0.00270 (0.03097) + + + --> STEP: 87/234 -- GLOBAL_STEP: 75435 + | > loss: -0.30470 (-0.34807) + | > log_mle: -0.39879 (-0.42843) + | > loss_dur: 0.09409 (0.08036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.88941 (11.82378) + | > current_lr: 0.00008 + | > step_time: 1.91040 (2.38019) + | > loader_time: 0.08310 (0.03124) + + + --> STEP: 92/234 -- GLOBAL_STEP: 75440 + | > loss: -0.30904 (-0.34547) + | > log_mle: -0.43235 (-0.42810) + | > loss_dur: 0.12330 (0.08263) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.32028 (12.02457) + | > current_lr: 0.00008 + | > step_time: 3.20220 (2.42605) + | > loader_time: 0.09360 (0.03590) + + + --> STEP: 97/234 -- GLOBAL_STEP: 75445 + | > loss: -0.30423 (-0.34356) + | > log_mle: -0.41969 (-0.42859) + | > loss_dur: 0.11546 (0.08503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.69023 (12.27741) + | > current_lr: 0.00008 + | > step_time: 0.87850 (2.42659) + | > loader_time: 0.00350 (0.03420) + + + --> STEP: 102/234 -- GLOBAL_STEP: 75450 + | > loss: -0.26940 (-0.34093) + | > log_mle: -0.40282 (-0.42837) + | > loss_dur: 0.13341 (0.08744) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.10162 (12.54316) + | > current_lr: 0.00008 + | > step_time: 2.90310 (2.39701) + | > loader_time: 0.09720 (0.03532) + + + --> STEP: 107/234 -- GLOBAL_STEP: 75455 + | > loss: -0.28416 (-0.33876) + | > log_mle: -0.43332 (-0.42896) + | > loss_dur: 0.14916 (0.09020) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.97257 (13.12605) + | > current_lr: 0.00008 + | > step_time: 1.90040 (2.40116) + | > loader_time: 0.08240 (0.03517) + + + --> STEP: 112/234 -- GLOBAL_STEP: 75460 + | > loss: -0.28799 (-0.33645) + | > log_mle: -0.44850 (-0.42959) + | > loss_dur: 0.16051 (0.09314) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.11157 (13.70135) + | > current_lr: 0.00008 + | > step_time: 1.60640 (2.37702) + | > loader_time: 0.08240 (0.03607) + + + --> STEP: 117/234 -- GLOBAL_STEP: 75465 + | > loss: -0.29765 (-0.33446) + | > log_mle: -0.44751 (-0.43028) + | > loss_dur: 0.14986 (0.09582) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.14867 (14.05488) + | > current_lr: 0.00008 + | > step_time: 5.50150 (2.42265) + | > loader_time: 0.10540 (0.03627) + + + --> STEP: 122/234 -- GLOBAL_STEP: 75470 + | > loss: -0.27167 (-0.33248) + | > log_mle: -0.41670 (-0.43031) + | > loss_dur: 0.14503 (0.09783) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.17311 (14.15132) + | > current_lr: 0.00008 + | > step_time: 1.60500 (2.39297) + | > loader_time: 0.00240 (0.03551) + + + --> STEP: 127/234 -- GLOBAL_STEP: 75475 + | > loss: -0.30018 (-0.33113) + | > log_mle: -0.48084 (-0.43163) + | > loss_dur: 0.18066 (0.10050) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.95257 (14.53502) + | > current_lr: 0.00008 + | > step_time: 2.41290 (2.40138) + | > loader_time: 0.09380 (0.03618) + + + --> STEP: 132/234 -- GLOBAL_STEP: 75480 + | > loss: -0.29925 (-0.33064) + | > log_mle: -0.45402 (-0.43347) + | > loss_dur: 0.15477 (0.10283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.26169 (15.25682) + | > current_lr: 0.00008 + | > step_time: 6.70410 (2.42143) + | > loader_time: 0.00780 (0.03497) + + + --> STEP: 137/234 -- GLOBAL_STEP: 75485 + | > loss: -0.28635 (-0.33024) + | > log_mle: -0.46957 (-0.43559) + | > loss_dur: 0.18321 (0.10535) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.69055 (16.13873) + | > current_lr: 0.00008 + | > step_time: 1.10330 (2.40806) + | > loader_time: 0.00300 (0.03446) + + + --> STEP: 142/234 -- GLOBAL_STEP: 75490 + | > loss: -0.30060 (-0.32959) + | > log_mle: -0.47972 (-0.43730) + | > loss_dur: 0.17912 (0.10771) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.52937 (16.61118) + | > current_lr: 0.00008 + | > step_time: 3.33330 (2.41113) + | > loader_time: 0.00240 (0.03335) + + + --> STEP: 147/234 -- GLOBAL_STEP: 75495 + | > loss: -0.31235 (-0.33031) + | > log_mle: -0.48448 (-0.44090) + | > loss_dur: 0.17213 (0.11058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.28654 (17.38187) + | > current_lr: 0.00008 + | > step_time: 2.49050 (2.39833) + | > loader_time: 0.00620 (0.03239) + + + --> STEP: 152/234 -- GLOBAL_STEP: 75500 + | > loss: -0.37324 (-0.33121) + | > log_mle: -0.57238 (-0.44422) + | > loss_dur: 0.19915 (0.11302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.38829 (18.25150) + | > current_lr: 0.00008 + | > step_time: 1.81270 (2.42938) + | > loader_time: 0.08750 (0.03376) + + + --> STEP: 157/234 -- GLOBAL_STEP: 75505 + | > loss: -0.33037 (-0.33274) + | > log_mle: -0.51793 (-0.44826) + | > loss_dur: 0.18757 (0.11552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.26825 (19.26196) + | > current_lr: 0.00008 + | > step_time: 3.10420 (2.43724) + | > loader_time: 0.20230 (0.03409) + + + --> STEP: 162/234 -- GLOBAL_STEP: 75510 + | > loss: -0.36948 (-0.33440) + | > log_mle: -0.55172 (-0.45219) + | > loss_dur: 0.18224 (0.11780) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.18097 (20.01818) + | > current_lr: 0.00008 + | > step_time: 5.50340 (2.44417) + | > loader_time: 0.09250 (0.03426) + + + --> STEP: 167/234 -- GLOBAL_STEP: 75515 + | > loss: -0.46447 (-0.33594) + | > log_mle: -0.65804 (-0.45590) + | > loss_dur: 0.19357 (0.11996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.40153 (20.95537) + | > current_lr: 0.00008 + | > step_time: 1.50050 (2.55126) + | > loader_time: 0.00740 (0.03547) + + + --> STEP: 172/234 -- GLOBAL_STEP: 75520 + | > loss: -0.43859 (-0.33820) + | > log_mle: -0.64941 (-0.46071) + | > loss_dur: 0.21082 (0.12251) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 50.35209 (21.89395) + | > current_lr: 0.00008 + | > step_time: 1.58990 (2.58411) + | > loader_time: 0.00300 (0.03605) + + + --> STEP: 177/234 -- GLOBAL_STEP: 75525 + | > loss: -0.39003 (-0.34013) + | > log_mle: -0.59652 (-0.46517) + | > loss_dur: 0.20650 (0.12503) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.82874 (22.85355) + | > current_lr: 0.00008 + | > step_time: 1.39280 (2.60305) + | > loader_time: 0.00240 (0.03560) + + + --> STEP: 182/234 -- GLOBAL_STEP: 75530 + | > loss: -0.38341 (-0.34179) + | > log_mle: -0.63355 (-0.46952) + | > loss_dur: 0.25014 (0.12774) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.32590 (23.81222) + | > current_lr: 0.00008 + | > step_time: 1.08820 (2.61515) + | > loader_time: 0.00590 (0.03580) + + + --> STEP: 187/234 -- GLOBAL_STEP: 75535 + | > loss: -0.43404 (-0.34387) + | > log_mle: -0.64972 (-0.47392) + | > loss_dur: 0.21568 (0.13005) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.40269 (24.83684) + | > current_lr: 0.00008 + | > step_time: 4.50750 (2.66835) + | > loader_time: 0.09590 (0.03698) + + + --> STEP: 192/234 -- GLOBAL_STEP: 75540 + | > loss: -0.45439 (-0.34561) + | > log_mle: -0.66739 (-0.47773) + | > loss_dur: 0.21300 (0.13212) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.87209 (26.22923) + | > current_lr: 0.00008 + | > step_time: 20.48900 (2.80615) + | > loader_time: 0.10530 (0.03864) + + + --> STEP: 197/234 -- GLOBAL_STEP: 75545 + | > loss: -0.44812 (-0.34772) + | > log_mle: -0.63894 (-0.48181) + | > loss_dur: 0.19082 (0.13409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.26689 (26.94291) + | > current_lr: 0.00008 + | > step_time: 2.71620 (2.84362) + | > loader_time: 0.40030 (0.04157) + + + --> STEP: 202/234 -- GLOBAL_STEP: 75550 + | > loss: -0.49974 (-0.34958) + | > log_mle: -0.70842 (-0.48585) + | > loss_dur: 0.20868 (0.13627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 104.49446 (28.30050) + | > current_lr: 0.00008 + | > step_time: 2.39970 (2.90908) + | > loader_time: 0.00440 (0.04295) + + + --> STEP: 207/234 -- GLOBAL_STEP: 75555 + | > loss: -0.48004 (-0.35153) + | > log_mle: -0.71469 (-0.48995) + | > loss_dur: 0.23465 (0.13842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 76.87788 (29.32207) + | > current_lr: 0.00008 + | > step_time: 7.09480 (2.95324) + | > loader_time: 0.11500 (0.04403) + + + --> STEP: 212/234 -- GLOBAL_STEP: 75560 + | > loss: -0.46620 (-0.35430) + | > log_mle: -0.69339 (-0.49489) + | > loss_dur: 0.22719 (0.14059) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.19241 (30.28856) + | > current_lr: 0.00008 + | > step_time: 3.21550 (3.00667) + | > loader_time: 0.09270 (0.04440) + + + --> STEP: 217/234 -- GLOBAL_STEP: 75565 + | > loss: -0.46919 (-0.35716) + | > log_mle: -0.72548 (-0.49990) + | > loss_dur: 0.25629 (0.14274) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.88814 (31.13419) + | > current_lr: 0.00008 + | > step_time: 4.89200 (3.03596) + | > loader_time: 0.00500 (0.04435) + + + --> STEP: 222/234 -- GLOBAL_STEP: 75570 + | > loss: -0.48618 (-0.36011) + | > log_mle: -0.74987 (-0.50488) + | > loss_dur: 0.26369 (0.14477) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.82824 (31.88663) + | > current_lr: 0.00008 + | > step_time: 0.24150 (3.01740) + | > loader_time: 0.00380 (0.04427) + + + --> STEP: 227/234 -- GLOBAL_STEP: 75575 + | > loss: -0.45881 (-0.36318) + | > log_mle: -0.71530 (-0.51010) + | > loss_dur: 0.25649 (0.14693) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.98363 (33.12373) + | > current_lr: 0.00008 + | > step_time: 0.24510 (2.95626) + | > loader_time: 0.00350 (0.04337) + + + --> STEP: 232/234 -- GLOBAL_STEP: 75580 + | > loss: -0.47780 (-0.36581) + | > log_mle: -0.94435 (-0.51692) + | > loss_dur: 0.46655 (0.15110) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 91.04508 (34.44626) + | > current_lr: 0.00008 + | > step_time: 0.35760 (2.89850) + | > loader_time: 0.04520 (0.04271) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.28080 (-0.01497) + | > avg_loss: -0.36773 (-0.00133) + | > avg_log_mle: -0.58622 (+0.00226) + | > avg_loss_dur: 0.21850 (-0.00359) + + > BEST MODEL : /root/TTS/run-April-27-2022_08+17AM-c410bc58/best_model_75582.pth + + > EPOCH: 323/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 05:18:43)  + + --> STEP: 3/234 -- GLOBAL_STEP: 75585 + | > loss: -0.28442 (-0.32687) + | > log_mle: -0.43248 (-0.44403) + | > loss_dur: 0.14806 (0.11716) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.94406 (20.82671) + | > current_lr: 0.00008 + | > step_time: 3.20620 (7.80127) + | > loader_time: 0.00510 (0.03391) + + + --> STEP: 8/234 -- GLOBAL_STEP: 75590 + | > loss: -0.37827 (-0.35654) + | > log_mle: -0.46373 (-0.44674) + | > loss_dur: 0.08546 (0.09020) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.92635 (18.80035) + | > current_lr: 0.00008 + | > step_time: 11.10060 (6.21215) + | > loader_time: 0.09610 (0.03850) + + + --> STEP: 13/234 -- GLOBAL_STEP: 75595 + | > loss: -0.42240 (-0.36622) + | > log_mle: -0.47890 (-0.45217) + | > loss_dur: 0.05650 (0.08595) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.71037 (17.32641) + | > current_lr: 0.00008 + | > step_time: 4.70230 (5.38439) + | > loader_time: 0.09200 (0.05184) + + + --> STEP: 18/234 -- GLOBAL_STEP: 75600 + | > loss: -0.38185 (-0.37717) + | > log_mle: -0.44725 (-0.45559) + | > loss_dur: 0.06539 (0.07842) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.16127 (15.82261) + | > current_lr: 0.00008 + | > step_time: 7.30410 (5.15464) + | > loader_time: 0.11140 (0.05333) + + + --> STEP: 23/234 -- GLOBAL_STEP: 75605 + | > loss: -0.42393 (-0.38266) + | > log_mle: -0.47903 (-0.45702) + | > loss_dur: 0.05510 (0.07437) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.32646 (14.87826) + | > current_lr: 0.00008 + | > step_time: 1.09580 (4.86452) + | > loader_time: 0.00130 (0.04625) + + + --> STEP: 28/234 -- GLOBAL_STEP: 75610 + | > loss: -0.43718 (-0.38413) + | > log_mle: -0.48461 (-0.45619) + | > loss_dur: 0.04744 (0.07205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.74763 (14.55196) + | > current_lr: 0.00008 + | > step_time: 1.98180 (4.24596) + | > loader_time: 0.00130 (0.04108) + + + --> STEP: 33/234 -- GLOBAL_STEP: 75615 + | > loss: -0.37655 (-0.38166) + | > log_mle: -0.43691 (-0.45344) + | > loss_dur: 0.06036 (0.07178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.69137 (14.17906) + | > current_lr: 0.00008 + | > step_time: 4.88690 (4.28493) + | > loader_time: 0.00130 (0.04342) + + + --> STEP: 38/234 -- GLOBAL_STEP: 75620 + | > loss: -0.35795 (-0.37754) + | > log_mle: -0.43453 (-0.44937) + | > loss_dur: 0.07658 (0.07183) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.91771 (14.15653) + | > current_lr: 0.00008 + | > step_time: 3.81110 (4.12385) + | > loader_time: 0.00240 (0.04265) + + + --> STEP: 43/234 -- GLOBAL_STEP: 75625 + | > loss: -0.33208 (-0.37377) + | > log_mle: -0.41851 (-0.44578) + | > loss_dur: 0.08643 (0.07200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.43296 (13.85129) + | > current_lr: 0.00008 + | > step_time: 1.89100 (4.08869) + | > loader_time: 0.09860 (0.04899) + + + --> STEP: 48/234 -- GLOBAL_STEP: 75630 + | > loss: -0.38252 (-0.37089) + | > log_mle: -0.44379 (-0.44384) + | > loss_dur: 0.06126 (0.07294) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.78719 (13.46604) + | > current_lr: 0.00008 + | > step_time: 1.62140 (3.91898) + | > loader_time: 0.00210 (0.04630) + + + --> STEP: 53/234 -- GLOBAL_STEP: 75635 + | > loss: -0.33556 (-0.36844) + | > log_mle: -0.41789 (-0.44171) + | > loss_dur: 0.08233 (0.07327) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.56768 (13.11314) + | > current_lr: 0.00008 + | > step_time: 1.20080 (3.67898) + | > loader_time: 0.00240 (0.04519) + + + --> STEP: 58/234 -- GLOBAL_STEP: 75640 + | > loss: -0.36586 (-0.36645) + | > log_mle: -0.42537 (-0.43976) + | > loss_dur: 0.05951 (0.07331) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.42407 (12.68456) + | > current_lr: 0.00008 + | > step_time: 1.20280 (3.50115) + | > loader_time: 0.08480 (0.04289) + + + --> STEP: 63/234 -- GLOBAL_STEP: 75645 + | > loss: -0.32448 (-0.36245) + | > log_mle: -0.40182 (-0.43762) + | > loss_dur: 0.07735 (0.07516) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.44350 (12.56888) + | > current_lr: 0.00008 + | > step_time: 3.48720 (3.35586) + | > loader_time: 0.00120 (0.03977) + + + --> STEP: 68/234 -- GLOBAL_STEP: 75650 + | > loss: -0.30471 (-0.35981) + | > log_mle: -0.39929 (-0.43542) + | > loss_dur: 0.09458 (0.07561) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.43927 (12.40647) + | > current_lr: 0.00008 + | > step_time: 1.37730 (3.25731) + | > loader_time: 0.00200 (0.03837) + + + --> STEP: 73/234 -- GLOBAL_STEP: 75655 + | > loss: -0.28271 (-0.35595) + | > log_mle: -0.40325 (-0.43323) + | > loss_dur: 0.12054 (0.07728) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.13589 (12.39964) + | > current_lr: 0.00008 + | > step_time: 1.38000 (3.15484) + | > loader_time: 0.00220 (0.03714) + + + --> STEP: 78/234 -- GLOBAL_STEP: 75660 + | > loss: -0.29678 (-0.35269) + | > log_mle: -0.39607 (-0.43116) + | > loss_dur: 0.09929 (0.07846) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.65769 (12.30088) + | > current_lr: 0.00008 + | > step_time: 1.11280 (3.04897) + | > loader_time: 0.07720 (0.03805) + + + --> STEP: 83/234 -- GLOBAL_STEP: 75665 + | > loss: -0.27706 (-0.34992) + | > log_mle: -0.39877 (-0.42956) + | > loss_dur: 0.12171 (0.07963) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.77535 (12.17325) + | > current_lr: 0.00008 + | > step_time: 3.59120 (2.99370) + | > loader_time: 0.00220 (0.03595) + + + --> STEP: 88/234 -- GLOBAL_STEP: 75670 + | > loss: -0.30629 (-0.34764) + | > log_mle: -0.43202 (-0.42838) + | > loss_dur: 0.12573 (0.08074) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.91463 (12.24554) + | > current_lr: 0.00008 + | > step_time: 2.11630 (2.91886) + | > loader_time: 0.00330 (0.03409) + + + --> STEP: 93/234 -- GLOBAL_STEP: 75675 + | > loss: -0.30310 (-0.34524) + | > log_mle: -0.44365 (-0.42821) + | > loss_dur: 0.14055 (0.08297) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.36171 (12.44116) + | > current_lr: 0.00008 + | > step_time: 1.78500 (2.89362) + | > loader_time: 0.00280 (0.03435) + + + --> STEP: 98/234 -- GLOBAL_STEP: 75680 + | > loss: -0.28326 (-0.34305) + | > log_mle: -0.38565 (-0.42804) + | > loss_dur: 0.10239 (0.08499) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.66151 (12.98811) + | > current_lr: 0.00008 + | > step_time: 1.50380 (2.86125) + | > loader_time: 0.08600 (0.03359) + + + --> STEP: 103/234 -- GLOBAL_STEP: 75685 + | > loss: -0.31747 (-0.34078) + | > log_mle: -0.46993 (-0.42867) + | > loss_dur: 0.15246 (0.08788) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.72449 (13.45254) + | > current_lr: 0.00008 + | > step_time: 1.49620 (2.81142) + | > loader_time: 0.00380 (0.03213) + + + --> STEP: 108/234 -- GLOBAL_STEP: 75690 + | > loss: -0.29483 (-0.33862) + | > log_mle: -0.41625 (-0.42889) + | > loss_dur: 0.12141 (0.09027) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.58986 (13.74715) + | > current_lr: 0.00008 + | > step_time: 2.30550 (2.76905) + | > loader_time: 0.08290 (0.03306) + + + --> STEP: 113/234 -- GLOBAL_STEP: 75695 + | > loss: -0.30245 (-0.33644) + | > log_mle: -0.45006 (-0.42981) + | > loss_dur: 0.14760 (0.09337) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.18286 (14.35971) + | > current_lr: 0.00008 + | > step_time: 1.11390 (2.73781) + | > loader_time: 0.09360 (0.03336) + + + --> STEP: 118/234 -- GLOBAL_STEP: 75700 + | > loss: -0.27587 (-0.33419) + | > log_mle: -0.43041 (-0.43027) + | > loss_dur: 0.15454 (0.09609) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.74261 (14.61419) + | > current_lr: 0.00008 + | > step_time: 1.29030 (2.69800) + | > loader_time: 0.00310 (0.03282) + + + --> STEP: 123/234 -- GLOBAL_STEP: 75705 + | > loss: -0.24855 (-0.33196) + | > log_mle: -0.39472 (-0.42994) + | > loss_dur: 0.14617 (0.09798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.52190 (14.76816) + | > current_lr: 0.00008 + | > step_time: 3.00070 (2.71290) + | > loader_time: 0.09520 (0.03463) + + + --> STEP: 128/234 -- GLOBAL_STEP: 75710 + | > loss: -0.30116 (-0.33104) + | > log_mle: -0.44572 (-0.43153) + | > loss_dur: 0.14456 (0.10048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.74358 (15.28156) + | > current_lr: 0.00008 + | > step_time: 5.99630 (2.71638) + | > loader_time: 0.00220 (0.03422) + + + --> STEP: 133/234 -- GLOBAL_STEP: 75715 + | > loss: -0.31818 (-0.33055) + | > log_mle: -0.48226 (-0.43352) + | > loss_dur: 0.16408 (0.10298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.98874 (16.09035) + | > current_lr: 0.00008 + | > step_time: 2.10170 (2.74003) + | > loader_time: 0.00290 (0.03603) + + + --> STEP: 138/234 -- GLOBAL_STEP: 75720 + | > loss: -0.27190 (-0.32971) + | > log_mle: -0.42901 (-0.43526) + | > loss_dur: 0.15711 (0.10554) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.86323 (16.67430) + | > current_lr: 0.00008 + | > step_time: 6.39320 (2.75449) + | > loader_time: 0.20140 (0.03750) + + + --> STEP: 143/234 -- GLOBAL_STEP: 75725 + | > loss: -0.35973 (-0.32980) + | > log_mle: -0.58480 (-0.43809) + | > loss_dur: 0.22507 (0.10829) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.90061 (17.31934) + | > current_lr: 0.00008 + | > step_time: 2.39800 (2.72951) + | > loader_time: 0.00420 (0.03813) + + + --> STEP: 148/234 -- GLOBAL_STEP: 75730 + | > loss: -0.33770 (-0.33034) + | > log_mle: -0.49114 (-0.44097) + | > loss_dur: 0.15344 (0.11063) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.03835 (17.93583) + | > current_lr: 0.00008 + | > step_time: 3.89670 (2.82779) + | > loader_time: 0.09880 (0.03941) + + + --> STEP: 153/234 -- GLOBAL_STEP: 75735 + | > loss: -0.43848 (-0.33193) + | > log_mle: -0.62644 (-0.44519) + | > loss_dur: 0.18796 (0.11326) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.24794 (18.79120) + | > current_lr: 0.00008 + | > step_time: 2.28830 (2.80962) + | > loader_time: 0.00240 (0.03933) + + + --> STEP: 158/234 -- GLOBAL_STEP: 75740 + | > loss: -0.34962 (-0.33285) + | > log_mle: -0.55225 (-0.44869) + | > loss_dur: 0.20263 (0.11584) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.51456 (19.71187) + | > current_lr: 0.00008 + | > step_time: 3.20790 (2.81788) + | > loader_time: 0.00280 (0.03874) + + + --> STEP: 163/234 -- GLOBAL_STEP: 75745 + | > loss: -0.33589 (-0.33416) + | > log_mle: -0.52044 (-0.45224) + | > loss_dur: 0.18455 (0.11808) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.73710 (20.54786) + | > current_lr: 0.00008 + | > step_time: 2.40770 (2.80826) + | > loader_time: 0.00330 (0.03826) + + + --> STEP: 168/234 -- GLOBAL_STEP: 75750 + | > loss: -0.38524 (-0.33595) + | > log_mle: -0.58702 (-0.45619) + | > loss_dur: 0.20178 (0.12024) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 44.82198 (21.33208) + | > current_lr: 0.00008 + | > step_time: 4.30360 (2.81771) + | > loader_time: 0.00380 (0.03888) + + + --> STEP: 173/234 -- GLOBAL_STEP: 75755 + | > loss: -0.38742 (-0.33792) + | > log_mle: -0.59219 (-0.46075) + | > loss_dur: 0.20477 (0.12283) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.23400 (22.30586) + | > current_lr: 0.00008 + | > step_time: 5.99530 (2.89288) + | > loader_time: 0.00610 (0.03896) + + + --> STEP: 178/234 -- GLOBAL_STEP: 75760 + | > loss: -0.42478 (-0.34002) + | > log_mle: -0.65058 (-0.46539) + | > loss_dur: 0.22580 (0.12537) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.68042 (23.11636) + | > current_lr: 0.00008 + | > step_time: 1.80460 (2.91387) + | > loader_time: 0.08700 (0.04006) + + + --> STEP: 183/234 -- GLOBAL_STEP: 75765 + | > loss: -0.44027 (-0.34126) + | > log_mle: -0.64398 (-0.46913) + | > loss_dur: 0.20371 (0.12787) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.13234 (24.28671) + | > current_lr: 0.00008 + | > step_time: 13.40230 (2.98241) + | > loader_time: 0.10920 (0.04010) + + + --> STEP: 188/234 -- GLOBAL_STEP: 75770 + | > loss: -0.42242 (-0.34249) + | > log_mle: -0.64530 (-0.47284) + | > loss_dur: 0.22288 (0.13035) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 90.09864 (25.42351) + | > current_lr: 0.00008 + | > step_time: 4.08710 (2.99892) + | > loader_time: 0.00400 (0.04044) + + + --> STEP: 193/234 -- GLOBAL_STEP: 75775 + | > loss: -0.44173 (-0.34438) + | > log_mle: -0.66306 (-0.47674) + | > loss_dur: 0.22133 (0.13236) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.75557 (26.10684) + | > current_lr: 0.00008 + | > step_time: 5.29330 (3.04871) + | > loader_time: 0.01610 (0.04045) + + + --> STEP: 198/234 -- GLOBAL_STEP: 75780 + | > loss: -0.42255 (-0.34607) + | > log_mle: -0.65261 (-0.48055) + | > loss_dur: 0.23006 (0.13448) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.77995 (26.91412) + | > current_lr: 0.00008 + | > step_time: 3.09430 (3.09090) + | > loader_time: 0.00800 (0.04074) + + + --> STEP: 203/234 -- GLOBAL_STEP: 75785 + | > loss: -0.35970 (-0.34756) + | > log_mle: -0.57298 (-0.48416) + | > loss_dur: 0.21328 (0.13660) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.06707 (27.73332) + | > current_lr: 0.00008 + | > step_time: 5.09250 (3.10342) + | > loader_time: 0.00440 (0.04074) + + + --> STEP: 208/234 -- GLOBAL_STEP: 75790 + | > loss: -0.42549 (-0.34969) + | > log_mle: -0.66211 (-0.48856) + | > loss_dur: 0.23663 (0.13887) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.81414 (28.68987) + | > current_lr: 0.00008 + | > step_time: 4.40620 (3.18947) + | > loader_time: 0.08780 (0.04345) + + + --> STEP: 213/234 -- GLOBAL_STEP: 75795 + | > loss: -0.49172 (-0.35260) + | > log_mle: -0.72267 (-0.49361) + | > loss_dur: 0.23095 (0.14100) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.54941 (29.44023) + | > current_lr: 0.00008 + | > step_time: 7.59410 (3.24936) + | > loader_time: 0.10290 (0.04430) + + + --> STEP: 218/234 -- GLOBAL_STEP: 75800 + | > loss: -0.43677 (-0.35521) + | > log_mle: -0.67923 (-0.49827) + | > loss_dur: 0.24246 (0.14306) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.26473 (30.17167) + | > current_lr: 0.00008 + | > step_time: 2.18510 (3.34546) + | > loader_time: 0.00460 (0.04559) + + + --> STEP: 223/234 -- GLOBAL_STEP: 75805 + | > loss: -0.48581 (-0.35798) + | > log_mle: -0.71976 (-0.50308) + | > loss_dur: 0.23396 (0.14510) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.41230 (31.12840) + | > current_lr: 0.00008 + | > step_time: 0.88030 (3.33534) + | > loader_time: 0.00480 (0.04509) + + + --> STEP: 228/234 -- GLOBAL_STEP: 75810 + | > loss: -0.46321 (-0.36091) + | > log_mle: -0.72126 (-0.50824) + | > loss_dur: 0.25804 (0.14733) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.29762 (31.82248) + | > current_lr: 0.00008 + | > step_time: 0.24260 (3.27635) + | > loader_time: 0.00420 (0.04420) + + + --> STEP: 233/234 -- GLOBAL_STEP: 75815 + | > loss: 0.05884 (-0.36107) + | > log_mle: -0.68330 (-0.51474) + | > loss_dur: 0.74214 (0.15367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.16256 (33.14721) + | > current_lr: 0.00008 + | > step_time: 0.19530 (3.21167) + | > loader_time: 0.00300 (0.04334) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.71036 (+0.42956) + | > avg_loss: -0.34441 (+0.02331) + | > avg_log_mle: -0.56504 (+0.02118) + | > avg_loss_dur: 0.22063 (+0.00213) + + + > EPOCH: 324/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 05:32:32)  + + --> STEP: 4/234 -- GLOBAL_STEP: 75820 + | > loss: -0.36476 (-0.34118) + | > log_mle: -0.44557 (-0.44417) + | > loss_dur: 0.08081 (0.10299) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.84586 (18.74529) + | > current_lr: 0.00008 + | > step_time: 1.79340 (5.24479) + | > loader_time: 0.00770 (0.00458) + + + --> STEP: 9/234 -- GLOBAL_STEP: 75825 + | > loss: -0.36166 (-0.35679) + | > log_mle: -0.46369 (-0.45011) + | > loss_dur: 0.10203 (0.09332) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.33688 (16.50273) + | > current_lr: 0.00008 + | > step_time: 1.41080 (4.04147) + | > loader_time: 0.09350 (0.02436) + + + --> STEP: 14/234 -- GLOBAL_STEP: 75830 + | > loss: -0.40627 (-0.37087) + | > log_mle: -0.46426 (-0.45451) + | > loss_dur: 0.05799 (0.08364) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.92040 (15.81657) + | > current_lr: 0.00008 + | > step_time: 1.38880 (4.07390) + | > loader_time: 0.00140 (0.03006) + + + --> STEP: 19/234 -- GLOBAL_STEP: 75835 + | > loss: -0.40552 (-0.38070) + | > log_mle: -0.46937 (-0.45825) + | > loss_dur: 0.06385 (0.07755) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.65089 (14.69995) + | > current_lr: 0.00008 + | > step_time: 2.59860 (3.70405) + | > loader_time: 0.00450 (0.03174) + + + --> STEP: 24/234 -- GLOBAL_STEP: 75840 + | > loss: -0.38428 (-0.38389) + | > log_mle: -0.44789 (-0.45810) + | > loss_dur: 0.06361 (0.07421) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.15030 (13.96546) + | > current_lr: 0.00008 + | > step_time: 1.48500 (3.60643) + | > loader_time: 0.00200 (0.03775) + + + --> STEP: 29/234 -- GLOBAL_STEP: 75845 + | > loss: -0.37745 (-0.38531) + | > log_mle: -0.44342 (-0.45702) + | > loss_dur: 0.06596 (0.07171) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.58572 (13.37743) + | > current_lr: 0.00008 + | > step_time: 3.80490 (3.34052) + | > loader_time: 0.08670 (0.03751) + + + --> STEP: 34/234 -- GLOBAL_STEP: 75850 + | > loss: -0.36069 (-0.38310) + | > log_mle: -0.43222 (-0.45437) + | > loss_dur: 0.07153 (0.07126) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.18477 (12.94511) + | > current_lr: 0.00008 + | > step_time: 2.19300 (3.29324) + | > loader_time: 0.00310 (0.03480) + + + --> STEP: 39/234 -- GLOBAL_STEP: 75855 + | > loss: -0.33268 (-0.37912) + | > log_mle: -0.41932 (-0.45072) + | > loss_dur: 0.08664 (0.07161) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.60100 (12.59843) + | > current_lr: 0.00008 + | > step_time: 1.03810 (3.14484) + | > loader_time: 0.00210 (0.03494) + + + --> STEP: 44/234 -- GLOBAL_STEP: 75860 + | > loss: -0.36927 (-0.37660) + | > log_mle: -0.42761 (-0.44809) + | > loss_dur: 0.05834 (0.07149) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56422 (12.22847) + | > current_lr: 0.00008 + | > step_time: 1.11870 (3.06091) + | > loader_time: 0.00170 (0.03741) + + + --> STEP: 49/234 -- GLOBAL_STEP: 75865 + | > loss: -0.38134 (-0.37572) + | > log_mle: -0.44591 (-0.44723) + | > loss_dur: 0.06458 (0.07152) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.23940 (11.88901) + | > current_lr: 0.00008 + | > step_time: 2.90900 (2.94042) + | > loader_time: 0.00220 (0.03558) + + + --> STEP: 54/234 -- GLOBAL_STEP: 75870 + | > loss: -0.35958 (-0.37327) + | > log_mle: -0.42645 (-0.44526) + | > loss_dur: 0.06686 (0.07199) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.30692 (11.60817) + | > current_lr: 0.00008 + | > step_time: 2.59420 (2.86950) + | > loader_time: 0.00230 (0.03398) + + + --> STEP: 59/234 -- GLOBAL_STEP: 75875 + | > loss: -0.33921 (-0.37124) + | > log_mle: -0.42318 (-0.44345) + | > loss_dur: 0.08397 (0.07221) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.13768 (11.41542) + | > current_lr: 0.00008 + | > step_time: 2.21020 (2.77379) + | > loader_time: 0.09290 (0.03572) + + + --> STEP: 64/234 -- GLOBAL_STEP: 75880 + | > loss: -0.33330 (-0.36683) + | > log_mle: -0.41383 (-0.44096) + | > loss_dur: 0.08053 (0.07413) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.14744 (11.53099) + | > current_lr: 0.00008 + | > step_time: 4.10750 (2.70083) + | > loader_time: 0.08400 (0.03697) + + + --> STEP: 69/234 -- GLOBAL_STEP: 75885 + | > loss: -0.31783 (-0.36324) + | > log_mle: -0.39490 (-0.43821) + | > loss_dur: 0.07708 (0.07497) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.55997 (11.76220) + | > current_lr: 0.00008 + | > step_time: 1.35600 (2.63799) + | > loader_time: 0.00220 (0.03562) + + + --> STEP: 74/234 -- GLOBAL_STEP: 75890 + | > loss: -0.30234 (-0.35849) + | > log_mle: -0.39399 (-0.43546) + | > loss_dur: 0.09166 (0.07697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.79092 (12.12085) + | > current_lr: 0.00008 + | > step_time: 1.81280 (2.57070) + | > loader_time: 0.08370 (0.03769) + + + --> STEP: 79/234 -- GLOBAL_STEP: 75895 + | > loss: -0.31097 (-0.35475) + | > log_mle: -0.40869 (-0.43330) + | > loss_dur: 0.09772 (0.07855) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.70195 (12.08996) + | > current_lr: 0.00008 + | > step_time: 1.72090 (2.53768) + | > loader_time: 0.00220 (0.03651) + + + --> STEP: 84/234 -- GLOBAL_STEP: 75900 + | > loss: -0.32079 (-0.35195) + | > log_mle: -0.40627 (-0.43157) + | > loss_dur: 0.08549 (0.07962) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.39258 (12.03069) + | > current_lr: 0.00008 + | > step_time: 3.49890 (2.53837) + | > loader_time: 0.00330 (0.03673) + + + --> STEP: 89/234 -- GLOBAL_STEP: 75905 + | > loss: -0.29939 (-0.34913) + | > log_mle: -0.42092 (-0.43041) + | > loss_dur: 0.12153 (0.08128) + | > amp_scaler: 2048.00000 (1035.50562) + | > grad_norm: 10.14983 (12.07235) + | > current_lr: 0.00008 + | > step_time: 1.84370 (2.51129) + | > loader_time: 0.00190 (0.03571) + + + --> STEP: 94/234 -- GLOBAL_STEP: 75910 + | > loss: -0.31552 (-0.34664) + | > log_mle: -0.43971 (-0.43025) + | > loss_dur: 0.12418 (0.08361) + | > amp_scaler: 2048.00000 (1089.36170) + | > grad_norm: 33.02395 (12.56506) + | > current_lr: 0.00008 + | > step_time: 1.69190 (2.46071) + | > loader_time: 0.00260 (0.03577) + + + --> STEP: 99/234 -- GLOBAL_STEP: 75915 + | > loss: -0.28638 (-0.34396) + | > log_mle: -0.46524 (-0.43008) + | > loss_dur: 0.17886 (0.08612) + | > amp_scaler: 2048.00000 (1137.77778) + | > grad_norm: 23.78785 (12.83909) + | > current_lr: 0.00008 + | > step_time: 1.01240 (2.43024) + | > loader_time: 0.08350 (0.03575) + + + --> STEP: 104/234 -- GLOBAL_STEP: 75920 + | > loss: -0.34100 (-0.34190) + | > log_mle: -0.48079 (-0.43062) + | > loss_dur: 0.13980 (0.08872) + | > amp_scaler: 2048.00000 (1181.53846) + | > grad_norm: 26.24836 (13.07920) + | > current_lr: 0.00008 + | > step_time: 2.09540 (2.41444) + | > loader_time: 0.00590 (0.03631) + + + --> STEP: 109/234 -- GLOBAL_STEP: 75925 + | > loss: -0.28057 (-0.33908) + | > log_mle: -0.45234 (-0.43048) + | > loss_dur: 0.17177 (0.09141) + | > amp_scaler: 2048.00000 (1221.28440) + | > grad_norm: 25.70061 (13.33221) + | > current_lr: 0.00008 + | > step_time: 2.19060 (2.42180) + | > loader_time: 0.00350 (0.03623) + + + --> STEP: 114/234 -- GLOBAL_STEP: 75930 + | > loss: -0.29430 (-0.33718) + | > log_mle: -0.43427 (-0.43132) + | > loss_dur: 0.13997 (0.09414) + | > amp_scaler: 2048.00000 (1257.54386) + | > grad_norm: 17.70572 (13.82294) + | > current_lr: 0.00008 + | > step_time: 1.60410 (2.39171) + | > loader_time: 0.00240 (0.03559) + + + --> STEP: 119/234 -- GLOBAL_STEP: 75935 + | > loss: -0.28964 (-0.33503) + | > log_mle: -0.42835 (-0.43169) + | > loss_dur: 0.13871 (0.09666) + | > amp_scaler: 2048.00000 (1290.75630) + | > grad_norm: 42.26739 (14.33962) + | > current_lr: 0.00008 + | > step_time: 2.19840 (2.37239) + | > loader_time: 0.00300 (0.03420) + + + --> STEP: 124/234 -- GLOBAL_STEP: 75940 + | > loss: -0.30541 (-0.33280) + | > log_mle: -0.45626 (-0.43127) + | > loss_dur: 0.15085 (0.09847) + | > amp_scaler: 2048.00000 (1321.29032) + | > grad_norm: 28.78110 (14.80938) + | > current_lr: 0.00008 + | > step_time: 3.81210 (2.38056) + | > loader_time: 0.18710 (0.03601) + + + --> STEP: 129/234 -- GLOBAL_STEP: 75945 + | > loss: -0.28138 (-0.33157) + | > log_mle: -0.44695 (-0.43256) + | > loss_dur: 0.16556 (0.10099) + | > amp_scaler: 2048.00000 (1349.45736) + | > grad_norm: 33.30700 (15.47952) + | > current_lr: 0.00008 + | > step_time: 1.89070 (2.36348) + | > loader_time: 0.00320 (0.03541) + + + --> STEP: 134/234 -- GLOBAL_STEP: 75950 + | > loss: -0.32160 (-0.33121) + | > log_mle: -0.50031 (-0.43471) + | > loss_dur: 0.17870 (0.10351) + | > amp_scaler: 2048.00000 (1375.52239) + | > grad_norm: 46.49191 (16.49474) + | > current_lr: 0.00008 + | > step_time: 1.10900 (2.35280) + | > loader_time: 0.00280 (0.03539) + + + --> STEP: 139/234 -- GLOBAL_STEP: 75955 + | > loss: -0.37850 (-0.33072) + | > log_mle: -0.56387 (-0.43675) + | > loss_dur: 0.18536 (0.10603) + | > amp_scaler: 2048.00000 (1399.71223) + | > grad_norm: 41.62246 (17.08053) + | > current_lr: 0.00008 + | > step_time: 2.70080 (2.35825) + | > loader_time: 0.00270 (0.03548) + + + --> STEP: 144/234 -- GLOBAL_STEP: 75960 + | > loss: -0.34361 (-0.33015) + | > log_mle: -0.53787 (-0.43905) + | > loss_dur: 0.19426 (0.10891) + | > amp_scaler: 2048.00000 (1422.22222) + | > grad_norm: 46.25364 (18.04960) + | > current_lr: 0.00008 + | > step_time: 4.39100 (2.41940) + | > loader_time: 0.10200 (0.03625) + + + --> STEP: 149/234 -- GLOBAL_STEP: 75965 + | > loss: -0.40323 (-0.33077) + | > log_mle: -0.58774 (-0.44204) + | > loss_dur: 0.18451 (0.11127) + | > amp_scaler: 2048.00000 (1443.22148) + | > grad_norm: 67.44240 (19.09649) + | > current_lr: 0.00008 + | > step_time: 2.81880 (2.42064) + | > loader_time: 0.08470 (0.03570) + + + --> STEP: 154/234 -- GLOBAL_STEP: 75970 + | > loss: -0.35976 (-0.33185) + | > log_mle: -0.54124 (-0.44573) + | > loss_dur: 0.18147 (0.11388) + | > amp_scaler: 2048.00000 (1462.85714) + | > grad_norm: 53.12139 (19.88065) + | > current_lr: 0.00008 + | > step_time: 3.59140 (2.44332) + | > loader_time: 0.00510 (0.03519) + + + --> STEP: 159/234 -- GLOBAL_STEP: 75975 + | > loss: -0.36391 (-0.33271) + | > log_mle: -0.55449 (-0.44902) + | > loss_dur: 0.19058 (0.11631) + | > amp_scaler: 2048.00000 (1481.25786) + | > grad_norm: 46.56969 (21.04093) + | > current_lr: 0.00008 + | > step_time: 6.10280 (2.46514) + | > loader_time: 0.09290 (0.03532) + + + --> STEP: 164/234 -- GLOBAL_STEP: 75980 + | > loss: -0.35180 (-0.33369) + | > log_mle: -0.55545 (-0.45225) + | > loss_dur: 0.20365 (0.11856) + | > amp_scaler: 2048.00000 (1498.53659) + | > grad_norm: 42.81822 (21.80430) + | > current_lr: 0.00008 + | > step_time: 2.00700 (2.46418) + | > loader_time: 0.10170 (0.03676) + + + --> STEP: 169/234 -- GLOBAL_STEP: 75985 + | > loss: -0.34920 (-0.33504) + | > log_mle: -0.56231 (-0.45601) + | > loss_dur: 0.21311 (0.12097) + | > amp_scaler: 2048.00000 (1514.79290) + | > grad_norm: 48.80169 (22.61540) + | > current_lr: 0.00008 + | > step_time: 3.20800 (2.49255) + | > loader_time: 0.00480 (0.03692) + + + --> STEP: 174/234 -- GLOBAL_STEP: 75990 + | > loss: -0.43560 (-0.33736) + | > log_mle: -0.63635 (-0.46084) + | > loss_dur: 0.20075 (0.12348) + | > amp_scaler: 2048.00000 (1530.11494) + | > grad_norm: 52.66391 (24.01199) + | > current_lr: 0.00008 + | > step_time: 3.89430 (2.60090) + | > loader_time: 0.00550 (0.03920) + + + --> STEP: 179/234 -- GLOBAL_STEP: 75995 + | > loss: -0.38590 (-0.33855) + | > log_mle: -0.62497 (-0.46468) + | > loss_dur: 0.23906 (0.12613) + | > amp_scaler: 2048.00000 (1544.58101) + | > grad_norm: 90.34406 (25.39193) + | > current_lr: 0.00008 + | > step_time: 4.80600 (2.71198) + | > loader_time: 0.18950 (0.04198) + + + --> STEP: 184/234 -- GLOBAL_STEP: 76000 + | > loss: -0.36900 (-0.33959) + | > log_mle: -0.58574 (-0.46820) + | > loss_dur: 0.21673 (0.12861) + | > amp_scaler: 2048.00000 (1558.26087) + | > grad_norm: 49.97551 (25.98562) + | > current_lr: 0.00008 + | > step_time: 8.30010 (2.76495) + | > loader_time: 0.29940 (0.04457) + + + --> STEP: 189/234 -- GLOBAL_STEP: 76005 + | > loss: -0.40601 (-0.34132) + | > log_mle: -0.60672 (-0.47228) + | > loss_dur: 0.20072 (0.13096) + | > amp_scaler: 2048.00000 (1571.21693) + | > grad_norm: 34.15443 (26.37505) + | > current_lr: 0.00008 + | > step_time: 1.78770 (2.80235) + | > loader_time: 0.00250 (0.04762) + + + --> STEP: 194/234 -- GLOBAL_STEP: 76010 + | > loss: -0.43261 (-0.34351) + | > log_mle: -0.63894 (-0.47638) + | > loss_dur: 0.20633 (0.13287) + | > amp_scaler: 2048.00000 (1583.50515) + | > grad_norm: 66.58664 (27.22992) + | > current_lr: 0.00008 + | > step_time: 3.51050 (2.84005) + | > loader_time: 0.08400 (0.04986) + + + --> STEP: 199/234 -- GLOBAL_STEP: 76015 + | > loss: -0.37700 (-0.34502) + | > log_mle: -0.59943 (-0.47997) + | > loss_dur: 0.22243 (0.13494) + | > amp_scaler: 1024.00000 (1579.73869) + | > grad_norm: 159.96887 (28.49849) + | > current_lr: 0.00008 + | > step_time: 1.80100 (2.88032) + | > loader_time: 0.00350 (0.05209) + + + --> STEP: 204/234 -- GLOBAL_STEP: 76020 + | > loss: -0.42645 (-0.34601) + | > log_mle: -0.67763 (-0.48328) + | > loss_dur: 0.25118 (0.13727) + | > amp_scaler: 1024.00000 (1566.11765) + | > grad_norm: 49.76844 (28.96309) + | > current_lr: 0.00008 + | > step_time: 8.00690 (2.96330) + | > loader_time: 0.00660 (0.05326) + + + --> STEP: 209/234 -- GLOBAL_STEP: 76025 + | > loss: -0.42734 (-0.34804) + | > log_mle: -0.64260 (-0.48744) + | > loss_dur: 0.21526 (0.13940) + | > amp_scaler: 1024.00000 (1553.14833) + | > grad_norm: 49.22520 (29.51393) + | > current_lr: 0.00008 + | > step_time: 6.61370 (3.06803) + | > loader_time: 0.08510 (0.05512) + + + --> STEP: 214/234 -- GLOBAL_STEP: 76030 + | > loss: -0.47212 (-0.35105) + | > log_mle: -0.67782 (-0.49258) + | > loss_dur: 0.20571 (0.14153) + | > amp_scaler: 1024.00000 (1540.78505) + | > grad_norm: 64.09822 (30.32668) + | > current_lr: 0.00008 + | > step_time: 5.20140 (3.15668) + | > loader_time: 0.00450 (0.05787) + + + --> STEP: 219/234 -- GLOBAL_STEP: 76035 + | > loss: -0.56262 (-0.35392) + | > log_mle: -0.78916 (-0.49761) + | > loss_dur: 0.22654 (0.14369) + | > amp_scaler: 1024.00000 (1528.98630) + | > grad_norm: 66.36785 (31.19675) + | > current_lr: 0.00008 + | > step_time: 3.91110 (3.17462) + | > loader_time: 0.19080 (0.05803) + + + --> STEP: 224/234 -- GLOBAL_STEP: 76040 + | > loss: -0.49859 (-0.35645) + | > log_mle: -0.72523 (-0.50224) + | > loss_dur: 0.22664 (0.14579) + | > amp_scaler: 1024.00000 (1517.71429) + | > grad_norm: 68.44299 (31.95892) + | > current_lr: 0.00008 + | > step_time: 1.33760 (3.14358) + | > loader_time: 0.08090 (0.05716) + + + --> STEP: 229/234 -- GLOBAL_STEP: 76045 + | > loss: -0.48329 (-0.35934) + | > log_mle: -0.77966 (-0.50765) + | > loss_dur: 0.29637 (0.14831) + | > amp_scaler: 1024.00000 (1506.93450) + | > grad_norm: 109.10800 (33.16211) + | > current_lr: 0.00008 + | > step_time: 0.25310 (3.08059) + | > loader_time: 0.00430 (0.05600) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.45641 (+0.74605) + | > avg_loss: -0.35480 (-0.01039) + | > avg_log_mle: -0.57561 (-0.01057) + | > avg_loss_dur: 0.22081 (+0.00018) + + + > EPOCH: 325/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 05:45:48)  + + --> STEP: 0/234 -- GLOBAL_STEP: 76050 + | > loss: -0.38502 (-0.38502) + | > log_mle: -0.52317 (-0.52317) + | > loss_dur: 0.13815 (0.13815) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.35086 (33.35086) + | > current_lr: 0.00008 + | > step_time: 3.40220 (3.40215) + | > loader_time: 5.61050 (5.61052) + + + --> STEP: 5/234 -- GLOBAL_STEP: 76055 + | > loss: -0.32429 (-0.34209) + | > log_mle: -0.44368 (-0.44570) + | > loss_dur: 0.11938 (0.10361) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.52113 (16.42968) + | > current_lr: 0.00008 + | > step_time: 3.10500 (4.37617) + | > loader_time: 0.00220 (0.02262) + + + --> STEP: 10/234 -- GLOBAL_STEP: 76060 + | > loss: -0.37056 (-0.36330) + | > log_mle: -0.45660 (-0.45245) + | > loss_dur: 0.08605 (0.08915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.97401 (15.62556) + | > current_lr: 0.00008 + | > step_time: 1.91020 (3.74965) + | > loader_time: 0.00120 (0.05998) + + + --> STEP: 15/234 -- GLOBAL_STEP: 76065 + | > loss: -0.41014 (-0.37685) + | > log_mle: -0.47097 (-0.45723) + | > loss_dur: 0.06083 (0.08037) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.99964 (14.95678) + | > current_lr: 0.00008 + | > step_time: 8.10000 (4.71275) + | > loader_time: 0.09430 (0.06665) + + + --> STEP: 20/234 -- GLOBAL_STEP: 76070 + | > loss: -0.41261 (-0.38479) + | > log_mle: -0.46940 (-0.45988) + | > loss_dur: 0.05679 (0.07509) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.74294 (13.73685) + | > current_lr: 0.00008 + | > step_time: 2.91690 (5.18027) + | > loader_time: 0.00140 (0.06096) + + + --> STEP: 25/234 -- GLOBAL_STEP: 76075 + | > loss: -0.38925 (-0.38597) + | > log_mle: -0.44586 (-0.45875) + | > loss_dur: 0.05661 (0.07279) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.16707 (13.19386) + | > current_lr: 0.00008 + | > step_time: 2.12560 (4.87647) + | > loader_time: 0.07920 (0.05645) + + + --> STEP: 30/234 -- GLOBAL_STEP: 76080 + | > loss: -0.37697 (-0.38729) + | > log_mle: -0.44387 (-0.45820) + | > loss_dur: 0.06689 (0.07091) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.12032 (12.54091) + | > current_lr: 0.00008 + | > step_time: 9.62780 (4.80338) + | > loader_time: 0.19460 (0.06019) + + + --> STEP: 35/234 -- GLOBAL_STEP: 76085 + | > loss: -0.32921 (-0.38360) + | > log_mle: -0.41711 (-0.45473) + | > loss_dur: 0.08789 (0.07112) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.50364 (12.34319) + | > current_lr: 0.00008 + | > step_time: 1.41940 (4.32622) + | > loader_time: 0.06630 (0.05637) + + + --> STEP: 40/234 -- GLOBAL_STEP: 76090 + | > loss: -0.34648 (-0.37987) + | > log_mle: -0.42454 (-0.45123) + | > loss_dur: 0.07805 (0.07136) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.28606 (12.11762) + | > current_lr: 0.00008 + | > step_time: 0.98610 (4.13447) + | > loader_time: 0.00260 (0.06367) + + + --> STEP: 45/234 -- GLOBAL_STEP: 76095 + | > loss: -0.34630 (-0.37707) + | > log_mle: -0.43475 (-0.44857) + | > loss_dur: 0.08845 (0.07149) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.88527 (11.84531) + | > current_lr: 0.00008 + | > step_time: 2.00230 (3.89467) + | > loader_time: 0.00150 (0.05895) + + + --> STEP: 50/234 -- GLOBAL_STEP: 76100 + | > loss: -0.36788 (-0.37639) + | > log_mle: -0.43317 (-0.44758) + | > loss_dur: 0.06529 (0.07119) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 5.38338 (11.41321) + | > current_lr: 0.00008 + | > step_time: 0.90680 (3.69593) + | > loader_time: 0.00270 (0.05501) + + + --> STEP: 55/234 -- GLOBAL_STEP: 76105 + | > loss: -0.36730 (-0.37397) + | > log_mle: -0.43340 (-0.44560) + | > loss_dur: 0.06609 (0.07163) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.22237 (11.16044) + | > current_lr: 0.00008 + | > step_time: 1.81710 (3.59621) + | > loader_time: 0.08420 (0.05178) + + + --> STEP: 60/234 -- GLOBAL_STEP: 76110 + | > loss: -0.30903 (-0.37096) + | > log_mle: -0.41826 (-0.44363) + | > loss_dur: 0.10923 (0.07267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.69786 (11.11209) + | > current_lr: 0.00008 + | > step_time: 1.90580 (3.45433) + | > loader_time: 0.08640 (0.05210) + + + --> STEP: 65/234 -- GLOBAL_STEP: 76115 + | > loss: -0.33422 (-0.36665) + | > log_mle: -0.40986 (-0.44086) + | > loss_dur: 0.07564 (0.07421) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.55857 (11.38687) + | > current_lr: 0.00008 + | > step_time: 3.12990 (3.32890) + | > loader_time: 0.07650 (0.05061) + + + --> STEP: 70/234 -- GLOBAL_STEP: 76120 + | > loss: -0.28517 (-0.36285) + | > log_mle: -0.38875 (-0.43804) + | > loss_dur: 0.10358 (0.07519) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.63322 (11.33571) + | > current_lr: 0.00008 + | > step_time: 3.20890 (3.24903) + | > loader_time: 0.08640 (0.04992) + + + --> STEP: 75/234 -- GLOBAL_STEP: 76125 + | > loss: -0.30549 (-0.35900) + | > log_mle: -0.40853 (-0.43605) + | > loss_dur: 0.10304 (0.07705) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.19722 (11.33101) + | > current_lr: 0.00008 + | > step_time: 1.39700 (3.16441) + | > loader_time: 0.00370 (0.04800) + + + --> STEP: 80/234 -- GLOBAL_STEP: 76130 + | > loss: -0.32445 (-0.35623) + | > log_mle: -0.40363 (-0.43407) + | > loss_dur: 0.07918 (0.07785) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.31502 (11.33338) + | > current_lr: 0.00008 + | > step_time: 1.46410 (3.11667) + | > loader_time: 0.00340 (0.04834) + + + --> STEP: 85/234 -- GLOBAL_STEP: 76135 + | > loss: -0.29700 (-0.35311) + | > log_mle: -0.39498 (-0.43226) + | > loss_dur: 0.09798 (0.07915) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.52919 (11.41959) + | > current_lr: 0.00008 + | > step_time: 3.29730 (3.09287) + | > loader_time: 0.02290 (0.04892) + + + --> STEP: 90/234 -- GLOBAL_STEP: 76140 + | > loss: -0.29424 (-0.35047) + | > log_mle: -0.41139 (-0.43134) + | > loss_dur: 0.11715 (0.08087) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.55498 (11.57419) + | > current_lr: 0.00008 + | > step_time: 2.30550 (3.03663) + | > loader_time: 0.00240 (0.04830) + + + --> STEP: 95/234 -- GLOBAL_STEP: 76145 + | > loss: -0.32036 (-0.34818) + | > log_mle: -0.47826 (-0.43184) + | > loss_dur: 0.15789 (0.08366) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.50360 (12.10226) + | > current_lr: 0.00008 + | > step_time: 2.01070 (3.00825) + | > loader_time: 0.00620 (0.04594) + + + --> STEP: 100/234 -- GLOBAL_STEP: 76150 + | > loss: -0.29721 (-0.34569) + | > log_mle: -0.41442 (-0.43109) + | > loss_dur: 0.11721 (0.08540) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.81759 (12.31450) + | > current_lr: 0.00008 + | > step_time: 1.20210 (3.00609) + | > loader_time: 0.00370 (0.04459) + + + --> STEP: 105/234 -- GLOBAL_STEP: 76155 + | > loss: -0.28474 (-0.34301) + | > log_mle: -0.39628 (-0.43099) + | > loss_dur: 0.11155 (0.08798) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.77934 (12.81689) + | > current_lr: 0.00008 + | > step_time: 2.41540 (2.97169) + | > loader_time: 0.00460 (0.04263) + + + --> STEP: 110/234 -- GLOBAL_STEP: 76160 + | > loss: -0.26823 (-0.33961) + | > log_mle: -0.40507 (-0.43050) + | > loss_dur: 0.13684 (0.09089) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.02161 (13.11732) + | > current_lr: 0.00008 + | > step_time: 1.62570 (2.92341) + | > loader_time: 0.00260 (0.04160) + + + --> STEP: 115/234 -- GLOBAL_STEP: 76165 + | > loss: -0.27239 (-0.33741) + | > log_mle: -0.43094 (-0.43114) + | > loss_dur: 0.15854 (0.09374) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.54598 (13.53194) + | > current_lr: 0.00008 + | > step_time: 1.68380 (2.87462) + | > loader_time: 0.00180 (0.04151) + + + --> STEP: 120/234 -- GLOBAL_STEP: 76170 + | > loss: -0.32339 (-0.33528) + | > log_mle: -0.48135 (-0.43164) + | > loss_dur: 0.15796 (0.09636) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.05393 (14.02166) + | > current_lr: 0.00008 + | > step_time: 4.89920 (2.86288) + | > loader_time: 0.00340 (0.04057) + + + --> STEP: 125/234 -- GLOBAL_STEP: 76175 + | > loss: -0.28521 (-0.33261) + | > log_mle: -0.46008 (-0.43096) + | > loss_dur: 0.17487 (0.09836) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.78054 (14.37298) + | > current_lr: 0.00008 + | > step_time: 3.07620 (2.85058) + | > loader_time: 0.00360 (0.03972) + + + --> STEP: 130/234 -- GLOBAL_STEP: 76180 + | > loss: -0.30185 (-0.33121) + | > log_mle: -0.47801 (-0.43232) + | > loss_dur: 0.17616 (0.10111) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.72707 (15.09338) + | > current_lr: 0.00008 + | > step_time: 3.09840 (2.80489) + | > loader_time: 0.00250 (0.03898) + + + --> STEP: 135/234 -- GLOBAL_STEP: 76185 + | > loss: -0.26738 (-0.33051) + | > log_mle: -0.40665 (-0.43380) + | > loss_dur: 0.13927 (0.10329) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 19.28425 (15.61982) + | > current_lr: 0.00008 + | > step_time: 4.09540 (2.85663) + | > loader_time: 0.20450 (0.04051) + + + --> STEP: 140/234 -- GLOBAL_STEP: 76190 + | > loss: -0.26507 (-0.32986) + | > log_mle: -0.44192 (-0.43599) + | > loss_dur: 0.17685 (0.10613) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 18.00964 (16.16339) + | > current_lr: 0.00008 + | > step_time: 8.44590 (2.91274) + | > loader_time: 0.09660 (0.04105) + + + --> STEP: 145/234 -- GLOBAL_STEP: 76195 + | > loss: -0.37543 (-0.33013) + | > log_mle: -0.54593 (-0.43910) + | > loss_dur: 0.17050 (0.10897) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.43306 (17.26437) + | > current_lr: 0.00008 + | > step_time: 2.09720 (2.88027) + | > loader_time: 0.00370 (0.04030) + + + --> STEP: 150/234 -- GLOBAL_STEP: 76200 + | > loss: -0.33097 (-0.33044) + | > log_mle: -0.53052 (-0.44195) + | > loss_dur: 0.19955 (0.11151) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.32455 (18.08189) + | > current_lr: 0.00008 + | > step_time: 1.49360 (2.84521) + | > loader_time: 0.00340 (0.04190) + + + --> STEP: 155/234 -- GLOBAL_STEP: 76205 + | > loss: -0.38995 (-0.33189) + | > log_mle: -0.58720 (-0.44598) + | > loss_dur: 0.19726 (0.11409) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.84892 (18.87954) + | > current_lr: 0.00008 + | > step_time: 1.50990 (2.81273) + | > loader_time: 0.08290 (0.04176) + + + --> STEP: 160/234 -- GLOBAL_STEP: 76210 + | > loss: -0.39452 (-0.33291) + | > log_mle: -0.59647 (-0.44939) + | > loss_dur: 0.20195 (0.11649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.90532 (19.49139) + | > current_lr: 0.00008 + | > step_time: 2.59600 (2.84063) + | > loader_time: 0.00980 (0.04182) + + + --> STEP: 165/234 -- GLOBAL_STEP: 76215 + | > loss: -0.39339 (-0.33391) + | > log_mle: -0.59042 (-0.45275) + | > loss_dur: 0.19703 (0.11884) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.49405 (20.09445) + | > current_lr: 0.00008 + | > step_time: 1.29800 (2.82597) + | > loader_time: 0.00420 (0.04185) + + + --> STEP: 170/234 -- GLOBAL_STEP: 76220 + | > loss: -0.39528 (-0.33493) + | > log_mle: -0.61637 (-0.45613) + | > loss_dur: 0.22109 (0.12120) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.05083 (21.21157) + | > current_lr: 0.00008 + | > step_time: 6.19050 (2.85757) + | > loader_time: 0.10820 (0.04184) + + + --> STEP: 175/234 -- GLOBAL_STEP: 76225 + | > loss: -0.38450 (-0.33686) + | > log_mle: -0.60116 (-0.46055) + | > loss_dur: 0.21666 (0.12369) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.59801 (22.04728) + | > current_lr: 0.00008 + | > step_time: 3.50330 (2.93362) + | > loader_time: 0.19160 (0.04355) + + + --> STEP: 180/234 -- GLOBAL_STEP: 76230 + | > loss: -0.40444 (-0.33858) + | > log_mle: -0.61138 (-0.46485) + | > loss_dur: 0.20695 (0.12627) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.66010 (23.02014) + | > current_lr: 0.00008 + | > step_time: 1.60250 (2.98082) + | > loader_time: 0.05140 (0.04587) + + + --> STEP: 185/234 -- GLOBAL_STEP: 76235 + | > loss: -0.41985 (-0.33978) + | > log_mle: -0.64602 (-0.46864) + | > loss_dur: 0.22618 (0.12885) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.22162 (23.99486) + | > current_lr: 0.00008 + | > step_time: 8.59460 (3.05377) + | > loader_time: 0.40060 (0.04953) + + + --> STEP: 190/234 -- GLOBAL_STEP: 76240 + | > loss: -0.41438 (-0.34126) + | > log_mle: -0.61690 (-0.47241) + | > loss_dur: 0.20252 (0.13116) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.87912 (24.60281) + | > current_lr: 0.00008 + | > step_time: 6.09660 (3.12601) + | > loader_time: 0.19600 (0.05136) + + + --> STEP: 195/234 -- GLOBAL_STEP: 76245 + | > loss: -0.40952 (-0.34358) + | > log_mle: -0.63128 (-0.47670) + | > loss_dur: 0.22176 (0.13312) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 79.53319 (25.40311) + | > current_lr: 0.00008 + | > step_time: 1.75860 (3.14901) + | > loader_time: 0.00510 (0.05915) + + + --> STEP: 200/234 -- GLOBAL_STEP: 76250 + | > loss: -0.39099 (-0.34503) + | > log_mle: -0.63383 (-0.48027) + | > loss_dur: 0.24284 (0.13523) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.44851 (26.43311) + | > current_lr: 0.00008 + | > step_time: 2.40680 (3.14645) + | > loader_time: 0.08300 (0.05940) + + + --> STEP: 205/234 -- GLOBAL_STEP: 76255 + | > loss: -0.40376 (-0.34666) + | > log_mle: -0.62889 (-0.48401) + | > loss_dur: 0.22513 (0.13735) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.45448 (27.07351) + | > current_lr: 0.00008 + | > step_time: 3.58800 (3.21845) + | > loader_time: 0.10730 (0.06176) + + + --> STEP: 210/234 -- GLOBAL_STEP: 76260 + | > loss: -0.47068 (-0.34915) + | > log_mle: -0.70463 (-0.48860) + | > loss_dur: 0.23395 (0.13945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 70.13126 (28.13211) + | > current_lr: 0.00008 + | > step_time: 3.90520 (3.28948) + | > loader_time: 0.00400 (0.06172) + + + --> STEP: 215/234 -- GLOBAL_STEP: 76265 + | > loss: -0.39762 (-0.35153) + | > log_mle: -0.64786 (-0.49320) + | > loss_dur: 0.25024 (0.14167) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.91033 (29.37539) + | > current_lr: 0.00008 + | > step_time: 6.99950 (3.36975) + | > loader_time: 0.20140 (0.06393) + + + --> STEP: 220/234 -- GLOBAL_STEP: 76270 + | > loss: -0.45947 (-0.35442) + | > log_mle: -0.69799 (-0.49821) + | > loss_dur: 0.23852 (0.14380) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 68.40803 (30.33841) + | > current_lr: 0.00008 + | > step_time: 3.19660 (3.35050) + | > loader_time: 0.00530 (0.06342) + + + --> STEP: 225/234 -- GLOBAL_STEP: 76275 + | > loss: -0.52489 (-0.35685) + | > log_mle: -0.77018 (-0.50273) + | > loss_dur: 0.24529 (0.14589) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.66010 (31.14508) + | > current_lr: 0.00008 + | > step_time: 0.99940 (3.31493) + | > loader_time: 0.00390 (0.06335) + + + --> STEP: 230/234 -- GLOBAL_STEP: 76280 + | > loss: -0.50335 (-0.35919) + | > log_mle: -0.82644 (-0.50791) + | > loss_dur: 0.32309 (0.14872) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 97.55849 (32.08853) + | > current_lr: 0.00008 + | > step_time: 0.26780 (3.25381) + | > loader_time: 0.00680 (0.06211) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.30055 (-1.15586) + | > avg_loss: -0.35253 (+0.00227) + | > avg_log_mle: -0.57128 (+0.00433) + | > avg_loss_dur: 0.21875 (-0.00206) + + + > EPOCH: 326/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 05:59:29)  + + --> STEP: 1/234 -- GLOBAL_STEP: 76285 + | > loss: -0.36024 (-0.36024) + | > log_mle: -0.44392 (-0.44392) + | > loss_dur: 0.08367 (0.08367) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.39243 (24.39243) + | > current_lr: 0.00008 + | > step_time: 6.49520 (6.49520) + | > loader_time: 0.00280 (0.00284) + + + --> STEP: 6/234 -- GLOBAL_STEP: 76290 + | > loss: -0.38952 (-0.35917) + | > log_mle: -0.44824 (-0.44566) + | > loss_dur: 0.05872 (0.08649) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.43130 (19.09569) + | > current_lr: 0.00008 + | > step_time: 0.46950 (5.75906) + | > loader_time: 0.00100 (0.07630) + + + --> STEP: 11/234 -- GLOBAL_STEP: 76295 + | > loss: -0.42399 (-0.37020) + | > log_mle: -0.47208 (-0.45318) + | > loss_dur: 0.04809 (0.08298) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.19099 (17.61406) + | > current_lr: 0.00008 + | > step_time: 0.96160 (3.63571) + | > loader_time: 0.00120 (0.04239) + + + --> STEP: 16/234 -- GLOBAL_STEP: 76300 + | > loss: -0.41620 (-0.37910) + | > log_mle: -0.47630 (-0.45720) + | > loss_dur: 0.06010 (0.07810) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.38788 (17.48022) + | > current_lr: 0.00008 + | > step_time: 1.00040 (2.90666) + | > loader_time: 0.00250 (0.02973) + + + --> STEP: 21/234 -- GLOBAL_STEP: 76305 + | > loss: -0.38617 (-0.38440) + | > log_mle: -0.44329 (-0.45743) + | > loss_dur: 0.05712 (0.07302) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.27756 (16.39807) + | > current_lr: 0.00008 + | > step_time: 1.30930 (2.49660) + | > loader_time: 0.00210 (0.02305) + + + --> STEP: 26/234 -- GLOBAL_STEP: 76310 + | > loss: -0.38484 (-0.38660) + | > log_mle: -0.44947 (-0.45718) + | > loss_dur: 0.06463 (0.07058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.64168 (15.41725) + | > current_lr: 0.00008 + | > step_time: 1.84170 (2.67343) + | > loader_time: 0.00210 (0.02539) + + + --> STEP: 31/234 -- GLOBAL_STEP: 76315 + | > loss: -0.35295 (-0.38658) + | > log_mle: -0.43636 (-0.45640) + | > loss_dur: 0.08341 (0.06982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.09350 (14.71869) + | > current_lr: 0.00008 + | > step_time: 1.79160 (2.49109) + | > loader_time: 0.00260 (0.02180) + + + --> STEP: 36/234 -- GLOBAL_STEP: 76320 + | > loss: -0.33944 (-0.38235) + | > log_mle: -0.41932 (-0.45254) + | > loss_dur: 0.07988 (0.07019) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.49228 (14.31379) + | > current_lr: 0.00008 + | > step_time: 1.50720 (2.59896) + | > loader_time: 0.00190 (0.02437) + + + --> STEP: 41/234 -- GLOBAL_STEP: 76325 + | > loss: -0.38137 (-0.37837) + | > log_mle: -0.43820 (-0.44878) + | > loss_dur: 0.05683 (0.07040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.48979 (13.91513) + | > current_lr: 0.00008 + | > step_time: 1.27400 (2.45992) + | > loader_time: 0.00160 (0.02394) + + + --> STEP: 46/234 -- GLOBAL_STEP: 76330 + | > loss: -0.35369 (-0.37438) + | > log_mle: -0.42534 (-0.44560) + | > loss_dur: 0.07165 (0.07122) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.13798 (13.70195) + | > current_lr: 0.00008 + | > step_time: 1.71120 (2.37460) + | > loader_time: 0.00260 (0.02161) + + + --> STEP: 51/234 -- GLOBAL_STEP: 76335 + | > loss: -0.35797 (-0.37380) + | > log_mle: -0.42690 (-0.44457) + | > loss_dur: 0.06893 (0.07077) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.70332 (13.05291) + | > current_lr: 0.00008 + | > step_time: 1.99120 (2.38191) + | > loader_time: 0.00330 (0.02154) + + + --> STEP: 56/234 -- GLOBAL_STEP: 76340 + | > loss: -0.34989 (-0.37150) + | > log_mle: -0.42468 (-0.44273) + | > loss_dur: 0.07479 (0.07122) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.88313 (12.61297) + | > current_lr: 0.00008 + | > step_time: 1.15100 (2.32747) + | > loader_time: 0.00240 (0.01982) + + + --> STEP: 61/234 -- GLOBAL_STEP: 76345 + | > loss: -0.33192 (-0.36862) + | > log_mle: -0.40897 (-0.44067) + | > loss_dur: 0.07706 (0.07205) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.29686 (12.40451) + | > current_lr: 0.00008 + | > step_time: 1.40700 (2.30379) + | > loader_time: 0.03550 (0.02320) + + + --> STEP: 66/234 -- GLOBAL_STEP: 76350 + | > loss: -0.33850 (-0.36500) + | > log_mle: -0.40908 (-0.43824) + | > loss_dur: 0.07057 (0.07325) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.55007 (12.24122) + | > current_lr: 0.00008 + | > step_time: 1.89540 (2.24744) + | > loader_time: 0.00420 (0.02327) + + + --> STEP: 71/234 -- GLOBAL_STEP: 76355 + | > loss: -0.30554 (-0.36101) + | > log_mle: -0.42173 (-0.43593) + | > loss_dur: 0.11619 (0.07492) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.31017 (12.08490) + | > current_lr: 0.00008 + | > step_time: 1.35780 (2.26869) + | > loader_time: 0.00240 (0.02572) + + + --> STEP: 76/234 -- GLOBAL_STEP: 76360 + | > loss: -0.30779 (-0.35742) + | > log_mle: -0.40954 (-0.43404) + | > loss_dur: 0.10175 (0.07662) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.15182 (11.91837) + | > current_lr: 0.00008 + | > step_time: 2.18980 (2.30391) + | > loader_time: 0.10010 (0.02874) + + + --> STEP: 81/234 -- GLOBAL_STEP: 76365 + | > loss: -0.32220 (-0.35502) + | > log_mle: -0.41597 (-0.43238) + | > loss_dur: 0.09377 (0.07736) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.39473 (11.78648) + | > current_lr: 0.00008 + | > step_time: 4.70730 (2.29865) + | > loader_time: 0.09060 (0.02918) + + + --> STEP: 86/234 -- GLOBAL_STEP: 76370 + | > loss: -0.31162 (-0.35189) + | > log_mle: -0.41084 (-0.43070) + | > loss_dur: 0.09923 (0.07881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.37207 (11.87117) + | > current_lr: 0.00008 + | > step_time: 2.52150 (2.28528) + | > loader_time: 0.19190 (0.03101) + + + --> STEP: 91/234 -- GLOBAL_STEP: 76375 + | > loss: -0.29399 (-0.34908) + | > log_mle: -0.41629 (-0.42988) + | > loss_dur: 0.12230 (0.08081) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.82026 (12.12984) + | > current_lr: 0.00008 + | > step_time: 3.10450 (2.29015) + | > loader_time: 0.00710 (0.03054) + + + --> STEP: 96/234 -- GLOBAL_STEP: 76380 + | > loss: -0.30043 (-0.34723) + | > log_mle: -0.40153 (-0.43038) + | > loss_dur: 0.10111 (0.08314) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.84197 (12.62847) + | > current_lr: 0.00008 + | > step_time: 1.60720 (2.28978) + | > loader_time: 0.00420 (0.03002) + + + --> STEP: 101/234 -- GLOBAL_STEP: 76385 + | > loss: -0.28000 (-0.34470) + | > log_mle: -0.43831 (-0.43014) + | > loss_dur: 0.15831 (0.08544) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.62555 (12.87850) + | > current_lr: 0.00008 + | > step_time: 3.59620 (2.29735) + | > loader_time: 0.09200 (0.03039) + + + --> STEP: 106/234 -- GLOBAL_STEP: 76390 + | > loss: -0.26965 (-0.34230) + | > log_mle: -0.43558 (-0.43044) + | > loss_dur: 0.16593 (0.08814) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.58856 (13.14930) + | > current_lr: 0.00008 + | > step_time: 2.11250 (2.30704) + | > loader_time: 0.07700 (0.03144) + + + --> STEP: 111/234 -- GLOBAL_STEP: 76395 + | > loss: -0.31822 (-0.33997) + | > log_mle: -0.49090 (-0.43096) + | > loss_dur: 0.17267 (0.09099) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.03678 (13.45144) + | > current_lr: 0.00008 + | > step_time: 1.71140 (2.29246) + | > loader_time: 0.00350 (0.03081) + + + --> STEP: 116/234 -- GLOBAL_STEP: 76400 + | > loss: -0.27874 (-0.33771) + | > log_mle: -0.45792 (-0.43162) + | > loss_dur: 0.17918 (0.09391) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 20.07873 (13.74399) + | > current_lr: 0.00008 + | > step_time: 1.91100 (2.29973) + | > loader_time: 0.00210 (0.03107) + + + --> STEP: 121/234 -- GLOBAL_STEP: 76405 + | > loss: -0.25133 (-0.33575) + | > log_mle: -0.37679 (-0.43173) + | > loss_dur: 0.12546 (0.09598) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.13125 (13.91751) + | > current_lr: 0.00008 + | > step_time: 0.91220 (2.27991) + | > loader_time: 0.00320 (0.02995) + + + --> STEP: 126/234 -- GLOBAL_STEP: 76410 + | > loss: -0.31572 (-0.33395) + | > log_mle: -0.49558 (-0.43227) + | > loss_dur: 0.17985 (0.09832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.20097 (14.30971) + | > current_lr: 0.00008 + | > step_time: 3.19430 (2.29815) + | > loader_time: 0.00310 (0.02892) + + + --> STEP: 131/234 -- GLOBAL_STEP: 76415 + | > loss: -0.38047 (-0.33340) + | > log_mle: -0.55797 (-0.43426) + | > loss_dur: 0.17750 (0.10086) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.14502 (14.78941) + | > current_lr: 0.00008 + | > step_time: 2.40420 (2.32714) + | > loader_time: 0.00300 (0.02923) + + + --> STEP: 136/234 -- GLOBAL_STEP: 76420 + | > loss: -0.40146 (-0.33303) + | > log_mle: -0.60109 (-0.43631) + | > loss_dur: 0.19963 (0.10329) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.91500 (15.53483) + | > current_lr: 0.00008 + | > step_time: 3.20710 (2.31762) + | > loader_time: 0.00310 (0.03052) + + + --> STEP: 141/234 -- GLOBAL_STEP: 76425 + | > loss: -0.32904 (-0.33214) + | > log_mle: -0.49934 (-0.43796) + | > loss_dur: 0.17030 (0.10583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.71671 (16.12977) + | > current_lr: 0.00008 + | > step_time: 1.80490 (2.32192) + | > loader_time: 0.00340 (0.03089) + + + --> STEP: 146/234 -- GLOBAL_STEP: 76430 + | > loss: -0.34617 (-0.33272) + | > log_mle: -0.54180 (-0.44150) + | > loss_dur: 0.19563 (0.10878) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.56639 (17.22869) + | > current_lr: 0.00008 + | > step_time: 1.70920 (2.29727) + | > loader_time: 0.09700 (0.03117) + + + --> STEP: 151/234 -- GLOBAL_STEP: 76435 + | > loss: -0.35015 (-0.33314) + | > log_mle: -0.51767 (-0.44420) + | > loss_dur: 0.16751 (0.11106) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 30.48966 (17.75372) + | > current_lr: 0.00008 + | > step_time: 2.30320 (2.36556) + | > loader_time: 0.00270 (0.03139) + + + --> STEP: 156/234 -- GLOBAL_STEP: 76440 + | > loss: -0.37281 (-0.33521) + | > log_mle: -0.56274 (-0.44871) + | > loss_dur: 0.18993 (0.11350) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.37498 (18.61986) + | > current_lr: 0.00008 + | > step_time: 1.78280 (2.35563) + | > loader_time: 0.00280 (0.03104) + + + --> STEP: 161/234 -- GLOBAL_STEP: 76445 + | > loss: -0.39383 (-0.33633) + | > log_mle: -0.58383 (-0.45237) + | > loss_dur: 0.19000 (0.11604) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 51.78945 (19.33436) + | > current_lr: 0.00008 + | > step_time: 1.08310 (2.37004) + | > loader_time: 0.00300 (0.03169) + + + --> STEP: 166/234 -- GLOBAL_STEP: 76450 + | > loss: -0.35387 (-0.33698) + | > log_mle: -0.52217 (-0.45522) + | > loss_dur: 0.16830 (0.11824) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.62789 (19.94488) + | > current_lr: 0.00008 + | > step_time: 3.01520 (2.37630) + | > loader_time: 0.08480 (0.03187) + + + --> STEP: 171/234 -- GLOBAL_STEP: 76455 + | > loss: -0.44583 (-0.33910) + | > log_mle: -0.64636 (-0.45990) + | > loss_dur: 0.20053 (0.12080) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.67838 (20.79377) + | > current_lr: 0.00008 + | > step_time: 1.50640 (2.36715) + | > loader_time: 0.00390 (0.03156) + + + --> STEP: 176/234 -- GLOBAL_STEP: 76460 + | > loss: -0.38543 (-0.34084) + | > log_mle: -0.59337 (-0.46430) + | > loss_dur: 0.20794 (0.12346) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.77648 (21.81086) + | > current_lr: 0.00008 + | > step_time: 5.39750 (2.38300) + | > loader_time: 0.00620 (0.03173) + + + --> STEP: 181/234 -- GLOBAL_STEP: 76465 + | > loss: -0.33979 (-0.34222) + | > log_mle: -0.53768 (-0.46822) + | > loss_dur: 0.19789 (0.12600) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.47097 (22.45696) + | > current_lr: 0.00008 + | > step_time: 5.69850 (2.44698) + | > loader_time: 0.09040 (0.03295) + + + --> STEP: 186/234 -- GLOBAL_STEP: 76470 + | > loss: -0.34929 (-0.34406) + | > log_mle: -0.57883 (-0.47257) + | > loss_dur: 0.22954 (0.12850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.43510 (23.16626) + | > current_lr: 0.00008 + | > step_time: 1.40230 (2.47258) + | > loader_time: 0.07370 (0.03306) + + + --> STEP: 191/234 -- GLOBAL_STEP: 76475 + | > loss: -0.40680 (-0.34595) + | > log_mle: -0.60091 (-0.47653) + | > loss_dur: 0.19411 (0.13058) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 45.46836 (23.98236) + | > current_lr: 0.00008 + | > step_time: 2.52090 (2.52380) + | > loader_time: 0.08820 (0.03558) + + + --> STEP: 196/234 -- GLOBAL_STEP: 76480 + | > loss: -0.35605 (-0.34771) + | > log_mle: -0.57947 (-0.48043) + | > loss_dur: 0.22342 (0.13271) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.09993 (25.63254) + | > current_lr: 0.00008 + | > step_time: 3.09980 (2.51559) + | > loader_time: 0.00310 (0.03614) + + + --> STEP: 201/234 -- GLOBAL_STEP: 76485 + | > loss: -0.33068 (-0.34897) + | > log_mle: -0.54550 (-0.48385) + | > loss_dur: 0.21482 (0.13488) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.44815 (26.53780) + | > current_lr: 0.00008 + | > step_time: 4.10250 (2.53909) + | > loader_time: 0.00600 (0.03626) + + + --> STEP: 206/234 -- GLOBAL_STEP: 76490 + | > loss: -0.45357 (-0.35114) + | > log_mle: -0.67383 (-0.48810) + | > loss_dur: 0.22027 (0.13697) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.76900 (27.07072) + | > current_lr: 0.00008 + | > step_time: 6.69370 (2.60119) + | > loader_time: 0.09980 (0.03638) + + + --> STEP: 211/234 -- GLOBAL_STEP: 76495 + | > loss: -0.48967 (-0.35366) + | > log_mle: -0.74091 (-0.49296) + | > loss_dur: 0.25124 (0.13930) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 94.23735 (28.06442) + | > current_lr: 0.00008 + | > step_time: 3.99940 (2.73107) + | > loader_time: 0.01360 (0.04022) + + + --> STEP: 216/234 -- GLOBAL_STEP: 76500 + | > loss: -0.47328 (-0.35605) + | > log_mle: -0.71553 (-0.49748) + | > loss_dur: 0.24226 (0.14144) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 67.58105 (28.91097) + | > current_lr: 0.00008 + | > step_time: 3.61680 (2.73788) + | > loader_time: 0.39990 (0.04383) + + + --> STEP: 221/234 -- GLOBAL_STEP: 76505 + | > loss: -0.41503 (-0.35884) + | > log_mle: -0.63661 (-0.50228) + | > loss_dur: 0.22159 (0.14344) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 87.60316 (29.63971) + | > current_lr: 0.00008 + | > step_time: 3.10240 (2.76554) + | > loader_time: 0.09480 (0.04466) + + + --> STEP: 226/234 -- GLOBAL_STEP: 76510 + | > loss: -0.51599 (-0.36184) + | > log_mle: -0.75205 (-0.50748) + | > loss_dur: 0.23606 (0.14564) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 71.23576 (30.69281) + | > current_lr: 0.00008 + | > step_time: 1.70370 (2.74557) + | > loader_time: 0.00270 (0.04415) + + + --> STEP: 231/234 -- GLOBAL_STEP: 76515 + | > loss: -0.46551 (-0.36432) + | > log_mle: -0.83944 (-0.51330) + | > loss_dur: 0.37393 (0.14898) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 73.80555 (31.69296) + | > current_lr: 0.00008 + | > step_time: 0.27420 (2.69640) + | > loader_time: 0.00470 (0.04329) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.32123 (+0.02068) + | > avg_loss: -0.36024 (-0.00771) + | > avg_log_mle: -0.58108 (-0.00980) + | > avg_loss_dur: 0.22084 (+0.00209) + + + > EPOCH: 327/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 06:11:09)  + + --> STEP: 2/234 -- GLOBAL_STEP: 76520 + | > loss: -0.37851 (-0.36934) + | > log_mle: -0.46182 (-0.45440) + | > loss_dur: 0.08331 (0.08507) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.56427 (20.81392) + | > current_lr: 0.00008 + | > step_time: 3.91430 (4.20604) + | > loader_time: 0.09120 (0.04737) + + + --> STEP: 7/234 -- GLOBAL_STEP: 76525 + | > loss: -0.39620 (-0.36191) + | > log_mle: -0.45744 (-0.44970) + | > loss_dur: 0.06124 (0.08779) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.21466 (16.88745) + | > current_lr: 0.00008 + | > step_time: 1.91310 (3.15538) + | > loader_time: 0.00120 (0.04178) + + + --> STEP: 12/234 -- GLOBAL_STEP: 76530 + | > loss: -0.36945 (-0.36977) + | > log_mle: -0.45097 (-0.45523) + | > loss_dur: 0.08153 (0.08545) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.37910 (16.99832) + | > current_lr: 0.00008 + | > step_time: 2.09890 (3.09663) + | > loader_time: 0.08510 (0.04033) + + + --> STEP: 17/234 -- GLOBAL_STEP: 76535 + | > loss: -0.41531 (-0.38135) + | > log_mle: -0.46944 (-0.45967) + | > loss_dur: 0.05413 (0.07832) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.39508 (16.10423) + | > current_lr: 0.00008 + | > step_time: 1.80080 (2.60330) + | > loader_time: 0.00370 (0.03386) + + + --> STEP: 22/234 -- GLOBAL_STEP: 76540 + | > loss: -0.38687 (-0.38461) + | > log_mle: -0.45680 (-0.45936) + | > loss_dur: 0.06993 (0.07475) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.33896 (15.29931) + | > current_lr: 0.00008 + | > step_time: 1.49520 (2.35504) + | > loader_time: 0.00160 (0.02685) + + + --> STEP: 27/234 -- GLOBAL_STEP: 76545 + | > loss: -0.37907 (-0.38648) + | > log_mle: -0.44349 (-0.45873) + | > loss_dur: 0.06442 (0.07225) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.98764 (14.48599) + | > current_lr: 0.00008 + | > step_time: 1.80420 (2.38589) + | > loader_time: 0.08850 (0.02580) + + + --> STEP: 32/234 -- GLOBAL_STEP: 76550 + | > loss: -0.37930 (-0.38695) + | > log_mle: -0.44775 (-0.45820) + | > loss_dur: 0.06846 (0.07125) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87440 (13.65822) + | > current_lr: 0.00008 + | > step_time: 1.64760 (2.23432) + | > loader_time: 0.00200 (0.02774) + + + --> STEP: 37/234 -- GLOBAL_STEP: 76555 + | > loss: -0.37032 (-0.38309) + | > log_mle: -0.42897 (-0.45449) + | > loss_dur: 0.05865 (0.07140) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.62072 (13.30886) + | > current_lr: 0.00008 + | > step_time: 1.77380 (2.15410) + | > loader_time: 0.00180 (0.02586) + + + --> STEP: 42/234 -- GLOBAL_STEP: 76560 + | > loss: -0.34625 (-0.37987) + | > log_mle: -0.41743 (-0.45141) + | > loss_dur: 0.07118 (0.07154) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.79953 (13.01832) + | > current_lr: 0.00008 + | > step_time: 1.20090 (2.12144) + | > loader_time: 0.00190 (0.02304) + + + --> STEP: 47/234 -- GLOBAL_STEP: 76565 + | > loss: -0.36141 (-0.37714) + | > log_mle: -0.43626 (-0.44924) + | > loss_dur: 0.07485 (0.07209) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.37774 (13.08272) + | > current_lr: 0.00008 + | > step_time: 1.29200 (2.06305) + | > loader_time: 0.00800 (0.02095) + + + --> STEP: 52/234 -- GLOBAL_STEP: 76570 + | > loss: -0.32682 (-0.37577) + | > log_mle: -0.41495 (-0.44776) + | > loss_dur: 0.08813 (0.07200) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.85922 (12.79699) + | > current_lr: 0.00008 + | > step_time: 3.79660 (2.09713) + | > loader_time: 0.00370 (0.02100) + + + --> STEP: 57/234 -- GLOBAL_STEP: 76575 + | > loss: -0.33282 (-0.37332) + | > log_mle: -0.41248 (-0.44575) + | > loss_dur: 0.07966 (0.07243) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.38886 (12.71564) + | > current_lr: 0.00008 + | > step_time: 2.60270 (2.09359) + | > loader_time: 0.00280 (0.02096) + + + --> STEP: 62/234 -- GLOBAL_STEP: 76580 + | > loss: -0.28898 (-0.36942) + | > log_mle: -0.41772 (-0.44364) + | > loss_dur: 0.12874 (0.07422) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.99720 (12.75550) + | > current_lr: 0.00008 + | > step_time: 1.96320 (2.10301) + | > loader_time: 0.00590 (0.02094) + + + --> STEP: 67/234 -- GLOBAL_STEP: 76585 + | > loss: -0.32231 (-0.36632) + | > log_mle: -0.41912 (-0.44116) + | > loss_dur: 0.09681 (0.07484) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.05834 (12.49441) + | > current_lr: 0.00008 + | > step_time: 4.09360 (2.12365) + | > loader_time: 0.10120 (0.02103) + + + --> STEP: 72/234 -- GLOBAL_STEP: 76590 + | > loss: -0.32065 (-0.36242) + | > log_mle: -0.41292 (-0.43866) + | > loss_dur: 0.09228 (0.07624) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.67109 (12.42365) + | > current_lr: 0.00008 + | > step_time: 2.52120 (2.10824) + | > loader_time: 0.00490 (0.02103) + + + --> STEP: 77/234 -- GLOBAL_STEP: 76595 + | > loss: -0.30239 (-0.35846) + | > log_mle: -0.40520 (-0.43655) + | > loss_dur: 0.10281 (0.07809) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.87586 (12.32709) + | > current_lr: 0.00008 + | > step_time: 2.00800 (2.08703) + | > loader_time: 0.08730 (0.02320) + + + --> STEP: 82/234 -- GLOBAL_STEP: 76600 + | > loss: -0.30662 (-0.35590) + | > log_mle: -0.40139 (-0.43479) + | > loss_dur: 0.09477 (0.07889) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.58396 (12.14556) + | > current_lr: 0.00008 + | > step_time: 3.00750 (2.11132) + | > loader_time: 0.00350 (0.02633) + + + --> STEP: 87/234 -- GLOBAL_STEP: 76605 + | > loss: -0.29456 (-0.35282) + | > log_mle: -0.40066 (-0.43314) + | > loss_dur: 0.10610 (0.08032) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.36977 (12.03314) + | > current_lr: 0.00008 + | > step_time: 1.31550 (2.07779) + | > loader_time: 0.09530 (0.02691) + + + --> STEP: 92/234 -- GLOBAL_STEP: 76610 + | > loss: -0.30661 (-0.35016) + | > log_mle: -0.43093 (-0.43265) + | > loss_dur: 0.12432 (0.08249) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.06829 (12.30627) + | > current_lr: 0.00008 + | > step_time: 1.50740 (2.07903) + | > loader_time: 0.00410 (0.02562) + + + --> STEP: 97/234 -- GLOBAL_STEP: 76615 + | > loss: -0.30149 (-0.34826) + | > log_mle: -0.42052 (-0.43304) + | > loss_dur: 0.11904 (0.08478) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.29932 (12.64328) + | > current_lr: 0.00008 + | > step_time: 3.20430 (2.07257) + | > loader_time: 0.00390 (0.02533) + + + --> STEP: 102/234 -- GLOBAL_STEP: 76620 + | > loss: -0.26945 (-0.34552) + | > log_mle: -0.40338 (-0.43266) + | > loss_dur: 0.13393 (0.08714) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.29221 (13.01316) + | > current_lr: 0.00008 + | > step_time: 1.70240 (2.08987) + | > loader_time: 0.00380 (0.02505) + + + --> STEP: 107/234 -- GLOBAL_STEP: 76625 + | > loss: -0.28729 (-0.34338) + | > log_mle: -0.43788 (-0.43335) + | > loss_dur: 0.15058 (0.08997) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.35553 (13.47144) + | > current_lr: 0.00008 + | > step_time: 4.50710 (2.11106) + | > loader_time: 0.00310 (0.02571) + + + --> STEP: 112/234 -- GLOBAL_STEP: 76630 + | > loss: -0.29197 (-0.34109) + | > log_mle: -0.45190 (-0.43391) + | > loss_dur: 0.15993 (0.09282) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.59400 (13.96068) + | > current_lr: 0.00008 + | > step_time: 2.69180 (2.17125) + | > loader_time: 0.00250 (0.02549) + + + --> STEP: 117/234 -- GLOBAL_STEP: 76635 + | > loss: -0.29988 (-0.33897) + | > log_mle: -0.44304 (-0.43449) + | > loss_dur: 0.14316 (0.09552) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 33.12605 (14.34618) + | > current_lr: 0.00008 + | > step_time: 1.69600 (2.14811) + | > loader_time: 0.00340 (0.02524) + + + --> STEP: 122/234 -- GLOBAL_STEP: 76640 + | > loss: -0.25950 (-0.33647) + | > log_mle: -0.40699 (-0.43404) + | > loss_dur: 0.14749 (0.09757) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.70314 (14.59775) + | > current_lr: 0.00008 + | > step_time: 1.60790 (2.15454) + | > loader_time: 0.00240 (0.02505) + + + --> STEP: 127/234 -- GLOBAL_STEP: 76645 + | > loss: -0.28333 (-0.33464) + | > log_mle: -0.47028 (-0.43492) + | > loss_dur: 0.18696 (0.10028) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.50330 (15.23137) + | > current_lr: 0.00008 + | > step_time: 1.10250 (2.16505) + | > loader_time: 0.00360 (0.02418) + + + --> STEP: 132/234 -- GLOBAL_STEP: 76650 + | > loss: -0.30316 (-0.33395) + | > log_mle: -0.45750 (-0.43656) + | > loss_dur: 0.15433 (0.10261) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.93757 (15.84437) + | > current_lr: 0.00008 + | > step_time: 3.61030 (2.17187) + | > loader_time: 0.08810 (0.02567) + + + --> STEP: 137/234 -- GLOBAL_STEP: 76655 + | > loss: -0.29364 (-0.33349) + | > log_mle: -0.47526 (-0.43870) + | > loss_dur: 0.18162 (0.10521) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.64213 (16.49342) + | > current_lr: 0.00008 + | > step_time: 1.38640 (2.18149) + | > loader_time: 0.00690 (0.02622) + + + --> STEP: 142/234 -- GLOBAL_STEP: 76660 + | > loss: -0.29940 (-0.33269) + | > log_mle: -0.48357 (-0.44033) + | > loss_dur: 0.18417 (0.10764) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 40.28453 (17.18990) + | > current_lr: 0.00008 + | > step_time: 5.42080 (2.21265) + | > loader_time: 0.08290 (0.02654) + + + --> STEP: 147/234 -- GLOBAL_STEP: 76665 + | > loss: -0.30607 (-0.33344) + | > log_mle: -0.48682 (-0.44385) + | > loss_dur: 0.18075 (0.11040) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 32.86126 (18.26943) + | > current_lr: 0.00008 + | > step_time: 4.14740 (2.21305) + | > loader_time: 0.02450 (0.02644) + + + --> STEP: 152/234 -- GLOBAL_STEP: 76670 + | > loss: -0.38067 (-0.33446) + | > log_mle: -0.57432 (-0.44711) + | > loss_dur: 0.19365 (0.11266) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.13509 (18.94364) + | > current_lr: 0.00008 + | > step_time: 1.39820 (2.20730) + | > loader_time: 0.09380 (0.02683) + + + --> STEP: 157/234 -- GLOBAL_STEP: 76675 + | > loss: -0.33636 (-0.33607) + | > log_mle: -0.51818 (-0.45111) + | > loss_dur: 0.18181 (0.11504) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 35.15938 (19.77540) + | > current_lr: 0.00008 + | > step_time: 2.30060 (2.21854) + | > loader_time: 0.08180 (0.02778) + + + --> STEP: 162/234 -- GLOBAL_STEP: 76680 + | > loss: -0.36506 (-0.33740) + | > log_mle: -0.55539 (-0.45495) + | > loss_dur: 0.19034 (0.11755) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.98553 (20.56240) + | > current_lr: 0.00008 + | > step_time: 1.09320 (2.20121) + | > loader_time: 0.00510 (0.02812) + + + --> STEP: 167/234 -- GLOBAL_STEP: 76685 + | > loss: -0.46321 (-0.33874) + | > log_mle: -0.65230 (-0.45848) + | > loss_dur: 0.18910 (0.11974) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.24149 (21.16186) + | > current_lr: 0.00008 + | > step_time: 1.47280 (2.19087) + | > loader_time: 0.00380 (0.02741) + + + --> STEP: 172/234 -- GLOBAL_STEP: 76690 + | > loss: -0.42716 (-0.34077) + | > log_mle: -0.64006 (-0.46310) + | > loss_dur: 0.21290 (0.12233) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 59.64028 (21.97375) + | > current_lr: 0.00008 + | > step_time: 1.51330 (2.17431) + | > loader_time: 0.07750 (0.02812) + + + --> STEP: 177/234 -- GLOBAL_STEP: 76695 + | > loss: -0.38211 (-0.34235) + | > log_mle: -0.58866 (-0.46710) + | > loss_dur: 0.20655 (0.12474) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 34.59269 (22.76101) + | > current_lr: 0.00008 + | > step_time: 4.69750 (2.23834) + | > loader_time: 0.09710 (0.02953) + + + --> STEP: 182/234 -- GLOBAL_STEP: 76700 + | > loss: -0.41143 (-0.34385) + | > log_mle: -0.64357 (-0.47129) + | > loss_dur: 0.23214 (0.12745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.89224 (23.49200) + | > current_lr: 0.00008 + | > step_time: 5.88940 (2.32467) + | > loader_time: 0.10040 (0.03071) + + + --> STEP: 187/234 -- GLOBAL_STEP: 76705 + | > loss: -0.43527 (-0.34582) + | > log_mle: -0.63727 (-0.47555) + | > loss_dur: 0.20200 (0.12972) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 54.78823 (24.22863) + | > current_lr: 0.00008 + | > step_time: 7.86330 (2.40402) + | > loader_time: 0.00490 (0.03248) + + + --> STEP: 192/234 -- GLOBAL_STEP: 76710 + | > loss: -0.46936 (-0.34816) + | > log_mle: -0.68083 (-0.47988) + | > loss_dur: 0.21147 (0.13173) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.76379 (24.86196) + | > current_lr: 0.00008 + | > step_time: 0.92100 (2.49246) + | > loader_time: 0.07480 (0.03366) + + + --> STEP: 197/234 -- GLOBAL_STEP: 76715 + | > loss: -0.44410 (-0.35038) + | > log_mle: -0.64006 (-0.48414) + | > loss_dur: 0.19596 (0.13376) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 77.66058 (25.76466) + | > current_lr: 0.00008 + | > step_time: 8.40690 (2.55673) + | > loader_time: 0.09120 (0.03429) + + + --> STEP: 202/234 -- GLOBAL_STEP: 76720 + | > loss: -0.54168 (-0.35245) + | > log_mle: -0.75614 (-0.48854) + | > loss_dur: 0.21445 (0.13609) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 81.38770 (26.73652) + | > current_lr: 0.00008 + | > step_time: 2.90390 (2.64260) + | > loader_time: 0.00280 (0.03736) + + + --> STEP: 207/234 -- GLOBAL_STEP: 76725 + | > loss: -0.49609 (-0.35484) + | > log_mle: -0.73147 (-0.49302) + | > loss_dur: 0.23538 (0.13818) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.47760 (27.58422) + | > current_lr: 0.00008 + | > step_time: 11.91160 (2.70300) + | > loader_time: 0.09750 (0.03820) + + + --> STEP: 212/234 -- GLOBAL_STEP: 76730 + | > loss: -0.43411 (-0.35712) + | > log_mle: -0.67533 (-0.49748) + | > loss_dur: 0.24122 (0.14036) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 82.62057 (28.97540) + | > current_lr: 0.00008 + | > step_time: 7.10020 (2.80039) + | > loader_time: 0.09900 (0.03962) + + + --> STEP: 217/234 -- GLOBAL_STEP: 76735 + | > loss: -0.49475 (-0.35979) + | > log_mle: -0.72958 (-0.50221) + | > loss_dur: 0.23483 (0.14242) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 62.86650 (29.83496) + | > current_lr: 0.00008 + | > step_time: 2.11770 (2.84465) + | > loader_time: 0.08710 (0.04279) + + + --> STEP: 222/234 -- GLOBAL_STEP: 76740 + | > loss: -0.46785 (-0.36249) + | > log_mle: -0.72226 (-0.50695) + | > loss_dur: 0.25441 (0.14446) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 65.37364 (30.79891) + | > current_lr: 0.00008 + | > step_time: 1.39950 (2.84324) + | > loader_time: 0.00380 (0.04319) + + + --> STEP: 227/234 -- GLOBAL_STEP: 76745 + | > loss: -0.44528 (-0.36546) + | > log_mle: -0.69938 (-0.51203) + | > loss_dur: 0.25410 (0.14657) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.67358 (31.63262) + | > current_lr: 0.00008 + | > step_time: 0.23920 (2.78576) + | > loader_time: 0.00430 (0.04232) + + + --> STEP: 232/234 -- GLOBAL_STEP: 76750 + | > loss: -0.46772 (-0.36804) + | > log_mle: -0.94382 (-0.51882) + | > loss_dur: 0.47610 (0.15077) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 113.48183 (32.83047) + | > current_lr: 0.00008 + | > step_time: 0.31410 (2.73138) + | > loader_time: 0.00800 (0.04152) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04487 (-0.27636) + | > avg_loss: -0.33371 (+0.02653) + | > avg_log_mle: -0.57376 (+0.00732) + | > avg_loss_dur: 0.24005 (+0.01921) + + + > EPOCH: 328/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 06:23:09)  + + --> STEP: 3/234 -- GLOBAL_STEP: 76755 + | > loss: -0.29674 (-0.34355) + | > log_mle: -0.44409 (-0.45188) + | > loss_dur: 0.14735 (0.10833) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.14183 (15.96194) + | > current_lr: 0.00008 + | > step_time: 7.49770 (6.16641) + | > loader_time: 0.00200 (0.00220) + + + --> STEP: 8/234 -- GLOBAL_STEP: 76760 + | > loss: -0.38805 (-0.36322) + | > log_mle: -0.47487 (-0.45420) + | > loss_dur: 0.08682 (0.09098) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.09435 (16.83275) + | > current_lr: 0.00008 + | > step_time: 0.49310 (5.36527) + | > loader_time: 0.00120 (0.01337) + + + --> STEP: 13/234 -- GLOBAL_STEP: 76765 + | > loss: -0.42145 (-0.37291) + | > log_mle: -0.48125 (-0.45798) + | > loss_dur: 0.05980 (0.08507) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.55456 (16.75580) + | > current_lr: 0.00008 + | > step_time: 1.16500 (3.89283) + | > loader_time: 0.00180 (0.01676) + + + --> STEP: 18/234 -- GLOBAL_STEP: 76770 + | > loss: -0.37543 (-0.38081) + | > log_mle: -0.44700 (-0.45962) + | > loss_dur: 0.07157 (0.07881) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.43472 (16.09114) + | > current_lr: 0.00008 + | > step_time: 1.33950 (3.16016) + | > loader_time: 0.00130 (0.01252) + + + --> STEP: 23/234 -- GLOBAL_STEP: 76775 + | > loss: -0.42750 (-0.38522) + | > log_mle: -0.48245 (-0.46054) + | > loss_dur: 0.05495 (0.07532) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.27927 (14.86870) + | > current_lr: 0.00008 + | > step_time: 1.34230 (2.81715) + | > loader_time: 0.00140 (0.01023) + + + --> STEP: 28/234 -- GLOBAL_STEP: 76780 + | > loss: -0.45022 (-0.38830) + | > log_mle: -0.49610 (-0.46071) + | > loss_dur: 0.04588 (0.07241) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.05639 (13.88112) + | > current_lr: 0.00008 + | > step_time: 1.86520 (2.60738) + | > loader_time: 0.00180 (0.00883) + + + --> STEP: 33/234 -- GLOBAL_STEP: 76785 + | > loss: -0.38717 (-0.38724) + | > log_mle: -0.44918 (-0.45901) + | > loss_dur: 0.06202 (0.07178) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.64766 (13.09012) + | > current_lr: 0.00008 + | > step_time: 2.33280 (2.49159) + | > loader_time: 0.08480 (0.01309) + + + --> STEP: 38/234 -- GLOBAL_STEP: 76790 + | > loss: -0.36260 (-0.38347) + | > log_mle: -0.43780 (-0.45522) + | > loss_dur: 0.07520 (0.07175) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.93025 (12.84290) + | > current_lr: 0.00008 + | > step_time: 2.52290 (2.43527) + | > loader_time: 0.08260 (0.01615) + + + --> STEP: 43/234 -- GLOBAL_STEP: 76795 + | > loss: -0.34907 (-0.37945) + | > log_mle: -0.42687 (-0.45165) + | > loss_dur: 0.07780 (0.07220) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.64893 (12.51961) + | > current_lr: 0.00008 + | > step_time: 1.11800 (2.31839) + | > loader_time: 0.08310 (0.01638) + + + --> STEP: 48/234 -- GLOBAL_STEP: 76800 + | > loss: -0.39152 (-0.37755) + | > log_mle: -0.44852 (-0.44990) + | > loss_dur: 0.05699 (0.07234) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 4.95090 (12.11303) + | > current_lr: 0.00008 + | > step_time: 1.49700 (2.24557) + | > loader_time: 0.00260 (0.01670) + + + --> STEP: 53/234 -- GLOBAL_STEP: 76805 + | > loss: -0.35426 (-0.37608) + | > log_mle: -0.42915 (-0.44815) + | > loss_dur: 0.07489 (0.07207) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.36532 (11.90932) + | > current_lr: 0.00008 + | > step_time: 1.90160 (2.19319) + | > loader_time: 0.08620 (0.01692) + + + --> STEP: 58/234 -- GLOBAL_STEP: 76810 + | > loss: -0.36563 (-0.37412) + | > log_mle: -0.43254 (-0.44631) + | > loss_dur: 0.06691 (0.07218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.73932 (11.70091) + | > current_lr: 0.00008 + | > step_time: 1.28760 (2.17981) + | > loader_time: 0.00300 (0.01716) + + + --> STEP: 63/234 -- GLOBAL_STEP: 76815 + | > loss: -0.32045 (-0.36973) + | > log_mle: -0.40337 (-0.44398) + | > loss_dur: 0.08293 (0.07425) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.20878 (11.89611) + | > current_lr: 0.00008 + | > step_time: 2.73480 (2.14755) + | > loader_time: 0.08680 (0.02003) + + + --> STEP: 68/234 -- GLOBAL_STEP: 76820 + | > loss: -0.29121 (-0.36651) + | > log_mle: -0.40149 (-0.44159) + | > loss_dur: 0.11029 (0.07508) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.53742 (11.78267) + | > current_lr: 0.00008 + | > step_time: 1.38580 (2.11661) + | > loader_time: 0.09550 (0.02013) + + + --> STEP: 73/234 -- GLOBAL_STEP: 76825 + | > loss: -0.29574 (-0.36259) + | > log_mle: -0.40868 (-0.43935) + | > loss_dur: 0.11294 (0.07675) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.07898 (11.81850) + | > current_lr: 0.00008 + | > step_time: 1.36520 (2.09206) + | > loader_time: 0.00170 (0.02013) + + + --> STEP: 78/234 -- GLOBAL_STEP: 76830 + | > loss: -0.30262 (-0.35910) + | > log_mle: -0.39564 (-0.43704) + | > loss_dur: 0.09302 (0.07794) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.34645 (11.84869) + | > current_lr: 0.00008 + | > step_time: 2.79390 (2.09414) + | > loader_time: 0.00220 (0.02137) + + + --> STEP: 83/234 -- GLOBAL_STEP: 76835 + | > loss: -0.26799 (-0.35604) + | > log_mle: -0.40208 (-0.43526) + | > loss_dur: 0.13409 (0.07922) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.18012 (11.81686) + | > current_lr: 0.00008 + | > step_time: 1.58340 (2.07594) + | > loader_time: 0.00170 (0.02131) + + + --> STEP: 88/234 -- GLOBAL_STEP: 76840 + | > loss: -0.30310 (-0.35335) + | > log_mle: -0.43794 (-0.43392) + | > loss_dur: 0.13484 (0.08057) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.03182 (11.83958) + | > current_lr: 0.00008 + | > step_time: 1.81290 (2.11022) + | > loader_time: 0.09580 (0.02131) + + + --> STEP: 93/234 -- GLOBAL_STEP: 76845 + | > loss: -0.31097 (-0.35060) + | > log_mle: -0.44475 (-0.43347) + | > loss_dur: 0.13378 (0.08287) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.14112 (12.00461) + | > current_lr: 0.00008 + | > step_time: 2.92350 (2.12649) + | > loader_time: 0.00250 (0.02122) + + + --> STEP: 98/234 -- GLOBAL_STEP: 76850 + | > loss: -0.29144 (-0.34850) + | > log_mle: -0.39052 (-0.43326) + | > loss_dur: 0.09908 (0.08476) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.50360 (12.17089) + | > current_lr: 0.00008 + | > step_time: 2.70650 (2.12227) + | > loader_time: 0.00240 (0.02105) + + + --> STEP: 103/234 -- GLOBAL_STEP: 76855 + | > loss: -0.31773 (-0.34610) + | > log_mle: -0.47228 (-0.43367) + | > loss_dur: 0.15455 (0.08757) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.63479 (12.84467) + | > current_lr: 0.00008 + | > step_time: 3.30540 (2.15729) + | > loader_time: 0.09120 (0.02220) + + + --> STEP: 108/234 -- GLOBAL_STEP: 76860 + | > loss: -0.29185 (-0.34377) + | > log_mle: -0.41708 (-0.43380) + | > loss_dur: 0.12523 (0.09003) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.86028 (13.32424) + | > current_lr: 0.00008 + | > step_time: 1.39310 (2.17489) + | > loader_time: 0.00230 (0.02131) + + + --> STEP: 113/234 -- GLOBAL_STEP: 76865 + | > loss: -0.30716 (-0.34181) + | > log_mle: -0.45701 (-0.43470) + | > loss_dur: 0.14984 (0.09289) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.04546 (13.91296) + | > current_lr: 0.00008 + | > step_time: 2.70890 (2.15812) + | > loader_time: 0.08460 (0.02122) + + + --> STEP: 118/234 -- GLOBAL_STEP: 76870 + | > loss: -0.26677 (-0.33934) + | > log_mle: -0.42911 (-0.43500) + | > loss_dur: 0.16234 (0.09566) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.74072 (14.21098) + | > current_lr: 0.00008 + | > step_time: 1.89860 (2.16335) + | > loader_time: 0.00220 (0.02271) + + + --> STEP: 123/234 -- GLOBAL_STEP: 76875 + | > loss: -0.25390 (-0.33693) + | > log_mle: -0.39542 (-0.43448) + | > loss_dur: 0.14153 (0.09755) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.50169 (14.39204) + | > current_lr: 0.00008 + | > step_time: 2.09480 (2.14845) + | > loader_time: 0.00330 (0.02334) + + + --> STEP: 128/234 -- GLOBAL_STEP: 76880 + | > loss: -0.31311 (-0.33577) + | > log_mle: -0.45420 (-0.43596) + | > loss_dur: 0.14109 (0.10018) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.91802 (15.03739) + | > current_lr: 0.00008 + | > step_time: 1.90580 (2.18656) + | > loader_time: 0.07700 (0.02382) + + + --> STEP: 133/234 -- GLOBAL_STEP: 76885 + | > loss: -0.32026 (-0.33519) + | > log_mle: -0.48563 (-0.43789) + | > loss_dur: 0.16537 (0.10270) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.57812 (15.95012) + | > current_lr: 0.00008 + | > step_time: 2.41510 (2.23837) + | > loader_time: 0.09270 (0.02526) + + + --> STEP: 138/234 -- GLOBAL_STEP: 76890 + | > loss: -0.27221 (-0.33421) + | > log_mle: -0.42330 (-0.43952) + | > loss_dur: 0.15108 (0.10530) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.04000 (16.57397) + | > current_lr: 0.00008 + | > step_time: 2.80550 (2.27524) + | > loader_time: 0.10200 (0.02595) + + + --> STEP: 143/234 -- GLOBAL_STEP: 76895 + | > loss: -0.36955 (-0.33401) + | > log_mle: -0.58397 (-0.44208) + | > loss_dur: 0.21442 (0.10807) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 41.66085 (17.40731) + | > current_lr: 0.00008 + | > step_time: 1.78800 (2.27546) + | > loader_time: 0.00300 (0.02624) + + + --> STEP: 148/234 -- GLOBAL_STEP: 76900 + | > loss: -0.31308 (-0.33422) + | > log_mle: -0.47537 (-0.44473) + | > loss_dur: 0.16229 (0.11052) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 37.21298 (18.28163) + | > current_lr: 0.00008 + | > step_time: 1.51790 (2.26215) + | > loader_time: 0.08790 (0.02602) + + + --> STEP: 153/234 -- GLOBAL_STEP: 76905 + | > loss: -0.45096 (-0.33568) + | > log_mle: -0.63038 (-0.44870) + | > loss_dur: 0.17942 (0.11303) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.12564 (19.19943) + | > current_lr: 0.00008 + | > step_time: 8.09610 (2.30460) + | > loader_time: 0.10840 (0.02661) + + + --> STEP: 158/234 -- GLOBAL_STEP: 76910 + | > loss: -0.34193 (-0.33620) + | > log_mle: -0.54327 (-0.45175) + | > loss_dur: 0.20134 (0.11555) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.35473 (20.36982) + | > current_lr: 0.00008 + | > step_time: 4.60710 (2.34815) + | > loader_time: 0.09290 (0.02697) + + + --> STEP: 163/234 -- GLOBAL_STEP: 76915 + | > loss: -0.32751 (-0.33715) + | > log_mle: -0.51628 (-0.45497) + | > loss_dur: 0.18877 (0.11782) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.53538 (21.03903) + | > current_lr: 0.00008 + | > step_time: 1.30290 (2.37131) + | > loader_time: 0.00340 (0.02745) + + + --> STEP: 168/234 -- GLOBAL_STEP: 76920 + | > loss: -0.37201 (-0.33852) + | > log_mle: -0.58481 (-0.45864) + | > loss_dur: 0.21280 (0.12013) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.27115 (21.66267) + | > current_lr: 0.00008 + | > step_time: 1.45970 (2.36260) + | > loader_time: 0.00330 (0.02722) + + + --> STEP: 173/234 -- GLOBAL_STEP: 76925 + | > loss: -0.38580 (-0.34046) + | > log_mle: -0.59414 (-0.46313) + | > loss_dur: 0.20833 (0.12267) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 38.86908 (22.32814) + | > current_lr: 0.00008 + | > step_time: 5.29370 (2.37395) + | > loader_time: 0.00290 (0.02653) + + + --> STEP: 178/234 -- GLOBAL_STEP: 76930 + | > loss: -0.43179 (-0.34255) + | > log_mle: -0.64736 (-0.46762) + | > loss_dur: 0.21557 (0.12507) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 106.59396 (23.48842) + | > current_lr: 0.00008 + | > step_time: 3.90120 (2.38596) + | > loader_time: 0.28710 (0.02902) + + + --> STEP: 183/234 -- GLOBAL_STEP: 76935 + | > loss: -0.45806 (-0.34413) + | > log_mle: -0.65587 (-0.47167) + | > loss_dur: 0.19781 (0.12754) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.27496 (24.77110) + | > current_lr: 0.00008 + | > step_time: 4.69190 (2.48481) + | > loader_time: 0.10600 (0.03084) + + + --> STEP: 188/234 -- GLOBAL_STEP: 76940 + | > loss: -0.45314 (-0.34598) + | > log_mle: -0.67251 (-0.47595) + | > loss_dur: 0.21937 (0.12996) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 58.21976 (25.94612) + | > current_lr: 0.00008 + | > step_time: 2.09520 (2.51246) + | > loader_time: 0.00460 (0.03266) + + + --> STEP: 193/234 -- GLOBAL_STEP: 76945 + | > loss: -0.45659 (-0.34818) + | > log_mle: -0.67343 (-0.48016) + | > loss_dur: 0.21683 (0.13199) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.81771 (26.71627) + | > current_lr: 0.00008 + | > step_time: 3.01130 (2.54941) + | > loader_time: 0.00540 (0.03497) + + + --> STEP: 198/234 -- GLOBAL_STEP: 76950 + | > loss: -0.44644 (-0.35023) + | > log_mle: -0.66644 (-0.48426) + | > loss_dur: 0.22000 (0.13403) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 66.93495 (27.60073) + | > current_lr: 0.00008 + | > step_time: 3.70560 (2.56184) + | > loader_time: 0.08580 (0.03543) + + + --> STEP: 203/234 -- GLOBAL_STEP: 76955 + | > loss: -0.37473 (-0.35187) + | > log_mle: -0.58077 (-0.48799) + | > loss_dur: 0.20604 (0.13612) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 43.47691 (28.64326) + | > current_lr: 0.00008 + | > step_time: 7.69660 (2.62818) + | > loader_time: 0.00550 (0.03705) + + + --> STEP: 208/234 -- GLOBAL_STEP: 76960 + | > loss: -0.43380 (-0.35420) + | > log_mle: -0.67219 (-0.49258) + | > loss_dur: 0.23839 (0.13838) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.80664 (29.73115) + | > current_lr: 0.00008 + | > step_time: 2.20230 (2.70255) + | > loader_time: 0.00430 (0.03859) + + + --> STEP: 213/234 -- GLOBAL_STEP: 76965 + | > loss: -0.48246 (-0.35717) + | > log_mle: -0.71313 (-0.49772) + | > loss_dur: 0.23067 (0.14055) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 63.48675 (30.71906) + | > current_lr: 0.00008 + | > step_time: 7.11450 (2.77769) + | > loader_time: 0.09110 (0.03960) + + + --> STEP: 218/234 -- GLOBAL_STEP: 76970 + | > loss: -0.45804 (-0.35965) + | > log_mle: -0.68167 (-0.50230) + | > loss_dur: 0.22363 (0.14265) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.47863 (31.43962) + | > current_lr: 0.00008 + | > step_time: 4.40880 (2.84308) + | > loader_time: 0.07710 (0.04133) + + + --> STEP: 223/234 -- GLOBAL_STEP: 76975 + | > loss: -0.49169 (-0.36262) + | > log_mle: -0.72572 (-0.50742) + | > loss_dur: 0.23403 (0.14480) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 85.35539 (32.22926) + | > current_lr: 0.00008 + | > step_time: 1.94740 (2.84458) + | > loader_time: 0.00370 (0.04172) + + + --> STEP: 228/234 -- GLOBAL_STEP: 76980 + | > loss: -0.47486 (-0.36567) + | > log_mle: -0.73119 (-0.51268) + | > loss_dur: 0.25633 (0.14701) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.70611 (33.11708) + | > current_lr: 0.00008 + | > step_time: 0.23630 (2.78735) + | > loader_time: 0.00280 (0.04088) + + + --> STEP: 233/234 -- GLOBAL_STEP: 76985 + | > loss: -0.05394 (-0.36654) + | > log_mle: -0.71672 (-0.51945) + | > loss_dur: 0.66278 (0.15292) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.15173 (34.14268) + | > current_lr: 0.00008 + | > step_time: 0.21590 (2.73325) + | > loader_time: 0.00310 (0.04039) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 1.12298 (+1.07811) + | > avg_loss: -0.34799 (-0.01429) + | > avg_log_mle: -0.57891 (-0.00516) + | > avg_loss_dur: 0.23092 (-0.00913) + + + > EPOCH: 329/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 06:34:59)  + + --> STEP: 4/234 -- GLOBAL_STEP: 76990 + | > loss: -0.35220 (-0.35594) + | > log_mle: -0.44339 (-0.44821) + | > loss_dur: 0.09119 (0.09228) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 22.67751 (21.87364) + | > current_lr: 0.00008 + | > step_time: 2.38990 (4.96198) + | > loader_time: 0.00200 (0.04543) + + + --> STEP: 9/234 -- GLOBAL_STEP: 76995 + | > loss: -0.35282 (-0.36690) + | > log_mle: -0.46396 (-0.45450) + | > loss_dur: 0.11113 (0.08760) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.64184 (18.45560) + | > current_lr: 0.00008 + | > step_time: 6.70440 (4.61465) + | > loader_time: 0.01020 (0.08919) + + + --> STEP: 14/234 -- GLOBAL_STEP: 77000 + | > loss: -0.39546 (-0.37850) + | > log_mle: -0.46315 (-0.45847) + | > loss_dur: 0.06769 (0.07998) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.92558 (16.98862) + | > current_lr: 0.00008 + | > step_time: 3.09710 (5.43924) + | > loader_time: 0.00550 (0.07841) + + + --> STEP: 19/234 -- GLOBAL_STEP: 77005 + | > loss: -0.41123 (-0.38584) + | > log_mle: -0.47147 (-0.46097) + | > loss_dur: 0.06024 (0.07513) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.94557 (15.61701) + | > current_lr: 0.00008 + | > step_time: 6.40090 (5.41893) + | > loader_time: 0.00230 (0.06367) + + + --> STEP: 24/234 -- GLOBAL_STEP: 77010 + | > loss: -0.39554 (-0.39013) + | > log_mle: -0.45617 (-0.46166) + | > loss_dur: 0.06063 (0.07153) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.92099 (14.24864) + | > current_lr: 0.00008 + | > step_time: 1.38010 (4.79788) + | > loader_time: 0.00130 (0.05822) + + + --> STEP: 29/234 -- GLOBAL_STEP: 77015 + | > loss: -0.40129 (-0.39295) + | > log_mle: -0.45668 (-0.46206) + | > loss_dur: 0.05539 (0.06911) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.36137 (13.59810) + | > current_lr: 0.00008 + | > step_time: 2.79720 (4.50256) + | > loader_time: 0.11030 (0.05814) + + + --> STEP: 34/234 -- GLOBAL_STEP: 77020 + | > loss: -0.37173 (-0.39152) + | > log_mle: -0.43547 (-0.46002) + | > loss_dur: 0.06374 (0.06850) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.56038 (12.90960) + | > current_lr: 0.00008 + | > step_time: 3.00050 (4.18683) + | > loader_time: 0.00210 (0.05006) + + + --> STEP: 39/234 -- GLOBAL_STEP: 77025 + | > loss: -0.34101 (-0.38676) + | > log_mle: -0.42252 (-0.45633) + | > loss_dur: 0.08151 (0.06957) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.16164 (12.53747) + | > current_lr: 0.00008 + | > step_time: 1.47860 (3.91165) + | > loader_time: 0.00140 (0.04393) + + + --> STEP: 44/234 -- GLOBAL_STEP: 77030 + | > loss: -0.35900 (-0.38324) + | > log_mle: -0.42264 (-0.45289) + | > loss_dur: 0.06364 (0.06965) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.37123 (12.49381) + | > current_lr: 0.00008 + | > step_time: 1.89750 (3.65984) + | > loader_time: 0.00150 (0.04162) + + + --> STEP: 49/234 -- GLOBAL_STEP: 77035 + | > loss: -0.37733 (-0.38136) + | > log_mle: -0.44089 (-0.45137) + | > loss_dur: 0.06357 (0.07001) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.54723 (12.15901) + | > current_lr: 0.00008 + | > step_time: 1.51790 (3.44147) + | > loader_time: 0.00180 (0.03961) + + + --> STEP: 54/234 -- GLOBAL_STEP: 77040 + | > loss: -0.34987 (-0.37864) + | > log_mle: -0.42537 (-0.44912) + | > loss_dur: 0.07550 (0.07048) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.71852 (11.94001) + | > current_lr: 0.00008 + | > step_time: 1.71500 (3.28975) + | > loader_time: 0.00170 (0.03773) + + + --> STEP: 59/234 -- GLOBAL_STEP: 77045 + | > loss: -0.34419 (-0.37661) + | > log_mle: -0.42215 (-0.44732) + | > loss_dur: 0.07796 (0.07072) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.34896 (11.72548) + | > current_lr: 0.00008 + | > step_time: 1.30630 (3.16429) + | > loader_time: 0.09110 (0.03759) + + + --> STEP: 64/234 -- GLOBAL_STEP: 77050 + | > loss: -0.33611 (-0.37183) + | > log_mle: -0.41151 (-0.44476) + | > loss_dur: 0.07541 (0.07293) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 6.55985 (11.79451) + | > current_lr: 0.00008 + | > step_time: 3.90240 (3.09251) + | > loader_time: 0.20210 (0.03923) + + + --> STEP: 69/234 -- GLOBAL_STEP: 77055 + | > loss: -0.32510 (-0.36841) + | > log_mle: -0.40806 (-0.44228) + | > loss_dur: 0.08297 (0.07387) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.03728 (11.65426) + | > current_lr: 0.00008 + | > step_time: 1.69540 (3.00796) + | > loader_time: 0.00480 (0.04024) + + + --> STEP: 74/234 -- GLOBAL_STEP: 77060 + | > loss: -0.30561 (-0.36422) + | > log_mle: -0.40095 (-0.43990) + | > loss_dur: 0.09534 (0.07567) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 11.30974 (11.73538) + | > current_lr: 0.00008 + | > step_time: 0.97300 (2.89951) + | > loader_time: 0.00180 (0.03995) + + + --> STEP: 79/234 -- GLOBAL_STEP: 77065 + | > loss: -0.32019 (-0.36098) + | > log_mle: -0.41360 (-0.43793) + | > loss_dur: 0.09341 (0.07696) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.90245 (11.58200) + | > current_lr: 0.00008 + | > step_time: 1.81020 (2.83115) + | > loader_time: 0.07990 (0.03855) + + + --> STEP: 84/234 -- GLOBAL_STEP: 77070 + | > loss: -0.32506 (-0.35819) + | > log_mle: -0.41105 (-0.43621) + | > loss_dur: 0.08599 (0.07802) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.32866 (11.55419) + | > current_lr: 0.00008 + | > step_time: 2.01310 (2.80691) + | > loader_time: 0.00280 (0.03862) + + + --> STEP: 89/234 -- GLOBAL_STEP: 77075 + | > loss: -0.30625 (-0.35525) + | > log_mle: -0.42467 (-0.43506) + | > loss_dur: 0.11842 (0.07981) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.28602 (11.58379) + | > current_lr: 0.00008 + | > step_time: 0.87400 (2.74878) + | > loader_time: 0.00200 (0.03763) + + + --> STEP: 94/234 -- GLOBAL_STEP: 77080 + | > loss: -0.32106 (-0.35271) + | > log_mle: -0.44170 (-0.43482) + | > loss_dur: 0.12064 (0.08210) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.71584 (11.93495) + | > current_lr: 0.00008 + | > step_time: 1.80390 (2.70295) + | > loader_time: 0.00290 (0.03584) + + + --> STEP: 99/234 -- GLOBAL_STEP: 77085 + | > loss: -0.31318 (-0.35047) + | > log_mle: -0.47309 (-0.43480) + | > loss_dur: 0.15991 (0.08433) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 28.56042 (12.21917) + | > current_lr: 0.00008 + | > step_time: 1.40230 (2.64918) + | > loader_time: 0.08600 (0.03676) + + + --> STEP: 104/234 -- GLOBAL_STEP: 77090 + | > loss: -0.32762 (-0.34813) + | > log_mle: -0.48047 (-0.43520) + | > loss_dur: 0.15285 (0.08707) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 23.30288 (12.53953) + | > current_lr: 0.00008 + | > step_time: 1.99410 (2.61821) + | > loader_time: 0.00420 (0.03665) + + + --> STEP: 109/234 -- GLOBAL_STEP: 77095 + | > loss: -0.28052 (-0.34534) + | > log_mle: -0.45158 (-0.43490) + | > loss_dur: 0.17105 (0.08956) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.73447 (12.78101) + | > current_lr: 0.00008 + | > step_time: 2.88740 (2.61621) + | > loader_time: 0.00390 (0.03860) + + + --> STEP: 114/234 -- GLOBAL_STEP: 77100 + | > loss: -0.29695 (-0.34334) + | > log_mle: -0.43485 (-0.43563) + | > loss_dur: 0.13790 (0.09229) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 24.10465 (13.14308) + | > current_lr: 0.00008 + | > step_time: 2.53850 (2.61542) + | > loader_time: 0.00190 (0.03933) + + + --> STEP: 119/234 -- GLOBAL_STEP: 77105 + | > loss: -0.28808 (-0.34096) + | > log_mle: -0.43237 (-0.43585) + | > loss_dur: 0.14430 (0.09490) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 16.79280 (13.38090) + | > current_lr: 0.00008 + | > step_time: 5.80050 (2.63569) + | > loader_time: 0.10760 (0.03869) + + + --> STEP: 124/234 -- GLOBAL_STEP: 77110 + | > loss: -0.31644 (-0.33869) + | > log_mle: -0.46478 (-0.43553) + | > loss_dur: 0.14834 (0.09683) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 21.91971 (13.62717) + | > current_lr: 0.00008 + | > step_time: 0.99830 (2.65438) + | > loader_time: 0.00310 (0.03724) + + + --> STEP: 129/234 -- GLOBAL_STEP: 77115 + | > loss: -0.28766 (-0.33737) + | > log_mle: -0.45012 (-0.43681) + | > loss_dur: 0.16246 (0.09945) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 36.06929 (14.36729) + | > current_lr: 0.00008 + | > step_time: 2.10230 (2.64091) + | > loader_time: 0.00190 (0.03717) + + + --> STEP: 134/234 -- GLOBAL_STEP: 77120 + | > loss: -0.30644 (-0.33646) + | > log_mle: -0.49457 (-0.43865) + | > loss_dur: 0.18813 (0.10218) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 31.81339 (15.63918) + | > current_lr: 0.00008 + | > step_time: 1.93860 (2.63058) + | > loader_time: 0.00250 (0.03660) + + + --> STEP: 139/234 -- GLOBAL_STEP: 77125 + | > loss: -0.37830 (-0.33571) + | > log_mle: -0.55510 (-0.44030) + | > loss_dur: 0.17680 (0.10459) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 42.87040 (16.33036) + | > current_lr: 0.00008 + | > step_time: 1.49820 (2.62266) + | > loader_time: 0.08280 (0.03596) + + + --> STEP: 144/234 -- GLOBAL_STEP: 77130 + | > loss: -0.33599 (-0.33480) + | > log_mle: -0.52989 (-0.44225) + | > loss_dur: 0.19391 (0.10745) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 29.41905 (17.17739) + | > current_lr: 0.00008 + | > step_time: 1.60540 (2.61440) + | > loader_time: 0.00350 (0.03595) + + + --> STEP: 149/234 -- GLOBAL_STEP: 77135 + | > loss: -0.36829 (-0.33472) + | > log_mle: -0.57254 (-0.44473) + | > loss_dur: 0.20425 (0.11000) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 48.14872 (18.09172) + | > current_lr: 0.00008 + | > step_time: 1.23700 (2.61578) + | > loader_time: 0.00270 (0.03487) + + + --> STEP: 154/234 -- GLOBAL_STEP: 77140 + | > loss: -0.35835 (-0.33556) + | > log_mle: -0.54536 (-0.44814) + | > loss_dur: 0.18700 (0.11258) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 27.50077 (18.67702) + | > current_lr: 0.00008 + | > step_time: 1.29160 (2.59895) + | > loader_time: 0.00250 (0.03560) + + + --> STEP: 159/234 -- GLOBAL_STEP: 77145 + | > loss: -0.35780 (-0.33647) + | > log_mle: -0.54765 (-0.45143) + | > loss_dur: 0.18985 (0.11496) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 60.46081 (19.96499) + | > current_lr: 0.00008 + | > step_time: 1.60300 (2.59206) + | > loader_time: 0.00480 (0.03512) + + + --> STEP: 164/234 -- GLOBAL_STEP: 77150 + | > loss: -0.34793 (-0.33739) + | > log_mle: -0.54619 (-0.45456) + | > loss_dur: 0.19825 (0.11718) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 46.66544 (20.64451) + | > current_lr: 0.00008 + | > step_time: 2.08380 (2.65271) + | > loader_time: 0.00220 (0.03583) + + + --> STEP: 169/234 -- GLOBAL_STEP: 77155 + | > loss: -0.34660 (-0.33863) + | > log_mle: -0.54686 (-0.45806) + | > loss_dur: 0.20026 (0.11944) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 55.23545 (21.53442) + | > current_lr: 0.00008 + | > step_time: 1.21240 (2.73586) + | > loader_time: 0.08960 (0.03927) + + + --> STEP: 174/234 -- GLOBAL_STEP: 77160 + | > loss: -0.45221 (-0.34091) + | > log_mle: -0.65524 (-0.46299) + | > loss_dur: 0.20302 (0.12208) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 52.02810 (22.36543) + | > current_lr: 0.00008 + | > step_time: 4.60030 (2.81762) + | > loader_time: 0.20080 (0.04150) + + + --> STEP: 179/234 -- GLOBAL_STEP: 77165 + | > loss: -0.40400 (-0.34263) + | > log_mle: -0.63466 (-0.46737) + | > loss_dur: 0.23066 (0.12473) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 56.45837 (23.33398) + | > current_lr: 0.00008 + | > step_time: 2.31960 (2.81442) + | > loader_time: 0.08050 (0.04100) + + + --> STEP: 184/234 -- GLOBAL_STEP: 77170 + | > loss: -0.38753 (-0.34404) + | > log_mle: -0.61287 (-0.47119) + | > loss_dur: 0.22534 (0.12715) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 57.57183 (24.19360) + | > current_lr: 0.00008 + | > step_time: 2.88660 (2.81357) + | > loader_time: 0.10010 (0.04137) + + + --> STEP: 189/234 -- GLOBAL_STEP: 77175 + | > loss: -0.38441 (-0.34564) + | > log_mle: -0.59584 (-0.47525) + | > loss_dur: 0.21143 (0.12961) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.76103 (25.33560) + | > current_lr: 0.00008 + | > step_time: 4.70610 (2.82901) + | > loader_time: 0.19740 (0.04144) + + + --> STEP: 194/234 -- GLOBAL_STEP: 77180 + | > loss: -0.44695 (-0.34798) + | > log_mle: -0.65016 (-0.47955) + | > loss_dur: 0.20321 (0.13157) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 39.24139 (26.03709) + | > current_lr: 0.00008 + | > step_time: 11.10840 (2.92782) + | > loader_time: 0.09700 (0.04284) + + + --> STEP: 199/234 -- GLOBAL_STEP: 77185 + | > loss: -0.44713 (-0.34978) + | > log_mle: -0.65620 (-0.48348) + | > loss_dur: 0.20907 (0.13370) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.94120 (26.65511) + | > current_lr: 0.00008 + | > step_time: 5.60140 (3.00346) + | > loader_time: 0.09450 (0.04285) + + + --> STEP: 204/234 -- GLOBAL_STEP: 77190 + | > loss: -0.45563 (-0.35135) + | > log_mle: -0.68855 (-0.48725) + | > loss_dur: 0.23291 (0.13590) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.97552 (27.38084) + | > current_lr: 0.00008 + | > step_time: 2.40310 (2.99232) + | > loader_time: 0.09350 (0.04319) + + + --> STEP: 209/234 -- GLOBAL_STEP: 77195 + | > loss: -0.43424 (-0.35361) + | > log_mle: -0.64446 (-0.49155) + | > loss_dur: 0.21022 (0.13795) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 53.88274 (28.04145) + | > current_lr: 0.00008 + | > step_time: 5.31490 (3.04904) + | > loader_time: 0.08610 (0.04448) + + + --> STEP: 214/234 -- GLOBAL_STEP: 77200 + | > loss: -0.48397 (-0.35675) + | > log_mle: -0.69090 (-0.49677) + | > loss_dur: 0.20693 (0.14002) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 49.02599 (28.74546) + | > current_lr: 0.00008 + | > step_time: 10.60800 (3.07829) + | > loader_time: 0.20930 (0.04527) + + + --> STEP: 219/234 -- GLOBAL_STEP: 77205 + | > loss: -0.56460 (-0.35955) + | > log_mle: -0.79180 (-0.50180) + | > loss_dur: 0.22720 (0.14226) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 64.75082 (29.57553) + | > current_lr: 0.00008 + | > step_time: 3.29620 (3.13866) + | > loader_time: 0.19630 (0.05091) + + + --> STEP: 224/234 -- GLOBAL_STEP: 77210 + | > loss: -0.50892 (-0.36197) + | > log_mle: -0.73459 (-0.50649) + | > loss_dur: 0.22567 (0.14452) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 69.96323 (30.43579) + | > current_lr: 0.00008 + | > step_time: 1.08320 (3.12335) + | > loader_time: 0.08390 (0.05063) + + + --> STEP: 229/234 -- GLOBAL_STEP: 77215 + | > loss: -0.49903 (-0.36493) + | > log_mle: -0.79116 (-0.51198) + | > loss_dur: 0.29213 (0.14704) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 72.15911 (31.40907) + | > current_lr: 0.00008 + | > step_time: 0.24580 (3.06041) + | > loader_time: 0.00310 (0.04960) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.01314 (-1.10984) + | > avg_loss: -0.33384 (+0.01416) + | > avg_log_mle: -0.55098 (+0.02793) + | > avg_loss_dur: 0.21714 (-0.01378) + + + > EPOCH: 330/1000 + --> /root/TTS/run-April-27-2022_08+17AM-c410bc58 + + > TRAINING (2022-04-30 06:48:01)  + + --> STEP: 0/234 -- GLOBAL_STEP: 77220 + | > loss: -0.34963 (-0.34963) + | > log_mle: -0.51993 (-0.51993) + | > loss_dur: 0.17031 (0.17031) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 47.47726 (47.47726) + | > current_lr: 0.00008 + | > step_time: 4.28510 (4.28513) + | > loader_time: 10.95850 (10.95851) + + + --> STEP: 5/234 -- GLOBAL_STEP: 77225 + | > loss: -0.33909 (-0.35136) + | > log_mle: -0.44569 (-0.44495) + | > loss_dur: 0.10659 (0.09360) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 17.36401 (19.70276) + | > current_lr: 0.00008 + | > step_time: 1.79470 (6.39492) + | > loader_time: 0.00150 (0.02265) + + + --> STEP: 10/234 -- GLOBAL_STEP: 77230 + | > loss: -0.38383 (-0.36663) + | > log_mle: -0.46054 (-0.45246) + | > loss_dur: 0.07671 (0.08583) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 15.35438 (18.30816) + | > current_lr: 0.00008 + | > step_time: 3.10780 (5.20959) + | > loader_time: 0.00270 (0.04092) + + + --> STEP: 15/234 -- GLOBAL_STEP: 77235 + | > loss: -0.39874 (-0.37812) + | > log_mle: -0.46785 (-0.45776) + | > loss_dur: 0.06911 (0.07964) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.98018 (16.88816) + | > current_lr: 0.00008 + | > step_time: 2.89470 (4.68697) + | > loader_time: 0.00100 (0.03433) + + + --> STEP: 20/234 -- GLOBAL_STEP: 77240 + | > loss: -0.41348 (-0.38609) + | > log_mle: -0.47366 (-0.46058) + | > loss_dur: 0.06018 (0.07450) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 13.08624 (15.99359) + | > current_lr: 0.00008 + | > step_time: 13.69500 (5.14432) + | > loader_time: 0.11120 (0.03197) + + + --> STEP: 25/234 -- GLOBAL_STEP: 77245 + | > loss: -0.39333 (-0.38847) + | > log_mle: -0.45405 (-0.46061) + | > loss_dur: 0.06072 (0.07213) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 8.81126 (15.16020) + | > current_lr: 0.00008 + | > step_time: 4.18960 (4.52546) + | > loader_time: 0.00270 (0.02603) + + + --> STEP: 30/234 -- GLOBAL_STEP: 77250 + | > loss: -0.38820 (-0.39119) + | > log_mle: -0.45565 (-0.46101) + | > loss_dur: 0.06745 (0.06982) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 9.07118 (14.20769) + | > current_lr: 0.00008 + | > step_time: 5.71270 (4.24745) + | > loader_time: 0.00190 (0.02482) + + + --> STEP: 35/234 -- GLOBAL_STEP: 77255 + | > loss: -0.34332 (-0.38843) + | > log_mle: -0.42194 (-0.45815) + | > loss_dur: 0.07862 (0.06972) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 12.62839 (13.67884) + | > current_lr: 0.00008 + | > step_time: 3.19130 (4.19127) + | > loader_time: 0.00210 (0.02425) + + + --> STEP: 40/234 -- GLOBAL_STEP: 77260 + | > loss: -0.33505 (-0.38416) + | > log_mle: -0.41929 (-0.45433) + | > loss_dur: 0.08424 (0.07017) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 10.34408 (13.51209) + | > current_lr: 0.00008 + | > step_time: 0.83920 (4.03794) + | > loader_time: 0.00220 (0.02593) + + + --> STEP: 45/234 -- GLOBAL_STEP: 77265 + | > loss: -0.33022 (-0.38029) + | > log_mle: -0.42316 (-0.45074) + | > loss_dur: 0.09295 (0.07045) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 14.49104 (13.38260) + | > current_lr: 0.00008 + | > step_time: 1.49880 (3.78364) + | > loader_time: 0.00180 (0.02333) + + + --> STEP: 50/234 -- GLOBAL_STEP: 77270 + | > loss: -0.36267 (-0.37838) + | > log_mle: -0.42793 (-0.44893) + | > loss_dur: 0.06526 (0.07055) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.13722 (13.01949) + | > current_lr: 0.00008 + | > step_time: 4.10130 (3.62749) + | > loader_time: 0.00290 (0.02281) + + + --> STEP: 55/234 -- GLOBAL_STEP: 77275 + | > loss: -0.36946 (-0.37571) + | > log_mle: -0.43015 (-0.44659) + | > loss_dur: 0.06068 (0.07088) + | > amp_scaler: 1024.00000 (1024.00000) + | > grad_norm: 7.78339 (12.73566) + | > current_lr: 0.00008 + | > step_time: 1.60020 (3.45706) + | > loader_time: 0.00290 (0.02231) +