diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,19630 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 4 + | > Num. of Torch Threads: 24 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Restoring from checkpoint_85000.pth ... + > Restoring Model... + > Restoring Optimizer... + > Model restored from step 85000 + + > Model has 86565676 parameters + + > EPOCH: 0/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 17:59:59)  + + --> TIME: 2024-07-13 18:00:51 -- STEP: 49/151 -- GLOBAL_STEP: 85050 + | > loss_disc: 2.270984411239624 (2.344344941937193) + | > loss_disc_real_0: 0.07595066726207733 (0.1293191237717259) + | > loss_disc_real_1: 0.19443166255950928 (0.20153363566009366) + | > loss_disc_real_2: 0.21397458016872406 (0.21202306175718502) + | > loss_disc_real_3: 0.20433378219604492 (0.2138762370664246) + | > loss_disc_real_4: 0.23436348140239716 (0.21386271623932585) + | > loss_disc_real_5: 0.2034481167793274 (0.22509207105150028) + | > loss_0: 2.270984411239624 (2.344344941937193) + | > grad_norm_0: tensor(31.0524, device='cuda:0') (tensor(27.1133, device='cuda:0')) + | > loss_gen: 2.3904876708984375 (2.5407091938719457) + | > loss_kl: 2.219207525253296 (1.8000880917724298) + | > loss_feat: 8.431201934814453 (8.107899305771811) + | > loss_mel: 17.31822395324707 (17.173261233738497) + | > loss_duration: 0.861293375492096 (0.9471519893529464) + | > loss_1: 31.220415115356445 (30.569109858298788) + | > grad_norm_1: tensor(548.5320, device='cuda:0') (tensor(474.7895, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.9901 (0.9427695614950997) + | > loader_time: 0.007 (0.00645761100613341) + + + --> TIME: 2024-07-13 18:01:42 -- STEP: 99/151 -- GLOBAL_STEP: 85100 + | > loss_disc: 2.4555039405822754 (2.35018313533128) + | > loss_disc_real_0: 0.09638208150863647 (0.13317064850619342) + | > loss_disc_real_1: 0.2800649404525757 (0.20355093012554476) + | > loss_disc_real_2: 0.1896459311246872 (0.2140977412763268) + | > loss_disc_real_3: 0.2935865521430969 (0.21058754735823834) + | > loss_disc_real_4: 0.2601836025714874 (0.212240163726036) + | > loss_disc_real_5: 0.16485488414764404 (0.22028085438892095) + | > loss_0: 2.4555039405822754 (2.35018313533128) + | > grad_norm_0: tensor(52.8882, device='cuda:0') (tensor(33.5202, device='cuda:0')) + | > loss_gen: 2.596668243408203 (2.5753634770711256) + | > loss_kl: 2.2814202308654785 (1.8577180884101174) + | > loss_feat: 8.010629653930664 (8.128030232708864) + | > loss_mel: 16.56311798095703 (17.148920415627835) + | > loss_duration: 1.0251718759536743 (0.9624096311704077) + | > loss_1: 30.477006912231445 (30.672441906399197) + | > grad_norm_1: tensor(818.5563, device='cuda:0') (tensor(514.9661, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.0683 (0.9713327306689639) + | > loader_time: 0.0075 (0.006682827015115757) + + + --> TIME: 2024-07-13 18:02:31 -- STEP: 149/151 -- GLOBAL_STEP: 85150 + | > loss_disc: 2.437288284301758 (2.359663883311638) + | > loss_disc_real_0: 0.15458367764949799 (0.13005156577533528) + | > loss_disc_real_1: 0.1364910900592804 (0.2047890981771802) + | > loss_disc_real_2: 0.2122194468975067 (0.21575130582255805) + | > loss_disc_real_3: 0.22852236032485962 (0.21199452322001425) + | > loss_disc_real_4: 0.22915241122245789 (0.2141940245692362) + | > loss_disc_real_5: 0.2717278003692627 (0.22117603260198695) + | > loss_0: 2.437288284301758 (2.359663883311638) + | > grad_norm_0: tensor(13.4101, device='cuda:0') (tensor(34.8486, device='cuda:0')) + | > loss_gen: 2.4546396732330322 (2.5672623023090737) + | > loss_kl: 1.8290010690689087 (1.8664817242014329) + | > loss_feat: 7.758839130401611 (8.061392048061293) + | > loss_mel: 16.93675422668457 (17.098031159215324) + | > loss_duration: 0.9814391136169434 (0.9625559645211137) + | > loss_1: 29.96067237854004 (30.5557232799146) + | > grad_norm_1: tensor(556.7095, device='cuda:0') (tensor(544.7836, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.8728 (0.9697136142909927) + | > loader_time: 0.0047 (0.006521536999900867) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033071114466740534 (+0) + | > avg_loss_disc: 2.516206246155959 (+0) + | > avg_loss_disc_real_0: 0.2600699410988734 (+0) + | > avg_loss_disc_real_1: 0.2083456963300705 (+0) + | > avg_loss_disc_real_2: 0.21439684927463531 (+0) + | > avg_loss_disc_real_3: 0.20721294329716608 (+0) + | > avg_loss_disc_real_4: 0.22410012781620026 (+0) + | > avg_loss_disc_real_5: 0.2721228576623477 (+0) + | > avg_loss_0: 2.516206246155959 (+0) + | > avg_loss_gen: 2.5573319288400502 (+0) + | > avg_loss_kl: 3.266693812150222 (+0) + | > avg_loss_feat: 7.817118277916541 (+0) + | > avg_loss_mel: 17.037521802462063 (+0) + | > avg_loss_duration: 1.225118692104633 (+0) + | > avg_loss_1: 31.90378497197078 (+0) + + > BEST MODEL : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/best_model_85152.pth + + > EPOCH: 1/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:02:46)  + + --> TIME: 2024-07-13 18:03:36 -- STEP: 48/151 -- GLOBAL_STEP: 85200 + | > loss_disc: 2.291482925415039 (2.337140128016472) + | > loss_disc_real_0: 0.08713032305240631 (0.1244243046579262) + | > loss_disc_real_1: 0.2207474559545517 (0.2053995244204998) + | > loss_disc_real_2: 0.17961956560611725 (0.21268792698780695) + | > loss_disc_real_3: 0.18684430420398712 (0.20644785619030395) + | > loss_disc_real_4: 0.17074066400527954 (0.2103780678783854) + | > loss_disc_real_5: 0.22951629757881165 (0.21822334577639899) + | > loss_0: 2.291482925415039 (2.337140128016472) + | > grad_norm_0: tensor(35.1502, device='cuda:0') (tensor(35.9111, device='cuda:0')) + | > loss_gen: 2.6931099891662598 (2.5599806209405274) + | > loss_kl: 2.169062852859497 (1.795740984380245) + | > loss_feat: 9.019967079162598 (8.07582836349805) + | > loss_mel: 17.394716262817383 (17.007847785949714) + | > loss_duration: 0.8977687954902649 (0.9776160592834154) + | > loss_1: 32.174625396728516 (30.41701368490855) + | > grad_norm_1: tensor(744.2356, device='cuda:0') (tensor(620.6928, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9143 (0.9439829289913177) + | > loader_time: 0.0101 (0.0060109943151474) + + + --> TIME: 2024-07-13 18:04:25 -- STEP: 98/151 -- GLOBAL_STEP: 85250 + | > loss_disc: 2.3556876182556152 (2.3679243253201854) + | > loss_disc_real_0: 0.1668941229581833 (0.13362428348283376) + | > loss_disc_real_1: 0.1987946778535843 (0.20541246189754836) + | > loss_disc_real_2: 0.23121164739131927 (0.21406117096847418) + | > loss_disc_real_3: 0.21170906722545624 (0.2110379178609167) + | > loss_disc_real_4: 0.19676923751831055 (0.21251810113994443) + | > loss_disc_real_5: 0.25027498602867126 (0.22262253870769422) + | > loss_0: 2.3556876182556152 (2.3679243253201854) + | > grad_norm_0: tensor(54.7695, device='cuda:0') (tensor(40.5022, device='cuda:0')) + | > loss_gen: 2.6753509044647217 (2.5653440271105095) + | > loss_kl: 2.046419382095337 (1.8629827024985333) + | > loss_feat: 8.970094680786133 (7.965650952592186) + | > loss_mel: 18.332874298095703 (17.100221945315) + | > loss_duration: 0.8754838705062866 (0.9751164828027997) + | > loss_1: 32.90022277832031 (30.4693160349009) + | > grad_norm_1: tensor(727.4832, device='cuda:0') (tensor(599.3464, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9292 (0.9577148909471473) + | > loader_time: 0.0072 (0.006329504811033911) + + + --> TIME: 2024-07-13 18:05:13 -- STEP: 148/151 -- GLOBAL_STEP: 85300 + | > loss_disc: 2.3887555599212646 (2.374854830471245) + | > loss_disc_real_0: 0.09432825446128845 (0.13117986197608555) + | > loss_disc_real_1: 0.17178091406822205 (0.20592449011432157) + | > loss_disc_real_2: 0.20341573655605316 (0.21382793894893415) + | > loss_disc_real_3: 0.19926588237285614 (0.21126394547723437) + | > loss_disc_real_4: 0.22698403894901276 (0.2154672916072446) + | > loss_disc_real_5: 0.2556914985179901 (0.2226588744166735) + | > loss_0: 2.3887555599212646 (2.374854830471245) + | > grad_norm_0: tensor(30.9386, device='cuda:0') (tensor(38.0835, device='cuda:0')) + | > loss_gen: 2.645131826400757 (2.545274417142611) + | > loss_kl: 1.8808661699295044 (1.842754584309217) + | > loss_feat: 9.748680114746094 (7.9876554785547995) + | > loss_mel: 17.656360626220703 (17.077846320899756) + | > loss_duration: 0.9488494396209717 (0.9690932180430438) + | > loss_1: 32.87989044189453 (30.42262380187576) + | > grad_norm_1: tensor(641.6022, device='cuda:0') (tensor(588.4418, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8953 (0.9571459148381207) + | > loader_time: 0.0054 (0.006222341511700605) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03442933009221004 (+0.0013582156254695055) + | > avg_loss_disc: 2.5450064952556906 (+0.028800249099731445) + | > avg_loss_disc_real_0: 0.24456198742756477 (-0.01550795367130861) + | > avg_loss_disc_real_1: 0.266665817453311 (+0.05832012112324053) + | > avg_loss_disc_real_2: 0.2593347246830281 (+0.04493787540839278) + | > avg_loss_disc_real_3: 0.24488911147301012 (+0.037676168175844044) + | > avg_loss_disc_real_4: 0.20635214562599474 (-0.017747982190205513) + | > avg_loss_disc_real_5: 0.24059694546919602 (-0.03152591219315165) + | > avg_loss_0: 2.5450064952556906 (+0.028800249099731445) + | > avg_loss_gen: 2.5621450497553897 (+0.004813120915339475) + | > avg_loss_kl: 3.205343118080726 (-0.06135069406949567) + | > avg_loss_feat: 7.618905324202317 (-0.19821295371422387) + | > avg_loss_mel: 17.08466148376465 (+0.0471396813025855) + | > avg_loss_duration: 1.2380147255384004 (+0.012896033433767329) + | > avg_loss_1: 31.70906947209285 (-0.1947154998779297) + + > BEST MODEL : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/best_model_85303.pth + + > EPOCH: 2/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:05:29)  + + --> TIME: 2024-07-13 18:06:19 -- STEP: 47/151 -- GLOBAL_STEP: 85350 + | > loss_disc: 2.4306905269622803 (2.3756229978926644) + | > loss_disc_real_0: 0.1312180757522583 (0.13026676152614838) + | > loss_disc_real_1: 0.24820753931999207 (0.20739339323754005) + | > loss_disc_real_2: 0.24570834636688232 (0.21501050096877078) + | > loss_disc_real_3: 0.19590647518634796 (0.2112911321381305) + | > loss_disc_real_4: 0.16437934339046478 (0.21596398822804716) + | > loss_disc_real_5: 0.24053968489170074 (0.22624933212361437) + | > loss_0: 2.4306905269622803 (2.3756229978926644) + | > grad_norm_0: tensor(12.0704, device='cuda:0') (tensor(33.6230, device='cuda:0')) + | > loss_gen: 2.3242900371551514 (2.5063495230167474) + | > loss_kl: 1.8219397068023682 (1.7411585691127371) + | > loss_feat: 7.689176082611084 (7.919756087851017) + | > loss_mel: 17.063894271850586 (16.939573511164237) + | > loss_duration: 0.9998279213905334 (0.9717166499888643) + | > loss_1: 29.899127960205078 (30.07855427518804) + | > grad_norm_1: tensor(263.8590, device='cuda:0') (tensor(552.3603, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8619 (1.0004666003775085) + | > loader_time: 0.0048 (0.006368155175067009) + + + --> TIME: 2024-07-13 18:07:06 -- STEP: 97/151 -- GLOBAL_STEP: 85400 + | > loss_disc: 2.3248627185821533 (2.366664178592643) + | > loss_disc_real_0: 0.18208090960979462 (0.12878648982834578) + | > loss_disc_real_1: 0.15297505259513855 (0.20380483813507042) + | > loss_disc_real_2: 0.17747394740581512 (0.21377153264493057) + | > loss_disc_real_3: 0.18881666660308838 (0.21149359320856864) + | > loss_disc_real_4: 0.187322735786438 (0.21389657566227863) + | > loss_disc_real_5: 0.24108664691448212 (0.22565288012175216) + | > loss_0: 2.3248627185821533 (2.366664178592643) + | > grad_norm_0: tensor(57.7924, device='cuda:0') (tensor(34.5359, device='cuda:0')) + | > loss_gen: 2.6300511360168457 (2.5318088851024183) + | > loss_kl: 1.6415892839431763 (1.7933832628210795) + | > loss_feat: 8.322636604309082 (7.9730433434555215) + | > loss_mel: 17.053272247314453 (17.06077589447966) + | > loss_duration: 0.9885591268539429 (0.9699887252345527) + | > loss_1: 30.6361083984375 (30.329000079754703) + | > grad_norm_1: tensor(755.8859, device='cuda:0') (tensor(556.0555, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8335 (0.9672263922150599) + | > loader_time: 0.0087 (0.00638045969697618) + + + --> TIME: 2024-07-13 18:07:56 -- STEP: 147/151 -- GLOBAL_STEP: 85450 + | > loss_disc: 2.214271306991577 (2.3689388677376466) + | > loss_disc_real_0: 0.11716435849666595 (0.13051126329671767) + | > loss_disc_real_1: 0.20707130432128906 (0.20474779828876055) + | > loss_disc_real_2: 0.2049092948436737 (0.21367589166375245) + | > loss_disc_real_3: 0.22696812450885773 (0.2115526269285046) + | > loss_disc_real_4: 0.2321663349866867 (0.21414283379202798) + | > loss_disc_real_5: 0.2134801596403122 (0.2261839399532396) + | > loss_0: 2.214271306991577 (2.3689388677376466) + | > grad_norm_0: tensor(8.9893, device='cuda:0') (tensor(36.0805, device='cuda:0')) + | > loss_gen: 2.6070542335510254 (2.527575852919597) + | > loss_kl: 2.1868038177490234 (1.7889570695202366) + | > loss_feat: 8.23537826538086 (7.8684507908464285) + | > loss_mel: 18.217737197875977 (16.953840411439245) + | > loss_duration: 1.0004446506500244 (0.9733249054474085) + | > loss_1: 32.24741744995117 (30.11214912829756) + | > grad_norm_1: tensor(587.2133, device='cuda:0') (tensor(575.2380, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9647 (0.9696395689127395) + | > loader_time: 0.0058 (0.00635003881389592) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032294401755699746 (-0.002134928336510293) + | > avg_loss_disc: 2.530505363757794 (-0.014501131497896669) + | > avg_loss_disc_real_0: 0.19752523990777823 (-0.04703674751978654) + | > avg_loss_disc_real_1: 0.21825703978538513 (-0.048408777667925895) + | > avg_loss_disc_real_2: 0.2517618938134267 (-0.007572830869601421) + | > avg_loss_disc_real_3: 0.25275376668343175 (+0.007864655210421623) + | > avg_loss_disc_real_4: 0.2248870306290113 (+0.01853488500301656) + | > avg_loss_disc_real_5: 0.1941643815774184 (-0.046432563891777634) + | > avg_loss_0: 2.530505363757794 (-0.014501131497896669) + | > avg_loss_gen: 2.343727937111488 (-0.21841711264390185) + | > avg_loss_kl: 3.231970952107356 (+0.026627834026629937) + | > avg_loss_feat: 7.609666200784536 (-0.009239123417780881) + | > avg_loss_mel: 17.34096307020921 (+0.256301586444561) + | > avg_loss_duration: 1.268319423382099 (+0.03030469784369849) + | > avg_loss_1: 31.794647950392502 (+0.08557847829965226) + + + > EPOCH: 3/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:08:09)  + + --> TIME: 2024-07-13 18:08:57 -- STEP: 46/151 -- GLOBAL_STEP: 85500 + | > loss_disc: 2.272672653198242 (2.361655696578649) + | > loss_disc_real_0: 0.04966231808066368 (0.13658618368208406) + | > loss_disc_real_1: 0.155034601688385 (0.19756281051946722) + | > loss_disc_real_2: 0.14979426562786102 (0.2124440420580947) + | > loss_disc_real_3: 0.1494610458612442 (0.20894766339789267) + | > loss_disc_real_4: 0.16989551484584808 (0.20905386332584464) + | > loss_disc_real_5: 0.1727299690246582 (0.22864162176847458) + | > loss_0: 2.272672653198242 (2.361655696578649) + | > grad_norm_0: tensor(42.7584, device='cuda:0') (tensor(44.0583, device='cuda:0')) + | > loss_gen: 2.45949649810791 (2.5493367184763374) + | > loss_kl: 2.2754955291748047 (1.8399419603140459) + | > loss_feat: 8.648070335388184 (7.8857210615406865) + | > loss_mel: 17.864734649658203 (17.086866171463683) + | > loss_duration: 0.8522070050239563 (0.9645773237166198) + | > loss_1: 32.10000228881836 (30.326443008754563) + | > grad_norm_1: tensor(539.5356, device='cuda:0') (tensor(607.6848, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8826 (0.9953850196755454) + | > loader_time: 0.0049 (0.0063909447711447) + + + --> TIME: 2024-07-13 18:09:45 -- STEP: 96/151 -- GLOBAL_STEP: 85550 + | > loss_disc: 2.423623561859131 (2.361676471928756) + | > loss_disc_real_0: 0.21994835138320923 (0.13622251917452863) + | > loss_disc_real_1: 0.20634987950325012 (0.2021476075363656) + | > loss_disc_real_2: 0.23821565508842468 (0.2146624163724482) + | > loss_disc_real_3: 0.24255071580410004 (0.20978012572353086) + | > loss_disc_real_4: 0.2745571732521057 (0.2085987031459808) + | > loss_disc_real_5: 0.2302924394607544 (0.22532315893719593) + | > loss_0: 2.423623561859131 (2.361676471928756) + | > grad_norm_0: tensor(36.5388, device='cuda:0') (tensor(35.8385, device='cuda:0')) + | > loss_gen: 2.6613712310791016 (2.5379595334331193) + | > loss_kl: 1.6948800086975098 (1.822296131402254) + | > loss_feat: 6.7168779373168945 (7.870067695776622) + | > loss_mel: 16.989255905151367 (17.05517413218816) + | > loss_duration: 1.0048863887786865 (0.9685815802464882) + | > loss_1: 29.067272186279297 (30.254078924655914) + | > grad_norm_1: tensor(434.0395, device='cuda:0') (tensor(541.8782, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.977 (0.9768321067094804) + | > loader_time: 0.0077 (0.00628662109375) + + + --> TIME: 2024-07-13 18:10:34 -- STEP: 146/151 -- GLOBAL_STEP: 85600 + | > loss_disc: 1.9726908206939697 (2.356752728762692) + | > loss_disc_real_0: 0.11803142726421356 (0.13277029003692822) + | > loss_disc_real_1: 0.1411200910806656 (0.2034386519904006) + | > loss_disc_real_2: 0.15673398971557617 (0.2143308836711596) + | > loss_disc_real_3: 0.21632879972457886 (0.2079782801335805) + | > loss_disc_real_4: 0.17708994448184967 (0.2083535295949407) + | > loss_disc_real_5: 0.15121059119701385 (0.2233416181925225) + | > loss_0: 1.9726908206939697 (2.356752728762692) + | > grad_norm_0: tensor(30.2826, device='cuda:0') (tensor(34.3997, device='cuda:0')) + | > loss_gen: 3.176170587539673 (2.5451303851114564) + | > loss_kl: 1.8994771242141724 (1.8026190502186348) + | > loss_feat: 10.49696159362793 (7.968362635129119) + | > loss_mel: 17.880081176757812 (17.07405488131797) + | > loss_duration: 0.9211073517799377 (0.9670695695158553) + | > loss_1: 34.37379837036133 (30.35723650945376) + | > grad_norm_1: tensor(542.9763, device='cuda:0') (tensor(535.5522, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8096 (0.9697384295398241) + | > loader_time: 0.0038 (0.006232000377080212) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03277703431936411 (+0.0004826325636643619) + | > avg_loss_disc: 2.5615615661327658 (+0.031056202374971864) + | > avg_loss_disc_real_0: 0.1977501419874338 (+0.00022490207965555897) + | > avg_loss_disc_real_1: 0.21868011011527136 (+0.00042307032988622706) + | > avg_loss_disc_real_2: 0.2314129231067804 (-0.020348970706646263) + | > avg_loss_disc_real_3: 0.2524316849616858 (-0.0003220817217459704) + | > avg_loss_disc_real_4: 0.2397850534090629 (+0.01489802278005159) + | > avg_loss_disc_real_5: 0.25436588434072643 (+0.060201502763308046) + | > avg_loss_0: 2.5615615661327658 (+0.031056202374971864) + | > avg_loss_gen: 2.4233520397773156 (+0.07962410266582776) + | > avg_loss_kl: 3.335407752257127 (+0.10343680014977075) + | > avg_loss_feat: 7.843881607055664 (+0.2342154062711277) + | > avg_loss_mel: 17.68033951979417 (+0.33937644958496094) + | > avg_loss_duration: 1.2809403309455285 (+0.012620907563429595) + | > avg_loss_1: 32.56392156160795 (+0.7692736112154464) + + + > EPOCH: 4/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:10:48)  + + --> TIME: 2024-07-13 18:11:34 -- STEP: 45/151 -- GLOBAL_STEP: 85650 + | > loss_disc: 2.2742137908935547 (2.3540068997277164) + | > loss_disc_real_0: 0.10329010337591171 (0.13301914359132444) + | > loss_disc_real_1: 0.24824723601341248 (0.2009595149093204) + | > loss_disc_real_2: 0.24010707437992096 (0.2126692450708813) + | > loss_disc_real_3: 0.17372140288352966 (0.20753841466373868) + | > loss_disc_real_4: 0.24359866976737976 (0.21090421146816676) + | > loss_disc_real_5: 0.20391637086868286 (0.22170643409093221) + | > loss_0: 2.2742137908935547 (2.3540068997277164) + | > grad_norm_0: tensor(20.3617, device='cuda:0') (tensor(41.0831, device='cuda:0')) + | > loss_gen: 2.752993583679199 (2.5647800922393804) + | > loss_kl: 2.0122690200805664 (1.817407653066847) + | > loss_feat: 8.183341979980469 (8.161592260996496) + | > loss_mel: 17.386598587036133 (17.114552052815746) + | > loss_duration: 0.9300900101661682 (0.9772878805796306) + | > loss_1: 31.265295028686523 (30.635619778103298) + | > grad_norm_1: tensor(516.5152, device='cuda:0') (tensor(582.0282, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9886 (0.9879918734232584) + | > loader_time: 0.006 (0.006361182530721029) + + + --> TIME: 2024-07-13 18:12:23 -- STEP: 95/151 -- GLOBAL_STEP: 85700 + | > loss_disc: 2.4250972270965576 (2.369670865410253) + | > loss_disc_real_0: 0.23182061314582825 (0.13465314954519272) + | > loss_disc_real_1: 0.25508397817611694 (0.20324624518030568) + | > loss_disc_real_2: 0.248159721493721 (0.21514020060238084) + | > loss_disc_real_3: 0.2400968074798584 (0.20865566761870133) + | > loss_disc_real_4: 0.21237464249134064 (0.21212901159336692) + | > loss_disc_real_5: 0.2062804251909256 (0.22487315676714245) + | > loss_0: 2.4250972270965576 (2.369670865410253) + | > grad_norm_0: tensor(55.9996, device='cuda:0') (tensor(35.2583, device='cuda:0')) + | > loss_gen: 2.7470011711120605 (2.5324689563951983) + | > loss_kl: 1.9841060638427734 (1.836852502822876) + | > loss_feat: 6.487278938293457 (8.004666724957916) + | > loss_mel: 16.773447036743164 (17.04957134849147) + | > loss_duration: 0.9591556191444397 (0.9725481516436527) + | > loss_1: 28.95098876953125 (30.396107663606344) + | > grad_norm_1: tensor(610.1434, device='cuda:0') (tensor(520.1939, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.886 (0.9752738726766486) + | > loader_time: 0.0059 (0.006472590095118472) + + + --> TIME: 2024-07-13 18:13:11 -- STEP: 145/151 -- GLOBAL_STEP: 85750 + | > loss_disc: 2.2784974575042725 (2.35552782190257) + | > loss_disc_real_0: 0.1375136375427246 (0.13060412062653176) + | > loss_disc_real_1: 0.18414561450481415 (0.2035006934712673) + | > loss_disc_real_2: 0.20079746842384338 (0.21500383235257248) + | > loss_disc_real_3: 0.2087894231081009 (0.20863633104439436) + | > loss_disc_real_4: 0.17193779349327087 (0.21041671130163916) + | > loss_disc_real_5: 0.24054723978042603 (0.22313021257005888) + | > loss_0: 2.2784974575042725 (2.35552782190257) + | > grad_norm_0: tensor(23.9220, device='cuda:0') (tensor(33.9173, device='cuda:0')) + | > loss_gen: 2.597346067428589 (2.5490854197535016) + | > loss_kl: 1.7985855340957642 (1.823840671572192) + | > loss_feat: 6.945088863372803 (8.051241177526014) + | > loss_mel: 16.432519912719727 (17.03744758079792) + | > loss_duration: 0.985998272895813 (0.9696045480925461) + | > loss_1: 28.759536743164062 (30.431219390342974) + | > grad_norm_1: tensor(728.3344, device='cuda:0') (tensor(523.8391, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8608 (0.9636011156542548) + | > loader_time: 0.0043 (0.006226301193237304) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03155987079326923 (-0.0012171635260948763) + | > avg_loss_disc: 2.6367573004502516 (+0.07519573431748583) + | > avg_loss_disc_real_0: 0.22118438665683454 (+0.02343424466940075) + | > avg_loss_disc_real_1: 0.2260683408150306 (+0.007388230699759246) + | > avg_loss_disc_real_2: 0.2207980453968048 (-0.010614877709975601) + | > avg_loss_disc_real_3: 0.22763815866066858 (-0.024793526301017194) + | > avg_loss_disc_real_4: 0.22037395720298475 (-0.019411096206078143) + | > avg_loss_disc_real_5: 0.22310982529933637 (-0.03125605904139006) + | > avg_loss_0: 2.6367573004502516 (+0.07519573431748583) + | > avg_loss_gen: 2.202641560481145 (-0.22071047929617071) + | > avg_loss_kl: 3.246712171114408 (-0.0886955811427188) + | > avg_loss_feat: 7.162334552178016 (-0.6815470548776483) + | > avg_loss_mel: 16.57621662433331 (-1.1041228954608613) + | > avg_loss_duration: 1.304986990415133 (+0.024046659469604492) + | > avg_loss_1: 30.49289160508376 (-2.071029956524189) + + > BEST MODEL : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/best_model_85756.pth + + > EPOCH: 5/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:13:30)  + + --> TIME: 2024-07-13 18:14:19 -- STEP: 44/151 -- GLOBAL_STEP: 85800 + | > loss_disc: 2.4278311729431152 (2.350394755601883) + | > loss_disc_real_0: 0.1663714051246643 (0.1289651500569148) + | > loss_disc_real_1: 0.20760740339756012 (0.20618569275194948) + | > loss_disc_real_2: 0.21454426646232605 (0.21359677443450148) + | > loss_disc_real_3: 0.20802830159664154 (0.2094451053576036) + | > loss_disc_real_4: 0.24495963752269745 (0.20909597995606335) + | > loss_disc_real_5: 0.1560090184211731 (0.22234707427295772) + | > loss_0: 2.4278311729431152 (2.350394755601883) + | > grad_norm_0: tensor(70.1052, device='cuda:0') (tensor(40.9964, device='cuda:0')) + | > loss_gen: 2.5169456005096436 (2.5360014059326867) + | > loss_kl: 1.714514136314392 (1.8036173609170048) + | > loss_feat: 7.723975658416748 (7.643759868361733) + | > loss_mel: 16.952171325683594 (16.806616739793256) + | > loss_duration: 0.9364227056503296 (0.9876962900161743) + | > loss_1: 29.84402847290039 (29.777691667730156) + | > grad_norm_1: tensor(688.1333, device='cuda:0') (tensor(602.7381, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8996 (1.0661976337432864) + | > loader_time: 0.0058 (0.006753049113533713) + + + --> TIME: 2024-07-13 18:15:05 -- STEP: 94/151 -- GLOBAL_STEP: 85850 + | > loss_disc: 2.340970516204834 (2.3345982559183818) + | > loss_disc_real_0: 0.08322817087173462 (0.12722802209727307) + | > loss_disc_real_1: 0.17604824900627136 (0.20487388033182063) + | > loss_disc_real_2: 0.16734261810779572 (0.21301860520814328) + | > loss_disc_real_3: 0.19864891469478607 (0.20841445108043385) + | > loss_disc_real_4: 0.20067012310028076 (0.2094670741006415) + | > loss_disc_real_5: 0.18622200191020966 (0.21922165678536637) + | > loss_0: 2.340970516204834 (2.3345982559183818) + | > grad_norm_0: tensor(33.2321, device='cuda:0') (tensor(39.7085, device='cuda:0')) + | > loss_gen: 2.4603796005249023 (2.5745263759126047) + | > loss_kl: 1.8132236003875732 (1.8101904506378985) + | > loss_feat: 9.192804336547852 (7.958134275801636) + | > loss_mel: 17.492555618286133 (17.048183431016632) + | > loss_duration: 0.925527036190033 (0.968936764813484) + | > loss_1: 31.884492874145508 (30.359971330520956) + | > grad_norm_1: tensor(392.9746, device='cuda:0') (tensor(596.4982, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8714 (0.9840277255849635) + | > loader_time: 0.0046 (0.006251619217243601) + + + --> TIME: 2024-07-13 18:15:55 -- STEP: 144/151 -- GLOBAL_STEP: 85900 + | > loss_disc: 2.3532731533050537 (2.3309248594774146) + | > loss_disc_real_0: 0.15600943565368652 (0.12880558135091433) + | > loss_disc_real_1: 0.19105248153209686 (0.20412513541264665) + | > loss_disc_real_2: 0.22631549835205078 (0.21153397609790167) + | > loss_disc_real_3: 0.21031741797924042 (0.20771674009867838) + | > loss_disc_real_4: 0.22593015432357788 (0.20888392197796035) + | > loss_disc_real_5: 0.21714097261428833 (0.21834006936599815) + | > loss_0: 2.3532731533050537 (2.3309248594774146) + | > grad_norm_0: tensor(12.7403, device='cuda:0') (tensor(35.5880, device='cuda:0')) + | > loss_gen: 2.6485743522644043 (2.5738460587130643) + | > loss_kl: 2.035264253616333 (1.8246904959281285) + | > loss_feat: 7.782230854034424 (8.009509997235405) + | > loss_mel: 17.579936981201172 (17.07455288701588) + | > loss_duration: 1.0095123052597046 (0.9679742116067145) + | > loss_1: 31.055519104003906 (30.450573629803127) + | > grad_norm_1: tensor(320.4420, device='cuda:0') (tensor(549.0327, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.99 (0.9805503189563751) + | > loader_time: 0.006 (0.00618420872423384) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031441266720111556 (-0.00011860407315767568) + | > avg_loss_disc: 2.469607169811542 (-0.16715013063870954) + | > avg_loss_disc_real_0: 0.25384677946567535 (+0.03266239280884081) + | > avg_loss_disc_real_1: 0.22780821988215813 (+0.0017398790671275255) + | > avg_loss_disc_real_2: 0.1903728865660154 (-0.030425158830789417) + | > avg_loss_disc_real_3: 0.2251917513517233 (-0.00244640730894527) + | > avg_loss_disc_real_4: 0.21883918344974518 (-0.001534773753239571) + | > avg_loss_disc_real_5: 0.21931901918007776 (-0.0037908061192586107) + | > avg_loss_0: 2.469607169811542 (-0.16715013063870954) + | > avg_loss_gen: 2.589128512602586 (+0.3864869521214409) + | > avg_loss_kl: 3.2825767627129188 (+0.03586459159851074) + | > avg_loss_feat: 8.180939197540283 (+1.0186046453622675) + | > avg_loss_mel: 17.163143157958984 (+0.5869265336256753) + | > avg_loss_duration: 1.3021098558719342 (-0.0028771345431988227) + | > avg_loss_1: 32.51789738581731 (+2.0250057807335473) + + + > EPOCH: 6/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:16:11)  + + --> TIME: 2024-07-13 18:16:57 -- STEP: 43/151 -- GLOBAL_STEP: 85950 + | > loss_disc: 2.1433968544006348 (2.318479665490084) + | > loss_disc_real_0: 0.07255455106496811 (0.12835740350013553) + | > loss_disc_real_1: 0.16748692095279694 (0.2015968845334164) + | > loss_disc_real_2: 0.1926363855600357 (0.21459403668725213) + | > loss_disc_real_3: 0.17442406713962555 (0.20529196047505668) + | > loss_disc_real_4: 0.19489121437072754 (0.20765655609064324) + | > loss_disc_real_5: 0.2079034000635147 (0.21560056784818338) + | > loss_0: 2.1433968544006348 (2.318479665490084) + | > grad_norm_0: tensor(23.3875, device='cuda:0') (tensor(31.7163, device='cuda:0')) + | > loss_gen: 2.64428448677063 (2.5823098116142806) + | > loss_kl: 2.0760927200317383 (1.8193124116853225) + | > loss_feat: 7.832683563232422 (8.11666117158047) + | > loss_mel: 17.983713150024414 (17.00668310564618) + | > loss_duration: 1.0400538444519043 (0.96359552061835) + | > loss_1: 31.57682991027832 (30.488562162532364) + | > grad_norm_1: tensor(559.5432, device='cuda:0') (tensor(533.6964, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9932 (1.005233781282292) + | > loader_time: 0.0066 (0.0064977601517078485) + + + --> TIME: 2024-07-13 18:17:46 -- STEP: 93/151 -- GLOBAL_STEP: 86000 + | > loss_disc: 2.3361918926239014 (2.345047609780424) + | > loss_disc_real_0: 0.06602006405591965 (0.1288765592360368) + | > loss_disc_real_1: 0.18704676628112793 (0.20440731397880024) + | > loss_disc_real_2: 0.19971199333667755 (0.21681097053712414) + | > loss_disc_real_3: 0.19557924568653107 (0.20653765368205246) + | > loss_disc_real_4: 0.193961501121521 (0.20970064738104421) + | > loss_disc_real_5: 0.21531380712985992 (0.21999360973476081) + | > loss_0: 2.3361918926239014 (2.345047609780424) + | > grad_norm_0: tensor(20.8704, device='cuda:0') (tensor(37.0218, device='cuda:0')) + | > loss_gen: 2.4144155979156494 (2.567456732514085) + | > loss_kl: 1.278347373008728 (1.8068782071913443) + | > loss_feat: 6.717871189117432 (7.974758291757234) + | > loss_mel: 15.713364601135254 (17.030709430735595) + | > loss_duration: 1.0663431882858276 (0.9620356796890177) + | > loss_1: 27.19034194946289 (30.34183830343267) + | > grad_norm_1: tensor(289.3896, device='cuda:0') (tensor(588.0049, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8032 (0.9841661658338321) + | > loader_time: 0.0044 (0.006300110970774006) + + + --> TIME: 2024-07-13 18:18:52 -- STEP: 143/151 -- GLOBAL_STEP: 86050 + | > loss_disc: 2.2386550903320312 (2.3427351271355894) + | > loss_disc_real_0: 0.08705215901136398 (0.13064019223088985) + | > loss_disc_real_1: 0.18115147948265076 (0.20434145056284395) + | > loss_disc_real_2: 0.2119380533695221 (0.2177158415942758) + | > loss_disc_real_3: 0.17051170766353607 (0.20780028387174737) + | > loss_disc_real_4: 0.16502977907657623 (0.20986014771294761) + | > loss_disc_real_5: 0.20097410678863525 (0.2180518449603261) + | > loss_0: 2.2386550903320312 (2.3427351271355894) + | > grad_norm_0: tensor(22.0708, device='cuda:0') (tensor(39.0762, device='cuda:0')) + | > loss_gen: 2.4426534175872803 (2.5847744941711426) + | > loss_kl: 1.5456931591033936 (1.8117478314813196) + | > loss_feat: 8.52739429473877 (8.038974858664133) + | > loss_mel: 16.486474990844727 (17.038807315426272) + | > loss_duration: 0.8798220157623291 (0.9663417852008259) + | > loss_1: 29.882038116455078 (30.44064627160559) + | > grad_norm_1: tensor(330.7585, device='cuda:0') (tensor(583.0942, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8509 (0.9915213968370344) + | > loader_time: 0.0042 (0.00629992251629596) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030828494292039137 (-0.0006127724280724188) + | > avg_loss_disc: 2.563803709470309 (+0.09419653965876673) + | > avg_loss_disc_real_0: 0.18433883786201477 (-0.06950794160366058) + | > avg_loss_disc_real_1: 0.21208698589068192 (-0.015721233991476208) + | > avg_loss_disc_real_2: 0.26072168120971095 (+0.07034879464369556) + | > avg_loss_disc_real_3: 0.22167812975553366 (-0.0035136215961896478) + | > avg_loss_disc_real_4: 0.2414577717964466 (+0.022618588346701413) + | > avg_loss_disc_real_5: 0.22846950819859138 (+0.009150489018513619) + | > avg_loss_0: 2.563803709470309 (+0.09419653965876673) + | > avg_loss_gen: 2.3237771621117225 (-0.26535135049086334) + | > avg_loss_kl: 3.322965200130756 (+0.04038843741783715) + | > avg_loss_feat: 7.418247809776893 (-0.7626913877633905) + | > avg_loss_mel: 17.08631412799542 (-0.07682902996356589) + | > avg_loss_duration: 1.295405938075139 (-0.0067039177967951336) + | > avg_loss_1: 31.446710439828728 (-1.071186945988579) + + + > EPOCH: 7/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:19:08)  + + --> TIME: 2024-07-13 18:19:52 -- STEP: 42/151 -- GLOBAL_STEP: 86100 + | > loss_disc: 2.3029568195343018 (2.379237186341059) + | > loss_disc_real_0: 0.08339662849903107 (0.12728864149678318) + | > loss_disc_real_1: 0.1881801038980484 (0.2110821959518251) + | > loss_disc_real_2: 0.1940440833568573 (0.21711507404134386) + | > loss_disc_real_3: 0.19685757160186768 (0.20884367238198007) + | > loss_disc_real_4: 0.15901318192481995 (0.2089413351246289) + | > loss_disc_real_5: 0.26555970311164856 (0.22560043739421026) + | > loss_0: 2.3029568195343018 (2.379237186341059) + | > grad_norm_0: tensor(37.9935, device='cuda:0') (tensor(35.2334, device='cuda:0')) + | > loss_gen: 2.6247880458831787 (2.4969259500503536) + | > loss_kl: 1.5264467000961304 (1.8254016822292691) + | > loss_feat: 9.265762329101562 (7.792644370169866) + | > loss_mel: 17.643478393554688 (16.57797234398978) + | > loss_duration: 0.9530174732208252 (0.9850284215949827) + | > loss_1: 32.013492584228516 (29.677972521100724) + | > grad_norm_1: tensor(673.4719, device='cuda:0') (tensor(492.8233, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.823 (1.00566684064411) + | > loader_time: 0.0044 (0.006544345901125953) + + + --> TIME: 2024-07-13 18:20:40 -- STEP: 92/151 -- GLOBAL_STEP: 86150 + | > loss_disc: 2.310433864593506 (2.358461898306142) + | > loss_disc_real_0: 0.09970615804195404 (0.13018368885083037) + | > loss_disc_real_1: 0.2709108293056488 (0.20768700540065765) + | > loss_disc_real_2: 0.21002110838890076 (0.21624060752599136) + | > loss_disc_real_3: 0.2682053744792938 (0.205716458675654) + | > loss_disc_real_4: 0.17409764230251312 (0.2083597810048124) + | > loss_disc_real_5: 0.23744510114192963 (0.22413064102115837) + | > loss_0: 2.310433864593506 (2.358461898306142) + | > grad_norm_0: tensor(23.7964, device='cuda:0') (tensor(38.5216, device='cuda:0')) + | > loss_gen: 2.6435706615448 (2.550471839697465) + | > loss_kl: 1.9274353981018066 (1.835373297981594) + | > loss_feat: 9.111540794372559 (7.966747815194338) + | > loss_mel: 17.53369140625 (16.827276478643) + | > loss_duration: 0.9951971769332886 (0.9716451349465742) + | > loss_1: 32.21143341064453 (30.151514447253682) + | > grad_norm_1: tensor(511.5360, device='cuda:0') (tensor(564.0400, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.886 (0.9730602917463883) + | > loader_time: 0.0064 (0.006259104479914126) + + + --> TIME: 2024-07-13 18:21:30 -- STEP: 142/151 -- GLOBAL_STEP: 86200 + | > loss_disc: 2.337104558944702 (2.359978910902856) + | > loss_disc_real_0: 0.11086778342723846 (0.12910359234772092) + | > loss_disc_real_1: 0.2268572747707367 (0.20687832777768794) + | > loss_disc_real_2: 0.2316567450761795 (0.21498609712006342) + | > loss_disc_real_3: 0.1690669059753418 (0.20636928249412856) + | > loss_disc_real_4: 0.16283537447452545 (0.21031008681780855) + | > loss_disc_real_5: 0.23287995159626007 (0.2253824084157675) + | > loss_0: 2.337104558944702 (2.359978910902856) + | > grad_norm_0: tensor(50.6006, device='cuda:0') (tensor(39.7003, device='cuda:0')) + | > loss_gen: 2.5851409435272217 (2.557172338727494) + | > loss_kl: 1.859470248222351 (1.8642116751469358) + | > loss_feat: 8.867143630981445 (7.981067531545397) + | > loss_mel: 16.896615982055664 (16.929119835437177) + | > loss_duration: 0.9189331531524658 (0.9638635528759217) + | > loss_1: 31.127304077148438 (30.295434844325964) + | > grad_norm_1: tensor(803.1762, device='cuda:0') (tensor(601.6689, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8703 (0.9742462383189672) + | > loader_time: 0.0048 (0.006223634934761155) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03593140382033128 (+0.005102909528292145) + | > avg_loss_disc: 2.5692049173208384 (+0.005401207850529666) + | > avg_loss_disc_real_0: 0.307557979455361 (+0.12321914159334624) + | > avg_loss_disc_real_1: 0.2502335860179021 (+0.038146600127220154) + | > avg_loss_disc_real_2: 0.24756477085443643 (-0.013156910355274526) + | > avg_loss_disc_real_3: 0.23821883705946115 (+0.016540707303927482) + | > avg_loss_disc_real_4: 0.23746396601200104 (-0.003993805784445553) + | > avg_loss_disc_real_5: 0.24827000384147352 (+0.019800495642882138) + | > avg_loss_0: 2.5692049173208384 (+0.005401207850529666) + | > avg_loss_gen: 2.6530673870673547 (+0.3292902249556322) + | > avg_loss_kl: 3.2671202146089993 (-0.05584498552175665) + | > avg_loss_feat: 7.544988082005427 (+0.12674027222853468) + | > avg_loss_mel: 16.94008621802697 (-0.1462279099684487) + | > avg_loss_duration: 1.2764102495633638 (-0.018995688511775244) + | > avg_loss_1: 31.681672756488506 (+0.23496231665977874) + + + > EPOCH: 8/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:21:47)  + + --> TIME: 2024-07-13 18:22:31 -- STEP: 41/151 -- GLOBAL_STEP: 86250 + | > loss_disc: 2.2777702808380127 (2.3872487370560815) + | > loss_disc_real_0: 0.1333281397819519 (0.14134701686661416) + | > loss_disc_real_1: 0.18606044352054596 (0.20462402628689277) + | > loss_disc_real_2: 0.2196936011314392 (0.21950321008519427) + | > loss_disc_real_3: 0.1419413685798645 (0.20562646083715486) + | > loss_disc_real_4: 0.2562776803970337 (0.20843899758850656) + | > loss_disc_real_5: 0.17910179495811462 (0.22964804601378558) + | > loss_0: 2.2777702808380127 (2.3872487370560815) + | > grad_norm_0: tensor(27.3428, device='cuda:0') (tensor(25.8606, device='cuda:0')) + | > loss_gen: 2.600320816040039 (2.4932977106513046) + | > loss_kl: 1.797868251800537 (1.8564997242718209) + | > loss_feat: 9.106929779052734 (7.827329170413133) + | > loss_mel: 17.487409591674805 (17.173372896706187) + | > loss_duration: 0.9171780347824097 (0.9517355604869563) + | > loss_1: 31.909706115722656 (30.302235068344487) + | > grad_norm_1: tensor(743.3965, device='cuda:0') (tensor(345.3649, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8708 (0.9952927798759647) + | > loader_time: 0.005 (0.006395229479161705) + + + --> TIME: 2024-07-13 18:23:20 -- STEP: 91/151 -- GLOBAL_STEP: 86300 + | > loss_disc: 2.520076274871826 (2.365261478738471) + | > loss_disc_real_0: 0.22237031161785126 (0.13248999961293656) + | > loss_disc_real_1: 0.19903957843780518 (0.20540010192237057) + | > loss_disc_real_2: 0.19159863889217377 (0.21416721239194766) + | > loss_disc_real_3: 0.20847192406654358 (0.20709922844237025) + | > loss_disc_real_4: 0.24466386437416077 (0.2115620045544027) + | > loss_disc_real_5: 0.2432231903076172 (0.22756316357261533) + | > loss_0: 2.520076274871826 (2.365261478738471) + | > grad_norm_0: tensor(70.9793, device='cuda:0') (tensor(30.3275, device='cuda:0')) + | > loss_gen: 2.603410482406616 (2.527635063443865) + | > loss_kl: 2.325080156326294 (1.8238129655083457) + | > loss_feat: 7.693119525909424 (7.961972970228929) + | > loss_mel: 17.655330657958984 (17.055952417981498) + | > loss_duration: 0.9769389629364014 (0.968244459602859) + | > loss_1: 31.25387954711914 (30.337617937025133) + | > grad_norm_1: tensor(684.9708, device='cuda:0') (tensor(475.5310, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9917 (0.9781508707738185) + | > loader_time: 0.0068 (0.006293233934339586) + + + --> TIME: 2024-07-13 18:24:09 -- STEP: 141/151 -- GLOBAL_STEP: 86350 + | > loss_disc: 2.429482936859131 (2.36419700730777) + | > loss_disc_real_0: 0.17567996680736542 (0.13392502058905062) + | > loss_disc_real_1: 0.21386711299419403 (0.20392026820926806) + | > loss_disc_real_2: 0.2160993069410324 (0.21377956698126824) + | > loss_disc_real_3: 0.19514475762844086 (0.2089756521138739) + | > loss_disc_real_4: 0.1918109953403473 (0.21021696449594296) + | > loss_disc_real_5: 0.22010111808776855 (0.22653454827501418) + | > loss_0: 2.429482936859131 (2.36419700730777) + | > grad_norm_0: tensor(19.6726, device='cuda:0') (tensor(34.5923, device='cuda:0')) + | > loss_gen: 2.370131492614746 (2.539140689457562) + | > loss_kl: 2.108902931213379 (1.829843152499368) + | > loss_feat: 6.99096155166626 (7.992498543245572) + | > loss_mel: 15.629376411437988 (17.047297193648966) + | > loss_duration: 0.985909104347229 (0.9692274057273323) + | > loss_1: 28.085281372070312 (30.378006968937868) + | > grad_norm_1: tensor(352.8918, device='cuda:0') (tensor(510.7053, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8811 (0.9773041204357824) + | > loader_time: 0.0049 (0.006203308173105227) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03296054326570951 (-0.002970860554621771) + | > avg_loss_disc: 2.5520541484539327 (-0.017150768866905697) + | > avg_loss_disc_real_0: 0.21714972876585448 (-0.09040825068950653) + | > avg_loss_disc_real_1: 0.22503516307243934 (-0.025198422945462734) + | > avg_loss_disc_real_2: 0.2581583032241234 (+0.010593532369686992) + | > avg_loss_disc_real_3: 0.23242965225990003 (-0.0057891847995611145) + | > avg_loss_disc_real_4: 0.2659251151176599 (+0.02846114910565889) + | > avg_loss_disc_real_5: 0.2372533117349331 (-0.01101669210654041) + | > avg_loss_0: 2.5520541484539327 (-0.017150768866905697) + | > avg_loss_gen: 2.481520726130559 (-0.17154666093679571) + | > avg_loss_kl: 3.542342791190514 (+0.27522257658151483) + | > avg_loss_feat: 7.815626107729399 (+0.27063802572397133) + | > avg_loss_mel: 17.174040134136494 (+0.23395391610952387) + | > avg_loss_duration: 1.2651653564893282 (-0.011244893074035645) + | > avg_loss_1: 32.27869532658504 (+0.5970225700965308) + + + > EPOCH: 9/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:24:28)  + + --> TIME: 2024-07-13 18:25:12 -- STEP: 40/151 -- GLOBAL_STEP: 86400 + | > loss_disc: 2.51235032081604 (2.383769637346268) + | > loss_disc_real_0: 0.2104438990354538 (0.1401992991566658) + | > loss_disc_real_1: 0.23494449257850647 (0.21115783601999283) + | > loss_disc_real_2: 0.2497776597738266 (0.21673843823373318) + | > loss_disc_real_3: 0.20603513717651367 (0.21379347741603852) + | > loss_disc_real_4: 0.2753753364086151 (0.2185913734138012) + | > loss_disc_real_5: 0.22186151146888733 (0.2230693321675062) + | > loss_0: 2.51235032081604 (2.383769637346268) + | > grad_norm_0: tensor(50.6522, device='cuda:0') (tensor(38.8230, device='cuda:0')) + | > loss_gen: 2.3445072174072266 (2.5276480555534357) + | > loss_kl: 1.7564479112625122 (1.813358873128891) + | > loss_feat: 6.683164596557617 (7.589479792118072) + | > loss_mel: 17.316452026367188 (16.701552176475523) + | > loss_duration: 1.0173923969268799 (0.9811985164880752) + | > loss_1: 29.117965698242188 (29.613237714767457) + | > grad_norm_1: tensor(285.7378, device='cuda:0') (tensor(563.4554, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9955 (1.0361795783042909) + | > loader_time: 0.0079 (0.006942474842071533) + + + --> TIME: 2024-07-13 18:25:59 -- STEP: 90/151 -- GLOBAL_STEP: 86450 + | > loss_disc: 2.1144890785217285 (2.3460335546069686) + | > loss_disc_real_0: 0.11299285292625427 (0.12871113709277582) + | > loss_disc_real_1: 0.17852699756622314 (0.20519533571269777) + | > loss_disc_real_2: 0.19403371214866638 (0.21309092177285088) + | > loss_disc_real_3: 0.145969957113266 (0.2070401981472969) + | > loss_disc_real_4: 0.17523029446601868 (0.215989038017061) + | > loss_disc_real_5: 0.20108816027641296 (0.22021309385697047) + | > loss_0: 2.1144890785217285 (2.3460335546069686) + | > grad_norm_0: tensor(33.2711, device='cuda:0') (tensor(36.4230, device='cuda:0')) + | > loss_gen: 2.903608798980713 (2.567719835705228) + | > loss_kl: 1.6658881902694702 (1.8281375004185572) + | > loss_feat: 9.82767105102539 (8.049494981765749) + | > loss_mel: 16.8330135345459 (16.99078480402629) + | > loss_duration: 0.9888594746589661 (0.9698948694599999) + | > loss_1: 32.21903991699219 (30.406032032436794) + | > grad_norm_1: tensor(651.4404, device='cuda:0') (tensor(600.2665, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8164 (0.9779671774970159) + | > loader_time: 0.0055 (0.0064205116695827905) + + + --> TIME: 2024-07-13 18:26:47 -- STEP: 140/151 -- GLOBAL_STEP: 86500 + | > loss_disc: 2.43283748626709 (2.349494763783048) + | > loss_disc_real_0: 0.12202173471450806 (0.12986487536025892) + | > loss_disc_real_1: 0.20088684558868408 (0.20665328183344434) + | > loss_disc_real_2: 0.2265734225511551 (0.2132148914039135) + | > loss_disc_real_3: 0.23904889822006226 (0.20803714314741747) + | > loss_disc_real_4: 0.22172920405864716 (0.21417534383279938) + | > loss_disc_real_5: 0.19803299009799957 (0.2204709650150367) + | > loss_0: 2.43283748626709 (2.349494763783048) + | > grad_norm_0: tensor(22.0023, device='cuda:0') (tensor(38.8904, device='cuda:0')) + | > loss_gen: 2.449376344680786 (2.5763381293841774) + | > loss_kl: 1.9833834171295166 (1.8375654233353478) + | > loss_feat: 6.606570720672607 (8.075444092069356) + | > loss_mel: 17.115144729614258 (17.07914878981455) + | > loss_duration: 1.086975336074829 (0.9681115410157612) + | > loss_1: 29.241451263427734 (30.53660798754011) + | > grad_norm_1: tensor(352.4106, device='cuda:0') (tensor(612.4237, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9773 (0.9662244200706482) + | > loader_time: 0.0058 (0.00627162115914481) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03275790581336388 (-0.00020263745234563174) + | > avg_loss_disc: 2.5562085004953237 (+0.0041543520413909185) + | > avg_loss_disc_real_0: 0.2464370269041795 (+0.029287298138325035) + | > avg_loss_disc_real_1: 0.24246963973228747 (+0.017434476659848125) + | > avg_loss_disc_real_2: 0.22051275578828958 (-0.03764554743583384) + | > avg_loss_disc_real_3: 0.21235305873247293 (-0.020076593527427106) + | > avg_loss_disc_real_4: 0.2376961513207509 (-0.028228963796909035) + | > avg_loss_disc_real_5: 0.2702892812398764 (+0.03303596950494328) + | > avg_loss_0: 2.5562085004953237 (+0.0041543520413909185) + | > avg_loss_gen: 2.5324008831611047 (+0.050880157030545714) + | > avg_loss_kl: 3.24794785793011 (-0.2943949332604041) + | > avg_loss_feat: 7.516834222353422 (-0.29879188537597656) + | > avg_loss_mel: 17.337688592764046 (+0.16364845862755217) + | > avg_loss_duration: 1.2686809301376343 (+0.003515573648306125) + | > avg_loss_1: 31.903551835280197 (-0.3751434913048399) + + + > EPOCH: 10/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:27:07)  + + --> TIME: 2024-07-13 18:27:47 -- STEP: 39/151 -- GLOBAL_STEP: 86550 + | > loss_disc: 2.5206823348999023 (2.3559469809898963) + | > loss_disc_real_0: 0.16713367402553558 (0.12847153001870865) + | > loss_disc_real_1: 0.27461645007133484 (0.20397312709918386) + | > loss_disc_real_2: 0.2524111270904541 (0.2143907061754129) + | > loss_disc_real_3: 0.19133807718753815 (0.2082799764779898) + | > loss_disc_real_4: 0.20213963091373444 (0.21184710546945915) + | > loss_disc_real_5: 0.22323186695575714 (0.22236069654807067) + | > loss_0: 2.5206823348999023 (2.3559469809898963) + | > grad_norm_0: tensor(36.7626, device='cuda:0') (tensor(36.1420, device='cuda:0')) + | > loss_gen: 2.4470224380493164 (2.5479224828573375) + | > loss_kl: 1.9781728982925415 (1.828045667746128) + | > loss_feat: 7.061949253082275 (7.854278491093562) + | > loss_mel: 17.268558502197266 (16.882046186006985) + | > loss_duration: 0.966853678226471 (0.9704337441004239) + | > loss_1: 29.722557067871094 (30.082726747561725) + | > grad_norm_1: tensor(640.1888, device='cuda:0') (tensor(590.5275, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0108 (0.9394131440382737) + | > loader_time: 0.0065 (0.0058646079821464345) + + + --> TIME: 2024-07-13 18:28:35 -- STEP: 89/151 -- GLOBAL_STEP: 86600 + | > loss_disc: 2.253260374069214 (2.350366461143065) + | > loss_disc_real_0: 0.08608099818229675 (0.12717170938012307) + | > loss_disc_real_1: 0.18238122761249542 (0.20473865042911488) + | > loss_disc_real_2: 0.20012813806533813 (0.21603153093477315) + | > loss_disc_real_3: 0.18876014649868011 (0.2080221015415834) + | > loss_disc_real_4: 0.18756838142871857 (0.21184955286176016) + | > loss_disc_real_5: 0.2049415558576584 (0.22438883212175262) + | > loss_0: 2.253260374069214 (2.350366461143065) + | > grad_norm_0: tensor(6.1984, device='cuda:0') (tensor(31.8880, device='cuda:0')) + | > loss_gen: 2.449099540710449 (2.5572245951448926) + | > loss_kl: 1.1423066854476929 (1.8272430093100902) + | > loss_feat: 8.684898376464844 (8.001488219486193) + | > loss_mel: 16.053895950317383 (17.130248980575733) + | > loss_duration: 0.9211667776107788 (0.9602692415205281) + | > loss_1: 29.251367568969727 (30.476474161898153) + | > grad_norm_1: tensor(191.5631, device='cuda:0') (tensor(524.7821, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8538 (0.9516961467400026) + | > loader_time: 0.0056 (0.006070809417896056) + + + --> TIME: 2024-07-13 18:29:25 -- STEP: 139/151 -- GLOBAL_STEP: 86650 + | > loss_disc: 2.24564528465271 (2.3445088228733426) + | > loss_disc_real_0: 0.07085144519805908 (0.12760112193419768) + | > loss_disc_real_1: 0.22482125461101532 (0.20377847793505346) + | > loss_disc_real_2: 0.20469865202903748 (0.21425189206497275) + | > loss_disc_real_3: 0.130844846367836 (0.20870205138227063) + | > loss_disc_real_4: 0.17788425087928772 (0.21051546266610674) + | > loss_disc_real_5: 0.22001975774765015 (0.22338056703694434) + | > loss_0: 2.24564528465271 (2.3445088228733426) + | > grad_norm_0: tensor(40.9086, device='cuda:0') (tensor(35.7776, device='cuda:0')) + | > loss_gen: 2.6472408771514893 (2.568612028368942) + | > loss_kl: 1.423481822013855 (1.8171704317168367) + | > loss_feat: 9.854520797729492 (8.058327469036735) + | > loss_mel: 17.036691665649414 (17.089123046655462) + | > loss_duration: 0.9067888855934143 (0.9622517078042888) + | > loss_1: 31.868724822998047 (30.495484743186896) + | > grad_norm_1: tensor(604.1261, device='cuda:0') (tensor(565.0790, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8145 (0.962599831519367) + | > loader_time: 0.0037 (0.006128143063552088) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03307208648094764 (+0.0003141806675837633) + | > avg_loss_disc: 2.5393379651583157 (-0.016870535337007997) + | > avg_loss_disc_real_0: 0.18032674204844695 (-0.06611028485573256) + | > avg_loss_disc_real_1: 0.25433908288295454 (+0.01186944315066707) + | > avg_loss_disc_real_2: 0.27060098946094513 (+0.05008823367265555) + | > avg_loss_disc_real_3: 0.28817529861743635 (+0.07582223988496342) + | > avg_loss_disc_real_4: 0.23530494822905615 (-0.0023912030916947435) + | > avg_loss_disc_real_5: 0.2300574515874569 (-0.040231829652419476) + | > avg_loss_0: 2.5393379651583157 (-0.016870535337007997) + | > avg_loss_gen: 2.5587460077725925 (+0.026345124611487858) + | > avg_loss_kl: 3.373235959273118 (+0.12528810134300805) + | > avg_loss_feat: 7.736286493448111 (+0.21945227109468846) + | > avg_loss_mel: 17.329860320458046 (-0.007828272305999917) + | > avg_loss_duration: 1.2800192466148963 (+0.011338316477262023) + | > avg_loss_1: 32.278147770808296 (+0.3745959355280988) + + + > EPOCH: 11/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:29:45)  + + --> TIME: 2024-07-13 18:30:25 -- STEP: 38/151 -- GLOBAL_STEP: 86700 + | > loss_disc: 2.3607177734375 (2.3668478288148567) + | > loss_disc_real_0: 0.15495318174362183 (0.1332782650072323) + | > loss_disc_real_1: 0.212725430727005 (0.20575615920518575) + | > loss_disc_real_2: 0.23456883430480957 (0.21750994222728828) + | > loss_disc_real_3: 0.20496279001235962 (0.20427221883284416) + | > loss_disc_real_4: 0.1976543813943863 (0.2115099759478318) + | > loss_disc_real_5: 0.22050859034061432 (0.23043506906220787) + | > loss_0: 2.3607177734375 (2.3668478288148567) + | > grad_norm_0: tensor(26.2129, device='cuda:0') (tensor(29.9789, device='cuda:0')) + | > loss_gen: 2.754102945327759 (2.546275301983482) + | > loss_kl: 1.65449059009552 (1.8572865536338405) + | > loss_feat: 8.336455345153809 (7.935320967122128) + | > loss_mel: 16.802249908447266 (16.832110731225264) + | > loss_duration: 0.9281701445579529 (0.9647336649267297) + | > loss_1: 30.475467681884766 (30.13572763141833) + | > grad_norm_1: tensor(139.1722, device='cuda:0') (tensor(485.2426, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8627 (0.9745102681611714) + | > loader_time: 0.0059 (0.006027296969765111) + + + --> TIME: 2024-07-13 18:31:12 -- STEP: 88/151 -- GLOBAL_STEP: 86750 + | > loss_disc: 2.4949631690979004 (2.350471651012246) + | > loss_disc_real_0: 0.18944434821605682 (0.12957671259275888) + | > loss_disc_real_1: 0.29367366433143616 (0.20570239729501993) + | > loss_disc_real_2: 0.2479991614818573 (0.21548106690699403) + | > loss_disc_real_3: 0.24370470643043518 (0.20702698149464346) + | > loss_disc_real_4: 0.22871506214141846 (0.2099992290816524) + | > loss_disc_real_5: 0.25333020091056824 (0.22490875744684177) + | > loss_0: 2.4949631690979004 (2.350471651012246) + | > grad_norm_0: tensor(18.8193, device='cuda:0') (tensor(30.8556, device='cuda:0')) + | > loss_gen: 2.5664713382720947 (2.557164682583375) + | > loss_kl: 2.114701747894287 (1.8559115813537077) + | > loss_feat: 6.932252407073975 (8.082137812267648) + | > loss_mel: 17.084014892578125 (16.92221589521928) + | > loss_duration: 1.0367012023925781 (0.9599265625531023) + | > loss_1: 29.734142303466797 (30.37735670263117) + | > grad_norm_1: tensor(359.9330, device='cuda:0') (tensor(473.9880, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.0804 (0.9520179412581704) + | > loader_time: 0.0069 (0.0059816918589852075) + + + --> TIME: 2024-07-13 18:32:03 -- STEP: 138/151 -- GLOBAL_STEP: 86800 + | > loss_disc: 2.452861785888672 (2.351970111114391) + | > loss_disc_real_0: 0.10087980329990387 (0.1321589311287887) + | > loss_disc_real_1: 0.22103580832481384 (0.20391910075061567) + | > loss_disc_real_2: 0.21254508197307587 (0.214238366679005) + | > loss_disc_real_3: 0.23972667753696442 (0.20714133826718814) + | > loss_disc_real_4: 0.2645616829395294 (0.2094740132814732) + | > loss_disc_real_5: 0.18147283792495728 (0.22593234760173853) + | > loss_0: 2.452861785888672 (2.351970111114391) + | > grad_norm_0: tensor(22.9515, device='cuda:0') (tensor(28.2617, device='cuda:0')) + | > loss_gen: 2.5966975688934326 (2.545613387356633) + | > loss_kl: 2.1035468578338623 (1.8562925354294155) + | > loss_feat: 7.139932632446289 (8.09272314500117) + | > loss_mel: 16.68901252746582 (17.022763335186507) + | > loss_duration: 1.0347026586532593 (0.9579231432382611) + | > loss_1: 29.56389045715332 (30.47531567449155) + | > grad_norm_1: tensor(418.7669, device='cuda:0') (tensor(414.2794, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.0695 (0.9696753716123276) + | > loader_time: 0.0071 (0.006093453669893569) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0323340709392841 (-0.000738015541663542) + | > avg_loss_disc: 2.4918392438154955 (-0.04749872134282018) + | > avg_loss_disc_real_0: 0.1931782617018773 (+0.012851519653430343) + | > avg_loss_disc_real_1: 0.16084000697502723 (-0.0934990759079273) + | > avg_loss_disc_real_2: 0.2065785641853626 (-0.06402242527558252) + | > avg_loss_disc_real_3: 0.24234562309888694 (-0.04582967551854941) + | > avg_loss_disc_real_4: 0.22286560558355772 (-0.012439342645498425) + | > avg_loss_disc_real_5: 0.20695751103071067 (-0.023099940556746246) + | > avg_loss_0: 2.4918392438154955 (-0.04749872134282018) + | > avg_loss_gen: 2.2412414550781254 (-0.3175045526944671) + | > avg_loss_kl: 3.1666993361253004 (-0.20653662314781762) + | > avg_loss_feat: 7.864365577697754 (+0.12807908424964332) + | > avg_loss_mel: 17.130166934086727 (-0.19969338637131884) + | > avg_loss_duration: 1.246323878948505 (-0.03369536766639136) + | > avg_loss_1: 31.64879666841947 (-0.6293511023888243) + + + > EPOCH: 12/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:32:24)  + + --> TIME: 2024-07-13 18:33:02 -- STEP: 37/151 -- GLOBAL_STEP: 86850 + | > loss_disc: 2.4936745166778564 (2.382033638052038) + | > loss_disc_real_0: 0.23623399436473846 (0.13540307039747368) + | > loss_disc_real_1: 0.2459201216697693 (0.20650135383412643) + | > loss_disc_real_2: 0.21171395480632782 (0.2223458314264143) + | > loss_disc_real_3: 0.22598019242286682 (0.20682555880095507) + | > loss_disc_real_4: 0.22378167510032654 (0.21193584074845184) + | > loss_disc_real_5: 0.25877055525779724 (0.22292469401617307) + | > loss_0: 2.4936745166778564 (2.382033638052038) + | > grad_norm_0: tensor(71.9078, device='cuda:0') (tensor(40.3640, device='cuda:0')) + | > loss_gen: 2.4766011238098145 (2.5517350789662956) + | > loss_kl: 1.7651546001434326 (1.81802947134585) + | > loss_feat: 6.907362937927246 (7.8375140138574535) + | > loss_mel: 17.335546493530273 (16.86192520244702) + | > loss_duration: 0.9103890657424927 (0.9794521090146657) + | > loss_1: 29.39505386352539 (30.048656102773304) + | > grad_norm_1: tensor(610.4353, device='cuda:0') (tensor(525.6979, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9668 (0.967665433883667) + | > loader_time: 0.0069 (0.006026029586791993) + + + --> TIME: 2024-07-13 18:33:51 -- STEP: 87/151 -- GLOBAL_STEP: 86900 + | > loss_disc: 2.295687675476074 (2.37083467943915) + | > loss_disc_real_0: 0.143681138753891 (0.13122650312966316) + | > loss_disc_real_1: 0.19179178774356842 (0.2070816572027645) + | > loss_disc_real_2: 0.18081097304821014 (0.21589631698597436) + | > loss_disc_real_3: 0.15512022376060486 (0.2062437599417807) + | > loss_disc_real_4: 0.16152581572532654 (0.21042254772679558) + | > loss_disc_real_5: 0.27284494042396545 (0.2268487048902731) + | > loss_0: 2.295687675476074 (2.37083467943915) + | > grad_norm_0: tensor(20.3609, device='cuda:0') (tensor(31.9390, device='cuda:0')) + | > loss_gen: 2.696866989135742 (2.536743613495224) + | > loss_kl: 2.0318634510040283 (1.8356270173500324) + | > loss_feat: 8.317280769348145 (7.933540914250516) + | > loss_mel: 18.01350975036621 (16.953926667399802) + | > loss_duration: 0.8472355604171753 (0.9792764063539177) + | > loss_1: 31.906755447387695 (30.23911473942899) + | > grad_norm_1: tensor(649.2879, device='cuda:0') (tensor(477.5268, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8711 (0.9609229920924395) + | > loader_time: 0.0049 (0.005983382806010628) + + + --> TIME: 2024-07-13 18:34:41 -- STEP: 137/151 -- GLOBAL_STEP: 86950 + | > loss_disc: 2.247399091720581 (2.3691358775117974) + | > loss_disc_real_0: 0.08508317917585373 (0.13240825520814764) + | > loss_disc_real_1: 0.16277873516082764 (0.20662585107514458) + | > loss_disc_real_2: 0.21682769060134888 (0.21561950107995612) + | > loss_disc_real_3: 0.1775740683078766 (0.2068878352424524) + | > loss_disc_real_4: 0.21425902843475342 (0.21151258127532735) + | > loss_disc_real_5: 0.21532295644283295 (0.22697190896873057) + | > loss_0: 2.247399091720581 (2.3691358775117974) + | > grad_norm_0: tensor(14.8959, device='cuda:0') (tensor(35.5905, device='cuda:0')) + | > loss_gen: 2.5528481006622314 (2.534924387061683) + | > loss_kl: 1.8192545175552368 (1.836897831328594) + | > loss_feat: 8.676602363586426 (7.8660638976271136) + | > loss_mel: 18.39240264892578 (16.989987707486137) + | > loss_duration: 0.9320616126060486 (0.9728514543414986) + | > loss_1: 32.3731689453125 (30.200725318741625) + | > grad_norm_1: tensor(361.7448, device='cuda:0') (tensor(489.3990, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8815 (0.9730445259678973) + | > loader_time: 0.0051 (0.006177091250454421) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030926227569580078 (-0.0014078433697040232) + | > avg_loss_disc: 2.5654899523808408 (+0.07365070856534528) + | > avg_loss_disc_real_0: 0.18450653782257667 (-0.008671723879300625) + | > avg_loss_disc_real_1: 0.20444983473190895 (+0.043609827756881714) + | > avg_loss_disc_real_2: 0.24561226597199073 (+0.03903370178662813) + | > avg_loss_disc_real_3: 0.22097956790373877 (-0.02136605519514817) + | > avg_loss_disc_real_4: 0.2551383272959636 (+0.03227272171240586) + | > avg_loss_disc_real_5: 0.23091286420822144 (+0.02395535317751077) + | > avg_loss_0: 2.5654899523808408 (+0.07365070856534528) + | > avg_loss_gen: 2.2974944481482873 (+0.05625299307016185) + | > avg_loss_kl: 3.273367845095121 (+0.1066685089698205) + | > avg_loss_feat: 7.447102033174955 (-0.41726354452279857) + | > avg_loss_mel: 17.261890778174767 (+0.1317238440880395) + | > avg_loss_duration: 1.3301990582392766 (+0.0838751792907717) + | > avg_loss_1: 31.61005372267503 (-0.038742945744441926) + + + > EPOCH: 13/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:35:03)  + + --> TIME: 2024-07-13 18:35:42 -- STEP: 36/151 -- GLOBAL_STEP: 87000 + | > loss_disc: 2.1670801639556885 (2.3892317414283752) + | > loss_disc_real_0: 0.08738599717617035 (0.13629993547995883) + | > loss_disc_real_1: 0.2257390320301056 (0.20801299437880516) + | > loss_disc_real_2: 0.1886676698923111 (0.2141879606578085) + | > loss_disc_real_3: 0.16440510749816895 (0.21006151516404417) + | > loss_disc_real_4: 0.13785693049430847 (0.21222755271527502) + | > loss_disc_real_5: 0.21460208296775818 (0.23045744912491906) + | > loss_0: 2.1670801639556885 (2.3892317414283752) + | > grad_norm_0: tensor(10.5724, device='cuda:0') (tensor(19.3914, device='cuda:0')) + | > loss_gen: 2.7795114517211914 (2.4581899576716952) + | > loss_kl: 2.262556314468384 (1.8791122684876125) + | > loss_feat: 7.8539276123046875 (7.678833736313714) + | > loss_mel: 17.694801330566406 (16.98622208171421) + | > loss_duration: 0.9572468996047974 (0.9721800535917282) + | > loss_1: 31.548044204711914 (29.974538220299614) + | > grad_norm_1: tensor(481.0546, device='cuda:0') (tensor(281.9739, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8975 (1.0096790591875713) + | > loader_time: 0.0056 (0.006418698363833957) + + + --> TIME: 2024-07-13 18:36:47 -- STEP: 86/151 -- GLOBAL_STEP: 87050 + | > loss_disc: 2.428290843963623 (2.3752169026884924) + | > loss_disc_real_0: 0.10775714367628098 (0.13238205115289195) + | > loss_disc_real_1: 0.2075979858636856 (0.2067617780940477) + | > loss_disc_real_2: 0.21119490265846252 (0.2155326845687489) + | > loss_disc_real_3: 0.243352010846138 (0.21378431737769482) + | > loss_disc_real_4: 0.23144011199474335 (0.21354274573021156) + | > loss_disc_real_5: 0.2192493975162506 (0.2266887814499611) + | > loss_0: 2.428290843963623 (2.3752169026884924) + | > grad_norm_0: tensor(14.6952, device='cuda:0') (tensor(26.0490, device='cuda:0')) + | > loss_gen: 2.3931424617767334 (2.5053734834804087) + | > loss_kl: 1.4212725162506104 (1.7957531102868014) + | > loss_feat: 8.537420272827148 (7.930685209673505) + | > loss_mel: 17.669830322265625 (16.978433365045593) + | > loss_duration: 0.909902036190033 (0.9722474848115167) + | > loss_1: 30.931568145751953 (30.18249285498331) + | > grad_norm_1: tensor(223.9241, device='cuda:0') (tensor(426.2457, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.894 (0.9988733058752015) + | > loader_time: 0.005 (0.006343994029732639) + + + --> TIME: 2024-07-13 18:37:36 -- STEP: 136/151 -- GLOBAL_STEP: 87100 + | > loss_disc: 2.3087046146392822 (2.3721394030486826) + | > loss_disc_real_0: 0.20935013890266418 (0.13242052774876362) + | > loss_disc_real_1: 0.2712250351905823 (0.20554424559368803) + | > loss_disc_real_2: 0.20884564518928528 (0.21463905636440306) + | > loss_disc_real_3: 0.21449148654937744 (0.21246611310497804) + | > loss_disc_real_4: 0.2348962426185608 (0.21258128489203312) + | > loss_disc_real_5: 0.2504715025424957 (0.22648648326011264) + | > loss_0: 2.3087046146392822 (2.3721394030486826) + | > grad_norm_0: tensor(47.9583, device='cuda:0') (tensor(27.5230, device='cuda:0')) + | > loss_gen: 2.9794564247131348 (2.50879594858955) + | > loss_kl: 1.4395204782485962 (1.7933891959050123) + | > loss_feat: 8.914423942565918 (7.902471729937722) + | > loss_mel: 17.06714630126953 (16.923364807577705) + | > loss_duration: 0.9580774903297424 (0.977719123749172) + | > loss_1: 31.358625411987305 (30.105740911820355) + | > grad_norm_1: tensor(625.7327, device='cuda:0') (tensor(436.0478, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8039 (0.9908062517642975) + | > loader_time: 0.0043 (0.006326093393213609) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030519320414616517 (-0.00040690715496356084) + | > avg_loss_disc: 2.6134325174184947 (+0.04794256503765393) + | > avg_loss_disc_real_0: 0.2486908997480686 (+0.06418436192549193) + | > avg_loss_disc_real_1: 0.21847524321996248 (+0.014025408488053531) + | > avg_loss_disc_real_2: 0.20373744460252616 (-0.04187482136946458) + | > avg_loss_disc_real_3: 0.20922831961741814 (-0.011751248286320626) + | > avg_loss_disc_real_4: 0.21194135684233445 (-0.043196970453629135) + | > avg_loss_disc_real_5: 0.24591022156752074 (+0.014997357359299301) + | > avg_loss_0: 2.6134325174184947 (+0.04794256503765393) + | > avg_loss_gen: 2.330732932457557 (+0.0332384843092699) + | > avg_loss_kl: 3.393847795633169 (+0.12047995053804827) + | > avg_loss_feat: 8.066120771261362 (+0.6190187380864067) + | > avg_loss_mel: 17.517897532536434 (+0.25600675436166753) + | > avg_loss_duration: 1.2709466402347271 (-0.0592524180045495) + | > avg_loss_1: 32.57954582801232 (+0.9694921053372916) + + + > EPOCH: 14/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:37:59)  + + --> TIME: 2024-07-13 18:38:35 -- STEP: 35/151 -- GLOBAL_STEP: 87150 + | > loss_disc: 2.686692237854004 (2.3432462624141146) + | > loss_disc_real_0: 0.319871187210083 (0.13102687980447497) + | > loss_disc_real_1: 0.18440406024456024 (0.20163693768637522) + | > loss_disc_real_2: 0.19222605228424072 (0.21242607023034776) + | > loss_disc_real_3: 0.20986849069595337 (0.20546721986361913) + | > loss_disc_real_4: 0.2148968130350113 (0.2088568776845932) + | > loss_disc_real_5: 0.26236122846603394 (0.22345869966915677) + | > loss_0: 2.686692237854004 (2.3432462624141146) + | > grad_norm_0: tensor(46.4614, device='cuda:0') (tensor(42.7871, device='cuda:0')) + | > loss_gen: 2.4179041385650635 (2.598629753930228) + | > loss_kl: 1.9996556043624878 (1.8224292823246546) + | > loss_feat: 6.873589515686035 (8.044194398607527) + | > loss_mel: 16.302019119262695 (16.83401189531599) + | > loss_duration: 0.8412706255912781 (0.9552070787974767) + | > loss_1: 28.434438705444336 (30.254472024100167) + | > grad_norm_1: tensor(252.3443, device='cuda:0') (tensor(582.0831, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9831 (0.9475516319274903) + | > loader_time: 0.0062 (0.005792978831699916) + + + --> TIME: 2024-07-13 18:39:25 -- STEP: 85/151 -- GLOBAL_STEP: 87200 + | > loss_disc: 2.443016767501831 (2.360284050773171) + | > loss_disc_real_0: 0.16014932096004486 (0.13460090712589376) + | > loss_disc_real_1: 0.2144605815410614 (0.2032980089678484) + | > loss_disc_real_2: 0.18774159252643585 (0.21197835294639364) + | > loss_disc_real_3: 0.17593230307102203 (0.20745396438766928) + | > loss_disc_real_4: 0.22819751501083374 (0.209035884457476) + | > loss_disc_real_5: 0.2108217179775238 (0.22274337849196266) + | > loss_0: 2.443016767501831 (2.360284050773171) + | > grad_norm_0: tensor(18.9507, device='cuda:0') (tensor(32.4847, device='cuda:0')) + | > loss_gen: 2.728605270385742 (2.5653870778925287) + | > loss_kl: 1.714463472366333 (1.8622379499323227) + | > loss_feat: 9.218210220336914 (8.09989667219274) + | > loss_mel: 18.21613883972168 (16.948091248904955) + | > loss_duration: 0.9065929055213928 (0.9624249661670011) + | > loss_1: 32.78400802612305 (30.438037692799288) + | > grad_norm_1: tensor(363.9892, device='cuda:0') (tensor(456.3405, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8668 (0.9725604309755214) + | > loader_time: 0.0048 (0.006181955337524414) + + + --> TIME: 2024-07-13 18:40:13 -- STEP: 135/151 -- GLOBAL_STEP: 87250 + | > loss_disc: 2.375812292098999 (2.3508530864009147) + | > loss_disc_real_0: 0.12529082596302032 (0.1320072121642254) + | > loss_disc_real_1: 0.2222091555595398 (0.20352800367055116) + | > loss_disc_real_2: 0.2118341624736786 (0.21286127467950186) + | > loss_disc_real_3: 0.23308953642845154 (0.20682894101849308) + | > loss_disc_real_4: 0.216904878616333 (0.2083605224335635) + | > loss_disc_real_5: 0.24341462552547455 (0.22296101748943328) + | > loss_0: 2.375812292098999 (2.3508530864009147) + | > grad_norm_0: tensor(36.0396, device='cuda:0') (tensor(33.4448, device='cuda:0')) + | > loss_gen: 2.607365608215332 (2.5690866487997552) + | > loss_kl: 1.850104570388794 (1.8672944541330692) + | > loss_feat: 7.145687103271484 (8.165008378911898) + | > loss_mel: 16.510971069335938 (17.101548357363093) + | > loss_duration: 0.943722128868103 (0.9581061367635374) + | > loss_1: 29.057849884033203 (30.661043887668185) + | > grad_norm_1: tensor(377.6035, device='cuda:0') (tensor(492.9333, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9289 (0.9590233732152869) + | > loader_time: 0.0054 (0.00603412875422725) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03428943340594952 (+0.003770112991333001) + | > avg_loss_disc: 2.507097941178542 (-0.10633457623995257) + | > avg_loss_disc_real_0: 0.1853974679341683 (-0.06329343181390029) + | > avg_loss_disc_real_1: 0.20424352700893694 (-0.014231716211025536) + | > avg_loss_disc_real_2: 0.2212063200198687 (+0.017468875417342544) + | > avg_loss_disc_real_3: 0.20055599510669708 (-0.008672324510721058) + | > avg_loss_disc_real_4: 0.23163788937605345 (+0.019696532533719002) + | > avg_loss_disc_real_5: 0.21956469691716707 (-0.02634552465035367) + | > avg_loss_0: 2.507097941178542 (-0.10633457623995257) + | > avg_loss_gen: 2.313341727623573 (-0.017391204833984375) + | > avg_loss_kl: 3.3584165206322303 (-0.0354312750009389) + | > avg_loss_feat: 8.194731345543495 (+0.12861057428213307) + | > avg_loss_mel: 17.707755749042217 (+0.18985821650578316) + | > avg_loss_duration: 1.2635353528536284 (-0.007411287381098752) + | > avg_loss_1: 32.83778087909405 (+0.2582350510817264) + + + > EPOCH: 15/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:40:39)  + + --> TIME: 2024-07-13 18:41:12 -- STEP: 34/151 -- GLOBAL_STEP: 87300 + | > loss_disc: 2.1196138858795166 (2.3016542897504912) + | > loss_disc_real_0: 0.10615187883377075 (0.12676160760662136) + | > loss_disc_real_1: 0.20599520206451416 (0.2015159077504102) + | > loss_disc_real_2: 0.2011599838733673 (0.2082099406158223) + | > loss_disc_real_3: 0.1848181188106537 (0.20077418601688216) + | > loss_disc_real_4: 0.2108728289604187 (0.20277742362197707) + | > loss_disc_real_5: 0.17722970247268677 (0.2158308314049945) + | > loss_0: 2.1196138858795166 (2.3016542897504912) + | > grad_norm_0: tensor(32.7843, device='cuda:0') (tensor(37.4508, device='cuda:0')) + | > loss_gen: 2.841792345046997 (2.6376552722033333) + | > loss_kl: 1.776386022567749 (1.765884956892799) + | > loss_feat: 9.298992156982422 (8.254201692693371) + | > loss_mel: 17.47504997253418 (17.021185510298785) + | > loss_duration: 0.9038351774215698 (0.9507658832213458) + | > loss_1: 32.29605484008789 (30.629693311803482) + | > grad_norm_1: tensor(880.8602, device='cuda:0') (tensor(610.5574, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8719 (0.9242062147925881) + | > loader_time: 0.0045 (0.005817588637856876) + + + --> TIME: 2024-07-13 18:42:04 -- STEP: 84/151 -- GLOBAL_STEP: 87350 + | > loss_disc: 2.283205032348633 (2.3642635458991643) + | > loss_disc_real_0: 0.15078896284103394 (0.13630756478579276) + | > loss_disc_real_1: 0.17369839549064636 (0.20610047007600465) + | > loss_disc_real_2: 0.143770232796669 (0.21355190074869565) + | > loss_disc_real_3: 0.1799490600824356 (0.2051368166825601) + | > loss_disc_real_4: 0.1556263118982315 (0.2084776625214588) + | > loss_disc_real_5: 0.2099781036376953 (0.2223891599902085) + | > loss_0: 2.283205032348633 (2.3642635458991643) + | > grad_norm_0: tensor(23.7366, device='cuda:0') (tensor(39.4015, device='cuda:0')) + | > loss_gen: 2.4315130710601807 (2.5593300773983896) + | > loss_kl: 1.8394335508346558 (1.8044539093971252) + | > loss_feat: 8.114994049072266 (8.000033066386269) + | > loss_mel: 17.276546478271484 (17.230520691190456) + | > loss_duration: 0.9425845742225647 (0.9586658130089442) + | > loss_1: 30.605072021484375 (30.553003697168258) + | > grad_norm_1: tensor(393.2594, device='cuda:0') (tensor(552.7702, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8547 (0.9772478739420573) + | > loader_time: 0.0082 (0.006234038443792434) + + + --> TIME: 2024-07-13 18:42:52 -- STEP: 134/151 -- GLOBAL_STEP: 87400 + | > loss_disc: 2.1790332794189453 (2.3595736204688227) + | > loss_disc_real_0: 0.10446944087743759 (0.13361518312968435) + | > loss_disc_real_1: 0.19370099902153015 (0.20547277302439534) + | > loss_disc_real_2: 0.19941800832748413 (0.2148959120151712) + | > loss_disc_real_3: 0.18046952784061432 (0.20620564949601444) + | > loss_disc_real_4: 0.22661472856998444 (0.20878491511763034) + | > loss_disc_real_5: 0.21933525800704956 (0.2237026683652579) + | > loss_0: 2.1790332794189453 (2.3595736204688227) + | > grad_norm_0: tensor(15.6040, device='cuda:0') (tensor(36.8069, device='cuda:0')) + | > loss_gen: 2.70949649810791 (2.563867615230047) + | > loss_kl: 1.6775953769683838 (1.7764693765497919) + | > loss_feat: 8.160524368286133 (8.019893610655371) + | > loss_mel: 15.997275352478027 (17.21680115230049) + | > loss_duration: 1.0669745206832886 (0.9610439009630858) + | > loss_1: 29.611865997314453 (30.538075802931147) + | > grad_norm_1: tensor(513.5450, device='cuda:0') (tensor(538.4869, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8198 (0.9691742178219468) + | > loader_time: 0.0043 (0.006092277925405929) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032611388426560625 (-0.001678044979388893) + | > avg_loss_disc: 2.585482597351074 (+0.0783846561725321) + | > avg_loss_disc_real_0: 0.2433496220753743 (+0.057952154141206) + | > avg_loss_disc_real_1: 0.2562469794200017 (+0.05200345241106474) + | > avg_loss_disc_real_2: 0.2787721340472882 (+0.057565814027419476) + | > avg_loss_disc_real_3: 0.23689570449865782 (+0.03633970939196074) + | > avg_loss_disc_real_4: 0.24193421464699966 (+0.010296325270946205) + | > avg_loss_disc_real_5: 0.21632903241194212 (-0.0032356645052249444) + | > avg_loss_0: 2.585482597351074 (+0.0783846561725321) + | > avg_loss_gen: 2.5656353877140927 (+0.2522936600905199) + | > avg_loss_kl: 3.3457161646622877 (-0.012700355969942567) + | > avg_loss_feat: 7.713601112365723 (-0.4811302331777725) + | > avg_loss_mel: 17.473427038926346 (-0.23432871011587153) + | > avg_loss_duration: 1.2618252772551317 (-0.0017100755984966742) + | > avg_loss_1: 32.360205576970024 (-0.47757530212402344) + + + > EPOCH: 16/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:43:18)  + + --> TIME: 2024-07-13 18:43:53 -- STEP: 33/151 -- GLOBAL_STEP: 87450 + | > loss_disc: 2.350949764251709 (2.3509280031377617) + | > loss_disc_real_0: 0.07054615020751953 (0.1268463906916705) + | > loss_disc_real_1: 0.19859984517097473 (0.19956844503229315) + | > loss_disc_real_2: 0.1978328377008438 (0.21449573383186804) + | > loss_disc_real_3: 0.2116452157497406 (0.21018779639041785) + | > loss_disc_real_4: 0.28441959619522095 (0.21405699442733417) + | > loss_disc_real_5: 0.20861390233039856 (0.2224930308081887) + | > loss_0: 2.350949764251709 (2.3509280031377617) + | > grad_norm_0: tensor(26.7628, device='cuda:0') (tensor(32.4307, device='cuda:0')) + | > loss_gen: 2.5662238597869873 (2.5307872584371847) + | > loss_kl: 1.714322566986084 (1.7990503094413064) + | > loss_feat: 8.34758186340332 (7.544449025934393) + | > loss_mel: 17.589977264404297 (16.86389801718972) + | > loss_duration: 0.8910602927207947 (0.9779364195736971) + | > loss_1: 31.10916519165039 (29.716121037801106) + | > grad_norm_1: tensor(447.8227, device='cuda:0') (tensor(497.3224, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9028 (0.9836148709961863) + | > loader_time: 0.0051 (0.006095618912667939) + + + --> TIME: 2024-07-13 18:44:42 -- STEP: 83/151 -- GLOBAL_STEP: 87500 + | > loss_disc: 2.203023910522461 (2.331232352429126) + | > loss_disc_real_0: 0.052618950605392456 (0.12531647283628763) + | > loss_disc_real_1: 0.158516526222229 (0.20147478562521642) + | > loss_disc_real_2: 0.1994953602552414 (0.21259135079671101) + | > loss_disc_real_3: 0.1455267369747162 (0.21050553759896606) + | > loss_disc_real_4: 0.17275825142860413 (0.20910699737359242) + | > loss_disc_real_5: 0.15049101412296295 (0.2218471043799297) + | > loss_0: 2.203023910522461 (2.331232352429126) + | > grad_norm_0: tensor(27.7455, device='cuda:0') (tensor(30.2624, device='cuda:0')) + | > loss_gen: 2.680079221725464 (2.5634011417986393) + | > loss_kl: 1.7427904605865479 (1.8132237045161694) + | > loss_feat: 9.652162551879883 (8.01784779077553) + | > loss_mel: 17.784992218017578 (16.993445499833815) + | > loss_duration: 0.9542832374572754 (0.9690939697874598) + | > loss_1: 32.814308166503906 (30.357012231665923) + | > grad_norm_1: tensor(791.4068, device='cuda:0') (tensor(507.9262, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8442 (0.980494625597115) + | > loader_time: 0.0047 (0.006110774465354091) + + + --> TIME: 2024-07-13 18:45:30 -- STEP: 133/151 -- GLOBAL_STEP: 87550 + | > loss_disc: 2.341062307357788 (2.3505869772201193) + | > loss_disc_real_0: 0.14296041429042816 (0.132176416681001) + | > loss_disc_real_1: 0.21682208776474 (0.20391246642833366) + | > loss_disc_real_2: 0.19927264750003815 (0.21378625268326665) + | > loss_disc_real_3: 0.2203216850757599 (0.21075115246432166) + | > loss_disc_real_4: 0.24680417776107788 (0.20976572635030388) + | > loss_disc_real_5: 0.2746734619140625 (0.22157106520538045) + | > loss_0: 2.341062307357788 (2.3505869772201193) + | > grad_norm_0: tensor(28.9043, device='cuda:0') (tensor(34.7147, device='cuda:0')) + | > loss_gen: 2.6168339252471924 (2.561344146728515) + | > loss_kl: 1.8345433473587036 (1.812303578495083) + | > loss_feat: 7.029324531555176 (8.010064053356203) + | > loss_mel: 16.47320556640625 (16.978378295898445) + | > loss_duration: 0.9252758026123047 (0.9655528117839555) + | > loss_1: 28.87918472290039 (30.32764289970685) + | > grad_norm_1: tensor(426.0196, device='cuda:0') (tensor(504.5382, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9504 (0.9695342752270233) + | > loader_time: 0.0058 (0.006026520764917358) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03415595568143404 (+0.0015445672548734177) + | > avg_loss_disc: 2.5065465523646426 (-0.07893604498643159) + | > avg_loss_disc_real_0: 0.18399048883181351 (-0.05935913324356079) + | > avg_loss_disc_real_1: 0.24727333509005034 (-0.008973644329951347) + | > avg_loss_disc_real_2: 0.2877874741187462 (+0.009015340071458045) + | > avg_loss_disc_real_3: 0.23939418334227341 (+0.0024984788436155925) + | > avg_loss_disc_real_4: 0.2532461675313803 (+0.011311952884380638) + | > avg_loss_disc_real_5: 0.20083458721637726 (-0.015494445195564865) + | > avg_loss_0: 2.5065465523646426 (-0.07893604498643159) + | > avg_loss_gen: 2.5082570772904615 (-0.057378310423631174) + | > avg_loss_kl: 3.335652443078848 (-0.010063721583439822) + | > avg_loss_feat: 7.494725740872896 (-0.21887537149282643) + | > avg_loss_mel: 16.98652634253869 (-0.4869006963876572) + | > avg_loss_duration: 1.2914505555079534 (+0.029625278252821685) + | > avg_loss_1: 31.616612361027645 (-0.7435932159423793) + + + > EPOCH: 17/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:45:58)  + + --> TIME: 2024-07-13 18:46:30 -- STEP: 32/151 -- GLOBAL_STEP: 87600 + | > loss_disc: 2.317781686782837 (2.328257694840431) + | > loss_disc_real_0: 0.09615561366081238 (0.12285287003032863) + | > loss_disc_real_1: 0.20543956756591797 (0.19771057227626443) + | > loss_disc_real_2: 0.24761709570884705 (0.21305325161665678) + | > loss_disc_real_3: 0.19081568717956543 (0.20221215346828103) + | > loss_disc_real_4: 0.21721969544887543 (0.20954212499782443) + | > loss_disc_real_5: 0.20649662613868713 (0.22001219727098942) + | > loss_0: 2.317781686782837 (2.328257694840431) + | > grad_norm_0: tensor(24.0627, device='cuda:0') (tensor(26.6552, device='cuda:0')) + | > loss_gen: 2.5746500492095947 (2.5537134632468224) + | > loss_kl: 1.7162413597106934 (1.7701582182198763) + | > loss_feat: 8.493054389953613 (8.34058229625225) + | > loss_mel: 17.250137329101562 (17.183322608470913) + | > loss_duration: 0.9703364968299866 (0.9130362551659346) + | > loss_1: 31.004419326782227 (30.760813057422638) + | > grad_norm_1: tensor(502.5339, device='cuda:0') (tensor(464.9001, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8285 (0.9045973718166351) + | > loader_time: 0.0043 (0.005348287522792816) + + + --> TIME: 2024-07-13 18:47:19 -- STEP: 82/151 -- GLOBAL_STEP: 87650 + | > loss_disc: 2.60341477394104 (2.3487613520971173) + | > loss_disc_real_0: 0.11981864273548126 (0.12874168925350762) + | > loss_disc_real_1: 0.2458084672689438 (0.20234136592324187) + | > loss_disc_real_2: 0.27887704968452454 (0.21482636325243043) + | > loss_disc_real_3: 0.21550343930721283 (0.2059209135974326) + | > loss_disc_real_4: 0.2467358261346817 (0.21122286105301322) + | > loss_disc_real_5: 0.2450045943260193 (0.22469756516014658) + | > loss_0: 2.60341477394104 (2.3487613520971173) + | > grad_norm_0: tensor(51.6694, device='cuda:0') (tensor(31.4730, device='cuda:0')) + | > loss_gen: 2.330209255218506 (2.551278009647276) + | > loss_kl: 1.4841526746749878 (1.8081591732618285) + | > loss_feat: 5.706779479980469 (8.14691794790873) + | > loss_mel: 14.175459861755371 (17.082607874056198) + | > loss_duration: 0.9167559742927551 (0.9454933993700074) + | > loss_1: 24.613357543945312 (30.534456369353503) + | > grad_norm_1: tensor(545.6897, device='cuda:0') (tensor(532.6953, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.854 (0.9471633812276329) + | > loader_time: 0.0057 (0.005971385211479373) + + + --> TIME: 2024-07-13 18:48:12 -- STEP: 132/151 -- GLOBAL_STEP: 87700 + | > loss_disc: 2.3924756050109863 (2.342264784104897) + | > loss_disc_real_0: 0.10282135009765625 (0.129980694530814) + | > loss_disc_real_1: 0.260856956243515 (0.20297519840074307) + | > loss_disc_real_2: 0.24262262880802155 (0.21462848737384332) + | > loss_disc_real_3: 0.19541168212890625 (0.20787062495946884) + | > loss_disc_real_4: 0.195766419172287 (0.21026965501633557) + | > loss_disc_real_5: 0.2384810447692871 (0.22185634889385916) + | > loss_0: 2.3924756050109863 (2.342264784104897) + | > grad_norm_0: tensor(19.6046, device='cuda:0') (tensor(32.4616, device='cuda:0')) + | > loss_gen: 2.476044178009033 (2.5582304289846705) + | > loss_kl: 2.298250913619995 (1.815262421965599) + | > loss_feat: 6.580550670623779 (8.109907072601905) + | > loss_mel: 17.18398666381836 (17.085848150831282) + | > loss_duration: 0.9833676815032959 (0.9585986990820278) + | > loss_1: 29.522199630737305 (30.52784675540346) + | > grad_norm_1: tensor(461.7007, device='cuda:0') (tensor(543.3187, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.439 (0.9804764299681692) + | > loader_time: 0.012 (0.006294389565785726) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03274930440462553 (-0.0014066512768085154) + | > avg_loss_disc: 2.630852222442627 (+0.12430567007798432) + | > avg_loss_disc_real_0: 0.17385864716309768 (-0.010131841668715835) + | > avg_loss_disc_real_1: 0.22066160119496858 (-0.026611733895081757) + | > avg_loss_disc_real_2: 0.2110574314227471 (-0.07673004269599912) + | > avg_loss_disc_real_3: 0.18876423285557672 (-0.05062995048669669) + | > avg_loss_disc_real_4: 0.24233371019363403 (-0.010912457337746262) + | > avg_loss_disc_real_5: 0.20432518307979292 (+0.0034905958634156575) + | > avg_loss_0: 2.630852222442627 (+0.12430567007798432) + | > avg_loss_gen: 2.07437710578625 (-0.4338799715042114) + | > avg_loss_kl: 3.2856708673330455 (-0.04998157574580242) + | > avg_loss_feat: 7.385706974909856 (-0.10901876596304039) + | > avg_loss_mel: 16.941151765676643 (-0.045374576862045046) + | > avg_loss_duration: 1.2428814722941472 (-0.04856908321380615) + | > avg_loss_1: 30.929787855881912 (-0.6868245051457329) + + + > EPOCH: 18/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:48:40)  + + --> TIME: 2024-07-13 18:49:12 -- STEP: 31/151 -- GLOBAL_STEP: 87750 + | > loss_disc: 2.5171563625335693 (2.3575940901233303) + | > loss_disc_real_0: 0.23178207874298096 (0.13907551428964057) + | > loss_disc_real_1: 0.21366755664348602 (0.2100876526486489) + | > loss_disc_real_2: 0.2810737192630768 (0.2163008730257711) + | > loss_disc_real_3: 0.24468214809894562 (0.20773651811384386) + | > loss_disc_real_4: 0.2106270045042038 (0.2132848116659349) + | > loss_disc_real_5: 0.25716495513916016 (0.21828663493356398) + | > loss_0: 2.5171563625335693 (2.3575940901233303) + | > grad_norm_0: tensor(11.3323, device='cuda:0') (tensor(35.7466, device='cuda:0')) + | > loss_gen: 2.283851385116577 (2.576351619535876) + | > loss_kl: 2.293217658996582 (1.74724841502405) + | > loss_feat: 6.861556053161621 (8.127912613653368) + | > loss_mel: 15.788459777832031 (16.743372271137858) + | > loss_duration: 0.9074230790138245 (0.9784645207466618) + | > loss_1: 28.13450813293457 (30.173349503547914) + | > grad_norm_1: tensor(209.7369, device='cuda:0') (tensor(524.5323, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9704 (0.9707720741148917) + | > loader_time: 0.0062 (0.006230608109504946) + + + --> TIME: 2024-07-13 18:50:03 -- STEP: 81/151 -- GLOBAL_STEP: 87800 + | > loss_disc: 2.459813117980957 (2.381459300900682) + | > loss_disc_real_0: 0.13358888030052185 (0.13525144898413138) + | > loss_disc_real_1: 0.1777249127626419 (0.20407737350022356) + | > loss_disc_real_2: 0.2459268569946289 (0.21768927813312153) + | > loss_disc_real_3: 0.2613775134086609 (0.21132196964305125) + | > loss_disc_real_4: 0.22816064953804016 (0.21183273711322267) + | > loss_disc_real_5: 0.2213139533996582 (0.22570762700504726) + | > loss_0: 2.459813117980957 (2.381459300900682) + | > grad_norm_0: tensor(25.7413, device='cuda:0') (tensor(29.0725, device='cuda:0')) + | > loss_gen: 2.4403417110443115 (2.5134617266831576) + | > loss_kl: 1.8007632493972778 (1.8038084359816562) + | > loss_feat: 8.154172897338867 (7.882962291623338) + | > loss_mel: 17.592294692993164 (16.825617613615808) + | > loss_duration: 0.8942931294441223 (0.9695879677195608) + | > loss_1: 30.881864547729492 (29.995438163663135) + | > grad_norm_1: tensor(591.3611, device='cuda:0') (tensor(451.2249, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8956 (0.9882571314588006) + | > loader_time: 0.0056 (0.006486822057653357) + + + --> TIME: 2024-07-13 18:50:52 -- STEP: 131/151 -- GLOBAL_STEP: 87850 + | > loss_disc: 2.4823007583618164 (2.3753619812827065) + | > loss_disc_real_0: 0.179425448179245 (0.13559501342304786) + | > loss_disc_real_1: 0.19902120530605316 (0.20484485539771222) + | > loss_disc_real_2: 0.2334219217300415 (0.21625124975925183) + | > loss_disc_real_3: 0.21369150280952454 (0.21078769999151012) + | > loss_disc_real_4: 0.2460198849439621 (0.21135827671480542) + | > loss_disc_real_5: 0.26961344480514526 (0.22533973744352356) + | > loss_0: 2.4823007583618164 (2.3753619812827065) + | > grad_norm_0: tensor(54.8923, device='cuda:0') (tensor(30.2994, device='cuda:0')) + | > loss_gen: 2.4437646865844727 (2.523515978842292) + | > loss_kl: 1.871037244796753 (1.8184698847413974) + | > loss_feat: 6.545725345611572 (7.911216819559344) + | > loss_mel: 17.175167083740234 (16.90099427900241) + | > loss_duration: 0.945818305015564 (0.9686588849730164) + | > loss_1: 28.98151206970215 (30.122855950857847) + | > grad_norm_1: tensor(563.7812, device='cuda:0') (tensor(449.2499, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.0252 (0.9860751610675841) + | > loader_time: 0.0063 (0.0064563623821462385) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034425240296583906 (+0.0016759358919583786) + | > avg_loss_disc: 2.557853331932655 (-0.0729988905099721) + | > avg_loss_disc_real_0: 0.22679156179611498 (+0.0529329146330173) + | > avg_loss_disc_real_1: 0.23136122753986946 (+0.010699626344900875) + | > avg_loss_disc_real_2: 0.22453252168802115 (+0.013475090265274048) + | > avg_loss_disc_real_3: 0.18594780105810899 (-0.002816431797467739) + | > avg_loss_disc_real_4: 0.2302257900054638 (-0.012107920188170224) + | > avg_loss_disc_real_5: 0.23512541560026315 (+0.030800232520470233) + | > avg_loss_0: 2.557853331932655 (-0.0729988905099721) + | > avg_loss_gen: 2.3299400806427 (+0.2555629748564501) + | > avg_loss_kl: 3.2679249873528113 (-0.017745879980234136) + | > avg_loss_feat: 7.582455048194299 (+0.19674807328444288) + | > avg_loss_mel: 17.48648592141958 (+0.5453341557429354) + | > avg_loss_duration: 1.2749510453297541 (+0.03206957303560687) + | > avg_loss_1: 31.941757202148434 (+1.0119693462665218) + + + > EPOCH: 19/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:51:21)  + + --> TIME: 2024-07-13 18:51:52 -- STEP: 30/151 -- GLOBAL_STEP: 87900 + | > loss_disc: 2.142838716506958 (2.34629491964976) + | > loss_disc_real_0: 0.10109194368124008 (0.12858947552740574) + | > loss_disc_real_1: 0.16142380237579346 (0.20799736628929774) + | > loss_disc_real_2: 0.16408762335777283 (0.2042665297786395) + | > loss_disc_real_3: 0.17270717024803162 (0.20877574235200883) + | > loss_disc_real_4: 0.20213451981544495 (0.21164580136537553) + | > loss_disc_real_5: 0.24267902970314026 (0.22156920184691747) + | > loss_0: 2.142838716506958 (2.34629491964976) + | > grad_norm_0: tensor(17.5079, device='cuda:0') (tensor(20.7991, device='cuda:0')) + | > loss_gen: 2.7006499767303467 (2.5332431157430007) + | > loss_kl: 2.024278163909912 (1.8664103746414185) + | > loss_feat: 9.282334327697754 (8.234426291783647) + | > loss_mel: 17.618520736694336 (17.080628363291428) + | > loss_duration: 0.9039335250854492 (0.9415947020053863) + | > loss_1: 32.52971649169922 (30.656302960713706) + | > grad_norm_1: tensor(106.6716, device='cuda:0') (tensor(234.3828, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8613 (0.9674325148264568) + | > loader_time: 0.0044 (0.005793237686157226) + + + --> TIME: 2024-07-13 18:52:42 -- STEP: 80/151 -- GLOBAL_STEP: 87950 + | > loss_disc: 2.567502021789551 (2.375423619151116) + | > loss_disc_real_0: 0.256682813167572 (0.13615562301129108) + | > loss_disc_real_1: 0.20285290479660034 (0.20852659083902836) + | > loss_disc_real_2: 0.2366425096988678 (0.2118758900091052) + | > loss_disc_real_3: 0.24728307127952576 (0.21037206724286078) + | > loss_disc_real_4: 0.19806835055351257 (0.21195117458701135) + | > loss_disc_real_5: 0.2867521047592163 (0.22581301108002663) + | > loss_0: 2.567502021789551 (2.375423619151116) + | > grad_norm_0: tensor(35.8889, device='cuda:0') (tensor(22.8758, device='cuda:0')) + | > loss_gen: 2.5670392513275146 (2.5095967561006547) + | > loss_kl: 1.9375898838043213 (1.8468353234231472) + | > loss_feat: 7.247156143188477 (7.9806733787059745) + | > loss_mel: 16.087928771972656 (16.969082260131845) + | > loss_duration: 0.8629917502403259 (0.9574604406952858) + | > loss_1: 28.70270538330078 (30.263648176193236) + | > grad_norm_1: tensor(335.7026, device='cuda:0') (tensor(318.9409, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9831 (0.9835371196269989) + | > loader_time: 0.0063 (0.006117168068885802) + + + --> TIME: 2024-07-13 18:53:31 -- STEP: 130/151 -- GLOBAL_STEP: 88000 + | > loss_disc: 2.3858935832977295 (2.3444348922142626) + | > loss_disc_real_0: 0.06682825833559036 (0.129481556151922) + | > loss_disc_real_1: 0.22781112790107727 (0.2054063257116538) + | > loss_disc_real_2: 0.21284329891204834 (0.2125863335453547) + | > loss_disc_real_3: 0.20539715886116028 (0.20894802854611325) + | > loss_disc_real_4: 0.241059809923172 (0.21059255577050723) + | > loss_disc_real_5: 0.18299497663974762 (0.2222604846725097) + | > loss_0: 2.3858935832977295 (2.3444348922142626) + | > grad_norm_0: tensor(27.8157, device='cuda:0') (tensor(28.5517, device='cuda:0')) + | > loss_gen: 2.644824504852295 (2.5686459137843203) + | > loss_kl: 1.6481505632400513 (1.8186712920665742) + | > loss_feat: 8.237253189086914 (8.164317604211657) + | > loss_mel: 16.045639038085938 (17.07873248320358) + | > loss_duration: 0.9506463408470154 (0.9573972257284018) + | > loss_1: 29.526512145996094 (30.58776453458346) + | > grad_norm_1: tensor(752.4386, device='cuda:0') (tensor(417.1966, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8902 (0.9771678887880766) + | > loader_time: 0.006 (0.006032690635094275) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03290376296410194 (-0.0015214773324819658) + | > avg_loss_disc: 2.6170336649968076 (+0.05918033306415271) + | > avg_loss_disc_real_0: 0.23456755738991958 (+0.007775995593804597) + | > avg_loss_disc_real_1: 0.16832918157944313 (-0.06303204596042633) + | > avg_loss_disc_real_2: 0.2048540975038822 (-0.019678424184138954) + | > avg_loss_disc_real_3: 0.23249884637502524 (+0.046551045316916256) + | > avg_loss_disc_real_4: 0.23349798298799074 (+0.003272192982526928) + | > avg_loss_disc_real_5: 0.2241350125807982 (-0.01099040301946494) + | > avg_loss_0: 2.6170336649968076 (+0.05918033306415271) + | > avg_loss_gen: 2.220497351426345 (-0.10944272921635534) + | > avg_loss_kl: 3.291458716759315 (+0.023533729406503667) + | > avg_loss_feat: 7.111919696514423 (-0.47053535167987537) + | > avg_loss_mel: 16.88233580956092 (-0.6041501118586581) + | > avg_loss_duration: 1.2524732718100915 (-0.022477773519662625) + | > avg_loss_1: 30.75868459848257 (-1.1830726036658632) + + + > EPOCH: 20/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:54:14)  + + --> TIME: 2024-07-13 18:54:46 -- STEP: 29/151 -- GLOBAL_STEP: 88050 + | > loss_disc: 1.9881173372268677 (2.294404255932775) + | > loss_disc_real_0: 0.0989580750465393 (0.12481613547123711) + | > loss_disc_real_1: 0.13403116166591644 (0.2009585109250299) + | > loss_disc_real_2: 0.2114536613225937 (0.21044519851947652) + | > loss_disc_real_3: 0.16571258008480072 (0.2085497050449766) + | > loss_disc_real_4: 0.17799584567546844 (0.20742086295423837) + | > loss_disc_real_5: 0.18046404421329498 (0.21222668937568007) + | > loss_0: 1.9881173372268677 (2.294404255932775) + | > grad_norm_0: tensor(49.8099, device='cuda:0') (tensor(27.4310, device='cuda:0')) + | > loss_gen: 2.827180862426758 (2.580575499041327) + | > loss_kl: 1.2878111600875854 (1.7905218354586898) + | > loss_feat: 9.957789421081543 (8.170173497035586) + | > loss_mel: 17.508380889892578 (16.872387918932688) + | > loss_duration: 0.8023737072944641 (0.9628169824337137) + | > loss_1: 32.38353729248047 (30.37647635361244) + | > grad_norm_1: tensor(887.3381, device='cuda:0') (tensor(442.7887, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.847 (1.014829167004289) + | > loader_time: 0.0041 (0.005918765890187231) + + + --> TIME: 2024-07-13 18:55:36 -- STEP: 79/151 -- GLOBAL_STEP: 88100 + | > loss_disc: 2.4125046730041504 (2.3120427629615685) + | > loss_disc_real_0: 0.16291725635528564 (0.11939815299797661) + | > loss_disc_real_1: 0.21361060440540314 (0.20446094851705093) + | > loss_disc_real_2: 0.21367406845092773 (0.21163470203740686) + | > loss_disc_real_3: 0.18076440691947937 (0.20939841357213032) + | > loss_disc_real_4: 0.15817801654338837 (0.2117844559723818) + | > loss_disc_real_5: 0.24179686605930328 (0.21691768437246728) + | > loss_0: 2.4125046730041504 (2.3120427629615685) + | > grad_norm_0: tensor(50.8076, device='cuda:0') (tensor(29.6697, device='cuda:0')) + | > loss_gen: 2.4473154544830322 (2.5930079297174378) + | > loss_kl: 1.7445870637893677 (1.8239458603194998) + | > loss_feat: 7.5693817138671875 (8.162090506734724) + | > loss_mel: 17.23086929321289 (16.925474263444737) + | > loss_duration: 0.9341256022453308 (0.9533613565601881) + | > loss_1: 29.926279067993164 (30.457880261578136) + | > grad_norm_1: tensor(640.4678, device='cuda:0') (tensor(532.6589, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.862 (1.0022429092020924) + | > loader_time: 0.0053 (0.006187743778470196) + + + --> TIME: 2024-07-13 18:56:24 -- STEP: 129/151 -- GLOBAL_STEP: 88150 + | > loss_disc: 2.460691452026367 (2.312040599741676) + | > loss_disc_real_0: 0.22509998083114624 (0.12179816213920135) + | > loss_disc_real_1: 0.20150215923786163 (0.202824094845343) + | > loss_disc_real_2: 0.20008763670921326 (0.21212862622599268) + | > loss_disc_real_3: 0.17559406161308289 (0.2084223697351855) + | > loss_disc_real_4: 0.17778386175632477 (0.21041011036366455) + | > loss_disc_real_5: 0.20456014573574066 (0.21503201606430747) + | > loss_0: 2.460691452026367 (2.312040599741676) + | > grad_norm_0: tensor(84.2368, device='cuda:0') (tensor(34.2313, device='cuda:0')) + | > loss_gen: 2.546311855316162 (2.603138062381005) + | > loss_kl: 1.8115631341934204 (1.8149514170580132) + | > loss_feat: 7.338997840881348 (8.148806531299913) + | > loss_mel: 17.474903106689453 (16.879581503165785) + | > loss_duration: 0.9766793251037598 (0.960763881372851) + | > loss_1: 30.148456573486328 (30.407241688218228) + | > grad_norm_1: tensor(1111.3438, device='cuda:0') (tensor(572.6088, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.0658 (0.9819671128147329) + | > loader_time: 0.0062 (0.006144939466964367) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03191546293405386 (-0.0009883000300480796) + | > avg_loss_disc: 2.564004531273475 (-0.0530291337233324) + | > avg_loss_disc_real_0: 0.24836253088254195 (+0.013794973492622375) + | > avg_loss_disc_real_1: 0.22336638890779936 (+0.055037207328356236) + | > avg_loss_disc_real_2: 0.19560887263371393 (-0.009245224870168267) + | > avg_loss_disc_real_3: 0.2363321758233584 (+0.003833329448333145) + | > avg_loss_disc_real_4: 0.2372484287390342 (+0.0037504457510434686) + | > avg_loss_disc_real_5: 0.239896854529014 (+0.015761841948215782) + | > avg_loss_0: 2.564004531273475 (-0.0530291337233324) + | > avg_loss_gen: 2.4271471316997824 (+0.2066497802734375) + | > avg_loss_kl: 3.3945128184098463 (+0.1030541016505313) + | > avg_loss_feat: 7.785496711730957 (+0.6735770152165337) + | > avg_loss_mel: 17.37600018427922 (+0.4936643747182998) + | > avg_loss_duration: 1.2767387628555298 (+0.024265491045438292) + | > avg_loss_1: 32.25989576486441 (+1.5012111663818395) + + + > EPOCH: 21/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:56:54)  + + --> TIME: 2024-07-13 18:57:22 -- STEP: 28/151 -- GLOBAL_STEP: 88200 + | > loss_disc: 2.453625440597534 (2.359728217124939) + | > loss_disc_real_0: 0.08568670600652695 (0.12792464997619388) + | > loss_disc_real_1: 0.21712082624435425 (0.20522922383887426) + | > loss_disc_real_2: 0.16973958909511566 (0.2171699053474835) + | > loss_disc_real_3: 0.23039129376411438 (0.21749064485941613) + | > loss_disc_real_4: 0.198330819606781 (0.21184447993125236) + | > loss_disc_real_5: 0.21237048506736755 (0.21996559043015754) + | > loss_0: 2.453625440597534 (2.359728217124939) + | > grad_norm_0: tensor(35.7463, device='cuda:0') (tensor(38.4835, device='cuda:0')) + | > loss_gen: 2.3646514415740967 (2.5877995916775296) + | > loss_kl: 1.9158908128738403 (1.7985849039895194) + | > loss_feat: 6.695183753967285 (8.230851531028746) + | > loss_mel: 15.90257453918457 (16.896580014910015) + | > loss_duration: 1.0018377304077148 (0.9632323490721839) + | > loss_1: 27.880138397216797 (30.47704839706421) + | > grad_norm_1: tensor(723.9379, device='cuda:0') (tensor(602.5258, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.0004 (0.9154068316732135) + | > loader_time: 0.0074 (0.0057803477559770855) + + + --> TIME: 2024-07-13 18:58:14 -- STEP: 78/151 -- GLOBAL_STEP: 88250 + | > loss_disc: 2.4362971782684326 (2.3447365821936197) + | > loss_disc_real_0: 0.07343876361846924 (0.13269052492120326) + | > loss_disc_real_1: 0.14982911944389343 (0.20281292287967143) + | > loss_disc_real_2: 0.1665460765361786 (0.21277088824755105) + | > loss_disc_real_3: 0.1519571840763092 (0.21065786385383362) + | > loss_disc_real_4: 0.1126239001750946 (0.20848059424987206) + | > loss_disc_real_5: 0.14064404368400574 (0.22139132424042776) + | > loss_0: 2.4362971782684326 (2.3447365821936197) + | > grad_norm_0: tensor(31.2627, device='cuda:0') (tensor(33.1668, device='cuda:0')) + | > loss_gen: 2.417469024658203 (2.5506242758188487) + | > loss_kl: 1.2623587846755981 (1.816435123101259) + | > loss_feat: 9.86097526550293 (7.961892103537535) + | > loss_mel: 17.590816497802734 (16.890988227648613) + | > loss_duration: 0.883642315864563 (0.96938883799773) + | > loss_1: 32.015262603759766 (30.18932860936874) + | > grad_norm_1: tensor(492.1114, device='cuda:0') (tensor(519.3096, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8512 (0.99018639478928) + | > loader_time: 0.0045 (0.006365543756729517) + + + --> TIME: 2024-07-13 18:59:03 -- STEP: 128/151 -- GLOBAL_STEP: 88300 + | > loss_disc: 2.316409111022949 (2.3369650561362505) + | > loss_disc_real_0: 0.11592566221952438 (0.13007539455429656) + | > loss_disc_real_1: 0.17775434255599976 (0.20214925718028098) + | > loss_disc_real_2: 0.18936361372470856 (0.21403703757096082) + | > loss_disc_real_3: 0.20892561972141266 (0.20889795012772083) + | > loss_disc_real_4: 0.20323839783668518 (0.20922848291229457) + | > loss_disc_real_5: 0.25244060158729553 (0.22014838428003713) + | > loss_0: 2.316409111022949 (2.3369650561362505) + | > grad_norm_0: tensor(18.9188, device='cuda:0') (tensor(32.9246, device='cuda:0')) + | > loss_gen: 2.459510326385498 (2.5650993380695573) + | > loss_kl: 1.8097704648971558 (1.8329088212922215) + | > loss_feat: 8.421921730041504 (8.039886962622404) + | > loss_mel: 17.49603271484375 (16.93426049500704) + | > loss_duration: 0.9368138909339905 (0.9647108796052635) + | > loss_1: 31.12405014038086 (30.33686664700508) + | > grad_norm_1: tensor(302.5688, device='cuda:0') (tensor(510.4679, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9275 (0.9825354292988777) + | > loader_time: 0.0067 (0.0063591524958610535) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033899820767916165 (+0.0019843578338623047) + | > avg_loss_disc: 2.5439712634453406 (-0.020033267828134527) + | > avg_loss_disc_real_0: 0.15924738290218207 (-0.08911514798035988) + | > avg_loss_disc_real_1: 0.22003142421062177 (-0.0033349646971775893) + | > avg_loss_disc_real_2: 0.28199855524760026 (+0.08638968261388633) + | > avg_loss_disc_real_3: 0.240357348552117 (+0.0040251727287586025) + | > avg_loss_disc_real_4: 0.25020770499339473 (+0.012959276254360524) + | > avg_loss_disc_real_5: 0.2527492825801556 (+0.012852428051141618) + | > avg_loss_0: 2.5439712634453406 (-0.020033267828134527) + | > avg_loss_gen: 2.3962778311509356 (-0.030869300548846734) + | > avg_loss_kl: 3.4167796281667857 (+0.0222668097569394) + | > avg_loss_feat: 7.5915954663203316 (-0.19390124541062548) + | > avg_loss_mel: 16.961871293874893 (-0.4141288904043279) + | > avg_loss_duration: 1.3180872477017915 (+0.041348484846261746) + | > avg_loss_1: 31.684612274169922 (-0.5752834906944884) + + + > EPOCH: 22/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 18:59:34)  + + --> TIME: 2024-07-13 19:00:03 -- STEP: 27/151 -- GLOBAL_STEP: 88350 + | > loss_disc: 2.4656710624694824 (2.3736727414307772) + | > loss_disc_real_0: 0.2382824420928955 (0.13812675299467864) + | > loss_disc_real_1: 0.22919219732284546 (0.2034345390620055) + | > loss_disc_real_2: 0.20955105125904083 (0.21439066142947585) + | > loss_disc_real_3: 0.210592120885849 (0.2123064194564466) + | > loss_disc_real_4: 0.22970537841320038 (0.21719754735628763) + | > loss_disc_real_5: 0.3272976577281952 (0.2354339913085655) + | > loss_0: 2.4656710624694824 (2.3736727414307772) + | > grad_norm_0: tensor(57.2152, device='cuda:0') (tensor(36.7072, device='cuda:0')) + | > loss_gen: 2.6358742713928223 (2.5596869433367693) + | > loss_kl: 1.8557524681091309 (1.8473550699375294) + | > loss_feat: 6.848704814910889 (7.908820010997631) + | > loss_mel: 17.247724533081055 (17.295544624328613) + | > loss_duration: 0.9592289328575134 (0.9458391578109177) + | > loss_1: 29.547285079956055 (30.55724546644423) + | > grad_norm_1: tensor(770.3238, device='cuda:0') (tensor(512.0547, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.952 (0.9838205090275517) + | > loader_time: 0.0056 (0.0061089904220015916) + + + --> TIME: 2024-07-13 19:00:52 -- STEP: 77/151 -- GLOBAL_STEP: 88400 + | > loss_disc: 2.443249464035034 (2.3636276629063993) + | > loss_disc_real_0: 0.1467350721359253 (0.1328512683220498) + | > loss_disc_real_1: 0.17960859835147858 (0.20568690052280178) + | > loss_disc_real_2: 0.1699284166097641 (0.2142403826311037) + | > loss_disc_real_3: 0.21122463047504425 (0.2095671139367215) + | > loss_disc_real_4: 0.2375207394361496 (0.21507200927703413) + | > loss_disc_real_5: 0.2855064868927002 (0.2290318078034884) + | > loss_0: 2.443249464035034 (2.3636276629063993) + | > grad_norm_0: tensor(20.2089, device='cuda:0') (tensor(31.3872, device='cuda:0')) + | > loss_gen: 2.3930864334106445 (2.541292571402215) + | > loss_kl: 2.0966506004333496 (1.8136964866093226) + | > loss_feat: 7.584315299987793 (7.870114574184665) + | > loss_mel: 17.638233184814453 (16.954262176117336) + | > loss_duration: 0.96080482006073 (0.9536755781669122) + | > loss_1: 30.67308807373047 (30.13304128275289) + | > grad_norm_1: tensor(139.1608, device='cuda:0') (tensor(468.0323, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.0581 (0.9754424931167008) + | > loader_time: 0.007 (0.006009944073565594) + + + --> TIME: 2024-07-13 19:01:42 -- STEP: 127/151 -- GLOBAL_STEP: 88450 + | > loss_disc: 2.4794058799743652 (2.357457149685838) + | > loss_disc_real_0: 0.1136668249964714 (0.13183539624758595) + | > loss_disc_real_1: 0.22335347533226013 (0.20569692317425736) + | > loss_disc_real_2: 0.21607984602451324 (0.2140622074444463) + | > loss_disc_real_3: 0.237254798412323 (0.20883075493996536) + | > loss_disc_real_4: 0.22605980932712555 (0.21178090220361245) + | > loss_disc_real_5: 0.23383180797100067 (0.22668297436293655) + | > loss_0: 2.4794058799743652 (2.357457149685838) + | > grad_norm_0: tensor(25.3110, device='cuda:0') (tensor(29.8378, device='cuda:0')) + | > loss_gen: 2.520963668823242 (2.542301750558568) + | > loss_kl: 1.75527822971344 (1.784226417072176) + | > loss_feat: 7.006092071533203 (7.894148282178744) + | > loss_mel: 16.001197814941406 (16.79279627762441) + | > loss_duration: 0.958076536655426 (0.9626731727066942) + | > loss_1: 28.241609573364258 (29.97614585696243) + | > grad_norm_1: tensor(537.4528, device='cuda:0') (tensor(470.3885, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8665 (0.97943393451961) + | > loader_time: 0.0049 (0.006080327071542815) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033451190361609824 (-0.0004486304063063412) + | > avg_loss_disc: 2.5252123245826135 (-0.01875893886272717) + | > avg_loss_disc_real_0: 0.17852623531451592 (+0.019278852412333847) + | > avg_loss_disc_real_1: 0.19780166447162628 (-0.02222975973899549) + | > avg_loss_disc_real_2: 0.1987026838155893 (-0.08329587143201095) + | > avg_loss_disc_real_3: 0.24706197931216314 (+0.006704630760046154) + | > avg_loss_disc_real_4: 0.22584450588776514 (-0.02436319910562959) + | > avg_loss_disc_real_5: 0.22135526629594657 (-0.03139401628420904) + | > avg_loss_0: 2.5252123245826135 (-0.01875893886272717) + | > avg_loss_gen: 2.2449040779700646 (-0.15137375318087098) + | > avg_loss_kl: 3.4548836121192346 (+0.03810398395244885) + | > avg_loss_feat: 7.801075751964863 (+0.20948028564453125) + | > avg_loss_mel: 17.432609264667217 (+0.4707379707923245) + | > avg_loss_duration: 1.2765352175785944 (-0.04155203012319708) + | > avg_loss_1: 32.21000774090107 (+0.5253954667311476) + + + > EPOCH: 23/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:02:14)  + + --> TIME: 2024-07-13 19:02:42 -- STEP: 26/151 -- GLOBAL_STEP: 88500 + | > loss_disc: 2.319988250732422 (2.3533433492367086) + | > loss_disc_real_0: 0.1844465136528015 (0.1264512622012542) + | > loss_disc_real_1: 0.22666724026203156 (0.20621985884813163) + | > loss_disc_real_2: 0.20228353142738342 (0.21417818333093935) + | > loss_disc_real_3: 0.23006656765937805 (0.20805440269983733) + | > loss_disc_real_4: 0.19233708083629608 (0.2093515396118164) + | > loss_disc_real_5: 0.190399631857872 (0.22240649392971626) + | > loss_0: 2.319988250732422 (2.3533433492367086) + | > grad_norm_0: tensor(33.9086, device='cuda:0') (tensor(41.9001, device='cuda:0')) + | > loss_gen: 2.796494483947754 (2.5891212683457594) + | > loss_kl: 2.0917980670928955 (1.6836053614432995) + | > loss_feat: 7.233240604400635 (8.069773600651668) + | > loss_mel: 16.50953483581543 (17.04663471075205) + | > loss_duration: 0.9719661474227905 (0.9528787594575149) + | > loss_1: 29.60303497314453 (30.342013725867638) + | > grad_norm_1: tensor(477.6436, device='cuda:0') (tensor(586.4908, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9451 (0.953087568283081) + | > loader_time: 0.0056 (0.005839714637169471) + + + --> TIME: 2024-07-13 19:03:32 -- STEP: 76/151 -- GLOBAL_STEP: 88550 + | > loss_disc: 2.362462043762207 (2.3615040026212992) + | > loss_disc_real_0: 0.0923733338713646 (0.1317256011657025) + | > loss_disc_real_1: 0.22294804453849792 (0.20395861252358086) + | > loss_disc_real_2: 0.22551867365837097 (0.21610365602138795) + | > loss_disc_real_3: 0.25895506143569946 (0.2127327964101967) + | > loss_disc_real_4: 0.1922483593225479 (0.21276853351216568) + | > loss_disc_real_5: 0.2011515498161316 (0.2234131340917788) + | > loss_0: 2.362462043762207 (2.3615040026212992) + | > grad_norm_0: tensor(30.5819, device='cuda:0') (tensor(30.6887, device='cuda:0')) + | > loss_gen: 2.3820555210113525 (2.5525564143532207) + | > loss_kl: 1.6773033142089844 (1.773829700915437) + | > loss_feat: 8.389267921447754 (7.961414481464188) + | > loss_mel: 18.257322311401367 (16.942423619722067) + | > loss_duration: 0.9343777894973755 (0.9578180571919993) + | > loss_1: 31.64032745361328 (30.18804211365549) + | > grad_norm_1: tensor(531.3103, device='cuda:0') (tensor(432.9211, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8735 (0.9768209865218714) + | > loader_time: 0.0061 (0.006116399639531186) + + + --> TIME: 2024-07-13 19:04:20 -- STEP: 126/151 -- GLOBAL_STEP: 88600 + | > loss_disc: 2.5197227001190186 (2.354196855000087) + | > loss_disc_real_0: 0.198042631149292 (0.12983174086917962) + | > loss_disc_real_1: 0.2179972380399704 (0.2042433297823346) + | > loss_disc_real_2: 0.2972608506679535 (0.21529051608272962) + | > loss_disc_real_3: 0.19372841715812683 (0.21091404628186) + | > loss_disc_real_4: 0.29754871129989624 (0.21157543704150214) + | > loss_disc_real_5: 0.220336452126503 (0.22288069874048233) + | > loss_0: 2.5197227001190186 (2.354196855000087) + | > grad_norm_0: tensor(45.1596, device='cuda:0') (tensor(26.8902, device='cuda:0')) + | > loss_gen: 2.485147714614868 (2.5358109001129403) + | > loss_kl: 2.0666379928588867 (1.7813612690993719) + | > loss_feat: 6.715600490570068 (7.992176899834288) + | > loss_mel: 15.81152057647705 (16.92219422355532) + | > loss_duration: 1.1264688968658447 (0.9525548498781901) + | > loss_1: 28.205373764038086 (30.184098001510378) + | > grad_norm_1: tensor(629.6414, device='cuda:0') (tensor(387.1993, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.4861 (0.9666345138398428) + | > loader_time: 0.0112 (0.006177270223223973) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030565940416776217 (-0.0028852499448336066) + | > avg_loss_disc: 2.5750079521766076 (+0.04979562759399414) + | > avg_loss_disc_real_0: 0.2601020106902489 (+0.08157577537573299) + | > avg_loss_disc_real_1: 0.2247697527591999 (+0.026968088287573605) + | > avg_loss_disc_real_2: 0.2562162004984342 (+0.05751351668284488) + | > avg_loss_disc_real_3: 0.20660807880071494 (-0.040453900511448204) + | > avg_loss_disc_real_4: 0.22224680735514715 (-0.0035976985326179878) + | > avg_loss_disc_real_5: 0.23128398450521323 (+0.009928718209266663) + | > avg_loss_0: 2.5750079521766076 (+0.04979562759399414) + | > avg_loss_gen: 2.516723944590642 (+0.27181986662057733) + | > avg_loss_kl: 3.567018857369056 (+0.11213524524982166) + | > avg_loss_feat: 8.195050753079927 (+0.39397500111506467) + | > avg_loss_mel: 17.4327215047983 (+0.00011224013108446229) + | > avg_loss_duration: 1.3040053661052997 (+0.027470148526705263) + | > avg_loss_1: 33.01551994910607 (+0.8055122082050019) + + + > EPOCH: 24/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:04:52)  + + --> TIME: 2024-07-13 19:05:18 -- STEP: 25/151 -- GLOBAL_STEP: 88650 + | > loss_disc: 2.399791955947876 (2.381889572143555) + | > loss_disc_real_0: 0.0857776626944542 (0.135815586745739) + | > loss_disc_real_1: 0.21234026551246643 (0.20684001326560975) + | > loss_disc_real_2: 0.19067002832889557 (0.22141911149024962) + | > loss_disc_real_3: 0.21830369532108307 (0.2113976538181305) + | > loss_disc_real_4: 0.20206217467784882 (0.2168295806646347) + | > loss_disc_real_5: 0.22480939328670502 (0.2244894152879715) + | > loss_0: 2.399791955947876 (2.381889572143555) + | > grad_norm_0: tensor(27.8219, device='cuda:0') (tensor(28.2624, device='cuda:0')) + | > loss_gen: 2.2828030586242676 (2.5498824977874763) + | > loss_kl: 1.726523756980896 (1.7160265731811524) + | > loss_feat: 7.762568473815918 (7.882194576263427) + | > loss_mel: 16.521560668945312 (16.668695297241214) + | > loss_duration: 0.8132888078689575 (0.9669518733024597) + | > loss_1: 29.10674476623535 (29.783751068115233) + | > grad_norm_1: tensor(249.8526, device='cuda:0') (tensor(460.4995, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8978 (0.9596849536895752) + | > loader_time: 0.0048 (0.006121139526367187) + + + --> TIME: 2024-07-13 19:06:07 -- STEP: 75/151 -- GLOBAL_STEP: 88700 + | > loss_disc: 2.3599822521209717 (2.362811632156372) + | > loss_disc_real_0: 0.15578636527061462 (0.13524712666869163) + | > loss_disc_real_1: 0.22185596823692322 (0.20487516363461813) + | > loss_disc_real_2: 0.2538627088069916 (0.21489287734031678) + | > loss_disc_real_3: 0.22452247142791748 (0.2113544525702795) + | > loss_disc_real_4: 0.18293572962284088 (0.2148966602484385) + | > loss_disc_real_5: 0.18548442423343658 (0.21967846512794495) + | > loss_0: 2.3599822521209717 (2.362811632156372) + | > grad_norm_0: tensor(55.2414, device='cuda:0') (tensor(32.0671, device='cuda:0')) + | > loss_gen: 2.590393304824829 (2.5539299297332763) + | > loss_kl: 1.784285068511963 (1.7595052647590637) + | > loss_feat: 8.042542457580566 (7.958631432851155) + | > loss_mel: 18.101224899291992 (16.892009277343742) + | > loss_duration: 0.9758481979370117 (0.9629914482434591) + | > loss_1: 31.494293212890625 (30.127067413330078) + | > grad_norm_1: tensor(597.6507, device='cuda:0') (tensor(443.6188, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0558 (0.9627383613586425) + | > loader_time: 0.0068 (0.0062098821004231775) + + + --> TIME: 2024-07-13 19:06:55 -- STEP: 125/151 -- GLOBAL_STEP: 88750 + | > loss_disc: 2.37412691116333 (2.351273529052735) + | > loss_disc_real_0: 0.13492242991924286 (0.13226854461431506) + | > loss_disc_real_1: 0.27221983671188354 (0.204723204433918) + | > loss_disc_real_2: 0.25249990820884705 (0.2146384677886963) + | > loss_disc_real_3: 0.20946113765239716 (0.21087154901027683) + | > loss_disc_real_4: 0.2564867436885834 (0.21251773273944854) + | > loss_disc_real_5: 0.18630751967430115 (0.2199168154001236) + | > loss_0: 2.37412691116333 (2.351273529052735) + | > grad_norm_0: tensor(9.7463, device='cuda:0') (tensor(27.4741, device='cuda:0')) + | > loss_gen: 2.4835376739501953 (2.5525146369934073) + | > loss_kl: 1.797499418258667 (1.779316948413849) + | > loss_feat: 7.9351582527160645 (8.062546958923344) + | > loss_mel: 17.967666625976562 (17.002521049499503) + | > loss_duration: 0.8888323903083801 (0.9679084157943726) + | > loss_1: 31.072694778442383 (30.36480822753906) + | > grad_norm_1: tensor(153.5048, device='cuda:0') (tensor(385.9819, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8736 (0.9612162132263183) + | > loader_time: 0.0049 (0.006153301239013673) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032782939764169544 (+0.0022169993473933267) + | > avg_loss_disc: 2.5178484733288107 (-0.05715947884779693) + | > avg_loss_disc_real_0: 0.23466161810434782 (-0.025440392585901084) + | > avg_loss_disc_real_1: 0.22368882596492767 (-0.0010809267942722134) + | > avg_loss_disc_real_2: 0.22585190144868997 (-0.03036429904974422) + | > avg_loss_disc_real_3: 0.21816901518748358 (+0.011560936386768639) + | > avg_loss_disc_real_4: 0.22119456414992994 (-0.0010522432052172126) + | > avg_loss_disc_real_5: 0.19779831285660082 (-0.03348567164861241) + | > avg_loss_0: 2.5178484733288107 (-0.05715947884779693) + | > avg_loss_gen: 2.366922195141132 (-0.14980174944951008) + | > avg_loss_kl: 3.3260037348820615 (-0.24101512248699475) + | > avg_loss_feat: 7.624883761772742 (-0.5701669913071852) + | > avg_loss_mel: 17.34266838660607 (-0.09005311819223039) + | > avg_loss_duration: 1.2676734374119685 (-0.03633192869333124) + | > avg_loss_1: 31.928151790912334 (-1.0873681581937369) + + + > EPOCH: 25/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:07:31)  + + --> TIME: 2024-07-13 19:07:56 -- STEP: 24/151 -- GLOBAL_STEP: 88800 + | > loss_disc: 2.4666407108306885 (2.3254354298114785) + | > loss_disc_real_0: 0.13957904279232025 (0.11669530067592859) + | > loss_disc_real_1: 0.26827484369277954 (0.20959518228967985) + | > loss_disc_real_2: 0.2813410460948944 (0.20981029607355595) + | > loss_disc_real_3: 0.2396717667579651 (0.21211413169900575) + | > loss_disc_real_4: 0.2563118636608124 (0.20517210786541304) + | > loss_disc_real_5: 0.24626269936561584 (0.2255710499982039) + | > loss_0: 2.4666407108306885 (2.3254354298114785) + | > grad_norm_0: tensor(39.8629, device='cuda:0') (tensor(22.0331, device='cuda:0')) + | > loss_gen: 2.7022078037261963 (2.5646484792232513) + | > loss_kl: 1.8691444396972656 (1.7639238809545834) + | > loss_feat: 7.045583724975586 (8.165046056111652) + | > loss_mel: 17.595504760742188 (17.131021261215206) + | > loss_duration: 0.9022411108016968 (0.9454992065827051) + | > loss_1: 30.114681243896484 (30.570138692855835) + | > grad_norm_1: tensor(422.9330, device='cuda:0') (tensor(332.4106, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9991 (0.9270976781845093) + | > loader_time: 0.0059 (0.005244523286819458) + + + --> TIME: 2024-07-13 19:08:45 -- STEP: 74/151 -- GLOBAL_STEP: 88850 + | > loss_disc: 2.23579478263855 (2.340457230000883) + | > loss_disc_real_0: 0.08357720077037811 (0.1258314387822473) + | > loss_disc_real_1: 0.22871702909469604 (0.20526100272262418) + | > loss_disc_real_2: 0.19053854048252106 (0.2129542140944584) + | > loss_disc_real_3: 0.19732466340065002 (0.21057092438678485) + | > loss_disc_real_4: 0.21575841307640076 (0.21165392749212886) + | > loss_disc_real_5: 0.2111196219921112 (0.22114385392617536) + | > loss_0: 2.23579478263855 (2.340457230000883) + | > grad_norm_0: tensor(27.6435, device='cuda:0') (tensor(22.1598, device='cuda:0')) + | > loss_gen: 2.530317544937134 (2.5495491350019295) + | > loss_kl: 1.8059974908828735 (1.7683155029206663) + | > loss_feat: 8.629532814025879 (8.041421626065228) + | > loss_mel: 17.677953720092773 (17.01856232978202) + | > loss_duration: 0.8836603164672852 (0.9500613945561487) + | > loss_1: 31.527462005615234 (30.32790980467925) + | > grad_norm_1: tensor(553.0062, device='cuda:0') (tensor(359.3148, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9312 (0.9557419660929087) + | > loader_time: 0.0063 (0.005835088523658545) + + + --> TIME: 2024-07-13 19:09:35 -- STEP: 124/151 -- GLOBAL_STEP: 88900 + | > loss_disc: 2.520878791809082 (2.345889793288323) + | > loss_disc_real_0: 0.15172651410102844 (0.13290882074544508) + | > loss_disc_real_1: 0.21860255300998688 (0.2058770944995265) + | > loss_disc_real_2: 0.21504400670528412 (0.2124916331902627) + | > loss_disc_real_3: 0.21854761242866516 (0.20947484888376727) + | > loss_disc_real_4: 0.20961424708366394 (0.2110383050335992) + | > loss_disc_real_5: 0.21447499096393585 (0.21982492812939228) + | > loss_0: 2.520878791809082 (2.345889793288323) + | > grad_norm_0: tensor(50.2245, device='cuda:0') (tensor(21.2735, device='cuda:0')) + | > loss_gen: 2.210008382797241 (2.5525652420136233) + | > loss_kl: 1.8194063901901245 (1.7881444399395297) + | > loss_feat: 6.775662422180176 (8.061370480445122) + | > loss_mel: 16.562009811401367 (16.991790271574438) + | > loss_duration: 0.9793808460235596 (0.9468765734664856) + | > loss_1: 28.346467971801758 (30.340746864195793) + | > grad_norm_1: tensor(286.7086, device='cuda:0') (tensor(308.5385, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9448 (0.9694174066666634) + | > loader_time: 0.0064 (0.00613541180087674) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03271392675546499 (-6.901300870455068e-05) + | > avg_loss_disc: 2.483803987503052 (-0.034044485825758475) + | > avg_loss_disc_real_0: 0.2635105745150493 (+0.028848956410701454) + | > avg_loss_disc_real_1: 0.18813563310183012 (-0.03555319286309755) + | > avg_loss_disc_real_2: 0.18353635531205398 (-0.04231554613663599) + | > avg_loss_disc_real_3: 0.20028494126521623 (-0.017884073922267346) + | > avg_loss_disc_real_4: 0.21183810555017912 (-0.009356458599750817) + | > avg_loss_disc_real_5: 0.200013945882137 (+0.002215633025536179) + | > avg_loss_0: 2.483803987503052 (-0.034044485825758475) + | > avg_loss_gen: 2.3844577532548175 (+0.017535558113685568) + | > avg_loss_kl: 3.426472260401799 (+0.10046852551973773) + | > avg_loss_feat: 7.885186965648942 (+0.2603032038761999) + | > avg_loss_mel: 16.8698028417734 (-0.47286554483267196) + | > avg_loss_duration: 1.2601727522336519 (-0.007500685178316591) + | > avg_loss_1: 31.82609264667218 (-0.10205914424015461) + + + > EPOCH: 26/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:10:12)  + + --> TIME: 2024-07-13 19:10:35 -- STEP: 23/151 -- GLOBAL_STEP: 88950 + | > loss_disc: 2.3295834064483643 (2.3124047362286113) + | > loss_disc_real_0: 0.12546688318252563 (0.12125928959120875) + | > loss_disc_real_1: 0.1924854815006256 (0.20026577102101367) + | > loss_disc_real_2: 0.2751513123512268 (0.21301349155280902) + | > loss_disc_real_3: 0.1753920018672943 (0.20894454613975857) + | > loss_disc_real_4: 0.22072243690490723 (0.21123669264109238) + | > loss_disc_real_5: 0.20532973110675812 (0.21930471386598505) + | > loss_0: 2.3295834064483643 (2.3124047362286113) + | > grad_norm_0: tensor(11.7778, device='cuda:0') (tensor(18.1279, device='cuda:0')) + | > loss_gen: 2.8359031677246094 (2.6257999668950625) + | > loss_kl: 1.8647562265396118 (1.7939942038577537) + | > loss_feat: 8.475083351135254 (8.414727314658787) + | > loss_mel: 16.908672332763672 (17.161782596422277) + | > loss_duration: 0.9937959909439087 (0.953778282455776) + | > loss_1: 31.078210830688477 (30.95008269600246) + | > grad_norm_1: tensor(509.8382, device='cuda:0') (tensor(379.2374, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.992 (0.911233321480129) + | > loader_time: 0.0062 (0.005502172138379968) + + + --> TIME: 2024-07-13 19:11:24 -- STEP: 73/151 -- GLOBAL_STEP: 89000 + | > loss_disc: 2.49963641166687 (2.3158310341508415) + | > loss_disc_real_0: 0.18407051265239716 (0.12703218954066708) + | > loss_disc_real_1: 0.1871548891067505 (0.20133901254771508) + | > loss_disc_real_2: 0.20275089144706726 (0.2122714054094602) + | > loss_disc_real_3: 0.23314456641674042 (0.20883792026402198) + | > loss_disc_real_4: 0.21063661575317383 (0.2123907402361909) + | > loss_disc_real_5: 0.22257541120052338 (0.2168905788088498) + | > loss_0: 2.49963641166687 (2.3158310341508415) + | > grad_norm_0: tensor(47.4213, device='cuda:0') (tensor(28.0558, device='cuda:0')) + | > loss_gen: 2.3696813583374023 (2.6154710919889683) + | > loss_kl: 2.089257001876831 (1.79929939034867) + | > loss_feat: 5.342071056365967 (8.271910980956196) + | > loss_mel: 15.457404136657715 (17.098863052995238) + | > loss_duration: 1.1363115310668945 (0.9602797162042905) + | > loss_1: 26.394725799560547 (30.745824369665694) + | > grad_norm_1: tensor(552.2282, device='cuda:0') (tensor(444.9398, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 2.0164 (0.9538918260025652) + | > loader_time: 0.0166 (0.005987536417294855) + + + --> TIME: 2024-07-13 19:12:27 -- STEP: 123/151 -- GLOBAL_STEP: 89050 + | > loss_disc: 2.2805097103118896 (2.3208477594018952) + | > loss_disc_real_0: 0.11531476676464081 (0.12417736619226331) + | > loss_disc_real_1: 0.16619910299777985 (0.20259724242415855) + | > loss_disc_real_2: 0.1744440644979477 (0.21224693154416432) + | > loss_disc_real_3: 0.18041163682937622 (0.2094931806006083) + | > loss_disc_real_4: 0.16774345934391022 (0.2104862590388554) + | > loss_disc_real_5: 0.23438334465026855 (0.21872684747222962) + | > loss_0: 2.2805097103118896 (2.3208477594018952) + | > grad_norm_0: tensor(20.4414, device='cuda:0') (tensor(27.5555, device='cuda:0')) + | > loss_gen: 2.441380739212036 (2.598522079669363) + | > loss_kl: 1.5095022916793823 (1.781089840865717) + | > loss_feat: 8.40842342376709 (8.216395416880038) + | > loss_mel: 15.609380722045898 (16.987203931420794) + | > loss_duration: 0.9099698066711426 (0.963369814361014) + | > loss_1: 28.8786563873291 (30.546581206282948) + | > grad_norm_1: tensor(234.6720, device='cuda:0') (tensor(451.0143, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8599 (0.9576729119308596) + | > loader_time: 0.0054 (0.005993779112653035) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03251147270202637 (-0.00020245405343862627) + | > avg_loss_disc: 2.637622429774358 (+0.1538184422713056) + | > avg_loss_disc_real_0: 0.1355742858006404 (-0.12793628871440887) + | > avg_loss_disc_real_1: 0.20213998166414407 (+0.014004348562313945) + | > avg_loss_disc_real_2: 0.2090825644823221 (+0.02554620917026812) + | > avg_loss_disc_real_3: 0.275139067035455 (+0.07485412577023876) + | > avg_loss_disc_real_4: 0.2650182315936455 (+0.05318012604346639) + | > avg_loss_disc_real_5: 0.2138163481767361 (+0.013802402294599087) + | > avg_loss_0: 2.637622429774358 (+0.1538184422713056) + | > avg_loss_gen: 2.1191807068311252 (-0.2652770464236922) + | > avg_loss_kl: 3.2327779072981615 (-0.1936943531036377) + | > avg_loss_feat: 7.434865474700928 (-0.45032149094801444) + | > avg_loss_mel: 17.150306334862343 (+0.2805034930889434) + | > avg_loss_duration: 1.2909508026563203 (+0.030778050422668457) + | > avg_loss_1: 31.22808104294997 (-0.5980116037222096) + + + > EPOCH: 27/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:13:05)  + + --> TIME: 2024-07-13 19:13:28 -- STEP: 22/151 -- GLOBAL_STEP: 89100 + | > loss_disc: 2.2768054008483887 (2.3280414559624414) + | > loss_disc_real_0: 0.07464787364006042 (0.13840533420443535) + | > loss_disc_real_1: 0.2026364654302597 (0.20307555320588025) + | > loss_disc_real_2: 0.2073524445295334 (0.21149776943705298) + | > loss_disc_real_3: 0.1761508285999298 (0.20859487761150708) + | > loss_disc_real_4: 0.21249711513519287 (0.2092412134463137) + | > loss_disc_real_5: 0.23931856453418732 (0.2167075520212) + | > loss_0: 2.2768054008483887 (2.3280414559624414) + | > grad_norm_0: tensor(9.5045, device='cuda:0') (tensor(25.3546, device='cuda:0')) + | > loss_gen: 2.5299177169799805 (2.542431181127375) + | > loss_kl: 1.8314214944839478 (1.7520660053599963) + | > loss_feat: 7.909651279449463 (8.000632936304266) + | > loss_mel: 16.89702796936035 (16.918596354397856) + | > loss_duration: 0.9198102951049805 (0.9390934570269152) + | > loss_1: 30.08782958984375 (30.15281972018155) + | > grad_norm_1: tensor(202.1489, device='cuda:0') (tensor(379.6005, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8578 (0.9454402815211903) + | > loader_time: 0.0054 (0.005808548493818803) + + + --> TIME: 2024-07-13 19:14:18 -- STEP: 72/151 -- GLOBAL_STEP: 89150 + | > loss_disc: 2.4614341259002686 (2.321126071943177) + | > loss_disc_real_0: 0.1918242871761322 (0.12920172512531275) + | > loss_disc_real_1: 0.2688959240913391 (0.20382795017212627) + | > loss_disc_real_2: 0.30007854104042053 (0.2115655868417687) + | > loss_disc_real_3: 0.23346596956253052 (0.20835473471217686) + | > loss_disc_real_4: 0.21643243730068207 (0.2104214294296172) + | > loss_disc_real_5: 0.2733803391456604 (0.21690810595949492) + | > loss_0: 2.4614341259002686 (2.321126071943177) + | > grad_norm_0: tensor(16.9695, device='cuda:0') (tensor(22.6741, device='cuda:0')) + | > loss_gen: 2.645641803741455 (2.575303372409608) + | > loss_kl: 2.190673589706421 (1.7543587883313496) + | > loss_feat: 7.246433258056641 (8.206654879781942) + | > loss_mel: 17.58814811706543 (17.07666744126215) + | > loss_duration: 1.0286744832992554 (0.9594039767980576) + | > loss_1: 30.69957160949707 (30.572388410568237) + | > grad_norm_1: tensor(510.6491, device='cuda:0') (tensor(349.6913, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.107 (0.9689461092154185) + | > loader_time: 0.0071 (0.005972640381919013) + + + --> TIME: 2024-07-13 19:15:09 -- STEP: 122/151 -- GLOBAL_STEP: 89200 + | > loss_disc: 2.2579593658447266 (2.3280280232429504) + | > loss_disc_real_0: 0.11979606002569199 (0.12963946120905095) + | > loss_disc_real_1: 0.21213537454605103 (0.20398657958282798) + | > loss_disc_real_2: 0.20497937500476837 (0.21124431197760538) + | > loss_disc_real_3: 0.21719855070114136 (0.20793337649742108) + | > loss_disc_real_4: 0.17116078734397888 (0.20986606956261103) + | > loss_disc_real_5: 0.22645536065101624 (0.21772456523336348) + | > loss_0: 2.2579593658447266 (2.3280280232429504) + | > grad_norm_0: tensor(26.7562, device='cuda:0') (tensor(25.6700, device='cuda:0')) + | > loss_gen: 2.705355644226074 (2.5647201987563584) + | > loss_kl: 1.855916142463684 (1.7873188755551324) + | > loss_feat: 8.114309310913086 (8.065369848345151) + | > loss_mel: 16.50725746154785 (17.007686786964307) + | > loss_duration: 0.929105281829834 (0.9590473429101413) + | > loss_1: 30.1119441986084 (30.384143047645445) + | > grad_norm_1: tensor(502.2136, device='cuda:0') (tensor(395.4495, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8576 (0.9857986383750791) + | > loader_time: 0.0047 (0.006241802309380202) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030969619750976562 (-0.0015418529510498047) + | > avg_loss_disc: 2.5922254965855527 (-0.04539693318880511) + | > avg_loss_disc_real_0: 0.2969805048062251 (+0.16140621900558472) + | > avg_loss_disc_real_1: 0.24203205681764162 (+0.03989207515349755) + | > avg_loss_disc_real_2: 0.2277034853513424 (+0.018620920869020313) + | > avg_loss_disc_real_3: 0.24916040094999167 (-0.025978666085463314) + | > avg_loss_disc_real_4: 0.2333480784526238 (-0.0316701531410217) + | > avg_loss_disc_real_5: 0.26476751841031587 (+0.050951170233579784) + | > avg_loss_0: 2.5922254965855527 (-0.04539693318880511) + | > avg_loss_gen: 2.570832839378944 (+0.45165213254781866) + | > avg_loss_kl: 3.4045391999758206 (+0.17176129267765905) + | > avg_loss_feat: 7.187800003932073 (-0.2470654707688551) + | > avg_loss_mel: 16.698944898752067 (-0.45136143611027535) + | > avg_loss_duration: 1.2741516186640813 (-0.01679918399223901) + | > avg_loss_1: 31.136267588688778 (-0.09181345426119236) + + + > EPOCH: 28/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:15:45)  + + --> TIME: 2024-07-13 19:16:09 -- STEP: 21/151 -- GLOBAL_STEP: 89250 + | > loss_disc: 2.2679848670959473 (2.3741393770490373) + | > loss_disc_real_0: 0.12377987802028656 (0.13778925012974513) + | > loss_disc_real_1: 0.16464853286743164 (0.20834354488622575) + | > loss_disc_real_2: 0.2235260009765625 (0.21741441601798647) + | > loss_disc_real_3: 0.2444203495979309 (0.20989543057623364) + | > loss_disc_real_4: 0.18204204738140106 (0.21215488868100302) + | > loss_disc_real_5: 0.16846974194049835 (0.22527118098168147) + | > loss_0: 2.2679848670959473 (2.3741393770490373) + | > grad_norm_0: tensor(39.3822, device='cuda:0') (tensor(23.1356, device='cuda:0')) + | > loss_gen: 2.5187530517578125 (2.5158191862560453) + | > loss_kl: 1.8563355207443237 (1.814760412488665) + | > loss_feat: 6.770241737365723 (7.63313509169079) + | > loss_mel: 17.50095558166504 (16.866465296064103) + | > loss_duration: 0.9603234529495239 (0.9640091203507923) + | > loss_1: 29.606609344482422 (29.794188998994372) + | > grad_norm_1: tensor(401.6534, device='cuda:0') (tensor(292.8967, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8822 (1.002994696299235) + | > loader_time: 0.0051 (0.006576186134701683) + + + --> TIME: 2024-07-13 19:17:00 -- STEP: 71/151 -- GLOBAL_STEP: 89300 + | > loss_disc: 2.217332124710083 (2.3651375300447697) + | > loss_disc_real_0: 0.0990261659026146 (0.13414242884642652) + | > loss_disc_real_1: 0.18555216491222382 (0.20347054344667515) + | > loss_disc_real_2: 0.17010335624217987 (0.21541530376588794) + | > loss_disc_real_3: 0.15407702326774597 (0.20920983292687106) + | > loss_disc_real_4: 0.1911989450454712 (0.21029402861292934) + | > loss_disc_real_5: 0.18615005910396576 (0.22677090827008367) + | > loss_0: 2.217332124710083 (2.3651375300447697) + | > grad_norm_0: tensor(25.8282, device='cuda:0') (tensor(23.5683, device='cuda:0')) + | > loss_gen: 2.4303297996520996 (2.5152171766254265) + | > loss_kl: 1.5273886919021606 (1.843653551289733) + | > loss_feat: 7.006995677947998 (7.8753516909102315) + | > loss_mel: 15.42601203918457 (17.08283741373412) + | > loss_duration: 0.9792113304138184 (0.9564298466897346) + | > loss_1: 27.369937896728516 (30.273489696878784) + | > grad_norm_1: tensor(599.3526, device='cuda:0') (tensor(352.2557, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8512 (1.015262556747651) + | > loader_time: 0.0054 (0.00649645294941647) + + + --> TIME: 2024-07-13 19:17:48 -- STEP: 121/151 -- GLOBAL_STEP: 89350 + | > loss_disc: 2.551178216934204 (2.3453960231513022) + | > loss_disc_real_0: 0.09572198987007141 (0.13193409434281114) + | > loss_disc_real_1: 0.18296891450881958 (0.2021829575792817) + | > loss_disc_real_2: 0.1567000150680542 (0.213277799531448) + | > loss_disc_real_3: 0.20795823633670807 (0.20678706701136818) + | > loss_disc_real_4: 0.2203769087791443 (0.20936514560348732) + | > loss_disc_real_5: 0.25622323155403137 (0.22372113291389686) + | > loss_0: 2.551178216934204 (2.3453960231513022) + | > grad_norm_0: tensor(14.2066, device='cuda:0') (tensor(25.0436, device='cuda:0')) + | > loss_gen: 2.1576404571533203 (2.554476229612493) + | > loss_kl: 1.8043192625045776 (1.825109389202654) + | > loss_feat: 7.422415256500244 (8.080360144623054) + | > loss_mel: 17.565946578979492 (17.123929071032318) + | > loss_duration: 0.9823567867279053 (0.9505403179767703) + | > loss_1: 29.93267822265625 (30.534415205648124) + | > grad_norm_1: tensor(140.4449, device='cuda:0') (tensor(386.6416, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8643 (0.9822222043660059) + | > loader_time: 0.005 (0.006210214835553131) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030667580091036283 (-0.00030203965994028) + | > avg_loss_disc: 2.529532505915715 (-0.06269299066983747) + | > avg_loss_disc_real_0: 0.2691305016095822 (-0.027850003196642936) + | > avg_loss_disc_real_1: 0.23825113704571357 (-0.003780919771928043) + | > avg_loss_disc_real_2: 0.2614689480799895 (+0.03376546272864708) + | > avg_loss_disc_real_3: 0.2587020316949257 (+0.009541630744934054) + | > avg_loss_disc_real_4: 0.25397184835030484 (+0.020623769897681027) + | > avg_loss_disc_real_5: 0.2714484093280939 (+0.006680890917778015) + | > avg_loss_0: 2.529532505915715 (-0.06269299066983747) + | > avg_loss_gen: 2.842653439595149 (+0.27182060021620513) + | > avg_loss_kl: 3.315367735349215 (-0.08917146462660552) + | > avg_loss_feat: 8.441278347602255 (+1.2534783436701824) + | > avg_loss_mel: 17.727431003863998 (+1.0284861051119307) + | > avg_loss_duration: 1.314277172088623 (+0.040125553424541716) + | > avg_loss_1: 33.64100764347957 (+2.5047400547907905) + + + > EPOCH: 29/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:18:26)  + + --> TIME: 2024-07-13 19:18:48 -- STEP: 20/151 -- GLOBAL_STEP: 89400 + | > loss_disc: 2.3891468048095703 (2.3393722057342528) + | > loss_disc_real_0: 0.13814373314380646 (0.13122593834996224) + | > loss_disc_real_1: 0.26211243867874146 (0.20894508361816405) + | > loss_disc_real_2: 0.2544742226600647 (0.21515194401144982) + | > loss_disc_real_3: 0.2298571616411209 (0.20695175379514694) + | > loss_disc_real_4: 0.21496295928955078 (0.21133029237389564) + | > loss_disc_real_5: 0.24118249118328094 (0.22374329268932341) + | > loss_0: 2.3891468048095703 (2.3393722057342528) + | > grad_norm_0: tensor(46.0664, device='cuda:0') (tensor(33.8970, device='cuda:0')) + | > loss_gen: 2.7106308937072754 (2.6448769211769103) + | > loss_kl: 2.163228750228882 (1.906265127658844) + | > loss_feat: 7.575900077819824 (8.017279624938965) + | > loss_mel: 17.68602180480957 (17.668684101104734) + | > loss_duration: 0.9374493956565857 (0.9501073449850083) + | > loss_1: 31.073230743408203 (31.187213230133057) + | > grad_norm_1: tensor(721.4847, device='cuda:0') (tensor(533.9555, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.0076 (1.0048822164535522) + | > loader_time: 0.0061 (0.006447780132293701) + + + --> TIME: 2024-07-13 19:19:38 -- STEP: 70/151 -- GLOBAL_STEP: 89450 + | > loss_disc: 2.388235092163086 (2.342650038855417) + | > loss_disc_real_0: 0.03780147060751915 (0.12882844034050187) + | > loss_disc_real_1: 0.19716337323188782 (0.2082309269479343) + | > loss_disc_real_2: 0.19274938106536865 (0.21161970517465045) + | > loss_disc_real_3: 0.21775692701339722 (0.20841274048600877) + | > loss_disc_real_4: 0.2282111793756485 (0.21062410878283636) + | > loss_disc_real_5: 0.17433321475982666 (0.22383683430297033) + | > loss_0: 2.388235092163086 (2.342650038855417) + | > grad_norm_0: tensor(36.7487, device='cuda:0') (tensor(37.1052, device='cuda:0')) + | > loss_gen: 2.5761966705322266 (2.637463331222534) + | > loss_kl: 1.8835798501968384 (1.8381381409508841) + | > loss_feat: 8.61109733581543 (8.127888011932376) + | > loss_mel: 18.170080184936523 (17.261807577950613) + | > loss_duration: 0.8579712510108948 (0.9596034288406372) + | > loss_1: 32.09892654418945 (30.824900572640555) + | > grad_norm_1: tensor(756.4721, device='cuda:0') (tensor(575.8945, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.937 (0.9913384880338396) + | > loader_time: 0.0067 (0.006283310481480189) + + + --> TIME: 2024-07-13 19:20:27 -- STEP: 120/151 -- GLOBAL_STEP: 89500 + | > loss_disc: 2.427060127258301 (2.3503109455108655) + | > loss_disc_real_0: 0.13648444414138794 (0.13826725759233038) + | > loss_disc_real_1: 0.20714333653450012 (0.20639728071788946) + | > loss_disc_real_2: 0.22902889549732208 (0.2104043496151765) + | > loss_disc_real_3: 0.18399569392204285 (0.20767025562624136) + | > loss_disc_real_4: 0.2141992449760437 (0.21032030892868836) + | > loss_disc_real_5: 0.18448950350284576 (0.22266051173210144) + | > loss_0: 2.427060127258301 (2.3503109455108655) + | > grad_norm_0: tensor(6.5530, device='cuda:0') (tensor(33.2487, device='cuda:0')) + | > loss_gen: 2.26737380027771 (2.588802550236383) + | > loss_kl: 1.42264723777771 (1.8312108531594276) + | > loss_feat: 7.40108585357666 (8.028316573301954) + | > loss_mel: 13.983641624450684 (17.031510774294546) + | > loss_duration: 0.9764145612716675 (0.9534591396649679) + | > loss_1: 26.051162719726562 (30.433299859364826) + | > grad_norm_1: tensor(184.1448, device='cuda:0') (tensor(475.2386, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8148 (0.981724061568578) + | > loader_time: 0.0054 (0.0062534173329671225) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032912602791419394 (+0.002245022700383112) + | > avg_loss_disc: 2.5654387107262244 (+0.03590620481050921) + | > avg_loss_disc_real_0: 0.23608628717752603 (-0.03304421443205616) + | > avg_loss_disc_real_1: 0.20537045483405775 (-0.032880682211655826) + | > avg_loss_disc_real_2: 0.24340850917192605 (-0.018060438908063442) + | > avg_loss_disc_real_3: 0.2589640571520879 (+0.00026202545716219605) + | > avg_loss_disc_real_4: 0.26367263610546404 (+0.009700787755159201) + | > avg_loss_disc_real_5: 0.23530846604934105 (-0.036139943278752834) + | > avg_loss_0: 2.5654387107262244 (+0.03590620481050921) + | > avg_loss_gen: 2.492646859242366 (-0.3500065803527832) + | > avg_loss_kl: 3.389585550014789 (+0.07421781466557409) + | > avg_loss_feat: 7.632409279163067 (-0.8088690684391882) + | > avg_loss_mel: 17.41924630678617 (-0.30818469707782725) + | > avg_loss_duration: 1.2986619472503662 (-0.015615224838256836) + | > avg_loss_1: 32.23254981407752 (-1.408457829402046) + + + > EPOCH: 30/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:21:05)  + + --> TIME: 2024-07-13 19:21:31 -- STEP: 19/151 -- GLOBAL_STEP: 89550 + | > loss_disc: 2.27219820022583 (2.32333641303213) + | > loss_disc_real_0: 0.17276045680046082 (0.14173152729084618) + | > loss_disc_real_1: 0.18958333134651184 (0.1925922906712482) + | > loss_disc_real_2: 0.15201716125011444 (0.21261213406136162) + | > loss_disc_real_3: 0.1351807564496994 (0.2084590180924064) + | > loss_disc_real_4: 0.20785216987133026 (0.20967756525466316) + | > loss_disc_real_5: 0.2026120275259018 (0.22204751717416862) + | > loss_0: 2.27219820022583 (2.32333641303213) + | > grad_norm_0: tensor(14.0998, device='cuda:0') (tensor(21.5530, device='cuda:0')) + | > loss_gen: 2.6084682941436768 (2.5442654082649634) + | > loss_kl: 2.194955587387085 (1.8909566402435303) + | > loss_feat: 8.210930824279785 (7.866330322466399) + | > loss_mel: 17.926429748535156 (16.901632911280583) + | > loss_duration: 0.960690438747406 (1.008096716905895) + | > loss_1: 31.90147590637207 (30.211282127781917) + | > grad_norm_1: tensor(516.1475, device='cuda:0') (tensor(373.0913, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.0025 (1.2205118003644442) + | > loader_time: 0.0062 (0.00846200240285773) + + + --> TIME: 2024-07-13 19:22:19 -- STEP: 69/151 -- GLOBAL_STEP: 89600 + | > loss_disc: 2.2224225997924805 (2.318247484124225) + | > loss_disc_real_0: 0.06385844200849533 (0.12366491556167608) + | > loss_disc_real_1: 0.16972951591014862 (0.20466985240362692) + | > loss_disc_real_2: 0.2131274938583374 (0.21663839499587598) + | > loss_disc_real_3: 0.19463570415973663 (0.20844090524790942) + | > loss_disc_real_4: 0.16815650463104248 (0.20949413854142893) + | > loss_disc_real_5: 0.1817266047000885 (0.2183386022629945) + | > loss_0: 2.2224225997924805 (2.318247484124225) + | > grad_norm_0: tensor(22.0149, device='cuda:0') (tensor(25.2837, device='cuda:0')) + | > loss_gen: 2.6866092681884766 (2.5934330415034634) + | > loss_kl: 1.5794941186904907 (1.8341584862142368) + | > loss_feat: 9.29401969909668 (8.17113520442576) + | > loss_mel: 17.10944175720215 (16.886220240938485) + | > loss_duration: 0.9481290578842163 (0.976911489514337) + | > loss_1: 31.617694854736328 (30.461858445319578) + | > grad_norm_1: tensor(727.3385, device='cuda:0') (tensor(468.5370, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8606 (1.023563025654226) + | > loader_time: 0.0045 (0.00658057046973187) + + + --> TIME: 2024-07-13 19:23:06 -- STEP: 119/151 -- GLOBAL_STEP: 89650 + | > loss_disc: 2.503298044204712 (2.32926053359729) + | > loss_disc_real_0: 0.1756133735179901 (0.12386659514002443) + | > loss_disc_real_1: 0.22512328624725342 (0.204747963853243) + | > loss_disc_real_2: 0.26066702604293823 (0.21739816546690563) + | > loss_disc_real_3: 0.23821501433849335 (0.20864849731701762) + | > loss_disc_real_4: 0.19870145618915558 (0.21010586256239594) + | > loss_disc_real_5: 0.19357813894748688 (0.2189953765448402) + | > loss_0: 2.503298044204712 (2.32926053359729) + | > grad_norm_0: tensor(34.4889, device='cuda:0') (tensor(30.3518, device='cuda:0')) + | > loss_gen: 2.3330373764038086 (2.605959836174461) + | > loss_kl: 1.6564465761184692 (1.8130716215662595) + | > loss_feat: 6.4144487380981445 (8.218795580022475) + | > loss_mel: 15.521875381469727 (17.001586849949934) + | > loss_duration: 0.9238580465316772 (0.9563009478464848) + | > loss_1: 26.849666595458984 (30.595714777457612) + | > grad_norm_1: tensor(739.8528, device='cuda:0') (tensor(514.3400, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9376 (0.9856675432509735) + | > loader_time: 0.0056 (0.0063240007192146876) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032298693290123574 (-0.0006139095012958201) + | > avg_loss_disc: 2.6108893981346717 (+0.045450687408447266) + | > avg_loss_disc_real_0: 0.33110976219177246 (+0.09502347501424643) + | > avg_loss_disc_real_1: 0.2410453466268686 (+0.03567489179281086) + | > avg_loss_disc_real_2: 0.24606425487078154 (+0.0026557456988554884) + | > avg_loss_disc_real_3: 0.2433950167435866 (-0.015569040408501322) + | > avg_loss_disc_real_4: 0.2741959530573625 (+0.010523316951898454) + | > avg_loss_disc_real_5: 0.21955590064709002 (-0.015752565402251034) + | > avg_loss_0: 2.6108893981346717 (+0.045450687408447266) + | > avg_loss_gen: 2.7281852685488186 (+0.23553840930645276) + | > avg_loss_kl: 3.4493025082808275 (+0.0597169582660384) + | > avg_loss_feat: 8.027072392977201 (+0.39466311381413455) + | > avg_loss_mel: 17.246128228994515 (-0.17311807779165633) + | > avg_loss_duration: 1.3204090503545909 (+0.021747103104224674) + | > avg_loss_1: 32.77109732994666 (+0.5385475158691406) + + + > EPOCH: 31/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:23:46)  + + --> TIME: 2024-07-13 19:24:07 -- STEP: 18/151 -- GLOBAL_STEP: 89700 + | > loss_disc: 2.205094337463379 (2.347811844613817) + | > loss_disc_real_0: 0.06375860422849655 (0.11934849640561475) + | > loss_disc_real_1: 0.19075541198253632 (0.19907227903604507) + | > loss_disc_real_2: 0.16943566501140594 (0.20477568192614448) + | > loss_disc_real_3: 0.16386373341083527 (0.2172576818201277) + | > loss_disc_real_4: 0.23236489295959473 (0.21786855657895407) + | > loss_disc_real_5: 0.1645856350660324 (0.23458634979195064) + | > loss_0: 2.205094337463379 (2.347811844613817) + | > grad_norm_0: tensor(17.4627, device='cuda:0') (tensor(27.6461, device='cuda:0')) + | > loss_gen: 2.894529342651367 (2.628975404633416) + | > loss_kl: 1.1352994441986084 (1.844161695904202) + | > loss_feat: 11.879220008850098 (8.637512233522203) + | > loss_mel: 18.6801700592041 (17.461336453755695) + | > loss_duration: 0.9092919826507568 (0.9528866211573283) + | > loss_1: 35.49850845336914 (31.524872567918564) + | > grad_norm_1: tensor(485.3527, device='cuda:0') (tensor(537.1707, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8485 (0.9694226847754585) + | > loader_time: 0.0044 (0.00554460949367947) + + + --> TIME: 2024-07-13 19:24:56 -- STEP: 68/151 -- GLOBAL_STEP: 89750 + | > loss_disc: 2.1746253967285156 (2.334586294258342) + | > loss_disc_real_0: 0.09456177800893784 (0.12943803787450575) + | > loss_disc_real_1: 0.14467257261276245 (0.20143459079896703) + | > loss_disc_real_2: 0.16869516670703888 (0.2137698161908809) + | > loss_disc_real_3: 0.22289074957370758 (0.20890471755581744) + | > loss_disc_real_4: 0.1767147332429886 (0.2096649399575065) + | > loss_disc_real_5: 0.16675199568271637 (0.22253861519343712) + | > loss_0: 2.1746253967285156 (2.334586294258342) + | > grad_norm_0: tensor(37.5553, device='cuda:0') (tensor(38.4572, device='cuda:0')) + | > loss_gen: 2.757733106613159 (2.6260682379498204) + | > loss_kl: 2.1795449256896973 (1.8422040185507607) + | > loss_feat: 9.567638397216797 (8.30455859969644) + | > loss_mel: 17.524904251098633 (17.187113719827995) + | > loss_duration: 0.9251169562339783 (0.9533630977658665) + | > loss_1: 32.95493698120117 (30.91330763872932) + | > grad_norm_1: tensor(884.6953, device='cuda:0') (tensor(589.5713, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8848 (0.9645940661430359) + | > loader_time: 0.0053 (0.005873571423923268) + + + --> TIME: 2024-07-13 19:25:45 -- STEP: 118/151 -- GLOBAL_STEP: 89800 + | > loss_disc: 2.2758936882019043 (2.3511464959484045) + | > loss_disc_real_0: 0.10014655441045761 (0.12976632428245022) + | > loss_disc_real_1: 0.15619264543056488 (0.2033820414947251) + | > loss_disc_real_2: 0.14594843983650208 (0.21365216178661686) + | > loss_disc_real_3: 0.1935739666223526 (0.20958501757201503) + | > loss_disc_real_4: 0.18344402313232422 (0.21170476394689688) + | > loss_disc_real_5: 0.26926693320274353 (0.22643055792077113) + | > loss_0: 2.2758936882019043 (2.3511464959484045) + | > grad_norm_0: tensor(51.5071, device='cuda:0') (tensor(34.6504, device='cuda:0')) + | > loss_gen: 2.493027687072754 (2.5790453482482394) + | > loss_kl: 1.3682113885879517 (1.8078701097076222) + | > loss_feat: 9.792198181152344 (8.088793160551687) + | > loss_mel: 18.40160369873047 (17.010718919463073) + | > loss_duration: 0.9570415019989014 (0.9557889689833431) + | > loss_1: 33.0120849609375 (30.44221648523363) + | > grad_norm_1: tensor(509.5776, device='cuda:0') (tensor(549.4406, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8186 (0.9665916471158044) + | > loader_time: 0.0039 (0.005999821727558718) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03302823580228365 (+0.0007295425121600779) + | > avg_loss_disc: 2.525087778384869 (-0.0858016197498026) + | > avg_loss_disc_real_0: 0.22118997000730956 (-0.1099197921844629) + | > avg_loss_disc_real_1: 0.1845679753101789 (-0.0564773713166897) + | > avg_loss_disc_real_2: 0.18307157319325668 (-0.06299268167752486) + | > avg_loss_disc_real_3: 0.21469241609940162 (-0.028702600644184978) + | > avg_loss_disc_real_4: 0.22454155179170462 (-0.04965440126565787) + | > avg_loss_disc_real_5: 0.20010261008372673 (-0.019453290563363285) + | > avg_loss_0: 2.525087778384869 (-0.0858016197498026) + | > avg_loss_gen: 2.2806861217205343 (-0.4474991468282843) + | > avg_loss_kl: 3.4341753079341006 (-0.015127200346726877) + | > avg_loss_feat: 7.672718671651987 (-0.35435372132521437) + | > avg_loss_mel: 17.04109940162072 (-0.2050288273737948) + | > avg_loss_duration: 1.2911969835941608 (-0.02921206676043009) + | > avg_loss_1: 31.719876362727238 (-1.0512209672194253) + + + > EPOCH: 32/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:26:27)  + + --> TIME: 2024-07-13 19:26:47 -- STEP: 17/151 -- GLOBAL_STEP: 89850 + | > loss_disc: 2.2569260597229004 (2.331792775322409) + | > loss_disc_real_0: 0.09064267575740814 (0.11846505434197538) + | > loss_disc_real_1: 0.21078820526599884 (0.203567475957029) + | > loss_disc_real_2: 0.1691308170557022 (0.2184404555489035) + | > loss_disc_real_3: 0.2233140468597412 (0.2036279518814648) + | > loss_disc_real_4: 0.24033348262310028 (0.21477202632847955) + | > loss_disc_real_5: 0.18129409849643707 (0.21292847307289348) + | > loss_0: 2.2569260597229004 (2.331792775322409) + | > grad_norm_0: tensor(26.4525, device='cuda:0') (tensor(36.1613, device='cuda:0')) + | > loss_gen: 2.469803810119629 (2.6045691826764275) + | > loss_kl: 1.977312445640564 (1.7132534840527702) + | > loss_feat: 7.896847724914551 (8.165176391601564) + | > loss_mel: 17.290058135986328 (16.9225352231194) + | > loss_duration: 0.8787026405334473 (0.9637675215216244) + | > loss_1: 30.512723922729492 (30.369301403270047) + | > grad_norm_1: tensor(769.5693, device='cuda:0') (tensor(591.5787, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9475 (0.9910662594963523) + | > loader_time: 0.0054 (0.005953676560345818) + + + --> TIME: 2024-07-13 19:27:36 -- STEP: 67/151 -- GLOBAL_STEP: 89900 + | > loss_disc: 2.3401412963867188 (2.3370949474733265) + | > loss_disc_real_0: 0.11193513870239258 (0.12382649424583161) + | > loss_disc_real_1: 0.16112551093101501 (0.1997999506655024) + | > loss_disc_real_2: 0.19725114107131958 (0.21726249336306727) + | > loss_disc_real_3: 0.21391209959983826 (0.2095299007287666) + | > loss_disc_real_4: 0.23615960776805878 (0.21258437233184702) + | > loss_disc_real_5: 0.2244863659143448 (0.21548637547599736) + | > loss_0: 2.3401412963867188 (2.3370949474733265) + | > grad_norm_0: tensor(17.2361, device='cuda:0') (tensor(35.7635, device='cuda:0')) + | > loss_gen: 2.6416754722595215 (2.567290113933051) + | > loss_kl: 1.9288299083709717 (1.7248164450944359) + | > loss_feat: 7.427775859832764 (8.090106209712243) + | > loss_mel: 17.329967498779297 (16.92965739520628) + | > loss_duration: 1.0075972080230713 (0.9587243092593862) + | > loss_1: 30.335845947265625 (30.270594468757288) + | > grad_norm_1: tensor(538.4393, device='cuda:0') (tensor(561.8384, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9844 (0.9756246894153197) + | > loader_time: 0.0071 (0.006073691951694773) + + + --> TIME: 2024-07-13 19:28:27 -- STEP: 117/151 -- GLOBAL_STEP: 89950 + | > loss_disc: 2.3435773849487305 (2.336480370953552) + | > loss_disc_real_0: 0.15474025905132294 (0.12443273043275899) + | > loss_disc_real_1: 0.19545511901378632 (0.20184529884758157) + | > loss_disc_real_2: 0.23510393500328064 (0.2167987275836814) + | > loss_disc_real_3: 0.21599867939949036 (0.2097387038744413) + | > loss_disc_real_4: 0.23859059810638428 (0.21351929594818345) + | > loss_disc_real_5: 0.24778945744037628 (0.21676048814740956) + | > loss_0: 2.3435773849487305 (2.336480370953552) + | > grad_norm_0: tensor(52.9023, device='cuda:0') (tensor(32.8846, device='cuda:0')) + | > loss_gen: 2.5618650913238525 (2.577399329242544) + | > loss_kl: 1.9324462413787842 (1.7476210665499048) + | > loss_feat: 8.60606575012207 (8.151796536567886) + | > loss_mel: 17.614910125732422 (17.002245739993885) + | > loss_duration: 0.9185307621955872 (0.9572218794089097) + | > loss_1: 31.633819580078125 (30.436284595065647) + | > grad_norm_1: tensor(655.2715, device='cuda:0') (tensor(550.1974, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9432 (0.9917433119227743) + | > loader_time: 0.006 (0.006302236491798336) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033567245189960196 (+0.0005390093876765437) + | > avg_loss_disc: 2.6215002353374777 (+0.09641245695260858) + | > avg_loss_disc_real_0: 0.24825620995118072 (+0.027066239943871168) + | > avg_loss_disc_real_1: 0.2332393893828759 (+0.048671414072696983) + | > avg_loss_disc_real_2: 0.22384105966641352 (+0.04076948647315684) + | > avg_loss_disc_real_3: 0.21994531498505518 (+0.005252898885653556) + | > avg_loss_disc_real_4: 0.23116483711279356 (+0.00662328532108894) + | > avg_loss_disc_real_5: 0.2934424647918114 (+0.09333985470808467) + | > avg_loss_0: 2.6215002353374777 (+0.09641245695260858) + | > avg_loss_gen: 2.3816876228039083 (+0.10100150108337402) + | > avg_loss_kl: 3.2717892206632175 (-0.1623860872708831) + | > avg_loss_feat: 7.39589368380033 (-0.2768249878516569) + | > avg_loss_mel: 16.444444069495567 (-0.5966553321251524) + | > avg_loss_duration: 1.2735939759474535 (-0.017603007646707303) + | > avg_loss_1: 30.767408517690804 (-0.9524678450364341) + + + > EPOCH: 33/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:29:10)  + + --> TIME: 2024-07-13 19:29:28 -- STEP: 16/151 -- GLOBAL_STEP: 90000 + | > loss_disc: 2.33536696434021 (2.3529329895973206) + | > loss_disc_real_0: 0.1287764459848404 (0.11875634174793959) + | > loss_disc_real_1: 0.20463724434375763 (0.20834503043442965) + | > loss_disc_real_2: 0.2134270817041397 (0.21775508858263493) + | > loss_disc_real_3: 0.17061041295528412 (0.20804663840681314) + | > loss_disc_real_4: 0.23818805813789368 (0.20972646586596966) + | > loss_disc_real_5: 0.22405055165290833 (0.22887746058404446) + | > loss_0: 2.33536696434021 (2.3529329895973206) + | > grad_norm_0: tensor(27.9232, device='cuda:0') (tensor(33.0958, device='cuda:0')) + | > loss_gen: 2.5932538509368896 (2.5535872429609294) + | > loss_kl: 1.6786442995071411 (1.8103372603654861) + | > loss_feat: 7.7180047035217285 (7.773738861083984) + | > loss_mel: 17.16191291809082 (16.947832524776462) + | > loss_duration: 1.0091828107833862 (0.9705919809639454) + | > loss_1: 30.160999298095703 (30.056087851524353) + | > grad_norm_1: tensor(367.4320, device='cuda:0') (tensor(577.9500, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9704 (0.9887115359306335) + | > loader_time: 0.0052 (0.0062647610902786255) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_90000.pth + + --> TIME: 2024-07-13 19:30:43 -- STEP: 66/151 -- GLOBAL_STEP: 90050 + | > loss_disc: 2.16387677192688 (2.3070029345425693) + | > loss_disc_real_0: 0.14198701083660126 (0.11849977725157232) + | > loss_disc_real_1: 0.23127128183841705 (0.20207411468480574) + | > loss_disc_real_2: 0.22595325112342834 (0.21216306225820022) + | > loss_disc_real_3: 0.16603931784629822 (0.20603898935245746) + | > loss_disc_real_4: 0.194981187582016 (0.20856474926977447) + | > loss_disc_real_5: 0.195952907204628 (0.21841193667866968) + | > loss_0: 2.16387677192688 (2.3070029345425693) + | > grad_norm_0: tensor(46.8832, device='cuda:0') (tensor(32.8770, device='cuda:0')) + | > loss_gen: 2.792217969894409 (2.598862431266091) + | > loss_kl: 1.5665611028671265 (1.7711456607688556) + | > loss_feat: 9.003936767578125 (8.150001533103712) + | > loss_mel: 16.203493118286133 (16.936562220255542) + | > loss_duration: 0.9328306913375854 (0.958911699779106) + | > loss_1: 30.499038696289062 (30.415483445832223) + | > grad_norm_1: tensor(564.9263, device='cuda:0') (tensor(597.0289, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8359 (0.9860335732951309) + | > loader_time: 0.004 (0.006010651588439941) + + + --> TIME: 2024-07-13 19:31:33 -- STEP: 116/151 -- GLOBAL_STEP: 90100 + | > loss_disc: 2.4162328243255615 (2.3155455938700964) + | > loss_disc_real_0: 0.14183276891708374 (0.12272297179904477) + | > loss_disc_real_1: 0.2514411211013794 (0.20137940636225812) + | > loss_disc_real_2: 0.19678552448749542 (0.21264734466014237) + | > loss_disc_real_3: 0.22021685540676117 (0.20747721053916832) + | > loss_disc_real_4: 0.179127499461174 (0.20814189957133655) + | > loss_disc_real_5: 0.24240675568580627 (0.21980273466685724) + | > loss_0: 2.4162328243255615 (2.3155455938700964) + | > grad_norm_0: tensor(19.4400, device='cuda:0') (tensor(30.1070, device='cuda:0')) + | > loss_gen: 2.293659210205078 (2.577270236508598) + | > loss_kl: 1.896979570388794 (1.7875609629113098) + | > loss_feat: 7.620588779449463 (8.121063100880587) + | > loss_mel: 16.820520401000977 (16.88171173786295) + | > loss_duration: 0.9579761028289795 (0.9659585228254055) + | > loss_1: 29.589723587036133 (30.33356451166087) + | > grad_norm_1: tensor(132.6326, device='cuda:0') (tensor(512.1117, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.893 (0.9822182840314405) + | > loader_time: 0.0067 (0.00623089075088501) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031415205735426686 (-0.0021520394545335095) + | > avg_loss_disc: 2.5487543436197133 (-0.07274589171776435) + | > avg_loss_disc_real_0: 0.25692236652741063 (+0.008666156576229905) + | > avg_loss_disc_real_1: 0.21954209758685186 (-0.013697291796024025) + | > avg_loss_disc_real_2: 0.21238837448450235 (-0.01145268518191117) + | > avg_loss_disc_real_3: 0.22619663293545061 (+0.006251317950395435) + | > avg_loss_disc_real_4: 0.25708856490942156 (+0.025923727796627993) + | > avg_loss_disc_real_5: 0.21826604581796205 (-0.07517641897384936) + | > avg_loss_0: 2.5487543436197133 (-0.07274589171776435) + | > avg_loss_gen: 2.5017035557673526 (+0.12001593296344426) + | > avg_loss_kl: 3.317220082649818 (+0.0454308619866004) + | > avg_loss_feat: 7.66993001791147 (+0.27403633411114026) + | > avg_loss_mel: 17.334280014038086 (+0.8898359445425186) + | > avg_loss_duration: 1.2788966802450328 (+0.005302704297579286) + | > avg_loss_1: 32.10203024057242 (+1.3346217228816144) + + + > EPOCH: 34/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:32:16)  + + --> TIME: 2024-07-13 19:32:32 -- STEP: 15/151 -- GLOBAL_STEP: 90150 + | > loss_disc: 2.393076181411743 (2.2559250354766847) + | > loss_disc_real_0: 0.1263350546360016 (0.11693528940280279) + | > loss_disc_real_1: 0.20486517250537872 (0.2006272961695989) + | > loss_disc_real_2: 0.23960302770137787 (0.2090474098920822) + | > loss_disc_real_3: 0.22271379828453064 (0.20609335203965504) + | > loss_disc_real_4: 0.27716049551963806 (0.21046718955039978) + | > loss_disc_real_5: 0.20264017581939697 (0.20913123985131582) + | > loss_0: 2.393076181411743 (2.2559250354766847) + | > grad_norm_0: tensor(29.1026, device='cuda:0') (tensor(31.9123, device='cuda:0')) + | > loss_gen: 2.5509116649627686 (2.667569637298584) + | > loss_kl: 1.8820821046829224 (1.7345863819122314) + | > loss_feat: 7.45930814743042 (8.562621688842773) + | > loss_mel: 16.594608306884766 (17.15527598063151) + | > loss_duration: 0.9088926911354065 (0.9551591356595357) + | > loss_1: 29.395803451538086 (31.07521260579427) + | > grad_norm_1: tensor(606.3742, device='cuda:0') (tensor(562.1517, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.0022 (0.892590316136678) + | > loader_time: 0.0058 (0.005327606201171875) + + + --> TIME: 2024-07-13 19:33:21 -- STEP: 65/151 -- GLOBAL_STEP: 90200 + | > loss_disc: 2.3928723335266113 (2.345022168526283) + | > loss_disc_real_0: 0.157667338848114 (0.12903862257416435) + | > loss_disc_real_1: 0.1759759783744812 (0.2023765279696538) + | > loss_disc_real_2: 0.2437637895345688 (0.2129554406954692) + | > loss_disc_real_3: 0.19580689072608948 (0.21101546952357658) + | > loss_disc_real_4: 0.1272101104259491 (0.21166275785519525) + | > loss_disc_real_5: 0.26601994037628174 (0.22332065953658178) + | > loss_0: 2.3928723335266113 (2.345022168526283) + | > grad_norm_0: tensor(47.4581, device='cuda:0') (tensor(37.0995, device='cuda:0')) + | > loss_gen: 2.518892765045166 (2.615450092462393) + | > loss_kl: 1.7785228490829468 (1.7971752854493948) + | > loss_feat: 8.304346084594727 (8.19495349297157) + | > loss_mel: 18.086227416992188 (17.176666010343112) + | > loss_duration: 0.9496583938598633 (0.9531786707731393) + | > loss_1: 31.637645721435547 (30.737423412616437) + | > grad_norm_1: tensor(293.5503, device='cuda:0') (tensor(521.1945, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8836 (0.9456610166109525) + | > loader_time: 0.0065 (0.006025761824387771) + + + --> TIME: 2024-07-13 19:34:09 -- STEP: 115/151 -- GLOBAL_STEP: 90250 + | > loss_disc: 2.2477941513061523 (2.335466550744099) + | > loss_disc_real_0: 0.10358957946300507 (0.12873283125784094) + | > loss_disc_real_1: 0.21828453242778778 (0.2030605615481086) + | > loss_disc_real_2: 0.25266724824905396 (0.2136704017934592) + | > loss_disc_real_3: 0.2291504144668579 (0.20803926587104798) + | > loss_disc_real_4: 0.1945657730102539 (0.2109555800323901) + | > loss_disc_real_5: 0.30772140622138977 (0.22179217843905738) + | > loss_0: 2.2477941513061523 (2.335466550744099) + | > grad_norm_0: tensor(35.6315, device='cuda:0') (tensor(36.1890, device='cuda:0')) + | > loss_gen: 3.002779483795166 (2.598253934279732) + | > loss_kl: 1.670241355895996 (1.7752047118933305) + | > loss_feat: 9.997637748718262 (8.14422856206479) + | > loss_mel: 18.35670280456543 (16.989736059437636) + | > loss_duration: 0.9176420569419861 (0.9564404186995134) + | > loss_1: 33.945003509521484 (30.46386367134426) + | > grad_norm_1: tensor(829.3920, device='cuda:0') (tensor(519.9073, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.87 (0.9534677443297013) + | > loader_time: 0.0047 (0.006004752283510955) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032138934502234824 (+0.0007237287668081377) + | > avg_loss_disc: 2.55479935499338 (+0.006045011373666753) + | > avg_loss_disc_real_0: 0.1963519946886943 (-0.06057037183871633) + | > avg_loss_disc_real_1: 0.23675668354217821 (+0.01721458595532635) + | > avg_loss_disc_real_2: 0.20564673726375288 (-0.006741637220749469) + | > avg_loss_disc_real_3: 0.2229991486439338 (-0.003197484291516811) + | > avg_loss_disc_real_4: 0.2632844471014463 (+0.006195882192024715) + | > avg_loss_disc_real_5: 0.25111817167355466 (+0.03285212585559261) + | > avg_loss_0: 2.55479935499338 (+0.006045011373666753) + | > avg_loss_gen: 2.4143692346719594 (-0.08733432109539319) + | > avg_loss_kl: 3.4746210208305945 (+0.1574009381807766) + | > avg_loss_feat: 7.370398227985088 (-0.2995317899263821) + | > avg_loss_mel: 16.62247980557955 (-0.7118002084585342) + | > avg_loss_duration: 1.2671565551024218 (-0.011740125142611024) + | > avg_loss_1: 31.149024816659782 (-0.9530054239126358) + + + > EPOCH: 35/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:34:55)  + + --> TIME: 2024-07-13 19:35:10 -- STEP: 14/151 -- GLOBAL_STEP: 90300 + | > loss_disc: 2.2172789573669434 (2.2696628740855624) + | > loss_disc_real_0: 0.11699997633695602 (0.11744343010442597) + | > loss_disc_real_1: 0.22863899171352386 (0.20217702431338175) + | > loss_disc_real_2: 0.21812394261360168 (0.20751289597579411) + | > loss_disc_real_3: 0.20385481417179108 (0.1939767184002059) + | > loss_disc_real_4: 0.17126667499542236 (0.19668121955224446) + | > loss_disc_real_5: 0.202492356300354 (0.22235192145620072) + | > loss_0: 2.2172789573669434 (2.2696628740855624) + | > grad_norm_0: tensor(13.4079, device='cuda:0') (tensor(27.6336, device='cuda:0')) + | > loss_gen: 2.67525577545166 (2.587056943348476) + | > loss_kl: 1.5446553230285645 (1.7341016956738062) + | > loss_feat: 7.907870292663574 (8.186942236764091) + | > loss_mel: 15.680695533752441 (16.908600670950754) + | > loss_duration: 0.8737981915473938 (0.9729700684547424) + | > loss_1: 28.682273864746094 (30.38967173440116) + | > grad_norm_1: tensor(300.6791, device='cuda:0') (tensor(492.8552, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8687 (0.9544819252831596) + | > loader_time: 0.0051 (0.00581582954951695) + + + --> TIME: 2024-07-13 19:36:00 -- STEP: 64/151 -- GLOBAL_STEP: 90350 + | > loss_disc: 2.3509981632232666 (2.3291184678673744) + | > loss_disc_real_0: 0.07920420169830322 (0.12588091229554266) + | > loss_disc_real_1: 0.2087460458278656 (0.20336281415075064) + | > loss_disc_real_2: 0.2385258972644806 (0.21388492570258677) + | > loss_disc_real_3: 0.20419535040855408 (0.20864421222358942) + | > loss_disc_real_4: 0.17751970887184143 (0.21039995294995606) + | > loss_disc_real_5: 0.26673462986946106 (0.22103399876505136) + | > loss_0: 2.3509981632232666 (2.3291184678673744) + | > grad_norm_0: tensor(11.1074, device='cuda:0') (tensor(29.5666, device='cuda:0')) + | > loss_gen: 2.476600408554077 (2.5671416446566586) + | > loss_kl: 1.9153918027877808 (1.7799355760216713) + | > loss_feat: 8.011824607849121 (8.012927494943142) + | > loss_mel: 17.608104705810547 (16.91649712622165) + | > loss_duration: 0.9168097376823425 (0.9613387966528535) + | > loss_1: 30.928730010986328 (30.237840712070465) + | > grad_norm_1: tensor(466.6915, device='cuda:0') (tensor(522.0565, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.0132 (0.9812351539731026) + | > loader_time: 0.0074 (0.006329234689474106) + + + --> TIME: 2024-07-13 19:36:50 -- STEP: 114/151 -- GLOBAL_STEP: 90400 + | > loss_disc: 2.2963051795959473 (2.31937271252013) + | > loss_disc_real_0: 0.12076824903488159 (0.12295544850068131) + | > loss_disc_real_1: 0.18925420939922333 (0.20338512421177143) + | > loss_disc_real_2: 0.19902460277080536 (0.21322774717159437) + | > loss_disc_real_3: 0.2634572982788086 (0.20914222064771149) + | > loss_disc_real_4: 0.2506953775882721 (0.2081697384516398) + | > loss_disc_real_5: 0.22545917332172394 (0.2198791552269668) + | > loss_0: 2.2963051795959473 (2.31937271252013) + | > grad_norm_0: tensor(47.9743, device='cuda:0') (tensor(31.6379, device='cuda:0')) + | > loss_gen: 2.6575703620910645 (2.5768961634552263) + | > loss_kl: 1.5768121480941772 (1.7400986285586106) + | > loss_feat: 7.4601006507873535 (8.022734512362565) + | > loss_mel: 15.57885456085205 (16.818199751669905) + | > loss_duration: 0.8634122014045715 (0.9584598912481677) + | > loss_1: 28.136751174926758 (30.116389073823626) + | > grad_norm_1: tensor(417.8548, device='cuda:0') (tensor(533.0908, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8837 (0.9797713150057876) + | > loader_time: 0.0049 (0.006252435215732507) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030882138472336992 (-0.0012567960298978317) + | > avg_loss_disc: 2.5469381625835714 (-0.007861192409808648) + | > avg_loss_disc_real_0: 0.20233799402530378 (+0.005985999336609482) + | > avg_loss_disc_real_1: 0.21335868078928727 (-0.023398002752890945) + | > avg_loss_disc_real_2: 0.22949067789774674 (+0.023843940633993865) + | > avg_loss_disc_real_3: 0.23319556162907526 (+0.010196412985141456) + | > avg_loss_disc_real_4: 0.23067255088916191 (-0.03261189621228436) + | > avg_loss_disc_real_5: 0.21175856773669904 (-0.039359603936855614) + | > avg_loss_0: 2.5469381625835714 (-0.007861192409808648) + | > avg_loss_gen: 2.3140881795149584 (-0.10028105515700103) + | > avg_loss_kl: 3.269211273926955 (-0.20540974690363933) + | > avg_loss_feat: 7.840267218076265 (+0.469868990091177) + | > avg_loss_mel: 16.689524283775917 (+0.06704447819636528) + | > avg_loss_duration: 1.30132873241718 (+0.0341721773147583) + | > avg_loss_1: 31.41441961435171 (+0.2653947976919291) + + + > EPOCH: 36/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:37:36)  + + --> TIME: 2024-07-13 19:37:52 -- STEP: 13/151 -- GLOBAL_STEP: 90450 + | > loss_disc: 2.3879923820495605 (2.3265395164489746) + | > loss_disc_real_0: 0.0947892814874649 (0.1204283289038218) + | > loss_disc_real_1: 0.21355779469013214 (0.20702076302124903) + | > loss_disc_real_2: 0.20319710671901703 (0.21397757759461036) + | > loss_disc_real_3: 0.23410587012767792 (0.20854145403091723) + | > loss_disc_real_4: 0.20417359471321106 (0.20672475260037643) + | > loss_disc_real_5: 0.17135192453861237 (0.21536515997006342) + | > loss_0: 2.3879923820495605 (2.3265395164489746) + | > grad_norm_0: tensor(31.1148, device='cuda:0') (tensor(25.5587, device='cuda:0')) + | > loss_gen: 2.3512589931488037 (2.5154154300689697) + | > loss_kl: 1.76881742477417 (1.8114142876404982) + | > loss_feat: 6.729456901550293 (7.740239455149724) + | > loss_mel: 14.619063377380371 (16.19587854238657) + | > loss_duration: 0.9298591613769531 (0.9569819844686068) + | > loss_1: 26.398454666137695 (29.219929915208084) + | > grad_norm_1: tensor(556.5389, device='cuda:0') (tensor(535.5561, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8446 (1.0350085588601918) + | > loader_time: 0.0034 (0.0062899039341853214) + + + --> TIME: 2024-07-13 19:38:39 -- STEP: 63/151 -- GLOBAL_STEP: 90500 + | > loss_disc: 2.424731731414795 (2.340205745091514) + | > loss_disc_real_0: 0.10770361870527267 (0.12819328437012345) + | > loss_disc_real_1: 0.2095305472612381 (0.2063626760528201) + | > loss_disc_real_2: 0.17182528972625732 (0.21407609752246312) + | > loss_disc_real_3: 0.22196979820728302 (0.20902244604769207) + | > loss_disc_real_4: 0.16427700221538544 (0.21109647031814333) + | > loss_disc_real_5: 0.22620779275894165 (0.22119398083951738) + | > loss_0: 2.424731731414795 (2.340205745091514) + | > grad_norm_0: tensor(23.9873, device='cuda:0') (tensor(27.4812, device='cuda:0')) + | > loss_gen: 2.209026575088501 (2.55926387650626) + | > loss_kl: 1.7381067276000977 (1.777522313216376) + | > loss_feat: 7.914119243621826 (7.951885416394188) + | > loss_mel: 17.59076690673828 (16.782287143525615) + | > loss_duration: 0.9323500990867615 (0.9510785691321842) + | > loss_1: 30.384368896484375 (30.022037385002015) + | > grad_norm_1: tensor(292.7789, device='cuda:0') (tensor(489.2919, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8976 (0.9483919143676758) + | > loader_time: 0.0063 (0.005761377395145477) + + + --> TIME: 2024-07-13 19:39:29 -- STEP: 113/151 -- GLOBAL_STEP: 90550 + | > loss_disc: 2.3354270458221436 (2.3448176257378233) + | > loss_disc_real_0: 0.1551884412765503 (0.12508330876584609) + | > loss_disc_real_1: 0.21884606778621674 (0.20593899264272333) + | > loss_disc_real_2: 0.19151146709918976 (0.21609579962966716) + | > loss_disc_real_3: 0.15007445216178894 (0.21016346247850265) + | > loss_disc_real_4: 0.21464011073112488 (0.21086764401566666) + | > loss_disc_real_5: 0.1781781017780304 (0.22347127138513379) + | > loss_0: 2.3354270458221436 (2.3448176257378233) + | > grad_norm_0: tensor(36.6003, device='cuda:0') (tensor(29.5232, device='cuda:0')) + | > loss_gen: 2.4459452629089355 (2.570074281861298) + | > loss_kl: 1.3682488203048706 (1.7841717780163857) + | > loss_feat: 8.051012992858887 (8.054858361725254) + | > loss_mel: 17.45652961730957 (16.859948504287594) + | > loss_duration: 0.8038756847381592 (0.9521833368107281) + | > loss_1: 30.125612258911133 (30.221236456811955) + | > grad_norm_1: tensor(948.6300, device='cuda:0') (tensor(530.7185, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8817 (0.9669270093462109) + | > loader_time: 0.0048 (0.006137183282227642) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03391408920288086 (+0.003031950730543867) + | > avg_loss_disc: 2.5772091425382175 (+0.030270979954646116) + | > avg_loss_disc_real_0: 0.20015705892672905 (-0.0021809350985747267) + | > avg_loss_disc_real_1: 0.19059448861158812 (-0.02276419217769915) + | > avg_loss_disc_real_2: 0.2046946516403785 (-0.024796026257368237) + | > avg_loss_disc_real_3: 0.2607611876267653 (+0.027565625997690052) + | > avg_loss_disc_real_4: 0.26599146081851077 (+0.03531890992934886) + | > avg_loss_disc_real_5: 0.24264084146572992 (+0.03088227372903088) + | > avg_loss_0: 2.5772091425382175 (+0.030270979954646116) + | > avg_loss_gen: 2.3158779327686014 (+0.001789753253643056) + | > avg_loss_kl: 3.4391509753007155 (+0.16993970137376024) + | > avg_loss_feat: 7.804312192476713 (-0.035955025599552215) + | > avg_loss_mel: 17.1315399316641 (+0.4420156478881836) + | > avg_loss_duration: 1.2760929144345796 (-0.02523581798260044) + | > avg_loss_1: 31.9669739649846 (+0.5525543506328887) + + + > EPOCH: 37/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:40:16)  + + --> TIME: 2024-07-13 19:40:29 -- STEP: 12/151 -- GLOBAL_STEP: 90600 + | > loss_disc: 2.3461074829101562 (2.367503901322683) + | > loss_disc_real_0: 0.14591318368911743 (0.14494513161480427) + | > loss_disc_real_1: 0.20319510996341705 (0.20443284511566162) + | > loss_disc_real_2: 0.2391882985830307 (0.21269813800851503) + | > loss_disc_real_3: 0.2560667097568512 (0.20456010103225708) + | > loss_disc_real_4: 0.2705727815628052 (0.21710951129595438) + | > loss_disc_real_5: 0.30061542987823486 (0.2179865650832653) + | > loss_0: 2.3461074829101562 (2.367503901322683) + | > grad_norm_0: tensor(19.0844, device='cuda:0') (tensor(29.8977, device='cuda:0')) + | > loss_gen: 2.8169403076171875 (2.5696789026260376) + | > loss_kl: 2.124340295791626 (1.753462682167689) + | > loss_feat: 8.332683563232422 (7.996687412261963) + | > loss_mel: 17.566953659057617 (17.133787949879963) + | > loss_duration: 0.8128140568733215 (0.9438390135765076) + | > loss_1: 31.653732299804688 (30.397456169128418) + | > grad_norm_1: tensor(290.5620, device='cuda:0') (tensor(361.9248, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9864 (0.9065731366475424) + | > loader_time: 0.0067 (0.005283077557881673) + + + --> TIME: 2024-07-13 19:41:21 -- STEP: 62/151 -- GLOBAL_STEP: 90650 + | > loss_disc: 2.3492352962493896 (2.3482400717273832) + | > loss_disc_real_0: 0.0920417532324791 (0.1336240466804274) + | > loss_disc_real_1: 0.20130519568920135 (0.20088467674870644) + | > loss_disc_real_2: 0.18767887353897095 (0.21087131673289883) + | > loss_disc_real_3: 0.21722844243049622 (0.2066889672029403) + | > loss_disc_real_4: 0.17222477495670319 (0.20816793280743784) + | > loss_disc_real_5: 0.19490814208984375 (0.22584295993851078) + | > loss_0: 2.3492352962493896 (2.3482400717273832) + | > grad_norm_0: tensor(12.2701, device='cuda:0') (tensor(32.9340, device='cuda:0')) + | > loss_gen: 2.543515682220459 (2.5689979522458968) + | > loss_kl: 1.3802233934402466 (1.8685043204215266) + | > loss_feat: 8.935667037963867 (8.089582989292763) + | > loss_mel: 17.002975463867188 (17.01841820439984) + | > loss_duration: 0.8940701484680176 (0.9595579793376308) + | > loss_1: 30.756452560424805 (30.50506139570667) + | > grad_norm_1: tensor(330.9850, device='cuda:0') (tensor(467.9425, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8373 (1.0060864110146797) + | > loader_time: 0.0042 (0.0064429736906482325) + + + --> TIME: 2024-07-13 19:42:08 -- STEP: 112/151 -- GLOBAL_STEP: 90700 + | > loss_disc: 2.2169299125671387 (2.3408626232828404) + | > loss_disc_real_0: 0.12137232720851898 (0.1300062195264868) + | > loss_disc_real_1: 0.2215835452079773 (0.20248259803546326) + | > loss_disc_real_2: 0.20784668624401093 (0.21254083900047202) + | > loss_disc_real_3: 0.1957632154226303 (0.2075024309701153) + | > loss_disc_real_4: 0.15844683349132538 (0.2078876492035176) + | > loss_disc_real_5: 0.199063241481781 (0.22356579253183945) + | > loss_0: 2.2169299125671387 (2.3408626232828404) + | > grad_norm_0: tensor(12.1114, device='cuda:0') (tensor(35.8459, device='cuda:0')) + | > loss_gen: 2.744266986846924 (2.580603525042534) + | > loss_kl: 1.4911882877349854 (1.7830893652779716) + | > loss_feat: 9.249225616455078 (8.10518006767546) + | > loss_mel: 18.037376403808594 (16.95871266296933) + | > loss_duration: 0.9303438067436218 (0.9602670350245067) + | > loss_1: 32.45240020751953 (30.387852481433324) + | > grad_norm_1: tensor(237.8660, device='cuda:0') (tensor(503.3897, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.868 (0.9729676395654678) + | > loader_time: 0.0059 (0.006163958992276873) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03359176562382624 (-0.0003223235790546161) + | > avg_loss_disc: 2.5061906484457164 (-0.07101849409250116) + | > avg_loss_disc_real_0: 0.23122943479281205 (+0.031072375866082996) + | > avg_loss_disc_real_1: 0.22220320197252127 (+0.031608713360933155) + | > avg_loss_disc_real_2: 0.23376074204078087 (+0.029066090400402367) + | > avg_loss_disc_real_3: 0.22810891843759096 (-0.032652269189174354) + | > avg_loss_disc_real_4: 0.2667535749765543 (+0.0007621141580435031) + | > avg_loss_disc_real_5: 0.2150946603371547 (-0.027546181128575237) + | > avg_loss_0: 2.5061906484457164 (-0.07101849409250116) + | > avg_loss_gen: 2.495522535764254 (+0.17964460299565266) + | > avg_loss_kl: 3.341605828358577 (-0.09754514694213867) + | > avg_loss_feat: 7.836008328657884 (+0.03169613618117051) + | > avg_loss_mel: 17.50112196115347 (+0.3695820294893686) + | > avg_loss_duration: 1.2929549308923574 (+0.01686201645777774) + | > avg_loss_1: 32.467213557316704 (+0.5002395923321039) + + + > EPOCH: 38/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:42:56)  + + --> TIME: 2024-07-13 19:43:08 -- STEP: 11/151 -- GLOBAL_STEP: 90750 + | > loss_disc: 2.2323546409606934 (2.2931644049557773) + | > loss_disc_real_0: 0.17175358533859253 (0.11745030500672081) + | > loss_disc_real_1: 0.20606938004493713 (0.19317043911327014) + | > loss_disc_real_2: 0.2318354845046997 (0.21324557336893948) + | > loss_disc_real_3: 0.2113601416349411 (0.20678182352672925) + | > loss_disc_real_4: 0.20382612943649292 (0.20607804574749686) + | > loss_disc_real_5: 0.20524749159812927 (0.21942530843344601) + | > loss_0: 2.2323546409606934 (2.2931644049557773) + | > grad_norm_0: tensor(61.9514, device='cuda:0') (tensor(29.3915, device='cuda:0')) + | > loss_gen: 2.8147964477539062 (2.5866765759207984) + | > loss_kl: 2.07368540763855 (1.8182593800804832) + | > loss_feat: 9.429561614990234 (8.199451229789041) + | > loss_mel: 18.494701385498047 (17.205134738575328) + | > loss_duration: 0.9104416370391846 (0.9187795303084634) + | > loss_1: 33.72318649291992 (30.728301828557793) + | > grad_norm_1: tensor(793.1439, device='cuda:0') (tensor(480.5895, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9468 (0.9322368665175005) + | > loader_time: 0.0058 (0.005564754659479315) + + + --> TIME: 2024-07-13 19:43:59 -- STEP: 61/151 -- GLOBAL_STEP: 90800 + | > loss_disc: 2.1476268768310547 (2.3147164407323624) + | > loss_disc_real_0: 0.0804273709654808 (0.12547862755714875) + | > loss_disc_real_1: 0.1296841949224472 (0.1966662841742156) + | > loss_disc_real_2: 0.14666762948036194 (0.20975227165417593) + | > loss_disc_real_3: 0.1479194611310959 (0.20721636004135258) + | > loss_disc_real_4: 0.14413857460021973 (0.20630118592840727) + | > loss_disc_real_5: 0.2344958782196045 (0.2205185489576371) + | > loss_0: 2.1476268768310547 (2.3147164407323624) + | > grad_norm_0: tensor(11.0380, device='cuda:0') (tensor(31.8125, device='cuda:0')) + | > loss_gen: 2.549560546875 (2.591204475183956) + | > loss_kl: 1.6115021705627441 (1.7894660781641476) + | > loss_feat: 8.937592506408691 (8.37790661952535) + | > loss_mel: 17.305747985839844 (17.196958151019984) + | > loss_duration: 0.94711834192276 (0.9443946695718609) + | > loss_1: 31.35152244567871 (30.899930453691326) + | > grad_norm_1: tensor(173.6676, device='cuda:0') (tensor(501.3556, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8327 (0.9885932773840229) + | > loader_time: 0.0042 (0.006130062165807505) + + + --> TIME: 2024-07-13 19:44:48 -- STEP: 111/151 -- GLOBAL_STEP: 90850 + | > loss_disc: 2.5879971981048584 (2.3286183997317487) + | > loss_disc_real_0: 0.24423430860042572 (0.1261178445708645) + | > loss_disc_real_1: 0.23036153614521027 (0.19984104370211694) + | > loss_disc_real_2: 0.28140923380851746 (0.21284492622624646) + | > loss_disc_real_3: 0.24381829798221588 (0.20896441798220883) + | > loss_disc_real_4: 0.3062712550163269 (0.2088320347908381) + | > loss_disc_real_5: 0.2698342800140381 (0.22044081301302523) + | > loss_0: 2.5879971981048584 (2.3286183997317487) + | > grad_norm_0: tensor(19.8003, device='cuda:0') (tensor(30.8560, device='cuda:0')) + | > loss_gen: 2.5171773433685303 (2.590851124342498) + | > loss_kl: 2.0705924034118652 (1.7933765821628742) + | > loss_feat: 5.702125549316406 (8.290972589372519) + | > loss_mel: 14.940643310546875 (17.152117608903755) + | > loss_duration: 1.1786646842956543 (0.9539666890024064) + | > loss_1: 26.409204483032227 (30.781284950874948) + | > grad_norm_1: tensor(254.4121, device='cuda:0') (tensor(488.9726, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 2.0927 (0.9825899235837093) + | > loader_time: 0.015 (0.006269070479246949) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03087379382206843 (-0.0027179718017578125) + | > avg_loss_disc: 2.534622540840736 (+0.02843189239501953) + | > avg_loss_disc_real_0: 0.21698947136218733 (-0.014239963430624725) + | > avg_loss_disc_real_1: 0.21673789161902207 (-0.005465310353499203) + | > avg_loss_disc_real_2: 0.2525297930607429 (+0.01876905101996204) + | > avg_loss_disc_real_3: 0.24057582250008216 (+0.012466904062491208) + | > avg_loss_disc_real_4: 0.2693208066316751 (+0.0025672316551208496) + | > avg_loss_disc_real_5: 0.24652105569839478 (+0.03142639536124009) + | > avg_loss_0: 2.534622540840736 (+0.02843189239501953) + | > avg_loss_gen: 2.4951148950136623 (-0.0004076407505917423) + | > avg_loss_kl: 3.3747034623072696 (+0.033097633948692806) + | > avg_loss_feat: 7.781021631681002 (-0.054986696976881966) + | > avg_loss_mel: 17.02312733576848 (-0.4779946253849907) + | > avg_loss_duration: 1.3274500278326182 (+0.03449509694026087) + | > avg_loss_1: 32.00141774691068 (-0.46579581040602136) + + + > EPOCH: 39/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:45:36)  + + --> TIME: 2024-07-13 19:45:47 -- STEP: 10/151 -- GLOBAL_STEP: 90900 + | > loss_disc: 2.0953733921051025 (2.3199864864349364) + | > loss_disc_real_0: 0.048283662647008896 (0.11792410202324391) + | > loss_disc_real_1: 0.1826280951499939 (0.20155064016580582) + | > loss_disc_real_2: 0.2022833675146103 (0.21242788136005403) + | > loss_disc_real_3: 0.22471095621585846 (0.21721865832805634) + | > loss_disc_real_4: 0.22084544599056244 (0.22595328092575073) + | > loss_disc_real_5: 0.2023984044790268 (0.22652529627084733) + | > loss_0: 2.0953733921051025 (2.3199864864349364) + | > grad_norm_0: tensor(7.6576, device='cuda:0') (tensor(21.1027, device='cuda:0')) + | > loss_gen: 2.71968936920166 (2.5633243322372437) + | > loss_kl: 1.74617338180542 (1.8703771591186524) + | > loss_feat: 8.859841346740723 (8.05165376663208) + | > loss_mel: 17.08893394470215 (16.809617137908937) + | > loss_duration: 0.9096851348876953 (0.9543775916099548) + | > loss_1: 31.324323654174805 (30.249350166320802) + | > grad_norm_1: tensor(178.9067, device='cuda:0') (tensor(380.2213, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.808 (0.9072500705718994) + | > loader_time: 0.0039 (0.00542900562286377) + + + --> TIME: 2024-07-13 19:46:34 -- STEP: 60/151 -- GLOBAL_STEP: 90950 + | > loss_disc: 2.306903600692749 (2.331056503454843) + | > loss_disc_real_0: 0.14429621398448944 (0.1233423630396525) + | > loss_disc_real_1: 0.19730764627456665 (0.20233204563458765) + | > loss_disc_real_2: 0.20775389671325684 (0.21221801961461703) + | > loss_disc_real_3: 0.20415902137756348 (0.21181778833270073) + | > loss_disc_real_4: 0.20477189123630524 (0.20837910262246925) + | > loss_disc_real_5: 0.20805029571056366 (0.22199015244841575) + | > loss_0: 2.306903600692749 (2.331056503454843) + | > grad_norm_0: tensor(33.9895, device='cuda:0') (tensor(31.3306, device='cuda:0')) + | > loss_gen: 2.608229637145996 (2.590359171231588) + | > loss_kl: 1.7634209394454956 (1.756718173623085) + | > loss_feat: 8.289752960205078 (8.058631157875057) + | > loss_mel: 16.597087860107422 (16.965539868672682) + | > loss_duration: 0.9101684093475342 (0.9637393285830815) + | > loss_1: 30.168659210205078 (30.334987990061443) + | > grad_norm_1: tensor(408.5661, device='cuda:0') (tensor(514.6774, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8417 (0.9346725742022196) + | > loader_time: 0.0043 (0.005740825335184733) + + + --> TIME: 2024-07-13 19:47:25 -- STEP: 110/151 -- GLOBAL_STEP: 91000 + | > loss_disc: 2.4693961143493652 (2.334962086244063) + | > loss_disc_real_0: 0.07093505561351776 (0.12431225072253839) + | > loss_disc_real_1: 0.1743542104959488 (0.2039019900289449) + | > loss_disc_real_2: 0.17822682857513428 (0.21303673454306343) + | > loss_disc_real_3: 0.18309207260608673 (0.21268916658379816) + | > loss_disc_real_4: 0.20722714066505432 (0.21022028929807923) + | > loss_disc_real_5: 0.2425961047410965 (0.22256125184622677) + | > loss_0: 2.4693961143493652 (2.334962086244063) + | > grad_norm_0: tensor(34.6234, device='cuda:0') (tensor(29.6837, device='cuda:0')) + | > loss_gen: 2.4248485565185547 (2.5734672871502973) + | > loss_kl: 1.9549721479415894 (1.7703379533507608) + | > loss_feat: 7.386575698852539 (8.032615665955978) + | > loss_mel: 17.862882614135742 (16.91804648312655) + | > loss_duration: 1.031233549118042 (0.9607834973118522) + | > loss_1: 30.660511016845703 (30.255251121520995) + | > grad_norm_1: tensor(487.7568, device='cuda:0') (tensor(502.0415, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9963 (0.9677675182169134) + | > loader_time: 0.0062 (0.006115635958584872) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03313020559457632 (+0.002256411772507891) + | > avg_loss_disc: 2.5302850833305945 (-0.004337457510141363) + | > avg_loss_disc_real_0: 0.19983405218674585 (-0.017155419175441472) + | > avg_loss_disc_real_1: 0.2578479590324255 (+0.04111006741340345) + | > avg_loss_disc_real_2: 0.2761230296813525 (+0.02359323662060958) + | > avg_loss_disc_real_3: 0.2645553969419919 (+0.023979574441909762) + | > avg_loss_disc_real_4: 0.2610310602646608 (-0.00828974636701435) + | > avg_loss_disc_real_5: 0.27285613921972424 (+0.02633508352132946) + | > avg_loss_0: 2.5302850833305945 (-0.004337457510141363) + | > avg_loss_gen: 2.636064859536978 (+0.14094996452331543) + | > avg_loss_kl: 3.407758896167462 (+0.033055433860192274) + | > avg_loss_feat: 7.463448817913349 (-0.31757281376765256) + | > avg_loss_mel: 16.904299662663387 (-0.11882767310509124) + | > avg_loss_duration: 1.2895989601428692 (-0.037851067689748996) + | > avg_loss_1: 31.701171288123497 (-0.3002464587871856) + + + > EPOCH: 40/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:48:36)  + + --> TIME: 2024-07-13 19:48:48 -- STEP: 9/151 -- GLOBAL_STEP: 91050 + | > loss_disc: 2.2289555072784424 (2.4010970327589245) + | > loss_disc_real_0: 0.08599996566772461 (0.1280581826965014) + | > loss_disc_real_1: 0.21793271601200104 (0.19974486198690203) + | > loss_disc_real_2: 0.20265516638755798 (0.22332545618216196) + | > loss_disc_real_3: 0.18203487992286682 (0.2155519293414222) + | > loss_disc_real_4: 0.14349128305912018 (0.20276913874679142) + | > loss_disc_real_5: 0.23108866810798645 (0.22717307342423332) + | > loss_0: 2.2289555072784424 (2.4010970327589245) + | > grad_norm_0: tensor(28.8780, device='cuda:0') (tensor(34.3930, device='cuda:0')) + | > loss_gen: 2.6334242820739746 (2.483080784479777) + | > loss_kl: 1.046122670173645 (1.7465030484729342) + | > loss_feat: 8.222908020019531 (7.467670016818577) + | > loss_mel: 16.391157150268555 (16.730709393819172) + | > loss_duration: 0.9300617575645447 (0.9831484953562418) + | > loss_1: 29.22367286682129 (29.41111119588216) + | > grad_norm_1: tensor(157.8756, device='cuda:0') (tensor(492.7487, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8408 (1.0800426006317139) + | > loader_time: 0.0046 (0.0069908565945095485) + + + --> TIME: 2024-07-13 19:49:37 -- STEP: 59/151 -- GLOBAL_STEP: 91100 + | > loss_disc: 2.589052677154541 (2.371195445626469) + | > loss_disc_real_0: 0.14259889721870422 (0.12679278048670897) + | > loss_disc_real_1: 0.20630834996700287 (0.20905625971697145) + | > loss_disc_real_2: 0.21750621497631073 (0.21602984277878778) + | > loss_disc_real_3: 0.20065513253211975 (0.21396184876813726) + | > loss_disc_real_4: 0.2539995312690735 (0.21487847248376427) + | > loss_disc_real_5: 0.23594944179058075 (0.2197752221155975) + | > loss_0: 2.589052677154541 (2.371195445626469) + | > grad_norm_0: tensor(22.7983, device='cuda:0') (tensor(30.9046, device='cuda:0')) + | > loss_gen: 2.1656734943389893 (2.54016727916265) + | > loss_kl: 1.5601307153701782 (1.7111668243246563) + | > loss_feat: 6.234750270843506 (7.796065193111613) + | > loss_mel: 16.609224319458008 (16.697188296560515) + | > loss_duration: 0.9518716931343079 (0.9676234186705897) + | > loss_1: 27.521652221679688 (29.712210703704315) + | > grad_norm_1: tensor(512.0398, device='cuda:0') (tensor(478.6484, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9007 (0.9916204072661314) + | > loader_time: 0.0059 (0.00617765976210772) + + + --> TIME: 2024-07-13 19:50:25 -- STEP: 109/151 -- GLOBAL_STEP: 91150 + | > loss_disc: 2.323190212249756 (2.345213262313002) + | > loss_disc_real_0: 0.13864299654960632 (0.12767417168398518) + | > loss_disc_real_1: 0.2507224380970001 (0.2055390363712923) + | > loss_disc_real_2: 0.24109908938407898 (0.21359460411268635) + | > loss_disc_real_3: 0.21397008001804352 (0.20976659559875455) + | > loss_disc_real_4: 0.23034624755382538 (0.21061215734263078) + | > loss_disc_real_5: 0.22626496851444244 (0.22110356117060426) + | > loss_0: 2.323190212249756 (2.345213262313002) + | > grad_norm_0: tensor(15.7684, device='cuda:0') (tensor(31.6884, device='cuda:0')) + | > loss_gen: 2.4597606658935547 (2.5682848768496744) + | > loss_kl: 1.914601445198059 (1.746991568749104) + | > loss_feat: 7.869426250457764 (8.067739285460306) + | > loss_mel: 17.913955688476562 (16.8207433770556) + | > loss_duration: 0.9942159652709961 (0.951715627394685) + | > loss_1: 31.151958465576172 (30.155474601535623) + | > grad_norm_1: tensor(368.6118, device='cuda:0') (tensor(494.4368, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.0879 (0.9675598735109379) + | > loader_time: 0.0076 (0.0060796343952143955) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032918306497427136 (-0.00021189909714918603) + | > avg_loss_disc: 2.6081714630126953 (+0.07788637968210077) + | > avg_loss_disc_real_0: 0.2668280658813623 (+0.06699401369461647) + | > avg_loss_disc_real_1: 0.20099123395406282 (-0.0568567250783627) + | > avg_loss_disc_real_2: 0.2594338678396665 (-0.016689161841686007) + | > avg_loss_disc_real_3: 0.23179917266735664 (-0.03275622427463529) + | > avg_loss_disc_real_4: 0.2564568313268515 (-0.004574228937809288) + | > avg_loss_disc_real_5: 0.26323290398487675 (-0.009623235234847483) + | > avg_loss_0: 2.6081714630126953 (+0.07788637968210077) + | > avg_loss_gen: 2.523024008824275 (-0.11304085071270276) + | > avg_loss_kl: 3.4759469765883226 (+0.06818808042086077) + | > avg_loss_feat: 7.728931316962609 (+0.26548249904926013) + | > avg_loss_mel: 17.00594579256498 (+0.10164612990159227) + | > avg_loss_duration: 1.2684699709598835 (-0.02112898918298578) + | > avg_loss_1: 32.00231801546537 (+0.3011467273418731) + + + > EPOCH: 41/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:51:17)  + + --> TIME: 2024-07-13 19:51:28 -- STEP: 8/151 -- GLOBAL_STEP: 91200 + | > loss_disc: 2.311424732208252 (2.3549682796001434) + | > loss_disc_real_0: 0.16023997962474823 (0.13750406634062529) + | > loss_disc_real_1: 0.21675942838191986 (0.21507045440375805) + | > loss_disc_real_2: 0.23043708503246307 (0.21297992952167988) + | > loss_disc_real_3: 0.19527973234653473 (0.2158798035234213) + | > loss_disc_real_4: 0.21062441170215607 (0.21608231216669083) + | > loss_disc_real_5: 0.2022870033979416 (0.23226745426654816) + | > loss_0: 2.311424732208252 (2.3549682796001434) + | > grad_norm_0: tensor(78.6608, device='cuda:0') (tensor(39.2108, device='cuda:0')) + | > loss_gen: 2.442389488220215 (2.5467857718467712) + | > loss_kl: 1.9533270597457886 (1.73163902759552) + | > loss_feat: 8.089095115661621 (8.014263212680817) + | > loss_mel: 18.24968910217285 (17.00841224193573) + | > loss_duration: 0.9425774812698364 (0.9510736241936684) + | > loss_1: 31.677078247070312 (30.25217318534851) + | > grad_norm_1: tensor(914.0994, device='cuda:0') (tensor(563.8934, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9766 (1.0422944724559784) + | > loader_time: 0.0061 (0.006874740123748779) + + + --> TIME: 2024-07-13 19:52:19 -- STEP: 58/151 -- GLOBAL_STEP: 91250 + | > loss_disc: 2.332066297531128 (2.3379593758747506) + | > loss_disc_real_0: 0.1237441897392273 (0.1259424755157067) + | > loss_disc_real_1: 0.18835057318210602 (0.2064108879401766) + | > loss_disc_real_2: 0.251814067363739 (0.2110345021917902) + | > loss_disc_real_3: 0.21755339205265045 (0.20718411484668994) + | > loss_disc_real_4: 0.25983983278274536 (0.2086706965647895) + | > loss_disc_real_5: 0.20502468943595886 (0.22801100745283323) + | > loss_0: 2.332066297531128 (2.3379593758747506) + | > grad_norm_0: tensor(43.5147, device='cuda:0') (tensor(38.8851, device='cuda:0')) + | > loss_gen: 2.5487430095672607 (2.6050582219814427) + | > loss_kl: 1.6266074180603027 (1.8128406898728733) + | > loss_feat: 7.820608139038086 (8.319117735172144) + | > loss_mel: 16.870471954345703 (17.116136830428548) + | > loss_duration: 0.8311244249343872 (0.9689043873342974) + | > loss_1: 29.697555541992188 (30.822057822654987) + | > grad_norm_1: tensor(735.0684, device='cuda:0') (tensor(535.3375, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8532 (1.0089903658833999) + | > loader_time: 0.0043 (0.006414553214763773) + + + --> TIME: 2024-07-13 19:53:07 -- STEP: 108/151 -- GLOBAL_STEP: 91300 + | > loss_disc: 2.48026180267334 (2.346347062676042) + | > loss_disc_real_0: 0.18500405550003052 (0.13119485832888766) + | > loss_disc_real_1: 0.23083555698394775 (0.20606092156635392) + | > loss_disc_real_2: 0.2565489113330841 (0.21121445408573858) + | > loss_disc_real_3: 0.269020676612854 (0.20797301456332207) + | > loss_disc_real_4: 0.23185744881629944 (0.2097868991808759) + | > loss_disc_real_5: 0.2723693251609802 (0.22564171502987543) + | > loss_0: 2.48026180267334 (2.346347062676042) + | > grad_norm_0: tensor(34.9251, device='cuda:0') (tensor(40.2995, device='cuda:0')) + | > loss_gen: 2.787978410720825 (2.5946910204710774) + | > loss_kl: 2.021867036819458 (1.8139339166658897) + | > loss_feat: 7.967855930328369 (8.149022552702162) + | > loss_mel: 16.56173324584961 (17.008039624602702) + | > loss_duration: 0.9107757210731506 (0.9551025579373041) + | > loss_1: 30.250211715698242 (30.520789640921134) + | > grad_norm_1: tensor(504.8013, device='cuda:0') (tensor(555.3426, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.879 (0.9796817589689184) + | > loader_time: 0.0057 (0.006198801376201489) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033605172083928034 (+0.0006868655865008982) + | > avg_loss_disc: 2.5769454882695126 (-0.031225974743182672) + | > avg_loss_disc_real_0: 0.21250413128962883 (-0.054323934591733486) + | > avg_loss_disc_real_1: 0.22794632613658905 (+0.02695509218252623) + | > avg_loss_disc_real_2: 0.2315073964687494 (-0.027926471370917083) + | > avg_loss_disc_real_3: 0.2479749104151359 (+0.01617573774777925) + | > avg_loss_disc_real_4: 0.24359657214238092 (-0.012860259184470563) + | > avg_loss_disc_real_5: 0.2505584256007121 (-0.012674478384164634) + | > avg_loss_0: 2.5769454882695126 (-0.031225974743182672) + | > avg_loss_gen: 2.382434973349938 (-0.1405890354743371) + | > avg_loss_kl: 3.24000307229849 (-0.23594390428983258) + | > avg_loss_feat: 7.650385599869948 (-0.0785457170926609) + | > avg_loss_mel: 16.786695847144493 (-0.21924994542048637) + | > avg_loss_duration: 1.2458767249034004 (-0.02259324605648305) + | > avg_loss_1: 31.305396153376652 (-0.6969218620887183) + + + > EPOCH: 42/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:53:58)  + + --> TIME: 2024-07-13 19:54:07 -- STEP: 7/151 -- GLOBAL_STEP: 91350 + | > loss_disc: 2.313535451889038 (2.3814492225646973) + | > loss_disc_real_0: 0.15404491126537323 (0.11773171808038439) + | > loss_disc_real_1: 0.2339213788509369 (0.21306566255433218) + | > loss_disc_real_2: 0.2695651054382324 (0.2232276052236557) + | > loss_disc_real_3: 0.26251670718193054 (0.20794985124043056) + | > loss_disc_real_4: 0.1814917027950287 (0.22920943796634674) + | > loss_disc_real_5: 0.26851925253868103 (0.2233571857213974) + | > loss_0: 2.313535451889038 (2.3814492225646973) + | > grad_norm_0: tensor(70.3981, device='cuda:0') (tensor(45.0067, device='cuda:0')) + | > loss_gen: 2.8524959087371826 (2.556755406515939) + | > loss_kl: 1.765101432800293 (1.7652413163866316) + | > loss_feat: 7.376683712005615 (7.414790289742606) + | > loss_mel: 17.367719650268555 (16.741090774536133) + | > loss_duration: 0.8867496252059937 (0.9204715490341187) + | > loss_1: 30.248748779296875 (29.398348672049387) + | > grad_norm_1: tensor(795.7157, device='cuda:0') (tensor(634.7446, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.0774 (0.9393775122506278) + | > loader_time: 0.0085 (0.0064961229051862445) + + + --> TIME: 2024-07-13 19:54:58 -- STEP: 57/151 -- GLOBAL_STEP: 91400 + | > loss_disc: 2.099566698074341 (2.349345428901806) + | > loss_disc_real_0: 0.08585228770971298 (0.12645026766940168) + | > loss_disc_real_1: 0.10246100276708603 (0.20350471544161178) + | > loss_disc_real_2: 0.16551826894283295 (0.21337779728989853) + | > loss_disc_real_3: 0.15384384989738464 (0.20798232837727196) + | > loss_disc_real_4: 0.18560071289539337 (0.21285443034088403) + | > loss_disc_real_5: 0.1883441060781479 (0.2227949236045804) + | > loss_0: 2.099566698074341 (2.349345428901806) + | > grad_norm_0: tensor(13.5648, device='cuda:0') (tensor(34.0633, device='cuda:0')) + | > loss_gen: 2.6279892921447754 (2.564288365213495) + | > loss_kl: 1.5193171501159668 (1.820070938060158) + | > loss_feat: 9.118581771850586 (8.029043105610628) + | > loss_mel: 17.449228286743164 (16.86899703845643) + | > loss_duration: 0.9538618326187134 (0.9554117436994586) + | > loss_1: 31.668977737426758 (30.237811138755397) + | > grad_norm_1: tensor(375.8165, device='cuda:0') (tensor(532.3174, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8375 (0.998545537915146) + | > loader_time: 0.0041 (0.0064387865233839605) + + + --> TIME: 2024-07-13 19:55:45 -- STEP: 107/151 -- GLOBAL_STEP: 91450 + | > loss_disc: 2.4798455238342285 (2.3553209505348573) + | > loss_disc_real_0: 0.12969619035720825 (0.1272367439537405) + | > loss_disc_real_1: 0.22404775023460388 (0.20533418063526956) + | > loss_disc_real_2: 0.22257350385189056 (0.21301359147111945) + | > loss_disc_real_3: 0.22246786952018738 (0.2112840242753519) + | > loss_disc_real_4: 0.23125450313091278 (0.21239360247816996) + | > loss_disc_real_5: 0.27867159247398376 (0.2254279964716635) + | > loss_0: 2.4798455238342285 (2.3553209505348573) + | > grad_norm_0: tensor(54.5148, device='cuda:0') (tensor(32.4578, device='cuda:0')) + | > loss_gen: 2.340134859085083 (2.550218218954924) + | > loss_kl: 1.975315809249878 (1.8279109457942928) + | > loss_feat: 7.590577125549316 (7.986438653179417) + | > loss_mel: 17.3800106048584 (16.773924248240814) + | > loss_duration: 0.9417972564697266 (0.9460024995224499) + | > loss_1: 30.22783660888672 (30.084494474892303) + | > grad_norm_1: tensor(678.7488, device='cuda:0') (tensor(516.1040, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9524 (0.9650548172888355) + | > loader_time: 0.0057 (0.006105487591752383) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033104511407705456 (-0.0005006606762225779) + | > avg_loss_disc: 2.669292761729314 (+0.0923472734598012) + | > avg_loss_disc_real_0: 0.341187616953483 (+0.12868348566385415) + | > avg_loss_disc_real_1: 0.26540196400422317 (+0.03745563786763412) + | > avg_loss_disc_real_2: 0.24461173781981835 (+0.013104341351068943) + | > avg_loss_disc_real_3: 0.2617402351819552 (+0.013765324766819326) + | > avg_loss_disc_real_4: 0.22211876282325158 (-0.02147780931912935) + | > avg_loss_disc_real_5: 0.24045776289242965 (-0.01010066270828247) + | > avg_loss_0: 2.669292761729314 (+0.0923472734598012) + | > avg_loss_gen: 2.613075403066782 (+0.23064042971684406) + | > avg_loss_kl: 3.2100954972780666 (-0.029907575020423405) + | > avg_loss_feat: 7.530901285318228 (-0.1194843145517206) + | > avg_loss_mel: 17.50436621445876 (+0.7176703673142661) + | > avg_loss_duration: 1.2569745962436383 (+0.011097871340237875) + | > avg_loss_1: 32.11541293217585 (+0.8100167787991985) + + + > EPOCH: 43/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:56:38)  + + --> TIME: 2024-07-13 19:56:46 -- STEP: 6/151 -- GLOBAL_STEP: 91500 + | > loss_disc: 2.4412970542907715 (2.4005313714345298) + | > loss_disc_real_0: 0.2320505827665329 (0.13883431628346443) + | > loss_disc_real_1: 0.26517054438591003 (0.21945215513308844) + | > loss_disc_real_2: 0.20571164786815643 (0.21488461643457413) + | > loss_disc_real_3: 0.272394061088562 (0.21562328934669495) + | > loss_disc_real_4: 0.2760213613510132 (0.2080558935801188) + | > loss_disc_real_5: 0.21040333807468414 (0.21998478968938193) + | > loss_0: 2.4412970542907715 (2.4005313714345298) + | > grad_norm_0: tensor(57.6757, device='cuda:0') (tensor(41.8149, device='cuda:0')) + | > loss_gen: 2.718655824661255 (2.5039668480555215) + | > loss_kl: 1.848527193069458 (1.9335530598958333) + | > loss_feat: 7.32686710357666 (7.549676815668742) + | > loss_mel: 17.165992736816406 (17.08549626668294) + | > loss_duration: 0.9216855764389038 (0.9335404634475708) + | > loss_1: 29.98172950744629 (30.006234169006348) + | > grad_norm_1: tensor(364.9059, device='cuda:0') (tensor(597.6357, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9699 (0.9471548795700073) + | > loader_time: 0.0058 (0.0067471663157145185) + + + --> TIME: 2024-07-13 19:57:37 -- STEP: 56/151 -- GLOBAL_STEP: 91550 + | > loss_disc: 2.327317714691162 (2.3457229946340825) + | > loss_disc_real_0: 0.24853937327861786 (0.13443390000611538) + | > loss_disc_real_1: 0.18186187744140625 (0.2059154619595834) + | > loss_disc_real_2: 0.26147380471229553 (0.2125372921249696) + | > loss_disc_real_3: 0.22753262519836426 (0.20760731984462058) + | > loss_disc_real_4: 0.22999635338783264 (0.2093478518405131) + | > loss_disc_real_5: 0.28736990690231323 (0.22112695180944034) + | > loss_0: 2.327317714691162 (2.3457229946340825) + | > grad_norm_0: tensor(34.5183, device='cuda:0') (tensor(38.7115, device='cuda:0')) + | > loss_gen: 2.8896913528442383 (2.5752076932362145) + | > loss_kl: 1.76198148727417 (1.7700877216245448) + | > loss_feat: 8.32984733581543 (8.084949493408201) + | > loss_mel: 17.1259765625 (16.89248522690364) + | > loss_duration: 0.9060019254684448 (0.9642010373728616) + | > loss_1: 31.013500213623047 (30.286931208201818) + | > grad_norm_1: tensor(420.8138, device='cuda:0') (tensor(557.4789, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9818 (1.001126502241407) + | > loader_time: 0.0056 (0.006536594458988735) + + + --> TIME: 2024-07-13 19:58:23 -- STEP: 106/151 -- GLOBAL_STEP: 91600 + | > loss_disc: 2.1329894065856934 (2.329993072545753) + | > loss_disc_real_0: 0.05593603104352951 (0.12656377385950307) + | > loss_disc_real_1: 0.1688544899225235 (0.2025671566151223) + | > loss_disc_real_2: 0.18366488814353943 (0.21285216580584365) + | > loss_disc_real_3: 0.1329798400402069 (0.2058638150118432) + | > loss_disc_real_4: 0.17087426781654358 (0.20823680862503233) + | > loss_disc_real_5: 0.20631995797157288 (0.2213396886890789) + | > loss_0: 2.1329894065856934 (2.329993072545753) + | > grad_norm_0: tensor(17.3658, device='cuda:0') (tensor(36.9747, device='cuda:0')) + | > loss_gen: 2.8735718727111816 (2.5833320550198824) + | > loss_kl: 1.825626015663147 (1.7679277245166167) + | > loss_feat: 9.433209419250488 (8.17618100598173) + | > loss_mel: 17.07729721069336 (17.016088386751576) + | > loss_duration: 1.0976974964141846 (0.955843243958815) + | > loss_1: 32.30739974975586 (30.499372446312094) + | > grad_norm_1: tensor(745.6934, device='cuda:0') (tensor(581.2236, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8104 (0.9620774107159309) + | > loader_time: 0.0049 (0.006283182018208054) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0304563228900616 (-0.0026481885176438576) + | > avg_loss_disc: 2.4788908041440525 (-0.19040195758526135) + | > avg_loss_disc_real_0: 0.19231461905516112 (-0.14887299789832187) + | > avg_loss_disc_real_1: 0.240074757200021 (-0.025327206804202168) + | > avg_loss_disc_real_2: 0.26052193572888005 (+0.015910197909061702) + | > avg_loss_disc_real_3: 0.1786773640375871 (-0.08306287114436811) + | > avg_loss_disc_real_4: 0.19291778825796568 (-0.029200974565285892) + | > avg_loss_disc_real_5: 0.21432290168908927 (-0.02613486120334038) + | > avg_loss_0: 2.4788908041440525 (-0.19040195758526135) + | > avg_loss_gen: 2.3780792309687686 (-0.23499617209801338) + | > avg_loss_kl: 3.468649754157433 (+0.2585542568793664) + | > avg_loss_feat: 7.841315342829777 (+0.31041405751154905) + | > avg_loss_mel: 17.21599190051739 (-0.28837431394136814) + | > avg_loss_duration: 1.3202468982109656 (+0.06327230196732736) + | > avg_loss_1: 32.22428292494554 (+0.10886999276969078) + + + > EPOCH: 44/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 19:59:17)  + + --> TIME: 2024-07-13 19:59:25 -- STEP: 5/151 -- GLOBAL_STEP: 91650 + | > loss_disc: 2.198798179626465 (2.276208591461182) + | > loss_disc_real_0: 0.08311809599399567 (0.1176862008869648) + | > loss_disc_real_1: 0.14431625604629517 (0.18471853733062743) + | > loss_disc_real_2: 0.15226644277572632 (0.20633840262889863) + | > loss_disc_real_3: 0.2034485638141632 (0.20025231540203095) + | > loss_disc_real_4: 0.21881812810897827 (0.2022827982902527) + | > loss_disc_real_5: 0.16984325647354126 (0.19534998536109924) + | > loss_0: 2.198798179626465 (2.276208591461182) + | > grad_norm_0: tensor(14.6594, device='cuda:0') (tensor(33.9212, device='cuda:0')) + | > loss_gen: 2.6340959072113037 (2.6144765853881835) + | > loss_kl: 1.3839597702026367 (1.8485881328582763) + | > loss_feat: 9.13572883605957 (8.516641616821289) + | > loss_mel: 16.54184341430664 (17.319461059570312) + | > loss_duration: 0.9325824975967407 (0.9901548981666565) + | > loss_1: 30.628210067749023 (31.289321517944337) + | > grad_norm_1: tensor(303.3645, device='cuda:0') (tensor(433.1073, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8318 (1.10728120803833) + | > loader_time: 0.0037 (0.00798206329345703) + + + --> TIME: 2024-07-13 20:00:14 -- STEP: 55/151 -- GLOBAL_STEP: 91700 + | > loss_disc: 2.403502941131592 (2.350172298604793) + | > loss_disc_real_0: 0.09103658050298691 (0.12829746001146056) + | > loss_disc_real_1: 0.22709302604198456 (0.202498752691529) + | > loss_disc_real_2: 0.21355660259723663 (0.21274675808169624) + | > loss_disc_real_3: 0.2527211904525757 (0.21004329824989493) + | > loss_disc_real_4: 0.1613713651895523 (0.20889110538092526) + | > loss_disc_real_5: 0.1898278146982193 (0.2210067258639769) + | > loss_0: 2.403502941131592 (2.350172298604793) + | > grad_norm_0: tensor(33.8676, device='cuda:0') (tensor(34.4662, device='cuda:0')) + | > loss_gen: 2.470017194747925 (2.5529692606492476) + | > loss_kl: 1.6796629428863525 (1.7374018040570345) + | > loss_feat: 7.1976237297058105 (8.00278801484542) + | > loss_mel: 16.198593139648438 (16.83463157307018) + | > loss_duration: 1.0193426609039307 (0.9650843533602628) + | > loss_1: 28.56523895263672 (30.092875012484463) + | > grad_norm_1: tensor(275.9930, device='cuda:0') (tensor(560.0060, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8142 (0.9919045621698555) + | > loader_time: 0.0052 (0.00602642839605158) + + + --> TIME: 2024-07-13 20:01:03 -- STEP: 105/151 -- GLOBAL_STEP: 91750 + | > loss_disc: 2.213381290435791 (2.358656474522182) + | > loss_disc_real_0: 0.116233229637146 (0.13218071020784838) + | > loss_disc_real_1: 0.22281956672668457 (0.20507208790097917) + | > loss_disc_real_2: 0.19213877618312836 (0.2138032627957208) + | > loss_disc_real_3: 0.24595743417739868 (0.20968576009784426) + | > loss_disc_real_4: 0.19429782032966614 (0.2111037295489084) + | > loss_disc_real_5: 0.2126265913248062 (0.22287580910183136) + | > loss_0: 2.213381290435791 (2.358656474522182) + | > grad_norm_0: tensor(14.4346, device='cuda:0') (tensor(33.7323, device='cuda:0')) + | > loss_gen: 2.6920347213745117 (2.5373710768563407) + | > loss_kl: 1.6809804439544678 (1.7766665311086745) + | > loss_feat: 8.75886344909668 (7.885101654416041) + | > loss_mel: 18.076684951782227 (16.884475653512148) + | > loss_duration: 0.9151608347892761 (0.957904208841778) + | > loss_1: 32.12372589111328 (30.041519183204286) + | > grad_norm_1: tensor(423.1219, device='cuda:0') (tensor(544.9041, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8756 (0.9765776134672619) + | > loader_time: 0.0054 (0.006166814622424898) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032407357142521784 (+0.0019510342524601855) + | > avg_loss_disc: 2.5663238855508657 (+0.08743308140681316) + | > avg_loss_disc_real_0: 0.20370472279878762 (+0.011390103743626506) + | > avg_loss_disc_real_1: 0.2096181993301098 (-0.030456557869911194) + | > avg_loss_disc_real_2: 0.2389229226570863 (-0.021599013071793738) + | > avg_loss_disc_real_3: 0.2267134006206806 (+0.048036036583093494) + | > avg_loss_disc_real_4: 0.24377265228674963 (+0.05085486402878395) + | > avg_loss_disc_real_5: 0.24819939984725073 (+0.03387649815816146) + | > avg_loss_0: 2.5663238855508657 (+0.08743308140681316) + | > avg_loss_gen: 2.417357243024386 (+0.03927801205561732) + | > avg_loss_kl: 3.376280747927152 (-0.09236900623028088) + | > avg_loss_feat: 7.757310243753286 (-0.08400509907649045) + | > avg_loss_mel: 16.88006723844088 (-0.3359246620765113) + | > avg_loss_duration: 1.3371153794802153 (+0.01686848126924967) + | > avg_loss_1: 31.768130669227013 (-0.45615225571852847) + + + > EPOCH: 45/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:01:57)  + + --> TIME: 2024-07-13 20:02:03 -- STEP: 4/151 -- GLOBAL_STEP: 91800 + | > loss_disc: 2.3068642616271973 (2.3586121797561646) + | > loss_disc_real_0: 0.13199985027313232 (0.14403797313570976) + | > loss_disc_real_1: 0.19727331399917603 (0.19483284652233124) + | > loss_disc_real_2: 0.1855289489030838 (0.19907134398818016) + | > loss_disc_real_3: 0.19213873147964478 (0.19555381312966347) + | > loss_disc_real_4: 0.195437490940094 (0.19132551550865173) + | > loss_disc_real_5: 0.19653990864753723 (0.22948938980698586) + | > loss_0: 2.3068642616271973 (2.3586121797561646) + | > grad_norm_0: tensor(24.9079, device='cuda:0') (tensor(28.6213, device='cuda:0')) + | > loss_gen: 2.6359434127807617 (2.514411449432373) + | > loss_kl: 1.8613511323928833 (1.8593519628047943) + | > loss_feat: 8.046378135681152 (7.540623188018799) + | > loss_mel: 17.52867889404297 (17.1604266166687) + | > loss_duration: 0.9138541221618652 (0.8944926559925079) + | > loss_1: 30.986207962036133 (29.96930742263794) + | > grad_norm_1: tensor(600.8823, device='cuda:0') (tensor(494.2953, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8749 (0.9230510592460632) + | > loader_time: 0.0061 (0.0057364702224731445) + + + --> TIME: 2024-07-13 20:02:51 -- STEP: 54/151 -- GLOBAL_STEP: 91850 + | > loss_disc: 2.480342388153076 (2.329331225819057) + | > loss_disc_real_0: 0.17692357301712036 (0.1264496349074222) + | > loss_disc_real_1: 0.24522832036018372 (0.2051532053285175) + | > loss_disc_real_2: 0.2752293050289154 (0.21055287453863356) + | > loss_disc_real_3: 0.26523739099502563 (0.20635650610482253) + | > loss_disc_real_4: 0.24954824149608612 (0.20931895739502376) + | > loss_disc_real_5: 0.24378250539302826 (0.21945885403288734) + | > loss_0: 2.480342388153076 (2.329331225819057) + | > grad_norm_0: tensor(27.0025, device='cuda:0') (tensor(33.9707, device='cuda:0')) + | > loss_gen: 2.491119384765625 (2.5892162941120294) + | > loss_kl: 1.9429718255996704 (1.837644621177956) + | > loss_feat: 6.696478366851807 (8.119106681258595) + | > loss_mel: 17.382848739624023 (16.948607762654625) + | > loss_duration: 0.9585988521575928 (0.9345133414974919) + | > loss_1: 29.472017288208008 (30.429088945742006) + | > grad_norm_1: tensor(683.2415, device='cuda:0') (tensor(489.0391, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9957 (0.9380642528887149) + | > loader_time: 0.0073 (0.005688985188802083) + + + --> TIME: 2024-07-13 20:03:44 -- STEP: 104/151 -- GLOBAL_STEP: 91900 + | > loss_disc: 2.4436864852905273 (2.348676369740413) + | > loss_disc_real_0: 0.11077161878347397 (0.1300960418839868) + | > loss_disc_real_1: 0.17206647992134094 (0.20510804524215367) + | > loss_disc_real_2: 0.200924813747406 (0.21423516264901712) + | > loss_disc_real_3: 0.2507181465625763 (0.2089087794033381) + | > loss_disc_real_4: 0.20545533299446106 (0.20930671964127287) + | > loss_disc_real_5: 0.17112037539482117 (0.22186606506315562) + | > loss_0: 2.4436864852905273 (2.348676369740413) + | > grad_norm_0: tensor(22.9509, device='cuda:0') (tensor(34.3789, device='cuda:0')) + | > loss_gen: 2.323321580886841 (2.57701272918628) + | > loss_kl: 1.9672257900238037 (1.8369428251798336) + | > loss_feat: 7.177728176116943 (8.118652884776775) + | > loss_mel: 16.777599334716797 (16.992423543563255) + | > loss_duration: 0.9469678401947021 (0.9553782859673867) + | > loss_1: 29.19284439086914 (30.48041035578801) + | > grad_norm_1: tensor(531.1736, device='cuda:0') (tensor(508.9706, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9848 (0.9905494405673101) + | > loader_time: 0.0071 (0.0062936177620520955) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03589441226078914 (+0.003487055118267357) + | > avg_loss_disc: 2.629291662803063 (+0.06296777725219727) + | > avg_loss_disc_real_0: 0.23621754806775314 (+0.03251282526896551) + | > avg_loss_disc_real_1: 0.24494271438855392 (+0.03532451505844411) + | > avg_loss_disc_real_2: 0.2221383395103308 (-0.016784583146755516) + | > avg_loss_disc_real_3: 0.19179271390804878 (-0.03492068671263182) + | > avg_loss_disc_real_4: 0.19113015096921188 (-0.052642501317537754) + | > avg_loss_disc_real_5: 0.2037973300768779 (-0.04440206977037284) + | > avg_loss_0: 2.629291662803063 (+0.06296777725219727) + | > avg_loss_gen: 2.1939936601198635 (-0.2233635829045224) + | > avg_loss_kl: 3.3613316462590146 (-0.014949101668137565) + | > avg_loss_feat: 7.753242015838623 (-0.004068227914663325) + | > avg_loss_mel: 16.882372709421013 (+0.0023054709801328954) + | > avg_loss_duration: 1.2450887514994695 (-0.09202662798074579) + | > avg_loss_1: 31.436028553889347 (-0.33210211533766554) + + + > EPOCH: 46/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:04:39)  + + --> TIME: 2024-07-13 20:04:44 -- STEP: 3/151 -- GLOBAL_STEP: 91950 + | > loss_disc: 2.2911088466644287 (2.4174416065216064) + | > loss_disc_real_0: 0.14287686347961426 (0.16196852922439575) + | > loss_disc_real_1: 0.2004612386226654 (0.24476741751035055) + | > loss_disc_real_2: 0.22344358265399933 (0.25225680073102313) + | > loss_disc_real_3: 0.20080353319644928 (0.2063789963722229) + | > loss_disc_real_4: 0.24159754812717438 (0.24102692802747092) + | > loss_disc_real_5: 0.19649188220500946 (0.1915058195590973) + | > loss_0: 2.2911088466644287 (2.4174416065216064) + | > grad_norm_0: tensor(15.5985, device='cuda:0') (tensor(22.1980, device='cuda:0')) + | > loss_gen: 2.5159718990325928 (2.4843161900838218) + | > loss_kl: 1.8787227869033813 (1.5679049293200176) + | > loss_feat: 7.145816802978516 (6.564359029134114) + | > loss_mel: 17.02615737915039 (15.660820643107096) + | > loss_duration: 0.9441004991531372 (0.991693913936615) + | > loss_1: 29.510770797729492 (27.269094467163086) + | > grad_norm_1: tensor(377.5514, device='cuda:0') (tensor(416.4159, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9232 (0.9308293660481771) + | > loader_time: 0.006 (0.0062643686930338545) + + + --> TIME: 2024-07-13 20:05:34 -- STEP: 53/151 -- GLOBAL_STEP: 92000 + | > loss_disc: 2.0865845680236816 (2.338616276687045) + | > loss_disc_real_0: 0.0842212364077568 (0.1258851195562561) + | > loss_disc_real_1: 0.16090697050094604 (0.2046015167011405) + | > loss_disc_real_2: 0.165117546916008 (0.21819684471724168) + | > loss_disc_real_3: 0.2214234173297882 (0.20805341623864085) + | > loss_disc_real_4: 0.17434605956077576 (0.21680734658016348) + | > loss_disc_real_5: 0.19384464621543884 (0.21526281029548286) + | > loss_0: 2.0865845680236816 (2.338616276687045) + | > grad_norm_0: tensor(21.9216, device='cuda:0') (tensor(32.6848, device='cuda:0')) + | > loss_gen: 2.744288921356201 (2.5655459412988626) + | > loss_kl: 1.4235665798187256 (1.7350156644605241) + | > loss_feat: 8.563281059265137 (8.119884383003665) + | > loss_mel: 16.852773666381836 (16.82193865866031) + | > loss_duration: 1.15872061252594 (0.959837628985351) + | > loss_1: 30.742631912231445 (30.20222242823187) + | > grad_norm_1: tensor(620.6176, device='cuda:0') (tensor(505.0901, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8209 (0.9846925690488996) + | > loader_time: 0.0048 (0.006280552666142302) + + + --> TIME: 2024-07-13 20:06:45 -- STEP: 103/151 -- GLOBAL_STEP: 92050 + | > loss_disc: 2.503957748413086 (2.334809641236241) + | > loss_disc_real_0: 0.18325375020503998 (0.12776901052125447) + | > loss_disc_real_1: 0.22134892642498016 (0.206260039968398) + | > loss_disc_real_2: 0.24105478823184967 (0.2153286269856888) + | > loss_disc_real_3: 0.2273654043674469 (0.20853192655785574) + | > loss_disc_real_4: 0.27194538712501526 (0.2130354291316375) + | > loss_disc_real_5: 0.3124833405017853 (0.218187498091494) + | > loss_0: 2.503957748413086 (2.334809641236241) + | > grad_norm_0: tensor(27.1425, device='cuda:0') (tensor(27.5069, device='cuda:0')) + | > loss_gen: 2.406352996826172 (2.56276404510424) + | > loss_kl: 2.313063859939575 (1.7515694504802666) + | > loss_feat: 5.831284523010254 (8.115437776139636) + | > loss_mel: 17.37698745727539 (16.78056767843303) + | > loss_duration: 1.1289571523666382 (0.9558571472908687) + | > loss_1: 29.0566463470459 (30.16619619350989) + | > grad_norm_1: tensor(95.0862, device='cuda:0') (tensor(391.5098, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.9755 (0.9952461163974503) + | > loader_time: 0.0141 (0.006333466872428226) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0324854483971229 (-0.0034089638636662437) + | > avg_loss_disc: 2.7020319608541636 (+0.07274029805110072) + | > avg_loss_disc_real_0: 0.26379132041564357 (+0.027573772347890435) + | > avg_loss_disc_real_1: 0.19279177486896515 (-0.05215093951958877) + | > avg_loss_disc_real_2: 0.23177938048656171 (+0.009641040976230919) + | > avg_loss_disc_real_3: 0.2057666022043962 (+0.013973888296347409) + | > avg_loss_disc_real_4: 0.21989598182531503 (+0.028765830856103153) + | > avg_loss_disc_real_5: 0.22092248155520514 (+0.017125151478327244) + | > avg_loss_0: 2.7020319608541636 (+0.07274029805110072) + | > avg_loss_gen: 2.1676341478641215 (-0.026359512255742068) + | > avg_loss_kl: 3.3887378802666297 (+0.027406234007615105) + | > avg_loss_feat: 7.47344735952524 (-0.27979465631338307) + | > avg_loss_mel: 16.894379248985878 (+0.012006539564865193) + | > avg_loss_duration: 1.2924327025046716 (+0.047343951005202056) + | > avg_loss_1: 31.21663108238807 (-0.21939747150127786) + + + > EPOCH: 47/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:07:39)  + + --> TIME: 2024-07-13 20:07:43 -- STEP: 2/151 -- GLOBAL_STEP: 92100 + | > loss_disc: 2.3505289554595947 (2.3309662342071533) + | > loss_disc_real_0: 0.11230099201202393 (0.1056244783103466) + | > loss_disc_real_1: 0.2359703928232193 (0.21647069603204727) + | > loss_disc_real_2: 0.20558199286460876 (0.1997442990541458) + | > loss_disc_real_3: 0.22328874468803406 (0.208233043551445) + | > loss_disc_real_4: 0.19220535457134247 (0.18977303057909012) + | > loss_disc_real_5: 0.18962828814983368 (0.19735970348119736) + | > loss_0: 2.3505289554595947 (2.3309662342071533) + | > grad_norm_0: tensor(55.3866, device='cuda:0') (tensor(50.7662, device='cuda:0')) + | > loss_gen: 2.884117364883423 (2.8644977807998657) + | > loss_kl: 1.6842916011810303 (1.736884355545044) + | > loss_feat: 9.855693817138672 (9.349364757537842) + | > loss_mel: 17.8282470703125 (17.752605438232422) + | > loss_duration: 0.9664926528930664 (0.9455730020999908) + | > loss_1: 33.218841552734375 (32.64892387390137) + | > grad_norm_1: tensor(775.0452, device='cuda:0') (tensor(701.6289, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8494 (0.8531793355941772) + | > loader_time: 0.0036 (0.003725886344909668) + + + --> TIME: 2024-07-13 20:08:30 -- STEP: 52/151 -- GLOBAL_STEP: 92150 + | > loss_disc: 2.6158320903778076 (2.3274915722700253) + | > loss_disc_real_0: 0.1611427366733551 (0.12501593256512514) + | > loss_disc_real_1: 0.2094019055366516 (0.20608163051880324) + | > loss_disc_real_2: 0.21529702842235565 (0.21105466181269059) + | > loss_disc_real_3: 0.2246418297290802 (0.20959986975559822) + | > loss_disc_real_4: 0.21597793698310852 (0.20749103793731102) + | > loss_disc_real_5: 0.21029861271381378 (0.22296041049636328) + | > loss_0: 2.6158320903778076 (2.3274915722700253) + | > grad_norm_0: tensor(28.6690, device='cuda:0') (tensor(37.4905, device='cuda:0')) + | > loss_gen: 2.2027640342712402 (2.644677941615764) + | > loss_kl: 2.270087480545044 (1.8041418125996223) + | > loss_feat: 5.627965450286865 (8.271185618180494) + | > loss_mel: 15.271675109863281 (16.941171205960785) + | > loss_duration: 1.153364658355713 (0.9521522774146154) + | > loss_1: 26.525856018066406 (30.613328823676476) + | > grad_norm_1: tensor(396.8413, device='cuda:0') (tensor(577.6279, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 2.0702 (0.9291278169705317) + | > loader_time: 0.0183 (0.0055367763225848835) + + + --> TIME: 2024-07-13 20:09:21 -- STEP: 102/151 -- GLOBAL_STEP: 92200 + | > loss_disc: 2.2505812644958496 (2.3508888342801253) + | > loss_disc_real_0: 0.13569225370883942 (0.13035359390664342) + | > loss_disc_real_1: 0.16806355118751526 (0.20401039325139103) + | > loss_disc_real_2: 0.20959268510341644 (0.21309877289276496) + | > loss_disc_real_3: 0.1867748647928238 (0.2077664616061192) + | > loss_disc_real_4: 0.2867202162742615 (0.21025715344676785) + | > loss_disc_real_5: 0.22318905591964722 (0.22606256764893437) + | > loss_0: 2.2505812644958496 (2.3508888342801253) + | > grad_norm_0: tensor(40.1602, device='cuda:0') (tensor(31.7652, device='cuda:0')) + | > loss_gen: 2.6360437870025635 (2.5885137833800966) + | > loss_kl: 1.9189597368240356 (1.8034217123891794) + | > loss_feat: 9.339088439941406 (8.155081690526472) + | > loss_mel: 18.014816284179688 (16.971890524321918) + | > loss_duration: 0.871954619884491 (0.9524315198262533) + | > loss_1: 32.78086471557617 (30.471339281867532) + | > grad_norm_1: tensor(481.3582, device='cuda:0') (tensor(488.3889, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8563 (0.9675981601079305) + | > loader_time: 0.0044 (0.006098906199137368) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03260185168339656 (+0.00011640328627366553) + | > avg_loss_disc: 2.544073618375338 (-0.15795834247882556) + | > avg_loss_disc_real_0: 0.19781329196233016 (-0.06597802845331341) + | > avg_loss_disc_real_1: 0.18442571277801806 (-0.00836606209094709) + | > avg_loss_disc_real_2: 0.22875033433620745 (-0.0030290461503542643) + | > avg_loss_disc_real_3: 0.2420486372250777 (+0.0362820350206815) + | > avg_loss_disc_real_4: 0.22282267419191507 (+0.0029266923666000366) + | > avg_loss_disc_real_5: 0.20296819393451399 (-0.01795428762069115) + | > avg_loss_0: 2.544073618375338 (-0.15795834247882556) + | > avg_loss_gen: 2.2604686480302076 (+0.09283450016608619) + | > avg_loss_kl: 3.3583101676060605 (-0.030427712660569206) + | > avg_loss_feat: 8.104529014000528 (+0.6310816544752882) + | > avg_loss_mel: 17.320070413442757 (+0.4256911644568788) + | > avg_loss_duration: 1.2772644299727223 (-0.0151682725319493) + | > avg_loss_1: 32.320642764751724 (+1.1040116823636552) + + + > EPOCH: 48/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:10:19)  + + --> TIME: 2024-07-13 20:10:22 -- STEP: 1/151 -- GLOBAL_STEP: 92250 + | > loss_disc: 2.364548444747925 (2.364548444747925) + | > loss_disc_real_0: 0.07640073448419571 (0.07640073448419571) + | > loss_disc_real_1: 0.2183011919260025 (0.2183011919260025) + | > loss_disc_real_2: 0.24665968120098114 (0.24665968120098114) + | > loss_disc_real_3: 0.21720285713672638 (0.21720285713672638) + | > loss_disc_real_4: 0.17878152430057526 (0.17878152430057526) + | > loss_disc_real_5: 0.19288289546966553 (0.19288289546966553) + | > loss_0: 2.364548444747925 (2.364548444747925) + | > grad_norm_0: tensor(31.3181, device='cuda:0') (tensor(31.3181, device='cuda:0')) + | > loss_gen: 2.4561798572540283 (2.4561798572540283) + | > loss_kl: 1.6742151975631714 (1.6742151975631714) + | > loss_feat: 7.911735534667969 (7.911735534667969) + | > loss_mel: 17.55632209777832 (17.55632209777832) + | > loss_duration: 0.8303520083427429 (0.8303520083427429) + | > loss_1: 30.42880630493164 (30.42880630493164) + | > grad_norm_1: tensor(552.9000, device='cuda:0') (tensor(552.9000, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9832 (0.9832015037536621) + | > loader_time: 0.0063 (0.0062716007232666016) + + + --> TIME: 2024-07-13 20:11:09 -- STEP: 51/151 -- GLOBAL_STEP: 92300 + | > loss_disc: 2.21182918548584 (2.323284955585703) + | > loss_disc_real_0: 0.09923118352890015 (0.12060860464093733) + | > loss_disc_real_1: 0.2297528088092804 (0.20927845642847173) + | > loss_disc_real_2: 0.2471369057893753 (0.2136908012862299) + | > loss_disc_real_3: 0.2099202573299408 (0.20588616646972357) + | > loss_disc_real_4: 0.13562113046646118 (0.2044104863615597) + | > loss_disc_real_5: 0.20507566630840302 (0.2219579982406953) + | > loss_0: 2.21182918548584 (2.323284955585703) + | > grad_norm_0: tensor(8.2294, device='cuda:0') (tensor(30.9120, device='cuda:0')) + | > loss_gen: 2.5912578105926514 (2.62313721226711) + | > loss_kl: 1.1827455759048462 (1.725829475066241) + | > loss_feat: 8.750479698181152 (8.264963280920888) + | > loss_mel: 16.35211753845215 (17.02057389652028) + | > loss_duration: 0.9417139291763306 (0.9412020526680291) + | > loss_1: 29.818313598632812 (30.575705846150715) + | > grad_norm_1: tensor(487.4452, device='cuda:0') (tensor(516.6694, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8601 (0.9430125367407706) + | > loader_time: 0.0048 (0.00549204209271599) + + + --> TIME: 2024-07-13 20:11:59 -- STEP: 101/151 -- GLOBAL_STEP: 92350 + | > loss_disc: 2.3192272186279297 (2.333079478528238) + | > loss_disc_real_0: 0.0935460552573204 (0.12422453910850062) + | > loss_disc_real_1: 0.1837444305419922 (0.20731464456213583) + | > loss_disc_real_2: 0.20334398746490479 (0.21269783303879275) + | > loss_disc_real_3: 0.19205380976200104 (0.206706600392809) + | > loss_disc_real_4: 0.18450357019901276 (0.20775465320537578) + | > loss_disc_real_5: 0.19003024697303772 (0.21931728145273605) + | > loss_0: 2.3192272186279297 (2.333079478528238) + | > grad_norm_0: tensor(10.9219, device='cuda:0') (tensor(35.3189, device='cuda:0')) + | > loss_gen: 2.3919878005981445 (2.5936416611813082) + | > loss_kl: 0.9076493978500366 (1.7468414837771122) + | > loss_feat: 7.504690647125244 (8.196300364957004) + | > loss_mel: 15.397418022155762 (16.882002660543613) + | > loss_duration: 1.0404049158096313 (0.9470183654586868) + | > loss_1: 27.242149353027344 (30.36580467224121) + | > grad_norm_1: tensor(141.2226, device='cuda:0') (tensor(487.9728, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8047 (0.9589332708037726) + | > loader_time: 0.0041 (0.005907899082297146) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032956875287569486 (+0.00035502360417292295) + | > avg_loss_disc: 2.506431909707876 (-0.03764170866746186) + | > avg_loss_disc_real_0: 0.21088954118581918 (+0.013076249223489017) + | > avg_loss_disc_real_1: 0.23120950964780954 (+0.04678379686979148) + | > avg_loss_disc_real_2: 0.23684754623816565 (+0.008097211901958196) + | > avg_loss_disc_real_3: 0.20920303120062902 (-0.032845606024448665) + | > avg_loss_disc_real_4: 0.23427146673202515 (+0.011448792540110081) + | > avg_loss_disc_real_5: 0.27539324989685643 (+0.07242505596234244) + | > avg_loss_0: 2.506431909707876 (-0.03764170866746186) + | > avg_loss_gen: 2.5179295356457043 (+0.2574608876154967) + | > avg_loss_kl: 3.456389830662654 (+0.09807966305659344) + | > avg_loss_feat: 7.945453020242544 (-0.15907599375798398) + | > avg_loss_mel: 17.580330041738655 (+0.26025962829589844) + | > avg_loss_duration: 1.272837418776292 (-0.004427011196430186) + | > avg_loss_1: 32.77293953528771 (+0.4522967705359875) + + + > EPOCH: 49/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:12:57)  + + --> TIME: 2024-07-13 20:12:59 -- STEP: 0/151 -- GLOBAL_STEP: 92400 + | > loss_disc: 2.482004165649414 (2.482004165649414) + | > loss_disc_real_0: 0.12363353371620178 (0.12363353371620178) + | > loss_disc_real_1: 0.25285181403160095 (0.25285181403160095) + | > loss_disc_real_2: 0.2632291316986084 (0.2632291316986084) + | > loss_disc_real_3: 0.212046280503273 (0.212046280503273) + | > loss_disc_real_4: 0.21510837972164154 (0.21510837972164154) + | > loss_disc_real_5: 0.31727203726768494 (0.31727203726768494) + | > loss_0: 2.482004165649414 (2.482004165649414) + | > grad_norm_0: tensor(33.0977, device='cuda:0') (tensor(33.0977, device='cuda:0')) + | > loss_gen: 2.490574598312378 (2.490574598312378) + | > loss_kl: 2.1448400020599365 (2.1448400020599365) + | > loss_feat: 6.4144287109375 (6.4144287109375) + | > loss_mel: 16.675779342651367 (16.675779342651367) + | > loss_duration: 0.9495736360549927 (0.9495736360549927) + | > loss_1: 28.675195693969727 (28.675195693969727) + | > grad_norm_1: tensor(229.0651, device='cuda:0') (tensor(229.0651, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0769 (1.0769140720367432) + | > loader_time: 1.0869 (1.0868988037109375) + + + --> TIME: 2024-07-13 20:13:50 -- STEP: 50/151 -- GLOBAL_STEP: 92450 + | > loss_disc: 2.4012742042541504 (2.3255287027359004) + | > loss_disc_real_0: 0.2537640929222107 (0.13084527008235455) + | > loss_disc_real_1: 0.23903325200080872 (0.20294343203306198) + | > loss_disc_real_2: 0.2592012882232666 (0.2098571166396141) + | > loss_disc_real_3: 0.21951177716255188 (0.20429557025432588) + | > loss_disc_real_4: 0.2236126959323883 (0.2076447120308876) + | > loss_disc_real_5: 0.2175256311893463 (0.22236385941505432) + | > loss_0: 2.4012742042541504 (2.3255287027359004) + | > grad_norm_0: tensor(23.6171, device='cuda:0') (tensor(18.2705, device='cuda:0')) + | > loss_gen: 2.6432080268859863 (2.586023201942444) + | > loss_kl: 1.8563706874847412 (1.8251794862747193) + | > loss_feat: 7.708130359649658 (8.214193611145017) + | > loss_mel: 17.931955337524414 (16.84701280593872) + | > loss_duration: 0.8931831121444702 (0.9586805486679078) + | > loss_1: 31.032846450805664 (30.431089782714842) + | > grad_norm_1: tensor(65.2195, device='cuda:0') (tensor(284.8462, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0749 (0.9949124622344969) + | > loader_time: 0.0079 (0.006197786331176758) + + + --> TIME: 2024-07-13 20:14:38 -- STEP: 100/151 -- GLOBAL_STEP: 92500 + | > loss_disc: 2.0833468437194824 (2.337411520481109) + | > loss_disc_real_0: 0.11047519743442535 (0.12912436243146655) + | > loss_disc_real_1: 0.19537824392318726 (0.20365252196788788) + | > loss_disc_real_2: 0.210841566324234 (0.2123364575207233) + | > loss_disc_real_3: 0.24474535882472992 (0.2105392475426197) + | > loss_disc_real_4: 0.14591673016548157 (0.20679649263620375) + | > loss_disc_real_5: 0.18351273238658905 (0.22250488221645356) + | > loss_0: 2.0833468437194824 (2.337411520481109) + | > grad_norm_0: tensor(38.5244, device='cuda:0') (tensor(23.1401, device='cuda:0')) + | > loss_gen: 2.9499826431274414 (2.566561195850371) + | > loss_kl: 1.7613353729248047 (1.8153822559118271) + | > loss_feat: 10.336607933044434 (8.136035718917846) + | > loss_mel: 17.125736236572266 (16.89736043930054) + | > loss_duration: 1.025956630706787 (0.9553666418790817) + | > loss_1: 33.199615478515625 (30.37070634841919) + | > grad_norm_1: tensor(679.8756, device='cuda:0') (tensor(347.5096, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8087 (0.9718369054794311) + | > loader_time: 0.0044 (0.006101326942443848) + + + --> TIME: 2024-07-13 20:15:27 -- STEP: 150/151 -- GLOBAL_STEP: 92550 + | > loss_disc: 2.2708663940429688 (2.3348091999689733) + | > loss_disc_real_0: 0.11596576869487762 (0.127835813636581) + | > loss_disc_real_1: 0.2095106989145279 (0.20266761620839435) + | > loss_disc_real_2: 0.22627080976963043 (0.21236198971668874) + | > loss_disc_real_3: 0.22115816175937653 (0.21023365875085195) + | > loss_disc_real_4: 0.1950126588344574 (0.20776878277460734) + | > loss_disc_real_5: 0.22454781830310822 (0.22352207839488983) + | > loss_0: 2.2708663940429688 (2.3348091999689733) + | > grad_norm_0: tensor(34.7479, device='cuda:0') (tensor(24.3582, device='cuda:0')) + | > loss_gen: 2.4561386108398438 (2.558379454612731) + | > loss_kl: 1.649288535118103 (1.8250724883874256) + | > loss_feat: 7.528180122375488 (8.086756353378297) + | > loss_mel: 17.672971725463867 (16.791492748260495) + | > loss_duration: 0.9526926875114441 (0.9542401123046875) + | > loss_1: 30.2592716217041 (30.21594130198161) + | > grad_norm_1: tensor(610.8632, device='cuda:0') (tensor(370.5521, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8722 (0.9757676680882772) + | > loader_time: 0.0051 (0.006115668614705405) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032903194427490234 (-5.368086007925127e-05) + | > avg_loss_disc: 2.5172266960144043 (+0.010794786306528081) + | > avg_loss_disc_real_0: 0.1742189251459562 (-0.03667061603986299) + | > avg_loss_disc_real_1: 0.21850731510382432 (-0.012702194543985218) + | > avg_loss_disc_real_2: 0.22683355326835924 (-0.010013992969806401) + | > avg_loss_disc_real_3: 0.2173096503202732 (+0.008106619119644165) + | > avg_loss_disc_real_4: 0.25087615274465996 (+0.016604686012634817) + | > avg_loss_disc_real_5: 0.21192335394712594 (-0.06346989594973049) + | > avg_loss_0: 2.5172266960144043 (+0.010794786306528081) + | > avg_loss_gen: 2.2966242753542385 (-0.2213052602914658) + | > avg_loss_kl: 3.4584474930396447 (+0.0020576623769907876) + | > avg_loss_feat: 7.935666414407583 (-0.009786605834960938) + | > avg_loss_mel: 17.25329853938176 (-0.3270315023568955) + | > avg_loss_duration: 1.281766552191514 (+0.008929133415221946) + | > avg_loss_1: 32.22580366868239 (-0.5471358666053234) + + + > EPOCH: 50/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:15:38)  + + --> TIME: 2024-07-13 20:16:31 -- STEP: 49/151 -- GLOBAL_STEP: 92600 + | > loss_disc: 2.4818427562713623 (2.3281895238526014) + | > loss_disc_real_0: 0.24785393476486206 (0.1303472413244295) + | > loss_disc_real_1: 0.2349786013364792 (0.2036698168637801) + | > loss_disc_real_2: 0.250031977891922 (0.2109486381618344) + | > loss_disc_real_3: 0.1946878284215927 (0.20670360296356435) + | > loss_disc_real_4: 0.1479356735944748 (0.20363723319403979) + | > loss_disc_real_5: 0.29371872544288635 (0.21910822665204807) + | > loss_0: 2.4818427562713623 (2.3281895238526014) + | > grad_norm_0: tensor(69.8153, device='cuda:0') (tensor(33.2204, device='cuda:0')) + | > loss_gen: 2.3866686820983887 (2.6272897185111543) + | > loss_kl: 2.2034270763397217 (1.8270116338924485) + | > loss_feat: 6.049211502075195 (8.288870324893873) + | > loss_mel: 16.563261032104492 (17.09591746816829) + | > loss_duration: 1.184409499168396 (0.9494997980643292) + | > loss_1: 28.386978149414062 (30.78858862117845) + | > grad_norm_1: tensor(453.1833, device='cuda:0') (tensor(493.1306, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.9979 (1.0077406386939847) + | > loader_time: 0.0156 (0.006424315121709083) + + + --> TIME: 2024-07-13 20:17:17 -- STEP: 99/151 -- GLOBAL_STEP: 92650 + | > loss_disc: 2.461498260498047 (2.3328453627499677) + | > loss_disc_real_0: 0.10327267646789551 (0.12601075573551526) + | > loss_disc_real_1: 0.1593097448348999 (0.204671171578494) + | > loss_disc_real_2: 0.16112373769283295 (0.21231595885874044) + | > loss_disc_real_3: 0.11932636052370071 (0.20738418257296687) + | > loss_disc_real_4: 0.14759963750839233 (0.20732920338409117) + | > loss_disc_real_5: 0.17678654193878174 (0.21993669160086698) + | > loss_0: 2.461498260498047 (2.3328453627499677) + | > grad_norm_0: tensor(13.0366, device='cuda:0') (tensor(31.2347, device='cuda:0')) + | > loss_gen: 2.3370165824890137 (2.609617334423645) + | > loss_kl: 1.9555190801620483 (1.8266715979335284) + | > loss_feat: 8.881258010864258 (8.285685558511751) + | > loss_mel: 17.749679565429688 (17.08078006782916) + | > loss_duration: 0.8308590650558472 (0.9439912022966327) + | > loss_1: 31.754331588745117 (30.746745446715693) + | > grad_norm_1: tensor(227.0225, device='cuda:0') (tensor(489.5528, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9016 (0.9614347544583407) + | > loader_time: 0.0064 (0.006168827866062972) + + + --> TIME: 2024-07-13 20:18:06 -- STEP: 149/151 -- GLOBAL_STEP: 92700 + | > loss_disc: 2.2281899452209473 (2.3250687234353697) + | > loss_disc_real_0: 0.11741088330745697 (0.12527908227287676) + | > loss_disc_real_1: 0.20457014441490173 (0.20344424177736242) + | > loss_disc_real_2: 0.17279212176799774 (0.21209497519787526) + | > loss_disc_real_3: 0.19698628783226013 (0.20769263628945253) + | > loss_disc_real_4: 0.1613374799489975 (0.20709149789490153) + | > loss_disc_real_5: 0.22269277274608612 (0.21997496875740538) + | > loss_0: 2.2281899452209473 (2.3250687234353697) + | > grad_norm_0: tensor(43.8792, device='cuda:0') (tensor(32.8467, device='cuda:0')) + | > loss_gen: 2.794497489929199 (2.6223740225670342) + | > loss_kl: 1.563349723815918 (1.8153032180446906) + | > loss_feat: 8.942657470703125 (8.310081977972247) + | > loss_mel: 17.45618438720703 (17.082998512575283) + | > loss_duration: 0.9312391877174377 (0.9476095626018192) + | > loss_1: 31.687931060791016 (30.77836701053901) + | > grad_norm_1: tensor(799.3261, device='cuda:0') (tensor(524.0266, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8437 (0.9674987312931342) + | > loader_time: 0.004 (0.006144277201402907) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031560659408569336 (-0.0013425350189208984) + | > avg_loss_disc: 2.589174050551195 (+0.0719473545367908) + | > avg_loss_disc_real_0: 0.25613153783174664 (+0.08191261268579045) + | > avg_loss_disc_real_1: 0.27101868505661303 (+0.05251136995278871) + | > avg_loss_disc_real_2: 0.28974870993540836 (+0.06291515666704911) + | > avg_loss_disc_real_3: 0.22078521664326006 (+0.0034755663229868727) + | > avg_loss_disc_real_4: 0.26200349972798276 (+0.011127346983322794) + | > avg_loss_disc_real_5: 0.20373495381612045 (-0.008188400131005497) + | > avg_loss_0: 2.589174050551195 (+0.0719473545367908) + | > avg_loss_gen: 2.6002485201908994 (+0.30362424483666084) + | > avg_loss_kl: 3.7073756731473484 (+0.24892818010770368) + | > avg_loss_feat: 8.156633413754976 (+0.22096699934739306) + | > avg_loss_mel: 17.43982065640963 (+0.18652211702787014) + | > avg_loss_duration: 1.355964339696444 (+0.07419778750493) + | > avg_loss_1: 33.260041603675255 (+1.0342379349928663) + + + > EPOCH: 51/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:18:18)  + + --> TIME: 2024-07-13 20:19:06 -- STEP: 48/151 -- GLOBAL_STEP: 92750 + | > loss_disc: 2.438415765762329 (2.341522390643755) + | > loss_disc_real_0: 0.11731873452663422 (0.11675513520215948) + | > loss_disc_real_1: 0.2248438000679016 (0.20978958687434593) + | > loss_disc_real_2: 0.20180249214172363 (0.21477435094614825) + | > loss_disc_real_3: 0.21767041087150574 (0.20442847007264695) + | > loss_disc_real_4: 0.217845618724823 (0.21161318508287272) + | > loss_disc_real_5: 0.2316080629825592 (0.22168165010710558) + | > loss_0: 2.438415765762329 (2.341522390643755) + | > grad_norm_0: tensor(41.2284, device='cuda:0') (tensor(28.8293, device='cuda:0')) + | > loss_gen: 2.3171255588531494 (2.5474998553593946) + | > loss_kl: 1.941406011581421 (1.7595943088332813) + | > loss_feat: 6.811250686645508 (8.084936529397966) + | > loss_mel: 16.386369705200195 (16.84800102313359) + | > loss_duration: 0.9318342208862305 (0.9330735132098198) + | > loss_1: 28.387985229492188 (30.173105080922443) + | > grad_norm_1: tensor(572.5978, device='cuda:0') (tensor(447.3026, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9804 (0.9402594268321991) + | > loader_time: 0.0072 (0.005836799740791321) + + + --> TIME: 2024-07-13 20:19:57 -- STEP: 98/151 -- GLOBAL_STEP: 92800 + | > loss_disc: 2.3889708518981934 (2.340138923148719) + | > loss_disc_real_0: 0.1670081913471222 (0.123842344790393) + | > loss_disc_real_1: 0.23670899868011475 (0.20719805572714126) + | > loss_disc_real_2: 0.26837289333343506 (0.2140547000631994) + | > loss_disc_real_3: 0.23337407410144806 (0.20614883653363406) + | > loss_disc_real_4: 0.2674439549446106 (0.21047893288184186) + | > loss_disc_real_5: 0.20434148609638214 (0.22250009571411172) + | > loss_0: 2.3889708518981934 (2.340138923148719) + | > grad_norm_0: tensor(25.0351, device='cuda:0') (tensor(29.9250, device='cuda:0')) + | > loss_gen: 2.6160624027252197 (2.55707518665158) + | > loss_kl: 2.0340423583984375 (1.8174598600183214) + | > loss_feat: 7.397563934326172 (7.965340857603112) + | > loss_mel: 16.98876953125 (16.799219150932462) + | > loss_duration: 0.9573264122009277 (0.9538192347604402) + | > loss_1: 29.993764877319336 (30.092914133655782) + | > grad_norm_1: tensor(578.3150, device='cuda:0') (tensor(475.6562, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.0153 (0.9776310969372185) + | > loader_time: 0.0066 (0.006304125396572813) + + + --> TIME: 2024-07-13 20:20:48 -- STEP: 148/151 -- GLOBAL_STEP: 92850 + | > loss_disc: 2.4588427543640137 (2.3486062280229616) + | > loss_disc_real_0: 0.2102544903755188 (0.12529975158237927) + | > loss_disc_real_1: 0.20786358416080475 (0.20790279787537208) + | > loss_disc_real_2: 0.20180724561214447 (0.21308095390732223) + | > loss_disc_real_3: 0.21967090666294098 (0.20608619570329387) + | > loss_disc_real_4: 0.2136346697807312 (0.2088378469484883) + | > loss_disc_real_5: 0.18316474556922913 (0.2248021759898276) + | > loss_0: 2.4588427543640137 (2.3486062280229616) + | > grad_norm_0: tensor(33.7857, device='cuda:0') (tensor(30.8145, device='cuda:0')) + | > loss_gen: 2.5034515857696533 (2.5503663485114627) + | > loss_kl: 2.1193552017211914 (1.8170969687603615) + | > loss_feat: 9.176203727722168 (7.939031066121282) + | > loss_mel: 17.521892547607422 (16.826968379922818) + | > loss_duration: 0.9204121828079224 (0.9592225644234065) + | > loss_1: 32.24131774902344 (30.092685287063187) + | > grad_norm_1: tensor(211.7927, device='cuda:0') (tensor(479.6953, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9527 (0.9843317769669198) + | > loader_time: 0.0052 (0.006259781283301276) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03168355501615085 (+0.00012289560758151075) + | > avg_loss_disc: 2.549330766384418 (-0.03984328416677707) + | > avg_loss_disc_real_0: 0.1939135640859604 (-0.06221797374578625) + | > avg_loss_disc_real_1: 0.1921915870446425 (-0.07882709801197052) + | > avg_loss_disc_real_2: 0.27076012927752274 (-0.018988580657885612) + | > avg_loss_disc_real_3: 0.24642859399318695 (+0.025643377349926888) + | > avg_loss_disc_real_4: 0.25762922717974734 (-0.004374272548235414) + | > avg_loss_disc_real_5: 0.26713977524867427 (+0.06340482143255383) + | > avg_loss_0: 2.549330766384418 (-0.03984328416677707) + | > avg_loss_gen: 2.433118068254911 (-0.1671304519359884) + | > avg_loss_kl: 3.3870037335615892 (-0.32037193958575916) + | > avg_loss_feat: 7.687677310063289 (-0.4689561036916876) + | > avg_loss_mel: 16.710036131051872 (-0.7297845253577577) + | > avg_loss_duration: 1.2616203381465032 (-0.09434400154994083) + | > avg_loss_1: 31.479455801156853 (-1.7805858025184023) + + + > EPOCH: 52/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:21:00)  + + --> TIME: 2024-07-13 20:21:50 -- STEP: 47/151 -- GLOBAL_STEP: 92900 + | > loss_disc: 2.209089756011963 (2.295812297374644) + | > loss_disc_real_0: 0.08189727365970612 (0.11832579883172158) + | > loss_disc_real_1: 0.14361466467380524 (0.2000899761915207) + | > loss_disc_real_2: 0.16641217470169067 (0.2094397205621638) + | > loss_disc_real_3: 0.13399723172187805 (0.20615881713146858) + | > loss_disc_real_4: 0.16616405546665192 (0.20908407486499625) + | > loss_disc_real_5: 0.14598791301250458 (0.2173990987716837) + | > loss_0: 2.209089756011963 (2.295812297374644) + | > grad_norm_0: tensor(20.1964, device='cuda:0') (tensor(33.6376, device='cuda:0')) + | > loss_gen: 2.6251978874206543 (2.621422057456159) + | > loss_kl: 1.0486476421356201 (1.7230617923939482) + | > loss_feat: 9.963716506958008 (8.2038438472342) + | > loss_mel: 18.22625732421875 (16.889113974063953) + | > loss_duration: 0.9577678442001343 (0.9579918207006252) + | > loss_1: 32.82158660888672 (30.395433385321436) + | > grad_norm_1: tensor(597.5549, device='cuda:0') (tensor(597.5153, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8523 (1.0005611409532262) + | > loader_time: 0.0052 (0.006471644056604263) + + + --> TIME: 2024-07-13 20:22:37 -- STEP: 97/151 -- GLOBAL_STEP: 92950 + | > loss_disc: 2.3703911304473877 (2.3137149896818334) + | > loss_disc_real_0: 0.15755099058151245 (0.1258703403405308) + | > loss_disc_real_1: 0.2087962031364441 (0.20278268545558772) + | > loss_disc_real_2: 0.19362591207027435 (0.21135697721206037) + | > loss_disc_real_3: 0.18352296948432922 (0.20850143881188227) + | > loss_disc_real_4: 0.18121762573719025 (0.20939589668180525) + | > loss_disc_real_5: 0.22392655909061432 (0.21481548226678493) + | > loss_0: 2.3703911304473877 (2.3137149896818334) + | > grad_norm_0: tensor(43.0845, device='cuda:0') (tensor(35.7709, device='cuda:0')) + | > loss_gen: 2.491123676300049 (2.6212175064480183) + | > loss_kl: 1.9297397136688232 (1.7495563687737454) + | > loss_feat: 8.114090919494629 (8.192422915979765) + | > loss_mel: 16.509408950805664 (16.891956152375215) + | > loss_duration: 0.9145189523696899 (0.9454683249758691) + | > loss_1: 29.958881378173828 (30.400621197887304) + | > grad_norm_1: tensor(462.6729, device='cuda:0') (tensor(561.2755, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8875 (0.9686848650273588) + | > loader_time: 0.0051 (0.006219770490508718) + + + --> TIME: 2024-07-13 20:23:23 -- STEP: 147/151 -- GLOBAL_STEP: 93000 + | > loss_disc: 2.381976366043091 (2.307549641245886) + | > loss_disc_real_0: 0.12217988073825836 (0.12143813362535165) + | > loss_disc_real_1: 0.22234125435352325 (0.2038227621795369) + | > loss_disc_real_2: 0.23808324337005615 (0.2105537224180844) + | > loss_disc_real_3: 0.20171286165714264 (0.20902444756760888) + | > loss_disc_real_4: 0.22074691951274872 (0.20784200586023782) + | > loss_disc_real_5: 0.2158713936805725 (0.21578083613089152) + | > loss_0: 2.381976366043091 (2.307549641245886) + | > grad_norm_0: tensor(38.2535, device='cuda:0') (tensor(34.3225, device='cuda:0')) + | > loss_gen: 2.505706548690796 (2.6302936255526377) + | > loss_kl: 1.7377490997314453 (1.772134424472342) + | > loss_feat: 7.276596546173096 (8.19962015930488) + | > loss_mel: 15.97620677947998 (16.89735513803911) + | > loss_duration: 0.9191522002220154 (0.9447940129812072) + | > loss_1: 28.415409088134766 (30.444197349807844) + | > grad_norm_1: tensor(626.3013, device='cuda:0') (tensor(573.0329, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8648 (0.951539759733239) + | > loader_time: 0.0046 (0.00601195478114952) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03304631893451397 (+0.0013627639183631246) + | > avg_loss_disc: 2.6193762742556057 (+0.07004550787118768) + | > avg_loss_disc_real_0: 0.2517675780333005 (+0.05785401394734013) + | > avg_loss_disc_real_1: 0.192661213187071 (+0.00046962614242848644) + | > avg_loss_disc_real_2: 0.2086905470261207 (-0.06206958225140205) + | > avg_loss_disc_real_3: 0.22084687306330755 (-0.025581720929879398) + | > avg_loss_disc_real_4: 0.25342972118120927 (-0.004199505998538078) + | > avg_loss_disc_real_5: 0.2246522605419159 (-0.04248751470675838) + | > avg_loss_0: 2.6193762742556057 (+0.07004550787118768) + | > avg_loss_gen: 2.3418882810152493 (-0.09122978723966169) + | > avg_loss_kl: 3.438260738666241 (+0.05125700510465192) + | > avg_loss_feat: 7.785255065331092 (+0.09757775526780321) + | > avg_loss_mel: 17.135695530818058 (+0.42565939976618594) + | > avg_loss_duration: 1.26559762771313 (+0.003977289566626796) + | > avg_loss_1: 31.96669710599459 (+0.4872413048377382) + + + > EPOCH: 53/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:23:57)  + + --> TIME: 2024-07-13 20:24:44 -- STEP: 46/151 -- GLOBAL_STEP: 93050 + | > loss_disc: 2.285301923751831 (2.340529454791027) + | > loss_disc_real_0: 0.0745324194431305 (0.12740150515152057) + | > loss_disc_real_1: 0.19350889325141907 (0.20561743527650833) + | > loss_disc_real_2: 0.15377889573574066 (0.2097675291740376) + | > loss_disc_real_3: 0.18383526802062988 (0.20815639003463413) + | > loss_disc_real_4: 0.17686808109283447 (0.20720890909433365) + | > loss_disc_real_5: 0.21598200500011444 (0.2248535842999168) + | > loss_0: 2.285301923751831 (2.340529454791027) + | > grad_norm_0: tensor(13.3278, device='cuda:0') (tensor(21.0906, device='cuda:0')) + | > loss_gen: 2.6959240436553955 (2.5435100223707114) + | > loss_kl: 1.9383337497711182 (1.756794150756753) + | > loss_feat: 10.144712448120117 (8.005472939947378) + | > loss_mel: 18.351350784301758 (16.952265449192204) + | > loss_duration: 0.8824370503425598 (0.955713099759558) + | > loss_1: 34.012760162353516 (30.213755441748578) + | > grad_norm_1: tensor(177.6653, device='cuda:0') (tensor(330.6862, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8456 (0.9674232887185138) + | > loader_time: 0.0045 (0.005855850551439368) + + + --> TIME: 2024-07-13 20:25:34 -- STEP: 96/151 -- GLOBAL_STEP: 93100 + | > loss_disc: 2.2461113929748535 (2.332814739396176) + | > loss_disc_real_0: 0.1908000409603119 (0.1259429599158466) + | > loss_disc_real_1: 0.21419647336006165 (0.2035106671974063) + | > loss_disc_real_2: 0.2202085703611374 (0.21211183288445076) + | > loss_disc_real_3: 0.15192727744579315 (0.20779261722539863) + | > loss_disc_real_4: 0.20737771689891815 (0.2082293105777353) + | > loss_disc_real_5: 0.21537266671657562 (0.22348941086481014) + | > loss_0: 2.2461113929748535 (2.332814739396176) + | > grad_norm_0: tensor(35.8492, device='cuda:0') (tensor(27.2830, device='cuda:0')) + | > loss_gen: 2.8890063762664795 (2.5716195652882257) + | > loss_kl: 1.684216856956482 (1.7689629377176364) + | > loss_feat: 8.643651962280273 (8.090607911348345) + | > loss_mel: 16.534603118896484 (16.851190805435166) + | > loss_duration: 0.9041386842727661 (0.956085130572319) + | > loss_1: 30.655616760253906 (30.23846634229024) + | > grad_norm_1: tensor(907.5273, device='cuda:0') (tensor(446.4738, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8789 (0.9799499064683914) + | > loader_time: 0.0058 (0.006060888369878133) + + + --> TIME: 2024-07-13 20:26:23 -- STEP: 146/151 -- GLOBAL_STEP: 93150 + | > loss_disc: 2.4492855072021484 (2.3195097030025633) + | > loss_disc_real_0: 0.13639764487743378 (0.12262394970716681) + | > loss_disc_real_1: 0.21797087788581848 (0.2019632512780085) + | > loss_disc_real_2: 0.23675291240215302 (0.21128639905420074) + | > loss_disc_real_3: 0.1956523358821869 (0.20690819835417892) + | > loss_disc_real_4: 0.26385247707366943 (0.20879695579817847) + | > loss_disc_real_5: 0.24794547259807587 (0.22220540567211908) + | > loss_0: 2.4492855072021484 (2.3195097030025633) + | > grad_norm_0: tensor(41.6839, device='cuda:0') (tensor(29.6156, device='cuda:0')) + | > loss_gen: 2.4275600910186768 (2.5936913131034536) + | > loss_kl: 1.938730239868164 (1.766974123781675) + | > loss_feat: 6.935459136962891 (8.183999166096726) + | > loss_mel: 15.956562995910645 (16.911444716257574) + | > loss_duration: 1.191504716873169 (0.9559625550492169) + | > loss_1: 28.449819564819336 (30.41207189429296) + | > grad_norm_1: tensor(609.7928, device='cuda:0') (tensor(512.2233, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 2.1478 (0.9758014205383928) + | > loader_time: 0.015 (0.006041451676251137) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03307076600881723 (+2.4447074303261818e-05) + | > avg_loss_disc: 2.536753746179434 (-0.08262252807617188) + | > avg_loss_disc_real_0: 0.23434252922351545 (-0.01742504880978507) + | > avg_loss_disc_real_1: 0.25025272369384766 (+0.05759151050677666) + | > avg_loss_disc_real_2: 0.261019942852167 (+0.052329395826046315) + | > avg_loss_disc_real_3: 0.25020354642317844 (+0.029356673359870883) + | > avg_loss_disc_real_4: 0.24606457696511194 (-0.007365144216097325) + | > avg_loss_disc_real_5: 0.23789752102815187 (+0.013245260486235977) + | > avg_loss_0: 2.536753746179434 (-0.08262252807617188) + | > avg_loss_gen: 2.5925340835864725 (+0.25064580257122326) + | > avg_loss_kl: 3.3089437851539025 (-0.12931695351233863) + | > avg_loss_feat: 7.545685951526348 (-0.23956911380474377) + | > avg_loss_mel: 16.795133884136497 (-0.34056164668156086) + | > avg_loss_duration: 1.2726456568791316 (+0.007048029166001557) + | > avg_loss_1: 31.51494334294246 (-0.4517537630521318) + + + > EPOCH: 54/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:26:38)  + + --> TIME: 2024-07-13 20:27:25 -- STEP: 45/151 -- GLOBAL_STEP: 93200 + | > loss_disc: 2.2389063835144043 (2.3201867103576657) + | > loss_disc_real_0: 0.12597763538360596 (0.12454588487744331) + | > loss_disc_real_1: 0.19903752207756042 (0.20090489519966973) + | > loss_disc_real_2: 0.21887998282909393 (0.20884084900220234) + | > loss_disc_real_3: 0.17922066152095795 (0.20649874607721966) + | > loss_disc_real_4: 0.20350812375545502 (0.20771982338693407) + | > loss_disc_real_5: 0.2520661950111389 (0.22337333725558386) + | > loss_0: 2.2389063835144043 (2.3201867103576657) + | > grad_norm_0: tensor(55.5142, device='cuda:0') (tensor(27.4957, device='cuda:0')) + | > loss_gen: 2.832444190979004 (2.5745892683664957) + | > loss_kl: 1.7433000802993774 (1.6968526283899943) + | > loss_feat: 8.788122177124023 (8.269272168477377) + | > loss_mel: 17.624855041503906 (16.96218679216173) + | > loss_duration: 0.9952292442321777 (0.9487168391545614) + | > loss_1: 31.983949661254883 (30.451617262098523) + | > grad_norm_1: tensor(848.3911, device='cuda:0') (tensor(409.3564, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.873 (0.9933055400848388) + | > loader_time: 0.0058 (0.0061738544040256076) + + + --> TIME: 2024-07-13 20:28:13 -- STEP: 95/151 -- GLOBAL_STEP: 93250 + | > loss_disc: 2.3413829803466797 (2.3214438287835377) + | > loss_disc_real_0: 0.1171434074640274 (0.12594515618525046) + | > loss_disc_real_1: 0.16662752628326416 (0.20364692885624736) + | > loss_disc_real_2: 0.22001251578330994 (0.21048344389388432) + | > loss_disc_real_3: 0.20451803505420685 (0.20515847457082648) + | > loss_disc_real_4: 0.20957612991333008 (0.2076540943039091) + | > loss_disc_real_5: 0.1964249312877655 (0.21966638721917806) + | > loss_0: 2.3413829803466797 (2.3214438287835377) + | > grad_norm_0: tensor(41.5921, device='cuda:0') (tensor(35.2498, device='cuda:0')) + | > loss_gen: 2.5763731002807617 (2.599371460864419) + | > loss_kl: 1.7984704971313477 (1.7088740882120634) + | > loss_feat: 8.140928268432617 (8.182611269699901) + | > loss_mel: 16.81056785583496 (16.90578051115337) + | > loss_duration: 0.9260767698287964 (0.957762809176194) + | > loss_1: 30.252416610717773 (30.354399932058232) + | > grad_norm_1: tensor(836.5886, device='cuda:0') (tensor(487.6156, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.9867 (0.973148724907323) + | > loader_time: 0.0059 (0.006087742353740491) + + + --> TIME: 2024-07-13 20:29:01 -- STEP: 145/151 -- GLOBAL_STEP: 93300 + | > loss_disc: 2.3440287113189697 (2.323351174387439) + | > loss_disc_real_0: 0.10658864676952362 (0.12536346552700822) + | > loss_disc_real_1: 0.20478230714797974 (0.203626578326883) + | > loss_disc_real_2: 0.19886046648025513 (0.211447952636357) + | > loss_disc_real_3: 0.19166044890880585 (0.2057605718744212) + | > loss_disc_real_4: 0.19188566505908966 (0.20784276256273532) + | > loss_disc_real_5: 0.1459217667579651 (0.22030758693300448) + | > loss_0: 2.3440287113189697 (2.323351174387439) + | > grad_norm_0: tensor(23.8224, device='cuda:0') (tensor(33.3091, device='cuda:0')) + | > loss_gen: 2.536774158477783 (2.591776528851739) + | > loss_kl: 1.7005960941314697 (1.7222154835174825) + | > loss_feat: 8.863052368164062 (8.131571796022609) + | > loss_mel: 16.15634536743164 (16.83460576287632) + | > loss_duration: 1.04813551902771 (0.9518218061019634) + | > loss_1: 30.304903030395508 (30.23199128775761) + | > grad_norm_1: tensor(540.5983, device='cuda:0') (tensor(493.3057, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8313 (0.9601096103931296) + | > loader_time: 0.0039 (0.005978360669366245) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03266136462871845 (-0.0004094013800987825) + | > avg_loss_disc: 2.4885907539954553 (-0.04816299218397857) + | > avg_loss_disc_real_0: 0.16314186958166269 (-0.07120065964185276) + | > avg_loss_disc_real_1: 0.2374577774451329 (-0.012794946248714745) + | > avg_loss_disc_real_2: 0.21649080629532153 (-0.04452913655684548) + | > avg_loss_disc_real_3: 0.17819155981907478 (-0.07201198660410366) + | > avg_loss_disc_real_4: 0.237761907852613 (-0.00830266911249894) + | > avg_loss_disc_real_5: 0.2511834055185318 (+0.013285884490379929) + | > avg_loss_0: 2.4885907539954553 (-0.04816299218397857) + | > avg_loss_gen: 2.3239264488220215 (-0.26860763476445104) + | > avg_loss_kl: 3.4233094912308912 (+0.1143657060769887) + | > avg_loss_feat: 7.9166029049799995 (+0.3709169534536514) + | > avg_loss_mel: 17.172479776235726 (+0.3773458920992283) + | > avg_loss_duration: 1.2845951043642485 (+0.011949447485116949) + | > avg_loss_1: 32.12091416579027 (+0.6059708228478087) + + + > EPOCH: 55/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:29:18)  + + --> TIME: 2024-07-13 20:30:04 -- STEP: 44/151 -- GLOBAL_STEP: 93350 + | > loss_disc: 2.3821017742156982 (2.2933633544228296) + | > loss_disc_real_0: 0.09454774111509323 (0.11600726605816321) + | > loss_disc_real_1: 0.23227187991142273 (0.20663013004443861) + | > loss_disc_real_2: 0.24054332077503204 (0.2109317776154388) + | > loss_disc_real_3: 0.2170134037733078 (0.20574944280087948) + | > loss_disc_real_4: 0.27045148611068726 (0.2108093872666359) + | > loss_disc_real_5: 0.2710919976234436 (0.2217205319214951) + | > loss_0: 2.3821017742156982 (2.2933633544228296) + | > grad_norm_0: tensor(45.7094, device='cuda:0') (tensor(29.0696, device='cuda:0')) + | > loss_gen: 2.439937114715576 (2.628247233954343) + | > loss_kl: 1.5607590675354004 (1.7689335454594006) + | > loss_feat: 7.428637504577637 (8.17073712565682) + | > loss_mel: 16.039657592773438 (16.773354140194975) + | > loss_duration: 0.8999524116516113 (0.952156665650281) + | > loss_1: 28.36894416809082 (30.293428767811168) + | > grad_norm_1: tensor(834.8265, device='cuda:0') (tensor(563.0153, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.881 (0.9897693395614627) + | > loader_time: 0.0051 (0.006096503951332786) + + + --> TIME: 2024-07-13 20:30:52 -- STEP: 94/151 -- GLOBAL_STEP: 93400 + | > loss_disc: 2.2050342559814453 (2.3005583387740125) + | > loss_disc_real_0: 0.05828016251325607 (0.1186478549733441) + | > loss_disc_real_1: 0.2352450042963028 (0.20509946195686113) + | > loss_disc_real_2: 0.19925381243228912 (0.21030567982729445) + | > loss_disc_real_3: 0.18347226083278656 (0.20710831222699044) + | > loss_disc_real_4: 0.2728765606880188 (0.2091098412712838) + | > loss_disc_real_5: 0.16599015891551971 (0.21939598467755825) + | > loss_0: 2.2050342559814453 (2.3005583387740125) + | > grad_norm_0: tensor(34.6895, device='cuda:0') (tensor(29.8289, device='cuda:0')) + | > loss_gen: 2.6772351264953613 (2.624732656681792) + | > loss_kl: 1.7132325172424316 (1.7819270542327394) + | > loss_feat: 10.185123443603516 (8.280125344053232) + | > loss_mel: 17.90966033935547 (16.76375366779084) + | > loss_duration: 1.140695333480835 (0.9600643737519041) + | > loss_1: 33.625946044921875 (30.410603158017423) + | > grad_norm_1: tensor(533.3561, device='cuda:0') (tensor(553.1568, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8341 (0.9646680532617773) + | > loader_time: 0.0044 (0.005982515659738094) + + + --> TIME: 2024-07-13 20:31:39 -- STEP: 144/151 -- GLOBAL_STEP: 93450 + | > loss_disc: 2.4415504932403564 (2.307567091451751) + | > loss_disc_real_0: 0.1375032663345337 (0.11910046484424835) + | > loss_disc_real_1: 0.2402392327785492 (0.2061347151692542) + | > loss_disc_real_2: 0.25104713439941406 (0.21212881203326914) + | > loss_disc_real_3: 0.20149850845336914 (0.20724544446501467) + | > loss_disc_real_4: 0.28828078508377075 (0.20875617432304558) + | > loss_disc_real_5: 0.23836179077625275 (0.21823592049380144) + | > loss_0: 2.4415504932403564 (2.307567091451751) + | > grad_norm_0: tensor(49.1929, device='cuda:0') (tensor(32.1504, device='cuda:0')) + | > loss_gen: 2.5021073818206787 (2.6269686768452334) + | > loss_kl: 1.787915825843811 (1.7626254103249974) + | > loss_feat: 7.7332539558410645 (8.26975253224374) + | > loss_mel: 16.648147583007812 (16.793728444311352) + | > loss_duration: 0.894996166229248 (0.9541967656049464) + | > loss_1: 29.566421508789062 (30.40727190176646) + | > grad_norm_1: tensor(611.9385, device='cuda:0') (tensor(574.7723, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9417 (0.9559586279922061) + | > loader_time: 0.0056 (0.005898156099849277) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.035032217319195084 (+0.002370852690476634) + | > avg_loss_disc: 2.534010960505559 (+0.0454202065101037) + | > avg_loss_disc_real_0: 0.20404842152045324 (+0.04090655193879056) + | > avg_loss_disc_real_1: 0.2303106005375202 (-0.007147176907612712) + | > avg_loss_disc_real_2: 0.2643873588396953 (+0.04789655254437375) + | > avg_loss_disc_real_3: 0.2291222226161223 (+0.05093066279704753) + | > avg_loss_disc_real_4: 0.25455493881152225 (+0.016793030958909244) + | > avg_loss_disc_real_5: 0.21501574378747207 (-0.03616766173105973) + | > avg_loss_0: 2.534010960505559 (+0.0454202065101037) + | > avg_loss_gen: 2.4110216177426858 (+0.08709516892066427) + | > avg_loss_kl: 3.3628838612483096 (-0.06042562998258161) + | > avg_loss_feat: 7.367110399099497 (-0.5494925058805027) + | > avg_loss_mel: 16.567361098069412 (-0.6051186781663134) + | > avg_loss_duration: 1.3021929630866418 (+0.017597858722393278) + | > avg_loss_1: 31.01056949908917 (-1.1103446667010992) + + + > EPOCH: 56/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:31:55)  + + --> TIME: 2024-07-13 20:32:40 -- STEP: 43/151 -- GLOBAL_STEP: 93500 + | > loss_disc: 2.415673017501831 (2.3477119068766745) + | > loss_disc_real_0: 0.17807869613170624 (0.12722512655133425) + | > loss_disc_real_1: 0.17856618762016296 (0.20279345921305722) + | > loss_disc_real_2: 0.22979454696178436 (0.21460346600344016) + | > loss_disc_real_3: 0.19174043834209442 (0.2089088902223942) + | > loss_disc_real_4: 0.21855944395065308 (0.2137655269267947) + | > loss_disc_real_5: 0.2912026047706604 (0.22958696165750192) + | > loss_0: 2.415673017501831 (2.3477119068766745) + | > grad_norm_0: tensor(33.7484, device='cuda:0') (tensor(31.3544, device='cuda:0')) + | > loss_gen: 2.391021490097046 (2.567031688468401) + | > loss_kl: 1.7899506092071533 (1.7544478704763014) + | > loss_feat: 8.072193145751953 (8.26959299486737) + | > loss_mel: 17.599435806274414 (16.842985485875325) + | > loss_duration: 0.925774097442627 (0.9275066062461498) + | > loss_1: 30.77837371826172 (30.361564813658248) + | > grad_norm_1: tensor(502.3973, device='cuda:0') (tensor(469.0464, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9675 (0.9852715259374574) + | > loader_time: 0.0062 (0.006176211113153502) + + + --> TIME: 2024-07-13 20:33:30 -- STEP: 93/151 -- GLOBAL_STEP: 93550 + | > loss_disc: 2.4417219161987305 (2.3474730599311098) + | > loss_disc_real_0: 0.2267405390739441 (0.12598621660983694) + | > loss_disc_real_1: 0.2261209934949875 (0.20372791187737577) + | > loss_disc_real_2: 0.1850016564130783 (0.21283027786080555) + | > loss_disc_real_3: 0.18888550996780396 (0.20776093935453763) + | > loss_disc_real_4: 0.27287012338638306 (0.21266253596992904) + | > loss_disc_real_5: 0.21176627278327942 (0.226771988214985) + | > loss_0: 2.4417219161987305 (2.3474730599311098) + | > grad_norm_0: tensor(45.4085, device='cuda:0') (tensor(33.5360, device='cuda:0')) + | > loss_gen: 2.56817364692688 (2.579787884989093) + | > loss_kl: 1.9475771188735962 (1.7747599386399793) + | > loss_feat: 7.228193283081055 (8.2092966828295) + | > loss_mel: 17.682411193847656 (16.947611060193793) + | > loss_duration: 1.0530234575271606 (0.9451783363537122) + | > loss_1: 30.479379653930664 (30.456634090792747) + | > grad_norm_1: tensor(512.2150, device='cuda:0') (tensor(506.1682, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.1297 (0.9883349839077201) + | > loader_time: 0.0086 (0.006193040519632319) + + + --> TIME: 2024-07-13 20:34:18 -- STEP: 143/151 -- GLOBAL_STEP: 93600 + | > loss_disc: 1.9296603202819824 (2.3377014406911143) + | > loss_disc_real_0: 0.0667969286441803 (0.12439227854455277) + | > loss_disc_real_1: 0.14474190771579742 (0.20422160067966763) + | > loss_disc_real_2: 0.18727713823318481 (0.21234378823033578) + | > loss_disc_real_3: 0.18046380579471588 (0.2075694958855222) + | > loss_disc_real_4: 0.15794774889945984 (0.20989870577008574) + | > loss_disc_real_5: 0.18420939147472382 (0.2243793643229491) + | > loss_0: 1.9296603202819824 (2.3377014406911143) + | > grad_norm_0: tensor(15.2867, device='cuda:0') (tensor(33.1185, device='cuda:0')) + | > loss_gen: 2.891942262649536 (2.587116416517672) + | > loss_kl: 1.584266185760498 (1.7625504965548748) + | > loss_feat: 8.724525451660156 (8.224497124865339) + | > loss_mel: 16.534282684326172 (16.92827255409082) + | > loss_duration: 1.0989569425582886 (0.9452942518087534) + | > loss_1: 30.833974838256836 (30.447731018066406) + | > grad_norm_1: tensor(557.0714, device='cuda:0') (tensor(512.7955, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8086 (0.9724667855909654) + | > loader_time: 0.0035 (0.0060493662640765) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03113401853121243 (-0.0038981987879826534) + | > avg_loss_disc: 2.6336656166956973 (+0.09965465619013836) + | > avg_loss_disc_real_0: 0.20194760194191566 (-0.0021008195785375827) + | > avg_loss_disc_real_1: 0.21527499304367945 (-0.015035607493840752) + | > avg_loss_disc_real_2: 0.22980189323425293 (-0.034585465605442345) + | > avg_loss_disc_real_3: 0.2208232317979519 (-0.008298990818170399) + | > avg_loss_disc_real_4: 0.19409868808893058 (-0.06045625072259167) + | > avg_loss_disc_real_5: 0.18543709585299858 (-0.029578647934473484) + | > avg_loss_0: 2.6336656166956973 (+0.09965465619013836) + | > avg_loss_gen: 2.127343489573552 (-0.2836781281691336) + | > avg_loss_kl: 3.5214818440950832 (+0.15859798284677362) + | > avg_loss_feat: 7.605563200437105 (+0.23845280133760838) + | > avg_loss_mel: 16.68299968426044 (+0.11563858619102874) + | > avg_loss_duration: 1.2691805637799776 (-0.03301239930666422) + | > avg_loss_1: 31.206568938035232 (+0.19599943894606398) + + + > EPOCH: 57/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:34:35)  + + --> TIME: 2024-07-13 20:35:17 -- STEP: 42/151 -- GLOBAL_STEP: 93650 + | > loss_disc: 2.4099442958831787 (2.361571062178839) + | > loss_disc_real_0: 0.08481617271900177 (0.1314051007585866) + | > loss_disc_real_1: 0.2358720898628235 (0.2058140786275977) + | > loss_disc_real_2: 0.2011997252702713 (0.21792147947209223) + | > loss_disc_real_3: 0.25088655948638916 (0.20900594088293256) + | > loss_disc_real_4: 0.21025390923023224 (0.20979423359746024) + | > loss_disc_real_5: 0.28895303606987 (0.22054838211763472) + | > loss_0: 2.4099442958831787 (2.361571062178839) + | > grad_norm_0: tensor(12.0711, device='cuda:0') (tensor(38.9452, device='cuda:0')) + | > loss_gen: 2.4059855937957764 (2.576498417627244) + | > loss_kl: 1.5521999597549438 (1.791390441712879) + | > loss_feat: 7.195796489715576 (8.185360340845016) + | > loss_mel: 16.292003631591797 (17.105942998613635) + | > loss_duration: 1.0047414302825928 (0.9477769831816355) + | > loss_1: 28.450727462768555 (30.60696910676502) + | > grad_norm_1: tensor(479.0600, device='cuda:0') (tensor(594.4381, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.852 (0.9350998231342861) + | > loader_time: 0.0044 (0.005844576018197196) + + + --> TIME: 2024-07-13 20:36:06 -- STEP: 92/151 -- GLOBAL_STEP: 93700 + | > loss_disc: 2.219404458999634 (2.3595351084418925) + | > loss_disc_real_0: 0.11091315746307373 (0.1297937086580888) + | > loss_disc_real_1: 0.1654096394777298 (0.20549602296365344) + | > loss_disc_real_2: 0.1767135113477707 (0.21500747634664827) + | > loss_disc_real_3: 0.204875648021698 (0.21050484024960062) + | > loss_disc_real_4: 0.1796405017375946 (0.210351567796391) + | > loss_disc_real_5: 0.2666222155094147 (0.2210849346997945) + | > loss_0: 2.219404458999634 (2.3595351084418925) + | > grad_norm_0: tensor(23.8979, device='cuda:0') (tensor(34.5363, device='cuda:0')) + | > loss_gen: 2.641476631164551 (2.5498149291328773) + | > loss_kl: 0.9894577264785767 (1.7608159689799598) + | > loss_feat: 9.281970024108887 (7.9947316491085525) + | > loss_mel: 16.74884605407715 (16.856658759324457) + | > loss_duration: 1.1558388471603394 (0.9493990015724431) + | > loss_1: 30.817588806152344 (30.111420258231785) + | > grad_norm_1: tensor(566.9988, device='cuda:0') (tensor(506.2773, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8157 (0.9539056990457617) + | > loader_time: 0.0039 (0.006144396636797034) + + + --> TIME: 2024-07-13 20:36:56 -- STEP: 142/151 -- GLOBAL_STEP: 93750 + | > loss_disc: 2.1493825912475586 (2.3530121118250036) + | > loss_disc_real_0: 0.11433251202106476 (0.12863282688801558) + | > loss_disc_real_1: 0.1965995579957962 (0.20390498013773434) + | > loss_disc_real_2: 0.18468506634235382 (0.21364579318274915) + | > loss_disc_real_3: 0.1707594245672226 (0.2109767777819029) + | > loss_disc_real_4: 0.13477180898189545 (0.2104157013687449) + | > loss_disc_real_5: 0.2008853256702423 (0.22077818340818647) + | > loss_0: 2.1493825912475586 (2.3530121118250036) + | > grad_norm_0: tensor(21.6714, device='cuda:0') (tensor(28.7021, device='cuda:0')) + | > loss_gen: 2.59567928314209 (2.5406614340526965) + | > loss_kl: 1.5769480466842651 (1.7687331706705225) + | > loss_feat: 7.784964084625244 (8.018732638426226) + | > loss_mel: 15.004199028015137 (16.79240310024209) + | > loss_duration: 1.072378158569336 (0.9513532792178678) + | > loss_1: 28.034170150756836 (30.0718836045601) + | > grad_norm_1: tensor(455.5623, device='cuda:0') (tensor(421.6252, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8015 (0.9663588245150069) + | > loader_time: 0.004 (0.006157099361151038) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032430098606989935 (+0.0012960800757775039) + | > avg_loss_disc: 2.6057323859288144 (-0.02793323076688292) + | > avg_loss_disc_real_0: 0.26385568426205563 (+0.06190808232013997) + | > avg_loss_disc_real_1: 0.1928875079521766 (-0.022387485091502846) + | > avg_loss_disc_real_2: 0.2726722932778872 (+0.042870400043634294) + | > avg_loss_disc_real_3: 0.2244536177470134 (+0.003630385949061482) + | > avg_loss_disc_real_4: 0.24135105655743525 (+0.04725236846850467) + | > avg_loss_disc_real_5: 0.21402927889273718 (+0.028592183039738595) + | > avg_loss_0: 2.6057323859288144 (-0.02793323076688292) + | > avg_loss_gen: 2.4162393533266506 (+0.2888958637530985) + | > avg_loss_kl: 3.4798439832834096 (-0.041637860811673644) + | > avg_loss_feat: 7.790362101334792 (+0.1847989008976869) + | > avg_loss_mel: 17.210651544424202 (+0.5276518601637612) + | > avg_loss_duration: 1.3315904232171865 (+0.06240985943720889) + | > avg_loss_1: 32.22868772653433 (+1.0221187884990997) + + + > EPOCH: 58/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:37:15)  + + --> TIME: 2024-07-13 20:37:58 -- STEP: 41/151 -- GLOBAL_STEP: 93800 + | > loss_disc: 2.206916570663452 (2.3241890232737465) + | > loss_disc_real_0: 0.11898033320903778 (0.11633877219950281) + | > loss_disc_real_1: 0.22259381413459778 (0.20540449604755495) + | > loss_disc_real_2: 0.2474311888217926 (0.21376833537729775) + | > loss_disc_real_3: 0.17736345529556274 (0.2092981465706011) + | > loss_disc_real_4: 0.19013875722885132 (0.20729403088732465) + | > loss_disc_real_5: 0.24633386731147766 (0.21832844869392673) + | > loss_0: 2.206916570663452 (2.3241890232737465) + | > grad_norm_0: tensor(9.6783, device='cuda:0') (tensor(26.8341, device='cuda:0')) + | > loss_gen: 2.8292222023010254 (2.5796257112084358) + | > loss_kl: 1.8804399967193604 (1.806812335805195) + | > loss_feat: 9.38667106628418 (8.24001793745088) + | > loss_mel: 17.399864196777344 (16.86579029734542) + | > loss_duration: 0.9072808027267456 (0.9675302214738799) + | > loss_1: 32.40347671508789 (30.459776482930998) + | > grad_norm_1: tensor(400.0973, device='cuda:0') (tensor(489.2452, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8584 (0.9941249766000887) + | > loader_time: 0.0042 (0.006553434744113829) + + + --> TIME: 2024-07-13 20:38:46 -- STEP: 91/151 -- GLOBAL_STEP: 93850 + | > loss_disc: 2.403444766998291 (2.3334242799779887) + | > loss_disc_real_0: 0.28269797563552856 (0.12445516556828887) + | > loss_disc_real_1: 0.19792431592941284 (0.20406958493557606) + | > loss_disc_real_2: 0.21299925446510315 (0.21421218196769337) + | > loss_disc_real_3: 0.20257893204689026 (0.20916087892684307) + | > loss_disc_real_4: 0.2002377063035965 (0.20703469658945942) + | > loss_disc_real_5: 0.22002936899662018 (0.21951884014920875) + | > loss_0: 2.403444766998291 (2.3334242799779887) + | > grad_norm_0: tensor(88.1153, device='cuda:0') (tensor(30.4124, device='cuda:0')) + | > loss_gen: 2.7626326084136963 (2.579809136443086) + | > loss_kl: 1.7727850675582886 (1.8182634246218337) + | > loss_feat: 7.819136142730713 (8.152956312829323) + | > loss_mel: 16.076614379882812 (16.847662957160026) + | > loss_duration: 1.040544033050537 (0.9606531282047649) + | > loss_1: 29.471712112426758 (30.359344880659503) + | > grad_norm_1: tensor(684.4243, device='cuda:0') (tensor(464.6416, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8554 (0.9720267830314216) + | > loader_time: 0.0048 (0.006256197835062887) + + + --> TIME: 2024-07-13 20:39:36 -- STEP: 141/151 -- GLOBAL_STEP: 93900 + | > loss_disc: 2.321338176727295 (2.346971943023358) + | > loss_disc_real_0: 0.1201334223151207 (0.1287070621888266) + | > loss_disc_real_1: 0.19401590526103973 (0.20402363661333178) + | > loss_disc_real_2: 0.22945095598697662 (0.21411045360649733) + | > loss_disc_real_3: 0.21433423459529877 (0.2101366786246604) + | > loss_disc_real_4: 0.19563932716846466 (0.20804612296270142) + | > loss_disc_real_5: 0.17490549385547638 (0.22021712605834856) + | > loss_0: 2.321338176727295 (2.346971943023358) + | > grad_norm_0: tensor(12.5342, device='cuda:0') (tensor(27.8324, device='cuda:0')) + | > loss_gen: 2.5297930240631104 (2.5604018116673695) + | > loss_kl: 1.5047956705093384 (1.8073452592741512) + | > loss_feat: 7.1920552253723145 (8.104565055657783) + | > loss_mel: 15.465402603149414 (16.840891067017893) + | > loss_duration: 0.9304705858230591 (0.9565572725965622) + | > loss_1: 27.62251853942871 (30.269760456491024) + | > grad_norm_1: tensor(260.8282, device='cuda:0') (tensor(393.8593, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8138 (0.9726061905529482) + | > loader_time: 0.0039 (0.006149675829190735) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03165246890141414 (-0.0007776297055757941) + | > avg_loss_disc: 2.585263032179612 (-0.020469353749202224) + | > avg_loss_disc_real_0: 0.2596813463247739 (-0.004174337937281725) + | > avg_loss_disc_real_1: 0.2084182512301665 (+0.015530743277989895) + | > avg_loss_disc_real_2: 0.23680345026346353 (-0.035868843014423696) + | > avg_loss_disc_real_3: 0.22663916876682869 (+0.002185551019815296) + | > avg_loss_disc_real_4: 0.26983631001069 (+0.028485253453254727) + | > avg_loss_disc_real_5: 0.19591534710847414 (-0.018113931784263043) + | > avg_loss_0: 2.585263032179612 (-0.020469353749202224) + | > avg_loss_gen: 2.432268179379977 (+0.01602882605332656) + | > avg_loss_kl: 3.5338834065657396 (+0.054039423282330024) + | > avg_loss_feat: 7.765109318953294 (-0.025252782381498307) + | > avg_loss_mel: 17.230874648460976 (+0.02022310403677352) + | > avg_loss_duration: 1.3253188224939199 (-0.0062716007232666016) + | > avg_loss_1: 32.28745416494516 (+0.05876643841082796) + + + > EPOCH: 59/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:39:54)  + + --> TIME: 2024-07-13 20:40:33 -- STEP: 40/151 -- GLOBAL_STEP: 93950 + | > loss_disc: 2.3756089210510254 (2.3247583389282225) + | > loss_disc_real_0: 0.19646011292934418 (0.12663164213299744) + | > loss_disc_real_1: 0.1926192194223404 (0.20440176613628863) + | > loss_disc_real_2: 0.18277330696582794 (0.20595677718520164) + | > loss_disc_real_3: 0.19526435434818268 (0.20401551462709905) + | > loss_disc_real_4: 0.15324334800243378 (0.20619998648762702) + | > loss_disc_real_5: 0.1988123506307602 (0.2221995174884796) + | > loss_0: 2.3756089210510254 (2.3247583389282225) + | > grad_norm_0: tensor(32.4963, device='cuda:0') (tensor(21.1879, device='cuda:0')) + | > loss_gen: 2.7629544734954834 (2.549663519859314) + | > loss_kl: 2.136035442352295 (1.763279674947262) + | > loss_feat: 9.899499893188477 (8.0224259018898) + | > loss_mel: 17.277463912963867 (16.540267896652217) + | > loss_duration: 0.8761005401611328 (0.9531272977590561) + | > loss_1: 32.952056884765625 (29.82876410484314) + | > grad_norm_1: tensor(351.4112, device='cuda:0') (tensor(329.8606, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9411 (0.9300494313240051) + | > loader_time: 0.0067 (0.005648183822631836) + + + --> TIME: 2024-07-13 20:41:24 -- STEP: 90/151 -- GLOBAL_STEP: 94000 + | > loss_disc: 2.319385528564453 (2.3188786096043055) + | > loss_disc_real_0: 0.10151685774326324 (0.12601056082381137) + | > loss_disc_real_1: 0.20681288838386536 (0.2031070081724061) + | > loss_disc_real_2: 0.2091463804244995 (0.21017568293544978) + | > loss_disc_real_3: 0.20739957690238953 (0.2072434360782306) + | > loss_disc_real_4: 0.22444143891334534 (0.20726601572500336) + | > loss_disc_real_5: 0.19726479053497314 (0.2178436486257447) + | > loss_0: 2.319385528564453 (2.3188786096043055) + | > grad_norm_0: tensor(36.4050, device='cuda:0') (tensor(25.6898, device='cuda:0')) + | > loss_gen: 2.6159071922302246 (2.5816411521699694) + | > loss_kl: 2.1979055404663086 (1.8215386463536156) + | > loss_feat: 8.309609413146973 (8.085181628333201) + | > loss_mel: 17.822690963745117 (16.710800255669476) + | > loss_duration: 0.9191220998764038 (0.9521101978090074) + | > loss_1: 31.865236282348633 (30.151271947224934) + | > grad_norm_1: tensor(735.3241, device='cuda:0') (tensor(408.3301, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.0181 (0.970820058716668) + | > loader_time: 0.0076 (0.006196832656860353) + + + --> TIME: 2024-07-13 20:42:34 -- STEP: 140/151 -- GLOBAL_STEP: 94050 + | > loss_disc: 2.1024975776672363 (2.323443088361195) + | > loss_disc_real_0: 0.11441781371831894 (0.12412763203361202) + | > loss_disc_real_1: 0.1754554659128189 (0.20213565049426896) + | > loss_disc_real_2: 0.1770593225955963 (0.211983485519886) + | > loss_disc_real_3: 0.2045181840658188 (0.20831940663712367) + | > loss_disc_real_4: 0.19918206334114075 (0.20915552297873155) + | > loss_disc_real_5: 0.17888769507408142 (0.21846983113459179) + | > loss_0: 2.1024975776672363 (2.323443088361195) + | > grad_norm_0: tensor(26.3553, device='cuda:0') (tensor(25.1194, device='cuda:0')) + | > loss_gen: 2.855860710144043 (2.576731273106166) + | > loss_kl: 1.7911272048950195 (1.8200193979910442) + | > loss_feat: 9.437390327453613 (8.07080812113626) + | > loss_mel: 18.178178787231445 (16.733734764371572) + | > loss_duration: 0.8678435683250427 (0.9486949392727443) + | > loss_1: 33.130401611328125 (30.149988569532123) + | > grad_norm_1: tensor(556.5198, device='cuda:0') (tensor(408.7228, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9009 (0.9748362115451268) + | > loader_time: 0.0047 (0.006223535537719728) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032002999232365534 (+0.00035053033095139297) + | > avg_loss_disc: 2.526002627152663 (-0.05926040502694896) + | > avg_loss_disc_real_0: 0.23664773198274466 (-0.023033614342029246) + | > avg_loss_disc_real_1: 0.21758878918794486 (+0.009170537957778363) + | > avg_loss_disc_real_2: 0.25545360147953033 (+0.018650151216066807) + | > avg_loss_disc_real_3: 0.24620157709488502 (+0.019562408328056335) + | > avg_loss_disc_real_4: 0.23394416272640228 (-0.03589214728428769) + | > avg_loss_disc_real_5: 0.24210171745373651 (+0.04618637034526238) + | > avg_loss_0: 2.526002627152663 (-0.05926040502694896) + | > avg_loss_gen: 2.5574559615208554 (+0.12518778214087822) + | > avg_loss_kl: 3.598717029278095 (+0.06483362271235515) + | > avg_loss_feat: 8.232494611006517 (+0.46738529205322354) + | > avg_loss_mel: 17.399084751422592 (+0.16821010296161631) + | > avg_loss_duration: 1.296851029762855 (-0.02846779273106481) + | > avg_loss_1: 33.08460323627178 (+0.797149071326622) + + + > EPOCH: 60/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:42:54)  + + --> TIME: 2024-07-13 20:43:38 -- STEP: 39/151 -- GLOBAL_STEP: 94100 + | > loss_disc: 2.1351161003112793 (2.3172506980406933) + | > loss_disc_real_0: 0.08147507905960083 (0.1193975652448642) + | > loss_disc_real_1: 0.15503287315368652 (0.20650664621438736) + | > loss_disc_real_2: 0.16650301218032837 (0.2121481620348417) + | > loss_disc_real_3: 0.17345532774925232 (0.20535308733964577) + | > loss_disc_real_4: 0.23868323862552643 (0.21008990361140326) + | > loss_disc_real_5: 0.2026555836200714 (0.21877318506057447) + | > loss_0: 2.1351161003112793 (2.3172506980406933) + | > grad_norm_0: tensor(30.0296, device='cuda:0') (tensor(25.7508, device='cuda:0')) + | > loss_gen: 2.675384044647217 (2.581907975368011) + | > loss_kl: 1.779860496520996 (1.8115142155916264) + | > loss_feat: 8.171807289123535 (7.972462250636174) + | > loss_mel: 16.972665786743164 (16.60922143398187) + | > loss_duration: 0.8561574220657349 (0.9607055202508584) + | > loss_1: 30.455875396728516 (29.935811458489834) + | > grad_norm_1: tensor(322.1914, device='cuda:0') (tensor(512.8520, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8738 (1.0789661957667422) + | > loader_time: 0.0058 (0.00683289919144068) + + + --> TIME: 2024-07-13 20:44:26 -- STEP: 89/151 -- GLOBAL_STEP: 94150 + | > loss_disc: 2.208649158477783 (2.3168661514025057) + | > loss_disc_real_0: 0.07070183753967285 (0.12465634473254171) + | > loss_disc_real_1: 0.2030443698167801 (0.2039926599585608) + | > loss_disc_real_2: 0.1720937043428421 (0.21277769696846438) + | > loss_disc_real_3: 0.17331959307193756 (0.20723881229255978) + | > loss_disc_real_4: 0.17530862987041473 (0.21105815436732903) + | > loss_disc_real_5: 0.15955857932567596 (0.21668221324347378) + | > loss_0: 2.208649158477783 (2.3168661514025057) + | > grad_norm_0: tensor(18.5447, device='cuda:0') (tensor(33.2763, device='cuda:0')) + | > loss_gen: 2.56361722946167 (2.6206842326046376) + | > loss_kl: 1.387769103050232 (1.7734104080146618) + | > loss_feat: 9.030972480773926 (8.163929682099418) + | > loss_mel: 17.157331466674805 (16.861847502461973) + | > loss_duration: 0.888757586479187 (0.9548174310266302) + | > loss_1: 31.0284481048584 (30.37468927361992) + | > grad_norm_1: tensor(302.1315, device='cuda:0') (tensor(548.3998, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8507 (1.0067231387234803) + | > loader_time: 0.006 (0.006556821673103933) + + + --> TIME: 2024-07-13 20:45:16 -- STEP: 139/151 -- GLOBAL_STEP: 94200 + | > loss_disc: 2.2470431327819824 (2.324910999202043) + | > loss_disc_real_0: 0.1084122359752655 (0.12456109771411196) + | > loss_disc_real_1: 0.18534284830093384 (0.2036628313630605) + | > loss_disc_real_2: 0.20796258747577667 (0.21320225480649108) + | > loss_disc_real_3: 0.18531496822834015 (0.20822381769581666) + | > loss_disc_real_4: 0.24250827729701996 (0.20963256082517637) + | > loss_disc_real_5: 0.22967907786369324 (0.21993968072983858) + | > loss_0: 2.2470431327819824 (2.324910999202043) + | > grad_norm_0: tensor(21.4365, device='cuda:0') (tensor(31.8295, device='cuda:0')) + | > loss_gen: 2.6116223335266113 (2.5968369348443665) + | > loss_kl: 1.8935080766677856 (1.7687120296114638) + | > loss_feat: 8.078498840332031 (8.155673196847493) + | > loss_mel: 17.1182861328125 (16.77037931346206) + | > loss_duration: 0.9514895081520081 (0.9578682385760245) + | > loss_1: 30.653406143188477 (30.249469770802012) + | > grad_norm_1: tensor(470.4563, device='cuda:0') (tensor(512.6955, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8736 (0.9955126210082345) + | > loader_time: 0.0049 (0.006458261887804209) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03169059753417969 (-0.0003124016981858463) + | > avg_loss_disc: 2.534423131209153 (+0.008420504056489975) + | > avg_loss_disc_real_0: 0.2654096335172653 (+0.028761901534520656) + | > avg_loss_disc_real_1: 0.1976209649672875 (-0.01996782422065735) + | > avg_loss_disc_real_2: 0.20256740886431474 (-0.0528861926152156) + | > avg_loss_disc_real_3: 0.22143795283941123 (-0.024763624255473793) + | > avg_loss_disc_real_4: 0.2160471918491217 (-0.017896970877280594) + | > avg_loss_disc_real_5: 0.22525591804431036 (-0.016845799409426154) + | > avg_loss_0: 2.534423131209153 (+0.008420504056489975) + | > avg_loss_gen: 2.4396480046785793 (-0.11780795684227607) + | > avg_loss_kl: 3.5625444925748386 (-0.03617253670325615) + | > avg_loss_feat: 8.108953145834116 (-0.12354146517240139) + | > avg_loss_mel: 17.12789696913499 (-0.2711877822876012) + | > avg_loss_duration: 1.365536011182345 (+0.06868498141948987) + | > avg_loss_1: 32.6045787517841 (-0.48002448448767865) + + + > EPOCH: 61/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:45:37)  + + --> TIME: 2024-07-13 20:46:16 -- STEP: 38/151 -- GLOBAL_STEP: 94250 + | > loss_disc: 2.258995294570923 (2.3258357236259863) + | > loss_disc_real_0: 0.10283760726451874 (0.12244334473813835) + | > loss_disc_real_1: 0.18284103274345398 (0.20741388633062965) + | > loss_disc_real_2: 0.16364362835884094 (0.20954079298596634) + | > loss_disc_real_3: 0.1616310030221939 (0.21001268609573967) + | > loss_disc_real_4: 0.2153322994709015 (0.20556837006619103) + | > loss_disc_real_5: 0.23724734783172607 (0.22463465834918775) + | > loss_0: 2.258995294570923 (2.3258357236259863) + | > grad_norm_0: tensor(19.6853, device='cuda:0') (tensor(27.2191, device='cuda:0')) + | > loss_gen: 2.401567220687866 (2.598263646426954) + | > loss_kl: 1.3817216157913208 (1.7341939430487783) + | > loss_feat: 8.611307144165039 (8.184853704352127) + | > loss_mel: 16.36176872253418 (16.83382905156989) + | > loss_duration: 0.8848732709884644 (0.9514867616327185) + | > loss_1: 29.641237258911133 (30.30262696115594) + | > grad_norm_1: tensor(244.0438, device='cuda:0') (tensor(523.7026, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8528 (0.9694606630425704) + | > loader_time: 0.0043 (0.005890281576859324) + + + --> TIME: 2024-07-13 20:47:07 -- STEP: 88/151 -- GLOBAL_STEP: 94300 + | > loss_disc: 2.1691904067993164 (2.3153568953275685) + | > loss_disc_real_0: 0.05109143257141113 (0.12210819150575182) + | > loss_disc_real_1: 0.20920328795909882 (0.203159975734624) + | > loss_disc_real_2: 0.17025311291217804 (0.21346955018287356) + | > loss_disc_real_3: 0.1945638209581375 (0.2083794657479633) + | > loss_disc_real_4: 0.18694978952407837 (0.20864059911532837) + | > loss_disc_real_5: 0.21003738045692444 (0.22010610506615855) + | > loss_0: 2.1691904067993164 (2.3153568953275685) + | > grad_norm_0: tensor(21.6530, device='cuda:0') (tensor(30.8723, device='cuda:0')) + | > loss_gen: 2.6002655029296875 (2.610554898327048) + | > loss_kl: 1.7821626663208008 (1.765873055566441) + | > loss_feat: 9.84249210357666 (8.205065431919962) + | > loss_mel: 16.75638198852539 (16.780973564494744) + | > loss_duration: 0.9348950386047363 (0.961453143845905) + | > loss_1: 31.916196823120117 (30.323920119892467) + | > grad_norm_1: tensor(320.8846, device='cuda:0') (tensor(552.3392, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8457 (0.9862968135963787) + | > loader_time: 0.0044 (0.0062497366558421745) + + + --> TIME: 2024-07-13 20:47:56 -- STEP: 138/151 -- GLOBAL_STEP: 94350 + | > loss_disc: 2.019866466522217 (2.316275847994763) + | > loss_disc_real_0: 0.06261640042066574 (0.1230760296617729) + | > loss_disc_real_1: 0.15822896361351013 (0.20370924893928608) + | > loss_disc_real_2: 0.17910948395729065 (0.21288223672604217) + | > loss_disc_real_3: 0.16846708953380585 (0.20745938713999762) + | > loss_disc_real_4: 0.1720186322927475 (0.2088545978717182) + | > loss_disc_real_5: 0.16461686789989471 (0.21933285632859106) + | > loss_0: 2.019866466522217 (2.316275847994763) + | > grad_norm_0: tensor(22.8758, device='cuda:0') (tensor(31.9986, device='cuda:0')) + | > loss_gen: 2.8064846992492676 (2.5971996905147163) + | > loss_kl: 1.533133864402771 (1.7856600863346155) + | > loss_feat: 9.816346168518066 (8.053762506747587) + | > loss_mel: 17.639883041381836 (16.758225171462364) + | > loss_duration: 0.8828802108764648 (0.9545371964357902) + | > loss_1: 32.67872619628906 (30.149384692095328) + | > grad_norm_1: tensor(547.2531, device='cuda:0') (tensor(545.1673, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8366 (0.9782682605411696) + | > loader_time: 0.0043 (0.006244424460590751) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03220360095684345 (+0.0005130034226637631) + | > avg_loss_disc: 2.5016413835378795 (-0.03278174767127373) + | > avg_loss_disc_real_0: 0.22460198860902053 (-0.04080764490824479) + | > avg_loss_disc_real_1: 0.19840389834000513 (+0.0007829333727176202) + | > avg_loss_disc_real_2: 0.2419828325510025 (+0.03941542368668777) + | > avg_loss_disc_real_3: 0.21392307029320642 (-0.007514882546204804) + | > avg_loss_disc_real_4: 0.1869733975483821 (-0.029073794300739586) + | > avg_loss_disc_real_5: 0.21726247439017662 (-0.007993443654133736) + | > avg_loss_0: 2.5016413835378795 (-0.03278174767127373) + | > avg_loss_gen: 2.337119744374202 (-0.10252826030437756) + | > avg_loss_kl: 3.537099526478694 (-0.02544496609614466) + | > avg_loss_feat: 7.995925426483154 (-0.11302771935096168) + | > avg_loss_mel: 17.306552740243767 (+0.17865577110877595) + | > avg_loss_duration: 1.2945864475690405 (-0.0709495636133044) + | > avg_loss_1: 32.47128398601826 (-0.13329476576584653) + + + > EPOCH: 62/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:48:17)  + + --> TIME: 2024-07-13 20:48:57 -- STEP: 37/151 -- GLOBAL_STEP: 94400 + | > loss_disc: 2.430581569671631 (2.3347018860481885) + | > loss_disc_real_0: 0.1580841839313507 (0.13075206609996584) + | > loss_disc_real_1: 0.23743949830532074 (0.20365550872441884) + | > loss_disc_real_2: 0.25118038058280945 (0.2131783958222415) + | > loss_disc_real_3: 0.233287513256073 (0.20547655949721466) + | > loss_disc_real_4: 0.2190316915512085 (0.2126448053765941) + | > loss_disc_real_5: 0.2491835504770279 (0.21619316291164709) + | > loss_0: 2.430581569671631 (2.3347018860481885) + | > grad_norm_0: tensor(34.7275, device='cuda:0') (tensor(34.1200, device='cuda:0')) + | > loss_gen: 2.666131019592285 (2.5918768354364343) + | > loss_kl: 1.9348523616790771 (1.824146522058023) + | > loss_feat: 6.871057033538818 (7.98853401235632) + | > loss_mel: 17.060640335083008 (16.72848232372387) + | > loss_duration: 0.9715455770492554 (0.9440235801645227) + | > loss_1: 29.504226684570312 (30.077063122311152) + | > grad_norm_1: tensor(219.4622, device='cuda:0') (tensor(498.5146, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9805 (1.007117915797878) + | > loader_time: 0.0058 (0.006347636918763857) + + + --> TIME: 2024-07-13 20:49:49 -- STEP: 87/151 -- GLOBAL_STEP: 94450 + | > loss_disc: 2.2408366203308105 (2.345417066552174) + | > loss_disc_real_0: 0.121103435754776 (0.128808319568634) + | > loss_disc_real_1: 0.18836061656475067 (0.20982281692411708) + | > loss_disc_real_2: 0.21255064010620117 (0.21572714461677375) + | > loss_disc_real_3: 0.24238455295562744 (0.20847996529834023) + | > loss_disc_real_4: 0.22616015374660492 (0.2124006784167783) + | > loss_disc_real_5: 0.2409786731004715 (0.21575437520427265) + | > loss_0: 2.2408366203308105 (2.345417066552174) + | > grad_norm_0: tensor(29.1780, device='cuda:0') (tensor(36.4933, device='cuda:0')) + | > loss_gen: 2.662651777267456 (2.5958156640502223) + | > loss_kl: 1.6407865285873413 (1.8269760430544273) + | > loss_feat: 7.959107398986816 (8.003366700534162) + | > loss_mel: 15.952020645141602 (16.776938602842133) + | > loss_duration: 0.8680222034454346 (0.9521934246194774) + | > loss_1: 29.08258819580078 (30.155290274784484) + | > grad_norm_1: tensor(572.5101, device='cuda:0') (tensor(562.0699, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8477 (1.016150389594593) + | > loader_time: 0.0043 (0.006603509530253794) + + + --> TIME: 2024-07-13 20:50:39 -- STEP: 137/151 -- GLOBAL_STEP: 94500 + | > loss_disc: 2.5343470573425293 (2.3263067276808473) + | > loss_disc_real_0: 0.1524786353111267 (0.12549243715122657) + | > loss_disc_real_1: 0.2314414530992508 (0.2074604485809368) + | > loss_disc_real_2: 0.24153636395931244 (0.21360938142250924) + | > loss_disc_real_3: 0.2634113132953644 (0.20650287750211074) + | > loss_disc_real_4: 0.2482019066810608 (0.2082953415539143) + | > loss_disc_real_5: 0.2428653985261917 (0.2166469463684263) + | > loss_0: 2.5343470573425293 (2.3263067276808473) + | > grad_norm_0: tensor(29.6934, device='cuda:0') (tensor(36.8871, device='cuda:0')) + | > loss_gen: 2.490084171295166 (2.6093919816678457) + | > loss_kl: 2.122706651687622 (1.7984145740522954) + | > loss_feat: 7.216625690460205 (8.15741818142633) + | > loss_mel: 16.988862991333008 (16.836551227708807) + | > loss_duration: 1.1047694683074951 (0.9573927747942236) + | > loss_1: 29.92304801940918 (30.35916867221359) + | > grad_norm_1: tensor(280.5429, device='cuda:0') (tensor(558.3369, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.422 (1.007469779383527) + | > loader_time: 0.0093 (0.006400642603853322) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031090332911564753 (-0.001113268045278698) + | > avg_loss_disc: 2.5140612308795633 (+0.012419847341683887) + | > avg_loss_disc_real_0: 0.14915668391264403 (-0.0754453046963765) + | > avg_loss_disc_real_1: 0.22849183013805976 (+0.030087931798054635) + | > avg_loss_disc_real_2: 0.23151067930918473 (-0.010472153241817772) + | > avg_loss_disc_real_3: 0.24077604481807122 (+0.026852974524864792) + | > avg_loss_disc_real_4: 0.24560624131789574 (+0.05863284376951364) + | > avg_loss_disc_real_5: 0.23444126087885636 (+0.017178786488679737) + | > avg_loss_0: 2.5140612308795633 (+0.012419847341683887) + | > avg_loss_gen: 2.3510348613445577 (+0.013915116970355967) + | > avg_loss_kl: 3.4993155736189623 (-0.037783952859731684) + | > avg_loss_feat: 7.715265824244573 (-0.28065960223858166) + | > avg_loss_mel: 17.149937629699707 (-0.15661511054405963) + | > avg_loss_duration: 1.292279234299293 (-0.0023072132697474768) + | > avg_loss_1: 32.00783348083496 (-0.463450505183296) + + + > EPOCH: 63/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:51:01)  + + --> TIME: 2024-07-13 20:51:38 -- STEP: 36/151 -- GLOBAL_STEP: 94550 + | > loss_disc: 2.338136911392212 (2.360825399557749) + | > loss_disc_real_0: 0.13693809509277344 (0.13729010998374885) + | > loss_disc_real_1: 0.21003656089305878 (0.20348113278547922) + | > loss_disc_real_2: 0.2156120240688324 (0.21001511274112594) + | > loss_disc_real_3: 0.2527746260166168 (0.20365719952517086) + | > loss_disc_real_4: 0.21199975907802582 (0.20874992509682974) + | > loss_disc_real_5: 0.229154571890831 (0.22408444351620144) + | > loss_0: 2.338136911392212 (2.360825399557749) + | > grad_norm_0: tensor(23.1444, device='cuda:0') (tensor(36.6880, device='cuda:0')) + | > loss_gen: 2.5378215312957764 (2.5589588946766324) + | > loss_kl: 1.421502947807312 (1.7786110275321536) + | > loss_feat: 8.839091300964355 (8.029916365941366) + | > loss_mel: 16.723773956298828 (16.91513445642259) + | > loss_duration: 0.9907830953598022 (0.9399796823660532) + | > loss_1: 30.512971878051758 (30.222600248124863) + | > grad_norm_1: tensor(294.2487, device='cuda:0') (tensor(482.0426, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8674 (0.9612902204195658) + | > loader_time: 0.0049 (0.0059217413266499834) + + + --> TIME: 2024-07-13 20:52:28 -- STEP: 86/151 -- GLOBAL_STEP: 94600 + | > loss_disc: 2.4045333862304688 (2.351834629857262) + | > loss_disc_real_0: 0.17816279828548431 (0.12904230210670206) + | > loss_disc_real_1: 0.21498985588550568 (0.20373837459225988) + | > loss_disc_real_2: 0.2535007894039154 (0.21260000626708186) + | > loss_disc_real_3: 0.23435012996196747 (0.20845832103906675) + | > loss_disc_real_4: 0.22176113724708557 (0.20793847757023434) + | > loss_disc_real_5: 0.2349674105644226 (0.22385797562987306) + | > loss_0: 2.4045333862304688 (2.351834629857262) + | > grad_norm_0: tensor(30.3563, device='cuda:0') (tensor(33.6785, device='cuda:0')) + | > loss_gen: 2.9419870376586914 (2.5627762051515806) + | > loss_kl: 2.079084873199463 (1.7699851123399513) + | > loss_feat: 8.99526596069336 (8.091203794922942) + | > loss_mel: 17.407987594604492 (16.858782990034232) + | > loss_duration: 0.9027102589607239 (0.9487801540729611) + | > loss_1: 32.32703399658203 (30.23152821563011) + | > grad_norm_1: tensor(462.0571, device='cuda:0') (tensor(509.2468, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9522 (0.9773210758386657) + | > loader_time: 0.0055 (0.006295969319898028) + + + --> TIME: 2024-07-13 20:53:17 -- STEP: 136/151 -- GLOBAL_STEP: 94650 + | > loss_disc: 2.2362377643585205 (2.339421058402342) + | > loss_disc_real_0: 0.14630398154258728 (0.12792588145855596) + | > loss_disc_real_1: 0.1819160431623459 (0.20315612852573395) + | > loss_disc_real_2: 0.24769632518291473 (0.21367700956761837) + | > loss_disc_real_3: 0.18774476647377014 (0.20767944130827398) + | > loss_disc_real_4: 0.2354835867881775 (0.20998066970530677) + | > loss_disc_real_5: 0.18547795712947845 (0.22206491483923266) + | > loss_0: 2.2362377643585205 (2.339421058402342) + | > grad_norm_0: tensor(37.1102, device='cuda:0') (tensor(31.7203, device='cuda:0')) + | > loss_gen: 2.7614192962646484 (2.572571693097844) + | > loss_kl: 2.129124164581299 (1.7790517579106724) + | > loss_feat: 8.589249610900879 (8.135662475053001) + | > loss_mel: 16.994035720825195 (16.87521971674527) + | > loss_duration: 0.9597235321998596 (0.9370154571883819) + | > loss_1: 31.43355369567871 (30.299520997440112) + | > grad_norm_1: tensor(840.1668, device='cuda:0') (tensor(490.6786, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9686 (0.9677675033316893) + | > loader_time: 0.0057 (0.006144781323040233) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03295790232144869 (+0.0018675694098839388) + | > avg_loss_disc: 2.5917216080885668 (+0.07766037720900343) + | > avg_loss_disc_real_0: 0.22111098009806412 (+0.0719542961854201) + | > avg_loss_disc_real_1: 0.23474942033107465 (+0.006257590193014889) + | > avg_loss_disc_real_2: 0.2269675341936258 (-0.004543145115558922) + | > avg_loss_disc_real_3: 0.24818785603229815 (+0.007411811214226932) + | > avg_loss_disc_real_4: 0.2912586744015033 (+0.045652433083607585) + | > avg_loss_disc_real_5: 0.2377594399910707 (+0.0033181791122143256) + | > avg_loss_0: 2.5917216080885668 (+0.07766037720900343) + | > avg_loss_gen: 2.4972048722780666 (+0.1461700109335089) + | > avg_loss_kl: 3.4604028371664195 (-0.038912736452542784) + | > avg_loss_feat: 7.9543477571927586 (+0.23908193294818592) + | > avg_loss_mel: 17.123425777141865 (-0.026511852557842275) + | > avg_loss_duration: 1.3228270273942213 (+0.030547793094928277) + | > avg_loss_1: 32.358208876389725 (+0.3503753955547637) + + + > EPOCH: 64/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:53:41)  + + --> TIME: 2024-07-13 20:54:20 -- STEP: 35/151 -- GLOBAL_STEP: 94700 + | > loss_disc: 2.3548367023468018 (2.3643754141671316) + | > loss_disc_real_0: 0.18919077515602112 (0.12757934033870696) + | > loss_disc_real_1: 0.19622182846069336 (0.2031339841229575) + | > loss_disc_real_2: 0.2277243584394455 (0.20975286449704852) + | > loss_disc_real_3: 0.20495538413524628 (0.20617695535932268) + | > loss_disc_real_4: 0.17446942627429962 (0.21639124495642525) + | > loss_disc_real_5: 0.16985251009464264 (0.2260005405970982) + | > loss_0: 2.3548367023468018 (2.3643754141671316) + | > grad_norm_0: tensor(34.9445, device='cuda:0') (tensor(32.9152, device='cuda:0')) + | > loss_gen: 2.540074348449707 (2.5413329328809464) + | > loss_kl: 1.8936058282852173 (1.8568351473127092) + | > loss_feat: 6.864365100860596 (7.915674332209996) + | > loss_mel: 16.436105728149414 (16.608701133728022) + | > loss_duration: 0.9884627461433411 (0.9755564076559884) + | > loss_1: 28.722612380981445 (29.898099735804966) + | > grad_norm_1: tensor(518.3620, device='cuda:0') (tensor(504.0085, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 1.0711 (1.0357063293457032) + | > loader_time: 0.0068 (0.006483963557652065) + + + --> TIME: 2024-07-13 20:55:09 -- STEP: 85/151 -- GLOBAL_STEP: 94750 + | > loss_disc: 2.3573219776153564 (2.3429470567142263) + | > loss_disc_real_0: 0.19780445098876953 (0.12570926858221784) + | > loss_disc_real_1: 0.2335289865732193 (0.20334915413576013) + | > loss_disc_real_2: 0.23507559299468994 (0.21189768454607796) + | > loss_disc_real_3: 0.23710864782333374 (0.20841474796042722) + | > loss_disc_real_4: 0.22264932096004486 (0.21152696223819958) + | > loss_disc_real_5: 0.2557492256164551 (0.2242073919843225) + | > loss_0: 2.3573219776153564 (2.3429470567142263) + | > grad_norm_0: tensor(34.7684, device='cuda:0') (tensor(30.5135, device='cuda:0')) + | > loss_gen: 2.470717191696167 (2.5437935913310334) + | > loss_kl: 1.5887056589126587 (1.8741654396057128) + | > loss_feat: 6.777416229248047 (7.982784209531896) + | > loss_mel: 17.2513484954834 (16.80137971990249) + | > loss_duration: 0.9710657000541687 (0.9608748786589678) + | > loss_1: 29.05925178527832 (30.16299772823558) + | > grad_norm_1: tensor(249.3319, device='cuda:0') (tensor(460.9542, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8673 (0.9998450082891129) + | > loader_time: 0.0049 (0.006492275350234088) + + + --> TIME: 2024-07-13 20:55:58 -- STEP: 135/151 -- GLOBAL_STEP: 94800 + | > loss_disc: 2.640493392944336 (2.346420162695426) + | > loss_disc_real_0: 0.4335010051727295 (0.12934256231895205) + | > loss_disc_real_1: 0.22779282927513123 (0.2025218328943959) + | > loss_disc_real_2: 0.18789616227149963 (0.21124705219710316) + | > loss_disc_real_3: 0.29500535130500793 (0.20832097662819757) + | > loss_disc_real_4: 0.23229527473449707 (0.20992844943647032) + | > loss_disc_real_5: 0.2425534725189209 (0.22426454215138047) + | > loss_0: 2.640493392944336 (2.346420162695426) + | > grad_norm_0: tensor(91.5283, device='cuda:0') (tensor(31.1841, device='cuda:0')) + | > loss_gen: 2.562366485595703 (2.558972351639359) + | > loss_kl: 1.8748146295547485 (1.8508024198037607) + | > loss_feat: 6.577829360961914 (8.049845808523672) + | > loss_mel: 17.335874557495117 (16.8785927525273) + | > loss_duration: 0.9080372452735901 (0.9566014824090181) + | > loss_1: 29.258922576904297 (30.294814795034902) + | > grad_norm_1: tensor(549.8940, device='cuda:0') (tensor(467.9747, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.992 (0.9858140080063431) + | > loader_time: 0.0056 (0.006284049705222801) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03223287142240084 (-0.0007250308990478516) + | > avg_loss_disc: 2.3735693051264835 (-0.21815230296208332) + | > avg_loss_disc_real_0: 0.1585167680795376 (-0.06259421201852652) + | > avg_loss_disc_real_1: 0.23023916093202737 (-0.004510259399047284) + | > avg_loss_disc_real_2: 0.20116348564624786 (-0.025804048547377945) + | > avg_loss_disc_real_3: 0.26997605653909534 (+0.021788200506797195) + | > avg_loss_disc_real_4: 0.24516998002162346 (-0.04608869437987986) + | > avg_loss_disc_real_5: 0.2123863364641483 (-0.025373103526922375) + | > avg_loss_0: 2.3735693051264835 (-0.21815230296208332) + | > avg_loss_gen: 2.715732354384202 (+0.21852748210613537) + | > avg_loss_kl: 3.415548012806819 (-0.04485482435960053) + | > avg_loss_feat: 8.49466217481173 (+0.5403144176189709) + | > avg_loss_mel: 17.515341538649338 (+0.3919157615074731) + | > avg_loss_duration: 1.266042791880094 (-0.05678423551412726) + | > avg_loss_1: 33.40732662494366 (+1.0491177485539325) + + + > EPOCH: 65/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:56:22)  + + --> TIME: 2024-07-13 20:56:57 -- STEP: 34/151 -- GLOBAL_STEP: 94850 + | > loss_disc: 2.387685775756836 (2.327587969162885) + | > loss_disc_real_0: 0.24757318198680878 (0.1268550142207567) + | > loss_disc_real_1: 0.18861104547977448 (0.1983383904923411) + | > loss_disc_real_2: 0.2405639886856079 (0.20933408509282506) + | > loss_disc_real_3: 0.21520580351352692 (0.20797639121027553) + | > loss_disc_real_4: 0.25662291049957275 (0.20995348911075032) + | > loss_disc_real_5: 0.24216139316558838 (0.2181586553068722) + | > loss_0: 2.387685775756836 (2.327587969162885) + | > grad_norm_0: tensor(35.8058, device='cuda:0') (tensor(31.9068, device='cuda:0')) + | > loss_gen: 2.870587110519409 (2.6248310593997735) + | > loss_kl: 1.8122798204421997 (1.7973720291081596) + | > loss_feat: 7.419354438781738 (8.377711085712205) + | > loss_mel: 16.701278686523438 (16.90947423261755) + | > loss_duration: 0.828940749168396 (0.9434563044239493) + | > loss_1: 29.6324405670166 (30.65284465341007) + | > grad_norm_1: tensor(267.1952, device='cuda:0') (tensor(480.7366, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.8803 (0.9614926436368156) + | > loader_time: 0.005 (0.005896491162917193) + + + --> TIME: 2024-07-13 20:57:47 -- STEP: 84/151 -- GLOBAL_STEP: 94900 + | > loss_disc: 2.2423148155212402 (2.334503094355265) + | > loss_disc_real_0: 0.1258305013179779 (0.12690307222129332) + | > loss_disc_real_1: 0.18934540450572968 (0.20238466782584077) + | > loss_disc_real_2: 0.17896969616413116 (0.21388508840685794) + | > loss_disc_real_3: 0.1803782880306244 (0.20825477104101867) + | > loss_disc_real_4: 0.20983688533306122 (0.21035881420331343) + | > loss_disc_real_5: 0.2168881893157959 (0.21910334626833597) + | > loss_0: 2.2423148155212402 (2.334503094355265) + | > grad_norm_0: tensor(42.5858, device='cuda:0') (tensor(29.7962, device='cuda:0')) + | > loss_gen: 2.7195374965667725 (2.577171754269372) + | > loss_kl: 1.1973979473114014 (1.791121313259715) + | > loss_feat: 7.945157051086426 (8.188301120485574) + | > loss_mel: 15.143418312072754 (16.905235381353474) + | > loss_duration: 1.1714893579483032 (0.9547482225156966) + | > loss_1: 28.177000045776367 (30.416577747889928) + | > grad_norm_1: tensor(669.6317, device='cuda:0') (tensor(431.8521, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.8175 (0.9778448712258112) + | > loader_time: 0.008 (0.00619452056430635) + + + --> TIME: 2024-07-13 20:58:37 -- STEP: 134/151 -- GLOBAL_STEP: 94950 + | > loss_disc: 2.3461806774139404 (2.333940123444172) + | > loss_disc_real_0: 0.1307671070098877 (0.12557344305426327) + | > loss_disc_real_1: 0.16306011378765106 (0.2023064813840745) + | > loss_disc_real_2: 0.2301131635904312 (0.21338917370607602) + | > loss_disc_real_3: 0.23456750810146332 (0.2090126463948791) + | > loss_disc_real_4: 0.21528664231300354 (0.210445908301357) + | > loss_disc_real_5: 0.2532048523426056 (0.21943800206949463) + | > loss_0: 2.3461806774139404 (2.333940123444172) + | > grad_norm_0: tensor(45.1792, device='cuda:0') (tensor(33.2546, device='cuda:0')) + | > loss_gen: 2.6664137840270996 (2.5919802633684075) + | > loss_kl: 1.7233244180679321 (1.7928572194789774) + | > loss_feat: 8.220824241638184 (8.22496579298333) + | > loss_mel: 15.214155197143555 (16.927178133779503) + | > loss_duration: 0.9454727172851562 (0.9499343099878795) + | > loss_1: 28.770191192626953 (30.486915744952302) + | > grad_norm_1: tensor(316.8333, device='cuda:0') (tensor(472.8578, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.8131 (0.9770100703879968) + | > loader_time: 0.0047 (0.006162703926883527) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033075772798978366 (+0.0008429013765775262) + | > avg_loss_disc: 2.6740592076228213 (+0.3004899024963379) + | > avg_loss_disc_real_0: 0.29305722965643954 (+0.13454046157690194) + | > avg_loss_disc_real_1: 0.24935713410377502 (+0.019117973171747654) + | > avg_loss_disc_real_2: 0.27891828692876375 (+0.07775480128251588) + | > avg_loss_disc_real_3: 0.1985235856129573 (-0.07145247092613805) + | > avg_loss_disc_real_4: 0.24707441261181465 (+0.0019044325901911852) + | > avg_loss_disc_real_5: 0.21157621305722457 (-0.0008101234069237406) + | > avg_loss_0: 2.6740592076228213 (+0.3004899024963379) + | > avg_loss_gen: 2.4347789837763867 (-0.28095337060781533) + | > avg_loss_kl: 3.7533667270953837 (+0.3378187142885647) + | > avg_loss_feat: 7.95990357032189 (-0.5347586044898396) + | > avg_loss_mel: 17.38741082411546 (-0.1279307145338784) + | > avg_loss_duration: 1.3027605276841383 (+0.03671773580404425) + | > avg_loss_1: 32.8382207430326 (-0.5691058819110566) + + + > EPOCH: 66/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 20:59:02)  + + --> TIME: 2024-07-13 20:59:35 -- STEP: 33/151 -- GLOBAL_STEP: 95000 + | > loss_disc: 2.3716862201690674 (2.3107052210605503) + | > loss_disc_real_0: 0.1468833088874817 (0.12140606727563971) + | > loss_disc_real_1: 0.19164250791072845 (0.2042353471571749) + | > loss_disc_real_2: 0.24288491904735565 (0.21547046213439017) + | > loss_disc_real_3: 0.1869852989912033 (0.2070866326491038) + | > loss_disc_real_4: 0.18982774019241333 (0.20721809204780695) + | > loss_disc_real_5: 0.19619859755039215 (0.2189968688921495) + | > loss_0: 2.3716862201690674 (2.3107052210605503) + | > grad_norm_0: tensor(31.0717, device='cuda:0') (tensor(41.5461, device='cuda:0')) + | > loss_gen: 2.5897364616394043 (2.663507317051743) + | > loss_kl: 1.7555005550384521 (1.737544383063461) + | > loss_feat: 8.66917896270752 (8.122507947863955) + | > loss_mel: 17.003116607666016 (17.030476656827062) + | > loss_duration: 0.9194120168685913 (0.9483435406829371) + | > loss_1: 30.93694496154785 (30.50237967751243) + | > grad_norm_1: tensor(870.2625, device='cuda:0') (tensor(681.2112, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.9003 (0.9367170333862305) + | > loader_time: 0.0054 (0.005713932441942619) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_95000.pth + + --> TIME: 2024-07-13 21:00:57 -- STEP: 83/151 -- GLOBAL_STEP: 95050 + | > loss_disc: 2.1618590354919434 (2.329644404261945) + | > loss_disc_real_0: 0.04941704496741295 (0.12481729618935695) + | > loss_disc_real_1: 0.14128471910953522 (0.20403866353164235) + | > loss_disc_real_2: 0.1843024343252182 (0.21135094519480166) + | > loss_disc_real_3: 0.13334394991397858 (0.2084554437413273) + | > loss_disc_real_4: 0.11890355497598648 (0.20830337227468032) + | > loss_disc_real_5: 0.2475290447473526 (0.22237806955733933) + | > loss_0: 2.1618590354919434 (2.329644404261945) + | > grad_norm_0: tensor(11.9192, device='cuda:0') (tensor(40.4067, device='cuda:0')) + | > loss_gen: 2.481627941131592 (2.5975947552416696) + | > loss_kl: 1.6431947946548462 (1.7790454489639007) + | > loss_feat: 9.213693618774414 (8.035418142755347) + | > loss_mel: 16.716434478759766 (16.79673177650177) + | > loss_duration: 1.0236035585403442 (0.9525114656930946) + | > loss_1: 31.078554153442383 (30.161301440503223) + | > grad_norm_1: tensor(85.6691, device='cuda:0') (tensor(602.8563, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.8114 (0.986424575369042) + | > loader_time: 0.0041 (0.006093835256185876) + + + --> TIME: 2024-07-13 21:01:46 -- STEP: 133/151 -- GLOBAL_STEP: 95100 + | > loss_disc: 2.1534829139709473 (2.3268828911888866) + | > loss_disc_real_0: 0.09607287496328354 (0.1256655176499284) + | > loss_disc_real_1: 0.23480701446533203 (0.20389507279584282) + | > loss_disc_real_2: 0.17007192969322205 (0.21147183833041586) + | > loss_disc_real_3: 0.21448346972465515 (0.20772693636722134) + | > loss_disc_real_4: 0.22854582965373993 (0.20780703470222933) + | > loss_disc_real_5: 0.2341960072517395 (0.22153765702606143) + | > loss_0: 2.1534829139709473 (2.3268828911888866) + | > grad_norm_0: tensor(8.5376, device='cuda:0') (tensor(34.7041, device='cuda:0')) + | > loss_gen: 2.7980403900146484 (2.5845695276905714) + | > loss_kl: 1.8193080425262451 (1.7539820962382437) + | > loss_feat: 9.072717666625977 (8.081705835528833) + | > loss_mel: 17.988971710205078 (16.732658995721568) + | > loss_duration: 0.9323780536651611 (0.9553535150406056) + | > loss_1: 32.61141586303711 (30.108269928093243) + | > grad_norm_1: tensor(274.1940, device='cuda:0') (tensor(505.1588, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.8896 (0.9824753201993784) + | > loader_time: 0.005 (0.006092594978504611) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03815801327045147 (+0.005082240471473105) + | > avg_loss_disc: 2.5658737145937405 (-0.10818549302908087) + | > avg_loss_disc_real_0: 0.19034535839007452 (-0.10271187126636502) + | > avg_loss_disc_real_1: 0.2025531564767544 (-0.046803977627020626) + | > avg_loss_disc_real_2: 0.23879013840968794 (-0.04012814851907581) + | > avg_loss_disc_real_3: 0.22486542279903704 (+0.026341837186079742) + | > avg_loss_disc_real_4: 0.21464676008774683 (-0.03242765252406782) + | > avg_loss_disc_real_5: 0.2418750260884945 (+0.03029881303126994) + | > avg_loss_0: 2.5658737145937405 (-0.10818549302908087) + | > avg_loss_gen: 2.259746258075421 (-0.17503272570096584) + | > avg_loss_kl: 3.5248194291041446 (-0.22854729799123907) + | > avg_loss_feat: 7.852011057046743 (-0.10789251327514648) + | > avg_loss_mel: 17.135104766258827 (-0.2523060578566323) + | > avg_loss_duration: 1.2966316846700816 (-0.006128843014056695) + | > avg_loss_1: 32.068312865037186 (-0.7699078779954149) + + + > EPOCH: 67/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:02:13)  + + --> TIME: 2024-07-13 21:02:50 -- STEP: 32/151 -- GLOBAL_STEP: 95150 + | > loss_disc: 2.4472639560699463 (2.376154601573944) + | > loss_disc_real_0: 0.13715234398841858 (0.1364318100968376) + | > loss_disc_real_1: 0.1515999436378479 (0.20679142931476235) + | > loss_disc_real_2: 0.17811673879623413 (0.21291370131075382) + | > loss_disc_real_3: 0.25785180926322937 (0.21271867863833904) + | > loss_disc_real_4: 0.15429329872131348 (0.20657183625735343) + | > loss_disc_real_5: 0.2542974650859833 (0.2244851877912879) + | > loss_0: 2.4472639560699463 (2.376154601573944) + | > grad_norm_0: tensor(17.0548, device='cuda:0') (tensor(22.5635, device='cuda:0')) + | > loss_gen: 2.3990767002105713 (2.5212764367461205) + | > loss_kl: 2.042039394378662 (1.758179645985365) + | > loss_feat: 7.7529730796813965 (7.965497702360154) + | > loss_mel: 17.507375717163086 (16.755266457796104) + | > loss_duration: 0.8621939420700073 (0.9701421465724707) + | > loss_1: 30.563657760620117 (29.97036212682724) + | > grad_norm_1: tensor(405.1271, device='cuda:0') (tensor(362.5465, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.8799 (1.0471096187829971) + | > loader_time: 0.0051 (0.006515786051750183) + + + --> TIME: 2024-07-13 21:03:35 -- STEP: 82/151 -- GLOBAL_STEP: 95200 + | > loss_disc: 2.2313475608825684 (2.349366717222261) + | > loss_disc_real_0: 0.08765998482704163 (0.1291127919969035) + | > loss_disc_real_1: 0.13307444751262665 (0.20313913475086048) + | > loss_disc_real_2: 0.2161817103624344 (0.2121650449386457) + | > loss_disc_real_3: 0.19863198697566986 (0.21305650490813138) + | > loss_disc_real_4: 0.17231254279613495 (0.20955375663754416) + | > loss_disc_real_5: 0.20346465706825256 (0.2233537519123496) + | > loss_0: 2.2313475608825684 (2.349366717222261) + | > grad_norm_0: tensor(25.5875, device='cuda:0') (tensor(24.7237, device='cuda:0')) + | > loss_gen: 2.5593626499176025 (2.54187088187148) + | > loss_kl: 1.923572063446045 (1.7620316525784934) + | > loss_feat: 8.336326599121094 (7.987466474858726) + | > loss_mel: 17.401107788085938 (16.659350720847524) + | > loss_duration: 0.9491077661514282 (0.9441193843760142) + | > loss_1: 31.169477462768555 (29.89483905420071) + | > grad_norm_1: tensor(705.9033, device='cuda:0') (tensor(411.4819, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.9344 (0.9618075155630345) + | > loader_time: 0.0068 (0.005925629197097408) + + + --> TIME: 2024-07-13 21:04:22 -- STEP: 132/151 -- GLOBAL_STEP: 95250 + | > loss_disc: 2.272077798843384 (2.3258542490727976) + | > loss_disc_real_0: 0.08007440716028214 (0.12228054869355577) + | > loss_disc_real_1: 0.20115987956523895 (0.2035140831475005) + | > loss_disc_real_2: 0.17685505747795105 (0.21262140267274596) + | > loss_disc_real_3: 0.20055821537971497 (0.20833183288799995) + | > loss_disc_real_4: 0.19408415257930756 (0.20911605413438697) + | > loss_disc_real_5: 0.2744460999965668 (0.21988777567942938) + | > loss_0: 2.272077798843384 (2.3258542490727976) + | > grad_norm_0: tensor(18.8700, device='cuda:0') (tensor(29.3046, device='cuda:0')) + | > loss_gen: 2.6722869873046875 (2.5996906703168694) + | > loss_kl: 1.63490629196167 (1.719945419918407) + | > loss_feat: 8.28961181640625 (8.154482494701035) + | > loss_mel: 17.208431243896484 (16.76942187367063) + | > loss_duration: 1.013892650604248 (0.950399793007157) + | > loss_1: 30.819129943847656 (30.193940206007525) + | > grad_norm_1: tensor(158.3275, device='cuda:0') (tensor(492.6655, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.8743 (0.9483048229506521) + | > loader_time: 0.0045 (0.005779044194654985) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03356486100416917 (-0.004593152266282298) + | > avg_loss_disc: 2.5508168110480676 (-0.015056903545672906) + | > avg_loss_disc_real_0: 0.21271613813363588 (+0.02237077974356136) + | > avg_loss_disc_real_1: 0.19733661642441383 (-0.005216540052340568) + | > avg_loss_disc_real_2: 0.24866745449029481 (+0.00987731608060688) + | > avg_loss_disc_real_3: 0.26054209012251633 (+0.035676667323479294) + | > avg_loss_disc_real_4: 0.3170264019415929 (+0.10237964185384604) + | > avg_loss_disc_real_5: 0.2654082351005994 (+0.0235332090121049) + | > avg_loss_0: 2.5508168110480676 (-0.015056903545672906) + | > avg_loss_gen: 2.591116941892184 (+0.33137068381676293) + | > avg_loss_kl: 3.4382571073678823 (-0.08656232173626233) + | > avg_loss_feat: 7.701483469742995 (-0.15052758730374816) + | > avg_loss_mel: 17.249159739567684 (+0.11405497330885694) + | > avg_loss_duration: 1.2771420112022986 (-0.019489673467782964) + | > avg_loss_1: 32.2571597466102 (+0.18884688157301355) + + + > EPOCH: 68/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:04:53)  + + --> TIME: 2024-07-13 21:05:27 -- STEP: 31/151 -- GLOBAL_STEP: 95300 + | > loss_disc: 2.264570474624634 (2.348215118531258) + | > loss_disc_real_0: 0.11409902572631836 (0.12743483123279387) + | > loss_disc_real_1: 0.22755347192287445 (0.2046860689597745) + | > loss_disc_real_2: 0.16396565735340118 (0.2123915913604921) + | > loss_disc_real_3: 0.14364057779312134 (0.20286429457126126) + | > loss_disc_real_4: 0.19509804248809814 (0.21253763427657466) + | > loss_disc_real_5: 0.24513942003250122 (0.22483739977882755) + | > loss_0: 2.264570474624634 (2.348215118531258) + | > grad_norm_0: tensor(12.3204, device='cuda:0') (tensor(24.0149, device='cuda:0')) + | > loss_gen: 2.629603147506714 (2.5374709636934343) + | > loss_kl: 1.173197627067566 (1.6478939325578752) + | > loss_feat: 9.764321327209473 (7.997338741056381) + | > loss_mel: 17.713890075683594 (16.903950629695768) + | > loss_duration: 0.9054619073867798 (0.9528636990054962) + | > loss_1: 32.18647384643555 (30.039517925631614) + | > grad_norm_1: tensor(108.6045, device='cuda:0') (tensor(390.3571, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.8554 (1.0122432170375701) + | > loader_time: 0.0043 (0.006482547329318139) + + + --> TIME: 2024-07-13 21:06:17 -- STEP: 81/151 -- GLOBAL_STEP: 95350 + | > loss_disc: 2.2067930698394775 (2.340443734769467) + | > loss_disc_real_0: 0.13902702927589417 (0.1293273671173755) + | > loss_disc_real_1: 0.1964673101902008 (0.20380655142628115) + | > loss_disc_real_2: 0.19866177439689636 (0.2115356324263561) + | > loss_disc_real_3: 0.16849136352539062 (0.20367831911569761) + | > loss_disc_real_4: 0.20081479847431183 (0.21052277741240866) + | > loss_disc_real_5: 0.2812686860561371 (0.22386866826334118) + | > loss_0: 2.2067930698394775 (2.340443734769467) + | > grad_norm_0: tensor(60.7752, device='cuda:0') (tensor(26.9585, device='cuda:0')) + | > loss_gen: 2.844208002090454 (2.569618342835227) + | > loss_kl: 1.2514008283615112 (1.6775166075906636) + | > loss_feat: 8.738096237182617 (8.157788100066009) + | > loss_mel: 16.68704605102539 (16.992914176281595) + | > loss_duration: 0.9208824038505554 (0.9561642378936579) + | > loss_1: 30.441635131835938 (30.354001527951088) + | > grad_norm_1: tensor(723.0186, device='cuda:0') (tensor(457.2910, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.8425 (0.9914273862485532) + | > loader_time: 0.0044 (0.006314430707766686) + + + --> TIME: 2024-07-13 21:07:04 -- STEP: 131/151 -- GLOBAL_STEP: 95400 + | > loss_disc: 2.3128976821899414 (2.3386483010444943) + | > loss_disc_real_0: 0.13562193512916565 (0.1303727231806016) + | > loss_disc_real_1: 0.2390754073858261 (0.20339160924421923) + | > loss_disc_real_2: 0.2214229851961136 (0.21095494448228647) + | > loss_disc_real_3: 0.21519392728805542 (0.20402757971341373) + | > loss_disc_real_4: 0.25767236948013306 (0.21046845700222117) + | > loss_disc_real_5: 0.28035297989845276 (0.22276975771853033) + | > loss_0: 2.3128976821899414 (2.3386483010444943) + | > grad_norm_0: tensor(39.1253, device='cuda:0') (tensor(31.0892, device='cuda:0')) + | > loss_gen: 2.797589063644409 (2.5918347617142077) + | > loss_kl: 1.3657857179641724 (1.697206689656236) + | > loss_feat: 9.0051851272583 (8.193754796763407) + | > loss_mel: 17.810291290283203 (17.06845573250576) + | > loss_duration: 0.9387767314910889 (0.9490528802835304) + | > loss_1: 31.917627334594727 (30.5003048991429) + | > grad_norm_1: tensor(134.8571, device='cuda:0') (tensor(494.0134, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.9064 (0.9676136697521647) + | > loader_time: 0.0062 (0.006132071254817584) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034034123787513144 (+0.0004692627833439711) + | > avg_loss_disc: 2.595411410698524 (+0.044594599650456423) + | > avg_loss_disc_real_0: 0.26755592914728016 (+0.05483979101364428) + | > avg_loss_disc_real_1: 0.27067023057204026 (+0.07333361414762643) + | > avg_loss_disc_real_2: 0.30230290500017315 (+0.053635450509878335) + | > avg_loss_disc_real_3: 0.23188877678834474 (-0.028653313334171593) + | > avg_loss_disc_real_4: 0.26279838383197784 (-0.05422801810961503) + | > avg_loss_disc_real_5: 0.2177586337694755 (-0.04764960133112392) + | > avg_loss_0: 2.595411410698524 (+0.044594599650456423) + | > avg_loss_gen: 2.6309226843026967 (+0.039805742410512934) + | > avg_loss_kl: 3.3570734537564793 (-0.081183653611403) + | > avg_loss_feat: 7.58216116978572 (-0.11932229995727539) + | > avg_loss_mel: 17.159547072190502 (-0.08961266737718177) + | > avg_loss_duration: 1.284846883553725 (+0.007704872351426362) + | > avg_loss_1: 32.014550869281464 (-0.2426088773287347) + + + > EPOCH: 69/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:07:34)  + + --> TIME: 2024-07-13 21:08:10 -- STEP: 30/151 -- GLOBAL_STEP: 95450 + | > loss_disc: 2.1487700939178467 (2.354416092236837) + | > loss_disc_real_0: 0.09363111108541489 (0.13047361585001152) + | > loss_disc_real_1: 0.17020127177238464 (0.20483459482590358) + | > loss_disc_real_2: 0.19078399240970612 (0.21718450238307316) + | > loss_disc_real_3: 0.18337658047676086 (0.20749837756156922) + | > loss_disc_real_4: 0.1774679571390152 (0.2118434066573779) + | > loss_disc_real_5: 0.2041574865579605 (0.21833021342754363) + | > loss_0: 2.1487700939178467 (2.354416092236837) + | > grad_norm_0: tensor(19.7470, device='cuda:0') (tensor(27.1242, device='cuda:0')) + | > loss_gen: 2.5520269870758057 (2.5274142106374105) + | > loss_kl: 1.2622849941253662 (1.7642939964930215) + | > loss_feat: 8.457859992980957 (8.051245848337809) + | > loss_mel: 16.641450881958008 (16.606535720825196) + | > loss_duration: 0.9925572276115417 (0.9675978243350982) + | > loss_1: 29.906179428100586 (29.91708780924479) + | > grad_norm_1: tensor(275.2482, device='cuda:0') (tensor(404.6497, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.8573 (1.097636596361796) + | > loader_time: 0.0046 (0.007154663403828939) + + + --> TIME: 2024-07-13 21:08:57 -- STEP: 80/151 -- GLOBAL_STEP: 95500 + | > loss_disc: 2.205526113510132 (2.306904411315918) + | > loss_disc_real_0: 0.1003151386976242 (0.12582574165426194) + | > loss_disc_real_1: 0.20032186806201935 (0.20081638237461447) + | > loss_disc_real_2: 0.15874189138412476 (0.20982194617390632) + | > loss_disc_real_3: 0.18817287683486938 (0.20430918531492354) + | > loss_disc_real_4: 0.15176837146282196 (0.20930168721824885) + | > loss_disc_real_5: 0.22430627048015594 (0.21485600862652063) + | > loss_0: 2.205526113510132 (2.306904411315918) + | > grad_norm_0: tensor(45.5446, device='cuda:0') (tensor(26.7659, device='cuda:0')) + | > loss_gen: 2.680208206176758 (2.581224483251572) + | > loss_kl: 1.7708730697631836 (1.768688052892685) + | > loss_feat: 7.787790298461914 (8.188395380973818) + | > loss_mel: 15.877650260925293 (16.848752927780154) + | > loss_duration: 1.2029175758361816 (0.9551097638905048) + | > loss_1: 29.319438934326172 (30.34217071533203) + | > grad_norm_1: tensor(521.5104, device='cuda:0') (tensor(421.9368, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.8087 (0.9966459006071091) + | > loader_time: 0.0049 (0.006433835625648499) + + + --> TIME: 2024-07-13 21:09:47 -- STEP: 130/151 -- GLOBAL_STEP: 95550 + | > loss_disc: 2.2431814670562744 (2.3081465244293216) + | > loss_disc_real_0: 0.10109148919582367 (0.12592337056994432) + | > loss_disc_real_1: 0.19519086182117462 (0.20216402978851244) + | > loss_disc_real_2: 0.25442904233932495 (0.21072506526341805) + | > loss_disc_real_3: 0.2612689435482025 (0.20560364946722984) + | > loss_disc_real_4: 0.2939966022968292 (0.2092920686189945) + | > loss_disc_real_5: 0.2810755670070648 (0.21679042228139364) + | > loss_0: 2.2431814670562744 (2.3081465244293216) + | > grad_norm_0: tensor(24.8346, device='cuda:0') (tensor(30.8005, device='cuda:0')) + | > loss_gen: 2.8140146732330322 (2.6001913199057944) + | > loss_kl: 1.379948377609253 (1.7856292935518119) + | > loss_feat: 9.267325401306152 (8.22712175662701) + | > loss_mel: 18.168733596801758 (16.880229429098286) + | > loss_duration: 0.8540178537368774 (0.9528109802649571) + | > loss_1: 32.484039306640625 (30.44598284501296) + | > grad_norm_1: tensor(457.6700, device='cuda:0') (tensor(464.3819, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.8608 (0.9917483641551091) + | > loader_time: 0.0046 (0.006313890677232009) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034921737817617565 (+0.0008876140301044208) + | > avg_loss_disc: 2.5881661818577695 (-0.007245228840754514) + | > avg_loss_disc_real_0: 0.31304513949614304 (+0.045489210348862885) + | > avg_loss_disc_real_1: 0.2047100823659163 (-0.06596014820612395) + | > avg_loss_disc_real_2: 0.19822278045690978 (-0.10408012454326337) + | > avg_loss_disc_real_3: 0.19976047025277063 (-0.03212830653557411) + | > avg_loss_disc_real_4: 0.20948380461105934 (-0.05331457922091851) + | > avg_loss_disc_real_5: 0.2187084280527555 (+0.0009497942832800144) + | > avg_loss_0: 2.5881661818577695 (-0.007245228840754514) + | > avg_loss_gen: 2.430668060596173 (-0.20025462370652392) + | > avg_loss_kl: 3.7142664102407603 (+0.35719295648428107) + | > avg_loss_feat: 8.025121578803429 (+0.44296040901770883) + | > avg_loss_mel: 17.28147946871244 (+0.1219323965219381) + | > avg_loss_duration: 1.253456867658175 (-0.031390015895550016) + | > avg_loss_1: 32.704992000873276 (+0.6904411315918111) + + + > EPOCH: 70/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:10:15)  + + --> TIME: 2024-07-13 21:10:45 -- STEP: 29/151 -- GLOBAL_STEP: 95600 + | > loss_disc: 2.3258347511291504 (2.3442195612808754) + | > loss_disc_real_0: 0.12058345973491669 (0.11814393064585225) + | > loss_disc_real_1: 0.19235777854919434 (0.20632627246708707) + | > loss_disc_real_2: 0.22691622376441956 (0.2138775042418776) + | > loss_disc_real_3: 0.18040652573108673 (0.20619228328096456) + | > loss_disc_real_4: 0.21415698528289795 (0.2117272209504555) + | > loss_disc_real_5: 0.2004605382680893 (0.21686682156447706) + | > loss_0: 2.3258347511291504 (2.3442195612808754) + | > grad_norm_0: tensor(40.2496, device='cuda:0') (tensor(37.1120, device='cuda:0')) + | > loss_gen: 2.6887781620025635 (2.5859759593832083) + | > loss_kl: 1.6986223459243774 (1.759893380362412) + | > loss_feat: 8.968419075012207 (8.109354939954036) + | > loss_mel: 16.561382293701172 (16.837298064396297) + | > loss_duration: 0.8950574398040771 (0.9589645800919369) + | > loss_1: 30.812259674072266 (30.251487008456525) + | > grad_norm_1: tensor(641.6401, device='cuda:0') (tensor(627.2192, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.8837 (0.9395163470301134) + | > loader_time: 0.0062 (0.005742697880185884) + + + --> TIME: 2024-07-13 21:11:35 -- STEP: 79/151 -- GLOBAL_STEP: 95650 + | > loss_disc: 2.051792860031128 (2.3285652142536786) + | > loss_disc_real_0: 0.06718867272138596 (0.12485994761691818) + | > loss_disc_real_1: 0.12044867873191833 (0.20073836494850206) + | > loss_disc_real_2: 0.14465510845184326 (0.21279830532737926) + | > loss_disc_real_3: 0.15710289776325226 (0.2064272689668439) + | > loss_disc_real_4: 0.1334570348262787 (0.2072769131841539) + | > loss_disc_real_5: 0.17281243205070496 (0.21809599803218357) + | > loss_0: 2.051792860031128 (2.3285652142536786) + | > grad_norm_0: tensor(31.6407, device='cuda:0') (tensor(32.7013, device='cuda:0')) + | > loss_gen: 2.9948151111602783 (2.587830401674101) + | > loss_kl: 1.844712495803833 (1.7935930637619164) + | > loss_feat: 11.766022682189941 (8.16194979148575) + | > loss_mel: 18.591150283813477 (16.838076712209965) + | > loss_duration: 0.9013628363609314 (0.9507485333877274) + | > loss_1: 36.09806442260742 (30.33219837237008) + | > grad_norm_1: tensor(611.6479, device='cuda:0') (tensor(553.0068, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.8179 (0.964071014259435) + | > loader_time: 0.0047 (0.0059518331213842465) + + + --> TIME: 2024-07-13 21:12:24 -- STEP: 129/151 -- GLOBAL_STEP: 95700 + | > loss_disc: 2.1891252994537354 (2.3324795342230966) + | > loss_disc_real_0: 0.061984919011592865 (0.12592476430211882) + | > loss_disc_real_1: 0.1577160358428955 (0.20287248596202495) + | > loss_disc_real_2: 0.15066668391227722 (0.21336306816385697) + | > loss_disc_real_3: 0.16496410965919495 (0.2055051622464676) + | > loss_disc_real_4: 0.19033163785934448 (0.207674262020015) + | > loss_disc_real_5: 0.1954278200864792 (0.22107504712518797) + | > loss_0: 2.1891252994537354 (2.3324795342230966) + | > grad_norm_0: tensor(17.4786, device='cuda:0') (tensor(33.0775, device='cuda:0')) + | > loss_gen: 2.6703641414642334 (2.5916103096895435) + | > loss_kl: 1.0839965343475342 (1.7971527481725973) + | > loss_feat: 9.733619689941406 (8.214567934819899) + | > loss_mel: 17.184968948364258 (16.940073737802432) + | > loss_duration: 1.1092878580093384 (0.950009940668594) + | > loss_1: 31.782238006591797 (30.493414590525074) + | > grad_norm_1: tensor(596.0566, device='cuda:0') (tensor(533.4188, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.8222 (0.9711572255275046) + | > loader_time: 0.0045 (0.006135225296020508) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033424634199876055 (-0.0014971036177415104) + | > avg_loss_disc: 2.543850201826829 (-0.044315980030940505) + | > avg_loss_disc_real_0: 0.2946375608444214 (-0.018407578651721657) + | > avg_loss_disc_real_1: 0.20998925085251147 (+0.005279168486595154) + | > avg_loss_disc_real_2: 0.2555042768900211 (+0.05728149643311131) + | > avg_loss_disc_real_3: 0.240010882799442 (+0.04025041254667136) + | > avg_loss_disc_real_4: 0.2517442347911688 (+0.04226043018010947) + | > avg_loss_disc_real_5: 0.2271773528594237 (+0.008468924806668193) + | > avg_loss_0: 2.543850201826829 (-0.044315980030940505) + | > avg_loss_gen: 2.646631460923415 (+0.21596340032724237) + | > avg_loss_kl: 3.4247723542726956 (-0.28949405596806477) + | > avg_loss_feat: 7.883618391477144 (-0.14150318732628442) + | > avg_loss_mel: 17.152019060575046 (-0.129460408137394) + | > avg_loss_duration: 1.285345875299894 (+0.03188900764171909) + | > avg_loss_1: 32.39238665654109 (-0.3126053443321837) + + + > EPOCH: 71/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:12:56)  + + --> TIME: 2024-07-13 21:13:25 -- STEP: 28/151 -- GLOBAL_STEP: 95750 + | > loss_disc: 2.25225567817688 (2.32456397158759) + | > loss_disc_real_0: 0.19944408535957336 (0.12853816976504667) + | > loss_disc_real_1: 0.20464712381362915 (0.19680159911513329) + | > loss_disc_real_2: 0.15156234800815582 (0.20860405904906137) + | > loss_disc_real_3: 0.1774403601884842 (0.2086577740098749) + | > loss_disc_real_4: 0.19614486396312714 (0.21007173082658223) + | > loss_disc_real_5: 0.20074838399887085 (0.21886531050716126) + | > loss_0: 2.25225567817688 (2.32456397158759) + | > grad_norm_0: tensor(33.6203, device='cuda:0') (tensor(22.9281, device='cuda:0')) + | > loss_gen: 2.9003591537475586 (2.5745706387928555) + | > loss_kl: 1.991230845451355 (1.7563392434801375) + | > loss_feat: 9.26655101776123 (8.513879401343209) + | > loss_mel: 17.886489868164062 (16.884327139173234) + | > loss_duration: 0.889057993888855 (0.9298792055674961) + | > loss_1: 32.93368911743164 (30.65899556023734) + | > grad_norm_1: tensor(571.9423, device='cuda:0') (tensor(390.5718, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.9491 (0.9370066608701434) + | > loader_time: 0.0057 (0.005919115883963448) + + + --> TIME: 2024-07-13 21:14:12 -- STEP: 78/151 -- GLOBAL_STEP: 95800 + | > loss_disc: 2.327052116394043 (2.3101637607965713) + | > loss_disc_real_0: 0.08412706851959229 (0.12372201222639817) + | > loss_disc_real_1: 0.18903657793998718 (0.20047692667979455) + | > loss_disc_real_2: 0.20717978477478027 (0.20955257404309052) + | > loss_disc_real_3: 0.19284527003765106 (0.20575363781207648) + | > loss_disc_real_4: 0.22012990713119507 (0.20810686070949602) + | > loss_disc_real_5: 0.2322678416967392 (0.22194035580525032) + | > loss_0: 2.327052116394043 (2.3101637607965713) + | > grad_norm_0: tensor(14.8779, device='cuda:0') (tensor(31.1486, device='cuda:0')) + | > loss_gen: 2.2847707271575928 (2.6213386456171666) + | > loss_kl: 1.773108720779419 (1.763502564185705) + | > loss_feat: 6.861054420471191 (8.438241017170443) + | > loss_mel: 15.394245147705078 (17.106045820774163) + | > loss_duration: 0.9469069838523865 (0.9394911519992046) + | > loss_1: 27.260086059570312 (30.868619307493553) + | > grad_norm_1: tensor(495.7411, device='cuda:0') (tensor(506.8951, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.8566 (0.9275119396356436) + | > loader_time: 0.0056 (0.005704412093529334) + + + --> TIME: 2024-07-13 21:15:04 -- STEP: 128/151 -- GLOBAL_STEP: 95850 + | > loss_disc: 2.433076858520508 (2.315614551305771) + | > loss_disc_real_0: 0.09275200217962265 (0.12540982913924384) + | > loss_disc_real_1: 0.24754947423934937 (0.20143816247582433) + | > loss_disc_real_2: 0.22705534100532532 (0.21134798438288271) + | > loss_disc_real_3: 0.17828616499900818 (0.2058915103552863) + | > loss_disc_real_4: 0.19107557833194733 (0.2091221121372655) + | > loss_disc_real_5: 0.1867389976978302 (0.2197465788340196) + | > loss_0: 2.433076858520508 (2.315614551305771) + | > grad_norm_0: tensor(21.5917, device='cuda:0') (tensor(34.4555, device='cuda:0')) + | > loss_gen: 2.3294148445129395 (2.624967653304338) + | > loss_kl: 2.023953914642334 (1.780779933091253) + | > loss_feat: 6.816054344177246 (8.334414198994642) + | > loss_mel: 17.139575958251953 (17.139151319861405) + | > loss_duration: 0.9703717231750488 (0.9491561213508248) + | > loss_1: 29.279369354248047 (30.82846926152706) + | > grad_norm_1: tensor(153.4121, device='cuda:0') (tensor(529.9149, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 1.0543 (0.9700990486890078) + | > loader_time: 0.0068 (0.006117817014455795) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03331094521742601 (-0.0001136889824500456) + | > avg_loss_disc: 2.6073112304394064 (+0.06346102861257741) + | > avg_loss_disc_real_0: 0.24325678096367762 (-0.05138077988074377) + | > avg_loss_disc_real_1: 0.21858161687850952 (+0.008592366025998055) + | > avg_loss_disc_real_2: 0.2423087518948775 (-0.013195524995143593) + | > avg_loss_disc_real_3: 0.271854518697812 (+0.031843635898370026) + | > avg_loss_disc_real_4: 0.26782856308496916 (+0.016084328293800354) + | > avg_loss_disc_real_5: 0.23147975825346434 (+0.004302405394040643) + | > avg_loss_0: 2.6073112304394064 (+0.06346102861257741) + | > avg_loss_gen: 2.4391451065356913 (-0.2074863543877239) + | > avg_loss_kl: 3.421067109474769 (-0.0037052447979264436) + | > avg_loss_feat: 7.229497982905461 (-0.654120408571683) + | > avg_loss_mel: 16.678324185884914 (-0.47369487469013194) + | > avg_loss_duration: 1.3417806717065663 (+0.05643479640667226) + | > avg_loss_1: 31.109814863938553 (-1.282571792602539) + + + > EPOCH: 72/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:15:36)  + + --> TIME: 2024-07-13 21:16:07 -- STEP: 27/151 -- GLOBAL_STEP: 95900 + | > loss_disc: 2.2430436611175537 (2.3281165052343304) + | > loss_disc_real_0: 0.07183828204870224 (0.1259675199786822) + | > loss_disc_real_1: 0.19124816358089447 (0.20387969635151051) + | > loss_disc_real_2: 0.18805883824825287 (0.2115501579311159) + | > loss_disc_real_3: 0.16919855773448944 (0.20697071899970373) + | > loss_disc_real_4: 0.17375794053077698 (0.20830714647416715) + | > loss_disc_real_5: 0.24357891082763672 (0.21623232463995615) + | > loss_0: 2.2430436611175537 (2.3281165052343304) + | > grad_norm_0: tensor(26.4950, device='cuda:0') (tensor(25.1116, device='cuda:0')) + | > loss_gen: 2.59194016456604 (2.5384376932073516) + | > loss_kl: 1.4384722709655762 (1.8181886584670455) + | > loss_feat: 8.777868270874023 (7.774200510095667) + | > loss_mel: 16.95703887939453 (16.656233504966455) + | > loss_duration: 0.9613832831382751 (0.9459419736155757) + | > loss_1: 30.726703643798828 (29.733002132839626) + | > grad_norm_1: tensor(542.1337, device='cuda:0') (tensor(456.3948, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.8336 (1.0533778490843597) + | > loader_time: 0.0041 (0.0068453506187156395) + + + --> TIME: 2024-07-13 21:16:54 -- STEP: 77/151 -- GLOBAL_STEP: 95950 + | > loss_disc: 2.146710157394409 (2.310155453620019) + | > loss_disc_real_0: 0.09377529472112656 (0.12367607875690832) + | > loss_disc_real_1: 0.17652848362922668 (0.20223736395309497) + | > loss_disc_real_2: 0.1963573843240738 (0.2119383208163373) + | > loss_disc_real_3: 0.1871161162853241 (0.20721687724838012) + | > loss_disc_real_4: 0.17479978501796722 (0.2061831165443767) + | > loss_disc_real_5: 0.2378624975681305 (0.21817480956579183) + | > loss_0: 2.146710157394409 (2.310155453620019) + | > grad_norm_0: tensor(37.2986, device='cuda:0') (tensor(32.6152, device='cuda:0')) + | > loss_gen: 2.6075193881988525 (2.610387978615699) + | > loss_kl: 1.0128861665725708 (1.75068425745159) + | > loss_feat: 9.511651992797852 (8.192213745860311) + | > loss_mel: 16.35051918029785 (16.835962146907658) + | > loss_duration: 0.9977815747261047 (0.9430038371643463) + | > loss_1: 30.480358123779297 (30.33225190794313) + | > grad_norm_1: tensor(613.1164, device='cuda:0') (tensor(520.2742, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.8072 (0.9765001179335949) + | > loader_time: 0.0042 (0.006165074063585949) + + + --> TIME: 2024-07-13 21:17:41 -- STEP: 127/151 -- GLOBAL_STEP: 96000 + | > loss_disc: 2.430490493774414 (2.3075365072160263) + | > loss_disc_real_0: 0.150640606880188 (0.12047283850082262) + | > loss_disc_real_1: 0.17273575067520142 (0.20160355800249447) + | > loss_disc_real_2: 0.19213329255580902 (0.21009483123858144) + | > loss_disc_real_3: 0.21413898468017578 (0.20823036967300054) + | > loss_disc_real_4: 0.20544952154159546 (0.20675661913522586) + | > loss_disc_real_5: 0.2259071320295334 (0.2195901366200034) + | > loss_0: 2.430490493774414 (2.3075365072160263) + | > grad_norm_0: tensor(30.1981, device='cuda:0') (tensor(32.3847, device='cuda:0')) + | > loss_gen: 2.331462860107422 (2.613557956350131) + | > loss_kl: 1.6268059015274048 (1.746631920807005) + | > loss_feat: 7.118470191955566 (8.221456794288216) + | > loss_mel: 15.072834014892578 (16.844606587267297) + | > loss_duration: 0.8693889379501343 (0.9457110190016078) + | > loss_1: 27.018962860107422 (30.371964251901222) + | > grad_norm_1: tensor(436.5545, device='cuda:0') (tensor(526.3688, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.8969 (0.9582954035030573) + | > loader_time: 0.0055 (0.006017696200393315) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.045716359065129206 (+0.012405413847703196) + | > avg_loss_disc: 2.50344861470736 (-0.10386261573204658) + | > avg_loss_disc_real_0: 0.19230189804847425 (-0.050954882915203364) + | > avg_loss_disc_real_1: 0.20413949283269736 (-0.01444212404581216) + | > avg_loss_disc_real_2: 0.2026059627532959 (-0.039702789141581596) + | > avg_loss_disc_real_3: 0.23129335504311782 (-0.0405611636546942) + | > avg_loss_disc_real_4: 0.2652400674728247 (-0.00258849561214447) + | > avg_loss_disc_real_5: 0.25539836516747105 (+0.023918606914006713) + | > avg_loss_0: 2.50344861470736 (-0.10386261573204658) + | > avg_loss_gen: 2.4125864689166727 (-0.026558637619018555) + | > avg_loss_kl: 3.4240177411299486 (+0.002950631655179503) + | > avg_loss_feat: 7.962269416222205 (+0.7327714333167439) + | > avg_loss_mel: 17.21913271683913 (+0.5408085309542159) + | > avg_loss_duration: 1.2909013766508832 (-0.050879295055683116) + | > avg_loss_1: 32.30890743549053 (+1.1990925715519793) + + + > EPOCH: 73/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:18:43)  + + --> TIME: 2024-07-13 21:19:12 -- STEP: 26/151 -- GLOBAL_STEP: 96050 + | > loss_disc: 2.232405662536621 (2.3189714642671437) + | > loss_disc_real_0: 0.10693612694740295 (0.12076885554079826) + | > loss_disc_real_1: 0.20755967497825623 (0.20421482450686967) + | > loss_disc_real_2: 0.22471977770328522 (0.2105238065123558) + | > loss_disc_real_3: 0.20403145253658295 (0.21145105190002) + | > loss_disc_real_4: 0.23585917055606842 (0.20938332493488604) + | > loss_disc_real_5: 0.20202594995498657 (0.21837318745943216) + | > loss_0: 2.232405662536621 (2.3189714642671437) + | > grad_norm_0: tensor(51.1376, device='cuda:0') (tensor(34.2700, device='cuda:0')) + | > loss_gen: 2.641990900039673 (2.5889377685693593) + | > loss_kl: 2.09443998336792 (1.7754106361132402) + | > loss_feat: 6.616915702819824 (8.088145952958325) + | > loss_mel: 16.40842056274414 (16.637261464045597) + | > loss_duration: 1.0030626058578491 (0.9499297348352579) + | > loss_1: 28.764829635620117 (30.039685909564678) + | > grad_norm_1: tensor(558.3171, device='cuda:0') (tensor(549.4313, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.9947 (1.004143632375277) + | > loader_time: 0.0062 (0.005985911075885479) + + + --> TIME: 2024-07-13 21:20:01 -- STEP: 76/151 -- GLOBAL_STEP: 96100 + | > loss_disc: 2.32305908203125 (2.2996723840111164) + | > loss_disc_real_0: 0.06886741518974304 (0.12440304565978684) + | > loss_disc_real_1: 0.14932918548583984 (0.20102542521137937) + | > loss_disc_real_2: 0.19624802470207214 (0.20984870097354838) + | > loss_disc_real_3: 0.25542742013931274 (0.20275671868339967) + | > loss_disc_real_4: 0.2543018162250519 (0.2055691953160261) + | > loss_disc_real_5: 0.20603081583976746 (0.2164721945790868) + | > loss_0: 2.32305908203125 (2.2996723840111164) + | > grad_norm_0: tensor(46.3946, device='cuda:0') (tensor(38.5237, device='cuda:0')) + | > loss_gen: 2.772674798965454 (2.6326098285223307) + | > loss_kl: 1.6752417087554932 (1.770442280330156) + | > loss_feat: 10.058996200561523 (8.356354688343245) + | > loss_mel: 17.746061325073242 (16.835559368133552) + | > loss_duration: 0.8469927310943604 (0.9583764672279358) + | > loss_1: 33.09996795654297 (30.553342794117174) + | > grad_norm_1: tensor(815.1874, device='cuda:0') (tensor(580.8748, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.8913 (0.9767522184472335) + | > loader_time: 0.0052 (0.0059147037957844) + + + --> TIME: 2024-07-13 21:20:49 -- STEP: 126/151 -- GLOBAL_STEP: 96150 + | > loss_disc: 2.184886932373047 (2.3071667343851114) + | > loss_disc_real_0: 0.07584699988365173 (0.1290482858580257) + | > loss_disc_real_1: 0.17766854166984558 (0.20223463870703226) + | > loss_disc_real_2: 0.18205900490283966 (0.2095711323476973) + | > loss_disc_real_3: 0.19832855463027954 (0.20410026592158137) + | > loss_disc_real_4: 0.20985887944698334 (0.20755911878650152) + | > loss_disc_real_5: 0.2251937836408615 (0.2170551653419222) + | > loss_0: 2.184886932373047 (2.3071667343851114) + | > grad_norm_0: tensor(30.6136, device='cuda:0') (tensor(40.2433, device='cuda:0')) + | > loss_gen: 2.6668097972869873 (2.640976544410464) + | > loss_kl: 1.4766461849212646 (1.7605634535115862) + | > loss_feat: 8.966196060180664 (8.35482844473824) + | > loss_mel: 16.27648162841797 (16.957895172966836) + | > loss_duration: 0.8936237096786499 (0.9504623029913221) + | > loss_1: 30.279756546020508 (30.664726151360405) + | > grad_norm_1: tensor(602.6964, device='cuda:0') (tensor(604.9470, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.8672 (0.970756589420258) + | > loader_time: 0.0061 (0.0058721057952396466) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0352170100578895 (-0.010499349007239707) + | > avg_loss_disc: 2.5269450407761793 (+0.02349642606881952) + | > avg_loss_disc_real_0: 0.1905541018797801 (-0.001747796168694138) + | > avg_loss_disc_real_1: 0.19936408675633943 (-0.00477540607635793) + | > avg_loss_disc_real_2: 0.2389563643015348 (+0.0363504015482389) + | > avg_loss_disc_real_3: 0.20772279455111578 (-0.02357056049200204) + | > avg_loss_disc_real_4: 0.23296227134191072 (-0.03227779613091397) + | > avg_loss_disc_real_5: 0.2086884631560399 (-0.04670990201143116) + | > avg_loss_0: 2.5269450407761793 (+0.02349642606881952) + | > avg_loss_gen: 2.289970929806049 (-0.12261553911062384) + | > avg_loss_kl: 3.480939975151649 (+0.05692223402170038) + | > avg_loss_feat: 8.086399775284987 (+0.12413035906278136) + | > avg_loss_mel: 17.1182918548584 (-0.10084086198073194) + | > avg_loss_duration: 1.2787145192806537 (-0.012186857370229509) + | > avg_loss_1: 32.25431735698994 (-0.054590078500595496) + + + > EPOCH: 74/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:21:25)  + + --> TIME: 2024-07-13 21:21:54 -- STEP: 25/151 -- GLOBAL_STEP: 96200 + | > loss_disc: 2.4562320709228516 (2.3506278133392335) + | > loss_disc_real_0: 0.13156314194202423 (0.12265807881951332) + | > loss_disc_real_1: 0.20393861830234528 (0.20201430201530457) + | > loss_disc_real_2: 0.2086067646741867 (0.2123407804965973) + | > loss_disc_real_3: 0.20118360221385956 (0.2126907801628113) + | > loss_disc_real_4: 0.17404776811599731 (0.21238344967365264) + | > loss_disc_real_5: 0.25266534090042114 (0.22055898129940033) + | > loss_0: 2.4562320709228516 (2.3506278133392335) + | > grad_norm_0: tensor(19.4856, device='cuda:0') (tensor(29.6434, device='cuda:0')) + | > loss_gen: 2.3934435844421387 (2.53029953956604) + | > loss_kl: 2.193570137023926 (1.7367288422584535) + | > loss_feat: 7.452971458435059 (7.790814456939697) + | > loss_mel: 17.228675842285156 (16.603540840148927) + | > loss_duration: 0.8893237709999084 (0.966836884021759) + | > loss_1: 30.15798568725586 (29.628220748901366) + | > grad_norm_1: tensor(551.1869, device='cuda:0') (tensor(516.5660, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.9215 (1.0612821006774902) + | > loader_time: 0.006 (0.0067021846771240234) + + + --> TIME: 2024-07-13 21:22:43 -- STEP: 75/151 -- GLOBAL_STEP: 96250 + | > loss_disc: 2.1717967987060547 (2.350463447570801) + | > loss_disc_real_0: 0.10665158927440643 (0.12342901418606445) + | > loss_disc_real_1: 0.1599700003862381 (0.21228614052136738) + | > loss_disc_real_2: 0.22388161718845367 (0.21462369521458943) + | > loss_disc_real_3: 0.20010310411453247 (0.20909949709971745) + | > loss_disc_real_4: 0.18994209170341492 (0.2108774201075236) + | > loss_disc_real_5: 0.19115696847438812 (0.21726796229680378) + | > loss_0: 2.1717967987060547 (2.350463447570801) + | > grad_norm_0: tensor(17.0386, device='cuda:0') (tensor(29.0659, device='cuda:0')) + | > loss_gen: 2.596921920776367 (2.565915962855021) + | > loss_kl: 1.6025160551071167 (1.779331918557485) + | > loss_feat: 8.4142484664917 (7.976077721913655) + | > loss_mel: 15.975427627563477 (16.77768109639485) + | > loss_duration: 0.9287015795707703 (0.9570276085535685) + | > loss_1: 29.51781463623047 (30.056034342447916) + | > grad_norm_1: tensor(401.8691, device='cuda:0') (tensor(521.7778, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.8074 (0.9974261442820231) + | > loader_time: 0.0041 (0.006234318415323893) + + + --> TIME: 2024-07-13 21:23:34 -- STEP: 125/151 -- GLOBAL_STEP: 96300 + | > loss_disc: 2.4968345165252686 (2.344206001281738) + | > loss_disc_real_0: 0.20241978764533997 (0.12418534150719646) + | > loss_disc_real_1: 0.2416960746049881 (0.20970605713129042) + | > loss_disc_real_2: 0.2815587818622589 (0.2142720513343811) + | > loss_disc_real_3: 0.2672187387943268 (0.20836318963766098) + | > loss_disc_real_4: 0.19645880162715912 (0.21010889554023743) + | > loss_disc_real_5: 0.2513238191604614 (0.2184292826652527) + | > loss_0: 2.4968345165252686 (2.344206001281738) + | > grad_norm_0: tensor(33.5263, device='cuda:0') (tensor(31.6670, device='cuda:0')) + | > loss_gen: 2.528581380844116 (2.5765586166381844) + | > loss_kl: 2.0490565299987793 (1.7856384825706482) + | > loss_feat: 5.973130226135254 (8.0298115158081) + | > loss_mel: 16.40308952331543 (16.83684992218016) + | > loss_duration: 1.1323727369308472 (0.9507317690849304) + | > loss_1: 28.08622932434082 (30.17959033203125) + | > grad_norm_1: tensor(476.8702, device='cuda:0') (tensor(546.5881, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 1.8235 (1.0012075557708742) + | > loader_time: 0.0147 (0.006406492233276368) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03460564980140099 (-0.0006113602564885093) + | > avg_loss_disc: 2.5537737883054294 (+0.02682874752925013) + | > avg_loss_disc_real_0: 0.24045744423682874 (+0.04990334235704863) + | > avg_loss_disc_real_1: 0.249568909406662 (+0.050204822650322556) + | > avg_loss_disc_real_2: 0.21490218089177057 (-0.02405418340976423) + | > avg_loss_disc_real_3: 0.24125536703146422 (+0.03353257248034844) + | > avg_loss_disc_real_4: 0.29053384409501004 (+0.05757157275309932) + | > avg_loss_disc_real_5: 0.2254302857013849 (+0.01674182254534501) + | > avg_loss_0: 2.5537737883054294 (+0.02682874752925013) + | > avg_loss_gen: 2.5733060103196363 (+0.28333508051358747) + | > avg_loss_kl: 3.6012228452242336 (+0.12028287007258465) + | > avg_loss_feat: 7.755935412186843 (-0.33046436309814364) + | > avg_loss_mel: 17.086061697739822 (-0.03223015711857613) + | > avg_loss_duration: 1.3074974280137281 (+0.02878290873307443) + | > avg_loss_1: 32.3240233201247 (+0.06970596313476562) + + + > EPOCH: 75/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:24:09)  + + --> TIME: 2024-07-13 21:24:35 -- STEP: 24/151 -- GLOBAL_STEP: 96350 + | > loss_disc: 2.3151559829711914 (2.3249104817708335) + | > loss_disc_real_0: 0.10463683307170868 (0.1160288102303942) + | > loss_disc_real_1: 0.24209772050380707 (0.20651799129943052) + | > loss_disc_real_2: 0.25736701488494873 (0.2159256705393394) + | > loss_disc_real_3: 0.27369123697280884 (0.21246431022882462) + | > loss_disc_real_4: 0.21924550831317902 (0.20723515128095946) + | > loss_disc_real_5: 0.296274870634079 (0.222512137144804) + | > loss_0: 2.3151559829711914 (2.3249104817708335) + | > grad_norm_0: tensor(37.1855, device='cuda:0') (tensor(37.1904, device='cuda:0')) + | > loss_gen: 2.761218309402466 (2.6352522472540536) + | > loss_kl: 1.8022303581237793 (1.8390220999717712) + | > loss_feat: 8.395910263061523 (8.22357859214147) + | > loss_mel: 17.832094192504883 (16.880394697189328) + | > loss_duration: 0.9278717041015625 (0.9504389613866806) + | > loss_1: 31.71932601928711 (30.5286869208018) + | > grad_norm_1: tensor(552.4919, device='cuda:0') (tensor(630.9279, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.9072 (0.976778248945872) + | > loader_time: 0.0058 (0.005832890669504802) + + + --> TIME: 2024-07-13 21:25:24 -- STEP: 74/151 -- GLOBAL_STEP: 96400 + | > loss_disc: 2.5864357948303223 (2.298708812610523) + | > loss_disc_real_0: 0.23673884570598602 (0.11431411155373664) + | > loss_disc_real_1: 0.20579595863819122 (0.20245940912816976) + | > loss_disc_real_2: 0.2391512542963028 (0.2134746695692475) + | > loss_disc_real_3: 0.2061813920736313 (0.20571288808777527) + | > loss_disc_real_4: 0.20183151960372925 (0.20707592891680227) + | > loss_disc_real_5: 0.23821227252483368 (0.21990592032670975) + | > loss_0: 2.5864357948303223 (2.298708812610523) + | > grad_norm_0: tensor(98.9900, device='cuda:0') (tensor(36.6469, device='cuda:0')) + | > loss_gen: 2.3619771003723145 (2.6402783136110046) + | > loss_kl: 2.1855857372283936 (1.826533452884571) + | > loss_feat: 6.11223030090332 (8.429172902493864) + | > loss_mel: 16.33943748474121 (17.089222418295368) + | > loss_duration: 1.076450228691101 (0.9368859134815835) + | > loss_1: 28.075679779052734 (30.922093159443623) + | > grad_norm_1: tensor(983.8759, device='cuda:0') (tensor(614.9913, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 1.5224 (0.9625506239968378) + | > loader_time: 0.0113 (0.0059453635602384) + + + --> TIME: 2024-07-13 21:26:13 -- STEP: 124/151 -- GLOBAL_STEP: 96450 + | > loss_disc: 2.1495351791381836 (2.305317796045733) + | > loss_disc_real_0: 0.05981352925300598 (0.12095701556292272) + | > loss_disc_real_1: 0.2106553614139557 (0.20349255883164943) + | > loss_disc_real_2: 0.16723351180553436 (0.21220716389436875) + | > loss_disc_real_3: 0.20743145048618317 (0.2058630466701523) + | > loss_disc_real_4: 0.17122749984264374 (0.20673129106721572) + | > loss_disc_real_5: 0.2533334493637085 (0.2179727235869054) + | > loss_0: 2.1495351791381836 (2.305317796045733) + | > grad_norm_0: tensor(17.5234, device='cuda:0') (tensor(40.2129, device='cuda:0')) + | > loss_gen: 3.0572714805603027 (2.650386921821101) + | > loss_kl: 2.0396528244018555 (1.8112315175033384) + | > loss_feat: 9.799646377563477 (8.380027444131915) + | > loss_mel: 17.801671981811523 (17.01992138739555) + | > loss_duration: 0.8593929409980774 (0.9389438696445958) + | > loss_1: 33.55763626098633 (30.800511314022927) + | > grad_norm_1: tensor(653.2739, device='cuda:0') (tensor(623.7084, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.8513 (0.9704674251617924) + | > loader_time: 0.004 (0.006008626953248055) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03229592396662785 (-0.002309725834773138) + | > avg_loss_disc: 2.513526274607732 (-0.040247513697697634) + | > avg_loss_disc_real_0: 0.21846002111068138 (-0.02199742312614736) + | > avg_loss_disc_real_1: 0.20897556382876176 (-0.04059334557790023) + | > avg_loss_disc_real_2: 0.25368307530879974 (+0.03878089441702917) + | > avg_loss_disc_real_3: 0.19383223354816437 (-0.04742313348329985) + | > avg_loss_disc_real_4: 0.22926011108435118 (-0.06127373301065886) + | > avg_loss_disc_real_5: 0.21499082331474012 (-0.010439462386644782) + | > avg_loss_0: 2.513526274607732 (-0.040247513697697634) + | > avg_loss_gen: 2.419823719904973 (-0.15348229041466332) + | > avg_loss_kl: 3.4603208211752086 (-0.14090202404902508) + | > avg_loss_feat: 7.612174144157996 (-0.1437612680288467) + | > avg_loss_mel: 16.92243341299204 (-0.16362828474778368) + | > avg_loss_duration: 1.292609031383808 (-0.014888396629920253) + | > avg_loss_1: 31.707361221313477 (-0.6166620988112257) + + + > EPOCH: 76/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:26:50)  + + --> TIME: 2024-07-13 21:27:15 -- STEP: 23/151 -- GLOBAL_STEP: 96500 + | > loss_disc: 2.5815255641937256 (2.3481779513151753) + | > loss_disc_real_0: 0.15530194342136383 (0.12226414340345756) + | > loss_disc_real_1: 0.2451641857624054 (0.2128371078035106) + | > loss_disc_real_2: 0.2897630035877228 (0.21538335408853448) + | > loss_disc_real_3: 0.26041021943092346 (0.21230296984962796) + | > loss_disc_real_4: 0.33414867520332336 (0.21516015218651813) + | > loss_disc_real_5: 0.25919729471206665 (0.21044859873211902) + | > loss_0: 2.5815255641937256 (2.3481779513151753) + | > grad_norm_0: tensor(79.3015, device='cuda:0') (tensor(45.6283, device='cuda:0')) + | > loss_gen: 2.4781317710876465 (2.618050191713416) + | > loss_kl: 2.156965494155884 (1.7921596972838691) + | > loss_feat: 6.08212947845459 (8.024928777114205) + | > loss_mel: 16.791751861572266 (16.94856083911398) + | > loss_duration: 1.14811110496521 (0.9370455690052198) + | > loss_1: 28.657089233398438 (30.32074546813965) + | > grad_norm_1: tensor(330.1783, device='cuda:0') (tensor(675.6716, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 2.1444 (0.9678928126459536) + | > loader_time: 0.0153 (0.00558906016142472) + + + --> TIME: 2024-07-13 21:28:04 -- STEP: 73/151 -- GLOBAL_STEP: 96550 + | > loss_disc: 2.349184513092041 (2.3118884106204933) + | > loss_disc_real_0: 0.18648724257946014 (0.11947561392228898) + | > loss_disc_real_1: 0.20358380675315857 (0.20486483978082057) + | > loss_disc_real_2: 0.23768369853496552 (0.2111082915982155) + | > loss_disc_real_3: 0.2619471848011017 (0.20775117490389575) + | > loss_disc_real_4: 0.245958149433136 (0.2068026404674739) + | > loss_disc_real_5: 0.2522342801094055 (0.21795210630109865) + | > loss_0: 2.349184513092041 (2.3118884106204933) + | > grad_norm_0: tensor(45.6149, device='cuda:0') (tensor(36.3075, device='cuda:0')) + | > loss_gen: 2.6182188987731934 (2.60777133784882) + | > loss_kl: 2.077099323272705 (1.7338491706815484) + | > loss_feat: 7.07626485824585 (8.217611319398232) + | > loss_mel: 17.2163143157959 (16.810032478750575) + | > loss_duration: 0.953789234161377 (0.943606294997751) + | > loss_1: 29.941686630249023 (30.312870548196035) + | > grad_norm_1: tensor(786.5743, device='cuda:0') (tensor(576.9376, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 1.0827 (0.9717152543263893) + | > loader_time: 0.0082 (0.005954846943894478) + + + --> TIME: 2024-07-13 21:28:56 -- STEP: 123/151 -- GLOBAL_STEP: 96600 + | > loss_disc: 2.4791274070739746 (2.3263924315692943) + | > loss_disc_real_0: 0.11393614113330841 (0.1248037683043054) + | > loss_disc_real_1: 0.22053393721580505 (0.20422147844380478) + | > loss_disc_real_2: 0.24649594724178314 (0.21243414173765882) + | > loss_disc_real_3: 0.23020264506340027 (0.2070436620615362) + | > loss_disc_real_4: 0.23173309862613678 (0.20856862981629565) + | > loss_disc_real_5: 0.2567060589790344 (0.21996736938391276) + | > loss_0: 2.4791274070739746 (2.3263924315692943) + | > grad_norm_0: tensor(15.4556, device='cuda:0') (tensor(38.4492, device='cuda:0')) + | > loss_gen: 2.4546890258789062 (2.6077932768720915) + | > loss_kl: 1.7751809358596802 (1.7613269495285624) + | > loss_feat: 7.8628950119018555 (8.152970806369941) + | > loss_mel: 17.307043075561523 (16.862978834446853) + | > loss_duration: 0.9100756645202637 (0.947144409989923) + | > loss_1: 30.30988311767578 (30.33221429344115) + | > grad_norm_1: tensor(536.6017, device='cuda:0') (tensor(586.0558, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 1.0112 (0.9993603849798683) + | > loader_time: 0.0075 (0.006323566281698584) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031646013259887695 (-0.000649910706740156) + | > avg_loss_disc: 2.512754825445322 (-0.0007714491624097874) + | > avg_loss_disc_real_0: 0.21049909522900215 (-0.007960925881679237) + | > avg_loss_disc_real_1: 0.209632274049979 (+0.0006567102212172438) + | > avg_loss_disc_real_2: 0.24008814073525941 (-0.01359493457354033) + | > avg_loss_disc_real_3: 0.24740734696388245 (+0.05357511341571808) + | > avg_loss_disc_real_4: 0.2786342581877342 (+0.049374147103383004) + | > avg_loss_disc_real_5: 0.2031941138781034 (-0.011796709436636715) + | > avg_loss_0: 2.512754825445322 (-0.0007714491624097874) + | > avg_loss_gen: 2.481378812056321 (+0.061555092151348134) + | > avg_loss_kl: 3.656963770206158 (+0.1966429490309496) + | > avg_loss_feat: 7.91633389546321 (+0.3041597513052139) + | > avg_loss_mel: 17.050319524911732 (+0.12788611191969323) + | > avg_loss_duration: 1.3313703628686757 (+0.03876133148486782) + | > avg_loss_1: 32.43636732835036 (+0.7290061070368807) + + + > EPOCH: 77/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:29:32)  + + --> TIME: 2024-07-13 21:29:58 -- STEP: 22/151 -- GLOBAL_STEP: 96650 + | > loss_disc: 2.2716832160949707 (2.2974325093356045) + | > loss_disc_real_0: 0.14138337969779968 (0.11608616398139433) + | > loss_disc_real_1: 0.2907145023345947 (0.21180787276137958) + | > loss_disc_real_2: 0.24384447932243347 (0.21916766532442786) + | > loss_disc_real_3: 0.22069455683231354 (0.20145933736454358) + | > loss_disc_real_4: 0.22279833257198334 (0.1974705999547785) + | > loss_disc_real_5: 0.22059515118598938 (0.21537215398116547) + | > loss_0: 2.2716832160949707 (2.2974325093356045) + | > grad_norm_0: tensor(16.3103, device='cuda:0') (tensor(35.1676, device='cuda:0')) + | > loss_gen: 2.764932632446289 (2.635969736359336) + | > loss_kl: 1.9697166681289673 (1.796530550176447) + | > loss_feat: 8.160417556762695 (8.397010283036666) + | > loss_mel: 17.526153564453125 (17.08507546511564) + | > loss_duration: 0.9630782604217529 (0.9637909098104998) + | > loss_1: 31.38429832458496 (30.87837722084739) + | > grad_norm_1: tensor(584.5913, device='cuda:0') (tensor(605.8556, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 1.1038 (1.0385471040552312) + | > loader_time: 0.0086 (0.006418033079667525) + + + --> TIME: 2024-07-13 21:30:44 -- STEP: 72/151 -- GLOBAL_STEP: 96700 + | > loss_disc: 2.329422950744629 (2.3017195628749) + | > loss_disc_real_0: 0.08454388380050659 (0.11958372758494483) + | > loss_disc_real_1: 0.18102827668190002 (0.20538386992282337) + | > loss_disc_real_2: 0.20901800692081451 (0.21388318099909362) + | > loss_disc_real_3: 0.1841445416212082 (0.20366101122150818) + | > loss_disc_real_4: 0.1952085793018341 (0.20441829909880954) + | > loss_disc_real_5: 0.20484183728694916 (0.21806343210240206) + | > loss_0: 2.329422950744629 (2.3017195628749) + | > grad_norm_0: tensor(35.3583, device='cuda:0') (tensor(29.4631, device='cuda:0')) + | > loss_gen: 2.512974739074707 (2.5937581161657968) + | > loss_kl: 1.7551209926605225 (1.77386198606756) + | > loss_feat: 8.858877182006836 (8.336585177315603) + | > loss_mel: 16.676435470581055 (16.92613837454054) + | > loss_duration: 0.964268147945404 (0.9287712607118819) + | > loss_1: 30.767677307128906 (30.559115171432495) + | > grad_norm_1: tensor(440.9454, device='cuda:0') (tensor(510.4155, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.8877 (0.9461490346325768) + | > loader_time: 0.0051 (0.005883604288101196) + + + --> TIME: 2024-07-13 21:31:36 -- STEP: 122/151 -- GLOBAL_STEP: 96750 + | > loss_disc: 2.3772449493408203 (2.313632789205333) + | > loss_disc_real_0: 0.1311359405517578 (0.12394301486430598) + | > loss_disc_real_1: 0.14611081779003143 (0.20354289292800623) + | > loss_disc_real_2: 0.1347675621509552 (0.21177083681352804) + | > loss_disc_real_3: 0.1991579830646515 (0.20597335928287663) + | > loss_disc_real_4: 0.18089862167835236 (0.20739076146092572) + | > loss_disc_real_5: 0.22667023539543152 (0.21814725474744548) + | > loss_0: 2.3772449493408203 (2.313632789205333) + | > grad_norm_0: tensor(16.5061, device='cuda:0') (tensor(29.9834, device='cuda:0')) + | > loss_gen: 2.4111878871917725 (2.5819468869537596) + | > loss_kl: 1.4331127405166626 (1.788785048684136) + | > loss_feat: 8.365928649902344 (8.183458789450215) + | > loss_mel: 17.275836944580078 (16.752862297120636) + | > loss_duration: 0.9525548219680786 (0.9452387358321518) + | > loss_1: 30.438621520996094 (30.252291913892403) + | > grad_norm_1: tensor(525.1503, device='cuda:0') (tensor(522.2039, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.8765 (0.9818206009317617) + | > loader_time: 0.0053 (0.006288030108467477) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03164859918447641 (+2.585924588717481e-06) + | > avg_loss_disc: 2.5556842180398793 (+0.0429293925945573) + | > avg_loss_disc_real_0: 0.255350146156091 (+0.044851050927088826) + | > avg_loss_disc_real_1: 0.1886414736509323 (-0.02099080039904669) + | > avg_loss_disc_real_2: 0.21307490422175482 (-0.027013236513504596) + | > avg_loss_disc_real_3: 0.23155455062022576 (-0.01585279634365669) + | > avg_loss_disc_real_4: 0.28285728509609515 (+0.004223026908360961) + | > avg_loss_disc_real_5: 0.22796636132093576 (+0.02477224744283235) + | > avg_loss_0: 2.5556842180398793 (+0.0429293925945573) + | > avg_loss_gen: 2.4460603273831882 (-0.03531848467313292) + | > avg_loss_kl: 3.5853462586036096 (-0.07161751160254859) + | > avg_loss_feat: 7.951958582951472 (+0.0356246874882622) + | > avg_loss_mel: 16.92838213993953 (-0.12193738497220252) + | > avg_loss_duration: 1.3105835364415095 (-0.020786826427166227) + | > avg_loss_1: 32.222329946664665 (-0.2140373816856922) + + + > EPOCH: 78/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:32:13)  + + --> TIME: 2024-07-13 21:32:34 -- STEP: 21/151 -- GLOBAL_STEP: 96800 + | > loss_disc: 2.0935020446777344 (2.2116942292168025) + | > loss_disc_real_0: 0.08347710222005844 (0.10689303119267736) + | > loss_disc_real_1: 0.2502365708351135 (0.1978371256873721) + | > loss_disc_real_2: 0.22103692591190338 (0.20611759097803206) + | > loss_disc_real_3: 0.1945497691631317 (0.19824394157954625) + | > loss_disc_real_4: 0.2083936482667923 (0.2021095199244363) + | > loss_disc_real_5: 0.20058777928352356 (0.20105224209172384) + | > loss_0: 2.0935020446777344 (2.2116942292168025) + | > grad_norm_0: tensor(29.3817, device='cuda:0') (tensor(30.6473, device='cuda:0')) + | > loss_gen: 2.892233371734619 (2.7116963749840144) + | > loss_kl: 1.6838022470474243 (1.7236820799963815) + | > loss_feat: 9.713741302490234 (8.669957819439116) + | > loss_mel: 16.91237449645996 (16.903279213678275) + | > loss_duration: 1.0400959253311157 (0.9357655019987197) + | > loss_1: 32.242244720458984 (30.944380987258185) + | > grad_norm_1: tensor(223.5394, device='cuda:0') (tensor(582.9247, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.8253 (0.8850919405619303) + | > loader_time: 0.0041 (0.005116394587925502) + + + --> TIME: 2024-07-13 21:33:24 -- STEP: 71/151 -- GLOBAL_STEP: 96850 + | > loss_disc: 2.1092965602874756 (2.2808979262768374) + | > loss_disc_real_0: 0.09854450076818466 (0.12133316632727502) + | > loss_disc_real_1: 0.19046850502490997 (0.2036443058873566) + | > loss_disc_real_2: 0.18751870095729828 (0.21285864962658413) + | > loss_disc_real_3: 0.11642354726791382 (0.20499892343937512) + | > loss_disc_real_4: 0.17257706820964813 (0.20131092298198755) + | > loss_disc_real_5: 0.12448001652956009 (0.20802263086530523) + | > loss_0: 2.1092965602874756 (2.2808979262768374) + | > grad_norm_0: tensor(19.0435, device='cuda:0') (tensor(28.1858, device='cuda:0')) + | > loss_gen: 2.6219654083251953 (2.6300831479086) + | > loss_kl: 1.6483968496322632 (1.7960431273554411) + | > loss_feat: 10.479235649108887 (8.414899114152076) + | > loss_mel: 18.439903259277344 (16.88005067261172) + | > loss_duration: 0.9273247718811035 (0.9270035246728172) + | > loss_1: 34.116825103759766 (30.64807961692273) + | > grad_norm_1: tensor(358.0409, device='cuda:0') (tensor(472.2063, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.8977 (0.9536946592196612) + | > loader_time: 0.0067 (0.005856151312169895) + + + --> TIME: 2024-07-13 21:34:16 -- STEP: 121/151 -- GLOBAL_STEP: 96900 + | > loss_disc: 2.080064058303833 (2.311539220415856) + | > loss_disc_real_0: 0.0914156436920166 (0.1268823380189494) + | > loss_disc_real_1: 0.21550756692886353 (0.2047639657889516) + | > loss_disc_real_2: 0.17850370705127716 (0.214872072427726) + | > loss_disc_real_3: 0.19453705847263336 (0.20718304603553014) + | > loss_disc_real_4: 0.22625239193439484 (0.2053740093530702) + | > loss_disc_real_5: 0.20218320190906525 (0.21322789708198595) + | > loss_0: 2.080064058303833 (2.311539220415856) + | > grad_norm_0: tensor(14.1719, device='cuda:0') (tensor(33.6683, device='cuda:0')) + | > loss_gen: 2.8631253242492676 (2.6292537795610653) + | > loss_kl: 1.6393321752548218 (1.7891683356821044) + | > loss_feat: 9.55472469329834 (8.244754810963775) + | > loss_mel: 16.092483520507812 (16.828765364717846) + | > loss_duration: 1.0157021284103394 (0.9424080292055429) + | > loss_1: 31.165367126464844 (30.43435028958912) + | > grad_norm_1: tensor(165.1259, device='cuda:0') (tensor(499.5284, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.8175 (0.9824010616491649) + | > loader_time: 0.004 (0.006194372807652497) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03264098901015062 (+0.0009923898256742059) + | > avg_loss_disc: 2.6032988291520334 (+0.04761461111215404) + | > avg_loss_disc_real_0: 0.21884130170712104 (-0.03650884444896993) + | > avg_loss_disc_real_1: 0.18055509603940523 (-0.008086377611527085) + | > avg_loss_disc_real_2: 0.24192492434611687 (+0.028850020124362052) + | > avg_loss_disc_real_3: 0.2407799638234652 (+0.009225413203239441) + | > avg_loss_disc_real_4: 0.2652829553072269 (-0.017574329788868248) + | > avg_loss_disc_real_5: 0.2443756770629149 (+0.016409315741979152) + | > avg_loss_0: 2.6032988291520334 (+0.04761461111215404) + | > avg_loss_gen: 2.341332343908457 (-0.10472798347473145) + | > avg_loss_kl: 3.5088319961841288 (-0.07651426241948078) + | > avg_loss_feat: 7.388634865100567 (-0.5633237178509054) + | > avg_loss_mel: 16.53052513415997 (-0.39785700577956007) + | > avg_loss_duration: 1.3220239786001353 (+0.01144044215862583) + | > avg_loss_1: 31.091348501352165 (-1.1309814453125) + + + > EPOCH: 79/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:34:54)  + + --> TIME: 2024-07-13 21:35:15 -- STEP: 20/151 -- GLOBAL_STEP: 96950 + | > loss_disc: 2.1381924152374268 (2.2826168656349184) + | > loss_disc_real_0: 0.08588766306638718 (0.1094106299802661) + | > loss_disc_real_1: 0.1757802665233612 (0.19594942443072796) + | > loss_disc_real_2: 0.13625141978263855 (0.20847576558589936) + | > loss_disc_real_3: 0.1703016757965088 (0.20786888860166072) + | > loss_disc_real_4: 0.19867844879627228 (0.21153229288756847) + | > loss_disc_real_5: 0.1939866691827774 (0.22084422335028647) + | > loss_0: 2.1381924152374268 (2.2826168656349184) + | > grad_norm_0: tensor(21.6866, device='cuda:0') (tensor(22.2657, device='cuda:0')) + | > loss_gen: 2.774217128753662 (2.651393437385559) + | > loss_kl: 1.1883357763290405 (1.6877390563488006) + | > loss_feat: 10.711751937866211 (8.702907013893128) + | > loss_mel: 17.475433349609375 (17.036139678955077) + | > loss_duration: 0.9252544641494751 (0.9120243638753891) + | > loss_1: 33.07499313354492 (30.990203857421875) + | > grad_norm_1: tensor(538.8981, device='cuda:0') (tensor(432.1515, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.8119 (0.9112649917602539) + | > loader_time: 0.0041 (0.005797386169433594) + + + --> TIME: 2024-07-13 21:36:05 -- STEP: 70/151 -- GLOBAL_STEP: 97000 + | > loss_disc: 2.1175899505615234 (2.304527512618473) + | > loss_disc_real_0: 0.11474041640758514 (0.12052457992519651) + | > loss_disc_real_1: 0.173127681016922 (0.19974428489804266) + | > loss_disc_real_2: 0.20001500844955444 (0.21175868085452487) + | > loss_disc_real_3: 0.23158054053783417 (0.20627811732036727) + | > loss_disc_real_4: 0.23341554403305054 (0.2081894777715206) + | > loss_disc_real_5: 0.17676103115081787 (0.22079479822090695) + | > loss_0: 2.1175899505615234 (2.304527512618473) + | > grad_norm_0: tensor(24.5216, device='cuda:0') (tensor(27.1203, device='cuda:0')) + | > loss_gen: 2.9777514934539795 (2.611719083786011) + | > loss_kl: 1.5957530736923218 (1.7102768744741168) + | > loss_feat: 9.492895126342773 (8.322804703031265) + | > loss_mel: 17.994319915771484 (16.979419340406153) + | > loss_duration: 0.9610285758972168 (0.9315877343927111) + | > loss_1: 33.02174758911133 (30.5558077948434) + | > grad_norm_1: tensor(1000.1837, device='cuda:0') (tensor(497.1436, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 1.0602 (0.9673368590218681) + | > loader_time: 0.0068 (0.006185265949794225) + + + --> TIME: 2024-07-13 21:37:24 -- STEP: 120/151 -- GLOBAL_STEP: 97050 + | > loss_disc: 1.9772298336029053 (2.3134933441877368) + | > loss_disc_real_0: 0.10409095138311386 (0.12126624354471763) + | > loss_disc_real_1: 0.13872815668582916 (0.19963782584915554) + | > loss_disc_real_2: 0.16261723637580872 (0.21244226718942325) + | > loss_disc_real_3: 0.13106024265289307 (0.20579114848126967) + | > loss_disc_real_4: 0.17221234738826752 (0.20891068385293085) + | > loss_disc_real_5: 0.21012787520885468 (0.2201553773134947) + | > loss_0: 1.9772298336029053 (2.3134933441877368) + | > grad_norm_0: tensor(10.2772, device='cuda:0') (tensor(32.3349, device='cuda:0')) + | > loss_gen: 2.913074254989624 (2.6248366673787444) + | > loss_kl: 1.2950481176376343 (1.7251259769002596) + | > loss_feat: 10.152530670166016 (8.363248308499657) + | > loss_mel: 17.64314079284668 (17.027986947695425) + | > loss_duration: 1.018279790878296 (0.9392849291364352) + | > loss_1: 33.022071838378906 (30.680482880274454) + | > grad_norm_1: tensor(420.2248, device='cuda:0') (tensor(514.2579, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.8234 (0.9986691077550254) + | > loader_time: 0.0048 (0.006413491566975912) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04416590470534105 (+0.01152491569519043) + | > avg_loss_disc: 2.7155527885143576 (+0.11225395936232418) + | > avg_loss_disc_real_0: 0.32944106138669527 (+0.11059975967957422) + | > avg_loss_disc_real_1: 0.24982406428227058 (+0.06926896824286535) + | > avg_loss_disc_real_2: 0.2606413352947969 (+0.018716410948680012) + | > avg_loss_disc_real_3: 0.31772797382794893 (+0.07694801000448373) + | > avg_loss_disc_real_4: 0.2979116646143106 (+0.032628709307083725) + | > avg_loss_disc_real_5: 0.27039766082396877 (+0.026021983761053857) + | > avg_loss_0: 2.7155527885143576 (+0.11225395936232418) + | > avg_loss_gen: 2.856606520139254 (+0.5152741762307973) + | > avg_loss_kl: 3.403902053833008 (-0.10492994235112096) + | > avg_loss_feat: 7.673596602219802 (+0.284961737119235) + | > avg_loss_mel: 16.944650429945725 (+0.4141252957857553) + | > avg_loss_duration: 1.288795333642226 (-0.03322864495790934) + | > avg_loss_1: 32.167551040649414 (+1.076202539297249) + + + > EPOCH: 80/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:38:03)  + + --> TIME: 2024-07-13 21:38:24 -- STEP: 19/151 -- GLOBAL_STEP: 97100 + | > loss_disc: 2.5973596572875977 (2.35648552994979) + | > loss_disc_real_0: 0.2656802237033844 (0.13637320285564972) + | > loss_disc_real_1: 0.2315361648797989 (0.20049646692840675) + | > loss_disc_real_2: 0.319945752620697 (0.22325807221625982) + | > loss_disc_real_3: 0.20985637605190277 (0.21322556310578397) + | > loss_disc_real_4: 0.2317325621843338 (0.2114049306041316) + | > loss_disc_real_5: 0.23414134979248047 (0.23062733051023984) + | > loss_0: 2.5973596572875977 (2.35648552994979) + | > grad_norm_0: tensor(37.8011, device='cuda:0') (tensor(24.7850, device='cuda:0')) + | > loss_gen: 2.3724677562713623 (2.5360710871847054) + | > loss_kl: 2.0802791118621826 (1.780217729116741) + | > loss_feat: 6.003378868103027 (7.962478763178775) + | > loss_mel: 16.988601684570312 (17.02040898172479) + | > loss_duration: 0.9814006090164185 (0.9367795555215133) + | > loss_1: 28.426128387451172 (30.235955890856292) + | > grad_norm_1: tensor(340.9307, device='cuda:0') (tensor(386.8434, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 1.1389 (0.9780777002635755) + | > loader_time: 0.0083 (0.0061726695612857214) + + + --> TIME: 2024-07-13 21:39:14 -- STEP: 69/151 -- GLOBAL_STEP: 97150 + | > loss_disc: 2.125053644180298 (2.2943820573281544) + | > loss_disc_real_0: 0.09436319768428802 (0.12121758264475975) + | > loss_disc_real_1: 0.2020588368177414 (0.19946376299080643) + | > loss_disc_real_2: 0.21216168999671936 (0.2130787646856861) + | > loss_disc_real_3: 0.19812002778053284 (0.20517763279486392) + | > loss_disc_real_4: 0.2209751158952713 (0.20614540458157443) + | > loss_disc_real_5: 0.22403110563755035 (0.22149331306201825) + | > loss_0: 2.125053644180298 (2.2943820573281544) + | > grad_norm_0: tensor(19.1505, device='cuda:0') (tensor(26.3556, device='cuda:0')) + | > loss_gen: 2.794053554534912 (2.6022997558980743) + | > loss_kl: 1.8078261613845825 (1.7361256385195083) + | > loss_feat: 11.417614936828613 (8.415328495744339) + | > loss_mel: 18.37638282775879 (16.99908803856891) + | > loss_duration: 0.807875394821167 (0.940110841523046) + | > loss_1: 35.20375442504883 (30.692952695100203) + | > grad_norm_1: tensor(590.9426, device='cuda:0') (tensor(463.4435, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.8995 (0.9942497239596602) + | > loader_time: 0.0059 (0.006200873333474864) + + + --> TIME: 2024-07-13 21:40:03 -- STEP: 119/151 -- GLOBAL_STEP: 97200 + | > loss_disc: 2.1678171157836914 (2.3050607312627203) + | > loss_disc_real_0: 0.06351615488529205 (0.12714113676998795) + | > loss_disc_real_1: 0.18300843238830566 (0.19859913085438624) + | > loss_disc_real_2: 0.15117156505584717 (0.21201843874795095) + | > loss_disc_real_3: 0.15575601160526276 (0.20535979551427505) + | > loss_disc_real_4: 0.155867338180542 (0.20676165474813527) + | > loss_disc_real_5: 0.19181573390960693 (0.21900559479699416) + | > loss_0: 2.1678171157836914 (2.3050607312627203) + | > grad_norm_0: tensor(11.8250, device='cuda:0') (tensor(29.9526, device='cuda:0')) + | > loss_gen: 2.5038273334503174 (2.604820906615058) + | > loss_kl: 1.4176242351531982 (1.7558968808470654) + | > loss_feat: 9.339527130126953 (8.318009184188204) + | > loss_mel: 16.943204879760742 (16.999409218796156) + | > loss_duration: 0.9319394826889038 (0.9452867282538855) + | > loss_1: 31.136123657226562 (30.623422782962063) + | > grad_norm_1: tensor(225.4150, device='cuda:0') (tensor(465.6597, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.8139 (0.9816948445905157) + | > loader_time: 0.0039 (0.006147011989304999) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03807541040273813 (-0.006090494302602917) + | > avg_loss_disc: 2.442310223212609 (-0.27324256530174873) + | > avg_loss_disc_real_0: 0.2414789360303145 (-0.08796212535638076) + | > avg_loss_disc_real_1: 0.2127764798127688 (-0.03704758446950179) + | > avg_loss_disc_real_2: 0.19401087554601523 (-0.06663045974878165) + | > avg_loss_disc_real_3: 0.17270173476292536 (-0.14502623906502357) + | > avg_loss_disc_real_4: 0.21946931458436525 (-0.07844235002994537) + | > avg_loss_disc_real_5: 0.21335646280875573 (-0.05704119801521304) + | > avg_loss_0: 2.442310223212609 (-0.27324256530174873) + | > avg_loss_gen: 2.440915217766395 (-0.415691302372859) + | > avg_loss_kl: 3.6973056609813986 (+0.29340360714839075) + | > avg_loss_feat: 8.331507426041822 (+0.6579108238220206) + | > avg_loss_mel: 17.57024515592135 (+0.625594725975624) + | > avg_loss_duration: 1.3108335366615882 (+0.022038203019362212) + | > avg_loss_1: 33.35080807025616 (+1.183257029606743) + + + > EPOCH: 81/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:40:44)  + + --> TIME: 2024-07-13 21:41:04 -- STEP: 18/151 -- GLOBAL_STEP: 97250 + | > loss_disc: 2.336667776107788 (2.3501671420203314) + | > loss_disc_real_0: 0.09927152842283249 (0.12491674514280425) + | > loss_disc_real_1: 0.21047839522361755 (0.21596799542506537) + | > loss_disc_real_2: 0.22595559060573578 (0.21737000428968006) + | > loss_disc_real_3: 0.23474465310573578 (0.20800361616743934) + | > loss_disc_real_4: 0.2256988137960434 (0.21318481779760784) + | > loss_disc_real_5: 0.23916320502758026 (0.21799449622631073) + | > loss_0: 2.336667776107788 (2.3501671420203314) + | > grad_norm_0: tensor(18.7821, device='cuda:0') (tensor(16.7629, device='cuda:0')) + | > loss_gen: 2.5535495281219482 (2.571605099572076) + | > loss_kl: 1.8171828985214233 (1.8473265965779622) + | > loss_feat: 8.515034675598145 (8.17615662680732) + | > loss_mel: 16.538415908813477 (16.972700542873806) + | > loss_duration: 0.8835243582725525 (0.9518956442674001) + | > loss_1: 30.30770492553711 (30.519684155782063) + | > grad_norm_1: tensor(242.7114, device='cuda:0') (tensor(344.0090, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.8634 (0.9956346750259399) + | > loader_time: 0.0044 (0.00666544172498915) + + + --> TIME: 2024-07-13 21:41:55 -- STEP: 68/151 -- GLOBAL_STEP: 97300 + | > loss_disc: 2.549154758453369 (2.3484655793975375) + | > loss_disc_real_0: 0.1949821561574936 (0.12807733554612188) + | > loss_disc_real_1: 0.2240404337644577 (0.20857037340893464) + | > loss_disc_real_2: 0.2528943717479706 (0.2147446572780609) + | > loss_disc_real_3: 0.24252523481845856 (0.20987405474571622) + | > loss_disc_real_4: 0.23462815582752228 (0.2112763440784286) + | > loss_disc_real_5: 0.24452915787696838 (0.2199914919979432) + | > loss_0: 2.549154758453369 (2.3484655793975375) + | > grad_norm_0: tensor(15.0752, device='cuda:0') (tensor(16.5874, device='cuda:0')) + | > loss_gen: 2.4164435863494873 (2.550299283336191) + | > loss_kl: 2.154937744140625 (1.7568439858801224) + | > loss_feat: 7.134641647338867 (8.178897612235131) + | > loss_mel: 16.9829158782959 (16.954232510398416) + | > loss_duration: 0.8748368620872498 (0.9400956727126065) + | > loss_1: 29.56377410888672 (30.38036890590892) + | > grad_norm_1: tensor(145.5815, device='cuda:0') (tensor(262.5900, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 1.0621 (1.002168953418732) + | > loader_time: 0.0065 (0.006497060551362879) + + + --> TIME: 2024-07-13 21:42:41 -- STEP: 118/151 -- GLOBAL_STEP: 97350 + | > loss_disc: 2.2006518840789795 (2.315151745990171) + | > loss_disc_real_0: 0.118018738925457 (0.12492978348696634) + | > loss_disc_real_1: 0.17623980343341827 (0.20410519484746253) + | > loss_disc_real_2: 0.16733922064304352 (0.21027973319514323) + | > loss_disc_real_3: 0.1882070153951645 (0.20831179694604066) + | > loss_disc_real_4: 0.2329632043838501 (0.20724655814090018) + | > loss_disc_real_5: 0.27291539311408997 (0.218243868295419) + | > loss_0: 2.2006518840789795 (2.315151745990171) + | > grad_norm_0: tensor(22.4439, device='cuda:0') (tensor(19.9330, device='cuda:0')) + | > loss_gen: 2.7073512077331543 (2.5817476268542014) + | > loss_kl: 1.3775275945663452 (1.742700117119288) + | > loss_feat: 10.010494232177734 (8.294378296803615) + | > loss_mel: 17.995927810668945 (16.89715055691994) + | > loss_duration: 1.0770618915557861 (0.9369097927869376) + | > loss_1: 33.16836166381836 (30.452886403617214) + | > grad_norm_1: tensor(337.6049, device='cuda:0') (tensor(320.3347, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.8157 (0.9627951787689986) + | > loader_time: 0.0042 (0.006086513147515766) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032495627036461465 (-0.005579783366276667) + | > avg_loss_disc: 2.6038445692795973 (+0.16153434606698847) + | > avg_loss_disc_real_0: 0.2515440308130704 (+0.010065094782755912) + | > avg_loss_disc_real_1: 0.26084383175923276 (+0.04806735194646397) + | > avg_loss_disc_real_2: 0.23273161970652068 (+0.038720744160505444) + | > avg_loss_disc_real_3: 0.2375320872435203 (+0.06483035248059493) + | > avg_loss_disc_real_4: 0.2426548829445472 (+0.023185568360181957) + | > avg_loss_disc_real_5: 0.216067475768236 (+0.0027110129594802856) + | > avg_loss_0: 2.6038445692795973 (+0.16153434606698847) + | > avg_loss_gen: 2.430848158322848 (-0.010067059443547244) + | > avg_loss_kl: 3.577342180105356 (-0.11996348087604236) + | > avg_loss_feat: 7.644081005683312 (-0.6874264203585101) + | > avg_loss_mel: 16.958454132080078 (-0.6117910238412705) + | > avg_loss_duration: 1.2985940346351035 (-0.012239502026484717) + | > avg_loss_1: 31.909319657545822 (-1.441488412710335) + + + > EPOCH: 82/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:43:23)  + + --> TIME: 2024-07-13 21:43:41 -- STEP: 17/151 -- GLOBAL_STEP: 97400 + | > loss_disc: 2.2848339080810547 (2.318608550464406) + | > loss_disc_real_0: 0.07541780918836594 (0.1161407017532517) + | > loss_disc_real_1: 0.21351605653762817 (0.19285539406187394) + | > loss_disc_real_2: 0.15860822796821594 (0.21350803094751694) + | > loss_disc_real_3: 0.22116205096244812 (0.20211460090735378) + | > loss_disc_real_4: 0.1954825520515442 (0.21353434727472417) + | > loss_disc_real_5: 0.25022825598716736 (0.22938065581461964) + | > loss_0: 2.2848339080810547 (2.318608550464406) + | > grad_norm_0: tensor(11.1281, device='cuda:0') (tensor(19.4372, device='cuda:0')) + | > loss_gen: 2.6792023181915283 (2.557417995789472) + | > loss_kl: 1.4797561168670654 (1.678743369439069) + | > loss_feat: 9.051007270812988 (8.333089491900274) + | > loss_mel: 16.34873390197754 (16.60660799811868) + | > loss_duration: 0.957771897315979 (0.9357745472122642) + | > loss_1: 30.51647186279297 (30.11163330078125) + | > grad_norm_1: tensor(387.2486, device='cuda:0') (tensor(330.5636, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.8506 (0.9516808004940257) + | > loader_time: 0.0043 (0.005812406539916992) + + + --> TIME: 2024-07-13 21:44:34 -- STEP: 67/151 -- GLOBAL_STEP: 97450 + | > loss_disc: 2.3506784439086914 (2.336773365291197) + | > loss_disc_real_0: 0.08670847117900848 (0.12678378814859176) + | > loss_disc_real_1: 0.1986325979232788 (0.2024451985732833) + | > loss_disc_real_2: 0.21075698733329773 (0.21711563530252942) + | > loss_disc_real_3: 0.18591061234474182 (0.2049420127895341) + | > loss_disc_real_4: 0.19346100091934204 (0.2110950733298686) + | > loss_disc_real_5: 0.1766880750656128 (0.22209089416176525) + | > loss_0: 2.3506784439086914 (2.336773365291197) + | > grad_norm_0: tensor(32.1157, device='cuda:0') (tensor(28.6292, device='cuda:0')) + | > loss_gen: 2.223358154296875 (2.578501776083191) + | > loss_kl: 1.7068006992340088 (1.754965162988919) + | > loss_feat: 7.687076091766357 (8.088600293913883) + | > loss_mel: 15.04826545715332 (16.630969332225284) + | > loss_duration: 0.9026269316673279 (0.951671420638241) + | > loss_1: 27.568126678466797 (30.00470793425147) + | > grad_norm_1: tensor(495.2626, device='cuda:0') (tensor(450.9624, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.8496 (1.0105411686114412) + | > loader_time: 0.0042 (0.00662381969281097) + + + --> TIME: 2024-07-13 21:45:25 -- STEP: 117/151 -- GLOBAL_STEP: 97500 + | > loss_disc: 2.0297255516052246 (2.3151942756440906) + | > loss_disc_real_0: 0.06472528725862503 (0.1253305158426619) + | > loss_disc_real_1: 0.1522257775068283 (0.20114444622881392) + | > loss_disc_real_2: 0.1690550446510315 (0.2137474746276171) + | > loss_disc_real_3: 0.16576644778251648 (0.20533376925776148) + | > loss_disc_real_4: 0.16832897067070007 (0.20881513066780868) + | > loss_disc_real_5: 0.15000732243061066 (0.21667108092552576) + | > loss_0: 2.0297255516052246 (2.3151942756440906) + | > grad_norm_0: tensor(34.1901, device='cuda:0') (tensor(30.8415, device='cuda:0')) + | > loss_gen: 2.936522960662842 (2.60375498706459) + | > loss_kl: 1.6557669639587402 (1.7415832658099315) + | > loss_feat: 10.8561372756958 (8.165483796698414) + | > loss_mel: 17.731319427490234 (16.75113467681102) + | > loss_duration: 0.9560145139694214 (0.9549978130903) + | > loss_1: 34.13575744628906 (30.21695443503877) + | > grad_norm_1: tensor(565.4963, device='cuda:0') (tensor(480.3235, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.8317 (1.0125857010865824) + | > loader_time: 0.0044 (0.006542330114250509) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03409075736999512 (+0.0015951303335336522) + | > avg_loss_disc: 2.635681519141564 (+0.03183694986196661) + | > avg_loss_disc_real_0: 0.22052385829962218 (-0.03102017251344824) + | > avg_loss_disc_real_1: 0.204810856626584 (-0.056032975132648766) + | > avg_loss_disc_real_2: 0.22391314804553986 (-0.00881847166098082) + | > avg_loss_disc_real_3: 0.21768142741460067 (-0.01985065982891962) + | > avg_loss_disc_real_4: 0.23765759628552657 (-0.004997286659020633) + | > avg_loss_disc_real_5: 0.27605714477025545 (+0.059989669002019436) + | > avg_loss_0: 2.635681519141564 (+0.03183694986196661) + | > avg_loss_gen: 2.303419534976666 (-0.12742862334618188) + | > avg_loss_kl: 3.5686463759495664 (-0.008695804155789855) + | > avg_loss_feat: 7.951448660630446 (+0.30736765494713403) + | > avg_loss_mel: 17.389977381779598 (+0.43152324969952005) + | > avg_loss_duration: 1.3293932951413667 (+0.03079926050626325) + | > avg_loss_1: 32.54288423978365 (+0.6335645822378311) + + + > EPOCH: 83/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:46:06)  + + --> TIME: 2024-07-13 21:46:25 -- STEP: 16/151 -- GLOBAL_STEP: 97550 + | > loss_disc: 2.3386154174804688 (2.3121108561754227) + | > loss_disc_real_0: 0.10904859006404877 (0.11906594154424965) + | > loss_disc_real_1: 0.17267346382141113 (0.2061798321083188) + | > loss_disc_real_2: 0.25124019384384155 (0.2113411221653223) + | > loss_disc_real_3: 0.20833203196525574 (0.2136164866387844) + | > loss_disc_real_4: 0.23722761869430542 (0.20475909486413002) + | > loss_disc_real_5: 0.23075705766677856 (0.21821956243366003) + | > loss_0: 2.3386154174804688 (2.3121108561754227) + | > grad_norm_0: tensor(37.4298, device='cuda:0') (tensor(28.1773, device='cuda:0')) + | > loss_gen: 2.4591526985168457 (2.575697883963585) + | > loss_kl: 1.8516985177993774 (1.6897185370326042) + | > loss_feat: 8.010437965393066 (8.099579185247421) + | > loss_mel: 17.456462860107422 (16.76962685585022) + | > loss_duration: 0.9168807864189148 (0.9800468645989895) + | > loss_1: 30.69463348388672 (30.114669680595398) + | > grad_norm_1: tensor(636.1401, device='cuda:0') (tensor(456.1815, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.9781 (1.0268616378307343) + | > loader_time: 0.0052 (0.006788685917854309) + + + --> TIME: 2024-07-13 21:47:15 -- STEP: 66/151 -- GLOBAL_STEP: 97600 + | > loss_disc: 2.283020257949829 (2.3115259156082613) + | > loss_disc_real_0: 0.1037171334028244 (0.11967695216563615) + | > loss_disc_real_1: 0.22457726299762726 (0.20369615947658365) + | > loss_disc_real_2: 0.21490977704524994 (0.21223386286786108) + | > loss_disc_real_3: 0.23860318958759308 (0.20567631315101276) + | > loss_disc_real_4: 0.22860229015350342 (0.2073649148823637) + | > loss_disc_real_5: 0.24188287556171417 (0.22085920585827393) + | > loss_0: 2.283020257949829 (2.3115259156082613) + | > grad_norm_0: tensor(27.2977, device='cuda:0') (tensor(29.9173, device='cuda:0')) + | > loss_gen: 2.7717316150665283 (2.609689221237645) + | > loss_kl: 1.7486802339553833 (1.7144351791251788) + | > loss_feat: 8.367098808288574 (8.212135358290238) + | > loss_mel: 16.839616775512695 (16.9696408329588) + | > loss_duration: 0.9024128913879395 (0.9502203103267786) + | > loss_1: 30.629539489746094 (30.456121011213824) + | > grad_norm_1: tensor(563.1000, device='cuda:0') (tensor(534.4404, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.8664 (0.9875802777030251) + | > loader_time: 0.0046 (0.006389628757130016) + + + --> TIME: 2024-07-13 21:48:06 -- STEP: 116/151 -- GLOBAL_STEP: 97650 + | > loss_disc: 2.519660472869873 (2.316255559181344) + | > loss_disc_real_0: 0.10639268159866333 (0.12116272012092943) + | > loss_disc_real_1: 0.2637213468551636 (0.20473669392281565) + | > loss_disc_real_2: 0.2496398687362671 (0.212659343049444) + | > loss_disc_real_3: 0.21738724410533905 (0.20680960421932154) + | > loss_disc_real_4: 0.19655071198940277 (0.20650806001805025) + | > loss_disc_real_5: 0.26350632309913635 (0.21983810604132456) + | > loss_0: 2.519660472869873 (2.316255559181344) + | > grad_norm_0: tensor(54.2767, device='cuda:0') (tensor(31.6744, device='cuda:0')) + | > loss_gen: 2.381422758102417 (2.599938378251832) + | > loss_kl: 1.9854965209960938 (1.7405968642440335) + | > loss_feat: 5.689911365509033 (8.138417223404193) + | > loss_mel: 14.022701263427734 (16.89233669741401) + | > loss_duration: 1.1354198455810547 (0.9444844619981174) + | > loss_1: 25.214950561523438 (30.31577365151767) + | > grad_norm_1: tensor(611.7684, device='cuda:0') (tensor(537.6716, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 2.0978 (1.0020242168985565) + | > loader_time: 0.0179 (0.0066038842858939335) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03382989076467661 (-0.0002608666053185091) + | > avg_loss_disc: 2.5773971997774563 (-0.058284319364107606) + | > avg_loss_disc_real_0: 0.2534716186615137 (+0.03294776036189151) + | > avg_loss_disc_real_1: 0.22584346395272475 (+0.021032607326140762) + | > avg_loss_disc_real_2: 0.23976405079548174 (+0.015850902749941886) + | > avg_loss_disc_real_3: 0.22188544846498048 (+0.004204021050379814) + | > avg_loss_disc_real_4: 0.25372544733377606 (+0.016067851048249482) + | > avg_loss_disc_real_5: 0.24740406756217664 (-0.028653077208078803) + | > avg_loss_0: 2.5773971997774563 (-0.058284319364107606) + | > avg_loss_gen: 2.4565915144406834 (+0.1531719794640174) + | > avg_loss_kl: 3.3066687950721154 (-0.26197758087745093) + | > avg_loss_feat: 7.664494294386643 (-0.286954366243803) + | > avg_loss_mel: 17.172799770648663 (-0.2171776111309356) + | > avg_loss_duration: 1.266134188725398 (-0.06325910641596866) + | > avg_loss_1: 31.86668850825383 (-0.6761957315298233) + + + > EPOCH: 84/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:48:48)  + + --> TIME: 2024-07-13 21:49:05 -- STEP: 15/151 -- GLOBAL_STEP: 97700 + | > loss_disc: 2.207719326019287 (2.3099159717559816) + | > loss_disc_real_0: 0.051906000822782516 (0.11660610909263293) + | > loss_disc_real_1: 0.17128843069076538 (0.20480211079120636) + | > loss_disc_real_2: 0.16393333673477173 (0.21370247801144918) + | > loss_disc_real_3: 0.13886544108390808 (0.20615572730700174) + | > loss_disc_real_4: 0.12756559252738953 (0.21593689620494844) + | > loss_disc_real_5: 0.2124902904033661 (0.2362707793712616) + | > loss_0: 2.207719326019287 (2.3099159717559816) + | > grad_norm_0: tensor(12.9485, device='cuda:0') (tensor(23.7020, device='cuda:0')) + | > loss_gen: 2.400693655014038 (2.6360246181488036) + | > loss_kl: 1.381030559539795 (1.7597954750061036) + | > loss_feat: 10.358519554138184 (8.566948541005452) + | > loss_mel: 17.3806209564209 (16.99619286855062) + | > loss_duration: 0.9294595718383789 (0.9362479408582052) + | > loss_1: 32.45032501220703 (30.895208994547527) + | > grad_norm_1: tensor(76.7605, device='cuda:0') (tensor(326.9802, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.8504 (1.0174136479695637) + | > loader_time: 0.0042 (0.006452163060506185) + + + --> TIME: 2024-07-13 21:49:53 -- STEP: 65/151 -- GLOBAL_STEP: 97750 + | > loss_disc: 2.511746883392334 (2.328695275233342) + | > loss_disc_real_0: 0.19345173239707947 (0.12032255879961527) + | > loss_disc_real_1: 0.19984987378120422 (0.20481205926491663) + | > loss_disc_real_2: 0.18139241635799408 (0.21551505281375005) + | > loss_disc_real_3: 0.25663045048713684 (0.2089712589979172) + | > loss_disc_real_4: 0.2387799471616745 (0.2097335206774565) + | > loss_disc_real_5: 0.2394789457321167 (0.22168303017432872) + | > loss_0: 2.511746883392334 (2.328695275233342) + | > grad_norm_0: tensor(48.4831, device='cuda:0') (tensor(28.1474, device='cuda:0')) + | > loss_gen: 2.5978569984436035 (2.589624470930833) + | > loss_kl: 1.774538516998291 (1.6850288858780493) + | > loss_feat: 7.508450508117676 (8.22361506682176) + | > loss_mel: 16.567150115966797 (16.79539475074182) + | > loss_duration: 0.8898147344589233 (0.9357907341076778) + | > loss_1: 29.337810516357422 (30.229453541682318) + | > grad_norm_1: tensor(632.3318, device='cuda:0') (tensor(469.6203, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.893 (0.9586755825923039) + | > loader_time: 0.0064 (0.0059784559103158805) + + + --> TIME: 2024-07-13 21:50:47 -- STEP: 115/151 -- GLOBAL_STEP: 97800 + | > loss_disc: 2.4062342643737793 (2.332315702023713) + | > loss_disc_real_0: 0.12304525077342987 (0.12389645242820616) + | > loss_disc_real_1: 0.24379955232143402 (0.20496760762256125) + | > loss_disc_real_2: 0.24008727073669434 (0.21428316403990205) + | > loss_disc_real_3: 0.22960464656352997 (0.21000228679698446) + | > loss_disc_real_4: 0.24531793594360352 (0.21020566037167673) + | > loss_disc_real_5: 0.2534712851047516 (0.21996316015720369) + | > loss_0: 2.4062342643737793 (2.332315702023713) + | > grad_norm_0: tensor(18.6279, device='cuda:0') (tensor(32.1151, device='cuda:0')) + | > loss_gen: 2.4106173515319824 (2.59000042003134) + | > loss_kl: 1.8590999841690063 (1.726979530894238) + | > loss_feat: 6.795355319976807 (8.105412935174028) + | > loss_mel: 17.363994598388672 (16.72016754979673) + | > loss_duration: 0.9351091384887695 (0.9508666940357374) + | > loss_1: 29.364177703857422 (30.093426911727242) + | > grad_norm_1: tensor(65.0078, device='cuda:0') (tensor(503.1095, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 1.0913 (1.0037578147390616) + | > loader_time: 0.008 (0.006479480992192807) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03237012716440054 (-0.0014597636002760678) + | > avg_loss_disc: 2.582280342395489 (+0.00488314261803291) + | > avg_loss_disc_real_0: 0.2062872166817005 (-0.04718440197981319) + | > avg_loss_disc_real_1: 0.2183073621529799 (-0.007536101799744843) + | > avg_loss_disc_real_2: 0.2153570021574314 (-0.02440704863805035) + | > avg_loss_disc_real_3: 0.24944405601574823 (+0.02755860755076775) + | > avg_loss_disc_real_4: 0.23915844697218674 (-0.01456700036158931) + | > avg_loss_disc_real_5: 0.2287186120565121 (-0.018685455505664555) + | > avg_loss_0: 2.582280342395489 (+0.00488314261803291) + | > avg_loss_gen: 2.292967814665575 (-0.16362369977510838) + | > avg_loss_kl: 3.438521458552434 (+0.13185266348031854) + | > avg_loss_feat: 7.4998219930208645 (-0.16467230136577893) + | > avg_loss_mel: 17.07903950030987 (-0.09376027033879097) + | > avg_loss_duration: 1.3270614330585186 (+0.06092724433312058) + | > avg_loss_1: 31.63741258474497 (-0.22927592350886172) + + + > EPOCH: 85/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:51:30)  + + --> TIME: 2024-07-13 21:51:45 -- STEP: 14/151 -- GLOBAL_STEP: 97850 + | > loss_disc: 2.247929573059082 (2.2604923929486955) + | > loss_disc_real_0: 0.10786128044128418 (0.11247139530522483) + | > loss_disc_real_1: 0.252648264169693 (0.19671102187463216) + | > loss_disc_real_2: 0.20134638249874115 (0.20347215341670172) + | > loss_disc_real_3: 0.19773302972316742 (0.205646778856005) + | > loss_disc_real_4: 0.21622949838638306 (0.20659738300102098) + | > loss_disc_real_5: 0.16477105021476746 (0.2119690967457635) + | > loss_0: 2.247929573059082 (2.2604923929486955) + | > grad_norm_0: tensor(39.1067, device='cuda:0') (tensor(26.9181, device='cuda:0')) + | > loss_gen: 2.732511043548584 (2.637593592916216) + | > loss_kl: 1.599137783050537 (1.7316971846989222) + | > loss_feat: 9.661805152893066 (8.46220966747829) + | > loss_mel: 17.406885147094727 (16.971064908163886) + | > loss_duration: 0.8333340883255005 (0.954071866614478) + | > loss_1: 32.233673095703125 (30.75663730076381) + | > grad_norm_1: tensor(587.3225, device='cuda:0') (tensor(444.2624, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.89 (0.9175858497619629) + | > loader_time: 0.0044 (0.005752546446663993) + + + --> TIME: 2024-07-13 21:52:36 -- STEP: 64/151 -- GLOBAL_STEP: 97900 + | > loss_disc: 2.2665019035339355 (2.297519639134407) + | > loss_disc_real_0: 0.15397845208644867 (0.1219220461207442) + | > loss_disc_real_1: 0.17300774157047272 (0.1992654229979962) + | > loss_disc_real_2: 0.21050454676151276 (0.20898177265189588) + | > loss_disc_real_3: 0.19798626005649567 (0.20595181756652892) + | > loss_disc_real_4: 0.20593099296092987 (0.2082278848392889) + | > loss_disc_real_5: 0.1933809369802475 (0.219519495498389) + | > loss_0: 2.2665019035339355 (2.297519639134407) + | > grad_norm_0: tensor(16.2003, device='cuda:0') (tensor(32.3666, device='cuda:0')) + | > loss_gen: 2.783190965652466 (2.6354485899209976) + | > loss_kl: 1.836446762084961 (1.7459468990564346) + | > loss_feat: 7.90302848815918 (8.32043855637312) + | > loss_mel: 15.024595260620117 (16.773346304893487) + | > loss_duration: 0.8792176246643066 (0.9559312956407666) + | > loss_1: 28.42647933959961 (30.43111163377762) + | > grad_norm_1: tensor(275.5566, device='cuda:0') (tensor(508.9453, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.892 (0.9820194765925407) + | > loader_time: 0.0065 (0.006299026310443878) + + + --> TIME: 2024-07-13 21:53:26 -- STEP: 114/151 -- GLOBAL_STEP: 97950 + | > loss_disc: 2.4585886001586914 (2.316601832707723) + | > loss_disc_real_0: 0.19464151561260223 (0.13029965588398149) + | > loss_disc_real_1: 0.21403688192367554 (0.20066412773571515) + | > loss_disc_real_2: 0.23966968059539795 (0.209570974111557) + | > loss_disc_real_3: 0.22676892578601837 (0.20727211027814632) + | > loss_disc_real_4: 0.20406441390514374 (0.2077697812857335) + | > loss_disc_real_5: 0.24867628514766693 (0.22087543823739939) + | > loss_0: 2.4585886001586914 (2.316601832707723) + | > grad_norm_0: tensor(13.2194, device='cuda:0') (tensor(32.8206, device='cuda:0')) + | > loss_gen: 2.4254965782165527 (2.6283249374021564) + | > loss_kl: 2.3542628288269043 (1.7410466989927122) + | > loss_feat: 7.445415019989014 (8.22619718836065) + | > loss_mel: 16.51789665222168 (16.848743229581594) + | > loss_duration: 0.8694013953208923 (0.9469597438971201) + | > loss_1: 29.612472534179688 (30.391271842153447) + | > grad_norm_1: tensor(210.1813, device='cuda:0') (tensor(461.5092, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 1.0012 (0.9857109680510404) + | > loader_time: 0.0057 (0.006280328098096345) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03173947334289551 (-0.0006306538215050325) + | > avg_loss_disc: 2.580695390701294 (-0.001584951694195258) + | > avg_loss_disc_real_0: 0.23647236365538377 (+0.03018514697368327) + | > avg_loss_disc_real_1: 0.21022889935053313 (-0.008078462802446784) + | > avg_loss_disc_real_2: 0.18618916204342476 (-0.029167840114006638) + | > avg_loss_disc_real_3: 0.18975084446943724 (-0.05969321154631099) + | > avg_loss_disc_real_4: 0.228727971131985 (-0.010430475840201736) + | > avg_loss_disc_real_5: 0.21813156169194442 (-0.010587050364567668) + | > avg_loss_0: 2.580695390701294 (-0.001584951694195258) + | > avg_loss_gen: 2.252808451652527 (-0.04015936301304812) + | > avg_loss_kl: 3.6856140356797438 (+0.2470925771273098) + | > avg_loss_feat: 8.071820956010084 (+0.5719989629892197) + | > avg_loss_mel: 16.79346040578989 (-0.2855790945199814) + | > avg_loss_duration: 1.3385958763269277 (+0.011534443268409023) + | > avg_loss_1: 32.14229935866136 (+0.5048867739163931) + + + > EPOCH: 86/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:54:11)  + + --> TIME: 2024-07-13 21:54:28 -- STEP: 13/151 -- GLOBAL_STEP: 98000 + | > loss_disc: 2.2758829593658447 (2.3854302626389723) + | > loss_disc_real_0: 0.16603979468345642 (0.1483634882248365) + | > loss_disc_real_1: 0.2016976922750473 (0.2069712051978478) + | > loss_disc_real_2: 0.21538777649402618 (0.21819881521738493) + | > loss_disc_real_3: 0.16357749700546265 (0.20355297051943266) + | > loss_disc_real_4: 0.18231603503227234 (0.20900209133441633) + | > loss_disc_real_5: 0.22411493957042694 (0.23204552439542916) + | > loss_0: 2.2758829593658447 (2.3854302626389723) + | > grad_norm_0: tensor(9.3776, device='cuda:0') (tensor(16.6637, device='cuda:0')) + | > loss_gen: 2.6754229068756104 (2.5613620281219482) + | > loss_kl: 1.6986169815063477 (1.7608120624835675) + | > loss_feat: 9.689281463623047 (8.09562741793119) + | > loss_mel: 17.79907989501953 (16.752437884990986) + | > loss_duration: 0.8594979047775269 (0.9874541621941787) + | > loss_1: 32.72189712524414 (30.157693716195915) + | > grad_norm_1: tensor(93.4054, device='cuda:0') (tensor(235.4012, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.8719 (1.1270252191103423) + | > loader_time: 0.0042 (0.00758066544165978) + + + --> TIME: 2024-07-13 21:55:41 -- STEP: 63/151 -- GLOBAL_STEP: 98050 + | > loss_disc: 2.4320359230041504 (2.3080386774880544) + | > loss_disc_real_0: 0.1428370177745819 (0.12654381960866945) + | > loss_disc_real_1: 0.20274442434310913 (0.20114090589303818) + | > loss_disc_real_2: 0.21896125376224518 (0.21128429614362262) + | > loss_disc_real_3: 0.2529931962490082 (0.20664388000492065) + | > loss_disc_real_4: 0.2585749626159668 (0.20982201054455743) + | > loss_disc_real_5: 0.21230077743530273 (0.21656783031565802) + | > loss_0: 2.4320359230041504 (2.3080386774880544) + | > grad_norm_0: tensor(30.7243, device='cuda:0') (tensor(17.2995, device='cuda:0')) + | > loss_gen: 2.6842241287231445 (2.600516796112061) + | > loss_kl: 1.7761057615280151 (1.815419492267427) + | > loss_feat: 7.881914138793945 (8.331558757358126) + | > loss_mel: 16.6804141998291 (16.81076307145376) + | > loss_duration: 0.9230744242668152 (0.9387436821347191) + | > loss_1: 29.94573402404785 (30.49700182960147) + | > grad_norm_1: tensor(601.8068, device='cuda:0') (tensor(270.8481, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.9094 (0.9765615917387462) + | > loader_time: 0.0062 (0.006092752729143415) + + + --> TIME: 2024-07-13 21:56:32 -- STEP: 113/151 -- GLOBAL_STEP: 98100 + | > loss_disc: 2.3861520290374756 (2.3017193743612916) + | > loss_disc_real_0: 0.11866607517004013 (0.12536347251945887) + | > loss_disc_real_1: 0.23708593845367432 (0.2020841002200557) + | > loss_disc_real_2: 0.22252191603183746 (0.2109698506846892) + | > loss_disc_real_3: 0.24779649078845978 (0.20647952811116665) + | > loss_disc_real_4: 0.26106327772140503 (0.20847644775578406) + | > loss_disc_real_5: 0.23830825090408325 (0.21826248233560966) + | > loss_0: 2.3861520290374756 (2.3017193743612916) + | > grad_norm_0: tensor(18.5300, device='cuda:0') (tensor(21.4547, device='cuda:0')) + | > loss_gen: 2.512526035308838 (2.61014652674177) + | > loss_kl: 1.8790242671966553 (1.8120377622874437) + | > loss_feat: 6.038663864135742 (8.305421854542415) + | > loss_mel: 15.42426586151123 (16.8242489215547) + | > loss_duration: 0.9673281311988831 (0.9378414027458799) + | > loss_1: 26.821807861328125 (30.489696485806355) + | > grad_norm_1: tensor(490.7614, device='cuda:0') (tensor(331.5587, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.9807 (0.9877366175693748) + | > loader_time: 0.0057 (0.006324394614295622) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03443705118619479 (+0.002697577843299284) + | > avg_loss_disc: 2.5197405998523417 (-0.06095479084895228) + | > avg_loss_disc_real_0: 0.2419741921699964 (+0.0055018285146126444) + | > avg_loss_disc_real_1: 0.2435181186749385 (+0.03328921932440537) + | > avg_loss_disc_real_2: 0.2886502536443563 (+0.10246109160093156) + | > avg_loss_disc_real_3: 0.2065089218891584 (+0.016758077419721157) + | > avg_loss_disc_real_4: 0.25472891330718994 (+0.026000942175204933) + | > avg_loss_disc_real_5: 0.2397765081662398 (+0.02164494647429538) + | > avg_loss_0: 2.5197405998523417 (-0.06095479084895228) + | > avg_loss_gen: 2.636163198030912 (+0.38335474637838507) + | > avg_loss_kl: 3.5350666229541483 (-0.15054741272559546) + | > avg_loss_feat: 7.775238623985877 (-0.29658233202420714) + | > avg_loss_mel: 16.93981067950909 (+0.14635027371920017) + | > avg_loss_duration: 1.3205480392162616 (-0.018047837110666043) + | > avg_loss_1: 32.20682745713454 (+0.06452809847317553) + + + > EPOCH: 87/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:57:19)  + + --> TIME: 2024-07-13 21:57:32 -- STEP: 12/151 -- GLOBAL_STEP: 98150 + | > loss_disc: 2.4371516704559326 (2.309111535549164) + | > loss_disc_real_0: 0.10943584144115448 (0.1222413145005703) + | > loss_disc_real_1: 0.2049284130334854 (0.20743614559372267) + | > loss_disc_real_2: 0.2541965842247009 (0.22028825928767523) + | > loss_disc_real_3: 0.2198241800069809 (0.21996797248721123) + | > loss_disc_real_4: 0.27062714099884033 (0.20014927660425505) + | > loss_disc_real_5: 0.22700190544128418 (0.2215863491098086) + | > loss_0: 2.4371516704559326 (2.309111535549164) + | > grad_norm_0: tensor(10.4970, device='cuda:0') (tensor(33.8514, device='cuda:0')) + | > loss_gen: 2.526820659637451 (2.635055502255758) + | > loss_kl: 2.1961143016815186 (1.7596217095851898) + | > loss_feat: 7.778011322021484 (8.188021659851074) + | > loss_mel: 17.202251434326172 (17.179765860239666) + | > loss_duration: 0.9071822762489319 (0.9354093621174494) + | > loss_1: 30.610380172729492 (30.69787375132243) + | > grad_norm_1: tensor(454.0939, device='cuda:0') (tensor(519.0498, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.9554 (0.9339989622433981) + | > loader_time: 0.0051 (0.006042202313741048) + + + --> TIME: 2024-07-13 21:58:20 -- STEP: 62/151 -- GLOBAL_STEP: 98200 + | > loss_disc: 2.2802810668945312 (2.2992514025780464) + | > loss_disc_real_0: 0.12229162454605103 (0.12120664041609532) + | > loss_disc_real_1: 0.19090186059474945 (0.2018307781507892) + | > loss_disc_real_2: 0.2256828248500824 (0.2099589027704731) + | > loss_disc_real_3: 0.240067720413208 (0.20899987268832423) + | > loss_disc_real_4: 0.24399824440479279 (0.20934746089962222) + | > loss_disc_real_5: 0.2650255560874939 (0.22078000657981442) + | > loss_0: 2.2802810668945312 (2.2992514025780464) + | > grad_norm_0: tensor(23.3930, device='cuda:0') (tensor(31.6347, device='cuda:0')) + | > loss_gen: 2.8938167095184326 (2.64722042699014) + | > loss_kl: 1.9890496730804443 (1.741524673277332) + | > loss_feat: 8.702098846435547 (8.451303643565025) + | > loss_mel: 18.053226470947266 (16.87378911049135) + | > loss_duration: 0.8624091148376465 (0.9370222543516467) + | > loss_1: 32.5005989074707 (30.65085998658211) + | > grad_norm_1: tensor(708.0560, device='cuda:0') (tensor(509.5520, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.901 (0.9368802847400788) + | > loader_time: 0.0054 (0.005969359028723932) + + + --> TIME: 2024-07-13 21:59:12 -- STEP: 112/151 -- GLOBAL_STEP: 98250 + | > loss_disc: 2.3744328022003174 (2.3058144748210894) + | > loss_disc_real_0: 0.12497492879629135 (0.12266465655661055) + | > loss_disc_real_1: 0.21266111731529236 (0.2027101985045842) + | > loss_disc_real_2: 0.27016016840934753 (0.2112927008420229) + | > loss_disc_real_3: 0.2683165669441223 (0.2065854048622506) + | > loss_disc_real_4: 0.23535408079624176 (0.2076338004719998) + | > loss_disc_real_5: 0.2750525176525116 (0.21908921735095127) + | > loss_0: 2.3744328022003174 (2.3058144748210894) + | > grad_norm_0: tensor(23.5877, device='cuda:0') (tensor(31.9737, device='cuda:0')) + | > loss_gen: 2.7651684284210205 (2.633429746542658) + | > loss_kl: 1.3878190517425537 (1.774046819124903) + | > loss_feat: 8.57271957397461 (8.289368872131615) + | > loss_mel: 16.622894287109375 (16.873949689524533) + | > loss_duration: 0.9579270482063293 (0.9434382234300885) + | > loss_1: 30.306529998779297 (30.514233299664088) + | > grad_norm_1: tensor(336.0343, device='cuda:0') (tensor(506.5892, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.854 (0.9809342665331704) + | > loader_time: 0.0044 (0.006306839840752738) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03395843505859376 (-0.00047861612760103506) + | > avg_loss_disc: 2.6543465761037974 (+0.13460597625145576) + | > avg_loss_disc_real_0: 0.4516538175252768 (+0.20967962535528037) + | > avg_loss_disc_real_1: 0.15369970351457596 (-0.08981841516036254) + | > avg_loss_disc_real_2: 0.2069956362247467 (-0.08165461741960961) + | > avg_loss_disc_real_3: 0.2112745688511775 (+0.0047656469620191155) + | > avg_loss_disc_real_4: 0.24306485973871672 (-0.01166405356847322) + | > avg_loss_disc_real_5: 0.22236321178766397 (-0.017413296378575832) + | > avg_loss_0: 2.6543465761037974 (+0.13460597625145576) + | > avg_loss_gen: 2.598023634690505 (-0.03813956334040691) + | > avg_loss_kl: 3.6721206261561465 (+0.13705400320199823) + | > avg_loss_feat: 7.8007378211388225 (+0.025499197152945463) + | > avg_loss_mel: 17.05592155456543 (+0.11611087505633932) + | > avg_loss_duration: 1.3064454702230601 (-0.014102568993201503) + | > avg_loss_1: 32.43324851989746 (+0.22642106276292395) + + + > EPOCH: 88/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 21:59:59)  + + --> TIME: 2024-07-13 22:00:13 -- STEP: 11/151 -- GLOBAL_STEP: 98300 + | > loss_disc: 2.020359516143799 (2.28240984136408) + | > loss_disc_real_0: 0.07598375529050827 (0.11807682500644164) + | > loss_disc_real_1: 0.21709416806697845 (0.20007393712347205) + | > loss_disc_real_2: 0.21351589262485504 (0.20817445624958386) + | > loss_disc_real_3: 0.1453915387392044 (0.20719725570895456) + | > loss_disc_real_4: 0.1588539183139801 (0.20401752130551773) + | > loss_disc_real_5: 0.15281330049037933 (0.22374329648234628) + | > loss_0: 2.020359516143799 (2.28240984136408) + | > grad_norm_0: tensor(20.1929, device='cuda:0') (tensor(23.2983, device='cuda:0')) + | > loss_gen: 2.850304126739502 (2.6117797981609) + | > loss_kl: 1.448257327079773 (1.8224241841923108) + | > loss_feat: 10.160362243652344 (8.362724911082875) + | > loss_mel: 16.184553146362305 (16.771396897055887) + | > loss_duration: 1.1274006366729736 (0.9780559648167003) + | > loss_1: 31.770877838134766 (30.546381170099433) + | > grad_norm_1: tensor(199.2849, device='cuda:0') (tensor(389.5464, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.8105 (1.055799050764604) + | > loader_time: 0.0032 (0.006867300380359997) + + + --> TIME: 2024-07-13 22:01:04 -- STEP: 61/151 -- GLOBAL_STEP: 98350 + | > loss_disc: 2.3518455028533936 (2.3299507547597416) + | > loss_disc_real_0: 0.12797893583774567 (0.1269460358336324) + | > loss_disc_real_1: 0.22602742910385132 (0.20434520625677266) + | > loss_disc_real_2: 0.21063213050365448 (0.2108891137310716) + | > loss_disc_real_3: 0.21042408049106598 (0.20671757957974418) + | > loss_disc_real_4: 0.22677360475063324 (0.20856703939985055) + | > loss_disc_real_5: 0.25795572996139526 (0.22346232880334385) + | > loss_0: 2.3518455028533936 (2.3299507547597416) + | > grad_norm_0: tensor(38.3930, device='cuda:0') (tensor(24.2768, device='cuda:0')) + | > loss_gen: 2.8607075214385986 (2.564903618859463) + | > loss_kl: 1.5371911525726318 (1.7720682000527617) + | > loss_feat: 8.26768970489502 (8.11359643936157) + | > loss_mel: 17.193456649780273 (16.762846196284062) + | > loss_duration: 0.9123560190200806 (0.9481936136230094) + | > loss_1: 30.771400451660156 (30.161607992453654) + | > grad_norm_1: tensor(600.3004, device='cuda:0') (tensor(361.0399, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.8861 (1.0064706997793227) + | > loader_time: 0.0051 (0.006437414982279793) + + + --> TIME: 2024-07-13 22:01:52 -- STEP: 111/151 -- GLOBAL_STEP: 98400 + | > loss_disc: 2.238987445831299 (2.332917462598095) + | > loss_disc_real_0: 0.11871437728404999 (0.12530415149422383) + | > loss_disc_real_1: 0.2003263682126999 (0.20401171024318213) + | > loss_disc_real_2: 0.1926531046628952 (0.2121391174224046) + | > loss_disc_real_3: 0.2083849459886551 (0.20605640972519781) + | > loss_disc_real_4: 0.17865435779094696 (0.20894314617187054) + | > loss_disc_real_5: 0.1827889084815979 (0.22302378512717583) + | > loss_0: 2.238987445831299 (2.332917462598095) + | > grad_norm_0: tensor(19.0414, device='cuda:0') (tensor(24.7116, device='cuda:0')) + | > loss_gen: 2.5488853454589844 (2.565949523771131) + | > loss_kl: 1.3082690238952637 (1.7596098901452244) + | > loss_feat: 7.6828155517578125 (8.176764900619917) + | > loss_mel: 15.191656112670898 (16.783275286356623) + | > loss_duration: 0.8766878843307495 (0.9392966364955043) + | > loss_1: 27.608314514160156 (30.224896284911008) + | > grad_norm_1: tensor(473.1666, device='cuda:0') (tensor(363.2070, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.8603 (0.9869687557220457) + | > loader_time: 0.0042 (0.006282252234381599) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033918178998506986 (-4.0256060086771295e-05) + | > avg_loss_disc: 2.5890272580660305 (-0.06531931803776692) + | > avg_loss_disc_real_0: 0.263320166331071 (-0.18833365119420575) + | > avg_loss_disc_real_1: 0.21542083070828363 (+0.061721127193707676) + | > avg_loss_disc_real_2: 0.24645636058770692 (+0.03946072436296022) + | > avg_loss_disc_real_3: 0.2508106506787814 (+0.039536081827603875) + | > avg_loss_disc_real_4: 0.2531910916933647 (+0.010126231954647957) + | > avg_loss_disc_real_5: 0.23155362560198858 (+0.009190413814324616) + | > avg_loss_0: 2.5890272580660305 (-0.06531931803776692) + | > avg_loss_gen: 2.5379769435295687 (-0.060046691160936305) + | > avg_loss_kl: 3.673938439442561 (+0.0018178132864146157) + | > avg_loss_feat: 7.810834077688364 (+0.010096256549541494) + | > avg_loss_mel: 16.84502520928016 (-0.21089634528527057) + | > avg_loss_duration: 1.3380692922152007 (+0.03162382199214053) + | > avg_loss_1: 32.20584355867826 (-0.22740496121920017) + + + > EPOCH: 89/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:02:42)  + + --> TIME: 2024-07-13 22:02:53 -- STEP: 10/151 -- GLOBAL_STEP: 98450 + | > loss_disc: 2.231278419494629 (2.215366315841675) + | > loss_disc_real_0: 0.11246781796216965 (0.10334292836487294) + | > loss_disc_real_1: 0.20580576360225677 (0.2043976753950119) + | > loss_disc_real_2: 0.17033109068870544 (0.19569979310035707) + | > loss_disc_real_3: 0.1888803392648697 (0.20360106676816941) + | > loss_disc_real_4: 0.2431688904762268 (0.19209390133619308) + | > loss_disc_real_5: 0.17449378967285156 (0.22330819219350814) + | > loss_0: 2.231278419494629 (2.215366315841675) + | > grad_norm_0: tensor(18.3535, device='cuda:0') (tensor(17.7527, device='cuda:0')) + | > loss_gen: 2.704237937927246 (2.6712908029556273) + | > loss_kl: 1.9506300687789917 (1.6175299167633057) + | > loss_feat: 9.440475463867188 (8.653404045104981) + | > loss_mel: 17.25210189819336 (16.517668342590333) + | > loss_duration: 0.8437357544898987 (0.9353172779083252) + | > loss_1: 32.19118118286133 (30.395210456848144) + | > grad_norm_1: tensor(182.6106, device='cuda:0') (tensor(373.3141, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.922 (0.8837427854537964) + | > loader_time: 0.0047 (0.005002212524414062) + + + --> TIME: 2024-07-13 22:03:42 -- STEP: 60/151 -- GLOBAL_STEP: 98500 + | > loss_disc: 2.26108455657959 (2.2937331954638154) + | > loss_disc_real_0: 0.10730421543121338 (0.11965140824516614) + | > loss_disc_real_1: 0.2295885980129242 (0.20233204551041126) + | > loss_disc_real_2: 0.26974543929100037 (0.20999110341072083) + | > loss_disc_real_3: 0.20047105848789215 (0.202594280987978) + | > loss_disc_real_4: 0.15955564379692078 (0.20605585599939027) + | > loss_disc_real_5: 0.1923128366470337 (0.21891142229239147) + | > loss_0: 2.26108455657959 (2.2937331954638154) + | > grad_norm_0: tensor(44.1023, device='cuda:0') (tensor(31.2747, device='cuda:0')) + | > loss_gen: 2.7072010040283203 (2.6289696772893274) + | > loss_kl: 2.075206756591797 (1.7477989474932352) + | > loss_feat: 8.532548904418945 (8.309872627258295) + | > loss_mel: 17.91427993774414 (16.776449108123778) + | > loss_duration: 0.9376245737075806 (0.9393343726793925) + | > loss_1: 32.16686248779297 (30.402424907684328) + | > grad_norm_1: tensor(479.5735, device='cuda:0') (tensor(471.8194, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 1.0753 (0.953506330649058) + | > loader_time: 0.0073 (0.006011374791463216) + + + --> TIME: 2024-07-13 22:04:33 -- STEP: 110/151 -- GLOBAL_STEP: 98550 + | > loss_disc: 2.308847427368164 (2.3188632509925156) + | > loss_disc_real_0: 0.09893539547920227 (0.12026804898950187) + | > loss_disc_real_1: 0.20412501692771912 (0.20428039004856893) + | > loss_disc_real_2: 0.24859729409217834 (0.21216978972608394) + | > loss_disc_real_3: 0.1565869003534317 (0.2055137955329635) + | > loss_disc_real_4: 0.25501272082328796 (0.2079906329512596) + | > loss_disc_real_5: 0.23614302277565002 (0.2207173454490575) + | > loss_0: 2.308847427368164 (2.3188632509925156) + | > grad_norm_0: tensor(14.1164, device='cuda:0') (tensor(28.7458, device='cuda:0')) + | > loss_gen: 2.6511218547821045 (2.5995411786166103) + | > loss_kl: 1.997241735458374 (1.7454298723827708) + | > loss_feat: 8.039874076843262 (8.147565033219076) + | > loss_mel: 17.38213539123535 (16.699575770985003) + | > loss_duration: 0.8963460326194763 (0.942147906260057) + | > loss_1: 30.966716766357422 (30.13425981348211) + | > grad_norm_1: tensor(342.0912, device='cuda:0') (tensor(414.8195, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.8978 (0.9750819878144698) + | > loader_time: 0.0064 (0.0064070701599121095) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03501002605144794 (+0.0010918470529409571) + | > avg_loss_disc: 2.6417083006638746 (+0.05268104259784412) + | > avg_loss_disc_real_0: 0.24588155173338377 (-0.017438614597687252) + | > avg_loss_disc_real_1: 0.19841583875509408 (-0.017004991953189552) + | > avg_loss_disc_real_2: 0.23224205581041482 (-0.014214304777292103) + | > avg_loss_disc_real_3: 0.23046258665048158 (-0.020348064028299806) + | > avg_loss_disc_real_4: 0.21826416483292213 (-0.03492692686044255) + | > avg_loss_disc_real_5: 0.2724144584857501 (+0.040860832883761494) + | > avg_loss_0: 2.6417083006638746 (+0.05268104259784412) + | > avg_loss_gen: 2.2730762041532078 (-0.26490073937636094) + | > avg_loss_kl: 3.4254015409029446 (-0.24853689853961658) + | > avg_loss_feat: 7.522677531609168 (-0.28815654607919594) + | > avg_loss_mel: 16.61993246812087 (-0.22509274115929045) + | > avg_loss_duration: 1.3096568401043231 (-0.028412452110877506) + | > avg_loss_1: 31.150744218092697 (-1.0550993405855635) + + + > EPOCH: 90/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:05:24)  + + --> TIME: 2024-07-13 22:05:36 -- STEP: 9/151 -- GLOBAL_STEP: 98600 + | > loss_disc: 2.416410446166992 (2.3533884949154324) + | > loss_disc_real_0: 0.10186980664730072 (0.13256588329871496) + | > loss_disc_real_1: 0.1624215990304947 (0.19654212892055511) + | > loss_disc_real_2: 0.1978043019771576 (0.22405016091134813) + | > loss_disc_real_3: 0.15434852242469788 (0.20878846446673074) + | > loss_disc_real_4: 0.229679673910141 (0.21662303639782798) + | > loss_disc_real_5: 0.23626463115215302 (0.2224565992752711) + | > loss_0: 2.416410446166992 (2.3533884949154324) + | > grad_norm_0: tensor(33.3357, device='cuda:0') (tensor(24.1146, device='cuda:0')) + | > loss_gen: 2.456984758377075 (2.556750906838311) + | > loss_kl: 2.0834591388702393 (1.7175570726394653) + | > loss_feat: 8.39727783203125 (8.084308942159018) + | > loss_mel: 17.729618072509766 (16.632997936672634) + | > loss_duration: 1.089390516281128 (1.0073668493164911) + | > loss_1: 31.756729125976562 (29.998981475830078) + | > grad_norm_1: tensor(462.9108, device='cuda:0') (tensor(461.9754, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 1.4538 (1.1091319455040827) + | > loader_time: 0.0112 (0.008638275994194878) + + + --> TIME: 2024-07-13 22:06:25 -- STEP: 59/151 -- GLOBAL_STEP: 98650 + | > loss_disc: 2.3154189586639404 (2.287969162908651) + | > loss_disc_real_0: 0.10973476618528366 (0.11726198932629521) + | > loss_disc_real_1: 0.21475011110305786 (0.1998006196345313) + | > loss_disc_real_2: 0.19600366055965424 (0.20864929738691299) + | > loss_disc_real_3: 0.17778290808200836 (0.20518637284383937) + | > loss_disc_real_4: 0.20643413066864014 (0.20739902682223563) + | > loss_disc_real_5: 0.23320691287517548 (0.21774322299633997) + | > loss_0: 2.3154189586639404 (2.287969162908651) + | > grad_norm_0: tensor(44.1438, device='cuda:0') (tensor(26.0349, device='cuda:0')) + | > loss_gen: 2.5964789390563965 (2.6349024368544756) + | > loss_kl: 1.118735671043396 (1.6593547039112801) + | > loss_feat: 7.408738136291504 (8.290670677766961) + | > loss_mel: 16.06035041809082 (16.7706851635949) + | > loss_duration: 0.9127731919288635 (0.9657551333055658) + | > loss_1: 28.097076416015625 (30.321367813368976) + | > grad_norm_1: tensor(502.3128, device='cuda:0') (tensor(460.2359, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.8548 (0.9970994965504792) + | > loader_time: 0.0047 (0.006644875316296594) + + + --> TIME: 2024-07-13 22:07:17 -- STEP: 109/151 -- GLOBAL_STEP: 98700 + | > loss_disc: 2.364696979522705 (2.3268642895812297) + | > loss_disc_real_0: 0.15304413437843323 (0.12820619450659923) + | > loss_disc_real_1: 0.20949339866638184 (0.20204509575979426) + | > loss_disc_real_2: 0.2489781379699707 (0.2117721922353867) + | > loss_disc_real_3: 0.20517101883888245 (0.208549230333862) + | > loss_disc_real_4: 0.214224711060524 (0.20943528208710732) + | > loss_disc_real_5: 0.2656237781047821 (0.21987002541165834) + | > loss_0: 2.364696979522705 (2.3268642895812297) + | > grad_norm_0: tensor(32.3529, device='cuda:0') (tensor(28.0055, device='cuda:0')) + | > loss_gen: 2.654134750366211 (2.603961872398307) + | > loss_kl: 1.6423321962356567 (1.709169918244038) + | > loss_feat: 7.077212810516357 (8.115936830503133) + | > loss_mel: 16.023176193237305 (16.7326726650973) + | > loss_duration: 0.9067269563674927 (0.9498563769760482) + | > loss_1: 28.3035831451416 (30.11159749862251) + | > grad_norm_1: tensor(540.5421, device='cuda:0') (tensor(451.8734, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.859 (1.0028563901918737) + | > loader_time: 0.0059 (0.006815455375461403) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033811110716599695 (-0.0011989153348482481) + | > avg_loss_disc: 2.652006057592539 (+0.010297756928664192) + | > avg_loss_disc_real_0: 0.25731455935881686 (+0.01143300762543309) + | > avg_loss_disc_real_1: 0.23772635826697716 (+0.03931051951188308) + | > avg_loss_disc_real_2: 0.24289290033853972 (+0.010650844528124898) + | > avg_loss_disc_real_3: 0.2536138089803549 (+0.023151222329873322) + | > avg_loss_disc_real_4: 0.25391357793257785 (+0.03564941309965572) + | > avg_loss_disc_real_5: 0.2383399846462103 (-0.034074473839539765) + | > avg_loss_0: 2.652006057592539 (+0.010297756928664192) + | > avg_loss_gen: 2.4768548928774328 (+0.203778688724225) + | > avg_loss_kl: 3.6596199732560377 (+0.23421843235309314) + | > avg_loss_feat: 7.656689937298115 (+0.13401240568894668) + | > avg_loss_mel: 17.09120853130634 (+0.47127606318547066) + | > avg_loss_duration: 1.2711376868761504 (-0.03851915322817279) + | > avg_loss_1: 32.15551082904522 (+1.0047666109525224) + + + > EPOCH: 91/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:08:07)  + + --> TIME: 2024-07-13 22:08:17 -- STEP: 8/151 -- GLOBAL_STEP: 98750 + | > loss_disc: 2.173612594604492 (2.3304774463176727) + | > loss_disc_real_0: 0.10398157685995102 (0.10521845333278179) + | > loss_disc_real_1: 0.1853611320257187 (0.21142436005175114) + | > loss_disc_real_2: 0.18151980638504028 (0.22166647017002106) + | > loss_disc_real_3: 0.24929165840148926 (0.23107475973665714) + | > loss_disc_real_4: 0.21849241852760315 (0.20884924195706844) + | > loss_disc_real_5: 0.1904079169034958 (0.2133941911160946) + | > loss_0: 2.173612594604492 (2.3304774463176727) + | > grad_norm_0: tensor(16.4915, device='cuda:0') (tensor(17.3839, device='cuda:0')) + | > loss_gen: 2.8098959922790527 (2.61776664853096) + | > loss_kl: 1.6897268295288086 (1.7628190368413925) + | > loss_feat: 9.098669052124023 (8.581208884716034) + | > loss_mel: 17.248170852661133 (16.701792359352112) + | > loss_duration: 0.8531795740127563 (0.9739148169755936) + | > loss_1: 31.699644088745117 (30.637501001358032) + | > grad_norm_1: tensor(346.1413, device='cuda:0') (tensor(333.8042, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.8819 (0.8980737626552582) + | > loader_time: 0.0054 (0.006161987781524658) + + + --> TIME: 2024-07-13 22:09:06 -- STEP: 58/151 -- GLOBAL_STEP: 98800 + | > loss_disc: 2.2012710571289062 (2.306163027368743) + | > loss_disc_real_0: 0.10700377076864243 (0.11673796587976916) + | > loss_disc_real_1: 0.14241071045398712 (0.20194616528420611) + | > loss_disc_real_2: 0.2030634731054306 (0.21533614448432264) + | > loss_disc_real_3: 0.17250390350818634 (0.21004172414541245) + | > loss_disc_real_4: 0.1551455855369568 (0.20582101262848954) + | > loss_disc_real_5: 0.1918376088142395 (0.22100651238499017) + | > loss_0: 2.2012710571289062 (2.306163027368743) + | > grad_norm_0: tensor(17.2205, device='cuda:0') (tensor(22.6106, device='cuda:0')) + | > loss_gen: 2.397977352142334 (2.6193547536586888) + | > loss_kl: 1.003957748413086 (1.7357601400079399) + | > loss_feat: 7.393486022949219 (8.49400306570119) + | > loss_mel: 14.072427749633789 (16.813675518693593) + | > loss_duration: 1.0480272769927979 (0.936640373591719) + | > loss_1: 25.915876388549805 (30.59943383315514) + | > grad_norm_1: tensor(178.3407, device='cuda:0') (tensor(398.5713, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.8176 (0.9579683953318102) + | > loader_time: 0.0056 (0.006297140285886568) + + + --> TIME: 2024-07-13 22:09:56 -- STEP: 108/151 -- GLOBAL_STEP: 98850 + | > loss_disc: 2.056741952896118 (2.3179863293965663) + | > loss_disc_real_0: 0.043872710317373276 (0.11934834742849623) + | > loss_disc_real_1: 0.16699841618537903 (0.20176253881719378) + | > loss_disc_real_2: 0.15897628664970398 (0.21416962933209208) + | > loss_disc_real_3: 0.1269223839044571 (0.2111497991890819) + | > loss_disc_real_4: 0.14346905052661896 (0.20739191535998275) + | > loss_disc_real_5: 0.18528346717357635 (0.21906124358927762) + | > loss_0: 2.056741952896118 (2.3179863293965663) + | > grad_norm_0: tensor(27.7129, device='cuda:0') (tensor(25.8974, device='cuda:0')) + | > loss_gen: 2.9923276901245117 (2.5977148634416087) + | > loss_kl: 1.97287118434906 (1.7769604568128232) + | > loss_feat: 10.466270446777344 (8.321165009781167) + | > loss_mel: 17.43787956237793 (16.781925139603793) + | > loss_duration: 1.109262228012085 (0.9390172494782342) + | > loss_1: 33.97861099243164 (30.41678276768437) + | > grad_norm_1: tensor(803.1949, device='cuda:0') (tensor(436.9654, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.8154 (0.9716880453957452) + | > loader_time: 0.0049 (0.006505387800711172) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03206537320063664 (-0.0017457375159630542) + | > avg_loss_disc: 2.619449707177969 (-0.032556350414569835) + | > avg_loss_disc_real_0: 0.15945762223922294 (-0.09785693711959392) + | > avg_loss_disc_real_1: 0.1839300118959867 (-0.053796346370990455) + | > avg_loss_disc_real_2: 0.21211284742905542 (-0.030780052909484296) + | > avg_loss_disc_real_3: 0.23143072655567756 (-0.02218308242467734) + | > avg_loss_disc_real_4: 0.24073193279596475 (-0.013181645136613102) + | > avg_loss_disc_real_5: 0.23119790393572587 (-0.007142080710484444) + | > avg_loss_0: 2.619449707177969 (-0.032556350414569835) + | > avg_loss_gen: 2.1050769090652466 (-0.3717779838121862) + | > avg_loss_kl: 3.6558089439685526 (-0.0038110292874851126) + | > avg_loss_feat: 7.556535684145414 (-0.10015425315270043) + | > avg_loss_mel: 16.73908651792086 (-0.352122013385479) + | > avg_loss_duration: 1.3405127433630137 (+0.06937505648686337) + | > avg_loss_1: 31.397020780123196 (-0.7584900489220239) + + + > EPOCH: 92/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:10:49)  + + --> TIME: 2024-07-13 22:10:59 -- STEP: 7/151 -- GLOBAL_STEP: 98900 + | > loss_disc: 2.132906675338745 (2.371034928730556) + | > loss_disc_real_0: 0.051577191799879074 (0.12662117768611228) + | > loss_disc_real_1: 0.13479574024677277 (0.2250342539378575) + | > loss_disc_real_2: 0.11493471264839172 (0.21306378713675908) + | > loss_disc_real_3: 0.1341492384672165 (0.20587026647159032) + | > loss_disc_real_4: 0.1811886578798294 (0.2003860878092902) + | > loss_disc_real_5: 0.15078173577785492 (0.22117251796381815) + | > loss_0: 2.132906675338745 (2.371034928730556) + | > grad_norm_0: tensor(26.6408, device='cuda:0') (tensor(29.4477, device='cuda:0')) + | > loss_gen: 2.7466862201690674 (2.5514851978846957) + | > loss_kl: 1.3403109312057495 (1.7813503912517004) + | > loss_feat: 9.77297306060791 (7.721022810254778) + | > loss_mel: 18.11380958557129 (16.932002476283483) + | > loss_duration: 0.8707235455513 (0.9420452288218907) + | > loss_1: 32.84450149536133 (29.927906581333705) + | > grad_norm_1: tensor(715.1296, device='cuda:0') (tensor(487.9086, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.8567 (1.1295556340898787) + | > loader_time: 0.0039 (0.008831875664847237) + + + --> TIME: 2024-07-13 22:11:50 -- STEP: 57/151 -- GLOBAL_STEP: 98950 + | > loss_disc: 2.2913644313812256 (2.3201183352554064) + | > loss_disc_real_0: 0.06651407480239868 (0.12123271920963334) + | > loss_disc_real_1: 0.1463925987482071 (0.20729352220108635) + | > loss_disc_real_2: 0.20623408257961273 (0.21278017027336255) + | > loss_disc_real_3: 0.1781008094549179 (0.20647110371736058) + | > loss_disc_real_4: 0.16289116442203522 (0.207914764421028) + | > loss_disc_real_5: 0.22163648903369904 (0.21972845599316715) + | > loss_0: 2.2913644313812256 (2.3201183352554064) + | > grad_norm_0: tensor(14.1498, device='cuda:0') (tensor(23.2504, device='cuda:0')) + | > loss_gen: 2.4859132766723633 (2.5867509089018164) + | > loss_kl: 1.8972563743591309 (1.7797630000532718) + | > loss_feat: 8.817834854125977 (8.144752468979155) + | > loss_mel: 16.188152313232422 (16.845672808195406) + | > loss_duration: 1.0382217168807983 (0.9390117766564352) + | > loss_1: 30.427377700805664 (30.295950939780788) + | > grad_norm_1: tensor(190.5352, device='cuda:0') (tensor(353.9256, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.8242 (1.023334130906222) + | > loader_time: 0.0042 (0.0065910397914418) + + + --> TIME: 2024-07-13 22:12:39 -- STEP: 107/151 -- GLOBAL_STEP: 99000 + | > loss_disc: 2.35166597366333 (2.3074526207469344) + | > loss_disc_real_0: 0.1425444781780243 (0.11970047687537201) + | > loss_disc_real_1: 0.1580386757850647 (0.20393075229965638) + | > loss_disc_real_2: 0.20473328232765198 (0.21086905980221582) + | > loss_disc_real_3: 0.18988358974456787 (0.20442617085771028) + | > loss_disc_real_4: 0.20222461223602295 (0.2082372107238413) + | > loss_disc_real_5: 0.23780493438243866 (0.22071826068040368) + | > loss_0: 2.35166597366333 (2.3074526207469344) + | > grad_norm_0: tensor(25.7133, device='cuda:0') (tensor(24.7554, device='cuda:0')) + | > loss_gen: 2.4864273071289062 (2.6090657488207936) + | > loss_kl: 1.5810253620147705 (1.747684062084305) + | > loss_feat: 7.431943416595459 (8.306183208929045) + | > loss_mel: 17.181310653686523 (16.924385195580587) + | > loss_duration: 0.9619885087013245 (0.93859678061209) + | > loss_1: 29.6426944732666 (30.52591509239696) + | > grad_norm_1: tensor(500.1200, device='cuda:0') (tensor(415.4227, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.8822 (0.9938894098050125) + | > loader_time: 0.0053 (0.006367358091835664) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.047554841408362754 (+0.015489468207726113) + | > avg_loss_disc: 2.6029611367445726 (-0.016488570433396355) + | > avg_loss_disc_real_0: 0.23392044237026802 (+0.07446282013104508) + | > avg_loss_disc_real_1: 0.20014166831970215 (+0.016211656423715443) + | > avg_loss_disc_real_2: 0.22732153764137855 (+0.015208690212323128) + | > avg_loss_disc_real_3: 0.24909726473001334 (+0.017666538174335777) + | > avg_loss_disc_real_4: 0.2529931137195 (+0.012261180923535259) + | > avg_loss_disc_real_5: 0.24984045670582697 (+0.0186425527701011) + | > avg_loss_0: 2.6029611367445726 (-0.016488570433396355) + | > avg_loss_gen: 2.4149147180410533 (+0.3098378089758067) + | > avg_loss_kl: 3.5730595405285177 (-0.0827494034400349) + | > avg_loss_feat: 8.09423542022705 (+0.5376997360816365) + | > avg_loss_mel: 17.05963552915133 (+0.32054901123046875) + | > avg_loss_duration: 1.2813793879288893 (-0.05913335543412446) + | > avg_loss_1: 32.42322452251728 (+1.0262037423940846) + + + > EPOCH: 93/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:14:00)  + + --> TIME: 2024-07-13 22:14:08 -- STEP: 6/151 -- GLOBAL_STEP: 99050 + | > loss_disc: 2.2691197395324707 (2.355372667312622) + | > loss_disc_real_0: 0.07332275807857513 (0.13671699663003287) + | > loss_disc_real_1: 0.1783222258090973 (0.21571558713912964) + | > loss_disc_real_2: 0.21749943494796753 (0.22869472205638885) + | > loss_disc_real_3: 0.19683416187763214 (0.21373001486063004) + | > loss_disc_real_4: 0.18804582953453064 (0.22027930120627084) + | > loss_disc_real_5: 0.1919865906238556 (0.21899189303318659) + | > loss_0: 2.2691197395324707 (2.355372667312622) + | > grad_norm_0: tensor(25.7881, device='cuda:0') (tensor(31.1639, device='cuda:0')) + | > loss_gen: 2.361470937728882 (2.6492799123128257) + | > loss_kl: 1.5872339010238647 (1.9140255252520244) + | > loss_feat: 8.795293807983398 (8.445679426193237) + | > loss_mel: 17.11393928527832 (17.58645502726237) + | > loss_duration: 0.7802425622940063 (0.8921960592269897) + | > loss_1: 30.638181686401367 (31.48763593037923) + | > grad_norm_1: tensor(449.5829, device='cuda:0') (tensor(563.4053, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.8583 (0.9469199180603027) + | > loader_time: 0.0042 (0.006665627161661784) + + + --> TIME: 2024-07-13 22:14:58 -- STEP: 56/151 -- GLOBAL_STEP: 99100 + | > loss_disc: 2.3084635734558105 (2.343433814389365) + | > loss_disc_real_0: 0.1115269884467125 (0.12763761476214444) + | > loss_disc_real_1: 0.2070244699716568 (0.20524074456521443) + | > loss_disc_real_2: 0.22957763075828552 (0.21604369833533252) + | > loss_disc_real_3: 0.22071048617362976 (0.2074218307222639) + | > loss_disc_real_4: 0.1802154928445816 (0.20881115033158235) + | > loss_disc_real_5: 0.16194230318069458 (0.22091079742780753) + | > loss_0: 2.3084635734558105 (2.343433814389365) + | > grad_norm_0: tensor(25.3863, device='cuda:0') (tensor(29.6532, device='cuda:0')) + | > loss_gen: 2.524426221847534 (2.5873356802122935) + | > loss_kl: 0.9527820348739624 (1.8089077706847871) + | > loss_feat: 8.25934886932373 (8.183372710432328) + | > loss_mel: 16.78719139099121 (16.7976906129292) + | > loss_duration: 1.0030627250671387 (0.9280693680047989) + | > loss_1: 29.526809692382812 (30.305376155035837) + | > grad_norm_1: tensor(510.0448, device='cuda:0') (tensor(467.3265, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.8359 (0.9921997700418744) + | > loader_time: 0.0048 (0.00626603194645473) + + + --> TIME: 2024-07-13 22:15:47 -- STEP: 106/151 -- GLOBAL_STEP: 99150 + | > loss_disc: 2.125188112258911 (2.3290244565819784) + | > loss_disc_real_0: 0.07475113868713379 (0.1226693709855372) + | > loss_disc_real_1: 0.21102459728717804 (0.20538991338239526) + | > loss_disc_real_2: 0.18243223428726196 (0.21357297243655854) + | > loss_disc_real_3: 0.239140123128891 (0.20708304207842304) + | > loss_disc_real_4: 0.20153425633907318 (0.20809266674068738) + | > loss_disc_real_5: 0.21770671010017395 (0.22315241494831053) + | > loss_0: 2.125188112258911 (2.3290244565819784) + | > grad_norm_0: tensor(7.2087, device='cuda:0') (tensor(28.8035, device='cuda:0')) + | > loss_gen: 2.741813898086548 (2.5969122558269873) + | > loss_kl: 1.6501117944717407 (1.7896505436807308) + | > loss_feat: 9.330368041992188 (8.190257207402647) + | > loss_mel: 18.068862915039062 (16.781064942197958) + | > loss_duration: 0.8734443783760071 (0.9389663535468983) + | > loss_1: 32.66460037231445 (30.296851248111366) + | > grad_norm_1: tensor(393.3448, device='cuda:0') (tensor(469.5429, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.8647 (0.9831704913445238) + | > loader_time: 0.0044 (0.006240201446245301) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03877173937284029 (-0.008783102035522461) + | > avg_loss_disc: 2.549272005374615 (-0.053689131369957455) + | > avg_loss_disc_real_0: 0.2520565058176334 (+0.018136063447365375) + | > avg_loss_disc_real_1: 0.23063829082709092 (+0.03049662250738877) + | > avg_loss_disc_real_2: 0.2160270271392969 (-0.011294510502081662) + | > avg_loss_disc_real_3: 0.20512334543925065 (-0.04397391929076269) + | > avg_loss_disc_real_4: 0.24790960894181177 (-0.005083504777688236) + | > avg_loss_disc_real_5: 0.20485848417648903 (-0.044981972529337944) + | > avg_loss_0: 2.549272005374615 (-0.053689131369957455) + | > avg_loss_gen: 2.4429582082308254 (+0.028043490189772147) + | > avg_loss_kl: 3.8278238589947042 (+0.2547643184661865) + | > avg_loss_feat: 8.061202599452091 (-0.033032820774959504) + | > avg_loss_mel: 17.207398341252254 (+0.14776281210092534) + | > avg_loss_duration: 1.3363546499839196 (+0.05497526205503034) + | > avg_loss_1: 32.875737997201774 (+0.4525134746844941) + + + > EPOCH: 94/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:16:42)  + + --> TIME: 2024-07-13 22:16:49 -- STEP: 5/151 -- GLOBAL_STEP: 99200 + | > loss_disc: 2.1950554847717285 (2.2050877094268797) + | > loss_disc_real_0: 0.11010044813156128 (0.10493173599243164) + | > loss_disc_real_1: 0.21648557484149933 (0.18758253157138824) + | > loss_disc_real_2: 0.2562861144542694 (0.19382581412792205) + | > loss_disc_real_3: 0.18023079633712769 (0.18348720967769622) + | > loss_disc_real_4: 0.20460844039916992 (0.1887097954750061) + | > loss_disc_real_5: 0.24341429769992828 (0.2011049598455429) + | > loss_0: 2.1950554847717285 (2.2050877094268797) + | > grad_norm_0: tensor(16.2112, device='cuda:0') (tensor(22.2410, device='cuda:0')) + | > loss_gen: 2.816093921661377 (2.705189609527588) + | > loss_kl: 1.7364786863327026 (1.7704758167266845) + | > loss_feat: 8.67136001586914 (8.800153827667236) + | > loss_mel: 17.123245239257812 (17.35065689086914) + | > loss_duration: 0.9431920647621155 (0.9212413311004639) + | > loss_1: 31.290369033813477 (31.547716903686524) + | > grad_norm_1: tensor(534.3207, device='cuda:0') (tensor(484.3907, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.929 (0.8909990310668945) + | > loader_time: 0.0053 (0.005249595642089844) + + + --> TIME: 2024-07-13 22:17:38 -- STEP: 55/151 -- GLOBAL_STEP: 99250 + | > loss_disc: 2.3004448413848877 (2.330524648319592) + | > loss_disc_real_0: 0.09706892818212509 (0.12993891862305731) + | > loss_disc_real_1: 0.2550072968006134 (0.20026294561949642) + | > loss_disc_real_2: 0.2609151303768158 (0.2081857678565112) + | > loss_disc_real_3: 0.197540283203125 (0.20462782762267373) + | > loss_disc_real_4: 0.2205207794904709 (0.20545173246752133) + | > loss_disc_real_5: 0.22056634724140167 (0.22743964222344484) + | > loss_0: 2.3004448413848877 (2.330524648319592) + | > grad_norm_0: tensor(17.2088, device='cuda:0') (tensor(23.6584, device='cuda:0')) + | > loss_gen: 2.6498851776123047 (2.580531111630526) + | > loss_kl: 1.4318442344665527 (1.8132568001747131) + | > loss_feat: 8.751933097839355 (8.177253359014335) + | > loss_mel: 17.932910919189453 (16.868793296813966) + | > loss_duration: 0.9124521613121033 (0.9355903701348738) + | > loss_1: 31.679025650024414 (30.375425095991655) + | > grad_norm_1: tensor(147.5877, device='cuda:0') (tensor(350.1756, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.884 (0.9598650845614347) + | > loader_time: 0.0049 (0.005995577031915838) + + + --> TIME: 2024-07-13 22:18:29 -- STEP: 105/151 -- GLOBAL_STEP: 99300 + | > loss_disc: 2.476317882537842 (2.317961338588171) + | > loss_disc_real_0: 0.09955000877380371 (0.127412183440867) + | > loss_disc_real_1: 0.22829900681972504 (0.20018767274561383) + | > loss_disc_real_2: 0.26902300119400024 (0.2100740461122422) + | > loss_disc_real_3: 0.2700994610786438 (0.20431341997214725) + | > loss_disc_real_4: 0.2937189042568207 (0.20685894198360896) + | > loss_disc_real_5: 0.2978304624557495 (0.2235911162126632) + | > loss_0: 2.476317882537842 (2.317961338588171) + | > grad_norm_0: tensor(14.3845, device='cuda:0') (tensor(27.2301, device='cuda:0')) + | > loss_gen: 2.727721691131592 (2.606585534413655) + | > loss_kl: 1.8864468336105347 (1.8061536715144202) + | > loss_feat: 7.106579780578613 (8.264876088641936) + | > loss_mel: 17.52322769165039 (16.8626284372239) + | > loss_duration: 1.0639383792877197 (0.9435742849395389) + | > loss_1: 30.30791473388672 (30.483818217686245) + | > grad_norm_1: tensor(553.3212, device='cuda:0') (tensor(426.0918, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 1.1935 (0.9846895172482445) + | > loader_time: 0.0085 (0.006397483462379093) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034082779517540567 (-0.004688959855299726) + | > avg_loss_disc: 2.629095957829402 (+0.0798239524547868) + | > avg_loss_disc_real_0: 0.21054010551709396 (-0.041516400300539436) + | > avg_loss_disc_real_1: 0.2578385678621439 (+0.027200277035052955) + | > avg_loss_disc_real_2: 0.25311101285310894 (+0.03708398571381205) + | > avg_loss_disc_real_3: 0.26612447889951557 (+0.06100113346026492) + | > avg_loss_disc_real_4: 0.2747860722816907 (+0.026876463339878948) + | > avg_loss_disc_real_5: 0.22328918026043817 (+0.01843069608394915) + | > avg_loss_0: 2.629095957829402 (+0.0798239524547868) + | > avg_loss_gen: 2.47447131230281 (+0.03151310407198471) + | > avg_loss_kl: 3.414480282710149 (-0.4133435762845554) + | > avg_loss_feat: 7.915056558755728 (-0.1461460406963635) + | > avg_loss_mel: 17.24591533954327 (+0.038516998291015625) + | > avg_loss_duration: 1.274108620790335 (-0.062246029193584684) + | > avg_loss_1: 32.32403197655311 (-0.5517060206486661) + + + > EPOCH: 95/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:19:25)  + + --> TIME: 2024-07-13 22:19:33 -- STEP: 4/151 -- GLOBAL_STEP: 99350 + | > loss_disc: 2.32650089263916 (2.297286331653595) + | > loss_disc_real_0: 0.2056490033864975 (0.12498107179999352) + | > loss_disc_real_1: 0.20014218986034393 (0.21136988326907158) + | > loss_disc_real_2: 0.19231025874614716 (0.21596822515130043) + | > loss_disc_real_3: 0.16458511352539062 (0.2032022587954998) + | > loss_disc_real_4: 0.19507969915866852 (0.19832409173250198) + | > loss_disc_real_5: 0.24060818552970886 (0.22018899396061897) + | > loss_0: 2.32650089263916 (2.297286331653595) + | > grad_norm_0: tensor(39.1704, device='cuda:0') (tensor(39.8568, device='cuda:0')) + | > loss_gen: 2.539672374725342 (2.6667312383651733) + | > loss_kl: 2.289524793624878 (1.9845047295093536) + | > loss_feat: 7.760545253753662 (8.991927742958069) + | > loss_mel: 17.673583984375 (17.970237255096436) + | > loss_duration: 1.1880836486816406 (0.9764586091041565) + | > loss_1: 31.4514102935791 (32.58986139297485) + | > grad_norm_1: tensor(490.7556, device='cuda:0') (tensor(619.9021, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 2.0876 (1.200623869895935) + | > loader_time: 0.0215 (0.009025096893310547) + + + --> TIME: 2024-07-13 22:20:21 -- STEP: 54/151 -- GLOBAL_STEP: 99400 + | > loss_disc: 2.6384007930755615 (2.3189292704617537) + | > loss_disc_real_0: 0.252045601606369 (0.11946382532241168) + | > loss_disc_real_1: 0.25227823853492737 (0.20776209635315118) + | > loss_disc_real_2: 0.26082316040992737 (0.2138488463781498) + | > loss_disc_real_3: 0.23564882576465607 (0.2079576206979928) + | > loss_disc_real_4: 0.22083733975887299 (0.20463676336738798) + | > loss_disc_real_5: 0.26433324813842773 (0.22346442661903523) + | > loss_0: 2.6384007930755615 (2.3189292704617537) + | > grad_norm_0: tensor(54.1838, device='cuda:0') (tensor(33.6335, device='cuda:0')) + | > loss_gen: 2.1597073078155518 (2.634382499588859) + | > loss_kl: 2.229975938796997 (1.7430543115845434) + | > loss_feat: 4.943878173828125 (8.44523300064934) + | > loss_mel: 12.40713882446289 (17.02387657871953) + | > loss_duration: 1.2001428604125977 (0.939825357110412) + | > loss_1: 22.940845489501953 (30.786372184753418) + | > grad_norm_1: tensor(634.2264, device='cuda:0') (tensor(580.6224, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 2.8561 (0.964438345697191) + | > loader_time: 0.0195 (0.005921699382640698) + + + --> TIME: 2024-07-13 22:21:11 -- STEP: 104/151 -- GLOBAL_STEP: 99450 + | > loss_disc: 2.28696608543396 (2.3266054873283104) + | > loss_disc_real_0: 0.13656090199947357 (0.12011131769619308) + | > loss_disc_real_1: 0.2315371185541153 (0.20599895436316729) + | > loss_disc_real_2: 0.26448073983192444 (0.21315773318593317) + | > loss_disc_real_3: 0.23806029558181763 (0.2077524193490927) + | > loss_disc_real_4: 0.30623868107795715 (0.20867892268758553) + | > loss_disc_real_5: 0.1554088592529297 (0.22126045335943884) + | > loss_0: 2.28696608543396 (2.3266054873283104) + | > grad_norm_0: tensor(47.2668, device='cuda:0') (tensor(35.3081, device='cuda:0')) + | > loss_gen: 2.801293134689331 (2.613864100896394) + | > loss_kl: 2.191042900085449 (1.7741773564081926) + | > loss_feat: 8.574135780334473 (8.335587116388172) + | > loss_mel: 16.810476303100586 (16.951292771559476) + | > loss_duration: 0.8728393912315369 (0.9427013832789201) + | > loss_1: 31.249786376953125 (30.617622962364784) + | > grad_norm_1: tensor(596.2190, device='cuda:0') (tensor(578.1416, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.9952 (0.972389973126925) + | > loader_time: 0.0076 (0.006121220496984629) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03409677285414476 (+1.3993336604192774e-05) + | > avg_loss_disc: 2.5296429303976207 (-0.0994530274317813) + | > avg_loss_disc_real_0: 0.2893876742858153 (+0.07884756876872134) + | > avg_loss_disc_real_1: 0.26094668530500853 (+0.003108117442864655) + | > avg_loss_disc_real_2: 0.2668522412960346 (+0.013741228442925635) + | > avg_loss_disc_real_3: 0.24208343831392434 (-0.024041040585591228) + | > avg_loss_disc_real_4: 0.24912097591620225 (-0.02566509636548847) + | > avg_loss_disc_real_5: 0.21793794746582323 (-0.005351232794614941) + | > avg_loss_0: 2.5296429303976207 (-0.0994530274317813) + | > avg_loss_gen: 2.726602389262273 (+0.2521310769594627) + | > avg_loss_kl: 3.701432943344116 (+0.2869526606339674) + | > avg_loss_feat: 7.945657693422758 (+0.030601134667030294) + | > avg_loss_mel: 16.994470889751724 (-0.25144444979154557) + | > avg_loss_duration: 1.3366809899990375 (+0.06257236920870257) + | > avg_loss_1: 32.70484513502855 (+0.3808131584754406) + + + > EPOCH: 96/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:22:06)  + + --> TIME: 2024-07-13 22:22:12 -- STEP: 3/151 -- GLOBAL_STEP: 99500 + | > loss_disc: 2.5252437591552734 (2.362300078074137) + | > loss_disc_real_0: 0.15526703000068665 (0.11752182493607204) + | > loss_disc_real_1: 0.1980416476726532 (0.2180913339058558) + | > loss_disc_real_2: 0.2150375247001648 (0.23217859864234924) + | > loss_disc_real_3: 0.23213845491409302 (0.19487635791301727) + | > loss_disc_real_4: 0.26352041959762573 (0.1949722319841385) + | > loss_disc_real_5: 0.24024324119091034 (0.19783840080102286) + | > loss_0: 2.5252437591552734 (2.362300078074137) + | > grad_norm_0: tensor(40.8497, device='cuda:0') (tensor(31.2094, device='cuda:0')) + | > loss_gen: 2.321678638458252 (2.4569523334503174) + | > loss_kl: 2.049376964569092 (2.00694731871287) + | > loss_feat: 6.077535152435303 (7.722841103871663) + | > loss_mel: 15.434249877929688 (16.65098508199056) + | > loss_duration: 0.9357485175132751 (0.9481823841730753) + | > loss_1: 26.818588256835938 (29.78590710957845) + | > grad_norm_1: tensor(473.5757, device='cuda:0') (tensor(504.7001, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 1.0658 (1.0230681896209717) + | > loader_time: 0.0079 (0.0066374937693278) + + + --> TIME: 2024-07-13 22:23:03 -- STEP: 53/151 -- GLOBAL_STEP: 99550 + | > loss_disc: 2.179093599319458 (2.3044727793279685) + | > loss_disc_real_0: 0.16818580031394958 (0.12341743271868184) + | > loss_disc_real_1: 0.2132875621318817 (0.20226207066257046) + | > loss_disc_real_2: 0.2102648764848709 (0.20867831780100768) + | > loss_disc_real_3: 0.20424655079841614 (0.20790597290362953) + | > loss_disc_real_4: 0.27261093258857727 (0.2040610709842646) + | > loss_disc_real_5: 0.1989016830921173 (0.21876949085941855) + | > loss_0: 2.179093599319458 (2.3044727793279685) + | > grad_norm_0: tensor(34.2048, device='cuda:0') (tensor(36.8753, device='cuda:0')) + | > loss_gen: 3.069669008255005 (2.631600438423876) + | > loss_kl: 1.9001230001449585 (1.8013549885659847) + | > loss_feat: 9.128328323364258 (8.31672655861333) + | > loss_mel: 17.541242599487305 (16.90482753177859) + | > loss_duration: 0.8774138689041138 (0.9497319426176682) + | > loss_1: 32.51677703857422 (30.60424153309948) + | > grad_norm_1: tensor(606.9886, device='cuda:0') (tensor(542.8964, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.8807 (1.0123928223016125) + | > loader_time: 0.0064 (0.006434224686532651) + + + --> TIME: 2024-07-13 22:23:51 -- STEP: 103/151 -- GLOBAL_STEP: 99600 + | > loss_disc: 2.396857976913452 (2.2934753524446947) + | > loss_disc_real_0: 0.11161693185567856 (0.12274609643568113) + | > loss_disc_real_1: 0.1807904839515686 (0.20039258405421545) + | > loss_disc_real_2: 0.19957919418811798 (0.20853865855527157) + | > loss_disc_real_3: 0.22046653926372528 (0.2076964619958285) + | > loss_disc_real_4: 0.22156314551830292 (0.20522268839831492) + | > loss_disc_real_5: 0.26485520601272583 (0.21975711790971386) + | > loss_0: 2.396857976913452 (2.2934753524446947) + | > grad_norm_0: tensor(11.2886, device='cuda:0') (tensor(32.1595, device='cuda:0')) + | > loss_gen: 2.2372591495513916 (2.633228299687211) + | > loss_kl: 1.5578787326812744 (1.7640228884891398) + | > loss_feat: 7.86918830871582 (8.394317645471071) + | > loss_mel: 15.175806999206543 (16.89264034530493) + | > loss_duration: 0.8657954931259155 (0.9402385967449077) + | > loss_1: 27.705928802490234 (30.624447739240036) + | > grad_norm_1: tensor(409.9521, device='cuda:0') (tensor(508.1618, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.8702 (0.9853772283757772) + | > loader_time: 0.0059 (0.006271522021988063) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03322058457594652 (-0.0008761882781982422) + | > avg_loss_disc: 2.5408729773301344 (+0.011230046932513726) + | > avg_loss_disc_real_0: 0.19379866352448097 (-0.09558901076133433) + | > avg_loss_disc_real_1: 0.1789410446698849 (-0.08200564063512364) + | > avg_loss_disc_real_2: 0.21656409593728873 (-0.050288145358745845) + | > avg_loss_disc_real_3: 0.22738178762105796 (-0.014701650692866386) + | > avg_loss_disc_real_4: 0.23779261570710403 (-0.01132836020909822) + | > avg_loss_disc_real_5: 0.26074334635184365 (+0.04280539888602042) + | > avg_loss_0: 2.5408729773301344 (+0.011230046932513726) + | > avg_loss_gen: 2.3594090388371396 (-0.36719335042513324) + | > avg_loss_kl: 3.790102243423462 (+0.0886693000793457) + | > avg_loss_feat: 7.802896132835975 (-0.14276156058678335) + | > avg_loss_mel: 17.277064836942234 (+0.28259394719050945) + | > avg_loss_duration: 1.3207584436123188 (-0.01592254638671875) + | > avg_loss_1: 32.55023017296424 (-0.1546149620643078) + + + > EPOCH: 97/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:24:48)  + + --> TIME: 2024-07-13 22:24:52 -- STEP: 2/151 -- GLOBAL_STEP: 99650 + | > loss_disc: 2.299185276031494 (2.2793092727661133) + | > loss_disc_real_0: 0.16267456114292145 (0.11059778742492199) + | > loss_disc_real_1: 0.19901826977729797 (0.17714812606573105) + | > loss_disc_real_2: 0.14262284338474274 (0.1584167256951332) + | > loss_disc_real_3: 0.18115749955177307 (0.17665956914424896) + | > loss_disc_real_4: 0.20257359743118286 (0.187040276825428) + | > loss_disc_real_5: 0.23839516937732697 (0.24351635575294495) + | > loss_0: 2.299185276031494 (2.2793092727661133) + | > grad_norm_0: tensor(41.6222, device='cuda:0') (tensor(37.0071, device='cuda:0')) + | > loss_gen: 2.886317253112793 (2.731426477432251) + | > loss_kl: 2.1280648708343506 (2.12895405292511) + | > loss_feat: 9.724761009216309 (8.631273746490479) + | > loss_mel: 17.964387893676758 (17.08915424346924) + | > loss_duration: 0.861720085144043 (0.8527727723121643) + | > loss_1: 33.56525421142578 (31.43358325958252) + | > grad_norm_1: tensor(749.1521, device='cuda:0') (tensor(509.2167, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.9579 (0.9341739416122437) + | > loader_time: 0.0114 (0.007647395133972168) + + + --> TIME: 2024-07-13 22:25:41 -- STEP: 52/151 -- GLOBAL_STEP: 99700 + | > loss_disc: 2.3923490047454834 (2.3166796473356395) + | > loss_disc_real_0: 0.08297732472419739 (0.12101787930497757) + | > loss_disc_real_1: 0.18620479106903076 (0.20539397803636697) + | > loss_disc_real_2: 0.16975054144859314 (0.20771452612601793) + | > loss_disc_real_3: 0.20027603209018707 (0.20584210868065173) + | > loss_disc_real_4: 0.18640781939029694 (0.20680389897181436) + | > loss_disc_real_5: 0.1965770274400711 (0.22106413084727067) + | > loss_0: 2.3923490047454834 (2.3166796473356395) + | > grad_norm_0: tensor(32.6811, device='cuda:0') (tensor(39.2492, device='cuda:0')) + | > loss_gen: 2.3866186141967773 (2.6426366475912246) + | > loss_kl: 1.8273887634277344 (1.7365832924842834) + | > loss_feat: 8.456110954284668 (8.350743678899908) + | > loss_mel: 16.504669189453125 (17.04918931080745) + | > loss_duration: 0.9233152866363525 (0.9289145882313068) + | > loss_1: 30.09810447692871 (30.708067930661716) + | > grad_norm_1: tensor(454.2194, device='cuda:0') (tensor(600.5000, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.9126 (0.9600696609570429) + | > loader_time: 0.0069 (0.005990991225609413) + + + --> TIME: 2024-07-13 22:26:31 -- STEP: 102/151 -- GLOBAL_STEP: 99750 + | > loss_disc: 2.35268235206604 (2.3051998498393047) + | > loss_disc_real_0: 0.22084824740886688 (0.11976346183641284) + | > loss_disc_real_1: 0.20970457792282104 (0.20397839622170316) + | > loss_disc_real_2: 0.2338913530111313 (0.21012301874511383) + | > loss_disc_real_3: 0.19653216004371643 (0.20385541635401108) + | > loss_disc_real_4: 0.20295901596546173 (0.20395070299798368) + | > loss_disc_real_5: 0.20955874025821686 (0.21969620781201943) + | > loss_0: 2.35268235206604 (2.3051998498393047) + | > grad_norm_0: tensor(31.8838, device='cuda:0') (tensor(33.5807, device='cuda:0')) + | > loss_gen: 2.5816123485565186 (2.635555377193525) + | > loss_kl: 2.242065191268921 (1.7487444375075547) + | > loss_feat: 7.76032018661499 (8.402001993328915) + | > loss_mel: 16.842191696166992 (17.082704226175927) + | > loss_duration: 0.9199440479278564 (0.9391398342216716) + | > loss_1: 30.346134185791016 (30.808146065356684) + | > grad_norm_1: tensor(376.3845, device='cuda:0') (tensor(548.9092, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.9986 (0.975431430573557) + | > loader_time: 0.0061 (0.006352291387670181) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032965623415433444 (-0.00025496116051307305) + | > avg_loss_disc: 2.5727859093592715 (+0.03191293202913714) + | > avg_loss_disc_real_0: 0.2171763044137221 (+0.02337764088924113) + | > avg_loss_disc_real_1: 0.18657084955618933 (+0.0076298048863044365) + | > avg_loss_disc_real_2: 0.21456374457249275 (-0.0020003513647959825) + | > avg_loss_disc_real_3: 0.22773801134182856 (+0.0003562237207705987) + | > avg_loss_disc_real_4: 0.2652633086993144 (+0.027470692992210388) + | > avg_loss_disc_real_5: 0.24257272481918335 (-0.018170621532660303) + | > avg_loss_0: 2.5727859093592715 (+0.03191293202913714) + | > avg_loss_gen: 2.339766263961792 (-0.0196427748753476) + | > avg_loss_kl: 3.730990318151621 (-0.05911192527184106) + | > avg_loss_feat: 7.686353023235615 (-0.11654310960035996) + | > avg_loss_mel: 16.8714599609375 (-0.40560487600473394) + | > avg_loss_duration: 1.2892007185862615 (-0.03155772502605725) + | > avg_loss_1: 31.917769945584812 (-0.6324602273794291) + + + > EPOCH: 98/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:27:29)  + + --> TIME: 2024-07-13 22:27:33 -- STEP: 1/151 -- GLOBAL_STEP: 99800 + | > loss_disc: 2.2615272998809814 (2.2615272998809814) + | > loss_disc_real_0: 0.23628592491149902 (0.23628592491149902) + | > loss_disc_real_1: 0.18825018405914307 (0.18825018405914307) + | > loss_disc_real_2: 0.1877928376197815 (0.1877928376197815) + | > loss_disc_real_3: 0.18726328015327454 (0.18726328015327454) + | > loss_disc_real_4: 0.14726202189922333 (0.14726202189922333) + | > loss_disc_real_5: 0.18043754994869232 (0.18043754994869232) + | > loss_0: 2.2615272998809814 (2.2615272998809814) + | > grad_norm_0: tensor(60.2528, device='cuda:0') (tensor(60.2528, device='cuda:0')) + | > loss_gen: 2.69472599029541 (2.69472599029541) + | > loss_kl: 2.097992420196533 (2.097992420196533) + | > loss_feat: 8.818373680114746 (8.818373680114746) + | > loss_mel: 18.6674861907959 (18.6674861907959) + | > loss_duration: 0.8292552828788757 (0.8292552828788757) + | > loss_1: 33.10783386230469 (33.10783386230469) + | > grad_norm_1: tensor(741.9359, device='cuda:0') (tensor(741.9359, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.9193 (0.9193089008331299) + | > loader_time: 0.008 (0.00801992416381836) + + + --> TIME: 2024-07-13 22:28:25 -- STEP: 51/151 -- GLOBAL_STEP: 99850 + | > loss_disc: 2.6411640644073486 (2.3500135786393117) + | > loss_disc_real_0: 0.20008337497711182 (0.13121220622869098) + | > loss_disc_real_1: 0.23699621856212616 (0.20932854127650166) + | > loss_disc_real_2: 0.2999444603919983 (0.21795493013718548) + | > loss_disc_real_3: 0.24427218735218048 (0.2038972436213026) + | > loss_disc_real_4: 0.24087035655975342 (0.21198614116977244) + | > loss_disc_real_5: 0.2320041060447693 (0.22063907630303325) + | > loss_0: 2.6411640644073486 (2.3500135786393117) + | > grad_norm_0: tensor(48.5708, device='cuda:0') (tensor(43.5346, device='cuda:0')) + | > loss_gen: 2.1621148586273193 (2.6243218113394344) + | > loss_kl: 1.9040687084197998 (1.7374012879296845) + | > loss_feat: 4.913634777069092 (8.125887403301167) + | > loss_mel: 14.102108001708984 (17.041524849685967) + | > loss_duration: 1.1923503875732422 (0.9411756630037346) + | > loss_1: 24.274276733398438 (30.47031077216653) + | > grad_norm_1: tensor(368.4617, device='cuda:0') (tensor(614.2548, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 2.1264 (1.0410231608970493) + | > loader_time: 0.0151 (0.006356201919854856) + + + --> TIME: 2024-07-13 22:29:13 -- STEP: 101/151 -- GLOBAL_STEP: 99900 + | > loss_disc: 2.1671299934387207 (2.3266071565080413) + | > loss_disc_real_0: 0.08016148209571838 (0.12257802925340017) + | > loss_disc_real_1: 0.18106545507907867 (0.20482144851495723) + | > loss_disc_real_2: 0.16547322273254395 (0.21530612742546762) + | > loss_disc_real_3: 0.14865067601203918 (0.20544394577788835) + | > loss_disc_real_4: 0.15951566398143768 (0.20811123246013527) + | > loss_disc_real_5: 0.20463769137859344 (0.22271577881114318) + | > loss_0: 2.1671299934387207 (2.3266071565080413) + | > grad_norm_0: tensor(15.1523, device='cuda:0') (tensor(35.7808, device='cuda:0')) + | > loss_gen: 2.3825128078460693 (2.6057765318615598) + | > loss_kl: 1.6906646490097046 (1.6666298331600604) + | > loss_feat: 8.21213150024414 (8.141561937804271) + | > loss_mel: 17.315092086791992 (16.905296958319035) + | > loss_duration: 0.8593337535858154 (0.9435101871443267) + | > loss_1: 30.459733963012695 (30.262775364488657) + | > grad_norm_1: tensor(99.7302, device='cuda:0') (tensor(536.9762, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.8916 (0.9902608087747407) + | > loader_time: 0.0051 (0.006117955292805587) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031951299080481894 (-0.0010143243349515496) + | > avg_loss_disc: 2.6218406787285438 (+0.04905476936927222) + | > avg_loss_disc_real_0: 0.2601866962817999 (+0.04301039186807781) + | > avg_loss_disc_real_1: 0.23665606746306786 (+0.05008521790687853) + | > avg_loss_disc_real_2: 0.2726752689251533 (+0.058111524352660565) + | > avg_loss_disc_real_3: 0.2195599629328801 (-0.008178048408948452) + | > avg_loss_disc_real_4: 0.27991752097239864 (+0.014654212273084222) + | > avg_loss_disc_real_5: 0.21156602066296798 (-0.03100670415621537) + | > avg_loss_0: 2.6218406787285438 (+0.04905476936927222) + | > avg_loss_gen: 2.5361943611731896 (+0.19642809721139765) + | > avg_loss_kl: 3.5697597723740797 (-0.16123054577754115) + | > avg_loss_feat: 7.850638022789588 (+0.16428499955397324) + | > avg_loss_mel: 16.752190369826096 (-0.11926959111140434) + | > avg_loss_duration: 1.3115479762737567 (+0.022347257687495237) + | > avg_loss_1: 32.020330282358024 (+0.10256033677321241) + + + > EPOCH: 99/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:30:11)  + + --> TIME: 2024-07-13 22:30:14 -- STEP: 0/151 -- GLOBAL_STEP: 99950 + | > loss_disc: 2.1880393028259277 (2.1880393028259277) + | > loss_disc_real_0: 0.10863156616687775 (0.10863156616687775) + | > loss_disc_real_1: 0.20593230426311493 (0.20593230426311493) + | > loss_disc_real_2: 0.21760840713977814 (0.21760840713977814) + | > loss_disc_real_3: 0.16384226083755493 (0.16384226083755493) + | > loss_disc_real_4: 0.20921112596988678 (0.20921112596988678) + | > loss_disc_real_5: 0.1901380717754364 (0.1901380717754364) + | > loss_0: 2.1880393028259277 (2.1880393028259277) + | > grad_norm_0: tensor(10.3045, device='cuda:0') (tensor(10.3045, device='cuda:0')) + | > loss_gen: 2.6439931392669678 (2.6439931392669678) + | > loss_kl: 1.8152070045471191 (1.8152070045471191) + | > loss_feat: 9.324139595031738 (9.324139595031738) + | > loss_mel: 17.237491607666016 (17.237491607666016) + | > loss_duration: 0.8634895086288452 (0.8634895086288452) + | > loss_1: 31.884321212768555 (31.884321212768555) + | > grad_norm_1: tensor(373.0717, device='cuda:0') (tensor(373.0717, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 1.0499 (1.0498723983764648) + | > loader_time: 1.185 (1.185046672821045) + + + --> TIME: 2024-07-13 22:31:06 -- STEP: 50/151 -- GLOBAL_STEP: 100000 + | > loss_disc: 2.4935860633850098 (2.338490521907806) + | > loss_disc_real_0: 0.15776203572750092 (0.11982759565114975) + | > loss_disc_real_1: 0.16946159303188324 (0.20523523658514023) + | > loss_disc_real_2: 0.20346495509147644 (0.21430501312017441) + | > loss_disc_real_3: 0.24389977753162384 (0.20868276447057724) + | > loss_disc_real_4: 0.23323623836040497 (0.21276747614145278) + | > loss_disc_real_5: 0.25742778182029724 (0.22137367904186248) + | > loss_0: 2.4935860633850098 (2.338490521907806) + | > grad_norm_0: tensor(64.2223, device='cuda:0') (tensor(28.8792, device='cuda:0')) + | > loss_gen: 2.4778666496276855 (2.589773292541504) + | > loss_kl: 1.9951205253601074 (1.7170866107940674) + | > loss_feat: 7.7835211753845215 (7.9713262462615955) + | > loss_mel: 15.928610801696777 (16.777506694793697) + | > loss_duration: 0.8908747434616089 (0.9462910342216492) + | > loss_1: 29.07599449157715 (30.00198402404785) + | > grad_norm_1: tensor(495.6104, device='cuda:0') (tensor(443.6942, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.935 (1.037567653656006) + | > loader_time: 0.0059 (0.0068083047866821286) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_100000.pth + + --> TIME: 2024-07-13 22:32:26 -- STEP: 100/151 -- GLOBAL_STEP: 100050 + | > loss_disc: 2.393012046813965 (2.310308426618576) + | > loss_disc_real_0: 0.09804826229810715 (0.119875990152359) + | > loss_disc_real_1: 0.22348929941654205 (0.2023841507732868) + | > loss_disc_real_2: 0.22196407616138458 (0.21203043833374977) + | > loss_disc_real_3: 0.2460131049156189 (0.20573321998119354) + | > loss_disc_real_4: 0.24197383224964142 (0.20730775192379952) + | > loss_disc_real_5: 0.20335176587104797 (0.21914708405733108) + | > loss_0: 2.393012046813965 (2.310308426618576) + | > grad_norm_0: tensor(20.5697, device='cuda:0') (tensor(31.3526, device='cuda:0')) + | > loss_gen: 2.485628366470337 (2.631845281124115) + | > loss_kl: 1.5104361772537231 (1.7300754857063294) + | > loss_feat: 9.091832160949707 (8.200655579566954) + | > loss_mel: 17.023544311523438 (16.906200551986693) + | > loss_duration: 0.878348708152771 (0.9398349446058273) + | > loss_1: 30.989789962768555 (30.408611965179443) + | > grad_norm_1: tensor(319.9598, device='cuda:0') (tensor(496.7718, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.8979 (1.0036252427101138) + | > loader_time: 0.0052 (0.0065141630172729495) + + + --> TIME: 2024-07-13 22:33:15 -- STEP: 150/151 -- GLOBAL_STEP: 100100 + | > loss_disc: 2.3462297916412354 (2.3046979689598084) + | > loss_disc_real_0: 0.06626132875680923 (0.12018331162631511) + | > loss_disc_real_1: 0.2023572474718094 (0.20301802337169647) + | > loss_disc_real_2: 0.1817111372947693 (0.21130585839351018) + | > loss_disc_real_3: 0.23249226808547974 (0.20623321240146955) + | > loss_disc_real_4: 0.21944090723991394 (0.20779616594314576) + | > loss_disc_real_5: 0.22894518077373505 (0.21754705319801967) + | > loss_0: 2.3462297916412354 (2.3046979689598084) + | > grad_norm_0: tensor(34.1284, device='cuda:0') (tensor(29.1091, device='cuda:0')) + | > loss_gen: 2.5098888874053955 (2.6299730968475337) + | > loss_kl: 2.1880717277526855 (1.7360672569274902) + | > loss_feat: 8.230463027954102 (8.256426426569623) + | > loss_mel: 17.664165496826172 (16.899541168212885) + | > loss_duration: 0.8526104688644409 (0.9375250593821207) + | > loss_1: 31.445199966430664 (30.45953317006429) + | > grad_norm_1: tensor(598.6821, device='cuda:0') (tensor(482.2193, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.9745 (0.9922524309158327) + | > loader_time: 0.0057 (0.006399442354838054) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03433885941138634 (+0.0023875603309044466) + | > avg_loss_disc: 2.5722243235661435 (-0.04961635516240026) + | > avg_loss_disc_real_0: 0.17214747460988852 (-0.08803922167191139) + | > avg_loss_disc_real_1: 0.18770887989264268 (-0.04894718757042518) + | > avg_loss_disc_real_2: 0.23868853312272292 (-0.03398673580243039) + | > avg_loss_disc_real_3: 0.24127030716492578 (+0.02171034423204568) + | > avg_loss_disc_real_4: 0.24011969566345215 (-0.03979782530894649) + | > avg_loss_disc_real_5: 0.2472736766705146 (+0.035707656007546634) + | > avg_loss_0: 2.5722243235661435 (-0.04961635516240026) + | > avg_loss_gen: 2.316986010624812 (-0.2192083505483775) + | > avg_loss_kl: 3.634554202740009 (+0.06479443036592913) + | > avg_loss_feat: 7.6787093969491815 (-0.17192862584040647) + | > avg_loss_mel: 17.247900155874397 (+0.49570978604830174) + | > avg_loss_duration: 1.327845573425293 (+0.016297597151536225) + | > avg_loss_1: 32.20599541297326 (+0.18566513061523438) + + + > EPOCH: 100/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:33:24)  + + --> TIME: 2024-07-13 22:34:19 -- STEP: 49/151 -- GLOBAL_STEP: 100150 + | > loss_disc: 2.136427640914917 (2.3199203695569715) + | > loss_disc_real_0: 0.09631678462028503 (0.1283635963894883) + | > loss_disc_real_1: 0.14751766622066498 (0.20219504863631968) + | > loss_disc_real_2: 0.13734321296215057 (0.21150886951660625) + | > loss_disc_real_3: 0.17014071345329285 (0.20498172482665705) + | > loss_disc_real_4: 0.15667147934436798 (0.20507947705229934) + | > loss_disc_real_5: 0.26182472705841064 (0.21928718321177423) + | > loss_0: 2.136427640914917 (2.3199203695569715) + | > grad_norm_0: tensor(53.0579, device='cuda:0') (tensor(32.6804, device='cuda:0')) + | > loss_gen: 2.648618459701538 (2.59495759983452) + | > loss_kl: 1.1248996257781982 (1.8457390775485916) + | > loss_feat: 9.244974136352539 (8.294614091211434) + | > loss_mel: 16.655118942260742 (16.936673300606863) + | > loss_duration: 1.089815378189087 (0.9393936638929405) + | > loss_1: 30.763427734375 (30.611377599288005) + | > grad_norm_1: tensor(537.4327, device='cuda:0') (tensor(496.4453, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.817 (1.0466330927245473) + | > loader_time: 0.004 (0.0068505588842898) + + + --> TIME: 2024-07-13 22:35:07 -- STEP: 99/151 -- GLOBAL_STEP: 100200 + | > loss_disc: 2.387479782104492 (2.2995040777957803) + | > loss_disc_real_0: 0.24048909544944763 (0.12457373083540887) + | > loss_disc_real_1: 0.2089393436908722 (0.2017604461824051) + | > loss_disc_real_2: 0.22556963562965393 (0.211209877873912) + | > loss_disc_real_3: 0.23140157759189606 (0.20507773230172166) + | > loss_disc_real_4: 0.22310757637023926 (0.2043589824678922) + | > loss_disc_real_5: 0.2435987889766693 (0.21651404719762132) + | > loss_0: 2.387479782104492 (2.2995040777957803) + | > grad_norm_0: tensor(77.5574, device='cuda:0') (tensor(34.9166, device='cuda:0')) + | > loss_gen: 2.6945393085479736 (2.6373996566040354) + | > loss_kl: 2.0931396484375 (1.821519714413267) + | > loss_feat: 7.891506195068359 (8.45781396615385) + | > loss_mel: 16.968019485473633 (16.927354186472268) + | > loss_duration: 0.8871756792068481 (0.9372437909396008) + | > loss_1: 30.534381866455078 (30.78133132240989) + | > grad_norm_1: tensor(572.7977, device='cuda:0') (tensor(516.5945, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.9034 (1.0020796554257174) + | > loader_time: 0.0064 (0.006559157612347844) + + + --> TIME: 2024-07-13 22:35:56 -- STEP: 149/151 -- GLOBAL_STEP: 100250 + | > loss_disc: 2.1721935272216797 (2.3008573231281053) + | > loss_disc_real_0: 0.09637852758169174 (0.12222232087046508) + | > loss_disc_real_1: 0.17112386226654053 (0.20113325019010755) + | > loss_disc_real_2: 0.1479848027229309 (0.21080906279135067) + | > loss_disc_real_3: 0.1643347293138504 (0.20425832721611) + | > loss_disc_real_4: 0.18585161864757538 (0.204888887053368) + | > loss_disc_real_5: 0.20508736371994019 (0.21823018839295283) + | > loss_0: 2.1721935272216797 (2.3008573231281053) + | > grad_norm_0: tensor(30.2560, device='cuda:0') (tensor(31.6777, device='cuda:0')) + | > loss_gen: 2.6551787853240967 (2.620595874402347) + | > loss_kl: 1.7559235095977783 (1.796113665471941) + | > loss_feat: 9.189626693725586 (8.406399384440988) + | > loss_mel: 17.2448787689209 (16.838431486347385) + | > loss_duration: 0.9706553220748901 (0.9381165228434057) + | > loss_1: 31.81626319885254 (30.599656981909835) + | > grad_norm_1: tensor(530.2438, device='cuda:0') (tensor(480.3318, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.8532 (0.9912868106125189) + | > loader_time: 0.0041 (0.006356095307625378) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0340429819547213 (-0.00029587745666503906) + | > avg_loss_disc: 2.620595400150006 (+0.048371076583862305) + | > avg_loss_disc_real_0: 0.2447537355698072 (+0.07260626095991868) + | > avg_loss_disc_real_1: 0.2317710851247494 (+0.044062205232106716) + | > avg_loss_disc_real_2: 0.22246225292866045 (-0.01622628019406247) + | > avg_loss_disc_real_3: 0.24428840325428888 (+0.003018096089363098) + | > avg_loss_disc_real_4: 0.27939191460609436 (+0.03927221894264221) + | > avg_loss_disc_real_5: 0.25537012288203614 (+0.00809644621152153) + | > avg_loss_0: 2.620595400150006 (+0.048371076583862305) + | > avg_loss_gen: 2.5236593026381273 (+0.2066732920133152) + | > avg_loss_kl: 3.716716014421903 (+0.08216181168189429) + | > avg_loss_feat: 7.7866914455707255 (+0.10798204862154392) + | > avg_loss_mel: 17.194015723008377 (-0.053884432866020404) + | > avg_loss_duration: 1.2413095098275404 (-0.08653606359775257) + | > avg_loss_1: 32.46239266028771 (+0.2563972473144531) + + + > EPOCH: 101/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:36:08)  + + --> TIME: 2024-07-13 22:37:00 -- STEP: 48/151 -- GLOBAL_STEP: 100300 + | > loss_disc: 2.68510103225708 (2.348334481318792) + | > loss_disc_real_0: 0.2209024876356125 (0.130292090975369) + | > loss_disc_real_1: 0.23596929013729095 (0.20409221605708203) + | > loss_disc_real_2: 0.2510491907596588 (0.2139068804681301) + | > loss_disc_real_3: 0.2796136736869812 (0.2073219626521071) + | > loss_disc_real_4: 0.2631029188632965 (0.2113436764727036) + | > loss_disc_real_5: 0.2552826404571533 (0.22421844955533743) + | > loss_0: 2.68510103225708 (2.348334481318792) + | > grad_norm_0: tensor(28.6499, device='cuda:0') (tensor(31.3169, device='cuda:0')) + | > loss_gen: 2.2203738689422607 (2.5906167129675555) + | > loss_kl: 2.1459591388702393 (1.7882067499061425) + | > loss_feat: 4.9714155197143555 (8.031699349482855) + | > loss_mel: 14.557968139648438 (16.775661945343018) + | > loss_duration: 0.9253580570220947 (0.9522815470894178) + | > loss_1: 24.821073532104492 (30.138466278711956) + | > grad_norm_1: tensor(267.8927, device='cuda:0') (tensor(414.1816, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 1.0694 (1.0222580681244533) + | > loader_time: 0.0079 (0.0067137877146403) + + + --> TIME: 2024-07-13 22:37:47 -- STEP: 98/151 -- GLOBAL_STEP: 100350 + | > loss_disc: 2.4547922611236572 (2.3292986799259583) + | > loss_disc_real_0: 0.10968371480703354 (0.12727989419838606) + | > loss_disc_real_1: 0.1580706089735031 (0.20518773474863597) + | > loss_disc_real_2: 0.18666663765907288 (0.21218421720728584) + | > loss_disc_real_3: 0.20880568027496338 (0.20649937661934872) + | > loss_disc_real_4: 0.22791588306427002 (0.20699119309381564) + | > loss_disc_real_5: 0.3077271282672882 (0.22194744296828095) + | > loss_0: 2.4547922611236572 (2.3292986799259583) + | > grad_norm_0: tensor(40.1833, device='cuda:0') (tensor(32.8939, device='cuda:0')) + | > loss_gen: 2.458204984664917 (2.6221081845614376) + | > loss_kl: 2.140766143798828 (1.759854217573088) + | > loss_feat: 8.163688659667969 (8.17226213824992) + | > loss_mel: 16.98726463317871 (16.802286693028044) + | > loss_duration: 0.9092814922332764 (0.9439229977374174) + | > loss_1: 30.659204483032227 (30.300434268250758) + | > grad_norm_1: tensor(384.0158, device='cuda:0') (tensor(459.2482, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.8892 (0.9808202398066618) + | > loader_time: 0.0052 (0.006351461215895049) + + + --> TIME: 2024-07-13 22:38:37 -- STEP: 148/151 -- GLOBAL_STEP: 100400 + | > loss_disc: 2.0999770164489746 (2.3244417478909365) + | > loss_disc_real_0: 0.08508503437042236 (0.12475761783787537) + | > loss_disc_real_1: 0.17703573405742645 (0.20364225806819428) + | > loss_disc_real_2: 0.1949337273836136 (0.21189392650046865) + | > loss_disc_real_3: 0.18942809104919434 (0.2069869018789079) + | > loss_disc_real_4: 0.16377276182174683 (0.2083290959834247) + | > loss_disc_real_5: 0.20635254681110382 (0.2210398508688888) + | > loss_0: 2.0999770164489746 (2.3244417478909365) + | > grad_norm_0: tensor(21.3435, device='cuda:0') (tensor(29.3317, device='cuda:0')) + | > loss_gen: 2.8177990913391113 (2.6092196702957153) + | > loss_kl: 1.7832374572753906 (1.767708868593783) + | > loss_feat: 9.070212364196777 (8.233651860340217) + | > loss_mel: 16.323810577392578 (16.780465892843303) + | > loss_duration: 1.0531806945800781 (0.9403674284348617) + | > loss_1: 31.048240661621094 (30.331413836092562) + | > grad_norm_1: tensor(449.2357, device='cuda:0') (tensor(430.1022, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.8427 (0.9789133071899414) + | > loader_time: 0.0045 (0.006262163858155946) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03310078840989333 (-0.0009421935448279753) + | > avg_loss_disc: 2.490589343584501 (-0.130006056565505) + | > avg_loss_disc_real_0: 0.20768887721575224 (-0.03706485835405496) + | > avg_loss_disc_real_1: 0.21600501812421358 (-0.015766067000535816) + | > avg_loss_disc_real_2: 0.21823831819570982 (-0.004223934732950629) + | > avg_loss_disc_real_3: 0.23640028559244597 (-0.007888117661842914) + | > avg_loss_disc_real_4: 0.2779163122177124 (-0.001475602388381958) + | > avg_loss_disc_real_5: 0.29238300025463104 (+0.0370128773725949) + | > avg_loss_0: 2.490589343584501 (-0.130006056565505) + | > avg_loss_gen: 2.636011105317336 (+0.11235180267920875) + | > avg_loss_kl: 3.7061055440169115 (-0.010610470404991634) + | > avg_loss_feat: 7.9970965385437 (+0.21040509297297483) + | > avg_loss_mel: 17.05517688164344 (-0.13883884136493663) + | > avg_loss_duration: 1.2627682869250958 (+0.021458777097555393) + | > avg_loss_1: 32.657157751230095 (+0.1947650909423828) + + + > EPOCH: 102/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:38:49)  + + --> TIME: 2024-07-13 22:39:39 -- STEP: 47/151 -- GLOBAL_STEP: 100450 + | > loss_disc: 2.2545037269592285 (2.3293173718959737) + | > loss_disc_real_0: 0.09963610023260117 (0.12154096277787332) + | > loss_disc_real_1: 0.18421413004398346 (0.20144470321371205) + | > loss_disc_real_2: 0.18422439694404602 (0.21037898704092553) + | > loss_disc_real_3: 0.22394126653671265 (0.20640402398210891) + | > loss_disc_real_4: 0.2686663269996643 (0.2115071897811078) + | > loss_disc_real_5: 0.22637490928173065 (0.21988226092876273) + | > loss_0: 2.2545037269592285 (2.3293173718959737) + | > grad_norm_0: tensor(34.5800, device='cuda:0') (tensor(25.6428, device='cuda:0')) + | > loss_gen: 2.7372617721557617 (2.5796995568782726) + | > loss_kl: 1.985750675201416 (1.718444268754188) + | > loss_feat: 8.663278579711914 (8.285820737798163) + | > loss_mel: 17.711383819580078 (16.747195994600336) + | > loss_duration: 0.8525260090827942 (0.9397833017592735) + | > loss_1: 31.9502010345459 (30.27094394602674) + | > grad_norm_1: tensor(477.2065, device='cuda:0') (tensor(442.8876, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.8875 (0.9998378449297968) + | > loader_time: 0.0051 (0.0063216939885565576) + + + --> TIME: 2024-07-13 22:40:27 -- STEP: 97/151 -- GLOBAL_STEP: 100500 + | > loss_disc: 1.9769957065582275 (2.321913333283258) + | > loss_disc_real_0: 0.05485157296061516 (0.12157218980113257) + | > loss_disc_real_1: 0.16046014428138733 (0.20232290536472478) + | > loss_disc_real_2: 0.15165959298610687 (0.21072167849417814) + | > loss_disc_real_3: 0.15701398253440857 (0.20403458485283807) + | > loss_disc_real_4: 0.13684561848640442 (0.20852088359827847) + | > loss_disc_real_5: 0.15157976746559143 (0.22161054611206055) + | > loss_0: 1.9769957065582275 (2.321913333283258) + | > grad_norm_0: tensor(26.7087, device='cuda:0') (tensor(29.7743, device='cuda:0')) + | > loss_gen: 2.7912750244140625 (2.6067182526146015) + | > loss_kl: 1.8025420904159546 (1.7349622771912014) + | > loss_feat: 9.806975364685059 (8.32660176090359) + | > loss_mel: 16.94972038269043 (16.789997543256316) + | > loss_duration: 1.0532290935516357 (0.9422805358454124) + | > loss_1: 32.403743743896484 (30.400560555998812) + | > grad_norm_1: tensor(654.4208, device='cuda:0') (tensor(478.4082, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.8141 (0.972108376394842) + | > loader_time: 0.0051 (0.006153499957212465) + + + --> TIME: 2024-07-13 22:41:18 -- STEP: 147/151 -- GLOBAL_STEP: 100550 + | > loss_disc: 2.6272921562194824 (2.3231936522892545) + | > loss_disc_real_0: 0.2534831762313843 (0.1200445711764754) + | > loss_disc_real_1: 0.2605932652950287 (0.2027028289173736) + | > loss_disc_real_2: 0.24712955951690674 (0.21081517545544368) + | > loss_disc_real_3: 0.2751491069793701 (0.20595681738285793) + | > loss_disc_real_4: 0.2546819746494293 (0.2088938220625832) + | > loss_disc_real_5: 0.2469344139099121 (0.22360982963828002) + | > loss_0: 2.6272921562194824 (2.3231936522892545) + | > grad_norm_0: tensor(75.2661, device='cuda:0') (tensor(31.0568, device='cuda:0')) + | > loss_gen: 2.188070058822632 (2.615144557693377) + | > loss_kl: 2.228024482727051 (1.7400479065317687) + | > loss_feat: 4.176786422729492 (8.312782566563616) + | > loss_mel: 13.422982215881348 (16.886215320249814) + | > loss_duration: 1.18117094039917 (0.9413868055051687) + | > loss_1: 23.19703483581543 (30.49557741645242) + | > grad_norm_1: tensor(258.9712, device='cuda:0') (tensor(478.8233, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 2.0865 (0.9798865269641487) + | > loader_time: 0.0151 (0.006106373404159025) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03213187364431528 (-0.0009689147655780467) + | > avg_loss_disc: 2.5238046646118164 (+0.033215321027315614) + | > avg_loss_disc_real_0: 0.2606892597216826 (+0.053000382505930366) + | > avg_loss_disc_real_1: 0.1834961737577732 (-0.03250884436644039) + | > avg_loss_disc_real_2: 0.17107810309300056 (-0.04716021510270926) + | > avg_loss_disc_real_3: 0.20080020336004403 (-0.035600082232401936) + | > avg_loss_disc_real_4: 0.2316682774287004 (-0.046248034789012016) + | > avg_loss_disc_real_5: 0.2453225335249534 (-0.04706046672967765) + | > avg_loss_0: 2.5238046646118164 (+0.033215321027315614) + | > avg_loss_gen: 2.429840638087346 (-0.20617046722999) + | > avg_loss_kl: 3.782503531529353 (+0.07639798751244165) + | > avg_loss_feat: 8.156411867875319 (+0.1593153293316183) + | > avg_loss_mel: 17.102606846736027 (+0.04742996509258646) + | > avg_loss_duration: 1.3437493305939894 (+0.08098104366889358) + | > avg_loss_1: 32.81511189387394 (+0.15795414264384533) + + + > EPOCH: 103/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:41:31)  + + --> TIME: 2024-07-13 22:42:18 -- STEP: 46/151 -- GLOBAL_STEP: 100600 + | > loss_disc: 2.3823180198669434 (2.3045677309450903) + | > loss_disc_real_0: 0.12249243259429932 (0.12168625758393951) + | > loss_disc_real_1: 0.21177414059638977 (0.2016591714776081) + | > loss_disc_real_2: 0.2428426891565323 (0.21315812287123306) + | > loss_disc_real_3: 0.23305940628051758 (0.20382393151521683) + | > loss_disc_real_4: 0.21051451563835144 (0.20516113904507263) + | > loss_disc_real_5: 0.24797549843788147 (0.21774790759967722) + | > loss_0: 2.3823180198669434 (2.3045677309450903) + | > grad_norm_0: tensor(13.1317, device='cuda:0') (tensor(25.2470, device='cuda:0')) + | > loss_gen: 2.675999641418457 (2.5717282813528315) + | > loss_kl: 2.278923749923706 (1.690198355394861) + | > loss_feat: 7.1346001625061035 (8.129009526708854) + | > loss_mel: 16.426036834716797 (16.370206563369084) + | > loss_duration: 0.9056186676025391 (0.9474927005560502) + | > loss_1: 29.421178817749023 (29.708635288736094) + | > grad_norm_1: tensor(380.3647, device='cuda:0') (tensor(279.4498, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.9659 (0.9795658536579298) + | > loader_time: 0.0056 (0.00584622569706129) + + + --> TIME: 2024-07-13 22:43:09 -- STEP: 96/151 -- GLOBAL_STEP: 100650 + | > loss_disc: 2.3344993591308594 (2.315466525653998) + | > loss_disc_real_0: 0.1125502809882164 (0.12306750678302099) + | > loss_disc_real_1: 0.18402311205863953 (0.20179955397422114) + | > loss_disc_real_2: 0.21326912939548492 (0.21290127366470793) + | > loss_disc_real_3: 0.21279433369636536 (0.20665796209747592) + | > loss_disc_real_4: 0.27900978922843933 (0.20699889453438422) + | > loss_disc_real_5: 0.2816486954689026 (0.22222884822015962) + | > loss_0: 2.3344993591308594 (2.315466525653998) + | > grad_norm_0: tensor(39.7841, device='cuda:0') (tensor(25.8893, device='cuda:0')) + | > loss_gen: 2.4487709999084473 (2.5721910074353227) + | > loss_kl: 1.6352401971817017 (1.733272568633159) + | > loss_feat: 7.086779594421387 (8.10460843642553) + | > loss_mel: 16.81375503540039 (16.55992893377939) + | > loss_duration: 0.9160164594650269 (0.9431740815440813) + | > loss_1: 28.90056037902832 (29.913175106048584) + | > grad_norm_1: tensor(599.7809, device='cuda:0') (tensor(361.4030, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.9295 (0.991094420353572) + | > loader_time: 0.0056 (0.006154932081699371) + + + --> TIME: 2024-07-13 22:43:58 -- STEP: 146/151 -- GLOBAL_STEP: 100700 + | > loss_disc: 2.1452479362487793 (2.3209910637711824) + | > loss_disc_real_0: 0.11903773248195648 (0.12152298914957536) + | > loss_disc_real_1: 0.1150876134634018 (0.20276348098908384) + | > loss_disc_real_2: 0.14725586771965027 (0.21180782829449601) + | > loss_disc_real_3: 0.1759616881608963 (0.2067959501522861) + | > loss_disc_real_4: 0.17010748386383057 (0.20739533767512397) + | > loss_disc_real_5: 0.23960593342781067 (0.2217834665350718) + | > loss_0: 2.1452479362487793 (2.3209910637711824) + | > grad_norm_0: tensor(48.8498, device='cuda:0') (tensor(27.8857, device='cuda:0')) + | > loss_gen: 2.7427549362182617 (2.5798746020826586) + | > loss_kl: 1.2206902503967285 (1.7274105218175337) + | > loss_feat: 9.483293533325195 (8.153712435944446) + | > loss_mel: 16.9247989654541 (16.638889476044533) + | > loss_duration: 1.0597467422485352 (0.9401506038561259) + | > loss_1: 31.43128204345703 (30.040037769160858) + | > grad_norm_1: tensor(495.0762, device='cuda:0') (tensor(408.3013, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.8165 (0.9848856517713368) + | > loader_time: 0.0044 (0.0060416427377152125) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03169448559100811 (-0.00043738805330716807) + | > avg_loss_disc: 2.5173795773432803 (-0.006425087268536078) + | > avg_loss_disc_real_0: 0.20689123410444993 (-0.05379802561723268) + | > avg_loss_disc_real_1: 0.22294191328378823 (+0.039445739526015045) + | > avg_loss_disc_real_2: 0.22373242790882403 (+0.05265432481582347) + | > avg_loss_disc_real_3: 0.23231754165429336 (+0.031517338294249325) + | > avg_loss_disc_real_4: 0.2722006061902413 (+0.04053232876154089) + | > avg_loss_disc_real_5: 0.20495694073346946 (-0.04036559279148394) + | > avg_loss_0: 2.5173795773432803 (-0.006425087268536078) + | > avg_loss_gen: 2.4982325847332296 (+0.06839194664588355) + | > avg_loss_kl: 3.828204393386841 (+0.045700861857487673) + | > avg_loss_feat: 8.146194861485409 (-0.010217006389909855) + | > avg_loss_mel: 17.26666553203876 (+0.16405868530273438) + | > avg_loss_duration: 1.3418088050989003 (-0.0019405254950890516) + | > avg_loss_1: 33.08110618591308 (+0.2659942920391387) + + + > EPOCH: 104/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:44:14)  + + --> TIME: 2024-07-13 22:45:01 -- STEP: 45/151 -- GLOBAL_STEP: 100750 + | > loss_disc: 2.217231273651123 (2.342654821607802) + | > loss_disc_real_0: 0.10969255864620209 (0.12815161943435674) + | > loss_disc_real_1: 0.18984819948673248 (0.20283647113376194) + | > loss_disc_real_2: 0.21060331165790558 (0.208265577422248) + | > loss_disc_real_3: 0.2085706889629364 (0.212189542916086) + | > loss_disc_real_4: 0.21148483455181122 (0.20924390620655484) + | > loss_disc_real_5: 0.20183877646923065 (0.22425729632377625) + | > loss_0: 2.217231273651123 (2.342654821607802) + | > grad_norm_0: tensor(39.2951, device='cuda:0') (tensor(35.5667, device='cuda:0')) + | > loss_gen: 2.6458511352539062 (2.5851557413736974) + | > loss_kl: 2.021200656890869 (1.7893441783057318) + | > loss_feat: 7.281535625457764 (8.187737782796223) + | > loss_mel: 15.82303237915039 (16.849849446614584) + | > loss_duration: 0.9365013837814331 (0.9381576577822367) + | > loss_1: 28.708120346069336 (30.350244776407877) + | > grad_norm_1: tensor(530.0013, device='cuda:0') (tensor(538.5668, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.9922 (0.9694699658287896) + | > loader_time: 0.0066 (0.005976173612806533) + + + --> TIME: 2024-07-13 22:45:50 -- STEP: 95/151 -- GLOBAL_STEP: 100800 + | > loss_disc: 2.4529781341552734 (2.3560381412506097) + | > loss_disc_real_0: 0.1353096067905426 (0.13661060341094655) + | > loss_disc_real_1: 0.22882239520549774 (0.20244183069781252) + | > loss_disc_real_2: 0.18691830337047577 (0.2114964108718069) + | > loss_disc_real_3: 0.22199925780296326 (0.2104246083058809) + | > loss_disc_real_4: 0.20554795861244202 (0.21122635571580184) + | > loss_disc_real_5: 0.26720044016838074 (0.2227003622211908) + | > loss_0: 2.4529781341552734 (2.3560381412506097) + | > grad_norm_0: tensor(38.2328, device='cuda:0') (tensor(37.2892, device='cuda:0')) + | > loss_gen: 2.423410415649414 (2.5959276776564733) + | > loss_kl: 2.300044059753418 (1.7786673545837401) + | > loss_feat: 8.237629890441895 (8.250049179478696) + | > loss_mel: 17.500816345214844 (16.927281861556217) + | > loss_duration: 0.9583516120910645 (0.9392431139945984) + | > loss_1: 31.420251846313477 (30.49116907621685) + | > grad_norm_1: tensor(617.0917, device='cuda:0') (tensor(503.2025, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 1.032 (0.9650015981573807) + | > loader_time: 0.0086 (0.006139306018227025) + + + --> TIME: 2024-07-13 22:46:40 -- STEP: 145/151 -- GLOBAL_STEP: 100850 + | > loss_disc: 2.238586187362671 (2.343580410398286) + | > loss_disc_real_0: 0.07854209840297699 (0.13262383164516806) + | > loss_disc_real_1: 0.20369549095630646 (0.20256673816976875) + | > loss_disc_real_2: 0.21395398676395416 (0.2116652250289917) + | > loss_disc_real_3: 0.17880631983280182 (0.20825114219353122) + | > loss_disc_real_4: 0.14092956483364105 (0.20895518352245462) + | > loss_disc_real_5: 0.20088860392570496 (0.22107650724978284) + | > loss_0: 2.238586187362671 (2.343580410398286) + | > grad_norm_0: tensor(14.5861, device='cuda:0') (tensor(34.7037, device='cuda:0')) + | > loss_gen: 2.704413414001465 (2.5972314176888287) + | > loss_kl: 1.34554922580719 (1.7900405448058556) + | > loss_feat: 9.633159637451172 (8.280880082886794) + | > loss_mel: 16.708925247192383 (16.912143536271728) + | > loss_duration: 1.0238518714904785 (0.9401787297479037) + | > loss_1: 31.4158992767334 (30.520474151085164) + | > grad_norm_1: tensor(232.7473, device='cuda:0') (tensor(486.6205, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.8149 (0.9748622516105915) + | > loader_time: 0.0037 (0.006258479480085702) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03230687288137583 (+0.0006123872903677149) + | > avg_loss_disc: 2.519591129743136 (+0.0022115523998555986) + | > avg_loss_disc_real_0: 0.21066375076770782 (+0.0037725166632578966) + | > avg_loss_disc_real_1: 0.20467118116525504 (-0.018270732118533195) + | > avg_loss_disc_real_2: 0.20570308887041533 (-0.018029339038408698) + | > avg_loss_disc_real_3: 0.21956542019660658 (-0.012752121457686783) + | > avg_loss_disc_real_4: 0.21895759380780733 (-0.05324301238243395) + | > avg_loss_disc_real_5: 0.23270036853276765 (+0.027743427799298198) + | > avg_loss_0: 2.519591129743136 (+0.0022115523998555986) + | > avg_loss_gen: 2.3705030771402216 (-0.12772950759300805) + | > avg_loss_kl: 3.5870508597447324 (-0.24115353364210845) + | > avg_loss_feat: 8.09525416447566 (-0.05094069700974835) + | > avg_loss_mel: 17.395631570082443 (+0.12896603804368212) + | > avg_loss_duration: 1.3147077010228083 (-0.02710110407609201) + | > avg_loss_1: 32.763147794283356 (-0.3179583916297233) + + + > EPOCH: 105/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:46:55)  + + --> TIME: 2024-07-13 22:47:43 -- STEP: 44/151 -- GLOBAL_STEP: 100900 + | > loss_disc: 2.4068479537963867 (2.315841070630334) + | > loss_disc_real_0: 0.10589935630559921 (0.12221969833428209) + | > loss_disc_real_1: 0.1949361115694046 (0.20467123118313876) + | > loss_disc_real_2: 0.20323100686073303 (0.21426396105777135) + | > loss_disc_real_3: 0.1597149521112442 (0.1999827613207427) + | > loss_disc_real_4: 0.1688840687274933 (0.2033068317581307) + | > loss_disc_real_5: 0.21415473520755768 (0.2174556776881218) + | > loss_0: 2.4068479537963867 (2.315841070630334) + | > grad_norm_0: tensor(23.2327, device='cuda:0') (tensor(30.2807, device='cuda:0')) + | > loss_gen: 2.4386661052703857 (2.5992545214566314) + | > loss_kl: 1.7795628309249878 (1.792942832816731) + | > loss_feat: 7.691218376159668 (8.178436333482916) + | > loss_mel: 17.740631103515625 (16.987086252732702) + | > loss_duration: 1.0424034595489502 (0.9571100798520175) + | > loss_1: 30.692481994628906 (30.514829982410777) + | > grad_norm_1: tensor(490.8456, device='cuda:0') (tensor(494.0519, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 1.0914 (1.026882599700581) + | > loader_time: 0.0069 (0.006728150627829812) + + + --> TIME: 2024-07-13 22:48:31 -- STEP: 94/151 -- GLOBAL_STEP: 100950 + | > loss_disc: 2.3124475479125977 (2.301723211369615) + | > loss_disc_real_0: 0.11020110547542572 (0.11984322124973257) + | > loss_disc_real_1: 0.17957331240177155 (0.20356958977719572) + | > loss_disc_real_2: 0.15653732419013977 (0.2116889108685737) + | > loss_disc_real_3: 0.20420576632022858 (0.2009254703496365) + | > loss_disc_real_4: 0.161329448223114 (0.20428641505063844) + | > loss_disc_real_5: 0.23172710835933685 (0.219095841208671) + | > loss_0: 2.3124475479125977 (2.301723211369615) + | > grad_norm_0: tensor(21.2632, device='cuda:0') (tensor(30.0928, device='cuda:0')) + | > loss_gen: 2.4787309169769287 (2.622674401770247) + | > loss_kl: 1.6020808219909668 (1.7569803709679461) + | > loss_feat: 8.736542701721191 (8.32202680567478) + | > loss_mel: 16.325090408325195 (16.971730810530634) + | > loss_duration: 0.767266035079956 (0.9415725374475439) + | > loss_1: 29.909709930419922 (30.614984958729845) + | > grad_norm_1: tensor(551.4756, device='cuda:0') (tensor(482.8976, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.8561 (0.9774631414007633) + | > loader_time: 0.0045 (0.006357882885222739) + + + --> TIME: 2024-07-13 22:49:19 -- STEP: 144/151 -- GLOBAL_STEP: 101000 + | > loss_disc: 2.1789164543151855 (2.3009166055255452) + | > loss_disc_real_0: 0.15406672656536102 (0.1203574322991901) + | > loss_disc_real_1: 0.1573856621980667 (0.20246768753147787) + | > loss_disc_real_2: 0.18217039108276367 (0.21193289601554474) + | > loss_disc_real_3: 0.16841086745262146 (0.20250289742317465) + | > loss_disc_real_4: 0.196753591299057 (0.2048429390415549) + | > loss_disc_real_5: 0.23832112550735474 (0.21959845545805162) + | > loss_0: 2.1789164543151855 (2.3009166055255452) + | > grad_norm_0: tensor(31.6425, device='cuda:0') (tensor(30.8080, device='cuda:0')) + | > loss_gen: 2.587900161743164 (2.629728423224555) + | > loss_kl: 2.302387237548828 (1.7632486613260374) + | > loss_feat: 8.039652824401855 (8.29974781473478) + | > loss_mel: 18.285154342651367 (16.94570273160933) + | > loss_duration: 0.823497474193573 (0.936139002856281) + | > loss_1: 32.03858947753906 (30.574566589461433) + | > grad_norm_1: tensor(641.7667, device='cuda:0') (tensor(504.2600, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 1.0789 (0.9733538809749815) + | > loader_time: 0.0064 (0.006293680932786729) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03173215572650616 (-0.0005747171548696681) + | > avg_loss_disc: 2.6157735677865834 (+0.0961824380434475) + | > avg_loss_disc_real_0: 0.18988704910645118 (-0.02077670166125664) + | > avg_loss_disc_real_1: 0.22771479189395905 (+0.023043610728704006) + | > avg_loss_disc_real_2: 0.25143366478956664 (+0.045730575919151306) + | > avg_loss_disc_real_3: 0.252159982919693 (+0.03259456272308642) + | > avg_loss_disc_real_4: 0.279873326420784 (+0.06091573261297667) + | > avg_loss_disc_real_5: 0.21539280506280753 (-0.017307563469960124) + | > avg_loss_0: 2.6157735677865834 (+0.0961824380434475) + | > avg_loss_gen: 2.3439596983102646 (-0.026543378829956943) + | > avg_loss_kl: 3.639016169768113 (+0.051965310023380784) + | > avg_loss_feat: 7.7935398908761835 (-0.3017142735994769) + | > avg_loss_mel: 17.351932305556076 (-0.04369926452636719) + | > avg_loss_duration: 1.3815962626383855 (+0.06688856161557721) + | > avg_loss_1: 32.51004395118127 (-0.2531038431020889) + + + > EPOCH: 106/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:50:03)  + + --> TIME: 2024-07-13 22:50:46 -- STEP: 43/151 -- GLOBAL_STEP: 101050 + | > loss_disc: 2.320380449295044 (2.251664666242377) + | > loss_disc_real_0: 0.098250612616539 (0.11271767422210338) + | > loss_disc_real_1: 0.2700357735157013 (0.20155836468519167) + | > loss_disc_real_2: 0.27381742000579834 (0.20885594535705654) + | > loss_disc_real_3: 0.2054865062236786 (0.19881705212038617) + | > loss_disc_real_4: 0.19655592739582062 (0.20332154007845146) + | > loss_disc_real_5: 0.18812333047389984 (0.21161106679328653) + | > loss_0: 2.320380449295044 (2.251664666242377) + | > grad_norm_0: tensor(14.6890, device='cuda:0') (tensor(30.4705, device='cuda:0')) + | > loss_gen: 2.6679859161376953 (2.70154984607253) + | > loss_kl: 1.6651595830917358 (1.7024186184239942) + | > loss_feat: 7.619451999664307 (8.762905176295794) + | > loss_mel: 17.560373306274414 (17.248581464900532) + | > loss_duration: 0.9232947826385498 (0.9346474228903304) + | > loss_1: 30.43626594543457 (31.35010266858478) + | > grad_norm_1: tensor(500.1150, device='cuda:0') (tensor(566.2053, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.9999 (0.9431502153707105) + | > loader_time: 0.0075 (0.005737798158512559) + + + --> TIME: 2024-07-13 22:51:36 -- STEP: 93/151 -- GLOBAL_STEP: 101100 + | > loss_disc: 2.3113486766815186 (2.2994601982896037) + | > loss_disc_real_0: 0.1483338326215744 (0.12160250107927989) + | > loss_disc_real_1: 0.20243149995803833 (0.20221055787737655) + | > loss_disc_real_2: 0.22763650119304657 (0.20969918674679214) + | > loss_disc_real_3: 0.22383464872837067 (0.20421922463242723) + | > loss_disc_real_4: 0.21311543881893158 (0.20573694218871413) + | > loss_disc_real_5: 0.23064394295215607 (0.21781596349131677) + | > loss_0: 2.3113486766815186 (2.2994601982896037) + | > grad_norm_0: tensor(52.5612, device='cuda:0') (tensor(33.1531, device='cuda:0')) + | > loss_gen: 2.7257306575775146 (2.6434073986545688) + | > loss_kl: 1.9665488004684448 (1.7245313608518211) + | > loss_feat: 6.756263732910156 (8.40319425828996) + | > loss_mel: 15.59058952331543 (17.00233735320389) + | > loss_duration: 0.9062471389770508 (0.933918345999974) + | > loss_1: 27.94538116455078 (30.707388806086716) + | > grad_norm_1: tensor(604.7479, device='cuda:0') (tensor(533.0610, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.9469 (0.9702054710798366) + | > loader_time: 0.0062 (0.006233561423517041) + + + --> TIME: 2024-07-13 22:52:26 -- STEP: 143/151 -- GLOBAL_STEP: 101150 + | > loss_disc: 2.389822483062744 (2.3068307213016315) + | > loss_disc_real_0: 0.18872855603694916 (0.12141817589323003) + | > loss_disc_real_1: 0.21116948127746582 (0.2034092973787468) + | > loss_disc_real_2: 0.22375528514385223 (0.21064092614850796) + | > loss_disc_real_3: 0.19501127302646637 (0.20518995925679906) + | > loss_disc_real_4: 0.2123507559299469 (0.207013457388311) + | > loss_disc_real_5: 0.23714503645896912 (0.22021392477559043) + | > loss_0: 2.389822483062744 (2.3068307213016315) + | > grad_norm_0: tensor(53.7548, device='cuda:0') (tensor(32.8600, device='cuda:0')) + | > loss_gen: 2.509577512741089 (2.628763290551993) + | > loss_kl: 1.9295405149459839 (1.7381457840646064) + | > loss_feat: 7.961154460906982 (8.320573276573134) + | > loss_mel: 16.894975662231445 (16.891361976836947) + | > loss_duration: 0.9219961166381836 (0.9354754098645457) + | > loss_1: 30.217243194580078 (30.51431975998245) + | > grad_norm_1: tensor(728.9396, device='cuda:0') (tensor(522.3710, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.9736 (0.9764876899185714) + | > loader_time: 0.0054 (0.006170429549850782) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032844213338998646 (+0.001112057612492487) + | > avg_loss_disc: 2.646931373156034 (+0.031157805369450564) + | > avg_loss_disc_real_0: 0.2978184727522043 (+0.10793142364575312) + | > avg_loss_disc_real_1: 0.18837364934957945 (-0.03934114254437959) + | > avg_loss_disc_real_2: 0.24100968700188857 (-0.010423977787678063) + | > avg_loss_disc_real_3: 0.2822296367241786 (+0.03006965380448562) + | > avg_loss_disc_real_4: 0.2809782291834171 (+0.0011049027626330865) + | > avg_loss_disc_real_5: 0.2420029995533136 (+0.026610194490506084) + | > avg_loss_0: 2.646931373156034 (+0.031157805369450564) + | > avg_loss_gen: 2.5719759464263916 (+0.22801624811612697) + | > avg_loss_kl: 3.6129157543182373 (-0.026100415449875847) + | > avg_loss_feat: 7.978664178114671 (+0.18512428723848728) + | > avg_loss_mel: 16.841549726632927 (-0.5103825789231493) + | > avg_loss_duration: 1.258784688436068 (-0.12281157420231748) + | > avg_loss_1: 32.26389063321627 (-0.24615331796499618) + + + > EPOCH: 107/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:52:45)  + + --> TIME: 2024-07-13 22:53:28 -- STEP: 42/151 -- GLOBAL_STEP: 101200 + | > loss_disc: 2.571294069290161 (2.3133742071333385) + | > loss_disc_real_0: 0.17728178203105927 (0.1270352928411393) + | > loss_disc_real_1: 0.2738039493560791 (0.208513220506055) + | > loss_disc_real_2: 0.2889314293861389 (0.21306359519561133) + | > loss_disc_real_3: 0.25006166100502014 (0.20839846418017433) + | > loss_disc_real_4: 0.3367411196231842 (0.20707704588061288) + | > loss_disc_real_5: 0.26306086778640747 (0.2196555155373755) + | > loss_0: 2.571294069290161 (2.3133742071333385) + | > grad_norm_0: tensor(26.5278, device='cuda:0') (tensor(26.0318, device='cuda:0')) + | > loss_gen: 2.568357229232788 (2.620913653146653) + | > loss_kl: 2.0300540924072266 (1.8423729141553242) + | > loss_feat: 6.335894584655762 (8.29837307475862) + | > loss_mel: 16.056222915649414 (17.02578519639514) + | > loss_duration: 0.9533032774925232 (0.9254552949042547) + | > loss_1: 27.943832397460938 (30.7129001163301) + | > grad_norm_1: tensor(387.4746, device='cuda:0') (tensor(466.7059, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 1.0195 (0.9679846990676153) + | > loader_time: 0.0062 (0.005976904006231399) + + + --> TIME: 2024-07-13 22:54:18 -- STEP: 92/151 -- GLOBAL_STEP: 101250 + | > loss_disc: 2.50707745552063 (2.3091216735217883) + | > loss_disc_real_0: 0.18195906281471252 (0.12513629282298294) + | > loss_disc_real_1: 0.2562672793865204 (0.20341189383812572) + | > loss_disc_real_2: 0.22947070002555847 (0.21019425583274468) + | > loss_disc_real_3: 0.17525547742843628 (0.2033118950607984) + | > loss_disc_real_4: 0.19172419607639313 (0.20766920952693277) + | > loss_disc_real_5: 0.21793824434280396 (0.2201903954796169) + | > loss_0: 2.50707745552063 (2.3091216735217883) + | > grad_norm_0: tensor(36.3969, device='cuda:0') (tensor(24.8293, device='cuda:0')) + | > loss_gen: 2.381098508834839 (2.611029578291851) + | > loss_kl: 2.2890655994415283 (1.8124931759160499) + | > loss_feat: 6.732520580291748 (8.2496303734572) + | > loss_mel: 16.611770629882812 (16.909694847853288) + | > loss_duration: 1.1795541048049927 (0.9332199012455733) + | > loss_1: 29.19400978088379 (30.51606781586357) + | > grad_norm_1: tensor(541.2774, device='cuda:0') (tensor(440.5793, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 2.0717 (0.9787490912105726) + | > loader_time: 0.0177 (0.0061551099238188346) + + + --> TIME: 2024-07-13 22:55:07 -- STEP: 142/151 -- GLOBAL_STEP: 101300 + | > loss_disc: 2.4724905490875244 (2.298338257091146) + | > loss_disc_real_0: 0.2120581865310669 (0.12039179863854194) + | > loss_disc_real_1: 0.2279321849346161 (0.20243607555896462) + | > loss_disc_real_2: 0.23871667683124542 (0.20987477762178636) + | > loss_disc_real_3: 0.22823980450630188 (0.20330014667460616) + | > loss_disc_real_4: 0.2203848659992218 (0.2071138583123684) + | > loss_disc_real_5: 0.2704179584980011 (0.22024596986216555) + | > loss_0: 2.4724905490875244 (2.298338257091146) + | > grad_norm_0: tensor(26.1876, device='cuda:0') (tensor(25.0163, device='cuda:0')) + | > loss_gen: 2.754230260848999 (2.6313663704294563) + | > loss_kl: 1.7975572347640991 (1.8018810606338609) + | > loss_feat: 6.990583419799805 (8.408704277495264) + | > loss_mel: 16.796123504638672 (16.92131581104978) + | > loss_duration: 0.8748744130134583 (0.9302221746511863) + | > loss_1: 29.213367462158203 (30.69348973287663) + | > grad_norm_1: tensor(444.8127, device='cuda:0') (tensor(460.1506, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.9631 (0.973530823076275) + | > loader_time: 0.0056 (0.006021780027470117) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030661986424372747 (-0.0021822269146258982) + | > avg_loss_disc: 2.682025726024921 (+0.03509435286888696) + | > avg_loss_disc_real_0: 0.29441582468839794 (-0.003402648063806357) + | > avg_loss_disc_real_1: 0.2413534843004667 (+0.052979834950887233) + | > avg_loss_disc_real_2: 0.252539730989016 (+0.011530043987127425) + | > avg_loss_disc_real_3: 0.2208749491434831 (-0.06135468758069551) + | > avg_loss_disc_real_4: 0.21818343836527604 (-0.06279479081814104) + | > avg_loss_disc_real_5: 0.2219824194908142 (-0.020020580062499405) + | > avg_loss_0: 2.682025726024921 (+0.03509435286888696) + | > avg_loss_gen: 2.3655222012446475 (-0.20645374518174409) + | > avg_loss_kl: 3.833550288127019 (+0.22063453380878162) + | > avg_loss_feat: 7.664259030268743 (-0.3144051478459282) + | > avg_loss_mel: 17.127461653489334 (+0.2859119268564072) + | > avg_loss_duration: 1.3310406574836144 (+0.07225596904754639) + | > avg_loss_1: 32.321834124051605 (+0.0579434908353349) + + + > EPOCH: 108/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:55:25)  + + --> TIME: 2024-07-13 22:56:07 -- STEP: 41/151 -- GLOBAL_STEP: 101350 + | > loss_disc: 2.2665467262268066 (2.2681018637447816) + | > loss_disc_real_0: 0.08521329611539841 (0.114887537661849) + | > loss_disc_real_1: 0.19884221255779266 (0.20177863829019593) + | > loss_disc_real_2: 0.20088598132133484 (0.2053643021641708) + | > loss_disc_real_3: 0.19882209599018097 (0.20720636299470577) + | > loss_disc_real_4: 0.2613682448863983 (0.2030645597998689) + | > loss_disc_real_5: 0.18018440902233124 (0.21818874085821757) + | > loss_0: 2.2665467262268066 (2.2681018637447816) + | > grad_norm_0: tensor(47.5199, device='cuda:0') (tensor(34.7453, device='cuda:0')) + | > loss_gen: 2.6083691120147705 (2.6684029858286786) + | > loss_kl: 1.5541080236434937 (1.7081848179421775) + | > loss_feat: 9.439281463623047 (8.355032106725181) + | > loss_mel: 17.132219314575195 (16.73002391908227) + | > loss_duration: 1.0010695457458496 (0.9340764938331232) + | > loss_1: 31.735048294067383 (30.395720505132907) + | > grad_norm_1: tensor(654.7913, device='cuda:0') (tensor(587.6799, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.8268 (0.9558227236677961) + | > loader_time: 0.0044 (0.006013311990877477) + + + --> TIME: 2024-07-13 22:56:55 -- STEP: 91/151 -- GLOBAL_STEP: 101400 + | > loss_disc: 2.5295310020446777 (2.2844143896312503) + | > loss_disc_real_0: 0.22864079475402832 (0.11975134950090241) + | > loss_disc_real_1: 0.1968333125114441 (0.1997128621889995) + | > loss_disc_real_2: 0.20291799306869507 (0.20977262923350698) + | > loss_disc_real_3: 0.1847020983695984 (0.20318691129540353) + | > loss_disc_real_4: 0.18755294382572174 (0.20368555089929602) + | > loss_disc_real_5: 0.270683228969574 (0.21875890712816637) + | > loss_0: 2.5295310020446777 (2.2844143896312503) + | > grad_norm_0: tensor(60.3252, device='cuda:0') (tensor(33.7015, device='cuda:0')) + | > loss_gen: 2.4457931518554688 (2.6507689979050193) + | > loss_kl: 1.262447714805603 (1.7177612296827547) + | > loss_feat: 7.571998596191406 (8.31503869675018) + | > loss_mel: 16.009233474731445 (16.691144220121608) + | > loss_duration: 0.9706023931503296 (0.9373304843902588) + | > loss_1: 28.260074615478516 (30.312043808318755) + | > grad_norm_1: tensor(537.9877, device='cuda:0') (tensor(537.3367, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.8561 (0.9564056658482813) + | > loader_time: 0.0044 (0.006145377735515217) + + + --> TIME: 2024-07-13 22:57:44 -- STEP: 141/151 -- GLOBAL_STEP: 101450 + | > loss_disc: 2.528830051422119 (2.2937067544206657) + | > loss_disc_real_0: 0.1244082972407341 (0.12224104173534306) + | > loss_disc_real_1: 0.21047231554985046 (0.2012330252439418) + | > loss_disc_real_2: 0.24286217987537384 (0.21033730342033058) + | > loss_disc_real_3: 0.24714137613773346 (0.20178826496744834) + | > loss_disc_real_4: 0.2498065084218979 (0.20325500872118255) + | > loss_disc_real_5: 0.2852219343185425 (0.22143482649368598) + | > loss_0: 2.528830051422119 (2.2937067544206657) + | > grad_norm_0: tensor(30.9580, device='cuda:0') (tensor(32.8661, device='cuda:0')) + | > loss_gen: 2.3556838035583496 (2.659879523811611) + | > loss_kl: 1.9312119483947754 (1.749634751191376) + | > loss_feat: 7.068725109100342 (8.385930643014028) + | > loss_mel: 17.234981536865234 (16.88702277596114) + | > loss_duration: 1.0067499876022339 (0.9423387997539331) + | > loss_1: 29.597352981567383 (30.62480662731414) + | > grad_norm_1: tensor(580.4385, device='cuda:0') (tensor(531.3739, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 1.0533 (0.9635480600045928) + | > loader_time: 0.0065 (0.006129236086040524) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03269037833580604 (+0.0020283919114332943) + | > avg_loss_disc: 2.5512332916259766 (-0.1307924343989444) + | > avg_loss_disc_real_0: 0.22165764295137846 (-0.07275818173701948) + | > avg_loss_disc_real_1: 0.21503049593705398 (-0.026322988363412708) + | > avg_loss_disc_real_2: 0.2107886809569139 (-0.04175105003210211) + | > avg_loss_disc_real_3: 0.17036830461942232 (-0.050506644524060784) + | > avg_loss_disc_real_4: 0.19620081725028846 (-0.021982621114987583) + | > avg_loss_disc_real_5: 0.24598817412669843 (+0.024005754635884224) + | > avg_loss_0: 2.5512332916259766 (-0.1307924343989444) + | > avg_loss_gen: 2.244277000427246 (-0.12124520081740142) + | > avg_loss_kl: 3.4522785590245175 (-0.3812717291025014) + | > avg_loss_feat: 7.71818549816425 (+0.05392646789550781) + | > avg_loss_mel: 17.30198471362774 (+0.17452306013840513) + | > avg_loss_duration: 1.3505150905022254 (+0.01947443301861096) + | > avg_loss_1: 32.067240935105545 (-0.2545931889460604) + + + > EPOCH: 109/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 22:58:05)  + + --> TIME: 2024-07-13 22:58:48 -- STEP: 40/151 -- GLOBAL_STEP: 101500 + | > loss_disc: 2.356917142868042 (2.3602044820785526) + | > loss_disc_real_0: 0.1689814031124115 (0.1313011590391397) + | > loss_disc_real_1: 0.23173467814922333 (0.21316576674580573) + | > loss_disc_real_2: 0.2469901293516159 (0.2130071461200714) + | > loss_disc_real_3: 0.20513920485973358 (0.2076661389321089) + | > loss_disc_real_4: 0.2554691433906555 (0.2060709957033396) + | > loss_disc_real_5: 0.31276947259902954 (0.21687549985945226) + | > loss_0: 2.356917142868042 (2.3602044820785526) + | > grad_norm_0: tensor(45.1291, device='cuda:0') (tensor(32.2137, device='cuda:0')) + | > loss_gen: 2.8671982288360596 (2.5754078805446623) + | > loss_kl: 1.4338945150375366 (1.7426426202058791) + | > loss_feat: 7.523021697998047 (7.975537872314453) + | > loss_mel: 17.83349609375 (16.968701028823855) + | > loss_duration: 0.8427697420120239 (0.9351781085133553) + | > loss_1: 30.500381469726562 (30.197467517852782) + | > grad_norm_1: tensor(571.2081, device='cuda:0') (tensor(491.7828, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.9141 (0.9655889093875885) + | > loader_time: 0.0056 (0.00605541467666626) + + + --> TIME: 2024-07-13 22:59:37 -- STEP: 90/151 -- GLOBAL_STEP: 101550 + | > loss_disc: 2.459484577178955 (2.3406357235378685) + | > loss_disc_real_0: 0.1661970019340515 (0.13208744393454655) + | > loss_disc_real_1: 0.16875497996807098 (0.20770947453048494) + | > loss_disc_real_2: 0.18747547268867493 (0.21349648435910543) + | > loss_disc_real_3: 0.1881333887577057 (0.20555951048930485) + | > loss_disc_real_4: 0.20663292706012726 (0.20230754034386741) + | > loss_disc_real_5: 0.24638845026493073 (0.220677469836341) + | > loss_0: 2.459484577178955 (2.3406357235378685) + | > grad_norm_0: tensor(13.1556, device='cuda:0') (tensor(31.5382, device='cuda:0')) + | > loss_gen: 2.245748519897461 (2.5934749550289573) + | > loss_kl: 1.9009826183319092 (1.73983506626553) + | > loss_feat: 6.328588485717773 (8.125350258085454) + | > loss_mel: 14.901103019714355 (16.92981939315797) + | > loss_duration: 0.8751000761985779 (0.9335806118117438) + | > loss_1: 26.251522064208984 (30.322060330708823) + | > grad_norm_1: tensor(106.9501, device='cuda:0') (tensor(437.3780, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.8927 (0.9660741567611695) + | > loader_time: 0.0066 (0.006250021192762587) + + + --> TIME: 2024-07-13 23:00:24 -- STEP: 140/151 -- GLOBAL_STEP: 101600 + | > loss_disc: 2.083799362182617 (2.322521821090154) + | > loss_disc_real_0: 0.10491203516721725 (0.12908419038036043) + | > loss_disc_real_1: 0.18157124519348145 (0.20414219796657565) + | > loss_disc_real_2: 0.22149553894996643 (0.21122363944138803) + | > loss_disc_real_3: 0.17992427945137024 (0.2049232461622783) + | > loss_disc_real_4: 0.18994587659835815 (0.2026658889438425) + | > loss_disc_real_5: 0.17249737679958344 (0.22196887039712498) + | > loss_0: 2.083799362182617 (2.322521821090154) + | > grad_norm_0: tensor(31.8492, device='cuda:0') (tensor(30.4809, device='cuda:0')) + | > loss_gen: 3.039830207824707 (2.6089026365961345) + | > loss_kl: 1.5140701532363892 (1.7424864428383964) + | > loss_feat: 9.439413070678711 (8.294241080965309) + | > loss_mel: 17.667638778686523 (16.90056770188469) + | > loss_duration: 0.9936536550521851 (0.9299355208873749) + | > loss_1: 32.654605865478516 (30.476133469172886) + | > grad_norm_1: tensor(729.0353, device='cuda:0') (tensor(444.1690, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.8153 (0.9534766622952052) + | > loader_time: 0.0038 (0.006045923914228167) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.035424452561598554 (+0.0027340742257925127) + | > avg_loss_disc: 2.553968833043025 (+0.0027355414170484593) + | > avg_loss_disc_real_0: 0.22749362656703362 (+0.005835983615655155) + | > avg_loss_disc_real_1: 0.21252739200225243 (-0.0025031039348015482) + | > avg_loss_disc_real_2: 0.2335442705796315 (+0.02275558962271762) + | > avg_loss_disc_real_3: 0.1900053723500325 (+0.01963706773061019) + | > avg_loss_disc_real_4: 0.24111050367355347 (+0.04490968642326501) + | > avg_loss_disc_real_5: 0.21775769614256346 (-0.028230477984134972) + | > avg_loss_0: 2.553968833043025 (+0.0027355414170484593) + | > avg_loss_gen: 2.3853962971613956 (+0.14111929673414947) + | > avg_loss_kl: 3.578715471120981 (+0.12643691209646368) + | > avg_loss_feat: 7.652155839479887 (-0.06602965868436339) + | > avg_loss_mel: 16.53058198782114 (-0.771402725806599) + | > avg_loss_duration: 1.2883551029058604 (-0.062159987596364985) + | > avg_loss_1: 31.435205166156475 (-0.6320357689490699) + + + > EPOCH: 110/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:00:46)  + + --> TIME: 2024-07-13 23:01:25 -- STEP: 39/151 -- GLOBAL_STEP: 101650 + | > loss_disc: 2.1385154724121094 (2.262514468951103) + | > loss_disc_real_0: 0.09312210977077484 (0.11267477866166677) + | > loss_disc_real_1: 0.2043134719133377 (0.19943571396363088) + | > loss_disc_real_2: 0.18237851560115814 (0.20916171639393538) + | > loss_disc_real_3: 0.14848773181438446 (0.1986832674115132) + | > loss_disc_real_4: 0.17882487177848816 (0.19988125333419213) + | > loss_disc_real_5: 0.1636703759431839 (0.21648084123929343) + | > loss_0: 2.1385154724121094 (2.262514468951103) + | > grad_norm_0: tensor(18.4349, device='cuda:0') (tensor(29.5766, device='cuda:0')) + | > loss_gen: 2.832819938659668 (2.6754913452344065) + | > loss_kl: 1.0624899864196777 (1.681853297429207) + | > loss_feat: 10.005016326904297 (8.467522095411251) + | > loss_mel: 17.1204833984375 (16.861581508929913) + | > loss_duration: 0.9698971509933472 (0.9451710306681119) + | > loss_1: 31.990705490112305 (30.631619233351486) + | > grad_norm_1: tensor(708.7604, device='cuda:0') (tensor(548.8881, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.8651 (0.9334410215035464) + | > loader_time: 0.0049 (0.0059759188921023635) + + + --> TIME: 2024-07-13 23:02:17 -- STEP: 89/151 -- GLOBAL_STEP: 101700 + | > loss_disc: 2.501012086868286 (2.319547872864798) + | > loss_disc_real_0: 0.17542223632335663 (0.13118298410364754) + | > loss_disc_real_1: 0.23935943841934204 (0.20215658190545072) + | > loss_disc_real_2: 0.26511693000793457 (0.21047709865516492) + | > loss_disc_real_3: 0.2532424330711365 (0.2045259749454059) + | > loss_disc_real_4: 0.23217053711414337 (0.20850814292939862) + | > loss_disc_real_5: 0.2293611466884613 (0.22057117186905292) + | > loss_0: 2.501012086868286 (2.319547872864798) + | > grad_norm_0: tensor(27.5050, device='cuda:0') (tensor(25.0112, device='cuda:0')) + | > loss_gen: 2.362839460372925 (2.5929157653551425) + | > loss_kl: 1.9875154495239258 (1.7315661237480935) + | > loss_feat: 6.689572811126709 (8.137796439481585) + | > loss_mel: 16.940393447875977 (16.669247155778862) + | > loss_duration: 0.8518968820571899 (0.945483995957321) + | > loss_1: 28.832216262817383 (30.07700941535864) + | > grad_norm_1: tensor(180.6635, device='cuda:0') (tensor(378.5384, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 1.0243 (0.9876181382811471) + | > loader_time: 0.0067 (0.006430917911315234) + + + --> TIME: 2024-07-13 23:03:04 -- STEP: 139/151 -- GLOBAL_STEP: 101750 + | > loss_disc: 2.1344640254974365 (2.314579777580371) + | > loss_disc_real_0: 0.09034576267004013 (0.1271325918249518) + | > loss_disc_real_1: 0.20269177854061127 (0.2018962657923321) + | > loss_disc_real_2: 0.19184327125549316 (0.21084120794594716) + | > loss_disc_real_3: 0.2075204998254776 (0.20362307924589662) + | > loss_disc_real_4: 0.19157159328460693 (0.2083720873585708) + | > loss_disc_real_5: 0.15387959778308868 (0.22023934919199498) + | > loss_0: 2.1344640254974365 (2.314579777580371) + | > grad_norm_0: tensor(37.3726, device='cuda:0') (tensor(25.2413, device='cuda:0')) + | > loss_gen: 2.8749141693115234 (2.593911572326002) + | > loss_kl: 1.8412740230560303 (1.7221262545036755) + | > loss_feat: 10.410524368286133 (8.20834408561103) + | > loss_mel: 17.26759147644043 (16.790369486637253) + | > loss_duration: 0.9206485152244568 (0.936270960372129) + | > loss_1: 33.31494903564453 (30.251022270257526) + | > grad_norm_1: tensor(694.5694, device='cuda:0') (tensor(401.9846, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.8513 (0.9677999945853254) + | > loader_time: 0.004 (0.0061805677070892115) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03351149192223182 (-0.0019129606393667314) + | > avg_loss_disc: 2.5419708031874437 (-0.011998029855581294) + | > avg_loss_disc_real_0: 0.24747863641152015 (+0.019985009844486534) + | > avg_loss_disc_real_1: 0.22681314899371222 (+0.014285756991459786) + | > avg_loss_disc_real_2: 0.22116805956913874 (-0.012376211010492771) + | > avg_loss_disc_real_3: 0.20697777775617746 (+0.016972405406144947) + | > avg_loss_disc_real_4: 0.23869106632012588 (-0.0024194373534275893) + | > avg_loss_disc_real_5: 0.21529780328273773 (-0.0024598928598257297) + | > avg_loss_0: 2.5419708031874437 (-0.011998029855581294) + | > avg_loss_gen: 2.399906195127047 (+0.014509897965651497) + | > avg_loss_kl: 3.7243934778066783 (+0.14567800668569708) + | > avg_loss_feat: 7.793069069202129 (+0.1409132297222424) + | > avg_loss_mel: 17.014140422527603 (+0.4835584347064632) + | > avg_loss_duration: 1.294137166096614 (+0.0057820631907534725) + | > avg_loss_1: 32.225645798903244 (+0.790440632746769) + + + > EPOCH: 111/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:03:27)  + + --> TIME: 2024-07-13 23:04:08 -- STEP: 38/151 -- GLOBAL_STEP: 101800 + | > loss_disc: 2.360361099243164 (2.3154756520923816) + | > loss_disc_real_0: 0.1370118260383606 (0.13526150545007304) + | > loss_disc_real_1: 0.2116793543100357 (0.20319878309965134) + | > loss_disc_real_2: 0.20231060683727264 (0.2114836852017202) + | > loss_disc_real_3: 0.22368048131465912 (0.20696461122286947) + | > loss_disc_real_4: 0.22018401324748993 (0.20761330347312124) + | > loss_disc_real_5: 0.32082685828208923 (0.22064217023159327) + | > loss_0: 2.360361099243164 (2.3154756520923816) + | > grad_norm_0: tensor(65.8366, device='cuda:0') (tensor(38.8914, device='cuda:0')) + | > loss_gen: 2.7556135654449463 (2.696670883580258) + | > loss_kl: 1.5502276420593262 (1.8137296218621104) + | > loss_feat: 8.493342399597168 (8.336023757332253) + | > loss_mel: 18.110645294189453 (16.885703839753802) + | > loss_duration: 0.9264872670173645 (0.9435614786650005) + | > loss_1: 31.836315155029297 (30.675689396105316) + | > grad_norm_1: tensor(756.6602, device='cuda:0') (tensor(482.0369, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.8579 (1.001256133380689) + | > loader_time: 0.0045 (0.00677863547676488) + + + --> TIME: 2024-07-13 23:04:57 -- STEP: 88/151 -- GLOBAL_STEP: 101850 + | > loss_disc: 2.459094524383545 (2.323533911596645) + | > loss_disc_real_0: 0.23559415340423584 (0.13644512814723628) + | > loss_disc_real_1: 0.2520427703857422 (0.2035895757038485) + | > loss_disc_real_2: 0.20738835632801056 (0.21170637485655872) + | > loss_disc_real_3: 0.2055211216211319 (0.2089250118217685) + | > loss_disc_real_4: 0.19034169614315033 (0.20716700990768988) + | > loss_disc_real_5: 0.20079511404037476 (0.21454382772472771) + | > loss_0: 2.459094524383545 (2.323533911596645) + | > grad_norm_0: tensor(36.4613, device='cuda:0') (tensor(34.5732, device='cuda:0')) + | > loss_gen: 2.4940786361694336 (2.6693086136471145) + | > loss_kl: 1.9787378311157227 (1.8119813725352287) + | > loss_feat: 7.085428714752197 (8.308808716860689) + | > loss_mel: 16.420513153076172 (16.938480257987973) + | > loss_duration: 0.9237323999404907 (0.9356795061718334) + | > loss_1: 28.90249252319336 (30.664258350025523) + | > grad_norm_1: tensor(370.3174, device='cuda:0') (tensor(486.4101, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.9158 (0.9811161891980604) + | > loader_time: 0.0066 (0.006501138210296631) + + + --> TIME: 2024-07-13 23:05:43 -- STEP: 138/151 -- GLOBAL_STEP: 101900 + | > loss_disc: 2.1414709091186523 (2.3094144662221283) + | > loss_disc_real_0: 0.08674304932355881 (0.1277453470435263) + | > loss_disc_real_1: 0.15353111922740936 (0.20270260812147803) + | > loss_disc_real_2: 0.19294030964374542 (0.2103655238946279) + | > loss_disc_real_3: 0.202492818236351 (0.20766866671434348) + | > loss_disc_real_4: 0.17049036920070648 (0.2066272610555524) + | > loss_disc_real_5: 0.22564798593521118 (0.2170887709311817) + | > loss_0: 2.1414709091186523 (2.3094144662221283) + | > grad_norm_0: tensor(41.1114, device='cuda:0') (tensor(31.9159, device='cuda:0')) + | > loss_gen: 2.6605405807495117 (2.6462266980737876) + | > loss_kl: 1.0003823041915894 (1.7809545916059744) + | > loss_feat: 7.65712308883667 (8.349829272947455) + | > loss_mel: 15.442814826965332 (16.92438972860143) + | > loss_duration: 1.1342886686325073 (0.9317876368329145) + | > loss_1: 27.89514923095703 (30.633187874503758) + | > grad_norm_1: tensor(452.2191, device='cuda:0') (tensor(479.0497, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.8101 (0.9605125724405482) + | > loader_time: 0.0035 (0.006236199019611746) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033427770321185775 (-8.372160104604814e-05) + | > avg_loss_disc: 2.6472648840684156 (+0.10529408088097192) + | > avg_loss_disc_real_0: 0.2277989719922726 (-0.019679664419247567) + | > avg_loss_disc_real_1: 0.2092645844587913 (-0.01754856453492093) + | > avg_loss_disc_real_2: 0.24962240572159106 (+0.02845434615245232) + | > avg_loss_disc_real_3: 0.2885844466777948 (+0.08160666892161736) + | > avg_loss_disc_real_4: 0.23904449091507837 (+0.000353424594952495) + | > avg_loss_disc_real_5: 0.2541324140933844 (+0.03883461081064665) + | > avg_loss_0: 2.6472648840684156 (+0.10529408088097192) + | > avg_loss_gen: 2.3883548883291392 (-0.011551306797907834) + | > avg_loss_kl: 3.618114086297842 (-0.10627939150883625) + | > avg_loss_feat: 7.722173397357647 (-0.07089567184448242) + | > avg_loss_mel: 16.96115024273212 (-0.05299017979548282) + | > avg_loss_duration: 1.2934521436691284 (-0.000685022427485471) + | > avg_loss_1: 31.983244822575497 (-0.24240097632774749) + + + > EPOCH: 112/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:06:07)  + + --> TIME: 2024-07-13 23:06:48 -- STEP: 37/151 -- GLOBAL_STEP: 101950 + | > loss_disc: 2.368906021118164 (2.3046156747921094) + | > loss_disc_real_0: 0.09249624609947205 (0.12216525444307842) + | > loss_disc_real_1: 0.2160368263721466 (0.19817508716841) + | > loss_disc_real_2: 0.17698591947555542 (0.20838334431519379) + | > loss_disc_real_3: 0.22265523672103882 (0.2057950216773394) + | > loss_disc_real_4: 0.2836226522922516 (0.20657291766759514) + | > loss_disc_real_5: 0.2441951334476471 (0.2239048146718257) + | > loss_0: 2.368906021118164 (2.3046156747921094) + | > grad_norm_0: tensor(19.0513, device='cuda:0') (tensor(21.9350, device='cuda:0')) + | > loss_gen: 2.54518985748291 (2.5862341120436385) + | > loss_kl: 2.0448782444000244 (1.7298681478242617) + | > loss_feat: 7.720114231109619 (8.376836866945832) + | > loss_mel: 17.466062545776367 (16.80101562190701) + | > loss_duration: 0.8877773284912109 (0.9489999332943478) + | > loss_1: 30.66402244567871 (30.44295491398992) + | > grad_norm_1: tensor(431.8864, device='cuda:0') (tensor(353.5315, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.9389 (1.0467459253362708) + | > loader_time: 0.0058 (0.006437172760834565) + + + --> TIME: 2024-07-13 23:07:37 -- STEP: 87/151 -- GLOBAL_STEP: 102000 + | > loss_disc: 2.514975070953369 (2.2933128387078483) + | > loss_disc_real_0: 0.16071034967899323 (0.11884469907174165) + | > loss_disc_real_1: 0.2135835736989975 (0.203057852627217) + | > loss_disc_real_2: 0.2179626226425171 (0.20756232498706073) + | > loss_disc_real_3: 0.2519662082195282 (0.20337282420917488) + | > loss_disc_real_4: 0.2559802234172821 (0.2067193292166995) + | > loss_disc_real_5: 0.23917356133460999 (0.22445788842508163) + | > loss_0: 2.514975070953369 (2.2933128387078483) + | > grad_norm_0: tensor(49.9652, device='cuda:0') (tensor(29.2429, device='cuda:0')) + | > loss_gen: 2.256269693374634 (2.658567546427936) + | > loss_kl: 1.8666331768035889 (1.7297999091532041) + | > loss_feat: 6.924269199371338 (8.573477734094379) + | > loss_mel: 15.251348495483398 (17.017400522341667) + | > loss_duration: 0.9601346254348755 (0.9420163905483553) + | > loss_1: 27.258655548095703 (30.921262149153083) + | > grad_norm_1: tensor(255.9047, device='cuda:0') (tensor(486.5314, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 1.0734 (0.996629569722318) + | > loader_time: 0.0069 (0.006293970963050579) + + + --> TIME: 2024-07-13 23:08:50 -- STEP: 137/151 -- GLOBAL_STEP: 102050 + | > loss_disc: 2.362874746322632 (2.2967721128115706) + | > loss_disc_real_0: 0.10110195726156235 (0.1173868201618647) + | > loss_disc_real_1: 0.21918968856334686 (0.20242259374065116) + | > loss_disc_real_2: 0.1665947437286377 (0.2089145927533616) + | > loss_disc_real_3: 0.1566799432039261 (0.20259831046318486) + | > loss_disc_real_4: 0.210209459066391 (0.20680655968667816) + | > loss_disc_real_5: 0.273359477519989 (0.22355315720077848) + | > loss_0: 2.362874746322632 (2.2967721128115706) + | > grad_norm_0: tensor(16.0485, device='cuda:0') (tensor(28.8094, device='cuda:0')) + | > loss_gen: 2.528453826904297 (2.636346973641946) + | > loss_kl: 1.0218758583068848 (1.7228508382818126) + | > loss_feat: 8.725083351135254 (8.423081408451948) + | > loss_mel: 16.213790893554688 (16.895627488185013) + | > loss_duration: 1.0080662965774536 (0.9374003057932332) + | > loss_1: 29.497270584106445 (30.615306896014804) + | > grad_norm_1: tensor(352.0211, device='cuda:0') (tensor(476.6469, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.8127 (0.973534958205954) + | > loader_time: 0.0041 (0.006030086183199917) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034074875024648815 (+0.0006471047034630403) + | > avg_loss_disc: 2.578468469473032 (-0.06879641459538366) + | > avg_loss_disc_real_0: 0.2818828179286077 (+0.05408384593633511) + | > avg_loss_disc_real_1: 0.2599809513642238 (+0.05071636690543249) + | > avg_loss_disc_real_2: 0.2680639326572418 (+0.018441526935650765) + | > avg_loss_disc_real_3: 0.2507028946509728 (-0.03788155202682203) + | > avg_loss_disc_real_4: 0.27171912216223204 (+0.03267463124715367) + | > avg_loss_disc_real_5: 0.23453237460209772 (-0.019600039491286664) + | > avg_loss_0: 2.578468469473032 (-0.06879641459538366) + | > avg_loss_gen: 2.668151231912466 (+0.27979634358332683) + | > avg_loss_kl: 3.540529856315026 (-0.07758422998281622) + | > avg_loss_feat: 7.787171143751878 (+0.06499774639423084) + | > avg_loss_mel: 17.069334763746994 (+0.10818452101487352) + | > avg_loss_duration: 1.279029442713811 (-0.014422700955317502) + | > avg_loss_1: 32.34421627338116 (+0.3609714508056605) + + + > EPOCH: 113/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:09:16)  + + --> TIME: 2024-07-13 23:09:55 -- STEP: 36/151 -- GLOBAL_STEP: 102100 + | > loss_disc: 2.216057300567627 (2.302310122383966) + | > loss_disc_real_0: 0.09840625524520874 (0.12277648815264304) + | > loss_disc_real_1: 0.20491035282611847 (0.20623035211529997) + | > loss_disc_real_2: 0.2047613114118576 (0.21429014474981362) + | > loss_disc_real_3: 0.1802942156791687 (0.20585175355275473) + | > loss_disc_real_4: 0.22001485526561737 (0.20706204055911964) + | > loss_disc_real_5: 0.24970905482769012 (0.21820178255438807) + | > loss_0: 2.216057300567627 (2.302310122383966) + | > grad_norm_0: tensor(29.4252, device='cuda:0') (tensor(25.9149, device='cuda:0')) + | > loss_gen: 2.551560401916504 (2.6091159184773765) + | > loss_kl: 1.3761097192764282 (1.6819351580407884) + | > loss_feat: 7.944499969482422 (8.464989688661364) + | > loss_mel: 16.509870529174805 (16.798022111256913) + | > loss_duration: 0.8291858434677124 (0.9443292915821075) + | > loss_1: 29.211227416992188 (30.49839194615682) + | > grad_norm_1: tensor(483.2342, device='cuda:0') (tensor(412.8368, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.8674 (1.0131725403997633) + | > loader_time: 0.0052 (0.006423526340060764) + + + --> TIME: 2024-07-13 23:10:45 -- STEP: 86/151 -- GLOBAL_STEP: 102150 + | > loss_disc: 2.600374460220337 (2.3057923663494213) + | > loss_disc_real_0: 0.10351824760437012 (0.12264935035518436) + | > loss_disc_real_1: 0.21694400906562805 (0.20396566157077634) + | > loss_disc_real_2: 0.2172183096408844 (0.21091629573425585) + | > loss_disc_real_3: 0.19538618624210358 (0.20585749592891958) + | > loss_disc_real_4: 0.25793886184692383 (0.20699433713804843) + | > loss_disc_real_5: 0.24561186134815216 (0.21978821865347928) + | > loss_0: 2.600374460220337 (2.3057923663494213) + | > grad_norm_0: tensor(98.2441, device='cuda:0') (tensor(31.9272, device='cuda:0')) + | > loss_gen: 2.1409659385681152 (2.629623967547748) + | > loss_kl: 2.129429817199707 (1.745036804398825) + | > loss_feat: 5.460824489593506 (8.386574229528737) + | > loss_mel: 12.668210983276367 (16.8382838271385) + | > loss_duration: 1.192350149154663 (0.9383664560872454) + | > loss_1: 23.591781616210938 (30.53788517796716) + | > grad_norm_1: tensor(370.6565, device='cuda:0') (tensor(491.2001, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 2.1308 (0.9912883680920267) + | > loader_time: 0.0183 (0.0063605585763620775) + + + --> TIME: 2024-07-13 23:11:32 -- STEP: 136/151 -- GLOBAL_STEP: 102200 + | > loss_disc: 2.282254457473755 (2.30217343568802) + | > loss_disc_real_0: 0.08723454922437668 (0.12538757361471653) + | > loss_disc_real_1: 0.23701421916484833 (0.2026456193998456) + | > loss_disc_real_2: 0.15677466988563538 (0.20907781333388653) + | > loss_disc_real_3: 0.2603738605976105 (0.20712549042175799) + | > loss_disc_real_4: 0.2601994276046753 (0.20765810991253922) + | > loss_disc_real_5: 0.2183062881231308 (0.2174050996408743) + | > loss_0: 2.282254457473755 (2.30217343568802) + | > grad_norm_0: tensor(15.9753, device='cuda:0') (tensor(32.3667, device='cuda:0')) + | > loss_gen: 2.905050754547119 (2.6519270521752976) + | > loss_kl: 1.336399793624878 (1.7265381646506928) + | > loss_feat: 10.08696174621582 (8.471275182331313) + | > loss_mel: 18.377342224121094 (16.933292213608244) + | > loss_duration: 0.8645690679550171 (0.9329735895290094) + | > loss_1: 33.57032775878906 (30.716006124720852) + | > grad_norm_1: tensor(219.5200, device='cuda:0') (tensor(484.6629, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.8597 (0.970658053370083) + | > loader_time: 0.0041 (0.006149675916222965) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031505584716796875 (-0.0025692903078519402) + | > avg_loss_disc: 2.597051638823289 (+0.018583169350256945) + | > avg_loss_disc_real_0: 0.2675701299538979 (-0.014312687974709803) + | > avg_loss_disc_real_1: 0.2219686542566006 (-0.03801229710762319) + | > avg_loss_disc_real_2: 0.23346524628309104 (-0.03459868637415078) + | > avg_loss_disc_real_3: 0.22865016299944657 (-0.022052731651526214) + | > avg_loss_disc_real_4: 0.22441661701752588 (-0.04730250514470616) + | > avg_loss_disc_real_5: 0.20955663117078635 (-0.02497574343131137) + | > avg_loss_0: 2.597051638823289 (+0.018583169350256945) + | > avg_loss_gen: 2.4349455650036154 (-0.2332056669088507) + | > avg_loss_kl: 3.7688768276801476 (+0.22834697136512183) + | > avg_loss_feat: 8.271954463078423 (+0.48478331932654495) + | > avg_loss_mel: 16.91494839008038 (-0.15438637366661467) + | > avg_loss_duration: 1.3273875163151667 (+0.0483580736013558) + | > avg_loss_1: 32.718113532433144 (+0.37389725905198645) + + + > EPOCH: 114/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:11:56)  + + --> TIME: 2024-07-13 23:12:37 -- STEP: 35/151 -- GLOBAL_STEP: 102250 + | > loss_disc: 2.2809269428253174 (2.337473658152989) + | > loss_disc_real_0: 0.12297167629003525 (0.13387053651469089) + | > loss_disc_real_1: 0.20623639225959778 (0.20372740924358368) + | > loss_disc_real_2: 0.23561470210552216 (0.20811110053743634) + | > loss_disc_real_3: 0.17615176737308502 (0.21571902079241617) + | > loss_disc_real_4: 0.23348727822303772 (0.20374855399131775) + | > loss_disc_real_5: 0.2320203334093094 (0.2151973043169294) + | > loss_0: 2.2809269428253174 (2.337473658152989) + | > grad_norm_0: tensor(15.7542, device='cuda:0') (tensor(20.5339, device='cuda:0')) + | > loss_gen: 2.5135579109191895 (2.5426700728280203) + | > loss_kl: 1.1478865146636963 (1.7854726552963256) + | > loss_feat: 8.102736473083496 (8.041210188184465) + | > loss_mel: 16.709638595581055 (16.836859921046667) + | > loss_duration: 1.0138020515441895 (0.9429768255778721) + | > loss_1: 29.487621307373047 (30.14918943132673) + | > grad_norm_1: tensor(413.4767, device='cuda:0') (tensor(364.8706, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.8283 (1.1049771785736084) + | > loader_time: 0.0046 (0.0073340415954589845) + + + --> TIME: 2024-07-13 23:13:23 -- STEP: 85/151 -- GLOBAL_STEP: 102300 + | > loss_disc: 2.1704540252685547 (2.297911366294412) + | > loss_disc_real_0: 0.09972082078456879 (0.1256536131834281) + | > loss_disc_real_1: 0.19778290390968323 (0.2007985176408993) + | > loss_disc_real_2: 0.26249754428863525 (0.20774037715266733) + | > loss_disc_real_3: 0.20054592192173004 (0.2088203309213414) + | > loss_disc_real_4: 0.20774976909160614 (0.20166038537726683) + | > loss_disc_real_5: 0.21250535547733307 (0.2166354544022504) + | > loss_0: 2.1704540252685547 (2.297911366294412) + | > grad_norm_0: tensor(9.6694, device='cuda:0') (tensor(19.2421, device='cuda:0')) + | > loss_gen: 2.9207942485809326 (2.603671808803784) + | > loss_kl: 1.908506155014038 (1.7351121762219597) + | > loss_feat: 9.482815742492676 (8.37794562508078) + | > loss_mel: 16.863927841186523 (16.93145387312945) + | > loss_duration: 0.9732799530029297 (0.9308604352614459) + | > loss_1: 32.149322509765625 (30.57904380349552) + | > grad_norm_1: tensor(280.6735, device='cuda:0') (tensor(304.5396, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.8615 (0.9910706211538874) + | > loader_time: 0.0057 (0.006567884893978343) + + + --> TIME: 2024-07-13 23:14:14 -- STEP: 135/151 -- GLOBAL_STEP: 102350 + | > loss_disc: 2.4189984798431396 (2.2994485254640926) + | > loss_disc_real_0: 0.10929806530475616 (0.12660555958195963) + | > loss_disc_real_1: 0.26623132824897766 (0.20121343566311736) + | > loss_disc_real_2: 0.18637588620185852 (0.210089835634938) + | > loss_disc_real_3: 0.1921621412038803 (0.20718623289355526) + | > loss_disc_real_4: 0.17649723589420319 (0.20178956378389287) + | > loss_disc_real_5: 0.2099677324295044 (0.2159915798240238) + | > loss_0: 2.4189984798431396 (2.2994485254640926) + | > grad_norm_0: tensor(35.3737, device='cuda:0') (tensor(24.0870, device='cuda:0')) + | > loss_gen: 2.7283430099487305 (2.6058380427183936) + | > loss_kl: 1.642223834991455 (1.7280438387835468) + | > loss_feat: 10.097403526306152 (8.272745573962174) + | > loss_mel: 17.66969108581543 (16.88055704611319) + | > loss_duration: 0.8521367907524109 (0.9318903494764257) + | > loss_1: 32.98979949951172 (30.419074701379845) + | > grad_norm_1: tensor(613.3614, device='cuda:0') (tensor(381.0178, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.8796 (0.9946705094090212) + | > loader_time: 0.0054 (0.006600756115383572) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03479313850402831 (+0.0032875537872314384) + | > avg_loss_disc: 2.5546452082120457 (-0.04240643061124327) + | > avg_loss_disc_real_0: 0.2530666784598277 (-0.014503451494070174) + | > avg_loss_disc_real_1: 0.18267865708241096 (-0.03928999717418963) + | > avg_loss_disc_real_2: 0.200846227315756 (-0.032619018967335045) + | > avg_loss_disc_real_3: 0.16505856812000272 (-0.06359159487944385) + | > avg_loss_disc_real_4: 0.21229961514472961 (-0.012117001872796268) + | > avg_loss_disc_real_5: 0.20264609845784995 (-0.006910532712936401) + | > avg_loss_0: 2.5546452082120457 (-0.04240643061124327) + | > avg_loss_gen: 2.222985084240253 (-0.21196048076336238) + | > avg_loss_kl: 3.7139983360583964 (-0.05487849162175129) + | > avg_loss_feat: 8.460370834057148 (+0.1884163709787252) + | > avg_loss_mel: 17.21074074965257 (+0.2957923595721894) + | > avg_loss_duration: 1.3478417763343225 (+0.020454260019155734) + | > avg_loss_1: 32.955936431884766 (+0.23782289945162205) + + + > EPOCH: 115/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:14:40)  + + --> TIME: 2024-07-13 23:15:18 -- STEP: 34/151 -- GLOBAL_STEP: 102400 + | > loss_disc: 2.406843423843384 (2.3002781447242286) + | > loss_disc_real_0: 0.2019277811050415 (0.13078076282844825) + | > loss_disc_real_1: 0.20303060114383698 (0.2010371636818437) + | > loss_disc_real_2: 0.21649688482284546 (0.20878633812946432) + | > loss_disc_real_3: 0.2542315423488617 (0.20808164468582938) + | > loss_disc_real_4: 0.21634088456630707 (0.20602189837133184) + | > loss_disc_real_5: 0.25794070959091187 (0.21247373258366303) + | > loss_0: 2.406843423843384 (2.3002781447242286) + | > grad_norm_0: tensor(59.4751, device='cuda:0') (tensor(39.6407, device='cuda:0')) + | > loss_gen: 2.608423948287964 (2.7026842622195972) + | > loss_kl: 1.9226711988449097 (1.7674429872456718) + | > loss_feat: 7.854990005493164 (8.348842312307921) + | > loss_mel: 17.453250885009766 (17.157225216136258) + | > loss_duration: 0.882407546043396 (0.9357213903875912) + | > loss_1: 30.721742630004883 (30.911915947409238) + | > grad_norm_1: tensor(707.2849, device='cuda:0') (tensor(639.3617, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.9741 (1.0231944953694059) + | > loader_time: 0.0065 (0.00681459202485926) + + + --> TIME: 2024-07-13 23:16:08 -- STEP: 84/151 -- GLOBAL_STEP: 102450 + | > loss_disc: 2.3173162937164307 (2.288717163460594) + | > loss_disc_real_0: 0.08421261608600616 (0.1245640761085919) + | > loss_disc_real_1: 0.16576257348060608 (0.20022633849155327) + | > loss_disc_real_2: 0.2016914039850235 (0.21024049304070927) + | > loss_disc_real_3: 0.2313140630722046 (0.2048769566629614) + | > loss_disc_real_4: 0.21322675049304962 (0.20403374465448512) + | > loss_disc_real_5: 0.2453838437795639 (0.2159231835532756) + | > loss_0: 2.3173162937164307 (2.288717163460594) + | > grad_norm_0: tensor(13.1697, device='cuda:0') (tensor(36.4238, device='cuda:0')) + | > loss_gen: 2.5502707958221436 (2.6892820625078113) + | > loss_kl: 1.6471481323242188 (1.7318945697375707) + | > loss_feat: 9.777475357055664 (8.373307795751664) + | > loss_mel: 17.797271728515625 (17.034017767224988) + | > loss_duration: 0.9160486459732056 (0.9386242024955296) + | > loss_1: 32.68821334838867 (30.767126492091588) + | > grad_norm_1: tensor(285.6661, device='cuda:0') (tensor(582.4500, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.8823 (1.005202154318491) + | > loader_time: 0.0086 (0.006758740970066616) + + + --> TIME: 2024-07-13 23:16:57 -- STEP: 134/151 -- GLOBAL_STEP: 102500 + | > loss_disc: 2.300568103790283 (2.2930846294360365) + | > loss_disc_real_0: 0.08793836832046509 (0.12308624392347549) + | > loss_disc_real_1: 0.15069247782230377 (0.20154288188735045) + | > loss_disc_real_2: 0.21141377091407776 (0.21077767792922347) + | > loss_disc_real_3: 0.21651466190814972 (0.20349999984253697) + | > loss_disc_real_4: 0.19357384741306305 (0.20500535067560063) + | > loss_disc_real_5: 0.2424078732728958 (0.2166627446439729) + | > loss_0: 2.300568103790283 (2.2930846294360365) + | > grad_norm_0: tensor(26.9633, device='cuda:0') (tensor(31.4828, device='cuda:0')) + | > loss_gen: 2.5474395751953125 (2.652785601900585) + | > loss_kl: 1.8048802614212036 (1.7424682948126722) + | > loss_feat: 8.038267135620117 (8.331318609750099) + | > loss_mel: 16.547523498535156 (16.88610968660952) + | > loss_duration: 0.8977134227752686 (0.944023327596152) + | > loss_1: 29.83582305908203 (30.556705617192964) + | > grad_norm_1: tensor(342.5298, device='cuda:0') (tensor(518.5579, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.9274 (0.9909706418193984) + | > loader_time: 0.0074 (0.006578157197183638) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032837280860314004 (-0.0019558576437143094) + | > avg_loss_disc: 2.5736374121445875 (+0.018992203932541862) + | > avg_loss_disc_real_0: 0.2237266439657945 (-0.029340034494033218) + | > avg_loss_disc_real_1: 0.21590703840439135 (+0.03322838132198039) + | > avg_loss_disc_real_2: 0.2202166227193979 (+0.01937039540364191) + | > avg_loss_disc_real_3: 0.20977663076840913 (+0.044718062648406415) + | > avg_loss_disc_real_4: 0.20867753945864165 (-0.0036220756860879666) + | > avg_loss_disc_real_5: 0.22942098172811362 (+0.026774883270263672) + | > avg_loss_0: 2.5736374121445875 (+0.018992203932541862) + | > avg_loss_gen: 2.2675544298612156 (+0.0445693456209626) + | > avg_loss_kl: 3.7810614109039307 (+0.06706307484553431) + | > avg_loss_feat: 7.991943139296311 (-0.46842769476083657) + | > avg_loss_mel: 16.933703715984638 (-0.27703703366793064) + | > avg_loss_duration: 1.3097422122955322 (-0.03809956403879022) + | > avg_loss_1: 32.28400523845966 (-0.6719311934251024) + + + > EPOCH: 116/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:17:23)  + + --> TIME: 2024-07-13 23:18:01 -- STEP: 33/151 -- GLOBAL_STEP: 102550 + | > loss_disc: 2.571500301361084 (2.2909041354150483) + | > loss_disc_real_0: 0.24927930533885956 (0.12307805477669745) + | > loss_disc_real_1: 0.2289735972881317 (0.20467390842509992) + | > loss_disc_real_2: 0.28475260734558105 (0.20980141921476883) + | > loss_disc_real_3: 0.26238054037094116 (0.20269714386174173) + | > loss_disc_real_4: 0.25880706310272217 (0.20184866767941098) + | > loss_disc_real_5: 0.3116595447063446 (0.21169570765712045) + | > loss_0: 2.571500301361084 (2.2909041354150483) + | > grad_norm_0: tensor(94.8502, device='cuda:0') (tensor(29.6327, device='cuda:0')) + | > loss_gen: 2.4268150329589844 (2.6151552922797925) + | > loss_kl: 2.208883047103882 (1.7406615596829038) + | > loss_feat: 5.406545639038086 (8.204664851679944) + | > loss_mel: 15.560269355773926 (16.68809749140884) + | > loss_duration: 1.1479235887527466 (0.9448306145090045) + | > loss_1: 26.750436782836914 (30.19340977524266) + | > grad_norm_1: tensor(828.1407, device='cuda:0') (tensor(484.8611, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 2.1176 (1.0448295564362498) + | > loader_time: 0.0152 (0.006790291179310192) + + + --> TIME: 2024-07-13 23:18:51 -- STEP: 83/151 -- GLOBAL_STEP: 102600 + | > loss_disc: 2.26957106590271 (2.2824381072837197) + | > loss_disc_real_0: 0.10896744579076767 (0.12078780039067728) + | > loss_disc_real_1: 0.19661709666252136 (0.20282269852707185) + | > loss_disc_real_2: 0.25764569640159607 (0.20890314930892853) + | > loss_disc_real_3: 0.19406676292419434 (0.20222895655287318) + | > loss_disc_real_4: 0.21434348821640015 (0.20502553987933927) + | > loss_disc_real_5: 0.2718730568885803 (0.21209657712873206) + | > loss_0: 2.26957106590271 (2.2824381072837197) + | > grad_norm_0: tensor(58.8671, device='cuda:0') (tensor(29.4007, device='cuda:0')) + | > loss_gen: 2.7742958068847656 (2.629614491060556) + | > loss_kl: 1.901577353477478 (1.7604697224605514) + | > loss_feat: 7.08704948425293 (8.337008367101834) + | > loss_mel: 16.463165283203125 (16.910021781921376) + | > loss_duration: 0.9154483675956726 (0.9411438897431615) + | > loss_1: 29.141536712646484 (30.57825823864305) + | > grad_norm_1: tensor(631.7355, device='cuda:0') (tensor(452.3809, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.9943 (1.0127290616552518) + | > loader_time: 0.0062 (0.006454829710075654) + + + --> TIME: 2024-07-13 23:19:40 -- STEP: 133/151 -- GLOBAL_STEP: 102650 + | > loss_disc: 2.2716445922851562 (2.2881514802014937) + | > loss_disc_real_0: 0.10526593774557114 (0.12302810833194203) + | > loss_disc_real_1: 0.20608213543891907 (0.20228621274008787) + | > loss_disc_real_2: 0.23758511245250702 (0.21010613856010868) + | > loss_disc_real_3: 0.2030625194311142 (0.20389434370331297) + | > loss_disc_real_4: 0.19898083806037903 (0.20476248569058297) + | > loss_disc_real_5: 0.17102086544036865 (0.21338246885995218) + | > loss_0: 2.2716445922851562 (2.2881514802014937) + | > grad_norm_0: tensor(17.6253, device='cuda:0') (tensor(26.6750, device='cuda:0')) + | > loss_gen: 2.397754430770874 (2.6331822674973577) + | > loss_kl: 1.9213454723358154 (1.7553293731875885) + | > loss_feat: 7.1166229248046875 (8.370560581522772) + | > loss_mel: 15.47163200378418 (16.847955094244238) + | > loss_duration: 0.8981116414070129 (0.9369468706891053) + | > loss_1: 27.80546760559082 (30.543974252571736) + | > grad_norm_1: tensor(110.5339, device='cuda:0') (tensor(422.9987, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.9141 (0.9955872342102512) + | > loader_time: 0.0056 (0.006356707192901382) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031833648681640625 (-0.001003632178673379) + | > avg_loss_disc: 2.585099385334895 (+0.011461973190307617) + | > avg_loss_disc_real_0: 0.260737276994265 (+0.03701063302847052) + | > avg_loss_disc_real_1: 0.2327295564688169 (+0.016822518064425557) + | > avg_loss_disc_real_2: 0.2698235706641124 (+0.049606947944714486) + | > avg_loss_disc_real_3: 0.22327283254036537 (+0.013496201771956234) + | > avg_loss_disc_real_4: 0.22845520537633163 (+0.01977766591768998) + | > avg_loss_disc_real_5: 0.21276413477384126 (-0.01665684695427236) + | > avg_loss_0: 2.585099385334895 (+0.011461973190307617) + | > avg_loss_gen: 2.4371249492351827 (+0.16957051937396717) + | > avg_loss_kl: 3.710248121848473 (-0.07081328905545758) + | > avg_loss_feat: 7.571643499227671 (-0.42029964006864073) + | > avg_loss_mel: 16.630610025846043 (-0.30309369013859566) + | > avg_loss_duration: 1.315559405546922 (+0.005817193251389741) + | > avg_loss_1: 31.665185928344727 (-0.6188193101149366) + + + > EPOCH: 117/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:20:06)  + + --> TIME: 2024-07-13 23:20:43 -- STEP: 32/151 -- GLOBAL_STEP: 102700 + | > loss_disc: 2.366009473800659 (2.3678114786744118) + | > loss_disc_real_0: 0.13177691400051117 (0.1277830795152113) + | > loss_disc_real_1: 0.2119002491235733 (0.2032735957764089) + | > loss_disc_real_2: 0.21983548998832703 (0.21277887281030416) + | > loss_disc_real_3: 0.19540129601955414 (0.21031396696344018) + | > loss_disc_real_4: 0.1835380643606186 (0.21173454029485583) + | > loss_disc_real_5: 0.23145002126693726 (0.22336755273863673) + | > loss_0: 2.366009473800659 (2.3678114786744118) + | > grad_norm_0: tensor(11.0566, device='cuda:0') (tensor(24.6969, device='cuda:0')) + | > loss_gen: 2.5464563369750977 (2.5143123641610146) + | > loss_kl: 1.5654140710830688 (1.8356340564787388) + | > loss_feat: 8.396882057189941 (7.927408173680305) + | > loss_mel: 16.21366310119629 (16.638619929552082) + | > loss_duration: 0.8075077533721924 (0.944210447371006) + | > loss_1: 29.529924392700195 (29.860184967517853) + | > grad_norm_1: tensor(171.8186, device='cuda:0') (tensor(390.1825, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.9399 (1.0424276813864708) + | > loader_time: 0.0055 (0.006976865231990814) + + + --> TIME: 2024-07-13 23:21:31 -- STEP: 82/151 -- GLOBAL_STEP: 102750 + | > loss_disc: 2.310136318206787 (2.330597723402629) + | > loss_disc_real_0: 0.10986147820949554 (0.1218369913900771) + | > loss_disc_real_1: 0.1851418912410736 (0.20203521011806116) + | > loss_disc_real_2: 0.22493451833724976 (0.21321528895599087) + | > loss_disc_real_3: 0.20283569395542145 (0.20770773827666186) + | > loss_disc_real_4: 0.24947689473628998 (0.20901851692214246) + | > loss_disc_real_5: 0.19307269155979156 (0.22281003616205075) + | > loss_0: 2.310136318206787 (2.330597723402629) + | > grad_norm_0: tensor(15.4057, device='cuda:0') (tensor(26.4028, device='cuda:0')) + | > loss_gen: 2.5502841472625732 (2.599443560693322) + | > loss_kl: 1.7104153633117676 (1.7803144716634982) + | > loss_feat: 7.45542573928833 (8.226353633694535) + | > loss_mel: 14.486844062805176 (16.801953804202196) + | > loss_duration: 0.8575283288955688 (0.9338294791012276) + | > loss_1: 27.060497283935547 (30.341895126714938) + | > grad_norm_1: tensor(361.1010, device='cuda:0') (tensor(439.6310, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.8493 (0.9902153567570012) + | > loader_time: 0.0044 (0.006475178206839214) + + + --> TIME: 2024-07-13 23:22:20 -- STEP: 132/151 -- GLOBAL_STEP: 102800 + | > loss_disc: 2.09955096244812 (2.3132198651631684) + | > loss_disc_real_0: 0.10536151379346848 (0.12129486645712997) + | > loss_disc_real_1: 0.18771162629127502 (0.2016776826119784) + | > loss_disc_real_2: 0.18050572276115417 (0.20952013315576495) + | > loss_disc_real_3: 0.18713410198688507 (0.20626993968405505) + | > loss_disc_real_4: 0.1964901238679886 (0.20675576873349422) + | > loss_disc_real_5: 0.2212320864200592 (0.2222009249257319) + | > loss_0: 2.09955096244812 (2.3132198651631684) + | > grad_norm_0: tensor(37.4591, device='cuda:0') (tensor(28.4342, device='cuda:0')) + | > loss_gen: 2.886608362197876 (2.6091935670737056) + | > loss_kl: 2.1221070289611816 (1.738004299501578) + | > loss_feat: 9.637946128845215 (8.293181202628396) + | > loss_mel: 17.37891387939453 (16.721899942918252) + | > loss_duration: 0.8712794780731201 (0.9409402157321121) + | > loss_1: 32.89685821533203 (30.303219405087557) + | > grad_norm_1: tensor(729.9612, device='cuda:0') (tensor(447.2366, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.8577 (0.979296705939553) + | > loader_time: 0.004 (0.006390325950853753) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03678275988652156 (+0.004949111204880938) + | > avg_loss_disc: 2.5388147830963135 (-0.04628460223858166) + | > avg_loss_disc_real_0: 0.21171494401418245 (-0.04902233298008257) + | > avg_loss_disc_real_1: 0.23392073122354654 (+0.0011911747547296292) + | > avg_loss_disc_real_2: 0.2344899487036925 (-0.03533362196041989) + | > avg_loss_disc_real_3: 0.26016470331412095 (+0.03689187077375558) + | > avg_loss_disc_real_4: 0.24052212330011222 (+0.01206691792378059) + | > avg_loss_disc_real_5: 0.2300865352153778 (+0.017322400441536545) + | > avg_loss_0: 2.5388147830963135 (-0.04628460223858166) + | > avg_loss_gen: 2.505262246498695 (+0.06813729726351214) + | > avg_loss_kl: 3.692148648775541 (-0.01809947307293225) + | > avg_loss_feat: 7.599019490755522 (+0.027375991527851085) + | > avg_loss_mel: 16.567479647122898 (-0.06313037872314453) + | > avg_loss_duration: 1.241383983538701 (-0.07417542200822091) + | > avg_loss_1: 31.605293567364033 (-0.05989236098069384) + + + > EPOCH: 118/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:22:48)  + + --> TIME: 2024-07-13 23:23:21 -- STEP: 31/151 -- GLOBAL_STEP: 102850 + | > loss_disc: 2.362868070602417 (2.2919784053679435) + | > loss_disc_real_0: 0.14192870259284973 (0.11804752087881488) + | > loss_disc_real_1: 0.18986473977565765 (0.20069323335924455) + | > loss_disc_real_2: 0.19897381961345673 (0.20551724924195197) + | > loss_disc_real_3: 0.306741863489151 (0.20328444002136106) + | > loss_disc_real_4: 0.24046669900417328 (0.20099339730316593) + | > loss_disc_real_5: 0.280073344707489 (0.22304992041280192) + | > loss_0: 2.362868070602417 (2.2919784053679435) + | > grad_norm_0: tensor(18.9646, device='cuda:0') (tensor(27.1584, device='cuda:0')) + | > loss_gen: 2.625718593597412 (2.624666736971947) + | > loss_kl: 1.8879822492599487 (1.7225172673502276) + | > loss_feat: 8.115361213684082 (8.371295713609264) + | > loss_mel: 17.513282775878906 (17.1257659850582) + | > loss_duration: 0.8960474729537964 (0.9266486571681115) + | > loss_1: 31.038393020629883 (30.770894573580833) + | > grad_norm_1: tensor(379.7815, device='cuda:0') (tensor(459.8277, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 1.1 (0.9670590816005584) + | > loader_time: 0.0074 (0.006135640605803459) + + + --> TIME: 2024-07-13 23:24:10 -- STEP: 81/151 -- GLOBAL_STEP: 102900 + | > loss_disc: 2.390747547149658 (2.3084401539814325) + | > loss_disc_real_0: 0.0921744704246521 (0.12390570259756511) + | > loss_disc_real_1: 0.19990308582782745 (0.20286935659838312) + | > loss_disc_real_2: 0.19464866816997528 (0.20727347113468028) + | > loss_disc_real_3: 0.14395974576473236 (0.2046256775473371) + | > loss_disc_real_4: 0.23327133059501648 (0.20486349318130517) + | > loss_disc_real_5: 0.21079696714878082 (0.22013177400753822) + | > loss_0: 2.390747547149658 (2.3084401539814325) + | > grad_norm_0: tensor(18.9812, device='cuda:0') (tensor(27.4277, device='cuda:0')) + | > loss_gen: 2.314595937728882 (2.600539207458496) + | > loss_kl: 1.8024708032608032 (1.7533836629655626) + | > loss_feat: 7.528947353363037 (8.103195508321123) + | > loss_mel: 15.0436372756958 (16.749592180605287) + | > loss_duration: 1.005020022392273 (0.9388591816395889) + | > loss_1: 27.694671630859375 (30.14556978955681) + | > grad_norm_1: tensor(466.5270, device='cuda:0') (tensor(453.4139, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.8617 (0.9695452054341634) + | > loader_time: 0.0064 (0.006170487698213553) + + + --> TIME: 2024-07-13 23:24:58 -- STEP: 131/151 -- GLOBAL_STEP: 102950 + | > loss_disc: 2.3909506797790527 (2.3010471495053233) + | > loss_disc_real_0: 0.10731242597103119 (0.11905934468252968) + | > loss_disc_real_1: 0.24166244268417358 (0.20197661482650814) + | > loss_disc_real_2: 0.30077916383743286 (0.2095773632972295) + | > loss_disc_real_3: 0.23069947957992554 (0.20419460131239345) + | > loss_disc_real_4: 0.2552451193332672 (0.20566879861227427) + | > loss_disc_real_5: 0.2003452181816101 (0.220595546237385) + | > loss_0: 2.3909506797790527 (2.3010471495053233) + | > grad_norm_0: tensor(15.2707, device='cuda:0') (tensor(28.3231, device='cuda:0')) + | > loss_gen: 3.0043652057647705 (2.6271627768305432) + | > loss_kl: 1.7935476303100586 (1.7733840596584873) + | > loss_feat: 7.352392673492432 (8.283698460527956) + | > loss_mel: 17.764026641845703 (16.861744247320043) + | > loss_duration: 0.9506751298904419 (0.9334335313498519) + | > loss_1: 30.865009307861328 (30.4794231735113) + | > grad_norm_1: tensor(663.3625, device='cuda:0') (tensor(496.1813, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 1.0794 (0.9644827132916632) + | > loader_time: 0.0068 (0.006109155771386532) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03269349611722506 (-0.004089263769296504) + | > avg_loss_disc: 2.7242618707510142 (+0.18544708765470075) + | > avg_loss_disc_real_0: 0.4083249133366805 (+0.19660996932249802) + | > avg_loss_disc_real_1: 0.1991535872220993 (-0.03476714400144723) + | > avg_loss_disc_real_2: 0.2613844252549685 (+0.026894476551276025) + | > avg_loss_disc_real_3: 0.27524168445513797 (+0.01507698114101702) + | > avg_loss_disc_real_4: 0.2864100726751181 (+0.04588794937500587) + | > avg_loss_disc_real_5: 0.2455734793956463 (+0.015486944180268497) + | > avg_loss_0: 2.7242618707510142 (+0.18544708765470075) + | > avg_loss_gen: 2.6420219311347375 (+0.1367596846360426) + | > avg_loss_kl: 3.8242192084972677 (+0.13207055972172688) + | > avg_loss_feat: 7.355227360358605 (-0.24379213039691638) + | > avg_loss_mel: 16.7626130764301 (+0.19513342930720157) + | > avg_loss_duration: 1.361715784439674 (+0.12033180090097284) + | > avg_loss_1: 31.94579740671011 (+0.3405038393460771) + + + > EPOCH: 119/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:25:29)  + + --> TIME: 2024-07-13 23:26:02 -- STEP: 30/151 -- GLOBAL_STEP: 103000 + | > loss_disc: 2.253519296646118 (2.2943463047345474) + | > loss_disc_real_0: 0.0799260139465332 (0.13080846071243285) + | > loss_disc_real_1: 0.20647290349006653 (0.20376674259702365) + | > loss_disc_real_2: 0.21555323898792267 (0.2093946894009908) + | > loss_disc_real_3: 0.19837301969528198 (0.20435588508844377) + | > loss_disc_real_4: 0.1800495982170105 (0.20451824416716893) + | > loss_disc_real_5: 0.2626551687717438 (0.21455132911602656) + | > loss_0: 2.253519296646118 (2.2943463047345474) + | > grad_norm_0: tensor(13.8293, device='cuda:0') (tensor(36.0938, device='cuda:0')) + | > loss_gen: 2.5477993488311768 (2.6707938194274896) + | > loss_kl: 1.7296322584152222 (1.7830458760261536) + | > loss_feat: 7.472627639770508 (8.437878863016765) + | > loss_mel: 16.824731826782227 (16.995840485890707) + | > loss_duration: 0.8166016340255737 (0.9303566694259644) + | > loss_1: 29.391393661499023 (30.817915407816567) + | > grad_norm_1: tensor(525.0322, device='cuda:0') (tensor(537.9477, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.8622 (1.0171144406000776) + | > loader_time: 0.0057 (0.006524729728698731) + + + --> TIME: 2024-07-13 23:27:18 -- STEP: 80/151 -- GLOBAL_STEP: 103050 + | > loss_disc: 2.2711546421051025 (2.3023797288537016) + | > loss_disc_real_0: 0.10757841914892197 (0.12231924026273192) + | > loss_disc_real_1: 0.22704747319221497 (0.2023267894051969) + | > loss_disc_real_2: 0.2368900030851364 (0.20873815836384887) + | > loss_disc_real_3: 0.20082814991474152 (0.206608503498137) + | > loss_disc_real_4: 0.16413778066635132 (0.20808050194755195) + | > loss_disc_real_5: 0.24623481929302216 (0.22021155748516322) + | > loss_0: 2.2711546421051025 (2.3023797288537016) + | > grad_norm_0: tensor(19.5674, device='cuda:0') (tensor(29.7511, device='cuda:0')) + | > loss_gen: 2.51863431930542 (2.6270354837179175) + | > loss_kl: 1.950262427330017 (1.7723880954086781) + | > loss_feat: 7.879600524902344 (8.370890098810197) + | > loss_mel: 17.560300827026367 (16.888992691040038) + | > loss_duration: 0.8755850195884705 (0.9279771007597446) + | > loss_1: 30.78438377380371 (30.587283325195312) + | > grad_norm_1: tensor(341.4635, device='cuda:0') (tensor(452.6042, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.9112 (0.9720676511526107) + | > loader_time: 0.0066 (0.006041023135185242) + + + --> TIME: 2024-07-13 23:28:08 -- STEP: 130/151 -- GLOBAL_STEP: 103100 + | > loss_disc: 2.4183456897735596 (2.3073280086884123) + | > loss_disc_real_0: 0.0712888091802597 (0.12096277879407773) + | > loss_disc_real_1: 0.20696228742599487 (0.20319124397176963) + | > loss_disc_real_2: 0.2432773858308792 (0.21055647312448572) + | > loss_disc_real_3: 0.29363083839416504 (0.2066181651674784) + | > loss_disc_real_4: 0.2413712590932846 (0.20693867051830658) + | > loss_disc_real_5: 0.2857249081134796 (0.22044464842631267) + | > loss_0: 2.4183456897735596 (2.3073280086884123) + | > grad_norm_0: tensor(24.1737, device='cuda:0') (tensor(26.3016, device='cuda:0')) + | > loss_gen: 2.4763267040252686 (2.6099144202012288) + | > loss_kl: 1.4069184064865112 (1.7888729677750514) + | > loss_feat: 8.44386100769043 (8.29356951713562) + | > loss_mel: 15.517931938171387 (16.800865804232078) + | > loss_duration: 0.9672044515609741 (0.9287635849072383) + | > loss_1: 28.81224250793457 (30.42198624244103) + | > grad_norm_1: tensor(409.7058, device='cuda:0') (tensor(427.2793, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.8214 (0.9806763098790096) + | > loader_time: 0.0052 (0.006254207170926607) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.047362639353825495 (+0.014669143236600436) + | > avg_loss_disc: 2.616828386600201 (-0.10743348415081311) + | > avg_loss_disc_real_0: 0.25411538435862613 (-0.15420952897805434) + | > avg_loss_disc_real_1: 0.2552286661588229 (+0.05607507893672359) + | > avg_loss_disc_real_2: 0.20976868157203382 (-0.0516157436829347) + | > avg_loss_disc_real_3: 0.24287403202973878 (-0.032367652425399185) + | > avg_loss_disc_real_4: 0.24920821075256055 (-0.03720186192255753) + | > avg_loss_disc_real_5: 0.18414066044183877 (-0.06143281895380753) + | > avg_loss_0: 2.616828386600201 (-0.10743348415081311) + | > avg_loss_gen: 2.4170785500453067 (-0.2249433810894308) + | > avg_loss_kl: 3.7811664984776425 (-0.043052710019625184) + | > avg_loss_feat: 8.138753120715801 (+0.7835257603571959) + | > avg_loss_mel: 17.492185445932243 (+0.7295723695021437) + | > avg_loss_duration: 1.278895845780006 (-0.08281993865966797) + | > avg_loss_1: 33.10807961684007 (+1.1622822101299626) + + + > EPOCH: 120/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:28:38)  + + --> TIME: 2024-07-13 23:29:09 -- STEP: 29/151 -- GLOBAL_STEP: 103150 + | > loss_disc: 2.2315566539764404 (2.332279887692682) + | > loss_disc_real_0: 0.11006385087966919 (0.1267008461571973) + | > loss_disc_real_1: 0.16236759722232819 (0.20455356815765643) + | > loss_disc_real_2: 0.15645165741443634 (0.21320964344616594) + | > loss_disc_real_3: 0.1717516928911209 (0.2107548944908997) + | > loss_disc_real_4: 0.16448025405406952 (0.20485795520502945) + | > loss_disc_real_5: 0.24723084270954132 (0.21664597145442305) + | > loss_0: 2.2315566539764404 (2.332279887692682) + | > grad_norm_0: tensor(46.5524, device='cuda:0') (tensor(19.2449, device='cuda:0')) + | > loss_gen: 2.630214214324951 (2.5426959004895435) + | > loss_kl: 1.2578474283218384 (1.7171011957628974) + | > loss_feat: 8.316526412963867 (8.097248274704505) + | > loss_mel: 17.40748405456543 (16.81509458607641) + | > loss_duration: 0.9491981267929077 (0.9283132388673979) + | > loss_1: 30.561269760131836 (30.100453541196625) + | > grad_norm_1: tensor(339.4307, device='cuda:0') (tensor(230.0346, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.8601 (0.9734264242238012) + | > loader_time: 0.0047 (0.005970658927128233) + + + --> TIME: 2024-07-13 23:29:58 -- STEP: 79/151 -- GLOBAL_STEP: 103200 + | > loss_disc: 2.4277000427246094 (2.3170802744129038) + | > loss_disc_real_0: 0.08196313679218292 (0.1273408858930763) + | > loss_disc_real_1: 0.198139488697052 (0.202206298897538) + | > loss_disc_real_2: 0.2460518181324005 (0.21101973875413962) + | > loss_disc_real_3: 0.2376747727394104 (0.20837501974045475) + | > loss_disc_real_4: 0.22173398733139038 (0.20860328346113616) + | > loss_disc_real_5: 0.24333128333091736 (0.21737519981740397) + | > loss_0: 2.4277000427246094 (2.3170802744129038) + | > grad_norm_0: tensor(20.0475, device='cuda:0') (tensor(25.9415, device='cuda:0')) + | > loss_gen: 2.3691415786743164 (2.6012584559525105) + | > loss_kl: 1.7682416439056396 (1.7084653053102614) + | > loss_feat: 7.248935699462891 (8.315217525144162) + | > loss_mel: 16.989044189453125 (16.867801316176795) + | > loss_duration: 0.9579635858535767 (0.9257575047167041) + | > loss_1: 29.33332633972168 (30.418500103527986) + | > grad_norm_1: tensor(331.9834, device='cuda:0') (tensor(348.7282, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.9416 (0.9744542731514456) + | > loader_time: 0.0058 (0.006183259094817728) + + + --> TIME: 2024-07-13 23:30:49 -- STEP: 129/151 -- GLOBAL_STEP: 103250 + | > loss_disc: 2.726700782775879 (2.317108084065044) + | > loss_disc_real_0: 0.26221439242362976 (0.12704016868920287) + | > loss_disc_real_1: 0.25990763306617737 (0.20379241989102476) + | > loss_disc_real_2: 0.23118631541728973 (0.21218407258044839) + | > loss_disc_real_3: 0.3766256868839264 (0.20710265105084855) + | > loss_disc_real_4: 0.2673991024494171 (0.20740523183530615) + | > loss_disc_real_5: 0.2823435068130493 (0.21687777253777482) + | > loss_0: 2.726700782775879 (2.317108084065044) + | > grad_norm_0: tensor(39.3687, device='cuda:0') (tensor(23.8810, device='cuda:0')) + | > loss_gen: 2.2048630714416504 (2.597794911658116) + | > loss_kl: 2.1194305419921875 (1.7121778684993123) + | > loss_feat: 4.405383110046387 (8.235018940859058) + | > loss_mel: 13.78160572052002 (16.795698454213696) + | > loss_duration: 1.2056535482406616 (0.93049462682517) + | > loss_1: 23.716936111450195 (30.27118483255076) + | > grad_norm_1: tensor(269.7606, device='cuda:0') (tensor(327.7450, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 2.2981 (0.9867790410684982) + | > loader_time: 0.0179 (0.006416745888170346) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03489873959467961 (-0.012463899759145886) + | > avg_loss_disc: 2.5164305430192213 (-0.1003978435809798) + | > avg_loss_disc_real_0: 0.1861329823732376 (-0.06798240198538852) + | > avg_loss_disc_real_1: 0.24014476056282336 (-0.015083905595999536) + | > avg_loss_disc_real_2: 0.24077284794587356 (+0.031004166373839737) + | > avg_loss_disc_real_3: 0.23444738067113435 (-0.008426651358604431) + | > avg_loss_disc_real_4: 0.2358686511333172 (-0.013339559619243352) + | > avg_loss_disc_real_5: 0.23187726162947142 (+0.04773660118763265) + | > avg_loss_0: 2.5164305430192213 (-0.1003978435809798) + | > avg_loss_gen: 2.4408965477576623 (+0.0238179977123556) + | > avg_loss_kl: 3.6668915565197286 (-0.11427494195791388) + | > avg_loss_feat: 7.934423299936148 (-0.20432982077965356) + | > avg_loss_mel: 17.041199537423942 (-0.4509859085083008) + | > avg_loss_duration: 1.2930566806059618 (+0.01416083482595587) + | > avg_loss_1: 32.37646748469426 (-0.7316121321458127) + + + > EPOCH: 121/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:31:20)  + + --> TIME: 2024-07-13 23:31:50 -- STEP: 28/151 -- GLOBAL_STEP: 103300 + | > loss_disc: 2.2794902324676514 (2.285566815308162) + | > loss_disc_real_0: 0.16346007585525513 (0.11759718374482223) + | > loss_disc_real_1: 0.2193351686000824 (0.20404369809797832) + | > loss_disc_real_2: 0.21093305945396423 (0.20956736164433615) + | > loss_disc_real_3: 0.16972103714942932 (0.20767607060926302) + | > loss_disc_real_4: 0.18321751058101654 (0.21188954902546747) + | > loss_disc_real_5: 0.2077208012342453 (0.21578033215233258) + | > loss_0: 2.2794902324676514 (2.285566815308162) + | > grad_norm_0: tensor(61.8219, device='cuda:0') (tensor(30.1976, device='cuda:0')) + | > loss_gen: 2.66934871673584 (2.653519323893956) + | > loss_kl: 1.961551308631897 (1.7825487852096558) + | > loss_feat: 9.440180778503418 (8.335239461490085) + | > loss_mel: 18.143796920776367 (16.874348163604733) + | > loss_duration: 0.8767129778862 (0.9207822935921806) + | > loss_1: 33.091590881347656 (30.566438606807164) + | > grad_norm_1: tensor(594.2667, device='cuda:0') (tensor(495.7648, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.9512 (0.9906212091445923) + | > loader_time: 0.0071 (0.006562760898045131) + + + --> TIME: 2024-07-13 23:32:40 -- STEP: 78/151 -- GLOBAL_STEP: 103350 + | > loss_disc: 2.1729865074157715 (2.293826836806077) + | > loss_disc_real_0: 0.06284981966018677 (0.11960359815603652) + | > loss_disc_real_1: 0.19176705181598663 (0.2047697093624335) + | > loss_disc_real_2: 0.1847439408302307 (0.2101669953419612) + | > loss_disc_real_3: 0.17895086109638214 (0.20621101797009125) + | > loss_disc_real_4: 0.21005558967590332 (0.20877581672408643) + | > loss_disc_real_5: 0.2374335080385208 (0.21352800726890558) + | > loss_0: 2.1729865074157715 (2.293826836806077) + | > grad_norm_0: tensor(23.4832, device='cuda:0') (tensor(28.0187, device='cuda:0')) + | > loss_gen: 2.7414817810058594 (2.6192119213250966) + | > loss_kl: 1.442333698272705 (1.7352020602959852) + | > loss_feat: 9.817549705505371 (8.266934144191254) + | > loss_mel: 17.260847091674805 (16.707970851506946) + | > loss_duration: 0.9508048295974731 (0.9327728350957235) + | > loss_1: 32.213016510009766 (30.26209190564278) + | > grad_norm_1: tensor(477.9134, device='cuda:0') (tensor(444.2705, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.8436 (0.9893629459234382) + | > loader_time: 0.0042 (0.006317887550745254) + + + --> TIME: 2024-07-13 23:33:28 -- STEP: 128/151 -- GLOBAL_STEP: 103400 + | > loss_disc: 2.142822742462158 (2.2876636814326052) + | > loss_disc_real_0: 0.059232693165540695 (0.11958103528013456) + | > loss_disc_real_1: 0.14857958257198334 (0.2024418479995802) + | > loss_disc_real_2: 0.14774955809116364 (0.21030138502828777) + | > loss_disc_real_3: 0.12295462191104889 (0.20550915139028803) + | > loss_disc_real_4: 0.16393856704235077 (0.20713814097689465) + | > loss_disc_real_5: 0.21107538044452667 (0.21311919298022983) + | > loss_0: 2.142822742462158 (2.2876636814326052) + | > grad_norm_0: tensor(12.1285, device='cuda:0') (tensor(26.2506, device='cuda:0')) + | > loss_gen: 2.680849552154541 (2.6250651143491264) + | > loss_kl: 1.8274517059326172 (1.74709755461663) + | > loss_feat: 10.026510238647461 (8.362366355955604) + | > loss_mel: 17.13005256652832 (16.647282920777805) + | > loss_duration: 0.8181536793708801 (0.9257998741231859) + | > loss_1: 32.48301696777344 (30.307611852884293) + | > grad_norm_1: tensor(241.6978, device='cuda:0') (tensor(413.0012, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.8784 (0.9677210804075002) + | > loader_time: 0.0057 (0.006159922108054161) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03476826961223896 (-0.00013046998244065078) + | > avg_loss_disc: 2.6505069732666016 (+0.13407643024738025) + | > avg_loss_disc_real_0: 0.3491749878113086 (+0.163042005438071) + | > avg_loss_disc_real_1: 0.23788592219352722 (-0.0022588383692961345) + | > avg_loss_disc_real_2: 0.24528604058118966 (+0.0045131926353161045) + | > avg_loss_disc_real_3: 0.24180523707316473 (+0.007357856402030377) + | > avg_loss_disc_real_4: 0.26701314747333527 (+0.031144496340018063) + | > avg_loss_disc_real_5: 0.22710129274771765 (-0.004775968881753773) + | > avg_loss_0: 2.6505069732666016 (+0.13407643024738025) + | > avg_loss_gen: 2.731170580937312 (+0.29027403317964984) + | > avg_loss_kl: 3.8891459795144887 (+0.22225442299476006) + | > avg_loss_feat: 7.801162573007437 (-0.13326072692871094) + | > avg_loss_mel: 16.652527148907 (-0.38867238851694097) + | > avg_loss_duration: 1.314798107514015 (+0.02174142690805314) + | > avg_loss_1: 32.38880406893217 (+0.012336584237907289) + + + > EPOCH: 122/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:34:00)  + + --> TIME: 2024-07-13 23:34:28 -- STEP: 27/151 -- GLOBAL_STEP: 103450 + | > loss_disc: 2.296172618865967 (2.2807871235741506) + | > loss_disc_real_0: 0.17527884244918823 (0.11636483227765118) + | > loss_disc_real_1: 0.18405267596244812 (0.19625518995302696) + | > loss_disc_real_2: 0.17799384891986847 (0.20668160142721953) + | > loss_disc_real_3: 0.18942569196224213 (0.20060496584132867) + | > loss_disc_real_4: 0.17628641426563263 (0.20299993952115378) + | > loss_disc_real_5: 0.19308017194271088 (0.22139121481665858) + | > loss_0: 2.296172618865967 (2.2807871235741506) + | > grad_norm_0: tensor(66.3504, device='cuda:0') (tensor(33.5023, device='cuda:0')) + | > loss_gen: 2.5100977420806885 (2.6393181571254027) + | > loss_kl: 1.481006145477295 (1.7043369655255918) + | > loss_feat: 7.753374099731445 (8.24964138313576) + | > loss_mel: 14.068085670471191 (16.659007178412544) + | > loss_duration: 1.150935411453247 (0.9313708852838587) + | > loss_1: 26.9635009765625 (30.18367498892325) + | > grad_norm_1: tensor(631.5052, device='cuda:0') (tensor(518.5519, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.8236 (0.9504931414568866) + | > loader_time: 0.0039 (0.005876929671676069) + + + --> TIME: 2024-07-13 23:35:18 -- STEP: 77/151 -- GLOBAL_STEP: 103500 + | > loss_disc: 2.4193758964538574 (2.3054382398531033) + | > loss_disc_real_0: 0.13677380979061127 (0.1221814885638751) + | > loss_disc_real_1: 0.21640551090240479 (0.1980133182429648) + | > loss_disc_real_2: 0.21390073001384735 (0.21083400292055948) + | > loss_disc_real_3: 0.19230134785175323 (0.2058336655427883) + | > loss_disc_real_4: 0.21154063940048218 (0.20921947294241422) + | > loss_disc_real_5: 0.3007798194885254 (0.22016858590113653) + | > loss_0: 2.4193758964538574 (2.3054382398531033) + | > grad_norm_0: tensor(14.0621, device='cuda:0') (tensor(28.7541, device='cuda:0')) + | > loss_gen: 2.5139575004577637 (2.6240194029622264) + | > loss_kl: 1.838870644569397 (1.7717222610077301) + | > loss_feat: 7.4241719245910645 (8.261955539901539) + | > loss_mel: 16.217580795288086 (16.7407428444206) + | > loss_duration: 0.8875584602355957 (0.9310677546959418) + | > loss_1: 28.882139205932617 (30.329507951612598) + | > grad_norm_1: tensor(166.7116, device='cuda:0') (tensor(438.7664, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.8888 (0.9700129496586787) + | > loader_time: 0.0061 (0.0062398786668653615) + + + --> TIME: 2024-07-13 23:36:08 -- STEP: 127/151 -- GLOBAL_STEP: 103550 + | > loss_disc: 2.261439800262451 (2.281405360679927) + | > loss_disc_real_0: 0.13479351997375488 (0.12005692195352607) + | > loss_disc_real_1: 0.1770670861005783 (0.1987425457304857) + | > loss_disc_real_2: 0.20700553059577942 (0.20983874058629584) + | > loss_disc_real_3: 0.17904502153396606 (0.20212462931636752) + | > loss_disc_real_4: 0.2920045852661133 (0.20796775688805919) + | > loss_disc_real_5: 0.2564665377140045 (0.21650476307850183) + | > loss_0: 2.261439800262451 (2.281405360679927) + | > grad_norm_0: tensor(10.7905, device='cuda:0') (tensor(27.4522, device='cuda:0')) + | > loss_gen: 2.816908836364746 (2.6523717294527795) + | > loss_kl: 2.003453254699707 (1.7683026579421337) + | > loss_feat: 9.555534362792969 (8.42532213466374) + | > loss_mel: 17.310047149658203 (16.774136731005083) + | > loss_duration: 0.8807531595230103 (0.9360144025697483) + | > loss_1: 32.56669616699219 (30.556147733072596) + | > grad_norm_1: tensor(461.3180, device='cuda:0') (tensor(453.6561, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.8859 (0.9772133170150397) + | > loader_time: 0.005 (0.006219201200590359) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03304312779353215 (-0.0017251418187068102) + | > avg_loss_disc: 2.5757321211007924 (-0.07477485216580915) + | > avg_loss_disc_real_0: 0.1976621815791497 (-0.15151280623215893) + | > avg_loss_disc_real_1: 0.2220458984375 (-0.01584002375602722) + | > avg_loss_disc_real_2: 0.2385897567638984 (-0.00669628381729126) + | > avg_loss_disc_real_3: 0.24319077340456155 (+0.0013855363313968194) + | > avg_loss_disc_real_4: 0.25843364802690655 (-0.008579499446428718) + | > avg_loss_disc_real_5: 0.2358085260941432 (+0.008707233346425564) + | > avg_loss_0: 2.5757321211007924 (-0.07477485216580915) + | > avg_loss_gen: 2.3595196100381703 (-0.3716509708991418) + | > avg_loss_kl: 3.6774632197159987 (-0.21168275979849005) + | > avg_loss_feat: 7.509915425227239 (-0.2912471477801981) + | > avg_loss_mel: 16.983762961167557 (+0.3312358122605552) + | > avg_loss_duration: 1.380241843370291 (+0.06544373585627605) + | > avg_loss_1: 31.910903197068436 (-0.4779008718637314) + + + > EPOCH: 123/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:36:41)  + + --> TIME: 2024-07-13 23:37:11 -- STEP: 26/151 -- GLOBAL_STEP: 103600 + | > loss_disc: 2.1003246307373047 (2.2844655972260695) + | > loss_disc_real_0: 0.08725504577159882 (0.13135323931391424) + | > loss_disc_real_1: 0.16920024156570435 (0.1993368290937864) + | > loss_disc_real_2: 0.1894717514514923 (0.20913896136558974) + | > loss_disc_real_3: 0.19855503737926483 (0.2064703883459935) + | > loss_disc_real_4: 0.13537798821926117 (0.20225194153877404) + | > loss_disc_real_5: 0.24922022223472595 (0.21694734577949232) + | > loss_0: 2.1003246307373047 (2.2844655972260695) + | > grad_norm_0: tensor(23.6554, device='cuda:0') (tensor(30.1205, device='cuda:0')) + | > loss_gen: 2.724578380584717 (2.6206511075680075) + | > loss_kl: 1.6600743532180786 (1.8501313374592707) + | > loss_feat: 9.573051452636719 (8.268821734648485) + | > loss_mel: 16.983596801757812 (16.631109897906963) + | > loss_duration: 0.9189357757568359 (0.9517006690685565) + | > loss_1: 31.86023712158203 (30.322414324833797) + | > grad_norm_1: tensor(454.6064, device='cuda:0') (tensor(484.5697, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.89 (0.9942086843343881) + | > loader_time: 0.0059 (0.005997566076425405) + + + --> TIME: 2024-07-13 23:37:59 -- STEP: 76/151 -- GLOBAL_STEP: 103650 + | > loss_disc: 2.6575253009796143 (2.3221786555490986) + | > loss_disc_real_0: 0.23057179152965546 (0.13418012029050214) + | > loss_disc_real_1: 0.24557098746299744 (0.20255742525975956) + | > loss_disc_real_2: 0.25110331177711487 (0.21201577861058085) + | > loss_disc_real_3: 0.1985127478837967 (0.20815801100903436) + | > loss_disc_real_4: 0.25629302859306335 (0.2078232808332694) + | > loss_disc_real_5: 0.2687702476978302 (0.21770862156623288) + | > loss_0: 2.6575253009796143 (2.3221786555490986) + | > grad_norm_0: tensor(45.3379, device='cuda:0') (tensor(32.1575, device='cuda:0')) + | > loss_gen: 2.1476926803588867 (2.618756234645843) + | > loss_kl: 2.0725514888763428 (1.7904528708834397) + | > loss_feat: 4.47705078125 (8.271709881330795) + | > loss_mel: 14.246273040771484 (16.71766932387101) + | > loss_duration: 1.1423345804214478 (0.936299771854752) + | > loss_1: 24.08590316772461 (30.334887906124717) + | > grad_norm_1: tensor(245.3927, device='cuda:0') (tensor(468.1428, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 2.0469 (0.9716045856475832) + | > loader_time: 0.0187 (0.006088153312080785) + + + --> TIME: 2024-07-13 23:38:48 -- STEP: 126/151 -- GLOBAL_STEP: 103700 + | > loss_disc: 2.3481993675231934 (2.3061362845557074) + | > loss_disc_real_0: 0.16334646940231323 (0.12876031710396682) + | > loss_disc_real_1: 0.24244900047779083 (0.20186371350335697) + | > loss_disc_real_2: 0.2727013826370239 (0.21135979524207493) + | > loss_disc_real_3: 0.3293672502040863 (0.2078998406373319) + | > loss_disc_real_4: 0.2740563452243805 (0.2060777455095261) + | > loss_disc_real_5: 0.23288245499134064 (0.21739704150056083) + | > loss_0: 2.3481993675231934 (2.3061362845557074) + | > grad_norm_0: tensor(12.5831, device='cuda:0') (tensor(27.5325, device='cuda:0')) + | > loss_gen: 2.8400793075561523 (2.612665057182312) + | > loss_kl: 2.1187660694122314 (1.7890653908252716) + | > loss_feat: 8.918638229370117 (8.267898934228084) + | > loss_mel: 17.33976173400879 (16.695758145952997) + | > loss_duration: 0.8928267955780029 (0.9347411819866726) + | > loss_1: 32.11007308959961 (30.300128649151514) + | > grad_norm_1: tensor(243.9825, device='cuda:0') (tensor(409.1411, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.9619 (0.9666957817380392) + | > loader_time: 0.0058 (0.006164861103845021) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033008538759671725 (-3.4589033860422946e-05) + | > avg_loss_disc: 2.57145986190209 (-0.004272259198702333) + | > avg_loss_disc_real_0: 0.2625516366500121 (+0.06488945507086238) + | > avg_loss_disc_real_1: 0.19194931135727808 (-0.03009658708022192) + | > avg_loss_disc_real_2: 0.236501823251064 (-0.0020879335128344) + | > avg_loss_disc_real_3: 0.2521362029589139 (+0.008945429554352374) + | > avg_loss_disc_real_4: 0.23373432457447052 (-0.02469932345243603) + | > avg_loss_disc_real_5: 0.23131143359037545 (-0.004497092503767758) + | > avg_loss_0: 2.57145986190209 (-0.004272259198702333) + | > avg_loss_gen: 2.3861517172593336 (+0.02663210722116327) + | > avg_loss_kl: 3.7194747741405783 (+0.0420115544245796) + | > avg_loss_feat: 7.453364152174729 (-0.05655127305250929) + | > avg_loss_mel: 16.659212112426758 (-0.32455084874079887) + | > avg_loss_duration: 1.3406538504820604 (-0.03958799288823056) + | > avg_loss_1: 31.558856523953953 (-0.3520466731144829) + + + > EPOCH: 124/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:39:23)  + + --> TIME: 2024-07-13 23:39:50 -- STEP: 25/151 -- GLOBAL_STEP: 103750 + | > loss_disc: 2.6227505207061768 (2.3222595119476317) + | > loss_disc_real_0: 0.17349587380886078 (0.1297661019861698) + | > loss_disc_real_1: 0.24625863134860992 (0.20587792992591858) + | > loss_disc_real_2: 0.2572244703769684 (0.20948122680187226) + | > loss_disc_real_3: 0.29952141642570496 (0.20993389546871186) + | > loss_disc_real_4: 0.22686824202537537 (0.2115449368953705) + | > loss_disc_real_5: 0.24753019213676453 (0.21870241463184356) + | > loss_0: 2.6227505207061768 (2.3222595119476317) + | > grad_norm_0: tensor(26.3720, device='cuda:0') (tensor(28.1252, device='cuda:0')) + | > loss_gen: 2.4363865852355957 (2.5889407062530525) + | > loss_kl: 2.1825623512268066 (1.8043068408966065) + | > loss_feat: 6.298237323760986 (7.814028358459472) + | > loss_mel: 17.296077728271484 (16.32823661804199) + | > loss_duration: 1.0629799365997314 (0.953336911201477) + | > loss_1: 29.2762451171875 (29.488849487304687) + | > grad_norm_1: tensor(396.8658, device='cuda:0') (tensor(401.2078, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 1.5684 (1.0030829906463623) + | > loader_time: 0.0116 (0.006520824432373047) + + + --> TIME: 2024-07-13 23:40:42 -- STEP: 75/151 -- GLOBAL_STEP: 103800 + | > loss_disc: 2.3544328212738037 (2.3278247531255087) + | > loss_disc_real_0: 0.1008375734090805 (0.12909150555729862) + | > loss_disc_real_1: 0.2712664306163788 (0.2083614053328832) + | > loss_disc_real_2: 0.219640851020813 (0.21218555867671965) + | > loss_disc_real_3: 0.24432174861431122 (0.20673863569895426) + | > loss_disc_real_4: 0.22746451199054718 (0.20847926517327628) + | > loss_disc_real_5: 0.22958387434482574 (0.21846558968226115) + | > loss_0: 2.3544328212738037 (2.3278247531255087) + | > grad_norm_0: tensor(13.7711, device='cuda:0') (tensor(29.9893, device='cuda:0')) + | > loss_gen: 2.778028964996338 (2.5978559684753426) + | > loss_kl: 1.837662935256958 (1.7993869415918986) + | > loss_feat: 8.147208213806152 (8.057622311909991) + | > loss_mel: 17.35186004638672 (16.739117851257323) + | > loss_duration: 0.8670347929000854 (0.9451668922106425) + | > loss_1: 30.981794357299805 (30.139149907430014) + | > grad_norm_1: tensor(144.2387, device='cuda:0') (tensor(430.5604, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.8604 (1.0186065546671543) + | > loader_time: 0.0049 (0.006673819224039714) + + + --> TIME: 2024-07-13 23:41:28 -- STEP: 125/151 -- GLOBAL_STEP: 103850 + | > loss_disc: 2.254765510559082 (2.3011171522140503) + | > loss_disc_real_0: 0.10520737618207932 (0.12416001692414282) + | > loss_disc_real_1: 0.23237787187099457 (0.20492160946130752) + | > loss_disc_real_2: 0.20351438224315643 (0.2114559428691864) + | > loss_disc_real_3: 0.17772336304187775 (0.2044518767595291) + | > loss_disc_real_4: 0.2335052788257599 (0.20740711641311646) + | > loss_disc_real_5: 0.192206472158432 (0.21695765006542206) + | > loss_0: 2.254765510559082 (2.3011171522140503) + | > grad_norm_0: tensor(27.2512, device='cuda:0') (tensor(29.7286, device='cuda:0')) + | > loss_gen: 2.604123592376709 (2.6233344821929934) + | > loss_kl: 1.7558581829071045 (1.7724216527938843) + | > loss_feat: 9.324691772460938 (8.240076313018792) + | > loss_mel: 16.89137840270996 (16.731941947937013) + | > loss_duration: 0.8793210983276367 (0.9288720345497131) + | > loss_1: 31.455371856689453 (30.296646270751953) + | > grad_norm_1: tensor(553.7162, device='cuda:0') (tensor(447.7012, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.9002 (0.974496088027954) + | > loader_time: 0.0056 (0.006229806900024414) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030741306451650765 (-0.0022672323080209604) + | > avg_loss_disc: 2.565396657356849 (-0.006063204545240897) + | > avg_loss_disc_real_0: 0.21911945939064026 (-0.04343217725937182) + | > avg_loss_disc_real_1: 0.23065843261205232 (+0.03870912125477424) + | > avg_loss_disc_real_2: 0.21962855068536905 (-0.016873272565694958) + | > avg_loss_disc_real_3: 0.19816643802019265 (-0.05396976493872127) + | > avg_loss_disc_real_4: 0.25812916228404414 (+0.024394837709573625) + | > avg_loss_disc_real_5: 0.19951563615065354 (-0.03179579743972191) + | > avg_loss_0: 2.565396657356849 (-0.006063204545240897) + | > avg_loss_gen: 2.3617041477790246 (-0.024447569480309017) + | > avg_loss_kl: 3.7618243327507606 (+0.0423495586101823) + | > avg_loss_feat: 8.378884608928974 (+0.9255204567542448) + | > avg_loss_mel: 17.25817922445444 (+0.5989671120276832) + | > avg_loss_duration: 1.3288635840782752 (-0.011790266403785221) + | > avg_loss_1: 33.08945626478929 (+1.5305997408353385) + + + > EPOCH: 125/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:42:04)  + + --> TIME: 2024-07-13 23:42:30 -- STEP: 24/151 -- GLOBAL_STEP: 103900 + | > loss_disc: 2.2322802543640137 (2.3084464271863303) + | > loss_disc_real_0: 0.11881691217422485 (0.1197075058395664) + | > loss_disc_real_1: 0.23430758714675903 (0.20870295353233814) + | > loss_disc_real_2: 0.236764594912529 (0.20979158394038677) + | > loss_disc_real_3: 0.16969004273414612 (0.19808791329463324) + | > loss_disc_real_4: 0.25588881969451904 (0.20688303653150797) + | > loss_disc_real_5: 0.20030146837234497 (0.21643136503795782) + | > loss_0: 2.2322802543640137 (2.3084464271863303) + | > grad_norm_0: tensor(38.5859, device='cuda:0') (tensor(29.8814, device='cuda:0')) + | > loss_gen: 2.8891892433166504 (2.599621891975403) + | > loss_kl: 1.6984857320785522 (1.6398056869705517) + | > loss_feat: 8.464995384216309 (8.105122923851015) + | > loss_mel: 16.738733291625977 (16.484728535016377) + | > loss_duration: 0.8493787050247192 (0.9239199211200079) + | > loss_1: 30.64078140258789 (29.753198703130085) + | > grad_norm_1: tensor(599.8029, device='cuda:0') (tensor(442.1857, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.8637 (0.9777426024278005) + | > loader_time: 0.0055 (0.00630075732866923) + + + --> TIME: 2024-07-13 23:43:19 -- STEP: 74/151 -- GLOBAL_STEP: 103950 + | > loss_disc: 2.335865020751953 (2.267599784039162) + | > loss_disc_real_0: 0.0754302591085434 (0.11396127521387629) + | > loss_disc_real_1: 0.15823052823543549 (0.20291594939457402) + | > loss_disc_real_2: 0.16575852036476135 (0.2096997217552082) + | > loss_disc_real_3: 0.20794035494327545 (0.20104286104843422) + | > loss_disc_real_4: 0.20896410942077637 (0.20348091423511505) + | > loss_disc_real_5: 0.2163243144750595 (0.2179732280405792) + | > loss_0: 2.335865020751953 (2.267599784039162) + | > grad_norm_0: tensor(30.7907, device='cuda:0') (tensor(27.9304, device='cuda:0')) + | > loss_gen: 2.4050965309143066 (2.6559572928660637) + | > loss_kl: 1.8136683702468872 (1.713566624634975) + | > loss_feat: 8.140131950378418 (8.434345876848376) + | > loss_mel: 16.535400390625 (16.717277359318096) + | > loss_duration: 0.8816471099853516 (0.9248220163422662) + | > loss_1: 29.775943756103516 (30.445968911454486) + | > grad_norm_1: tensor(367.7216, device='cuda:0') (tensor(481.0763, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.8907 (0.9696577884055473) + | > loader_time: 0.0059 (0.006179152308283625) + + + --> TIME: 2024-07-13 23:44:07 -- STEP: 124/151 -- GLOBAL_STEP: 104000 + | > loss_disc: 2.231750011444092 (2.276084275014939) + | > loss_disc_real_0: 0.11329479515552521 (0.11670047212992946) + | > loss_disc_real_1: 0.2118268758058548 (0.20140305666192884) + | > loss_disc_real_2: 0.2078600823879242 (0.2096997529508606) + | > loss_disc_real_3: 0.22949698567390442 (0.20177092114763875) + | > loss_disc_real_4: 0.23307934403419495 (0.2050422118195603) + | > loss_disc_real_5: 0.21850691735744476 (0.21976107755495655) + | > loss_0: 2.231750011444092 (2.276084275014939) + | > grad_norm_0: tensor(30.0386, device='cuda:0') (tensor(25.9871, device='cuda:0')) + | > loss_gen: 2.928081512451172 (2.648885123191342) + | > loss_kl: 1.3923088312149048 (1.710745251947834) + | > loss_feat: 8.453861236572266 (8.434651320980445) + | > loss_mel: 15.830758094787598 (16.718650410252234) + | > loss_duration: 0.9434152841567993 (0.9249639842779406) + | > loss_1: 29.548425674438477 (30.437895913277902) + | > grad_norm_1: tensor(610.5037, device='cuda:0') (tensor(444.7728, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.8595 (0.9610499220509683) + | > loader_time: 0.0047 (0.006095401702388641) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034064182868370645 (+0.0033228764167198797) + | > avg_loss_disc: 2.583407530417809 (+0.01801087306095983) + | > avg_loss_disc_real_0: 0.3400548994541168 (+0.12093544006347656) + | > avg_loss_disc_real_1: 0.21904925199655387 (-0.011609180615498454) + | > avg_loss_disc_real_2: 0.21756791609984177 (-0.002060634585527271) + | > avg_loss_disc_real_3: 0.22414239324056184 (+0.02597595522036919) + | > avg_loss_disc_real_4: 0.2994011858334908 (+0.041272023549446646) + | > avg_loss_disc_real_5: 0.24282630590292123 (+0.04331066975226769) + | > avg_loss_0: 2.583407530417809 (+0.01801087306095983) + | > avg_loss_gen: 2.706668945459219 (+0.3449647976801944) + | > avg_loss_kl: 3.76577903674199 (+0.003954703991229547) + | > avg_loss_feat: 8.139644035926231 (-0.2392405730027427) + | > avg_loss_mel: 17.084932327270508 (-0.17324689718393316) + | > avg_loss_duration: 1.2982210104282086 (-0.030642573650066618) + | > avg_loss_1: 32.9952455667349 (-0.09421069805438975) + + + > EPOCH: 126/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:45:12)  + + --> TIME: 2024-07-13 23:45:37 -- STEP: 23/151 -- GLOBAL_STEP: 104050 + | > loss_disc: 2.344351291656494 (2.2818582783574644) + | > loss_disc_real_0: 0.11726334691047668 (0.11317356740650923) + | > loss_disc_real_1: 0.22795827686786652 (0.20068353414535522) + | > loss_disc_real_2: 0.18090802431106567 (0.20972293226615243) + | > loss_disc_real_3: 0.1915249079465866 (0.2058241095231927) + | > loss_disc_real_4: 0.21097253262996674 (0.20144611823817957) + | > loss_disc_real_5: 0.21085073053836823 (0.21758524166501086) + | > loss_0: 2.344351291656494 (2.2818582783574644) + | > grad_norm_0: tensor(17.6775, device='cuda:0') (tensor(19.4571, device='cuda:0')) + | > loss_gen: 2.351529598236084 (2.613862690718278) + | > loss_kl: 1.9478061199188232 (1.6183329302331675) + | > loss_feat: 7.738176345825195 (8.426707972650943) + | > loss_mel: 17.086463928222656 (16.494155178899348) + | > loss_duration: 0.9211791753768921 (0.9381456271461819) + | > loss_1: 30.045154571533203 (30.091204270072605) + | > grad_norm_1: tensor(165.6642, device='cuda:0') (tensor(334.8993, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.9554 (0.9441277877144192) + | > loader_time: 0.0065 (0.005841265554013459) + + + --> TIME: 2024-07-13 23:46:24 -- STEP: 73/151 -- GLOBAL_STEP: 104100 + | > loss_disc: 2.221430540084839 (2.29357480349606) + | > loss_disc_real_0: 0.1323530226945877 (0.11727543476305596) + | > loss_disc_real_1: 0.2164105474948883 (0.20413706482273258) + | > loss_disc_real_2: 0.23796887695789337 (0.2107539145301466) + | > loss_disc_real_3: 0.18509933352470398 (0.20380387383781068) + | > loss_disc_real_4: 0.2351510226726532 (0.20718006065038785) + | > loss_disc_real_5: 0.23138366639614105 (0.21893686933876716) + | > loss_0: 2.221430540084839 (2.29357480349606) + | > grad_norm_0: tensor(49.1573, device='cuda:0') (tensor(25.2477, device='cuda:0')) + | > loss_gen: 2.771296977996826 (2.6464416817443017) + | > loss_kl: 1.3592549562454224 (1.7234449680537394) + | > loss_feat: 8.799789428710938 (8.482792508112235) + | > loss_mel: 17.545501708984375 (16.872181134681178) + | > loss_duration: 0.8861290216445923 (0.919685180873087) + | > loss_1: 31.36197280883789 (30.644545411410398) + | > grad_norm_1: tensor(452.5215, device='cuda:0') (tensor(379.7887, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.8521 (0.94121165798135) + | > loader_time: 0.006 (0.005927046684369648) + + + --> TIME: 2024-07-13 23:47:15 -- STEP: 123/151 -- GLOBAL_STEP: 104150 + | > loss_disc: 2.103358507156372 (2.2899021365778225) + | > loss_disc_real_0: 0.04099435359239578 (0.11845474063259799) + | > loss_disc_real_1: 0.16232669353485107 (0.20266473014664843) + | > loss_disc_real_2: 0.17099981009960175 (0.21062746795454645) + | > loss_disc_real_3: 0.16717346012592316 (0.20687527009626713) + | > loss_disc_real_4: 0.15900883078575134 (0.20478523740681212) + | > loss_disc_real_5: 0.17569132149219513 (0.2141413420923357) + | > loss_0: 2.103358507156372 (2.2899021365778225) + | > grad_norm_0: tensor(27.7754, device='cuda:0') (tensor(25.8701, device='cuda:0')) + | > loss_gen: 2.5892364978790283 (2.6336909910527675) + | > loss_kl: 1.4946359395980835 (1.7037908609320478) + | > loss_feat: 10.32106876373291 (8.449356424130075) + | > loss_mel: 16.713722229003906 (16.826831794366605) + | > loss_duration: 0.9108184576034546 (0.929327630899786) + | > loss_1: 32.029483795166016 (30.54299773239508) + | > grad_norm_1: tensor(310.6164, device='cuda:0') (tensor(386.3466, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.8245 (0.9619061035838553) + | > loader_time: 0.0041 (0.006146591853320113) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03569344373849722 (+0.0016292608701265754) + | > avg_loss_disc: 2.4500231742858887 (-0.13338435613192035) + | > avg_loss_disc_real_0: 0.17467732211718193 (-0.1653775773369349) + | > avg_loss_disc_real_1: 0.24330474436283112 (+0.024255492366277248) + | > avg_loss_disc_real_2: 0.22050144580694345 (+0.002933529707101673) + | > avg_loss_disc_real_3: 0.20791870584854713 (-0.016223687392014713) + | > avg_loss_disc_real_4: 0.2332411351112219 (-0.06616005072226888) + | > avg_loss_disc_real_5: 0.2096342948766855 (-0.03319201102623573) + | > avg_loss_0: 2.4500231742858887 (-0.13338435613192035) + | > avg_loss_gen: 2.461056085733267 (-0.24561285972595215) + | > avg_loss_kl: 3.6164756554823656 (-0.1493033812596245) + | > avg_loss_feat: 7.979913454789382 (-0.15973058113684946) + | > avg_loss_mel: 17.342238499568058 (+0.25730617229755026) + | > avg_loss_duration: 1.3001888715303858 (+0.001967861102177171) + | > avg_loss_1: 32.69987238370455 (-0.29537318303034965) + + + > EPOCH: 127/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:47:52)  + + --> TIME: 2024-07-13 23:48:16 -- STEP: 22/151 -- GLOBAL_STEP: 104200 + | > loss_disc: 2.3956072330474854 (2.3113131089643995) + | > loss_disc_real_0: 0.14590781927108765 (0.12468896975571458) + | > loss_disc_real_1: 0.21369370818138123 (0.2015794745900414) + | > loss_disc_real_2: 0.26095104217529297 (0.21115710179914127) + | > loss_disc_real_3: 0.25932660698890686 (0.20731822469017722) + | > loss_disc_real_4: 0.250053733587265 (0.2101751139218157) + | > loss_disc_real_5: 0.2840333878993988 (0.21806052191690964) + | > loss_0: 2.3956072330474854 (2.3113131089643995) + | > grad_norm_0: tensor(26.8517, device='cuda:0') (tensor(35.2217, device='cuda:0')) + | > loss_gen: 2.669206380844116 (2.653342778032476) + | > loss_kl: 1.9987164735794067 (1.725964990529147) + | > loss_feat: 7.791279315948486 (8.399551630020142) + | > loss_mel: 17.103456497192383 (16.965665470470082) + | > loss_duration: 0.91135174036026 (0.92361980676651) + | > loss_1: 30.474010467529297 (30.668144832957875) + | > grad_norm_1: tensor(633.3575, device='cuda:0') (tensor(519.4127, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 1.0874 (0.9769177328456532) + | > loader_time: 0.0087 (0.00601173530925404) + + + --> TIME: 2024-07-13 23:49:05 -- STEP: 72/151 -- GLOBAL_STEP: 104250 + | > loss_disc: 2.1586358547210693 (2.276615271965662) + | > loss_disc_real_0: 0.07343821227550507 (0.11925178083280723) + | > loss_disc_real_1: 0.1685217022895813 (0.20090168010857373) + | > loss_disc_real_2: 0.15426480770111084 (0.20578167132205435) + | > loss_disc_real_3: 0.12996554374694824 (0.20440470323794419) + | > loss_disc_real_4: 0.10352925211191177 (0.20432293135672808) + | > loss_disc_real_5: 0.15622062981128693 (0.216683572365178) + | > loss_0: 2.1586358547210693 (2.276615271965662) + | > grad_norm_0: tensor(44.4553, device='cuda:0') (tensor(31.8566, device='cuda:0')) + | > loss_gen: 2.6933231353759766 (2.6753561927212606) + | > loss_kl: 1.991622805595398 (1.7290900515185461) + | > loss_feat: 11.691926002502441 (8.601184560192955) + | > loss_mel: 17.15894889831543 (16.938535028033776) + | > loss_duration: 0.8584665060043335 (0.9374030662907494) + | > loss_1: 34.394287109375 (30.8815688557095) + | > grad_norm_1: tensor(538.7432, device='cuda:0') (tensor(502.7331, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.8148 (0.9686407181951735) + | > loader_time: 0.004 (0.006100604931513469) + + + --> TIME: 2024-07-13 23:49:57 -- STEP: 122/151 -- GLOBAL_STEP: 104300 + | > loss_disc: 2.3866591453552246 (2.2794721155870157) + | > loss_disc_real_0: 0.0989624485373497 (0.11739480565683763) + | > loss_disc_real_1: 0.22727803885936737 (0.2021430650206863) + | > loss_disc_real_2: 0.28731584548950195 (0.20914407184378048) + | > loss_disc_real_3: 0.2540152072906494 (0.2046690564907965) + | > loss_disc_real_4: 0.23715205490589142 (0.2042771721350365) + | > loss_disc_real_5: 0.18776974081993103 (0.2184934213024671) + | > loss_0: 2.3866591453552246 (2.2794721155870157) + | > grad_norm_0: tensor(29.4121, device='cuda:0') (tensor(28.5353, device='cuda:0')) + | > loss_gen: 2.523073673248291 (2.654209240538174) + | > loss_kl: 1.655642032623291 (1.7540383617408941) + | > loss_feat: 7.3794050216674805 (8.521322758471376) + | > loss_mel: 18.128948211669922 (16.844967373081882) + | > loss_duration: 0.913113534450531 (0.9441052733874712) + | > loss_1: 30.600181579589844 (30.718642797626433) + | > grad_norm_1: tensor(680.8638, device='cuda:0') (tensor(454.7749, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.9489 (0.9900787697463739) + | > loader_time: 0.0055 (0.006378863678603876) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033962524854219876 (-0.0017309188842773438) + | > avg_loss_disc: 2.5792916187873254 (+0.1292684445014367) + | > avg_loss_disc_real_0: 0.35135889741090626 (+0.17668157529372433) + | > avg_loss_disc_real_1: 0.21425876594506776 (-0.029045978417763352) + | > avg_loss_disc_real_2: 0.22247361907592186 (+0.0019721732689784166) + | > avg_loss_disc_real_3: 0.26560465074502504 (+0.05768594489647791) + | > avg_loss_disc_real_4: 0.2933330341027333 (+0.060091898991511405) + | > avg_loss_disc_real_5: 0.23016204627660605 (+0.020527751399920552) + | > avg_loss_0: 2.5792916187873254 (+0.1292684445014367) + | > avg_loss_gen: 2.7274795312147875 (+0.26642344548152064) + | > avg_loss_kl: 3.590171758945172 (-0.026303896537193783) + | > avg_loss_feat: 7.913916074312651 (-0.06599738047673132) + | > avg_loss_mel: 17.19505075307993 (-0.14718774648812882) + | > avg_loss_duration: 1.2806983544276311 (-0.019490517102754623) + | > avg_loss_1: 32.70731632526104 (+0.007443941556488198) + + + > EPOCH: 128/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:50:33)  + + --> TIME: 2024-07-13 23:50:59 -- STEP: 21/151 -- GLOBAL_STEP: 104350 + | > loss_disc: 2.416559934616089 (2.3131599766867503) + | > loss_disc_real_0: 0.2147044837474823 (0.13152856450705303) + | > loss_disc_real_1: 0.1948062926530838 (0.20344194698901402) + | > loss_disc_real_2: 0.18780599534511566 (0.21183987316631137) + | > loss_disc_real_3: 0.2242787778377533 (0.2069308842931475) + | > loss_disc_real_4: 0.24684076011180878 (0.2082685104438237) + | > loss_disc_real_5: 0.2565966546535492 (0.21447990692797161) + | > loss_0: 2.416559934616089 (2.3131599766867503) + | > grad_norm_0: tensor(30.5553, device='cuda:0') (tensor(25.7449, device='cuda:0')) + | > loss_gen: 2.4365615844726562 (2.593311627705892) + | > loss_kl: 1.8055392503738403 (1.7328931887944539) + | > loss_feat: 6.650722503662109 (8.047756649198986) + | > loss_mel: 16.865339279174805 (16.660344350905646) + | > loss_duration: 0.8320789337158203 (0.9368929891359239) + | > loss_1: 28.590242385864258 (29.97119903564453) + | > grad_norm_1: tensor(144.7823, device='cuda:0') (tensor(414.6753, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.9403 (1.0239305269150507) + | > loader_time: 0.007 (0.006575152987525577) + + + --> TIME: 2024-07-13 23:51:47 -- STEP: 71/151 -- GLOBAL_STEP: 104400 + | > loss_disc: 2.132174491882324 (2.3175071763320707) + | > loss_disc_real_0: 0.06406168639659882 (0.12875357417153643) + | > loss_disc_real_1: 0.18066495656967163 (0.20115382969379425) + | > loss_disc_real_2: 0.1789821982383728 (0.21097051362756272) + | > loss_disc_real_3: 0.18332742154598236 (0.205413098696252) + | > loss_disc_real_4: 0.1769511103630066 (0.20993012903441846) + | > loss_disc_real_5: 0.16831056773662567 (0.2144474021985497) + | > loss_0: 2.132174491882324 (2.3175071763320707) + | > grad_norm_0: tensor(17.6014, device='cuda:0') (tensor(27.4997, device='cuda:0')) + | > loss_gen: 2.6008594036102295 (2.5955189886227457) + | > loss_kl: 1.5434763431549072 (1.7432530295680946) + | > loss_feat: 9.148452758789062 (8.166470735845433) + | > loss_mel: 15.784578323364258 (16.682243615808627) + | > loss_duration: 1.0465720891952515 (0.9329842068779637) + | > loss_1: 30.123937606811523 (30.120470638006505) + | > grad_norm_1: tensor(441.2778, device='cuda:0') (tensor(400.4581, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.8105 (0.9795960271862193) + | > loader_time: 0.0041 (0.006193644563916703) + + + --> TIME: 2024-07-13 23:52:33 -- STEP: 121/151 -- GLOBAL_STEP: 104450 + | > loss_disc: 2.1594431400299072 (2.2863146460745933) + | > loss_disc_real_0: 0.08335624635219574 (0.12057236311110583) + | > loss_disc_real_1: 0.17595188319683075 (0.19961745969273825) + | > loss_disc_real_2: 0.2141972929239273 (0.21019206990387815) + | > loss_disc_real_3: 0.17412692308425903 (0.20253509738721137) + | > loss_disc_real_4: 0.18115301430225372 (0.20543004227571252) + | > loss_disc_real_5: 0.15567424893379211 (0.21597803020772854) + | > loss_0: 2.1594431400299072 (2.2863146460745933) + | > grad_norm_0: tensor(7.0102, device='cuda:0') (tensor(25.9743, device='cuda:0')) + | > loss_gen: 2.7581634521484375 (2.625623100060077) + | > loss_kl: 1.247620701789856 (1.741553272097564) + | > loss_feat: 9.185916900634766 (8.440458892790739) + | > loss_mel: 16.582521438598633 (16.748211245891483) + | > loss_duration: 0.9371410608291626 (0.9318803893633125) + | > loss_1: 30.71136474609375 (30.487726983945233) + | > grad_norm_1: tensor(240.4815, device='cuda:0') (tensor(409.4250, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.8208 (0.9476689070709482) + | > loader_time: 0.0045 (0.0059137797552692006) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03352543023916391 (-0.0004370946150559635) + | > avg_loss_disc: 2.550474625367385 (-0.028816993419940484) + | > avg_loss_disc_real_0: 0.2614507113511746 (-0.08990818605973167) + | > avg_loss_disc_real_1: 0.2230137586593628 (+0.008754992714295029) + | > avg_loss_disc_real_2: 0.21064482629299164 (-0.011828792782930225) + | > avg_loss_disc_real_3: 0.24022605900581068 (-0.025378591739214362) + | > avg_loss_disc_real_4: 0.2405396207020833 (-0.052793413400650024) + | > avg_loss_disc_real_5: 0.25652687136943525 (+0.026364825092829197) + | > avg_loss_0: 2.550474625367385 (-0.028816993419940484) + | > avg_loss_gen: 2.507865538963905 (-0.21961399225088263) + | > avg_loss_kl: 3.5967480769524207 (+0.006576318007248894) + | > avg_loss_feat: 7.906496964968168 (-0.007419109344482422) + | > avg_loss_mel: 16.897145124582146 (-0.2979056284977837) + | > avg_loss_duration: 1.3471002120238085 (+0.06640185759617734) + | > avg_loss_1: 32.25535627511832 (-0.45196005014272345) + + + > EPOCH: 129/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:53:14)  + + --> TIME: 2024-07-13 23:53:37 -- STEP: 20/151 -- GLOBAL_STEP: 104500 + | > loss_disc: 2.5655176639556885 (2.3190453171730043) + | > loss_disc_real_0: 0.1315435767173767 (0.1278277065604925) + | > loss_disc_real_1: 0.1823340505361557 (0.2019017182290554) + | > loss_disc_real_2: 0.21145810186862946 (0.21355659291148185) + | > loss_disc_real_3: 0.26405903697013855 (0.21339772194623946) + | > loss_disc_real_4: 0.2509368062019348 (0.20667065382003785) + | > loss_disc_real_5: 0.21201559901237488 (0.21352729946374893) + | > loss_0: 2.5655176639556885 (2.3190453171730043) + | > grad_norm_0: tensor(13.1261, device='cuda:0') (tensor(29.8595, device='cuda:0')) + | > loss_gen: 2.1812851428985596 (2.56503449678421) + | > loss_kl: 1.4685639142990112 (1.765163493156433) + | > loss_feat: 7.485080718994141 (8.07325267791748) + | > loss_mel: 15.273005485534668 (16.565730428695677) + | > loss_duration: 0.8990061283111572 (0.9248276770114898) + | > loss_1: 27.306941986083984 (29.894008731842042) + | > grad_norm_1: tensor(372.8119, device='cuda:0') (tensor(372.2147, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.8917 (1.0274000287055967) + | > loader_time: 0.0061 (0.006934213638305664) + + + --> TIME: 2024-07-13 23:54:27 -- STEP: 70/151 -- GLOBAL_STEP: 104550 + | > loss_disc: 2.106689691543579 (2.321448244367327) + | > loss_disc_real_0: 0.07760678231716156 (0.12136930791395048) + | > loss_disc_real_1: 0.17163263261318207 (0.20191516195024764) + | > loss_disc_real_2: 0.18580946326255798 (0.21142780174102102) + | > loss_disc_real_3: 0.18751710653305054 (0.20785236358642578) + | > loss_disc_real_4: 0.18123947083950043 (0.209710898569652) + | > loss_disc_real_5: 0.25188496708869934 (0.22048821662153517) + | > loss_0: 2.106689691543579 (2.321448244367327) + | > grad_norm_0: tensor(16.1175, device='cuda:0') (tensor(28.0546, device='cuda:0')) + | > loss_gen: 2.9657363891601562 (2.610712729181562) + | > loss_kl: 1.4700626134872437 (1.7721493363380432) + | > loss_feat: 9.506394386291504 (8.231705754143853) + | > loss_mel: 16.556625366210938 (16.6563711983817) + | > loss_duration: 0.9325648546218872 (0.9267707134996142) + | > loss_1: 31.43138313293457 (30.197709710257396) + | > grad_norm_1: tensor(339.9298, device='cuda:0') (tensor(422.7115, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.8184 (0.9897916078567505) + | > loader_time: 0.0057 (0.006416123253958566) + + + --> TIME: 2024-07-13 23:55:16 -- STEP: 120/151 -- GLOBAL_STEP: 104600 + | > loss_disc: 2.3693199157714844 (2.319829438130061) + | > loss_disc_real_0: 0.1306471824645996 (0.12016968724007407) + | > loss_disc_real_1: 0.2085973620414734 (0.20232752288381256) + | > loss_disc_real_2: 0.18011236190795898 (0.21168802846223117) + | > loss_disc_real_3: 0.20725849270820618 (0.20861642037828762) + | > loss_disc_real_4: 0.1725502610206604 (0.2094883798311154) + | > loss_disc_real_5: 0.26164087653160095 (0.22185990897317728) + | > loss_0: 2.3693199157714844 (2.319829438130061) + | > grad_norm_0: tensor(48.0322, device='cuda:0') (tensor(27.7891, device='cuda:0')) + | > loss_gen: 2.422252655029297 (2.6144772787888835) + | > loss_kl: 1.4884209632873535 (1.7638516187667848) + | > loss_feat: 7.322947978973389 (8.270101141929633) + | > loss_mel: 15.231143951416016 (16.729323124885557) + | > loss_duration: 0.8690294027328491 (0.929480475684007) + | > loss_1: 27.33379554748535 (30.307233651479084) + | > grad_norm_1: tensor(276.4501, device='cuda:0') (tensor(429.1660, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.8645 (0.9863625665505728) + | > loader_time: 0.0048 (0.006376798947652181) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033611260927640475 (+8.58306884765625e-05) + | > avg_loss_disc: 2.4291296188647933 (-0.12134500650259161) + | > avg_loss_disc_real_0: 0.26175659321821654 (+0.0003058818670419461) + | > avg_loss_disc_real_1: 0.22871169104025915 (+0.005697932380896359) + | > avg_loss_disc_real_2: 0.2396301913719911 (+0.02898536507899946) + | > avg_loss_disc_real_3: 0.2176092118024826 (-0.022616847203328072) + | > avg_loss_disc_real_4: 0.2640944581765395 (+0.02355483747445619) + | > avg_loss_disc_real_5: 0.20721106231212616 (-0.04931580905730909) + | > avg_loss_0: 2.4291296188647933 (-0.12134500650259161) + | > avg_loss_gen: 2.7082903935359073 (+0.20042485457200243) + | > avg_loss_kl: 3.8235932680276723 (+0.22684519107525158) + | > avg_loss_feat: 8.027613713191105 (+0.12111674822293672) + | > avg_loss_mel: 17.10819303072416 (+0.21104790614201363) + | > avg_loss_duration: 1.3267634098346417 (-0.02033680218916678) + | > avg_loss_1: 32.99445372361403 (+0.7390974484957127) + + + > EPOCH: 130/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:55:56)  + + --> TIME: 2024-07-13 23:56:17 -- STEP: 19/151 -- GLOBAL_STEP: 104650 + | > loss_disc: 2.105990409851074 (2.2730914166099145) + | > loss_disc_real_0: 0.10588578879833221 (0.11432632254926782) + | > loss_disc_real_1: 0.1501973271369934 (0.1946804139174913) + | > loss_disc_real_2: 0.16621650755405426 (0.20735805050322884) + | > loss_disc_real_3: 0.16115550696849823 (0.1981569508972921) + | > loss_disc_real_4: 0.18473735451698303 (0.2070008884919317) + | > loss_disc_real_5: 0.21458527445793152 (0.2131635938820086) + | > loss_0: 2.105990409851074 (2.2730914166099145) + | > grad_norm_0: tensor(26.0477, device='cuda:0') (tensor(33.9743, device='cuda:0')) + | > loss_gen: 2.858961582183838 (2.673115027578254) + | > loss_kl: 1.5007596015930176 (1.725845073398791) + | > loss_feat: 9.790560722351074 (8.619108200073242) + | > loss_mel: 16.51935577392578 (16.97330951690674) + | > loss_duration: 0.8945408463478088 (0.9382809839750591) + | > loss_1: 31.564178466796875 (30.929658889770508) + | > grad_norm_1: tensor(564.5711, device='cuda:0') (tensor(461.6847, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.8681 (0.9796754184522127) + | > loader_time: 0.0057 (0.006173472655446906) + + + --> TIME: 2024-07-13 23:57:08 -- STEP: 69/151 -- GLOBAL_STEP: 104700 + | > loss_disc: 2.287090301513672 (2.2712176295294277) + | > loss_disc_real_0: 0.10810874402523041 (0.10832796402383542) + | > loss_disc_real_1: 0.16892890632152557 (0.19915184616178705) + | > loss_disc_real_2: 0.23032057285308838 (0.2109467566445254) + | > loss_disc_real_3: 0.22489099204540253 (0.20235769742208976) + | > loss_disc_real_4: 0.18444634974002838 (0.20705751729184302) + | > loss_disc_real_5: 0.29143911600112915 (0.21911194510218027) + | > loss_0: 2.287090301513672 (2.2712176295294277) + | > grad_norm_0: tensor(19.4782, device='cuda:0') (tensor(30.4202, device='cuda:0')) + | > loss_gen: 2.6385293006896973 (2.663952523383541) + | > loss_kl: 1.6897573471069336 (1.7325816923293516) + | > loss_feat: 8.701557159423828 (8.42649807446245) + | > loss_mel: 17.650833129882812 (16.764904063680902) + | > loss_duration: 0.8831278085708618 (0.9279673643734144) + | > loss_1: 31.563804626464844 (30.51590369404226) + | > grad_norm_1: tensor(451.6303, device='cuda:0') (tensor(501.1003, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.9009 (0.9875691766324255) + | > loader_time: 0.0055 (0.006232154542121334) + + + --> TIME: 2024-07-13 23:57:58 -- STEP: 119/151 -- GLOBAL_STEP: 104750 + | > loss_disc: 2.344377040863037 (2.285269965644645) + | > loss_disc_real_0: 0.19616059958934784 (0.11349900126331995) + | > loss_disc_real_1: 0.2812708616256714 (0.20194328144317916) + | > loss_disc_real_2: 0.27552375197410583 (0.21157256106869513) + | > loss_disc_real_3: 0.2459121197462082 (0.2031086872981376) + | > loss_disc_real_4: 0.2788517475128174 (0.20774469055047556) + | > loss_disc_real_5: 0.20128367841243744 (0.21799204404614553) + | > loss_0: 2.344377040863037 (2.285269965644645) + | > grad_norm_0: tensor(66.3519, device='cuda:0') (tensor(30.4086, device='cuda:0')) + | > loss_gen: 2.931154727935791 (2.653267880447772) + | > loss_kl: 1.9902124404907227 (1.7480460079778142) + | > loss_feat: 8.24323558807373 (8.365580274277377) + | > loss_mel: 18.33475685119629 (16.796256722522372) + | > loss_duration: 0.9396805763244629 (0.9343306071618024) + | > loss_1: 32.43904113769531 (30.497481546482117) + | > grad_norm_1: tensor(556.3752, device='cuda:0') (tensor(488.8141, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 1.0762 (0.9935020258446705) + | > loader_time: 0.0063 (0.00634876820219665) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03218656319838304 (-0.0014246977292574348) + | > avg_loss_disc: 2.5840128201704764 (+0.15488320130568312) + | > avg_loss_disc_real_0: 0.2643071940312018 (+0.0025506008129852487) + | > avg_loss_disc_real_1: 0.22890423696774703 (+0.00019254592748788046) + | > avg_loss_disc_real_2: 0.23910828966360825 (-0.0005219017083828437) + | > avg_loss_disc_real_3: 0.24679154272262865 (+0.02918233092014605) + | > avg_loss_disc_real_4: 0.22640100350746742 (-0.03769345466907206) + | > avg_loss_disc_real_5: 0.2693931127970035 (+0.06218205048487735) + | > avg_loss_0: 2.5840128201704764 (+0.15488320130568312) + | > avg_loss_gen: 2.5652863795940695 (-0.14300401394183782) + | > avg_loss_kl: 3.863136053085327 (+0.03954278505765485) + | > avg_loss_feat: 7.841527021848238 (-0.18608669134286693) + | > avg_loss_mel: 16.831970288203312 (-0.2762227425208472) + | > avg_loss_duration: 1.3323149130894587 (+0.005551503254817014) + | > avg_loss_1: 32.434235059298004 (-0.5602186643160252) + + + > EPOCH: 131/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-13 23:58:39)  + + --> TIME: 2024-07-13 23:58:58 -- STEP: 18/151 -- GLOBAL_STEP: 104800 + | > loss_disc: 2.16696834564209 (2.3081993659337363) + | > loss_disc_real_0: 0.07406985759735107 (0.10548742550114791) + | > loss_disc_real_1: 0.13796627521514893 (0.20192784070968628) + | > loss_disc_real_2: 0.181422621011734 (0.21296222673522103) + | > loss_disc_real_3: 0.1891983598470688 (0.1988846684495608) + | > loss_disc_real_4: 0.15280701220035553 (0.20673420114649665) + | > loss_disc_real_5: 0.19774313271045685 (0.22522777277562353) + | > loss_0: 2.16696834564209 (2.3081993659337363) + | > grad_norm_0: tensor(19.7474, device='cuda:0') (tensor(26.6846, device='cuda:0')) + | > loss_gen: 2.6456005573272705 (2.662661419974433) + | > loss_kl: 1.0426057577133179 (1.6458669635984633) + | > loss_feat: 9.589574813842773 (8.662506553861832) + | > loss_mel: 15.395345687866211 (16.972655720180935) + | > loss_duration: 1.0630910396575928 (0.926155322127872) + | > loss_1: 29.73621940612793 (30.86984592013889) + | > grad_norm_1: tensor(190.0136, device='cuda:0') (tensor(466.9914, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.8164 (0.8955520656373765) + | > loader_time: 0.0045 (0.0051153235965304905) + + + --> TIME: 2024-07-13 23:59:49 -- STEP: 68/151 -- GLOBAL_STEP: 104850 + | > loss_disc: 2.1578707695007324 (2.312605451135074) + | > loss_disc_real_0: 0.055133044719696045 (0.11911267721477677) + | > loss_disc_real_1: 0.19178345799446106 (0.2035001691649942) + | > loss_disc_real_2: 0.1748037338256836 (0.2104163759333246) + | > loss_disc_real_3: 0.1746475249528885 (0.20672794234226732) + | > loss_disc_real_4: 0.1965048462152481 (0.20999649276628213) + | > loss_disc_real_5: 0.15780435502529144 (0.21842562790740938) + | > loss_0: 2.1578707695007324 (2.312605451135074) + | > grad_norm_0: tensor(26.0981, device='cuda:0') (tensor(30.5786, device='cuda:0')) + | > loss_gen: 2.5007731914520264 (2.5904912562931286) + | > loss_kl: 1.3626230955123901 (1.723190434715327) + | > loss_feat: 8.849392890930176 (8.174289801541498) + | > loss_mel: 16.406557083129883 (16.468788483563593) + | > loss_duration: 1.07376229763031 (0.9369121491909027) + | > loss_1: 30.19310760498047 (29.893672101637897) + | > grad_norm_1: tensor(442.6701, device='cuda:0') (tensor(511.6098, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.818 (0.9865546892671024) + | > loader_time: 0.0055 (0.006303538294399486) + + + --> TIME: 2024-07-14 00:00:39 -- STEP: 118/151 -- GLOBAL_STEP: 104900 + | > loss_disc: 2.0915584564208984 (2.301392476437455) + | > loss_disc_real_0: 0.054602816700935364 (0.11897885581573188) + | > loss_disc_real_1: 0.18339934945106506 (0.2012121835249966) + | > loss_disc_real_2: 0.14556805789470673 (0.21006019804942405) + | > loss_disc_real_3: 0.18664981424808502 (0.2058327223916175) + | > loss_disc_real_4: 0.2002677470445633 (0.2065504070813373) + | > loss_disc_real_5: 0.19090792536735535 (0.21920354405449607) + | > loss_0: 2.0915584564208984 (2.301392476437455) + | > grad_norm_0: tensor(25.1809, device='cuda:0') (tensor(32.6045, device='cuda:0')) + | > loss_gen: 2.761350631713867 (2.63300031322544) + | > loss_kl: 1.7682430744171143 (1.7475533844050715) + | > loss_feat: 10.218904495239258 (8.395807581432788) + | > loss_mel: 18.100515365600586 (16.68917725449902) + | > loss_duration: 0.9518202543258667 (0.9321011628134775) + | > loss_1: 33.80083465576172 (30.397639759516313) + | > grad_norm_1: tensor(642.4824, device='cuda:0') (tensor(545.4798, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.8613 (0.9862049494759508) + | > loader_time: 0.006 (0.006378349611314677) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032614671266995944 (+0.0004281080686129035) + | > avg_loss_disc: 2.5823453206282396 (-0.0016674995422367722) + | > avg_loss_disc_real_0: 0.20164924229566866 (-0.06265795173553312) + | > avg_loss_disc_real_1: 0.25059556044065034 (+0.021691323472903312) + | > avg_loss_disc_real_2: 0.24277078417631295 (+0.0036624945127047004) + | > avg_loss_disc_real_3: 0.23238589786566222 (-0.014405644856966437) + | > avg_loss_disc_real_4: 0.2593278701488788 (+0.032926866641411395) + | > avg_loss_disc_real_5: 0.21925860414138207 (-0.05013450865562144) + | > avg_loss_0: 2.5823453206282396 (-0.0016674995422367722) + | > avg_loss_gen: 2.4046657268817606 (-0.16062065271230885) + | > avg_loss_kl: 3.824009895324707 (-0.03912615776062012) + | > avg_loss_feat: 7.817445791684664 (-0.02408123016357422) + | > avg_loss_mel: 16.42013116983267 (-0.4118391183706436) + | > avg_loss_duration: 1.3475341154978826 (+0.015219202408423893) + | > avg_loss_1: 31.81378746032715 (-0.6204475989708556) + + + > EPOCH: 132/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:01:22)  + + --> TIME: 2024-07-14 00:01:40 -- STEP: 17/151 -- GLOBAL_STEP: 104950 + | > loss_disc: 2.159003973007202 (2.3052089775309845) + | > loss_disc_real_0: 0.10823914408683777 (0.11288385005558238) + | > loss_disc_real_1: 0.18310871720314026 (0.1982067998717813) + | > loss_disc_real_2: 0.2018839567899704 (0.21803564709775589) + | > loss_disc_real_3: 0.1891501098871231 (0.20985312233952916) + | > loss_disc_real_4: 0.22401711344718933 (0.21574077360770283) + | > loss_disc_real_5: 0.25563010573387146 (0.21717049269115224) + | > loss_0: 2.159003973007202 (2.3052089775309845) + | > grad_norm_0: tensor(44.3946, device='cuda:0') (tensor(33.7776, device='cuda:0')) + | > loss_gen: 3.062774419784546 (2.67547973464517) + | > loss_kl: 1.5949984788894653 (1.7302319232155294) + | > loss_feat: 9.82007122039795 (8.378194107728847) + | > loss_mel: 17.70321273803711 (16.767014166888067) + | > loss_duration: 0.9212675094604492 (0.8929801372920766) + | > loss_1: 33.102325439453125 (30.44390016443589) + | > grad_norm_1: tensor(594.3109, device='cuda:0') (tensor(566.5367, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.8632 (0.9351187313304228) + | > loader_time: 0.0049 (0.005457878112792969) + + + --> TIME: 2024-07-14 00:02:30 -- STEP: 67/151 -- GLOBAL_STEP: 105000 + | > loss_disc: 2.388489246368408 (2.291884130506373) + | > loss_disc_real_0: 0.1361631453037262 (0.11623409974263675) + | > loss_disc_real_1: 0.2387765645980835 (0.20253184148624762) + | > loss_disc_real_2: 0.26069536805152893 (0.21234715985718058) + | > loss_disc_real_3: 0.2611233592033386 (0.20669135851646536) + | > loss_disc_real_4: 0.2340848445892334 (0.20967875332085054) + | > loss_disc_real_5: 0.2726430296897888 (0.21299846247950596) + | > loss_0: 2.388489246368408 (2.291884130506373) + | > grad_norm_0: tensor(44.4050, device='cuda:0') (tensor(36.1204, device='cuda:0')) + | > loss_gen: 2.787754774093628 (2.670636771330193) + | > loss_kl: 2.1737451553344727 (1.7025890795152578) + | > loss_feat: 7.418806076049805 (8.350496989577561) + | > loss_mel: 17.738515853881836 (16.82255074515271) + | > loss_duration: 1.1951574087142944 (0.9265959725451114) + | > loss_1: 31.31397819519043 (30.472869588367974) + | > grad_norm_1: tensor(864.1500, device='cuda:0') (tensor(580.2767, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 1.9576 (0.9616832519645122) + | > loader_time: 0.0145 (0.0060513838013606285) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_105000.pth + + --> TIME: 2024-07-14 00:03:53 -- STEP: 117/151 -- GLOBAL_STEP: 105050 + | > loss_disc: 2.286026954650879 (2.27882216934465) + | > loss_disc_real_0: 0.0631011500954628 (0.11625582342728591) + | > loss_disc_real_1: 0.1738019734621048 (0.20172213170772943) + | > loss_disc_real_2: 0.18857155740261078 (0.21120910715853047) + | > loss_disc_real_3: 0.23817473649978638 (0.20439283887290546) + | > loss_disc_real_4: 0.19360874593257904 (0.20803445170068333) + | > loss_disc_real_5: 0.1854170262813568 (0.21298732576716659) + | > loss_0: 2.286026954650879 (2.27882216934465) + | > grad_norm_0: tensor(35.1409, device='cuda:0') (tensor(37.6713, device='cuda:0')) + | > loss_gen: 2.6493189334869385 (2.6789858443105334) + | > loss_kl: 1.8080763816833496 (1.7111198861374814) + | > loss_feat: 8.953558921813965 (8.391501479678677) + | > loss_mel: 17.614818572998047 (16.790513943403198) + | > loss_duration: 0.9316171407699585 (0.9360844044603853) + | > loss_1: 31.95738983154297 (30.508205527933235) + | > grad_norm_1: tensor(680.1683, device='cuda:0') (tensor(587.8948, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.9677 (0.9984820092845167) + | > loader_time: 0.0062 (0.006352059861533662) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03569944088275616 (+0.0030847696157602147) + | > avg_loss_disc: 2.586847341977633 (+0.00450202134939337) + | > avg_loss_disc_real_0: 0.29437847091601443 (+0.09272922862034577) + | > avg_loss_disc_real_1: 0.23617850358669573 (-0.014417056853954613) + | > avg_loss_disc_real_2: 0.2374444088110557 (-0.005326375365257263) + | > avg_loss_disc_real_3: 0.2919643670320511 (+0.05957846916638887) + | > avg_loss_disc_real_4: 0.24166662991046906 (-0.01766124023840976) + | > avg_loss_disc_real_5: 0.26829453156544614 (+0.04903592742406407) + | > avg_loss_0: 2.586847341977633 (+0.00450202134939337) + | > avg_loss_gen: 2.7712922463050256 (+0.36662651942326496) + | > avg_loss_kl: 3.6695340413313646 (-0.1544758539933424) + | > avg_loss_feat: 8.294240511380709 (+0.4767947196960449) + | > avg_loss_mel: 17.047689364506645 (+0.627558194673977) + | > avg_loss_duration: 1.3122338239963238 (-0.03530029150155878) + | > avg_loss_1: 33.094989629892204 (+1.2812021695650557) + + + > EPOCH: 133/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:04:36)  + + --> TIME: 2024-07-14 00:04:54 -- STEP: 16/151 -- GLOBAL_STEP: 105100 + | > loss_disc: 2.499433994293213 (2.335826963186264) + | > loss_disc_real_0: 0.11493031680583954 (0.13157042115926743) + | > loss_disc_real_1: 0.2347024381160736 (0.1997591434046626) + | > loss_disc_real_2: 0.23099468648433685 (0.21078194119036198) + | > loss_disc_real_3: 0.2255261391401291 (0.21352963335812092) + | > loss_disc_real_4: 0.22475619614124298 (0.20098542515188456) + | > loss_disc_real_5: 0.1963612139225006 (0.2237534737214446) + | > loss_0: 2.499433994293213 (2.335826963186264) + | > grad_norm_0: tensor(25.0428, device='cuda:0') (tensor(18.4111, device='cuda:0')) + | > loss_gen: 2.363372564315796 (2.5776021629571915) + | > loss_kl: 1.8032711744308472 (1.8067328184843063) + | > loss_feat: 7.0427656173706055 (8.145249724388123) + | > loss_mel: 14.817642211914062 (16.497654736042023) + | > loss_duration: 0.8418774008750916 (0.9287351556122303) + | > loss_1: 26.868928909301758 (29.95597505569458) + | > grad_norm_1: tensor(189.0440, device='cuda:0') (tensor(181.7272, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.8832 (0.960673451423645) + | > loader_time: 0.004 (0.006006717681884766) + + + --> TIME: 2024-07-14 00:05:45 -- STEP: 66/151 -- GLOBAL_STEP: 105150 + | > loss_disc: 2.6039252281188965 (2.324572769078341) + | > loss_disc_real_0: 0.250691294670105 (0.1280515978507924) + | > loss_disc_real_1: 0.18921302258968353 (0.2040953974832188) + | > loss_disc_real_2: 0.19555264711380005 (0.2122623288270199) + | > loss_disc_real_3: 0.18917423486709595 (0.20887818458405408) + | > loss_disc_real_4: 0.2155141979455948 (0.2073451250114224) + | > loss_disc_real_5: 0.20929677784442902 (0.21729738694248776) + | > loss_0: 2.6039252281188965 (2.324572769078341) + | > grad_norm_0: tensor(55.5808, device='cuda:0') (tensor(25.5429, device='cuda:0')) + | > loss_gen: 2.2637081146240234 (2.572496024045077) + | > loss_kl: 2.26230525970459 (1.7509247714822942) + | > loss_feat: 6.9312567710876465 (8.062027244856864) + | > loss_mel: 15.587377548217773 (16.56069720875134) + | > loss_duration: 1.1915024518966675 (0.9401587058197368) + | > loss_1: 28.236148834228516 (29.88630416176536) + | > grad_norm_1: tensor(528.4909, device='cuda:0') (tensor(354.3400, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 2.1189 (1.0090592918973977) + | > loader_time: 0.0169 (0.00648347536722819) + + + --> TIME: 2024-07-14 00:06:34 -- STEP: 116/151 -- GLOBAL_STEP: 105200 + | > loss_disc: 2.340869426727295 (2.296710683354016) + | > loss_disc_real_0: 0.08200791478157043 (0.1220371417444328) + | > loss_disc_real_1: 0.19783391058444977 (0.20165040138466606) + | > loss_disc_real_2: 0.16042731702327728 (0.20936479958994636) + | > loss_disc_real_3: 0.15672993659973145 (0.205674583156561) + | > loss_disc_real_4: 0.1750311702489853 (0.20427641503769775) + | > loss_disc_real_5: 0.17263217270374298 (0.21710682801645378) + | > loss_0: 2.340869426727295 (2.296710683354016) + | > grad_norm_0: tensor(12.7863, device='cuda:0') (tensor(27.0064, device='cuda:0')) + | > loss_gen: 2.3902087211608887 (2.6118088344047807) + | > loss_kl: 1.8333441019058228 (1.77205336813269) + | > loss_feat: 8.212180137634277 (8.284649914708629) + | > loss_mel: 16.73322296142578 (16.662459209047523) + | > loss_duration: 0.8836590051651001 (0.9367391559584387) + | > loss_1: 30.052616119384766 (30.267710669287318) + | > grad_norm_1: tensor(233.9489, device='cuda:0') (tensor(424.1903, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.88 (0.9845568640478725) + | > loader_time: 0.0064 (0.006417151155142948) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0347441159761869 (-0.0009553249065692576) + | > avg_loss_disc: 2.4832850786355825 (-0.10356226334205054) + | > avg_loss_disc_real_0: 0.21530875334372887 (-0.07906971757228556) + | > avg_loss_disc_real_1: 0.19519062913381136 (-0.040987874452884376) + | > avg_loss_disc_real_2: 0.23718388034747198 (-0.00026052846358370907) + | > avg_loss_disc_real_3: 0.23217264849406022 (-0.05979171853799087) + | > avg_loss_disc_real_4: 0.26473752237283266 (+0.0230708924623636) + | > avg_loss_disc_real_5: 0.25389044559918916 (-0.014404085966256974) + | > avg_loss_0: 2.4832850786355825 (-0.10356226334205054) + | > avg_loss_gen: 2.567482984983004 (-0.20380926132202148) + | > avg_loss_kl: 3.731568079728347 (+0.06203403839698218) + | > avg_loss_feat: 7.794042990757869 (-0.50019752062284) + | > avg_loss_mel: 17.015841410710262 (-0.031847953796383166) + | > avg_loss_duration: 1.3578241146527803 (+0.04559029065645648) + | > avg_loss_1: 32.466759608342095 (-0.6282300215501095) + + + > EPOCH: 134/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:07:18)  + + --> TIME: 2024-07-14 00:07:34 -- STEP: 15/151 -- GLOBAL_STEP: 105250 + | > loss_disc: 2.374159812927246 (2.2366087118784588) + | > loss_disc_real_0: 0.11328445374965668 (0.12407652710874875) + | > loss_disc_real_1: 0.21163296699523926 (0.19550626476605734) + | > loss_disc_real_2: 0.273465096950531 (0.2030210683743159) + | > loss_disc_real_3: 0.2567737400531769 (0.19393850564956666) + | > loss_disc_real_4: 0.2618565261363983 (0.1974824185172717) + | > loss_disc_real_5: 0.2570502460002899 (0.21257973710695902) + | > loss_0: 2.374159812927246 (2.2366087118784588) + | > grad_norm_0: tensor(34.5532, device='cuda:0') (tensor(33.4605, device='cuda:0')) + | > loss_gen: 2.7975869178771973 (2.749610010782878) + | > loss_kl: 1.6849191188812256 (1.7330660025278728) + | > loss_feat: 9.498856544494629 (8.69888973236084) + | > loss_mel: 17.321226119995117 (17.179630152384444) + | > loss_duration: 0.8844118714332581 (0.8770051638285319) + | > loss_1: 32.1870002746582 (31.238200378417968) + | > grad_norm_1: tensor(523.9597, device='cuda:0') (tensor(584.6033, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.8843 (0.9220918337504069) + | > loader_time: 0.0041 (0.005572223663330078) + + + --> TIME: 2024-07-14 00:08:25 -- STEP: 65/151 -- GLOBAL_STEP: 105300 + | > loss_disc: 2.4128451347351074 (2.2893777975669276) + | > loss_disc_real_0: 0.15833720564842224 (0.12586874233988612) + | > loss_disc_real_1: 0.21169525384902954 (0.19943387164519383) + | > loss_disc_real_2: 0.21265748143196106 (0.20947100256498044) + | > loss_disc_real_3: 0.22656671702861786 (0.2034597240961515) + | > loss_disc_real_4: 0.22128403186798096 (0.2049479373372518) + | > loss_disc_real_5: 0.2577970027923584 (0.21702673526910635) + | > loss_0: 2.4128451347351074 (2.2893777975669276) + | > grad_norm_0: tensor(48.7120, device='cuda:0') (tensor(38.1266, device='cuda:0')) + | > loss_gen: 2.6036620140075684 (2.6963325317089373) + | > loss_kl: 1.4352940320968628 (1.6993233589025645) + | > loss_feat: 6.066873550415039 (8.537584906357985) + | > loss_mel: 15.737393379211426 (17.042357033949628) + | > loss_duration: 0.8955367803573608 (0.9320614631359394) + | > loss_1: 26.738759994506836 (30.907659149169923) + | > grad_norm_1: tensor(611.4214, device='cuda:0') (tensor(549.2028, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.8472 (0.9843600786649264) + | > loader_time: 0.0043 (0.006216137225811298) + + + --> TIME: 2024-07-14 00:09:14 -- STEP: 115/151 -- GLOBAL_STEP: 105350 + | > loss_disc: 2.290433883666992 (2.2934625563414204) + | > loss_disc_real_0: 0.12897387146949768 (0.1234875805352045) + | > loss_disc_real_1: 0.18943201005458832 (0.20214196702708367) + | > loss_disc_real_2: 0.23105297982692719 (0.21013558500486876) + | > loss_disc_real_3: 0.19398613274097443 (0.20364822162234264) + | > loss_disc_real_4: 0.254896342754364 (0.20624307841062545) + | > loss_disc_real_5: 0.2033955603837967 (0.21663231810797814) + | > loss_0: 2.290433883666992 (2.2934625563414204) + | > grad_norm_0: tensor(41.6966, device='cuda:0') (tensor(37.4199, device='cuda:0')) + | > loss_gen: 2.5828464031219482 (2.6634781630142883) + | > loss_kl: 1.767392635345459 (1.7323453623315563) + | > loss_feat: 7.111428260803223 (8.405345340397046) + | > loss_mel: 15.393926620483398 (16.98956942350968) + | > loss_duration: 0.9450112581253052 (0.9317476153373718) + | > loss_1: 27.80060386657715 (30.722485749617867) + | > grad_norm_1: tensor(676.9152, device='cuda:0') (tensor(553.0142, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.9452 (0.9759149053822394) + | > loader_time: 0.0058 (0.00623909908792247) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03498466198260968 (+0.00024054600642278018) + | > avg_loss_disc: 2.586716945354755 (+0.10343186671917248) + | > avg_loss_disc_real_0: 0.23949537712794083 (+0.024186623784211964) + | > avg_loss_disc_real_1: 0.2571532050004372 (+0.061962575866625846) + | > avg_loss_disc_real_2: 0.2891770349099086 (+0.05199315456243661) + | > avg_loss_disc_real_3: 0.2660515010356903 (+0.03387885254163009) + | > avg_loss_disc_real_4: 0.23220186967116135 (-0.0325356527016713) + | > avg_loss_disc_real_5: 0.24111571334875548 (-0.012774732250433685) + | > avg_loss_0: 2.586716945354755 (+0.10343186671917248) + | > avg_loss_gen: 2.6052568509028506 (+0.03777386591984655) + | > avg_loss_kl: 3.5584760996011586 (-0.17309198012718818) + | > avg_loss_feat: 7.527320861816406 (-0.2667221289414625) + | > avg_loss_mel: 16.771906485924355 (-0.24393492478590773) + | > avg_loss_duration: 1.315293312072754 (-0.042530802580026394) + | > avg_loss_1: 31.778253995455227 (-0.6885056128868676) + + + > EPOCH: 135/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:10:00)  + + --> TIME: 2024-07-14 00:10:15 -- STEP: 14/151 -- GLOBAL_STEP: 105400 + | > loss_disc: 2.2784018516540527 (2.3384149926049367) + | > loss_disc_real_0: 0.10468794405460358 (0.12686404506010668) + | > loss_disc_real_1: 0.1820451021194458 (0.20349157920905522) + | > loss_disc_real_2: 0.2689233124256134 (0.22335510700941086) + | > loss_disc_real_3: 0.25503259897232056 (0.21617612136261805) + | > loss_disc_real_4: 0.3033404052257538 (0.2192805249776159) + | > loss_disc_real_5: 0.2901633083820343 (0.23485448317868368) + | > loss_0: 2.2784018516540527 (2.3384149926049367) + | > grad_norm_0: tensor(41.5443, device='cuda:0') (tensor(34.2416, device='cuda:0')) + | > loss_gen: 2.797112464904785 (2.6142273460115706) + | > loss_kl: 1.78614342212677 (1.7820243835449219) + | > loss_feat: 9.609789848327637 (8.173622267586845) + | > loss_mel: 18.729066848754883 (16.90368720463344) + | > loss_duration: 0.9415640234947205 (0.9077405461243221) + | > loss_1: 33.86367416381836 (30.381301607404435) + | > grad_norm_1: tensor(613.3209, device='cuda:0') (tensor(537.7055, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.8781 (0.9245151962552752) + | > loader_time: 0.0044 (0.005486130714416504) + + + --> TIME: 2024-07-14 00:11:02 -- STEP: 64/151 -- GLOBAL_STEP: 105450 + | > loss_disc: 2.458031415939331 (2.2719030268490314) + | > loss_disc_real_0: 0.19581034779548645 (0.11631393653806299) + | > loss_disc_real_1: 0.25933903455734253 (0.19973691739141944) + | > loss_disc_real_2: 0.24414139986038208 (0.21092351619154215) + | > loss_disc_real_3: 0.26552125811576843 (0.20189692871645093) + | > loss_disc_real_4: 0.26845842599868774 (0.21198954014107585) + | > loss_disc_real_5: 0.2502075433731079 (0.21560714300721884) + | > loss_0: 2.458031415939331 (2.2719030268490314) + | > grad_norm_0: tensor(31.6936, device='cuda:0') (tensor(34.1568, device='cuda:0')) + | > loss_gen: 2.6173744201660156 (2.6689480803906913) + | > loss_kl: 1.7424112558364868 (1.7731008976697922) + | > loss_feat: 6.783576965332031 (8.421652413904669) + | > loss_mel: 16.719100952148438 (16.824498817324642) + | > loss_duration: 0.9487736821174622 (0.9257669122889638) + | > loss_1: 28.81123924255371 (30.613967269659042) + | > grad_norm_1: tensor(369.5424, device='cuda:0') (tensor(587.0996, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 1.0008 (0.9310168996453285) + | > loader_time: 0.0061 (0.005770012736320496) + + + --> TIME: 2024-07-14 00:11:55 -- STEP: 114/151 -- GLOBAL_STEP: 105500 + | > loss_disc: 2.24029803276062 (2.2898998574206715) + | > loss_disc_real_0: 0.12559042870998383 (0.11533313211903237) + | > loss_disc_real_1: 0.20592495799064636 (0.20216528692266397) + | > loss_disc_real_2: 0.1913391351699829 (0.210317188971921) + | > loss_disc_real_3: 0.16695725917816162 (0.20520017505214924) + | > loss_disc_real_4: 0.16898873448371887 (0.21318356431367105) + | > loss_disc_real_5: 0.2739601731300354 (0.21651967105112577) + | > loss_0: 2.24029803276062 (2.2898998574206715) + | > grad_norm_0: tensor(42.9534, device='cuda:0') (tensor(34.2032, device='cuda:0')) + | > loss_gen: 2.6674697399139404 (2.6503879229227705) + | > loss_kl: 1.9160984754562378 (1.7626144148801501) + | > loss_feat: 8.72022819519043 (8.332021968406542) + | > loss_mel: 17.871511459350586 (16.76202915425887) + | > loss_duration: 0.8817145824432373 (0.9369917433512839) + | > loss_1: 32.05702209472656 (30.444045301069295) + | > grad_norm_1: tensor(947.9547, device='cuda:0') (tensor(583.5139, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.9432 (0.9759682542399356) + | > loader_time: 0.0063 (0.006246903486419142) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03374112569368803 (-0.001243536288921654) + | > avg_loss_disc: 2.5693677021906924 (-0.0173492431640625) + | > avg_loss_disc_real_0: 0.22182038082526281 (-0.01767499630267802) + | > avg_loss_disc_real_1: 0.23099523553481469 (-0.026157969465622516) + | > avg_loss_disc_real_2: 0.257777968278298 (-0.03139906663161057) + | > avg_loss_disc_real_3: 0.27566855458112866 (+0.009617053545438348) + | > avg_loss_disc_real_4: 0.2666450131397981 (+0.03444314346863675) + | > avg_loss_disc_real_5: 0.22442355981239906 (-0.01669215353635642) + | > avg_loss_0: 2.5693677021906924 (-0.0173492431640625) + | > avg_loss_gen: 2.574475801908053 (-0.03078104899479772) + | > avg_loss_kl: 3.767833636357234 (+0.20935753675607538) + | > avg_loss_feat: 7.797353927905743 (+0.2700330660893364) + | > avg_loss_mel: 17.054586997398964 (+0.2826805114746094) + | > avg_loss_duration: 1.3466248237169707 (+0.03133151164421677) + | > avg_loss_1: 32.54087477463941 (+0.7626207791841857) + + + > EPOCH: 136/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:12:40)  + + --> TIME: 2024-07-14 00:12:56 -- STEP: 13/151 -- GLOBAL_STEP: 105550 + | > loss_disc: 2.467702865600586 (2.3538950039790225) + | > loss_disc_real_0: 0.08875288814306259 (0.12004059897019313) + | > loss_disc_real_1: 0.18548265099525452 (0.20658090596015638) + | > loss_disc_real_2: 0.20580628514289856 (0.21667753274624163) + | > loss_disc_real_3: 0.21229320764541626 (0.2175073898755587) + | > loss_disc_real_4: 0.27924278378486633 (0.2189303980423854) + | > loss_disc_real_5: 0.2362193614244461 (0.22201933998327988) + | > loss_0: 2.467702865600586 (2.3538950039790225) + | > grad_norm_0: tensor(64.7981, device='cuda:0') (tensor(39.2433, device='cuda:0')) + | > loss_gen: 2.3412280082702637 (2.6389539608588586) + | > loss_kl: 1.497047781944275 (1.7073273888001075) + | > loss_feat: 6.9047393798828125 (8.036906425769512) + | > loss_mel: 15.34587287902832 (16.674310757563667) + | > loss_duration: 0.8372607827186584 (0.9529250447566693) + | > loss_1: 26.926149368286133 (30.010423073401817) + | > grad_norm_1: tensor(765.4160, device='cuda:0') (tensor(632.5784, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.9528 (1.049629669923049) + | > loader_time: 0.0049 (0.007541014597966121) + + + --> TIME: 2024-07-14 00:13:44 -- STEP: 63/151 -- GLOBAL_STEP: 105600 + | > loss_disc: 2.4320244789123535 (2.3160096898911493) + | > loss_disc_real_0: 0.14927469193935394 (0.12100536790159014) + | > loss_disc_real_1: 0.2635675370693207 (0.205040409924492) + | > loss_disc_real_2: 0.22289112210273743 (0.21316689891474588) + | > loss_disc_real_3: 0.2379589080810547 (0.20693754132777925) + | > loss_disc_real_4: 0.1957724392414093 (0.20696575703128936) + | > loss_disc_real_5: 0.23646245896816254 (0.21792120474671564) + | > loss_0: 2.4320244789123535 (2.3160096898911493) + | > grad_norm_0: tensor(22.6201, device='cuda:0') (tensor(35.0919, device='cuda:0')) + | > loss_gen: 2.8331801891326904 (2.6423130527375243) + | > loss_kl: 1.8188793659210205 (1.681585240931738) + | > loss_feat: 9.964929580688477 (8.226302396683465) + | > loss_mel: 18.011207580566406 (16.677277731517005) + | > loss_duration: 0.9558577537536621 (0.9232390835171654) + | > loss_1: 33.58405303955078 (30.15071747794984) + | > grad_norm_1: tensor(547.8098, device='cuda:0') (tensor(529.6365, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.9306 (0.9577070001571899) + | > loader_time: 0.0067 (0.006105214830428835) + + + --> TIME: 2024-07-14 00:14:34 -- STEP: 113/151 -- GLOBAL_STEP: 105650 + | > loss_disc: 2.2878451347351074 (2.3104036012582014) + | > loss_disc_real_0: 0.11127429455518723 (0.12020112165307577) + | > loss_disc_real_1: 0.19015394151210785 (0.20388250026555188) + | > loss_disc_real_2: 0.2128034085035324 (0.21234800317118654) + | > loss_disc_real_3: 0.20673540234565735 (0.20486839352983288) + | > loss_disc_real_4: 0.19354170560836792 (0.20762564512216938) + | > loss_disc_real_5: 0.25711166858673096 (0.21823878217059958) + | > loss_0: 2.2878451347351074 (2.3104036012582014) + | > grad_norm_0: tensor(10.6723, device='cuda:0') (tensor(31.9384, device='cuda:0')) + | > loss_gen: 2.8292551040649414 (2.62672302575238) + | > loss_kl: 1.9468474388122559 (1.6822691074514808) + | > loss_feat: 10.477564811706543 (8.236481455575047) + | > loss_mel: 17.616838455200195 (16.677866513750196) + | > loss_duration: 0.8090267181396484 (0.935991567848003) + | > loss_1: 33.679534912109375 (30.15933178589407) + | > grad_norm_1: tensor(318.9335, device='cuda:0') (tensor(463.0919, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.8897 (0.9748113640641745) + | > loader_time: 0.0049 (0.0063198794305852025) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033308799450214095 (-0.0004323262434739322) + | > avg_loss_disc: 2.5732666895939755 (+0.003898987403283094) + | > avg_loss_disc_real_0: 0.22017876230753386 (-0.0016416185177289544) + | > avg_loss_disc_real_1: 0.20966030771915728 (-0.021334927815657406) + | > avg_loss_disc_real_2: 0.22769690706179693 (-0.030081061216501087) + | > avg_loss_disc_real_3: 0.20954166925870454 (-0.06612688532242411) + | > avg_loss_disc_real_4: 0.19503299433451432 (-0.07161201880528378) + | > avg_loss_disc_real_5: 0.2127473675287687 (-0.011676192283630371) + | > avg_loss_0: 2.5732666895939755 (+0.003898987403283094) + | > avg_loss_gen: 2.2685377322710476 (-0.3059380696370053) + | > avg_loss_kl: 3.69231156202463 (-0.07552207433260394) + | > avg_loss_feat: 8.062349796295166 (+0.26499586838942335) + | > avg_loss_mel: 16.96873327401968 (-0.08585372337928376) + | > avg_loss_duration: 1.3508872068845308 (+0.004262383167560113) + | > avg_loss_1: 32.34281906714806 (-0.19805570749134915) + + + > EPOCH: 137/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:15:21)  + + --> TIME: 2024-07-14 00:15:35 -- STEP: 12/151 -- GLOBAL_STEP: 105700 + | > loss_disc: 2.258877754211426 (2.2549067139625554) + | > loss_disc_real_0: 0.06554247438907623 (0.11942798520127933) + | > loss_disc_real_1: 0.15154603123664856 (0.19307947531342506) + | > loss_disc_real_2: 0.19834546744823456 (0.20697084193428358) + | > loss_disc_real_3: 0.19255763292312622 (0.20603241895635924) + | > loss_disc_real_4: 0.1588926464319229 (0.1990117368598779) + | > loss_disc_real_5: 0.22646665573120117 (0.21410385643442473) + | > loss_0: 2.258877754211426 (2.2549067139625554) + | > grad_norm_0: tensor(11.1264, device='cuda:0') (tensor(30.5410, device='cuda:0')) + | > loss_gen: 2.736116409301758 (2.734443426132202) + | > loss_kl: 1.896508812904358 (1.6805114646752675) + | > loss_feat: 9.40189266204834 (8.826600511868795) + | > loss_mel: 17.499170303344727 (17.281420866648357) + | > loss_duration: 0.8942248225212097 (0.8967408140500387) + | > loss_1: 32.427913665771484 (31.41971715291341) + | > grad_norm_1: tensor(457.8946, device='cuda:0') (tensor(535.6528, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.8853 (0.9323338667551676) + | > loader_time: 0.004 (0.005928536256154378) + + + --> TIME: 2024-07-14 00:16:28 -- STEP: 62/151 -- GLOBAL_STEP: 105750 + | > loss_disc: 2.2662532329559326 (2.2972257598753902) + | > loss_disc_real_0: 0.10921137779951096 (0.12370527631813483) + | > loss_disc_real_1: 0.2343873232603073 (0.20299593984119355) + | > loss_disc_real_2: 0.22253109514713287 (0.21196296330421202) + | > loss_disc_real_3: 0.2840014398097992 (0.20564642164015) + | > loss_disc_real_4: 0.23970557749271393 (0.20452101480576299) + | > loss_disc_real_5: 0.2087976634502411 (0.21523056996445503) + | > loss_0: 2.2662532329559326 (2.2972257598753902) + | > grad_norm_0: tensor(14.8406, device='cuda:0') (tensor(33.8330, device='cuda:0')) + | > loss_gen: 2.749190330505371 (2.661918194063248) + | > loss_kl: 2.1664164066314697 (1.7505141208248753) + | > loss_feat: 7.710055828094482 (8.336463828240671) + | > loss_mel: 16.59617805480957 (16.77277102008942) + | > loss_duration: 0.8778407573699951 (0.9283314087698536) + | > loss_1: 30.099679946899414 (30.449998455662882) + | > grad_norm_1: tensor(98.4008, device='cuda:0') (tensor(546.5000, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.8897 (1.02011840574203) + | > loader_time: 0.0067 (0.006548427766369235) + + + --> TIME: 2024-07-14 00:17:17 -- STEP: 112/151 -- GLOBAL_STEP: 105800 + | > loss_disc: 2.507498264312744 (2.2925179089818712) + | > loss_disc_real_0: 0.17296943068504333 (0.11927338556519577) + | > loss_disc_real_1: 0.21017304062843323 (0.20079297372805222) + | > loss_disc_real_2: 0.1685338020324707 (0.20917199631886824) + | > loss_disc_real_3: 0.26759710907936096 (0.20498786168172956) + | > loss_disc_real_4: 0.2628372013568878 (0.20578303314479335) + | > loss_disc_real_5: 0.25060516595840454 (0.21782100812665053) + | > loss_0: 2.507498264312744 (2.2925179089818712) + | > grad_norm_0: tensor(34.9798, device='cuda:0') (tensor(29.9552, device='cuda:0')) + | > loss_gen: 2.2987210750579834 (2.636201360395976) + | > loss_kl: 1.7501018047332764 (1.746121914259025) + | > loss_feat: 6.9044189453125 (8.27214819618634) + | > loss_mel: 14.815635681152344 (16.6591538786888) + | > loss_duration: 0.8883105516433716 (0.9359935245343617) + | > loss_1: 26.657190322875977 (30.249618887901306) + | > grad_norm_1: tensor(591.0710, device='cuda:0') (tensor(496.1351, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.9646 (0.9969158832515991) + | > loader_time: 0.0055 (0.006422027945518494) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03304041348970853 (-0.0002683859605055669) + | > avg_loss_disc: 2.5053377701685977 (-0.06792891942537782) + | > avg_loss_disc_real_0: 0.1708186142719709 (-0.04936014803556296) + | > avg_loss_disc_real_1: 0.2113000750541687 (+0.0016397673350114217) + | > avg_loss_disc_real_2: 0.19844812040145582 (-0.029248786660341114) + | > avg_loss_disc_real_3: 0.19212101858395797 (-0.017420650674746574) + | > avg_loss_disc_real_4: 0.20251362254986396 (+0.007480628215349644) + | > avg_loss_disc_real_5: 0.2089778150503452 (-0.003769552478423477) + | > avg_loss_0: 2.5053377701685977 (-0.06792891942537782) + | > avg_loss_gen: 2.212238495166485 (-0.05629923710456275) + | > avg_loss_kl: 3.8109913972707896 (+0.11867983524615955) + | > avg_loss_feat: 8.117432080782377 (+0.0550822844872112) + | > avg_loss_mel: 17.119711068960335 (+0.15097779494065477) + | > avg_loss_duration: 1.3282844470097468 (-0.022602759874783995) + | > avg_loss_1: 32.58865737915039 (+0.2458383120023271) + + + > EPOCH: 138/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:18:03)  + + --> TIME: 2024-07-14 00:18:18 -- STEP: 11/151 -- GLOBAL_STEP: 105850 + | > loss_disc: 2.4195666313171387 (2.3265776200727983) + | > loss_disc_real_0: 0.11868387460708618 (0.1399682414802638) + | > loss_disc_real_1: 0.15621189773082733 (0.19352122193033044) + | > loss_disc_real_2: 0.16943581402301788 (0.20475242083722894) + | > loss_disc_real_3: 0.2129366397857666 (0.2053883265365254) + | > loss_disc_real_4: 0.21791590750217438 (0.2054598033428192) + | > loss_disc_real_5: 0.2765745520591736 (0.22066469896923413) + | > loss_0: 2.4195666313171387 (2.3265776200727983) + | > grad_norm_0: tensor(32.6043, device='cuda:0') (tensor(36.3382, device='cuda:0')) + | > loss_gen: 2.4455578327178955 (2.601227131756869) + | > loss_kl: 2.078981399536133 (1.8745528567921033) + | > loss_feat: 8.534567832946777 (8.305956146933816) + | > loss_mel: 16.631444931030273 (17.044344208457254) + | > loss_duration: 0.9934121370315552 (0.9433810331604697) + | > loss_1: 30.6839656829834 (30.769461024891246) + | > grad_norm_1: tensor(605.7126, device='cuda:0') (tensor(431.6183, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 1.1597 (1.1162561069835315) + | > loader_time: 0.0087 (0.007985938679088245) + + + --> TIME: 2024-07-14 00:19:06 -- STEP: 61/151 -- GLOBAL_STEP: 105900 + | > loss_disc: 2.3300259113311768 (2.30494502333344) + | > loss_disc_real_0: 0.18266229331493378 (0.12342033901664076) + | > loss_disc_real_1: 0.2305988073348999 (0.199977380819008) + | > loss_disc_real_2: 0.18853537738323212 (0.20976632369346307) + | > loss_disc_real_3: 0.2502442002296448 (0.20503501300929022) + | > loss_disc_real_4: 0.1915317326784134 (0.20675629865927775) + | > loss_disc_real_5: 0.20621412992477417 (0.2186676530075855) + | > loss_0: 2.3300259113311768 (2.30494502333344) + | > grad_norm_0: tensor(34.4475, device='cuda:0') (tensor(33.8160, device='cuda:0')) + | > loss_gen: 2.576524019241333 (2.620535193896684) + | > loss_kl: 1.9984279870986938 (1.7070354008283772) + | > loss_feat: 8.005240440368652 (8.202393164400194) + | > loss_mel: 17.572423934936523 (16.660544583054836) + | > loss_duration: 0.9115108251571655 (0.9333615009901953) + | > loss_1: 31.06412696838379 (30.123869755229013) + | > grad_norm_1: tensor(696.9926, device='cuda:0') (tensor(466.7413, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.9844 (0.9753203861048965) + | > loader_time: 0.0073 (0.00615415807630195) + + + --> TIME: 2024-07-14 00:19:54 -- STEP: 111/151 -- GLOBAL_STEP: 105950 + | > loss_disc: 2.367264747619629 (2.2998481387490624) + | > loss_disc_real_0: 0.09853625297546387 (0.12011153700652423) + | > loss_disc_real_1: 0.23988761007785797 (0.20019131187383118) + | > loss_disc_real_2: 0.20394746959209442 (0.21059166727302311) + | > loss_disc_real_3: 0.21338140964508057 (0.20349325857183956) + | > loss_disc_real_4: 0.2179962694644928 (0.2071739834573892) + | > loss_disc_real_5: 0.27951890230178833 (0.22076812426786166) + | > loss_0: 2.367264747619629 (2.2998481387490624) + | > grad_norm_0: tensor(30.5758, device='cuda:0') (tensor(29.5231, device='cuda:0')) + | > loss_gen: 2.4875845909118652 (2.6257543434967867) + | > loss_kl: 1.8805739879608154 (1.740397166561436) + | > loss_feat: 7.890824794769287 (8.435450489456587) + | > loss_mel: 16.983280181884766 (16.878249408962485) + | > loss_duration: 0.9136908650398254 (0.9301939338177174) + | > loss_1: 30.155954360961914 (30.610045304169525) + | > grad_norm_1: tensor(420.9101, device='cuda:0') (tensor(423.8877, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.895 (0.9672738302935351) + | > loader_time: 0.0052 (0.006188525809898033) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03547219129709097 (+0.0024317778073824417) + | > avg_loss_disc: 2.5567328196305494 (+0.051395049461951725) + | > avg_loss_disc_real_0: 0.26487650206455815 (+0.09405788779258725) + | > avg_loss_disc_real_1: 0.2366888397015058 (+0.02538876464733711) + | > avg_loss_disc_real_2: 0.2717965967380084 (+0.07334847633655256) + | > avg_loss_disc_real_3: 0.24487124383449554 (+0.052750225250537575) + | > avg_loss_disc_real_4: 0.23609936925081107 (+0.033585746700947106) + | > avg_loss_disc_real_5: 0.21531773186646974 (+0.00633991681612453) + | > avg_loss_0: 2.5567328196305494 (+0.051395049461951725) + | > avg_loss_gen: 2.621951470008263 (+0.40971297484177827) + | > avg_loss_kl: 3.7947640419006348 (-0.01622735537015485) + | > avg_loss_feat: 7.684265833634597 (-0.4331662471477804) + | > avg_loss_mel: 17.11534412090595 (-0.004366948054386199) + | > avg_loss_duration: 1.3266522609270537 (-0.00163218608269311) + | > avg_loss_1: 32.542977993304916 (-0.045679385845474485) + + + > EPOCH: 139/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:20:43)  + + --> TIME: 2024-07-14 00:20:54 -- STEP: 10/151 -- GLOBAL_STEP: 106000 + | > loss_disc: 2.090123414993286 (2.157809484004974) + | > loss_disc_real_0: 0.09429915249347687 (0.1128342442214489) + | > loss_disc_real_1: 0.16545149683952332 (0.1936730608344078) + | > loss_disc_real_2: 0.18106608092784882 (0.2024053752422333) + | > loss_disc_real_3: 0.1551382690668106 (0.19626486599445342) + | > loss_disc_real_4: 0.183312326669693 (0.18951224982738496) + | > loss_disc_real_5: 0.24939565360546112 (0.21808741390705108) + | > loss_0: 2.090123414993286 (2.157809484004974) + | > grad_norm_0: tensor(15.2991, device='cuda:0') (tensor(17.8095, device='cuda:0')) + | > loss_gen: 2.727593183517456 (2.752052593231201) + | > loss_kl: 0.8719764947891235 (1.5874411106109618) + | > loss_feat: 9.781303405761719 (9.000337743759156) + | > loss_mel: 17.204212188720703 (16.366531085968017) + | > loss_duration: 1.0423498153686523 (0.9343002676963806) + | > loss_1: 31.62743377685547 (30.64066219329834) + | > grad_norm_1: tensor(142.6757, device='cuda:0') (tensor(267.6649, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.8279 (0.8694961071014404) + | > loader_time: 0.0045 (0.00511014461517334) + + + --> TIME: 2024-07-14 00:22:13 -- STEP: 60/151 -- GLOBAL_STEP: 106050 + | > loss_disc: 2.1796975135803223 (2.306249545017878) + | > loss_disc_real_0: 0.1761106699705124 (0.12815112049380936) + | > loss_disc_real_1: 0.22000780701637268 (0.20287717307607334) + | > loss_disc_real_2: 0.24254503846168518 (0.2114914819598198) + | > loss_disc_real_3: 0.166866272687912 (0.20551785764594874) + | > loss_disc_real_4: 0.20970837771892548 (0.2017074853181839) + | > loss_disc_real_5: 0.19129303097724915 (0.22089070479075115) + | > loss_0: 2.1796975135803223 (2.306249545017878) + | > grad_norm_0: tensor(40.1055, device='cuda:0') (tensor(28.0403, device='cuda:0')) + | > loss_gen: 2.7892303466796875 (2.6339051961898803) + | > loss_kl: 1.8726227283477783 (1.7322238524754843) + | > loss_feat: 9.44621467590332 (8.291055107116701) + | > loss_mel: 17.5794620513916 (16.758730363845824) + | > loss_duration: 0.8949942588806152 (0.9297318528095881) + | > loss_1: 32.582523345947266 (30.34564628601074) + | > grad_norm_1: tensor(839.2161, device='cuda:0') (tensor(438.7759, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.9258 (1.0057962298393248) + | > loader_time: 0.0054 (0.006273154417673747) + + + --> TIME: 2024-07-14 00:23:00 -- STEP: 110/151 -- GLOBAL_STEP: 106100 + | > loss_disc: 2.095907688140869 (2.288757279786197) + | > loss_disc_real_0: 0.09759627282619476 (0.12054983892224057) + | > loss_disc_real_1: 0.18310140073299408 (0.20138732113621452) + | > loss_disc_real_2: 0.19569623470306396 (0.2108331195332787) + | > loss_disc_real_3: 0.16423605382442474 (0.20426231439818032) + | > loss_disc_real_4: 0.15278516709804535 (0.2023260233077136) + | > loss_disc_real_5: 0.17291639745235443 (0.21777976670048454) + | > loss_0: 2.095907688140869 (2.288757279786197) + | > grad_norm_0: tensor(16.7997, device='cuda:0') (tensor(27.2609, device='cuda:0')) + | > loss_gen: 2.9671928882598877 (2.644399686293169) + | > loss_kl: 1.347303867340088 (1.7115761681036517) + | > loss_feat: 9.652727127075195 (8.436172168905081) + | > loss_mel: 16.67759132385254 (16.75375704331831) + | > loss_duration: 1.007498025894165 (0.9326322165402499) + | > loss_1: 31.652313232421875 (30.47853723005815) + | > grad_norm_1: tensor(546.0331, device='cuda:0') (tensor(465.8293, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.8197 (0.9658920179713856) + | > loader_time: 0.0045 (0.0060613805597478695) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.038939897830669694 (+0.003467706533578724) + | > avg_loss_disc: 2.5221118927001953 (-0.03462092693035412) + | > avg_loss_disc_real_0: 0.21311328961299017 (-0.05176321245156798) + | > avg_loss_disc_real_1: 0.2470031472352835 (+0.010314307533777684) + | > avg_loss_disc_real_2: 0.2287887575534674 (-0.043007839184540986) + | > avg_loss_disc_real_3: 0.23090730492885297 (-0.01396393890564257) + | > avg_loss_disc_real_4: 0.22835500767597786 (-0.007744361574833214) + | > avg_loss_disc_real_5: 0.24540431797504425 (+0.03008658610857451) + | > avg_loss_0: 2.5221118927001953 (-0.03462092693035412) + | > avg_loss_gen: 2.526864473636334 (-0.09508699637192919) + | > avg_loss_kl: 3.712606613452618 (-0.08215742844801666) + | > avg_loss_feat: 8.18067220541147 (+0.49640637177687363) + | > avg_loss_mel: 17.327688217163086 (+0.21234409625713724) + | > avg_loss_duration: 1.316746317423307 (-0.00990594350374674) + | > avg_loss_1: 33.06457856985239 (+0.5216005765474705) + + + > EPOCH: 140/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:23:50)  + + --> TIME: 2024-07-14 00:24:02 -- STEP: 9/151 -- GLOBAL_STEP: 106150 + | > loss_disc: 2.378329277038574 (2.3406894471910267) + | > loss_disc_real_0: 0.0853443369269371 (0.14303992440303168) + | > loss_disc_real_1: 0.1933245062828064 (0.20191762844721475) + | > loss_disc_real_2: 0.175758957862854 (0.19812391698360443) + | > loss_disc_real_3: 0.1957799792289734 (0.20659352673424614) + | > loss_disc_real_4: 0.1941520720720291 (0.2182212769985199) + | > loss_disc_real_5: 0.1603895127773285 (0.18816181851757896) + | > loss_0: 2.378329277038574 (2.3406894471910267) + | > grad_norm_0: tensor(23.8193, device='cuda:0') (tensor(40.2275, device='cuda:0')) + | > loss_gen: 2.466887950897217 (2.5940168963538275) + | > loss_kl: 1.8783352375030518 (1.785020695792304) + | > loss_feat: 8.15231990814209 (8.058067427741157) + | > loss_mel: 16.3890323638916 (17.043083826700848) + | > loss_duration: 0.9023658633232117 (0.8997300002310011) + | > loss_1: 29.788942337036133 (30.379919687906902) + | > grad_norm_1: tensor(282.9074, device='cuda:0') (tensor(473.7260, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.8982 (0.9402232699924045) + | > loader_time: 0.0046 (0.006075090832180447) + + + --> TIME: 2024-07-14 00:24:51 -- STEP: 59/151 -- GLOBAL_STEP: 106200 + | > loss_disc: 2.3803510665893555 (2.3130434933355297) + | > loss_disc_real_0: 0.04244993254542351 (0.1266395617592133) + | > loss_disc_real_1: 0.18739473819732666 (0.20029348084482096) + | > loss_disc_real_2: 0.20944081246852875 (0.21082398345914938) + | > loss_disc_real_3: 0.1908658742904663 (0.20496672884387485) + | > loss_disc_real_4: 0.22910098731517792 (0.20944733235795618) + | > loss_disc_real_5: 0.23549863696098328 (0.2180509888014551) + | > loss_0: 2.3803510665893555 (2.3130434933355297) + | > grad_norm_0: tensor(14.9352, device='cuda:0') (tensor(26.6071, device='cuda:0')) + | > loss_gen: 2.5118367671966553 (2.6254930738675397) + | > loss_kl: 1.5791295766830444 (1.7659738629551258) + | > loss_feat: 8.71160888671875 (8.319394919831872) + | > loss_mel: 17.0169620513916 (16.787269382153525) + | > loss_duration: 0.9110159873962402 (0.9114726266618502) + | > loss_1: 30.730554580688477 (30.409603991750945) + | > grad_norm_1: tensor(71.4654, device='cuda:0') (tensor(390.9168, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.8393 (0.961443682848397) + | > loader_time: 0.0044 (0.0060814922138796015) + + + --> TIME: 2024-07-14 00:25:41 -- STEP: 109/151 -- GLOBAL_STEP: 106250 + | > loss_disc: 2.542196750640869 (2.3057015335887945) + | > loss_disc_real_0: 0.19846010208129883 (0.12549487207460838) + | > loss_disc_real_1: 0.25295019149780273 (0.20074066656445144) + | > loss_disc_real_2: 0.3005353808403015 (0.211194899240765) + | > loss_disc_real_3: 0.2529503405094147 (0.20431055272117668) + | > loss_disc_real_4: 0.26808884739875793 (0.20883804279456444) + | > loss_disc_real_5: 0.22841738164424896 (0.21834238844180326) + | > loss_0: 2.542196750640869 (2.3057015335887945) + | > grad_norm_0: tensor(27.1827, device='cuda:0') (tensor(26.7879, device='cuda:0')) + | > loss_gen: 2.3556087017059326 (2.628556758985607) + | > loss_kl: 2.1779439449310303 (1.7626121875342973) + | > loss_feat: 5.582223892211914 (8.296236624411481) + | > loss_mel: 14.165523529052734 (16.66779111284728) + | > loss_duration: 1.1126811504364014 (0.9229691980081961) + | > loss_1: 25.393980026245117 (30.278165887255188) + | > grad_norm_1: tensor(156.5959, device='cuda:0') (tensor(406.0815, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 1.8799 (0.9751436819723986) + | > loader_time: 0.014 (0.006202973357034386) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03558986003582294 (-0.003350037794846751) + | > avg_loss_disc: 2.624090469800509 (+0.10197857710031366) + | > avg_loss_disc_real_0: 0.2683268739626958 (+0.05521358434970561) + | > avg_loss_disc_real_1: 0.2347836299584462 (-0.012219517276837288) + | > avg_loss_disc_real_2: 0.24251813957324395 (+0.013729382019776554) + | > avg_loss_disc_real_3: 0.19527269441347855 (-0.03563461051537442) + | > avg_loss_disc_real_4: 0.22749871130173022 (-0.0008562963742476393) + | > avg_loss_disc_real_5: 0.22799532000835127 (-0.017408997966692985) + | > avg_loss_0: 2.624090469800509 (+0.10197857710031366) + | > avg_loss_gen: 2.4462000590104322 (-0.08066441462590168) + | > avg_loss_kl: 3.5987534523010254 (-0.11385316115159272) + | > avg_loss_feat: 7.986132365006667 (-0.19453984040480332) + | > avg_loss_mel: 17.069061939532936 (-0.25862627763014956) + | > avg_loss_duration: 1.2825145813135 (-0.03423173610980701) + | > avg_loss_1: 32.38266211289626 (-0.6819164569561238) + + + > EPOCH: 141/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:26:32)  + + --> TIME: 2024-07-14 00:26:43 -- STEP: 8/151 -- GLOBAL_STEP: 106300 + | > loss_disc: 2.388997793197632 (2.2826536744832993) + | > loss_disc_real_0: 0.1244278997182846 (0.10604151245206594) + | > loss_disc_real_1: 0.22115999460220337 (0.19927546568214893) + | > loss_disc_real_2: 0.2667156457901001 (0.209366200491786) + | > loss_disc_real_3: 0.2273862659931183 (0.21311170421540737) + | > loss_disc_real_4: 0.2433720976114273 (0.21663709171116352) + | > loss_disc_real_5: 0.224086731672287 (0.20867222547531128) + | > loss_0: 2.388997793197632 (2.2826536744832993) + | > grad_norm_0: tensor(22.1661, device='cuda:0') (tensor(22.8685, device='cuda:0')) + | > loss_gen: 2.3698580265045166 (2.669634699821472) + | > loss_kl: 1.9872022867202759 (1.6720626950263977) + | > loss_feat: 6.362631320953369 (8.348811626434326) + | > loss_mel: 16.61351776123047 (16.72200870513916) + | > loss_duration: 0.9413751363754272 (0.9232428893446922) + | > loss_1: 28.274585723876953 (30.335761308670044) + | > grad_norm_1: tensor(309.2219, device='cuda:0') (tensor(508.0627, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 1.155 (0.9211803078651428) + | > loader_time: 0.0076 (0.005137413740158081) + + + --> TIME: 2024-07-14 00:27:34 -- STEP: 58/151 -- GLOBAL_STEP: 106350 + | > loss_disc: 2.1671414375305176 (2.273787582742757) + | > loss_disc_real_0: 0.0933946967124939 (0.1116312782055345) + | > loss_disc_real_1: 0.2211170643568039 (0.1977494929371209) + | > loss_disc_real_2: 0.23625849187374115 (0.20697622746229172) + | > loss_disc_real_3: 0.17140963673591614 (0.20185326116866079) + | > loss_disc_real_4: 0.16586855053901672 (0.20669756187447186) + | > loss_disc_real_5: 0.2449033260345459 (0.2194048996115553) + | > loss_0: 2.1671414375305176 (2.273787582742757) + | > grad_norm_0: tensor(20.4550, device='cuda:0') (tensor(28.6534, device='cuda:0')) + | > loss_gen: 2.9426488876342773 (2.6551500106679984) + | > loss_kl: 1.2351542711257935 (1.7134010185455453) + | > loss_feat: 9.551347732543945 (8.351216250452504) + | > loss_mel: 16.282960891723633 (16.581549776011503) + | > loss_duration: 0.891960859298706 (0.9325071172467594) + | > loss_1: 30.904071807861328 (30.233824203754292) + | > grad_norm_1: tensor(542.3316, device='cuda:0') (tensor(514.4755, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.8547 (0.9914426762482217) + | > loader_time: 0.0054 (0.006197810173034668) + + + --> TIME: 2024-07-14 00:28:22 -- STEP: 108/151 -- GLOBAL_STEP: 106400 + | > loss_disc: 2.33815860748291 (2.2800250881248036) + | > loss_disc_real_0: 0.13549858331680298 (0.11140961224144255) + | > loss_disc_real_1: 0.22350706160068512 (0.1991149247796447) + | > loss_disc_real_2: 0.20407621562480927 (0.20925003483339588) + | > loss_disc_real_3: 0.22338896989822388 (0.20523274579533826) + | > loss_disc_real_4: 0.19703450798988342 (0.20704140410655075) + | > loss_disc_real_5: 0.22927358746528625 (0.219225256945248) + | > loss_0: 2.33815860748291 (2.2800250881248036) + | > grad_norm_0: tensor(11.3198, device='cuda:0') (tensor(27.9147, device='cuda:0')) + | > loss_gen: 2.482163667678833 (2.6571573637149957) + | > loss_kl: 1.5005110502243042 (1.7468558323604089) + | > loss_feat: 8.028702735900879 (8.368427832921352) + | > loss_mel: 16.62171745300293 (16.656215756027805) + | > loss_duration: 0.8708269596099854 (0.928285054034657) + | > loss_1: 29.503923416137695 (30.35694196489122) + | > grad_norm_1: tensor(138.5056, device='cuda:0') (tensor(491.9333, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.8771 (0.9715350181968124) + | > loader_time: 0.0058 (0.0060382529541298195) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03505229949951172 (-0.000537560536311224) + | > avg_loss_disc: 2.6163102113283596 (-0.0077802584721493595) + | > avg_loss_disc_real_0: 0.33624372344750625 (+0.06791684948481047) + | > avg_loss_disc_real_1: 0.24873237311840057 (+0.013948743159954369) + | > avg_loss_disc_real_2: 0.2601589755370067 (+0.01764083596376273) + | > avg_loss_disc_real_3: 0.2751602392930251 (+0.07988754487954652) + | > avg_loss_disc_real_4: 0.2767610228978671 (+0.049262311596136865) + | > avg_loss_disc_real_5: 0.20486167875620034 (-0.02313364125215092) + | > avg_loss_0: 2.6163102113283596 (-0.0077802584721493595) + | > avg_loss_gen: 2.758510112762451 (+0.3123100537520189) + | > avg_loss_kl: 3.8530690303215613 (+0.25431557802053595) + | > avg_loss_feat: 7.9793506769033575 (-0.006781688103309591) + | > avg_loss_mel: 17.207971279437725 (+0.1389093399047887) + | > avg_loss_duration: 1.3331532386633067 (+0.05063865734980677) + | > avg_loss_1: 33.13205469571627 (+0.7493925828200076) + + + > EPOCH: 142/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:29:12)  + + --> TIME: 2024-07-14 00:29:21 -- STEP: 7/151 -- GLOBAL_STEP: 106450 + | > loss_disc: 2.3332343101501465 (2.249547617776053) + | > loss_disc_real_0: 0.1016935333609581 (0.0957507672054427) + | > loss_disc_real_1: 0.18253929913043976 (0.19324979186058044) + | > loss_disc_real_2: 0.1941290944814682 (0.19159671238490514) + | > loss_disc_real_3: 0.22262708842754364 (0.21092897653579712) + | > loss_disc_real_4: 0.23670880496501923 (0.21384571279798234) + | > loss_disc_real_5: 0.24990756809711456 (0.22570826751845224) + | > loss_0: 2.3332343101501465 (2.249547617776053) + | > grad_norm_0: tensor(15.3526, device='cuda:0') (tensor(14.8439, device='cuda:0')) + | > loss_gen: 2.567758083343506 (2.6288637774331227) + | > loss_kl: 1.4666407108306885 (1.6887269530977522) + | > loss_feat: 8.191758155822754 (9.017668451581683) + | > loss_mel: 16.530452728271484 (16.544854845319474) + | > loss_duration: 1.0041496753692627 (0.9444816197667804) + | > loss_1: 29.760759353637695 (30.824595315115793) + | > grad_norm_1: tensor(392.1957, device='cuda:0') (tensor(277.5138, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.8614 (0.8584450994219098) + | > loader_time: 0.004 (0.005622080394199916) + + + --> TIME: 2024-07-14 00:30:12 -- STEP: 57/151 -- GLOBAL_STEP: 106500 + | > loss_disc: 2.2531888484954834 (2.267300128936768) + | > loss_disc_real_0: 0.13655981421470642 (0.11351877679688889) + | > loss_disc_real_1: 0.1466262936592102 (0.2022122581276977) + | > loss_disc_real_2: 0.1796068251132965 (0.2086302787065506) + | > loss_disc_real_3: 0.1818830668926239 (0.20457740428677776) + | > loss_disc_real_4: 0.15287356078624725 (0.20222401331391252) + | > loss_disc_real_5: 0.22392621636390686 (0.2157032576046492) + | > loss_0: 2.2531888484954834 (2.267300128936768) + | > grad_norm_0: tensor(22.1412, device='cuda:0') (tensor(25.4132, device='cuda:0')) + | > loss_gen: 2.575875759124756 (2.686421247950772) + | > loss_kl: 1.9277137517929077 (1.780218517571165) + | > loss_feat: 9.132821083068848 (8.6694761912028) + | > loss_mel: 16.24991226196289 (16.991605457506687) + | > loss_duration: 0.9602965116500854 (0.9399863962541547) + | > loss_1: 30.846620559692383 (31.0677076640882) + | > grad_norm_1: tensor(210.1105, device='cuda:0') (tensor(368.5353, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.8644 (0.9968362858420924) + | > loader_time: 0.0063 (0.006563383236266019) + + + --> TIME: 2024-07-14 00:31:01 -- STEP: 107/151 -- GLOBAL_STEP: 106550 + | > loss_disc: 2.152928590774536 (2.2720471119212218) + | > loss_disc_real_0: 0.05180227383971214 (0.11239358476389234) + | > loss_disc_real_1: 0.1861889362335205 (0.20057119359479886) + | > loss_disc_real_2: 0.19078007340431213 (0.2101904270804931) + | > loss_disc_real_3: 0.1587955802679062 (0.20662444182366968) + | > loss_disc_real_4: 0.12912867963314056 (0.2048419293399169) + | > loss_disc_real_5: 0.1984374225139618 (0.21285336722280376) + | > loss_0: 2.152928590774536 (2.2720471119212218) + | > grad_norm_0: tensor(45.0736, device='cuda:0') (tensor(29.0485, device='cuda:0')) + | > loss_gen: 2.810004949569702 (2.6825334825248364) + | > loss_kl: 2.051776647567749 (1.7838257640321677) + | > loss_feat: 9.82983112335205 (8.624576907291589) + | > loss_mel: 17.536865234375 (16.940469198137794) + | > loss_duration: 0.8783209323883057 (0.936191910338179) + | > loss_1: 33.1068000793457 (30.967597177095502) + | > grad_norm_1: tensor(746.1375, device='cuda:0') (tensor(462.4663, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.8826 (0.9845457411258021) + | > loader_time: 0.0044 (0.006388514955467153) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03381840999309833 (-0.0012338895064133854) + | > avg_loss_disc: 2.5803989813877988 (-0.03591122994056084) + | > avg_loss_disc_real_0: 0.21413484788857973 (-0.12210887555892652) + | > avg_loss_disc_real_1: 0.2368584951529136 (-0.011873877965486973) + | > avg_loss_disc_real_2: 0.2520463329095107 (-0.008112642627496003) + | > avg_loss_disc_real_3: 0.25876978498238784 (-0.016390454310637237) + | > avg_loss_disc_real_4: 0.26949566488082594 (-0.007265358017041146) + | > avg_loss_disc_real_5: 0.25202430670077985 (+0.04716262794457951) + | > avg_loss_0: 2.5803989813877988 (-0.03591122994056084) + | > avg_loss_gen: 2.5213282841902513 (-0.23718182857219983) + | > avg_loss_kl: 3.8718783855438232 (+0.018809355222261903) + | > avg_loss_feat: 8.117890357971191 (+0.13853968106783388) + | > avg_loss_mel: 16.851526700533352 (-0.3564445789043731) + | > avg_loss_duration: 1.2964593447171724 (-0.036693893946134315) + | > avg_loss_1: 32.65908255943885 (-0.47297213627741996) + + + > EPOCH: 143/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:31:53)  + + --> TIME: 2024-07-14 00:32:02 -- STEP: 6/151 -- GLOBAL_STEP: 106600 + | > loss_disc: 2.2531347274780273 (2.321485996246338) + | > loss_disc_real_0: 0.10972601175308228 (0.12221050014098485) + | > loss_disc_real_1: 0.2233581840991974 (0.19211717694997787) + | > loss_disc_real_2: 0.2264205664396286 (0.204225925107797) + | > loss_disc_real_3: 0.22967611253261566 (0.21719052145878473) + | > loss_disc_real_4: 0.2570017874240875 (0.2164998600880305) + | > loss_disc_real_5: 0.2466317117214203 (0.23010056465864182) + | > loss_0: 2.2531347274780273 (2.321485996246338) + | > grad_norm_0: tensor(11.9404, device='cuda:0') (tensor(22.4071, device='cuda:0')) + | > loss_gen: 2.688291072845459 (2.5473098754882812) + | > loss_kl: 1.4121736288070679 (1.7157978018124898) + | > loss_feat: 8.310891151428223 (7.890711784362793) + | > loss_mel: 15.546928405761719 (15.836011250813803) + | > loss_duration: 0.8180598616600037 (0.9434781769911448) + | > loss_1: 28.776344299316406 (28.933308283487957) + | > grad_norm_1: tensor(392.1415, device='cuda:0') (tensor(303.9718, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.8673 (1.0662355820337932) + | > loader_time: 0.0045 (0.007114609082539876) + + + --> TIME: 2024-07-14 00:32:51 -- STEP: 56/151 -- GLOBAL_STEP: 106650 + | > loss_disc: 2.4231107234954834 (2.320151094879422) + | > loss_disc_real_0: 0.10399683564901352 (0.12316058788980756) + | > loss_disc_real_1: 0.1726190447807312 (0.20280132283057486) + | > loss_disc_real_2: 0.2225409597158432 (0.21153845185680048) + | > loss_disc_real_3: 0.1749269813299179 (0.20765089775834764) + | > loss_disc_real_4: 0.19290512800216675 (0.21106270088681153) + | > loss_disc_real_5: 0.21634922921657562 (0.21741745434701443) + | > loss_0: 2.4231107234954834 (2.320151094879422) + | > grad_norm_0: tensor(19.8709, device='cuda:0') (tensor(28.9001, device='cuda:0')) + | > loss_gen: 2.179755926132202 (2.587085864373616) + | > loss_kl: 1.6458556652069092 (1.7679861006992204) + | > loss_feat: 7.531978130340576 (8.073666112763542) + | > loss_mel: 15.007159233093262 (16.469893523624968) + | > loss_duration: 0.9515055418014526 (0.9233631832259042) + | > loss_1: 27.316255569458008 (29.821994645254954) + | > grad_norm_1: tensor(285.3855, device='cuda:0') (tensor(407.5316, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.8467 (0.9900980719498228) + | > loader_time: 0.0046 (0.006310411861964634) + + + --> TIME: 2024-07-14 00:33:39 -- STEP: 106/151 -- GLOBAL_STEP: 106700 + | > loss_disc: 2.3134820461273193 (2.302593872232257) + | > loss_disc_real_0: 0.15291543304920197 (0.11715084149168348) + | > loss_disc_real_1: 0.25118014216423035 (0.20221595086579053) + | > loss_disc_real_2: 0.24037359654903412 (0.209765465754383) + | > loss_disc_real_3: 0.2923791706562042 (0.2057124223068075) + | > loss_disc_real_4: 0.200312539935112 (0.20878101533876275) + | > loss_disc_real_5: 0.2294550985097885 (0.22108923671942837) + | > loss_0: 2.3134820461273193 (2.302593872232257) + | > grad_norm_0: tensor(28.6276, device='cuda:0') (tensor(27.3803, device='cuda:0')) + | > loss_gen: 2.752488851547241 (2.6202360324139864) + | > loss_kl: 1.788649559020996 (1.7199837132444922) + | > loss_feat: 7.872006416320801 (8.319561665912845) + | > loss_mel: 16.61003875732422 (16.583089072749303) + | > loss_duration: 0.9106449484825134 (0.9239413085973488) + | > loss_1: 29.933828353881836 (30.16681176311565) + | > grad_norm_1: tensor(628.7020, device='cuda:0') (tensor(435.3947, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.9078 (0.972403312629124) + | > loader_time: 0.0065 (0.0062168004377832955) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03367180090684157 (-0.00014660908625676455) + | > avg_loss_disc: 2.58891329398522 (+0.008514312597421192) + | > avg_loss_disc_real_0: 0.20779862426794493 (-0.0063362236206347955) + | > avg_loss_disc_real_1: 0.22396150689858657 (-0.01289698825432703) + | > avg_loss_disc_real_2: 0.2722862523335677 (+0.020239919424057007) + | > avg_loss_disc_real_3: 0.26922375193009007 (+0.010453966947702231) + | > avg_loss_disc_real_4: 0.23222497678720033 (-0.037270688093625604) + | > avg_loss_disc_real_5: 0.23893912251179034 (-0.013085184188989518) + | > avg_loss_0: 2.58891329398522 (+0.008514312597421192) + | > avg_loss_gen: 2.399509833409236 (-0.12181845078101539) + | > avg_loss_kl: 3.7302401616023135 (-0.1416382239415097) + | > avg_loss_feat: 7.701848066770113 (-0.4160422912010784) + | > avg_loss_mel: 16.815369899456318 (-0.03615680107703412) + | > avg_loss_duration: 1.325292367201585 (+0.028833022484412663) + | > avg_loss_1: 31.972260255080002 (-0.6868223043588486) + + + > EPOCH: 144/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:34:34)  + + --> TIME: 2024-07-14 00:34:42 -- STEP: 5/151 -- GLOBAL_STEP: 106750 + | > loss_disc: 2.5395402908325195 (2.3832858562469483) + | > loss_disc_real_0: 0.1559138149023056 (0.1417843982577324) + | > loss_disc_real_1: 0.2181718945503235 (0.2078112930059433) + | > loss_disc_real_2: 0.20578840374946594 (0.23285734951496123) + | > loss_disc_real_3: 0.26355963945388794 (0.2414366215467453) + | > loss_disc_real_4: 0.22426879405975342 (0.21085373163223267) + | > loss_disc_real_5: 0.29303520917892456 (0.2482717365026474) + | > loss_0: 2.5395402908325195 (2.3832858562469483) + | > grad_norm_0: tensor(19.8252, device='cuda:0') (tensor(26.3031, device='cuda:0')) + | > loss_gen: 2.4204254150390625 (2.5834954738616944) + | > loss_kl: 2.0909721851348877 (1.8102863311767579) + | > loss_feat: 7.4032087326049805 (7.719868564605713) + | > loss_mel: 16.775754928588867 (16.379817008972168) + | > loss_duration: 0.9757556915283203 (0.9777934312820434) + | > loss_1: 29.66611671447754 (29.47126121520996) + | > grad_norm_1: tensor(135.7318, device='cuda:0') (tensor(254.8595, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 1.1679 (1.1896369934082032) + | > loader_time: 0.008 (0.009270381927490235) + + + --> TIME: 2024-07-14 00:35:31 -- STEP: 55/151 -- GLOBAL_STEP: 106800 + | > loss_disc: 2.276912212371826 (2.3167613332921815) + | > loss_disc_real_0: 0.08909580111503601 (0.12591258964755317) + | > loss_disc_real_1: 0.22122855484485626 (0.2024013419042934) + | > loss_disc_real_2: 0.25677523016929626 (0.21005348698659376) + | > loss_disc_real_3: 0.22082151472568512 (0.21029324694113297) + | > loss_disc_real_4: 0.21576383709907532 (0.20981622257015922) + | > loss_disc_real_5: 0.21431615948677063 (0.2260785709727894) + | > loss_0: 2.276912212371826 (2.3167613332921815) + | > grad_norm_0: tensor(13.1082, device='cuda:0') (tensor(18.5292, device='cuda:0')) + | > loss_gen: 2.7765772342681885 (2.602567026831887) + | > loss_kl: 1.886810064315796 (1.7772287542169745) + | > loss_feat: 7.946867942810059 (8.296149956096306) + | > loss_mel: 16.59709358215332 (16.795527492869986) + | > loss_duration: 0.814024806022644 (0.9295480457219211) + | > loss_1: 30.021371841430664 (30.401021229137076) + | > grad_norm_1: tensor(359.5486, device='cuda:0') (tensor(223.8606, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.8665 (0.9894592111760917) + | > loader_time: 0.0047 (0.006384264339100231) + + + --> TIME: 2024-07-14 00:36:21 -- STEP: 105/151 -- GLOBAL_STEP: 106850 + | > loss_disc: 2.4649953842163086 (2.305460904893421) + | > loss_disc_real_0: 0.12210236489772797 (0.12208454736641472) + | > loss_disc_real_1: 0.24833059310913086 (0.20170401945000604) + | > loss_disc_real_2: 0.24560412764549255 (0.2090923431373778) + | > loss_disc_real_3: 0.2115115225315094 (0.20721712296917322) + | > loss_disc_real_4: 0.20215903222560883 (0.20820063111327944) + | > loss_disc_real_5: 0.248836487531662 (0.22369394969372522) + | > loss_0: 2.4649953842163086 (2.305460904893421) + | > grad_norm_0: tensor(20.2685, device='cuda:0') (tensor(19.5909, device='cuda:0')) + | > loss_gen: 2.2839467525482178 (2.596546009608677) + | > loss_kl: 1.9489784240722656 (1.754025331565312) + | > loss_feat: 6.7816033363342285 (8.285647501264306) + | > loss_mel: 14.444469451904297 (16.75063988821847) + | > loss_duration: 0.908037543296814 (0.9230128628867013) + | > loss_1: 26.367034912109375 (30.30987161908831) + | > grad_norm_1: tensor(240.1076, device='cuda:0') (tensor(265.7114, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 1.0221 (0.98329914410909) + | > loader_time: 0.0081 (0.0063214688074021115) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031447263864370495 (-0.002224537042471074) + | > avg_loss_disc: 2.6020277646871715 (+0.013114470701951486) + | > avg_loss_disc_real_0: 0.22844235025919402 (+0.020643725991249084) + | > avg_loss_disc_real_1: 0.2727911082597879 (+0.04882960136120135) + | > avg_loss_disc_real_2: 0.2534467600859128 (-0.018839492247654854) + | > avg_loss_disc_real_3: 0.2204337246142901 (-0.04879002731579998) + | > avg_loss_disc_real_4: 0.2097225418457618 (-0.022502434941438526) + | > avg_loss_disc_real_5: 0.2178367181466176 (-0.021102404365172744) + | > avg_loss_0: 2.6020277646871715 (+0.013114470701951486) + | > avg_loss_gen: 2.397720355253953 (-0.0017894781552829642) + | > avg_loss_kl: 3.7329461941352258 (+0.0027060325329122392) + | > avg_loss_feat: 7.622290684626653 (-0.07955738214346031) + | > avg_loss_mel: 17.10805181356577 (+0.2926819141094512) + | > avg_loss_duration: 1.3654997898982122 (+0.04020742269662714) + | > avg_loss_1: 32.22650850736178 (+0.2542482522817764) + + + > EPOCH: 145/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:37:17)  + + --> TIME: 2024-07-14 00:37:23 -- STEP: 4/151 -- GLOBAL_STEP: 106900 + | > loss_disc: 2.281804084777832 (2.317200541496277) + | > loss_disc_real_0: 0.08093202114105225 (0.1134742833673954) + | > loss_disc_real_1: 0.1723172813653946 (0.21069782972335815) + | > loss_disc_real_2: 0.16436119377613068 (0.17343441396951675) + | > loss_disc_real_3: 0.24868205189704895 (0.20856520906090736) + | > loss_disc_real_4: 0.20682409405708313 (0.17940406501293182) + | > loss_disc_real_5: 0.24868687987327576 (0.21336562186479568) + | > loss_0: 2.281804084777832 (2.317200541496277) + | > grad_norm_0: tensor(21.7313, device='cuda:0') (tensor(28.9856, device='cuda:0')) + | > loss_gen: 2.8683512210845947 (2.630144417285919) + | > loss_kl: 1.3081303834915161 (1.5904574990272522) + | > loss_feat: 10.322173118591309 (9.194341897964478) + | > loss_mel: 17.499576568603516 (16.75883436203003) + | > loss_duration: 0.8541849255561829 (0.8818759471178055) + | > loss_1: 32.852413177490234 (31.055654048919678) + | > grad_norm_1: tensor(597.1026, device='cuda:0') (tensor(416.4364, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.8609 (0.8904790282249451) + | > loader_time: 0.0042 (0.005310475826263428) + + + --> TIME: 2024-07-14 00:38:12 -- STEP: 54/151 -- GLOBAL_STEP: 106950 + | > loss_disc: 2.438751697540283 (2.3382041277708834) + | > loss_disc_real_0: 0.10383262485265732 (0.12497944108865879) + | > loss_disc_real_1: 0.144682377576828 (0.20578207296353798) + | > loss_disc_real_2: 0.19649742543697357 (0.21194138129552206) + | > loss_disc_real_3: 0.2415386438369751 (0.20782066053814358) + | > loss_disc_real_4: 0.20209889113903046 (0.20615738989026458) + | > loss_disc_real_5: 0.2731289267539978 (0.2219228242282514) + | > loss_0: 2.438751697540283 (2.3382041277708834) + | > grad_norm_0: tensor(58.4107, device='cuda:0') (tensor(31.8164, device='cuda:0')) + | > loss_gen: 2.448347568511963 (2.5919014436227306) + | > loss_kl: 1.8337849378585815 (1.74344508294706) + | > loss_feat: 8.269943237304688 (8.141961353796498) + | > loss_mel: 17.55638885498047 (16.829699092441132) + | > loss_duration: 0.8779553174972534 (0.9183631186132077) + | > loss_1: 30.986419677734375 (30.225369877285427) + | > grad_norm_1: tensor(772.1200, device='cuda:0') (tensor(473.2338, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.882 (0.9696029512970535) + | > loader_time: 0.0052 (0.006236977047390408) + + + --> TIME: 2024-07-14 00:39:02 -- STEP: 104/151 -- GLOBAL_STEP: 107000 + | > loss_disc: 2.247948408126831 (2.3313561609158153) + | > loss_disc_real_0: 0.07859373092651367 (0.12455893820151685) + | > loss_disc_real_1: 0.20978491008281708 (0.206008906977681) + | > loss_disc_real_2: 0.18639780580997467 (0.21075101067813543) + | > loss_disc_real_3: 0.19953739643096924 (0.20675840099843648) + | > loss_disc_real_4: 0.19741322100162506 (0.2061439984692977) + | > loss_disc_real_5: 0.24876657128334045 (0.22415280585678724) + | > loss_0: 2.247948408126831 (2.3313561609158153) + | > grad_norm_0: tensor(32.9807, device='cuda:0') (tensor(29.0161, device='cuda:0')) + | > loss_gen: 2.613245725631714 (2.5891751990868497) + | > loss_kl: 1.710629940032959 (1.7297294053893824) + | > loss_feat: 8.646645545959473 (8.151279027645405) + | > loss_mel: 16.94733428955078 (16.769405676768365) + | > loss_duration: 0.8816077709197998 (0.9287870824337006) + | > loss_1: 30.799461364746094 (30.168376262371357) + | > grad_norm_1: tensor(526.8820, device='cuda:0') (tensor(413.3116, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.8637 (0.9808236337625064) + | > loader_time: 0.0049 (0.006304873869969294) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.04544300299424391 (+0.013995739129873418) + | > avg_loss_disc: 2.5337445735931396 (-0.0682831910940318) + | > avg_loss_disc_real_0: 0.20906642767099234 (-0.01937592258820167) + | > avg_loss_disc_real_1: 0.22592332156804892 (-0.046867786691738994) + | > avg_loss_disc_real_2: 0.22074102896910447 (-0.032705731116808356) + | > avg_loss_disc_real_3: 0.2136327876494481 (-0.0068009369648419915) + | > avg_loss_disc_real_4: 0.21399127863920653 (+0.004268736793444722) + | > avg_loss_disc_real_5: 0.22118910115498763 (+0.0033523830083700412) + | > avg_loss_0: 2.5337445735931396 (-0.0682831910940318) + | > avg_loss_gen: 2.368782410254845 (-0.028937944999107845) + | > avg_loss_kl: 3.9049289043133077 (+0.17198271017808198) + | > avg_loss_feat: 8.47993751672598 (+0.857646832099328) + | > avg_loss_mel: 16.9916960642888 (-0.11635574927696979) + | > avg_loss_duration: 1.3600768401072576 (-0.00542294979095459) + | > avg_loss_1: 33.10542150644156 (+0.8789129990797804) + + + > EPOCH: 146/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:40:27)  + + --> TIME: 2024-07-14 00:40:32 -- STEP: 3/151 -- GLOBAL_STEP: 107050 + | > loss_disc: 2.4931392669677734 (2.3903862635294595) + | > loss_disc_real_0: 0.1942049264907837 (0.1422756165266037) + | > loss_disc_real_1: 0.19505158066749573 (0.19265736639499664) + | > loss_disc_real_2: 0.19362333416938782 (0.17677888770898184) + | > loss_disc_real_3: 0.21022994816303253 (0.17790395518143973) + | > loss_disc_real_4: 0.25569623708724976 (0.20947089791297913) + | > loss_disc_real_5: 0.22361166775226593 (0.23794562121232352) + | > loss_0: 2.4931392669677734 (2.3903862635294595) + | > grad_norm_0: tensor(16.4053, device='cuda:0') (tensor(24.8231, device='cuda:0')) + | > loss_gen: 2.383117437362671 (2.482581615447998) + | > loss_kl: 1.609432578086853 (1.568538506825765) + | > loss_feat: 6.260425090789795 (7.315061251322429) + | > loss_mel: 15.316929817199707 (16.40949471791585) + | > loss_duration: 0.8889757990837097 (0.9096505641937256) + | > loss_1: 26.458879470825195 (28.685326258341473) + | > grad_norm_1: tensor(376.2113, device='cuda:0') (tensor(402.5518, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 1.0134 (0.9274330933888754) + | > loader_time: 0.0081 (0.005465030670166016) + + + --> TIME: 2024-07-14 00:41:20 -- STEP: 53/151 -- GLOBAL_STEP: 107100 + | > loss_disc: 2.2881417274475098 (2.326085556228207) + | > loss_disc_real_0: 0.13760101795196533 (0.12134271297814711) + | > loss_disc_real_1: 0.2166779488325119 (0.20090377021510647) + | > loss_disc_real_2: 0.2155998796224594 (0.20872811157748383) + | > loss_disc_real_3: 0.1889970302581787 (0.20701192038239175) + | > loss_disc_real_4: 0.1831267923116684 (0.20985307305488946) + | > loss_disc_real_5: 0.23534521460533142 (0.22442758476959085) + | > loss_0: 2.2881417274475098 (2.326085556228207) + | > grad_norm_0: tensor(14.7790, device='cuda:0') (tensor(28.8972, device='cuda:0')) + | > loss_gen: 2.5144786834716797 (2.6116632290606234) + | > loss_kl: 1.180777907371521 (1.7041157124177464) + | > loss_feat: 6.639585018157959 (8.227149297606266) + | > loss_mel: 15.719736099243164 (16.618624291330008) + | > loss_duration: 0.9446228742599487 (0.9268541077397904) + | > loss_1: 26.99920082092285 (30.088406652774452) + | > grad_norm_1: tensor(218.2816, device='cuda:0') (tensor(447.9923, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.8286 (0.9615607846458003) + | > loader_time: 0.0047 (0.0059941489741487325) + + + --> TIME: 2024-07-14 00:42:11 -- STEP: 103/151 -- GLOBAL_STEP: 107150 + | > loss_disc: 1.9975569248199463 (2.3224439470513363) + | > loss_disc_real_0: 0.04898925870656967 (0.11923790401191388) + | > loss_disc_real_1: 0.18044528365135193 (0.20151546597480774) + | > loss_disc_real_2: 0.16909578442573547 (0.20853335446524388) + | > loss_disc_real_3: 0.1541323959827423 (0.2062157595041886) + | > loss_disc_real_4: 0.1871408224105835 (0.21030418053992744) + | > loss_disc_real_5: 0.20172934234142303 (0.22396222943241156) + | > loss_0: 1.9975569248199463 (2.3224439470513363) + | > grad_norm_0: tensor(17.7578, device='cuda:0') (tensor(26.8235, device='cuda:0')) + | > loss_gen: 2.9203858375549316 (2.6031388032783584) + | > loss_kl: 1.2225044965744019 (1.7229600971184889) + | > loss_feat: 10.411745071411133 (8.203343738629975) + | > loss_mel: 16.866804122924805 (16.63580325969213) + | > loss_duration: 1.0434271097183228 (0.9304905151857913) + | > loss_1: 32.464866638183594 (30.09573642027031) + | > grad_norm_1: tensor(345.9978, device='cuda:0') (tensor(430.5688, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.8099 (0.9783794717881286) + | > loader_time: 0.004 (0.006166282209377844) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03784805077772873 (-0.007594952216515183) + | > avg_loss_disc: 2.617817603624784 (+0.08407303003164435) + | > avg_loss_disc_real_0: 0.21519941893907693 (+0.006132991268084587) + | > avg_loss_disc_real_1: 0.2157264741567465 (-0.010196847411302418) + | > avg_loss_disc_real_2: 0.32053087078608 (+0.09978984181697553) + | > avg_loss_disc_real_3: 0.24254037669071785 (+0.02890758904126975) + | > avg_loss_disc_real_4: 0.24709709791036752 (+0.03310581927116099) + | > avg_loss_disc_real_5: 0.2620792996424895 (+0.04089019848750186) + | > avg_loss_0: 2.617817603624784 (+0.08407303003164435) + | > avg_loss_gen: 2.532667636871338 (+0.16388522661649274) + | > avg_loss_kl: 3.7810167716099667 (-0.123912132703341) + | > avg_loss_feat: 7.813958241389348 (-0.6659792753366327) + | > avg_loss_mel: 17.03431664980375 (+0.04262058551494974) + | > avg_loss_duration: 1.3548775636232817 (-0.00519927648397589) + | > avg_loss_1: 32.51683675325834 (-0.5885847531832198) + + + > EPOCH: 147/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:43:08)  + + --> TIME: 2024-07-14 00:43:12 -- STEP: 2/151 -- GLOBAL_STEP: 107200 + | > loss_disc: 2.4673800468444824 (2.476098418235779) + | > loss_disc_real_0: 0.08841300010681152 (0.09752145409584045) + | > loss_disc_real_1: 0.23610638082027435 (0.23205706477165222) + | > loss_disc_real_2: 0.23113976418972015 (0.24933137744665146) + | > loss_disc_real_3: 0.2524288296699524 (0.2515142112970352) + | > loss_disc_real_4: 0.19730828702449799 (0.20583129674196243) + | > loss_disc_real_5: 0.22517123818397522 (0.2689311355352402) + | > loss_0: 2.4673800468444824 (2.476098418235779) + | > grad_norm_0: tensor(20.9384, device='cuda:0') (tensor(37.3302, device='cuda:0')) + | > loss_gen: 2.4463694095611572 (2.4526790380477905) + | > loss_kl: 2.113022565841675 (2.160359263420105) + | > loss_feat: 8.238997459411621 (7.338177680969238) + | > loss_mel: 16.395828247070312 (15.733609676361084) + | > loss_duration: 0.8109405636787415 (0.849124401807785) + | > loss_1: 30.005157470703125 (28.533949851989746) + | > grad_norm_1: tensor(257.0639, device='cuda:0') (tensor(344.9854, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.8674 (0.9233828783035278) + | > loader_time: 0.0051 (0.005443453788757324) + + + --> TIME: 2024-07-14 00:44:00 -- STEP: 52/151 -- GLOBAL_STEP: 107250 + | > loss_disc: 2.3142917156219482 (2.282339210693654) + | > loss_disc_real_0: 0.1263839304447174 (0.11965437885373831) + | > loss_disc_real_1: 0.17737597227096558 (0.20002231001853943) + | > loss_disc_real_2: 0.21461501717567444 (0.20713157149461597) + | > loss_disc_real_3: 0.17440015077590942 (0.2031452383559484) + | > loss_disc_real_4: 0.2253996878862381 (0.20535375034579864) + | > loss_disc_real_5: 0.19378022849559784 (0.21999906060787347) + | > loss_0: 2.3142917156219482 (2.282339210693654) + | > grad_norm_0: tensor(45.0167, device='cuda:0') (tensor(29.4503, device='cuda:0')) + | > loss_gen: 2.7410731315612793 (2.6638042926788335) + | > loss_kl: 1.8252984285354614 (1.7536133252657378) + | > loss_feat: 10.208728790283203 (8.633697445576011) + | > loss_mel: 17.645044326782227 (16.96770057311425) + | > loss_duration: 0.9050731658935547 (0.9149525119708135) + | > loss_1: 33.325218200683594 (30.933768382439247) + | > grad_norm_1: tensor(717.8188, device='cuda:0') (tensor(466.0757, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.9231 (0.9406283131012549) + | > loader_time: 0.0056 (0.005747772180117094) + + + --> TIME: 2024-07-14 00:44:50 -- STEP: 102/151 -- GLOBAL_STEP: 107300 + | > loss_disc: 2.1890194416046143 (2.2830459335271054) + | > loss_disc_real_0: 0.08909973502159119 (0.11645722874969829) + | > loss_disc_real_1: 0.19920068979263306 (0.20035207534537597) + | > loss_disc_real_2: 0.20894499123096466 (0.2071336953371179) + | > loss_disc_real_3: 0.19607792794704437 (0.20158899692343732) + | > loss_disc_real_4: 0.23578868806362152 (0.2054035222413493) + | > loss_disc_real_5: 0.18879935145378113 (0.2218337327826257) + | > loss_0: 2.1890194416046143 (2.2830459335271054) + | > grad_norm_0: tensor(8.0323, device='cuda:0') (tensor(28.1478, device='cuda:0')) + | > loss_gen: 2.7954747676849365 (2.649456552430695) + | > loss_kl: 1.5121551752090454 (1.7201423983947903) + | > loss_feat: 9.84066390991211 (8.528681843888524) + | > loss_mel: 17.3892822265625 (16.866676339916154) + | > loss_duration: 0.8596693873405457 (0.9273015465222153) + | > loss_1: 32.397247314453125 (30.692258778740378) + | > grad_norm_1: tensor(444.6495, device='cuda:0') (tensor(456.8461, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.8664 (0.9637497009015551) + | > loader_time: 0.0043 (0.0059910217920939104) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03642872663644644 (-0.001419324141282291) + | > avg_loss_disc: 2.671694663854746 (+0.05387706022996186) + | > avg_loss_disc_real_0: 0.3464159736266503 (+0.13121655468757334) + | > avg_loss_disc_real_1: 0.21724821283267096 (+0.00152173867592445) + | > avg_loss_disc_real_2: 0.25007893030460066 (-0.07045194048147935) + | > avg_loss_disc_real_3: 0.2451400882922686 (+0.002599711601550758) + | > avg_loss_disc_real_4: 0.284147988145168 (+0.03705089023480049) + | > avg_loss_disc_real_5: 0.2303551401083286 (-0.031724159534160884) + | > avg_loss_0: 2.671694663854746 (+0.05387706022996186) + | > avg_loss_gen: 2.589394441017738 (+0.05672680414639997) + | > avg_loss_kl: 3.6598311754373403 (-0.12118559617262648) + | > avg_loss_feat: 7.738762378692627 (-0.07519586269672107) + | > avg_loss_mel: 16.864939762995792 (-0.1693768868079566) + | > avg_loss_duration: 1.364779884998615 (+0.009902321375333312) + | > avg_loss_1: 32.2177069737361 (-0.2991297795222394) + + + > EPOCH: 148/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:45:49)  + + --> TIME: 2024-07-14 00:45:53 -- STEP: 1/151 -- GLOBAL_STEP: 107350 + | > loss_disc: 2.3185200691223145 (2.3185200691223145) + | > loss_disc_real_0: 0.15697918832302094 (0.15697918832302094) + | > loss_disc_real_1: 0.16298460960388184 (0.16298460960388184) + | > loss_disc_real_2: 0.2271498590707779 (0.2271498590707779) + | > loss_disc_real_3: 0.229110985994339 (0.229110985994339) + | > loss_disc_real_4: 0.28269603848457336 (0.28269603848457336) + | > loss_disc_real_5: 0.2978491485118866 (0.2978491485118866) + | > loss_0: 2.3185200691223145 (2.3185200691223145) + | > grad_norm_0: tensor(75.4488, device='cuda:0') (tensor(75.4488, device='cuda:0')) + | > loss_gen: 2.787799835205078 (2.787799835205078) + | > loss_kl: 1.6414527893066406 (1.6414527893066406) + | > loss_feat: 8.960563659667969 (8.960563659667969) + | > loss_mel: 18.220050811767578 (18.220050811767578) + | > loss_duration: 0.928726315498352 (0.928726315498352) + | > loss_1: 32.53859329223633 (32.53859329223633) + | > grad_norm_1: tensor(790.9176, device='cuda:0') (tensor(790.9176, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.9289 (0.9288995265960693) + | > loader_time: 0.0046 (0.0046002864837646484) + + + --> TIME: 2024-07-14 00:46:44 -- STEP: 51/151 -- GLOBAL_STEP: 107400 + | > loss_disc: 2.1620566844940186 (2.25847864151001) + | > loss_disc_real_0: 0.04635940119624138 (0.11236026846602853) + | > loss_disc_real_1: 0.1464795619249344 (0.20115614901570714) + | > loss_disc_real_2: 0.16204440593719482 (0.20936132704510407) + | > loss_disc_real_3: 0.1672373265028 (0.20063689672479443) + | > loss_disc_real_4: 0.18340177834033966 (0.2038202496135936) + | > loss_disc_real_5: 0.16742192208766937 (0.21510429738783368) + | > loss_0: 2.1620566844940186 (2.25847864151001) + | > grad_norm_0: tensor(22.0098, device='cuda:0') (tensor(30.0432, device='cuda:0')) + | > loss_gen: 2.57155704498291 (2.684522871877633) + | > loss_kl: 1.1926376819610596 (1.7474492928561043) + | > loss_feat: 9.504608154296875 (8.67031156315523) + | > loss_mel: 17.18389320373535 (16.960454435909494) + | > loss_duration: 0.9685868620872498 (0.9307423420980865) + | > loss_1: 31.421279907226562 (30.993480383181105) + | > grad_norm_1: tensor(286.6219, device='cuda:0') (tensor(513.4250, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.8458 (0.9948385322795198) + | > loader_time: 0.0041 (0.006046926274019129) + + + --> TIME: 2024-07-14 00:47:31 -- STEP: 101/151 -- GLOBAL_STEP: 107450 + | > loss_disc: 2.1885786056518555 (2.28066110374904) + | > loss_disc_real_0: 0.0748753622174263 (0.11676743261442327) + | > loss_disc_real_1: 0.24816782772541046 (0.2018375603279265) + | > loss_disc_real_2: 0.2170407623052597 (0.21003130491417232) + | > loss_disc_real_3: 0.23566661775112152 (0.20231878639447806) + | > loss_disc_real_4: 0.2106933891773224 (0.20427769857763065) + | > loss_disc_real_5: 0.1897011250257492 (0.2202804582838965) + | > loss_0: 2.1885786056518555 (2.28066110374904) + | > grad_norm_0: tensor(23.1982, device='cuda:0') (tensor(26.9148, device='cuda:0')) + | > loss_gen: 2.666133403778076 (2.6582994673511764) + | > loss_kl: 1.7746611833572388 (1.7415919138653444) + | > loss_feat: 8.613298416137695 (8.54608775129413) + | > loss_mel: 17.348403930664062 (16.942080157818168) + | > loss_duration: 0.907041072845459 (0.9289361208972364) + | > loss_1: 31.309537887573242 (30.81699539411186) + | > grad_norm_1: tensor(676.8604, device='cuda:0') (tensor(425.0215, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.8866 (0.9664259948352777) + | > loader_time: 0.0062 (0.006013882042157768) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03304677743178148 (-0.0033819492046649607) + | > avg_loss_disc: 2.512604309962346 (-0.15909035389239978) + | > avg_loss_disc_real_0: 0.14019610618169492 (-0.20621986744495535) + | > avg_loss_disc_real_1: 0.18430704910021561 (-0.03294116373245534) + | > avg_loss_disc_real_2: 0.2121320401246731 (-0.037946890179927556) + | > avg_loss_disc_real_3: 0.24340467498852655 (-0.0017354133037420505) + | > avg_loss_disc_real_4: 0.2503232875695595 (-0.03382470057560849) + | > avg_loss_disc_real_5: 0.21342929280721223 (-0.016925847301116376) + | > avg_loss_0: 2.512604309962346 (-0.15909035389239978) + | > avg_loss_gen: 2.281810155281654 (-0.307584285736084) + | > avg_loss_kl: 3.788206320542556 (+0.12837514510521553) + | > avg_loss_feat: 8.05422647182758 (+0.3154640931349526) + | > avg_loss_mel: 16.497665845430813 (-0.3672739175649795) + | > avg_loss_duration: 1.364100144459651 (-0.0006797405389640243) + | > avg_loss_1: 31.986009157620945 (-0.2316978161151546) + + + > EPOCH: 149/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:48:30)  + + --> TIME: 2024-07-14 00:48:33 -- STEP: 0/151 -- GLOBAL_STEP: 107500 + | > loss_disc: 2.393794298171997 (2.393794298171997) + | > loss_disc_real_0: 0.07130800932645798 (0.07130800932645798) + | > loss_disc_real_1: 0.17750152945518494 (0.17750152945518494) + | > loss_disc_real_2: 0.197434663772583 (0.197434663772583) + | > loss_disc_real_3: 0.24977362155914307 (0.24977362155914307) + | > loss_disc_real_4: 0.2363283634185791 (0.2363283634185791) + | > loss_disc_real_5: 0.22492361068725586 (0.22492361068725586) + | > loss_0: 2.393794298171997 (2.393794298171997) + | > grad_norm_0: tensor(17.8176, device='cuda:0') (tensor(17.8176, device='cuda:0')) + | > loss_gen: 2.5210955142974854 (2.5210955142974854) + | > loss_kl: 1.9001978635787964 (1.9001978635787964) + | > loss_feat: 7.723910331726074 (7.723910331726074) + | > loss_mel: 16.01972198486328 (16.01972198486328) + | > loss_duration: 0.8727570176124573 (0.8727570176124573) + | > loss_1: 29.037681579589844 (29.037681579589844) + | > grad_norm_1: tensor(501.0659, device='cuda:0') (tensor(501.0659, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 1.1314 (1.131396770477295) + | > loader_time: 1.1131 (1.1130626201629639) + + + --> TIME: 2024-07-14 00:49:23 -- STEP: 50/151 -- GLOBAL_STEP: 107550 + | > loss_disc: 2.471487522125244 (2.286027183532715) + | > loss_disc_real_0: 0.16444693505764008 (0.1203943944722414) + | > loss_disc_real_1: 0.26664847135543823 (0.20048449724912643) + | > loss_disc_real_2: 0.28958627581596375 (0.21227379620075226) + | > loss_disc_real_3: 0.2649567723274231 (0.20589170545339586) + | > loss_disc_real_4: 0.29657670855522156 (0.20476607084274293) + | > loss_disc_real_5: 0.2969281077384949 (0.21757690787315367) + | > loss_0: 2.471487522125244 (2.286027183532715) + | > grad_norm_0: tensor(40.9334, device='cuda:0') (tensor(31.5495, device='cuda:0')) + | > loss_gen: 2.7845237255096436 (2.651790885925294) + | > loss_kl: 1.9113601446151733 (1.6973187565803527) + | > loss_feat: 7.760750770568848 (8.466302242279049) + | > loss_mel: 17.82044219970703 (16.726455955505372) + | > loss_duration: 0.994331955909729 (0.9251055109500885) + | > loss_1: 31.271408081054688 (30.466973266601563) + | > grad_norm_1: tensor(581.7266, device='cuda:0') (tensor(471.3290, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 1.0212 (1.0011791276931763) + | > loader_time: 0.0065 (0.006449317932128907) + + + --> TIME: 2024-07-14 00:50:12 -- STEP: 100/151 -- GLOBAL_STEP: 107600 + | > loss_disc: 2.349884271621704 (2.288648610115052) + | > loss_disc_real_0: 0.13359707593917847 (0.11873721785843372) + | > loss_disc_real_1: 0.22437778115272522 (0.20186802685260774) + | > loss_disc_real_2: 0.21812427043914795 (0.21111871659755707) + | > loss_disc_real_3: 0.21984580159187317 (0.2050250232964754) + | > loss_disc_real_4: 0.1979219913482666 (0.20486460149288177) + | > loss_disc_real_5: 0.22338511049747467 (0.21992933943867685) + | > loss_0: 2.349884271621704 (2.288648610115052) + | > grad_norm_0: tensor(25.3551, device='cuda:0') (tensor(27.1311, device='cuda:0')) + | > loss_gen: 2.3518600463867188 (2.629918658733369) + | > loss_kl: 2.219179391860962 (1.7147104090452194) + | > loss_feat: 6.870385646820068 (8.355558776855469) + | > loss_mel: 16.439716339111328 (16.72438322067261) + | > loss_duration: 0.9408515691757202 (0.9317664647102356) + | > loss_1: 28.821992874145508 (30.356337566375732) + | > grad_norm_1: tensor(696.8579, device='cuda:0') (tensor(426.5771, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.994 (0.9790399837493895) + | > loader_time: 0.0065 (0.006344306468963623) + + + --> TIME: 2024-07-14 00:51:01 -- STEP: 150/151 -- GLOBAL_STEP: 107650 + | > loss_disc: 2.187690496444702 (2.2892430202166247) + | > loss_disc_real_0: 0.06907787173986435 (0.11657849701742332) + | > loss_disc_real_1: 0.17013688385486603 (0.2004380053281784) + | > loss_disc_real_2: 0.26393863558769226 (0.2101627243558566) + | > loss_disc_real_3: 0.1557648777961731 (0.20515467301011087) + | > loss_disc_real_4: 0.19217009842395782 (0.20587656488021214) + | > loss_disc_real_5: 0.23160390555858612 (0.21996537158886587) + | > loss_0: 2.187690496444702 (2.2892430202166247) + | > grad_norm_0: tensor(15.3639, device='cuda:0') (tensor(27.2808, device='cuda:0')) + | > loss_gen: 2.681938648223877 (2.628756957054139) + | > loss_kl: 1.5271217823028564 (1.7076257566610973) + | > loss_feat: 8.684860229492188 (8.36891743342082) + | > loss_mel: 17.471111297607422 (16.731165326436358) + | > loss_duration: 0.8760789632797241 (0.9290859945615133) + | > loss_1: 31.241111755371094 (30.365551541646322) + | > grad_norm_1: tensor(607.3628, device='cuda:0') (tensor(442.6149, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.8559 (0.9756355841954549) + | > loader_time: 0.0042 (0.006265276273091634) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03213302905742938 (-0.0009137483743520969) + | > avg_loss_disc: 2.588117526127742 (+0.07551321616539575) + | > avg_loss_disc_real_0: 0.25973497101893794 (+0.11953886483724302) + | > avg_loss_disc_real_1: 0.21181583175292382 (+0.027508782652708202) + | > avg_loss_disc_real_2: 0.24358782630700332 (+0.03145578618233022) + | > avg_loss_disc_real_3: 0.24130749702453613 (-0.002097177963990421) + | > avg_loss_disc_real_4: 0.3009064220465147 (+0.05058313447695517) + | > avg_loss_disc_real_5: 0.2537647027235765 (+0.04033540991636425) + | > avg_loss_0: 2.588117526127742 (+0.07551321616539575) + | > avg_loss_gen: 2.5942294781024637 (+0.31241932282080986) + | > avg_loss_kl: 3.8980292540330153 (+0.10982293349045946) + | > avg_loss_feat: 7.440649766188401 (-0.6135767056391783) + | > avg_loss_mel: 16.048351287841797 (-0.44931455758901606) + | > avg_loss_duration: 1.4214070118390596 (+0.0573068673794086) + | > avg_loss_1: 31.40266667879545 (-0.5833424788254966) + + + > EPOCH: 150/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:51:12)  + + --> TIME: 2024-07-14 00:52:03 -- STEP: 49/151 -- GLOBAL_STEP: 107700 + | > loss_disc: 2.1969425678253174 (2.259605419879057) + | > loss_disc_real_0: 0.07257872819900513 (0.11642318621886019) + | > loss_disc_real_1: 0.20755352079868317 (0.19935921594804648) + | > loss_disc_real_2: 0.19181057810783386 (0.20549920931154367) + | > loss_disc_real_3: 0.1910320222377777 (0.2001124310250185) + | > loss_disc_real_4: 0.1888916939496994 (0.202866508340349) + | > loss_disc_real_5: 0.2321619838476181 (0.21409902919311913) + | > loss_0: 2.1969425678253174 (2.259605419879057) + | > grad_norm_0: tensor(15.3322, device='cuda:0') (tensor(35.6294, device='cuda:0')) + | > loss_gen: 2.7746729850769043 (2.685823270252772) + | > loss_kl: 1.5031155347824097 (1.748083970984634) + | > loss_feat: 8.588652610778809 (8.557395195474435) + | > loss_mel: 15.662450790405273 (16.74424786470374) + | > loss_duration: 0.9138838052749634 (0.9262810167001219) + | > loss_1: 29.44277572631836 (30.661831369205398) + | > grad_norm_1: tensor(462.4633, device='cuda:0') (tensor(561.1404, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.8508 (0.9944645677294053) + | > loader_time: 0.0041 (0.0062754835401262555) + + + --> TIME: 2024-07-14 00:52:52 -- STEP: 99/151 -- GLOBAL_STEP: 107750 + | > loss_disc: 2.0795061588287354 (2.276544344545615) + | > loss_disc_real_0: 0.059042543172836304 (0.11717337068885264) + | > loss_disc_real_1: 0.18864989280700684 (0.19828729105718207) + | > loss_disc_real_2: 0.21350745856761932 (0.20742157748853318) + | > loss_disc_real_3: 0.13240790367126465 (0.20252051791458414) + | > loss_disc_real_4: 0.09043479710817337 (0.20421197792195311) + | > loss_disc_real_5: 0.18475341796875 (0.21880478517274665) + | > loss_0: 2.0795061588287354 (2.276544344545615) + | > grad_norm_0: tensor(23.4647, device='cuda:0') (tensor(33.2355, device='cuda:0')) + | > loss_gen: 2.8424112796783447 (2.656828824919883) + | > loss_kl: 1.1582320928573608 (1.7014387352298004) + | > loss_feat: 10.511187553405762 (8.448616263842347) + | > loss_mel: 18.371809005737305 (16.77666249901357) + | > loss_duration: 0.8312317728996277 (0.9306092942603911) + | > loss_1: 33.71487045288086 (30.514155705769856) + | > grad_norm_1: tensor(332.7149, device='cuda:0') (tensor(539.0422, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.8468 (0.9755622232803192) + | > loader_time: 0.0043 (0.006283654106987849) + + + --> TIME: 2024-07-14 00:53:41 -- STEP: 149/151 -- GLOBAL_STEP: 107800 + | > loss_disc: 2.2333736419677734 (2.27944242554223) + | > loss_disc_real_0: 0.08676101267337799 (0.11635954870573627) + | > loss_disc_real_1: 0.1823175549507141 (0.20016942038232047) + | > loss_disc_real_2: 0.179510697722435 (0.2078371954044239) + | > loss_disc_real_3: 0.1459302455186844 (0.2029553724815381) + | > loss_disc_real_4: 0.18911132216453552 (0.2039559689044152) + | > loss_disc_real_5: 0.182453915476799 (0.21948533095169387) + | > loss_0: 2.2333736419677734 (2.27944242554223) + | > grad_norm_0: tensor(18.9227, device='cuda:0') (tensor(32.6711, device='cuda:0')) + | > loss_gen: 2.442284107208252 (2.6448294700392134) + | > loss_kl: 1.946932077407837 (1.6963752940197119) + | > loss_feat: 9.10888385772705 (8.414654395724313) + | > loss_mel: 17.62812614440918 (16.759162934834524) + | > loss_duration: 0.8883423209190369 (0.9318995951806139) + | > loss_1: 32.01456832885742 (30.446921777405194) + | > grad_norm_1: tensor(658.4059, device='cuda:0') (tensor(536.0369, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.8752 (0.976553092866936) + | > loader_time: 0.0046 (0.006168050253951309) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03372483987074632 (+0.0015918108133169406) + | > avg_loss_disc: 2.5743562074807973 (-0.01376131864694452) + | > avg_loss_disc_real_0: 0.31614465438402617 (+0.056409683365088226) + | > avg_loss_disc_real_1: 0.19508725633988014 (-0.016728575413043678) + | > avg_loss_disc_real_2: 0.21871369160138643 (-0.02487413470561689) + | > avg_loss_disc_real_3: 0.20955042999524337 (-0.03175706702929276) + | > avg_loss_disc_real_4: 0.21621175224964434 (-0.08469466979687035) + | > avg_loss_disc_real_5: 0.199456213758542 (-0.054308488965034485) + | > avg_loss_0: 2.5743562074807973 (-0.01376131864694452) + | > avg_loss_gen: 2.393203093455388 (-0.20102638464707567) + | > avg_loss_kl: 3.7823269550616923 (-0.115702298971323) + | > avg_loss_feat: 8.331702048961933 (+0.8910522827735319) + | > avg_loss_mel: 17.517767612750713 (+1.4694163249089165) + | > avg_loss_duration: 1.3348279549525335 (-0.08657905688652612) + | > avg_loss_1: 33.35982806865986 (+1.9571613898644102) + + + > EPOCH: 151/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:53:53)  + + --> TIME: 2024-07-14 00:54:41 -- STEP: 48/151 -- GLOBAL_STEP: 107850 + | > loss_disc: 2.1110050678253174 (2.274367821713288) + | > loss_disc_real_0: 0.11623263359069824 (0.11712366513287027) + | > loss_disc_real_1: 0.15499256551265717 (0.19986623184134564) + | > loss_disc_real_2: 0.18718281388282776 (0.20972528091321388) + | > loss_disc_real_3: 0.18808668851852417 (0.20089373365044594) + | > loss_disc_real_4: 0.15869955718517303 (0.2046053965265552) + | > loss_disc_real_5: 0.21251940727233887 (0.21732255288710198) + | > loss_0: 2.1110050678253174 (2.274367821713288) + | > grad_norm_0: tensor(9.3397, device='cuda:0') (tensor(27.1041, device='cuda:0')) + | > loss_gen: 2.889786958694458 (2.6334416617949805) + | > loss_kl: 1.7632079124450684 (1.6489065376420815) + | > loss_feat: 8.605377197265625 (8.300752123196919) + | > loss_mel: 16.849388122558594 (16.653950134913135) + | > loss_duration: 0.8924227952957153 (0.9358560827871164) + | > loss_1: 31.000185012817383 (30.17290687561035) + | > grad_norm_1: tensor(199.7549, device='cuda:0') (tensor(460.8290, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.8906 (0.9468802114327749) + | > loader_time: 0.0061 (0.006166492899258931) + + + --> TIME: 2024-07-14 00:55:32 -- STEP: 98/151 -- GLOBAL_STEP: 107900 + | > loss_disc: 2.4387683868408203 (2.288558700863195) + | > loss_disc_real_0: 0.20703986287117004 (0.11771570526215495) + | > loss_disc_real_1: 0.215119406580925 (0.20267898984709565) + | > loss_disc_real_2: 0.20063866674900055 (0.2089465301559896) + | > loss_disc_real_3: 0.19241885840892792 (0.20533877170207548) + | > loss_disc_real_4: 0.24467629194259644 (0.20770509328160966) + | > loss_disc_real_5: 0.2625633776187897 (0.22079971584738517) + | > loss_0: 2.4387683868408203 (2.288558700863195) + | > grad_norm_0: tensor(42.9243, device='cuda:0') (tensor(30.3771, device='cuda:0')) + | > loss_gen: 2.6275625228881836 (2.639717805142304) + | > loss_kl: 2.3070523738861084 (1.699306915609204) + | > loss_feat: 8.981338500976562 (8.35325671215446) + | > loss_mel: 18.205400466918945 (16.787622373931267) + | > loss_duration: 0.9496148824691772 (0.9276129281034275) + | > loss_1: 33.07096862792969 (30.407516946597976) + | > grad_norm_1: tensor(461.6457, device='cuda:0') (tensor(484.1169, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.8969 (0.9713337445745662) + | > loader_time: 0.0055 (0.006312630614455864) + + + --> TIME: 2024-07-14 00:56:22 -- STEP: 148/151 -- GLOBAL_STEP: 107950 + | > loss_disc: 2.286328077316284 (2.2979473479696213) + | > loss_disc_real_0: 0.09231188148260117 (0.11796910392523215) + | > loss_disc_real_1: 0.1807619035243988 (0.2019088735954987) + | > loss_disc_real_2: 0.21944352984428406 (0.21052822631758614) + | > loss_disc_real_3: 0.2092958241701126 (0.2060661438751865) + | > loss_disc_real_4: 0.18864041566848755 (0.20667983994290634) + | > loss_disc_real_5: 0.23061318695545197 (0.22155635592502518) + | > loss_0: 2.286328077316284 (2.2979473479696213) + | > grad_norm_0: tensor(12.8039, device='cuda:0') (tensor(29.8299, device='cuda:0')) + | > loss_gen: 2.617867946624756 (2.632747207138989) + | > loss_kl: 2.0633606910705566 (1.7141369504703057) + | > loss_feat: 8.013340950012207 (8.309564368144883) + | > loss_mel: 16.796953201293945 (16.74194459657412) + | > loss_duration: 0.8915388584136963 (0.9287299324531813) + | > loss_1: 30.3830623626709 (30.32712324245556) + | > grad_norm_1: tensor(338.5495, device='cuda:0') (tensor(480.9221, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.9259 (0.9781817922721038) + | > loader_time: 0.0051 (0.006320932426968135) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031093083895169772 (-0.0026317559755765496) + | > avg_loss_disc: 2.580536842346192 (+0.006180634865394552) + | > avg_loss_disc_real_0: 0.28879617613095504 (-0.02734847825307113) + | > avg_loss_disc_real_1: 0.25751550381000227 (+0.06242824747012213) + | > avg_loss_disc_real_2: 0.23334602896983808 (+0.014632337368451653) + | > avg_loss_disc_real_3: 0.2282537416769908 (+0.018703311681747437) + | > avg_loss_disc_real_4: 0.2264678398003945 (+0.010256087550750165) + | > avg_loss_disc_real_5: 0.2333685835966697 (+0.033912369838127704) + | > avg_loss_0: 2.580536842346192 (+0.006180634865394552) + | > avg_loss_gen: 2.572441302813017 (+0.1792382093576288) + | > avg_loss_kl: 3.872228292318491 (+0.08990133725679872) + | > avg_loss_feat: 8.067284143888033 (-0.2644179050739002) + | > avg_loss_mel: 16.79222605778621 (-0.7255415549645043) + | > avg_loss_duration: 1.3511781050608709 (+0.016350150108337402) + | > avg_loss_1: 32.65535780099722 (-0.7044702676626358) + + + > EPOCH: 152/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:56:34)  + + --> TIME: 2024-07-14 00:57:25 -- STEP: 47/151 -- GLOBAL_STEP: 108000 + | > loss_disc: 2.3950233459472656 (2.288860151108275) + | > loss_disc_real_0: 0.14151468873023987 (0.11677831070537263) + | > loss_disc_real_1: 0.2002725601196289 (0.19865898138031046) + | > loss_disc_real_2: 0.24266304075717926 (0.21089037745556932) + | > loss_disc_real_3: 0.21320994198322296 (0.20135832023113331) + | > loss_disc_real_4: 0.22470182180404663 (0.207609364802533) + | > loss_disc_real_5: 0.26677176356315613 (0.2226604591024683) + | > loss_0: 2.3950233459472656 (2.288860151108275) + | > grad_norm_0: tensor(41.9015, device='cuda:0') (tensor(29.4624, device='cuda:0')) + | > loss_gen: 2.6303279399871826 (2.6279755805401095) + | > loss_kl: 1.5352851152420044 (1.726073701330956) + | > loss_feat: 6.941796779632568 (8.13439193685004) + | > loss_mel: 16.000030517578125 (16.688124819004784) + | > loss_duration: 0.8195767402648926 (0.9322867482266528) + | > loss_1: 27.92701530456543 (30.10885279229347) + | > grad_norm_1: tensor(202.9926, device='cuda:0') (tensor(509.1604, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.8881 (1.0101423365004516) + | > loader_time: 0.0049 (0.006387558389217295) + + + --> TIME: 2024-07-14 00:58:41 -- STEP: 97/151 -- GLOBAL_STEP: 108050 + | > loss_disc: 2.2964489459991455 (2.3018776964895498) + | > loss_disc_real_0: 0.1040155291557312 (0.11703932523420177) + | > loss_disc_real_1: 0.21229785680770874 (0.2020714801457739) + | > loss_disc_real_2: 0.1799783706665039 (0.21085727552777708) + | > loss_disc_real_3: 0.14031948149204254 (0.20356231620631268) + | > loss_disc_real_4: 0.14856715500354767 (0.20653964125925733) + | > loss_disc_real_5: 0.1940155029296875 (0.221514443300434) + | > loss_0: 2.2964489459991455 (2.3018776964895498) + | > grad_norm_0: tensor(29.3588, device='cuda:0') (tensor(29.7088, device='cuda:0')) + | > loss_gen: 2.5571227073669434 (2.6357442447819666) + | > loss_kl: 1.6896860599517822 (1.690925072148903) + | > loss_feat: 8.56781005859375 (8.278392172351321) + | > loss_mel: 17.1328182220459 (16.79321034421626) + | > loss_duration: 0.8532332181930542 (0.9296774796603882) + | > loss_1: 30.800670623779297 (30.327949386282064) + | > grad_norm_1: tensor(589.8514, device='cuda:0') (tensor(491.2904, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.9387 (0.9948398083755653) + | > loader_time: 0.0069 (0.006295609719974478) + + + --> TIME: 2024-07-14 00:59:30 -- STEP: 147/151 -- GLOBAL_STEP: 108100 + | > loss_disc: 2.571894645690918 (2.2931868583977626) + | > loss_disc_real_0: 0.191584512591362 (0.11484556412007534) + | > loss_disc_real_1: 0.24193717539310455 (0.20082468894265942) + | > loss_disc_real_2: 0.2306256741285324 (0.210150529940923) + | > loss_disc_real_3: 0.21302790939807892 (0.20513038458872818) + | > loss_disc_real_4: 0.23742923140525818 (0.20722225729097316) + | > loss_disc_real_5: 0.21034187078475952 (0.22059074642301416) + | > loss_0: 2.571894645690918 (2.2931868583977626) + | > grad_norm_0: tensor(18.4891, device='cuda:0') (tensor(27.7940, device='cuda:0')) + | > loss_gen: 2.491652488708496 (2.636717116751639) + | > loss_kl: 2.171232223510742 (1.7012399653999173) + | > loss_feat: 6.906096935272217 (8.290919368769838) + | > loss_mel: 16.115516662597656 (16.764593643396072) + | > loss_duration: 1.1853917837142944 (0.9242881370239517) + | > loss_1: 28.869890213012695 (30.317758183901002) + | > grad_norm_1: tensor(334.7313, device='cuda:0') (tensor(476.0045, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 2.3083 (0.9831800347282775) + | > loader_time: 0.0159 (0.006190384326337956) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0397171790783222 (+0.008624095183152429) + | > avg_loss_disc: 2.5754412504342885 (-0.005095591911903341) + | > avg_loss_disc_real_0: 0.24457170756963584 (-0.0442244685613192) + | > avg_loss_disc_real_1: 0.22660771241554847 (-0.030907791394453793) + | > avg_loss_disc_real_2: 0.2427770713200936 (+0.00943104235025552) + | > avg_loss_disc_real_3: 0.2386897481404818 (+0.010436006463490993) + | > avg_loss_disc_real_4: 0.27111454881154573 (+0.044646709011151225) + | > avg_loss_disc_real_5: 0.21843220064273247 (-0.014936382953937233) + | > avg_loss_0: 2.5754412504342885 (-0.005095591911903341) + | > avg_loss_gen: 2.5582169936253476 (-0.01422430918766926) + | > avg_loss_kl: 3.884398937225342 (+0.012170644906850825) + | > avg_loss_feat: 8.127347029172457 (+0.06006288528442383) + | > avg_loss_mel: 17.290730549738957 (+0.4985044919527475) + | > avg_loss_duration: 1.313271870979896 (-0.03790623408097482) + | > avg_loss_1: 33.17396530738244 (+0.5186075063852158) + + + > EPOCH: 153/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 00:59:42)  + + --> TIME: 2024-07-14 01:00:30 -- STEP: 46/151 -- GLOBAL_STEP: 108150 + | > loss_disc: 2.0970568656921387 (2.2800487539042598) + | > loss_disc_real_0: 0.05263770371675491 (0.11957358860451242) + | > loss_disc_real_1: 0.16523854434490204 (0.19819600734373793) + | > loss_disc_real_2: 0.1756865382194519 (0.2098710148230843) + | > loss_disc_real_3: 0.19923897087574005 (0.20121412335530572) + | > loss_disc_real_4: 0.16733193397521973 (0.1999787173193434) + | > loss_disc_real_5: 0.18355053663253784 (0.22089218056720236) + | > loss_0: 2.0970568656921387 (2.2800487539042598) + | > grad_norm_0: tensor(18.9627, device='cuda:0') (tensor(21.0072, device='cuda:0')) + | > loss_gen: 2.868644952774048 (2.661297964013141) + | > loss_kl: 1.7205040454864502 (1.7711018259110658) + | > loss_feat: 10.09757137298584 (8.689386243405552) + | > loss_mel: 17.223018646240234 (16.996501404306166) + | > loss_duration: 0.8771834373474121 (0.9100695550441742) + | > loss_1: 32.786922454833984 (31.02835704969323) + | > grad_norm_1: tensor(490.6951, device='cuda:0') (tensor(305.2135, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.8449 (0.9725643137226934) + | > loader_time: 0.0047 (0.006538904231527577) + + + --> TIME: 2024-07-14 01:01:17 -- STEP: 96/151 -- GLOBAL_STEP: 108200 + | > loss_disc: 2.2438673973083496 (2.27600537861387) + | > loss_disc_real_0: 0.0804930180311203 (0.1189412868116051) + | > loss_disc_real_1: 0.20271748304367065 (0.19930883349540332) + | > loss_disc_real_2: 0.20377951860427856 (0.20874935551546514) + | > loss_disc_real_3: 0.1970420479774475 (0.20039821423900625) + | > loss_disc_real_4: 0.1699633002281189 (0.2025502665589253) + | > loss_disc_real_5: 0.18207287788391113 (0.219235856551677) + | > loss_0: 2.2438673973083496 (2.27600537861387) + | > grad_norm_0: tensor(9.0925, device='cuda:0') (tensor(25.3120, device='cuda:0')) + | > loss_gen: 2.616943836212158 (2.653557193775971) + | > loss_kl: 1.4269672632217407 (1.7425635332862537) + | > loss_feat: 7.531112194061279 (8.568095892667774) + | > loss_mel: 17.485563278198242 (16.94161248207091) + | > loss_duration: 0.979786217212677 (0.9167754712204138) + | > loss_1: 30.04037094116211 (30.822604497273762) + | > grad_norm_1: tensor(122.5121, device='cuda:0') (tensor(397.3395, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.9083 (0.9555885468920072) + | > loader_time: 0.0056 (0.006332519153753917) + + + --> TIME: 2024-07-14 01:02:06 -- STEP: 146/151 -- GLOBAL_STEP: 108250 + | > loss_disc: 2.429807186126709 (2.286265383844506) + | > loss_disc_real_0: 0.19678911566734314 (0.11961368935769551) + | > loss_disc_real_1: 0.24259759485721588 (0.20006563534883604) + | > loss_disc_real_2: 0.22204181551933289 (0.20932404875551183) + | > loss_disc_real_3: 0.1969749629497528 (0.20098978765819167) + | > loss_disc_real_4: 0.2558499276638031 (0.20224977009100456) + | > loss_disc_real_5: 0.23506425321102142 (0.22267783218867157) + | > loss_0: 2.429807186126709 (2.286265383844506) + | > grad_norm_0: tensor(23.2887, device='cuda:0') (tensor(24.8986, device='cuda:0')) + | > loss_gen: 2.6965103149414062 (2.64268447275031) + | > loss_kl: 1.4073892831802368 (1.7294676826424793) + | > loss_feat: 6.311660289764404 (8.476359925857963) + | > loss_mel: 15.08195972442627 (16.849673218923073) + | > loss_duration: 0.9454058408737183 (0.9226267652152336) + | > loss_1: 26.44292640686035 (30.620812076411834) + | > grad_norm_1: tensor(240.2202, device='cuda:0') (tensor(394.2119, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.8626 (0.9574645218783862) + | > loader_time: 0.0047 (0.006264294663520708) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03426652688246507 (-0.005450652195857129) + | > avg_loss_disc: 2.5774142191960254 (+0.001972968761736915) + | > avg_loss_disc_real_0: 0.23182062346201676 (-0.012751084107619076) + | > avg_loss_disc_real_1: 0.19967474272617927 (-0.0269329696893692) + | > avg_loss_disc_real_2: 0.2211436124948355 (-0.021633458825258106) + | > avg_loss_disc_real_3: 0.21201831102371216 (-0.026671437116769642) + | > avg_loss_disc_real_4: 0.23692681468450105 (-0.03418773412704468) + | > avg_loss_disc_real_5: 0.21595643575374895 (-0.002475764888983517) + | > avg_loss_0: 2.5774142191960254 (+0.001972968761736915) + | > avg_loss_gen: 2.2968869576087365 (-0.2613300360166111) + | > avg_loss_kl: 3.939509208385761 (+0.055110271160419) + | > avg_loss_feat: 8.049009983356182 (-0.07833704581627465) + | > avg_loss_mel: 17.166178923386795 (-0.12455162635216155) + | > avg_loss_duration: 1.3519004308260405 (+0.03862855984614444) + | > avg_loss_1: 32.80348557692308 (-0.37047973045935834) + + + > EPOCH: 154/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:02:22)  + + --> TIME: 2024-07-14 01:03:11 -- STEP: 45/151 -- GLOBAL_STEP: 108300 + | > loss_disc: 2.134528160095215 (2.3068339877658413) + | > loss_disc_real_0: 0.04949040338397026 (0.12057605824536748) + | > loss_disc_real_1: 0.18291877210140228 (0.20363677574528588) + | > loss_disc_real_2: 0.14417526125907898 (0.20715675983164045) + | > loss_disc_real_3: 0.16239088773727417 (0.20138737824228076) + | > loss_disc_real_4: 0.133828803896904 (0.2055961807568868) + | > loss_disc_real_5: 0.1128518283367157 (0.22198067704836527) + | > loss_0: 2.134528160095215 (2.3068339877658413) + | > grad_norm_0: tensor(16.7197, device='cuda:0') (tensor(32.2408, device='cuda:0')) + | > loss_gen: 2.6858229637145996 (2.64290951622857) + | > loss_kl: 1.501924991607666 (1.7889939069747924) + | > loss_feat: 10.020488739013672 (8.30666602452596) + | > loss_mel: 17.702951431274414 (17.003954633076983) + | > loss_duration: 0.9405797123908997 (0.9211394243770176) + | > loss_1: 32.851768493652344 (30.66366352505154) + | > grad_norm_1: tensor(580.0056, device='cuda:0') (tensor(521.7728, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.8685 (1.015962383482191) + | > loader_time: 0.0059 (0.006732119454277886) + + + --> TIME: 2024-07-14 01:03:58 -- STEP: 95/151 -- GLOBAL_STEP: 108350 + | > loss_disc: 2.206281900405884 (2.294095787249113) + | > loss_disc_real_0: 0.10803398489952087 (0.11526155158093102) + | > loss_disc_real_1: 0.2124226838350296 (0.20283158813652238) + | > loss_disc_real_2: 0.20726948976516724 (0.20939033611824637) + | > loss_disc_real_3: 0.18269915878772736 (0.20227787933851543) + | > loss_disc_real_4: 0.19755461812019348 (0.20509648056406724) + | > loss_disc_real_5: 0.22959424555301666 (0.2225956794462706) + | > loss_0: 2.206281900405884 (2.294095787249113) + | > grad_norm_0: tensor(28.5763, device='cuda:0') (tensor(32.4578, device='cuda:0')) + | > loss_gen: 2.7222487926483154 (2.6608096298418555) + | > loss_kl: 1.551418423652649 (1.7693762314947028) + | > loss_feat: 9.225095748901367 (8.387597922274939) + | > loss_mel: 17.38358497619629 (16.791100502014157) + | > loss_duration: 0.8926141262054443 (0.9269657831442983) + | > loss_1: 31.774961471557617 (30.535850002891138) + | > grad_norm_1: tensor(369.9189, device='cuda:0') (tensor(552.5233, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.8637 (0.9734174552716706) + | > loader_time: 0.0058 (0.006398088053653115) + + + --> TIME: 2024-07-14 01:04:47 -- STEP: 145/151 -- GLOBAL_STEP: 108400 + | > loss_disc: 2.413201332092285 (2.2993939120194007) + | > loss_disc_real_0: 0.1229666993021965 (0.11686635068778334) + | > loss_disc_real_1: 0.31179672479629517 (0.2026282978468928) + | > loss_disc_real_2: 0.2426767349243164 (0.20885250681433185) + | > loss_disc_real_3: 0.20529669523239136 (0.20301483674296014) + | > loss_disc_real_4: 0.27625709772109985 (0.20533990865123686) + | > loss_disc_real_5: 0.246648371219635 (0.22353754310772336) + | > loss_0: 2.413201332092285 (2.2993939120194007) + | > grad_norm_0: tensor(54.9671, device='cuda:0') (tensor(33.2058, device='cuda:0')) + | > loss_gen: 2.771322250366211 (2.6498182543392836) + | > loss_kl: 1.782528042793274 (1.7464859140330344) + | > loss_feat: 8.565948486328125 (8.360712054680128) + | > loss_mel: 17.729324340820312 (16.778630427656505) + | > loss_duration: 0.8586653470993042 (0.9252265243694698) + | > loss_1: 31.707788467407227 (30.460873057924466) + | > grad_norm_1: tensor(532.2037, device='cuda:0') (tensor(532.6786, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.9498 (0.9694974767750707) + | > loader_time: 0.0053 (0.006229727843712116) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03380469175485464 (-0.0004618351276104299) + | > avg_loss_disc: 2.5426966593815736 (-0.03471755981445179) + | > avg_loss_disc_real_0: 0.2978730293420645 (+0.06605240588004774) + | > avg_loss_disc_real_1: 0.19821323912877303 (-0.0014615035974062385) + | > avg_loss_disc_real_2: 0.25997580931736874 (+0.03883219682253325) + | > avg_loss_disc_real_3: 0.23195713758468628 (+0.01993882656097412) + | > avg_loss_disc_real_4: 0.2282572125013058 (-0.008669602183195263) + | > avg_loss_disc_real_5: 0.2218194672694573 (+0.005863031515708356) + | > avg_loss_0: 2.5426966593815736 (-0.03471755981445179) + | > avg_loss_gen: 2.6392466104947605 (+0.342359652886024) + | > avg_loss_kl: 3.8870156911703257 (-0.05249351721543505) + | > avg_loss_feat: 8.30677186525785 (+0.2577618819016685) + | > avg_loss_mel: 17.150377566997822 (-0.015801356388973176) + | > avg_loss_duration: 1.3308568367591271 (-0.021043594066913363) + | > avg_loss_1: 33.314268552339996 (+0.510782975416916) + + + > EPOCH: 155/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:05:03)  + + --> TIME: 2024-07-14 01:05:52 -- STEP: 44/151 -- GLOBAL_STEP: 108450 + | > loss_disc: 2.2185723781585693 (2.2898279482668102) + | > loss_disc_real_0: 0.06689488887786865 (0.12071697642518715) + | > loss_disc_real_1: 0.17622266709804535 (0.20224034650759262) + | > loss_disc_real_2: 0.19996114075183868 (0.20566661723635413) + | > loss_disc_real_3: 0.1548595130443573 (0.20537497374144467) + | > loss_disc_real_4: 0.23078031837940216 (0.20595110309394923) + | > loss_disc_real_5: 0.189575657248497 (0.22055399113080718) + | > loss_0: 2.2185723781585693 (2.2898279482668102) + | > grad_norm_0: tensor(39.6764, device='cuda:0') (tensor(38.8540, device='cuda:0')) + | > loss_gen: 2.513949394226074 (2.6532460017637765) + | > loss_kl: 1.4388293027877808 (1.7601189098574899) + | > loss_feat: 10.299415588378906 (8.43986760486256) + | > loss_mel: 17.934589385986328 (16.873593720522795) + | > loss_duration: 0.9961285591125488 (0.9256775189529766) + | > loss_1: 33.18291091918945 (30.65250383723866) + | > grad_norm_1: tensor(295.4355, device='cuda:0') (tensor(538.7960, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.8228 (1.0341445478526028) + | > loader_time: 0.0056 (0.006719459186900745) + + + --> TIME: 2024-07-14 01:06:39 -- STEP: 94/151 -- GLOBAL_STEP: 108500 + | > loss_disc: 2.288891077041626 (2.287980036532625) + | > loss_disc_real_0: 0.12658897042274475 (0.11817674263519176) + | > loss_disc_real_1: 0.20342588424682617 (0.1996623385459818) + | > loss_disc_real_2: 0.21623367071151733 (0.20731721113019805) + | > loss_disc_real_3: 0.2474498450756073 (0.2056670369620019) + | > loss_disc_real_4: 0.22521713376045227 (0.2057321257096656) + | > loss_disc_real_5: 0.29988202452659607 (0.22336765981100976) + | > loss_0: 2.288891077041626 (2.287980036532625) + | > grad_norm_0: tensor(5.3786, device='cuda:0') (tensor(34.4365, device='cuda:0')) + | > loss_gen: 2.6648857593536377 (2.6626586153152134) + | > loss_kl: 1.9271591901779175 (1.755990172954316) + | > loss_feat: 9.87796688079834 (8.538187260323383) + | > loss_mel: 17.724117279052734 (16.90333343059459) + | > loss_duration: 0.8845217823982239 (0.9229376645798378) + | > loss_1: 33.078651428222656 (30.783107108258186) + | > grad_norm_1: tensor(280.6899, device='cuda:0') (tensor(521.2048, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.9125 (0.9862009971699817) + | > loader_time: 0.0052 (0.0062880541415924725) + + + --> TIME: 2024-07-14 01:07:28 -- STEP: 144/151 -- GLOBAL_STEP: 108550 + | > loss_disc: 2.5960347652435303 (2.2914708389176264) + | > loss_disc_real_0: 0.18241262435913086 (0.11774121708650556) + | > loss_disc_real_1: 0.24651296436786652 (0.19880281016230575) + | > loss_disc_real_2: 0.25217896699905396 (0.20829375010604664) + | > loss_disc_real_3: 0.2714831233024597 (0.2051664240554803) + | > loss_disc_real_4: 0.274357408285141 (0.2054081823055943) + | > loss_disc_real_5: 0.27007463574409485 (0.22274743527587917) + | > loss_0: 2.5960347652435303 (2.2914708389176264) + | > grad_norm_0: tensor(33.8613, device='cuda:0') (tensor(32.8266, device='cuda:0')) + | > loss_gen: 2.4725048542022705 (2.6520547784037056) + | > loss_kl: 2.043390989303589 (1.779263376361794) + | > loss_feat: 6.569127082824707 (8.512068314684761) + | > loss_mel: 16.063980102539062 (16.881959100564316) + | > loss_duration: 0.9863741397857666 (0.9218201641407278) + | > loss_1: 28.135377883911133 (30.747165693177116) + | > grad_norm_1: tensor(278.6852, device='cuda:0') (tensor(528.5399, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 1.0934 (0.9788356340593762) + | > loader_time: 0.0065 (0.006144551767243279) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034193185659555286 (+0.00038849390470064443) + | > avg_loss_disc: 2.474213178341205 (-0.0684834810403685) + | > avg_loss_disc_real_0: 0.23899041230861956 (-0.05888261703344494) + | > avg_loss_disc_real_1: 0.2511952015069815 (+0.05298196237820846) + | > avg_loss_disc_real_2: 0.23660542643987215 (-0.023370382877496598) + | > avg_loss_disc_real_3: 0.23014734914669624 (-0.0018097884379900397) + | > avg_loss_disc_real_4: 0.22788202304106492 (-0.0003751894602408712) + | > avg_loss_disc_real_5: 0.2319798721716954 (+0.010160404902238102) + | > avg_loss_0: 2.474213178341205 (-0.0684834810403685) + | > avg_loss_gen: 2.6745911561525784 (+0.03534454565781786) + | > avg_loss_kl: 3.8498589258927565 (-0.03715676527756928) + | > avg_loss_feat: 8.172411258404072 (-0.13436060685377882) + | > avg_loss_mel: 17.132331407987156 (-0.018046159010665974) + | > avg_loss_duration: 1.3364039476101215 (+0.005547110850994352) + | > avg_loss_1: 33.165596595177284 (-0.14867195716271198) + + + > EPOCH: 156/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:07:44)  + + --> TIME: 2024-07-14 01:08:29 -- STEP: 43/151 -- GLOBAL_STEP: 108600 + | > loss_disc: 2.4471254348754883 (2.3331169583076656) + | > loss_disc_real_0: 0.12507586181163788 (0.13646268446085064) + | > loss_disc_real_1: 0.2128027081489563 (0.20297640423442043) + | > loss_disc_real_2: 0.26343873143196106 (0.2106158532375513) + | > loss_disc_real_3: 0.20701134204864502 (0.20782711963320888) + | > loss_disc_real_4: 0.2385278195142746 (0.2045702147622441) + | > loss_disc_real_5: 0.2694585621356964 (0.2171554399091144) + | > loss_0: 2.4471254348754883 (2.3331169583076656) + | > grad_norm_0: tensor(13.2107, device='cuda:0') (tensor(29.9920, device='cuda:0')) + | > loss_gen: 2.3543403148651123 (2.604953871216885) + | > loss_kl: 1.7758803367614746 (1.8227815614190213) + | > loss_feat: 7.812849044799805 (8.284707612769548) + | > loss_mel: 17.14206886291504 (16.86995180263076) + | > loss_duration: 0.8293889164924622 (0.9266616161479506) + | > loss_1: 29.914527893066406 (30.50905657923499) + | > grad_norm_1: tensor(134.1672, device='cuda:0') (tensor(404.3712, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.8945 (0.9783654268397841) + | > loader_time: 0.0049 (0.006124684976976972) + + + --> TIME: 2024-07-14 01:09:18 -- STEP: 93/151 -- GLOBAL_STEP: 108650 + | > loss_disc: 2.3498082160949707 (2.32731262201904) + | > loss_disc_real_0: 0.13092505931854248 (0.1285268636720797) + | > loss_disc_real_1: 0.19976502656936646 (0.20403518679962362) + | > loss_disc_real_2: 0.2209634929895401 (0.21101786580777937) + | > loss_disc_real_3: 0.2807372212409973 (0.20649246567039078) + | > loss_disc_real_4: 0.23433150351047516 (0.20611175430077378) + | > loss_disc_real_5: 0.26141348481178284 (0.22237412807761983) + | > loss_0: 2.3498082160949707 (2.32731262201904) + | > grad_norm_0: tensor(11.5336, device='cuda:0') (tensor(24.6322, device='cuda:0')) + | > loss_gen: 2.5005221366882324 (2.606545732867332) + | > loss_kl: 2.046542167663574 (1.7559625185945982) + | > loss_feat: 8.2744140625 (8.344337832543152) + | > loss_mel: 16.854433059692383 (16.876411776388846) + | > loss_duration: 0.887019157409668 (0.919533823767016) + | > loss_1: 30.562931060791016 (30.502791661088185) + | > grad_norm_1: tensor(250.1408, device='cuda:0') (tensor(345.7987, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.8874 (0.9704858436379381) + | > loader_time: 0.0052 (0.00606651972698909) + + + --> TIME: 2024-07-14 01:10:08 -- STEP: 143/151 -- GLOBAL_STEP: 108700 + | > loss_disc: 2.4136314392089844 (2.319500695575367) + | > loss_disc_real_0: 0.08039975166320801 (0.12420029500564506) + | > loss_disc_real_1: 0.22556646168231964 (0.20391420436489) + | > loss_disc_real_2: 0.21900472044944763 (0.20995703980430858) + | > loss_disc_real_3: 0.1836051046848297 (0.20565962770602084) + | > loss_disc_real_4: 0.21471603214740753 (0.20601684772051299) + | > loss_disc_real_5: 0.23219777643680573 (0.22263985924370644) + | > loss_0: 2.4136314392089844 (2.319500695575367) + | > grad_norm_0: tensor(27.9550, device='cuda:0') (tensor(25.8842, device='cuda:0')) + | > loss_gen: 2.4707770347595215 (2.60352664727431) + | > loss_kl: 2.0272443294525146 (1.7578586102365612) + | > loss_feat: 7.311144828796387 (8.331957837084786) + | > loss_mel: 17.105499267578125 (16.81767587728433) + | > loss_duration: 0.9488216638565063 (0.9278217024736471) + | > loss_1: 29.863489151000977 (30.438840679355433) + | > grad_norm_1: tensor(566.2179, device='cuda:0') (tensor(380.6955, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 1.4627 (0.9808563335792168) + | > loader_time: 0.0059 (0.006088118453125854) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03307562607985277 (-0.0011175595797025192) + | > avg_loss_disc: 2.656256363942073 (+0.18204318560086774) + | > avg_loss_disc_real_0: 0.3287719018184222 (+0.08978148950980264) + | > avg_loss_disc_real_1: 0.2084172792159594 (-0.04277792229102209) + | > avg_loss_disc_real_2: 0.21589772288615888 (-0.020707703553713264) + | > avg_loss_disc_real_3: 0.22666841516127953 (-0.00347893398541671) + | > avg_loss_disc_real_4: 0.2500002739521173 (+0.022118250911052406) + | > avg_loss_disc_real_5: 0.23619925058805025 (+0.004219378416354835) + | > avg_loss_0: 2.656256363942073 (+0.18204318560086774) + | > avg_loss_gen: 2.424852132797241 (-0.24973902335533715) + | > avg_loss_kl: 3.7510832089644213 (-0.0987757169283352) + | > avg_loss_feat: 7.5877341490525465 (-0.5846771093515253) + | > avg_loss_mel: 17.034084980304424 (-0.09824642768273151) + | > avg_loss_duration: 1.3868243235808153 (+0.05042037597069382) + | > avg_loss_1: 32.18457896892841 (-0.981017626248871) + + + > EPOCH: 157/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:10:25)  + + --> TIME: 2024-07-14 01:11:07 -- STEP: 42/151 -- GLOBAL_STEP: 108750 + | > loss_disc: 2.2170515060424805 (2.2427010309128534) + | > loss_disc_real_0: 0.09905131906270981 (0.1068498691810029) + | > loss_disc_real_1: 0.215704545378685 (0.19923518988348188) + | > loss_disc_real_2: 0.22706876695156097 (0.20598344114564715) + | > loss_disc_real_3: 0.21953745186328888 (0.19894778994577272) + | > loss_disc_real_4: 0.1935487538576126 (0.2003826059046246) + | > loss_disc_real_5: 0.20622733235359192 (0.22425283598048346) + | > loss_0: 2.2170515060424805 (2.2427010309128534) + | > grad_norm_0: tensor(17.7712, device='cuda:0') (tensor(30.1692, device='cuda:0')) + | > loss_gen: 2.6812219619750977 (2.6755779300417224) + | > loss_kl: 2.118192195892334 (1.78381893606413) + | > loss_feat: 8.138176918029785 (8.460654951277233) + | > loss_mel: 17.438417434692383 (16.73327427818662) + | > loss_duration: 0.894856333732605 (0.9370731030191694) + | > loss_1: 31.270862579345703 (30.590399060930526) + | > grad_norm_1: tensor(461.4319, device='cuda:0') (tensor(510.7746, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.9593 (0.9337212925865537) + | > loader_time: 0.0072 (0.005725168046497163) + + + --> TIME: 2024-07-14 01:11:57 -- STEP: 92/151 -- GLOBAL_STEP: 108800 + | > loss_disc: 2.36104416847229 (2.2615363675615066) + | > loss_disc_real_0: 0.1539674997329712 (0.110322296578923) + | > loss_disc_real_1: 0.24252967536449432 (0.19920795184114706) + | > loss_disc_real_2: 0.21417410671710968 (0.2065282285861347) + | > loss_disc_real_3: 0.26831477880477905 (0.20329703220530695) + | > loss_disc_real_4: 0.219252809882164 (0.20395319706395917) + | > loss_disc_real_5: 0.21345126628875732 (0.22234700879325037) + | > loss_0: 2.36104416847229 (2.2615363675615066) + | > grad_norm_0: tensor(26.4006, device='cuda:0') (tensor(28.3255, device='cuda:0')) + | > loss_gen: 2.706690549850464 (2.659676404102988) + | > loss_kl: 1.9880151748657227 (1.7718815583249796) + | > loss_feat: 8.236653327941895 (8.424490192662116) + | > loss_mel: 17.543190002441406 (16.843866617783263) + | > loss_duration: 0.9682081341743469 (0.9220915559841238) + | > loss_1: 31.44275665283203 (30.6220062504644) + | > grad_norm_1: tensor(377.3102, device='cuda:0') (tensor(507.8791, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.9674 (0.9610253546548926) + | > loader_time: 0.0063 (0.006116696026014245) + + + --> TIME: 2024-07-14 01:12:46 -- STEP: 142/151 -- GLOBAL_STEP: 108850 + | > loss_disc: 2.0744035243988037 (2.271036552711272) + | > loss_disc_real_0: 0.05021633952856064 (0.1141782536787886) + | > loss_disc_real_1: 0.15530207753181458 (0.1980941117229596) + | > loss_disc_real_2: 0.16058100759983063 (0.2063792177908857) + | > loss_disc_real_3: 0.1436716914176941 (0.20308077225173024) + | > loss_disc_real_4: 0.17384156584739685 (0.20575800050102488) + | > loss_disc_real_5: 0.16258370876312256 (0.2223892920243908) + | > loss_0: 2.0744035243988037 (2.271036552711272) + | > grad_norm_0: tensor(19.4781, device='cuda:0') (tensor(29.3124, device='cuda:0')) + | > loss_gen: 2.740734577178955 (2.654364726912807) + | > loss_kl: 1.0976154804229736 (1.7629890089303677) + | > loss_feat: 10.000316619873047 (8.405363015725579) + | > loss_mel: 17.785259246826172 (16.839605727665873) + | > loss_duration: 1.1086344718933105 (0.9239360518018964) + | > loss_1: 32.73255920410156 (30.58625843155552) + | > grad_norm_1: tensor(270.7451, device='cuda:0') (tensor(495.7937, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.8093 (0.9631085194332499) + | > loader_time: 0.004 (0.006058691253124826) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03244324830862192 (-0.0006323777712308465) + | > avg_loss_disc: 2.6520221600165734 (-0.004234203925499447) + | > avg_loss_disc_real_0: 0.34896384752713716 (+0.020191945708714965) + | > avg_loss_disc_real_1: 0.2167929204610678 (+0.008375641245108395) + | > avg_loss_disc_real_2: 0.27650626232990855 (+0.060608539443749665) + | > avg_loss_disc_real_3: 0.18603876175788733 (-0.040629653403392196) + | > avg_loss_disc_real_4: 0.24212013185024261 (-0.00788014210187471) + | > avg_loss_disc_real_5: 0.20569070027424738 (-0.030508550313802868) + | > avg_loss_0: 2.6520221600165734 (-0.004234203925499447) + | > avg_loss_gen: 2.5403102177840013 (+0.11545808498676013) + | > avg_loss_kl: 3.9579441914191613 (+0.20686098245474005) + | > avg_loss_feat: 8.199761427365816 (+0.6120272783132696) + | > avg_loss_mel: 17.064166142390324 (+0.030081162085899393) + | > avg_loss_duration: 1.365469033901508 (-0.02135528967930722) + | > avg_loss_1: 33.127650627723106 (+0.9430716587946932) + + + > EPOCH: 158/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:13:05)  + + --> TIME: 2024-07-14 01:13:47 -- STEP: 41/151 -- GLOBAL_STEP: 108900 + | > loss_disc: 2.1453447341918945 (2.277688404408897) + | > loss_disc_real_0: 0.05132667347788811 (0.11449518899728613) + | > loss_disc_real_1: 0.15622608363628387 (0.19835564330583666) + | > loss_disc_real_2: 0.17168527841567993 (0.20589606554769888) + | > loss_disc_real_3: 0.20249108970165253 (0.20218514905470172) + | > loss_disc_real_4: 0.1409439891576767 (0.20088670493626012) + | > loss_disc_real_5: 0.14258183538913727 (0.2225299819213588) + | > loss_0: 2.1453447341918945 (2.277688404408897) + | > grad_norm_0: tensor(12.9432, device='cuda:0') (tensor(19.6453, device='cuda:0')) + | > loss_gen: 2.495795965194702 (2.6248573268332134) + | > loss_kl: 2.0963985919952393 (1.7658613731221455) + | > loss_feat: 10.613990783691406 (8.779916484181472) + | > loss_mel: 17.736635208129883 (16.923741689542446) + | > loss_duration: 0.8361575603485107 (0.924232064223871) + | > loss_1: 33.77898025512695 (31.018609070196383) + | > grad_norm_1: tensor(438.2510, device='cuda:0') (tensor(287.2402, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.8843 (0.9652198465859018) + | > loader_time: 0.0061 (0.0058184716759658445) + + + --> TIME: 2024-07-14 01:14:37 -- STEP: 91/151 -- GLOBAL_STEP: 108950 + | > loss_disc: 2.227100372314453 (2.293802756529587) + | > loss_disc_real_0: 0.09400475025177002 (0.11684003054768174) + | > loss_disc_real_1: 0.19543790817260742 (0.19964762727965363) + | > loss_disc_real_2: 0.2465835064649582 (0.20937693945981645) + | > loss_disc_real_3: 0.17160187661647797 (0.20205541339876887) + | > loss_disc_real_4: 0.19031275808811188 (0.2029499555682088) + | > loss_disc_real_5: 0.18311023712158203 (0.21979662219246665) + | > loss_0: 2.227100372314453 (2.293802756529587) + | > grad_norm_0: tensor(17.6818, device='cuda:0') (tensor(24.0964, device='cuda:0')) + | > loss_gen: 2.776247024536133 (2.6205406005565943) + | > loss_kl: 2.0577139854431152 (1.7587964809857881) + | > loss_feat: 7.846510410308838 (8.559651794014394) + | > loss_mel: 16.00684928894043 (16.72073281466306) + | > loss_duration: 1.0265859365463257 (0.9284370823220892) + | > loss_1: 29.71390724182129 (30.588158848521473) + | > grad_norm_1: tensor(312.7937, device='cuda:0') (tensor(367.1477, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 1.0895 (0.9707978976951851) + | > loader_time: 0.0073 (0.006123338426862443) + + + --> TIME: 2024-07-14 01:15:27 -- STEP: 141/151 -- GLOBAL_STEP: 109000 + | > loss_disc: 2.253288745880127 (2.2885918853976195) + | > loss_disc_real_0: 0.19533169269561768 (0.11749210842746369) + | > loss_disc_real_1: 0.20375439524650574 (0.20024951991248632) + | > loss_disc_real_2: 0.2712317407131195 (0.20974941958561008) + | > loss_disc_real_3: 0.23218858242034912 (0.2037406157411582) + | > loss_disc_real_4: 0.22839011251926422 (0.2031567872416043) + | > loss_disc_real_5: 0.21204884350299835 (0.2198642871692671) + | > loss_0: 2.253288745880127 (2.2885918853976195) + | > grad_norm_0: tensor(59.2410, device='cuda:0') (tensor(25.9739, device='cuda:0')) + | > loss_gen: 2.904921531677246 (2.635372811175408) + | > loss_kl: 1.7223246097564697 (1.7647213060805138) + | > loss_feat: 8.426445007324219 (8.487441705473774) + | > loss_mel: 16.846433639526367 (16.713168448590228) + | > loss_duration: 0.8365657925605774 (0.9308731437574888) + | > loss_1: 30.7366886138916 (30.531577374072786) + | > grad_norm_1: tensor(516.8538, device='cuda:0') (tensor(417.2609, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.8965 (0.9783517840906237) + | > loader_time: 0.0046 (0.006196835362319404) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03408611737764799 (+0.0016428690690260678) + | > avg_loss_disc: 2.703152161378127 (+0.051130001361553656) + | > avg_loss_disc_real_0: 0.34517383116942185 (-0.003790016357715309) + | > avg_loss_disc_real_1: 0.18750684077923113 (-0.029286079681836663) + | > avg_loss_disc_real_2: 0.2163502975152089 (-0.06015596481469965) + | > avg_loss_disc_real_3: 0.22542399282638842 (+0.039385231068501086) + | > avg_loss_disc_real_4: 0.2531773642851756 (+0.011057232434933006) + | > avg_loss_disc_real_5: 0.2355432166503026 (+0.02985251637605521) + | > avg_loss_0: 2.703152161378127 (+0.051130001361553656) + | > avg_loss_gen: 2.3456726441016564 (-0.1946375736823449) + | > avg_loss_kl: 3.7180445927840013 (-0.23989959863515997) + | > avg_loss_feat: 8.175292895390438 (-0.024468531975378127) + | > avg_loss_mel: 17.433768492478592 (+0.36960235008826814) + | > avg_loss_duration: 1.3593448033699622 (-0.006124230531545871) + | > avg_loss_1: 33.03212297879732 (-0.09552764892578836) + + + > EPOCH: 159/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:16:14)  + + --> TIME: 2024-07-14 01:16:56 -- STEP: 40/151 -- GLOBAL_STEP: 109050 + | > loss_disc: 2.085581064224243 (2.266315379738807) + | > loss_disc_real_0: 0.07011131942272186 (0.11688092574477196) + | > loss_disc_real_1: 0.17787373065948486 (0.19988899752497674) + | > loss_disc_real_2: 0.1395457535982132 (0.21153629422187806) + | > loss_disc_real_3: 0.17660580575466156 (0.1991736598312855) + | > loss_disc_real_4: 0.17249217629432678 (0.20281828194856644) + | > loss_disc_real_5: 0.2089693397283554 (0.21865376234054565) + | > loss_0: 2.085581064224243 (2.266315379738807) + | > grad_norm_0: tensor(8.9517, device='cuda:0') (tensor(28.2361, device='cuda:0')) + | > loss_gen: 2.5740838050842285 (2.6898230552673335) + | > loss_kl: 1.721104383468628 (1.661128693819046) + | > loss_feat: 8.39399242401123 (8.552066731452943) + | > loss_mel: 16.89166259765625 (17.06876275539398) + | > loss_duration: 0.8983942270278931 (0.9341505751013756) + | > loss_1: 30.479236602783203 (30.9059317111969) + | > grad_norm_1: tensor(334.3386, device='cuda:0') (tensor(402.7086, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.8473 (0.9802992939949038) + | > loader_time: 0.0045 (0.006009483337402343) + + + --> TIME: 2024-07-14 01:17:46 -- STEP: 90/151 -- GLOBAL_STEP: 109100 + | > loss_disc: 2.347299337387085 (2.294942274358536) + | > loss_disc_real_0: 0.05967527627944946 (0.11951883344186677) + | > loss_disc_real_1: 0.13130083680152893 (0.19980083008607227) + | > loss_disc_real_2: 0.14237043261528015 (0.2110635229282909) + | > loss_disc_real_3: 0.2046436220407486 (0.20461162726084392) + | > loss_disc_real_4: 0.15690846741199493 (0.20476673593123754) + | > loss_disc_real_5: 0.19588987529277802 (0.21859563969903523) + | > loss_0: 2.347299337387085 (2.294942274358536) + | > grad_norm_0: tensor(22.0156, device='cuda:0') (tensor(25.2518, device='cuda:0')) + | > loss_gen: 2.318655252456665 (2.6405861801571313) + | > loss_kl: 1.0741162300109863 (1.722955294450124) + | > loss_feat: 7.352782249450684 (8.422822544309824) + | > loss_mel: 16.20008087158203 (16.825137074788415) + | > loss_duration: 1.126117467880249 (0.9293511576122708) + | > loss_1: 28.071752548217773 (30.54085227118598) + | > grad_norm_1: tensor(177.7117, device='cuda:0') (tensor(384.1360, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.8212 (0.9898835764990912) + | > loader_time: 0.004 (0.006220057275560167) + + + --> TIME: 2024-07-14 01:18:37 -- STEP: 140/151 -- GLOBAL_STEP: 109150 + | > loss_disc: 2.030691385269165 (2.297594153881073) + | > loss_disc_real_0: 0.059338610619306564 (0.11929960540894952) + | > loss_disc_real_1: 0.21727000176906586 (0.20085245040910585) + | > loss_disc_real_2: 0.15214841067790985 (0.21050121720348086) + | > loss_disc_real_3: 0.16428084671497345 (0.20491524072630066) + | > loss_disc_real_4: 0.15283331274986267 (0.20653454970036234) + | > loss_disc_real_5: 0.2098725289106369 (0.21921306954962866) + | > loss_0: 2.030691385269165 (2.297594153881073) + | > grad_norm_0: tensor(15.8908, device='cuda:0') (tensor(25.6474, device='cuda:0')) + | > loss_gen: 2.7852323055267334 (2.631660088471004) + | > loss_kl: 0.9516749382019043 (1.7118922391108105) + | > loss_feat: 9.860486030578613 (8.375425263813558) + | > loss_mel: 17.05740737915039 (16.75395727157592) + | > loss_duration: 0.9531348943710327 (0.9314555078744888) + | > loss_1: 31.607934951782227 (30.404390348706926) + | > grad_norm_1: tensor(574.8665, device='cuda:0') (tensor(406.6664, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.8449 (0.996272511141641) + | > loader_time: 0.0043 (0.0062827808516366145) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03310944483830379 (-0.0009766725393441991) + | > avg_loss_disc: 2.6430246829986572 (-0.060127478379469856) + | > avg_loss_disc_real_0: 0.1832197102216574 (-0.16195412094776446) + | > avg_loss_disc_real_1: 0.25424623374755567 (+0.06673939296832454) + | > avg_loss_disc_real_2: 0.29426621015255267 (+0.07791591263734376) + | > avg_loss_disc_real_3: 0.25087048227970415 (+0.025446489453315735) + | > avg_loss_disc_real_4: 0.2615737571166112 (+0.008396392831435562) + | > avg_loss_disc_real_5: 0.23410858099277204 (-0.0014346356575305474) + | > avg_loss_0: 2.6430246829986572 (-0.060127478379469856) + | > avg_loss_gen: 2.447500412280743 (+0.10182776817908668) + | > avg_loss_kl: 3.637002027951754 (-0.08104256483224725) + | > avg_loss_feat: 7.834088728978084 (-0.3412041664123544) + | > avg_loss_mel: 16.93952890542837 (-0.4942395870502203) + | > avg_loss_duration: 1.3410003918867845 (-0.01834441148317767) + | > avg_loss_1: 32.19912118178148 (-0.8330017970158394) + + + > EPOCH: 160/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:18:56)  + + --> TIME: 2024-07-14 01:19:37 -- STEP: 39/151 -- GLOBAL_STEP: 109200 + | > loss_disc: 2.3157379627227783 (2.286180465649336) + | > loss_disc_real_0: 0.06960703432559967 (0.12313750472206336) + | > loss_disc_real_1: 0.21944260597229004 (0.20163397911267403) + | > loss_disc_real_2: 0.18886403739452362 (0.20262495714884537) + | > loss_disc_real_3: 0.25817957520484924 (0.20606363125336477) + | > loss_disc_real_4: 0.23262475430965424 (0.20291985647800642) + | > loss_disc_real_5: 0.24310611188411713 (0.22104493929789618) + | > loss_0: 2.3157379627227783 (2.286180465649336) + | > grad_norm_0: tensor(14.1437, device='cuda:0') (tensor(34.6777, device='cuda:0')) + | > loss_gen: 2.604326009750366 (2.673425894517165) + | > loss_kl: 1.7007691860198975 (1.6058014432589214) + | > loss_feat: 8.949291229248047 (8.352022293286442) + | > loss_mel: 16.26249122619629 (16.654361553681202) + | > loss_duration: 0.8843209743499756 (0.9319885327265813) + | > loss_1: 30.401199340820312 (30.217599673148914) + | > grad_norm_1: tensor(304.4304, device='cuda:0') (tensor(531.9730, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.8668 (0.9777067746871557) + | > loader_time: 0.0056 (0.006314271535628881) + + + --> TIME: 2024-07-14 01:20:28 -- STEP: 89/151 -- GLOBAL_STEP: 109250 + | > loss_disc: 2.0067765712738037 (2.3185247571280825) + | > loss_disc_real_0: 0.0689442902803421 (0.1250996870522419) + | > loss_disc_real_1: 0.16762660443782806 (0.20297658610879704) + | > loss_disc_real_2: 0.16017118096351624 (0.2089851212300611) + | > loss_disc_real_3: 0.1643255352973938 (0.20696680716584237) + | > loss_disc_real_4: 0.1628214567899704 (0.20625364864140414) + | > loss_disc_real_5: 0.16893191635608673 (0.2231663092133704) + | > loss_0: 2.0067765712738037 (2.3185247571280825) + | > grad_norm_0: tensor(19.6160, device='cuda:0') (tensor(29.8729, device='cuda:0')) + | > loss_gen: 2.680612564086914 (2.614255843537578) + | > loss_kl: 1.9548290967941284 (1.6989372010981099) + | > loss_feat: 9.614115715026855 (8.174878972299979) + | > loss_mel: 16.838665008544922 (16.663081340575495) + | > loss_duration: 0.7683843374252319 (0.9306147962473752) + | > loss_1: 31.85660743713379 (30.08176820733574) + | > grad_norm_1: tensor(458.7290, device='cuda:0') (tensor(425.4174, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.8452 (0.9860705364955945) + | > loader_time: 0.0046 (0.006406342045644696) + + + --> TIME: 2024-07-14 01:21:15 -- STEP: 139/151 -- GLOBAL_STEP: 109300 + | > loss_disc: 2.256136417388916 (2.30152122494128) + | > loss_disc_real_0: 0.09274142980575562 (0.11977482830984992) + | > loss_disc_real_1: 0.19788722693920135 (0.20182192968807633) + | > loss_disc_real_2: 0.21480870246887207 (0.20837352537422726) + | > loss_disc_real_3: 0.166005939245224 (0.2060893829777944) + | > loss_disc_real_4: 0.2034301608800888 (0.20359680303137936) + | > loss_disc_real_5: 0.22500494122505188 (0.2227612978262867) + | > loss_0: 2.256136417388916 (2.30152122494128) + | > grad_norm_0: tensor(9.3713, device='cuda:0') (tensor(26.0737, device='cuda:0')) + | > loss_gen: 2.495532989501953 (2.6283201433771812) + | > loss_kl: 1.5590450763702393 (1.7073490992724467) + | > loss_feat: 9.551480293273926 (8.354847729634885) + | > loss_mel: 16.482173919677734 (16.736196497361437) + | > loss_duration: 0.9191787242889404 (0.9277732063540451) + | > loss_1: 31.007410049438477 (30.35448676733662) + | > grad_norm_1: tensor(111.0086, device='cuda:0') (tensor(394.1603, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.8731 (0.9706504602226422) + | > loader_time: 0.0044 (0.006119916764952296) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03165652201725887 (-0.0014529228210449219) + | > avg_loss_disc: 2.574327743970431 (-0.06869693902822638) + | > avg_loss_disc_real_0: 0.2523502054122778 (+0.06913049519062042) + | > avg_loss_disc_real_1: 0.20410334376188424 (-0.05014288998567143) + | > avg_loss_disc_real_2: 0.2540995157681979 (-0.04016669438435477) + | > avg_loss_disc_real_3: 0.22451650293973777 (-0.026353979339966388) + | > avg_loss_disc_real_4: 0.23949349041168505 (-0.022080266704926133) + | > avg_loss_disc_real_5: 0.23115315574866074 (-0.0029554252441112983) + | > avg_loss_0: 2.574327743970431 (-0.06869693902822638) + | > avg_loss_gen: 2.4822671963618346 (+0.03476678408109146) + | > avg_loss_kl: 3.758585031215961 (+0.1215830032642069) + | > avg_loss_feat: 8.370198579934927 (+0.5361098509568434) + | > avg_loss_mel: 17.12831658583421 (+0.18878768040583793) + | > avg_loss_duration: 1.3580074860499456 (+0.01700709416316104) + | > avg_loss_1: 33.09737454927885 (+0.8982533674973681) + + + > EPOCH: 161/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:21:37)  + + --> TIME: 2024-07-14 01:22:17 -- STEP: 38/151 -- GLOBAL_STEP: 109350 + | > loss_disc: 2.2666096687316895 (2.3156879512887256) + | > loss_disc_real_0: 0.08409618586301804 (0.11339263323890536) + | > loss_disc_real_1: 0.19010969996452332 (0.20823382154891365) + | > loss_disc_real_2: 0.2657985985279083 (0.21026647326193357) + | > loss_disc_real_3: 0.18126589059829712 (0.2048754454835465) + | > loss_disc_real_4: 0.22179676592350006 (0.21152121181550773) + | > loss_disc_real_5: 0.24699236452579498 (0.224124750024394) + | > loss_0: 2.2666096687316895 (2.3156879512887256) + | > grad_norm_0: tensor(16.4193, device='cuda:0') (tensor(29.4987, device='cuda:0')) + | > loss_gen: 2.710914373397827 (2.626721501350403) + | > loss_kl: 1.4395439624786377 (1.6916505317938955) + | > loss_feat: 8.012389183044434 (8.338443869038638) + | > loss_mel: 16.513826370239258 (16.74893866087261) + | > loss_duration: 0.8975154161453247 (0.9262271570531946) + | > loss_1: 29.574190139770508 (30.331981809515703) + | > grad_norm_1: tensor(410.6852, device='cuda:0') (tensor(425.5212, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.8849 (0.9667793575086092) + | > loader_time: 0.0052 (0.005983578531365646) + + + --> TIME: 2024-07-14 01:23:08 -- STEP: 88/151 -- GLOBAL_STEP: 109400 + | > loss_disc: 2.166940927505493 (2.321582756259224) + | > loss_disc_real_0: 0.08608074486255646 (0.11735866997729648) + | > loss_disc_real_1: 0.15921708941459656 (0.20595388012853535) + | > loss_disc_real_2: 0.19463792443275452 (0.21156211908568034) + | > loss_disc_real_3: 0.1935667246580124 (0.206246830269017) + | > loss_disc_real_4: 0.20096947252750397 (0.20796203494749277) + | > loss_disc_real_5: 0.24133826792240143 (0.22401408106088638) + | > loss_0: 2.166940927505493 (2.321582756259224) + | > grad_norm_0: tensor(16.8387, device='cuda:0') (tensor(29.0413, device='cuda:0')) + | > loss_gen: 2.792585849761963 (2.6134297766468735) + | > loss_kl: 1.5913513898849487 (1.7437355044213207) + | > loss_feat: 9.105937004089355 (8.180162446065387) + | > loss_mel: 17.257524490356445 (16.737363262609993) + | > loss_duration: 0.8593407869338989 (0.9204661019823768) + | > loss_1: 31.606739044189453 (30.19515709443526) + | > grad_norm_1: tensor(219.1875, device='cuda:0') (tensor(439.4063, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.8667 (0.99328259175474) + | > loader_time: 0.0058 (0.0062781409783796835) + + + --> TIME: 2024-07-14 01:23:57 -- STEP: 138/151 -- GLOBAL_STEP: 109450 + | > loss_disc: 2.141995429992676 (2.3105322139850557) + | > loss_disc_real_0: 0.08450769633054733 (0.11906125647542269) + | > loss_disc_real_1: 0.17556548118591309 (0.2019619140504063) + | > loss_disc_real_2: 0.1758849024772644 (0.210583856680255) + | > loss_disc_real_3: 0.16673026978969574 (0.20550938701068144) + | > loss_disc_real_4: 0.18473118543624878 (0.20747491782126215) + | > loss_disc_real_5: 0.17397402226924896 (0.22355335756488467) + | > loss_0: 2.141995429992676 (2.3105322139850557) + | > grad_norm_0: tensor(10.5836, device='cuda:0') (tensor(28.5397, device='cuda:0')) + | > loss_gen: 2.434046983718872 (2.6218963394994312) + | > loss_kl: 1.3625589609146118 (1.7198722155197808) + | > loss_feat: 8.93668270111084 (8.240230017814081) + | > loss_mel: 17.16507339477539 (16.714417575062182) + | > loss_duration: 0.9389922618865967 (0.9294932942459548) + | > loss_1: 30.83735466003418 (30.22590941277103) + | > grad_norm_1: tensor(75.8179, device='cuda:0') (tensor(435.6235, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.8378 (0.9830873755441196) + | > loader_time: 0.0047 (0.006277030792789183) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032286955760075495 (+0.0006304337428166273) + | > avg_loss_disc: 2.528072577256423 (-0.04625516671400787) + | > avg_loss_disc_real_0: 0.23541304698357216 (-0.016937158428705662) + | > avg_loss_disc_real_1: 0.22589785777605498 (+0.021794514014170735) + | > avg_loss_disc_real_2: 0.21806904329703405 (-0.03603047247116384) + | > avg_loss_disc_real_3: 0.21419370747529543 (-0.010322795464442341) + | > avg_loss_disc_real_4: 0.1902953157058129 (-0.04919817470587215) + | > avg_loss_disc_real_5: 0.2203959604868522 (-0.010757195261808544) + | > avg_loss_0: 2.528072577256423 (-0.04625516671400787) + | > avg_loss_gen: 2.425429490896372 (-0.05683770546546274) + | > avg_loss_kl: 3.751926458798922 (-0.006658572417038933) + | > avg_loss_feat: 7.9141872846163235 (-0.4560112953186035) + | > avg_loss_mel: 16.712226647597095 (-0.41608993823711415) + | > avg_loss_duration: 1.3424020730532134 (-0.015605412996732237) + | > avg_loss_1: 32.14617112966684 (-0.9512034196120069) + + + > EPOCH: 162/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:24:21)  + + --> TIME: 2024-07-14 01:25:00 -- STEP: 37/151 -- GLOBAL_STEP: 109500 + | > loss_disc: 2.1883909702301025 (2.307981033582945) + | > loss_disc_real_0: 0.11237792670726776 (0.1213757947490022) + | > loss_disc_real_1: 0.194699227809906 (0.2007478403078543) + | > loss_disc_real_2: 0.18942119181156158 (0.20785559915207527) + | > loss_disc_real_3: 0.17203880846500397 (0.20395521416857437) + | > loss_disc_real_4: 0.26316145062446594 (0.20794510196995092) + | > loss_disc_real_5: 0.20900581777095795 (0.22622970431237607) + | > loss_0: 2.1883909702301025 (2.307981033582945) + | > grad_norm_0: tensor(18.1839, device='cuda:0') (tensor(29.0237, device='cuda:0')) + | > loss_gen: 2.810941696166992 (2.6275044131923364) + | > loss_kl: 1.7746341228485107 (1.7838015395241815) + | > loss_feat: 8.608122825622559 (8.265907983522158) + | > loss_mel: 17.28314208984375 (16.705413173984837) + | > loss_duration: 0.9127886891365051 (0.9204370943275658) + | > loss_1: 31.389629364013672 (30.303064294763512) + | > grad_norm_1: tensor(727.8923, device='cuda:0') (tensor(437.3197, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.9064 (0.9949593866193617) + | > loader_time: 0.0062 (0.006805265271985853) + + + --> TIME: 2024-07-14 01:25:47 -- STEP: 87/151 -- GLOBAL_STEP: 109550 + | > loss_disc: 2.226085901260376 (2.302344182441975) + | > loss_disc_real_0: 0.11913817375898361 (0.11630956164893062) + | > loss_disc_real_1: 0.14791440963745117 (0.20188214792602363) + | > loss_disc_real_2: 0.1661631464958191 (0.2075267041037823) + | > loss_disc_real_3: 0.2179795503616333 (0.20433291604464082) + | > loss_disc_real_4: 0.1891099363565445 (0.20678372467043757) + | > loss_disc_real_5: 0.23077937960624695 (0.22377663336951156) + | > loss_0: 2.226085901260376 (2.302344182441975) + | > grad_norm_0: tensor(13.0059, device='cuda:0') (tensor(24.2155, device='cuda:0')) + | > loss_gen: 2.6440699100494385 (2.6301995118459067) + | > loss_kl: 1.8543577194213867 (1.7459069106770662) + | > loss_feat: 8.83897876739502 (8.439700702141074) + | > loss_mel: 17.719318389892578 (16.76849137229481) + | > loss_duration: 0.9010705947875977 (0.9269131976982643) + | > loss_1: 31.957794189453125 (30.511211767963978) + | > grad_norm_1: tensor(103.7691, device='cuda:0') (tensor(398.9841, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.892 (0.949717318874666) + | > loader_time: 0.0053 (0.0062310777861496495) + + + --> TIME: 2024-07-14 01:26:39 -- STEP: 137/151 -- GLOBAL_STEP: 109600 + | > loss_disc: 2.4705758094787598 (2.3000498313973434) + | > loss_disc_real_0: 0.20463711023330688 (0.11715151456585765) + | > loss_disc_real_1: 0.20245341956615448 (0.20330971729581374) + | > loss_disc_real_2: 0.26294493675231934 (0.2090789789374728) + | > loss_disc_real_3: 0.17409397661685944 (0.20517304463543162) + | > loss_disc_real_4: 0.19283150136470795 (0.2049221575912768) + | > loss_disc_real_5: 0.20798908174037933 (0.22068865390589637) + | > loss_0: 2.4705758094787598 (2.3000498313973434) + | > grad_norm_0: tensor(36.7768, device='cuda:0') (tensor(25.5325, device='cuda:0')) + | > loss_gen: 2.6305112838745117 (2.633022224816093) + | > loss_kl: 1.5875208377838135 (1.7582966785361298) + | > loss_feat: 7.521496772766113 (8.349776809232955) + | > loss_mel: 16.938617706298828 (16.717345801583154) + | > loss_duration: 0.9305406808853149 (0.9269357384556401) + | > loss_1: 29.608686447143555 (30.385377257409758) + | > grad_norm_1: tensor(592.2986, device='cuda:0') (tensor(429.6495, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.968 (0.9782310872182359) + | > loader_time: 0.0055 (0.006395926440719271) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033864608177771933 (+0.0015776524176964388) + | > avg_loss_disc: 2.6886787047752967 (+0.1606061275188737) + | > avg_loss_disc_real_0: 0.3761928883882669 (+0.14077984140469477) + | > avg_loss_disc_real_1: 0.22146231738420633 (-0.0044355403918486525) + | > avg_loss_disc_real_2: 0.24349232934988463 (+0.025423286052850574) + | > avg_loss_disc_real_3: 0.25269597883407885 (+0.038502271358783424) + | > avg_loss_disc_real_4: 0.25624939570060146 (+0.06595407999478856) + | > avg_loss_disc_real_5: 0.24367935497027177 (+0.023283394483419567) + | > avg_loss_0: 2.6886787047752967 (+0.1606061275188737) + | > avg_loss_gen: 2.6237720342782826 (+0.19834254338191082) + | > avg_loss_kl: 3.8949986421144924 (+0.14307218331557037) + | > avg_loss_feat: 7.90009095118596 (-0.01409633343036365) + | > avg_loss_mel: 16.75760555267334 (+0.045378905076244536) + | > avg_loss_duration: 1.324887835062467 (-0.01751423799074625) + | > avg_loss_1: 32.50135509784405 (+0.355183968177208) + + + > EPOCH: 163/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:27:01)  + + --> TIME: 2024-07-14 01:27:40 -- STEP: 36/151 -- GLOBAL_STEP: 109650 + | > loss_disc: 2.1683542728424072 (2.2997922102610273) + | > loss_disc_real_0: 0.04483708366751671 (0.1166173176219066) + | > loss_disc_real_1: 0.17425023019313812 (0.20417138975527552) + | > loss_disc_real_2: 0.1639956533908844 (0.20965496781799528) + | > loss_disc_real_3: 0.2308368980884552 (0.20563074201345444) + | > loss_disc_real_4: 0.21071700751781464 (0.20748950292666754) + | > loss_disc_real_5: 0.18643292784690857 (0.2187934642036756) + | > loss_0: 2.1683542728424072 (2.2997922102610273) + | > grad_norm_0: tensor(40.8799, device='cuda:0') (tensor(28.9711, device='cuda:0')) + | > loss_gen: 2.636197566986084 (2.616694675551521) + | > loss_kl: 1.539278268814087 (1.7215487129158444) + | > loss_feat: 8.879985809326172 (8.179254134496054) + | > loss_mel: 16.448110580444336 (16.474929359224106) + | > loss_duration: 0.9929097890853882 (0.9371639870935016) + | > loss_1: 30.496482849121094 (29.929591019948322) + | > grad_norm_1: tensor(679.8901, device='cuda:0') (tensor(515.0630, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.8192 (1.0067398813035753) + | > loader_time: 0.0044 (0.006477011574639214) + + + --> TIME: 2024-07-14 01:28:30 -- STEP: 86/151 -- GLOBAL_STEP: 109700 + | > loss_disc: 2.2700483798980713 (2.2697691307511434) + | > loss_disc_real_0: 0.1537538468837738 (0.11573785855326542) + | > loss_disc_real_1: 0.21532994508743286 (0.1996823292832042) + | > loss_disc_real_2: 0.21849989891052246 (0.20728676402291588) + | > loss_disc_real_3: 0.21493278443813324 (0.20236151093660396) + | > loss_disc_real_4: 0.19788967072963715 (0.2046601212648458) + | > loss_disc_real_5: 0.1783723682165146 (0.21953268262536027) + | > loss_0: 2.2700483798980713 (2.2697691307511434) + | > grad_norm_0: tensor(19.4976, device='cuda:0') (tensor(26.9569, device='cuda:0')) + | > loss_gen: 2.6070263385772705 (2.6471405389697047) + | > loss_kl: 1.7235803604125977 (1.7233618276063787) + | > loss_feat: 7.824462890625 (8.390436543974767) + | > loss_mel: 16.364164352416992 (16.605677560318348) + | > loss_duration: 0.8805012702941895 (0.9293600334677585) + | > loss_1: 29.399736404418945 (30.295976660972418) + | > grad_norm_1: tensor(553.3685, device='cuda:0') (tensor(459.8083, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.8983 (0.9865694295528327) + | > loader_time: 0.007 (0.006417152493499046) + + + --> TIME: 2024-07-14 01:29:19 -- STEP: 136/151 -- GLOBAL_STEP: 109750 + | > loss_disc: 2.3551180362701416 (2.281711501233719) + | > loss_disc_real_0: 0.1268199384212494 (0.11519043122911278) + | > loss_disc_real_1: 0.22727635502815247 (0.1999526962857036) + | > loss_disc_real_2: 0.22590003907680511 (0.20817198385210597) + | > loss_disc_real_3: 0.1932341605424881 (0.20393830515882544) + | > loss_disc_real_4: 0.2331748902797699 (0.2048722628284903) + | > loss_disc_real_5: 0.19011682271957397 (0.22045654047499685) + | > loss_0: 2.3551180362701416 (2.281711501233719) + | > grad_norm_0: tensor(53.9521, device='cuda:0') (tensor(28.6529, device='cuda:0')) + | > loss_gen: 2.4006714820861816 (2.6518267673604625) + | > loss_kl: 2.2768027782440186 (1.732672491494347) + | > loss_feat: 5.661243915557861 (8.40309490175808) + | > loss_mel: 14.377052307128906 (16.60690634390887) + | > loss_duration: 1.1924986839294434 (0.9309528684791396) + | > loss_1: 25.908267974853516 (30.32545353384579) + | > grad_norm_1: tensor(334.9993, device='cuda:0') (tensor(486.1601, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 2.2766 (0.9799400007023533) + | > loader_time: 0.017 (0.006274928064907297) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03620663056006798 (+0.002342022382296048) + | > avg_loss_disc: 2.535354944375845 (-0.15332376039945173) + | > avg_loss_disc_real_0: 0.2818802480514233 (-0.09431264033684361) + | > avg_loss_disc_real_1: 0.22785587035692656 (+0.0063935529727202345) + | > avg_loss_disc_real_2: 0.24711245756882888 (+0.003620128218944252) + | > avg_loss_disc_real_3: 0.24596486183313224 (-0.006731117000946613) + | > avg_loss_disc_real_4: 0.258529469370842 (+0.0022800736702405233) + | > avg_loss_disc_real_5: 0.2614505417071856 (+0.017771186736913858) + | > avg_loss_0: 2.535354944375845 (-0.15332376039945173) + | > avg_loss_gen: 2.768803523137019 (+0.1450314888587365) + | > avg_loss_kl: 3.8083840700296254 (-0.086614572084867) + | > avg_loss_feat: 7.682140497060923 (-0.21795045412503722) + | > avg_loss_mel: 16.101166651799126 (-0.656438900874214) + | > avg_loss_duration: 1.2938016102864192 (-0.031086224776047944) + | > avg_loss_1: 31.65429526108962 (-0.8470598367544291) + + + > EPOCH: 164/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:29:41)  + + --> TIME: 2024-07-14 01:30:19 -- STEP: 35/151 -- GLOBAL_STEP: 109800 + | > loss_disc: 2.064201593399048 (2.2801404339926585) + | > loss_disc_real_0: 0.09103893488645554 (0.1192699013011796) + | > loss_disc_real_1: 0.14602193236351013 (0.1935905784368515) + | > loss_disc_real_2: 0.1807783544063568 (0.20617302571024212) + | > loss_disc_real_3: 0.13360075652599335 (0.2011029383965901) + | > loss_disc_real_4: 0.14105089008808136 (0.202694918853896) + | > loss_disc_real_5: 0.16576462984085083 (0.2154451221227646) + | > loss_0: 2.064201593399048 (2.2801404339926585) + | > grad_norm_0: tensor(18.9051, device='cuda:0') (tensor(32.8573, device='cuda:0')) + | > loss_gen: 3.0248308181762695 (2.692404556274414) + | > loss_kl: 2.03354549407959 (1.762510687964303) + | > loss_feat: 10.15442943572998 (8.511924784524101) + | > loss_mel: 17.500520706176758 (16.796391432625907) + | > loss_duration: 0.9922036528587341 (0.9363022548811776) + | > loss_1: 33.705528259277344 (30.699533680507116) + | > grad_norm_1: tensor(97.9131, device='cuda:0') (tensor(505.6329, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.8191 (1.0010754108428956) + | > loader_time: 0.004 (0.006583888190133231) + + + --> TIME: 2024-07-14 01:31:05 -- STEP: 85/151 -- GLOBAL_STEP: 109850 + | > loss_disc: 2.251098155975342 (2.282841626335593) + | > loss_disc_real_0: 0.08181287348270416 (0.1169745176592294) + | > loss_disc_real_1: 0.22345636785030365 (0.19994623748695148) + | > loss_disc_real_2: 0.24447672069072723 (0.2087224020677454) + | > loss_disc_real_3: 0.22840644419193268 (0.20488247520783373) + | > loss_disc_real_4: 0.2055797576904297 (0.2056100300129722) + | > loss_disc_real_5: 0.2676257789134979 (0.22048759215018324) + | > loss_0: 2.251098155975342 (2.282841626335593) + | > grad_norm_0: tensor(22.6844, device='cuda:0') (tensor(26.6782, device='cuda:0')) + | > loss_gen: 2.5922908782958984 (2.6674671734080593) + | > loss_kl: 1.9566816091537476 (1.7301376980893752) + | > loss_feat: 8.6701021194458 (8.55273232740515) + | > loss_mel: 17.040145874023438 (16.697805988087367) + | > loss_duration: 0.920513391494751 (0.9321037103148068) + | > loss_1: 31.179733276367188 (30.58024668974035) + | > grad_norm_1: tensor(477.1144, device='cuda:0') (tensor(422.6113, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.8761 (0.9498622080859017) + | > loader_time: 0.0059 (0.006236250260296988) + + + --> TIME: 2024-07-14 01:31:56 -- STEP: 135/151 -- GLOBAL_STEP: 109900 + | > loss_disc: 2.241427183151245 (2.2826296718032273) + | > loss_disc_real_0: 0.11963154375553131 (0.11481664525689902) + | > loss_disc_real_1: 0.14777036011219025 (0.20097714960575103) + | > loss_disc_real_2: 0.23380330204963684 (0.20903366627516567) + | > loss_disc_real_3: 0.24632108211517334 (0.20474385111420243) + | > loss_disc_real_4: 0.18270540237426758 (0.20586033275833837) + | > loss_disc_real_5: 0.21145115792751312 (0.22009114711372935) + | > loss_0: 2.241427183151245 (2.2826296718032273) + | > grad_norm_0: tensor(45.1714, device='cuda:0') (tensor(24.7424, device='cuda:0')) + | > loss_gen: 2.8277065753936768 (2.657692889814023) + | > loss_kl: 1.6678466796875 (1.7454658221315456) + | > loss_feat: 8.986555099487305 (8.484244862309216) + | > loss_mel: 17.390884399414062 (16.704074365121347) + | > loss_duration: 0.8611024618148804 (0.9285050652645253) + | > loss_1: 31.73409652709961 (30.519982754742657) + | > grad_norm_1: tensor(892.0859, device='cuda:0') (tensor(416.8540, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.8496 (0.9661143832736545) + | > loader_time: 0.0044 (0.00634871412206579) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03429044209993803 (-0.0019161884601299542) + | > avg_loss_disc: 2.5373375049004188 (+0.0019825605245737954) + | > avg_loss_disc_real_0: 0.2611928880214691 (-0.020687360029954194) + | > avg_loss_disc_real_1: 0.18693242623255804 (-0.04092344412436852) + | > avg_loss_disc_real_2: 0.2053468685883742 (-0.04176558898045468) + | > avg_loss_disc_real_3: 0.24426904779214126 (-0.0016958140409909783) + | > avg_loss_disc_real_4: 0.25247450746022737 (-0.006054961910614609) + | > avg_loss_disc_real_5: 0.21731078051603758 (-0.04413976119114804) + | > avg_loss_0: 2.5373375049004188 (+0.0019825605245737954) + | > avg_loss_gen: 2.433102112550002 (-0.3357014105870171) + | > avg_loss_kl: 3.7225068715902476 (-0.0858771984393778) + | > avg_loss_feat: 7.630295460040752 (-0.05184503702017018) + | > avg_loss_mel: 16.390132097097542 (+0.28896544529841606) + | > avg_loss_duration: 1.340456880055941 (+0.046655269769521945) + | > avg_loss_1: 31.516492990347054 (-0.13780227074256501) + + + > EPOCH: 165/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:32:22)  + + --> TIME: 2024-07-14 01:32:58 -- STEP: 34/151 -- GLOBAL_STEP: 109950 + | > loss_disc: 2.179481267929077 (2.288175645996542) + | > loss_disc_real_0: 0.17499086260795593 (0.12174484922605402) + | > loss_disc_real_1: 0.18182876706123352 (0.19797628548215418) + | > loss_disc_real_2: 0.17169925570487976 (0.20499972582739942) + | > loss_disc_real_3: 0.2028503119945526 (0.20583358626155293) + | > loss_disc_real_4: 0.2322099506855011 (0.21003188981729395) + | > loss_disc_real_5: 0.15901267528533936 (0.2179082601385958) + | > loss_0: 2.179481267929077 (2.288175645996542) + | > grad_norm_0: tensor(33.8610, device='cuda:0') (tensor(29.3375, device='cuda:0')) + | > loss_gen: 2.7362027168273926 (2.6547498212141147) + | > loss_kl: 1.707736849784851 (1.7847506737007814) + | > loss_feat: 8.148542404174805 (8.338255461524515) + | > loss_mel: 16.018226623535156 (16.795243796180277) + | > loss_duration: 0.8227227330207825 (0.9285566228277543) + | > loss_1: 29.43343162536621 (30.5015567891738) + | > grad_norm_1: tensor(159.0739, device='cuda:0') (tensor(439.0441, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.868 (0.9851669563966639) + | > loader_time: 0.0057 (0.006202473359949449) + + + --> TIME: 2024-07-14 01:33:47 -- STEP: 84/151 -- GLOBAL_STEP: 110000 + | > loss_disc: 2.0974676609039307 (2.2877581914265943) + | > loss_disc_real_0: 0.10421749949455261 (0.11773674846405074) + | > loss_disc_real_1: 0.1637127697467804 (0.20028659541692057) + | > loss_disc_real_2: 0.1886526495218277 (0.20812093182688668) + | > loss_disc_real_3: 0.15708215534687042 (0.20540842538078627) + | > loss_disc_real_4: 0.1621803194284439 (0.20689228345595656) + | > loss_disc_real_5: 0.2118629664182663 (0.21997279602856862) + | > loss_0: 2.0974676609039307 (2.2877581914265943) + | > grad_norm_0: tensor(13.0211, device='cuda:0') (tensor(30.4912, device='cuda:0')) + | > loss_gen: 2.6329877376556396 (2.6577973961830144) + | > loss_kl: 1.5673412084579468 (1.7308043908505213) + | > loss_feat: 9.044281005859375 (8.434274934587025) + | > loss_mel: 16.470970153808594 (16.703272138323097) + | > loss_duration: 0.9171582460403442 (0.9248486246381488) + | > loss_1: 30.63273811340332 (30.4509977159046) + | > grad_norm_1: tensor(123.7416, device='cuda:0') (tensor(419.6010, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.8411 (0.966554806345985) + | > loader_time: 0.0071 (0.006062922023591541) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_110000.pth + + --> TIME: 2024-07-14 01:35:07 -- STEP: 134/151 -- GLOBAL_STEP: 110050 + | > loss_disc: 2.352943181991577 (2.2879371714236125) + | > loss_disc_real_0: 0.13472880423069 (0.11656798416776444) + | > loss_disc_real_1: 0.21796295046806335 (0.20083829816152804) + | > loss_disc_real_2: 0.21423299610614777 (0.20881756039252922) + | > loss_disc_real_3: 0.2311900407075882 (0.20657428665392435) + | > loss_disc_real_4: 0.18398547172546387 (0.20545681938529015) + | > loss_disc_real_5: 0.24867337942123413 (0.21952594260671246) + | > loss_0: 2.352943181991577 (2.2879371714236125) + | > grad_norm_0: tensor(14.4001, device='cuda:0') (tensor(26.7531, device='cuda:0')) + | > loss_gen: 2.6705915927886963 (2.6463792733292086) + | > loss_kl: 1.5860077142715454 (1.7225825123822511) + | > loss_feat: 8.457662582397461 (8.402822811212117) + | > loss_mel: 17.00739288330078 (16.735153191125214) + | > loss_duration: 0.8659349679946899 (0.92367498243033) + | > loss_1: 30.587589263916016 (30.430612763362145) + | > grad_norm_1: tensor(451.3918, device='cuda:0') (tensor(401.2712, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.9598 (0.9804589463703668) + | > loader_time: 0.0071 (0.006224456118113959) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03415474524864784 (-0.00013569685129018877) + | > avg_loss_disc: 2.659684804769663 (+0.12234729986924409) + | > avg_loss_disc_real_0: 0.3463703967057742 (+0.08517750868430507) + | > avg_loss_disc_real_1: 0.19392026960849762 (+0.006987843375939579) + | > avg_loss_disc_real_2: 0.2557202554666079 (+0.05037338687823373) + | > avg_loss_disc_real_3: 0.2839775417859738 (+0.03970849399383253) + | > avg_loss_disc_real_4: 0.2834309706321129 (+0.03095646317188555) + | > avg_loss_disc_real_5: 0.251158655836032 (+0.03384787531999439) + | > avg_loss_0: 2.659684804769663 (+0.12234729986924409) + | > avg_loss_gen: 2.736633172402015 (+0.3035310598520131) + | > avg_loss_kl: 3.712160642330463 (-0.010346229259784678) + | > avg_loss_feat: 8.106379655691294 (+0.4760841956505413) + | > avg_loss_mel: 16.870167145362267 (+0.48003504826472465) + | > avg_loss_duration: 1.3119697387401874 (-0.028487141315753695) + | > avg_loss_1: 32.7373104095459 (+1.2208174191988448) + + + > EPOCH: 166/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:35:31)  + + --> TIME: 2024-07-14 01:36:06 -- STEP: 33/151 -- GLOBAL_STEP: 110100 + | > loss_disc: 2.436633586883545 (2.305410471829501) + | > loss_disc_real_0: 0.1768963634967804 (0.12414862621914256) + | > loss_disc_real_1: 0.22999827563762665 (0.19585574756969104) + | > loss_disc_real_2: 0.23116594552993774 (0.20914274679891992) + | > loss_disc_real_3: 0.20375803112983704 (0.2026992878227523) + | > loss_disc_real_4: 0.19353140890598297 (0.20285974397803797) + | > loss_disc_real_5: 0.28691205382347107 (0.2243855893611908) + | > loss_0: 2.436633586883545 (2.305410471829501) + | > grad_norm_0: tensor(12.0879, device='cuda:0') (tensor(17.9034, device='cuda:0')) + | > loss_gen: 2.455744743347168 (2.615150039846247) + | > loss_kl: 2.2537195682525635 (1.6790152968782368) + | > loss_feat: 6.039551734924316 (8.289819442864621) + | > loss_mel: 15.666549682617188 (16.447862856315844) + | > loss_duration: 0.8622821569442749 (0.9250604636741407) + | > loss_1: 27.277847290039062 (29.95690813931552) + | > grad_norm_1: tensor(160.4180, device='cuda:0') (tensor(273.2780, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 1.0417 (0.9679378379474987) + | > loader_time: 0.0078 (0.006311329928311434) + + + --> TIME: 2024-07-14 01:36:57 -- STEP: 83/151 -- GLOBAL_STEP: 110150 + | > loss_disc: 2.3045246601104736 (2.319894302322204) + | > loss_disc_real_0: 0.15612593293190002 (0.12761910928480596) + | > loss_disc_real_1: 0.26553329825401306 (0.20095362074403877) + | > loss_disc_real_2: 0.24349135160446167 (0.211388802672007) + | > loss_disc_real_3: 0.2327859103679657 (0.2028380937605019) + | > loss_disc_real_4: 0.1842511147260666 (0.2055021810782961) + | > loss_disc_real_5: 0.19607625901699066 (0.22284279440540866) + | > loss_0: 2.3045246601104736 (2.319894302322204) + | > grad_norm_0: tensor(23.2099, device='cuda:0') (tensor(16.9353, device='cuda:0')) + | > loss_gen: 2.568479537963867 (2.602611265986799) + | > loss_kl: 1.4990590810775757 (1.691244937569262) + | > loss_feat: 8.240769386291504 (8.310745744820101) + | > loss_mel: 15.496298789978027 (16.560267241604357) + | > loss_duration: 0.931660532951355 (0.9397121114903185) + | > loss_1: 28.736265182495117 (30.104581258383142) + | > grad_norm_1: tensor(254.6836, device='cuda:0') (tensor(231.8766, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.8705 (0.9979519240827446) + | > loader_time: 0.0048 (0.006549171654574841) + + + --> TIME: 2024-07-14 01:37:45 -- STEP: 133/151 -- GLOBAL_STEP: 110200 + | > loss_disc: 2.1183509826660156 (2.3164505779295044) + | > loss_disc_real_0: 0.08359585702419281 (0.12332653789255857) + | > loss_disc_real_1: 0.21410903334617615 (0.20143241306444756) + | > loss_disc_real_2: 0.18103832006454468 (0.2108081596014195) + | > loss_disc_real_3: 0.21041277050971985 (0.20493584546379576) + | > loss_disc_real_4: 0.16701851785182953 (0.2051980046737463) + | > loss_disc_real_5: 0.26299765706062317 (0.22264944644350754) + | > loss_0: 2.1183509826660156 (2.3164505779295044) + | > grad_norm_0: tensor(6.5162, device='cuda:0') (tensor(17.4737, device='cuda:0')) + | > loss_gen: 2.8535995483398438 (2.595150789820162) + | > loss_kl: 1.5410170555114746 (1.682564051527726) + | > loss_feat: 9.469538688659668 (8.337572775389019) + | > loss_mel: 18.61777114868164 (16.62962965140664) + | > loss_duration: 0.9151537418365479 (0.9299015743391854) + | > loss_1: 33.39707946777344 (30.174818827693624) + | > grad_norm_1: tensor(334.6162, device='cuda:0') (tensor(244.6504, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.8697 (0.9764172242100078) + | > loader_time: 0.0062 (0.006448026886559967) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.034187812071580156 (+3.3066822932317774e-05) + | > avg_loss_disc: 2.6507087304041934 (-0.008976074365469433) + | > avg_loss_disc_real_0: 0.28487183382877934 (-0.061498562876994844) + | > avg_loss_disc_real_1: 0.20181976831876314 (+0.007899498710265518) + | > avg_loss_disc_real_2: 0.2371360453275534 (-0.018584210139054536) + | > avg_loss_disc_real_3: 0.2508440544972053 (-0.03313348728876847) + | > avg_loss_disc_real_4: 0.2675304619165567 (-0.015900508715556205) + | > avg_loss_disc_real_5: 0.2372599450441507 (-0.013898710791881264) + | > avg_loss_0: 2.6507087304041934 (-0.008976074365469433) + | > avg_loss_gen: 2.4248461539928723 (-0.3117870184091429) + | > avg_loss_kl: 3.8630508276132436 (+0.15089018528278064) + | > avg_loss_feat: 7.585373438321627 (-0.521006217369667) + | > avg_loss_mel: 16.56020501943735 (-0.30996212592491545) + | > avg_loss_duration: 1.3852378588456373 (+0.07326812010544992) + | > avg_loss_1: 31.8187135549692 (-0.9185968545766983) + + + > EPOCH: 167/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:38:12)  + + --> TIME: 2024-07-14 01:38:44 -- STEP: 32/151 -- GLOBAL_STEP: 110250 + | > loss_disc: 2.335293769836426 (2.2292805686593065) + | > loss_disc_real_0: 0.14164802432060242 (0.10404595115687698) + | > loss_disc_real_1: 0.2284732609987259 (0.19428450101986527) + | > loss_disc_real_2: 0.23299704492092133 (0.20250833919271827) + | > loss_disc_real_3: 0.23198077082633972 (0.2027915152721107) + | > loss_disc_real_4: 0.23905044794082642 (0.20130500430241227) + | > loss_disc_real_5: 0.24686454236507416 (0.21699115680530667) + | > loss_0: 2.335293769836426 (2.2292805686593065) + | > grad_norm_0: tensor(49.4045, device='cuda:0') (tensor(25.4839, device='cuda:0')) + | > loss_gen: 2.752811908721924 (2.6967228055000305) + | > loss_kl: 1.0985240936279297 (1.6217959970235825) + | > loss_feat: 6.673009872436523 (8.596434161067007) + | > loss_mel: 14.112911224365234 (16.75667560100555) + | > loss_duration: 0.8826904892921448 (0.9025776442140341) + | > loss_1: 25.519947052001953 (30.57420653104782) + | > grad_norm_1: tensor(750.0756, device='cuda:0') (tensor(492.7090, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.8812 (0.9050854817032814) + | > loader_time: 0.0058 (0.005601033568382263) + + + --> TIME: 2024-07-14 01:39:32 -- STEP: 82/151 -- GLOBAL_STEP: 110300 + | > loss_disc: 2.01310396194458 (2.258698300617496) + | > loss_disc_real_0: 0.05084855481982231 (0.10903564763323563) + | > loss_disc_real_1: 0.1439586728811264 (0.20043539510267536) + | > loss_disc_real_2: 0.13797152042388916 (0.20630002675986872) + | > loss_disc_real_3: 0.18775448203086853 (0.20476537543099102) + | > loss_disc_real_4: 0.1559813767671585 (0.20139679948731168) + | > loss_disc_real_5: 0.17540140450000763 (0.220128224753752) + | > loss_0: 2.01310396194458 (2.258698300617496) + | > grad_norm_0: tensor(31.6251, device='cuda:0') (tensor(28.8183, device='cuda:0')) + | > loss_gen: 2.6602208614349365 (2.69404972471842) + | > loss_kl: 1.4765875339508057 (1.667838537111515) + | > loss_feat: 9.985335350036621 (8.634978620017446) + | > loss_mel: 16.512733459472656 (16.81254300838564) + | > loss_duration: 1.0153883695602417 (0.9176016180980496) + | > loss_1: 31.650264739990234 (30.727011517780582) + | > grad_norm_1: tensor(560.0772, device='cuda:0') (tensor(517.0384, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.8223 (0.9327133806740365) + | > loader_time: 0.0054 (0.005872450223783168) + + + --> TIME: 2024-07-14 01:40:25 -- STEP: 132/151 -- GLOBAL_STEP: 110350 + | > loss_disc: 2.4278104305267334 (2.2714418073495226) + | > loss_disc_real_0: 0.24899479746818542 (0.11463920614033034) + | > loss_disc_real_1: 0.24743333458900452 (0.19974452686129193) + | > loss_disc_real_2: 0.24548183381557465 (0.20687333234783375) + | > loss_disc_real_3: 0.24087105691432953 (0.20610347085378386) + | > loss_disc_real_4: 0.23233497142791748 (0.20277472043579275) + | > loss_disc_real_5: 0.23312002420425415 (0.2197596202745582) + | > loss_0: 2.4278104305267334 (2.2714418073495226) + | > grad_norm_0: tensor(58.9162, device='cuda:0') (tensor(28.4128, device='cuda:0')) + | > loss_gen: 2.6434130668640137 (2.6725065942966575) + | > loss_kl: 2.0131289958953857 (1.7009242511156835) + | > loss_feat: 7.3056111335754395 (8.519029512549894) + | > loss_mel: 17.960678100585938 (16.780796014901366) + | > loss_duration: 0.9618821144104004 (0.9268630146980286) + | > loss_1: 30.884714126586914 (30.60011946071278) + | > grad_norm_1: tensor(681.8187, device='cuda:0') (tensor(491.2057, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.9626 (0.9803151975978505) + | > loader_time: 0.0076 (0.006414814428849654) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0337530649625338 (-0.0004347471090463545) + | > avg_loss_disc: 2.63743728857774 (-0.013271441826453234) + | > avg_loss_disc_real_0: 0.28985860026799715 (+0.004986766439217805) + | > avg_loss_disc_real_1: 0.25763498132045454 (+0.0558152130016914) + | > avg_loss_disc_real_2: 0.29036765144421506 (+0.05323160611666167) + | > avg_loss_disc_real_3: 0.228152056153004 (-0.022691998344201325) + | > avg_loss_disc_real_4: 0.2532389312982559 (-0.014291530618300796) + | > avg_loss_disc_real_5: 0.2617207307081956 (+0.024460785664044915) + | > avg_loss_0: 2.63743728857774 (-0.013271441826453234) + | > avg_loss_gen: 2.7017722680018497 (+0.2769261140089774) + | > avg_loss_kl: 3.9722317182100735 (+0.1091808905968299) + | > avg_loss_feat: 8.203896412482628 (+0.6185229741610012) + | > avg_loss_mel: 16.973461151123047 (+0.41325613168569575) + | > avg_loss_duration: 1.35055895952078 (-0.03467889932485724) + | > avg_loss_1: 33.20191970238319 (+1.3832061474139863) + + + > EPOCH: 168/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:40:53)  + + --> TIME: 2024-07-14 01:41:25 -- STEP: 31/151 -- GLOBAL_STEP: 110400 + | > loss_disc: 2.2934348583221436 (2.326951757554085) + | > loss_disc_real_0: 0.13969865441322327 (0.11760992917322344) + | > loss_disc_real_1: 0.2579367160797119 (0.20361124195398822) + | > loss_disc_real_2: 0.269622266292572 (0.211818705643377) + | > loss_disc_real_3: 0.17179565131664276 (0.2069195206126859) + | > loss_disc_real_4: 0.1934863030910492 (0.2108434578103404) + | > loss_disc_real_5: 0.23602963984012604 (0.22564184569543408) + | > loss_0: 2.2934348583221436 (2.326951757554085) + | > grad_norm_0: tensor(11.1150, device='cuda:0') (tensor(23.5946, device='cuda:0')) + | > loss_gen: 2.8935647010803223 (2.583776912381572) + | > loss_kl: 1.2489302158355713 (1.6209071797709311) + | > loss_feat: 9.473066329956055 (8.048523549110657) + | > loss_mel: 16.790761947631836 (16.285704458913496) + | > loss_duration: 0.8312585949897766 (0.9459573761109383) + | > loss_1: 31.237581253051758 (29.484869680097027) + | > grad_norm_1: tensor(57.4579, device='cuda:0') (tensor(348.0646, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.8664 (0.9657382349814138) + | > loader_time: 0.0046 (0.006193091792445029) + + + --> TIME: 2024-07-14 01:42:16 -- STEP: 81/151 -- GLOBAL_STEP: 110450 + | > loss_disc: 2.102329730987549 (2.3033004601796465) + | > loss_disc_real_0: 0.061684973537921906 (0.11494603702499542) + | > loss_disc_real_1: 0.18460339307785034 (0.20076646674194454) + | > loss_disc_real_2: 0.19249166548252106 (0.21189168389932608) + | > loss_disc_real_3: 0.21269969642162323 (0.20660530343467806) + | > loss_disc_real_4: 0.19885103404521942 (0.2078520181922265) + | > loss_disc_real_5: 0.2169537991285324 (0.2247596839327871) + | > loss_0: 2.102329730987549 (2.3033004601796465) + | > grad_norm_0: tensor(18.3391, device='cuda:0') (tensor(24.1760, device='cuda:0')) + | > loss_gen: 2.914515972137451 (2.6269066922458597) + | > loss_kl: 1.4269616603851318 (1.6750460826320412) + | > loss_feat: 10.04090690612793 (8.38246789390658) + | > loss_mel: 16.241674423217773 (16.63728401984697) + | > loss_duration: 0.9234656095504761 (0.932169046666887) + | > loss_1: 31.54752540588379 (30.253873825073242) + | > grad_norm_1: tensor(507.9913, device='cuda:0') (tensor(421.2408, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.8482 (0.9843462072772747) + | > loader_time: 0.0044 (0.00636587025206766) + + + --> TIME: 2024-07-14 01:43:05 -- STEP: 131/151 -- GLOBAL_STEP: 110500 + | > loss_disc: 2.4665627479553223 (2.2956583991305513) + | > loss_disc_real_0: 0.1719239056110382 (0.11692901720641223) + | > loss_disc_real_1: 0.21670527756214142 (0.20101604258058636) + | > loss_disc_real_2: 0.2633424997329712 (0.2101081558538757) + | > loss_disc_real_3: 0.26091468334198 (0.20599587608612221) + | > loss_disc_real_4: 0.23641471564769745 (0.20641310990766715) + | > loss_disc_real_5: 0.23841196298599243 (0.22165583449465628) + | > loss_0: 2.4665627479553223 (2.2956583991305513) + | > grad_norm_0: tensor(18.6268, device='cuda:0') (tensor(27.1192, device='cuda:0')) + | > loss_gen: 2.5458908081054688 (2.6570458230171496) + | > loss_kl: 1.8800619840621948 (1.691712347150759) + | > loss_feat: 7.568118095397949 (8.49329296869176) + | > loss_mel: 16.87368392944336 (16.676292026315934) + | > loss_duration: 1.0313868522644043 (0.9303099358354816) + | > loss_1: 29.89914321899414 (30.448653170170676) + | > grad_norm_1: tensor(463.3600, device='cuda:0') (tensor(441.5765, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 1.1155 (0.9822700169250258) + | > loader_time: 0.01 (0.006384329031441958) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03258983905498798 (-0.0011632259075458201) + | > avg_loss_disc: 2.6163200048299937 (-0.021117283747746463) + | > avg_loss_disc_real_0: 0.21387486045177168 (-0.07598373981622547) + | > avg_loss_disc_real_1: 0.22695847543386313 (-0.030676505886591404) + | > avg_loss_disc_real_2: 0.255372085250341 (-0.03499556619387406) + | > avg_loss_disc_real_3: 0.29909347341610837 (+0.07094141726310438) + | > avg_loss_disc_real_4: 0.2858118231479938 (+0.032572891849737884) + | > avg_loss_disc_real_5: 0.25323488964484286 (-0.008485841063352761) + | > avg_loss_0: 2.6163200048299937 (-0.021117283747746463) + | > avg_loss_gen: 2.59688348036546 (-0.10488878763638976) + | > avg_loss_kl: 3.7391758882082424 (-0.23305583000183105) + | > avg_loss_feat: 8.011379645420956 (-0.1925167670616723) + | > avg_loss_mel: 16.729914371783916 (-0.24354677933913038) + | > avg_loss_duration: 1.3317091098198524 (-0.018849849700927734) + | > avg_loss_1: 32.40906231219952 (-0.79285739018367) + + + > EPOCH: 169/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:43:34)  + + --> TIME: 2024-07-14 01:44:07 -- STEP: 30/151 -- GLOBAL_STEP: 110550 + | > loss_disc: 2.167422294616699 (2.262700812021891) + | > loss_disc_real_0: 0.10004054009914398 (0.11108448865513007) + | > loss_disc_real_1: 0.1774861216545105 (0.19810085942347844) + | > loss_disc_real_2: 0.17507168650627136 (0.20119791527589162) + | > loss_disc_real_3: 0.16646519303321838 (0.20910897304614384) + | > loss_disc_real_4: 0.20866285264492035 (0.20819280569752058) + | > loss_disc_real_5: 0.15753284096717834 (0.21486751933892567) + | > loss_0: 2.167422294616699 (2.262700812021891) + | > grad_norm_0: tensor(17.3252, device='cuda:0') (tensor(29.1718, device='cuda:0')) + | > loss_gen: 2.7231693267822266 (2.6505765914916988) + | > loss_kl: 1.715540885925293 (1.6947434465090434) + | > loss_feat: 10.005064010620117 (8.528141943613688) + | > loss_mel: 16.33053207397461 (16.68255945841472) + | > loss_duration: 0.9185584783554077 (0.9322901904582978) + | > loss_1: 31.69286346435547 (30.488311449686687) + | > grad_norm_1: tensor(610.1276, device='cuda:0') (tensor(493.0378, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.9496 (1.0062745968500773) + | > loader_time: 0.0054 (0.006504718462626139) + + + --> TIME: 2024-07-14 01:44:54 -- STEP: 80/151 -- GLOBAL_STEP: 110600 + | > loss_disc: 2.3065731525421143 (2.277510806918144) + | > loss_disc_real_0: 0.1352892965078354 (0.11565595483407379) + | > loss_disc_real_1: 0.23847872018814087 (0.20166886765509845) + | > loss_disc_real_2: 0.24519425630569458 (0.20784233491867782) + | > loss_disc_real_3: 0.2514653205871582 (0.20723930597305298) + | > loss_disc_real_4: 0.27384093403816223 (0.2063303293660283) + | > loss_disc_real_5: 0.20353013277053833 (0.21554417368024587) + | > loss_0: 2.3065731525421143 (2.277510806918144) + | > grad_norm_0: tensor(9.6268, device='cuda:0') (tensor(26.7191, device='cuda:0')) + | > loss_gen: 2.763458728790283 (2.653643694519996) + | > loss_kl: 1.660182237625122 (1.6829511299729352) + | > loss_feat: 9.776748657226562 (8.545702558755874) + | > loss_mel: 17.248985290527344 (16.723243880271912) + | > loss_duration: 0.830640435218811 (0.9183680973947048) + | > loss_1: 32.28001403808594 (30.523909401893615) + | > grad_norm_1: tensor(88.5653, device='cuda:0') (tensor(453.4803, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.9689 (0.96387699842453) + | > loader_time: 0.006 (0.0063114076852798465) + + + --> TIME: 2024-07-14 01:45:46 -- STEP: 130/151 -- GLOBAL_STEP: 110650 + | > loss_disc: 2.323800802230835 (2.284364454562848) + | > loss_disc_real_0: 0.09432964771986008 (0.11683220453560353) + | > loss_disc_real_1: 0.18173544108867645 (0.20189837790452517) + | > loss_disc_real_2: 0.24480049312114716 (0.2090690072912436) + | > loss_disc_real_3: 0.2343272715806961 (0.2060945456990829) + | > loss_disc_real_4: 0.24498625099658966 (0.2061557795565862) + | > loss_disc_real_5: 0.2686014175415039 (0.21817247294462644) + | > loss_0: 2.323800802230835 (2.284364454562848) + | > grad_norm_0: tensor(25.7719, device='cuda:0') (tensor(27.0082, device='cuda:0')) + | > loss_gen: 2.6766326427459717 (2.645570127780621) + | > loss_kl: 1.6480478048324585 (1.6636072140473588) + | > loss_feat: 7.763760566711426 (8.488058673418486) + | > loss_mel: 17.562177658081055 (16.72040224808913) + | > loss_duration: 0.8305082321166992 (0.9265837802336766) + | > loss_1: 30.481124877929688 (30.444222142146184) + | > grad_norm_1: tensor(501.3943, device='cuda:0') (tensor(460.6064, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.9595 (0.9884329942556528) + | > loader_time: 0.007 (0.006552784259502704) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03306739146892841 (+0.0004775524139404297) + | > avg_loss_disc: 2.620536969258235 (+0.0042169644282412655) + | > avg_loss_disc_real_0: 0.2733701169490814 (+0.059495256497309745) + | > avg_loss_disc_real_1: 0.23121575438059294 (+0.004257278946729809) + | > avg_loss_disc_real_2: 0.22722888107483202 (-0.028143204175508973) + | > avg_loss_disc_real_3: 0.22694571316242218 (-0.07214776025368619) + | > avg_loss_disc_real_4: 0.25650005501050216 (-0.029311768137491645) + | > avg_loss_disc_real_5: 0.23023777970900902 (-0.022997109935833843) + | > avg_loss_0: 2.620536969258235 (+0.0042169644282412655) + | > avg_loss_gen: 2.4271273062779355 (-0.16975617408752441) + | > avg_loss_kl: 3.664520263671875 (-0.07465562453636743) + | > avg_loss_feat: 7.975649907038762 (-0.03572973838219351) + | > avg_loss_mel: 17.130240220289963 (+0.40032584850604636) + | > avg_loss_duration: 1.323632341164809 (-0.008076768655043365) + | > avg_loss_1: 32.521170102632965 (+0.11210779043344843) + + + > EPOCH: 170/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:46:15)  + + --> TIME: 2024-07-14 01:46:45 -- STEP: 29/151 -- GLOBAL_STEP: 110700 + | > loss_disc: 2.289858341217041 (2.2889770885993697) + | > loss_disc_real_0: 0.08021270483732224 (0.1197658336625017) + | > loss_disc_real_1: 0.2237815111875534 (0.1989000372845551) + | > loss_disc_real_2: 0.24169817566871643 (0.20922775823494483) + | > loss_disc_real_3: 0.2650678753852844 (0.20365984378189878) + | > loss_disc_real_4: 0.2344280481338501 (0.20589134354015876) + | > loss_disc_real_5: 0.2594045102596283 (0.22057814690573463) + | > loss_0: 2.289858341217041 (2.2889770885993697) + | > grad_norm_0: tensor(16.2512, device='cuda:0') (tensor(29.3662, device='cuda:0')) + | > loss_gen: 2.8400981426239014 (2.6718343044149466) + | > loss_kl: 1.878376841545105 (1.7321477224086892) + | > loss_feat: 9.185186386108398 (8.604886729141763) + | > loss_mel: 17.927539825439453 (16.748164571564768) + | > loss_duration: 0.9554309248924255 (0.9039029088513605) + | > loss_1: 32.7866325378418 (30.66093609250825) + | > grad_norm_1: tensor(296.4322, device='cuda:0') (tensor(558.6716, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 1.0106 (0.9695363291378679) + | > loader_time: 0.0061 (0.006041748770352067) + + + --> TIME: 2024-07-14 01:47:37 -- STEP: 79/151 -- GLOBAL_STEP: 110750 + | > loss_disc: 2.1610782146453857 (2.2979902345922936) + | > loss_disc_real_0: 0.07017121464014053 (0.12338441346265089) + | > loss_disc_real_1: 0.17877256870269775 (0.19985457113649274) + | > loss_disc_real_2: 0.1964995563030243 (0.2086530976657626) + | > loss_disc_real_3: 0.1781734824180603 (0.20696084625736066) + | > loss_disc_real_4: 0.14778481423854828 (0.2072247276200524) + | > loss_disc_real_5: 0.24513618648052216 (0.21915179132660734) + | > loss_0: 2.1610782146453857 (2.2979902345922936) + | > grad_norm_0: tensor(11.6758, device='cuda:0') (tensor(26.7058, device='cuda:0')) + | > loss_gen: 2.612276077270508 (2.635979299303852) + | > loss_kl: 1.2836015224456787 (1.7469232474701315) + | > loss_feat: 8.393489837646484 (8.386590028110938) + | > loss_mel: 17.06618881225586 (16.664441627792158) + | > loss_duration: 1.0441641807556152 (0.9262278985373581) + | > loss_1: 30.399721145629883 (30.360162107250357) + | > grad_norm_1: tensor(87.9109, device='cuda:0') (tensor(467.6490, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.822 (0.9950791944431353) + | > loader_time: 0.0041 (0.006485667409776132) + + + --> TIME: 2024-07-14 01:48:26 -- STEP: 129/151 -- GLOBAL_STEP: 110800 + | > loss_disc: 2.3717005252838135 (2.2860322377478437) + | > loss_disc_real_0: 0.11862488090991974 (0.11837458278435141) + | > loss_disc_real_1: 0.27153488993644714 (0.19999278024878614) + | > loss_disc_real_2: 0.2612268626689911 (0.20874578881171324) + | > loss_disc_real_3: 0.21038880944252014 (0.20509579223255778) + | > loss_disc_real_4: 0.20798727869987488 (0.20462948291800742) + | > loss_disc_real_5: 0.23012641072273254 (0.21947068264780117) + | > loss_0: 2.3717005252838135 (2.2860322377478437) + | > grad_norm_0: tensor(21.0329, device='cuda:0') (tensor(25.2520, device='cuda:0')) + | > loss_gen: 2.526977777481079 (2.6499894312185837) + | > loss_kl: 1.8177838325500488 (1.7409350553224252) + | > loss_feat: 7.639811038970947 (8.521844642106874) + | > loss_mel: 16.396413803100586 (16.774718587712716) + | > loss_duration: 0.9800896048545837 (0.9255583332490551) + | > loss_1: 29.361074447631836 (30.613045951192692) + | > grad_norm_1: tensor(259.5406, device='cuda:0') (tensor(456.1658, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 1.1133 (0.9873638726020044) + | > loader_time: 0.0074 (0.006419453510018282) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03388854173513559 (+0.000821150266207181) + | > avg_loss_disc: 2.5627205555255594 (-0.05781641373267554) + | > avg_loss_disc_real_0: 0.26292579678388744 (-0.010444320165193977) + | > avg_loss_disc_real_1: 0.1904168174817012 (-0.04079893689889175) + | > avg_loss_disc_real_2: 0.22558801105389228 (-0.0016408700209397387) + | > avg_loss_disc_real_3: 0.26027051645975846 (+0.03332480329733628) + | > avg_loss_disc_real_4: 0.20082556857512548 (-0.055674486435376674) + | > avg_loss_disc_real_5: 0.24145829792206103 (+0.011220518213052005) + | > avg_loss_0: 2.5627205555255594 (-0.05781641373267554) + | > avg_loss_gen: 2.4624038476210375 (+0.035276541343101986) + | > avg_loss_kl: 3.888634388263409 (+0.22411412459153413) + | > avg_loss_feat: 8.14734227840717 (+0.17169237136840731) + | > avg_loss_mel: 16.82403989938589 (-0.3062003209040718) + | > avg_loss_duration: 1.3538301357856164 (+0.030197794620807406) + | > avg_loss_1: 32.676250604482796 (+0.1550805018498309) + + + > EPOCH: 171/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:48:56)  + + --> TIME: 2024-07-14 01:49:25 -- STEP: 28/151 -- GLOBAL_STEP: 110850 + | > loss_disc: 2.428999900817871 (2.300355932542256) + | > loss_disc_real_0: 0.2023724764585495 (0.11566188146493264) + | > loss_disc_real_1: 0.2263643443584442 (0.20085925236344337) + | > loss_disc_real_2: 0.24000181257724762 (0.20922457320349558) + | > loss_disc_real_3: 0.19889570772647858 (0.2082227644111429) + | > loss_disc_real_4: 0.18838194012641907 (0.2079574710556439) + | > loss_disc_real_5: 0.1895124614238739 (0.22021347603627614) + | > loss_0: 2.428999900817871 (2.300355932542256) + | > grad_norm_0: tensor(23.3356, device='cuda:0') (tensor(24.6157, device='cuda:0')) + | > loss_gen: 2.517421245574951 (2.6467399852616444) + | > loss_kl: 1.938670039176941 (1.7229780554771423) + | > loss_feat: 7.658501625061035 (8.521470614842006) + | > loss_mel: 17.066118240356445 (16.606711523873464) + | > loss_duration: 1.0812921524047852 (0.9252057884420667) + | > loss_1: 30.262004852294922 (30.423105784824916) + | > grad_norm_1: tensor(491.1585, device='cuda:0') (tensor(433.7529, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 1.2224 (0.9611508590834481) + | > loader_time: 0.0079 (0.00593942403793335) + + + --> TIME: 2024-07-14 01:50:14 -- STEP: 78/151 -- GLOBAL_STEP: 110900 + | > loss_disc: 2.0888209342956543 (2.301774460535783) + | > loss_disc_real_0: 0.04474572464823723 (0.11586072448736583) + | > loss_disc_real_1: 0.17209668457508087 (0.20375071370448822) + | > loss_disc_real_2: 0.13094624876976013 (0.20943115193110245) + | > loss_disc_real_3: 0.17711099982261658 (0.20934455956404024) + | > loss_disc_real_4: 0.18521231412887573 (0.20678162307311326) + | > loss_disc_real_5: 0.18445993959903717 (0.2180760825673739) + | > loss_0: 2.0888209342956543 (2.301774460535783) + | > grad_norm_0: tensor(31.6119, device='cuda:0') (tensor(22.8724, device='cuda:0')) + | > loss_gen: 2.7058286666870117 (2.6277311123334446) + | > loss_kl: 1.1165754795074463 (1.691290020942688) + | > loss_feat: 10.311991691589355 (8.510659278967442) + | > loss_mel: 17.934955596923828 (16.68963449429243) + | > loss_duration: 1.1725969314575195 (0.9255753327638675) + | > loss_1: 33.241947174072266 (30.444890168996956) + | > grad_norm_1: tensor(386.1929, device='cuda:0') (tensor(361.2374, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.8149 (0.9686640195357494) + | > loader_time: 0.0044 (0.006253914955334786) + + + --> TIME: 2024-07-14 01:51:03 -- STEP: 128/151 -- GLOBAL_STEP: 110950 + | > loss_disc: 2.155663013458252 (2.287751770578324) + | > loss_disc_real_0: 0.07192005217075348 (0.11197758678463288) + | > loss_disc_real_1: 0.2207554578781128 (0.20346614124719054) + | > loss_disc_real_2: 0.2053263783454895 (0.2084865003125742) + | > loss_disc_real_3: 0.20204980671405792 (0.2074363853316754) + | > loss_disc_real_4: 0.18529509007930756 (0.20518720126710832) + | > loss_disc_real_5: 0.23924081027507782 (0.21840884815901518) + | > loss_0: 2.155663013458252 (2.287751770578324) + | > grad_norm_0: tensor(28.6297, device='cuda:0') (tensor(24.2356, device='cuda:0')) + | > loss_gen: 2.852189540863037 (2.636786945164203) + | > loss_kl: 1.2584649324417114 (1.6835437063127756) + | > loss_feat: 9.327027320861816 (8.458937492221596) + | > loss_mel: 15.874580383300781 (16.68833824247121) + | > loss_duration: 1.1058428287506104 (0.9241978535428643) + | > loss_1: 30.41810417175293 (30.391804173588753) + | > grad_norm_1: tensor(644.8761, device='cuda:0') (tensor(411.3865, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.8137 (0.9644563980400562) + | > loader_time: 0.0043 (0.006151529029011726) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032722106346717246 (-0.0011664353884183465) + | > avg_loss_disc: 2.6816597168262186 (+0.11893916130065918) + | > avg_loss_disc_real_0: 0.2936231940984726 (+0.03069739731458515) + | > avg_loss_disc_real_1: 0.20212340354919434 (+0.011706586067493141) + | > avg_loss_disc_real_2: 0.2222932634445337 (-0.003294747609358578) + | > avg_loss_disc_real_3: 0.27381068353469556 (+0.013540167074937104) + | > avg_loss_disc_real_4: 0.2922269633183113 (+0.09140139474318579) + | > avg_loss_disc_real_5: 0.2515888259961055 (+0.010130528074044465) + | > avg_loss_0: 2.6816597168262186 (+0.11893916130065918) + | > avg_loss_gen: 2.4582634338965788 (-0.004140413724458725) + | > avg_loss_kl: 3.78363508444566 (-0.10499930381774902) + | > avg_loss_feat: 7.630548917330229 (-0.5167933610769406) + | > avg_loss_mel: 16.901752031766453 (+0.07771213238056163) + | > avg_loss_duration: 1.32091918358436 (-0.032910952201256505) + | > avg_loss_1: 32.09511844928448 (-0.5811321551983184) + + + > EPOCH: 172/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:51:37)  + + --> TIME: 2024-07-14 01:52:05 -- STEP: 27/151 -- GLOBAL_STEP: 111000 + | > loss_disc: 2.5455360412597656 (2.3161481398123285) + | > loss_disc_real_0: 0.14343762397766113 (0.11763370106065715) + | > loss_disc_real_1: 0.25238120555877686 (0.20090864764319527) + | > loss_disc_real_2: 0.2685239613056183 (0.21202361418141258) + | > loss_disc_real_3: 0.2659103274345398 (0.20281686219904158) + | > loss_disc_real_4: 0.32313668727874756 (0.21197306447558933) + | > loss_disc_real_5: 0.2739749550819397 (0.22361609118956108) + | > loss_0: 2.5455360412597656 (2.3161481398123285) + | > grad_norm_0: tensor(45.8444, device='cuda:0') (tensor(31.5190, device='cuda:0')) + | > loss_gen: 2.5408451557159424 (2.6184652381473117) + | > loss_kl: 1.9085396528244019 (1.6661050496277985) + | > loss_feat: 7.328134536743164 (8.196457544962565) + | > loss_mel: 15.516132354736328 (16.595718631037958) + | > loss_duration: 0.8617637753486633 (0.9008291937686779) + | > loss_1: 28.15541648864746 (29.97757551405165) + | > grad_norm_1: tensor(748.0247, device='cuda:0') (tensor(488.4871, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.9764 (0.9426870434372513) + | > loader_time: 0.0056 (0.0060322196395308884) + + + --> TIME: 2024-07-14 01:53:22 -- STEP: 77/151 -- GLOBAL_STEP: 111050 + | > loss_disc: 2.605952739715576 (2.307237990490802) + | > loss_disc_real_0: 0.2557219862937927 (0.12360917646195982) + | > loss_disc_real_1: 0.2512691617012024 (0.19981112979449234) + | > loss_disc_real_2: 0.23514951765537262 (0.20971172409398214) + | > loss_disc_real_3: 0.21012723445892334 (0.20298824914090044) + | > loss_disc_real_4: 0.23814302682876587 (0.2068141441453587) + | > loss_disc_real_5: 0.26019155979156494 (0.22188990379308726) + | > loss_0: 2.605952739715576 (2.307237990490802) + | > grad_norm_0: tensor(60.2823, device='cuda:0') (tensor(31.9486, device='cuda:0')) + | > loss_gen: 2.4642250537872314 (2.6464221880033416) + | > loss_kl: 1.7330431938171387 (1.7136980767373915) + | > loss_feat: 7.645983695983887 (8.267994298563377) + | > loss_mel: 17.733915328979492 (16.686908375133175) + | > loss_duration: 0.8894047737121582 (0.9106899925640651) + | > loss_1: 30.466571807861328 (30.22571291242327) + | > grad_norm_1: tensor(328.3178, device='cuda:0') (tensor(497.5135, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.9989 (0.9809693330294126) + | > loader_time: 0.0071 (0.006204156132487507) + + + --> TIME: 2024-07-14 01:54:11 -- STEP: 127/151 -- GLOBAL_STEP: 111100 + | > loss_disc: 2.120164394378662 (2.295368705208847) + | > loss_disc_real_0: 0.11843623965978622 (0.12447567932366384) + | > loss_disc_real_1: 0.15484723448753357 (0.19949003485009426) + | > loss_disc_real_2: 0.17574112117290497 (0.2092633953713995) + | > loss_disc_real_3: 0.18468725681304932 (0.20283422501772408) + | > loss_disc_real_4: 0.1700730174779892 (0.20474798684045087) + | > loss_disc_real_5: 0.1728648990392685 (0.2207355871209948) + | > loss_0: 2.120164394378662 (2.295368705208847) + | > grad_norm_0: tensor(26.6991, device='cuda:0') (tensor(33.4077, device='cuda:0')) + | > loss_gen: 2.539370536804199 (2.654103378611287) + | > loss_kl: 1.6086924076080322 (1.7005053772701053) + | > loss_feat: 8.070722579956055 (8.394660998517137) + | > loss_mel: 16.82280921936035 (16.766667929221327) + | > loss_duration: 0.8779048323631287 (0.9183484761733708) + | > loss_1: 29.91950035095215 (30.434286147590697) + | > grad_norm_1: tensor(548.0800, device='cuda:0') (tensor(492.1795, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.8973 (0.9773295207286444) + | > loader_time: 0.005 (0.006231229136309285) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03337425452012282 (+0.0006521481734055729) + | > avg_loss_disc: 2.664938266460712 (-0.01672145036550665) + | > avg_loss_disc_real_0: 0.350753149160972 (+0.057129955062499405) + | > avg_loss_disc_real_1: 0.22983551483887893 (+0.027712111289684593) + | > avg_loss_disc_real_2: 0.24415004138763136 (+0.02185677794309765) + | > avg_loss_disc_real_3: 0.27852464066101956 (+0.004713957126323998) + | > avg_loss_disc_real_4: 0.3190444295222943 (+0.02681746620398301) + | > avg_loss_disc_real_5: 0.27998263904681575 (+0.02839381305071026) + | > avg_loss_0: 2.664938266460712 (-0.01672145036550665) + | > avg_loss_gen: 2.930254312661978 (+0.471990878765399) + | > avg_loss_kl: 3.801377241428082 (+0.017742156982421875) + | > avg_loss_feat: 8.016514888176552 (+0.385965970846323) + | > avg_loss_mel: 16.803040724534256 (-0.09871130723219679) + | > avg_loss_duration: 1.3668784911815937 (+0.04595930759723377) + | > avg_loss_1: 32.918065731342026 (+0.8229472820575481) + + + > EPOCH: 173/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:54:44)  + + --> TIME: 2024-07-14 01:55:13 -- STEP: 26/151 -- GLOBAL_STEP: 111150 + | > loss_disc: 2.101936101913452 (2.3102012414198656) + | > loss_disc_real_0: 0.06297027319669724 (0.11611963478991619) + | > loss_disc_real_1: 0.18823711574077606 (0.19778821989893913) + | > loss_disc_real_2: 0.21092842519283295 (0.20745962800887915) + | > loss_disc_real_3: 0.2102670818567276 (0.19999749958515167) + | > loss_disc_real_4: 0.22402702271938324 (0.2074872811253254) + | > loss_disc_real_5: 0.21102465689182281 (0.2209317500774677) + | > loss_0: 2.101936101913452 (2.3102012414198656) + | > grad_norm_0: tensor(11.6220, device='cuda:0') (tensor(18.1346, device='cuda:0')) + | > loss_gen: 2.9593920707702637 (2.5941658203418436) + | > loss_kl: 1.487533688545227 (1.7714679539203644) + | > loss_feat: 9.17542839050293 (8.678737860459547) + | > loss_mel: 16.774932861328125 (16.589635959038368) + | > loss_duration: 0.9308381080627441 (0.9408754935631385) + | > loss_1: 31.328125 (30.574883167560284) + | > grad_norm_1: tensor(291.8340, device='cuda:0') (tensor(246.1700, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.8649 (0.9898210855630728) + | > loader_time: 0.0045 (0.0061803505970881535) + + + --> TIME: 2024-07-14 01:56:03 -- STEP: 76/151 -- GLOBAL_STEP: 111200 + | > loss_disc: 2.031266689300537 (2.286071402461905) + | > loss_disc_real_0: 0.0664571225643158 (0.11510005166852161) + | > loss_disc_real_1: 0.14236757159233093 (0.20018748909627135) + | > loss_disc_real_2: 0.15345412492752075 (0.2081434542411252) + | > loss_disc_real_3: 0.12772037088871002 (0.20212322855858425) + | > loss_disc_real_4: 0.1390775740146637 (0.2056986562123424) + | > loss_disc_real_5: 0.2320965975522995 (0.21932514011859894) + | > loss_0: 2.031266689300537 (2.286071402461905) + | > grad_norm_0: tensor(7.9930, device='cuda:0') (tensor(20.0397, device='cuda:0')) + | > loss_gen: 2.7922706604003906 (2.622481230058168) + | > loss_kl: 1.6552460193634033 (1.7952422468285811) + | > loss_feat: 10.2003755569458 (8.531592707884938) + | > loss_mel: 17.730281829833984 (16.615317946986146) + | > loss_duration: 0.9810385704040527 (0.9358596033171603) + | > loss_1: 33.35921096801758 (30.500493902909128) + | > grad_norm_1: tensor(256.1852, device='cuda:0') (tensor(335.3475, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.8499 (0.9944009561287729) + | > loader_time: 0.0059 (0.006381693639253316) + + + --> TIME: 2024-07-14 01:56:51 -- STEP: 126/151 -- GLOBAL_STEP: 111250 + | > loss_disc: 2.4258921146392822 (2.2703373214555165) + | > loss_disc_real_0: 0.16154569387435913 (0.1123709725659518) + | > loss_disc_real_1: 0.22078196704387665 (0.19999520250019573) + | > loss_disc_real_2: 0.19307662546634674 (0.20669732112733144) + | > loss_disc_real_3: 0.18167397379875183 (0.20142310656725415) + | > loss_disc_real_4: 0.2074194997549057 (0.20514455626881312) + | > loss_disc_real_5: 0.20901009440422058 (0.21799927263978927) + | > loss_0: 2.4258921146392822 (2.2703373214555165) + | > grad_norm_0: tensor(16.9108, device='cuda:0') (tensor(22.5421, device='cuda:0')) + | > loss_gen: 2.4894821643829346 (2.64043484793769) + | > loss_kl: 2.0370399951934814 (1.766588016161843) + | > loss_feat: 8.235078811645508 (8.56556137402852) + | > loss_mel: 16.971506118774414 (16.662104213048533) + | > loss_duration: 0.8612295389175415 (0.9230069976950449) + | > loss_1: 30.594337463378906 (30.557695479620072) + | > grad_norm_1: tensor(236.4822, device='cuda:0') (tensor(388.4912, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.9011 (0.9732015530268351) + | > loader_time: 0.0052 (0.006201547289651538) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03325885992783766 (-0.00011539459228515625) + | > avg_loss_disc: 2.595441488119272 (-0.06949677834143975) + | > avg_loss_disc_real_0: 0.29046937135549694 (-0.06028377780547506) + | > avg_loss_disc_real_1: 0.2217092330639179 (-0.008126281774961025) + | > avg_loss_disc_real_2: 0.24217563294447386 (-0.0019744084431574938) + | > avg_loss_disc_real_3: 0.21156779046242052 (-0.06695685019859904) + | > avg_loss_disc_real_4: 0.20094490739015433 (-0.11809952213213995) + | > avg_loss_disc_real_5: 0.2232000197355564 (-0.05678261931125936) + | > avg_loss_0: 2.595441488119272 (-0.06949677834143975) + | > avg_loss_gen: 2.4088797752673807 (-0.521374537394597) + | > avg_loss_kl: 3.9455131934239316 (+0.1441359519958496) + | > avg_loss_feat: 7.835010198446421 (-0.18150468973013112) + | > avg_loss_mel: 16.800915864797737 (-0.0021248597365186583) + | > avg_loss_duration: 1.351643287218534 (-0.015235203963059663) + | > avg_loss_1: 32.34196208073543 (-0.5761036506065977) + + + > EPOCH: 174/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 01:57:25)  + + --> TIME: 2024-07-14 01:57:50 -- STEP: 25/151 -- GLOBAL_STEP: 111300 + | > loss_disc: 2.369572639465332 (2.245951862335205) + | > loss_disc_real_0: 0.1266673505306244 (0.11940197601914405) + | > loss_disc_real_1: 0.2332073599100113 (0.19479702174663543) + | > loss_disc_real_2: 0.23449580371379852 (0.20787497341632843) + | > loss_disc_real_3: 0.2448524683713913 (0.2041548955440521) + | > loss_disc_real_4: 0.25125423073768616 (0.20462255239486693) + | > loss_disc_real_5: 0.23720762133598328 (0.22026686012744903) + | > loss_0: 2.369572639465332 (2.245951862335205) + | > grad_norm_0: tensor(24.5887, device='cuda:0') (tensor(35.0795, device='cuda:0')) + | > loss_gen: 2.5195558071136475 (2.7407291507720943) + | > loss_kl: 1.952889084815979 (1.607507872581482) + | > loss_feat: 7.496430397033691 (8.533165397644042) + | > loss_mel: 17.008316040039062 (16.687452201843268) + | > loss_duration: 0.9683209657669067 (0.9300724053382874) + | > loss_1: 29.945512771606445 (30.49892707824707) + | > grad_norm_1: tensor(755.6268, device='cuda:0') (tensor(558.4453, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 1.1691 (0.9097849559783936) + | > loader_time: 0.008 (0.005370130538940429) + + + --> TIME: 2024-07-14 01:58:44 -- STEP: 75/151 -- GLOBAL_STEP: 111350 + | > loss_disc: 2.422034740447998 (2.2792451651891072) + | > loss_disc_real_0: 0.17512741684913635 (0.11843181505799294) + | > loss_disc_real_1: 0.18029767274856567 (0.1954263335466385) + | > loss_disc_real_2: 0.17270508408546448 (0.2075759603579839) + | > loss_disc_real_3: 0.1862085908651352 (0.20551859835783637) + | > loss_disc_real_4: 0.20896171033382416 (0.20467180321613948) + | > loss_disc_real_5: 0.2655029594898224 (0.22018743177254996) + | > loss_0: 2.422034740447998 (2.2792451651891072) + | > grad_norm_0: tensor(43.0430, device='cuda:0') (tensor(30.5665, device='cuda:0')) + | > loss_gen: 2.4354100227355957 (2.6744163449605303) + | > loss_kl: 1.8524280786514282 (1.7166732041041057) + | > loss_feat: 7.9350786209106445 (8.50285624186198) + | > loss_mel: 17.147680282592773 (16.623233133951825) + | > loss_duration: 0.8636202812194824 (0.9347711126009624) + | > loss_1: 30.23421859741211 (30.451950098673503) + | > grad_norm_1: tensor(681.4584, device='cuda:0') (tensor(526.9492, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 1.0653 (1.012369845708211) + | > loader_time: 0.0081 (0.00645486831665039) + + + --> TIME: 2024-07-14 01:59:32 -- STEP: 125/151 -- GLOBAL_STEP: 111400 + | > loss_disc: 2.1053216457366943 (2.297818529129029) + | > loss_disc_real_0: 0.05247434228658676 (0.11956185686588287) + | > loss_disc_real_1: 0.1454750895500183 (0.1988618161678314) + | > loss_disc_real_2: 0.1626787632703781 (0.20843474233150483) + | > loss_disc_real_3: 0.1401766836643219 (0.2054756610393524) + | > loss_disc_real_4: 0.16077964007854462 (0.20626821023225783) + | > loss_disc_real_5: 0.18509413301944733 (0.222202908039093) + | > loss_0: 2.1053216457366943 (2.297818529129029) + | > grad_norm_0: tensor(26.2182, device='cuda:0') (tensor(31.1876, device='cuda:0')) + | > loss_gen: 2.5097460746765137 (2.6480472393035894) + | > loss_kl: 1.309751272201538 (1.7279884338378906) + | > loss_feat: 9.264126777648926 (8.401553283691406) + | > loss_mel: 15.924028396606445 (16.68920346832275) + | > loss_duration: 0.9718471765518188 (0.9264616875648498) + | > loss_1: 29.97949981689453 (30.39325405883789) + | > grad_norm_1: tensor(296.4456, device='cuda:0') (tensor(525.5433, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.8232 (0.9897737083435056) + | > loader_time: 0.0054 (0.006402183532714844) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032593176915095404 (-0.0006656830127422589) + | > avg_loss_disc: 2.6770393665020285 (+0.08159787838275623) + | > avg_loss_disc_real_0: 0.29387147380755496 (+0.00340210245205802) + | > avg_loss_disc_real_1: 0.2513726961154204 (+0.029663463051502498) + | > avg_loss_disc_real_2: 0.26030570841752565 (+0.018130075473051788) + | > avg_loss_disc_real_3: 0.27362533945303696 (+0.06205754899061644) + | > avg_loss_disc_real_4: 0.29735444486141205 (+0.09640953747125772) + | > avg_loss_disc_real_5: 0.2287107087098635 (+0.005510688974307121) + | > avg_loss_0: 2.6770393665020285 (+0.08159787838275623) + | > avg_loss_gen: 2.601496568092933 (+0.19261679282555244) + | > avg_loss_kl: 3.7415542602539062 (-0.20395893317002534) + | > avg_loss_feat: 7.861138453850379 (+0.026128255403958356) + | > avg_loss_mel: 16.87117151113657 (+0.07025564633883263) + | > avg_loss_duration: 1.3704338257129376 (+0.018790538494403597) + | > avg_loss_1: 32.4457944723276 (+0.10383239159217084) + + + > EPOCH: 175/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:00:09)  + + --> TIME: 2024-07-14 02:00:37 -- STEP: 24/151 -- GLOBAL_STEP: 111450 + | > loss_disc: 2.4360451698303223 (2.3451242248217263) + | > loss_disc_real_0: 0.07928632944822311 (0.12118044247229894) + | > loss_disc_real_1: 0.24297696352005005 (0.2104043004413446) + | > loss_disc_real_2: 0.2569099962711334 (0.21525228582322598) + | > loss_disc_real_3: 0.2555193305015564 (0.21621035349865755) + | > loss_disc_real_4: 0.2287595272064209 (0.21094098687171936) + | > loss_disc_real_5: 0.2518322169780731 (0.22187135368585587) + | > loss_0: 2.4360451698303223 (2.3451242248217263) + | > grad_norm_0: tensor(20.3484, device='cuda:0') (tensor(25.3286, device='cuda:0')) + | > loss_gen: 2.460203170776367 (2.644049853086472) + | > loss_kl: 1.839612364768982 (1.8451604942480724) + | > loss_feat: 7.7741007804870605 (8.388035953044893) + | > loss_mel: 17.498584747314453 (17.049865444501243) + | > loss_duration: 0.9433188438415527 (0.9280832310517629) + | > loss_1: 30.515819549560547 (30.855194807052612) + | > grad_norm_1: tensor(408.4315, device='cuda:0') (tensor(458.6982, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.9514 (1.0758197804292045) + | > loader_time: 0.0071 (0.00679585337638855) + + + --> TIME: 2024-07-14 02:01:26 -- STEP: 74/151 -- GLOBAL_STEP: 111500 + | > loss_disc: 2.352865219116211 (2.3314710662171647) + | > loss_disc_real_0: 0.1282099336385727 (0.11678163032676722) + | > loss_disc_real_1: 0.20201484858989716 (0.20785397371730288) + | > loss_disc_real_2: 0.2328564077615738 (0.2120885464387971) + | > loss_disc_real_3: 0.20588602125644684 (0.20936184477161718) + | > loss_disc_real_4: 0.20048396289348602 (0.21094490124567136) + | > loss_disc_real_5: 0.19086754322052002 (0.22237211002691373) + | > loss_0: 2.352865219116211 (2.3314710662171647) + | > grad_norm_0: tensor(34.5310, device='cuda:0') (tensor(24.4476, device='cuda:0')) + | > loss_gen: 2.4794530868530273 (2.5974020313572246) + | > loss_kl: 1.4980082511901855 (1.7687200227299251) + | > loss_feat: 7.971347808837891 (8.203164435721732) + | > loss_mel: 15.962306022644043 (16.652209977845885) + | > loss_duration: 0.8429089784622192 (0.9292305748204928) + | > loss_1: 28.754026412963867 (30.150726833859004) + | > grad_norm_1: tensor(238.0264, device='cuda:0') (tensor(412.0846, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.8828 (1.0086282137277969) + | > loader_time: 0.0064 (0.006572842597961428) + + + --> TIME: 2024-07-14 02:02:15 -- STEP: 124/151 -- GLOBAL_STEP: 111550 + | > loss_disc: 2.411566734313965 (2.2999788272765374) + | > loss_disc_real_0: 0.16570524871349335 (0.1141344886873999) + | > loss_disc_real_1: 0.2157496064901352 (0.20340235639483698) + | > loss_disc_real_2: 0.2233261615037918 (0.20924860839882203) + | > loss_disc_real_3: 0.18799583613872528 (0.20587342340619333) + | > loss_disc_real_4: 0.2565092444419861 (0.2089974746829079) + | > loss_disc_real_5: 0.24812939763069153 (0.2187866613028511) + | > loss_0: 2.411566734313965 (2.2999788272765374) + | > grad_norm_0: tensor(59.8870, device='cuda:0') (tensor(25.6565, device='cuda:0')) + | > loss_gen: 2.5275609493255615 (2.6302182539816834) + | > loss_kl: 1.7988147735595703 (1.7595836750922664) + | > loss_feat: 6.1778717041015625 (8.360603417119668) + | > loss_mel: 15.080520629882812 (16.692987818871774) + | > loss_duration: 0.9804161787033081 (0.9234843566533057) + | > loss_1: 26.565183639526367 (30.36687744817426) + | > grad_norm_1: tensor(466.1166, device='cuda:0') (tensor(438.1672, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.9958 (0.9874419954515277) + | > loader_time: 0.0064 (0.006314218044281008) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03458633789649376 (+0.001993160981398359) + | > avg_loss_disc: 2.5924076483799863 (-0.08463171812204218) + | > avg_loss_disc_real_0: 0.34743540103618914 (+0.05356392722863418) + | > avg_loss_disc_real_1: 0.19742836172764117 (-0.053944334387779236) + | > avg_loss_disc_real_2: 0.24135625477020556 (-0.01894945364732009) + | > avg_loss_disc_real_3: 0.21677191670124346 (-0.0568534227517935) + | > avg_loss_disc_real_4: 0.24055978885063758 (-0.056794656010774464) + | > avg_loss_disc_real_5: 0.249524508531277 (+0.020813799821413487) + | > avg_loss_0: 2.5924076483799863 (-0.08463171812204218) + | > avg_loss_gen: 2.6522978269136868 (+0.05080125882075359) + | > avg_loss_kl: 3.737371463042039 (-0.004182797211867317) + | > avg_loss_feat: 7.920171847710242 (+0.05903339385986328) + | > avg_loss_mel: 17.089469616229717 (+0.2182981050931474) + | > avg_loss_duration: 1.3742097799594586 (+0.003775954246520996) + | > avg_loss_1: 32.77351995614859 (+0.3277254838209913) + + + > EPOCH: 176/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:02:51)  + + --> TIME: 2024-07-14 02:03:16 -- STEP: 23/151 -- GLOBAL_STEP: 111600 + | > loss_disc: 2.2513067722320557 (2.2943492868672246) + | > loss_disc_real_0: 0.1285613477230072 (0.11266098074291063) + | > loss_disc_real_1: 0.28437310457229614 (0.20340782533521237) + | > loss_disc_real_2: 0.2300691306591034 (0.21381691422151483) + | > loss_disc_real_3: 0.23192143440246582 (0.2112315446138382) + | > loss_disc_real_4: 0.2301107794046402 (0.20989059624464615) + | > loss_disc_real_5: 0.19764722883701324 (0.21546610736328622) + | > loss_0: 2.2513067722320557 (2.2943492868672246) + | > grad_norm_0: tensor(35.0131, device='cuda:0') (tensor(34.1379, device='cuda:0')) + | > loss_gen: 2.7223966121673584 (2.7113952325737993) + | > loss_kl: 1.3115630149841309 (1.7518566328546274) + | > loss_feat: 8.327714920043945 (8.558200317880381) + | > loss_mel: 17.20354652404785 (16.635964849720832) + | > loss_duration: 0.7999489307403564 (0.935294931349547) + | > loss_1: 30.365171432495117 (30.592712236487348) + | > grad_norm_1: tensor(596.2717, device='cuda:0') (tensor(589.4625, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.9071 (1.0006632701210354) + | > loader_time: 0.0048 (0.006562077480813731) + + + --> TIME: 2024-07-14 02:04:06 -- STEP: 73/151 -- GLOBAL_STEP: 111650 + | > loss_disc: 2.2236099243164062 (2.303906927370045) + | > loss_disc_real_0: 0.07974898815155029 (0.11824788260337425) + | > loss_disc_real_1: 0.17982615530490875 (0.19950079958732814) + | > loss_disc_real_2: 0.1404091715812683 (0.20862831093677103) + | > loss_disc_real_3: 0.1707305759191513 (0.20861209663626262) + | > loss_disc_real_4: 0.2091400921344757 (0.21067041913940482) + | > loss_disc_real_5: 0.3051741123199463 (0.22192922704023857) + | > loss_0: 2.2236099243164062 (2.303906927370045) + | > grad_norm_0: tensor(7.7213, device='cuda:0') (tensor(34.2820, device='cuda:0')) + | > loss_gen: 2.7629950046539307 (2.6485908782645446) + | > loss_kl: 0.975540280342102 (1.7248376689545095) + | > loss_feat: 9.130411148071289 (8.277291219528408) + | > loss_mel: 17.593753814697266 (16.769925875206514) + | > loss_duration: 0.9976749420166016 (0.9192927055162926) + | > loss_1: 31.46037483215332 (30.339938464230055) + | > grad_norm_1: tensor(488.7671, device='cuda:0') (tensor(531.1846, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.8149 (0.9953487997185694) + | > loader_time: 0.0043 (0.0064184502379535) + + + --> TIME: 2024-07-14 02:04:55 -- STEP: 123/151 -- GLOBAL_STEP: 111700 + | > loss_disc: 2.368640422821045 (2.3040993863004977) + | > loss_disc_real_0: 0.15302008390426636 (0.11811145870545046) + | > loss_disc_real_1: 0.22746232151985168 (0.20363382195554128) + | > loss_disc_real_2: 0.23884589970111847 (0.209271180435894) + | > loss_disc_real_3: 0.23516611754894257 (0.2080598019245194) + | > loss_disc_real_4: 0.21282300353050232 (0.2091964370593792) + | > loss_disc_real_5: 0.2217707633972168 (0.22095147405213456) + | > loss_0: 2.368640422821045 (2.3040993863004977) + | > grad_norm_0: tensor(13.0229, device='cuda:0') (tensor(32.8201, device='cuda:0')) + | > loss_gen: 2.4198033809661865 (2.6512117599084126) + | > loss_kl: 1.7468407154083252 (1.7132311705651322) + | > loss_feat: 9.448144912719727 (8.320073972872601) + | > loss_mel: 15.370437622070312 (16.687401585462617) + | > loss_duration: 0.9134236574172974 (0.9206900916448454) + | > loss_1: 29.898649215698242 (30.292608586753285) + | > grad_norm_1: tensor(373.1614, device='cuda:0') (tensor(515.1585, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.8664 (0.9825040092313193) + | > loader_time: 0.0058 (0.006346260629049162) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03456917175879845 (-1.71661376953125e-05) + | > avg_loss_disc: 2.5727510085472693 (-0.019656639832716927) + | > avg_loss_disc_real_0: 0.2647109352625333 (-0.08272446577365583) + | > avg_loss_disc_real_1: 0.1882503663118069 (-0.009177995415834278) + | > avg_loss_disc_real_2: 0.24133548255150133 (-2.0772218704223633e-05) + | > avg_loss_disc_real_3: 0.19787370241605318 (-0.018898214285190285) + | > avg_loss_disc_real_4: 0.2872023788782267 (+0.04664259002758911) + | > avg_loss_disc_real_5: 0.2183706141435183 (-0.0311538943877587) + | > avg_loss_0: 2.5727510085472693 (-0.019656639832716927) + | > avg_loss_gen: 2.3832009205451383 (-0.26909690636854844) + | > avg_loss_kl: 3.8553273494427023 (+0.11795588640066335) + | > avg_loss_feat: 7.751796428973858 (-0.1683754187363844) + | > avg_loss_mel: 17.018572880671577 (-0.07089673555814002) + | > avg_loss_duration: 1.343325899197505 (-0.030883880761953586) + | > avg_loss_1: 32.35222405653734 (-0.4212958996112519) + + + > EPOCH: 177/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:05:32)  + + --> TIME: 2024-07-14 02:05:56 -- STEP: 22/151 -- GLOBAL_STEP: 111750 + | > loss_disc: 2.1931450366973877 (2.293891343203458) + | > loss_disc_real_0: 0.16494861245155334 (0.11678748374635523) + | > loss_disc_real_1: 0.19402432441711426 (0.19612314145673404) + | > loss_disc_real_2: 0.19345049560070038 (0.206631869755008) + | > loss_disc_real_3: 0.18133005499839783 (0.20477359335530887) + | > loss_disc_real_4: 0.18453410267829895 (0.20488672297109256) + | > loss_disc_real_5: 0.23914803564548492 (0.22528159686110236) + | > loss_0: 2.1931450366973877 (2.293891343203458) + | > grad_norm_0: tensor(40.7199, device='cuda:0') (tensor(22.6712, device='cuda:0')) + | > loss_gen: 2.786153793334961 (2.5933754552494395) + | > loss_kl: 1.7926278114318848 (1.7477245330810547) + | > loss_feat: 8.985608100891113 (8.172884312543003) + | > loss_mel: 16.95760154724121 (16.622766971588135) + | > loss_duration: 0.8835972547531128 (0.9072154001756148) + | > loss_1: 31.405588150024414 (30.043966813520953) + | > grad_norm_1: tensor(599.8763, device='cuda:0') (tensor(382.4046, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.9366 (0.9686460169878873) + | > loader_time: 0.0055 (0.006403511220758612) + + + --> TIME: 2024-07-14 02:06:46 -- STEP: 72/151 -- GLOBAL_STEP: 111800 + | > loss_disc: 2.2223904132843018 (2.294511028461985) + | > loss_disc_real_0: 0.04521866887807846 (0.11528785381880072) + | > loss_disc_real_1: 0.19045567512512207 (0.19790097646829155) + | > loss_disc_real_2: 0.1950959414243698 (0.20840044567982355) + | > loss_disc_real_3: 0.11781638115644455 (0.20772263335271013) + | > loss_disc_real_4: 0.14435502886772156 (0.20510058891442087) + | > loss_disc_real_5: 0.1647876352071762 (0.222049569297168) + | > loss_0: 2.2223904132843018 (2.294511028461985) + | > grad_norm_0: tensor(9.3490, device='cuda:0') (tensor(26.9546, device='cuda:0')) + | > loss_gen: 2.608818292617798 (2.6301058597034883) + | > loss_kl: 1.3733339309692383 (1.7319331078065767) + | > loss_feat: 10.301154136657715 (8.259792096085013) + | > loss_mel: 16.793241500854492 (16.538856930202908) + | > loss_duration: 1.0298290252685547 (0.9274031412270334) + | > loss_1: 32.10637664794922 (30.08809102906121) + | > grad_norm_1: tensor(149.0772, device='cuda:0') (tensor(449.5023, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.8235 (0.9817081623607213) + | > loader_time: 0.0056 (0.006456779109107124) + + + --> TIME: 2024-07-14 02:07:35 -- STEP: 122/151 -- GLOBAL_STEP: 111850 + | > loss_disc: 2.2144932746887207 (2.29577602030801) + | > loss_disc_real_0: 0.086969755589962 (0.11677756142176565) + | > loss_disc_real_1: 0.1970663219690323 (0.20104867543597688) + | > loss_disc_real_2: 0.19624468684196472 (0.20836211118053216) + | > loss_disc_real_3: 0.23060019314289093 (0.20766867605633424) + | > loss_disc_real_4: 0.19297470152378082 (0.20523571357375286) + | > loss_disc_real_5: 0.2163028120994568 (0.22144570751268355) + | > loss_0: 2.2144932746887207 (2.29577602030801) + | > grad_norm_0: tensor(11.6365, device='cuda:0') (tensor(29.2877, device='cuda:0')) + | > loss_gen: 2.6621739864349365 (2.6399101214330702) + | > loss_kl: 1.9252396821975708 (1.7429597841911628) + | > loss_feat: 8.595311164855957 (8.313919798272542) + | > loss_mel: 17.806804656982422 (16.57425465349291) + | > loss_duration: 0.8788530826568604 (0.9190041916292222) + | > loss_1: 31.86838150024414 (30.19004849918553) + | > grad_norm_1: tensor(444.9666, device='cuda:0') (tensor(474.1189, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 1.456 (0.978561643694268) + | > loader_time: 0.0056 (0.006432672015956191) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03308859238257775 (-0.0014805793762207031) + | > avg_loss_disc: 2.4098109281980076 (-0.16294008034926177) + | > avg_loss_disc_real_0: 0.1462900535418437 (-0.1184208817206896) + | > avg_loss_disc_real_1: 0.2646773698238226 (+0.07642700351201573) + | > avg_loss_disc_real_2: 0.26797762513160706 (+0.02664214258010572) + | > avg_loss_disc_real_3: 0.24046917603566095 (+0.04259547361960778) + | > avg_loss_disc_real_4: 0.26814138086942524 (-0.019060998008801455) + | > avg_loss_disc_real_5: 0.1945400318274131 (-0.023830582316105187) + | > avg_loss_0: 2.4098109281980076 (-0.16294008034926177) + | > avg_loss_gen: 2.6821100528423605 (+0.29890913229722216) + | > avg_loss_kl: 3.8313410098736105 (-0.023986339569091797) + | > avg_loss_feat: 8.58376117853018 (+0.831964749556322) + | > avg_loss_mel: 17.26707179729755 (+0.248498916625973) + | > avg_loss_duration: 1.323311915764442 (-0.020013983433063043) + | > avg_loss_1: 33.68759580758902 (+1.3353717510516816) + + + > EPOCH: 178/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:08:15)  + + --> TIME: 2024-07-14 02:08:36 -- STEP: 21/151 -- GLOBAL_STEP: 111900 + | > loss_disc: 2.2059874534606934 (2.249554713567098) + | > loss_disc_real_0: 0.12541207671165466 (0.10883048034849621) + | > loss_disc_real_1: 0.1849246770143509 (0.19998479173296974) + | > loss_disc_real_2: 0.19957715272903442 (0.2091985401653108) + | > loss_disc_real_3: 0.2279355525970459 (0.20218278538613094) + | > loss_disc_real_4: 0.1757587045431137 (0.20101321524097807) + | > loss_disc_real_5: 0.17554080486297607 (0.20968846409093767) + | > loss_0: 2.2059874534606934 (2.249554713567098) + | > grad_norm_0: tensor(36.4007, device='cuda:0') (tensor(33.3956, device='cuda:0')) + | > loss_gen: 2.8364744186401367 (2.795995598747617) + | > loss_kl: 1.5683937072753906 (1.703981950169518) + | > loss_feat: 9.378276824951172 (9.000787734985352) + | > loss_mel: 17.54037094116211 (17.004732359023322) + | > loss_duration: 0.8720261454582214 (0.9235255008652097) + | > loss_1: 32.19554138183594 (31.42902319771903) + | > grad_norm_1: tensor(152.0559, device='cuda:0') (tensor(587.9187, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.9882 (0.9106997535342262) + | > loader_time: 0.0074 (0.005760374523344495) + + + --> TIME: 2024-07-14 02:09:27 -- STEP: 71/151 -- GLOBAL_STEP: 111950 + | > loss_disc: 2.3885247707366943 (2.286892786831923) + | > loss_disc_real_0: 0.0804007276892662 (0.11339505016803741) + | > loss_disc_real_1: 0.2142205834388733 (0.20063020121043837) + | > loss_disc_real_2: 0.21740403771400452 (0.20804959797943143) + | > loss_disc_real_3: 0.24636565148830414 (0.20485453066271794) + | > loss_disc_real_4: 0.2232191562652588 (0.20418127216923404) + | > loss_disc_real_5: 0.2538154721260071 (0.22175573589096606) + | > loss_0: 2.3885247707366943 (2.286892786831923) + | > grad_norm_0: tensor(15.3951, device='cuda:0') (tensor(33.6273, device='cuda:0')) + | > loss_gen: 2.60805082321167 (2.6759122761202527) + | > loss_kl: 1.4952641725540161 (1.7214611709957393) + | > loss_feat: 7.730364799499512 (8.418405935797892) + | > loss_mel: 16.771196365356445 (16.889719157151777) + | > loss_duration: 0.8670597076416016 (0.9290438949222296) + | > loss_1: 29.47193717956543 (30.634542384617767) + | > grad_norm_1: tensor(434.7584, device='cuda:0') (tensor(567.5491, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.8493 (0.9727217714551469) + | > loader_time: 0.0045 (0.006397620053358481) + + + --> TIME: 2024-07-14 02:10:16 -- STEP: 121/151 -- GLOBAL_STEP: 112000 + | > loss_disc: 2.330782890319824 (2.2966095140157656) + | > loss_disc_real_0: 0.07355339080095291 (0.11354693154658167) + | > loss_disc_real_1: 0.1879005879163742 (0.20266893863185378) + | > loss_disc_real_2: 0.2280462682247162 (0.209327391233326) + | > loss_disc_real_3: 0.2318064421415329 (0.20486701704746435) + | > loss_disc_real_4: 0.1945323348045349 (0.2060498773805366) + | > loss_disc_real_5: 0.24562948942184448 (0.22092863116875167) + | > loss_0: 2.330782890319824 (2.2966095140157656) + | > grad_norm_0: tensor(22.6005, device='cuda:0') (tensor(34.2316, device='cuda:0')) + | > loss_gen: 2.7051687240600586 (2.6685573995605973) + | > loss_kl: 1.9899886846542358 (1.7208661042954312) + | > loss_feat: 7.837191104888916 (8.402501074735781) + | > loss_mel: 16.44948959350586 (16.87163404984909) + | > loss_duration: 0.9150246977806091 (0.9259484055613684) + | > loss_1: 29.896862030029297 (30.589507016268644) + | > grad_norm_1: tensor(547.8556, device='cuda:0') (tensor(574.8130, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.8664 (0.9729466714149664) + | > loader_time: 0.005 (0.0063613288658709586) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03413257232079139 (+0.001043979938213646) + | > avg_loss_disc: 2.688855684720553 (+0.27904475652254535) + | > avg_loss_disc_real_0: 0.4150263208609361 (+0.2687362673190924) + | > avg_loss_disc_real_1: 0.25091058703569274 (-0.013766782788129872) + | > avg_loss_disc_real_2: 0.27010563015937805 (+0.002128005027770996) + | > avg_loss_disc_real_3: 0.25635876334630525 (+0.0158895873106443) + | > avg_loss_disc_real_4: 0.2336179671379236 (-0.03452341373150164) + | > avg_loss_disc_real_5: 0.2470755588549834 (+0.05253552702757028) + | > avg_loss_0: 2.688855684720553 (+0.27904475652254535) + | > avg_loss_gen: 2.8363151366894064 (+0.1542050838470459) + | > avg_loss_kl: 3.877345085144043 (+0.04600407527043249) + | > avg_loss_feat: 7.826888744647686 (-0.7568724338824939) + | > avg_loss_mel: 16.942064578716572 (-0.3250072185809785) + | > avg_loss_duration: 1.4078070658903856 (+0.08449515012594366) + | > avg_loss_1: 32.89042062025804 (-0.7971751873309785) + + + > EPOCH: 179/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:11:23)  + + --> TIME: 2024-07-14 02:11:45 -- STEP: 20/151 -- GLOBAL_STEP: 112050 + | > loss_disc: 2.273786783218384 (2.243605947494507) + | > loss_disc_real_0: 0.16930733621120453 (0.11465875394642353) + | > loss_disc_real_1: 0.27573201060295105 (0.19328751638531685) + | > loss_disc_real_2: 0.24813269078731537 (0.19925968125462531) + | > loss_disc_real_3: 0.22153979539871216 (0.1969185508787632) + | > loss_disc_real_4: 0.24919289350509644 (0.20885959267616272) + | > loss_disc_real_5: 0.24082084000110626 (0.21258432567119598) + | > loss_0: 2.273786783218384 (2.243605947494507) + | > grad_norm_0: tensor(22.0516, device='cuda:0') (tensor(20.4968, device='cuda:0')) + | > loss_gen: 2.9000470638275146 (2.635944771766663) + | > loss_kl: 1.8959784507751465 (1.6410558104515076) + | > loss_feat: 7.507968902587891 (8.394288039207458) + | > loss_mel: 16.939321517944336 (16.245563220977782) + | > loss_duration: 0.9886904358863831 (0.9422499388456345) + | > loss_1: 30.23200798034668 (29.859102153778075) + | > grad_norm_1: tensor(313.9704, device='cuda:0') (tensor(323.5126, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 1.0719 (0.9818068861961364) + | > loader_time: 0.007 (0.006055521965026856) + + + --> TIME: 2024-07-14 02:12:34 -- STEP: 70/151 -- GLOBAL_STEP: 112100 + | > loss_disc: 2.1806507110595703 (2.281454934392656) + | > loss_disc_real_0: 0.12426495552062988 (0.11891743860074452) + | > loss_disc_real_1: 0.24143029749393463 (0.1990712868315833) + | > loss_disc_real_2: 0.23580868542194366 (0.20956915095448495) + | > loss_disc_real_3: 0.2005871683359146 (0.20242978206702641) + | > loss_disc_real_4: 0.17398601770401 (0.20418860667518207) + | > loss_disc_real_5: 0.16810889542102814 (0.2176060859646116) + | > loss_0: 2.1806507110595703 (2.281454934392656) + | > grad_norm_0: tensor(14.4406, device='cuda:0') (tensor(24.5686, device='cuda:0')) + | > loss_gen: 2.6511354446411133 (2.620208028384617) + | > loss_kl: 1.546773910522461 (1.6976165771484373) + | > loss_feat: 7.729424953460693 (8.256916890825542) + | > loss_mel: 16.93649673461914 (16.300571441650387) + | > loss_duration: 0.9007689952850342 (0.9301537275314331) + | > loss_1: 29.764598846435547 (29.805466542925153) + | > grad_norm_1: tensor(217.5243, device='cuda:0') (tensor(363.9169, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.8968 (0.9791757345199585) + | > loader_time: 0.0057 (0.006279914719717843) + + + --> TIME: 2024-07-14 02:13:22 -- STEP: 120/151 -- GLOBAL_STEP: 112150 + | > loss_disc: 2.2368993759155273 (2.2756418824195843) + | > loss_disc_real_0: 0.06171954795718193 (0.11674326003218691) + | > loss_disc_real_1: 0.15692216157913208 (0.1992412811766068) + | > loss_disc_real_2: 0.20637820661067963 (0.2076797608286142) + | > loss_disc_real_3: 0.20188993215560913 (0.2033266716947158) + | > loss_disc_real_4: 0.2230159044265747 (0.2030668690800667) + | > loss_disc_real_5: 0.2328692227602005 (0.21958068708578746) + | > loss_0: 2.2368993759155273 (2.2756418824195843) + | > grad_norm_0: tensor(16.9740, device='cuda:0') (tensor(24.1142, device='cuda:0')) + | > loss_gen: 2.604912042617798 (2.625373371442159) + | > loss_kl: 1.2848137617111206 (1.6962485844890276) + | > loss_feat: 9.1842622756958 (8.330323855082186) + | > loss_mel: 17.00303077697754 (16.368278288841246) + | > loss_duration: 0.8304367065429688 (0.926695444683234) + | > loss_1: 30.907455444335938 (29.946919425328574) + | > grad_norm_1: tensor(413.1902, device='cuda:0') (tensor(360.4007, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.8788 (0.9639685730139415) + | > loader_time: 0.0049 (0.00616111159324646) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0346527283008282 (+0.00052015598003681) + | > avg_loss_disc: 2.4427273456866923 (-0.24612833903386067) + | > avg_loss_disc_real_0: 0.20507594255300668 (-0.20995037830792943) + | > avg_loss_disc_real_1: 0.2140620190363664 (-0.03684856799932634) + | > avg_loss_disc_real_2: 0.1981249715273197 (-0.07198065863205835) + | > avg_loss_disc_real_3: 0.22266855492041662 (-0.03369020842588863) + | > avg_loss_disc_real_4: 0.21953801352244157 (-0.014079953615482033) + | > avg_loss_disc_real_5: 0.2245033589693216 (-0.02257219988566178) + | > avg_loss_0: 2.4427273456866923 (-0.24612833903386067) + | > avg_loss_gen: 2.4479551315307617 (-0.38836000515864466) + | > avg_loss_kl: 3.760557926618136 (-0.11678715852590704) + | > avg_loss_feat: 8.209744966947115 (+0.38285622229942895) + | > avg_loss_mel: 17.09796142578125 (+0.1558968470646782) + | > avg_loss_duration: 1.3259811309667735 (-0.08182593492361212) + | > avg_loss_1: 32.84220049931453 (-0.0482201209435118) + + + > EPOCH: 180/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:14:00)  + + --> TIME: 2024-07-14 02:14:21 -- STEP: 19/151 -- GLOBAL_STEP: 112200 + | > loss_disc: 1.990883469581604 (2.2525159622493542) + | > loss_disc_real_0: 0.0677293986082077 (0.10750127583742142) + | > loss_disc_real_1: 0.18855313956737518 (0.1953624302619382) + | > loss_disc_real_2: 0.17326517403125763 (0.20997889810486844) + | > loss_disc_real_3: 0.172141894698143 (0.19994799714339406) + | > loss_disc_real_4: 0.18854273855686188 (0.20699278226024226) + | > loss_disc_real_5: 0.183777317404747 (0.21351846582011172) + | > loss_0: 1.990883469581604 (2.2525159622493542) + | > grad_norm_0: tensor(27.1362, device='cuda:0') (tensor(28.3136, device='cuda:0')) + | > loss_gen: 3.057584285736084 (2.6865699542196175) + | > loss_kl: 1.1364455223083496 (1.672910451889038) + | > loss_feat: 10.524129867553711 (8.568467767615067) + | > loss_mel: 17.516185760498047 (16.90988375011243) + | > loss_duration: 1.0148340463638306 (0.8888890680513883) + | > loss_1: 33.24917984008789 (30.726721111096833) + | > grad_norm_1: tensor(669.9517, device='cuda:0') (tensor(504.6775, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.8191 (0.9522803582643208) + | > loader_time: 0.0045 (0.0060756331995913855) + + + --> TIME: 2024-07-14 02:15:14 -- STEP: 69/151 -- GLOBAL_STEP: 112250 + | > loss_disc: 2.3593344688415527 (2.288671697395435) + | > loss_disc_real_0: 0.14821574091911316 (0.11516512682040532) + | > loss_disc_real_1: 0.17633971571922302 (0.2039788787563642) + | > loss_disc_real_2: 0.2348863035440445 (0.2103935197211694) + | > loss_disc_real_3: 0.22695094347000122 (0.2044790482175523) + | > loss_disc_real_4: 0.21152406930923462 (0.20764734714791394) + | > loss_disc_real_5: 0.2883963882923126 (0.21879736992759982) + | > loss_0: 2.3593344688415527 (2.288671697395435) + | > grad_norm_0: tensor(42.0960, device='cuda:0') (tensor(30.7120, device='cuda:0')) + | > loss_gen: 2.644784450531006 (2.6549659639165015) + | > loss_kl: 1.7689101696014404 (1.717574599860371) + | > loss_feat: 8.003901481628418 (8.324696886366695) + | > loss_mel: 17.11917495727539 (16.646358420883406) + | > loss_duration: 0.8566684722900391 (0.9276485278986503) + | > loss_1: 30.39344024658203 (30.2712441596432) + | > grad_norm_1: tensor(533.0857, device='cuda:0') (tensor(492.3079, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.9764 (1.026441059250762) + | > loader_time: 0.0055 (0.0066784015600232114) + + + --> TIME: 2024-07-14 02:16:01 -- STEP: 119/151 -- GLOBAL_STEP: 112300 + | > loss_disc: 2.37424898147583 (2.2939593331152652) + | > loss_disc_real_0: 0.06978068500757217 (0.11797469942008748) + | > loss_disc_real_1: 0.1816592961549759 (0.20391018719983703) + | > loss_disc_real_2: 0.1999502032995224 (0.2092072788406821) + | > loss_disc_real_3: 0.1533333659172058 (0.20563776215084462) + | > loss_disc_real_4: 0.18447282910346985 (0.2066780232331332) + | > loss_disc_real_5: 0.2928307056427002 (0.21880935627121886) + | > loss_0: 2.37424898147583 (2.2939593331152652) + | > grad_norm_0: tensor(14.7729, device='cuda:0') (tensor(28.4423, device='cuda:0')) + | > loss_gen: 2.3220643997192383 (2.6517904345728778) + | > loss_kl: 1.2317864894866943 (1.7451839366880786) + | > loss_feat: 8.502513885498047 (8.460100758977298) + | > loss_mel: 16.420644760131836 (16.770096778869625) + | > loss_duration: 0.833819568157196 (0.9123708771056488) + | > loss_1: 29.310829162597656 (30.539542751151973) + | > grad_norm_1: tensor(314.1394, device='cuda:0') (tensor(454.5997, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.8764 (0.9878991271267417) + | > loader_time: 0.0049 (0.0064388523582650835) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0347759540264423 (+0.00012322572561409423) + | > avg_loss_disc: 2.6855882497934194 (+0.24286090410672712) + | > avg_loss_disc_real_0: 0.34659412732491124 (+0.14151818477190456) + | > avg_loss_disc_real_1: 0.26509279700425953 (+0.051030777967893126) + | > avg_loss_disc_real_2: 0.2664310955084287 (+0.068306123981109) + | > avg_loss_disc_real_3: 0.2288660601927684 (+0.006197505272351772) + | > avg_loss_disc_real_4: 0.2644146359883822 (+0.04487662246594065) + | > avg_loss_disc_real_5: 0.2566913813352585 (+0.032188022365936875) + | > avg_loss_0: 2.6855882497934194 (+0.24286090410672712) + | > avg_loss_gen: 2.75274186867934 (+0.3047867371485782) + | > avg_loss_kl: 3.9177279472351074 (+0.1571700206169715) + | > avg_loss_feat: 7.983056288499099 (-0.2266886784480162) + | > avg_loss_mel: 16.93061975332407 (-0.16734167245718012) + | > avg_loss_duration: 1.3778850665459266 (+0.05190393557915307) + | > avg_loss_1: 32.96203070420485 (+0.11983020489032015) + + + > EPOCH: 181/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:16:42)  + + --> TIME: 2024-07-14 02:17:01 -- STEP: 18/151 -- GLOBAL_STEP: 112350 + | > loss_disc: 2.2417194843292236 (2.2489873038397894) + | > loss_disc_real_0: 0.08517785370349884 (0.10901405393249458) + | > loss_disc_real_1: 0.2158115953207016 (0.19609662973218495) + | > loss_disc_real_2: 0.2045421600341797 (0.2135029604865445) + | > loss_disc_real_3: 0.22495907545089722 (0.2062536622915003) + | > loss_disc_real_4: 0.18307170271873474 (0.20202606957819727) + | > loss_disc_real_5: 0.27704641222953796 (0.22231635120179918) + | > loss_0: 2.2417194843292236 (2.2489873038397894) + | > grad_norm_0: tensor(24.0834, device='cuda:0') (tensor(23.8806, device='cuda:0')) + | > loss_gen: 2.4961471557617188 (2.680730316374037) + | > loss_kl: 1.6141873598098755 (1.6813640991846721) + | > loss_feat: 7.766902923583984 (8.7219557762146) + | > loss_mel: 17.24214744567871 (16.882088979085285) + | > loss_duration: 0.9391281604766846 (0.8914852738380432) + | > loss_1: 30.058513641357422 (30.857625007629395) + | > grad_norm_1: tensor(481.9440, device='cuda:0') (tensor(381.7832, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.8881 (0.9013402064641317) + | > loader_time: 0.0063 (0.005436711841159397) + + + --> TIME: 2024-07-14 02:17:54 -- STEP: 68/151 -- GLOBAL_STEP: 112400 + | > loss_disc: 1.9766881465911865 (2.2678194799843965) + | > loss_disc_real_0: 0.06084984540939331 (0.11476816669763888) + | > loss_disc_real_1: 0.13591444492340088 (0.1982715037815711) + | > loss_disc_real_2: 0.1684061735868454 (0.20927724176469972) + | > loss_disc_real_3: 0.18666379153728485 (0.2036592019174029) + | > loss_disc_real_4: 0.1439932882785797 (0.20262069791993675) + | > loss_disc_real_5: 0.18980443477630615 (0.21698923956821947) + | > loss_0: 1.9766881465911865 (2.2678194799843965) + | > grad_norm_0: tensor(18.1301, device='cuda:0') (tensor(25.9139, device='cuda:0')) + | > loss_gen: 2.9313461780548096 (2.6529422051766347) + | > loss_kl: 0.7776232361793518 (1.7149385503109764) + | > loss_feat: 9.375648498535156 (8.481603769695054) + | > loss_mel: 16.277772903442383 (16.729270261876735) + | > loss_duration: 1.1035187244415283 (0.9223017166642582) + | > loss_1: 30.465911865234375 (30.50105650284711) + | > grad_norm_1: tensor(407.4140, device='cuda:0') (tensor(370.5166, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.8168 (1.0077424259746777) + | > loader_time: 0.0045 (0.006673651583054487) + + + --> TIME: 2024-07-14 02:18:43 -- STEP: 118/151 -- GLOBAL_STEP: 112450 + | > loss_disc: 2.489309072494507 (2.2626832319518266) + | > loss_disc_real_0: 0.12530368566513062 (0.115049897866734) + | > loss_disc_real_1: 0.20427562296390533 (0.19847837696641182) + | > loss_disc_real_2: 0.27547889947891235 (0.20965060781119232) + | > loss_disc_real_3: 0.24449273943901062 (0.203035567713491) + | > loss_disc_real_4: 0.228817418217659 (0.20359138399362564) + | > loss_disc_real_5: 0.26431038975715637 (0.21706621475138907) + | > loss_0: 2.489309072494507 (2.2626832319518266) + | > grad_norm_0: tensor(29.9970, device='cuda:0') (tensor(26.0309, device='cuda:0')) + | > loss_gen: 2.4176251888275146 (2.6664906113834705) + | > loss_kl: 1.9909937381744385 (1.718931724964562) + | > loss_feat: 6.718188285827637 (8.565930140220507) + | > loss_mel: 15.771636009216309 (16.74153117002067) + | > loss_duration: 0.9225656986236572 (0.9192160726603815) + | > loss_1: 27.821008682250977 (30.6120997445058) + | > grad_norm_1: tensor(369.2391, device='cuda:0') (tensor(397.5243, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 1.0306 (0.9866698616642064) + | > loader_time: 0.0072 (0.006472379474316613) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03444833021897536 (-0.00032762380746693964) + | > avg_loss_disc: 2.5662314158219557 (-0.11935683397146368) + | > avg_loss_disc_real_0: 0.23525241246590248 (-0.11134171485900876) + | > avg_loss_disc_real_1: 0.19077542424201965 (-0.07431737276223987) + | > avg_loss_disc_real_2: 0.252966060088231 (-0.013465035420197669) + | > avg_loss_disc_real_3: 0.22659932535428268 (-0.0022667348384857178) + | > avg_loss_disc_real_4: 0.22632038707916552 (-0.0380942489092167) + | > avg_loss_disc_real_5: 0.21682483760210183 (-0.03986654373315665) + | > avg_loss_0: 2.5662314158219557 (-0.11935683397146368) + | > avg_loss_gen: 2.3486838340759277 (-0.40405803460341216) + | > avg_loss_kl: 3.865162886106051 (-0.052565061129056456) + | > avg_loss_feat: 8.01073950987596 (+0.027683221376860523) + | > avg_loss_mel: 16.835208379305325 (-0.09541137401874522) + | > avg_loss_duration: 1.3473131289848914 (-0.030571937561035156) + | > avg_loss_1: 32.40710816016564 (-0.5549225440392078) + + + > EPOCH: 182/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:19:24)  + + --> TIME: 2024-07-14 02:19:43 -- STEP: 17/151 -- GLOBAL_STEP: 112500 + | > loss_disc: 2.219580888748169 (2.260980592054479) + | > loss_disc_real_0: 0.08170001208782196 (0.11050035453894559) + | > loss_disc_real_1: 0.17760345339775085 (0.20335210772121654) + | > loss_disc_real_2: 0.1616039276123047 (0.21100745569257176) + | > loss_disc_real_3: 0.1589154452085495 (0.1993423069224638) + | > loss_disc_real_4: 0.15866059064865112 (0.20274987904464498) + | > loss_disc_real_5: 0.22236165404319763 (0.21265649970840006) + | > loss_0: 2.219580888748169 (2.260980592054479) + | > grad_norm_0: tensor(25.9022, device='cuda:0') (tensor(28.9473, device='cuda:0')) + | > loss_gen: 2.612529993057251 (2.668866690467386) + | > loss_kl: 1.5038684606552124 (1.7031411072787117) + | > loss_feat: 9.8912353515625 (8.587079440846162) + | > loss_mel: 17.063247680664062 (16.77395528905532) + | > loss_duration: 0.7350317239761353 (0.9178904961137211) + | > loss_1: 31.805912017822266 (30.65093287299661) + | > grad_norm_1: tensor(109.3713, device='cuda:0') (tensor(519.9869, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.8759 (0.9905012214885038) + | > loader_time: 0.0048 (0.00607748592601103) + + + --> TIME: 2024-07-14 02:20:34 -- STEP: 67/151 -- GLOBAL_STEP: 112550 + | > loss_disc: 2.217078685760498 (2.248677305321195) + | > loss_disc_real_0: 0.17888377606868744 (0.1122100595662843) + | > loss_disc_real_1: 0.2712690532207489 (0.20114543010939412) + | > loss_disc_real_2: 0.2309769243001938 (0.20730097547395906) + | > loss_disc_real_3: 0.21410474181175232 (0.20144985560605774) + | > loss_disc_real_4: 0.23863568902015686 (0.20001276638080825) + | > loss_disc_real_5: 0.15635555982589722 (0.21457537875246646) + | > loss_0: 2.217078685760498 (2.248677305321195) + | > grad_norm_0: tensor(38.6196, device='cuda:0') (tensor(30.5804, device='cuda:0')) + | > loss_gen: 2.8083159923553467 (2.699826240539551) + | > loss_kl: 1.7883042097091675 (1.6930187860531594) + | > loss_feat: 7.879496097564697 (8.672584839721228) + | > loss_mel: 16.03490447998047 (16.71806195956558) + | > loss_duration: 0.903491199016571 (0.9352965292645924) + | > loss_1: 29.414512634277344 (30.718788175440547) + | > grad_norm_1: tensor(600.6048, device='cuda:0') (tensor(515.9590, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.9682 (1.0001117115590101) + | > loader_time: 0.0071 (0.00650478121060044) + + + --> TIME: 2024-07-14 02:21:24 -- STEP: 117/151 -- GLOBAL_STEP: 112600 + | > loss_disc: 2.520315408706665 (2.2745103000575657) + | > loss_disc_real_0: 0.2029750794172287 (0.1166983506618402) + | > loss_disc_real_1: 0.16925059258937836 (0.20205720978924352) + | > loss_disc_real_2: 0.1876659095287323 (0.20740081650069636) + | > loss_disc_real_3: 0.22476516664028168 (0.20338030267729718) + | > loss_disc_real_4: 0.24192827939987183 (0.20268351400000414) + | > loss_disc_real_5: 0.21402192115783691 (0.2158046279452805) + | > loss_0: 2.520315408706665 (2.2745103000575657) + | > grad_norm_0: tensor(38.6477, device='cuda:0') (tensor(31.0777, device='cuda:0')) + | > loss_gen: 2.2205944061279297 (2.67528367246318) + | > loss_kl: 2.117255926132202 (1.7277811816614914) + | > loss_feat: 6.588576316833496 (8.596241755363264) + | > loss_mel: 13.589608192443848 (16.720852199782676) + | > loss_duration: 1.1509207487106323 (0.9239374699755611) + | > loss_1: 25.666955947875977 (30.64409616258409) + | > grad_norm_1: tensor(480.1664, device='cuda:0') (tensor(500.6917, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 2.1089 (0.9947947762970226) + | > loader_time: 0.0154 (0.006462814461471688) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03292630268977238 (-0.0015220275292029753) + | > avg_loss_disc: 2.5939051921551046 (+0.027673776333148936) + | > avg_loss_disc_real_0: 0.2642141878604889 (+0.028961775394586414) + | > avg_loss_disc_real_1: 0.23119454429699823 (+0.04041912005497858) + | > avg_loss_disc_real_2: 0.24552766520243424 (-0.007438394885796784) + | > avg_loss_disc_real_3: 0.21843897952483252 (-0.00816034582945016) + | > avg_loss_disc_real_4: 0.24401617623292482 (+0.0176957891537593) + | > avg_loss_disc_real_5: 0.21089290082454681 (-0.005931936777555019) + | > avg_loss_0: 2.5939051921551046 (+0.027673776333148936) + | > avg_loss_gen: 2.4474669969998875 (+0.09878316292395972) + | > avg_loss_kl: 3.6906259610102725 (-0.17453692509577845) + | > avg_loss_feat: 7.826039204230676 (-0.1847003056452836) + | > avg_loss_mel: 16.810080308180588 (-0.025128071124736806) + | > avg_loss_duration: 1.3102334187580988 (-0.03707971022679257) + | > avg_loss_1: 32.0844456599309 (-0.32266250023474186) + + + > EPOCH: 183/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:22:07)  + + --> TIME: 2024-07-14 02:22:27 -- STEP: 16/151 -- GLOBAL_STEP: 112650 + | > loss_disc: 2.151242256164551 (2.3595560491085052) + | > loss_disc_real_0: 0.09255701303482056 (0.15675766998901963) + | > loss_disc_real_1: 0.22155168652534485 (0.20957346726208925) + | > loss_disc_real_2: 0.18574048578739166 (0.2082693064585328) + | > loss_disc_real_3: 0.2372726947069168 (0.2145986407995224) + | > loss_disc_real_4: 0.16237476468086243 (0.20971516612917185) + | > loss_disc_real_5: 0.15267178416252136 (0.21857332065701485) + | > loss_0: 2.151242256164551 (2.3595560491085052) + | > grad_norm_0: tensor(39.9029, device='cuda:0') (tensor(39.2941, device='cuda:0')) + | > loss_gen: 2.6061718463897705 (2.6006892174482346) + | > loss_kl: 1.4522098302841187 (1.8690972551703453) + | > loss_feat: 9.511150360107422 (7.751545429229736) + | > loss_mel: 18.29016876220703 (16.718613147735596) + | > loss_duration: 0.8005114197731018 (0.9412485770881176) + | > loss_1: 32.660213470458984 (29.88119375705719) + | > grad_norm_1: tensor(543.4494, device='cuda:0') (tensor(519.7840, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.8814 (1.111670508980751) + | > loader_time: 0.0047 (0.007257789373397828) + + + --> TIME: 2024-07-14 02:23:14 -- STEP: 66/151 -- GLOBAL_STEP: 112700 + | > loss_disc: 2.4508473873138428 (2.2842579502047915) + | > loss_disc_real_0: 0.16260944306850433 (0.1250349892811342) + | > loss_disc_real_1: 0.26091596484184265 (0.20296638165459488) + | > loss_disc_real_2: 0.24239423871040344 (0.20741273982055258) + | > loss_disc_real_3: 0.25529056787490845 (0.2044611151019732) + | > loss_disc_real_4: 0.2770446240901947 (0.20265871354124762) + | > loss_disc_real_5: 0.24003693461418152 (0.21648714014075018) + | > loss_0: 2.4508473873138428 (2.2842579502047915) + | > grad_norm_0: tensor(24.5948, device='cuda:0') (tensor(30.8918, device='cuda:0')) + | > loss_gen: 2.7789642810821533 (2.675167921817664) + | > loss_kl: 1.9271628856658936 (1.7668138408299647) + | > loss_feat: 8.88965892791748 (8.456031120184695) + | > loss_mel: 17.166879653930664 (16.76832878228389) + | > loss_duration: 0.8614792823791504 (0.9193712238109473) + | > loss_1: 31.6241455078125 (30.58571312644265) + | > grad_norm_1: tensor(423.6117, device='cuda:0') (tensor(446.6498, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.8913 (0.9737242713119042) + | > loader_time: 0.0066 (0.006184494856632118) + + + --> TIME: 2024-07-14 02:24:04 -- STEP: 116/151 -- GLOBAL_STEP: 112750 + | > loss_disc: 2.21848201751709 (2.2837568603712946) + | > loss_disc_real_0: 0.09348084777593613 (0.1217937891467892) + | > loss_disc_real_1: 0.17444047331809998 (0.2025615140024958) + | > loss_disc_real_2: 0.17573182284832 (0.20760585200683823) + | > loss_disc_real_3: 0.19692803919315338 (0.203704464538344) + | > loss_disc_real_4: 0.2128424346446991 (0.20122485569324985) + | > loss_disc_real_5: 0.18292659521102905 (0.21912806577466684) + | > loss_0: 2.21848201751709 (2.2837568603712946) + | > grad_norm_0: tensor(28.2778, device='cuda:0') (tensor(28.3195, device='cuda:0')) + | > loss_gen: 2.618955373764038 (2.6584094187309004) + | > loss_kl: 1.7829970121383667 (1.7560297286716) + | > loss_feat: 8.622735977172852 (8.540932219603961) + | > loss_mel: 16.66660499572754 (16.7454535139018) + | > loss_duration: 0.8839151859283447 (0.9170932266218909) + | > loss_1: 30.57520866394043 (30.617918162510314) + | > grad_norm_1: tensor(588.2635, device='cuda:0') (tensor(420.9248, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.9407 (0.979658963351414) + | > loader_time: 0.0066 (0.00634900660350405) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03341238315288837 (+0.00048608046311598985) + | > avg_loss_disc: 2.626077523598304 (+0.0321723314431992) + | > avg_loss_disc_real_0: 0.26584457204892087 (+0.001630384188431977) + | > avg_loss_disc_real_1: 0.24494201861895049 (+0.013747474321952252) + | > avg_loss_disc_real_2: 0.28429817580259764 (+0.0387705106001634) + | > avg_loss_disc_real_3: 0.2584668558377486 (+0.04002787631291607) + | > avg_loss_disc_real_4: 0.25881031499459195 (+0.01479413876166713) + | > avg_loss_disc_real_5: 0.24067030732448286 (+0.029777406499936043) + | > avg_loss_0: 2.626077523598304 (+0.0321723314431992) + | > avg_loss_gen: 2.61112249814547 (+0.1636555011455827) + | > avg_loss_kl: 3.8425790713383603 (+0.15195311032808778) + | > avg_loss_feat: 7.481641072493333 (-0.3443981317373428) + | > avg_loss_mel: 16.646008638235237 (-0.1640716699453506) + | > avg_loss_duration: 1.3486261642896211 (+0.03839274553152228) + | > avg_loss_1: 31.929977270273064 (-0.15446838965783627) + + + > EPOCH: 184/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:24:49)  + + --> TIME: 2024-07-14 02:25:06 -- STEP: 15/151 -- GLOBAL_STEP: 112800 + | > loss_disc: 2.2947025299072266 (2.277593143781026) + | > loss_disc_real_0: 0.09379594027996063 (0.11320981259147327) + | > loss_disc_real_1: 0.207110196352005 (0.19410769740740458) + | > loss_disc_real_2: 0.22278684377670288 (0.20671276648839315) + | > loss_disc_real_3: 0.15001577138900757 (0.19925757398207983) + | > loss_disc_real_4: 0.1878468096256256 (0.2001479392250379) + | > loss_disc_real_5: 0.19561895728111267 (0.226237690448761) + | > loss_0: 2.2947025299072266 (2.277593143781026) + | > grad_norm_0: tensor(31.0047, device='cuda:0') (tensor(21.7907, device='cuda:0')) + | > loss_gen: 2.5084896087646484 (2.638200283050537) + | > loss_kl: 1.592286467552185 (1.6813786268234252) + | > loss_feat: 8.14763355255127 (8.514268271128337) + | > loss_mel: 16.02407455444336 (16.519406890869142) + | > loss_duration: 0.830925464630127 (0.9577895561854045) + | > loss_1: 29.103408813476562 (30.311043548583985) + | > grad_norm_1: tensor(473.7592, device='cuda:0') (tensor(439.8060, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.8686 (0.9749049504597982) + | > loader_time: 0.0044 (0.00618441899617513) + + + --> TIME: 2024-07-14 02:25:55 -- STEP: 65/151 -- GLOBAL_STEP: 112850 + | > loss_disc: 2.11838436126709 (2.2847968559998733) + | > loss_disc_real_0: 0.12891435623168945 (0.11348316440215478) + | > loss_disc_real_1: 0.18571721017360687 (0.19954760395563564) + | > loss_disc_real_2: 0.1863728016614914 (0.20531927874455086) + | > loss_disc_real_3: 0.18338516354560852 (0.20460670418464222) + | > loss_disc_real_4: 0.13781525194644928 (0.20671300418101823) + | > loss_disc_real_5: 0.16950877010822296 (0.22122283302820644) + | > loss_0: 2.11838436126709 (2.2847968559998733) + | > grad_norm_0: tensor(40.2156, device='cuda:0') (tensor(29.2475, device='cuda:0')) + | > loss_gen: 2.9040818214416504 (2.6787118691664475) + | > loss_kl: 1.2179871797561646 (1.7402600160011878) + | > loss_feat: 10.956453323364258 (8.605416862781235) + | > loss_mel: 17.369762420654297 (16.75765769665058) + | > loss_duration: 0.9117478132247925 (0.9206965070504408) + | > loss_1: 33.36003112792969 (30.70274297274076) + | > grad_norm_1: tensor(436.1373, device='cuda:0') (tensor(480.7103, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.8495 (0.9668126949897179) + | > loader_time: 0.0049 (0.0063085849468524635) + + + --> TIME: 2024-07-14 02:26:46 -- STEP: 115/151 -- GLOBAL_STEP: 112900 + | > loss_disc: 2.2431793212890625 (2.2906173996303387) + | > loss_disc_real_0: 0.08001122623682022 (0.11405966589632241) + | > loss_disc_real_1: 0.21261177957057953 (0.20217023919457974) + | > loss_disc_real_2: 0.24034574627876282 (0.20637301439824313) + | > loss_disc_real_3: 0.16585195064544678 (0.2064816220299057) + | > loss_disc_real_4: 0.1498575210571289 (0.2071711166397385) + | > loss_disc_real_5: 0.1989060342311859 (0.22024480920770895) + | > loss_0: 2.2431793212890625 (2.2906173996303387) + | > grad_norm_0: tensor(21.6149, device='cuda:0') (tensor(30.4003, device='cuda:0')) + | > loss_gen: 2.529069423675537 (2.671898715392403) + | > loss_kl: 1.5377421379089355 (1.7624442992003069) + | > loss_feat: 7.3164262771606445 (8.54027717839117) + | > loss_mel: 16.864116668701172 (16.77247455431067) + | > loss_duration: 0.8678861856460571 (0.919897376972696) + | > loss_1: 29.1152400970459 (30.66699215432872) + | > grad_norm_1: tensor(499.7624, device='cuda:0') (tensor(486.7092, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.8953 (0.9836043274920919) + | > loader_time: 0.005 (0.00652086009149966) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032594735805804916 (-0.0008176473470834569) + | > avg_loss_disc: 2.5839701432448168 (-0.042107380353487045) + | > avg_loss_disc_real_0: 0.2968086050106929 (+0.030964032961772026) + | > avg_loss_disc_real_1: 0.2209235246364887 (-0.02401849398246178) + | > avg_loss_disc_real_2: 0.2513288133419477 (-0.032969362460649965) + | > avg_loss_disc_real_3: 0.23370510454361254 (-0.024761751294136047) + | > avg_loss_disc_real_4: 0.2717054704060921 (+0.012895155411500159) + | > avg_loss_disc_real_5: 0.2651532097504689 (+0.024482902425986053) + | > avg_loss_0: 2.5839701432448168 (-0.042107380353487045) + | > avg_loss_gen: 2.7421483259934645 (+0.13102582784799433) + | > avg_loss_kl: 3.964133996229905 (+0.12155492489154485) + | > avg_loss_feat: 8.134740352630615 (+0.6530992801372824) + | > avg_loss_mel: 16.876209332392765 (+0.23020069415752786) + | > avg_loss_duration: 1.397012316263639 (+0.048386151974017855) + | > avg_loss_1: 33.11424387418307 (+1.1842666039100038) + + + > EPOCH: 185/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:27:30)  + + --> TIME: 2024-07-14 02:27:49 -- STEP: 14/151 -- GLOBAL_STEP: 112950 + | > loss_disc: 1.9217889308929443 (2.222748909677778) + | > loss_disc_real_0: 0.06368568539619446 (0.10583216590540749) + | > loss_disc_real_1: 0.2048196792602539 (0.1958737884249006) + | > loss_disc_real_2: 0.18635766208171844 (0.20195542488779342) + | > loss_disc_real_3: 0.15569959580898285 (0.19134983207498277) + | > loss_disc_real_4: 0.13794375956058502 (0.20546846304621016) + | > loss_disc_real_5: 0.18145184218883514 (0.22505912504025868) + | > loss_0: 1.9217889308929443 (2.222748909677778) + | > grad_norm_0: tensor(8.9107, device='cuda:0') (tensor(33.1903, device='cuda:0')) + | > loss_gen: 3.107999086380005 (2.7551209926605225) + | > loss_kl: 1.1943905353546143 (1.7834135123661585) + | > loss_feat: 10.059175491333008 (8.70993617602757) + | > loss_mel: 15.428030967712402 (16.84099394934518) + | > loss_duration: 1.1559371948242188 (0.9720935949257442) + | > loss_1: 30.945533752441406 (31.061557906014578) + | > grad_norm_1: tensor(347.9716, device='cuda:0') (tensor(545.8456, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.8569 (1.075739792415074) + | > loader_time: 0.0044 (0.0069075993129185265) + + + --> TIME: 2024-07-14 02:28:39 -- STEP: 64/151 -- GLOBAL_STEP: 113000 + | > loss_disc: 2.0914230346679688 (2.282593682408333) + | > loss_disc_real_0: 0.07250963151454926 (0.118689245544374) + | > loss_disc_real_1: 0.19673898816108704 (0.19863102468661964) + | > loss_disc_real_2: 0.19012662768363953 (0.20621673786081374) + | > loss_disc_real_3: 0.15787972509860992 (0.2062002478633076) + | > loss_disc_real_4: 0.13607966899871826 (0.20675611612387002) + | > loss_disc_real_5: 0.17682993412017822 (0.22016126802191138) + | > loss_0: 2.0914230346679688 (2.282593682408333) + | > grad_norm_0: tensor(13.4951, device='cuda:0') (tensor(27.3918, device='cuda:0')) + | > loss_gen: 2.790382146835327 (2.6585861407220364) + | > loss_kl: 1.310139775276184 (1.7135493699461222) + | > loss_feat: 9.075201988220215 (8.514620386064053) + | > loss_mel: 15.607131958007812 (16.79897551238536) + | > loss_duration: 1.1914656162261963 (0.9473271248862147) + | > loss_1: 29.974321365356445 (30.633058577775955) + | > grad_norm_1: tensor(238.4066, device='cuda:0') (tensor(428.3604, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.8152 (1.0026032887399197) + | > loader_time: 0.0053 (0.006293397396802902) + + + --> TIME: 2024-07-14 02:29:53 -- STEP: 114/151 -- GLOBAL_STEP: 113050 + | > loss_disc: 2.4789042472839355 (2.2872158560836535) + | > loss_disc_real_0: 0.09263446182012558 (0.11577631667125643) + | > loss_disc_real_1: 0.20203983783721924 (0.20125683854546464) + | > loss_disc_real_2: 0.2755966782569885 (0.20977929156077535) + | > loss_disc_real_3: 0.23877529799938202 (0.20610914150612397) + | > loss_disc_real_4: 0.2634057104587555 (0.2062326853996829) + | > loss_disc_real_5: 0.2294185608625412 (0.2196092133720716) + | > loss_0: 2.4789042472839355 (2.2872158560836535) + | > grad_norm_0: tensor(20.0659, device='cuda:0') (tensor(25.1700, device='cuda:0')) + | > loss_gen: 2.521031379699707 (2.6511808257353935) + | > loss_kl: 1.827696681022644 (1.7165418099938778) + | > loss_feat: 9.029061317443848 (8.519629612303621) + | > loss_mel: 16.961626052856445 (16.70645368308351) + | > loss_duration: 0.8368650078773499 (0.927217986500054) + | > loss_1: 31.176280975341797 (30.521024018003228) + | > grad_norm_1: tensor(343.9222, device='cuda:0') (tensor(414.5942, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.9431 (0.9728670015669706) + | > loader_time: 0.0054 (0.006133679758038437) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0368506358220027 (+0.004255900016197786) + | > avg_loss_disc: 2.5988819232353797 (+0.014911779990562923) + | > avg_loss_disc_real_0: 0.30305848786464107 (+0.006249882853948174) + | > avg_loss_disc_real_1: 0.22075413969846872 (-0.00016938493801998966) + | > avg_loss_disc_real_2: 0.23667220427439764 (-0.014656609067550036) + | > avg_loss_disc_real_3: 0.18945776499234712 (-0.04424733955126542) + | > avg_loss_disc_real_4: 0.2058930156322626 (-0.06581245477382952) + | > avg_loss_disc_real_5: 0.2591759722966414 (-0.0059772374538275) + | > avg_loss_0: 2.5988819232353797 (+0.014911779990562923) + | > avg_loss_gen: 2.445458705608661 (-0.2966896203848033) + | > avg_loss_kl: 3.9468517853663516 (-0.017282210863553527) + | > avg_loss_feat: 8.150587448706993 (+0.01584709607637791) + | > avg_loss_mel: 17.353550250713642 (+0.477340918320877) + | > avg_loss_duration: 1.3883740260050847 (-0.008638290258554226) + | > avg_loss_1: 33.28482217055101 (+0.17057829636794253) + + + > EPOCH: 186/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:30:39)  + + --> TIME: 2024-07-14 02:30:54 -- STEP: 13/151 -- GLOBAL_STEP: 113100 + | > loss_disc: 2.3736565113067627 (2.2539901366600623) + | > loss_disc_real_0: 0.1273731291294098 (0.11225753105603732) + | > loss_disc_real_1: 0.20752950012683868 (0.19635799068671006) + | > loss_disc_real_2: 0.19757576286792755 (0.20395067219550794) + | > loss_disc_real_3: 0.2754189074039459 (0.20486340614465567) + | > loss_disc_real_4: 0.2399049997329712 (0.1966927830989544) + | > loss_disc_real_5: 0.21927575767040253 (0.21142172813415527) + | > loss_0: 2.3736565113067627 (2.2539901366600623) + | > grad_norm_0: tensor(18.9583, device='cuda:0') (tensor(32.9786, device='cuda:0')) + | > loss_gen: 2.8471267223358154 (2.7364782186654897) + | > loss_kl: 1.7783950567245483 (1.832967235491826) + | > loss_feat: 7.948641777038574 (8.46405054972722) + | > loss_mel: 15.666061401367188 (16.372134868915264) + | > loss_duration: 0.898732602596283 (0.8691439307652987) + | > loss_1: 29.13895606994629 (30.2747745513916) + | > grad_norm_1: tensor(529.8593, device='cuda:0') (tensor(540.4224, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.9402 (0.9133424575512226) + | > loader_time: 0.0046 (0.0059987031496488135) + + + --> TIME: 2024-07-14 02:31:44 -- STEP: 63/151 -- GLOBAL_STEP: 113150 + | > loss_disc: 2.0526392459869385 (2.2717592224242193) + | > loss_disc_real_0: 0.0541452094912529 (0.1179600978299739) + | > loss_disc_real_1: 0.17223083972930908 (0.20024972231615157) + | > loss_disc_real_2: 0.1596238911151886 (0.2106284596144207) + | > loss_disc_real_3: 0.17262357473373413 (0.2065668891346644) + | > loss_disc_real_4: 0.2310108095407486 (0.20299649297718017) + | > loss_disc_real_5: 0.20523148775100708 (0.21486039199526347) + | > loss_0: 2.0526392459869385 (2.2717592224242193) + | > grad_norm_0: tensor(10.2880, device='cuda:0') (tensor(36.0512, device='cuda:0')) + | > loss_gen: 2.7249913215637207 (2.7273911521548313) + | > loss_kl: 1.876582384109497 (1.7278679807980855) + | > loss_feat: 9.178645133972168 (8.628665636456205) + | > loss_mel: 17.52420425415039 (16.660011836460654) + | > loss_duration: 0.9620730876922607 (0.9257071538577004) + | > loss_1: 32.26649475097656 (30.66964364430261) + | > grad_norm_1: tensor(177.9180, device='cuda:0') (tensor(561.0144, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.8575 (0.9734408968970889) + | > loader_time: 0.0044 (0.006232923931545681) + + + --> TIME: 2024-07-14 02:32:35 -- STEP: 113/151 -- GLOBAL_STEP: 113200 + | > loss_disc: 2.2913026809692383 (2.2816608310800732) + | > loss_disc_real_0: 0.11222188174724579 (0.11642306363951843) + | > loss_disc_real_1: 0.22748520970344543 (0.2009004288542587) + | > loss_disc_real_2: 0.1915043145418167 (0.20859247722984414) + | > loss_disc_real_3: 0.21417537331581116 (0.2064250432020795) + | > loss_disc_real_4: 0.243364155292511 (0.20374584349647032) + | > loss_disc_real_5: 0.2650345265865326 (0.21828067487320013) + | > loss_0: 2.2913026809692383 (2.2816608310800732) + | > grad_norm_0: tensor(9.4366, device='cuda:0') (tensor(33.1927, device='cuda:0')) + | > loss_gen: 2.862839698791504 (2.686506146878268) + | > loss_kl: 1.760539174079895 (1.7058214518876202) + | > loss_feat: 9.259367942810059 (8.50202244361945) + | > loss_mel: 17.823368072509766 (16.608880811032986) + | > loss_duration: 0.8762753009796143 (0.9275025999651546) + | > loss_1: 32.58238983154297 (30.43073340221844) + | > grad_norm_1: tensor(418.2387, device='cuda:0') (tensor(504.3527, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.9106 (0.993106407401836) + | > loader_time: 0.0059 (0.006428001201258296) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03742162997906025 (+0.0005709941570575458) + | > avg_loss_disc: 2.515726401255681 (-0.08315552197969867) + | > avg_loss_disc_real_0: 0.23603907571389124 (-0.06701941215074983) + | > avg_loss_disc_real_1: 0.22361348684017474 (+0.00285934714170602) + | > avg_loss_disc_real_2: 0.23405708143344292 (-0.00261512284095472) + | > avg_loss_disc_real_3: 0.22990744618269113 (+0.040449681190344006) + | > avg_loss_disc_real_4: 0.22615338747317976 (+0.02026037184091717) + | > avg_loss_disc_real_5: 0.24389412311407235 (-0.015281849182569057) + | > avg_loss_0: 2.515726401255681 (-0.08315552197969867) + | > avg_loss_gen: 2.5024777559133677 (+0.05701905030470655) + | > avg_loss_kl: 3.847975969314575 (-0.09887581605177642) + | > avg_loss_feat: 8.206808493687557 (+0.05622104498056402) + | > avg_loss_mel: 17.25136353419377 (-0.10218671651987066) + | > avg_loss_duration: 1.3436380624771118 (-0.04473596352797293) + | > avg_loss_1: 33.152263788076546 (-0.13255838247446405) + + + > EPOCH: 187/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:33:21)  + + --> TIME: 2024-07-14 02:33:34 -- STEP: 12/151 -- GLOBAL_STEP: 113250 + | > loss_disc: 2.5858144760131836 (2.2984774708747864) + | > loss_disc_real_0: 0.1497229039669037 (0.11925501562654972) + | > loss_disc_real_1: 0.205635204911232 (0.21441129222512245) + | > loss_disc_real_2: 0.2624225616455078 (0.21842710425456366) + | > loss_disc_real_3: 0.2911194860935211 (0.2051230991880099) + | > loss_disc_real_4: 0.25304311513900757 (0.20854337885975838) + | > loss_disc_real_5: 0.292526513338089 (0.21967008958260217) + | > loss_0: 2.5858144760131836 (2.2984774708747864) + | > grad_norm_0: tensor(25.7885, device='cuda:0') (tensor(23.6491, device='cuda:0')) + | > loss_gen: 2.366616725921631 (2.63059401512146) + | > loss_kl: 1.7727103233337402 (1.5217478573322296) + | > loss_feat: 7.1421403884887695 (8.073814630508423) + | > loss_mel: 16.564495086669922 (16.472562710444134) + | > loss_duration: 0.9986550807952881 (0.9381227493286133) + | > loss_1: 28.84461784362793 (29.636842250823975) + | > grad_norm_1: tensor(335.7149, device='cuda:0') (tensor(374.9745, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 1.1723 (0.9289463559786478) + | > loader_time: 0.0089 (0.005834817886352539) + + + --> TIME: 2024-07-14 02:34:28 -- STEP: 62/151 -- GLOBAL_STEP: 113300 + | > loss_disc: 2.4091014862060547 (2.3122402583399135) + | > loss_disc_real_0: 0.24482151865959167 (0.12300233002151212) + | > loss_disc_real_1: 0.24948546290397644 (0.20409648336710468) + | > loss_disc_real_2: 0.237983837723732 (0.2084681833944013) + | > loss_disc_real_3: 0.2903667688369751 (0.20543401231688838) + | > loss_disc_real_4: 0.2121843695640564 (0.2085859732281777) + | > loss_disc_real_5: 0.2817111909389496 (0.22504648590280163) + | > loss_0: 2.4091014862060547 (2.3122402583399135) + | > grad_norm_0: tensor(62.0489, device='cuda:0') (tensor(31.5764, device='cuda:0')) + | > loss_gen: 2.8275980949401855 (2.6393823046838083) + | > loss_kl: 2.0183141231536865 (1.7138141268684017) + | > loss_feat: 7.326781749725342 (8.232846336980023) + | > loss_mel: 16.93256187438965 (16.494858941724225) + | > loss_duration: 0.8571893572807312 (0.9371483191367118) + | > loss_1: 29.962444305419922 (30.018050009204494) + | > grad_norm_1: tensor(535.2446, device='cuda:0') (tensor(466.4431, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 1.0094 (1.0275112275154359) + | > loader_time: 0.0086 (0.006951885838662425) + + + --> TIME: 2024-07-14 02:35:17 -- STEP: 112/151 -- GLOBAL_STEP: 113350 + | > loss_disc: 2.2373878955841064 (2.3063378940735553) + | > loss_disc_real_0: 0.12336723506450653 (0.1238340399161513) + | > loss_disc_real_1: 0.24466711282730103 (0.20397565779941423) + | > loss_disc_real_2: 0.1918671578168869 (0.20833746889340027) + | > loss_disc_real_3: 0.22920939326286316 (0.2043639478672828) + | > loss_disc_real_4: 0.2968904972076416 (0.20651067487363303) + | > loss_disc_real_5: 0.19873324036598206 (0.22410680699561322) + | > loss_0: 2.2373878955841064 (2.3063378940735553) + | > grad_norm_0: tensor(15.3106, device='cuda:0') (tensor(27.4107, device='cuda:0')) + | > loss_gen: 2.7747654914855957 (2.6347311713865826) + | > loss_kl: 1.6289093494415283 (1.7100225926509924) + | > loss_feat: 8.527276039123535 (8.338959617274146) + | > loss_mel: 17.792861938476562 (16.701043214116776) + | > loss_duration: 0.8522815108299255 (0.9283261144799846) + | > loss_1: 31.576093673706055 (30.31308271203722) + | > grad_norm_1: tensor(113.2748, device='cuda:0') (tensor(383.9818, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.8968 (1.0018826190914427) + | > loader_time: 0.0049 (0.006630590983799526) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03495605175311749 (-0.0024655782259427605) + | > avg_loss_disc: 2.564844461587759 (+0.049118060332077995) + | > avg_loss_disc_real_0: 0.23947569269400376 (+0.0034366169801125224) + | > avg_loss_disc_real_1: 0.22114971853219545 (-0.002463768307979286) + | > avg_loss_disc_real_2: 0.2442136464210657 (+0.010156564987622768) + | > avg_loss_disc_real_3: 0.26622184079426986 (+0.03631439461157873) + | > avg_loss_disc_real_4: 0.27181479793328506 (+0.0456614104601053) + | > avg_loss_disc_real_5: 0.25232059680498564 (+0.008426473690913289) + | > avg_loss_0: 2.564844461587759 (+0.049118060332077995) + | > avg_loss_gen: 2.5475265979766846 (+0.04504884206331683) + | > avg_loss_kl: 3.8055970118596005 (-0.042378957454974664) + | > avg_loss_feat: 7.550562968620887 (-0.6562455250666703) + | > avg_loss_mel: 16.614702738248386 (-0.6366607959453852) + | > avg_loss_duration: 1.3512482826526349 (+0.007610220175523041) + | > avg_loss_1: 31.86963756267841 (-1.2826262253981362) + + + > EPOCH: 188/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:36:03)  + + --> TIME: 2024-07-14 02:36:16 -- STEP: 11/151 -- GLOBAL_STEP: 113400 + | > loss_disc: 2.1685760021209717 (2.2308318723331797) + | > loss_disc_real_0: 0.07536717504262924 (0.11169267445802689) + | > loss_disc_real_1: 0.14301547408103943 (0.1959516704082489) + | > loss_disc_real_2: 0.1692368984222412 (0.20240959389643234) + | > loss_disc_real_3: 0.1667013168334961 (0.20169751752506604) + | > loss_disc_real_4: 0.15534472465515137 (0.20443665303967215) + | > loss_disc_real_5: 0.18503466248512268 (0.2209814664992419) + | > loss_0: 2.1685760021209717 (2.2308318723331797) + | > grad_norm_0: tensor(14.0012, device='cuda:0') (tensor(21.4764, device='cuda:0')) + | > loss_gen: 2.4765219688415527 (2.657840035178445) + | > loss_kl: 1.380965232849121 (1.7750910954041914) + | > loss_feat: 8.402868270874023 (8.311318570917303) + | > loss_mel: 15.302549362182617 (15.934199853376908) + | > loss_duration: 0.938122034072876 (0.9439553293314847) + | > loss_1: 28.501028060913086 (29.622405138882723) + | > grad_norm_1: tensor(331.5442, device='cuda:0') (tensor(326.8841, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.8313 (0.9675783894278787) + | > loader_time: 0.0045 (0.006491335955533114) + + + --> TIME: 2024-07-14 02:37:05 -- STEP: 61/151 -- GLOBAL_STEP: 113450 + | > loss_disc: 2.4394092559814453 (2.262339781542293) + | > loss_disc_real_0: 0.20647530257701874 (0.11675349048903731) + | > loss_disc_real_1: 0.23848465085029602 (0.19789877411771994) + | > loss_disc_real_2: 0.23132279515266418 (0.20342250266036052) + | > loss_disc_real_3: 0.2405841201543808 (0.20132696677426823) + | > loss_disc_real_4: 0.2220149040222168 (0.2014849673284859) + | > loss_disc_real_5: 0.19398844242095947 (0.21978990122920178) + | > loss_0: 2.4394092559814453 (2.262339781542293) + | > grad_norm_0: tensor(38.9159, device='cuda:0') (tensor(22.4064, device='cuda:0')) + | > loss_gen: 2.6342687606811523 (2.6549726665997118) + | > loss_kl: 1.7344565391540527 (1.7270559824881007) + | > loss_feat: 7.850487232208252 (8.641124193785624) + | > loss_mel: 14.79705810546875 (16.621835114525965) + | > loss_duration: 0.9052010774612427 (0.9238485263996437) + | > loss_1: 27.921470642089844 (30.568836305962233) + | > grad_norm_1: tensor(455.6723, device='cuda:0') (tensor(328.6894, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.9103 (0.9652618462922143) + | > loader_time: 0.0064 (0.0063752932626692975) + + + --> TIME: 2024-07-14 02:37:56 -- STEP: 111/151 -- GLOBAL_STEP: 113500 + | > loss_disc: 2.432448387145996 (2.2861189315984918) + | > loss_disc_real_0: 0.11533038318157196 (0.11944780109433441) + | > loss_disc_real_1: 0.2305639386177063 (0.20149914551157136) + | > loss_disc_real_2: 0.23507067561149597 (0.20657588440824198) + | > loss_disc_real_3: 0.23146528005599976 (0.2044598657537151) + | > loss_disc_real_4: 0.25753605365753174 (0.20363047255857572) + | > loss_disc_real_5: 0.23343199491500854 (0.22008528513414366) + | > loss_0: 2.432448387145996 (2.2861189315984918) + | > grad_norm_0: tensor(14.1453, device='cuda:0') (tensor(24.0242, device='cuda:0')) + | > loss_gen: 2.278876543045044 (2.6313620386896894) + | > loss_kl: 1.690993309020996 (1.7131583561768402) + | > loss_feat: 6.638693332672119 (8.474032290346988) + | > loss_mel: 15.637557983398438 (16.69641926052333) + | > loss_duration: 0.8645139932632446 (0.9236153476947063) + | > loss_1: 27.11063575744629 (30.4385871028041) + | > grad_norm_1: tensor(297.1298, device='cuda:0') (tensor(368.4142, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.9112 (0.9811254729021777) + | > loader_time: 0.0065 (0.00644590188791086) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.033695330986609824 (-0.0012607207665076628) + | > avg_loss_disc: 2.5186526775360107 (-0.046191784051748286) + | > avg_loss_disc_real_0: 0.17474492925864 (-0.06473076343536377) + | > avg_loss_disc_real_1: 0.23136569444949812 (+0.010215975917302667) + | > avg_loss_disc_real_2: 0.27693959726737094 (+0.03272595084630525) + | > avg_loss_disc_real_3: 0.23497781272117907 (-0.03124402807309079) + | > avg_loss_disc_real_4: 0.2505475019033139 (-0.021267296029971183) + | > avg_loss_disc_real_5: 0.2661750144683398 (+0.013854417663354157) + | > avg_loss_0: 2.5186526775360107 (-0.046191784051748286) + | > avg_loss_gen: 2.5414951031024637 (-0.006031494874220833) + | > avg_loss_kl: 4.009676676530105 (+0.2040796646705041) + | > avg_loss_feat: 7.944390993851882 (+0.39382802523099514) + | > avg_loss_mel: 16.968965750474197 (+0.354263012225811) + | > avg_loss_duration: 1.342228706066425 (-0.009019576586209777) + | > avg_loss_1: 32.80675726670485 (+0.9371197040264398) + + + > EPOCH: 189/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:38:46)  + + --> TIME: 2024-07-14 02:38:59 -- STEP: 10/151 -- GLOBAL_STEP: 113550 + | > loss_disc: 2.354419469833374 (2.351771330833435) + | > loss_disc_real_0: 0.1466926485300064 (0.12843537516891956) + | > loss_disc_real_1: 0.2005605697631836 (0.19710431843996049) + | > loss_disc_real_2: 0.19945286214351654 (0.2014206051826477) + | > loss_disc_real_3: 0.2412409782409668 (0.21199704706668854) + | > loss_disc_real_4: 0.19309234619140625 (0.19404325038194656) + | > loss_disc_real_5: 0.21823440492153168 (0.21741097271442414) + | > loss_0: 2.354419469833374 (2.351771330833435) + | > grad_norm_0: tensor(15.9017, device='cuda:0') (tensor(22.8186, device='cuda:0')) + | > loss_gen: 2.7392349243164062 (2.5039521217346192) + | > loss_kl: 2.081393003463745 (1.8002689838409425) + | > loss_feat: 8.056800842285156 (7.804389429092407) + | > loss_mel: 17.216428756713867 (16.308200740814208) + | > loss_duration: 0.9230831861495972 (0.9233410716056824) + | > loss_1: 31.016939163208008 (29.340152168273924) + | > grad_norm_1: tensor(409.2295, device='cuda:0') (tensor(250.7511, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 1.0872 (1.0644185066223144) + | > loader_time: 0.0061 (0.006763911247253418) + + + --> TIME: 2024-07-14 02:39:48 -- STEP: 60/151 -- GLOBAL_STEP: 113600 + | > loss_disc: 2.3030285835266113 (2.268053797880808) + | > loss_disc_real_0: 0.159457266330719 (0.11659131099780401) + | > loss_disc_real_1: 0.29233309626579285 (0.1990300565958023) + | > loss_disc_real_2: 0.2691430449485779 (0.20538063794374467) + | > loss_disc_real_3: 0.23267221450805664 (0.2010875252385934) + | > loss_disc_real_4: 0.21098928153514862 (0.20445393150051436) + | > loss_disc_real_5: 0.23047317564487457 (0.21619241908192635) + | > loss_0: 2.3030285835266113 (2.268053797880808) + | > grad_norm_0: tensor(20.2922, device='cuda:0') (tensor(19.4303, device='cuda:0')) + | > loss_gen: 2.9372386932373047 (2.6245003660519917) + | > loss_kl: 1.9532636404037476 (1.7181980888048807) + | > loss_feat: 8.490277290344238 (8.539528687795004) + | > loss_mel: 16.908832550048828 (16.515828164418533) + | > loss_duration: 0.8953394889831543 (0.9209603836139043) + | > loss_1: 31.184951782226562 (30.31901559829712) + | > grad_norm_1: tensor(537.2507, device='cuda:0') (tensor(260.9526, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 1.0028 (0.9904924233754476) + | > loader_time: 0.0086 (0.006159802277882894) + + + --> TIME: 2024-07-14 02:40:36 -- STEP: 110/151 -- GLOBAL_STEP: 113650 + | > loss_disc: 2.064038038253784 (2.27820677323775) + | > loss_disc_real_0: 0.10031803697347641 (0.11891226484016938) + | > loss_disc_real_1: 0.157943993806839 (0.19978985583240338) + | > loss_disc_real_2: 0.1696147322654724 (0.2081942005590959) + | > loss_disc_real_3: 0.15036222338676453 (0.20186447927897627) + | > loss_disc_real_4: 0.16948914527893066 (0.20475482331080871) + | > loss_disc_real_5: 0.1774207055568695 (0.21628167994997718) + | > loss_0: 2.064038038253784 (2.27820677323775) + | > grad_norm_0: tensor(28.2015, device='cuda:0') (tensor(22.7317, device='cuda:0')) + | > loss_gen: 2.7830986976623535 (2.6371614781292996) + | > loss_kl: 1.560936450958252 (1.7190544805743477) + | > loss_feat: 10.345632553100586 (8.522221313823351) + | > loss_mel: 16.068252563476562 (16.618558034029874) + | > loss_duration: 0.8217589855194092 (0.9205281246792186) + | > loss_1: 31.57967758178711 (30.417523418773303) + | > grad_norm_1: tensor(468.8365, device='cuda:0') (tensor(322.9033, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.8629 (0.9691563476215709) + | > loader_time: 0.0046 (0.006105182387612083) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031313217603243314 (-0.0023821133833665104) + | > avg_loss_disc: 2.6938388714423547 (+0.17518619390634393) + | > avg_loss_disc_real_0: 0.23559779960375565 (+0.06085287034511566) + | > avg_loss_disc_real_1: 0.24912055524495932 (+0.017754860795461208) + | > avg_loss_disc_real_2: 0.23270304615680987 (-0.04423655111056107) + | > avg_loss_disc_real_3: 0.2833596765995025 (+0.04838186387832344) + | > avg_loss_disc_real_4: 0.259677318426279 (+0.009129816522965128) + | > avg_loss_disc_real_5: 0.24749871973808 (-0.018676294730259807) + | > avg_loss_0: 2.6938388714423547 (+0.17518619390634393) + | > avg_loss_gen: 2.3949736815232496 (-0.1465214215792141) + | > avg_loss_kl: 3.942023203923152 (-0.06765347260695265) + | > avg_loss_feat: 7.555963442875789 (-0.38842755097609327) + | > avg_loss_mel: 16.912824997535118 (-0.056140752939079164) + | > avg_loss_duration: 1.3357391174022968 (-0.0064895886641282985) + | > avg_loss_1: 32.14152438823994 (-0.6652328784649129) + + + > EPOCH: 190/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:41:27)  + + --> TIME: 2024-07-14 02:41:40 -- STEP: 9/151 -- GLOBAL_STEP: 113700 + | > loss_disc: 2.3104164600372314 (2.3196966118282742) + | > loss_disc_real_0: 0.15578126907348633 (0.11723676075537999) + | > loss_disc_real_1: 0.24435250461101532 (0.20658008588684929) + | > loss_disc_real_2: 0.2214985340833664 (0.2031788867380884) + | > loss_disc_real_3: 0.2011614441871643 (0.2017820676167806) + | > loss_disc_real_4: 0.2622535228729248 (0.2015433990293079) + | > loss_disc_real_5: 0.22998425364494324 (0.20291128589047325) + | > loss_0: 2.3104164600372314 (2.3196966118282742) + | > grad_norm_0: tensor(25.4297, device='cuda:0') (tensor(26.0939, device='cuda:0')) + | > loss_gen: 2.8977670669555664 (2.6436958048078747) + | > loss_kl: 1.7227087020874023 (1.670435176955329) + | > loss_feat: 7.801485538482666 (8.428749879201254) + | > loss_mel: 16.80453109741211 (16.67295371161567) + | > loss_duration: 0.9172175526618958 (0.9530805945396423) + | > loss_1: 30.14371109008789 (30.368915557861328) + | > grad_norm_1: tensor(186.1081, device='cuda:0') (tensor(386.3569, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.8964 (1.1084466245439317) + | > loader_time: 0.0043 (0.007398313946194119) + + + --> TIME: 2024-07-14 02:42:32 -- STEP: 59/151 -- GLOBAL_STEP: 113750 + | > loss_disc: 2.4179258346557617 (2.3209026223522127) + | > loss_disc_real_0: 0.1745634377002716 (0.12444779529409894) + | > loss_disc_real_1: 0.20190443098545074 (0.2023838026038671) + | > loss_disc_real_2: 0.17772135138511658 (0.2071777558175184) + | > loss_disc_real_3: 0.20179836452007294 (0.20379291803149854) + | > loss_disc_real_4: 0.24345219135284424 (0.20599705182899863) + | > loss_disc_real_5: 0.24682991206645966 (0.22748234463950334) + | > loss_0: 2.4179258346557617 (2.3209026223522127) + | > grad_norm_0: tensor(11.1620, device='cuda:0') (tensor(21.8871, device='cuda:0')) + | > loss_gen: 2.495919942855835 (2.5875385090456167) + | > loss_kl: 1.663571834564209 (1.7012745576389765) + | > loss_feat: 6.569729328155518 (8.197117506447485) + | > loss_mel: 15.305643081665039 (16.524388103161826) + | > loss_duration: 0.874333918094635 (0.9420777842149897) + | > loss_1: 26.909198760986328 (29.952396392822266) + | > grad_norm_1: tensor(400.6818, device='cuda:0') (tensor(341.5903, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.8821 (1.0413469581280719) + | > loader_time: 0.0052 (0.006849887007373875) + + + --> TIME: 2024-07-14 02:43:19 -- STEP: 109/151 -- GLOBAL_STEP: 113800 + | > loss_disc: 2.3836073875427246 (2.3083311527147217) + | > loss_disc_real_0: 0.07791437208652496 (0.12028250467339786) + | > loss_disc_real_1: 0.23001153767108917 (0.20078989795041738) + | > loss_disc_real_2: 0.20616751909255981 (0.20414020524385873) + | > loss_disc_real_3: 0.18524466454982758 (0.20588190785241783) + | > loss_disc_real_4: 0.15723054111003876 (0.2061823540187757) + | > loss_disc_real_5: 0.22319231927394867 (0.22631828131478862) + | > loss_0: 2.3836073875427246 (2.3083311527147217) + | > grad_norm_0: tensor(30.9904, device='cuda:0') (tensor(24.7717, device='cuda:0')) + | > loss_gen: 2.725348711013794 (2.6148550991618307) + | > loss_kl: 1.898948073387146 (1.6933704816966975) + | > loss_feat: 8.825554847717285 (8.348846824890975) + | > loss_mel: 16.422395706176758 (16.629030245159736) + | > loss_duration: 0.8987247943878174 (0.9209766568393882) + | > loss_1: 30.770973205566406 (30.207079143699158) + | > grad_norm_1: tensor(84.1863, device='cuda:0') (tensor(377.0329, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.8545 (0.9871271422149935) + | > loader_time: 0.0041 (0.006286601407812274) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03240398260263296 (+0.0010907649993896484) + | > avg_loss_disc: 2.597480058670044 (-0.09635881277231073) + | > avg_loss_disc_real_0: 0.23180816838374504 (-0.0037896312200106086) + | > avg_loss_disc_real_1: 0.1902985515502783 (-0.05882200369468102) + | > avg_loss_disc_real_2: 0.20576368616177484 (-0.026939359995035023) + | > avg_loss_disc_real_3: 0.24652302723664504 (-0.036836649362857465) + | > avg_loss_disc_real_4: 0.2635259880469396 (+0.0038486696206606053) + | > avg_loss_disc_real_5: 0.2354791244635215 (-0.012019595274558481) + | > avg_loss_0: 2.597480058670044 (-0.09635881277231073) + | > avg_loss_gen: 2.3742370788867655 (-0.020736602636484136) + | > avg_loss_kl: 4.131580902979924 (+0.18955769905677222) + | > avg_loss_feat: 8.26390394797692 (+0.7079405051011305) + | > avg_loss_mel: 17.038534824664776 (+0.12570982712965773) + | > avg_loss_duration: 1.352957377066979 (+0.017218259664682156) + | > avg_loss_1: 33.16121482849121 (+1.0196904402512743) + + + > EPOCH: 191/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:44:10)  + + --> TIME: 2024-07-14 02:44:23 -- STEP: 8/151 -- GLOBAL_STEP: 113850 + | > loss_disc: 2.545830249786377 (2.3274614214897156) + | > loss_disc_real_0: 0.1225663274526596 (0.13866693433374166) + | > loss_disc_real_1: 0.22062820196151733 (0.19723714143037796) + | > loss_disc_real_2: 0.1460637003183365 (0.20155935734510422) + | > loss_disc_real_3: 0.27575066685676575 (0.19337640423327684) + | > loss_disc_real_4: 0.2548392117023468 (0.21120226383209229) + | > loss_disc_real_5: 0.23446890711784363 (0.21546494960784912) + | > loss_0: 2.545830249786377 (2.3274614214897156) + | > grad_norm_0: tensor(38.6461, device='cuda:0') (tensor(26.7717, device='cuda:0')) + | > loss_gen: 2.1526596546173096 (2.553057551383972) + | > loss_kl: 2.2970194816589355 (1.8987266719341278) + | > loss_feat: 4.913227081298828 (7.365127503871918) + | > loss_mel: 13.411683082580566 (15.713525056838991) + | > loss_duration: 1.1554088592529297 (1.0173647329211235) + | > loss_1: 23.92999839782715 (28.54780125617981) + | > grad_norm_1: tensor(482.7528, device='cuda:0') (tensor(361.0099, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 2.0981 (1.2862018048763275) + | > loader_time: 0.0171 (0.00929868221282959) + + + --> TIME: 2024-07-14 02:45:11 -- STEP: 58/151 -- GLOBAL_STEP: 113900 + | > loss_disc: 2.370281457901001 (2.317725736519385) + | > loss_disc_real_0: 0.17370936274528503 (0.12136928390326171) + | > loss_disc_real_1: 0.1771518439054489 (0.20279541354754876) + | > loss_disc_real_2: 0.22936107218265533 (0.20780158299824286) + | > loss_disc_real_3: 0.20341850817203522 (0.20337204956288996) + | > loss_disc_real_4: 0.1982523798942566 (0.20645485972536018) + | > loss_disc_real_5: 0.23846913874149323 (0.22462876809054408) + | > loss_0: 2.370281457901001 (2.317725736519385) + | > grad_norm_0: tensor(78.0060, device='cuda:0') (tensor(28.2766, device='cuda:0')) + | > loss_gen: 2.7169103622436523 (2.6158919909904754) + | > loss_kl: 1.824129343032837 (1.7911545408183132) + | > loss_feat: 8.170355796813965 (8.124776585348718) + | > loss_mel: 17.053869247436523 (16.531637158887143) + | > loss_duration: 0.764854371547699 (0.9106101198443051) + | > loss_1: 30.530120849609375 (29.974070318813983) + | > grad_norm_1: tensor(470.2432, device='cuda:0') (tensor(426.3150, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.8671 (1.003091105099382) + | > loader_time: 0.0046 (0.006485955468539534) + + + --> TIME: 2024-07-14 02:45:58 -- STEP: 108/151 -- GLOBAL_STEP: 113950 + | > loss_disc: 2.434748411178589 (2.305212908320957) + | > loss_disc_real_0: 0.14350275695323944 (0.12499279684076704) + | > loss_disc_real_1: 0.17443722486495972 (0.20290128403791674) + | > loss_disc_real_2: 0.21306446194648743 (0.20813704764953367) + | > loss_disc_real_3: 0.2535368800163269 (0.20139859471884036) + | > loss_disc_real_4: 0.25146785378456116 (0.20535876532947575) + | > loss_disc_real_5: 0.23245014250278473 (0.22273670702620788) + | > loss_0: 2.434748411178589 (2.305212908320957) + | > grad_norm_0: tensor(10.9950, device='cuda:0') (tensor(28.7318, device='cuda:0')) + | > loss_gen: 2.5099806785583496 (2.651764463495326) + | > loss_kl: 1.8695857524871826 (1.7551775596759938) + | > loss_feat: 8.216772079467773 (8.338521895585236) + | > loss_mel: 15.709854125976562 (16.69116121751291) + | > loss_duration: 0.774290144443512 (0.9211877358180506) + | > loss_1: 29.080482482910156 (30.357812775505913) + | > grad_norm_1: tensor(366.7962, device='cuda:0') (tensor(421.5812, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.8947 (0.9712023911652741) + | > loader_time: 0.006 (0.006232071805883337) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03400202897878794 (+0.001598046376154981) + | > avg_loss_disc: 2.6011887880472035 (+0.003708729377159603) + | > avg_loss_disc_real_0: 0.20904186826485854 (-0.0227663001188865) + | > avg_loss_disc_real_1: 0.22711989398186022 (+0.036821342431581916) + | > avg_loss_disc_real_2: 0.23524639125053698 (+0.029482705088762134) + | > avg_loss_disc_real_3: 0.2991894414791694 (+0.052666414242524384) + | > avg_loss_disc_real_4: 0.2681519641326024 (+0.004625976085662786) + | > avg_loss_disc_real_5: 0.24403311885320222 (+0.008553994389680714) + | > avg_loss_0: 2.6011887880472035 (+0.003708729377159603) + | > avg_loss_gen: 2.4849513310652513 (+0.11071425217848585) + | > avg_loss_kl: 3.8143809942098765 (-0.31719990877004767) + | > avg_loss_feat: 7.848597013033354 (-0.4153069349435654) + | > avg_loss_mel: 17.101375799912674 (+0.062840975247898) + | > avg_loss_duration: 1.3460432932927058 (-0.006914083774273161) + | > avg_loss_1: 32.595348798311676 (-0.5658660301795351) + + + > EPOCH: 192/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:46:51)  + + --> TIME: 2024-07-14 02:47:00 -- STEP: 7/151 -- GLOBAL_STEP: 114000 + | > loss_disc: 2.299529552459717 (2.158086793763297) + | > loss_disc_real_0: 0.12111850827932358 (0.10790092444845609) + | > loss_disc_real_1: 0.19839873909950256 (0.18264495687825338) + | > loss_disc_real_2: 0.25467050075531006 (0.20928082721573965) + | > loss_disc_real_3: 0.19685091078281403 (0.19209687624658858) + | > loss_disc_real_4: 0.2568281292915344 (0.20051983850342886) + | > loss_disc_real_5: 0.22604748606681824 (0.2077160371201379) + | > loss_0: 2.299529552459717 (2.158086793763297) + | > grad_norm_0: tensor(11.3150, device='cuda:0') (tensor(18.5331, device='cuda:0')) + | > loss_gen: 2.6450276374816895 (2.7775582586015974) + | > loss_kl: 1.5878995656967163 (1.6011403458459037) + | > loss_feat: 9.648139953613281 (8.927618367331368) + | > loss_mel: 16.755016326904297 (16.82560838971819) + | > loss_duration: 0.8908302783966064 (0.9054968782833644) + | > loss_1: 31.526912689208984 (31.037421907697404) + | > grad_norm_1: tensor(68.6080, device='cuda:0') (tensor(276.3663, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.9217 (0.873523337500436) + | > loader_time: 0.0088 (0.005290303911481585) + + + --> TIME: 2024-07-14 02:48:17 -- STEP: 57/151 -- GLOBAL_STEP: 114050 + | > loss_disc: 2.1376123428344727 (2.24961895691721) + | > loss_disc_real_0: 0.035220012068748474 (0.10784957440275895) + | > loss_disc_real_1: 0.25104933977127075 (0.19842926883383802) + | > loss_disc_real_2: 0.2021082043647766 (0.20786735990591215) + | > loss_disc_real_3: 0.19177532196044922 (0.20085736926187547) + | > loss_disc_real_4: 0.21932131052017212 (0.19898234752186558) + | > loss_disc_real_5: 0.19878631830215454 (0.21841120066349967) + | > loss_0: 2.1376123428344727 (2.24961895691721) + | > grad_norm_0: tensor(23.8308, device='cuda:0') (tensor(22.4623, device='cuda:0')) + | > loss_gen: 2.9507179260253906 (2.6655839158777606) + | > loss_kl: 1.4742494821548462 (1.6746179508535486) + | > loss_feat: 9.884263038635254 (8.64029452675267) + | > loss_mel: 16.957359313964844 (16.618904649165646) + | > loss_duration: 1.0009667873382568 (0.9291414733518634) + | > loss_1: 32.267555236816406 (30.528542568809108) + | > grad_norm_1: tensor(455.8374, device='cuda:0') (tensor(351.2894, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.8163 (0.9905453313860979) + | > loader_time: 0.0042 (0.006020023111711468) + + + --> TIME: 2024-07-14 02:49:06 -- STEP: 107/151 -- GLOBAL_STEP: 114100 + | > loss_disc: 2.2783639430999756 (2.273086385192156) + | > loss_disc_real_0: 0.11347882449626923 (0.11289863096915673) + | > loss_disc_real_1: 0.18055510520935059 (0.19937139700784862) + | > loss_disc_real_2: 0.18514089286327362 (0.20716448949876232) + | > loss_disc_real_3: 0.19783855974674225 (0.20124643094071715) + | > loss_disc_real_4: 0.18780694901943207 (0.2029452322520942) + | > loss_disc_real_5: 0.22351738810539246 (0.22052982365973642) + | > loss_0: 2.2783639430999756 (2.273086385192156) + | > grad_norm_0: tensor(7.5987, device='cuda:0') (tensor(23.0901, device='cuda:0')) + | > loss_gen: 2.641977310180664 (2.64001251158313) + | > loss_kl: 1.5915066003799438 (1.7033385670630732) + | > loss_feat: 10.068551063537598 (8.529033139487296) + | > loss_mel: 16.358442306518555 (16.566113365030738) + | > loss_duration: 0.9083372354507446 (0.9253637083222933) + | > loss_1: 31.568815231323242 (30.36386133354401) + | > grad_norm_1: tensor(285.2120, device='cuda:0') (tensor(356.8012, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.8543 (0.9809178281053205) + | > loader_time: 0.0046 (0.006184560116206374) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0337373476762038 (-0.000264681302584141) + | > avg_loss_disc: 2.5886994691995473 (-0.01248931884765625) + | > avg_loss_disc_real_0: 0.28025969633689296 (+0.07121782807203442) + | > avg_loss_disc_real_1: 0.22088804497168615 (-0.0062318490101740676) + | > avg_loss_disc_real_2: 0.220862104342534 (-0.014384286908002974) + | > avg_loss_disc_real_3: 0.235084667801857 (-0.06410477367731243) + | > avg_loss_disc_real_4: 0.22494576527522162 (-0.04320619885738078) + | > avg_loss_disc_real_5: 0.20545794642888582 (-0.038575172424316406) + | > avg_loss_0: 2.5886994691995473 (-0.01248931884765625) + | > avg_loss_gen: 2.449237548387968 (-0.03571378267728331) + | > avg_loss_kl: 3.734673316662128 (-0.07970767754774855) + | > avg_loss_feat: 8.127572499788725 (+0.2789754867553711) + | > avg_loss_mel: 17.222125126765324 (+0.12074932685264983) + | > avg_loss_duration: 1.3833378094893236 (+0.03729451619661783) + | > avg_loss_1: 32.91694582425631 (+0.3215970259446337) + + + > EPOCH: 193/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:49:58)  + + --> TIME: 2024-07-14 02:50:06 -- STEP: 6/151 -- GLOBAL_STEP: 114150 + | > loss_disc: 2.2102930545806885 (2.143575131893158) + | > loss_disc_real_0: 0.07820221781730652 (0.10212949973841508) + | > loss_disc_real_1: 0.20826563239097595 (0.19755718111991882) + | > loss_disc_real_2: 0.165188729763031 (0.19892778992652893) + | > loss_disc_real_3: 0.22193390130996704 (0.1976554567615191) + | > loss_disc_real_4: 0.21604730188846588 (0.18466316163539886) + | > loss_disc_real_5: 0.20212972164154053 (0.20764374236265817) + | > loss_0: 2.2102930545806885 (2.143575131893158) + | > grad_norm_0: tensor(12.0112, device='cuda:0') (tensor(20.9997, device='cuda:0')) + | > loss_gen: 2.7071168422698975 (2.821356018384298) + | > loss_kl: 1.8945331573486328 (1.8291542728741963) + | > loss_feat: 8.617785453796387 (9.111460049947103) + | > loss_mel: 16.769405364990234 (17.128427505493164) + | > loss_duration: 0.9125570058822632 (0.9338197112083435) + | > loss_1: 30.901397705078125 (31.82421652475993) + | > grad_norm_1: tensor(175.7121, device='cuda:0') (tensor(373.8441, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.8747 (0.9020150105158488) + | > loader_time: 0.0089 (0.006718198458353679) + + + --> TIME: 2024-07-14 02:50:57 -- STEP: 56/151 -- GLOBAL_STEP: 114200 + | > loss_disc: 2.428278923034668 (2.263399666973523) + | > loss_disc_real_0: 0.1276417225599289 (0.12053246237337589) + | > loss_disc_real_1: 0.18857137858867645 (0.1964538394074355) + | > loss_disc_real_2: 0.2228781282901764 (0.20740841169442453) + | > loss_disc_real_3: 0.21442493796348572 (0.20073296994503056) + | > loss_disc_real_4: 0.2528857886791229 (0.20484552492520638) + | > loss_disc_real_5: 0.24182960391044617 (0.21449523153049604) + | > loss_0: 2.428278923034668 (2.263399666973523) + | > grad_norm_0: tensor(19.6398, device='cuda:0') (tensor(27.6346, device='cuda:0')) + | > loss_gen: 2.4641568660736084 (2.6772565884249553) + | > loss_kl: 1.692145824432373 (1.699542112648487) + | > loss_feat: 7.3623857498168945 (8.562588691711428) + | > loss_mel: 16.418088912963867 (16.830013615744445) + | > loss_duration: 0.9326608180999756 (0.9389323251588004) + | > loss_1: 28.86943817138672 (30.70833318574088) + | > grad_norm_1: tensor(127.5878, device='cuda:0') (tensor(398.7421, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.9848 (0.9925309887954167) + | > loader_time: 0.0061 (0.00619729927607945) + + + --> TIME: 2024-07-14 02:51:47 -- STEP: 106/151 -- GLOBAL_STEP: 114250 + | > loss_disc: 2.4683849811553955 (2.268775959059877) + | > loss_disc_real_0: 0.16146688163280487 (0.11843789481329468) + | > loss_disc_real_1: 0.20622281730175018 (0.19670742897773688) + | > loss_disc_real_2: 0.2685750722885132 (0.20672186576532872) + | > loss_disc_real_3: 0.27282366156578064 (0.20228659396745124) + | > loss_disc_real_4: 0.16238588094711304 (0.20334220227767835) + | > loss_disc_real_5: 0.2967449128627777 (0.21910211739112745) + | > loss_0: 2.4683849811553955 (2.268775959059877) + | > grad_norm_0: tensor(48.5857, device='cuda:0') (tensor(24.7658, device='cuda:0')) + | > loss_gen: 2.6678614616394043 (2.6673065176549944) + | > loss_kl: 1.9261654615402222 (1.6978826595927186) + | > loss_feat: 6.836719989776611 (8.54183710746045) + | > loss_mel: 17.402639389038086 (16.800874053307293) + | > loss_duration: 0.9215854406356812 (0.9257389790606949) + | > loss_1: 29.75497055053711 (30.63363920967534) + | > grad_norm_1: tensor(641.8387, device='cuda:0') (tensor(368.5071, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.9587 (0.9927269197859854) + | > loader_time: 0.0064 (0.0064118943124447225) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03249835968017578 (-0.001238987996028021) + | > avg_loss_disc: 2.5995751160841722 (+0.010875646884624945) + | > avg_loss_disc_real_0: 0.2804213005762834 (+0.00016160423939043378) + | > avg_loss_disc_real_1: 0.2163117161163917 (-0.004576328855294465) + | > avg_loss_disc_real_2: 0.22420305128280932 (+0.0033409469402753134) + | > avg_loss_disc_real_3: 0.21966373118070456 (-0.015420936621152431) + | > avg_loss_disc_real_4: 0.257366894529416 (+0.03242112925419441) + | > avg_loss_disc_real_5: 0.21639513969421387 (+0.010937193265328049) + | > avg_loss_0: 2.5995751160841722 (+0.010875646884624945) + | > avg_loss_gen: 2.567867242372953 (+0.11862969398498491) + | > avg_loss_kl: 3.907767020739042 (+0.17309370407691382) + | > avg_loss_feat: 8.22127177165105 (+0.09369927186232552) + | > avg_loss_mel: 16.892732620239254 (-0.32939250652606944) + | > avg_loss_duration: 1.3702395329108605 (-0.013098276578463075) + | > avg_loss_1: 32.959878774789665 (+0.04293295053335555) + + + > EPOCH: 194/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:52:39)  + + --> TIME: 2024-07-14 02:52:48 -- STEP: 5/151 -- GLOBAL_STEP: 114300 + | > loss_disc: 2.3846113681793213 (2.2999292373657227) + | > loss_disc_real_0: 0.09080614894628525 (0.11278442293405533) + | > loss_disc_real_1: 0.22764812409877777 (0.20240197479724883) + | > loss_disc_real_2: 0.2546089291572571 (0.22570578753948212) + | > loss_disc_real_3: 0.22034353017807007 (0.210959729552269) + | > loss_disc_real_4: 0.18310077488422394 (0.20477677881717682) + | > loss_disc_real_5: 0.20286564528942108 (0.21918928921222686) + | > loss_0: 2.3846113681793213 (2.2999292373657227) + | > grad_norm_0: tensor(21.7010, device='cuda:0') (tensor(27.1792, device='cuda:0')) + | > loss_gen: 2.4260575771331787 (2.586742877960205) + | > loss_kl: 1.8045865297317505 (1.7893605470657348) + | > loss_feat: 7.0164475440979 (7.979699039459229) + | > loss_mel: 16.397476196289062 (16.48150806427002) + | > loss_duration: 0.8868198394775391 (0.9554015636444092) + | > loss_1: 28.531387329101562 (29.792712020874024) + | > grad_norm_1: tensor(451.7470, device='cuda:0') (tensor(493.2328, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 1.0214 (1.3428752422332764) + | > loader_time: 0.0057 (0.010454463958740234) + + + --> TIME: 2024-07-14 02:53:36 -- STEP: 55/151 -- GLOBAL_STEP: 114350 + | > loss_disc: 2.317147731781006 (2.2915799401023165) + | > loss_disc_real_0: 0.10317382216453552 (0.11462890465151179) + | > loss_disc_real_1: 0.29135629534721375 (0.1998519171367992) + | > loss_disc_real_2: 0.25009673833847046 (0.2086611883206801) + | > loss_disc_real_3: 0.26806119084358215 (0.20439798953858287) + | > loss_disc_real_4: 0.25832799077033997 (0.20409942323511296) + | > loss_disc_real_5: 0.19878830015659332 (0.21923699568618427) + | > loss_0: 2.317147731781006 (2.2915799401023165) + | > grad_norm_0: tensor(15.2736, device='cuda:0') (tensor(20.0504, device='cuda:0')) + | > loss_gen: 2.976971387863159 (2.6185957258397883) + | > loss_kl: 2.156858444213867 (1.7316047939387236) + | > loss_feat: 8.910202026367188 (8.306006128137758) + | > loss_mel: 16.962617874145508 (16.504062115062364) + | > loss_duration: 0.8407089710235596 (0.9260968598452481) + | > loss_1: 31.84735870361328 (30.0863657171076) + | > grad_norm_1: tensor(114.3826, device='cuda:0') (tensor(290.8987, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.8971 (0.9789975209669635) + | > loader_time: 0.006 (0.006316276030106978) + + + --> TIME: 2024-07-14 02:54:24 -- STEP: 105/151 -- GLOBAL_STEP: 114400 + | > loss_disc: 2.2305030822753906 (2.2760179826191496) + | > loss_disc_real_0: 0.0730360671877861 (0.11207548961752938) + | > loss_disc_real_1: 0.17924268543720245 (0.1988163337821052) + | > loss_disc_real_2: 0.2158532738685608 (0.2072462293363752) + | > loss_disc_real_3: 0.22423793375492096 (0.20365187859251385) + | > loss_disc_real_4: 0.19938606023788452 (0.20416820957547144) + | > loss_disc_real_5: 0.2461463361978531 (0.22076282118047988) + | > loss_0: 2.2305030822753906 (2.2760179826191496) + | > grad_norm_0: tensor(23.6890, device='cuda:0') (tensor(21.1051, device='cuda:0')) + | > loss_gen: 2.8268308639526367 (2.628024130775815) + | > loss_kl: 1.9668606519699097 (1.7542145916393825) + | > loss_feat: 8.760567665100098 (8.402942939031686) + | > loss_mel: 17.81391143798828 (16.607462855747762) + | > loss_duration: 0.8383832573890686 (0.9202436895597549) + | > loss_1: 32.2065544128418 (30.312888172694613) + | > grad_norm_1: tensor(436.6185, device='cuda:0') (tensor(302.1167, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.8974 (0.9643437135787237) + | > loader_time: 0.0063 (0.006152157556442987) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03195298635042631 (-0.0005453733297494726) + | > avg_loss_disc: 2.666641033612765 (+0.06706591752859259) + | > avg_loss_disc_real_0: 0.3581281625307523 (+0.07770686195446891) + | > avg_loss_disc_real_1: 0.22281075899417585 (+0.006499042877784161) + | > avg_loss_disc_real_2: 0.21743330588707557 (-0.006769745395733745) + | > avg_loss_disc_real_3: 0.21963305542102227 (-3.067575968229708e-05) + | > avg_loss_disc_real_4: 0.23385131932221925 (-0.02351557520719677) + | > avg_loss_disc_real_5: 0.2600328910809297 (+0.04363775138671583) + | > avg_loss_0: 2.666641033612765 (+0.06706591752859259) + | > avg_loss_gen: 2.5571735272040734 (-0.010693715168879514) + | > avg_loss_kl: 3.951410953815167 (+0.04364393307612513) + | > avg_loss_feat: 8.31620403436514 (+0.0949322627140905) + | > avg_loss_mel: 17.434725027817947 (+0.541992407578693) + | > avg_loss_duration: 1.3804085713166456 (+0.010169038405785091) + | > avg_loss_1: 33.639922215388374 (+0.680043440598709) + + + > EPOCH: 195/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:55:19)  + + --> TIME: 2024-07-14 02:55:25 -- STEP: 4/151 -- GLOBAL_STEP: 114450 + | > loss_disc: 2.5272064208984375 (2.4159273505210876) + | > loss_disc_real_0: 0.0747096985578537 (0.15111125633120537) + | > loss_disc_real_1: 0.18193216621875763 (0.21299982070922852) + | > loss_disc_real_2: 0.20244500041007996 (0.2524770721793175) + | > loss_disc_real_3: 0.23670712113380432 (0.24136899039149284) + | > loss_disc_real_4: 0.28165197372436523 (0.2555212117731571) + | > loss_disc_real_5: 0.24773521721363068 (0.23765596002340317) + | > loss_0: 2.5272064208984375 (2.4159273505210876) + | > grad_norm_0: tensor(29.8815, device='cuda:0') (tensor(42.7540, device='cuda:0')) + | > loss_gen: 2.2775471210479736 (2.602120280265808) + | > loss_kl: 1.6877496242523193 (1.8371731638908386) + | > loss_feat: 7.297959804534912 (7.661354660987854) + | > loss_mel: 17.401521682739258 (17.279263496398926) + | > loss_duration: 0.8639898300170898 (0.926074430346489) + | > loss_1: 29.528766632080078 (30.30598545074463) + | > grad_norm_1: tensor(349.2905, device='cuda:0') (tensor(535.8617, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.9455 (1.087763786315918) + | > loader_time: 0.0049 (0.007546901702880859) + + + --> TIME: 2024-07-14 02:56:12 -- STEP: 54/151 -- GLOBAL_STEP: 114500 + | > loss_disc: 2.219937801361084 (2.2898285344794944) + | > loss_disc_real_0: 0.0865471288561821 (0.1189657776719994) + | > loss_disc_real_1: 0.16903138160705566 (0.20310404924330888) + | > loss_disc_real_2: 0.19734235107898712 (0.21360134029829944) + | > loss_disc_real_3: 0.27372509241104126 (0.2028764577375518) + | > loss_disc_real_4: 0.23188850283622742 (0.20065553983052573) + | > loss_disc_real_5: 0.2142455130815506 (0.22062123070160547) + | > loss_0: 2.219937801361084 (2.2898285344794944) + | > grad_norm_0: tensor(16.4270, device='cuda:0') (tensor(21.8386, device='cuda:0')) + | > loss_gen: 2.849491834640503 (2.665643338803892) + | > loss_kl: 1.5241972208023071 (1.696172261679614) + | > loss_feat: 9.455388069152832 (8.582353079760516) + | > loss_mel: 17.412858963012695 (16.712237587681514) + | > loss_duration: 0.8491409420967102 (0.9080267272613667) + | > loss_1: 32.0910758972168 (30.56443295655427) + | > grad_norm_1: tensor(303.9225, device='cuda:0') (tensor(290.7908, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.8767 (0.9268262562928377) + | > loader_time: 0.0047 (0.005707484704476816) + + + --> TIME: 2024-07-14 02:57:03 -- STEP: 104/151 -- GLOBAL_STEP: 114550 + | > loss_disc: 2.146092176437378 (2.2951107449256445) + | > loss_disc_real_0: 0.047875892370939255 (0.11685174653449884) + | > loss_disc_real_1: 0.19560155272483826 (0.20531082096008152) + | > loss_disc_real_2: 0.19629155099391937 (0.21201739732462627) + | > loss_disc_real_3: 0.19696560502052307 (0.20466503386314094) + | > loss_disc_real_4: 0.16793763637542725 (0.20145310046008) + | > loss_disc_real_5: 0.24918946623802185 (0.2214785679601706) + | > loss_0: 2.146092176437378 (2.2951107449256445) + | > grad_norm_0: tensor(12.8641, device='cuda:0') (tensor(22.7084, device='cuda:0')) + | > loss_gen: 2.645801305770874 (2.6482706872316513) + | > loss_kl: 1.8509620428085327 (1.72055826909267) + | > loss_feat: 9.225916862487793 (8.527381475155172) + | > loss_mel: 17.739521026611328 (16.6949619604991) + | > loss_duration: 0.9220076203346252 (0.922125403697674) + | > loss_1: 32.38420867919922 (30.513297704549935) + | > grad_norm_1: tensor(71.3228, device='cuda:0') (tensor(328.2757, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.8767 (0.9732154103425833) + | > loader_time: 0.0052 (0.006283727975992056) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031938864634587213 (-1.4121715839095217e-05) + | > avg_loss_disc: 2.604333840883695 (-0.062307192729069705) + | > avg_loss_disc_real_0: 0.30609824794989365 (-0.05202991458085865) + | > avg_loss_disc_real_1: 0.22785949133909666 (+0.005048732344920814) + | > avg_loss_disc_real_2: 0.2426755726337433 (+0.025242266746667713) + | > avg_loss_disc_real_3: 0.2524214650575931 (+0.03278840963657084) + | > avg_loss_disc_real_4: 0.24301013006613806 (+0.009158810743918805) + | > avg_loss_disc_real_5: 0.21760533635432905 (-0.04242755472660065) + | > avg_loss_0: 2.604333840883695 (-0.062307192729069705) + | > avg_loss_gen: 2.5403221203730655 (-0.016851406831007942) + | > avg_loss_kl: 3.955873177601741 (+0.0044622237865739045) + | > avg_loss_feat: 8.111267199883095 (-0.20493683448204614) + | > avg_loss_mel: 16.641051292419434 (-0.7936737353985137) + | > avg_loss_duration: 1.3530528178581824 (-0.027355753458463195) + | > avg_loss_1: 32.60156616797814 (-1.0383560474102325) + + + > EPOCH: 196/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 02:58:00)  + + --> TIME: 2024-07-14 02:58:06 -- STEP: 3/151 -- GLOBAL_STEP: 114600 + | > loss_disc: 2.4744086265563965 (2.3840879599253335) + | > loss_disc_real_0: 0.12401368468999863 (0.11224564164876938) + | > loss_disc_real_1: 0.2123248279094696 (0.22574069102605185) + | > loss_disc_real_2: 0.21769163012504578 (0.2240035136540731) + | > loss_disc_real_3: 0.2481997311115265 (0.2207555671532949) + | > loss_disc_real_4: 0.24335148930549622 (0.23068304856618246) + | > loss_disc_real_5: 0.2261458784341812 (0.212713951865832) + | > loss_0: 2.4744086265563965 (2.3840879599253335) + | > grad_norm_0: tensor(14.2395, device='cuda:0') (tensor(15.1470, device='cuda:0')) + | > loss_gen: 2.405363082885742 (2.5084376335144043) + | > loss_kl: 2.1307179927825928 (2.222886562347412) + | > loss_feat: 8.80650520324707 (8.116012414296469) + | > loss_mel: 16.287738800048828 (16.930118560791016) + | > loss_duration: 0.8513966798782349 (0.9438446760177612) + | > loss_1: 30.481721878051758 (30.72129948933919) + | > grad_norm_1: tensor(91.0331, device='cuda:0') (tensor(127.0270, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.9654 (1.09432586034139) + | > loader_time: 0.0051 (0.008714119593302408) + + + --> TIME: 2024-07-14 02:58:55 -- STEP: 53/151 -- GLOBAL_STEP: 114650 + | > loss_disc: 2.488628625869751 (2.296840609244582) + | > loss_disc_real_0: 0.15168651938438416 (0.11155596923715663) + | > loss_disc_real_1: 0.2089989185333252 (0.20111044538470935) + | > loss_disc_real_2: 0.1810130476951599 (0.20815317726360177) + | > loss_disc_real_3: 0.1928783357143402 (0.20884385451955614) + | > loss_disc_real_4: 0.22331209480762482 (0.2079937500211428) + | > loss_disc_real_5: 0.2420850545167923 (0.22930892739655837) + | > loss_0: 2.488628625869751 (2.296840609244582) + | > grad_norm_0: tensor(19.7588, device='cuda:0') (tensor(18.1809, device='cuda:0')) + | > loss_gen: 2.3873472213745117 (2.6178743119509713) + | > loss_kl: 1.3521950244903564 (1.7298008439675816) + | > loss_feat: 6.648374557495117 (8.47254924054416) + | > loss_mel: 14.543527603149414 (16.489525848964472) + | > loss_duration: 0.8427520990371704 (0.9374819251726259) + | > loss_1: 25.774198532104492 (30.247231897318137) + | > grad_norm_1: tensor(147.5797, device='cuda:0') (tensor(268.1790, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.8517 (0.9893703235770172) + | > loader_time: 0.0045 (0.006483042015219634) + + + --> TIME: 2024-07-14 02:59:45 -- STEP: 103/151 -- GLOBAL_STEP: 114700 + | > loss_disc: 2.4895966053009033 (2.3158047291839026) + | > loss_disc_real_0: 0.1422363817691803 (0.11498813182838911) + | > loss_disc_real_1: 0.27917030453681946 (0.20263879406220706) + | > loss_disc_real_2: 0.29601404070854187 (0.21013044963762598) + | > loss_disc_real_3: 0.22800198197364807 (0.20879385728859207) + | > loss_disc_real_4: 0.2258710116147995 (0.20773262581200275) + | > loss_disc_real_5: 0.26199308037757874 (0.22401851777312826) + | > loss_0: 2.4895966053009033 (2.3158047291839026) + | > grad_norm_0: tensor(28.0105, device='cuda:0') (tensor(18.0427, device='cuda:0')) + | > loss_gen: 2.770596981048584 (2.5855141727669726) + | > loss_kl: 2.1220741271972656 (1.7304829839363838) + | > loss_feat: 7.561337947845459 (8.388361129945922) + | > loss_mel: 16.814369201660156 (16.50793721837904) + | > loss_duration: 1.0164499282836914 (0.9287769400957718) + | > loss_1: 30.284828186035156 (30.14107233806721) + | > grad_norm_1: tensor(159.4459, device='cuda:0') (tensor(272.4340, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 1.1452 (0.9814801679074185) + | > loader_time: 0.0075 (0.006351352895347818) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03239275858952449 (+0.0004538939549372789) + | > avg_loss_disc: 2.584816785959097 (-0.01951705492459821) + | > avg_loss_disc_real_0: 0.23538722327122322 (-0.07071102467867044) + | > avg_loss_disc_real_1: 0.20347981842664573 (-0.02437967291245094) + | > avg_loss_disc_real_2: 0.20362540391775277 (-0.03905016871599051) + | > avg_loss_disc_real_3: 0.17578635765956 (-0.07663510739803311) + | > avg_loss_disc_real_4: 0.2067262725188182 (-0.03628385754731986) + | > avg_loss_disc_real_5: 0.178215164404649 (-0.03939017194968006) + | > avg_loss_0: 2.584816785959097 (-0.01951705492459821) + | > avg_loss_gen: 2.156831301175631 (-0.38349081919743444) + | > avg_loss_kl: 3.7635559302109938 (-0.19231724739074707) + | > avg_loss_feat: 8.330374607673058 (+0.21910740778996285) + | > avg_loss_mel: 17.04620464031513 (+0.4051533478956948) + | > avg_loss_duration: 1.3389060588983388 (-0.01414675895984363) + | > avg_loss_1: 32.63587232736441 (+0.034306159386268575) + + + > EPOCH: 197/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:00:42)  + + --> TIME: 2024-07-14 03:00:46 -- STEP: 2/151 -- GLOBAL_STEP: 114750 + | > loss_disc: 2.3761861324310303 (2.3081973791122437) + | > loss_disc_real_0: 0.2473156452178955 (0.18501023575663567) + | > loss_disc_real_1: 0.23951521515846252 (0.19792763888835907) + | > loss_disc_real_2: 0.2342711091041565 (0.17802836373448372) + | > loss_disc_real_3: 0.25177279114723206 (0.1919771134853363) + | > loss_disc_real_4: 0.2379537969827652 (0.18354074656963348) + | > loss_disc_real_5: 0.26043644547462463 (0.20575637370347977) + | > loss_0: 2.3761861324310303 (2.3081973791122437) + | > grad_norm_0: tensor(21.8777, device='cuda:0') (tensor(15.1028, device='cuda:0')) + | > loss_gen: 3.0606000423431396 (2.8338656425476074) + | > loss_kl: 2.073861837387085 (1.625447154045105) + | > loss_feat: 8.222034454345703 (8.750253200531006) + | > loss_mel: 16.926172256469727 (16.37436008453369) + | > loss_duration: 0.8766311407089233 (0.9993111491203308) + | > loss_1: 31.159299850463867 (30.583237648010254) + | > grad_norm_1: tensor(79.2183, device='cuda:0') (tensor(113.8932, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.9999 (0.9338482618331909) + | > loader_time: 0.0077 (0.006219148635864258) + + + --> TIME: 2024-07-14 03:01:36 -- STEP: 52/151 -- GLOBAL_STEP: 114800 + | > loss_disc: 2.267503023147583 (2.339584832008068) + | > loss_disc_real_0: 0.08710946887731552 (0.1237208022950934) + | > loss_disc_real_1: 0.19651958346366882 (0.20323030679271772) + | > loss_disc_real_2: 0.19564755260944366 (0.21367475318794066) + | > loss_disc_real_3: 0.1762407124042511 (0.20786502298254234) + | > loss_disc_real_4: 0.20283520221710205 (0.20728153305558059) + | > loss_disc_real_5: 0.21992237865924835 (0.22893295637690103) + | > loss_0: 2.267503023147583 (2.339584832008068) + | > grad_norm_0: tensor(36.3767, device='cuda:0') (tensor(24.4619, device='cuda:0')) + | > loss_gen: 2.4885802268981934 (2.5819968581199646) + | > loss_kl: 2.105768918991089 (1.723530846146437) + | > loss_feat: 7.1557393074035645 (8.232499526097227) + | > loss_mel: 16.75716209411621 (16.733707886475784) + | > loss_duration: 0.9446561336517334 (0.9441094685059327) + | > loss_1: 29.451906204223633 (30.215844521155724) + | > grad_norm_1: tensor(345.5208, device='cuda:0') (tensor(341.7210, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 1.0928 (0.9859639406204224) + | > loader_time: 0.0082 (0.006312860892369197) + + + --> TIME: 2024-07-14 03:02:24 -- STEP: 102/151 -- GLOBAL_STEP: 114850 + | > loss_disc: 2.0674822330474854 (2.321940849809086) + | > loss_disc_real_0: 0.06130772829055786 (0.1189346090689594) + | > loss_disc_real_1: 0.15160183608531952 (0.2020593183297737) + | > loss_disc_real_2: 0.15033330023288727 (0.2103830154473875) + | > loss_disc_real_3: 0.14940078556537628 (0.20796174176183402) + | > loss_disc_real_4: 0.18230928480625153 (0.20970717905198827) + | > loss_disc_real_5: 0.18732674419879913 (0.22605784295820722) + | > loss_0: 2.0674822330474854 (2.321940849809086) + | > grad_norm_0: tensor(36.5241, device='cuda:0') (tensor(23.5412, device='cuda:0')) + | > loss_gen: 2.7846181392669678 (2.5959548809949085) + | > loss_kl: 1.1505998373031616 (1.7207959434565376) + | > loss_feat: 9.405189514160156 (8.341965137743484) + | > loss_mel: 16.239925384521484 (16.584424766839707) + | > loss_duration: 1.1345577239990234 (0.9336186296799603) + | > loss_1: 30.71489143371582 (30.176759458055685) + | > grad_norm_1: tensor(667.8793, device='cuda:0') (tensor(344.9613, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.8098 (0.9653092038397696) + | > loader_time: 0.0041 (0.006222474808786427) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03427035991962139 (+0.0018776013300969008) + | > avg_loss_disc: 2.609948213283832 (+0.025131427324735167) + | > avg_loss_disc_real_0: 0.27405264056645906 (+0.03866541729523584) + | > avg_loss_disc_real_1: 0.21729730298885933 (+0.0138174845622136) + | > avg_loss_disc_real_2: 0.19773507347473732 (-0.005890330443015457) + | > avg_loss_disc_real_3: 0.23971160902426794 (+0.06392525136470795) + | > avg_loss_disc_real_4: 0.2534991686160748 (+0.0467728960972566) + | > avg_loss_disc_real_5: 0.2391581008067498 (+0.0609429364021008) + | > avg_loss_0: 2.609948213283832 (+0.025131427324735167) + | > avg_loss_gen: 2.4719652762779822 (+0.3151339751023512) + | > avg_loss_kl: 3.8968762434445896 (+0.13332031323359583) + | > avg_loss_feat: 8.164941971118633 (-0.1654326365544243) + | > avg_loss_mel: 17.256330856910118 (+0.21012621659498976) + | > avg_loss_duration: 1.3436898451585035 (+0.004783786260164735) + | > avg_loss_1: 33.133804027850815 (+0.49793170048640434) + + + > EPOCH: 198/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:03:23)  + + --> TIME: 2024-07-14 03:03:26 -- STEP: 1/151 -- GLOBAL_STEP: 114900 + | > loss_disc: 2.3118743896484375 (2.3118743896484375) + | > loss_disc_real_0: 0.12066392600536346 (0.12066392600536346) + | > loss_disc_real_1: 0.181849405169487 (0.181849405169487) + | > loss_disc_real_2: 0.17740286886692047 (0.17740286886692047) + | > loss_disc_real_3: 0.2016546130180359 (0.2016546130180359) + | > loss_disc_real_4: 0.2145252525806427 (0.2145252525806427) + | > loss_disc_real_5: 0.2228742092847824 (0.2228742092847824) + | > loss_0: 2.3118743896484375 (2.3118743896484375) + | > grad_norm_0: tensor(16.2557, device='cuda:0') (tensor(16.2557, device='cuda:0')) + | > loss_gen: 2.5168297290802 (2.5168297290802) + | > loss_kl: 1.707512617111206 (1.707512617111206) + | > loss_feat: 7.902730941772461 (7.902730941772461) + | > loss_mel: 16.220426559448242 (16.220426559448242) + | > loss_duration: 0.8723589181900024 (0.8723589181900024) + | > loss_1: 29.219858169555664 (29.219858169555664) + | > grad_norm_1: tensor(528.6561, device='cuda:0') (tensor(528.6561, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.9279 (0.9278652667999268) + | > loader_time: 0.0101 (0.010092973709106445) + + + --> TIME: 2024-07-14 03:04:17 -- STEP: 51/151 -- GLOBAL_STEP: 114950 + | > loss_disc: 2.27114200592041 (2.3018568543826827) + | > loss_disc_real_0: 0.12765878438949585 (0.11242933176896151) + | > loss_disc_real_1: 0.2239731401205063 (0.20260088642438254) + | > loss_disc_real_2: 0.20746839046478271 (0.21397319699034972) + | > loss_disc_real_3: 0.2010839879512787 (0.20844152978822295) + | > loss_disc_real_4: 0.2843617796897888 (0.21157707493094838) + | > loss_disc_real_5: 0.20185764133930206 (0.22291470304423688) + | > loss_0: 2.27114200592041 (2.3018568543826827) + | > grad_norm_0: tensor(20.5447, device='cuda:0') (tensor(23.6283, device='cuda:0')) + | > loss_gen: 2.7856509685516357 (2.6571329691830803) + | > loss_kl: 1.7517200708389282 (1.7422984043757122) + | > loss_feat: 9.57412052154541 (8.560688140345553) + | > loss_mel: 17.291763305664062 (16.748634506674374) + | > loss_duration: 0.8148568868637085 (0.9122871172194388) + | > loss_1: 32.218109130859375 (30.62104109221814) + | > grad_norm_1: tensor(332.3958, device='cuda:0') (tensor(391.4199, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.8656 (1.0007367648330385) + | > loader_time: 0.0046 (0.006674069984286439) + + + --> TIME: 2024-07-14 03:05:04 -- STEP: 101/151 -- GLOBAL_STEP: 115000 + | > loss_disc: 2.095337152481079 (2.299075695547726) + | > loss_disc_real_0: 0.09520260989665985 (0.11259910571250585) + | > loss_disc_real_1: 0.21364349126815796 (0.20292254913561414) + | > loss_disc_real_2: 0.203091561794281 (0.21337019541476032) + | > loss_disc_real_3: 0.19753587245941162 (0.20859168292862354) + | > loss_disc_real_4: 0.12488875538110733 (0.2076802177004295) + | > loss_disc_real_5: 0.19805648922920227 (0.22065272337139244) + | > loss_0: 2.095337152481079 (2.299075695547726) + | > grad_norm_0: tensor(29.9457, device='cuda:0') (tensor(22.9494, device='cuda:0')) + | > loss_gen: 2.803736686706543 (2.6376378311969253) + | > loss_kl: 1.6106659173965454 (1.743100125010651) + | > loss_feat: 9.916729927062988 (8.54090584858809) + | > loss_mel: 17.525007247924805 (16.578299144707096) + | > loss_duration: 0.860275387763977 (0.922619637876454) + | > loss_1: 32.71641540527344 (30.422562589739808) + | > grad_norm_1: tensor(489.8272, device='cuda:0') (tensor(390.3357, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.8759 (0.9715503442405471) + | > loader_time: 0.0065 (0.006400105976822353) + + + > CHECKPOINT : /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a/checkpoint_115000.pth + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03651648301344652 (+0.002246123093825124) + | > avg_loss_disc: 2.6137020404522238 (+0.003753827168391677) + | > avg_loss_disc_real_0: 0.3063129713902107 (+0.032260330823751626) + | > avg_loss_disc_real_1: 0.2135944779102619 (-0.003702825078597427) + | > avg_loss_disc_real_2: 0.2904067841859964 (+0.0926717107112591) + | > avg_loss_disc_real_3: 0.3175777426132789 (+0.07786613358901098) + | > avg_loss_disc_real_4: 0.2980041859241632 (+0.044505017308088424) + | > avg_loss_disc_real_5: 0.19777033879206732 (-0.04138776201468247) + | > avg_loss_0: 2.6137020404522238 (+0.003753827168391677) + | > avg_loss_gen: 2.8170719696925235 (+0.34510669341454125) + | > avg_loss_kl: 3.9990569078005276 (+0.10218066435593798) + | > avg_loss_feat: 8.042073543255146 (-0.12286842786348728) + | > avg_loss_mel: 17.00066903921274 (-0.2556618176973764) + | > avg_loss_duration: 1.3705729979735155 (+0.026883152815011968) + | > avg_loss_1: 33.2294440636268 (+0.09564003577598612) + + + > EPOCH: 199/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:06:34)  + + --> TIME: 2024-07-14 03:06:36 -- STEP: 0/151 -- GLOBAL_STEP: 115050 + | > loss_disc: 2.1901021003723145 (2.1901021003723145) + | > loss_disc_real_0: 0.1053585410118103 (0.1053585410118103) + | > loss_disc_real_1: 0.1695149689912796 (0.1695149689912796) + | > loss_disc_real_2: 0.22405046224594116 (0.22405046224594116) + | > loss_disc_real_3: 0.30734172463417053 (0.30734172463417053) + | > loss_disc_real_4: 0.22874392569065094 (0.22874392569065094) + | > loss_disc_real_5: 0.18695451319217682 (0.18695451319217682) + | > loss_0: 2.1901021003723145 (2.1901021003723145) + | > grad_norm_0: tensor(36.9545, device='cuda:0') (tensor(36.9545, device='cuda:0')) + | > loss_gen: 2.921330451965332 (2.921330451965332) + | > loss_kl: 1.515198826789856 (1.515198826789856) + | > loss_feat: 9.415578842163086 (9.415578842163086) + | > loss_mel: 17.22412872314453 (17.22412872314453) + | > loss_duration: 0.8561736345291138 (0.8561736345291138) + | > loss_1: 31.932411193847656 (31.932411193847656) + | > grad_norm_1: tensor(438.7794, device='cuda:0') (tensor(438.7794, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 1.0631 (1.0630605220794678) + | > loader_time: 1.163 (1.1630406379699707) + + + --> TIME: 2024-07-14 03:07:27 -- STEP: 50/151 -- GLOBAL_STEP: 115100 + | > loss_disc: 2.4717910289764404 (2.2791142416000363) + | > loss_disc_real_0: 0.19394032657146454 (0.10743825443089008) + | > loss_disc_real_1: 0.24626225233078003 (0.20160066708922386) + | > loss_disc_real_2: 0.2930724024772644 (0.20634535133838652) + | > loss_disc_real_3: 0.281555712223053 (0.2044232365489006) + | > loss_disc_real_4: 0.19941966235637665 (0.21059895634651185) + | > loss_disc_real_5: 0.2833006680011749 (0.22281405955553055) + | > loss_0: 2.4717910289764404 (2.2791142416000363) + | > grad_norm_0: tensor(25.2721, device='cuda:0') (tensor(19.8913, device='cuda:0')) + | > loss_gen: 2.5951428413391113 (2.641552500724793) + | > loss_kl: 1.923478603363037 (1.7721198618412017) + | > loss_feat: 7.125554084777832 (8.578052129745487) + | > loss_mel: 16.627559661865234 (16.573046188354496) + | > loss_duration: 0.9048860669136047 (0.9399964892864228) + | > loss_1: 29.176620483398438 (30.50476718902588) + | > grad_norm_1: tensor(236.3494, device='cuda:0') (tensor(317.5033, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.884 (0.9966937875747679) + | > loader_time: 0.005 (0.006353726387023926) + + + --> TIME: 2024-07-14 03:08:17 -- STEP: 100/151 -- GLOBAL_STEP: 115150 + | > loss_disc: 2.189690589904785 (2.280568276643754) + | > loss_disc_real_0: 0.11028727144002914 (0.10761824034154416) + | > loss_disc_real_1: 0.1921699196100235 (0.20206025414168835) + | > loss_disc_real_2: 0.16663198173046112 (0.20700670674443244) + | > loss_disc_real_3: 0.18772730231285095 (0.2044736462831497) + | > loss_disc_real_4: 0.19266726076602936 (0.21026303261518478) + | > loss_disc_real_5: 0.23457258939743042 (0.22278553858399391) + | > loss_0: 2.189690589904785 (2.280568276643754) + | > grad_norm_0: tensor(36.4173, device='cuda:0') (tensor(24.9939, device='cuda:0')) + | > loss_gen: 2.6652636528015137 (2.649296066761016) + | > loss_kl: 1.3133047819137573 (1.7849100416898727) + | > loss_feat: 8.18205451965332 (8.541878528594973) + | > loss_mel: 15.044867515563965 (16.570829677581784) + | > loss_duration: 1.1978569030761719 (0.9370553088188172) + | > loss_1: 28.40334701538086 (30.48396966934204) + | > grad_norm_1: tensor(580.3722, device='cuda:0') (tensor(373.4617, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.8165 (0.9960351228713991) + | > loader_time: 0.0043 (0.006452131271362305) + + + --> TIME: 2024-07-14 03:09:07 -- STEP: 150/151 -- GLOBAL_STEP: 115200 + | > loss_disc: 2.1577465534210205 (2.2906763672828676) + | > loss_disc_real_0: 0.07835951447486877 (0.11104826370875041) + | > loss_disc_real_1: 0.1833418905735016 (0.20360492333769795) + | > loss_disc_real_2: 0.2201155126094818 (0.2075804455081622) + | > loss_disc_real_3: 0.2472635954618454 (0.20661067992448806) + | > loss_disc_real_4: 0.195761576294899 (0.20801751444737115) + | > loss_disc_real_5: 0.19558808207511902 (0.22237769812345504) + | > loss_0: 2.1577465534210205 (2.2906763672828676) + | > grad_norm_0: tensor(16.6730, device='cuda:0') (tensor(22.7096, device='cuda:0')) + | > loss_gen: 2.9903926849365234 (2.6325086959203072) + | > loss_kl: 1.5556424856185913 (1.7646705643335978) + | > loss_feat: 9.460153579711914 (8.473945109049485) + | > loss_mel: 15.941197395324707 (16.553306058247887) + | > loss_duration: 0.8900861144065857 (0.9318377820650736) + | > loss_1: 30.83747100830078 (30.356268259684246) + | > grad_norm_1: tensor(561.6707, device='cuda:0') (tensor(339.8963, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.8219 (0.9947318442662557) + | > loader_time: 0.0038 (0.006372882525126138) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03542989950913649 (-0.0010865835043100264) + | > avg_loss_disc: 2.5705510652982273 (-0.04315097515399646) + | > avg_loss_disc_real_0: 0.2981514667089169 (-0.008161504681293785) + | > avg_loss_disc_real_1: 0.1763661836202328 (-0.03722829429002911) + | > avg_loss_disc_real_2: 0.2617260871025232 (-0.028680697083473206) + | > avg_loss_disc_real_3: 0.25514441384718967 (-0.06243332876608926) + | > avg_loss_disc_real_4: 0.26639217940660614 (-0.03161200651755708) + | > avg_loss_disc_real_5: 0.22575016205127424 (+0.02797982325920692) + | > avg_loss_0: 2.5705510652982273 (-0.04315097515399646) + | > avg_loss_gen: 2.6330278469966006 (-0.18404412269592285) + | > avg_loss_kl: 3.964796231343196 (-0.03426067645733166) + | > avg_loss_feat: 8.248036127824049 (+0.20596258456890304) + | > avg_loss_mel: 16.614267495962288 (-0.3864015432504537) + | > avg_loss_duration: 1.3697371391149669 (-0.0008358588585486437) + | > avg_loss_1: 32.82986508882963 (-0.39957897479717275) + + + > EPOCH: 200/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:09:18)  + + --> TIME: 2024-07-14 03:10:08 -- STEP: 49/151 -- GLOBAL_STEP: 115250 + | > loss_disc: 2.515821695327759 (2.295128355220873) + | > loss_disc_real_0: 0.1868760883808136 (0.1171231109420864) + | > loss_disc_real_1: 0.19066111743450165 (0.20502151427220325) + | > loss_disc_real_2: 0.18127429485321045 (0.20824197086752677) + | > loss_disc_real_3: 0.24339568614959717 (0.2105339297226497) + | > loss_disc_real_4: 0.25432634353637695 (0.2065989445058667) + | > loss_disc_real_5: 0.2759900689125061 (0.22149611675009437) + | > loss_0: 2.515821695327759 (2.295128355220873) + | > grad_norm_0: tensor(40.7138, device='cuda:0') (tensor(17.9633, device='cuda:0')) + | > loss_gen: 2.3972573280334473 (2.61725098259595) + | > loss_kl: 1.7690716981887817 (1.7389911218565337) + | > loss_feat: 6.761438369750977 (8.513259099454295) + | > loss_mel: 14.503000259399414 (16.468763156813015) + | > loss_duration: 0.8602550029754639 (0.9079116777497895) + | > loss_1: 26.2910213470459 (30.246176155246033) + | > grad_norm_1: tensor(403.0656, device='cuda:0') (tensor(272.6598, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.997 (0.9497496351903799) + | > loader_time: 0.0061 (0.005886326030809052) + + + --> TIME: 2024-07-14 03:10:59 -- STEP: 99/151 -- GLOBAL_STEP: 115300 + | > loss_disc: 2.327526092529297 (2.3144589000278044) + | > loss_disc_real_0: 0.11848853528499603 (0.11719356695509921) + | > loss_disc_real_1: 0.24882054328918457 (0.2052665798351018) + | > loss_disc_real_2: 0.22233793139457703 (0.20986881776891572) + | > loss_disc_real_3: 0.24626603722572327 (0.20801690657331487) + | > loss_disc_real_4: 0.2657000720500946 (0.2071798919427274) + | > loss_disc_real_5: 0.2218565046787262 (0.2261553823646873) + | > loss_0: 2.327526092529297 (2.3144589000278044) + | > grad_norm_0: tensor(19.8001, device='cuda:0') (tensor(22.2153, device='cuda:0')) + | > loss_gen: 2.7824478149414062 (2.6132740878095526) + | > loss_kl: 1.7117481231689453 (1.7514415365276914) + | > loss_feat: 8.085189819335938 (8.41338996694546) + | > loss_mel: 16.374244689941406 (16.46710666502365) + | > loss_duration: 0.9388964176177979 (0.922465119699035) + | > loss_1: 29.892526626586914 (30.16767736878058) + | > grad_norm_1: tensor(205.4438, device='cuda:0') (tensor(348.5399, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.9033 (0.9845577297788678) + | > loader_time: 0.0062 (0.006399896409776475) + + + --> TIME: 2024-07-14 03:11:48 -- STEP: 149/151 -- GLOBAL_STEP: 115350 + | > loss_disc: 2.234623908996582 (2.303572992350432) + | > loss_disc_real_0: 0.13606321811676025 (0.11481932058130334) + | > loss_disc_real_1: 0.19392971694469452 (0.203692828348819) + | > loss_disc_real_2: 0.18967647850513458 (0.20904090610526554) + | > loss_disc_real_3: 0.19857211410999298 (0.20649361010365844) + | > loss_disc_real_4: 0.20380885899066925 (0.20723181042895222) + | > loss_disc_real_5: 0.2578354477882385 (0.22514202940784045) + | > loss_0: 2.234623908996582 (2.303572992350432) + | > grad_norm_0: tensor(31.3560, device='cuda:0') (tensor(24.5530, device='cuda:0')) + | > loss_gen: 2.9615890979766846 (2.6231550802320447) + | > loss_kl: 1.1004310846328735 (1.7384406472212517) + | > loss_feat: 9.160879135131836 (8.459878591883108) + | > loss_mel: 16.576736450195312 (16.487313014548896) + | > loss_duration: 0.9292270541191101 (0.9233675111060174) + | > loss_1: 30.728864669799805 (30.23215483339041) + | > grad_norm_1: tensor(331.6307, device='cuda:0') (tensor(379.4110, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.864 (0.9753787629556336) + | > loader_time: 0.0043 (0.00618536200299359) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03321629304152269 (-0.0022136064676138018) + | > avg_loss_disc: 2.5789408133580136 (+0.008389748059786317) + | > avg_loss_disc_real_0: 0.2527600652896441 (-0.04539140141927278) + | > avg_loss_disc_real_1: 0.27046035803281343 (+0.09409417441258064) + | > avg_loss_disc_real_2: 0.2533022715495183 (-0.00842381555300492) + | > avg_loss_disc_real_3: 0.2671531140804291 (+0.012008700233239411) + | > avg_loss_disc_real_4: 0.24201849905344155 (-0.02437368035316459) + | > avg_loss_disc_real_5: 0.20136341681847206 (-0.02438674523280218) + | > avg_loss_0: 2.5789408133580136 (+0.008389748059786317) + | > avg_loss_gen: 2.6508382100325365 (+0.01781036303593586) + | > avg_loss_kl: 3.8428031297830434 (-0.12199310156015253) + | > avg_loss_feat: 8.212376191065864 (-0.035659936758184685) + | > avg_loss_mel: 16.71092458871695 (+0.09665709275466128) + | > avg_loss_duration: 1.346171314899738 (-0.023565824215228792) + | > avg_loss_1: 32.763113462007965 (-0.06675162682166302) + + + > EPOCH: 201/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:11:59)  + + --> TIME: 2024-07-14 03:12:50 -- STEP: 48/151 -- GLOBAL_STEP: 115400 + | > loss_disc: 2.134016752243042 (2.2702607015768685) + | > loss_disc_real_0: 0.09571094810962677 (0.10697081320298214) + | > loss_disc_real_1: 0.15719108283519745 (0.20059344545006752) + | > loss_disc_real_2: 0.1969238817691803 (0.20961919597660503) + | > loss_disc_real_3: 0.1789836883544922 (0.20787873677909374) + | > loss_disc_real_4: 0.2092212587594986 (0.20237217377871275) + | > loss_disc_real_5: 0.227729931473732 (0.22460056530932584) + | > loss_0: 2.134016752243042 (2.2702607015768685) + | > grad_norm_0: tensor(37.8736, device='cuda:0') (tensor(29.4765, device='cuda:0')) + | > loss_gen: 2.676295757293701 (2.674525464574496) + | > loss_kl: 1.5596588850021362 (1.7479980116089184) + | > loss_feat: 9.627699851989746 (8.50677807132403) + | > loss_mel: 16.316999435424805 (16.80261619885762) + | > loss_duration: 0.9742550253868103 (0.9165160382787386) + | > loss_1: 31.154909133911133 (30.648433883984882) + | > grad_norm_1: tensor(557.6309, device='cuda:0') (tensor(507.4344, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.8514 (1.0071286360422769) + | > loader_time: 0.0047 (0.006507501006126404) + + + --> TIME: 2024-07-14 03:13:39 -- STEP: 98/151 -- GLOBAL_STEP: 115450 + | > loss_disc: 2.486271858215332 (2.293806336363967) + | > loss_disc_real_0: 0.16231685876846313 (0.11159044610602516) + | > loss_disc_real_1: 0.21292409300804138 (0.19965629507692492) + | > loss_disc_real_2: 0.2561660408973694 (0.208708044597689) + | > loss_disc_real_3: 0.253451943397522 (0.2094247953928247) + | > loss_disc_real_4: 0.20387879014015198 (0.20580409847351974) + | > loss_disc_real_5: 0.23436792194843292 (0.22578055365961425) + | > loss_0: 2.486271858215332 (2.293806336363967) + | > grad_norm_0: tensor(31.1705, device='cuda:0') (tensor(30.2467, device='cuda:0')) + | > loss_gen: 2.3420724868774414 (2.635450932444358) + | > loss_kl: 2.4255011081695557 (1.7785136541541742) + | > loss_feat: 6.383927345275879 (8.352951186043876) + | > loss_mel: 15.086915969848633 (16.705056521357324) + | > loss_duration: 1.1151611804962158 (0.9179967471531459) + | > loss_1: 27.35357666015625 (30.389969086160466) + | > grad_norm_1: tensor(430.2294, device='cuda:0') (tensor(505.3835, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 1.2562 (0.9824739937879599) + | > loader_time: 0.0086 (0.006291754391728615) + + + --> TIME: 2024-07-14 03:14:29 -- STEP: 148/151 -- GLOBAL_STEP: 115500 + | > loss_disc: 2.182786464691162 (2.301524668126492) + | > loss_disc_real_0: 0.10081585496664047 (0.11476726346724742) + | > loss_disc_real_1: 0.23116524517536163 (0.20066775935324463) + | > loss_disc_real_2: 0.2042258232831955 (0.20889602284375072) + | > loss_disc_real_3: 0.22220827639102936 (0.20861278333373975) + | > loss_disc_real_4: 0.23831404745578766 (0.20736859477049602) + | > loss_disc_real_5: 0.1861037313938141 (0.22515215060195407) + | > loss_0: 2.182786464691162 (2.301524668126492) + | > grad_norm_0: tensor(45.2155, device='cuda:0') (tensor(31.1825, device='cuda:0')) + | > loss_gen: 2.6463205814361572 (2.631079694709262) + | > loss_kl: 1.8230410814285278 (1.7617869570448588) + | > loss_feat: 7.803673267364502 (8.348622940682079) + | > loss_mel: 15.906553268432617 (16.718172447101495) + | > loss_duration: 0.9265978932380676 (0.9168038932052819) + | > loss_1: 29.106185913085938 (30.376465939186716) + | > grad_norm_1: tensor(669.7562, device='cuda:0') (tensor(506.6445, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 1.0062 (0.9827016592025755) + | > loader_time: 0.0064 (0.00616477631233834) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03296885123619667 (-0.00024744180532602217) + | > avg_loss_disc: 2.525042955691998 (-0.053897857666015625) + | > avg_loss_disc_real_0: 0.23346027502646813 (-0.019299790263175992) + | > avg_loss_disc_real_1: 0.1915592195896002 (-0.07890113844321323) + | > avg_loss_disc_real_2: 0.22359343216969416 (-0.02970883937982413) + | > avg_loss_disc_real_3: 0.18676882065259492 (-0.08038429342783415) + | > avg_loss_disc_real_4: 0.24332942756322715 (+0.0013109285097855916) + | > avg_loss_disc_real_5: 0.20204087862601647 (+0.0006774618075444105) + | > avg_loss_0: 2.525042955691998 (-0.053897857666015625) + | > avg_loss_gen: 2.2796796835385837 (-0.37115852649395276) + | > avg_loss_kl: 3.816836503835825 (-0.025966625947218436) + | > avg_loss_feat: 7.990402698516846 (-0.22197349254901866) + | > avg_loss_mel: 16.60513870532696 (-0.10578588338998784) + | > avg_loss_duration: 1.3746588688630323 (+0.028487553963294276) + | > avg_loss_1: 32.066717001108024 (-0.6963964608999405) + + + > EPOCH: 202/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:14:42)  + + --> TIME: 2024-07-14 03:15:33 -- STEP: 47/151 -- GLOBAL_STEP: 115550 + | > loss_disc: 2.401961326599121 (2.311493670686762) + | > loss_disc_real_0: 0.10256563127040863 (0.11111158948946506) + | > loss_disc_real_1: 0.21088853478431702 (0.20608211831843598) + | > loss_disc_real_2: 0.18461021780967712 (0.20752265985975873) + | > loss_disc_real_3: 0.1631043702363968 (0.2121996575213493) + | > loss_disc_real_4: 0.2134958952665329 (0.2097004620318717) + | > loss_disc_real_5: 0.2251877635717392 (0.22412649557945577) + | > loss_0: 2.401961326599121 (2.311493670686762) + | > grad_norm_0: tensor(20.7187, device='cuda:0') (tensor(27.4604, device='cuda:0')) + | > loss_gen: 2.343519926071167 (2.6031263229694774) + | > loss_kl: 1.1787325143814087 (1.7185991951759825) + | > loss_feat: 7.627691268920898 (8.385995611231376) + | > loss_mel: 16.78495979309082 (16.700560042198667) + | > loss_duration: 0.86208176612854 (0.9005493014416797) + | > loss_1: 28.796985626220703 (30.30883013948481) + | > grad_norm_1: tensor(256.1265, device='cuda:0') (tensor(430.4165, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.8855 (0.9844823035788027) + | > loader_time: 0.0066 (0.006247114627919299) + + + --> TIME: 2024-07-14 03:16:21 -- STEP: 97/151 -- GLOBAL_STEP: 115600 + | > loss_disc: 2.3663859367370605 (2.2965427624810597) + | > loss_disc_real_0: 0.06293603777885437 (0.11171607550272007) + | > loss_disc_real_1: 0.19124287366867065 (0.2043967517380862) + | > loss_disc_real_2: 0.18431952595710754 (0.20898643673695239) + | > loss_disc_real_3: 0.24083712697029114 (0.20949177413257128) + | > loss_disc_real_4: 0.25625884532928467 (0.20674751560712595) + | > loss_disc_real_5: 0.23868130147457123 (0.22335158150220655) + | > loss_0: 2.3663859367370605 (2.2965427624810597) + | > grad_norm_0: tensor(24.6932, device='cuda:0') (tensor(25.0621, device='cuda:0')) + | > loss_gen: 2.48934268951416 (2.6141818774115184) + | > loss_kl: 1.5503599643707275 (1.7005493032563592) + | > loss_feat: 7.89035701751709 (8.423414957892033) + | > loss_mel: 17.158445358276367 (16.52123327353565) + | > loss_duration: 0.9004627466201782 (0.9144237195093607) + | > loss_1: 29.988967895507812 (30.173802739566135) + | > grad_norm_1: tensor(508.9483, device='cuda:0') (tensor(400.6810, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.901 (0.9671144288839754) + | > loader_time: 0.0056 (0.006333235612849598) + + + --> TIME: 2024-07-14 03:17:10 -- STEP: 147/151 -- GLOBAL_STEP: 115650 + | > loss_disc: 2.5192039012908936 (2.2956851518073047) + | > loss_disc_real_0: 0.13264405727386475 (0.11321421400928984) + | > loss_disc_real_1: 0.24738545715808868 (0.20410144754818507) + | > loss_disc_real_2: 0.2504394054412842 (0.2085286845155314) + | > loss_disc_real_3: 0.23433758318424225 (0.2079780117995074) + | > loss_disc_real_4: 0.2151971310377121 (0.20735302871587322) + | > loss_disc_real_5: 0.20362289249897003 (0.22283447275356372) + | > loss_0: 2.5192039012908936 (2.2956851518073047) + | > grad_norm_0: tensor(108.6449, device='cuda:0') (tensor(25.2630, device='cuda:0')) + | > loss_gen: 2.20456862449646 (2.614226475864852) + | > loss_kl: 2.1951141357421875 (1.7093064922864745) + | > loss_feat: 5.570214748382568 (8.36162018613751) + | > loss_mel: 13.684441566467285 (16.438997041611437) + | > loss_duration: 1.1821191310882568 (0.9147267313230605) + | > loss_1: 24.836458206176758 (30.038876630822006) + | > grad_norm_1: tensor(558.2408, device='cuda:0') (tensor(409.1814, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 2.1613 (0.9682485230114996) + | > loader_time: 0.0167 (0.006215204186990958) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0331427134000338 (+0.00017386216383713515) + | > avg_loss_disc: 2.543609930918767 (+0.018566975226768978) + | > avg_loss_disc_real_0: 0.31437252920407516 (+0.08091225417760703) + | > avg_loss_disc_real_1: 0.18380054487631872 (-0.007758674713281483) + | > avg_loss_disc_real_2: 0.2327068276130236 (+0.009113395443329453) + | > avg_loss_disc_real_3: 0.265708320415937 (+0.07893949976334208) + | > avg_loss_disc_real_4: 0.25662753329827237 (+0.013298105735045224) + | > avg_loss_disc_real_5: 0.243980633524748 (+0.04193975489873153) + | > avg_loss_0: 2.543609930918767 (+0.018566975226768978) + | > avg_loss_gen: 2.7344205379486084 (+0.45474085441002465) + | > avg_loss_kl: 3.864705434212318 (+0.04786893037649298) + | > avg_loss_feat: 8.285853459284855 (+0.29545076076800925) + | > avg_loss_mel: 17.407492270836464 (+0.8023535655095024) + | > avg_loss_duration: 1.367718531535222 (-0.0069403373278102976) + | > avg_loss_1: 33.66018999539889 (+1.5934729942908632) + + + > EPOCH: 203/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:17:24)  + + --> TIME: 2024-07-14 03:18:10 -- STEP: 46/151 -- GLOBAL_STEP: 115700 + | > loss_disc: 2.1225855350494385 (2.29422894768093) + | > loss_disc_real_0: 0.10398425906896591 (0.11083336976235328) + | > loss_disc_real_1: 0.1628948301076889 (0.19971197454825693) + | > loss_disc_real_2: 0.23281478881835938 (0.20948377672744833) + | > loss_disc_real_3: 0.214432492852211 (0.2040884621117426) + | > loss_disc_real_4: 0.13733910024166107 (0.2017128075594487) + | > loss_disc_real_5: 0.26218971610069275 (0.22734051165373428) + | > loss_0: 2.1225855350494385 (2.29422894768093) + | > grad_norm_0: tensor(16.7477, device='cuda:0') (tensor(17.0004, device='cuda:0')) + | > loss_gen: 2.604964256286621 (2.6165692391602886) + | > loss_kl: 2.198852777481079 (1.7223463797050973) + | > loss_feat: 8.711498260498047 (8.455034318177596) + | > loss_mel: 16.188125610351562 (16.838909128437876) + | > loss_duration: 0.8912263512611389 (0.904774687860323) + | > loss_1: 30.594667434692383 (30.53763356416122) + | > grad_norm_1: tensor(372.7024, device='cuda:0') (tensor(258.0410, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.9346 (0.9399421163227247) + | > loader_time: 0.0056 (0.005622407664423404) + + + --> TIME: 2024-07-14 03:19:01 -- STEP: 96/151 -- GLOBAL_STEP: 115750 + | > loss_disc: 2.281886339187622 (2.313462304572262) + | > loss_disc_real_0: 0.10722555965185165 (0.11748650239314884) + | > loss_disc_real_1: 0.273587167263031 (0.20278765245651206) + | > loss_disc_real_2: 0.24869582056999207 (0.20910878060385585) + | > loss_disc_real_3: 0.16371941566467285 (0.20386219195400676) + | > loss_disc_real_4: 0.21415209770202637 (0.20505894053106502) + | > loss_disc_real_5: 0.17224378883838654 (0.22548986660937467) + | > loss_0: 2.281886339187622 (2.313462304572262) + | > grad_norm_0: tensor(13.1242, device='cuda:0') (tensor(19.0334, device='cuda:0')) + | > loss_gen: 2.7829132080078125 (2.5905668437480913) + | > loss_kl: 2.2974791526794434 (1.7445356150468194) + | > loss_feat: 9.370243072509766 (8.308825979630159) + | > loss_mel: 17.765531539916992 (16.74887187282244) + | > loss_duration: 0.860038161277771 (0.9197085319707791) + | > loss_1: 33.07620620727539 (30.312508583068848) + | > grad_norm_1: tensor(355.8784, device='cuda:0') (tensor(254.8706, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.9303 (0.9807562207182249) + | > loader_time: 0.0054 (0.006154229243596395) + + + --> TIME: 2024-07-14 03:19:51 -- STEP: 146/151 -- GLOBAL_STEP: 115800 + | > loss_disc: 2.361402988433838 (2.3054219696619733) + | > loss_disc_real_0: 0.202729731798172 (0.11702995961659575) + | > loss_disc_real_1: 0.2443050891160965 (0.2029739815282495) + | > loss_disc_real_2: 0.24475282430648804 (0.20994746827915922) + | > loss_disc_real_3: 0.22488808631896973 (0.2052184273935344) + | > loss_disc_real_4: 0.18840254843235016 (0.20481739744339902) + | > loss_disc_real_5: 0.2546026110649109 (0.2238705480751926) + | > loss_0: 2.361402988433838 (2.3054219696619733) + | > grad_norm_0: tensor(57.4208, device='cuda:0') (tensor(20.7580, device='cuda:0')) + | > loss_gen: 2.855154037475586 (2.6065895018512246) + | > loss_kl: 1.432347297668457 (1.7389241999959293) + | > loss_feat: 9.12559700012207 (8.353402405569007) + | > loss_mel: 17.028654098510742 (16.68051395024338) + | > loss_duration: 0.8815304636955261 (0.9213076046068375) + | > loss_1: 31.32328224182129 (30.300737459365635) + | > grad_norm_1: tensor(485.3307, device='cuda:0') (tensor(298.8691, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.8738 (0.9814166206203095) + | > loader_time: 0.0043 (0.00616828056230937) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03185422603900616 (-0.0012884873610276432) + | > avg_loss_disc: 2.5985203889700084 (+0.054910458051241395) + | > avg_loss_disc_real_0: 0.2652573688672139 (-0.04911516033686125) + | > avg_loss_disc_real_1: 0.21102296389066255 (+0.02722241901434383) + | > avg_loss_disc_real_2: 0.2317021878866049 (-0.0010046397264187046) + | > avg_loss_disc_real_3: 0.1919060727724662 (-0.07380224764347079) + | > avg_loss_disc_real_4: 0.2365965063755329 (-0.020031026922739475) + | > avg_loss_disc_real_5: 0.24910714305364168 (+0.00512650952889368) + | > avg_loss_0: 2.5985203889700084 (+0.054910458051241395) + | > avg_loss_gen: 2.4288693574758677 (-0.30555118047274066) + | > avg_loss_kl: 3.6168890549586368 (-0.24781637925368116) + | > avg_loss_feat: 7.744303226470947 (-0.5415502328139077) + | > avg_loss_mel: 16.58428852374737 (-0.8232037470890923) + | > avg_loss_duration: 1.387417564025292 (+0.01969903249006988) + | > avg_loss_1: 31.761767167311447 (-1.8984228280874405) + + + > EPOCH: 204/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:20:06)  + + --> TIME: 2024-07-14 03:20:51 -- STEP: 45/151 -- GLOBAL_STEP: 115850 + | > loss_disc: 2.215169906616211 (2.283465443717109) + | > loss_disc_real_0: 0.0591161847114563 (0.10992496154374547) + | > loss_disc_real_1: 0.24868081510066986 (0.2048511063059171) + | > loss_disc_real_2: 0.1966186910867691 (0.20795295006699033) + | > loss_disc_real_3: 0.1649813950061798 (0.20207147068447537) + | > loss_disc_real_4: 0.1504739224910736 (0.20655048025978937) + | > loss_disc_real_5: 0.19063803553581238 (0.21807468467288546) + | > loss_0: 2.215169906616211 (2.283465443717109) + | > grad_norm_0: tensor(16.8917, device='cuda:0') (tensor(20.2084, device='cuda:0')) + | > loss_gen: 2.4800798892974854 (2.635719161563449) + | > loss_kl: 1.2751688957214355 (1.66754093170166) + | > loss_feat: 7.808302879333496 (8.563747204674614) + | > loss_mel: 15.53239917755127 (16.507486682467988) + | > loss_duration: 1.2390899658203125 (0.9274546557002598) + | > loss_1: 28.335041046142578 (30.30194837782118) + | > grad_norm_1: tensor(487.3745, device='cuda:0') (tensor(350.2823, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.815 (0.9462020927005343) + | > loader_time: 0.0054 (0.0060983710818820515) + + + --> TIME: 2024-07-14 03:21:43 -- STEP: 95/151 -- GLOBAL_STEP: 115900 + | > loss_disc: 2.008065938949585 (2.275311851501465) + | > loss_disc_real_0: 0.08194199204444885 (0.10696935296842927) + | > loss_disc_real_1: 0.15241152048110962 (0.20537426338384027) + | > loss_disc_real_2: 0.22261491417884827 (0.2092942071588416) + | > loss_disc_real_3: 0.12992678582668304 (0.2045964518659993) + | > loss_disc_real_4: 0.16612648963928223 (0.20664417359389756) + | > loss_disc_real_5: 0.13102154433727264 (0.2158355712890625) + | > loss_0: 2.008065938949585 (2.275311851501465) + | > grad_norm_0: tensor(7.1807, device='cuda:0') (tensor(22.6582, device='cuda:0')) + | > loss_gen: 3.126840353012085 (2.647621385674727) + | > loss_kl: 1.6370996236801147 (1.701526873990109) + | > loss_feat: 11.35952091217041 (8.587975406646732) + | > loss_mel: 17.52676010131836 (16.538540438601835) + | > loss_duration: 0.8409141898155212 (0.9276035252370333) + | > loss_1: 34.49113464355469 (30.403267468904193) + | > grad_norm_1: tensor(469.2671, device='cuda:0') (tensor(387.4455, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.8694 (0.9855507775356895) + | > loader_time: 0.0044 (0.006319467644942434) + + + --> TIME: 2024-07-14 03:22:32 -- STEP: 145/151 -- GLOBAL_STEP: 115950 + | > loss_disc: 2.412693500518799 (2.2785100122977955) + | > loss_disc_real_0: 0.11161820590496063 (0.10968102828182023) + | > loss_disc_real_1: 0.19795086979866028 (0.20317267883440543) + | > loss_disc_real_2: 0.23914381861686707 (0.20893446926412915) + | > loss_disc_real_3: 0.2168913334608078 (0.20522840074424087) + | > loss_disc_real_4: 0.26645880937576294 (0.20724881974787548) + | > loss_disc_real_5: 0.2116049975156784 (0.21844256992997793) + | > loss_0: 2.412693500518799 (2.2785100122977955) + | > grad_norm_0: tensor(35.1361, device='cuda:0') (tensor(23.8390, device='cuda:0')) + | > loss_gen: 2.329636812210083 (2.651119845488976) + | > loss_kl: 2.004901170730591 (1.700811612194982) + | > loss_feat: 6.018004417419434 (8.538235710407127) + | > loss_mel: 14.990922927856445 (16.597653060123843) + | > loss_duration: 1.1523888111114502 (0.9214588559907059) + | > loss_1: 26.4958553314209 (30.40927901432432) + | > grad_norm_1: tensor(322.8725, device='cuda:0') (tensor(388.5364, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 2.043 (0.98201647462516) + | > loader_time: 0.0152 (0.00620364813969053) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031997350546029896 (+0.0001431245070237369) + | > avg_loss_disc: 2.651126348055326 (+0.05260595908531762) + | > avg_loss_disc_real_0: 0.3249741827066128 (+0.05971681383939892) + | > avg_loss_disc_real_1: 0.23196455950920397 (+0.02094159561854142) + | > avg_loss_disc_real_2: 0.2660838250930493 (+0.03438163720644438) + | > avg_loss_disc_real_3: 0.25273547034997207 (+0.060829397577505856) + | > avg_loss_disc_real_4: 0.2772536300695859 (+0.04065712369405303) + | > avg_loss_disc_real_5: 0.24454030509178454 (-0.00456683796185714) + | > avg_loss_0: 2.651126348055326 (+0.05260595908531762) + | > avg_loss_gen: 2.712217624370868 (+0.2833482668950005) + | > avg_loss_kl: 3.9433396596174974 (+0.32645060465886067) + | > avg_loss_feat: 7.426373481750488 (-0.317929744720459) + | > avg_loss_mel: 16.662049000079815 (+0.07776047633244332) + | > avg_loss_duration: 1.3585731891485362 (-0.02884437487675573) + | > avg_loss_1: 32.10255314753606 (+0.3407859802246094) + + + > EPOCH: 205/1000 + --> /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a + + > TRAINING (2024-07-14 03:22:47)  + + --> TIME: 2024-07-14 03:23:35 -- STEP: 44/151 -- GLOBAL_STEP: 116000 + | > loss_disc: 2.1309359073638916 (2.3514896577054802) + | > loss_disc_real_0: 0.0771031528711319 (0.11891941514543512) + | > loss_disc_real_1: 0.16086755692958832 (0.20669212801889938) + | > loss_disc_real_2: 0.17756837606430054 (0.21390224011106926) + | > loss_disc_real_3: 0.21952374279499054 (0.21074855226007375) + | > loss_disc_real_4: 0.22627362608909607 (0.21621805158528415) + | > loss_disc_real_5: 0.2135108858346939 (0.22443535382097418) + | > loss_0: 2.1309359073638916 (2.3514896577054802) + | > grad_norm_0: tensor(23.6146, device='cuda:0') (tensor(21.1154, device='cuda:0')) + | > loss_gen: 2.698040723800659 (2.568016837943684) + | > loss_kl: 1.3431384563446045 (1.749802909114144) + | > loss_feat: 8.553112030029297 (8.122337092052808) + | > loss_mel: 15.586190223693848 (16.293468106876713) + | > loss_duration: 1.017145037651062 (0.9347549880092795) + | > loss_1: 29.1976261138916 (29.668379696932707) + | > grad_norm_1: tensor(452.8250, device='cuda:0') (tensor(344.7641, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.8279 (1.0292408683083276) + | > loader_time: 0.0042 (0.006661848588423295) + + ! Run is kept in /home/kk/storage/YourTTS-finetuned-from-yourtts-base-male-finetuned-female_only-July-13-2024_05+59PM-dbf1a08a