diff --git "a/sf_log.txt" "b/sf_log.txt" new file mode 100644--- /dev/null +++ "b/sf_log.txt" @@ -0,0 +1,22423 @@ +[2023-03-07 15:31:17,415][236315] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json... +[2023-03-07 15:31:17,416][236315] Rollout worker 0 uses device cpu +[2023-03-07 15:31:17,417][236315] Rollout worker 1 uses device cpu +[2023-03-07 15:31:17,417][236315] Rollout worker 2 uses device cpu +[2023-03-07 15:31:17,418][236315] Rollout worker 3 uses device cpu +[2023-03-07 15:31:17,419][236315] Rollout worker 4 uses device cpu +[2023-03-07 15:31:17,419][236315] Rollout worker 5 uses device cpu +[2023-03-07 15:31:17,419][236315] Rollout worker 6 uses device cpu +[2023-03-07 15:31:17,420][236315] Rollout worker 7 uses device cpu +[2023-03-07 15:31:17,420][236315] Rollout worker 8 uses device cpu +[2023-03-07 15:31:17,421][236315] Rollout worker 9 uses device cpu +[2023-03-07 15:31:17,421][236315] Rollout worker 10 uses device cpu +[2023-03-07 15:31:17,421][236315] Rollout worker 11 uses device cpu +[2023-03-07 15:31:17,422][236315] Rollout worker 12 uses device cpu +[2023-03-07 15:31:17,422][236315] Rollout worker 13 uses device cpu +[2023-03-07 15:31:17,422][236315] Rollout worker 14 uses device cpu +[2023-03-07 15:31:17,423][236315] Rollout worker 15 uses device cpu +[2023-03-07 15:31:17,423][236315] Rollout worker 16 uses device cpu +[2023-03-07 15:31:17,424][236315] Rollout worker 17 uses device cpu +[2023-03-07 15:31:17,424][236315] Rollout worker 18 uses device cpu +[2023-03-07 15:31:17,424][236315] Rollout worker 19 uses device cpu +[2023-03-07 15:31:17,425][236315] Rollout worker 20 uses device cpu +[2023-03-07 15:31:17,425][236315] Rollout worker 21 uses device cpu +[2023-03-07 15:31:17,426][236315] Rollout worker 22 uses device cpu +[2023-03-07 15:31:17,426][236315] Rollout worker 23 uses device cpu +[2023-03-07 15:31:17,426][236315] Rollout worker 24 uses device cpu +[2023-03-07 15:31:17,427][236315] Rollout worker 25 uses device cpu +[2023-03-07 15:31:17,427][236315] Rollout worker 26 uses device cpu +[2023-03-07 15:31:17,427][236315] Rollout worker 27 uses device cpu +[2023-03-07 15:31:17,428][236315] Rollout worker 28 uses device cpu +[2023-03-07 15:31:17,428][236315] Rollout worker 29 uses device cpu +[2023-03-07 15:31:17,431][236315] Rollout worker 30 uses device cpu +[2023-03-07 15:31:17,431][236315] Rollout worker 31 uses device cpu +[2023-03-07 15:31:17,432][236315] Rollout worker 32 uses device cpu +[2023-03-07 15:31:17,432][236315] Rollout worker 33 uses device cpu +[2023-03-07 15:31:17,432][236315] Rollout worker 34 uses device cpu +[2023-03-07 15:31:17,433][236315] Rollout worker 35 uses device cpu +[2023-03-07 15:31:17,433][236315] Rollout worker 36 uses device cpu +[2023-03-07 15:31:17,433][236315] Rollout worker 37 uses device cpu +[2023-03-07 15:31:17,434][236315] Rollout worker 38 uses device cpu +[2023-03-07 15:31:17,435][236315] Rollout worker 39 uses device cpu +[2023-03-07 15:31:17,435][236315] Rollout worker 40 uses device cpu +[2023-03-07 15:31:17,436][236315] Rollout worker 41 uses device cpu +[2023-03-07 15:31:17,436][236315] Rollout worker 42 uses device cpu +[2023-03-07 15:31:17,436][236315] Rollout worker 43 uses device cpu +[2023-03-07 15:31:17,437][236315] Rollout worker 44 uses device cpu +[2023-03-07 15:31:17,437][236315] Rollout worker 45 uses device cpu +[2023-03-07 15:31:17,438][236315] Rollout worker 46 uses device cpu +[2023-03-07 15:31:17,438][236315] Rollout worker 47 uses device cpu +[2023-03-07 15:31:17,438][236315] Rollout worker 48 uses device cpu +[2023-03-07 15:31:17,439][236315] Rollout worker 49 uses device cpu +[2023-03-07 15:31:17,439][236315] Rollout worker 50 uses device cpu +[2023-03-07 15:31:17,440][236315] Rollout worker 51 uses device cpu +[2023-03-07 15:31:17,440][236315] Rollout worker 52 uses device cpu +[2023-03-07 15:31:17,440][236315] Rollout worker 53 uses device cpu +[2023-03-07 15:31:17,441][236315] Rollout worker 54 uses device cpu +[2023-03-07 15:31:17,441][236315] Rollout worker 55 uses device cpu +[2023-03-07 15:31:17,442][236315] Rollout worker 56 uses device cpu +[2023-03-07 15:31:17,442][236315] Rollout worker 57 uses device cpu +[2023-03-07 15:31:17,444][236315] Rollout worker 58 uses device cpu +[2023-03-07 15:31:17,444][236315] Rollout worker 59 uses device cpu +[2023-03-07 15:31:17,445][236315] Rollout worker 60 uses device cpu +[2023-03-07 15:31:17,445][236315] Rollout worker 61 uses device cpu +[2023-03-07 15:31:17,445][236315] Rollout worker 62 uses device cpu +[2023-03-07 15:31:17,446][236315] Rollout worker 63 uses device cpu +[2023-03-07 15:31:17,447][236315] Rollout worker 64 uses device cpu +[2023-03-07 15:31:17,447][236315] Rollout worker 65 uses device cpu +[2023-03-07 15:31:17,447][236315] Rollout worker 66 uses device cpu +[2023-03-07 15:31:17,448][236315] Rollout worker 67 uses device cpu +[2023-03-07 15:31:17,448][236315] Rollout worker 68 uses device cpu +[2023-03-07 15:31:17,449][236315] Rollout worker 69 uses device cpu +[2023-03-07 15:31:17,449][236315] Rollout worker 70 uses device cpu +[2023-03-07 15:31:17,450][236315] Rollout worker 71 uses device cpu +[2023-03-07 15:31:17,450][236315] Rollout worker 72 uses device cpu +[2023-03-07 15:31:17,451][236315] Rollout worker 73 uses device cpu +[2023-03-07 15:31:17,451][236315] Rollout worker 74 uses device cpu +[2023-03-07 15:31:17,452][236315] Rollout worker 75 uses device cpu +[2023-03-07 15:31:17,452][236315] Rollout worker 76 uses device cpu +[2023-03-07 15:31:17,453][236315] Rollout worker 77 uses device cpu +[2023-03-07 15:31:17,453][236315] Rollout worker 78 uses device cpu +[2023-03-07 15:31:17,454][236315] Rollout worker 79 uses device cpu +[2023-03-07 15:31:17,454][236315] Rollout worker 80 uses device cpu +[2023-03-07 15:31:17,455][236315] Rollout worker 81 uses device cpu +[2023-03-07 15:31:17,455][236315] Rollout worker 82 uses device cpu +[2023-03-07 15:31:17,456][236315] Rollout worker 83 uses device cpu +[2023-03-07 15:31:17,458][236315] Rollout worker 84 uses device cpu +[2023-03-07 15:31:17,458][236315] Rollout worker 85 uses device cpu +[2023-03-07 15:31:17,459][236315] Rollout worker 86 uses device cpu +[2023-03-07 15:31:17,459][236315] Rollout worker 87 uses device cpu +[2023-03-07 15:31:17,460][236315] Rollout worker 88 uses device cpu +[2023-03-07 15:31:17,460][236315] Rollout worker 89 uses device cpu +[2023-03-07 15:31:17,461][236315] Rollout worker 90 uses device cpu +[2023-03-07 15:31:17,461][236315] Rollout worker 91 uses device cpu +[2023-03-07 15:31:17,462][236315] Rollout worker 92 uses device cpu +[2023-03-07 15:31:17,462][236315] Rollout worker 93 uses device cpu +[2023-03-07 15:31:17,463][236315] Rollout worker 94 uses device cpu +[2023-03-07 15:31:17,463][236315] Rollout worker 95 uses device cpu +[2023-03-07 15:31:17,465][236315] Rollout worker 96 uses device cpu +[2023-03-07 15:31:17,465][236315] Rollout worker 97 uses device cpu +[2023-03-07 15:31:17,466][236315] Rollout worker 98 uses device cpu +[2023-03-07 15:31:17,466][236315] Rollout worker 99 uses device cpu +[2023-03-07 15:31:17,467][236315] Rollout worker 100 uses device cpu +[2023-03-07 15:31:17,467][236315] Rollout worker 101 uses device cpu +[2023-03-07 15:31:17,468][236315] Rollout worker 102 uses device cpu +[2023-03-07 15:31:17,468][236315] Rollout worker 103 uses device cpu +[2023-03-07 15:31:17,469][236315] Rollout worker 104 uses device cpu +[2023-03-07 15:31:17,469][236315] Rollout worker 105 uses device cpu +[2023-03-07 15:31:17,470][236315] Rollout worker 106 uses device cpu +[2023-03-07 15:31:17,470][236315] Rollout worker 107 uses device cpu +[2023-03-07 15:31:17,471][236315] Rollout worker 108 uses device cpu +[2023-03-07 15:31:17,471][236315] Rollout worker 109 uses device cpu +[2023-03-07 15:31:17,472][236315] Rollout worker 110 uses device cpu +[2023-03-07 15:31:17,472][236315] Rollout worker 111 uses device cpu +[2023-03-07 15:31:17,473][236315] Rollout worker 112 uses device cpu +[2023-03-07 15:31:17,473][236315] Rollout worker 113 uses device cpu +[2023-03-07 15:31:17,474][236315] Rollout worker 114 uses device cpu +[2023-03-07 15:31:17,474][236315] Rollout worker 115 uses device cpu +[2023-03-07 15:31:17,475][236315] Rollout worker 116 uses device cpu +[2023-03-07 15:31:17,475][236315] Rollout worker 117 uses device cpu +[2023-03-07 15:31:17,476][236315] Rollout worker 118 uses device cpu +[2023-03-07 15:31:17,476][236315] Rollout worker 119 uses device cpu +[2023-03-07 15:31:17,769][236315] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:31:17,769][236315] InferenceWorker_p0-w0: min num requests: 40 +[2023-03-07 15:31:18,081][236315] Starting all processes... +[2023-03-07 15:31:18,082][236315] Starting process learner_proc0 +[2023-03-07 15:31:18,967][236315] Starting all processes... +[2023-03-07 15:31:18,968][238955] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:31:18,968][238955] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0 +[2023-03-07 15:31:18,974][236315] Starting process inference_proc0-0 +[2023-03-07 15:31:18,974][236315] Starting process rollout_proc1 +[2023-03-07 15:31:18,977][238955] Num visible devices: 1 +[2023-03-07 15:31:18,982][238955] Starting seed is not provided +[2023-03-07 15:31:18,982][238955] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:31:18,982][238955] Initializing actor-critic model on device cuda:0 +[2023-03-07 15:31:18,982][238955] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:31:18,975][236315] Starting process rollout_proc3 +[2023-03-07 15:31:18,983][238955] RunningMeanStd input shape: (1,) +[2023-03-07 15:31:18,976][236315] Starting process rollout_proc5 +[2023-03-07 15:31:18,976][236315] Starting process rollout_proc7 +[2023-03-07 15:31:18,993][238955] ConvEncoder: input_channels=3 +[2023-03-07 15:31:18,980][236315] Starting process rollout_proc9 +[2023-03-07 15:31:18,981][236315] Starting process rollout_proc11 +[2023-03-07 15:31:18,986][236315] Starting process rollout_proc13 +[2023-03-07 15:31:18,988][236315] Starting process rollout_proc15 +[2023-03-07 15:31:18,988][236315] Starting process rollout_proc17 +[2023-03-07 15:31:18,990][236315] Starting process rollout_proc19 +[2023-03-07 15:31:18,990][236315] Starting process rollout_proc21 +[2023-03-07 15:31:18,991][236315] Starting process rollout_proc23 +[2023-03-07 15:31:18,993][236315] Starting process rollout_proc25 +[2023-03-07 15:31:18,995][236315] Starting process rollout_proc27 +[2023-03-07 15:31:18,997][236315] Starting process rollout_proc29 +[2023-03-07 15:31:19,012][236315] Starting process rollout_proc2 +[2023-03-07 15:31:19,016][236315] Starting process rollout_proc4 +[2023-03-07 15:31:19,022][236315] Starting process rollout_proc6 +[2023-03-07 15:31:19,030][236315] Starting process rollout_proc8 +[2023-03-07 15:31:19,036][236315] Starting process rollout_proc10 +[2023-03-07 15:31:19,043][236315] Starting process rollout_proc14 +[2023-03-07 15:31:19,043][236315] Starting process rollout_proc12 +[2023-03-07 15:31:19,056][236315] Starting process rollout_proc18 +[2023-03-07 15:31:19,068][236315] Starting process rollout_proc16 +[2023-03-07 15:31:19,068][236315] Starting process rollout_proc20 +[2023-03-07 15:31:19,081][236315] Starting process rollout_proc22 +[2023-03-07 15:31:19,085][236315] Starting process rollout_proc24 +[2023-03-07 15:31:19,105][238955] Conv encoder output size: 512 +[2023-03-07 15:31:19,105][238955] Policy head output size: 512 +[2023-03-07 15:31:19,092][236315] Starting process rollout_proc28 +[2023-03-07 15:31:19,095][236315] Starting process rollout_proc26 +[2023-03-07 15:31:19,095][236315] Starting process rollout_proc30 +[2023-03-07 15:31:19,122][238955] Created Actor Critic model with architecture: +[2023-03-07 15:31:19,122][238955] ActorCriticSharedWeights( + (obs_normalizer): ObservationNormalizer( + (running_mean_std): RunningMeanStdDictInPlace( + (running_mean_std): ModuleDict( + (obs): RunningMeanStdInPlace() + ) + ) + ) + (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace) + (encoder): VizdoomEncoder( + (basic_encoder): ConvEncoder( + (enc): RecursiveScriptModule( + original_name=ConvEncoderImpl + (conv_head): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Conv2d) + (1): RecursiveScriptModule(original_name=ELU) + (2): RecursiveScriptModule(original_name=Conv2d) + (3): RecursiveScriptModule(original_name=ELU) + (4): RecursiveScriptModule(original_name=Conv2d) + (5): RecursiveScriptModule(original_name=ELU) + ) + (mlp_layers): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Linear) + (1): RecursiveScriptModule(original_name=ELU) + ) + ) + ) + ) + (core): ModelCoreRNN( + (core): GRU(512, 512) + ) + (decoder): MlpDecoder( + (mlp): Identity() + ) + (critic_linear): Linear(in_features=512, out_features=1, bias=True) + (action_parameterization): ActionParameterizationDefault( + (distribution_linear): Linear(in_features=512, out_features=5, bias=True) + ) +) +[2023-03-07 15:31:19,104][236315] Starting process rollout_proc31 +[2023-03-07 15:31:19,111][236315] Starting process rollout_proc33 +[2023-03-07 15:31:19,114][236315] Starting process rollout_proc35 +[2023-03-07 15:31:19,119][236315] Starting process rollout_proc37 +[2023-03-07 15:31:19,120][236315] Starting process rollout_proc39 +[2023-03-07 15:31:19,130][236315] Starting process rollout_proc41 +[2023-03-07 15:31:19,140][236315] Starting process rollout_proc43 +[2023-03-07 15:31:19,143][236315] Starting process rollout_proc45 +[2023-03-07 15:31:19,147][236315] Starting process rollout_proc47 +[2023-03-07 15:31:19,147][236315] Starting process rollout_proc49 +[2023-03-07 15:31:19,151][236315] Starting process rollout_proc51 +[2023-03-07 15:31:19,162][236315] Starting process rollout_proc53 +[2023-03-07 15:31:19,166][236315] Starting process rollout_proc55 +[2023-03-07 15:31:19,167][236315] Starting process rollout_proc57 +[2023-03-07 15:31:19,167][236315] Starting process rollout_proc59 +[2023-03-07 15:31:19,171][236315] Starting process rollout_proc34 +[2023-03-07 15:31:19,171][236315] Starting process rollout_proc32 +[2023-03-07 15:31:19,177][236315] Starting process rollout_proc36 +[2023-03-07 15:31:19,180][236315] Starting process rollout_proc38 +[2023-03-07 15:31:19,189][236315] Starting process rollout_proc40 +[2023-03-07 15:31:19,194][236315] Starting process rollout_proc42 +[2023-03-07 15:31:19,199][236315] Starting process rollout_proc44 +[2023-03-07 15:31:19,201][236315] Starting process rollout_proc46 +[2023-03-07 15:31:19,206][236315] Starting process rollout_proc48 +[2023-03-07 15:31:19,212][236315] Starting process rollout_proc50 +[2023-03-07 15:31:19,221][236315] Starting process rollout_proc52 +[2023-03-07 15:31:19,224][236315] Starting process rollout_proc54 +[2023-03-07 15:31:19,229][236315] Starting process rollout_proc56 +[2023-03-07 15:31:19,239][236315] Starting process rollout_proc58 +[2023-03-07 15:31:19,251][236315] Starting process rollout_proc61 +[2023-03-07 15:31:19,251][236315] Starting process rollout_proc63 +[2023-03-07 15:31:19,252][236315] Starting process rollout_proc60 +[2023-03-07 15:31:19,259][236315] Starting process rollout_proc65 +[2023-03-07 15:31:19,262][236315] Starting process rollout_proc67 +[2023-03-07 15:31:19,268][236315] Starting process rollout_proc69 +[2023-03-07 15:31:19,277][236315] Starting process rollout_proc71 +[2023-03-07 15:31:19,279][236315] Starting process rollout_proc73 +[2023-03-07 15:31:19,282][236315] Starting process rollout_proc75 +[2023-03-07 15:31:19,291][236315] Starting process rollout_proc77 +[2023-03-07 15:31:19,300][236315] Starting process rollout_proc79 +[2023-03-07 15:31:19,308][236315] Starting process rollout_proc81 +[2023-03-07 15:31:19,312][236315] Starting process rollout_proc83 +[2023-03-07 15:31:19,315][236315] Starting process rollout_proc85 +[2023-03-07 15:31:19,322][236315] Starting process rollout_proc87 +[2023-03-07 15:31:19,331][236315] Starting process rollout_proc62 +[2023-03-07 15:31:19,350][236315] Starting process rollout_proc64 +[2023-03-07 15:31:19,361][236315] Starting process rollout_proc89 +[2023-03-07 15:31:19,366][236315] Starting process rollout_proc66 +[2023-03-07 15:31:19,379][236315] Starting process rollout_proc68 +[2023-03-07 15:31:19,392][236315] Starting process rollout_proc72 +[2023-03-07 15:31:19,396][236315] Starting process rollout_proc70 +[2023-03-07 15:31:19,400][236315] Starting process rollout_proc74 +[2023-03-07 15:31:19,407][236315] Starting process rollout_proc78 +[2023-03-07 15:31:19,481][236315] Starting process rollout_proc82 +[2023-03-07 15:31:19,483][236315] Starting process rollout_proc80 +[2023-03-07 15:31:19,487][236315] Starting process rollout_proc91 +[2023-03-07 15:31:19,488][236315] Starting process rollout_proc84 +[2023-03-07 15:31:19,488][236315] Starting process rollout_proc93 +[2023-03-07 15:31:19,489][236315] Starting process rollout_proc76 +[2023-03-07 15:31:19,493][236315] Starting process rollout_proc95 +[2023-03-07 15:31:19,493][236315] Starting process rollout_proc97 +[2023-03-07 15:31:19,495][236315] Starting process rollout_proc86 +[2023-03-07 15:31:19,497][236315] Starting process rollout_proc99 +[2023-03-07 15:31:19,500][236315] Starting process rollout_proc101 +[2023-03-07 15:31:19,504][236315] Starting process rollout_proc90 +[2023-03-07 15:31:19,507][236315] Starting process rollout_proc88 +[2023-03-07 15:31:19,510][236315] Starting process rollout_proc103 +[2023-03-07 15:31:19,519][236315] Starting process rollout_proc105 +[2023-03-07 15:31:19,535][236315] Starting process rollout_proc107 +[2023-03-07 15:31:19,571][236315] Starting process rollout_proc109 +[2023-03-07 15:31:19,602][236315] Starting process rollout_proc111 +[2023-03-07 15:31:19,603][236315] Starting process rollout_proc92 +[2023-03-07 15:31:19,614][236315] Starting process rollout_proc113 +[2023-03-07 15:31:19,640][236315] Starting process rollout_proc94 +[2023-03-07 15:31:19,660][236315] Starting process rollout_proc115 +[2023-03-07 15:31:19,671][236315] Starting process rollout_proc100 +[2023-03-07 15:31:19,685][236315] Starting process rollout_proc98 +[2023-03-07 15:31:19,700][236315] Starting process rollout_proc104 +[2023-03-07 15:31:19,704][236315] Starting process rollout_proc102 +[2023-03-07 15:31:19,718][236315] Starting process rollout_proc117 +[2023-03-07 15:31:19,746][236315] Starting process rollout_proc108 +[2023-03-07 15:31:19,747][236315] Starting process rollout_proc96 +[2023-03-07 15:31:19,814][236315] Starting process rollout_proc106 +[2023-03-07 15:31:19,815][236315] Starting process rollout_proc110 +[2023-03-07 15:31:19,821][236315] Starting process rollout_proc119 +[2023-03-07 15:31:19,844][236315] Starting process rollout_proc112 +[2023-03-07 15:31:19,944][236315] Starting process rollout_proc114 +[2023-03-07 15:31:19,962][236315] Starting process rollout_proc116 +[2023-03-07 15:31:19,963][236315] Starting process rollout_proc118 +[2023-03-07 15:31:21,306][239186] Worker 4 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,410][239108] Worker 13 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,604][239112] Worker 19 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,607][239106] Worker 7 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,708][239187] Worker 6 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,754][239107] Worker 9 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,781][239191] Worker 12 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,786][239105] Worker 5 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,797][239104] Worker 3 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,808][239185] Worker 2 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,850][239111] Worker 15 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,852][239109] Worker 11 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,864][239198] Worker 26 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,869][239103] Worker 1 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,881][239110] Worker 17 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,882][239181] Worker 27 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,887][239183] Worker 29 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,976][239194] Worker 18 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:21,982][239195] Worker 22 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,000][239177] Worker 21 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,005][239207] Worker 43 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,042][239190] Worker 14 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,061][239178] Worker 23 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,078][239204] Worker 37 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,127][239211] Worker 49 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,139][239193] Worker 16 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,143][239205] Worker 39 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,150][239184] Worker 25 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,186][239192] Worker 20 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,238][239221] Worker 38 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,288][239199] Worker 28 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,291][239231] Worker 54 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,313][239197] Worker 24 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,364][239223] Worker 42 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,382][236315] Starting process rollout_proc0 +[2023-03-07 15:31:22,402][239101] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:31:22,402][239101] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0 +[2023-03-07 15:31:22,403][239201] Worker 33 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,413][239236] Worker 65 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,417][239101] Num visible devices: 1 +[2023-03-07 15:31:22,427][239233] Worker 61 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,428][239213] Worker 53 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,429][239212] Worker 51 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,440][239210] Worker 47 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,451][239214] Worker 55 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,457][239220] Worker 34 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,460][239188] Worker 10 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,463][239227] Worker 48 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,473][239215] Worker 57 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,483][239189] Worker 8 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,500][239206] Worker 41 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,511][239234] Worker 63 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,517][239232] Worker 58 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,578][239203] Worker 35 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,579][239235] Worker 60 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,597][239200] Worker 30 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,599][239243] Worker 79 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,598][239222] Worker 32 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,603][239229] Worker 52 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,633][239216] Worker 59 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,633][239226] Worker 46 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,677][239209] Worker 45 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,694][239217] Worker 36 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,696][239225] Worker 44 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,696][239254] Worker 87 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,700][239228] Worker 50 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,708][239224] Worker 40 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,711][239202] Worker 31 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,720][239242] Worker 81 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,722][239252] Worker 70 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,724][239237] Worker 67 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,730][239238] Worker 71 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,734][239250] Worker 85 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,736][239240] Worker 73 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,741][239256] Worker 66 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,745][239253] Worker 89 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,751][239255] Worker 78 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,763][239846] Worker 100 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,777][239249] Worker 68 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,806][239554] Worker 101 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,816][239257] Worker 82 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,824][239245] Worker 72 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,830][239260] Worker 91 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,832][240209] Worker 106 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,855][239241] Worker 77 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,867][239247] Worker 75 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,870][239230] Worker 56 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,871][239239] Worker 69 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,873][239244] Worker 74 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,890][239391] Worker 97 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,893][239586] Worker 107 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,899][240101] Worker 102 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,920][239828] Worker 92 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,924][239585] Worker 88 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,924][239392] Worker 95 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,935][239259] Worker 84 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,941][239584] Worker 103 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,948][239251] Worker 62 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,949][239456] Worker 90 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,952][239248] Worker 64 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,955][240103] Worker 108 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,979][239246] Worker 83 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,982][239325] Worker 76 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,983][239843] Worker 113 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:22,992][239258] Worker 80 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,011][239845] Worker 115 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,015][239324] Worker 86 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,022][239588] Worker 105 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,043][240297] Worker 112 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,044][240273] Worker 110 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,050][239587] Worker 109 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,054][239844] Worker 94 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,078][239961] Worker 104 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,080][239826] Worker 111 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,088][240550] Worker 116 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,106][240296] Worker 119 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,122][240104] Worker 96 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,126][239327] Worker 99 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,170][239878] Worker 98 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,190][240102] Worker 117 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,194][239326] Worker 93 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,214][240579] Worker 118 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,236][240430] Worker 114 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,489][238955] Using optimizer +[2023-03-07 15:31:23,490][238955] No checkpoints found +[2023-03-07 15:31:23,490][238955] Did not load from checkpoint, starting from scratch! +[2023-03-07 15:31:23,490][238955] Initialized policy 0 weights for model version 0 +[2023-03-07 15:31:23,491][238955] LearnerWorker_p0 finished initialization! +[2023-03-07 15:31:23,492][238955] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:31:23,560][239101] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:31:23,560][239101] RunningMeanStd input shape: (1,) +[2023-03-07 15:31:23,570][239101] ConvEncoder: input_channels=3 +[2023-03-07 15:31:23,596][248936] Worker 0 uses CPU cores [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] +[2023-03-07 15:31:23,643][239101] Conv encoder output size: 512 +[2023-03-07 15:31:23,643][239101] Policy head output size: 512 +[2023-03-07 15:31:24,477][236315] Inference worker 0-0 is ready! +[2023-03-07 15:31:24,478][236315] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:31:24,532][240550] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,535][239828] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,537][239586] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,541][239234] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,546][240296] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,547][240209] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,547][239231] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,551][239191] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,552][239193] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,554][239585] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,554][239192] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,555][248936] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,555][239178] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,556][239213] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,557][239203] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,557][239105] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,558][239878] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,559][239961] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,560][239183] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,560][239211] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,560][239206] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,561][239846] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,561][239205] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,561][239103] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,561][239112] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,561][239187] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,562][239232] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,562][239243] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,562][239200] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239554] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239225] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239248] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239109] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239240] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][240102] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,563][239220] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,564][239239] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,564][239199] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,564][239209] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,565][239215] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,565][239198] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,565][239185] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,565][239844] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,565][239584] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,566][239244] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,566][239224] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,566][239392] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,566][239222] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,566][239326] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,567][239247] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,567][240273] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,567][239217] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,567][239202] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,567][239235] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][239250] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][240101] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][239241] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][239236] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][239327] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,568][240430] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239588] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239587] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239826] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239223] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239845] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][240297] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,570][239325] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,570][240104] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,570][239324] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,570][239104] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,571][239230] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,571][239242] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,570][239237] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,569][239258] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,571][239181] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,571][239238] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,571][239257] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,572][239245] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,572][239194] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,572][239210] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,573][239843] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,574][239255] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,574][240103] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,575][239249] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,575][239227] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,576][239107] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,577][239197] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,577][239229] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,577][239207] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,578][239252] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,578][239246] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,578][239204] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,578][239186] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,579][239110] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,581][239254] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][239228] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][239212] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][239190] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][240579] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][239456] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239233] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,582][239259] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239214] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239189] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239260] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239106] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,583][239201] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,584][239391] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,584][239253] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,584][239177] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,584][239251] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,584][239216] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,585][239221] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,586][239184] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,586][239226] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,586][239195] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,588][239108] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,589][239188] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,590][239256] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:24,590][239111] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:31:25,021][239585] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,022][239247] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,023][239828] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,023][239234] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,024][239185] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,025][240550] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,026][240296] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,027][239846] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,028][240209] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,028][239586] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,204][239234] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,204][240550] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,209][239205] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,214][239225] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,214][239112] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,215][239191] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,232][239203] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,232][240297] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,274][239244] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,274][239845] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,396][239112] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,412][239188] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,412][236315] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:31:25,412][240296] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,420][239184] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,422][239585] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,423][239828] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,423][239178] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,423][239213] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,455][239247] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,460][239241] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,602][239225] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,603][239178] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,607][239240] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,608][239244] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,610][239191] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,611][239554] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,613][239231] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,615][239845] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,656][239257] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,665][239215] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,780][239828] Decorrelating experience for 64 frames... +[2023-03-07 15:31:25,790][239205] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,791][239188] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,792][239241] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,803][239247] Decorrelating experience for 64 frames... +[2023-03-07 15:31:25,807][239185] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,809][239187] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,812][239961] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,851][239845] Decorrelating experience for 64 frames... +[2023-03-07 15:31:25,862][239181] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,955][239248] Decorrelating experience for 0 frames... +[2023-03-07 15:31:25,981][239585] Decorrelating experience for 64 frames... +[2023-03-07 15:31:25,985][239257] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,988][239187] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,989][239961] Decorrelating experience for 32 frames... +[2023-03-07 15:31:25,993][239828] Decorrelating experience for 96 frames... +[2023-03-07 15:31:26,003][239247] Decorrelating experience for 96 frames... +[2023-03-07 15:31:26,009][239326] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,058][239193] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,061][239202] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,152][239192] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,162][239185] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,170][239456] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,175][239554] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,186][239110] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,200][239244] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,214][239249] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,224][239584] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,236][239205] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,243][239326] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,325][239240] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,338][239181] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,354][239191] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,357][239184] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,361][239197] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,412][240103] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,413][240102] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,418][239224] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,430][239199] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,438][239205] Decorrelating experience for 96 frames... +[2023-03-07 15:31:26,503][239213] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,521][239108] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,538][239187] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,556][239225] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,563][248936] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,587][239257] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,588][239197] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,604][239199] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,615][239195] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,617][239211] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,680][239192] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,699][239241] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,717][239214] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,738][248936] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,766][239843] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,781][239231] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,786][239210] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,786][239254] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,794][239326] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,811][239243] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,854][239112] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,882][239197] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,895][239249] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,912][239103] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,939][239195] Decorrelating experience for 32 frames... +[2023-03-07 15:31:26,966][239191] Decorrelating experience for 96 frames... +[2023-03-07 15:31:26,968][240430] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,976][239240] Decorrelating experience for 64 frames... +[2023-03-07 15:31:26,979][239235] Decorrelating experience for 0 frames... +[2023-03-07 15:31:26,984][248936] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,037][239257] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,073][239245] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,094][239243] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,098][240102] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,116][239846] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,144][239202] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,145][239211] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,153][239456] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,165][240273] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,169][239206] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,213][239255] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,253][239584] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,290][239197] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,294][248936] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,295][239243] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,320][239185] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,334][239231] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,343][239193] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,347][239843] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,351][239326] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,387][239240] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,452][239206] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,486][239236] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,486][239192] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,486][239325] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,496][239187] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,516][239216] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,518][239327] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,551][239248] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,585][239217] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,586][239220] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,645][239214] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,661][239845] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,665][239112] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,669][239210] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,683][240296] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,691][239255] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,703][239211] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,754][239241] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,764][239217] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,774][239234] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,844][239214] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,844][239231] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,848][240430] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,853][239456] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,870][239210] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,872][239109] Decorrelating experience for 0 frames... +[2023-03-07 15:31:27,903][239245] Decorrelating experience for 32 frames... +[2023-03-07 15:31:27,943][239255] Decorrelating experience for 64 frames... +[2023-03-07 15:31:27,945][239192] Decorrelating experience for 96 frames... +[2023-03-07 15:31:27,954][239215] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,019][239392] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,020][239198] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,040][239195] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,049][239109] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,049][239327] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,060][239178] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,103][239456] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,121][239213] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,123][239246] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,146][240430] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,197][239103] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,198][240273] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,237][239237] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,246][239210] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,249][239327] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,250][239109] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,301][239198] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,303][239245] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,311][239235] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,328][239826] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,371][239587] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,378][239111] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,419][239184] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,425][239204] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,451][240273] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,451][239193] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,499][239255] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,503][240430] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,505][239198] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,513][239211] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,548][239587] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,554][239214] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,607][239206] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,607][239248] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,662][238955] Signal inference workers to stop experience collection... +[2023-03-07 15:31:28,666][239843] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,666][239109] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,671][239101] InferenceWorker_p0-w0: stopping experience collection +[2023-03-07 15:31:28,677][239186] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,709][240273] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,712][239327] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,716][239204] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,729][239200] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,739][239846] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,794][239554] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,825][239232] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,857][239103] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,864][239585] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,866][239237] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,907][239181] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,908][240102] Decorrelating experience for 64 frames... +[2023-03-07 15:31:28,912][239245] Decorrelating experience for 96 frames... +[2023-03-07 15:31:28,919][239220] Decorrelating experience for 32 frames... +[2023-03-07 15:31:28,927][239222] Decorrelating experience for 0 frames... +[2023-03-07 15:31:28,971][239204] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,001][239186] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,048][239846] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,051][239232] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,066][239217] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,092][239234] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,106][239246] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,108][239243] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,110][240102] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,125][239220] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,147][239392] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,183][240101] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,226][239244] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,238][239184] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,268][239188] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,274][239254] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,295][240296] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,303][239232] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,307][239961] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,330][239253] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,335][239324] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,362][239237] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,403][239222] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,428][240209] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,447][239587] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,472][239217] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,477][238955] Signal inference workers to resume experience collection... +[2023-03-07 15:31:29,477][239101] InferenceWorker_p0-w0: resuming experience collection +[2023-03-07 15:31:29,483][239843] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,485][239203] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,488][239213] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,511][239202] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,515][240101] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,543][239392] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,579][239227] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,606][239246] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,627][239220] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,650][239195] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,661][239111] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,668][239232] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,670][239193] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,702][239186] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,703][239222] Decorrelating experience for 64 frames... +[2023-03-07 15:31:29,734][239209] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,757][239200] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,786][239106] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,806][239554] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,833][239586] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,860][239587] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,871][239248] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,875][239188] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,911][239250] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,913][239256] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,937][239324] Decorrelating experience for 32 frames... +[2023-03-07 15:31:29,961][239878] Decorrelating experience for 0 frames... +[2023-03-07 15:31:29,976][239961] Decorrelating experience for 96 frames... +[2023-03-07 15:31:29,990][239228] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,012][239200] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,045][239203] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,061][239222] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,078][239236] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,086][239225] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,111][239216] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,147][239844] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,148][239878] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,159][239227] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,167][239228] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,184][240297] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,267][239106] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,276][240101] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,276][239236] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,283][239235] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,293][239229] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,335][239200] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,341][239223] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,347][239194] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,369][240103] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,369][239111] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,412][236315] Fps is (10 sec: 3276.7, 60 sec: 3276.7, 300 sec: 3276.7). Total num frames: 16384. Throughput: 0: 143.6. Samples: 718. Policy #0 lag: (min: 0.0, avg: 0.0, max: 0.0) +[2023-03-07 15:31:30,413][236315] Avg episode reward: [(0, '1.876')] +[2023-03-07 15:31:30,488][239227] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,489][239228] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,502][239584] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,521][239844] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,547][239229] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,575][239391] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,576][239246] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,596][239588] Decorrelating experience for 0 frames... +[2023-03-07 15:31:30,605][239198] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,634][239101] Updated weights for policy 0, policy_version 10 (0.0310) +[2023-03-07 15:31:30,691][239253] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,701][239103] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,705][239325] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,736][239584] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,741][239223] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,749][239254] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,770][239878] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,794][239826] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,802][240550] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,818][239586] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,907][239194] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,909][239237] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,914][239178] Decorrelating experience for 96 frames... +[2023-03-07 15:31:30,953][240103] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,974][239110] Decorrelating experience for 32 frames... +[2023-03-07 15:31:30,977][239325] Decorrelating experience for 64 frames... +[2023-03-07 15:31:30,992][239254] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,003][239878] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,034][239181] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,058][239229] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,111][240101] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,122][240104] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,123][240209] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,156][239206] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,181][239194] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,183][239186] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,187][239101] Updated weights for policy 0, policy_version 20 (0.0009) +[2023-03-07 15:31:31,222][239104] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,237][239224] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,261][239212] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,309][239238] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,322][239256] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,340][240104] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,348][240209] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,368][240579] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,396][239209] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,406][239229] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,432][239227] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,454][239249] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,507][239215] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,523][239107] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,536][239203] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,547][239216] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,570][240579] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,575][239104] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,617][239325] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,620][239189] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,650][239108] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,661][239212] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,717][239324] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,719][239107] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,732][239249] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,745][239256] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,768][239239] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,781][239101] Updated weights for policy 0, policy_version 30 (0.0010) +[2023-03-07 15:31:31,807][240104] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,819][239190] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,855][239588] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,877][239108] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,889][239392] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,933][239216] Decorrelating experience for 96 frames... +[2023-03-07 15:31:31,937][239207] Decorrelating experience for 0 frames... +[2023-03-07 15:31:31,955][240579] Decorrelating experience for 64 frames... +[2023-03-07 15:31:31,978][239189] Decorrelating experience for 32 frames... +[2023-03-07 15:31:31,983][239251] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,021][239239] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,041][239260] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,061][239106] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,097][239212] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,130][239108] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,167][239844] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,170][239104] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,187][239223] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,195][239588] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,240][239101] Updated weights for policy 0, policy_version 40 (0.0009) +[2023-03-07 15:31:32,247][239201] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,251][239252] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,271][239110] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,287][239190] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,306][239239] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,383][239250] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,408][239844] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,411][240103] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,425][239199] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,461][239215] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,492][239242] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,493][239212] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,519][239110] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,520][239104] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,558][239207] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,596][239391] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,629][239238] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,639][239588] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,643][239826] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,675][239190] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,680][239101] Updated weights for policy 0, policy_version 50 (0.0010) +[2023-03-07 15:31:32,705][239106] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,715][239250] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,738][239177] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,809][239260] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,816][239252] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,867][239201] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,868][239242] Decorrelating experience for 32 frames... +[2023-03-07 15:31:32,892][240297] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,895][239207] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,920][239190] Decorrelating experience for 96 frames... +[2023-03-07 15:31:32,939][239391] Decorrelating experience for 64 frames... +[2023-03-07 15:31:32,940][239230] Decorrelating experience for 0 frames... +[2023-03-07 15:31:32,976][239586] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,028][239111] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,061][239324] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,091][239194] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,097][239101] Updated weights for policy 0, policy_version 60 (0.0010) +[2023-03-07 15:31:33,108][239252] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,111][239239] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,112][239201] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,154][240579] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,193][239207] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,195][239226] Decorrelating experience for 0 frames... +[2023-03-07 15:31:33,245][239826] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,286][239391] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,326][239238] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,331][239189] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,345][239235] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,350][239252] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,352][239228] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,380][239223] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,434][239202] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,447][239256] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,457][240550] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,507][239226] Decorrelating experience for 32 frames... +[2023-03-07 15:31:33,553][239201] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,558][240104] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,560][239250] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,595][239177] Decorrelating experience for 32 frames... +[2023-03-07 15:31:33,602][239242] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,605][239101] Updated weights for policy 0, policy_version 70 (0.0010) +[2023-03-07 15:31:33,630][239105] Decorrelating experience for 0 frames... +[2023-03-07 15:31:33,663][239189] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,684][239224] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,729][239226] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,761][239107] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,768][239199] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,799][239230] Decorrelating experience for 32 frames... +[2023-03-07 15:31:33,801][239251] Decorrelating experience for 32 frames... +[2023-03-07 15:31:33,835][239242] Decorrelating experience for 96 frames... +[2023-03-07 15:31:33,874][239177] Decorrelating experience for 64 frames... +[2023-03-07 15:31:33,888][239221] Decorrelating experience for 0 frames... +[2023-03-07 15:31:33,963][239226] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,009][239238] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,026][239224] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,034][239233] Decorrelating experience for 0 frames... +[2023-03-07 15:31:34,096][239101] Updated weights for policy 0, policy_version 80 (0.0010) +[2023-03-07 15:31:34,100][239221] Decorrelating experience for 32 frames... +[2023-03-07 15:31:34,101][239251] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,107][239204] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,210][239107] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,247][239105] Decorrelating experience for 32 frames... +[2023-03-07 15:31:34,261][239183] Decorrelating experience for 0 frames... +[2023-03-07 15:31:34,331][239221] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,347][239233] Decorrelating experience for 32 frames... +[2023-03-07 15:31:34,467][239183] Decorrelating experience for 32 frames... +[2023-03-07 15:31:34,570][239101] Updated weights for policy 0, policy_version 91 (0.0011) +[2023-03-07 15:31:34,585][239221] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,586][239105] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,593][239233] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,696][239230] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,717][239183] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,838][239105] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,842][239233] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,898][239101] Updated weights for policy 0, policy_version 101 (0.0010) +[2023-03-07 15:31:34,918][239260] Decorrelating experience for 64 frames... +[2023-03-07 15:31:34,951][239230] Decorrelating experience for 96 frames... +[2023-03-07 15:31:34,959][239183] Decorrelating experience for 96 frames... +[2023-03-07 15:31:35,131][239177] Decorrelating experience for 96 frames... +[2023-03-07 15:31:35,151][239260] Decorrelating experience for 96 frames... +[2023-03-07 15:31:35,266][239101] Updated weights for policy 0, policy_version 111 (0.0010) +[2023-03-07 15:31:35,336][240297] Decorrelating experience for 96 frames... +[2023-03-07 15:31:35,412][236315] Fps is (10 sec: 46693.8, 60 sec: 46693.8, 300 sec: 46693.8). Total num frames: 466944. Throughput: 0: 7749.7. Samples: 77498. Policy #0 lag: (min: 1.0, avg: 5.4, max: 11.0) +[2023-03-07 15:31:35,413][236315] Avg episode reward: [(0, '4.504')] +[2023-03-07 15:31:35,445][238955] Saving new best policy, reward=4.504! +[2023-03-07 15:31:35,558][239251] Decorrelating experience for 96 frames... +[2023-03-07 15:31:35,669][239101] Updated weights for policy 0, policy_version 121 (0.0011) +[2023-03-07 15:31:35,741][238955] Signal inference workers to stop experience collection... (50 times) +[2023-03-07 15:31:35,748][238955] Signal inference workers to resume experience collection... (50 times) +[2023-03-07 15:31:35,757][239101] InferenceWorker_p0-w0: stopping experience collection (50 times) +[2023-03-07 15:31:35,757][239101] InferenceWorker_p0-w0: resuming experience collection (50 times) +[2023-03-07 15:31:35,806][239258] Decorrelating experience for 0 frames... +[2023-03-07 15:31:36,002][239101] Updated weights for policy 0, policy_version 131 (0.0011) +[2023-03-07 15:31:36,012][239258] Decorrelating experience for 32 frames... +[2023-03-07 15:31:36,249][239258] Decorrelating experience for 64 frames... +[2023-03-07 15:31:36,261][239259] Decorrelating experience for 0 frames... +[2023-03-07 15:31:36,389][239101] Updated weights for policy 0, policy_version 142 (0.0011) +[2023-03-07 15:31:36,470][239259] Decorrelating experience for 32 frames... +[2023-03-07 15:31:36,493][239258] Decorrelating experience for 96 frames... +[2023-03-07 15:31:36,519][239209] Decorrelating experience for 64 frames... +[2023-03-07 15:31:36,703][239259] Decorrelating experience for 64 frames... +[2023-03-07 15:31:36,762][239101] Updated weights for policy 0, policy_version 152 (0.0011) +[2023-03-07 15:31:36,767][239253] Decorrelating experience for 64 frames... +[2023-03-07 15:31:36,768][239209] Decorrelating experience for 96 frames... +[2023-03-07 15:31:37,012][239259] Decorrelating experience for 96 frames... +[2023-03-07 15:31:37,016][239253] Decorrelating experience for 96 frames... +[2023-03-07 15:31:37,021][239236] Decorrelating experience for 96 frames... +[2023-03-07 15:31:37,099][239101] Updated weights for policy 0, policy_version 162 (0.0010) +[2023-03-07 15:31:37,393][239101] Updated weights for policy 0, policy_version 172 (0.0010) +[2023-03-07 15:31:37,763][236315] Heartbeat connected on Batcher_0 +[2023-03-07 15:31:37,771][236315] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 15:31:37,772][239101] Updated weights for policy 0, policy_version 182 (0.0009) +[2023-03-07 15:31:37,773][236315] Heartbeat connected on RolloutWorker_w0 +[2023-03-07 15:31:37,776][236315] Heartbeat connected on RolloutWorker_w1 +[2023-03-07 15:31:37,781][236315] Heartbeat connected on RolloutWorker_w3 +[2023-03-07 15:31:37,783][236315] Heartbeat connected on RolloutWorker_w2 +[2023-03-07 15:31:37,783][236315] Heartbeat connected on RolloutWorker_w4 +[2023-03-07 15:31:37,786][236315] Heartbeat connected on RolloutWorker_w5 +[2023-03-07 15:31:37,788][236315] Heartbeat connected on RolloutWorker_w6 +[2023-03-07 15:31:37,790][236315] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 15:31:37,795][236315] Heartbeat connected on RolloutWorker_w7 +[2023-03-07 15:31:37,796][236315] Heartbeat connected on RolloutWorker_w9 +[2023-03-07 15:31:37,797][236315] Heartbeat connected on RolloutWorker_w8 +[2023-03-07 15:31:37,798][236315] Heartbeat connected on RolloutWorker_w10 +[2023-03-07 15:31:37,800][236315] Heartbeat connected on RolloutWorker_w11 +[2023-03-07 15:31:37,802][236315] Heartbeat connected on RolloutWorker_w12 +[2023-03-07 15:31:37,804][236315] Heartbeat connected on RolloutWorker_w13 +[2023-03-07 15:31:37,806][236315] Heartbeat connected on RolloutWorker_w14 +[2023-03-07 15:31:37,811][236315] Heartbeat connected on RolloutWorker_w16 +[2023-03-07 15:31:37,812][236315] Heartbeat connected on RolloutWorker_w15 +[2023-03-07 15:31:37,829][236315] Heartbeat connected on RolloutWorker_w17 +[2023-03-07 15:31:37,832][236315] Heartbeat connected on RolloutWorker_w18 +[2023-03-07 15:31:37,834][236315] Heartbeat connected on RolloutWorker_w19 +[2023-03-07 15:31:37,837][236315] Heartbeat connected on RolloutWorker_w20 +[2023-03-07 15:31:37,839][236315] Heartbeat connected on RolloutWorker_w21 +[2023-03-07 15:31:37,840][236315] Heartbeat connected on RolloutWorker_w22 +[2023-03-07 15:31:37,841][236315] Heartbeat connected on RolloutWorker_w23 +[2023-03-07 15:31:37,848][236315] Heartbeat connected on RolloutWorker_w24 +[2023-03-07 15:31:37,849][236315] Heartbeat connected on RolloutWorker_w25 +[2023-03-07 15:31:37,850][236315] Heartbeat connected on RolloutWorker_w26 +[2023-03-07 15:31:37,850][236315] Heartbeat connected on RolloutWorker_w27 +[2023-03-07 15:31:37,854][236315] Heartbeat connected on RolloutWorker_w29 +[2023-03-07 15:31:37,855][236315] Heartbeat connected on RolloutWorker_w28 +[2023-03-07 15:31:37,856][236315] Heartbeat connected on RolloutWorker_w30 +[2023-03-07 15:31:37,858][236315] Heartbeat connected on RolloutWorker_w31 +[2023-03-07 15:31:37,862][236315] Heartbeat connected on RolloutWorker_w33 +[2023-03-07 15:31:37,863][236315] Heartbeat connected on RolloutWorker_w32 +[2023-03-07 15:31:37,864][236315] Heartbeat connected on RolloutWorker_w34 +[2023-03-07 15:31:37,866][236315] Heartbeat connected on RolloutWorker_w35 +[2023-03-07 15:31:37,869][236315] Heartbeat connected on RolloutWorker_w36 +[2023-03-07 15:31:37,871][236315] Heartbeat connected on RolloutWorker_w37 +[2023-03-07 15:31:37,873][236315] Heartbeat connected on RolloutWorker_w38 +[2023-03-07 15:31:37,880][236315] Heartbeat connected on RolloutWorker_w39 +[2023-03-07 15:31:37,883][236315] Heartbeat connected on RolloutWorker_w40 +[2023-03-07 15:31:37,884][236315] Heartbeat connected on RolloutWorker_w41 +[2023-03-07 15:31:37,885][236315] Heartbeat connected on RolloutWorker_w43 +[2023-03-07 15:31:37,885][236315] Heartbeat connected on RolloutWorker_w42 +[2023-03-07 15:31:37,886][236315] Heartbeat connected on RolloutWorker_w44 +[2023-03-07 15:31:37,891][236315] Heartbeat connected on RolloutWorker_w45 +[2023-03-07 15:31:37,892][236315] Heartbeat connected on RolloutWorker_w46 +[2023-03-07 15:31:37,892][236315] Heartbeat connected on RolloutWorker_w47 +[2023-03-07 15:31:37,894][236315] Heartbeat connected on RolloutWorker_w48 +[2023-03-07 15:31:37,898][236315] Heartbeat connected on RolloutWorker_w50 +[2023-03-07 15:31:37,899][236315] Heartbeat connected on RolloutWorker_w49 +[2023-03-07 15:31:37,900][236315] Heartbeat connected on RolloutWorker_w51 +[2023-03-07 15:31:37,905][236315] Heartbeat connected on RolloutWorker_w53 +[2023-03-07 15:31:37,905][236315] Heartbeat connected on RolloutWorker_w52 +[2023-03-07 15:31:37,906][236315] Heartbeat connected on RolloutWorker_w54 +[2023-03-07 15:31:37,909][236315] Heartbeat connected on RolloutWorker_w55 +[2023-03-07 15:31:37,912][236315] Heartbeat connected on RolloutWorker_w56 +[2023-03-07 15:31:37,913][236315] Heartbeat connected on RolloutWorker_w57 +[2023-03-07 15:31:37,915][236315] Heartbeat connected on RolloutWorker_w58 +[2023-03-07 15:31:37,919][236315] Heartbeat connected on RolloutWorker_w59 +[2023-03-07 15:31:37,920][236315] Heartbeat connected on RolloutWorker_w60 +[2023-03-07 15:31:37,921][236315] Heartbeat connected on RolloutWorker_w61 +[2023-03-07 15:31:37,925][236315] Heartbeat connected on RolloutWorker_w63 +[2023-03-07 15:31:37,926][236315] Heartbeat connected on RolloutWorker_w62 +[2023-03-07 15:31:37,927][236315] Heartbeat connected on RolloutWorker_w64 +[2023-03-07 15:31:37,929][236315] Heartbeat connected on RolloutWorker_w65 +[2023-03-07 15:31:37,931][236315] Heartbeat connected on RolloutWorker_w66 +[2023-03-07 15:31:37,934][236315] Heartbeat connected on RolloutWorker_w67 +[2023-03-07 15:31:37,936][236315] Heartbeat connected on RolloutWorker_w68 +[2023-03-07 15:31:37,938][236315] Heartbeat connected on RolloutWorker_w69 +[2023-03-07 15:31:37,940][236315] Heartbeat connected on RolloutWorker_w70 +[2023-03-07 15:31:37,942][236315] Heartbeat connected on RolloutWorker_w71 +[2023-03-07 15:31:37,946][236315] Heartbeat connected on RolloutWorker_w72 +[2023-03-07 15:31:37,947][236315] Heartbeat connected on RolloutWorker_w73 +[2023-03-07 15:31:37,951][236315] Heartbeat connected on RolloutWorker_w75 +[2023-03-07 15:31:37,953][236315] Heartbeat connected on RolloutWorker_w74 +[2023-03-07 15:31:37,953][236315] Heartbeat connected on RolloutWorker_w76 +[2023-03-07 15:31:37,955][236315] Heartbeat connected on RolloutWorker_w77 +[2023-03-07 15:31:37,959][236315] Heartbeat connected on RolloutWorker_w79 +[2023-03-07 15:31:37,961][236315] Heartbeat connected on RolloutWorker_w78 +[2023-03-07 15:31:37,961][236315] Heartbeat connected on RolloutWorker_w80 +[2023-03-07 15:31:37,999][236315] Heartbeat connected on RolloutWorker_w82 +[2023-03-07 15:31:38,000][236315] Heartbeat connected on RolloutWorker_w81 +[2023-03-07 15:31:38,002][236315] Heartbeat connected on RolloutWorker_w83 +[2023-03-07 15:31:38,003][236315] Heartbeat connected on RolloutWorker_w84 +[2023-03-07 15:31:38,006][236315] Heartbeat connected on RolloutWorker_w85 +[2023-03-07 15:31:38,007][236315] Heartbeat connected on RolloutWorker_w86 +[2023-03-07 15:31:38,009][236315] Heartbeat connected on RolloutWorker_w87 +[2023-03-07 15:31:38,014][236315] Heartbeat connected on RolloutWorker_w89 +[2023-03-07 15:31:38,015][236315] Heartbeat connected on RolloutWorker_w90 +[2023-03-07 15:31:38,017][236315] Heartbeat connected on RolloutWorker_w88 +[2023-03-07 15:31:38,019][236315] Heartbeat connected on RolloutWorker_w91 +[2023-03-07 15:31:38,020][236315] Heartbeat connected on RolloutWorker_w92 +[2023-03-07 15:31:38,022][236315] Heartbeat connected on RolloutWorker_w93 +[2023-03-07 15:31:38,024][236315] Heartbeat connected on RolloutWorker_w94 +[2023-03-07 15:31:38,028][236315] Heartbeat connected on RolloutWorker_w96 +[2023-03-07 15:31:38,030][236315] Heartbeat connected on RolloutWorker_w97 +[2023-03-07 15:31:38,031][236315] Heartbeat connected on RolloutWorker_w95 +[2023-03-07 15:31:38,034][236315] Heartbeat connected on RolloutWorker_w98 +[2023-03-07 15:31:38,036][236315] Heartbeat connected on RolloutWorker_w99 +[2023-03-07 15:31:38,036][236315] Heartbeat connected on RolloutWorker_w100 +[2023-03-07 15:31:38,038][236315] Heartbeat connected on RolloutWorker_w101 +[2023-03-07 15:31:38,042][236315] Heartbeat connected on RolloutWorker_w103 +[2023-03-07 15:31:38,044][236315] Heartbeat connected on RolloutWorker_w104 +[2023-03-07 15:31:38,045][236315] Heartbeat connected on RolloutWorker_w102 +[2023-03-07 15:31:38,046][236315] Heartbeat connected on RolloutWorker_w105 +[2023-03-07 15:31:38,052][236315] Heartbeat connected on RolloutWorker_w106 +[2023-03-07 15:31:38,052][236315] Heartbeat connected on RolloutWorker_w107 +[2023-03-07 15:31:38,053][236315] Heartbeat connected on RolloutWorker_w108 +[2023-03-07 15:31:38,054][236315] Heartbeat connected on RolloutWorker_w109 +[2023-03-07 15:31:38,059][236315] Heartbeat connected on RolloutWorker_w111 +[2023-03-07 15:31:38,059][236315] Heartbeat connected on RolloutWorker_w110 +[2023-03-07 15:31:38,061][236315] Heartbeat connected on RolloutWorker_w112 +[2023-03-07 15:31:38,063][239101] Updated weights for policy 0, policy_version 192 (0.0011) +[2023-03-07 15:31:38,063][236315] Heartbeat connected on RolloutWorker_w113 +[2023-03-07 15:31:38,065][236315] Heartbeat connected on RolloutWorker_w114 +[2023-03-07 15:31:38,067][236315] Heartbeat connected on RolloutWorker_w115 +[2023-03-07 15:31:38,069][236315] Heartbeat connected on RolloutWorker_w116 +[2023-03-07 15:31:38,072][236315] Heartbeat connected on RolloutWorker_w117 +[2023-03-07 15:31:38,073][236315] Heartbeat connected on RolloutWorker_w118 +[2023-03-07 15:31:38,075][236315] Heartbeat connected on RolloutWorker_w119 +[2023-03-07 15:31:38,471][239101] Updated weights for policy 0, policy_version 202 (0.0010) +[2023-03-07 15:31:38,833][239101] Updated weights for policy 0, policy_version 213 (0.0012) +[2023-03-07 15:31:39,177][239101] Updated weights for policy 0, policy_version 223 (0.0010) +[2023-03-07 15:31:39,222][238955] Signal inference workers to stop experience collection... (100 times) +[2023-03-07 15:31:39,227][238955] Signal inference workers to resume experience collection... (100 times) +[2023-03-07 15:31:39,229][239101] InferenceWorker_p0-w0: stopping experience collection (100 times) +[2023-03-07 15:31:39,237][239101] InferenceWorker_p0-w0: resuming experience collection (100 times) +[2023-03-07 15:31:39,573][239101] Updated weights for policy 0, policy_version 233 (0.0010) +[2023-03-07 15:31:39,899][239101] Updated weights for policy 0, policy_version 243 (0.0011) +[2023-03-07 15:31:40,211][239101] Updated weights for policy 0, policy_version 253 (0.0010) +[2023-03-07 15:31:40,412][236315] Fps is (10 sec: 104445.5, 60 sec: 70722.7, 300 sec: 70722.7). Total num frames: 1060864. Throughput: 0: 16313.9. Samples: 244714. Policy #0 lag: (min: 1.0, avg: 8.7, max: 16.0) +[2023-03-07 15:31:40,414][236315] Avg episode reward: [(0, '4.624')] +[2023-03-07 15:31:40,415][238955] Saving new best policy, reward=4.624! +[2023-03-07 15:31:40,625][239101] Updated weights for policy 0, policy_version 264 (0.0009) +[2023-03-07 15:31:40,933][239101] Updated weights for policy 0, policy_version 274 (0.0011) +[2023-03-07 15:31:41,276][239101] Updated weights for policy 0, policy_version 284 (0.0010) +[2023-03-07 15:31:41,646][239101] Updated weights for policy 0, policy_version 294 (0.0010) +[2023-03-07 15:31:41,960][239101] Updated weights for policy 0, policy_version 304 (0.0010) +[2023-03-07 15:31:42,191][238955] Signal inference workers to stop experience collection... (150 times) +[2023-03-07 15:31:42,198][238955] Signal inference workers to resume experience collection... (150 times) +[2023-03-07 15:31:42,200][239101] InferenceWorker_p0-w0: stopping experience collection (150 times) +[2023-03-07 15:31:42,208][239101] InferenceWorker_p0-w0: resuming experience collection (150 times) +[2023-03-07 15:31:42,282][239101] Updated weights for policy 0, policy_version 314 (0.0010) +[2023-03-07 15:31:42,556][239101] Updated weights for policy 0, policy_version 324 (0.0010) +[2023-03-07 15:31:42,880][239101] Updated weights for policy 0, policy_version 334 (0.0010) +[2023-03-07 15:31:43,202][239101] Updated weights for policy 0, policy_version 344 (0.0010) +[2023-03-07 15:31:43,558][239101] Updated weights for policy 0, policy_version 354 (0.0010) +[2023-03-07 15:31:43,879][239101] Updated weights for policy 0, policy_version 364 (0.0011) +[2023-03-07 15:31:44,201][239101] Updated weights for policy 0, policy_version 374 (0.0010) +[2023-03-07 15:31:44,559][239101] Updated weights for policy 0, policy_version 384 (0.0011) +[2023-03-07 15:31:44,866][239101] Updated weights for policy 0, policy_version 394 (0.0010) +[2023-03-07 15:31:45,190][239101] Updated weights for policy 0, policy_version 404 (0.0010) +[2023-03-07 15:31:45,261][238955] Signal inference workers to stop experience collection... (200 times) +[2023-03-07 15:31:45,273][238955] Signal inference workers to resume experience collection... (200 times) +[2023-03-07 15:31:45,275][239101] InferenceWorker_p0-w0: stopping experience collection (200 times) +[2023-03-07 15:31:45,283][239101] InferenceWorker_p0-w0: resuming experience collection (200 times) +[2023-03-07 15:31:45,412][236315] Fps is (10 sec: 121651.9, 60 sec: 84172.4, 300 sec: 84172.4). Total num frames: 1683456. Throughput: 0: 16881.5. Samples: 337632. Policy #0 lag: (min: 1.0, avg: 8.6, max: 16.0) +[2023-03-07 15:31:45,414][236315] Avg episode reward: [(0, '4.457')] +[2023-03-07 15:31:45,513][239101] Updated weights for policy 0, policy_version 414 (0.0011) +[2023-03-07 15:31:45,833][239101] Updated weights for policy 0, policy_version 424 (0.0010) +[2023-03-07 15:31:46,157][239101] Updated weights for policy 0, policy_version 434 (0.0011) +[2023-03-07 15:31:46,524][239101] Updated weights for policy 0, policy_version 444 (0.0010) +[2023-03-07 15:31:46,848][239101] Updated weights for policy 0, policy_version 454 (0.0011) +[2023-03-07 15:31:47,165][239101] Updated weights for policy 0, policy_version 464 (0.0009) +[2023-03-07 15:31:47,433][239101] Updated weights for policy 0, policy_version 474 (0.0011) +[2023-03-07 15:31:47,743][239101] Updated weights for policy 0, policy_version 484 (0.0011) +[2023-03-07 15:31:48,066][239101] Updated weights for policy 0, policy_version 494 (0.0009) +[2023-03-07 15:31:48,345][239101] Updated weights for policy 0, policy_version 504 (0.0009) +[2023-03-07 15:31:48,619][238955] Signal inference workers to stop experience collection... (250 times) +[2023-03-07 15:31:48,629][238955] Signal inference workers to resume experience collection... (250 times) +[2023-03-07 15:31:48,632][239101] InferenceWorker_p0-w0: stopping experience collection (250 times) +[2023-03-07 15:31:48,640][239101] InferenceWorker_p0-w0: resuming experience collection (250 times) +[2023-03-07 15:31:48,659][239101] Updated weights for policy 0, policy_version 514 (0.0010) +[2023-03-07 15:31:48,980][239101] Updated weights for policy 0, policy_version 524 (0.0011) +[2023-03-07 15:31:49,261][239101] Updated weights for policy 0, policy_version 534 (0.0011) +[2023-03-07 15:31:49,577][239101] Updated weights for policy 0, policy_version 544 (0.0010) +[2023-03-07 15:31:49,972][239101] Updated weights for policy 0, policy_version 554 (0.0011) +[2023-03-07 15:31:50,251][239101] Updated weights for policy 0, policy_version 564 (0.0010) +[2023-03-07 15:31:50,412][236315] Fps is (10 sec: 126570.0, 60 sec: 93060.9, 300 sec: 93060.9). Total num frames: 2326528. Throughput: 0: 21161.7. Samples: 529044. Policy #0 lag: (min: 1.0, avg: 9.3, max: 17.0) +[2023-03-07 15:31:50,413][236315] Avg episode reward: [(0, '4.936')] +[2023-03-07 15:31:50,427][238955] Saving new best policy, reward=4.936! +[2023-03-07 15:31:50,602][239101] Updated weights for policy 0, policy_version 574 (0.0010) +[2023-03-07 15:31:50,969][239101] Updated weights for policy 0, policy_version 585 (0.0010) +[2023-03-07 15:31:51,258][239101] Updated weights for policy 0, policy_version 595 (0.0010) +[2023-03-07 15:31:51,549][239101] Updated weights for policy 0, policy_version 605 (0.0011) +[2023-03-07 15:31:51,875][239101] Updated weights for policy 0, policy_version 615 (0.0010) +[2023-03-07 15:31:51,969][238955] Signal inference workers to stop experience collection... (300 times) +[2023-03-07 15:31:51,972][238955] Signal inference workers to resume experience collection... (300 times) +[2023-03-07 15:31:51,978][239101] InferenceWorker_p0-w0: stopping experience collection (300 times) +[2023-03-07 15:31:51,985][239101] InferenceWorker_p0-w0: resuming experience collection (300 times) +[2023-03-07 15:31:52,164][239101] Updated weights for policy 0, policy_version 625 (0.0011) +[2023-03-07 15:31:52,472][239101] Updated weights for policy 0, policy_version 635 (0.0011) +[2023-03-07 15:31:52,791][239101] Updated weights for policy 0, policy_version 645 (0.0010) +[2023-03-07 15:31:53,076][239101] Updated weights for policy 0, policy_version 655 (0.0010) +[2023-03-07 15:31:53,361][239101] Updated weights for policy 0, policy_version 665 (0.0011) +[2023-03-07 15:31:53,700][239101] Updated weights for policy 0, policy_version 675 (0.0011) +[2023-03-07 15:31:54,060][239101] Updated weights for policy 0, policy_version 685 (0.0010) +[2023-03-07 15:31:54,331][239101] Updated weights for policy 0, policy_version 695 (0.0009) +[2023-03-07 15:31:54,675][239101] Updated weights for policy 0, policy_version 705 (0.0010) +[2023-03-07 15:31:55,061][239101] Updated weights for policy 0, policy_version 716 (0.0013) +[2023-03-07 15:31:55,207][238955] Signal inference workers to stop experience collection... (350 times) +[2023-03-07 15:31:55,215][239101] InferenceWorker_p0-w0: stopping experience collection (350 times) +[2023-03-07 15:31:55,216][238955] Signal inference workers to resume experience collection... (350 times) +[2023-03-07 15:31:55,223][239101] InferenceWorker_p0-w0: resuming experience collection (350 times) +[2023-03-07 15:31:55,332][239101] Updated weights for policy 0, policy_version 726 (0.0009) +[2023-03-07 15:31:55,412][236315] Fps is (10 sec: 129432.7, 60 sec: 99259.2, 300 sec: 99259.2). Total num frames: 2977792. Throughput: 0: 24166.7. Samples: 725004. Policy #0 lag: (min: 0.0, avg: 8.9, max: 16.0) +[2023-03-07 15:31:55,413][236315] Avg episode reward: [(0, '5.088')] +[2023-03-07 15:31:55,438][238955] Saving new best policy, reward=5.088! +[2023-03-07 15:31:55,669][239101] Updated weights for policy 0, policy_version 736 (0.0011) +[2023-03-07 15:31:55,978][239101] Updated weights for policy 0, policy_version 746 (0.0012) +[2023-03-07 15:31:56,252][239101] Updated weights for policy 0, policy_version 756 (0.0010) +[2023-03-07 15:31:56,594][239101] Updated weights for policy 0, policy_version 766 (0.0010) +[2023-03-07 15:31:56,944][239101] Updated weights for policy 0, policy_version 776 (0.0012) +[2023-03-07 15:31:57,216][239101] Updated weights for policy 0, policy_version 786 (0.0011) +[2023-03-07 15:31:57,553][239101] Updated weights for policy 0, policy_version 796 (0.0010) +[2023-03-07 15:31:57,863][239101] Updated weights for policy 0, policy_version 806 (0.0010) +[2023-03-07 15:31:57,901][238955] Signal inference workers to stop experience collection... (400 times) +[2023-03-07 15:31:57,901][238955] Signal inference workers to resume experience collection... (400 times) +[2023-03-07 15:31:57,913][239101] InferenceWorker_p0-w0: stopping experience collection (400 times) +[2023-03-07 15:31:57,913][239101] InferenceWorker_p0-w0: resuming experience collection (400 times) +[2023-03-07 15:31:58,126][239101] Updated weights for policy 0, policy_version 816 (0.0011) +[2023-03-07 15:31:58,467][239101] Updated weights for policy 0, policy_version 826 (0.0010) +[2023-03-07 15:31:58,772][239101] Updated weights for policy 0, policy_version 836 (0.0010) +[2023-03-07 15:31:59,061][239101] Updated weights for policy 0, policy_version 846 (0.0011) +[2023-03-07 15:31:59,389][239101] Updated weights for policy 0, policy_version 856 (0.0011) +[2023-03-07 15:31:59,722][239101] Updated weights for policy 0, policy_version 866 (0.0011) +[2023-03-07 15:32:00,004][239101] Updated weights for policy 0, policy_version 876 (0.0012) +[2023-03-07 15:32:00,334][239101] Updated weights for policy 0, policy_version 886 (0.0010) +[2023-03-07 15:32:00,412][236315] Fps is (10 sec: 131482.6, 60 sec: 104038.4, 300 sec: 104038.4). Total num frames: 3641344. Throughput: 0: 23495.1. Samples: 822328. Policy #0 lag: (min: 2.0, avg: 8.9, max: 17.0) +[2023-03-07 15:32:00,413][236315] Avg episode reward: [(0, '5.710')] +[2023-03-07 15:32:00,436][238955] Saving new best policy, reward=5.710! +[2023-03-07 15:32:00,641][239101] Updated weights for policy 0, policy_version 896 (0.0009) +[2023-03-07 15:32:00,782][238955] Signal inference workers to stop experience collection... (450 times) +[2023-03-07 15:32:00,791][238955] Signal inference workers to resume experience collection... (450 times) +[2023-03-07 15:32:00,793][239101] InferenceWorker_p0-w0: stopping experience collection (450 times) +[2023-03-07 15:32:00,803][239101] InferenceWorker_p0-w0: resuming experience collection (450 times) +[2023-03-07 15:32:00,920][239101] Updated weights for policy 0, policy_version 906 (0.0010) +[2023-03-07 15:32:01,258][239101] Updated weights for policy 0, policy_version 916 (0.0010) +[2023-03-07 15:32:01,563][239101] Updated weights for policy 0, policy_version 926 (0.0008) +[2023-03-07 15:32:01,840][239101] Updated weights for policy 0, policy_version 936 (0.0010) +[2023-03-07 15:32:02,171][239101] Updated weights for policy 0, policy_version 946 (0.0014) +[2023-03-07 15:32:02,484][239101] Updated weights for policy 0, policy_version 956 (0.0010) +[2023-03-07 15:32:02,771][239101] Updated weights for policy 0, policy_version 966 (0.0009) +[2023-03-07 15:32:03,152][239101] Updated weights for policy 0, policy_version 976 (0.0010) +[2023-03-07 15:32:03,199][238955] Stopping Batcher_0... +[2023-03-07 15:32:03,200][238955] Loop batcher_evt_loop terminating... +[2023-03-07 15:32:03,201][236315] Component Batcher_0 stopped! +[2023-03-07 15:32:03,211][239101] Weights refcount: 2 0 +[2023-03-07 15:32:03,213][239101] Stopping InferenceWorker_p0-w0... +[2023-03-07 15:32:03,213][239101] Loop inference_proc0-0_evt_loop terminating... +[2023-03-07 15:32:03,213][236315] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 15:32:03,221][238955] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:32:03,337][239828] Stopping RolloutWorker_w92... +[2023-03-07 15:32:03,338][239828] Loop rollout_proc92_evt_loop terminating... +[2023-03-07 15:32:03,338][240550] Stopping RolloutWorker_w116... +[2023-03-07 15:32:03,338][240550] Loop rollout_proc116_evt_loop terminating... +[2023-03-07 15:32:03,338][236315] Component RolloutWorker_w92 stopped! +[2023-03-07 15:32:03,340][240297] Stopping RolloutWorker_w112... +[2023-03-07 15:32:03,340][240297] Loop rollout_proc112_evt_loop terminating... +[2023-03-07 15:32:03,341][236315] Component RolloutWorker_w116 stopped! +[2023-03-07 15:32:03,341][239207] Stopping RolloutWorker_w43... +[2023-03-07 15:32:03,341][239255] Stopping RolloutWorker_w78... +[2023-03-07 15:32:03,342][239207] Loop rollout_proc43_evt_loop terminating... +[2023-03-07 15:32:03,341][239105] Stopping RolloutWorker_w5... +[2023-03-07 15:32:03,342][239255] Loop rollout_proc78_evt_loop terminating... +[2023-03-07 15:32:03,342][239183] Stopping RolloutWorker_w29... +[2023-03-07 15:32:03,342][239105] Loop rollout_proc5_evt_loop terminating... +[2023-03-07 15:32:03,342][239183] Loop rollout_proc29_evt_loop terminating... +[2023-03-07 15:32:03,342][239254] Stopping RolloutWorker_w87... +[2023-03-07 15:32:03,342][239241] Stopping RolloutWorker_w77... +[2023-03-07 15:32:03,342][239254] Loop rollout_proc87_evt_loop terminating... +[2023-03-07 15:32:03,342][239257] Stopping RolloutWorker_w82... +[2023-03-07 15:32:03,342][239241] Loop rollout_proc77_evt_loop terminating... +[2023-03-07 15:32:03,343][239257] Loop rollout_proc82_evt_loop terminating... +[2023-03-07 15:32:03,343][239248] Stopping RolloutWorker_w64... +[2023-03-07 15:32:03,343][239205] Stopping RolloutWorker_w39... +[2023-03-07 15:32:03,343][239248] Loop rollout_proc64_evt_loop terminating... +[2023-03-07 15:32:03,344][239205] Loop rollout_proc39_evt_loop terminating... +[2023-03-07 15:32:03,344][239213] Stopping RolloutWorker_w53... +[2023-03-07 15:32:03,344][239240] Stopping RolloutWorker_w73... +[2023-03-07 15:32:03,345][239243] Stopping RolloutWorker_w79... +[2023-03-07 15:32:03,345][239391] Stopping RolloutWorker_w97... +[2023-03-07 15:32:03,345][239213] Loop rollout_proc53_evt_loop terminating... +[2023-03-07 15:32:03,345][239220] Stopping RolloutWorker_w34... +[2023-03-07 15:32:03,345][240101] Stopping RolloutWorker_w102... +[2023-03-07 15:32:03,345][239240] Loop rollout_proc73_evt_loop terminating... +[2023-03-07 15:32:03,345][239586] Stopping RolloutWorker_w107... +[2023-03-07 15:32:03,345][239243] Loop rollout_proc79_evt_loop terminating... +[2023-03-07 15:32:03,345][239391] Loop rollout_proc97_evt_loop terminating... +[2023-03-07 15:32:03,345][239234] Stopping RolloutWorker_w63... +[2023-03-07 15:32:03,345][239220] Loop rollout_proc34_evt_loop terminating... +[2023-03-07 15:32:03,345][240101] Loop rollout_proc102_evt_loop terminating... +[2023-03-07 15:32:03,343][236315] Component RolloutWorker_w112 stopped! +[2023-03-07 15:32:03,345][239586] Loop rollout_proc107_evt_loop terminating... +[2023-03-07 15:32:03,345][239234] Loop rollout_proc63_evt_loop terminating... +[2023-03-07 15:32:03,346][239112] Stopping RolloutWorker_w19... +[2023-03-07 15:32:03,346][239227] Stopping RolloutWorker_w48... +[2023-03-07 15:32:03,346][239112] Loop rollout_proc19_evt_loop terminating... +[2023-03-07 15:32:03,346][239227] Loop rollout_proc48_evt_loop terminating... +[2023-03-07 15:32:03,347][239249] Stopping RolloutWorker_w68... +[2023-03-07 15:32:03,347][239249] Loop rollout_proc68_evt_loop terminating... +[2023-03-07 15:32:03,346][236315] Component RolloutWorker_w43 stopped! +[2023-03-07 15:32:03,349][239106] Stopping RolloutWorker_w7... +[2023-03-07 15:32:03,348][236315] Component RolloutWorker_w78 stopped! +[2023-03-07 15:32:03,349][239106] Loop rollout_proc7_evt_loop terminating... +[2023-03-07 15:32:03,350][239209] Stopping RolloutWorker_w45... +[2023-03-07 15:32:03,349][236315] Component RolloutWorker_w5 stopped! +[2023-03-07 15:32:03,350][239209] Loop rollout_proc45_evt_loop terminating... +[2023-03-07 15:32:03,351][236315] Component RolloutWorker_w29 stopped! +[2023-03-07 15:32:03,352][239232] Stopping RolloutWorker_w58... +[2023-03-07 15:32:03,353][239232] Loop rollout_proc58_evt_loop terminating... +[2023-03-07 15:32:03,352][236315] Component RolloutWorker_w87 stopped! +[2023-03-07 15:32:03,354][239212] Stopping RolloutWorker_w51... +[2023-03-07 15:32:03,354][239216] Stopping RolloutWorker_w59... +[2023-03-07 15:32:03,354][239259] Stopping RolloutWorker_w84... +[2023-03-07 15:32:03,354][239216] Loop rollout_proc59_evt_loop terminating... +[2023-03-07 15:32:03,354][239259] Loop rollout_proc84_evt_loop terminating... +[2023-03-07 15:32:03,354][239212] Loop rollout_proc51_evt_loop terminating... +[2023-03-07 15:32:03,356][240103] Stopping RolloutWorker_w108... +[2023-03-07 15:32:03,356][240103] Loop rollout_proc108_evt_loop terminating... +[2023-03-07 15:32:03,356][239229] Stopping RolloutWorker_w52... +[2023-03-07 15:32:03,357][239229] Loop rollout_proc52_evt_loop terminating... +[2023-03-07 15:32:03,357][239107] Stopping RolloutWorker_w9... +[2023-03-07 15:32:03,355][236315] Component RolloutWorker_w77 stopped! +[2023-03-07 15:32:03,357][239107] Loop rollout_proc9_evt_loop terminating... +[2023-03-07 15:32:03,357][239826] Stopping RolloutWorker_w111... +[2023-03-07 15:32:03,357][240296] Stopping RolloutWorker_w119... +[2023-03-07 15:32:03,357][239326] Stopping RolloutWorker_w93... +[2023-03-07 15:32:03,358][239826] Loop rollout_proc111_evt_loop terminating... +[2023-03-07 15:32:03,358][240296] Loop rollout_proc119_evt_loop terminating... +[2023-03-07 15:32:03,358][239326] Loop rollout_proc93_evt_loop terminating... +[2023-03-07 15:32:03,358][239239] Stopping RolloutWorker_w69... +[2023-03-07 15:32:03,358][239324] Stopping RolloutWorker_w86... +[2023-03-07 15:32:03,358][239225] Stopping RolloutWorker_w44... +[2023-03-07 15:32:03,358][239239] Loop rollout_proc69_evt_loop terminating... +[2023-03-07 15:32:03,358][239253] Stopping RolloutWorker_w89... +[2023-03-07 15:32:03,358][239324] Loop rollout_proc86_evt_loop terminating... +[2023-03-07 15:32:03,358][240430] Stopping RolloutWorker_w114... +[2023-03-07 15:32:03,358][239225] Loop rollout_proc44_evt_loop terminating... +[2023-03-07 15:32:03,358][239244] Stopping RolloutWorker_w74... +[2023-03-07 15:32:03,358][239327] Stopping RolloutWorker_w99... +[2023-03-07 15:32:03,359][239253] Loop rollout_proc89_evt_loop terminating... +[2023-03-07 15:32:03,359][239233] Stopping RolloutWorker_w61... +[2023-03-07 15:32:03,359][240430] Loop rollout_proc114_evt_loop terminating... +[2023-03-07 15:32:03,359][239244] Loop rollout_proc74_evt_loop terminating... +[2023-03-07 15:32:03,359][239327] Loop rollout_proc99_evt_loop terminating... +[2023-03-07 15:32:03,359][239233] Loop rollout_proc61_evt_loop terminating... +[2023-03-07 15:32:03,358][236315] Component RolloutWorker_w82 stopped! +[2023-03-07 15:32:03,359][239456] Stopping RolloutWorker_w90... +[2023-03-07 15:32:03,361][240273] Stopping RolloutWorker_w110... +[2023-03-07 15:32:03,361][239193] Stopping RolloutWorker_w16... +[2023-03-07 15:32:03,361][239844] Stopping RolloutWorker_w94... +[2023-03-07 15:32:03,360][239456] Loop rollout_proc90_evt_loop terminating... +[2023-03-07 15:32:03,361][239224] Stopping RolloutWorker_w40... +[2023-03-07 15:32:03,361][239185] Stopping RolloutWorker_w2... +[2023-03-07 15:32:03,361][240102] Stopping RolloutWorker_w117... +[2023-03-07 15:32:03,361][239221] Stopping RolloutWorker_w38... +[2023-03-07 15:32:03,361][239246] Stopping RolloutWorker_w83... +[2023-03-07 15:32:03,361][239235] Stopping RolloutWorker_w60... +[2023-03-07 15:32:03,361][239194] Stopping RolloutWorker_w18... +[2023-03-07 15:32:03,362][240273] Loop rollout_proc110_evt_loop terminating... +[2023-03-07 15:32:03,361][239191] Stopping RolloutWorker_w12... +[2023-03-07 15:32:03,362][239193] Loop rollout_proc16_evt_loop terminating... +[2023-03-07 15:32:03,362][239187] Stopping RolloutWorker_w6... +[2023-03-07 15:32:03,362][239224] Loop rollout_proc40_evt_loop terminating... +[2023-03-07 15:32:03,362][239185] Loop rollout_proc2_evt_loop terminating... +[2023-03-07 15:32:03,362][239103] Stopping RolloutWorker_w1... +[2023-03-07 15:32:03,362][239844] Loop rollout_proc94_evt_loop terminating... +[2023-03-07 15:32:03,362][240102] Loop rollout_proc117_evt_loop terminating... +[2023-03-07 15:32:03,362][239221] Loop rollout_proc38_evt_loop terminating... +[2023-03-07 15:32:03,362][239246] Loop rollout_proc83_evt_loop terminating... +[2023-03-07 15:32:03,362][239235] Loop rollout_proc60_evt_loop terminating... +[2023-03-07 15:32:03,362][239194] Loop rollout_proc18_evt_loop terminating... +[2023-03-07 15:32:03,362][239110] Stopping RolloutWorker_w17... +[2023-03-07 15:32:03,361][236315] Component RolloutWorker_w64 stopped! +[2023-03-07 15:32:03,362][239191] Loop rollout_proc12_evt_loop terminating... +[2023-03-07 15:32:03,362][239231] Stopping RolloutWorker_w54... +[2023-03-07 15:32:03,362][239187] Loop rollout_proc6_evt_loop terminating... +[2023-03-07 15:32:03,362][239103] Loop rollout_proc1_evt_loop terminating... +[2023-03-07 15:32:03,362][240104] Stopping RolloutWorker_w96... +[2023-03-07 15:32:03,362][239198] Stopping RolloutWorker_w26... +[2023-03-07 15:32:03,362][239231] Loop rollout_proc54_evt_loop terminating... +[2023-03-07 15:32:03,362][239110] Loop rollout_proc17_evt_loop terminating... +[2023-03-07 15:32:03,362][239845] Stopping RolloutWorker_w115... +[2023-03-07 15:32:03,362][240104] Loop rollout_proc96_evt_loop terminating... +[2023-03-07 15:32:03,362][239211] Stopping RolloutWorker_w49... +[2023-03-07 15:32:03,363][239198] Loop rollout_proc26_evt_loop terminating... +[2023-03-07 15:32:03,363][239214] Stopping RolloutWorker_w55... +[2023-03-07 15:32:03,363][240579] Stopping RolloutWorker_w118... +[2023-03-07 15:32:03,362][239260] Stopping RolloutWorker_w91... +[2023-03-07 15:32:03,363][239211] Loop rollout_proc49_evt_loop terminating... +[2023-03-07 15:32:03,363][239845] Loop rollout_proc115_evt_loop terminating... +[2023-03-07 15:32:03,363][240579] Loop rollout_proc118_evt_loop terminating... +[2023-03-07 15:32:03,363][239214] Loop rollout_proc55_evt_loop terminating... +[2023-03-07 15:32:03,363][239588] Stopping RolloutWorker_w105... +[2023-03-07 15:32:03,363][239961] Stopping RolloutWorker_w104... +[2023-03-07 15:32:03,363][239260] Loop rollout_proc91_evt_loop terminating... +[2023-03-07 15:32:03,363][239961] Loop rollout_proc104_evt_loop terminating... +[2023-03-07 15:32:03,363][239588] Loop rollout_proc105_evt_loop terminating... +[2023-03-07 15:32:03,364][239111] Stopping RolloutWorker_w15... +[2023-03-07 15:32:03,364][239111] Loop rollout_proc15_evt_loop terminating... +[2023-03-07 15:32:03,364][236315] Component RolloutWorker_w39 stopped! +[2023-03-07 15:32:03,365][236315] Component RolloutWorker_w53 stopped! +[2023-03-07 15:32:03,366][239186] Stopping RolloutWorker_w4... +[2023-03-07 15:32:03,366][239197] Stopping RolloutWorker_w24... +[2023-03-07 15:32:03,366][239189] Stopping RolloutWorker_w8... +[2023-03-07 15:32:03,366][239190] Stopping RolloutWorker_w14... +[2023-03-07 15:32:03,367][239186] Loop rollout_proc4_evt_loop terminating... +[2023-03-07 15:32:03,366][239258] Stopping RolloutWorker_w80... +[2023-03-07 15:32:03,366][239204] Stopping RolloutWorker_w37... +[2023-03-07 15:32:03,367][239197] Loop rollout_proc24_evt_loop terminating... +[2023-03-07 15:32:03,366][239188] Stopping RolloutWorker_w10... +[2023-03-07 15:32:03,367][239584] Stopping RolloutWorker_w103... +[2023-03-07 15:32:03,367][239190] Loop rollout_proc14_evt_loop terminating... +[2023-03-07 15:32:03,367][239189] Loop rollout_proc8_evt_loop terminating... +[2023-03-07 15:32:03,367][239258] Loop rollout_proc80_evt_loop terminating... +[2023-03-07 15:32:03,367][239204] Loop rollout_proc37_evt_loop terminating... +[2023-03-07 15:32:03,367][239188] Loop rollout_proc10_evt_loop terminating... +[2023-03-07 15:32:03,367][239584] Loop rollout_proc103_evt_loop terminating... +[2023-03-07 15:32:03,367][239554] Stopping RolloutWorker_w101... +[2023-03-07 15:32:03,367][239587] Stopping RolloutWorker_w109... +[2023-03-07 15:32:03,367][239223] Stopping RolloutWorker_w42... +[2023-03-07 15:32:03,367][239878] Stopping RolloutWorker_w98... +[2023-03-07 15:32:03,367][239109] Stopping RolloutWorker_w11... +[2023-03-07 15:32:03,367][239210] Stopping RolloutWorker_w47... +[2023-03-07 15:32:03,367][239245] Stopping RolloutWorker_w72... +[2023-03-07 15:32:03,367][239256] Stopping RolloutWorker_w66... +[2023-03-07 15:32:03,367][239184] Stopping RolloutWorker_w25... +[2023-03-07 15:32:03,367][239242] Stopping RolloutWorker_w81... +[2023-03-07 15:32:03,367][239247] Stopping RolloutWorker_w75... +[2023-03-07 15:32:03,367][239325] Stopping RolloutWorker_w76... +[2023-03-07 15:32:03,367][239238] Stopping RolloutWorker_w71... +[2023-03-07 15:32:03,367][240209] Stopping RolloutWorker_w106... +[2023-03-07 15:32:03,367][239217] Stopping RolloutWorker_w36... +[2023-03-07 15:32:03,367][239392] Stopping RolloutWorker_w95... +[2023-03-07 15:32:03,367][239200] Stopping RolloutWorker_w30... +[2023-03-07 15:32:03,368][239587] Loop rollout_proc109_evt_loop terminating... +[2023-03-07 15:32:03,368][239223] Loop rollout_proc42_evt_loop terminating... +[2023-03-07 15:32:03,368][239210] Loop rollout_proc47_evt_loop terminating... +[2023-03-07 15:32:03,368][239554] Loop rollout_proc101_evt_loop terminating... +[2023-03-07 15:32:03,367][239237] Stopping RolloutWorker_w67... +[2023-03-07 15:32:03,367][239226] Stopping RolloutWorker_w46... +[2023-03-07 15:32:03,367][239230] Stopping RolloutWorker_w56... +[2023-03-07 15:32:03,367][239192] Stopping RolloutWorker_w20... +[2023-03-07 15:32:03,367][239177] Stopping RolloutWorker_w21... +[2023-03-07 15:32:03,367][239108] Stopping RolloutWorker_w13... +[2023-03-07 15:32:03,367][239250] Stopping RolloutWorker_w85... +[2023-03-07 15:32:03,368][239109] Loop rollout_proc11_evt_loop terminating... +[2023-03-07 15:32:03,368][239878] Loop rollout_proc98_evt_loop terminating... +[2023-03-07 15:32:03,368][239242] Loop rollout_proc81_evt_loop terminating... +[2023-03-07 15:32:03,368][239245] Loop rollout_proc72_evt_loop terminating... +[2023-03-07 15:32:03,368][239256] Loop rollout_proc66_evt_loop terminating... +[2023-03-07 15:32:03,368][239184] Loop rollout_proc25_evt_loop terminating... +[2023-03-07 15:32:03,368][239201] Stopping RolloutWorker_w33... +[2023-03-07 15:32:03,368][248936] Stopping RolloutWorker_w0... +[2023-03-07 15:32:03,368][239247] Loop rollout_proc75_evt_loop terminating... +[2023-03-07 15:32:03,368][239325] Loop rollout_proc76_evt_loop terminating... +[2023-03-07 15:32:03,368][240209] Loop rollout_proc106_evt_loop terminating... +[2023-03-07 15:32:03,368][239217] Loop rollout_proc36_evt_loop terminating... +[2023-03-07 15:32:03,368][239238] Loop rollout_proc71_evt_loop terminating... +[2023-03-07 15:32:03,368][239104] Stopping RolloutWorker_w3... +[2023-03-07 15:32:03,368][239392] Loop rollout_proc95_evt_loop terminating... +[2023-03-07 15:32:03,368][239252] Stopping RolloutWorker_w70... +[2023-03-07 15:32:03,368][239200] Loop rollout_proc30_evt_loop terminating... +[2023-03-07 15:32:03,368][239226] Loop rollout_proc46_evt_loop terminating... +[2023-03-07 15:32:03,368][239237] Loop rollout_proc67_evt_loop terminating... +[2023-03-07 15:32:03,368][239108] Loop rollout_proc13_evt_loop terminating... +[2023-03-07 15:32:03,368][239203] Stopping RolloutWorker_w35... +[2023-03-07 15:32:03,368][239230] Loop rollout_proc56_evt_loop terminating... +[2023-03-07 15:32:03,368][239192] Loop rollout_proc20_evt_loop terminating... +[2023-03-07 15:32:03,368][239177] Loop rollout_proc21_evt_loop terminating... +[2023-03-07 15:32:03,368][239250] Loop rollout_proc85_evt_loop terminating... +[2023-03-07 15:32:03,368][239201] Loop rollout_proc33_evt_loop terminating... +[2023-03-07 15:32:03,368][248936] Loop rollout_proc0_evt_loop terminating... +[2023-03-07 15:32:03,368][239178] Stopping RolloutWorker_w23... +[2023-03-07 15:32:03,368][239195] Stopping RolloutWorker_w22... +[2023-03-07 15:32:03,368][239846] Stopping RolloutWorker_w100... +[2023-03-07 15:32:03,368][239104] Loop rollout_proc3_evt_loop terminating... +[2023-03-07 15:32:03,368][239252] Loop rollout_proc70_evt_loop terminating... +[2023-03-07 15:32:03,368][239585] Stopping RolloutWorker_w88... +[2023-03-07 15:32:03,368][239203] Loop rollout_proc35_evt_loop terminating... +[2023-03-07 15:32:03,368][239228] Stopping RolloutWorker_w50... +[2023-03-07 15:32:03,368][239222] Stopping RolloutWorker_w32... +[2023-03-07 15:32:03,368][239215] Stopping RolloutWorker_w57... +[2023-03-07 15:32:03,368][239181] Stopping RolloutWorker_w27... +[2023-03-07 15:32:03,368][239206] Stopping RolloutWorker_w41... +[2023-03-07 15:32:03,368][239199] Stopping RolloutWorker_w28... +[2023-03-07 15:32:03,369][239195] Loop rollout_proc22_evt_loop terminating... +[2023-03-07 15:32:03,369][239178] Loop rollout_proc23_evt_loop terminating... +[2023-03-07 15:32:03,369][239585] Loop rollout_proc88_evt_loop terminating... +[2023-03-07 15:32:03,368][239843] Stopping RolloutWorker_w113... +[2023-03-07 15:32:03,369][239846] Loop rollout_proc100_evt_loop terminating... +[2023-03-07 15:32:03,369][239228] Loop rollout_proc50_evt_loop terminating... +[2023-03-07 15:32:03,369][239222] Loop rollout_proc32_evt_loop terminating... +[2023-03-07 15:32:03,369][239181] Loop rollout_proc27_evt_loop terminating... +[2023-03-07 15:32:03,369][239206] Loop rollout_proc41_evt_loop terminating... +[2023-03-07 15:32:03,369][239199] Loop rollout_proc28_evt_loop terminating... +[2023-03-07 15:32:03,369][239843] Loop rollout_proc113_evt_loop terminating... +[2023-03-07 15:32:03,368][239202] Stopping RolloutWorker_w31... +[2023-03-07 15:32:03,369][239215] Loop rollout_proc57_evt_loop terminating... +[2023-03-07 15:32:03,369][239202] Loop rollout_proc31_evt_loop terminating... +[2023-03-07 15:32:03,371][239236] Stopping RolloutWorker_w65... +[2023-03-07 15:32:03,372][239236] Loop rollout_proc65_evt_loop terminating... +[2023-03-07 15:32:03,372][239251] Stopping RolloutWorker_w62... +[2023-03-07 15:32:03,373][239251] Loop rollout_proc62_evt_loop terminating... +[2023-03-07 15:32:03,368][236315] Component RolloutWorker_w73 stopped! +[2023-03-07 15:32:03,377][236315] Component RolloutWorker_w79 stopped! +[2023-03-07 15:32:03,378][236315] Component RolloutWorker_w97 stopped! +[2023-03-07 15:32:03,379][236315] Component RolloutWorker_w34 stopped! +[2023-03-07 15:32:03,380][236315] Component RolloutWorker_w102 stopped! +[2023-03-07 15:32:03,381][236315] Component RolloutWorker_w107 stopped! +[2023-03-07 15:32:03,381][236315] Component RolloutWorker_w63 stopped! +[2023-03-07 15:32:03,384][236315] Component RolloutWorker_w19 stopped! +[2023-03-07 15:32:03,386][236315] Component RolloutWorker_w48 stopped! +[2023-03-07 15:32:03,388][236315] Component RolloutWorker_w68 stopped! +[2023-03-07 15:32:03,389][236315] Component RolloutWorker_w7 stopped! +[2023-03-07 15:32:03,391][238955] Saving new best policy, reward=7.182! +[2023-03-07 15:32:03,390][236315] Component RolloutWorker_w45 stopped! +[2023-03-07 15:32:03,392][236315] Component RolloutWorker_w58 stopped! +[2023-03-07 15:32:03,393][236315] Component RolloutWorker_w51 stopped! +[2023-03-07 15:32:03,393][236315] Component RolloutWorker_w59 stopped! +[2023-03-07 15:32:03,395][236315] Component RolloutWorker_w84 stopped! +[2023-03-07 15:32:03,396][236315] Component RolloutWorker_w108 stopped! +[2023-03-07 15:32:03,397][236315] Component RolloutWorker_w52 stopped! +[2023-03-07 15:32:03,397][236315] Component RolloutWorker_w9 stopped! +[2023-03-07 15:32:03,399][236315] Component RolloutWorker_w111 stopped! +[2023-03-07 15:32:03,399][236315] Component RolloutWorker_w93 stopped! +[2023-03-07 15:32:03,400][236315] Component RolloutWorker_w119 stopped! +[2023-03-07 15:32:03,401][236315] Component RolloutWorker_w69 stopped! +[2023-03-07 15:32:03,402][236315] Component RolloutWorker_w86 stopped! +[2023-03-07 15:32:03,403][236315] Component RolloutWorker_w44 stopped! +[2023-03-07 15:32:03,404][236315] Component RolloutWorker_w89 stopped! +[2023-03-07 15:32:03,404][236315] Component RolloutWorker_w114 stopped! +[2023-03-07 15:32:03,405][236315] Component RolloutWorker_w74 stopped! +[2023-03-07 15:32:03,406][236315] Component RolloutWorker_w99 stopped! +[2023-03-07 15:32:03,407][236315] Component RolloutWorker_w61 stopped! +[2023-03-07 15:32:03,408][236315] Component RolloutWorker_w90 stopped! +[2023-03-07 15:32:03,409][236315] Component RolloutWorker_w110 stopped! +[2023-03-07 15:32:03,409][236315] Component RolloutWorker_w16 stopped! +[2023-03-07 15:32:03,410][236315] Component RolloutWorker_w94 stopped! +[2023-03-07 15:32:03,411][236315] Component RolloutWorker_w83 stopped! +[2023-03-07 15:32:03,412][236315] Component RolloutWorker_w40 stopped! +[2023-03-07 15:32:03,413][236315] Component RolloutWorker_w2 stopped! +[2023-03-07 15:32:03,413][236315] Component RolloutWorker_w38 stopped! +[2023-03-07 15:32:03,414][236315] Component RolloutWorker_w117 stopped! +[2023-03-07 15:32:03,415][236315] Component RolloutWorker_w60 stopped! +[2023-03-07 15:32:03,416][236315] Component RolloutWorker_w12 stopped! +[2023-03-07 15:32:03,417][236315] Component RolloutWorker_w18 stopped! +[2023-03-07 15:32:03,417][236315] Component RolloutWorker_w6 stopped! +[2023-03-07 15:32:03,418][236315] Component RolloutWorker_w1 stopped! +[2023-03-07 15:32:03,419][236315] Component RolloutWorker_w17 stopped! +[2023-03-07 15:32:03,420][236315] Component RolloutWorker_w54 stopped! +[2023-03-07 15:32:03,421][236315] Component RolloutWorker_w26 stopped! +[2023-03-07 15:32:03,421][236315] Component RolloutWorker_w96 stopped! +[2023-03-07 15:32:03,422][236315] Component RolloutWorker_w115 stopped! +[2023-03-07 15:32:03,423][236315] Component RolloutWorker_w91 stopped! +[2023-03-07 15:32:03,424][236315] Component RolloutWorker_w49 stopped! +[2023-03-07 15:32:03,424][236315] Component RolloutWorker_w55 stopped! +[2023-03-07 15:32:03,425][236315] Component RolloutWorker_w118 stopped! +[2023-03-07 15:32:03,426][236315] Component RolloutWorker_w105 stopped! +[2023-03-07 15:32:03,427][236315] Component RolloutWorker_w104 stopped! +[2023-03-07 15:32:03,427][236315] Component RolloutWorker_w15 stopped! +[2023-03-07 15:32:03,428][236315] Component RolloutWorker_w4 stopped! +[2023-03-07 15:32:03,429][236315] Component RolloutWorker_w24 stopped! +[2023-03-07 15:32:03,430][236315] Component RolloutWorker_w8 stopped! +[2023-03-07 15:32:03,430][236315] Component RolloutWorker_w14 stopped! +[2023-03-07 15:32:03,431][236315] Component RolloutWorker_w37 stopped! +[2023-03-07 15:32:03,432][236315] Component RolloutWorker_w80 stopped! +[2023-03-07 15:32:03,432][236315] Component RolloutWorker_w10 stopped! +[2023-03-07 15:32:03,433][236315] Component RolloutWorker_w103 stopped! +[2023-03-07 15:32:03,434][236315] Component RolloutWorker_w101 stopped! +[2023-03-07 15:32:03,435][236315] Component RolloutWorker_w109 stopped! +[2023-03-07 15:32:03,435][236315] Component RolloutWorker_w98 stopped! +[2023-03-07 15:32:03,436][236315] Component RolloutWorker_w66 stopped! +[2023-03-07 15:32:03,437][236315] Component RolloutWorker_w72 stopped! +[2023-03-07 15:32:03,437][236315] Component RolloutWorker_w11 stopped! +[2023-03-07 15:32:03,438][236315] Component RolloutWorker_w25 stopped! +[2023-03-07 15:32:03,439][236315] Component RolloutWorker_w42 stopped! +[2023-03-07 15:32:03,440][236315] Component RolloutWorker_w81 stopped! +[2023-03-07 15:32:03,440][236315] Component RolloutWorker_w47 stopped! +[2023-03-07 15:32:03,441][236315] Component RolloutWorker_w75 stopped! +[2023-03-07 15:32:03,442][236315] Component RolloutWorker_w76 stopped! +[2023-03-07 15:32:03,443][236315] Component RolloutWorker_w106 stopped! +[2023-03-07 15:32:03,444][236315] Component RolloutWorker_w71 stopped! +[2023-03-07 15:32:03,444][236315] Component RolloutWorker_w95 stopped! +[2023-03-07 15:32:03,445][236315] Component RolloutWorker_w36 stopped! +[2023-03-07 15:32:03,446][236315] Component RolloutWorker_w30 stopped! +[2023-03-07 15:32:03,446][236315] Component RolloutWorker_w56 stopped! +[2023-03-07 15:32:03,447][236315] Component RolloutWorker_w20 stopped! +[2023-03-07 15:32:03,448][236315] Component RolloutWorker_w67 stopped! +[2023-03-07 15:32:03,448][236315] Component RolloutWorker_w46 stopped! +[2023-03-07 15:32:03,449][236315] Component RolloutWorker_w21 stopped! +[2023-03-07 15:32:03,450][236315] Component RolloutWorker_w85 stopped! +[2023-03-07 15:32:03,450][236315] Component RolloutWorker_w13 stopped! +[2023-03-07 15:32:03,451][236315] Component RolloutWorker_w33 stopped! +[2023-03-07 15:32:03,452][236315] Component RolloutWorker_w0 stopped! +[2023-03-07 15:32:03,453][236315] Component RolloutWorker_w3 stopped! +[2023-03-07 15:32:03,453][236315] Component RolloutWorker_w70 stopped! +[2023-03-07 15:32:03,454][236315] Component RolloutWorker_w35 stopped! +[2023-03-07 15:32:03,455][236315] Component RolloutWorker_w100 stopped! +[2023-03-07 15:32:03,455][236315] Component RolloutWorker_w23 stopped! +[2023-03-07 15:32:03,455][236315] Component RolloutWorker_w22 stopped! +[2023-03-07 15:32:03,456][236315] Component RolloutWorker_w31 stopped! +[2023-03-07 15:32:03,456][236315] Component RolloutWorker_w88 stopped! +[2023-03-07 15:32:03,457][236315] Component RolloutWorker_w32 stopped! +[2023-03-07 15:32:03,457][236315] Component RolloutWorker_w57 stopped! +[2023-03-07 15:32:03,458][236315] Component RolloutWorker_w50 stopped! +[2023-03-07 15:32:03,458][236315] Component RolloutWorker_w27 stopped! +[2023-03-07 15:32:03,459][236315] Component RolloutWorker_w28 stopped! +[2023-03-07 15:32:03,459][236315] Component RolloutWorker_w41 stopped! +[2023-03-07 15:32:03,459][236315] Component RolloutWorker_w113 stopped! +[2023-03-07 15:32:03,460][236315] Component RolloutWorker_w65 stopped! +[2023-03-07 15:32:03,460][236315] Component RolloutWorker_w62 stopped! +[2023-03-07 15:32:03,481][238955] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:32:03,607][238955] Stopping LearnerWorker_p0... +[2023-03-07 15:32:03,608][238955] Loop learner_proc0_evt_loop terminating... +[2023-03-07 15:32:03,608][236315] Component LearnerWorker_p0 stopped! +[2023-03-07 15:32:03,609][236315] Waiting for process learner_proc0 to stop... +[2023-03-07 15:32:04,738][236315] Waiting for process inference_proc0-0 to join... +[2023-03-07 15:32:04,740][236315] Waiting for process rollout_proc0 to join... +[2023-03-07 15:32:04,741][236315] Waiting for process rollout_proc1 to join... +[2023-03-07 15:32:04,741][236315] Waiting for process rollout_proc2 to join... +[2023-03-07 15:32:04,742][236315] Waiting for process rollout_proc3 to join... +[2023-03-07 15:32:04,743][236315] Waiting for process rollout_proc4 to join... +[2023-03-07 15:32:04,744][236315] Waiting for process rollout_proc5 to join... +[2023-03-07 15:32:04,744][236315] Waiting for process rollout_proc6 to join... +[2023-03-07 15:32:04,745][236315] Waiting for process rollout_proc7 to join... +[2023-03-07 15:32:04,746][236315] Waiting for process rollout_proc8 to join... +[2023-03-07 15:32:04,747][236315] Waiting for process rollout_proc9 to join... +[2023-03-07 15:32:04,748][236315] Waiting for process rollout_proc10 to join... +[2023-03-07 15:32:04,748][236315] Waiting for process rollout_proc11 to join... +[2023-03-07 15:32:04,749][236315] Waiting for process rollout_proc12 to join... +[2023-03-07 15:32:04,750][236315] Waiting for process rollout_proc13 to join... +[2023-03-07 15:32:04,751][236315] Waiting for process rollout_proc14 to join... +[2023-03-07 15:32:04,751][236315] Waiting for process rollout_proc15 to join... +[2023-03-07 15:32:04,752][236315] Waiting for process rollout_proc16 to join... +[2023-03-07 15:32:04,753][236315] Waiting for process rollout_proc17 to join... +[2023-03-07 15:32:04,754][236315] Waiting for process rollout_proc18 to join... +[2023-03-07 15:32:04,755][236315] Waiting for process rollout_proc19 to join... +[2023-03-07 15:32:04,756][236315] Waiting for process rollout_proc20 to join... +[2023-03-07 15:32:04,756][236315] Waiting for process rollout_proc21 to join... +[2023-03-07 15:32:04,757][236315] Waiting for process rollout_proc22 to join... +[2023-03-07 15:32:04,758][236315] Waiting for process rollout_proc23 to join... +[2023-03-07 15:32:04,759][236315] Waiting for process rollout_proc24 to join... +[2023-03-07 15:32:04,759][236315] Waiting for process rollout_proc25 to join... +[2023-03-07 15:32:04,760][236315] Waiting for process rollout_proc26 to join... +[2023-03-07 15:32:04,761][236315] Waiting for process rollout_proc27 to join... +[2023-03-07 15:32:04,762][236315] Waiting for process rollout_proc28 to join... +[2023-03-07 15:32:04,763][236315] Waiting for process rollout_proc29 to join... +[2023-03-07 15:32:04,764][236315] Waiting for process rollout_proc30 to join... +[2023-03-07 15:32:04,765][236315] Waiting for process rollout_proc31 to join... +[2023-03-07 15:32:04,765][236315] Waiting for process rollout_proc32 to join... +[2023-03-07 15:32:04,766][236315] Waiting for process rollout_proc33 to join... +[2023-03-07 15:32:04,767][236315] Waiting for process rollout_proc34 to join... +[2023-03-07 15:32:04,768][236315] Waiting for process rollout_proc35 to join... +[2023-03-07 15:32:04,769][236315] Waiting for process rollout_proc36 to join... +[2023-03-07 15:32:04,770][236315] Waiting for process rollout_proc37 to join... +[2023-03-07 15:32:04,771][236315] Waiting for process rollout_proc38 to join... +[2023-03-07 15:32:04,772][236315] Waiting for process rollout_proc39 to join... +[2023-03-07 15:32:04,772][236315] Waiting for process rollout_proc40 to join... +[2023-03-07 15:32:04,773][236315] Waiting for process rollout_proc41 to join... +[2023-03-07 15:32:04,776][236315] Waiting for process rollout_proc42 to join... +[2023-03-07 15:32:04,777][236315] Waiting for process rollout_proc43 to join... +[2023-03-07 15:32:04,777][236315] Waiting for process rollout_proc44 to join... +[2023-03-07 15:32:04,778][236315] Waiting for process rollout_proc45 to join... +[2023-03-07 15:32:04,779][236315] Waiting for process rollout_proc46 to join... +[2023-03-07 15:32:04,780][236315] Waiting for process rollout_proc47 to join... +[2023-03-07 15:32:04,781][236315] Waiting for process rollout_proc48 to join... +[2023-03-07 15:32:04,781][236315] Waiting for process rollout_proc49 to join... +[2023-03-07 15:32:04,782][236315] Waiting for process rollout_proc50 to join... +[2023-03-07 15:32:04,783][236315] Waiting for process rollout_proc51 to join... +[2023-03-07 15:32:04,784][236315] Waiting for process rollout_proc52 to join... +[2023-03-07 15:32:04,785][236315] Waiting for process rollout_proc53 to join... +[2023-03-07 15:32:04,786][236315] Waiting for process rollout_proc54 to join... +[2023-03-07 15:32:04,789][236315] Waiting for process rollout_proc55 to join... +[2023-03-07 15:32:04,790][236315] Waiting for process rollout_proc56 to join... +[2023-03-07 15:32:04,791][236315] Waiting for process rollout_proc57 to join... +[2023-03-07 15:32:04,792][236315] Waiting for process rollout_proc58 to join... +[2023-03-07 15:32:04,793][236315] Waiting for process rollout_proc59 to join... +[2023-03-07 15:32:04,793][236315] Waiting for process rollout_proc60 to join... +[2023-03-07 15:32:04,798][236315] Waiting for process rollout_proc61 to join... +[2023-03-07 15:32:04,799][236315] Waiting for process rollout_proc62 to join... +[2023-03-07 15:32:04,799][236315] Waiting for process rollout_proc63 to join... +[2023-03-07 15:32:04,801][236315] Waiting for process rollout_proc64 to join... +[2023-03-07 15:32:04,802][236315] Waiting for process rollout_proc65 to join... +[2023-03-07 15:32:04,805][236315] Waiting for process rollout_proc66 to join... +[2023-03-07 15:32:04,805][236315] Waiting for process rollout_proc67 to join... +[2023-03-07 15:32:04,807][236315] Waiting for process rollout_proc68 to join... +[2023-03-07 15:32:04,807][236315] Waiting for process rollout_proc69 to join... +[2023-03-07 15:32:04,808][236315] Waiting for process rollout_proc70 to join... +[2023-03-07 15:32:04,809][236315] Waiting for process rollout_proc71 to join... +[2023-03-07 15:32:04,811][236315] Waiting for process rollout_proc72 to join... +[2023-03-07 15:32:04,812][236315] Waiting for process rollout_proc73 to join... +[2023-03-07 15:32:04,813][236315] Waiting for process rollout_proc74 to join... +[2023-03-07 15:32:04,814][236315] Waiting for process rollout_proc75 to join... +[2023-03-07 15:32:04,815][236315] Waiting for process rollout_proc76 to join... +[2023-03-07 15:32:04,815][236315] Waiting for process rollout_proc77 to join... +[2023-03-07 15:32:04,816][236315] Waiting for process rollout_proc78 to join... +[2023-03-07 15:32:04,817][236315] Waiting for process rollout_proc79 to join... +[2023-03-07 15:32:04,818][236315] Waiting for process rollout_proc80 to join... +[2023-03-07 15:32:04,820][236315] Waiting for process rollout_proc81 to join... +[2023-03-07 15:32:04,821][236315] Waiting for process rollout_proc82 to join... +[2023-03-07 15:32:04,822][236315] Waiting for process rollout_proc83 to join... +[2023-03-07 15:32:04,823][236315] Waiting for process rollout_proc84 to join... +[2023-03-07 15:32:04,823][236315] Waiting for process rollout_proc85 to join... +[2023-03-07 15:32:04,824][236315] Waiting for process rollout_proc86 to join... +[2023-03-07 15:32:04,825][236315] Waiting for process rollout_proc87 to join... +[2023-03-07 15:32:04,828][236315] Waiting for process rollout_proc88 to join... +[2023-03-07 15:32:04,828][236315] Waiting for process rollout_proc89 to join... +[2023-03-07 15:32:04,829][236315] Waiting for process rollout_proc90 to join... +[2023-03-07 15:32:04,830][236315] Waiting for process rollout_proc91 to join... +[2023-03-07 15:32:04,831][236315] Waiting for process rollout_proc92 to join... +[2023-03-07 15:32:04,832][236315] Waiting for process rollout_proc93 to join... +[2023-03-07 15:32:04,833][236315] Waiting for process rollout_proc94 to join... +[2023-03-07 15:32:04,833][236315] Waiting for process rollout_proc95 to join... +[2023-03-07 15:32:04,837][236315] Waiting for process rollout_proc96 to join... +[2023-03-07 15:32:04,838][236315] Waiting for process rollout_proc97 to join... +[2023-03-07 15:32:04,839][236315] Waiting for process rollout_proc98 to join... +[2023-03-07 15:32:04,840][236315] Waiting for process rollout_proc99 to join... +[2023-03-07 15:32:04,840][236315] Waiting for process rollout_proc100 to join... +[2023-03-07 15:32:04,841][236315] Waiting for process rollout_proc101 to join... +[2023-03-07 15:32:04,843][236315] Waiting for process rollout_proc102 to join... +[2023-03-07 15:32:04,844][236315] Waiting for process rollout_proc103 to join... +[2023-03-07 15:32:04,845][236315] Waiting for process rollout_proc104 to join... +[2023-03-07 15:32:04,846][236315] Waiting for process rollout_proc105 to join... +[2023-03-07 15:32:04,846][236315] Waiting for process rollout_proc106 to join... +[2023-03-07 15:32:04,847][236315] Waiting for process rollout_proc107 to join... +[2023-03-07 15:32:04,848][236315] Waiting for process rollout_proc108 to join... +[2023-03-07 15:32:04,849][236315] Waiting for process rollout_proc109 to join... +[2023-03-07 15:32:04,852][236315] Waiting for process rollout_proc110 to join... +[2023-03-07 15:32:04,853][236315] Waiting for process rollout_proc111 to join... +[2023-03-07 15:32:04,854][236315] Waiting for process rollout_proc112 to join... +[2023-03-07 15:32:04,855][236315] Waiting for process rollout_proc113 to join... +[2023-03-07 15:32:04,855][236315] Waiting for process rollout_proc114 to join... +[2023-03-07 15:32:04,856][236315] Waiting for process rollout_proc115 to join... +[2023-03-07 15:32:04,857][236315] Waiting for process rollout_proc116 to join... +[2023-03-07 15:32:04,860][236315] Waiting for process rollout_proc117 to join... +[2023-03-07 15:32:04,860][236315] Waiting for process rollout_proc118 to join... +[2023-03-07 15:32:04,861][236315] Waiting for process rollout_proc119 to join... +[2023-03-07 15:32:04,862][236315] Batcher 0 profile tree view: +batching: 8.5475, releasing_batches: 1.7587 +[2023-03-07 15:32:04,863][236315] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0003 + wait_policy_total: 4.9626 +update_model: 1.5047 + weight_update: 0.0010 +one_step: 0.0059 + handle_policy_step: 30.2144 + deserialize: 4.9301, stack: 0.1299, obs_to_device_normalize: 6.0325, forward: 7.5373, send_messages: 7.4461 + prepare_outputs: 3.3605 + to_cpu: 1.5272 +[2023-03-07 15:32:04,863][236315] Learner 0 profile tree view: +misc: 0.0042, prepare_batch: 7.8980 +train: 14.9381 + epoch_init: 0.0042, minibatch_init: 0.0042, losses_postprocess: 0.2660, kl_divergence: 0.2326, after_optimizer: 0.3681 + calculate_losses: 5.7767 + losses_init: 0.0028, forward_head: 0.6161, bptt_initial: 2.7001, tail: 0.4777, advantages_returns: 0.1425, losses: 0.8200 + bptt: 0.8765 + bptt_forward_core: 0.8383 + update: 7.9928 + clip: 1.1126 +[2023-03-07 15:32:04,864][236315] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0117, enqueue_policy_requests: 0.6162, env_step: 7.9088, overhead: 0.6936, complete_rollouts: 0.0178 +save_policy_outputs: 0.7882 + split_output_tensors: 0.3733 +[2023-03-07 15:32:04,865][236315] RolloutWorker_w119 profile tree view: +wait_for_trajectories: 0.0110, enqueue_policy_requests: 0.6116, env_step: 7.6528, overhead: 0.6713, complete_rollouts: 0.0168 +save_policy_outputs: 0.7615 + split_output_tensors: 0.3602 +[2023-03-07 15:32:04,868][236315] Loop Runner_EvtLoop terminating... +[2023-03-07 15:32:04,870][236315] Runner profile tree view: +main_loop: 46.7885 +[2023-03-07 15:32:04,871][236315] Collected {0: 4009984}, FPS: 85704.5 +[2023-03-07 15:32:18,532][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:32:18,533][236315] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 15:32:18,533][236315] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:32:18,534][236315] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:32:18,535][236315] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:32:18,535][236315] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:32:18,536][236315] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:32:18,536][236315] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:32:18,537][236315] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 15:32:18,537][236315] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 15:32:18,538][236315] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:32:18,539][236315] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:32:18,539][236315] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:32:18,540][236315] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:32:18,540][236315] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:32:18,547][236315] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:32:18,548][236315] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:32:18,549][236315] RunningMeanStd input shape: (1,) +[2023-03-07 15:32:18,561][236315] ConvEncoder: input_channels=3 +[2023-03-07 15:32:18,650][236315] Conv encoder output size: 512 +[2023-03-07 15:32:18,651][236315] Policy head output size: 512 +[2023-03-07 15:32:20,222][236315] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:32:21,110][236315] Num frames 100... +[2023-03-07 15:32:21,193][236315] Num frames 200... +[2023-03-07 15:32:21,275][236315] Num frames 300... +[2023-03-07 15:32:21,357][236315] Num frames 400... +[2023-03-07 15:32:21,442][236315] Num frames 500... +[2023-03-07 15:32:21,506][236315] Avg episode rewards: #0: 8.120, true rewards: #0: 5.120 +[2023-03-07 15:32:21,507][236315] Avg episode reward: 8.120, avg true_objective: 5.120 +[2023-03-07 15:32:21,607][236315] Num frames 600... +[2023-03-07 15:32:21,691][236315] Num frames 700... +[2023-03-07 15:32:21,775][236315] Num frames 800... +[2023-03-07 15:32:21,861][236315] Num frames 900... +[2023-03-07 15:32:21,945][236315] Num frames 1000... +[2023-03-07 15:32:22,046][236315] Avg episode rewards: #0: 8.780, true rewards: #0: 5.280 +[2023-03-07 15:32:22,047][236315] Avg episode reward: 8.780, avg true_objective: 5.280 +[2023-03-07 15:32:22,101][236315] Num frames 1100... +[2023-03-07 15:32:22,186][236315] Num frames 1200... +[2023-03-07 15:32:22,270][236315] Num frames 1300... +[2023-03-07 15:32:22,354][236315] Num frames 1400... +[2023-03-07 15:32:22,437][236315] Num frames 1500... +[2023-03-07 15:32:22,522][236315] Num frames 1600... +[2023-03-07 15:32:22,610][236315] Num frames 1700... +[2023-03-07 15:32:22,694][236315] Num frames 1800... +[2023-03-07 15:32:22,768][236315] Avg episode rewards: #0: 11.413, true rewards: #0: 6.080 +[2023-03-07 15:32:22,769][236315] Avg episode reward: 11.413, avg true_objective: 6.080 +[2023-03-07 15:32:22,832][236315] Num frames 1900... +[2023-03-07 15:32:22,919][236315] Num frames 2000... +[2023-03-07 15:32:23,004][236315] Num frames 2100... +[2023-03-07 15:32:23,088][236315] Num frames 2200... +[2023-03-07 15:32:23,203][236315] Avg episode rewards: #0: 9.930, true rewards: #0: 5.680 +[2023-03-07 15:32:23,204][236315] Avg episode reward: 9.930, avg true_objective: 5.680 +[2023-03-07 15:32:23,240][236315] Num frames 2300... +[2023-03-07 15:32:23,335][236315] Num frames 2400... +[2023-03-07 15:32:23,418][236315] Num frames 2500... +[2023-03-07 15:32:23,501][236315] Num frames 2600... +[2023-03-07 15:32:23,585][236315] Num frames 2700... +[2023-03-07 15:32:23,709][236315] Avg episode rewards: #0: 9.368, true rewards: #0: 5.568 +[2023-03-07 15:32:23,711][236315] Avg episode reward: 9.368, avg true_objective: 5.568 +[2023-03-07 15:32:23,729][236315] Num frames 2800... +[2023-03-07 15:32:23,829][236315] Num frames 2900... +[2023-03-07 15:32:23,911][236315] Num frames 3000... +[2023-03-07 15:32:23,993][236315] Num frames 3100... +[2023-03-07 15:32:24,075][236315] Num frames 3200... +[2023-03-07 15:32:24,157][236315] Num frames 3300... +[2023-03-07 15:32:24,234][236315] Avg episode rewards: #0: 9.047, true rewards: #0: 5.547 +[2023-03-07 15:32:24,235][236315] Avg episode reward: 9.047, avg true_objective: 5.547 +[2023-03-07 15:32:24,319][236315] Num frames 3400... +[2023-03-07 15:32:24,402][236315] Num frames 3500... +[2023-03-07 15:32:24,486][236315] Num frames 3600... +[2023-03-07 15:32:24,570][236315] Num frames 3700... +[2023-03-07 15:32:24,655][236315] Num frames 3800... +[2023-03-07 15:32:24,769][236315] Avg episode rewards: #0: 8.817, true rewards: #0: 5.531 +[2023-03-07 15:32:24,770][236315] Avg episode reward: 8.817, avg true_objective: 5.531 +[2023-03-07 15:32:24,805][236315] Num frames 3900... +[2023-03-07 15:32:24,899][236315] Num frames 4000... +[2023-03-07 15:32:24,981][236315] Num frames 4100... +[2023-03-07 15:32:25,065][236315] Num frames 4200... +[2023-03-07 15:32:25,152][236315] Num frames 4300... +[2023-03-07 15:32:25,223][236315] Avg episode rewards: #0: 8.400, true rewards: #0: 5.400 +[2023-03-07 15:32:25,224][236315] Avg episode reward: 8.400, avg true_objective: 5.400 +[2023-03-07 15:32:25,314][236315] Num frames 4400... +[2023-03-07 15:32:25,400][236315] Num frames 4500... +[2023-03-07 15:32:25,485][236315] Num frames 4600... +[2023-03-07 15:32:25,569][236315] Num frames 4700... +[2023-03-07 15:32:25,653][236315] Num frames 4800... +[2023-03-07 15:32:25,762][236315] Avg episode rewards: #0: 8.293, true rewards: #0: 5.404 +[2023-03-07 15:32:25,763][236315] Avg episode reward: 8.293, avg true_objective: 5.404 +[2023-03-07 15:32:25,798][236315] Num frames 4900... +[2023-03-07 15:32:25,883][236315] Num frames 5000... +[2023-03-07 15:32:25,969][236315] Num frames 5100... +[2023-03-07 15:32:26,053][236315] Num frames 5200... +[2023-03-07 15:32:26,139][236315] Num frames 5300... +[2023-03-07 15:32:26,225][236315] Num frames 5400... +[2023-03-07 15:32:26,310][236315] Num frames 5500... +[2023-03-07 15:32:26,395][236315] Num frames 5600... +[2023-03-07 15:32:26,478][236315] Num frames 5700... +[2023-03-07 15:32:26,609][236315] Avg episode rewards: #0: 9.092, true rewards: #0: 5.792 +[2023-03-07 15:32:26,610][236315] Avg episode reward: 9.092, avg true_objective: 5.792 +[2023-03-07 15:32:33,222][236315] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 15:35:32,240][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:35:32,241][236315] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 15:35:32,242][236315] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:35:32,242][236315] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:35:32,243][236315] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:35:32,244][236315] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:35:32,244][236315] Adding new argument 'max_num_frames'=100000 that is not in the saved config file! +[2023-03-07 15:35:32,246][236315] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:35:32,247][236315] Adding new argument 'push_to_hub'=True that is not in the saved config file! +[2023-03-07 15:35:32,247][236315] Adding new argument 'hf_repository'='ThomasSimonini/rl_course_vizdoom_health_gathering_supreme' that is not in the saved config file! +[2023-03-07 15:35:32,248][236315] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:35:32,249][236315] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:35:32,250][236315] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:35:32,250][236315] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:35:32,251][236315] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:35:32,261][236315] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:35:32,262][236315] RunningMeanStd input shape: (1,) +[2023-03-07 15:35:32,279][236315] ConvEncoder: input_channels=3 +[2023-03-07 15:35:32,314][236315] Conv encoder output size: 512 +[2023-03-07 15:35:32,315][236315] Policy head output size: 512 +[2023-03-07 15:35:32,345][236315] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:35:32,882][236315] Num frames 100... +[2023-03-07 15:35:32,966][236315] Num frames 200... +[2023-03-07 15:35:33,051][236315] Num frames 300... +[2023-03-07 15:35:33,136][236315] Num frames 400... +[2023-03-07 15:35:33,220][236315] Num frames 500... +[2023-03-07 15:35:33,339][236315] Avg episode rewards: #0: 10.760, true rewards: #0: 5.760 +[2023-03-07 15:35:33,340][236315] Avg episode reward: 10.760, avg true_objective: 5.760 +[2023-03-07 15:35:33,372][236315] Num frames 600... +[2023-03-07 15:35:33,469][236315] Num frames 700... +[2023-03-07 15:35:33,553][236315] Num frames 800... +[2023-03-07 15:35:33,637][236315] Num frames 900... +[2023-03-07 15:35:33,722][236315] Num frames 1000... +[2023-03-07 15:35:33,806][236315] Num frames 1100... +[2023-03-07 15:35:33,889][236315] Num frames 1200... +[2023-03-07 15:35:33,957][236315] Avg episode rewards: #0: 10.580, true rewards: #0: 6.080 +[2023-03-07 15:35:33,958][236315] Avg episode reward: 10.580, avg true_objective: 6.080 +[2023-03-07 15:35:34,029][236315] Num frames 1300... +[2023-03-07 15:35:34,111][236315] Num frames 1400... +[2023-03-07 15:35:34,195][236315] Num frames 1500... +[2023-03-07 15:35:34,282][236315] Num frames 1600... +[2023-03-07 15:35:34,334][236315] Avg episode rewards: #0: 8.333, true rewards: #0: 5.333 +[2023-03-07 15:35:34,334][236315] Avg episode reward: 8.333, avg true_objective: 5.333 +[2023-03-07 15:35:34,419][236315] Num frames 1700... +[2023-03-07 15:35:34,501][236315] Num frames 1800... +[2023-03-07 15:35:34,584][236315] Num frames 1900... +[2023-03-07 15:35:34,720][236315] Avg episode rewards: #0: 7.998, true rewards: #0: 4.997 +[2023-03-07 15:35:34,722][236315] Avg episode reward: 7.998, avg true_objective: 4.997 +[2023-03-07 15:35:34,723][236315] Num frames 2000... +[2023-03-07 15:35:34,816][236315] Num frames 2100... +[2023-03-07 15:35:34,898][236315] Num frames 2200... +[2023-03-07 15:35:34,982][236315] Num frames 2300... +[2023-03-07 15:35:35,064][236315] Num frames 2400... +[2023-03-07 15:35:35,181][236315] Avg episode rewards: #0: 7.558, true rewards: #0: 4.958 +[2023-03-07 15:35:35,182][236315] Avg episode reward: 7.558, avg true_objective: 4.958 +[2023-03-07 15:35:35,207][236315] Num frames 2500... +[2023-03-07 15:35:35,297][236315] Num frames 2600... +[2023-03-07 15:35:35,379][236315] Num frames 2700... +[2023-03-07 15:35:35,461][236315] Num frames 2800... +[2023-03-07 15:35:35,543][236315] Num frames 2900... +[2023-03-07 15:35:35,671][236315] Avg episode rewards: #0: 7.485, true rewards: #0: 4.985 +[2023-03-07 15:35:35,672][236315] Avg episode reward: 7.485, avg true_objective: 4.985 +[2023-03-07 15:35:35,682][236315] Num frames 3000... +[2023-03-07 15:35:35,783][236315] Num frames 3100... +[2023-03-07 15:35:35,867][236315] Num frames 3200... +[2023-03-07 15:35:35,952][236315] Num frames 3300... +[2023-03-07 15:35:36,069][236315] Avg episode rewards: #0: 6.964, true rewards: #0: 4.821 +[2023-03-07 15:35:36,070][236315] Avg episode reward: 6.964, avg true_objective: 4.821 +[2023-03-07 15:35:36,099][236315] Num frames 3400... +[2023-03-07 15:35:36,192][236315] Num frames 3500... +[2023-03-07 15:35:36,276][236315] Num frames 3600... +[2023-03-07 15:35:36,361][236315] Num frames 3700... +[2023-03-07 15:35:36,442][236315] Num frames 3800... +[2023-03-07 15:35:36,523][236315] Num frames 3900... +[2023-03-07 15:35:36,592][236315] Avg episode rewards: #0: 7.024, true rewards: #0: 4.899 +[2023-03-07 15:35:36,593][236315] Avg episode reward: 7.024, avg true_objective: 4.899 +[2023-03-07 15:35:36,673][236315] Num frames 4000... +[2023-03-07 15:35:36,755][236315] Num frames 4100... +[2023-03-07 15:35:36,836][236315] Num frames 4200... +[2023-03-07 15:35:36,919][236315] Num frames 4300... +[2023-03-07 15:35:37,029][236315] Avg episode rewards: #0: 6.852, true rewards: #0: 4.852 +[2023-03-07 15:35:37,030][236315] Avg episode reward: 6.852, avg true_objective: 4.852 +[2023-03-07 15:35:37,057][236315] Num frames 4400... +[2023-03-07 15:35:37,140][236315] Num frames 4500... +[2023-03-07 15:35:37,224][236315] Num frames 4600... +[2023-03-07 15:35:37,309][236315] Num frames 4700... +[2023-03-07 15:35:37,394][236315] Num frames 4800... +[2023-03-07 15:35:37,488][236315] Avg episode rewards: #0: 6.847, true rewards: #0: 4.847 +[2023-03-07 15:35:37,489][236315] Avg episode reward: 6.847, avg true_objective: 4.847 +[2023-03-07 15:35:42,929][236315] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 15:35:54,788][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:35:54,789][236315] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 15:35:54,790][236315] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:35:54,790][236315] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:35:54,791][236315] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:35:54,791][236315] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:35:54,792][236315] Adding new argument 'max_num_frames'=100000 that is not in the saved config file! +[2023-03-07 15:35:54,792][236315] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:35:54,793][236315] Adding new argument 'push_to_hub'=True that is not in the saved config file! +[2023-03-07 15:35:54,793][236315] Adding new argument 'hf_repository'='Rolo/rl_course_vizdoom_health_gathering_supreme' that is not in the saved config file! +[2023-03-07 15:35:54,794][236315] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:35:54,795][236315] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:35:54,795][236315] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:35:54,796][236315] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:35:54,796][236315] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:35:54,803][236315] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:35:54,804][236315] RunningMeanStd input shape: (1,) +[2023-03-07 15:35:54,812][236315] ConvEncoder: input_channels=3 +[2023-03-07 15:35:54,840][236315] Conv encoder output size: 512 +[2023-03-07 15:35:54,841][236315] Policy head output size: 512 +[2023-03-07 15:35:54,861][236315] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:35:55,380][236315] Num frames 100... +[2023-03-07 15:35:55,462][236315] Num frames 200... +[2023-03-07 15:35:55,545][236315] Num frames 300... +[2023-03-07 15:35:55,629][236315] Num frames 400... +[2023-03-07 15:35:55,711][236315] Num frames 500... +[2023-03-07 15:35:55,794][236315] Num frames 600... +[2023-03-07 15:35:55,855][236315] Avg episode rewards: #0: 9.080, true rewards: #0: 6.080 +[2023-03-07 15:35:55,856][236315] Avg episode reward: 9.080, avg true_objective: 6.080 +[2023-03-07 15:35:55,956][236315] Num frames 700... +[2023-03-07 15:35:56,041][236315] Num frames 800... +[2023-03-07 15:35:56,126][236315] Num frames 900... +[2023-03-07 15:35:56,211][236315] Num frames 1000... +[2023-03-07 15:35:56,312][236315] Avg episode rewards: #0: 7.280, true rewards: #0: 5.280 +[2023-03-07 15:35:56,313][236315] Avg episode reward: 7.280, avg true_objective: 5.280 +[2023-03-07 15:35:56,353][236315] Num frames 1100... +[2023-03-07 15:35:56,436][236315] Num frames 1200... +[2023-03-07 15:35:56,519][236315] Num frames 1300... +[2023-03-07 15:35:56,601][236315] Num frames 1400... +[2023-03-07 15:35:56,683][236315] Num frames 1500... +[2023-03-07 15:35:56,768][236315] Num frames 1600... +[2023-03-07 15:35:56,903][236315] Avg episode rewards: #0: 8.320, true rewards: #0: 5.653 +[2023-03-07 15:35:56,904][236315] Avg episode reward: 8.320, avg true_objective: 5.653 +[2023-03-07 15:35:56,908][236315] Num frames 1700... +[2023-03-07 15:35:56,992][236315] Num frames 1800... +[2023-03-07 15:35:57,076][236315] Num frames 1900... +[2023-03-07 15:35:57,159][236315] Num frames 2000... +[2023-03-07 15:35:57,244][236315] Num frames 2100... +[2023-03-07 15:35:57,336][236315] Avg episode rewards: #0: 7.610, true rewards: #0: 5.360 +[2023-03-07 15:35:57,337][236315] Avg episode reward: 7.610, avg true_objective: 5.360 +[2023-03-07 15:35:57,405][236315] Num frames 2200... +[2023-03-07 15:35:57,487][236315] Num frames 2300... +[2023-03-07 15:35:57,571][236315] Num frames 2400... +[2023-03-07 15:35:57,657][236315] Num frames 2500... +[2023-03-07 15:35:57,742][236315] Num frames 2600... +[2023-03-07 15:35:57,827][236315] Num frames 2700... +[2023-03-07 15:35:57,910][236315] Num frames 2800... +[2023-03-07 15:35:57,995][236315] Num frames 2900... +[2023-03-07 15:35:58,080][236315] Num frames 3000... +[2023-03-07 15:35:58,164][236315] Num frames 3100... +[2023-03-07 15:35:58,249][236315] Num frames 3200... +[2023-03-07 15:35:58,356][236315] Avg episode rewards: #0: 10.328, true rewards: #0: 6.528 +[2023-03-07 15:35:58,357][236315] Avg episode reward: 10.328, avg true_objective: 6.528 +[2023-03-07 15:35:58,389][236315] Num frames 3300... +[2023-03-07 15:35:58,473][236315] Num frames 3400... +[2023-03-07 15:35:58,557][236315] Num frames 3500... +[2023-03-07 15:35:58,642][236315] Num frames 3600... +[2023-03-07 15:35:58,726][236315] Num frames 3700... +[2023-03-07 15:35:58,810][236315] Num frames 3800... +[2023-03-07 15:35:58,926][236315] Avg episode rewards: #0: 10.120, true rewards: #0: 6.453 +[2023-03-07 15:35:58,927][236315] Avg episode reward: 10.120, avg true_objective: 6.453 +[2023-03-07 15:35:58,964][236315] Num frames 3900... +[2023-03-07 15:35:59,063][236315] Num frames 4000... +[2023-03-07 15:35:59,146][236315] Num frames 4100... +[2023-03-07 15:35:59,230][236315] Num frames 4200... +[2023-03-07 15:35:59,314][236315] Num frames 4300... +[2023-03-07 15:35:59,412][236315] Avg episode rewards: #0: 9.646, true rewards: #0: 6.217 +[2023-03-07 15:35:59,413][236315] Avg episode reward: 9.646, avg true_objective: 6.217 +[2023-03-07 15:35:59,463][236315] Num frames 4400... +[2023-03-07 15:35:59,546][236315] Num frames 4500... +[2023-03-07 15:35:59,630][236315] Num frames 4600... +[2023-03-07 15:35:59,712][236315] Num frames 4700... +[2023-03-07 15:35:59,797][236315] Num frames 4800... +[2023-03-07 15:35:59,905][236315] Avg episode rewards: #0: 9.330, true rewards: #0: 6.080 +[2023-03-07 15:35:59,906][236315] Avg episode reward: 9.330, avg true_objective: 6.080 +[2023-03-07 15:35:59,950][236315] Num frames 4900... +[2023-03-07 15:36:00,036][236315] Num frames 5000... +[2023-03-07 15:36:00,120][236315] Num frames 5100... +[2023-03-07 15:36:00,204][236315] Num frames 5200... +[2023-03-07 15:36:00,298][236315] Avg episode rewards: #0: 8.720, true rewards: #0: 5.831 +[2023-03-07 15:36:00,299][236315] Avg episode reward: 8.720, avg true_objective: 5.831 +[2023-03-07 15:36:00,344][236315] Num frames 5300... +[2023-03-07 15:36:00,428][236315] Num frames 5400... +[2023-03-07 15:36:00,511][236315] Num frames 5500... +[2023-03-07 15:36:00,598][236315] Num frames 5600... +[2023-03-07 15:36:00,649][236315] Avg episode rewards: #0: 8.300, true rewards: #0: 5.600 +[2023-03-07 15:36:00,650][236315] Avg episode reward: 8.300, avg true_objective: 5.600 +[2023-03-07 15:36:07,089][236315] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 15:36:47,372][236315] The model has been pushed to https://huggingface.co/Rolo/rl_course_vizdoom_health_gathering_supreme +[2023-03-07 15:38:37,596][236315] Loading legacy config file train_dir/doom_health_gathering_supreme_2222/cfg.json instead of train_dir/doom_health_gathering_supreme_2222/config.json +[2023-03-07 15:38:37,597][236315] Loading existing experiment configuration from train_dir/doom_health_gathering_supreme_2222/config.json +[2023-03-07 15:38:37,598][236315] Overriding arg 'experiment' with value 'doom_health_gathering_supreme_2222' passed from command line +[2023-03-07 15:38:37,599][236315] Overriding arg 'train_dir' with value 'train_dir' passed from command line +[2023-03-07 15:38:37,600][236315] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 15:38:37,601][236315] Adding new argument 'lr_adaptive_min'=1e-06 that is not in the saved config file! +[2023-03-07 15:38:37,602][236315] Adding new argument 'lr_adaptive_max'=0.01 that is not in the saved config file! +[2023-03-07 15:38:37,603][236315] Adding new argument 'env_gpu_observations'=True that is not in the saved config file! +[2023-03-07 15:38:37,603][236315] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:38:37,604][236315] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:38:37,605][236315] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:38:37,606][236315] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:38:37,606][236315] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:38:37,607][236315] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:38:37,608][236315] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 15:38:37,609][236315] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 15:38:37,610][236315] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:38:37,611][236315] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:38:37,611][236315] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:38:37,612][236315] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:38:37,613][236315] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:38:37,618][236315] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:38:37,619][236315] RunningMeanStd input shape: (1,) +[2023-03-07 15:38:37,635][236315] ConvEncoder: input_channels=3 +[2023-03-07 15:38:37,675][236315] Conv encoder output size: 512 +[2023-03-07 15:38:37,676][236315] Policy head output size: 512 +[2023-03-07 15:38:37,708][236315] Loading state from checkpoint train_dir/doom_health_gathering_supreme_2222/checkpoint_p0/checkpoint_000539850_4422451200.pth... +[2023-03-07 15:38:38,225][236315] Num frames 100... +[2023-03-07 15:38:38,309][236315] Num frames 200... +[2023-03-07 15:38:38,393][236315] Num frames 300... +[2023-03-07 15:38:38,477][236315] Num frames 400... +[2023-03-07 15:38:38,566][236315] Num frames 500... +[2023-03-07 15:38:38,651][236315] Num frames 600... +[2023-03-07 15:38:38,734][236315] Num frames 700... +[2023-03-07 15:38:38,819][236315] Num frames 800... +[2023-03-07 15:38:38,903][236315] Num frames 900... +[2023-03-07 15:38:38,987][236315] Num frames 1000... +[2023-03-07 15:38:39,071][236315] Num frames 1100... +[2023-03-07 15:38:39,158][236315] Num frames 1200... +[2023-03-07 15:38:39,245][236315] Num frames 1300... +[2023-03-07 15:38:39,330][236315] Num frames 1400... +[2023-03-07 15:38:39,414][236315] Num frames 1500... +[2023-03-07 15:38:39,500][236315] Num frames 1600... +[2023-03-07 15:38:39,588][236315] Num frames 1700... +[2023-03-07 15:38:39,674][236315] Num frames 1800... +[2023-03-07 15:38:39,760][236315] Num frames 1900... +[2023-03-07 15:38:39,845][236315] Num frames 2000... +[2023-03-07 15:38:39,935][236315] Num frames 2100... +[2023-03-07 15:38:39,986][236315] Avg episode rewards: #0: 59.999, true rewards: #0: 21.000 +[2023-03-07 15:38:39,987][236315] Avg episode reward: 59.999, avg true_objective: 21.000 +[2023-03-07 15:38:40,072][236315] Num frames 2200... +[2023-03-07 15:38:40,157][236315] Num frames 2300... +[2023-03-07 15:38:40,243][236315] Num frames 2400... +[2023-03-07 15:38:40,327][236315] Num frames 2500... +[2023-03-07 15:38:40,411][236315] Num frames 2600... +[2023-03-07 15:38:40,496][236315] Num frames 2700... +[2023-03-07 15:38:40,580][236315] Num frames 2800... +[2023-03-07 15:38:40,665][236315] Num frames 2900... +[2023-03-07 15:38:40,751][236315] Num frames 3000... +[2023-03-07 15:38:40,837][236315] Num frames 3100... +[2023-03-07 15:38:40,921][236315] Num frames 3200... +[2023-03-07 15:38:41,006][236315] Num frames 3300... +[2023-03-07 15:38:41,090][236315] Num frames 3400... +[2023-03-07 15:38:41,181][236315] Num frames 3500... +[2023-03-07 15:38:41,269][236315] Num frames 3600... +[2023-03-07 15:38:41,356][236315] Num frames 3700... +[2023-03-07 15:38:41,442][236315] Num frames 3800... +[2023-03-07 15:38:41,529][236315] Num frames 3900... +[2023-03-07 15:38:41,614][236315] Num frames 4000... +[2023-03-07 15:38:41,699][236315] Num frames 4100... +[2023-03-07 15:38:41,788][236315] Num frames 4200... +[2023-03-07 15:38:41,840][236315] Avg episode rewards: #0: 62.499, true rewards: #0: 21.000 +[2023-03-07 15:38:41,840][236315] Avg episode reward: 62.499, avg true_objective: 21.000 +[2023-03-07 15:38:41,943][236315] Num frames 4300... +[2023-03-07 15:38:42,028][236315] Num frames 4400... +[2023-03-07 15:38:42,113][236315] Num frames 4500... +[2023-03-07 15:38:42,198][236315] Num frames 4600... +[2023-03-07 15:38:42,282][236315] Num frames 4700... +[2023-03-07 15:38:42,365][236315] Num frames 4800... +[2023-03-07 15:38:42,450][236315] Num frames 4900... +[2023-03-07 15:38:42,535][236315] Num frames 5000... +[2023-03-07 15:38:42,620][236315] Num frames 5100... +[2023-03-07 15:38:42,702][236315] Num frames 5200... +[2023-03-07 15:38:42,787][236315] Num frames 5300... +[2023-03-07 15:38:42,873][236315] Num frames 5400... +[2023-03-07 15:38:42,958][236315] Num frames 5500... +[2023-03-07 15:38:43,042][236315] Num frames 5600... +[2023-03-07 15:38:43,127][236315] Num frames 5700... +[2023-03-07 15:38:43,214][236315] Num frames 5800... +[2023-03-07 15:38:43,302][236315] Num frames 5900... +[2023-03-07 15:38:43,387][236315] Num frames 6000... +[2023-03-07 15:38:43,473][236315] Num frames 6100... +[2023-03-07 15:38:43,560][236315] Num frames 6200... +[2023-03-07 15:38:43,649][236315] Num frames 6300... +[2023-03-07 15:38:43,700][236315] Avg episode rewards: #0: 63.999, true rewards: #0: 21.000 +[2023-03-07 15:38:43,701][236315] Avg episode reward: 63.999, avg true_objective: 21.000 +[2023-03-07 15:38:43,803][236315] Num frames 6400... +[2023-03-07 15:38:43,887][236315] Num frames 6500... +[2023-03-07 15:38:43,973][236315] Num frames 6600... +[2023-03-07 15:38:44,059][236315] Num frames 6700... +[2023-03-07 15:38:44,145][236315] Num frames 6800... +[2023-03-07 15:38:44,230][236315] Num frames 6900... +[2023-03-07 15:38:44,315][236315] Num frames 7000... +[2023-03-07 15:38:44,401][236315] Num frames 7100... +[2023-03-07 15:38:44,488][236315] Num frames 7200... +[2023-03-07 15:38:44,574][236315] Num frames 7300... +[2023-03-07 15:38:44,661][236315] Num frames 7400... +[2023-03-07 15:38:44,747][236315] Num frames 7500... +[2023-03-07 15:38:44,834][236315] Num frames 7600... +[2023-03-07 15:38:44,921][236315] Num frames 7700... +[2023-03-07 15:38:45,007][236315] Num frames 7800... +[2023-03-07 15:38:45,094][236315] Num frames 7900... +[2023-03-07 15:38:45,181][236315] Num frames 8000... +[2023-03-07 15:38:45,269][236315] Num frames 8100... +[2023-03-07 15:38:45,355][236315] Num frames 8200... +[2023-03-07 15:38:45,441][236315] Num frames 8300... +[2023-03-07 15:38:45,532][236315] Num frames 8400... +[2023-03-07 15:38:45,584][236315] Avg episode rewards: #0: 63.249, true rewards: #0: 21.000 +[2023-03-07 15:38:45,585][236315] Avg episode reward: 63.249, avg true_objective: 21.000 +[2023-03-07 15:38:45,689][236315] Num frames 8500... +[2023-03-07 15:38:45,773][236315] Num frames 8600... +[2023-03-07 15:38:45,858][236315] Num frames 8700... +[2023-03-07 15:38:45,944][236315] Num frames 8800... +[2023-03-07 15:38:46,030][236315] Num frames 8900... +[2023-03-07 15:38:46,116][236315] Num frames 9000... +[2023-03-07 15:38:46,202][236315] Num frames 9100... +[2023-03-07 15:38:46,287][236315] Num frames 9200... +[2023-03-07 15:38:46,373][236315] Num frames 9300... +[2023-03-07 15:38:46,458][236315] Num frames 9400... +[2023-03-07 15:38:46,544][236315] Num frames 9500... +[2023-03-07 15:38:46,631][236315] Num frames 9600... +[2023-03-07 15:38:46,717][236315] Num frames 9700... +[2023-03-07 15:38:46,802][236315] Num frames 9800... +[2023-03-07 15:38:46,887][236315] Num frames 9900... +[2023-03-07 15:38:46,973][236315] Num frames 10000... +[2023-03-07 15:38:47,063][236315] Num frames 10100... +[2023-03-07 15:38:47,149][236315] Num frames 10200... +[2023-03-07 15:38:47,238][236315] Num frames 10300... +[2023-03-07 15:38:47,326][236315] Num frames 10400... +[2023-03-07 15:38:47,415][236315] Num frames 10500... +[2023-03-07 15:38:47,466][236315] Avg episode rewards: #0: 63.999, true rewards: #0: 21.000 +[2023-03-07 15:38:47,467][236315] Avg episode reward: 63.999, avg true_objective: 21.000 +[2023-03-07 15:38:47,573][236315] Num frames 10600... +[2023-03-07 15:38:47,659][236315] Num frames 10700... +[2023-03-07 15:38:47,743][236315] Num frames 10800... +[2023-03-07 15:38:47,828][236315] Num frames 10900... +[2023-03-07 15:38:47,915][236315] Num frames 11000... +[2023-03-07 15:38:48,001][236315] Num frames 11100... +[2023-03-07 15:38:48,087][236315] Num frames 11200... +[2023-03-07 15:38:48,173][236315] Num frames 11300... +[2023-03-07 15:38:48,258][236315] Num frames 11400... +[2023-03-07 15:38:48,343][236315] Num frames 11500... +[2023-03-07 15:38:48,430][236315] Num frames 11600... +[2023-03-07 15:38:48,517][236315] Num frames 11700... +[2023-03-07 15:38:48,603][236315] Num frames 11800... +[2023-03-07 15:38:48,688][236315] Num frames 11900... +[2023-03-07 15:38:48,774][236315] Num frames 12000... +[2023-03-07 15:38:48,861][236315] Num frames 12100... +[2023-03-07 15:38:48,949][236315] Num frames 12200... +[2023-03-07 15:38:49,036][236315] Num frames 12300... +[2023-03-07 15:38:49,121][236315] Num frames 12400... +[2023-03-07 15:38:49,208][236315] Num frames 12500... +[2023-03-07 15:38:49,300][236315] Num frames 12600... +[2023-03-07 15:38:49,351][236315] Avg episode rewards: #0: 63.832, true rewards: #0: 21.000 +[2023-03-07 15:38:49,352][236315] Avg episode reward: 63.832, avg true_objective: 21.000 +[2023-03-07 15:38:49,447][236315] Num frames 12700... +[2023-03-07 15:38:49,533][236315] Num frames 12800... +[2023-03-07 15:38:49,619][236315] Num frames 12900... +[2023-03-07 15:38:49,704][236315] Num frames 13000... +[2023-03-07 15:38:49,789][236315] Num frames 13100... +[2023-03-07 15:38:49,875][236315] Num frames 13200... +[2023-03-07 15:38:49,961][236315] Num frames 13300... +[2023-03-07 15:38:50,048][236315] Num frames 13400... +[2023-03-07 15:38:50,134][236315] Num frames 13500... +[2023-03-07 15:38:50,220][236315] Num frames 13600... +[2023-03-07 15:38:50,306][236315] Num frames 13700... +[2023-03-07 15:38:50,391][236315] Num frames 13800... +[2023-03-07 15:38:50,477][236315] Num frames 13900... +[2023-03-07 15:38:50,561][236315] Num frames 14000... +[2023-03-07 15:38:50,648][236315] Num frames 14100... +[2023-03-07 15:38:50,735][236315] Num frames 14200... +[2023-03-07 15:38:50,823][236315] Num frames 14300... +[2023-03-07 15:38:50,909][236315] Num frames 14400... +[2023-03-07 15:38:50,995][236315] Num frames 14500... +[2023-03-07 15:38:51,081][236315] Num frames 14600... +[2023-03-07 15:38:51,171][236315] Num frames 14700... +[2023-03-07 15:38:51,222][236315] Avg episode rewards: #0: 63.284, true rewards: #0: 21.000 +[2023-03-07 15:38:51,223][236315] Avg episode reward: 63.284, avg true_objective: 21.000 +[2023-03-07 15:38:51,330][236315] Num frames 14800... +[2023-03-07 15:38:51,415][236315] Num frames 14900... +[2023-03-07 15:38:51,501][236315] Num frames 15000... +[2023-03-07 15:38:51,587][236315] Num frames 15100... +[2023-03-07 15:38:51,673][236315] Num frames 15200... +[2023-03-07 15:38:51,759][236315] Num frames 15300... +[2023-03-07 15:38:51,844][236315] Num frames 15400... +[2023-03-07 15:38:51,930][236315] Num frames 15500... +[2023-03-07 15:38:52,018][236315] Num frames 15600... +[2023-03-07 15:38:52,106][236315] Num frames 15700... +[2023-03-07 15:38:52,193][236315] Num frames 15800... +[2023-03-07 15:38:52,278][236315] Num frames 15900... +[2023-03-07 15:38:52,363][236315] Num frames 16000... +[2023-03-07 15:38:52,450][236315] Num frames 16100... +[2023-03-07 15:38:52,537][236315] Num frames 16200... +[2023-03-07 15:38:52,624][236315] Num frames 16300... +[2023-03-07 15:38:52,710][236315] Num frames 16400... +[2023-03-07 15:38:52,795][236315] Num frames 16500... +[2023-03-07 15:38:52,883][236315] Num frames 16600... +[2023-03-07 15:38:52,971][236315] Num frames 16700... +[2023-03-07 15:38:53,096][236315] Avg episode rewards: #0: 63.349, true rewards: #0: 20.975 +[2023-03-07 15:38:53,097][236315] Avg episode reward: 63.349, avg true_objective: 20.975 +[2023-03-07 15:38:53,114][236315] Num frames 16800... +[2023-03-07 15:38:53,204][236315] Num frames 16900... +[2023-03-07 15:38:53,290][236315] Num frames 17000... +[2023-03-07 15:38:53,376][236315] Num frames 17100... +[2023-03-07 15:38:53,461][236315] Num frames 17200... +[2023-03-07 15:38:53,546][236315] Num frames 17300... +[2023-03-07 15:38:53,631][236315] Num frames 17400... +[2023-03-07 15:38:53,716][236315] Num frames 17500... +[2023-03-07 15:38:53,802][236315] Num frames 17600... +[2023-03-07 15:38:53,887][236315] Num frames 17700... +[2023-03-07 15:38:53,971][236315] Num frames 17800... +[2023-03-07 15:38:54,059][236315] Num frames 17900... +[2023-03-07 15:38:54,146][236315] Num frames 18000... +[2023-03-07 15:38:54,233][236315] Num frames 18100... +[2023-03-07 15:38:54,320][236315] Num frames 18200... +[2023-03-07 15:38:54,407][236315] Num frames 18300... +[2023-03-07 15:38:54,493][236315] Num frames 18400... +[2023-03-07 15:38:54,580][236315] Num frames 18500... +[2023-03-07 15:38:54,666][236315] Num frames 18600... +[2023-03-07 15:38:54,752][236315] Num frames 18700... +[2023-03-07 15:38:54,838][236315] Num frames 18800... +[2023-03-07 15:38:54,962][236315] Avg episode rewards: #0: 63.532, true rewards: #0: 20.978 +[2023-03-07 15:38:54,963][236315] Avg episode reward: 63.532, avg true_objective: 20.978 +[2023-03-07 15:38:54,990][236315] Num frames 18900... +[2023-03-07 15:38:55,088][236315] Num frames 19000... +[2023-03-07 15:38:55,174][236315] Num frames 19100... +[2023-03-07 15:38:55,260][236315] Num frames 19200... +[2023-03-07 15:38:55,344][236315] Num frames 19300... +[2023-03-07 15:38:55,432][236315] Num frames 19400... +[2023-03-07 15:38:55,518][236315] Num frames 19500... +[2023-03-07 15:38:55,603][236315] Num frames 19600... +[2023-03-07 15:38:55,689][236315] Num frames 19700... +[2023-03-07 15:38:55,776][236315] Num frames 19800... +[2023-03-07 15:38:55,861][236315] Num frames 19900... +[2023-03-07 15:38:55,948][236315] Num frames 20000... +[2023-03-07 15:38:56,033][236315] Num frames 20100... +[2023-03-07 15:38:56,118][236315] Num frames 20200... +[2023-03-07 15:38:56,204][236315] Num frames 20300... +[2023-03-07 15:38:56,291][236315] Num frames 20400... +[2023-03-07 15:38:56,377][236315] Num frames 20500... +[2023-03-07 15:38:56,463][236315] Num frames 20600... +[2023-03-07 15:38:56,552][236315] Num frames 20700... +[2023-03-07 15:38:56,640][236315] Num frames 20800... +[2023-03-07 15:38:56,727][236315] Num frames 20900... +[2023-03-07 15:38:56,849][236315] Avg episode rewards: #0: 64.379, true rewards: #0: 20.980 +[2023-03-07 15:38:56,850][236315] Avg episode reward: 64.379, avg true_objective: 20.980 +[2023-03-07 15:39:22,780][236315] Replay video saved to train_dir/doom_health_gathering_supreme_2222/replay.mp4! +[2023-03-07 15:42:58,492][236315] Environment doom_basic already registered, overwriting... +[2023-03-07 15:42:58,494][236315] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 15:42:58,495][236315] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 15:42:58,496][236315] Environment doom_dm already registered, overwriting... +[2023-03-07 15:42:58,497][236315] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 15:42:58,497][236315] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 15:42:58,499][236315] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 15:42:58,499][236315] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 15:42:58,500][236315] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 15:42:58,501][236315] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 15:42:58,502][236315] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 15:42:58,503][236315] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 15:42:58,503][236315] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 15:42:58,504][236315] Environment doom_battle already registered, overwriting... +[2023-03-07 15:42:58,505][236315] Environment doom_battle2 already registered, overwriting... +[2023-03-07 15:42:58,505][236315] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 15:42:58,507][236315] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 15:42:58,508][236315] Environment doom_duel already registered, overwriting... +[2023-03-07 15:42:58,508][236315] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 15:42:58,509][236315] Environment doom_benchmark already registered, overwriting... +[2023-03-07 15:42:58,510][236315] register_encoder_factory: +[2023-03-07 15:42:58,526][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:42:58,527][236315] Overriding arg 'num_workers' with value 128 passed from command line +[2023-03-07 15:42:58,527][236315] Overriding arg 'num_envs_per_worker' with value 32 passed from command line +[2023-03-07 15:42:58,532][236315] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/default_experiment already exists! +[2023-03-07 15:42:58,533][236315] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/default_experiment... +[2023-03-07 15:42:58,534][236315] Weights and Biases integration disabled +[2023-03-07 15:42:58,537][236315] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 15:42:59,597][236315] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=default_experiment +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=120 --num_envs_per_worker=4 --train_for_env_steps=4000000 +cli_args={'env': 'doom_health_gathering_supreme', 'num_workers': 120, 'num_envs_per_worker': 4, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 15:42:59,598][236315] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json... +[2023-03-07 15:42:59,599][236315] Rollout worker 0 uses device cpu +[2023-03-07 15:42:59,599][236315] Rollout worker 1 uses device cpu +[2023-03-07 15:42:59,600][236315] Rollout worker 2 uses device cpu +[2023-03-07 15:42:59,600][236315] Rollout worker 3 uses device cpu +[2023-03-07 15:42:59,601][236315] Rollout worker 4 uses device cpu +[2023-03-07 15:42:59,601][236315] Rollout worker 5 uses device cpu +[2023-03-07 15:42:59,602][236315] Rollout worker 6 uses device cpu +[2023-03-07 15:42:59,603][236315] Rollout worker 7 uses device cpu +[2023-03-07 15:42:59,603][236315] Rollout worker 8 uses device cpu +[2023-03-07 15:42:59,604][236315] Rollout worker 9 uses device cpu +[2023-03-07 15:42:59,604][236315] Rollout worker 10 uses device cpu +[2023-03-07 15:42:59,605][236315] Rollout worker 11 uses device cpu +[2023-03-07 15:42:59,605][236315] Rollout worker 12 uses device cpu +[2023-03-07 15:42:59,606][236315] Rollout worker 13 uses device cpu +[2023-03-07 15:42:59,606][236315] Rollout worker 14 uses device cpu +[2023-03-07 15:42:59,607][236315] Rollout worker 15 uses device cpu +[2023-03-07 15:42:59,607][236315] Rollout worker 16 uses device cpu +[2023-03-07 15:42:59,608][236315] Rollout worker 17 uses device cpu +[2023-03-07 15:42:59,608][236315] Rollout worker 18 uses device cpu +[2023-03-07 15:42:59,609][236315] Rollout worker 19 uses device cpu +[2023-03-07 15:42:59,609][236315] Rollout worker 20 uses device cpu +[2023-03-07 15:42:59,610][236315] Rollout worker 21 uses device cpu +[2023-03-07 15:42:59,610][236315] Rollout worker 22 uses device cpu +[2023-03-07 15:42:59,611][236315] Rollout worker 23 uses device cpu +[2023-03-07 15:42:59,611][236315] Rollout worker 24 uses device cpu +[2023-03-07 15:42:59,612][236315] Rollout worker 25 uses device cpu +[2023-03-07 15:42:59,612][236315] Rollout worker 26 uses device cpu +[2023-03-07 15:42:59,613][236315] Rollout worker 27 uses device cpu +[2023-03-07 15:42:59,613][236315] Rollout worker 28 uses device cpu +[2023-03-07 15:42:59,614][236315] Rollout worker 29 uses device cpu +[2023-03-07 15:42:59,614][236315] Rollout worker 30 uses device cpu +[2023-03-07 15:42:59,615][236315] Rollout worker 31 uses device cpu +[2023-03-07 15:42:59,616][236315] Rollout worker 32 uses device cpu +[2023-03-07 15:42:59,616][236315] Rollout worker 33 uses device cpu +[2023-03-07 15:42:59,617][236315] Rollout worker 34 uses device cpu +[2023-03-07 15:42:59,617][236315] Rollout worker 35 uses device cpu +[2023-03-07 15:42:59,618][236315] Rollout worker 36 uses device cpu +[2023-03-07 15:42:59,618][236315] Rollout worker 37 uses device cpu +[2023-03-07 15:42:59,619][236315] Rollout worker 38 uses device cpu +[2023-03-07 15:42:59,619][236315] Rollout worker 39 uses device cpu +[2023-03-07 15:42:59,620][236315] Rollout worker 40 uses device cpu +[2023-03-07 15:42:59,620][236315] Rollout worker 41 uses device cpu +[2023-03-07 15:42:59,621][236315] Rollout worker 42 uses device cpu +[2023-03-07 15:42:59,621][236315] Rollout worker 43 uses device cpu +[2023-03-07 15:42:59,622][236315] Rollout worker 44 uses device cpu +[2023-03-07 15:42:59,622][236315] Rollout worker 45 uses device cpu +[2023-03-07 15:42:59,623][236315] Rollout worker 46 uses device cpu +[2023-03-07 15:42:59,623][236315] Rollout worker 47 uses device cpu +[2023-03-07 15:42:59,624][236315] Rollout worker 48 uses device cpu +[2023-03-07 15:42:59,624][236315] Rollout worker 49 uses device cpu +[2023-03-07 15:42:59,624][236315] Rollout worker 50 uses device cpu +[2023-03-07 15:42:59,625][236315] Rollout worker 51 uses device cpu +[2023-03-07 15:42:59,625][236315] Rollout worker 52 uses device cpu +[2023-03-07 15:42:59,625][236315] Rollout worker 53 uses device cpu +[2023-03-07 15:42:59,626][236315] Rollout worker 54 uses device cpu +[2023-03-07 15:42:59,626][236315] Rollout worker 55 uses device cpu +[2023-03-07 15:42:59,626][236315] Rollout worker 56 uses device cpu +[2023-03-07 15:42:59,627][236315] Rollout worker 57 uses device cpu +[2023-03-07 15:42:59,627][236315] Rollout worker 58 uses device cpu +[2023-03-07 15:42:59,627][236315] Rollout worker 59 uses device cpu +[2023-03-07 15:42:59,628][236315] Rollout worker 60 uses device cpu +[2023-03-07 15:42:59,628][236315] Rollout worker 61 uses device cpu +[2023-03-07 15:42:59,628][236315] Rollout worker 62 uses device cpu +[2023-03-07 15:42:59,628][236315] Rollout worker 63 uses device cpu +[2023-03-07 15:42:59,629][236315] Rollout worker 64 uses device cpu +[2023-03-07 15:42:59,629][236315] Rollout worker 65 uses device cpu +[2023-03-07 15:42:59,629][236315] Rollout worker 66 uses device cpu +[2023-03-07 15:42:59,630][236315] Rollout worker 67 uses device cpu +[2023-03-07 15:42:59,630][236315] Rollout worker 68 uses device cpu +[2023-03-07 15:42:59,630][236315] Rollout worker 69 uses device cpu +[2023-03-07 15:42:59,631][236315] Rollout worker 70 uses device cpu +[2023-03-07 15:42:59,631][236315] Rollout worker 71 uses device cpu +[2023-03-07 15:42:59,631][236315] Rollout worker 72 uses device cpu +[2023-03-07 15:42:59,632][236315] Rollout worker 73 uses device cpu +[2023-03-07 15:42:59,632][236315] Rollout worker 74 uses device cpu +[2023-03-07 15:42:59,632][236315] Rollout worker 75 uses device cpu +[2023-03-07 15:42:59,633][236315] Rollout worker 76 uses device cpu +[2023-03-07 15:42:59,633][236315] Rollout worker 77 uses device cpu +[2023-03-07 15:42:59,633][236315] Rollout worker 78 uses device cpu +[2023-03-07 15:42:59,634][236315] Rollout worker 79 uses device cpu +[2023-03-07 15:42:59,634][236315] Rollout worker 80 uses device cpu +[2023-03-07 15:42:59,634][236315] Rollout worker 81 uses device cpu +[2023-03-07 15:42:59,635][236315] Rollout worker 82 uses device cpu +[2023-03-07 15:42:59,635][236315] Rollout worker 83 uses device cpu +[2023-03-07 15:42:59,635][236315] Rollout worker 84 uses device cpu +[2023-03-07 15:42:59,636][236315] Rollout worker 85 uses device cpu +[2023-03-07 15:42:59,642][236315] Rollout worker 86 uses device cpu +[2023-03-07 15:42:59,643][236315] Rollout worker 87 uses device cpu +[2023-03-07 15:42:59,643][236315] Rollout worker 88 uses device cpu +[2023-03-07 15:42:59,644][236315] Rollout worker 89 uses device cpu +[2023-03-07 15:42:59,644][236315] Rollout worker 90 uses device cpu +[2023-03-07 15:42:59,644][236315] Rollout worker 91 uses device cpu +[2023-03-07 15:42:59,644][236315] Rollout worker 92 uses device cpu +[2023-03-07 15:42:59,645][236315] Rollout worker 93 uses device cpu +[2023-03-07 15:42:59,645][236315] Rollout worker 94 uses device cpu +[2023-03-07 15:42:59,645][236315] Rollout worker 95 uses device cpu +[2023-03-07 15:42:59,646][236315] Rollout worker 96 uses device cpu +[2023-03-07 15:42:59,646][236315] Rollout worker 97 uses device cpu +[2023-03-07 15:42:59,646][236315] Rollout worker 98 uses device cpu +[2023-03-07 15:42:59,647][236315] Rollout worker 99 uses device cpu +[2023-03-07 15:42:59,647][236315] Rollout worker 100 uses device cpu +[2023-03-07 15:42:59,647][236315] Rollout worker 101 uses device cpu +[2023-03-07 15:42:59,648][236315] Rollout worker 102 uses device cpu +[2023-03-07 15:42:59,648][236315] Rollout worker 103 uses device cpu +[2023-03-07 15:42:59,648][236315] Rollout worker 104 uses device cpu +[2023-03-07 15:42:59,649][236315] Rollout worker 105 uses device cpu +[2023-03-07 15:42:59,649][236315] Rollout worker 106 uses device cpu +[2023-03-07 15:42:59,649][236315] Rollout worker 107 uses device cpu +[2023-03-07 15:42:59,650][236315] Rollout worker 108 uses device cpu +[2023-03-07 15:42:59,650][236315] Rollout worker 109 uses device cpu +[2023-03-07 15:42:59,650][236315] Rollout worker 110 uses device cpu +[2023-03-07 15:42:59,651][236315] Rollout worker 111 uses device cpu +[2023-03-07 15:42:59,651][236315] Rollout worker 112 uses device cpu +[2023-03-07 15:42:59,651][236315] Rollout worker 113 uses device cpu +[2023-03-07 15:42:59,652][236315] Rollout worker 114 uses device cpu +[2023-03-07 15:42:59,652][236315] Rollout worker 115 uses device cpu +[2023-03-07 15:42:59,652][236315] Rollout worker 116 uses device cpu +[2023-03-07 15:42:59,653][236315] Rollout worker 117 uses device cpu +[2023-03-07 15:42:59,653][236315] Rollout worker 118 uses device cpu +[2023-03-07 15:42:59,653][236315] Rollout worker 119 uses device cpu +[2023-03-07 15:42:59,654][236315] Rollout worker 120 uses device cpu +[2023-03-07 15:42:59,654][236315] Rollout worker 121 uses device cpu +[2023-03-07 15:42:59,657][236315] Rollout worker 122 uses device cpu +[2023-03-07 15:42:59,657][236315] Rollout worker 123 uses device cpu +[2023-03-07 15:42:59,658][236315] Rollout worker 124 uses device cpu +[2023-03-07 15:42:59,658][236315] Rollout worker 125 uses device cpu +[2023-03-07 15:42:59,658][236315] Rollout worker 126 uses device cpu +[2023-03-07 15:42:59,659][236315] Rollout worker 127 uses device cpu +[2023-03-07 15:43:01,805][236315] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:43:01,806][236315] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:43:02,064][236315] Starting all processes... +[2023-03-07 15:43:02,065][236315] Starting process learner_proc0 +[2023-03-07 15:43:02,950][236315] Starting all processes... +[2023-03-07 15:43:02,952][281427] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:43:02,952][281427] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0 +[2023-03-07 15:43:02,955][236315] Starting process inference_proc0-0 +[2023-03-07 15:43:02,956][236315] Starting process rollout_proc2 +[2023-03-07 15:43:02,961][281427] Num visible devices: 1 +[2023-03-07 15:43:02,957][236315] Starting process rollout_proc5 +[2023-03-07 15:43:02,957][236315] Starting process rollout_proc8 +[2023-03-07 15:43:02,965][281427] Starting seed is not provided +[2023-03-07 15:43:02,967][281427] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:43:02,967][281427] Initializing actor-critic model on device cuda:0 +[2023-03-07 15:43:02,967][281427] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:43:02,968][281427] RunningMeanStd input shape: (1,) +[2023-03-07 15:43:02,958][236315] Starting process rollout_proc11 +[2023-03-07 15:43:02,959][236315] Starting process rollout_proc14 +[2023-03-07 15:43:02,978][281427] ConvEncoder: input_channels=3 +[2023-03-07 15:43:02,963][236315] Starting process rollout_proc17 +[2023-03-07 15:43:02,964][236315] Starting process rollout_proc20 +[2023-03-07 15:43:02,965][236315] Starting process rollout_proc23 +[2023-03-07 15:43:02,967][236315] Starting process rollout_proc26 +[2023-03-07 15:43:02,970][236315] Starting process rollout_proc29 +[2023-03-07 15:43:02,973][236315] Starting process rollout_proc32 +[2023-03-07 15:43:02,975][236315] Starting process rollout_proc35 +[2023-03-07 15:43:02,977][236315] Starting process rollout_proc38 +[2023-03-07 15:43:02,977][236315] Starting process rollout_proc41 +[2023-03-07 15:43:02,978][236315] Starting process rollout_proc44 +[2023-03-07 15:43:03,000][236315] Starting process rollout_proc3 +[2023-03-07 15:43:03,001][236315] Starting process rollout_proc6 +[2023-03-07 15:43:03,007][236315] Starting process rollout_proc9 +[2023-03-07 15:43:03,018][236315] Starting process rollout_proc12 +[2023-03-07 15:43:03,020][236315] Starting process rollout_proc21 +[2023-03-07 15:43:03,020][236315] Starting process rollout_proc15 +[2023-03-07 15:43:03,064][281427] Conv encoder output size: 512 +[2023-03-07 15:43:03,064][281427] Policy head output size: 512 +[2023-03-07 15:43:03,026][236315] Starting process rollout_proc18 +[2023-03-07 15:43:03,034][236315] Starting process rollout_proc27 +[2023-03-07 15:43:03,076][281427] Created Actor Critic model with architecture: +[2023-03-07 15:43:03,035][236315] Starting process rollout_proc24 +[2023-03-07 15:43:03,076][281427] ActorCriticSharedWeights( + (obs_normalizer): ObservationNormalizer( + (running_mean_std): RunningMeanStdDictInPlace( + (running_mean_std): ModuleDict( + (obs): RunningMeanStdInPlace() + ) + ) + ) + (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace) + (encoder): VizdoomEncoder( + (basic_encoder): ConvEncoder( + (enc): RecursiveScriptModule( + original_name=ConvEncoderImpl + (conv_head): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Conv2d) + (1): RecursiveScriptModule(original_name=ELU) + (2): RecursiveScriptModule(original_name=Conv2d) + (3): RecursiveScriptModule(original_name=ELU) + (4): RecursiveScriptModule(original_name=Conv2d) + (5): RecursiveScriptModule(original_name=ELU) + ) + (mlp_layers): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Linear) + (1): RecursiveScriptModule(original_name=ELU) + ) + ) + ) + ) + (core): ModelCoreRNN( + (core): GRU(512, 512) + ) + (decoder): MlpDecoder( + (mlp): Identity() + ) + (critic_linear): Linear(in_features=512, out_features=1, bias=True) + (action_parameterization): ActionParameterizationDefault( + (distribution_linear): Linear(in_features=512, out_features=5, bias=True) + ) +) +[2023-03-07 15:43:03,044][236315] Starting process rollout_proc30 +[2023-03-07 15:43:03,053][236315] Starting process rollout_proc33 +[2023-03-07 15:43:03,067][236315] Starting process rollout_proc36 +[2023-03-07 15:43:03,079][236315] Starting process rollout_proc42 +[2023-03-07 15:43:03,082][236315] Starting process rollout_proc39 +[2023-03-07 15:43:03,098][236315] Starting process rollout_proc7 +[2023-03-07 15:43:03,098][236315] Starting process rollout_proc4 +[2023-03-07 15:43:03,109][236315] Starting process rollout_proc13 +[2023-03-07 15:43:03,109][236315] Starting process rollout_proc10 +[2023-03-07 15:43:03,110][236315] Starting process rollout_proc45 +[2023-03-07 15:43:03,118][236315] Starting process rollout_proc22 +[2023-03-07 15:43:03,122][236315] Starting process rollout_proc19 +[2023-03-07 15:43:03,128][236315] Starting process rollout_proc16 +[2023-03-07 15:43:03,131][236315] Starting process rollout_proc28 +[2023-03-07 15:43:03,137][236315] Starting process rollout_proc31 +[2023-03-07 15:43:03,143][236315] Starting process rollout_proc25 +[2023-03-07 15:43:03,145][236315] Starting process rollout_proc34 +[2023-03-07 15:43:03,150][236315] Starting process rollout_proc37 +[2023-03-07 15:43:03,159][236315] Starting process rollout_proc43 +[2023-03-07 15:43:03,165][236315] Starting process rollout_proc47 +[2023-03-07 15:43:03,173][236315] Starting process rollout_proc40 +[2023-03-07 15:43:03,178][236315] Starting process rollout_proc50 +[2023-03-07 15:43:03,186][236315] Starting process rollout_proc53 +[2023-03-07 15:43:03,186][236315] Starting process rollout_proc56 +[2023-03-07 15:43:03,191][236315] Starting process rollout_proc59 +[2023-03-07 15:43:03,192][236315] Starting process rollout_proc46 +[2023-03-07 15:43:03,199][236315] Starting process rollout_proc62 +[2023-03-07 15:43:03,204][236315] Starting process rollout_proc65 +[2023-03-07 15:43:03,214][236315] Starting process rollout_proc68 +[2023-03-07 15:43:03,216][236315] Starting process rollout_proc71 +[2023-03-07 15:43:03,221][236315] Starting process rollout_proc74 +[2023-03-07 15:43:03,231][236315] Starting process rollout_proc77 +[2023-03-07 15:43:03,237][236315] Starting process rollout_proc80 +[2023-03-07 15:43:03,243][236315] Starting process rollout_proc83 +[2023-03-07 15:43:03,261][236315] Starting process rollout_proc86 +[2023-03-07 15:43:03,317][236315] Starting process rollout_proc75 +[2023-03-07 15:43:03,321][236315] Starting process rollout_proc48 +[2023-03-07 15:43:03,322][236315] Starting process rollout_proc72 +[2023-03-07 15:43:03,325][236315] Starting process rollout_proc81 +[2023-03-07 15:43:03,327][236315] Starting process rollout_proc84 +[2023-03-07 15:43:03,330][236315] Starting process rollout_proc60 +[2023-03-07 15:43:03,334][236315] Starting process rollout_proc63 +[2023-03-07 15:43:03,338][236315] Starting process rollout_proc54 +[2023-03-07 15:43:03,340][236315] Starting process rollout_proc51 +[2023-03-07 15:43:03,340][236315] Starting process rollout_proc87 +[2023-03-07 15:43:03,341][236315] Starting process rollout_proc89 +[2023-03-07 15:43:03,344][236315] Starting process rollout_proc66 +[2023-03-07 15:43:03,347][236315] Starting process rollout_proc69 +[2023-03-07 15:43:03,358][236315] Starting process rollout_proc57 +[2023-03-07 15:43:03,358][236315] Starting process rollout_proc78 +[2023-03-07 15:43:03,368][236315] Starting process rollout_proc76 +[2023-03-07 15:43:03,378][236315] Starting process rollout_proc49 +[2023-03-07 15:43:03,405][236315] Starting process rollout_proc82 +[2023-03-07 15:43:03,405][236315] Starting process rollout_proc73 +[2023-03-07 15:43:03,422][236315] Starting process rollout_proc85 +[2023-03-07 15:43:03,430][236315] Starting process rollout_proc61 +[2023-03-07 15:43:03,438][236315] Starting process rollout_proc64 +[2023-03-07 15:43:03,438][236315] Starting process rollout_proc55 +[2023-03-07 15:43:03,449][236315] Starting process rollout_proc52 +[2023-03-07 15:43:03,461][236315] Starting process rollout_proc90 +[2023-03-07 15:43:03,467][236315] Starting process rollout_proc67 +[2023-03-07 15:43:03,472][236315] Starting process rollout_proc70 +[2023-03-07 15:43:03,477][236315] Starting process rollout_proc58 +[2023-03-07 15:43:03,478][236315] Starting process rollout_proc88 +[2023-03-07 15:43:03,479][236315] Starting process rollout_proc79 +[2023-03-07 15:43:03,500][236315] Starting process rollout_proc92 +[2023-03-07 15:43:03,508][236315] Starting process rollout_proc95 +[2023-03-07 15:43:03,516][236315] Starting process rollout_proc98 +[2023-03-07 15:43:03,530][236315] Starting process rollout_proc101 +[2023-03-07 15:43:03,530][236315] Starting process rollout_proc104 +[2023-03-07 15:43:03,582][236315] Starting process rollout_proc107 +[2023-03-07 15:43:03,655][236315] Starting process rollout_proc110 +[2023-03-07 15:43:03,715][236315] Starting process rollout_proc116 +[2023-03-07 15:43:03,707][236315] Starting process rollout_proc113 +[2023-03-07 15:43:03,706][236315] Starting process rollout_proc91 +[2023-03-07 15:43:03,734][236315] Starting process rollout_proc119 +[2023-03-07 15:43:03,752][236315] Starting process rollout_proc122 +[2023-03-07 15:43:03,752][236315] Starting process rollout_proc125 +[2023-03-07 15:43:03,756][236315] Starting process rollout_proc93 +[2023-03-07 15:43:03,776][236315] Starting process rollout_proc96 +[2023-03-07 15:43:03,793][236315] Starting process rollout_proc105 +[2023-03-07 15:43:03,843][236315] Starting process rollout_proc102 +[2023-03-07 15:43:03,863][236315] Starting process rollout_proc99 +[2023-03-07 15:43:03,864][236315] Starting process rollout_proc117 +[2023-03-07 15:43:03,882][236315] Starting process rollout_proc108 +[2023-03-07 15:43:03,934][236315] Starting process rollout_proc114 +[2023-03-07 15:43:03,959][236315] Starting process rollout_proc111 +[2023-03-07 15:43:03,973][236315] Starting process rollout_proc123 +[2023-03-07 15:43:03,996][236315] Starting process rollout_proc94 +[2023-03-07 15:43:04,003][236315] Starting process rollout_proc126 +[2023-03-07 15:43:04,003][236315] Starting process rollout_proc120 +[2023-03-07 15:43:04,021][236315] Starting process rollout_proc97 +[2023-03-07 15:43:04,047][236315] Starting process rollout_proc103 +[2023-03-07 15:43:04,047][236315] Starting process rollout_proc100 +[2023-03-07 15:43:04,051][236315] Starting process rollout_proc109 +[2023-03-07 15:43:04,069][236315] Starting process rollout_proc115 +[2023-03-07 15:43:04,069][236315] Starting process rollout_proc106 +[2023-03-07 15:43:04,126][236315] Starting process rollout_proc118 +[2023-03-07 15:43:04,190][236315] Starting process rollout_proc112 +[2023-03-07 15:43:04,213][236315] Starting process rollout_proc124 +[2023-03-07 15:43:04,213][236315] Starting process rollout_proc121 +[2023-03-07 15:43:04,239][236315] Starting process rollout_proc127 +[2023-03-07 15:43:04,887][281661] Worker 15 uses CPU cores [15] +[2023-03-07 15:43:05,170][281581] Worker 17 uses CPU cores [17] +[2023-03-07 15:43:05,351][281583] Worker 26 uses CPU cores [26] +[2023-03-07 15:43:05,354][281669] Worker 42 uses CPU cores [42] +[2023-03-07 15:43:05,360][281657] Worker 44 uses CPU cores [44] +[2023-03-07 15:43:05,403][236315] Starting process rollout_proc0 +[2023-03-07 15:43:05,411][281574] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:43:05,411][281574] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0 +[2023-03-07 15:43:05,423][281574] Num visible devices: 1 +[2023-03-07 15:43:05,462][281580] Worker 20 uses CPU cores [20] +[2023-03-07 15:43:05,514][281575] Worker 5 uses CPU cores [5] +[2023-03-07 15:43:05,532][281582] Worker 23 uses CPU cores [23] +[2023-03-07 15:43:05,568][236315] Starting process rollout_proc1 +[2023-03-07 15:43:05,634][281649] Worker 35 uses CPU cores [35] +[2023-03-07 15:43:05,658][281584] Worker 29 uses CPU cores [29] +[2023-03-07 15:43:05,703][281602] Worker 32 uses CPU cores [32] +[2023-03-07 15:43:05,713][281576] Worker 2 uses CPU cores [2] +[2023-03-07 15:43:05,754][281952] Worker 59 uses CPU cores [59] +[2023-03-07 15:43:05,826][281578] Worker 11 uses CPU cores [11] +[2023-03-07 15:43:05,838][281577] Worker 8 uses CPU cores [8] +[2023-03-07 15:43:05,858][281658] Worker 21 uses CPU cores [21] +[2023-03-07 15:43:05,858][281655] Worker 12 uses CPU cores [12] +[2023-03-07 15:43:05,867][281799] Worker 39 uses CPU cores [39] +[2023-03-07 15:43:05,882][281579] Worker 14 uses CPU cores [14] +[2023-03-07 15:43:05,928][281660] Worker 18 uses CPU cores [18] +[2023-03-07 15:43:05,984][281800] Worker 4 uses CPU cores [4] +[2023-03-07 15:43:06,000][281654] Worker 6 uses CPU cores [6] +[2023-03-07 15:43:06,138][281804] Worker 45 uses CPU cores [45] +[2023-03-07 15:43:06,146][281960] Worker 56 uses CPU cores [56] +[2023-03-07 15:43:06,150][281665] Worker 33 uses CPU cores [33] +[2023-03-07 15:43:06,170][281663] Worker 30 uses CPU cores [30] +[2023-03-07 15:43:06,180][281656] Worker 9 uses CPU cores [9] +[2023-03-07 15:43:06,198][281652] Worker 38 uses CPU cores [38] +[2023-03-07 15:43:06,202][281805] Worker 19 uses CPU cores [19] +[2023-03-07 15:43:06,206][281953] Worker 62 uses CPU cores [62] +[2023-03-07 15:43:06,232][281427] Using optimizer +[2023-03-07 15:43:06,233][281427] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth... +[2023-03-07 15:43:06,239][281664] Worker 24 uses CPU cores [24] +[2023-03-07 15:43:06,254][281667] Worker 36 uses CPU cores [36] +[2023-03-07 15:43:06,260][281427] Loading model from checkpoint +[2023-03-07 15:43:06,266][281427] Loaded experiment state at self.train_step=979, self.env_steps=4009984 +[2023-03-07 15:43:06,266][281427] Initialized policy 0 weights for model version 979 +[2023-03-07 15:43:06,266][281809] Worker 28 uses CPU cores [28] +[2023-03-07 15:43:06,292][281427] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:43:06,302][281959] Worker 68 uses CPU cores [68] +[2023-03-07 15:43:06,306][281427] LearnerWorker_p0 finished initialization! +[2023-03-07 15:43:06,376][281662] Worker 27 uses CPU cores [27] +[2023-03-07 15:43:06,401][281815] Worker 43 uses CPU cores [43] +[2023-03-07 15:43:06,444][281801] Worker 13 uses CPU cores [13] +[2023-03-07 15:43:06,473][281966] Worker 84 uses CPU cores [84] +[2023-03-07 15:43:06,522][281574] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:43:06,533][281574] RunningMeanStd input shape: (1,) +[2023-03-07 15:43:06,541][281670] Worker 7 uses CPU cores [7] +[2023-03-07 15:43:06,548][281954] Worker 50 uses CPU cores [50] +[2023-03-07 15:43:06,549][281803] Worker 10 uses CPU cores [10] +[2023-03-07 15:43:06,570][281808] Worker 25 uses CPU cores [25] +[2023-03-07 15:43:06,575][281574] ConvEncoder: input_channels=3 +[2023-03-07 15:43:06,586][281951] Worker 83 uses CPU cores [83] +[2023-03-07 15:43:06,607][281950] Worker 80 uses CPU cores [80] +[2023-03-07 15:43:06,610][281806] Worker 16 uses CPU cores [16] +[2023-03-07 15:43:06,630][281945] Worker 74 uses CPU cores [74] +[2023-03-07 15:43:06,634][281946] Worker 71 uses CPU cores [71] +[2023-03-07 15:43:06,642][281651] Worker 41 uses CPU cores [41] +[2023-03-07 15:43:06,642][281802] Worker 22 uses CPU cores [22] +[2023-03-07 15:43:06,646][283352] Worker 117 uses CPU cores [117] +[2023-03-07 15:43:06,654][282558] Worker 58 uses CPU cores [58] +[2023-03-07 15:43:06,658][281653] Worker 3 uses CPU cores [3] +[2023-03-07 15:43:06,666][281810] Worker 34 uses CPU cores [34] +[2023-03-07 15:43:06,690][281944] Worker 40 uses CPU cores [40] +[2023-03-07 15:43:06,709][281961] Worker 77 uses CPU cores [77] +[2023-03-07 15:43:06,725][281574] Conv encoder output size: 512 +[2023-03-07 15:43:06,725][281574] Policy head output size: 512 +[2023-03-07 15:43:06,758][281965] Worker 72 uses CPU cores [72] +[2023-03-07 15:43:06,758][281969] Worker 63 uses CPU cores [63] +[2023-03-07 15:43:06,762][281968] Worker 54 uses CPU cores [54] +[2023-03-07 15:43:06,802][282565] Worker 52 uses CPU cores [52] +[2023-03-07 15:43:06,802][281962] Worker 75 uses CPU cores [75] +[2023-03-07 15:43:06,814][281955] Worker 53 uses CPU cores [53] +[2023-03-07 15:43:06,818][281973] Worker 87 uses CPU cores [87] +[2023-03-07 15:43:06,822][281813] Worker 37 uses CPU cores [37] +[2023-03-07 15:43:06,826][281957] Worker 46 uses CPU cores [46] +[2023-03-07 15:43:06,829][281963] Worker 48 uses CPU cores [48] +[2023-03-07 15:43:06,829][281991] Worker 61 uses CPU cores [61] +[2023-03-07 15:43:06,830][281958] Worker 65 uses CPU cores [65] +[2023-03-07 15:43:06,832][281948] Worker 47 uses CPU cores [47] +[2023-03-07 15:43:06,846][282283] Worker 55 uses CPU cores [55] +[2023-03-07 15:43:06,847][281992] Worker 85 uses CPU cores [85] +[2023-03-07 15:43:06,854][281990] Worker 82 uses CPU cores [82] +[2023-03-07 15:43:06,862][281967] Worker 60 uses CPU cores [60] +[2023-03-07 15:43:06,876][281971] Worker 89 uses CPU cores [89] +[2023-03-07 15:43:06,878][283150] Worker 96 uses CPU cores [96] +[2023-03-07 15:43:06,881][281989] Worker 73 uses CPU cores [73] +[2023-03-07 15:43:06,886][281964] Worker 81 uses CPU cores [81] +[2023-03-07 15:43:06,891][282574] Worker 67 uses CPU cores [67] +[2023-03-07 15:43:06,911][281984] Worker 49 uses CPU cores [49] +[2023-03-07 15:43:06,921][281975] Worker 57 uses CPU cores [57] +[2023-03-07 15:43:06,934][281993] Worker 64 uses CPU cores [64] +[2023-03-07 15:43:06,935][281956] Worker 86 uses CPU cores [86] +[2023-03-07 15:43:06,954][281807] Worker 31 uses CPU cores [31] +[2023-03-07 15:43:06,966][284304] Worker 124 uses CPU cores [124] +[2023-03-07 15:43:06,982][282697] Worker 95 uses CPU cores [95] +[2023-03-07 15:43:06,985][283854] Worker 120 uses CPU cores [120] +[2023-03-07 15:43:06,986][281976] Worker 78 uses CPU cores [78] +[2023-03-07 15:43:06,993][282695] Worker 79 uses CPU cores [79] +[2023-03-07 15:43:07,031][281988] Worker 76 uses CPU cores [76] +[2023-03-07 15:43:07,046][282698] Worker 101 uses CPU cores [101] +[2023-03-07 15:43:07,058][281970] Worker 51 uses CPU cores [51] +[2023-03-07 15:43:07,066][283278] Worker 102 uses CPU cores [102] +[2023-03-07 15:43:07,073][283641] Worker 94 uses CPU cores [94] +[2023-03-07 15:43:07,090][282763] Worker 107 uses CPU cores [107] +[2023-03-07 15:43:07,090][281974] Worker 69 uses CPU cores [69] +[2023-03-07 15:43:07,094][282184] Worker 70 uses CPU cores [70] +[2023-03-07 15:43:07,096][283727] Worker 126 uses CPU cores [126] +[2023-03-07 15:43:07,098][282183] Worker 90 uses CPU cores [90] +[2023-03-07 15:43:07,104][283396] Worker 114 uses CPU cores [114] +[2023-03-07 15:43:07,105][283085] Worker 91 uses CPU cores [91] +[2023-03-07 15:43:07,110][281972] Worker 66 uses CPU cores [66] +[2023-03-07 15:43:07,122][283281] Worker 99 uses CPU cores [99] +[2023-03-07 15:43:07,134][283890] Worker 100 uses CPU cores [100] +[2023-03-07 15:43:07,134][283408] Worker 108 uses CPU cores [108] +[2023-03-07 15:43:07,135][283019] Worker 122 uses CPU cores [122] +[2023-03-07 15:43:07,149][283084] Worker 119 uses CPU cores [119] +[2023-03-07 15:43:07,153][282694] Worker 92 uses CPU cores [92] +[2023-03-07 15:43:07,153][282696] Worker 98 uses CPU cores [98] +[2023-03-07 15:43:07,162][282904] Worker 113 uses CPU cores [113] +[2023-03-07 15:43:07,164][284053] Worker 118 uses CPU cores [118] +[2023-03-07 15:43:07,175][284309] Worker 121 uses CPU cores [121] +[2023-03-07 15:43:07,177][282699] Worker 104 uses CPU cores [104] +[2023-03-07 15:43:07,182][283469] Worker 123 uses CPU cores [123] +[2023-03-07 15:43:07,202][282891] Worker 116 uses CPU cores [116] +[2023-03-07 15:43:07,224][282827] Worker 110 uses CPU cores [110] +[2023-03-07 15:43:07,225][284047] Worker 115 uses CPU cores [115] +[2023-03-07 15:43:07,249][283879] Worker 97 uses CPU cores [97] +[2023-03-07 15:43:07,249][284048] Worker 109 uses CPU cores [109] +[2023-03-07 15:43:07,258][284200] Worker 106 uses CPU cores [106] +[2023-03-07 15:43:07,262][284166] Worker 112 uses CPU cores [112] +[2023-03-07 15:43:07,266][283149] Worker 93 uses CPU cores [93] +[2023-03-07 15:43:07,271][284345] Worker 127 uses CPU cores [127] +[2023-03-07 15:43:07,286][283960] Worker 103 uses CPU cores [103] +[2023-03-07 15:43:07,294][283277] Worker 105 uses CPU cores [105] +[2023-03-07 15:43:07,302][283020] Worker 125 uses CPU cores [125] +[2023-03-07 15:43:07,308][283417] Worker 111 uses CPU cores [111] +[2023-03-07 15:43:07,316][282693] Worker 88 uses CPU cores [88] +[2023-03-07 15:43:07,508][287383] Worker 0 uses CPU cores [0] +[2023-03-07 15:43:07,521][288232] Worker 1 uses CPU cores [1] +[2023-03-07 15:43:07,778][236315] Inference worker 0-0 is ready! +[2023-03-07 15:43:07,779][236315] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:43:07,838][281953] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,838][281968] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,841][282283] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,842][281958] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,844][281984] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,844][281665] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,844][281808] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,846][281963] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,847][281804] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,847][281662] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,848][281993] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,849][281991] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,849][281975] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,850][282558] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,851][281810] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,853][281809] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,853][281649] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,853][281970] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,853][281954] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,853][281969] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,854][284304] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,854][281957] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,854][281584] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,855][281948] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,855][281952] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,855][283854] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,855][282697] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,855][281799] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,856][281807] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,856][281575] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,856][281944] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,856][281667] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,856][282183] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,857][281654] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,857][281663] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,857][281967] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,857][281669] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,857][281992] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,858][281651] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,858][281657] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,858][281664] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][281576] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][281971] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][282891] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][281946] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][281670] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,859][284166] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281602] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281815] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][283150] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281801] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281972] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281960] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281656] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,860][281583] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][282574] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][281956] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][283890] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][282565] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][281580] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][281988] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,861][283352] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281813] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281964] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281653] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][284200] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281961] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][282763] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281800] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][283408] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281955] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,862][281577] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,863][281989] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,863][281959] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,863][281652] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][281973] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][282698] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][283278] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][281658] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][281965] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,864][281990] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,865][281802] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,865][281976] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,865][281803] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,865][281578] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,866][281962] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,866][281966] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,866][281945] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,866][281655] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,866][281660] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,868][281806] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,868][281661] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,868][281581] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,869][283396] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,871][281950] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,871][281579] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,882][281582] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,892][281951] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,896][282695] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,905][283641] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,927][281974] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,928][282184] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,939][283727] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,953][283281] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:07,971][281805] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,009][283084] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,009][283019] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,010][282904] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,010][282696] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,016][283085] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,024][284309] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,026][282699] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,026][283469] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,027][284053] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,082][284047] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,097][284048] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,116][283149] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,126][284345] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,127][282694] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,145][283960] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,156][283020] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,171][283417] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,171][282827] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,239][283879] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,242][283277] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,365][288232] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,395][282693] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,461][287383] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:43:08,538][236315] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 4009984. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:08,872][281948] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,872][281957] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,874][281651] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,929][281963] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,932][281993] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,932][281802] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,932][283150] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,932][281958] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,933][281664] Decorrelating experience for 0 frames... +[2023-03-07 15:43:08,933][281804] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,051][284200] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,114][283150] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,117][283960] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,118][283469] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,119][281993] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,121][281800] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,123][283277] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,126][283890] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,127][281649] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,128][283281] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,230][282184] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,292][281576] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,296][283960] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,302][281580] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,302][281658] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,307][281953] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,310][281945] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,312][281984] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,380][281649] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,381][283281] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,407][281661] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,468][282693] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,473][281969] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,485][281989] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,487][281976] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,488][281974] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,490][282574] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,495][281954] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,555][283408] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,560][281970] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,581][281813] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,659][284166] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,664][281976] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,665][282695] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,670][281960] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,675][281662] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,689][283396] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,690][281663] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,734][283960] Decorrelating experience for 64 frames... +[2023-03-07 15:43:09,759][281806] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,765][283641] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,841][282695] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,843][281815] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,844][281602] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,849][282891] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,858][283020] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,868][281951] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,876][281801] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,910][281664] Decorrelating experience for 32 frames... +[2023-03-07 15:43:09,942][284053] Decorrelating experience for 0 frames... +[2023-03-07 15:43:09,943][284309] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,021][281946] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,023][281990] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,023][283417] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,029][283408] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,051][282696] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,051][283277] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,054][283149] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,083][281959] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,122][281954] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,147][283879] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,195][281664] Decorrelating experience for 64 frames... +[2023-03-07 15:43:10,203][281801] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,204][281948] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,225][282696] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,230][283149] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,232][281972] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,235][281665] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,258][281799] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,293][281946] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,322][282574] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,371][281579] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,377][282695] Decorrelating experience for 64 frames... +[2023-03-07 15:43:10,387][282698] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,404][281971] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,408][282694] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,414][281665] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,433][287383] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,448][283084] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,477][281968] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,531][282283] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,563][284345] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,579][281971] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,598][282763] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,612][281579] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,624][281976] Decorrelating experience for 64 frames... +[2023-03-07 15:43:10,632][282694] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,642][281575] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,651][282699] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,673][281974] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,738][281661] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,745][282283] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,758][282574] Decorrelating experience for 64 frames... +[2023-03-07 15:43:10,789][281654] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,804][281968] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,819][283084] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,820][281804] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,840][282697] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,849][281960] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,860][281583] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,921][281663] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,937][281807] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,959][282183] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,962][282698] Decorrelating experience for 32 frames... +[2023-03-07 15:43:10,985][283085] Decorrelating experience for 0 frames... +[2023-03-07 15:43:10,996][281964] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,009][281952] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,024][284053] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,025][281975] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,034][281658] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,107][283352] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,109][281651] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,135][281578] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,145][281948] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,164][283890] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,169][284048] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,184][281952] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,198][282698] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,199][281975] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,208][281662] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,284][281810] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,287][282283] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,312][281972] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,319][284304] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,343][281652] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,362][287383] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,371][282565] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,379][282827] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,385][281975] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,390][281965] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,458][281810] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,462][283085] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,485][282183] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,498][281972] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,517][282184] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,537][281946] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,547][281651] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,557][283417] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,560][281582] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,564][281807] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,638][281656] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,652][281970] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,661][282574] Decorrelating experience for 96 frames... +[2023-03-07 15:43:11,671][281658] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,695][281806] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,715][281581] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,735][282695] Decorrelating experience for 96 frames... +[2023-03-07 15:43:11,739][281582] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,740][281975] Decorrelating experience for 96 frames... +[2023-03-07 15:43:11,742][281799] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,812][281578] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,828][281958] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,841][281577] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,850][281807] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,866][282574] Decorrelating experience for 128 frames... +[2023-03-07 15:43:11,892][283417] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,913][282904] Decorrelating experience for 0 frames... +[2023-03-07 15:43:11,915][281656] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,922][282696] Decorrelating experience for 64 frames... +[2023-03-07 15:43:11,939][281581] Decorrelating experience for 32 frames... +[2023-03-07 15:43:11,985][281946] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,006][282827] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,047][281970] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,068][281813] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,083][281974] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,091][281649] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,093][281575] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,093][281960] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,102][281662] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,114][281663] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,160][282693] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,188][281976] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,223][281993] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,250][281972] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,261][282558] Decorrelating experience for 0 frames... +[2023-03-07 15:43:12,270][281957] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,271][284053] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,275][281658] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,277][283408] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,289][281960] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,334][281963] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,374][281989] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,398][281961] Decorrelating experience for 0 frames... +[2023-03-07 15:43:12,426][281952] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,437][281660] Decorrelating experience for 0 frames... +[2023-03-07 15:43:12,454][283020] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,455][281976] Decorrelating experience for 128 frames... +[2023-03-07 15:43:12,458][281968] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,466][281663] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,487][283085] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,516][281969] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,570][282696] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,590][281662] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,602][282698] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,611][281660] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,636][282694] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,638][281806] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,639][283278] Decorrelating experience for 0 frames... +[2023-03-07 15:43:12,650][282693] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,660][281972] Decorrelating experience for 128 frames... +[2023-03-07 15:43:12,691][282574] Decorrelating experience for 160 frames... +[2023-03-07 15:43:12,744][281583] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,788][281575] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,790][281802] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,816][281665] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,818][283084] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,827][283408] Decorrelating experience for 96 frames... +[2023-03-07 15:43:12,835][281815] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,845][284200] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,872][281963] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,886][282763] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,920][281577] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,965][283278] Decorrelating experience for 32 frames... +[2023-03-07 15:43:12,971][281989] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,994][283281] Decorrelating experience for 64 frames... +[2023-03-07 15:43:12,997][281967] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,000][281810] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,038][281576] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,047][282827] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,051][281960] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,063][282698] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,095][283727] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,149][281660] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,176][282574] Decorrelating experience for 192 frames... +[2023-03-07 15:43:13,188][282696] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,196][288232] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,226][281958] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,232][281663] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,242][282699] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,255][284304] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,301][283020] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,325][281576] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,338][281991] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,353][282693] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,363][281951] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,383][281583] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,405][281948] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,409][284166] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,434][282698] Decorrelating experience for 160 frames... +[2023-03-07 15:43:13,463][282184] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,478][281961] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,506][281662] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,512][283641] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,529][281952] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:13,558][281969] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,559][281953] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,586][281971] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,621][281652] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,646][282827] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,666][281948] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,682][282697] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,691][284047] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,698][283278] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,710][281975] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,736][281657] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,760][281577] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,765][281664] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,805][281651] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,850][281649] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,860][284345] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,877][288232] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,892][282694] Decorrelating experience for 96 frames... +[2023-03-07 15:43:13,912][283019] Decorrelating experience for 0 frames... +[2023-03-07 15:43:13,929][282827] Decorrelating experience for 128 frames... +[2023-03-07 15:43:13,936][281957] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,941][284309] Decorrelating experience for 32 frames... +[2023-03-07 15:43:13,944][281953] Decorrelating experience for 64 frames... +[2023-03-07 15:43:13,981][281966] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,054][282904] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,057][281664] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,078][281988] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,084][281584] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,110][281577] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,114][283020] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,131][281579] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,134][283149] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,162][283277] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,168][282693] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,228][281967] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,253][281951] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,264][281970] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,276][281804] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,295][281957] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,297][281953] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,313][281649] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,313][281582] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,349][281964] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,388][281575] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,427][284309] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,440][284200] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,459][281974] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,460][283277] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,478][281990] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,479][283020] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,489][281583] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,503][283396] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,531][281965] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,562][281973] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,602][281957] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,616][281809] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,637][281654] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,638][281662] Decorrelating experience for 160 frames... +[2023-03-07 15:43:14,653][281951] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,680][281960] Decorrelating experience for 160 frames... +[2023-03-07 15:43:14,714][281807] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,723][281584] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,737][282699] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,748][283149] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,775][284309] Decorrelating experience for 96 frames... +[2023-03-07 15:43:14,793][281972] Decorrelating experience for 160 frames... +[2023-03-07 15:43:14,820][282565] Decorrelating experience for 32 frames... +[2023-03-07 15:43:14,821][281965] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,831][282574] Decorrelating experience for 224 frames... +[2023-03-07 15:43:14,872][281654] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,903][281583] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,929][281956] Decorrelating experience for 0 frames... +[2023-03-07 15:43:14,958][282694] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,963][281799] Decorrelating experience for 64 frames... +[2023-03-07 15:43:14,971][282695] Decorrelating experience for 128 frames... +[2023-03-07 15:43:14,979][283960] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,001][283408] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,005][283278] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,010][281660] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,054][283020] Decorrelating experience for 160 frames... +[2023-03-07 15:43:15,083][281579] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,104][282283] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,138][281993] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,146][281581] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,154][282558] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,171][288232] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,182][282694] Decorrelating experience for 160 frames... +[2023-03-07 15:43:15,186][283727] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,188][282699] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,233][282763] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,267][281657] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,285][281992] Decorrelating experience for 0 frames... +[2023-03-07 15:43:15,314][283278] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,330][283469] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,334][281576] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,362][283417] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,369][281655] Decorrelating experience for 0 frames... +[2023-03-07 15:43:15,370][281951] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,376][282698] Decorrelating experience for 192 frames... +[2023-03-07 15:43:15,415][281960] Decorrelating experience for 192 frames... +[2023-03-07 15:43:15,441][284200] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,459][283085] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,505][281806] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,510][281956] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,548][281655] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,573][284166] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,573][283417] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,579][283019] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,591][282574] Decorrelating experience for 256 frames... +[2023-03-07 15:43:15,608][281963] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,624][281967] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,678][282558] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,684][282763] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,687][281989] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,727][281583] Decorrelating experience for 160 frames... +[2023-03-07 15:43:15,748][282695] Decorrelating experience for 160 frames... +[2023-03-07 15:43:15,756][281803] Decorrelating experience for 0 frames... +[2023-03-07 15:43:15,761][281960] Decorrelating experience for 224 frames... +[2023-03-07 15:43:15,782][281972] Decorrelating experience for 192 frames... +[2023-03-07 15:43:15,789][284309] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,808][284345] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,854][281952] Decorrelating experience for 128 frames... +[2023-03-07 15:43:15,863][283641] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,868][281665] Decorrelating experience for 96 frames... +[2023-03-07 15:43:15,905][281664] Decorrelating experience for 160 frames... +[2023-03-07 15:43:15,923][281990] Decorrelating experience for 64 frames... +[2023-03-07 15:43:15,940][281602] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,942][284047] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,965][281992] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,980][281803] Decorrelating experience for 32 frames... +[2023-03-07 15:43:15,987][281957] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,028][283277] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,038][287383] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,060][283641] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,080][284166] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,098][281956] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,124][283150] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,125][283019] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,145][281815] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,152][284200] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,168][282283] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,204][281992] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,215][283149] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,237][281984] Decorrelating experience for 32 frames... +[2023-03-07 15:43:16,260][281805] Decorrelating experience for 0 frames... +[2023-03-07 15:43:16,301][283960] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,325][281957] Decorrelating experience for 192 frames... +[2023-03-07 15:43:16,329][281969] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,331][281972] Decorrelating experience for 224 frames... +[2023-03-07 15:43:16,347][281956] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,377][284345] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,379][281654] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,392][281958] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,414][284309] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,436][283149] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,475][281578] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,502][284053] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,513][281993] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,528][281967] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,557][281808] Decorrelating experience for 0 frames... +[2023-03-07 15:43:16,566][283854] Decorrelating experience for 0 frames... +[2023-03-07 15:43:16,569][281583] Decorrelating experience for 192 frames... +[2023-03-07 15:43:16,593][283278] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,615][281971] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,623][281649] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,655][282693] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,701][281973] Decorrelating experience for 32 frames... +[2023-03-07 15:43:16,715][288232] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,734][281956] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,736][281602] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,763][282574] Decorrelating experience for 288 frames... +[2023-03-07 15:43:16,774][281957] Decorrelating experience for 224 frames... +[2023-03-07 15:43:16,791][281967] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,806][281958] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,831][282558] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,850][281802] Decorrelating experience for 64 frames... +[2023-03-07 15:43:16,885][281665] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,893][283854] Decorrelating experience for 32 frames... +[2023-03-07 15:43:16,913][281658] Decorrelating experience for 128 frames... +[2023-03-07 15:43:16,931][282698] Decorrelating experience for 224 frames... +[2023-03-07 15:43:16,943][287383] Decorrelating experience for 96 frames... +[2023-03-07 15:43:16,954][283277] Decorrelating experience for 160 frames... +[2023-03-07 15:43:16,990][281973] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,008][281969] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,016][281653] Decorrelating experience for 0 frames... +[2023-03-07 15:43:17,029][281958] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,064][281967] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,073][281956] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,086][281580] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,130][281971] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,141][282574] Decorrelating experience for 320 frames... +[2023-03-07 15:43:17,163][281661] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,188][283727] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,192][281805] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,207][281989] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,241][281993] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,244][282695] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,250][281815] Decorrelating experience for 96 frames... +[2023-03-07 15:43:17,276][281962] Decorrelating experience for 0 frames... +[2023-03-07 15:43:17,306][281577] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,334][281969] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,338][283277] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,366][281657] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,377][281966] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,386][281952] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,427][281946] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,431][281805] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,449][281661] Decorrelating experience for 96 frames... +[2023-03-07 15:43:17,453][281655] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,483][281945] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,517][281958] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,520][283879] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,547][282693] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,556][281975] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,563][281649] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,603][281962] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,608][281956] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,632][281584] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,636][281957] Decorrelating experience for 256 frames... +[2023-03-07 15:43:17,671][281952] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,698][281654] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,718][281966] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,724][283352] Decorrelating experience for 32 frames... +[2023-03-07 15:43:17,733][282694] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,739][282574] Decorrelating experience for 352 frames... +[2023-03-07 15:43:17,779][281953] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,802][281804] Decorrelating experience for 96 frames... +[2023-03-07 15:43:17,811][281581] Decorrelating experience for 96 frames... +[2023-03-07 15:43:17,819][283396] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,850][281946] Decorrelating experience for 160 frames... +[2023-03-07 15:43:17,914][281649] Decorrelating experience for 224 frames... +[2023-03-07 15:43:17,915][282763] Decorrelating experience for 128 frames... +[2023-03-07 15:43:17,919][281657] Decorrelating experience for 96 frames... +[2023-03-07 15:43:17,962][282183] Decorrelating experience for 64 frames... +[2023-03-07 15:43:17,966][282693] Decorrelating experience for 224 frames... +[2023-03-07 15:43:17,975][281664] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,981][281993] Decorrelating experience for 192 frames... +[2023-03-07 15:43:17,989][283854] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,017][281948] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,030][283641] Decorrelating experience for 128 frames... +[2023-03-07 15:43:18,093][283727] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,137][281652] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,151][283890] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,157][284309] Decorrelating experience for 192 frames... +[2023-03-07 15:43:18,162][281813] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,166][282698] Decorrelating experience for 256 frames... +[2023-03-07 15:43:18,171][281815] Decorrelating experience for 128 frames... +[2023-03-07 15:43:18,172][283020] Decorrelating experience for 192 frames... +[2023-03-07 15:43:18,210][283879] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,250][282695] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,271][281945] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,376][283281] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,384][281661] Decorrelating experience for 128 frames... +[2023-03-07 15:43:18,384][283854] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,384][283149] Decorrelating experience for 192 frames... +[2023-03-07 15:43:18,387][281993] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,388][281580] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,388][281989] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,400][284309] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,467][283020] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:18,582][281958] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,587][282565] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,587][281667] Decorrelating experience for 0 frames... +[2023-03-07 15:43:18,587][281663] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,588][281807] Decorrelating experience for 128 frames... +[2023-03-07 15:43:18,592][283281] Decorrelating experience for 128 frames... +[2023-03-07 15:43:18,624][281584] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,655][281578] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,656][281962] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,670][283150] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,776][282763] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,781][283417] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,781][281799] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,782][281956] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,826][281652] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,840][283149] Decorrelating experience for 224 frames... +[2023-03-07 15:43:18,841][281958] Decorrelating experience for 256 frames... +[2023-03-07 15:43:18,872][283469] Decorrelating experience for 64 frames... +[2023-03-07 15:43:18,885][283278] Decorrelating experience for 192 frames... +[2023-03-07 15:43:18,960][281661] Decorrelating experience for 160 frames... +[2023-03-07 15:43:18,962][284309] Decorrelating experience for 256 frames... +[2023-03-07 15:43:18,977][281965] Decorrelating experience for 96 frames... +[2023-03-07 15:43:18,978][283727] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,011][283879] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,025][282693] Decorrelating experience for 256 frames... +[2023-03-07 15:43:19,029][281974] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,071][283469] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,076][281955] Decorrelating experience for 0 frames... +[2023-03-07 15:43:19,102][283084] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,162][282763] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,170][281663] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,171][283417] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,190][281965] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,206][281807] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,212][281958] Decorrelating experience for 288 frames... +[2023-03-07 15:43:19,218][282283] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,254][283281] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,282][284309] Decorrelating experience for 288 frames... +[2023-03-07 15:43:19,282][283020] Decorrelating experience for 256 frames... +[2023-03-07 15:43:19,342][281584] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,369][281962] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,380][283890] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,386][281976] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,393][281580] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,404][283854] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,427][281948] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,429][283278] Decorrelating experience for 224 frames... +[2023-03-07 15:43:19,480][281946] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,530][281970] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,548][283417] Decorrelating experience for 224 frames... +[2023-03-07 15:43:19,557][282763] Decorrelating experience for 224 frames... +[2023-03-07 15:43:19,578][281968] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,583][284048] Decorrelating experience for 32 frames... +[2023-03-07 15:43:19,588][281966] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,629][283277] Decorrelating experience for 224 frames... +[2023-03-07 15:43:19,638][283084] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,658][281974] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,670][282283] Decorrelating experience for 192 frames... +[2023-03-07 15:43:19,714][281584] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,731][281962] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,735][281944] Decorrelating experience for 0 frames... +[2023-03-07 15:43:19,765][281965] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,772][284048] Decorrelating experience for 64 frames... +[2023-03-07 15:43:19,780][281602] Decorrelating experience for 96 frames... +[2023-03-07 15:43:19,807][281970] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,829][283879] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,835][281579] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,883][282693] Decorrelating experience for 288 frames... +[2023-03-07 15:43:19,903][282698] Decorrelating experience for 288 frames... +[2023-03-07 15:43:19,939][283277] Decorrelating experience for 256 frames... +[2023-03-07 15:43:19,945][283408] Decorrelating experience for 160 frames... +[2023-03-07 15:43:19,955][281581] Decorrelating experience for 128 frames... +[2023-03-07 15:43:19,957][283417] Decorrelating experience for 256 frames... +[2023-03-07 15:43:19,990][283278] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,001][281664] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,015][281667] Decorrelating experience for 32 frames... +[2023-03-07 15:43:20,018][281580] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,068][281663] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,079][283854] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,134][282183] Decorrelating experience for 96 frames... +[2023-03-07 15:43:20,140][281990] Decorrelating experience for 96 frames... +[2023-03-07 15:43:20,141][281945] Decorrelating experience for 96 frames... +[2023-03-07 15:43:20,146][281958] Decorrelating experience for 320 frames... +[2023-03-07 15:43:20,180][282283] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,184][281808] Decorrelating experience for 32 frames... +[2023-03-07 15:43:20,193][281968] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,198][281956] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,250][281965] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,257][281652] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,320][281664] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,333][281960] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,352][281579] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,355][283879] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,365][281944] Decorrelating experience for 32 frames... +[2023-03-07 15:43:20,379][281663] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,415][281802] Decorrelating experience for 96 frames... +[2023-03-07 15:43:20,439][281946] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,440][284166] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,443][281976] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,496][281807] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,538][281656] Decorrelating experience for 64 frames... +[2023-03-07 15:43:20,540][281651] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,548][281602] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,549][281965] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,566][281806] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,596][281974] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,618][283854] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,622][282763] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,631][281581] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,685][283408] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,724][284309] Decorrelating experience for 320 frames... +[2023-03-07 15:43:20,734][281652] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,736][283417] Decorrelating experience for 288 frames... +[2023-03-07 15:43:20,745][281580] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,777][283960] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,778][281802] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,792][281807] Decorrelating experience for 224 frames... +[2023-03-07 15:43:20,811][281990] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,861][281993] Decorrelating experience for 256 frames... +[2023-03-07 15:43:20,871][281799] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,906][281945] Decorrelating experience for 128 frames... +[2023-03-07 15:43:20,918][284166] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,970][281975] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,972][281581] Decorrelating experience for 192 frames... +[2023-03-07 15:43:20,975][281602] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,994][281654] Decorrelating experience for 160 frames... +[2023-03-07 15:43:20,999][281577] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,028][281965] Decorrelating experience for 256 frames... +[2023-03-07 15:43:21,039][281662] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,048][281575] Decorrelating experience for 128 frames... +[2023-03-07 15:43:21,152][281803] Decorrelating experience for 64 frames... +[2023-03-07 15:43:21,156][281665] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,157][281946] Decorrelating experience for 256 frames... +[2023-03-07 15:43:21,161][283879] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,179][284048] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,189][281657] Decorrelating experience for 128 frames... +[2023-03-07 15:43:21,191][283019] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,212][283149] Decorrelating experience for 256 frames... +[2023-03-07 15:43:21,217][281651] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,239][282699] Decorrelating experience for 128 frames... +[2023-03-07 15:43:21,338][281993] Decorrelating experience for 288 frames... +[2023-03-07 15:43:21,339][288232] Decorrelating experience for 128 frames... +[2023-03-07 15:43:21,343][283960] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,344][281989] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,363][281579] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,369][283417] Decorrelating experience for 320 frames... +[2023-03-07 15:43:21,394][283084] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,395][282694] Decorrelating experience for 224 frames... +[2023-03-07 15:43:21,398][282763] Decorrelating experience for 288 frames... +[2023-03-07 15:43:21,413][281577] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,516][283020] Decorrelating experience for 288 frames... +[2023-03-07 15:43:21,522][281665] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,533][281808] Decorrelating experience for 64 frames... +[2023-03-07 15:43:21,535][281662] Decorrelating experience for 224 frames... +[2023-03-07 15:43:21,557][281958] Decorrelating experience for 352 frames... +[2023-03-07 15:43:21,569][281955] Decorrelating experience for 32 frames... +[2023-03-07 15:43:21,572][283278] Decorrelating experience for 288 frames... +[2023-03-07 15:43:21,580][281575] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,593][282184] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,594][283641] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,691][281973] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,700][281799] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,713][282699] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,734][281808] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,754][281803] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,766][281579] Decorrelating experience for 224 frames... +[2023-03-07 15:43:21,767][283417] Decorrelating experience for 352 frames... +[2023-03-07 15:43:21,773][281667] Decorrelating experience for 64 frames... +[2023-03-07 15:43:21,776][288232] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,800][281984] Decorrelating experience for 64 frames... +[2023-03-07 15:43:21,801][236315] Heartbeat connected on Batcher_0 +[2023-03-07 15:43:21,803][236315] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 15:43:21,834][236315] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 15:43:21,869][283727] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,881][283084] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,894][281662] Decorrelating experience for 256 frames... +[2023-03-07 15:43:21,911][281661] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,946][281990] Decorrelating experience for 160 frames... +[2023-03-07 15:43:21,947][281805] Decorrelating experience for 96 frames... +[2023-03-07 15:43:21,952][281800] Decorrelating experience for 32 frames... +[2023-03-07 15:43:21,955][283281] Decorrelating experience for 192 frames... +[2023-03-07 15:43:21,967][281665] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,000][281988] Decorrelating experience for 32 frames... +[2023-03-07 15:43:22,045][281653] Decorrelating experience for 32 frames... +[2023-03-07 15:43:22,059][283890] Decorrelating experience for 128 frames... +[2023-03-07 15:43:22,098][284048] Decorrelating experience for 128 frames... +[2023-03-07 15:43:22,099][281669] Decorrelating experience for 0 frames... +[2023-03-07 15:43:22,126][281965] Decorrelating experience for 288 frames... +[2023-03-07 15:43:22,129][281581] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,133][283641] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,154][282565] Decorrelating experience for 96 frames... +[2023-03-07 15:43:22,156][281661] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,175][283084] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,239][281799] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,242][281800] Decorrelating experience for 64 frames... +[2023-03-07 15:43:22,287][282763] Decorrelating experience for 320 frames... +[2023-03-07 15:43:22,291][281990] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,305][281974] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,312][281651] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,313][281973] Decorrelating experience for 128 frames... +[2023-03-07 15:43:22,338][281945] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,353][284309] Decorrelating experience for 352 frames... +[2023-03-07 15:43:22,382][282283] Decorrelating experience for 256 frames... +[2023-03-07 15:43:22,423][281950] Decorrelating experience for 0 frames... +[2023-03-07 15:43:22,424][284048] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,470][282565] Decorrelating experience for 128 frames... +[2023-03-07 15:43:22,489][281799] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,490][283890] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,492][283879] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,497][281581] Decorrelating experience for 256 frames... +[2023-03-07 15:43:22,516][281661] Decorrelating experience for 256 frames... +[2023-03-07 15:43:22,539][281990] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,566][281662] Decorrelating experience for 288 frames... +[2023-03-07 15:43:22,602][281951] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,641][281976] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,674][281664] Decorrelating experience for 288 frames... +[2023-03-07 15:43:22,685][283278] Decorrelating experience for 320 frames... +[2023-03-07 15:43:22,696][281946] Decorrelating experience for 288 frames... +[2023-03-07 15:43:22,698][283408] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,709][281964] Decorrelating experience for 64 frames... +[2023-03-07 15:43:22,713][281974] Decorrelating experience for 256 frames... +[2023-03-07 15:43:22,751][282283] Decorrelating experience for 288 frames... +[2023-03-07 15:43:22,752][281579] Decorrelating experience for 256 frames... +[2023-03-07 15:43:22,817][281577] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,851][281652] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,851][281989] Decorrelating experience for 224 frames... +[2023-03-07 15:43:22,864][283019] Decorrelating experience for 128 frames... +[2023-03-07 15:43:22,892][281973] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,902][282565] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,921][282693] Decorrelating experience for 320 frames... +[2023-03-07 15:43:22,931][283890] Decorrelating experience for 192 frames... +[2023-03-07 15:43:22,936][281968] Decorrelating experience for 160 frames... +[2023-03-07 15:43:22,996][281667] Decorrelating experience for 96 frames... +[2023-03-07 15:43:22,998][281810] Decorrelating experience for 96 frames... +[2023-03-07 15:43:23,030][283277] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,041][281653] Decorrelating experience for 64 frames... +[2023-03-07 15:43:23,050][281990] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,085][281993] Decorrelating experience for 320 frames... +[2023-03-07 15:43:23,087][282283] Decorrelating experience for 320 frames... +[2023-03-07 15:43:23,102][282558] Decorrelating experience for 128 frames... +[2023-03-07 15:43:23,108][283281] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,114][281663] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,179][281975] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,205][281575] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,216][281802] Decorrelating experience for 160 frames... +[2023-03-07 15:43:23,222][284166] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,227][281657] Decorrelating experience for 160 frames... +[2023-03-07 15:43:23,265][281580] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,269][281955] Decorrelating experience for 64 frames... +[2023-03-07 15:43:23,281][283854] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,329][281799] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,341][281974] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,356][281806] Decorrelating experience for 160 frames... +[2023-03-07 15:43:23,390][281946] Decorrelating experience for 320 frames... +[2023-03-07 15:43:23,394][283149] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,421][284048] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,459][281802] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,460][281800] Decorrelating experience for 96 frames... +[2023-03-07 15:43:23,461][281971] Decorrelating experience for 160 frames... +[2023-03-07 15:43:23,466][283960] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,519][281970] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,529][281649] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,535][283641] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:23,571][281990] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,618][281653] Decorrelating experience for 96 frames... +[2023-03-07 15:43:23,642][281975] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,657][281662] Decorrelating experience for 320 frames... +[2023-03-07 15:43:23,663][283879] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,664][281582] Decorrelating experience for 96 frames... +[2023-03-07 15:43:23,682][283281] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,698][281971] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,717][281989] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,748][282574] Decorrelating experience for 384 frames... +[2023-03-07 15:43:23,749][281988] Decorrelating experience for 64 frames... +[2023-03-07 15:43:23,821][283727] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,837][281669] Decorrelating experience for 32 frames... +[2023-03-07 15:43:23,854][284309] Decorrelating experience for 384 frames... +[2023-03-07 15:43:23,873][281970] Decorrelating experience for 224 frames... +[2023-03-07 15:43:23,896][281661] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,911][281806] Decorrelating experience for 192 frames... +[2023-03-07 15:43:23,912][281649] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,928][283960] Decorrelating experience for 256 frames... +[2023-03-07 15:43:23,928][281950] Decorrelating experience for 32 frames... +[2023-03-07 15:43:23,935][283879] Decorrelating experience for 288 frames... +[2023-03-07 15:43:23,997][281663] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,014][283150] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,066][281583] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,075][283084] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,078][281961] Decorrelating experience for 64 frames... +[2023-03-07 15:43:24,097][283019] Decorrelating experience for 160 frames... +[2023-03-07 15:43:24,097][281577] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,106][281652] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,117][283890] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,117][283408] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,174][284309] Decorrelating experience for 416 frames... +[2023-03-07 15:43:24,223][284048] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,259][281990] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,272][283879] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,273][281580] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,279][281991] Decorrelating experience for 32 frames... +[2023-03-07 15:43:24,289][281945] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,306][283960] Decorrelating experience for 288 frames... +[2023-03-07 15:43:24,309][281967] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,310][282558] Decorrelating experience for 160 frames... +[2023-03-07 15:43:24,349][281971] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,429][281667] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,440][281970] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,462][283085] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,476][281965] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,477][282574] Decorrelating experience for 416 frames... +[2023-03-07 15:43:24,479][288232] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,492][283150] Decorrelating experience for 160 frames... +[2023-03-07 15:43:24,495][281649] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,498][283084] Decorrelating experience for 288 frames... +[2023-03-07 15:43:24,529][281800] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,624][281810] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,633][281974] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,652][283019] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,656][281990] Decorrelating experience for 352 frames... +[2023-03-07 15:43:24,676][281657] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,679][283727] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,685][281968] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,692][281945] Decorrelating experience for 224 frames... +[2023-03-07 15:43:24,714][287383] Decorrelating experience for 128 frames... +[2023-03-07 15:43:24,715][281577] Decorrelating experience for 288 frames... +[2023-03-07 15:43:24,806][281667] Decorrelating experience for 160 frames... +[2023-03-07 15:43:24,836][281584] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,837][283084] Decorrelating experience for 320 frames... +[2023-03-07 15:43:24,840][284048] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,859][281988] Decorrelating experience for 96 frames... +[2023-03-07 15:43:24,880][281975] Decorrelating experience for 288 frames... +[2023-03-07 15:43:24,881][281991] Decorrelating experience for 64 frames... +[2023-03-07 15:43:24,897][281973] Decorrelating experience for 192 frames... +[2023-03-07 15:43:24,897][282695] Decorrelating experience for 256 frames... +[2023-03-07 15:43:24,903][281944] Decorrelating experience for 64 frames... +[2023-03-07 15:43:24,984][284345] Decorrelating experience for 128 frames... +[2023-03-07 15:43:25,014][282696] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,038][281579] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,041][282699] Decorrelating experience for 192 frames... +[2023-03-07 15:43:25,050][281577] Decorrelating experience for 320 frames... +[2023-03-07 15:43:25,067][281967] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,078][281658] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,081][282693] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,084][281991] Decorrelating experience for 96 frames... +[2023-03-07 15:43:25,089][281602] Decorrelating experience for 192 frames... +[2023-03-07 15:43:25,164][282697] Decorrelating experience for 64 frames... +[2023-03-07 15:43:25,192][281649] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,219][281808] Decorrelating experience for 128 frames... +[2023-03-07 15:43:25,229][281974] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,237][283278] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,262][281945] Decorrelating experience for 256 frames... +[2023-03-07 15:43:25,266][284345] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,268][281975] Decorrelating experience for 320 frames... +[2023-03-07 15:43:25,289][281805] Decorrelating experience for 128 frames... +[2023-03-07 15:43:25,303][283150] Decorrelating experience for 192 frames... +[2023-03-07 15:43:25,345][281976] Decorrelating experience for 256 frames... +[2023-03-07 15:43:25,368][281578] Decorrelating experience for 128 frames... +[2023-03-07 15:43:25,402][283854] Decorrelating experience for 256 frames... +[2023-03-07 15:43:25,424][281971] Decorrelating experience for 256 frames... +[2023-03-07 15:43:25,456][282699] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,457][281663] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,458][281664] Decorrelating experience for 320 frames... +[2023-03-07 15:43:25,480][281803] Decorrelating experience for 128 frames... +[2023-03-07 15:43:25,486][281968] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,494][281799] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,527][282827] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,547][281973] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,580][281967] Decorrelating experience for 256 frames... +[2023-03-07 15:43:25,605][281951] Decorrelating experience for 192 frames... +[2023-03-07 15:43:25,639][283084] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,641][281584] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,649][283408] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,684][281944] Decorrelating experience for 96 frames... +[2023-03-07 15:43:25,694][283085] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,712][282693] Decorrelating experience for 384 frames... +[2023-03-07 15:43:25,718][282697] Decorrelating experience for 96 frames... +[2023-03-07 15:43:25,726][281984] Decorrelating experience for 96 frames... +[2023-03-07 15:43:25,757][281989] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,783][281964] Decorrelating experience for 96 frames... +[2023-03-07 15:43:25,825][283854] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,855][281945] Decorrelating experience for 288 frames... +[2023-03-07 15:43:25,863][281800] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,869][283352] Decorrelating experience for 64 frames... +[2023-03-07 15:43:25,900][281578] Decorrelating experience for 160 frames... +[2023-03-07 15:43:25,905][281993] Decorrelating experience for 352 frames... +[2023-03-07 15:43:25,935][281951] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,936][288232] Decorrelating experience for 224 frames... +[2023-03-07 15:43:25,968][282565] Decorrelating experience for 192 frames... +[2023-03-07 15:43:25,973][281803] Decorrelating experience for 160 frames... +[2023-03-07 15:43:26,035][282183] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,053][281665] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,063][284309] Decorrelating experience for 448 frames... +[2023-03-07 15:43:26,077][281805] Decorrelating experience for 160 frames... +[2023-03-07 15:43:26,083][281651] Decorrelating experience for 224 frames... +[2023-03-07 15:43:26,099][281971] Decorrelating experience for 288 frames... +[2023-03-07 15:43:26,118][284048] Decorrelating experience for 288 frames... +[2023-03-07 15:43:26,128][283408] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,156][281800] Decorrelating experience for 192 frames... +[2023-03-07 15:43:26,157][281649] Decorrelating experience for 384 frames... +[2023-03-07 15:43:26,215][281663] Decorrelating experience for 384 frames... +[2023-03-07 15:43:26,230][281988] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,272][281976] Decorrelating experience for 288 frames... +[2023-03-07 15:43:26,275][283352] Decorrelating experience for 96 frames... +[2023-03-07 15:43:26,282][281945] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,297][281815] Decorrelating experience for 160 frames... +[2023-03-07 15:43:26,325][281975] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,345][281984] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,360][281804] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,392][281989] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,411][281946] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,453][287383] Decorrelating experience for 160 frames... +[2023-03-07 15:43:26,457][281800] Decorrelating experience for 224 frames... +[2023-03-07 15:43:26,462][281973] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,513][281805] Decorrelating experience for 192 frames... +[2023-03-07 15:43:26,522][281660] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,541][281958] Decorrelating experience for 384 frames... +[2023-03-07 15:43:26,553][281976] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,573][281803] Decorrelating experience for 192 frames... +[2023-03-07 15:43:26,590][282694] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,590][281990] Decorrelating experience for 384 frames... +[2023-03-07 15:43:26,654][283727] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,698][281651] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,702][282283] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,715][281807] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,715][282697] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,719][281965] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,751][281964] Decorrelating experience for 128 frames... +[2023-03-07 15:43:26,773][282827] Decorrelating experience for 192 frames... +[2023-03-07 15:43:26,775][281946] Decorrelating experience for 384 frames... +[2023-03-07 15:43:26,802][284200] Decorrelating experience for 160 frames... +[2023-03-07 15:43:26,860][282763] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,884][282695] Decorrelating experience for 288 frames... +[2023-03-07 15:43:26,887][284048] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,895][281976] Decorrelating experience for 352 frames... +[2023-03-07 15:43:26,902][283149] Decorrelating experience for 320 frames... +[2023-03-07 15:43:26,930][281972] Decorrelating experience for 256 frames... +[2023-03-07 15:43:26,943][281669] Decorrelating experience for 64 frames... +[2023-03-07 15:43:26,958][284309] Decorrelating experience for 480 frames... +[2023-03-07 15:43:27,043][281984] Decorrelating experience for 160 frames... +[2023-03-07 15:43:27,046][282904] Decorrelating experience for 64 frames... +[2023-03-07 15:43:27,066][281807] Decorrelating experience for 288 frames... +[2023-03-07 15:43:27,076][281800] Decorrelating experience for 256 frames... +[2023-03-07 15:43:27,082][281658] Decorrelating experience for 192 frames... +[2023-03-07 15:43:27,083][282565] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,086][281801] Decorrelating experience for 64 frames... +[2023-03-07 15:43:27,130][281957] Decorrelating experience for 288 frames... +[2023-03-07 15:43:27,137][281657] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,145][281584] Decorrelating experience for 256 frames... +[2023-03-07 15:43:27,232][281802] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,252][282184] Decorrelating experience for 128 frames... +[2023-03-07 15:43:27,255][281976] Decorrelating experience for 384 frames... +[2023-03-07 15:43:27,264][281990] Decorrelating experience for 416 frames... +[2023-03-07 15:43:27,270][281656] Decorrelating experience for 96 frames... +[2023-03-07 15:43:27,279][282574] Decorrelating experience for 448 frames... +[2023-03-07 15:43:27,311][282904] Decorrelating experience for 96 frames... +[2023-03-07 15:43:27,325][281964] Decorrelating experience for 160 frames... +[2023-03-07 15:43:27,392][282827] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,400][281581] Decorrelating experience for 288 frames... +[2023-03-07 15:43:27,415][281958] Decorrelating experience for 416 frames... +[2023-03-07 15:43:27,428][281649] Decorrelating experience for 416 frames... +[2023-03-07 15:43:27,461][281989] Decorrelating experience for 352 frames... +[2023-03-07 15:43:27,465][284200] Decorrelating experience for 192 frames... +[2023-03-07 15:43:27,481][281658] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,494][281578] Decorrelating experience for 192 frames... +[2023-03-07 15:43:27,514][284309] Decorrelating experience for 512 frames... +[2023-03-07 15:43:27,516][281957] Decorrelating experience for 320 frames... +[2023-03-07 15:43:27,586][284047] Decorrelating experience for 64 frames... +[2023-03-07 15:43:27,589][284048] Decorrelating experience for 352 frames... +[2023-03-07 15:43:27,599][282699] Decorrelating experience for 256 frames... +[2023-03-07 15:43:27,606][283960] Decorrelating experience for 320 frames... +[2023-03-07 15:43:27,645][281946] Decorrelating experience for 416 frames... +[2023-03-07 15:43:27,676][282574] Decorrelating experience for 480 frames... +[2023-03-07 15:43:27,700][281656] Decorrelating experience for 128 frames... +[2023-03-07 15:43:27,702][281813] Decorrelating experience for 96 frames... +[2023-03-07 15:43:27,702][282565] Decorrelating experience for 256 frames... +[2023-03-07 15:43:27,764][281975] Decorrelating experience for 384 frames... +[2023-03-07 15:43:27,768][281991] Decorrelating experience for 128 frames... +[2023-03-07 15:43:27,768][284345] Decorrelating experience for 192 frames... +[2023-03-07 15:43:27,790][281581] Decorrelating experience for 320 frames... +[2023-03-07 15:43:27,792][284047] Decorrelating experience for 96 frames... +[2023-03-07 15:43:27,824][282695] Decorrelating experience for 320 frames... +[2023-03-07 15:43:27,892][282184] Decorrelating experience for 160 frames... +[2023-03-07 15:43:27,894][283408] Decorrelating experience for 352 frames... +[2023-03-07 15:43:27,895][283727] Decorrelating experience for 288 frames... +[2023-03-07 15:43:27,918][284200] Decorrelating experience for 224 frames... +[2023-03-07 15:43:27,947][284048] Decorrelating experience for 384 frames... +[2023-03-07 15:43:27,955][283890] Decorrelating experience for 256 frames... +[2023-03-07 15:43:27,974][283019] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,003][284309] Decorrelating experience for 544 frames... +[2023-03-07 15:43:28,013][281653] Decorrelating experience for 128 frames... +[2023-03-07 15:43:28,018][282697] Decorrelating experience for 160 frames... +[2023-03-07 15:43:28,077][282699] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,080][282565] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,089][283149] Decorrelating experience for 352 frames... +[2023-03-07 15:43:28,119][284345] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,129][283854] Decorrelating experience for 320 frames... +[2023-03-07 15:43:28,143][281973] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,152][283281] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,186][282827] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,193][281652] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,200][281669] Decorrelating experience for 96 frames... +[2023-03-07 15:43:28,258][281959] Decorrelating experience for 32 frames... +[2023-03-07 15:43:28,279][282184] Decorrelating experience for 192 frames... +[2023-03-07 15:43:28,309][281958] Decorrelating experience for 448 frames... +[2023-03-07 15:43:28,312][284166] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,325][282696] Decorrelating experience for 192 frames... +[2023-03-07 15:43:28,328][281976] Decorrelating experience for 416 frames... +[2023-03-07 15:43:28,331][281803] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,367][284048] Decorrelating experience for 416 frames... +[2023-03-07 15:43:28,377][281656] Decorrelating experience for 160 frames... +[2023-03-07 15:43:28,381][284345] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,474][284053] Decorrelating experience for 128 frames... +[2023-03-07 15:43:28,496][281809] Decorrelating experience for 32 frames... +[2023-03-07 15:43:28,499][281964] Decorrelating experience for 192 frames... +[2023-03-07 15:43:28,509][281582] Decorrelating experience for 128 frames... +[2023-03-07 15:43:28,517][282699] Decorrelating experience for 320 frames... +[2023-03-07 15:43:28,518][281652] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:28,546][281658] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,563][281961] Decorrelating experience for 96 frames... +[2023-03-07 15:43:28,569][281968] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,573][282574] Decorrelating experience for 512 frames... +[2023-03-07 15:43:28,651][283352] Decorrelating experience for 128 frames... +[2023-03-07 15:43:28,673][281808] Decorrelating experience for 160 frames... +[2023-03-07 15:43:28,696][284048] Decorrelating experience for 448 frames... +[2023-03-07 15:43:28,704][281948] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,708][281670] Another process currently holds the lock /tmp/sf2_rolo/doom_007.lockfile, attempt: 1 +[2023-03-07 15:43:28,732][282696] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,749][281954] Decorrelating experience for 64 frames... +[2023-03-07 15:43:28,750][283879] Decorrelating experience for 352 frames... +[2023-03-07 15:43:28,774][281584] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,790][281654] Decorrelating experience for 192 frames... +[2023-03-07 15:43:28,854][281950] Decorrelating experience for 64 frames... +[2023-03-07 15:43:28,875][283352] Decorrelating experience for 160 frames... +[2023-03-07 15:43:28,896][282184] Decorrelating experience for 224 frames... +[2023-03-07 15:43:28,918][282699] Decorrelating experience for 352 frames... +[2023-03-07 15:43:28,919][288232] Decorrelating experience for 256 frames... +[2023-03-07 15:43:28,934][281582] Decorrelating experience for 160 frames... +[2023-03-07 15:43:28,957][281946] Decorrelating experience for 448 frames... +[2023-03-07 15:43:28,958][284345] Decorrelating experience for 288 frames... +[2023-03-07 15:43:28,970][281974] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,034][281602] Decorrelating experience for 224 frames... +[2023-03-07 15:43:29,052][281652] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,078][281800] Decorrelating experience for 288 frames... +[2023-03-07 15:43:29,109][281584] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,116][283084] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,120][281654] Decorrelating experience for 224 frames... +[2023-03-07 15:43:29,142][281975] Decorrelating experience for 416 frames... +[2023-03-07 15:43:29,145][284048] Decorrelating experience for 480 frames... +[2023-03-07 15:43:29,148][281973] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,151][281945] Decorrelating experience for 352 frames... +[2023-03-07 15:43:29,211][282697] Decorrelating experience for 192 frames... +[2023-03-07 15:43:29,256][281955] Decorrelating experience for 96 frames... +[2023-03-07 15:43:29,285][282184] Decorrelating experience for 256 frames... +[2023-03-07 15:43:29,295][281992] Decorrelating experience for 96 frames... +[2023-03-07 15:43:29,322][281944] Decorrelating experience for 128 frames... +[2023-03-07 15:43:29,333][281988] Decorrelating experience for 160 frames... +[2023-03-07 15:43:29,336][281805] Decorrelating experience for 224 frames... +[2023-03-07 15:43:29,338][281993] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,392][284309] Decorrelating experience for 576 frames... +[2023-03-07 15:43:29,430][283854] Decorrelating experience for 352 frames... +[2023-03-07 15:43:29,436][283890] Decorrelating experience for 288 frames... +[2023-03-07 15:43:29,464][281954] Decorrelating experience for 96 frames... +[2023-03-07 15:43:29,515][281807] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,517][283352] Decorrelating experience for 192 frames... +[2023-03-07 15:43:29,522][282283] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,526][281969] Decorrelating experience for 192 frames... +[2023-03-07 15:43:29,541][281960] Decorrelating experience for 288 frames... +[2023-03-07 15:43:29,569][281988] Decorrelating experience for 192 frames... +[2023-03-07 15:43:29,591][281809] Decorrelating experience for 64 frames... +[2023-03-07 15:43:29,610][281583] Decorrelating experience for 256 frames... +[2023-03-07 15:43:29,617][282827] Decorrelating experience for 288 frames... +[2023-03-07 15:43:29,643][281974] Decorrelating experience for 416 frames... +[2023-03-07 15:43:29,696][281665] Decorrelating experience for 288 frames... +[2023-03-07 15:43:29,708][283277] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,720][281962] Decorrelating experience for 160 frames... +[2023-03-07 15:43:29,727][284345] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,739][281955] Decorrelating experience for 128 frames... +[2023-03-07 15:43:29,773][283890] Decorrelating experience for 320 frames... +[2023-03-07 15:43:29,798][283019] Decorrelating experience for 256 frames... +[2023-03-07 15:43:29,802][283396] Decorrelating experience for 96 frames... +[2023-03-07 15:43:29,806][283149] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,867][284309] Decorrelating experience for 608 frames... +[2023-03-07 15:43:29,873][282891] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:29,891][282763] Decorrelating experience for 384 frames... +[2023-03-07 15:43:29,896][281808] Decorrelating experience for 192 frames... +[2023-03-07 15:43:29,901][281581] Decorrelating experience for 352 frames... +[2023-03-07 15:43:29,907][281993] Decorrelating experience for 416 frames... +[2023-03-07 15:43:29,992][282693] Decorrelating experience for 416 frames... +[2023-03-07 15:43:29,992][283352] Decorrelating experience for 224 frames... +[2023-03-07 15:43:29,993][283641] Decorrelating experience for 256 frames... +[2023-03-07 15:43:29,998][281975] Decorrelating experience for 448 frames... +[2023-03-07 15:43:30,003][281803] Decorrelating experience for 256 frames... +[2023-03-07 15:43:30,046][281968] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,074][283890] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,076][281813] Decorrelating experience for 128 frames... +[2023-03-07 15:43:30,082][282565] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,099][281969] Decorrelating experience for 224 frames... +[2023-03-07 15:43:30,178][284047] Decorrelating experience for 128 frames... +[2023-03-07 15:43:30,180][281976] Decorrelating experience for 448 frames... +[2023-03-07 15:43:30,184][283469] Decorrelating experience for 128 frames... +[2023-03-07 15:43:30,209][281946] Decorrelating experience for 480 frames... +[2023-03-07 15:43:30,233][281809] Decorrelating experience for 96 frames... +[2023-03-07 15:43:30,254][281960] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,257][281799] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,267][281807] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,274][281973] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,328][282694] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,363][281971] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,372][281602] Decorrelating experience for 256 frames... +[2023-03-07 15:43:30,418][283019] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,418][282763] Decorrelating experience for 416 frames... +[2023-03-07 15:43:30,420][281955] Decorrelating experience for 160 frames... +[2023-03-07 15:43:30,437][281583] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,451][281657] Decorrelating experience for 256 frames... +[2023-03-07 15:43:30,454][281653] Decorrelating experience for 160 frames... +[2023-03-07 15:43:30,514][281945] Decorrelating experience for 384 frames... +[2023-03-07 15:43:30,517][282558] Decorrelating experience for 192 frames... +[2023-03-07 15:43:30,542][281660] Decorrelating experience for 160 frames... +[2023-03-07 15:43:30,556][281662] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,611][281946] Decorrelating experience for 512 frames... +[2023-03-07 15:43:30,621][283641] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,632][284345] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,636][281968] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,637][281973] Decorrelating experience for 384 frames... +[2023-03-07 15:43:30,639][281665] Decorrelating experience for 320 frames... +[2023-03-07 15:43:30,697][283854] Decorrelating experience for 384 frames... +[2023-03-07 15:43:30,718][281992] Decorrelating experience for 128 frames... +[2023-03-07 15:43:30,724][281657] Decorrelating experience for 288 frames... +[2023-03-07 15:43:30,735][283150] Decorrelating experience for 224 frames... +[2023-03-07 15:43:30,790][282183] Decorrelating experience for 160 frames... +[2023-03-07 15:43:30,816][281971] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,822][281958] Decorrelating experience for 480 frames... +[2023-03-07 15:43:30,829][281988] Decorrelating experience for 224 frames... +[2023-03-07 15:43:30,841][282565] Decorrelating experience for 352 frames... +[2023-03-07 15:43:30,875][281670] Decorrelating experience for 0 frames... +[2023-03-07 15:43:30,878][281807] Decorrelating experience for 384 frames... +[2023-03-07 15:43:30,917][281663] Decorrelating experience for 416 frames... +[2023-03-07 15:43:30,919][281804] Decorrelating experience for 160 frames... +[2023-03-07 15:43:30,923][281654] Decorrelating experience for 256 frames... +[2023-03-07 15:43:30,980][281660] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,004][283150] Decorrelating experience for 256 frames... +[2023-03-07 15:43:31,008][282696] Decorrelating experience for 256 frames... +[2023-03-07 15:43:31,016][281581] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,050][281653] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,056][281973] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,081][281808] Decorrelating experience for 224 frames... +[2023-03-07 15:43:31,109][281655] Decorrelating experience for 96 frames... +[2023-03-07 15:43:31,115][281962] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,118][283854] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,165][281662] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,193][283019] Decorrelating experience for 320 frames... +[2023-03-07 15:43:31,197][281961] Decorrelating experience for 128 frames... +[2023-03-07 15:43:31,210][281657] Decorrelating experience for 320 frames... +[2023-03-07 15:43:31,240][282699] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,261][281576] Decorrelating experience for 128 frames... +[2023-03-07 15:43:31,275][281988] Decorrelating experience for 256 frames... +[2023-03-07 15:43:31,296][281575] Decorrelating experience for 224 frames... +[2023-03-07 15:43:31,305][283890] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,325][287383] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,345][281966] Decorrelating experience for 128 frames... +[2023-03-07 15:43:31,382][281958] Decorrelating experience for 512 frames... +[2023-03-07 15:43:31,385][281976] Decorrelating experience for 480 frames... +[2023-03-07 15:43:31,402][281653] Decorrelating experience for 224 frames... +[2023-03-07 15:43:31,422][282694] Decorrelating experience for 320 frames... +[2023-03-07 15:43:31,450][282283] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,462][283641] Decorrelating experience for 320 frames... +[2023-03-07 15:43:31,476][283879] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,484][281662] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,529][281602] Decorrelating experience for 288 frames... +[2023-03-07 15:43:31,533][281660] Decorrelating experience for 224 frames... +[2023-03-07 15:43:31,566][284047] Decorrelating experience for 160 frames... +[2023-03-07 15:43:31,571][281944] Decorrelating experience for 160 frames... +[2023-03-07 15:43:31,602][281993] Decorrelating experience for 448 frames... +[2023-03-07 15:43:31,629][283890] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,629][283278] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,654][281965] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,667][281969] Decorrelating experience for 256 frames... +[2023-03-07 15:43:31,722][282694] Decorrelating experience for 352 frames... +[2023-03-07 15:43:31,752][281958] Decorrelating experience for 544 frames... +[2023-03-07 15:43:31,759][282565] Decorrelating experience for 384 frames... +[2023-03-07 15:43:31,763][283352] Decorrelating experience for 256 frames... +[2023-03-07 15:43:31,785][281950] Decorrelating experience for 96 frames... +[2023-03-07 15:43:31,807][283150] Decorrelating experience for 288 frames... +[2023-03-07 15:43:31,808][281804] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,835][281656] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,854][281988] Decorrelating experience for 288 frames... +[2023-03-07 15:43:31,910][281602] Decorrelating experience for 320 frames... +[2023-03-07 15:43:31,911][283854] Decorrelating experience for 448 frames... +[2023-03-07 15:43:31,968][282574] Decorrelating experience for 544 frames... +[2023-03-07 15:43:31,973][281945] Decorrelating experience for 416 frames... +[2023-03-07 15:43:31,989][281669] Decorrelating experience for 128 frames... +[2023-03-07 15:43:31,993][281955] Decorrelating experience for 192 frames... +[2023-03-07 15:43:31,998][281973] Decorrelating experience for 448 frames... +[2023-03-07 15:43:32,030][284048] Decorrelating experience for 512 frames... +[2023-03-07 15:43:32,094][281976] Decorrelating experience for 512 frames... +[2023-03-07 15:43:32,101][283278] Decorrelating experience for 416 frames... +[2023-03-07 15:43:32,102][281803] Decorrelating experience for 288 frames... +[2023-03-07 15:43:32,163][281670] Decorrelating experience for 32 frames... +[2023-03-07 15:43:32,178][284309] Decorrelating experience for 640 frames... +[2023-03-07 15:43:32,178][281809] Decorrelating experience for 128 frames... +[2023-03-07 15:43:32,184][281958] Decorrelating experience for 576 frames... +[2023-03-07 15:43:32,210][281575] Decorrelating experience for 256 frames... +[2023-03-07 15:43:32,238][281582] Decorrelating experience for 192 frames... +[2023-03-07 15:43:32,287][282565] Decorrelating experience for 416 frames... +[2023-03-07 15:43:32,292][281972] Decorrelating experience for 288 frames... +[2023-03-07 15:43:32,292][281580] Decorrelating experience for 256 frames... +[2023-03-07 15:43:32,348][282574] Decorrelating experience for 576 frames... +[2023-03-07 15:43:32,357][281654] Decorrelating experience for 288 frames... +[2023-03-07 15:43:32,365][282699] Decorrelating experience for 416 frames... +[2023-03-07 15:43:32,372][281802] Decorrelating experience for 256 frames... +[2023-03-07 15:43:32,394][281963] Decorrelating experience for 128 frames... +[2023-03-07 15:43:32,416][281805] Decorrelating experience for 256 frames... +[2023-03-07 15:43:32,460][281976] Decorrelating experience for 544 frames... +[2023-03-07 15:43:32,475][283277] Decorrelating experience for 352 frames... +[2023-03-07 15:43:32,487][284053] Decorrelating experience for 160 frames... +[2023-03-07 15:43:32,528][281804] Decorrelating experience for 224 frames... +[2023-03-07 15:43:32,544][281581] Decorrelating experience for 416 frames... +[2023-03-07 15:43:32,554][281670] Decorrelating experience for 64 frames... +[2023-03-07 15:43:32,575][281578] Decorrelating experience for 224 frames... +[2023-03-07 15:43:32,577][281991] Decorrelating experience for 160 frames... +[2023-03-07 15:43:32,597][281954] Decorrelating experience for 128 frames... +[2023-03-07 15:43:32,673][281662] Decorrelating experience for 448 frames... +[2023-03-07 15:43:32,674][281963] Decorrelating experience for 160 frames... +[2023-03-07 15:43:32,688][281805] Decorrelating experience for 288 frames... +[2023-03-07 15:43:32,722][281945] Decorrelating experience for 448 frames... +[2023-03-07 15:43:32,739][282694] Decorrelating experience for 384 frames... +[2023-03-07 15:43:32,747][281802] Decorrelating experience for 288 frames... +[2023-03-07 15:43:32,757][283879] Decorrelating experience for 416 frames... +[2023-03-07 15:43:32,759][282558] Decorrelating experience for 224 frames... +[2023-03-07 15:43:32,783][281966] Decorrelating experience for 160 frames... +[2023-03-07 15:43:32,786][281803] Decorrelating experience for 320 frames... +[2023-03-07 15:43:32,853][284345] Decorrelating experience for 384 frames... +[2023-03-07 15:43:32,881][281958] Decorrelating experience for 608 frames... +[2023-03-07 15:43:32,890][281578] Decorrelating experience for 256 frames... +[2023-03-07 15:43:32,901][282695] Decorrelating experience for 352 frames... +[2023-03-07 15:43:32,921][282698] Decorrelating experience for 320 frames... +[2023-03-07 15:43:32,942][283277] Decorrelating experience for 384 frames... +[2023-03-07 15:43:32,947][281968] Decorrelating experience for 352 frames... +[2023-03-07 15:43:32,970][281583] Decorrelating experience for 320 frames... +[2023-03-07 15:43:32,971][281799] Decorrelating experience for 352 frames... +[2023-03-07 15:43:32,992][281580] Decorrelating experience for 288 frames... +[2023-03-07 15:43:33,083][281965] Decorrelating experience for 416 frames... +[2023-03-07 15:43:33,086][283879] Decorrelating experience for 448 frames... +[2023-03-07 15:43:33,100][281800] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,111][282183] Decorrelating experience for 192 frames... +[2023-03-07 15:43:33,126][283890] Decorrelating experience for 448 frames... +[2023-03-07 15:43:33,129][283085] Decorrelating experience for 192 frames... +[2023-03-07 15:43:33,155][282699] Decorrelating experience for 448 frames... +[2023-03-07 15:43:33,157][281993] Decorrelating experience for 480 frames... +[2023-03-07 15:43:33,168][283854] Decorrelating experience for 480 frames... +[2023-03-07 15:43:33,264][281578] Decorrelating experience for 288 frames... +[2023-03-07 15:43:33,299][281658] Decorrelating experience for 288 frames... +[2023-03-07 15:43:33,303][284304] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:33,312][282695] Decorrelating experience for 384 frames... +[2023-03-07 15:43:33,313][281670] Decorrelating experience for 96 frames... +[2023-03-07 15:43:33,331][281583] Decorrelating experience for 352 frames... +[2023-03-07 15:43:33,338][281806] Decorrelating experience for 224 frames... +[2023-03-07 15:43:33,343][281656] Decorrelating experience for 224 frames... +[2023-03-07 15:43:33,344][281948] Decorrelating experience for 256 frames... +[2023-03-07 15:43:33,413][283417] Decorrelating experience for 384 frames... +[2023-03-07 15:43:33,442][281584] Decorrelating experience for 352 frames... +[2023-03-07 15:43:33,478][281580] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,499][281576] Decorrelating experience for 160 frames... +[2023-03-07 15:43:33,520][282694] Decorrelating experience for 416 frames... +[2023-03-07 15:43:33,533][284345] Decorrelating experience for 416 frames... +[2023-03-07 15:43:33,534][281670] Decorrelating experience for 128 frames... +[2023-03-07 15:43:33,536][283352] Decorrelating experience for 288 frames... +[2023-03-07 15:43:33,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:33,542][281992] Decorrelating experience for 160 frames... +[2023-03-07 15:43:33,579][282183] Decorrelating experience for 224 frames... +[2023-03-07 15:43:33,590][283150] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,621][281662] Decorrelating experience for 480 frames... +[2023-03-07 15:43:33,664][283396] Decorrelating experience for 128 frames... +[2023-03-07 15:43:33,686][281958] Decorrelating experience for 640 frames... +[2023-03-07 15:43:33,702][281649] Decorrelating experience for 448 frames... +[2023-03-07 15:43:33,725][281944] Decorrelating experience for 192 frames... +[2023-03-07 15:43:33,728][282698] Decorrelating experience for 352 frames... +[2023-03-07 15:43:33,728][281991] Decorrelating experience for 192 frames... +[2023-03-07 15:43:33,748][281806] Decorrelating experience for 256 frames... +[2023-03-07 15:43:33,772][281652] Decorrelating experience for 352 frames... +[2023-03-07 15:43:33,804][281578] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,807][283417] Decorrelating experience for 416 frames... +[2023-03-07 15:43:33,847][281800] Decorrelating experience for 352 frames... +[2023-03-07 15:43:33,873][281963] Decorrelating experience for 192 frames... +[2023-03-07 15:43:33,881][281964] Decorrelating experience for 224 frames... +[2023-03-07 15:43:33,910][284048] Decorrelating experience for 544 frames... +[2023-03-07 15:43:33,916][281658] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,918][281809] Decorrelating experience for 160 frames... +[2023-03-07 15:43:33,975][282827] Decorrelating experience for 320 frames... +[2023-03-07 15:43:33,988][281802] Decorrelating experience for 320 frames... +[2023-03-07 15:43:34,027][281806] Decorrelating experience for 288 frames... +[2023-03-07 15:43:34,030][281944] Decorrelating experience for 224 frames... +[2023-03-07 15:43:34,080][283352] Decorrelating experience for 320 frames... +[2023-03-07 15:43:34,090][281971] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,090][281945] Decorrelating experience for 480 frames... +[2023-03-07 15:43:34,100][282696] Decorrelating experience for 288 frames... +[2023-03-07 15:43:34,117][281965] Decorrelating experience for 448 frames... +[2023-03-07 15:43:34,130][281808] Decorrelating experience for 256 frames... +[2023-03-07 15:43:34,159][281800] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,163][281670] Decorrelating experience for 160 frames... +[2023-03-07 15:43:34,229][281652] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,266][281658] Decorrelating experience for 352 frames... +[2023-03-07 15:43:34,276][281807] Decorrelating experience for 416 frames... +[2023-03-07 15:43:34,279][282827] Decorrelating experience for 352 frames... +[2023-03-07 15:43:34,282][281669] Decorrelating experience for 160 frames... +[2023-03-07 15:43:34,306][283396] Decorrelating experience for 160 frames... +[2023-03-07 15:43:34,317][281968] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,317][281806] Decorrelating experience for 320 frames... +[2023-03-07 15:43:34,344][281990] Decorrelating experience for 448 frames... +[2023-03-07 15:43:34,356][281961] Decorrelating experience for 160 frames... +[2023-03-07 15:43:34,442][282698] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,457][281969] Decorrelating experience for 288 frames... +[2023-03-07 15:43:34,479][281963] Decorrelating experience for 224 frames... +[2023-03-07 15:43:34,488][283278] Decorrelating experience for 448 frames... +[2023-03-07 15:43:34,492][284048] Decorrelating experience for 576 frames... +[2023-03-07 15:43:34,501][281988] Decorrelating experience for 320 frames... +[2023-03-07 15:43:34,523][281967] Decorrelating experience for 288 frames... +[2023-03-07 15:43:34,528][281801] Decorrelating experience for 96 frames... +[2023-03-07 15:43:34,536][281583] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,537][281815] Decorrelating experience for 192 frames... +[2023-03-07 15:43:34,625][281658] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,640][283879] Decorrelating experience for 480 frames... +[2023-03-07 15:43:34,668][281971] Decorrelating experience for 416 frames... +[2023-03-07 15:43:34,682][281975] Decorrelating experience for 480 frames... +[2023-03-07 15:43:34,692][282827] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,710][281584] Decorrelating experience for 384 frames... +[2023-03-07 15:43:34,710][281945] Decorrelating experience for 512 frames... +[2023-03-07 15:43:34,736][282558] Decorrelating experience for 256 frames... +[2023-03-07 15:43:34,738][284166] Decorrelating experience for 256 frames... +[2023-03-07 15:43:34,764][281946] Decorrelating experience for 544 frames... +[2023-03-07 15:43:34,823][281993] Decorrelating experience for 512 frames... +[2023-03-07 15:43:34,874][281670] Decorrelating experience for 192 frames... +[2023-03-07 15:43:34,885][284048] Decorrelating experience for 608 frames... +[2023-03-07 15:43:34,893][281664] Decorrelating experience for 352 frames... +[2023-03-07 15:43:34,903][281954] Decorrelating experience for 160 frames... +[2023-03-07 15:43:34,914][281807] Decorrelating experience for 448 frames... +[2023-03-07 15:43:34,921][281803] Decorrelating experience for 352 frames... +[2023-03-07 15:43:34,930][281965] Decorrelating experience for 480 frames... +[2023-03-07 15:43:34,942][281992] Decorrelating experience for 192 frames... +[2023-03-07 15:43:34,951][281984] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,001][281971] Decorrelating experience for 448 frames... +[2023-03-07 15:43:35,068][281656] Decorrelating experience for 256 frames... +[2023-03-07 15:43:35,080][283084] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,087][281960] Decorrelating experience for 352 frames... +[2023-03-07 15:43:35,093][283396] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,107][281958] Decorrelating experience for 672 frames... +[2023-03-07 15:43:35,142][281951] Decorrelating experience for 256 frames... +[2023-03-07 15:43:35,181][283281] Decorrelating experience for 320 frames... +[2023-03-07 15:43:35,222][284053] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,223][281968] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,250][281658] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,259][282184] Decorrelating experience for 288 frames... +[2023-03-07 15:43:35,305][281652] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,306][281815] Decorrelating experience for 224 frames... +[2023-03-07 15:43:35,307][283085] Decorrelating experience for 224 frames... +[2023-03-07 15:43:35,344][281975] Decorrelating experience for 512 frames... +[2023-03-07 15:43:35,354][283396] Decorrelating experience for 224 frames... +[2023-03-07 15:43:35,363][281809] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,414][281583] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,415][281807] Decorrelating experience for 480 frames... +[2023-03-07 15:43:35,439][282891] Decorrelating experience for 32 frames... +[2023-03-07 15:43:35,446][281803] Decorrelating experience for 384 frames... +[2023-03-07 15:43:35,496][282696] Decorrelating experience for 320 frames... +[2023-03-07 15:43:35,500][283879] Decorrelating experience for 512 frames... +[2023-03-07 15:43:35,559][281956] Decorrelating experience for 288 frames... +[2023-03-07 15:43:35,560][283278] Decorrelating experience for 480 frames... +[2023-03-07 15:43:35,561][282184] Decorrelating experience for 320 frames... +[2023-03-07 15:43:35,561][281969] Decorrelating experience for 320 frames... +[2023-03-07 15:43:35,594][281990] Decorrelating experience for 480 frames... +[2023-03-07 15:43:35,598][281960] Decorrelating experience for 384 frames... +[2023-03-07 15:43:35,631][283281] Decorrelating experience for 352 frames... +[2023-03-07 15:43:35,656][284166] Decorrelating experience for 288 frames... +[2023-03-07 15:43:35,680][282891] Decorrelating experience for 64 frames... +[2023-03-07 15:43:35,745][281656] Decorrelating experience for 288 frames... +[2023-03-07 15:43:35,745][281965] Decorrelating experience for 512 frames... +[2023-03-07 15:43:35,750][281576] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,771][281963] Decorrelating experience for 256 frames... +[2023-03-07 15:43:35,783][282693] Decorrelating experience for 448 frames... +[2023-03-07 15:43:35,818][281670] Decorrelating experience for 224 frames... +[2023-03-07 15:43:35,822][281652] Decorrelating experience for 448 frames... +[2023-03-07 15:43:35,864][284309] Decorrelating experience for 672 frames... +[2023-03-07 15:43:35,878][283149] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,900][281584] Decorrelating experience for 416 frames... +[2023-03-07 15:43:35,923][281961] Decorrelating experience for 192 frames... +[2023-03-07 15:43:35,941][281580] Decorrelating experience for 352 frames... +[2023-03-07 15:43:35,945][281802] Decorrelating experience for 352 frames... +[2023-03-07 15:43:35,969][281974] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,006][281945] Decorrelating experience for 544 frames... +[2023-03-07 15:43:36,024][283085] Decorrelating experience for 256 frames... +[2023-03-07 15:43:36,048][281581] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,067][282183] Decorrelating experience for 256 frames... +[2023-03-07 15:43:36,082][282699] Decorrelating experience for 480 frames... +[2023-03-07 15:43:36,087][281803] Decorrelating experience for 416 frames... +[2023-03-07 15:43:36,108][281988] Decorrelating experience for 352 frames... +[2023-03-07 15:43:36,126][281806] Decorrelating experience for 352 frames... +[2023-03-07 15:43:36,155][281992] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,157][283408] Decorrelating experience for 384 frames... +[2023-03-07 15:43:36,217][281965] Decorrelating experience for 544 frames... +[2023-03-07 15:43:36,232][281656] Decorrelating experience for 320 frames... +[2023-03-07 15:43:36,237][281958] Decorrelating experience for 704 frames... +[2023-03-07 15:43:36,267][281976] Decorrelating experience for 576 frames... +[2023-03-07 15:43:36,273][281967] Decorrelating experience for 320 frames... +[2023-03-07 15:43:36,293][281583] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,294][282763] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,314][284053] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,354][284048] Decorrelating experience for 640 frames... +[2023-03-07 15:43:36,354][281584] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,423][281946] Decorrelating experience for 576 frames... +[2023-03-07 15:43:36,424][281809] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,439][281658] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,449][281952] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,485][282184] Decorrelating experience for 352 frames... +[2023-03-07 15:43:36,486][282698] Decorrelating experience for 416 frames... +[2023-03-07 15:43:36,486][283019] Decorrelating experience for 352 frames... +[2023-03-07 15:43:36,534][281576] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,536][281806] Decorrelating experience for 384 frames... +[2023-03-07 15:43:36,560][281802] Decorrelating experience for 384 frames... +[2023-03-07 15:43:36,633][281652] Decorrelating experience for 480 frames... +[2023-03-07 15:43:36,635][281815] Decorrelating experience for 256 frames... +[2023-03-07 15:43:36,643][281803] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,665][281972] Decorrelating experience for 320 frames... +[2023-03-07 15:43:36,673][281988] Decorrelating experience for 384 frames... +[2023-03-07 15:43:36,684][283408] Decorrelating experience for 416 frames... +[2023-03-07 15:43:36,722][281578] Decorrelating experience for 352 frames... +[2023-03-07 15:43:36,722][281582] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,744][281992] Decorrelating experience for 256 frames... +[2023-03-07 15:43:36,771][281575] Decorrelating experience for 288 frames... +[2023-03-07 15:43:36,836][283149] Decorrelating experience for 448 frames... +[2023-03-07 15:43:36,841][281808] Decorrelating experience for 288 frames... +[2023-03-07 15:43:36,848][282184] Decorrelating experience for 384 frames... +[2023-03-07 15:43:36,863][281956] Decorrelating experience for 320 frames... +[2023-03-07 15:43:36,889][281658] Decorrelating experience for 480 frames... +[2023-03-07 15:43:36,910][281576] Decorrelating experience for 256 frames... +[2023-03-07 15:43:36,910][281801] Decorrelating experience for 128 frames... +[2023-03-07 15:43:36,924][281955] Decorrelating experience for 224 frames... +[2023-03-07 15:43:36,959][281815] Decorrelating experience for 288 frames... +[2023-03-07 15:43:36,975][281971] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,023][282694] Decorrelating experience for 448 frames... +[2023-03-07 15:43:37,037][281656] Decorrelating experience for 352 frames... +[2023-03-07 15:43:37,044][281583] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,046][281810] Decorrelating experience for 160 frames... +[2023-03-07 15:43:37,082][281958] Decorrelating experience for 736 frames... +[2023-03-07 15:43:37,104][283019] Decorrelating experience for 384 frames... +[2023-03-07 15:43:37,130][281972] Decorrelating experience for 352 frames... +[2023-03-07 15:43:37,150][281802] Decorrelating experience for 416 frames... +[2023-03-07 15:43:37,170][281974] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,182][281992] Decorrelating experience for 288 frames... +[2023-03-07 15:43:37,203][281965] Decorrelating experience for 576 frames... +[2023-03-07 15:43:37,218][281969] Decorrelating experience for 352 frames... +[2023-03-07 15:43:37,228][281665] Decorrelating experience for 352 frames... +[2023-03-07 15:43:37,286][281961] Decorrelating experience for 224 frames... +[2023-03-07 15:43:37,312][281806] Decorrelating experience for 416 frames... +[2023-03-07 15:43:37,350][281808] Decorrelating experience for 320 frames... +[2023-03-07 15:43:37,350][281656] Decorrelating experience for 384 frames... +[2023-03-07 15:43:37,368][281976] Decorrelating experience for 608 frames... +[2023-03-07 15:43:37,374][282558] Decorrelating experience for 288 frames... +[2023-03-07 15:43:37,377][283084] Decorrelating experience for 448 frames... +[2023-03-07 15:43:37,400][281815] Decorrelating experience for 320 frames... +[2023-03-07 15:43:37,411][281580] Decorrelating experience for 384 frames... +[2023-03-07 15:43:37,461][281950] Decorrelating experience for 128 frames... +[2023-03-07 15:43:37,465][281649] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,517][281952] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,541][281955] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,552][281961] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,556][281581] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,562][281658] Decorrelating experience for 512 frames... +[2023-03-07 15:43:37,581][281974] Decorrelating experience for 512 frames... +[2023-03-07 15:43:37,584][281969] Decorrelating experience for 384 frames... +[2023-03-07 15:43:37,590][281670] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,646][284166] Decorrelating experience for 320 frames... +[2023-03-07 15:43:37,648][281965] Decorrelating experience for 608 frames... +[2023-03-07 15:43:37,726][283084] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,733][281580] Decorrelating experience for 416 frames... +[2023-03-07 15:43:37,748][281652] Decorrelating experience for 512 frames... +[2023-03-07 15:43:37,748][281803] Decorrelating experience for 480 frames... +[2023-03-07 15:43:37,765][283150] Decorrelating experience for 352 frames... +[2023-03-07 15:43:37,779][282698] Decorrelating experience for 448 frames... +[2023-03-07 15:43:37,787][282183] Decorrelating experience for 288 frames... +[2023-03-07 15:43:37,831][281953] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:37,831][281961] Decorrelating experience for 288 frames... +[2023-03-07 15:43:37,834][281807] Decorrelating experience for 512 frames... +[2023-03-07 15:43:37,839][281660] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,926][281976] Decorrelating experience for 640 frames... +[2023-03-07 15:43:37,933][281655] Decorrelating experience for 128 frames... +[2023-03-07 15:43:37,958][283396] Decorrelating experience for 256 frames... +[2023-03-07 15:43:37,960][281972] Decorrelating experience for 384 frames... +[2023-03-07 15:43:38,011][281810] Decorrelating experience for 192 frames... +[2023-03-07 15:43:38,019][281670] Decorrelating experience for 288 frames... +[2023-03-07 15:43:38,026][283408] Decorrelating experience for 448 frames... +[2023-03-07 15:43:38,029][282696] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,054][281974] Decorrelating experience for 544 frames... +[2023-03-07 15:43:38,058][281945] Decorrelating experience for 576 frames... +[2023-03-07 15:43:38,110][281584] Decorrelating experience for 480 frames... +[2023-03-07 15:43:38,123][281955] Decorrelating experience for 288 frames... +[2023-03-07 15:43:38,198][281948] Decorrelating experience for 288 frames... +[2023-03-07 15:43:38,227][283281] Decorrelating experience for 384 frames... +[2023-03-07 15:43:38,229][281969] Decorrelating experience for 416 frames... +[2023-03-07 15:43:38,229][281663] Decorrelating experience for 448 frames... +[2023-03-07 15:43:38,230][281965] Decorrelating experience for 640 frames... +[2023-03-07 15:43:38,234][283417] Decorrelating experience for 448 frames... +[2023-03-07 15:43:38,250][281944] Decorrelating experience for 256 frames... +[2023-03-07 15:43:38,266][281807] Decorrelating experience for 544 frames... +[2023-03-07 15:43:38,298][281815] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,308][281583] Decorrelating experience for 512 frames... +[2023-03-07 15:43:38,383][281972] Decorrelating experience for 416 frames... +[2023-03-07 15:43:38,413][282565] Decorrelating experience for 448 frames... +[2023-03-07 15:43:38,425][281961] Decorrelating experience for 320 frames... +[2023-03-07 15:43:38,448][284166] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,467][282699] Decorrelating experience for 512 frames... +[2023-03-07 15:43:38,481][283352] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,486][281658] Decorrelating experience for 544 frames... +[2023-03-07 15:43:38,486][281955] Decorrelating experience for 320 frames... +[2023-03-07 15:43:38,491][283879] Decorrelating experience for 544 frames... +[2023-03-07 15:43:38,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:38,566][281950] Decorrelating experience for 160 frames... +[2023-03-07 15:43:38,594][281976] Decorrelating experience for 672 frames... +[2023-03-07 15:43:38,618][283417] Decorrelating experience for 480 frames... +[2023-03-07 15:43:38,652][283281] Decorrelating experience for 416 frames... +[2023-03-07 15:43:38,662][284309] Decorrelating experience for 704 frames... +[2023-03-07 15:43:38,671][282558] Decorrelating experience for 320 frames... +[2023-03-07 15:43:38,677][283408] Decorrelating experience for 480 frames... +[2023-03-07 15:43:38,682][281944] Decorrelating experience for 288 frames... +[2023-03-07 15:43:38,752][281582] Decorrelating experience for 256 frames... +[2023-03-07 15:43:38,759][282183] Decorrelating experience for 320 frames... +[2023-03-07 15:43:38,759][281583] Decorrelating experience for 544 frames... +[2023-03-07 15:43:38,776][281967] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,833][281654] Decorrelating experience for 320 frames... +[2023-03-07 15:43:38,849][283352] Decorrelating experience for 384 frames... +[2023-03-07 15:43:38,859][283278] Decorrelating experience for 512 frames... +[2023-03-07 15:43:38,882][281984] Decorrelating experience for 224 frames... +[2023-03-07 15:43:38,928][281945] Decorrelating experience for 608 frames... +[2023-03-07 15:43:38,935][281803] Decorrelating experience for 512 frames... +[2023-03-07 15:43:38,949][282827] Decorrelating experience for 416 frames... +[2023-03-07 15:43:38,951][281951] Decorrelating experience for 288 frames... +[2023-03-07 15:43:38,979][281961] Decorrelating experience for 352 frames... +[2023-03-07 15:43:38,982][281813] Decorrelating experience for 160 frames... +[2023-03-07 15:43:39,019][281660] Decorrelating experience for 288 frames... +[2023-03-07 15:43:39,045][284053] Decorrelating experience for 256 frames... +[2023-03-07 15:43:39,113][281967] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,123][282696] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,128][282698] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,134][283150] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,136][281992] Decorrelating experience for 320 frames... +[2023-03-07 15:43:39,163][281579] Decorrelating experience for 320 frames... +[2023-03-07 15:43:39,203][281960] Decorrelating experience for 416 frames... +[2023-03-07 15:43:39,231][281663] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,292][283641] Decorrelating experience for 352 frames... +[2023-03-07 15:43:39,318][284053] Decorrelating experience for 288 frames... +[2023-03-07 15:43:39,327][281581] Decorrelating experience for 512 frames... +[2023-03-07 15:43:39,343][283149] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,343][281580] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,388][281810] Decorrelating experience for 224 frames... +[2023-03-07 15:43:39,393][281578] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,393][281800] Decorrelating experience for 416 frames... +[2023-03-07 15:43:39,417][281813] Decorrelating experience for 192 frames... +[2023-03-07 15:43:39,434][282827] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,485][281991] Decorrelating experience for 224 frames... +[2023-03-07 15:43:39,539][281992] Decorrelating experience for 352 frames... +[2023-03-07 15:43:39,542][281576] Decorrelating experience for 288 frames... +[2023-03-07 15:43:39,582][281579] Decorrelating experience for 352 frames... +[2023-03-07 15:43:39,600][281976] Decorrelating experience for 704 frames... +[2023-03-07 15:43:39,605][282565] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,606][281583] Decorrelating experience for 576 frames... +[2023-03-07 15:43:39,606][283469] Decorrelating experience for 160 frames... +[2023-03-07 15:43:39,611][283150] Decorrelating experience for 416 frames... +[2023-03-07 15:43:39,643][283281] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,697][281664] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,726][281800] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,737][281802] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,767][281578] Decorrelating experience for 416 frames... +[2023-03-07 15:43:39,791][281989] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,799][281955] Decorrelating experience for 352 frames... +[2023-03-07 15:43:39,801][281961] Decorrelating experience for 384 frames... +[2023-03-07 15:43:39,805][281972] Decorrelating experience for 448 frames... +[2023-03-07 15:43:39,821][281663] Decorrelating experience for 512 frames... +[2023-03-07 15:43:39,852][283149] Decorrelating experience for 512 frames... +[2023-03-07 15:43:39,891][283890] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,946][283417] Decorrelating experience for 512 frames... +[2023-03-07 15:43:39,947][283084] Decorrelating experience for 512 frames... +[2023-03-07 15:43:39,979][281813] Decorrelating experience for 224 frames... +[2023-03-07 15:43:39,986][282827] Decorrelating experience for 480 frames... +[2023-03-07 15:43:39,994][281966] Decorrelating experience for 192 frames... +[2023-03-07 15:43:39,997][283278] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,027][281945] Decorrelating experience for 640 frames... +[2023-03-07 15:43:40,058][281576] Decorrelating experience for 320 frames... +[2023-03-07 15:43:40,092][281657] Decorrelating experience for 352 frames... +[2023-03-07 15:43:40,096][281579] Decorrelating experience for 384 frames... +[2023-03-07 15:43:40,145][281602] Decorrelating experience for 352 frames... +[2023-03-07 15:43:40,147][281658] Decorrelating experience for 576 frames... +[2023-03-07 15:43:40,179][281800] Decorrelating experience for 480 frames... +[2023-03-07 15:43:40,195][282183] Decorrelating experience for 352 frames... +[2023-03-07 15:43:40,199][284048] Decorrelating experience for 672 frames... +[2023-03-07 15:43:40,219][281989] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,238][281976] Decorrelating experience for 736 frames... +[2023-03-07 15:43:40,248][281813] Decorrelating experience for 256 frames... +[2023-03-07 15:43:40,291][281669] Decorrelating experience for 192 frames... +[2023-03-07 15:43:40,309][283890] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,349][281984] Decorrelating experience for 256 frames... +[2023-03-07 15:43:40,349][281808] Decorrelating experience for 352 frames... +[2023-03-07 15:43:40,393][283278] Decorrelating experience for 576 frames... +[2023-03-07 15:43:40,395][281802] Decorrelating experience for 480 frames... +[2023-03-07 15:43:40,398][281967] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,450][282565] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,460][284053] Decorrelating experience for 320 frames... +[2023-03-07 15:43:40,461][281992] Decorrelating experience for 384 frames... +[2023-03-07 15:43:40,473][283417] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,497][282827] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,547][281663] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,548][281950] Decorrelating experience for 192 frames... +[2023-03-07 15:43:40,592][281800] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,597][281649] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,602][282184] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,644][283879] Decorrelating experience for 576 frames... +[2023-03-07 15:43:40,660][281602] Decorrelating experience for 384 frames... +[2023-03-07 15:43:40,662][282698] Decorrelating experience for 512 frames... +[2023-03-07 15:43:40,676][281984] Decorrelating experience for 288 frames... +[2023-03-07 15:43:40,681][281961] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,740][283890] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,762][281652] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,805][281962] Decorrelating experience for 224 frames... +[2023-03-07 15:43:40,805][281656] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,806][281988] Decorrelating experience for 416 frames... +[2023-03-07 15:43:40,838][284048] Decorrelating experience for 704 frames... +[2023-03-07 15:43:40,845][281658] Decorrelating experience for 608 frames... +[2023-03-07 15:43:40,867][281576] Decorrelating experience for 352 frames... +[2023-03-07 15:43:40,876][282827] Decorrelating experience for 544 frames... +[2023-03-07 15:43:40,877][281660] Decorrelating experience for 320 frames... +[2023-03-07 15:43:40,937][282184] Decorrelating experience for 448 frames... +[2023-03-07 15:43:40,950][282558] Decorrelating experience for 352 frames... +[2023-03-07 15:43:41,002][281963] Decorrelating experience for 288 frames... +[2023-03-07 15:43:41,003][282283] Decorrelating experience for 448 frames... +[2023-03-07 15:43:41,030][281801] Decorrelating experience for 160 frames... +[2023-03-07 15:43:41,033][281813] Decorrelating experience for 288 frames... +[2023-03-07 15:43:41,052][282699] Decorrelating experience for 544 frames... +[2023-03-07 15:43:41,068][281808] Decorrelating experience for 384 frames... +[2023-03-07 15:43:41,069][284053] Decorrelating experience for 352 frames... +[2023-03-07 15:43:41,081][281579] Decorrelating experience for 416 frames... +[2023-03-07 15:43:41,142][283879] Decorrelating experience for 608 frames... +[2023-03-07 15:43:41,175][281961] Decorrelating experience for 448 frames... +[2023-03-07 15:43:41,197][281652] Decorrelating experience for 576 frames... +[2023-03-07 15:43:41,212][281657] Decorrelating experience for 384 frames... +[2023-03-07 15:43:41,253][283085] Decorrelating experience for 288 frames... +[2023-03-07 15:43:41,253][281658] Decorrelating experience for 640 frames... +[2023-03-07 15:43:41,261][281964] Decorrelating experience for 256 frames... +[2023-03-07 15:43:41,289][281581] Decorrelating experience for 544 frames... +[2023-03-07 15:43:41,302][282698] Decorrelating experience for 544 frames... +[2023-03-07 15:43:41,323][283641] Decorrelating experience for 384 frames... +[2023-03-07 15:43:41,379][284053] Decorrelating experience for 384 frames... +[2023-03-07 15:43:41,385][283890] Decorrelating experience for 576 frames... +[2023-03-07 15:43:41,407][283396] Decorrelating experience for 288 frames... +[2023-03-07 15:43:41,416][281579] Decorrelating experience for 448 frames... +[2023-03-07 15:43:41,437][282184] Decorrelating experience for 480 frames... +[2023-03-07 15:43:41,452][281655] Decorrelating experience for 160 frames... +[2023-03-07 15:43:41,457][283020] Decorrelating experience for 320 frames... +[2023-03-07 15:43:41,486][282693] Decorrelating experience for 480 frames... +[2023-03-07 15:43:41,493][281656] Decorrelating experience for 448 frames... +[2023-03-07 15:43:41,542][281966] Decorrelating experience for 224 frames... +[2023-03-07 15:43:41,566][281810] Decorrelating experience for 256 frames... +[2023-03-07 15:43:41,601][281801] Decorrelating experience for 192 frames... +[2023-03-07 15:43:41,602][281963] Decorrelating experience for 320 frames... +[2023-03-07 15:43:41,622][281984] Decorrelating experience for 320 frames... +[2023-03-07 15:43:41,642][281652] Decorrelating experience for 608 frames... +[2023-03-07 15:43:41,650][281804] Decorrelating experience for 256 frames... +[2023-03-07 15:43:41,670][281658] Decorrelating experience for 672 frames... +[2023-03-07 15:43:41,686][281657] Decorrelating experience for 416 frames... +[2023-03-07 15:43:41,721][281955] Decorrelating experience for 384 frames... +[2023-03-07 15:43:41,752][281954] Decorrelating experience for 192 frames... +[2023-03-07 15:43:41,784][281649] Decorrelating experience for 544 frames... +[2023-03-07 15:43:41,797][281655] Decorrelating experience for 192 frames... +[2023-03-07 15:43:41,848][281967] Decorrelating experience for 448 frames... +[2023-03-07 15:43:41,868][282699] Decorrelating experience for 576 frames... +[2023-03-07 15:43:41,889][284047] Decorrelating experience for 192 frames... +[2023-03-07 15:43:41,925][284048] Decorrelating experience for 736 frames... +[2023-03-07 15:43:41,947][281961] Decorrelating experience for 480 frames... +[2023-03-07 15:43:41,956][281813] Decorrelating experience for 320 frames... +[2023-03-07 15:43:41,990][281801] Decorrelating experience for 224 frames... +[2023-03-07 15:43:42,003][282558] Decorrelating experience for 384 frames... +[2023-03-07 15:43:42,019][281656] Decorrelating experience for 480 frames... +[2023-03-07 15:43:42,045][281576] Decorrelating experience for 384 frames... +[2023-03-07 15:43:42,045][281804] Decorrelating experience for 288 frames... +[2023-03-07 15:43:42,060][284309] Decorrelating experience for 736 frames... +[2023-03-07 15:43:42,084][281660] Decorrelating experience for 352 frames... +[2023-03-07 15:43:42,116][281658] Decorrelating experience for 704 frames... +[2023-03-07 15:43:42,124][281966] Decorrelating experience for 256 frames... +[2023-03-07 15:43:42,145][281808] Decorrelating experience for 416 frames... +[2023-03-07 15:43:42,178][281984] Decorrelating experience for 352 frames... +[2023-03-07 15:43:42,208][281946] Decorrelating experience for 608 frames... +[2023-03-07 15:43:42,209][281964] Decorrelating experience for 288 frames... +[2023-03-07 15:43:42,255][284047] Decorrelating experience for 224 frames... +[2023-03-07 15:43:42,255][282698] Decorrelating experience for 576 frames... +[2023-03-07 15:43:42,255][283890] Decorrelating experience for 608 frames... +[2023-03-07 15:43:42,275][281815] Decorrelating experience for 384 frames... +[2023-03-07 15:43:42,303][281988] Decorrelating experience for 448 frames... +[2023-03-07 15:43:42,339][281800] Decorrelating experience for 544 frames... +[2023-03-07 15:43:42,364][281951] Decorrelating experience for 320 frames... +[2023-03-07 15:43:42,378][281581] Decorrelating experience for 576 frames... +[2023-03-07 15:43:42,448][281810] Decorrelating experience for 288 frames... +[2023-03-07 15:43:42,450][281804] Decorrelating experience for 320 frames... +[2023-03-07 15:43:42,459][281966] Decorrelating experience for 288 frames... +[2023-03-07 15:43:42,465][281669] Decorrelating experience for 224 frames... +[2023-03-07 15:43:42,484][284053] Decorrelating experience for 416 frames... +[2023-03-07 15:43:42,485][281968] Decorrelating experience for 448 frames... +[2023-03-07 15:43:42,488][281663] Decorrelating experience for 576 frames... +[2023-03-07 15:43:42,521][284047] Decorrelating experience for 256 frames... +[2023-03-07 15:43:42,553][283879] Decorrelating experience for 640 frames... +[2023-03-07 15:43:42,635][282699] Decorrelating experience for 608 frames... +[2023-03-07 15:43:42,638][281655] Decorrelating experience for 224 frames... +[2023-03-07 15:43:42,659][281661] Decorrelating experience for 320 frames... +[2023-03-07 15:43:42,671][281960] Decorrelating experience for 448 frames... +[2023-03-07 15:43:42,672][281946] Decorrelating experience for 640 frames... +[2023-03-07 15:43:42,679][282558] Decorrelating experience for 416 frames... +[2023-03-07 15:43:42,690][281992] Decorrelating experience for 416 frames... +[2023-03-07 15:43:42,740][282565] Decorrelating experience for 544 frames... +[2023-03-07 15:43:42,748][281988] Decorrelating experience for 480 frames... +[2023-03-07 15:43:42,776][281658] Decorrelating experience for 736 frames... +[2023-03-07 15:43:42,822][283417] Decorrelating experience for 576 frames... +[2023-03-07 15:43:42,829][281576] Decorrelating experience for 416 frames... +[2023-03-07 15:43:42,867][281950] Decorrelating experience for 224 frames... +[2023-03-07 15:43:42,874][281583] Decorrelating experience for 608 frames... +[2023-03-07 15:43:42,884][281663] Decorrelating experience for 608 frames... +[2023-03-07 15:43:42,906][281665] Decorrelating experience for 384 frames... +[2023-03-07 15:43:42,948][283149] Decorrelating experience for 544 frames... +[2023-03-07 15:43:42,949][281657] Decorrelating experience for 448 frames... +[2023-03-07 15:43:42,969][283150] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,010][281654] Decorrelating experience for 352 frames... +[2023-03-07 15:43:43,019][281960] Decorrelating experience for 480 frames... +[2023-03-07 15:43:43,070][281959] Decorrelating experience for 64 frames... +[2023-03-07 15:43:43,090][281810] Decorrelating experience for 320 frames... +[2023-03-07 15:43:43,102][281984] Decorrelating experience for 384 frames... +[2023-03-07 15:43:43,127][282891] Decorrelating experience for 96 frames... +[2023-03-07 15:43:43,133][281950] Decorrelating experience for 256 frames... +[2023-03-07 15:43:43,154][281815] Decorrelating experience for 416 frames... +[2023-03-07 15:43:43,203][281801] Decorrelating experience for 256 frames... +[2023-03-07 15:43:43,208][282565] Decorrelating experience for 576 frames... +[2023-03-07 15:43:43,210][281670] Decorrelating experience for 320 frames... +[2023-03-07 15:43:43,282][281583] Decorrelating experience for 640 frames... +[2023-03-07 15:43:43,286][281962] Decorrelating experience for 256 frames... +[2023-03-07 15:43:43,316][281973] Decorrelating experience for 480 frames... +[2023-03-07 15:43:43,320][281989] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,320][281951] Decorrelating experience for 352 frames... +[2023-03-07 15:43:43,389][281669] Decorrelating experience for 256 frames... +[2023-03-07 15:43:43,398][281656] Decorrelating experience for 512 frames... +[2023-03-07 15:43:43,398][281663] Decorrelating experience for 640 frames... +[2023-03-07 15:43:43,419][281580] Decorrelating experience for 480 frames... +[2023-03-07 15:43:43,471][283879] Decorrelating experience for 672 frames... +[2023-03-07 15:43:43,471][281969] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,499][281664] Decorrelating experience for 416 frames... +[2023-03-07 15:43:43,511][281807] Decorrelating experience for 576 frames... +[2023-03-07 15:43:43,523][283020] Decorrelating experience for 352 frames... +[2023-03-07 15:43:43,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:43,578][281984] Decorrelating experience for 416 frames... +[2023-03-07 15:43:43,589][281976] Decorrelating experience for 768 frames... +[2023-03-07 15:43:43,590][283278] Decorrelating experience for 608 frames... +[2023-03-07 15:43:43,605][282565] Decorrelating experience for 608 frames... +[2023-03-07 15:43:43,658][284047] Decorrelating experience for 288 frames... +[2023-03-07 15:43:43,662][283352] Decorrelating experience for 416 frames... +[2023-03-07 15:43:43,686][284048] Decorrelating experience for 768 frames... +[2023-03-07 15:43:43,725][281946] Decorrelating experience for 672 frames... +[2023-03-07 15:43:43,725][281578] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,765][281649] Decorrelating experience for 576 frames... +[2023-03-07 15:43:43,778][283417] Decorrelating experience for 608 frames... +[2023-03-07 15:43:43,842][281664] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,846][281992] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,854][281583] Decorrelating experience for 672 frames... +[2023-03-07 15:43:43,857][281652] Decorrelating experience for 640 frames... +[2023-03-07 15:43:43,874][282558] Decorrelating experience for 448 frames... +[2023-03-07 15:43:43,881][281670] Decorrelating experience for 352 frames... +[2023-03-07 15:43:43,966][282184] Decorrelating experience for 512 frames... +[2023-03-07 15:43:43,969][281955] Decorrelating experience for 416 frames... +[2023-03-07 15:43:43,985][284166] Decorrelating experience for 384 frames... +[2023-03-07 15:43:43,995][281963] Decorrelating experience for 352 frames... +[2023-03-07 15:43:44,062][283879] Decorrelating experience for 704 frames... +[2023-03-07 15:43:44,070][281807] Decorrelating experience for 608 frames... +[2023-03-07 15:43:44,071][281976] Decorrelating experience for 800 frames... +[2023-03-07 15:43:44,073][281984] Decorrelating experience for 448 frames... +[2023-03-07 15:43:44,096][281806] Decorrelating experience for 448 frames... +[2023-03-07 15:43:44,151][284048] Decorrelating experience for 800 frames... +[2023-03-07 15:43:44,172][282693] Decorrelating experience for 512 frames... +[2023-03-07 15:43:44,183][281658] Decorrelating experience for 768 frames... +[2023-03-07 15:43:44,183][281580] Decorrelating experience for 512 frames... +[2023-03-07 15:43:44,257][281657] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,261][282699] Decorrelating experience for 640 frames... +[2023-03-07 15:43:44,296][283352] Decorrelating experience for 448 frames... +[2023-03-07 15:43:44,300][281973] Decorrelating experience for 512 frames... +[2023-03-07 15:43:44,300][281652] Decorrelating experience for 672 frames... +[2023-03-07 15:43:44,302][281971] Decorrelating experience for 512 frames... +[2023-03-07 15:43:44,338][281810] Decorrelating experience for 352 frames... +[2023-03-07 15:43:44,374][281963] Decorrelating experience for 384 frames... +[2023-03-07 15:43:44,391][281670] Decorrelating experience for 384 frames... +[2023-03-07 15:43:44,393][281576] Decorrelating experience for 448 frames... +[2023-03-07 15:43:44,449][281806] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,449][281667] Decorrelating experience for 192 frames... +[2023-03-07 15:43:44,492][281968] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,524][282697] Decorrelating experience for 224 frames... +[2023-03-07 15:43:44,538][281654] Decorrelating experience for 384 frames... +[2023-03-07 15:43:44,556][281970] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:44,559][281992] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,560][283879] Decorrelating experience for 736 frames... +[2023-03-07 15:43:44,586][281801] Decorrelating experience for 288 frames... +[2023-03-07 15:43:44,649][281989] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,649][282891] Decorrelating experience for 128 frames... +[2023-03-07 15:43:44,649][281807] Decorrelating experience for 640 frames... +[2023-03-07 15:43:44,684][281803] Decorrelating experience for 544 frames... +[2023-03-07 15:43:44,743][284166] Decorrelating experience for 416 frames... +[2023-03-07 15:43:44,753][281810] Decorrelating experience for 384 frames... +[2023-03-07 15:43:44,792][284048] Decorrelating experience for 832 frames... +[2023-03-07 15:43:44,799][281971] Decorrelating experience for 544 frames... +[2023-03-07 15:43:44,812][281808] Decorrelating experience for 448 frames... +[2023-03-07 15:43:44,842][282699] Decorrelating experience for 672 frames... +[2023-03-07 15:43:44,850][281946] Decorrelating experience for 704 frames... +[2023-03-07 15:43:44,851][281578] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,875][281973] Decorrelating experience for 544 frames... +[2023-03-07 15:43:44,900][281968] Decorrelating experience for 512 frames... +[2023-03-07 15:43:44,940][281664] Decorrelating experience for 480 frames... +[2023-03-07 15:43:44,946][282695] Decorrelating experience for 416 frames... +[2023-03-07 15:43:44,995][281950] Decorrelating experience for 288 frames... +[2023-03-07 15:43:44,997][282184] Decorrelating experience for 544 frames... +[2023-03-07 15:43:45,014][283020] Decorrelating experience for 384 frames... +[2023-03-07 15:43:45,054][281966] Decorrelating experience for 320 frames... +[2023-03-07 15:43:45,055][281984] Decorrelating experience for 480 frames... +[2023-03-07 15:43:45,056][281576] Decorrelating experience for 480 frames... +[2023-03-07 15:43:45,086][282891] Decorrelating experience for 160 frames... +[2023-03-07 15:43:45,146][281649] Decorrelating experience for 608 frames... +[2023-03-07 15:43:45,149][281951] Decorrelating experience for 384 frames... +[2023-03-07 15:43:45,210][281577] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:45,223][284053] Decorrelating experience for 448 frames... +[2023-03-07 15:43:45,257][281988] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,258][281660] Decorrelating experience for 384 frames... +[2023-03-07 15:43:45,265][281960] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,267][282693] Decorrelating experience for 544 frames... +[2023-03-07 15:43:45,268][281971] Decorrelating experience for 576 frames... +[2023-03-07 15:43:45,280][281991] Decorrelating experience for 256 frames... +[2023-03-07 15:43:45,290][282696] Decorrelating experience for 416 frames... +[2023-03-07 15:43:45,342][281962] Decorrelating experience for 288 frames... +[2023-03-07 15:43:45,367][284048] Decorrelating experience for 864 frames... +[2023-03-07 15:43:45,407][281966] Decorrelating experience for 352 frames... +[2023-03-07 15:43:45,444][281582] Decorrelating experience for 288 frames... +[2023-03-07 15:43:45,458][281667] Decorrelating experience for 224 frames... +[2023-03-07 15:43:45,477][281951] Decorrelating experience for 416 frames... +[2023-03-07 15:43:45,490][283352] Decorrelating experience for 480 frames... +[2023-03-07 15:43:45,494][281945] Decorrelating experience for 672 frames... +[2023-03-07 15:43:45,504][283020] Decorrelating experience for 416 frames... +[2023-03-07 15:43:45,518][281804] Decorrelating experience for 352 frames... +[2023-03-07 15:43:45,565][281989] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,606][281959] Decorrelating experience for 96 frames... +[2023-03-07 15:43:45,612][281955] Decorrelating experience for 448 frames... +[2023-03-07 15:43:45,631][281657] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,648][281649] Decorrelating experience for 640 frames... +[2023-03-07 15:43:45,675][281807] Decorrelating experience for 672 frames... +[2023-03-07 15:43:45,679][282696] Decorrelating experience for 448 frames... +[2023-03-07 15:43:45,695][281815] Decorrelating experience for 448 frames... +[2023-03-07 15:43:45,722][281670] Decorrelating experience for 416 frames... +[2023-03-07 15:43:45,752][281991] Decorrelating experience for 288 frames... +[2023-03-07 15:43:45,771][281966] Decorrelating experience for 384 frames... +[2023-03-07 15:43:45,791][281950] Decorrelating experience for 320 frames... +[2023-03-07 15:43:45,797][281584] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,822][281992] Decorrelating experience for 512 frames... +[2023-03-07 15:43:45,860][281968] Decorrelating experience for 544 frames... +[2023-03-07 15:43:45,868][281971] Decorrelating experience for 608 frames... +[2023-03-07 15:43:45,888][281582] Decorrelating experience for 320 frames... +[2023-03-07 15:43:45,896][283890] Decorrelating experience for 640 frames... +[2023-03-07 15:43:45,931][281801] Decorrelating experience for 320 frames... +[2023-03-07 15:43:45,965][281951] Decorrelating experience for 448 frames... +[2023-03-07 15:43:45,985][282891] Decorrelating experience for 192 frames... +[2023-03-07 15:43:45,998][281944] Decorrelating experience for 320 frames... +[2023-03-07 15:43:46,014][281657] Decorrelating experience for 544 frames... +[2023-03-07 15:43:46,014][281652] Decorrelating experience for 704 frames... +[2023-03-07 15:43:46,056][281664] Decorrelating experience for 512 frames... +[2023-03-07 15:43:46,082][281580] Decorrelating experience for 544 frames... +[2023-03-07 15:43:46,087][282696] Decorrelating experience for 480 frames... +[2023-03-07 15:43:46,125][284048] Decorrelating experience for 896 frames... +[2023-03-07 15:43:46,135][281991] Decorrelating experience for 320 frames... +[2023-03-07 15:43:46,170][283879] Decorrelating experience for 768 frames... +[2023-03-07 15:43:46,197][281950] Decorrelating experience for 352 frames... +[2023-03-07 15:43:46,200][283641] Decorrelating experience for 416 frames... +[2023-03-07 15:43:46,212][283020] Decorrelating experience for 448 frames... +[2023-03-07 15:43:46,241][281582] Decorrelating experience for 352 frames... +[2023-03-07 15:43:46,256][281808] Decorrelating experience for 480 frames... +[2023-03-07 15:43:46,266][281584] Decorrelating experience for 544 frames... +[2023-03-07 15:43:46,282][281801] Decorrelating experience for 352 frames... +[2023-03-07 15:43:46,311][281670] Decorrelating experience for 448 frames... +[2023-03-07 15:43:46,350][283084] Decorrelating experience for 544 frames... +[2023-03-07 15:43:46,379][281577] Decorrelating experience for 352 frames... +[2023-03-07 15:43:46,417][281655] Decorrelating experience for 256 frames... +[2023-03-07 15:43:46,451][284166] Decorrelating experience for 448 frames... +[2023-03-07 15:43:46,458][281988] Decorrelating experience for 544 frames... +[2023-03-07 15:43:46,459][281652] Decorrelating experience for 736 frames... +[2023-03-07 15:43:46,472][281804] Decorrelating experience for 384 frames... +[2023-03-07 15:43:46,475][281576] Decorrelating experience for 512 frames... +[2023-03-07 15:43:46,497][282695] Decorrelating experience for 448 frames... +[2023-03-07 15:43:46,529][281991] Decorrelating experience for 352 frames... +[2023-03-07 15:43:46,551][281964] Decorrelating experience for 320 frames... +[2023-03-07 15:43:46,564][284304] Decorrelating experience for 64 frames... +[2023-03-07 15:43:46,603][282827] Decorrelating experience for 576 frames... +[2023-03-07 15:43:46,637][282699] Decorrelating experience for 704 frames... +[2023-03-07 15:43:46,654][281971] Decorrelating experience for 640 frames... +[2023-03-07 15:43:46,681][281654] Decorrelating experience for 416 frames... +[2023-03-07 15:43:46,684][283890] Decorrelating experience for 672 frames... +[2023-03-07 15:43:46,701][283879] Decorrelating experience for 800 frames... +[2023-03-07 15:43:46,710][281962] Decorrelating experience for 320 frames... +[2023-03-07 15:43:46,733][283084] Decorrelating experience for 576 frames... +[2023-03-07 15:43:46,751][282696] Decorrelating experience for 512 frames... +[2023-03-07 15:43:46,757][284047] Decorrelating experience for 320 frames... +[2023-03-07 15:43:46,802][281963] Decorrelating experience for 416 frames... +[2023-03-07 15:43:46,821][281651] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:46,866][281655] Decorrelating experience for 288 frames... +[2023-03-07 15:43:46,904][281658] Decorrelating experience for 800 frames... +[2023-03-07 15:43:46,912][281578] Decorrelating experience for 512 frames... +[2023-03-07 15:43:46,917][283641] Decorrelating experience for 448 frames... +[2023-03-07 15:43:46,918][281577] Decorrelating experience for 384 frames... +[2023-03-07 15:43:46,935][281991] Decorrelating experience for 384 frames... +[2023-03-07 15:43:46,989][284200] Decorrelating experience for 256 frames... +[2023-03-07 15:43:46,994][281657] Decorrelating experience for 576 frames... +[2023-03-07 15:43:46,995][281584] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,032][282184] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,078][281944] Decorrelating experience for 352 frames... +[2023-03-07 15:43:47,099][283352] Decorrelating experience for 512 frames... +[2023-03-07 15:43:47,107][281960] Decorrelating experience for 544 frames... +[2023-03-07 15:43:47,127][284304] Decorrelating experience for 96 frames... +[2023-03-07 15:43:47,130][281804] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,149][282891] Decorrelating experience for 224 frames... +[2023-03-07 15:43:47,185][282827] Decorrelating experience for 608 frames... +[2023-03-07 15:43:47,205][281965] Decorrelating experience for 672 frames... +[2023-03-07 15:43:47,208][283879] Decorrelating experience for 832 frames... +[2023-03-07 15:43:47,220][281652] Decorrelating experience for 768 frames... +[2023-03-07 15:43:47,262][288232] Decorrelating experience for 288 frames... +[2023-03-07 15:43:47,294][281810] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,313][281950] Decorrelating experience for 384 frames... +[2023-03-07 15:43:47,326][284053] Decorrelating experience for 480 frames... +[2023-03-07 15:43:47,362][281991] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,388][282904] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:47,396][281584] Decorrelating experience for 608 frames... +[2023-03-07 15:43:47,398][281971] Decorrelating experience for 672 frames... +[2023-03-07 15:43:47,410][281815] Decorrelating experience for 480 frames... +[2023-03-07 15:43:47,412][281961] Decorrelating experience for 512 frames... +[2023-03-07 15:43:47,419][281968] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,453][284048] Decorrelating experience for 928 frames... +[2023-03-07 15:43:47,481][281649] Decorrelating experience for 672 frames... +[2023-03-07 15:43:47,503][281576] Decorrelating experience for 544 frames... +[2023-03-07 15:43:47,508][281582] Decorrelating experience for 384 frames... +[2023-03-07 15:43:47,589][281993] Decorrelating experience for 544 frames... +[2023-03-07 15:43:47,598][281658] Decorrelating experience for 832 frames... +[2023-03-07 15:43:47,604][281660] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,610][283408] Decorrelating experience for 512 frames... +[2023-03-07 15:43:47,638][284047] Decorrelating experience for 352 frames... +[2023-03-07 15:43:47,677][281957] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:43:47,678][281962] Decorrelating experience for 352 frames... +[2023-03-07 15:43:47,687][283149] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,697][281944] Decorrelating experience for 384 frames... +[2023-03-07 15:43:47,701][281991] Decorrelating experience for 448 frames... +[2023-03-07 15:43:47,729][283960] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:47,777][281809] Decorrelating experience for 256 frames... +[2023-03-07 15:43:47,786][281961] Decorrelating experience for 544 frames... +[2023-03-07 15:43:47,829][283396] Decorrelating experience for 320 frames... +[2023-03-07 15:43:47,829][283890] Decorrelating experience for 704 frames... +[2023-03-07 15:43:47,831][281966] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,863][281665] Decorrelating experience for 416 frames... +[2023-03-07 15:43:47,879][281973] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,885][282184] Decorrelating experience for 608 frames... +[2023-03-07 15:43:47,891][281576] Decorrelating experience for 576 frames... +[2023-03-07 15:43:47,932][281652] Decorrelating experience for 800 frames... +[2023-03-07 15:43:47,960][281579] Decorrelating experience for 480 frames... +[2023-03-07 15:43:47,979][282696] Decorrelating experience for 544 frames... +[2023-03-07 15:43:48,019][281952] Decorrelating experience for 288 frames... +[2023-03-07 15:43:48,022][281815] Decorrelating experience for 512 frames... +[2023-03-07 15:43:48,029][283727] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:48,045][281657] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,075][283641] Decorrelating experience for 480 frames... +[2023-03-07 15:43:48,086][283278] Decorrelating experience for 640 frames... +[2023-03-07 15:43:48,089][282698] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,122][281944] Decorrelating experience for 416 frames... +[2023-03-07 15:43:48,156][281809] Decorrelating experience for 288 frames... +[2023-03-07 15:43:48,162][282183] Decorrelating experience for 384 frames... +[2023-03-07 15:43:48,215][283084] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,215][284047] Decorrelating experience for 384 frames... +[2023-03-07 15:43:48,235][284048] Decorrelating experience for 960 frames... +[2023-03-07 15:43:48,276][282891] Decorrelating experience for 256 frames... +[2023-03-07 15:43:48,279][281992] Decorrelating experience for 544 frames... +[2023-03-07 15:43:48,312][281801] Decorrelating experience for 384 frames... +[2023-03-07 15:43:48,344][281576] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,358][281665] Decorrelating experience for 448 frames... +[2023-03-07 15:43:48,365][281973] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,366][281582] Decorrelating experience for 416 frames... +[2023-03-07 15:43:48,400][281991] Decorrelating experience for 480 frames... +[2023-03-07 15:43:48,430][281584] Decorrelating experience for 640 frames... +[2023-03-07 15:43:48,457][281657] Decorrelating experience for 640 frames... +[2023-03-07 15:43:48,469][281658] Decorrelating experience for 864 frames... +[2023-03-07 15:43:48,498][281988] Decorrelating experience for 576 frames... +[2023-03-07 15:43:48,534][284053] Decorrelating experience for 512 frames... +[2023-03-07 15:43:48,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:48,549][282904] Decorrelating experience for 128 frames... +[2023-03-07 15:43:48,588][283890] Decorrelating experience for 736 frames... +[2023-03-07 15:43:48,607][281968] Decorrelating experience for 608 frames... +[2023-03-07 15:43:48,618][283879] Decorrelating experience for 864 frames... +[2023-03-07 15:43:48,647][281652] Decorrelating experience for 832 frames... +[2023-03-07 15:43:48,690][281993] Decorrelating experience for 576 frames... +[2023-03-07 15:43:48,697][281984] Decorrelating experience for 512 frames... +[2023-03-07 15:43:48,708][283020] Decorrelating experience for 480 frames... +[2023-03-07 15:43:48,733][283084] Decorrelating experience for 640 frames... +[2023-03-07 15:43:48,736][281976] Decorrelating experience for 832 frames... +[2023-03-07 15:43:48,781][284200] Decorrelating experience for 288 frames... +[2023-03-07 15:43:48,797][283396] Decorrelating experience for 352 frames... +[2023-03-07 15:43:48,810][281952] Decorrelating experience for 320 frames... +[2023-03-07 15:43:48,852][281973] Decorrelating experience for 640 frames... +[2023-03-07 15:43:48,888][281964] Decorrelating experience for 352 frames... +[2023-03-07 15:43:48,900][281945] Decorrelating experience for 704 frames... +[2023-03-07 15:43:48,900][281801] Decorrelating experience for 416 frames... +[2023-03-07 15:43:48,923][283854] Decorrelating experience for 512 frames... +[2023-03-07 15:43:48,947][281810] Decorrelating experience for 448 frames... +[2023-03-07 15:43:48,949][281961] Decorrelating experience for 576 frames... +[2023-03-07 15:43:48,985][281657] Decorrelating experience for 672 frames... +[2023-03-07 15:43:48,988][281951] Decorrelating experience for 480 frames... +[2023-03-07 15:43:48,998][281806] Decorrelating experience for 512 frames... +[2023-03-07 15:43:49,054][281988] Decorrelating experience for 608 frames... +[2023-03-07 15:43:49,075][281813] Decorrelating experience for 352 frames... +[2023-03-07 15:43:49,093][283352] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,102][281584] Decorrelating experience for 672 frames... +[2023-03-07 15:43:49,116][281952] Decorrelating experience for 352 frames... +[2023-03-07 15:43:49,144][282183] Decorrelating experience for 416 frames... +[2023-03-07 15:43:49,177][284200] Decorrelating experience for 320 frames... +[2023-03-07 15:43:49,181][283396] Decorrelating experience for 384 frames... +[2023-03-07 15:43:49,195][281582] Decorrelating experience for 448 frames... +[2023-03-07 15:43:49,196][281993] Decorrelating experience for 608 frames... +[2023-03-07 15:43:49,264][281959] Decorrelating experience for 128 frames... +[2023-03-07 15:43:49,300][283854] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,306][282904] Decorrelating experience for 160 frames... +[2023-03-07 15:43:49,333][281950] Decorrelating experience for 416 frames... +[2023-03-07 15:43:49,348][281992] Decorrelating experience for 576 frames... +[2023-03-07 15:43:49,372][288232] Decorrelating experience for 320 frames... +[2023-03-07 15:43:49,374][283020] Decorrelating experience for 512 frames... +[2023-03-07 15:43:49,384][281966] Decorrelating experience for 448 frames... +[2023-03-07 15:43:49,397][281810] Decorrelating experience for 480 frames... +[2023-03-07 15:43:49,404][281951] Decorrelating experience for 512 frames... +[2023-03-07 15:43:49,447][282697] Decorrelating experience for 256 frames... +[2023-03-07 15:43:49,499][281813] Decorrelating experience for 384 frames... +[2023-03-07 15:43:49,505][281809] Decorrelating experience for 320 frames... +[2023-03-07 15:43:49,519][281990] Decorrelating experience for 512 frames... +[2023-03-07 15:43:49,543][281582] Decorrelating experience for 480 frames... +[2023-03-07 15:43:49,571][281988] Decorrelating experience for 640 frames... +[2023-03-07 15:43:49,573][281663] Decorrelating experience for 672 frames... +[2023-03-07 15:43:49,588][281976] Decorrelating experience for 864 frames... +[2023-03-07 15:43:49,597][281959] Decorrelating experience for 160 frames... +[2023-03-07 15:43:49,603][281578] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,636][281649] Decorrelating experience for 704 frames... +[2023-03-07 15:43:49,716][281964] Decorrelating experience for 384 frames... +[2023-03-07 15:43:49,717][281993] Decorrelating experience for 640 frames... +[2023-03-07 15:43:49,728][284304] Decorrelating experience for 128 frames... +[2023-03-07 15:43:49,757][282698] Decorrelating experience for 640 frames... +[2023-03-07 15:43:49,787][281966] Decorrelating experience for 480 frames... +[2023-03-07 15:43:49,826][281806] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,828][282763] Decorrelating experience for 480 frames... +[2023-03-07 15:43:49,832][281991] Decorrelating experience for 512 frames... +[2023-03-07 15:43:49,843][281950] Decorrelating experience for 448 frames... +[2023-03-07 15:43:49,862][283020] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,904][283469] Decorrelating experience for 192 frames... +[2023-03-07 15:43:49,922][281989] Decorrelating experience for 544 frames... +[2023-03-07 15:43:49,923][281961] Decorrelating experience for 608 frames... +[2023-03-07 15:43:49,960][281809] Decorrelating experience for 352 frames... +[2023-03-07 15:43:49,989][282183] Decorrelating experience for 448 frames... +[2023-03-07 15:43:50,018][281960] Decorrelating experience for 576 frames... +[2023-03-07 15:43:50,028][281975] Decorrelating experience for 544 frames... +[2023-03-07 15:43:50,036][288232] Decorrelating experience for 352 frames... +[2023-03-07 15:43:50,040][283396] Decorrelating experience for 416 frames... +[2023-03-07 15:43:50,117][281962] Decorrelating experience for 384 frames... +[2023-03-07 15:43:50,122][281663] Decorrelating experience for 704 frames... +[2023-03-07 15:43:50,125][281988] Decorrelating experience for 672 frames... +[2023-03-07 15:43:50,160][281655] Decorrelating experience for 320 frames... +[2023-03-07 15:43:50,175][282699] Decorrelating experience for 736 frames... +[2023-03-07 15:43:50,199][281990] Decorrelating experience for 544 frames... +[2023-03-07 15:43:50,206][281804] Decorrelating experience for 448 frames... +[2023-03-07 15:43:50,224][283417] Decorrelating experience for 640 frames... +[2023-03-07 15:43:50,238][281575] Decorrelating experience for 320 frames... +[2023-03-07 15:43:50,244][282763] Decorrelating experience for 512 frames... +[2023-03-07 15:43:50,307][281803] Decorrelating experience for 576 frames... +[2023-03-07 15:43:50,317][284304] Decorrelating experience for 160 frames... +[2023-03-07 15:43:50,319][281810] Decorrelating experience for 512 frames... +[2023-03-07 15:43:50,348][281799] Decorrelating experience for 384 frames... +[2023-03-07 15:43:50,400][281584] Decorrelating experience for 704 frames... +[2023-03-07 15:43:50,404][281966] Decorrelating experience for 512 frames... +[2023-03-07 15:43:50,412][284047] Decorrelating experience for 416 frames... +[2023-03-07 15:43:50,431][281809] Decorrelating experience for 384 frames... +[2023-03-07 15:43:50,444][281962] Decorrelating experience for 416 frames... +[2023-03-07 15:43:50,460][281944] Decorrelating experience for 448 frames... +[2023-03-07 15:43:50,507][283396] Decorrelating experience for 448 frames... +[2023-03-07 15:43:50,512][282184] Decorrelating experience for 640 frames... +[2023-03-07 15:43:50,561][281801] Decorrelating experience for 448 frames... +[2023-03-07 15:43:50,588][283960] Decorrelating experience for 352 frames... +[2023-03-07 15:43:50,614][282904] Decorrelating experience for 192 frames... +[2023-03-07 15:43:50,615][283854] Decorrelating experience for 576 frames... +[2023-03-07 15:43:50,623][281663] Decorrelating experience for 736 frames... +[2023-03-07 15:43:50,651][281806] Decorrelating experience for 576 frames... +[2023-03-07 15:43:50,657][281961] Decorrelating experience for 640 frames... +[2023-03-07 15:43:50,696][284200] Decorrelating experience for 352 frames... +[2023-03-07 15:43:50,701][282891] Decorrelating experience for 288 frames... +[2023-03-07 15:43:50,702][281964] Decorrelating experience for 416 frames... +[2023-03-07 15:43:50,744][281991] Decorrelating experience for 544 frames... +[2023-03-07 15:43:50,804][281583] Decorrelating experience for 704 frames... +[2023-03-07 15:43:50,807][281655] Decorrelating experience for 352 frames... +[2023-03-07 15:43:50,853][281584] Decorrelating experience for 736 frames... +[2023-03-07 15:43:50,856][281965] Decorrelating experience for 704 frames... +[2023-03-07 15:43:50,865][281815] Decorrelating experience for 544 frames... +[2023-03-07 15:43:50,876][281960] Decorrelating experience for 608 frames... +[2023-03-07 15:43:50,908][281658] Decorrelating experience for 896 frames... +[2023-03-07 15:43:50,908][283084] Decorrelating experience for 672 frames... +[2023-03-07 15:43:50,933][282184] Decorrelating experience for 672 frames... +[2023-03-07 15:43:50,950][281804] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,057][283020] Decorrelating experience for 576 frames... +[2023-03-07 15:43:51,075][281576] Decorrelating experience for 640 frames... +[2023-03-07 15:43:51,076][281660] Decorrelating experience for 448 frames... +[2023-03-07 15:43:51,077][281667] Decorrelating experience for 256 frames... +[2023-03-07 15:43:51,082][282763] Decorrelating experience for 544 frames... +[2023-03-07 15:43:51,106][283278] Decorrelating experience for 672 frames... +[2023-03-07 15:43:51,106][281799] Decorrelating experience for 416 frames... +[2023-03-07 15:43:51,119][281950] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,143][281663] Decorrelating experience for 768 frames... +[2023-03-07 15:43:51,153][281973] Decorrelating experience for 672 frames... +[2023-03-07 15:43:51,270][281809] Decorrelating experience for 416 frames... +[2023-03-07 15:43:51,279][283396] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,283][281655] Decorrelating experience for 384 frames... +[2023-03-07 15:43:51,296][281960] Decorrelating experience for 640 frames... +[2023-03-07 15:43:51,308][282693] Decorrelating experience for 576 frames... +[2023-03-07 15:43:51,321][282283] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,327][281806] Decorrelating experience for 608 frames... +[2023-03-07 15:43:51,346][281964] Decorrelating experience for 448 frames... +[2023-03-07 15:43:51,356][284048] Decorrelating experience for 992 frames... +[2023-03-07 15:43:51,381][283960] Decorrelating experience for 384 frames... +[2023-03-07 15:43:51,422][287383] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:51,455][281955] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,493][282763] Decorrelating experience for 576 frames... +[2023-03-07 15:43:51,522][281653] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:51,533][284166] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,533][284047] Decorrelating experience for 448 frames... +[2023-03-07 15:43:51,533][281975] Decorrelating experience for 576 frames... +[2023-03-07 15:43:51,534][281803] Decorrelating experience for 608 frames... +[2023-03-07 15:43:51,559][282891] Decorrelating experience for 320 frames... +[2023-03-07 15:43:51,560][282698] Decorrelating experience for 672 frames... +[2023-03-07 15:43:51,573][282184] Decorrelating experience for 704 frames... +[2023-03-07 15:43:51,576][281799] Decorrelating experience for 448 frames... +[2023-03-07 15:43:51,658][283150] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,728][236315] Heartbeat connected on RolloutWorker_w109 +[2023-03-07 15:43:51,744][281801] Decorrelating experience for 480 frames... +[2023-03-07 15:43:51,749][281973] Decorrelating experience for 704 frames... +[2023-03-07 15:43:51,756][281815] Decorrelating experience for 576 frames... +[2023-03-07 15:43:51,768][283879] Decorrelating experience for 896 frames... +[2023-03-07 15:43:51,769][282693] Decorrelating experience for 608 frames... +[2023-03-07 15:43:51,776][281960] Decorrelating experience for 672 frames... +[2023-03-07 15:43:51,780][283396] Decorrelating experience for 512 frames... +[2023-03-07 15:43:51,789][283641] Decorrelating experience for 512 frames... +[2023-03-07 15:43:51,809][281663] Decorrelating experience for 800 frames... +[2023-03-07 15:43:51,854][281576] Decorrelating experience for 672 frames... +[2023-03-07 15:43:51,951][283278] Decorrelating experience for 704 frames... +[2023-03-07 15:43:51,967][281657] Decorrelating experience for 704 frames... +[2023-03-07 15:43:51,971][283020] Decorrelating experience for 608 frames... +[2023-03-07 15:43:51,984][281799] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,009][282698] Decorrelating experience for 704 frames... +[2023-03-07 15:43:52,012][281803] Decorrelating experience for 640 frames... +[2023-03-07 15:43:52,014][284166] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,045][281670] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,045][281946] Decorrelating experience for 736 frames... +[2023-03-07 15:43:52,056][281602] Decorrelating experience for 416 frames... +[2023-03-07 15:43:52,172][281801] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,173][281965] Decorrelating experience for 736 frames... +[2023-03-07 15:43:52,173][283352] Decorrelating experience for 576 frames... +[2023-03-07 15:43:52,187][282904] Decorrelating experience for 224 frames... +[2023-03-07 15:43:52,212][281968] Decorrelating experience for 640 frames... +[2023-03-07 15:43:52,236][282693] Decorrelating experience for 640 frames... +[2023-03-07 15:43:52,256][281583] Decorrelating experience for 736 frames... +[2023-03-07 15:43:52,295][281990] Decorrelating experience for 576 frames... +[2023-03-07 15:43:52,298][281576] Decorrelating experience for 704 frames... +[2023-03-07 15:43:52,301][283408] Decorrelating experience for 544 frames... +[2023-03-07 15:43:52,371][281976] Decorrelating experience for 896 frames... +[2023-03-07 15:43:52,371][283281] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,414][281989] Decorrelating experience for 576 frames... +[2023-03-07 15:43:52,445][283879] Decorrelating experience for 928 frames... +[2023-03-07 15:43:52,458][281955] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,464][283641] Decorrelating experience for 544 frames... +[2023-03-07 15:43:52,466][281660] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,484][283020] Decorrelating experience for 640 frames... +[2023-03-07 15:43:52,499][282283] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,510][283278] Decorrelating experience for 736 frames... +[2023-03-07 15:43:52,578][281959] Decorrelating experience for 192 frames... +[2023-03-07 15:43:52,580][282574] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:52,582][281670] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,615][281580] Decorrelating experience for 576 frames... +[2023-03-07 15:43:52,677][281944] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,678][281582] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,679][282904] Decorrelating experience for 256 frames... +[2023-03-07 15:43:52,690][281803] Decorrelating experience for 672 frames... +[2023-03-07 15:43:52,720][281799] Decorrelating experience for 512 frames... +[2023-03-07 15:43:52,720][281649] Decorrelating experience for 736 frames... +[2023-03-07 15:43:52,724][281969] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,769][282695] Decorrelating experience for 480 frames... +[2023-03-07 15:43:52,777][281667] Decorrelating experience for 288 frames... +[2023-03-07 15:43:52,817][281578] Decorrelating experience for 576 frames... +[2023-03-07 15:43:52,831][281805] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:52,871][284309] Decorrelating experience for 768 frames... +[2023-03-07 15:43:52,874][282693] Decorrelating experience for 672 frames... +[2023-03-07 15:43:52,903][281988] Decorrelating experience for 704 frames... +[2023-03-07 15:43:52,918][283020] Decorrelating experience for 672 frames... +[2023-03-07 15:43:52,943][282283] Decorrelating experience for 544 frames... +[2023-03-07 15:43:52,943][281989] Decorrelating experience for 608 frames... +[2023-03-07 15:43:52,985][283084] Decorrelating experience for 704 frames... +[2023-03-07 15:43:53,018][281993] Decorrelating experience for 672 frames... +[2023-03-07 15:43:53,040][281655] Decorrelating experience for 416 frames... +[2023-03-07 15:43:53,044][281661] Decorrelating experience for 352 frames... +[2023-03-07 15:43:53,068][282904] Decorrelating experience for 288 frames... +[2023-03-07 15:43:53,091][283150] Decorrelating experience for 512 frames... +[2023-03-07 15:43:53,103][281582] Decorrelating experience for 544 frames... +[2023-03-07 15:43:53,134][283277] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:53,137][281656] Decorrelating experience for 544 frames... +[2023-03-07 15:43:53,144][281944] Decorrelating experience for 512 frames... +[2023-03-07 15:43:53,174][283879] Decorrelating experience for 960 frames... +[2023-03-07 15:43:53,175][281667] Decorrelating experience for 320 frames... +[2023-03-07 15:43:53,223][283417] Decorrelating experience for 672 frames... +[2023-03-07 15:43:53,246][281961] Decorrelating experience for 672 frames... +[2023-03-07 15:43:53,271][281578] Decorrelating experience for 608 frames... +[2023-03-07 15:43:53,281][282891] Decorrelating experience for 352 frames... +[2023-03-07 15:43:53,294][281955] Decorrelating experience for 544 frames... +[2023-03-07 15:43:53,301][281665] Decorrelating experience for 480 frames... +[2023-03-07 15:43:53,331][281575] Decorrelating experience for 352 frames... +[2023-03-07 15:43:53,357][284309] Decorrelating experience for 800 frames... +[2023-03-07 15:43:53,382][284200] Decorrelating experience for 384 frames... +[2023-03-07 15:43:53,387][281649] Decorrelating experience for 768 frames... +[2023-03-07 15:43:53,414][282695] Decorrelating experience for 512 frames... +[2023-03-07 15:43:53,449][281959] Decorrelating experience for 224 frames... +[2023-03-07 15:43:53,478][281966] Decorrelating experience for 544 frames... +[2023-03-07 15:43:53,480][283150] Decorrelating experience for 544 frames... +[2023-03-07 15:43:53,498][281969] Decorrelating experience for 512 frames... +[2023-03-07 15:43:53,522][283019] Decorrelating experience for 416 frames... +[2023-03-07 15:43:53,537][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4009984. Throughput: 0: 19.2. Samples: 864. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:43:53,538][236315] Avg episode reward: [(0, '1.214')] +[2023-03-07 15:43:53,561][283020] Decorrelating experience for 704 frames... +[2023-03-07 15:43:53,570][283352] Decorrelating experience for 608 frames... +[2023-03-07 15:43:53,610][281576] Decorrelating experience for 736 frames... +[2023-03-07 15:43:53,650][283084] Decorrelating experience for 736 frames... +[2023-03-07 15:43:53,650][282283] Decorrelating experience for 576 frames... +[2023-03-07 15:43:53,658][284047] Decorrelating experience for 480 frames... +[2023-03-07 15:43:53,675][283469] Decorrelating experience for 224 frames... +[2023-03-07 15:43:53,690][281660] Decorrelating experience for 512 frames... +[2023-03-07 15:43:53,690][282763] Decorrelating experience for 608 frames... +[2023-03-07 15:43:53,724][281962] Decorrelating experience for 448 frames... +[2023-03-07 15:43:53,737][284345] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:53,754][281575] Decorrelating experience for 384 frames... +[2023-03-07 15:43:53,765][283641] Decorrelating experience for 576 frames... +[2023-03-07 15:43:53,800][282698] Decorrelating experience for 736 frames... +[2023-03-07 15:43:53,827][281427] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000980_4014080.pth... +[2023-03-07 15:43:53,827][281427] Stopping Batcher_0... +[2023-03-07 15:43:53,835][236315] Component Batcher_0 stopped! +[2023-03-07 15:43:53,843][281427] Loop batcher_evt_loop terminating... +[2023-03-07 15:43:53,853][281946] Decorrelating experience for 768 frames... +[2023-03-07 15:43:53,867][281427] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000980_4014080.pth... +[2023-03-07 15:43:53,870][283879] Decorrelating experience for 992 frames... +[2023-03-07 15:43:53,888][281662] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:53,894][281574] Weights refcount: 2 0 +[2023-03-07 15:43:53,895][281574] Stopping InferenceWorker_p0-w0... +[2023-03-07 15:43:53,896][281574] Loop inference_proc0-0_evt_loop terminating... +[2023-03-07 15:43:53,896][236315] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 15:43:53,903][281649] Decorrelating experience for 800 frames... +[2023-03-07 15:43:53,917][283408] Decorrelating experience for 576 frames... +[2023-03-07 15:43:53,920][281990] Decorrelating experience for 608 frames... +[2023-03-07 15:43:53,924][281654] Decorrelating experience for 448 frames... +[2023-03-07 15:43:53,930][284048] Stopping RolloutWorker_w109... +[2023-03-07 15:43:53,931][284048] Loop rollout_proc109_evt_loop terminating... +[2023-03-07 15:43:53,931][236315] Component RolloutWorker_w109 stopped! +[2023-03-07 15:43:53,960][281577] Decorrelating experience for 416 frames... +[2023-03-07 15:43:53,961][281427] Stopping LearnerWorker_p0... +[2023-03-07 15:43:53,961][281989] Decorrelating experience for 640 frames... +[2023-03-07 15:43:53,961][281427] Loop learner_proc0_evt_loop terminating... +[2023-03-07 15:43:53,961][236315] Component LearnerWorker_p0 stopped! +[2023-03-07 15:43:53,964][281815] Decorrelating experience for 608 frames... +[2023-03-07 15:43:54,057][283469] Decorrelating experience for 256 frames... +[2023-03-07 15:43:54,065][283281] Decorrelating experience for 512 frames... +[2023-03-07 15:43:54,077][281945] Decorrelating experience for 736 frames... +[2023-03-07 15:43:54,116][282891] Decorrelating experience for 384 frames... +[2023-03-07 15:43:54,118][281801] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,130][281661] Decorrelating experience for 384 frames... +[2023-03-07 15:43:54,170][282693] Decorrelating experience for 704 frames... +[2023-03-07 15:43:54,183][281988] Decorrelating experience for 736 frames... +[2023-03-07 15:43:54,183][281959] Decorrelating experience for 256 frames... +[2023-03-07 15:43:54,259][284309] Decorrelating experience for 832 frames... +[2023-03-07 15:43:54,262][281976] Decorrelating experience for 928 frames... +[2023-03-07 15:43:54,268][283084] Decorrelating experience for 768 frames... +[2023-03-07 15:43:54,313][281584] Decorrelating experience for 768 frames... +[2023-03-07 15:43:54,316][281955] Decorrelating experience for 576 frames... +[2023-03-07 15:43:54,339][283879] Stopping RolloutWorker_w97... +[2023-03-07 15:43:54,339][283879] Loop rollout_proc97_evt_loop terminating... +[2023-03-07 15:43:54,340][236315] Component RolloutWorker_w97 stopped! +[2023-03-07 15:43:54,348][283396] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,376][283020] Decorrelating experience for 736 frames... +[2023-03-07 15:43:54,378][281670] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,410][282565] Decorrelating experience for 640 frames... +[2023-03-07 15:43:54,449][281575] Decorrelating experience for 416 frames... +[2023-03-07 15:43:54,465][284166] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,504][281968] Decorrelating experience for 672 frames... +[2023-03-07 15:43:54,523][281664] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,537][281578] Decorrelating experience for 640 frames... +[2023-03-07 15:43:54,575][281655] Decorrelating experience for 448 frames... +[2023-03-07 15:43:54,579][283019] Decorrelating experience for 448 frames... +[2023-03-07 15:43:54,582][281950] Decorrelating experience for 512 frames... +[2023-03-07 15:43:54,598][283469] Decorrelating experience for 288 frames... +[2023-03-07 15:43:54,642][281803] Decorrelating experience for 704 frames... +[2023-03-07 15:43:54,668][284200] Decorrelating experience for 416 frames... +[2023-03-07 15:43:54,699][281948] Decorrelating experience for 320 frames... +[2023-03-07 15:43:54,728][282283] Decorrelating experience for 608 frames... +[2023-03-07 15:43:54,733][283281] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,777][281656] Decorrelating experience for 576 frames... +[2023-03-07 15:43:54,787][281654] Decorrelating experience for 480 frames... +[2023-03-07 15:43:54,791][281575] Decorrelating experience for 448 frames... +[2023-03-07 15:43:54,851][282904] Decorrelating experience for 320 frames... +[2023-03-07 15:43:54,852][281984] Decorrelating experience for 544 frames... +[2023-03-07 15:43:54,857][282693] Decorrelating experience for 736 frames... +[2023-03-07 15:43:54,870][281959] Decorrelating experience for 288 frames... +[2023-03-07 15:43:54,927][281657] Decorrelating experience for 736 frames... +[2023-03-07 15:43:54,972][281988] Decorrelating experience for 768 frames... +[2023-03-07 15:43:54,986][281962] Decorrelating experience for 480 frames... +[2023-03-07 15:43:54,993][282763] Decorrelating experience for 640 frames... +[2023-03-07 15:43:55,005][281948] Decorrelating experience for 352 frames... +[2023-03-07 15:43:55,009][281655] Decorrelating experience for 480 frames... +[2023-03-07 15:43:55,036][281804] Decorrelating experience for 512 frames... +[2023-03-07 15:43:55,044][283352] Decorrelating experience for 640 frames... +[2023-03-07 15:43:55,074][281670] Decorrelating experience for 576 frames... +[2023-03-07 15:43:55,119][281664] Decorrelating experience for 576 frames... +[2023-03-07 15:43:55,128][282891] Decorrelating experience for 416 frames... +[2023-03-07 15:43:55,163][282694] Decorrelating experience for 480 frames... +[2023-03-07 15:43:55,193][284047] Decorrelating experience for 512 frames... +[2023-03-07 15:43:55,202][283469] Decorrelating experience for 320 frames... +[2023-03-07 15:43:55,207][281945] Decorrelating experience for 768 frames... +[2023-03-07 15:43:55,236][284304] Decorrelating experience for 192 frames... +[2023-03-07 15:43:55,276][283417] Decorrelating experience for 704 frames... +[2023-03-07 15:43:55,277][282904] Decorrelating experience for 352 frames... +[2023-03-07 15:43:55,311][281577] Decorrelating experience for 448 frames... +[2023-03-07 15:43:55,328][281959] Decorrelating experience for 320 frames... +[2023-03-07 15:43:55,356][281805] Decorrelating experience for 320 frames... +[2023-03-07 15:43:55,395][283019] Decorrelating experience for 480 frames... +[2023-03-07 15:43:55,398][281657] Decorrelating experience for 768 frames... +[2023-03-07 15:43:55,414][284166] Decorrelating experience for 576 frames... +[2023-03-07 15:43:55,439][281583] Decorrelating experience for 768 frames... +[2023-03-07 15:43:55,473][284309] Decorrelating experience for 864 frames... +[2023-03-07 15:43:55,473][283352] Decorrelating experience for 672 frames... +[2023-03-07 15:43:55,504][281961] Decorrelating experience for 704 frames... +[2023-03-07 15:43:55,515][281806] Decorrelating experience for 640 frames... +[2023-03-07 15:43:55,537][281655] Decorrelating experience for 512 frames... +[2023-03-07 15:43:55,555][281575] Decorrelating experience for 480 frames... +[2023-03-07 15:43:55,601][281989] Decorrelating experience for 672 frames... +[2023-03-07 15:43:55,617][283469] Decorrelating experience for 352 frames... +[2023-03-07 15:43:55,645][281660] Decorrelating experience for 544 frames... +[2023-03-07 15:43:55,657][282698] Decorrelating experience for 768 frames... +[2023-03-07 15:43:55,682][281602] Decorrelating experience for 448 frames... +[2023-03-07 15:43:55,689][281948] Decorrelating experience for 384 frames... +[2023-03-07 15:43:55,692][282699] Decorrelating experience for 768 frames... +[2023-03-07 15:43:55,743][282694] Decorrelating experience for 512 frames... +[2023-03-07 15:43:55,754][281803] Decorrelating experience for 736 frames... +[2023-03-07 15:43:55,773][281813] Decorrelating experience for 416 frames... +[2023-03-07 15:43:55,803][284200] Decorrelating experience for 448 frames... +[2023-03-07 15:43:55,866][281664] Decorrelating experience for 608 frames... +[2023-03-07 15:43:55,876][281805] Decorrelating experience for 352 frames... +[2023-03-07 15:43:55,877][282565] Decorrelating experience for 672 frames... +[2023-03-07 15:43:55,879][281657] Decorrelating experience for 800 frames... +[2023-03-07 15:43:55,903][283084] Decorrelating experience for 800 frames... +[2023-03-07 15:43:55,962][284047] Decorrelating experience for 544 frames... +[2023-03-07 15:43:55,963][281955] Decorrelating experience for 608 frames... +[2023-03-07 15:43:55,966][281967] Decorrelating experience for 480 frames... +[2023-03-07 15:43:55,975][281945] Decorrelating experience for 800 frames... +[2023-03-07 15:43:55,995][281806] Decorrelating experience for 672 frames... +[2023-03-07 15:43:56,054][281988] Decorrelating experience for 800 frames... +[2023-03-07 15:43:56,093][281602] Decorrelating experience for 480 frames... +[2023-03-07 15:43:56,093][284309] Decorrelating experience for 896 frames... +[2023-03-07 15:43:56,106][281661] Decorrelating experience for 416 frames... +[2023-03-07 15:43:56,187][281670] Decorrelating experience for 608 frames... +[2023-03-07 15:43:56,189][282183] Decorrelating experience for 480 frames... +[2023-03-07 15:43:56,190][283417] Decorrelating experience for 736 frames... +[2023-03-07 15:43:56,190][281809] Decorrelating experience for 448 frames... +[2023-03-07 15:43:56,196][283469] Decorrelating experience for 384 frames... +[2023-03-07 15:43:56,199][282891] Decorrelating experience for 448 frames... +[2023-03-07 15:43:56,248][281805] Decorrelating experience for 384 frames... +[2023-03-07 15:43:56,310][284200] Decorrelating experience for 480 frames... +[2023-03-07 15:43:56,361][281950] Decorrelating experience for 544 frames... +[2023-03-07 15:43:56,397][281955] Decorrelating experience for 640 frames... +[2023-03-07 15:43:56,411][282698] Decorrelating experience for 800 frames... +[2023-03-07 15:43:56,411][283150] Decorrelating experience for 576 frames... +[2023-03-07 15:43:56,412][281580] Decorrelating experience for 608 frames... +[2023-03-07 15:43:56,412][281660] Decorrelating experience for 576 frames... +[2023-03-07 15:43:56,415][281804] Decorrelating experience for 544 frames... +[2023-03-07 15:43:56,459][282763] Decorrelating experience for 672 frames... +[2023-03-07 15:43:56,462][282565] Decorrelating experience for 704 frames... +[2023-03-07 15:43:56,600][281661] Decorrelating experience for 448 frames... +[2023-03-07 15:43:56,634][281809] Decorrelating experience for 480 frames... +[2023-03-07 15:43:56,635][282183] Decorrelating experience for 512 frames... +[2023-03-07 15:43:56,635][283085] Decorrelating experience for 320 frames... +[2023-03-07 15:43:56,636][282693] Decorrelating experience for 768 frames... +[2023-03-07 15:43:56,636][281946] Decorrelating experience for 800 frames... +[2023-03-07 15:43:56,636][281584] Decorrelating experience for 800 frames... +[2023-03-07 15:43:56,662][284047] Decorrelating experience for 576 frames... +[2023-03-07 15:43:56,725][281602] Decorrelating experience for 512 frames... +[2023-03-07 15:43:56,808][281656] Decorrelating experience for 608 frames... +[2023-03-07 15:43:56,836][281580] Decorrelating experience for 640 frames... +[2023-03-07 15:43:56,850][281803] Decorrelating experience for 768 frames... +[2023-03-07 15:43:56,851][281665] Decorrelating experience for 512 frames... +[2023-03-07 15:43:56,851][283469] Decorrelating experience for 416 frames... +[2023-03-07 15:43:56,855][283417] Decorrelating experience for 768 frames... +[2023-03-07 15:43:56,912][282698] Decorrelating experience for 832 frames... +[2023-03-07 15:43:56,918][281664] Decorrelating experience for 640 frames... +[2023-03-07 15:43:56,928][281583] Decorrelating experience for 800 frames... +[2023-03-07 15:43:56,974][282694] Decorrelating experience for 544 frames... +[2023-03-07 15:43:57,013][281974] Decorrelating experience for 576 frames... +[2023-03-07 15:43:57,027][281956] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:43:57,029][282699] Decorrelating experience for 800 frames... +[2023-03-07 15:43:57,053][283084] Decorrelating experience for 832 frames... +[2023-03-07 15:43:57,059][281968] Decorrelating experience for 704 frames... +[2023-03-07 15:43:57,061][282565] Decorrelating experience for 736 frames... +[2023-03-07 15:43:57,109][281661] Decorrelating experience for 480 frames... +[2023-03-07 15:43:57,123][281577] Decorrelating experience for 480 frames... +[2023-03-07 15:43:57,125][281813] Decorrelating experience for 448 frames... +[2023-03-07 15:43:57,206][281948] Decorrelating experience for 416 frames... +[2023-03-07 15:43:57,213][281952] Decorrelating experience for 384 frames... +[2023-03-07 15:43:57,266][281602] Decorrelating experience for 544 frames... +[2023-03-07 15:43:57,270][281657] Decorrelating experience for 832 frames... +[2023-03-07 15:43:57,276][283020] Decorrelating experience for 768 frames... +[2023-03-07 15:43:57,293][281955] Decorrelating experience for 672 frames... +[2023-03-07 15:43:57,318][283085] Decorrelating experience for 352 frames... +[2023-03-07 15:43:57,378][281958] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:43:57,379][284304] Decorrelating experience for 224 frames... +[2023-03-07 15:43:57,401][283408] Decorrelating experience for 608 frames... +[2023-03-07 15:43:57,426][282558] Decorrelating experience for 480 frames... +[2023-03-07 15:43:57,461][281805] Decorrelating experience for 416 frames... +[2023-03-07 15:43:57,465][281804] Decorrelating experience for 576 frames... +[2023-03-07 15:43:57,480][281576] Decorrelating experience for 768 frames... +[2023-03-07 15:43:57,487][281664] Decorrelating experience for 672 frames... +[2023-03-07 15:43:57,546][281967] Decorrelating experience for 512 frames... +[2023-03-07 15:43:57,566][281661] Decorrelating experience for 512 frames... +[2023-03-07 15:43:57,587][283396] Decorrelating experience for 576 frames... +[2023-03-07 15:43:57,590][281960] Decorrelating experience for 704 frames... +[2023-03-07 15:43:57,613][281799] Decorrelating experience for 544 frames... +[2023-03-07 15:43:57,654][282763] Decorrelating experience for 704 frames... +[2023-03-07 15:43:57,660][281602] Decorrelating experience for 576 frames... +[2023-03-07 15:43:57,670][281976] Decorrelating experience for 960 frames... +[2023-03-07 15:43:57,732][281584] Decorrelating experience for 832 frames... +[2023-03-07 15:43:57,753][284304] Decorrelating experience for 256 frames... +[2023-03-07 15:43:57,778][281813] Decorrelating experience for 480 frames... +[2023-03-07 15:43:57,781][281577] Decorrelating experience for 512 frames... +[2023-03-07 15:43:57,803][281805] Decorrelating experience for 448 frames... +[2023-03-07 15:43:57,851][283469] Decorrelating experience for 448 frames... +[2023-03-07 15:43:57,856][282699] Decorrelating experience for 832 frames... +[2023-03-07 15:43:57,881][281953] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 2 +[2023-03-07 15:43:57,921][281654] Decorrelating experience for 512 frames... +[2023-03-07 15:43:57,926][281952] Decorrelating experience for 416 frames... +[2023-03-07 15:43:57,972][281804] Decorrelating experience for 608 frames... +[2023-03-07 15:43:57,974][281652] Decorrelating experience for 864 frames... +[2023-03-07 15:43:57,981][283084] Decorrelating experience for 864 frames... +[2023-03-07 15:43:57,989][281575] Decorrelating experience for 512 frames... +[2023-03-07 15:43:58,058][281799] Decorrelating experience for 576 frames... +[2023-03-07 15:43:58,067][281655] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,119][281966] Decorrelating experience for 576 frames... +[2023-03-07 15:43:58,123][283396] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,179][283352] Decorrelating experience for 704 frames... +[2023-03-07 15:43:58,183][282698] Decorrelating experience for 864 frames... +[2023-03-07 15:43:58,203][281805] Decorrelating experience for 480 frames... +[2023-03-07 15:43:58,222][281584] Decorrelating experience for 864 frames... +[2023-03-07 15:43:58,259][281577] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,260][281989] Decorrelating experience for 704 frames... +[2023-03-07 15:43:58,300][283469] Decorrelating experience for 480 frames... +[2023-03-07 15:43:58,373][281653] Decorrelating experience for 256 frames... +[2023-03-07 15:43:58,388][283150] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,388][281602] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,425][281944] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,454][281655] Decorrelating experience for 576 frames... +[2023-03-07 15:43:58,470][282699] Decorrelating experience for 864 frames... +[2023-03-07 15:43:58,470][281649] Decorrelating experience for 832 frames... +[2023-03-07 15:43:58,508][281952] Decorrelating experience for 448 frames... +[2023-03-07 15:43:58,509][281799] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,583][281948] Decorrelating experience for 448 frames... +[2023-03-07 15:43:58,584][282183] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,611][287383] Decorrelating experience for 224 frames... +[2023-03-07 15:43:58,667][281805] Decorrelating experience for 512 frames... +[2023-03-07 15:43:58,670][281575] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,681][284053] Decorrelating experience for 544 frames... +[2023-03-07 15:43:58,704][281968] Decorrelating experience for 736 frames... +[2023-03-07 15:43:58,721][281966] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,738][282698] Decorrelating experience for 896 frames... +[2023-03-07 15:43:58,778][282283] Decorrelating experience for 640 frames... +[2023-03-07 15:43:58,778][281815] Decorrelating experience for 640 frames... +[2023-03-07 15:43:58,798][281803] Decorrelating experience for 800 frames... +[2023-03-07 15:43:58,874][281972] Decorrelating experience for 480 frames... +[2023-03-07 15:43:58,877][281660] Decorrelating experience for 608 frames... +[2023-03-07 15:43:58,898][281960] Decorrelating experience for 736 frames... +[2023-03-07 15:43:58,954][283396] Decorrelating experience for 640 frames... +[2023-03-07 15:43:58,964][281944] Decorrelating experience for 576 frames... +[2023-03-07 15:43:58,968][281602] Decorrelating experience for 640 frames... +[2023-03-07 15:43:58,978][282904] Decorrelating experience for 384 frames... +[2023-03-07 15:43:58,984][283019] Decorrelating experience for 512 frames... +[2023-03-07 15:43:59,068][281649] Decorrelating experience for 864 frames... +[2023-03-07 15:43:59,069][281967] Decorrelating experience for 544 frames... +[2023-03-07 15:43:59,096][281955] Decorrelating experience for 704 frames... +[2023-03-07 15:43:59,140][281966] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,162][281575] Decorrelating experience for 576 frames... +[2023-03-07 15:43:59,167][281653] Decorrelating experience for 288 frames... +[2023-03-07 15:43:59,217][282283] Decorrelating experience for 672 frames... +[2023-03-07 15:43:59,218][281576] Decorrelating experience for 800 frames... +[2023-03-07 15:43:59,221][281805] Decorrelating experience for 544 frames... +[2023-03-07 15:43:59,240][281951] Decorrelating experience for 544 frames... +[2023-03-07 15:43:59,257][281658] Decorrelating experience for 928 frames... +[2023-03-07 15:43:59,278][281584] Decorrelating experience for 896 frames... +[2023-03-07 15:43:59,285][281664] Decorrelating experience for 704 frames... +[2023-03-07 15:43:59,357][281654] Decorrelating experience for 544 frames... +[2023-03-07 15:43:59,438][281813] Decorrelating experience for 512 frames... +[2023-03-07 15:43:59,438][283019] Decorrelating experience for 544 frames... +[2023-03-07 15:43:59,439][281944] Decorrelating experience for 608 frames... +[2023-03-07 15:43:59,444][281968] Decorrelating experience for 768 frames... +[2023-03-07 15:43:59,448][281662] Decorrelating experience for 512 frames... +[2023-03-07 15:43:59,449][284053] Decorrelating experience for 576 frames... +[2023-03-07 15:43:59,451][281950] Decorrelating experience for 576 frames... +[2023-03-07 15:43:59,476][283150] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,494][281962] Decorrelating experience for 512 frames... +[2023-03-07 15:43:59,548][281955] Decorrelating experience for 736 frames... +[2023-03-07 15:43:59,642][282183] Decorrelating experience for 576 frames... +[2023-03-07 15:43:59,648][281651] Decorrelating experience for 288 frames... +[2023-03-07 15:43:59,649][281807] Decorrelating experience for 704 frames... +[2023-03-07 15:43:59,655][281660] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,684][281988] Decorrelating experience for 832 frames... +[2023-03-07 15:43:59,703][281578] Decorrelating experience for 672 frames... +[2023-03-07 15:43:59,726][281602] Decorrelating experience for 672 frames... +[2023-03-07 15:43:59,742][281664] Decorrelating experience for 736 frames... +[2023-03-07 15:43:59,747][282699] Decorrelating experience for 896 frames... +[2023-03-07 15:43:59,748][281804] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,843][281954] Decorrelating experience for 224 frames... +[2023-03-07 15:43:59,849][283641] Decorrelating experience for 608 frames... +[2023-03-07 15:43:59,852][281576] Decorrelating experience for 832 frames... +[2023-03-07 15:43:59,885][281803] Decorrelating experience for 832 frames... +[2023-03-07 15:43:59,898][283408] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,929][281799] Decorrelating experience for 640 frames... +[2023-03-07 15:43:59,972][283352] Decorrelating experience for 736 frames... +[2023-03-07 15:43:59,976][281945] Decorrelating experience for 832 frames... +[2023-03-07 15:43:59,991][281654] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,024][281962] Decorrelating experience for 544 frames... +[2023-03-07 15:44:00,045][281582] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,081][281951] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,101][282184] Decorrelating experience for 736 frames... +[2023-03-07 15:44:00,174][282283] Decorrelating experience for 704 frames... +[2023-03-07 15:44:00,174][282698] Decorrelating experience for 928 frames... +[2023-03-07 15:44:00,178][281972] Decorrelating experience for 512 frames... +[2023-03-07 15:44:00,181][281966] Decorrelating experience for 672 frames... +[2023-03-07 15:44:00,194][281602] Decorrelating experience for 704 frames... +[2023-03-07 15:44:00,217][281575] Decorrelating experience for 608 frames... +[2023-03-07 15:44:00,234][284345] Decorrelating experience for 448 frames... +[2023-03-07 15:44:00,292][284309] Decorrelating experience for 928 frames... +[2023-03-07 15:44:00,385][282904] Decorrelating experience for 416 frames... +[2023-03-07 15:44:00,391][281967] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,391][283408] Decorrelating experience for 672 frames... +[2023-03-07 15:44:00,418][281955] Decorrelating experience for 768 frames... +[2023-03-07 15:44:00,419][281583] Decorrelating experience for 832 frames... +[2023-03-07 15:44:00,427][281989] Decorrelating experience for 736 frames... +[2023-03-07 15:44:00,504][281576] Decorrelating experience for 864 frames... +[2023-03-07 15:44:00,529][281969] Decorrelating experience for 544 frames... +[2023-03-07 15:44:00,581][281962] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,617][282565] Decorrelating experience for 768 frames... +[2023-03-07 15:44:00,617][284053] Decorrelating experience for 608 frames... +[2023-03-07 15:44:00,638][282184] Decorrelating experience for 768 frames... +[2023-03-07 15:44:00,639][281813] Decorrelating experience for 544 frames... +[2023-03-07 15:44:00,648][281988] Decorrelating experience for 864 frames... +[2023-03-07 15:44:00,651][281802] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:00,683][281655] Decorrelating experience for 608 frames... +[2023-03-07 15:44:00,729][281805] Decorrelating experience for 576 frames... +[2023-03-07 15:44:00,774][281652] Decorrelating experience for 896 frames... +[2023-03-07 15:44:00,817][281584] Decorrelating experience for 928 frames... +[2023-03-07 15:44:00,821][281602] Decorrelating experience for 736 frames... +[2023-03-07 15:44:00,833][281804] Decorrelating experience for 672 frames... +[2023-03-07 15:44:00,848][281575] Decorrelating experience for 640 frames... +[2023-03-07 15:44:00,850][283408] Decorrelating experience for 704 frames... +[2023-03-07 15:44:00,875][281653] Decorrelating experience for 320 frames... +[2023-03-07 15:44:00,924][281583] Decorrelating experience for 864 frames... +[2023-03-07 15:44:00,940][282698] Decorrelating experience for 960 frames... +[2023-03-07 15:44:00,967][281577] Decorrelating experience for 576 frames... +[2023-03-07 15:44:01,005][281656] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,031][281969] Decorrelating experience for 576 frames... +[2023-03-07 15:44:01,035][282699] Decorrelating experience for 928 frames... +[2023-03-07 15:44:01,052][281962] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,081][281954] Decorrelating experience for 256 frames... +[2023-03-07 15:44:01,117][282565] Decorrelating experience for 800 frames... +[2023-03-07 15:44:01,120][281578] Decorrelating experience for 704 frames... +[2023-03-07 15:44:01,120][281992] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,162][281582] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,196][281579] Decorrelating experience for 512 frames... +[2023-03-07 15:44:01,235][281813] Decorrelating experience for 576 frames... +[2023-03-07 15:44:01,236][281670] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,280][282184] Decorrelating experience for 800 frames... +[2023-03-07 15:44:01,280][281655] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,311][284200] Decorrelating experience for 512 frames... +[2023-03-07 15:44:01,311][281944] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,360][281951] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,363][283890] Decorrelating experience for 768 frames... +[2023-03-07 15:44:01,388][281968] Decorrelating experience for 800 frames... +[2023-03-07 15:44:01,449][281969] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,450][282283] Decorrelating experience for 736 frames... +[2023-03-07 15:44:01,481][281576] Decorrelating experience for 896 frames... +[2023-03-07 15:44:01,487][283641] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,490][281584] Decorrelating experience for 960 frames... +[2023-03-07 15:44:01,511][281971] Decorrelating experience for 704 frames... +[2023-03-07 15:44:01,512][281989] Decorrelating experience for 768 frames... +[2023-03-07 15:44:01,590][282891] Decorrelating experience for 480 frames... +[2023-03-07 15:44:01,590][281577] Decorrelating experience for 608 frames... +[2023-03-07 15:44:01,593][281963] Decorrelating experience for 448 frames... +[2023-03-07 15:44:01,645][283281] Decorrelating experience for 576 frames... +[2023-03-07 15:44:01,692][281960] Decorrelating experience for 768 frames... +[2023-03-07 15:44:01,693][281582] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,723][284200] Decorrelating experience for 544 frames... +[2023-03-07 15:44:01,724][281955] Decorrelating experience for 800 frames... +[2023-03-07 15:44:01,727][284304] Decorrelating experience for 288 frames... +[2023-03-07 15:44:01,794][283020] Decorrelating experience for 800 frames... +[2023-03-07 15:44:01,798][281579] Decorrelating experience for 544 frames... +[2023-03-07 15:44:01,805][281951] Decorrelating experience for 640 frames... +[2023-03-07 15:44:01,811][282698] Decorrelating experience for 992 frames... +[2023-03-07 15:44:01,890][281575] Decorrelating experience for 672 frames... +[2023-03-07 15:44:01,892][281653] Decorrelating experience for 352 frames... +[2023-03-07 15:44:01,921][281809] Decorrelating experience for 512 frames... +[2023-03-07 15:44:01,925][281649] Decorrelating experience for 896 frames... +[2023-03-07 15:44:01,985][281813] Decorrelating experience for 608 frames... +[2023-03-07 15:44:02,014][282565] Decorrelating experience for 832 frames... +[2023-03-07 15:44:02,015][283727] Decorrelating experience for 320 frames... +[2023-03-07 15:44:02,018][281577] Decorrelating experience for 640 frames... +[2023-03-07 15:44:02,092][281972] Decorrelating experience for 544 frames... +[2023-03-07 15:44:02,108][281602] Decorrelating experience for 768 frames... +[2023-03-07 15:44:02,113][283469] Decorrelating experience for 512 frames... +[2023-03-07 15:44:02,122][281799] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,176][284304] Decorrelating experience for 320 frames... +[2023-03-07 15:44:02,216][283890] Decorrelating experience for 800 frames... +[2023-03-07 15:44:02,254][281953] Decorrelating experience for 160 frames... +[2023-03-07 15:44:02,283][282184] Decorrelating experience for 832 frames... +[2023-03-07 15:44:02,284][281805] Decorrelating experience for 608 frames... +[2023-03-07 15:44:02,309][281663] Decorrelating experience for 832 frames... +[2023-03-07 15:44:02,312][281951] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,313][281658] Decorrelating experience for 960 frames... +[2023-03-07 15:44:02,369][281971] Decorrelating experience for 736 frames... +[2023-03-07 15:44:02,426][282698] Stopping RolloutWorker_w101... +[2023-03-07 15:44:02,426][282698] Loop rollout_proc101_evt_loop terminating... +[2023-03-07 15:44:02,427][236315] Component RolloutWorker_w101 stopped! +[2023-03-07 15:44:02,455][281976] Decorrelating experience for 992 frames... +[2023-03-07 15:44:02,460][281649] Decorrelating experience for 928 frames... +[2023-03-07 15:44:02,462][284200] Decorrelating experience for 576 frames... +[2023-03-07 15:44:02,482][281582] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,483][281993] Decorrelating experience for 704 frames... +[2023-03-07 15:44:02,511][281953] Decorrelating experience for 192 frames... +[2023-03-07 15:44:02,512][281990] Decorrelating experience for 640 frames... +[2023-03-07 15:44:02,527][283020] Decorrelating experience for 832 frames... +[2023-03-07 15:44:02,563][281800] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:02,566][282565] Decorrelating experience for 864 frames... +[2023-03-07 15:44:02,570][281669] Decorrelating experience for 288 frames... +[2023-03-07 15:44:02,624][281581] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:02,650][282574] Decorrelating experience for 608 frames... +[2023-03-07 15:44:02,676][281963] Decorrelating experience for 480 frames... +[2023-03-07 15:44:02,679][281580] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,713][281974] Decorrelating experience for 608 frames... +[2023-03-07 15:44:02,720][281813] Decorrelating experience for 640 frames... +[2023-03-07 15:44:02,740][281972] Decorrelating experience for 576 frames... +[2023-03-07 15:44:02,755][283281] Decorrelating experience for 608 frames... +[2023-03-07 15:44:02,765][281988] Decorrelating experience for 896 frames... +[2023-03-07 15:44:02,768][282693] Decorrelating experience for 800 frames... +[2023-03-07 15:44:02,843][281803] Decorrelating experience for 864 frames... +[2023-03-07 15:44:02,878][281658] Decorrelating experience for 992 frames... +[2023-03-07 15:44:02,884][281652] Decorrelating experience for 928 frames... +[2023-03-07 15:44:02,884][281578] Decorrelating experience for 736 frames... +[2023-03-07 15:44:02,917][283150] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,930][281976] Stopping RolloutWorker_w78... +[2023-03-07 15:44:02,930][281976] Loop rollout_proc78_evt_loop terminating... +[2023-03-07 15:44:02,930][236315] Component RolloutWorker_w78 stopped! +[2023-03-07 15:44:02,951][283469] Decorrelating experience for 544 frames... +[2023-03-07 15:44:02,960][282697] Decorrelating experience for 288 frames... +[2023-03-07 15:44:02,966][281577] Decorrelating experience for 672 frames... +[2023-03-07 15:44:02,996][281669] Decorrelating experience for 320 frames... +[2023-03-07 15:44:03,040][283020] Decorrelating experience for 864 frames... +[2023-03-07 15:44:03,041][282558] Decorrelating experience for 512 frames... +[2023-03-07 15:44:03,082][281799] Decorrelating experience for 704 frames... +[2023-03-07 15:44:03,096][281954] Decorrelating experience for 288 frames... +[2023-03-07 15:44:03,110][287383] Decorrelating experience for 256 frames... +[2023-03-07 15:44:03,155][281813] Decorrelating experience for 672 frames... +[2023-03-07 15:44:03,173][284200] Decorrelating experience for 608 frames... +[2023-03-07 15:44:03,197][281960] Decorrelating experience for 800 frames... +[2023-03-07 15:44:03,228][281992] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,231][281661] Decorrelating experience for 544 frames... +[2023-03-07 15:44:03,256][281602] Decorrelating experience for 800 frames... +[2023-03-07 15:44:03,276][282574] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,282][281580] Decorrelating experience for 704 frames... +[2023-03-07 15:44:03,286][281670] Decorrelating experience for 672 frames... +[2023-03-07 15:44:03,302][281989] Decorrelating experience for 800 frames... +[2023-03-07 15:44:03,361][281803] Decorrelating experience for 896 frames... +[2023-03-07 15:44:03,394][281579] Decorrelating experience for 576 frames... +[2023-03-07 15:44:03,401][281578] Decorrelating experience for 768 frames... +[2023-03-07 15:44:03,420][281581] Decorrelating experience for 608 frames... +[2023-03-07 15:44:03,425][281952] Decorrelating experience for 480 frames... +[2023-03-07 15:44:03,448][281584] Decorrelating experience for 992 frames... +[2023-03-07 15:44:03,483][281944] Decorrelating experience for 672 frames... +[2023-03-07 15:44:03,488][281658] Stopping RolloutWorker_w21... +[2023-03-07 15:44:03,488][281658] Loop rollout_proc21_evt_loop terminating... +[2023-03-07 15:44:03,488][236315] Component RolloutWorker_w21 stopped! +[2023-03-07 15:44:03,495][281664] Decorrelating experience for 768 frames... +[2023-03-07 15:44:03,511][282565] Decorrelating experience for 896 frames... +[2023-03-07 15:44:03,544][281799] Decorrelating experience for 736 frames... +[2023-03-07 15:44:03,589][281652] Decorrelating experience for 960 frames... +[2023-03-07 15:44:03,594][281974] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,618][281577] Decorrelating experience for 704 frames... +[2023-03-07 15:44:03,642][283281] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,664][282695] Decorrelating experience for 544 frames... +[2023-03-07 15:44:03,692][282693] Decorrelating experience for 832 frames... +[2023-03-07 15:44:03,705][284053] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,713][282697] Decorrelating experience for 320 frames... +[2023-03-07 15:44:03,756][281661] Decorrelating experience for 576 frames... +[2023-03-07 15:44:03,795][281960] Decorrelating experience for 832 frames... +[2023-03-07 15:44:03,809][282574] Decorrelating experience for 672 frames... +[2023-03-07 15:44:03,834][283150] Decorrelating experience for 704 frames... +[2023-03-07 15:44:03,838][282558] Decorrelating experience for 544 frames... +[2023-03-07 15:44:03,855][281951] Decorrelating experience for 704 frames... +[2023-03-07 15:44:03,886][281969] Decorrelating experience for 640 frames... +[2023-03-07 15:44:03,903][281669] Decorrelating experience for 352 frames... +[2023-03-07 15:44:03,908][281584] Stopping RolloutWorker_w29... +[2023-03-07 15:44:03,908][281584] Loop rollout_proc29_evt_loop terminating... +[2023-03-07 15:44:03,908][236315] Component RolloutWorker_w29 stopped! +[2023-03-07 15:44:03,919][281946] Decorrelating experience for 832 frames... +[2023-03-07 15:44:03,964][284304] Decorrelating experience for 352 frames... +[2023-03-07 15:44:03,998][281989] Decorrelating experience for 832 frames... +[2023-03-07 15:44:04,043][281803] Decorrelating experience for 928 frames... +[2023-03-07 15:44:04,050][282694] Decorrelating experience for 576 frames... +[2023-03-07 15:44:04,060][281993] Decorrelating experience for 736 frames... +[2023-03-07 15:44:04,080][281952] Decorrelating experience for 512 frames... +[2023-03-07 15:44:04,093][281580] Decorrelating experience for 736 frames... +[2023-03-07 15:44:04,122][281959] Decorrelating experience for 352 frames... +[2023-03-07 15:44:04,151][281967] Decorrelating experience for 608 frames... +[2023-03-07 15:44:04,178][282695] Decorrelating experience for 576 frames... +[2023-03-07 15:44:04,219][281664] Decorrelating experience for 800 frames... +[2023-03-07 15:44:04,229][283469] Decorrelating experience for 576 frames... +[2023-03-07 15:44:04,253][281581] Decorrelating experience for 640 frames... +[2023-03-07 15:44:04,254][282184] Decorrelating experience for 864 frames... +[2023-03-07 15:44:04,258][281990] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,310][283019] Decorrelating experience for 576 frames... +[2023-03-07 15:44:04,315][281969] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,324][281948] Decorrelating experience for 480 frames... +[2023-03-07 15:44:04,337][281815] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,435][281974] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,446][281655] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,458][281602] Decorrelating experience for 832 frames... +[2023-03-07 15:44:04,467][281954] Decorrelating experience for 320 frames... +[2023-03-07 15:44:04,488][281945] Decorrelating experience for 864 frames... +[2023-03-07 15:44:04,502][281959] Decorrelating experience for 384 frames... +[2023-03-07 15:44:04,535][283084] Decorrelating experience for 896 frames... +[2023-03-07 15:44:04,535][283281] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,578][281970] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 2 +[2023-03-07 15:44:04,635][281946] Decorrelating experience for 864 frames... +[2023-03-07 15:44:04,662][283020] Decorrelating experience for 896 frames... +[2023-03-07 15:44:04,684][281657] Decorrelating experience for 864 frames... +[2023-03-07 15:44:04,723][282694] Decorrelating experience for 608 frames... +[2023-03-07 15:44:04,734][281990] Decorrelating experience for 704 frames... +[2023-03-07 15:44:04,736][283396] Decorrelating experience for 672 frames... +[2023-03-07 15:44:04,739][283277] Decorrelating experience for 416 frames... +[2023-03-07 15:44:04,739][282697] Decorrelating experience for 352 frames... +[2023-03-07 15:44:04,776][281815] Decorrelating experience for 704 frames... +[2023-03-07 15:44:04,841][281952] Decorrelating experience for 544 frames... +[2023-03-07 15:44:04,867][281969] Decorrelating experience for 704 frames... +[2023-03-07 15:44:04,877][281653] Decorrelating experience for 384 frames... +[2023-03-07 15:44:04,885][281954] Decorrelating experience for 352 frames... +[2023-03-07 15:44:04,918][281951] Decorrelating experience for 736 frames... +[2023-03-07 15:44:04,932][281576] Decorrelating experience for 928 frames... +[2023-03-07 15:44:04,941][281959] Decorrelating experience for 416 frames... +[2023-03-07 15:44:04,953][281955] Decorrelating experience for 832 frames... +[2023-03-07 15:44:05,016][281670] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,036][281974] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,070][281662] Decorrelating experience for 544 frames... +[2023-03-07 15:44:05,078][281602] Decorrelating experience for 864 frames... +[2023-03-07 15:44:05,112][282697] Decorrelating experience for 384 frames... +[2023-03-07 15:44:05,128][281970] Decorrelating experience for 288 frames... +[2023-03-07 15:44:05,146][281582] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,155][281948] Decorrelating experience for 512 frames... +[2023-03-07 15:44:05,191][281809] Decorrelating experience for 544 frames... +[2023-03-07 15:44:05,219][282694] Decorrelating experience for 640 frames... +[2023-03-07 15:44:05,241][281952] Decorrelating experience for 576 frames... +[2023-03-07 15:44:05,242][282184] Decorrelating experience for 896 frames... +[2023-03-07 15:44:05,273][281580] Decorrelating experience for 768 frames... +[2023-03-07 15:44:05,274][281656] Decorrelating experience for 672 frames... +[2023-03-07 15:44:05,331][284166] Decorrelating experience for 608 frames... +[2023-03-07 15:44:05,344][283020] Decorrelating experience for 928 frames... +[2023-03-07 15:44:05,387][281651] Decorrelating experience for 320 frames... +[2023-03-07 15:44:05,388][283084] Decorrelating experience for 928 frames... +[2023-03-07 15:44:05,391][283085] Decorrelating experience for 384 frames... +[2023-03-07 15:44:05,482][281960] Decorrelating experience for 864 frames... +[2023-03-07 15:44:05,498][281576] Decorrelating experience for 960 frames... +[2023-03-07 15:44:05,499][281799] Decorrelating experience for 768 frames... +[2023-03-07 15:44:05,499][282565] Decorrelating experience for 928 frames... +[2023-03-07 15:44:05,502][281649] Decorrelating experience for 960 frames... +[2023-03-07 15:44:05,532][281950] Decorrelating experience for 608 frames... +[2023-03-07 15:44:05,546][281974] Decorrelating experience for 736 frames... +[2023-03-07 15:44:05,593][283417] Decorrelating experience for 800 frames... +[2023-03-07 15:44:05,594][288232] Decorrelating experience for 384 frames... +[2023-03-07 15:44:05,648][283281] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,672][281969] Decorrelating experience for 736 frames... +[2023-03-07 15:44:05,698][282695] Decorrelating experience for 608 frames... +[2023-03-07 15:44:05,701][282558] Decorrelating experience for 576 frames... +[2023-03-07 15:44:05,765][283396] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,769][281951] Decorrelating experience for 768 frames... +[2023-03-07 15:44:05,795][283469] Decorrelating experience for 608 frames... +[2023-03-07 15:44:05,796][283890] Decorrelating experience for 832 frames... +[2023-03-07 15:44:05,803][281805] Decorrelating experience for 640 frames... +[2023-03-07 15:44:05,865][281813] Decorrelating experience for 704 frames... +[2023-03-07 15:44:05,899][281651] Decorrelating experience for 352 frames... +[2023-03-07 15:44:05,930][281948] Decorrelating experience for 544 frames... +[2023-03-07 15:44:05,953][281972] Decorrelating experience for 608 frames... +[2023-03-07 15:44:05,963][281950] Decorrelating experience for 640 frames... +[2023-03-07 15:44:05,999][281991] Decorrelating experience for 576 frames... +[2023-03-07 15:44:06,000][282696] Decorrelating experience for 576 frames... +[2023-03-07 15:44:06,004][281960] Decorrelating experience for 896 frames... +[2023-03-07 15:44:06,047][281583] Decorrelating experience for 896 frames... +[2023-03-07 15:44:06,064][281656] Decorrelating experience for 704 frames... +[2023-03-07 15:44:06,064][282183] Decorrelating experience for 608 frames... +[2023-03-07 15:44:06,101][282891] Decorrelating experience for 512 frames... +[2023-03-07 15:44:06,157][281580] Decorrelating experience for 800 frames... +[2023-03-07 15:44:06,164][281581] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,181][288232] Decorrelating experience for 416 frames... +[2023-03-07 15:44:06,204][281975] Decorrelating experience for 608 frames... +[2023-03-07 15:44:06,205][281993] Decorrelating experience for 768 frames... +[2023-03-07 15:44:06,238][281970] Decorrelating experience for 320 frames... +[2023-03-07 15:44:06,264][281575] Decorrelating experience for 704 frames... +[2023-03-07 15:44:06,265][281602] Decorrelating experience for 896 frames... +[2023-03-07 15:44:06,305][282283] Decorrelating experience for 768 frames... +[2023-03-07 15:44:06,366][281805] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,379][281992] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,382][281799] Decorrelating experience for 800 frames... +[2023-03-07 15:44:06,401][281944] Decorrelating experience for 704 frames... +[2023-03-07 15:44:06,406][281660] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,436][281967] Decorrelating experience for 640 frames... +[2023-03-07 15:44:06,468][283408] Decorrelating experience for 736 frames... +[2023-03-07 15:44:06,472][283469] Decorrelating experience for 640 frames... +[2023-03-07 15:44:06,490][281808] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:06,506][282697] Decorrelating experience for 416 frames... +[2023-03-07 15:44:06,562][281991] Decorrelating experience for 608 frames... +[2023-03-07 15:44:06,574][282695] Decorrelating experience for 640 frames... +[2023-03-07 15:44:06,586][281950] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,603][281948] Decorrelating experience for 576 frames... +[2023-03-07 15:44:06,604][281959] Decorrelating experience for 448 frames... +[2023-03-07 15:44:06,661][282693] Decorrelating experience for 864 frames... +[2023-03-07 15:44:06,674][281653] Decorrelating experience for 416 frames... +[2023-03-07 15:44:06,687][281583] Decorrelating experience for 928 frames... +[2023-03-07 15:44:06,694][283084] Decorrelating experience for 960 frames... +[2023-03-07 15:44:06,706][281962] Decorrelating experience for 640 frames... +[2023-03-07 15:44:06,760][282696] Decorrelating experience for 608 frames... +[2023-03-07 15:44:06,784][288232] Decorrelating experience for 448 frames... +[2023-03-07 15:44:06,802][281577] Decorrelating experience for 736 frames... +[2023-03-07 15:44:06,805][283641] Decorrelating experience for 672 frames... +[2023-03-07 15:44:06,855][283417] Decorrelating experience for 832 frames... +[2023-03-07 15:44:06,855][281580] Decorrelating experience for 832 frames... +[2023-03-07 15:44:06,907][281660] Decorrelating experience for 704 frames... +[2023-03-07 15:44:06,931][282574] Decorrelating experience for 704 frames... +[2023-03-07 15:44:06,949][281576] Decorrelating experience for 992 frames... +[2023-03-07 15:44:06,965][282283] Decorrelating experience for 800 frames... +[2023-03-07 15:44:06,989][281960] Decorrelating experience for 928 frames... +[2023-03-07 15:44:06,992][281602] Decorrelating experience for 928 frames... +[2023-03-07 15:44:07,028][281954] Decorrelating experience for 384 frames... +[2023-03-07 15:44:07,062][281944] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,072][281972] Decorrelating experience for 640 frames... +[2023-03-07 15:44:07,117][282183] Decorrelating experience for 640 frames... +[2023-03-07 15:44:07,144][288232] Decorrelating experience for 480 frames... +[2023-03-07 15:44:07,194][281950] Decorrelating experience for 704 frames... +[2023-03-07 15:44:07,197][281651] Decorrelating experience for 384 frames... +[2023-03-07 15:44:07,205][282695] Decorrelating experience for 672 frames... +[2023-03-07 15:44:07,223][281649] Decorrelating experience for 992 frames... +[2023-03-07 15:44:07,226][283277] Decorrelating experience for 448 frames... +[2023-03-07 15:44:07,236][283396] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,268][281989] Decorrelating experience for 864 frames... +[2023-03-07 15:44:07,322][281945] Decorrelating experience for 896 frames... +[2023-03-07 15:44:07,374][281803] Decorrelating experience for 960 frames... +[2023-03-07 15:44:07,378][282693] Decorrelating experience for 896 frames... +[2023-03-07 15:44:07,394][281970] Decorrelating experience for 352 frames... +[2023-03-07 15:44:07,410][283020] Decorrelating experience for 960 frames... +[2023-03-07 15:44:07,410][283641] Decorrelating experience for 704 frames... +[2023-03-07 15:44:07,417][281959] Decorrelating experience for 480 frames... +[2023-03-07 15:44:07,427][281655] Decorrelating experience for 704 frames... +[2023-03-07 15:44:07,455][282827] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:07,502][283281] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,546][281807] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,553][281661] Decorrelating experience for 608 frames... +[2023-03-07 15:44:07,600][281576] Stopping RolloutWorker_w2... +[2023-03-07 15:44:07,601][281576] Loop rollout_proc2_evt_loop terminating... +[2023-03-07 15:44:07,600][236315] Component RolloutWorker_w2 stopped! +[2023-03-07 15:44:07,608][281667] Decorrelating experience for 352 frames... +[2023-03-07 15:44:07,609][281993] Decorrelating experience for 800 frames... +[2023-03-07 15:44:07,627][288232] Decorrelating experience for 512 frames... +[2023-03-07 15:44:07,662][281582] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,663][284053] Decorrelating experience for 672 frames... +[2023-03-07 15:44:07,663][281954] Decorrelating experience for 416 frames... +[2023-03-07 15:44:07,685][281972] Decorrelating experience for 672 frames... +[2023-03-07 15:44:07,695][281960] Decorrelating experience for 960 frames... +[2023-03-07 15:44:07,706][281957] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 2 +[2023-03-07 15:44:07,757][281809] Decorrelating experience for 576 frames... +[2023-03-07 15:44:07,758][283277] Decorrelating experience for 480 frames... +[2023-03-07 15:44:07,804][281815] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,851][281649] Stopping RolloutWorker_w35... +[2023-03-07 15:44:07,851][281649] Loop rollout_proc35_evt_loop terminating... +[2023-03-07 15:44:07,851][236315] Component RolloutWorker_w35 stopped! +[2023-03-07 15:44:07,858][282283] Decorrelating experience for 832 frames... +[2023-03-07 15:44:07,859][281989] Decorrelating experience for 896 frames... +[2023-03-07 15:44:07,880][281950] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,885][283641] Decorrelating experience for 736 frames... +[2023-03-07 15:44:07,893][281959] Decorrelating experience for 512 frames... +[2023-03-07 15:44:07,896][281974] Decorrelating experience for 768 frames... +[2023-03-07 15:44:07,909][281801] Decorrelating experience for 576 frames... +[2023-03-07 15:44:07,961][283149] Decorrelating experience for 608 frames... +[2023-03-07 15:44:07,962][281944] Decorrelating experience for 768 frames... +[2023-03-07 15:44:08,003][282565] Decorrelating experience for 960 frames... +[2023-03-07 15:44:08,073][281954] Decorrelating experience for 448 frames... +[2023-03-07 15:44:08,095][281803] Decorrelating experience for 992 frames... +[2023-03-07 15:44:08,109][281655] Decorrelating experience for 736 frames... +[2023-03-07 15:44:08,123][281580] Decorrelating experience for 864 frames... +[2023-03-07 15:44:08,124][281955] Decorrelating experience for 864 frames... +[2023-03-07 15:44:08,124][284053] Decorrelating experience for 704 frames... +[2023-03-07 15:44:08,159][281948] Decorrelating experience for 608 frames... +[2023-03-07 15:44:08,199][282558] Decorrelating experience for 608 frames... +[2023-03-07 15:44:08,209][282763] Decorrelating experience for 736 frames... +[2023-03-07 15:44:08,228][283417] Decorrelating experience for 864 frames... +[2023-03-07 15:44:08,286][281959] Decorrelating experience for 544 frames... +[2023-03-07 15:44:08,300][281813] Decorrelating experience for 736 frames... +[2023-03-07 15:44:08,323][282693] Decorrelating experience for 928 frames... +[2023-03-07 15:44:08,334][283281] Decorrelating experience for 768 frames... +[2023-03-07 15:44:08,343][283277] Decorrelating experience for 512 frames... +[2023-03-07 15:44:08,358][281583] Decorrelating experience for 960 frames... +[2023-03-07 15:44:08,362][283890] Decorrelating experience for 864 frames... +[2023-03-07 15:44:08,406][281665] Decorrelating experience for 544 frames... +[2023-03-07 15:44:08,410][281971] Decorrelating experience for 768 frames... +[2023-03-07 15:44:08,434][281970] Decorrelating experience for 384 frames... +[2023-03-07 15:44:08,494][283149] Decorrelating experience for 640 frames... +[2023-03-07 15:44:08,509][281652] Decorrelating experience for 992 frames... +[2023-03-07 15:44:08,522][281801] Decorrelating experience for 608 frames... +[2023-03-07 15:44:08,543][281954] Decorrelating experience for 480 frames... +[2023-03-07 15:44:08,559][281975] Decorrelating experience for 640 frames... +[2023-03-07 15:44:08,559][283469] Decorrelating experience for 672 frames... +[2023-03-07 15:44:08,571][281803] Stopping RolloutWorker_w10... +[2023-03-07 15:44:08,572][281803] Loop rollout_proc10_evt_loop terminating... +[2023-03-07 15:44:08,571][236315] Component RolloutWorker_w10 stopped! +[2023-03-07 15:44:08,578][281950] Decorrelating experience for 768 frames... +[2023-03-07 15:44:08,610][281802] Decorrelating experience for 512 frames... +[2023-03-07 15:44:08,611][282699] Decorrelating experience for 960 frames... +[2023-03-07 15:44:08,655][281815] Decorrelating experience for 768 frames... +[2023-03-07 15:44:08,708][282697] Decorrelating experience for 448 frames... +[2023-03-07 15:44:08,765][281948] Decorrelating experience for 640 frames... +[2023-03-07 15:44:08,766][283277] Decorrelating experience for 544 frames... +[2023-03-07 15:44:08,780][281989] Decorrelating experience for 928 frames... +[2023-03-07 15:44:08,781][281955] Decorrelating experience for 896 frames... +[2023-03-07 15:44:08,781][281799] Decorrelating experience for 832 frames... +[2023-03-07 15:44:08,787][281970] Decorrelating experience for 416 frames... +[2023-03-07 15:44:08,817][281966] Decorrelating experience for 704 frames... +[2023-03-07 15:44:08,818][283150] Decorrelating experience for 736 frames... +[2023-03-07 15:44:08,852][281946] Decorrelating experience for 896 frames... +[2023-03-07 15:44:08,906][281653] Decorrelating experience for 448 frames... +[2023-03-07 15:44:08,982][281652] Stopping RolloutWorker_w38... +[2023-03-07 15:44:08,982][281652] Loop rollout_proc38_evt_loop terminating... +[2023-03-07 15:44:08,982][236315] Component RolloutWorker_w38 stopped! +[2023-03-07 15:44:08,996][281583] Decorrelating experience for 992 frames... +[2023-03-07 15:44:08,997][282283] Decorrelating experience for 864 frames... +[2023-03-07 15:44:08,997][283281] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,000][282693] Decorrelating experience for 960 frames... +[2023-03-07 15:44:09,018][283727] Decorrelating experience for 352 frames... +[2023-03-07 15:44:09,019][281957] Decorrelating experience for 352 frames... +[2023-03-07 15:44:09,049][281944] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,054][281971] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,071][283417] Decorrelating experience for 896 frames... +[2023-03-07 15:44:09,097][281801] Decorrelating experience for 640 frames... +[2023-03-07 15:44:09,192][283149] Decorrelating experience for 672 frames... +[2023-03-07 15:44:09,202][281802] Decorrelating experience for 544 frames... +[2023-03-07 15:44:09,216][281810] Decorrelating experience for 544 frames... +[2023-03-07 15:44:09,220][281665] Decorrelating experience for 576 frames... +[2023-03-07 15:44:09,228][281580] Decorrelating experience for 896 frames... +[2023-03-07 15:44:09,252][281959] Decorrelating experience for 576 frames... +[2023-03-07 15:44:09,271][282827] Decorrelating experience for 640 frames... +[2023-03-07 15:44:09,275][281950] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,284][281815] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,289][281974] Decorrelating experience for 800 frames... +[2023-03-07 15:44:09,398][282565] Decorrelating experience for 992 frames... +[2023-03-07 15:44:09,411][284345] Decorrelating experience for 480 frames... +[2023-03-07 15:44:09,458][281583] Stopping RolloutWorker_w26... +[2023-03-07 15:44:09,458][281583] Loop rollout_proc26_evt_loop terminating... +[2023-03-07 15:44:09,458][236315] Component RolloutWorker_w26 stopped! +[2023-03-07 15:44:09,474][281657] Decorrelating experience for 896 frames... +[2023-03-07 15:44:09,527][281945] Decorrelating experience for 928 frames... +[2023-03-07 15:44:09,530][281582] Decorrelating experience for 768 frames... +[2023-03-07 15:44:09,530][281970] Decorrelating experience for 448 frames... +[2023-03-07 15:44:09,531][281660] Decorrelating experience for 736 frames... +[2023-03-07 15:44:09,531][282763] Decorrelating experience for 768 frames... +[2023-03-07 15:44:09,533][281990] Decorrelating experience for 736 frames... +[2023-03-07 15:44:09,538][281966] Decorrelating experience for 736 frames... +[2023-03-07 15:44:09,598][282699] Decorrelating experience for 992 frames... +[2023-03-07 15:44:09,602][281802] Decorrelating experience for 576 frames... +[2023-03-07 15:44:09,670][281653] Decorrelating experience for 480 frames... +[2023-03-07 15:44:09,724][282697] Decorrelating experience for 480 frames... +[2023-03-07 15:44:09,734][281993] Decorrelating experience for 832 frames... +[2023-03-07 15:44:09,750][281989] Decorrelating experience for 960 frames... +[2023-03-07 15:44:09,756][281946] Decorrelating experience for 928 frames... +[2023-03-07 15:44:09,757][281957] Decorrelating experience for 384 frames... +[2023-03-07 15:44:09,801][284304] Decorrelating experience for 384 frames... +[2023-03-07 15:44:09,804][281967] Decorrelating experience for 672 frames... +[2023-03-07 15:44:09,815][281971] Decorrelating experience for 832 frames... +[2023-03-07 15:44:09,833][282693] Decorrelating experience for 992 frames... +[2023-03-07 15:44:09,863][283278] Decorrelating experience for 768 frames... +[2023-03-07 15:44:09,888][282565] Stopping RolloutWorker_w52... +[2023-03-07 15:44:09,888][282565] Loop rollout_proc52_evt_loop terminating... +[2023-03-07 15:44:09,888][236315] Component RolloutWorker_w52 stopped! +[2023-03-07 15:44:09,925][283417] Decorrelating experience for 928 frames... +[2023-03-07 15:44:09,939][283277] Decorrelating experience for 576 frames... +[2023-03-07 15:44:09,948][283469] Decorrelating experience for 704 frames... +[2023-03-07 15:44:09,977][283641] Decorrelating experience for 768 frames... +[2023-03-07 15:44:09,980][281950] Decorrelating experience for 832 frames... +[2023-03-07 15:44:09,999][281661] Decorrelating experience for 640 frames... +[2023-03-07 15:44:10,026][281964] Decorrelating experience for 480 frames... +[2023-03-07 15:44:10,030][284166] Decorrelating experience for 640 frames... +[2023-03-07 15:44:10,040][281665] Decorrelating experience for 608 frames... +[2023-03-07 15:44:10,078][282827] Decorrelating experience for 672 frames... +[2023-03-07 15:44:10,095][282699] Stopping RolloutWorker_w104... +[2023-03-07 15:44:10,095][282699] Loop rollout_proc104_evt_loop terminating... +[2023-03-07 15:44:10,096][236315] Component RolloutWorker_w104 stopped! +[2023-03-07 15:44:10,157][281801] Decorrelating experience for 672 frames... +[2023-03-07 15:44:10,180][281810] Decorrelating experience for 576 frames... +[2023-03-07 15:44:10,183][282763] Decorrelating experience for 800 frames... +[2023-03-07 15:44:10,211][281970] Decorrelating experience for 480 frames... +[2023-03-07 15:44:10,215][281948] Decorrelating experience for 672 frames... +[2023-03-07 15:44:10,220][281944] Decorrelating experience for 832 frames... +[2023-03-07 15:44:10,223][281962] Decorrelating experience for 672 frames... +[2023-03-07 15:44:10,244][281974] Decorrelating experience for 832 frames... +[2023-03-07 15:44:10,273][281807] Decorrelating experience for 768 frames... +[2023-03-07 15:44:10,274][281582] Decorrelating experience for 800 frames... +[2023-03-07 15:44:10,312][282693] Stopping RolloutWorker_w88... +[2023-03-07 15:44:10,313][282693] Loop rollout_proc88_evt_loop terminating... +[2023-03-07 15:44:10,313][236315] Component RolloutWorker_w88 stopped! +[2023-03-07 15:44:10,384][281945] Decorrelating experience for 960 frames... +[2023-03-07 15:44:10,414][281579] Decorrelating experience for 608 frames... +[2023-03-07 15:44:10,447][281969] Decorrelating experience for 768 frames... +[2023-03-07 15:44:10,453][283277] Decorrelating experience for 608 frames... +[2023-03-07 15:44:10,453][281802] Decorrelating experience for 608 frames... +[2023-03-07 15:44:10,453][284200] Decorrelating experience for 640 frames... +[2023-03-07 15:44:10,457][281993] Decorrelating experience for 864 frames... +[2023-03-07 15:44:10,457][282697] Decorrelating experience for 512 frames... +[2023-03-07 15:44:10,470][281808] Decorrelating experience for 512 frames... +[2023-03-07 15:44:10,473][281663] Decorrelating experience for 864 frames... +[2023-03-07 15:44:10,607][281801] Decorrelating experience for 704 frames... +[2023-03-07 15:44:10,628][281815] Decorrelating experience for 832 frames... +[2023-03-07 15:44:10,647][281813] Decorrelating experience for 768 frames... +[2023-03-07 15:44:10,650][281972] Decorrelating experience for 704 frames... +[2023-03-07 15:44:10,667][281946] Decorrelating experience for 960 frames... +[2023-03-07 15:44:10,673][281953] Decorrelating experience for 224 frames... +[2023-03-07 15:44:10,681][283890] Decorrelating experience for 896 frames... +[2023-03-07 15:44:10,687][282827] Decorrelating experience for 704 frames... +[2023-03-07 15:44:10,692][283417] Decorrelating experience for 960 frames... +[2023-03-07 15:44:10,703][283396] Decorrelating experience for 768 frames... +[2023-03-07 15:44:10,802][281799] Decorrelating experience for 864 frames... +[2023-03-07 15:44:10,818][281950] Decorrelating experience for 864 frames... +[2023-03-07 15:44:10,868][283469] Decorrelating experience for 736 frames... +[2023-03-07 15:44:10,877][281802] Decorrelating experience for 640 frames... +[2023-03-07 15:44:10,899][284200] Decorrelating experience for 672 frames... +[2023-03-07 15:44:10,903][284345] Decorrelating experience for 512 frames... +[2023-03-07 15:44:10,911][281964] Decorrelating experience for 512 frames... +[2023-03-07 15:44:10,916][287383] Decorrelating experience for 288 frames... +[2023-03-07 15:44:10,926][283854] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:44:10,933][281948] Decorrelating experience for 704 frames... +[2023-03-07 15:44:10,999][281953] Decorrelating experience for 256 frames... +[2023-03-07 15:44:11,066][281665] Decorrelating experience for 640 frames... +[2023-03-07 15:44:11,081][281969] Decorrelating experience for 800 frames... +[2023-03-07 15:44:11,087][281810] Decorrelating experience for 608 frames... +[2023-03-07 15:44:11,137][281808] Decorrelating experience for 544 frames... +[2023-03-07 15:44:11,138][283278] Decorrelating experience for 800 frames... +[2023-03-07 15:44:11,139][281801] Decorrelating experience for 736 frames... +[2023-03-07 15:44:11,139][282558] Decorrelating experience for 640 frames... +[2023-03-07 15:44:11,191][288232] Decorrelating experience for 544 frames... +[2023-03-07 15:44:11,194][281581] Decorrelating experience for 704 frames... +[2023-03-07 15:44:11,274][282694] Decorrelating experience for 672 frames... +[2023-03-07 15:44:11,277][281944] Decorrelating experience for 864 frames... +[2023-03-07 15:44:11,292][281953] Decorrelating experience for 288 frames... +[2023-03-07 15:44:11,299][281960] Decorrelating experience for 992 frames... +[2023-03-07 15:44:11,358][281799] Decorrelating experience for 896 frames... +[2023-03-07 15:44:11,359][281577] Decorrelating experience for 768 frames... +[2023-03-07 15:44:11,360][281806] Decorrelating experience for 704 frames... +[2023-03-07 15:44:11,360][283641] Decorrelating experience for 800 frames... +[2023-03-07 15:44:11,386][284166] Decorrelating experience for 672 frames... +[2023-03-07 15:44:11,420][281954] Decorrelating experience for 512 frames... +[2023-03-07 15:44:11,474][282891] Decorrelating experience for 544 frames... +[2023-03-07 15:44:11,486][283417] Decorrelating experience for 992 frames... +[2023-03-07 15:44:11,566][283149] Decorrelating experience for 704 frames... +[2023-03-07 15:44:11,566][281810] Decorrelating experience for 640 frames... +[2023-03-07 15:44:11,581][283281] Decorrelating experience for 832 frames... +[2023-03-07 15:44:11,582][283960] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:11,596][281582] Decorrelating experience for 832 frames... +[2023-03-07 15:44:11,597][281662] Decorrelating experience for 576 frames... +[2023-03-07 15:44:11,601][283396] Decorrelating experience for 800 frames... +[2023-03-07 15:44:11,602][281953] Decorrelating experience for 320 frames... +[2023-03-07 15:44:11,620][281957] Decorrelating experience for 416 frames... +[2023-03-07 15:44:11,676][281951] Decorrelating experience for 800 frames... +[2023-03-07 15:44:11,695][281660] Decorrelating experience for 768 frames... +[2023-03-07 15:44:11,748][281960] Stopping RolloutWorker_w56... +[2023-03-07 15:44:11,748][281960] Loop rollout_proc56_evt_loop terminating... +[2023-03-07 15:44:11,748][236315] Component RolloutWorker_w56 stopped! +[2023-03-07 15:44:11,767][283150] Decorrelating experience for 768 frames... +[2023-03-07 15:44:11,777][281657] Decorrelating experience for 928 frames... +[2023-03-07 15:44:11,782][282574] Decorrelating experience for 736 frames... +[2023-03-07 15:44:11,820][281802] Decorrelating experience for 672 frames... +[2023-03-07 15:44:11,820][281966] Decorrelating experience for 768 frames... +[2023-03-07 15:44:11,860][283727] Decorrelating experience for 384 frames... +[2023-03-07 15:44:11,869][281950] Decorrelating experience for 896 frames... +[2023-03-07 15:44:11,876][281651] Decorrelating experience for 416 frames... +[2023-03-07 15:44:11,899][281806] Decorrelating experience for 736 frames... +[2023-03-07 15:44:11,902][284200] Decorrelating experience for 704 frames... +[2023-03-07 15:44:11,955][283417] Stopping RolloutWorker_w111... +[2023-03-07 15:44:11,956][283417] Loop rollout_proc111_evt_loop terminating... +[2023-03-07 15:44:11,956][236315] Component RolloutWorker_w111 stopped! +[2023-03-07 15:44:11,978][281989] Decorrelating experience for 992 frames... +[2023-03-07 15:44:12,035][281973] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:44:12,038][283149] Decorrelating experience for 736 frames... +[2023-03-07 15:44:12,039][281964] Decorrelating experience for 544 frames... +[2023-03-07 15:44:12,042][281667] Decorrelating experience for 384 frames... +[2023-03-07 15:44:12,047][283890] Decorrelating experience for 928 frames... +[2023-03-07 15:44:12,099][281954] Decorrelating experience for 544 frames... +[2023-03-07 15:44:12,127][281953] Decorrelating experience for 352 frames... +[2023-03-07 15:44:12,141][281580] Decorrelating experience for 928 frames... +[2023-03-07 15:44:12,165][281653] Decorrelating experience for 512 frames... +[2023-03-07 15:44:12,182][281660] Decorrelating experience for 800 frames... +[2023-03-07 15:44:12,186][281962] Decorrelating experience for 704 frames... +[2023-03-07 15:44:12,230][282763] Decorrelating experience for 832 frames... +[2023-03-07 15:44:12,246][288232] Decorrelating experience for 576 frames... +[2023-03-07 15:44:12,253][281993] Decorrelating experience for 896 frames... +[2023-03-07 15:44:12,316][281661] Decorrelating experience for 672 frames... +[2023-03-07 15:44:12,319][283727] Decorrelating experience for 416 frames... +[2023-03-07 15:44:12,324][282694] Decorrelating experience for 704 frames... +[2023-03-07 15:44:12,332][284047] Decorrelating experience for 608 frames... +[2023-03-07 15:44:12,382][281651] Decorrelating experience for 448 frames... +[2023-03-07 15:44:12,385][284345] Decorrelating experience for 544 frames... +[2023-03-07 15:44:12,426][281657] Decorrelating experience for 960 frames... +[2023-03-07 15:44:12,444][283278] Decorrelating experience for 832 frames... +[2023-03-07 15:44:12,451][281989] Stopping RolloutWorker_w73... +[2023-03-07 15:44:12,452][281989] Loop rollout_proc73_evt_loop terminating... +[2023-03-07 15:44:12,452][236315] Component RolloutWorker_w73 stopped! +[2023-03-07 15:44:12,462][287383] Decorrelating experience for 320 frames... +[2023-03-07 15:44:12,488][281965] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:12,517][281662] Decorrelating experience for 608 frames... +[2023-03-07 15:44:12,521][282695] Decorrelating experience for 704 frames... +[2023-03-07 15:44:12,536][281948] Decorrelating experience for 736 frames... +[2023-03-07 15:44:12,543][282183] Decorrelating experience for 672 frames... +[2023-03-07 15:44:12,544][284304] Decorrelating experience for 416 frames... +[2023-03-07 15:44:12,579][282558] Decorrelating experience for 672 frames... +[2023-03-07 15:44:12,641][281966] Decorrelating experience for 800 frames... +[2023-03-07 15:44:12,646][281808] Decorrelating experience for 576 frames... +[2023-03-07 15:44:12,656][282574] Decorrelating experience for 768 frames... +[2023-03-07 15:44:12,680][281815] Decorrelating experience for 864 frames... +[2023-03-07 15:44:12,725][281801] Decorrelating experience for 768 frames... +[2023-03-07 15:44:12,726][281952] Decorrelating experience for 608 frames... +[2023-03-07 15:44:12,758][282891] Decorrelating experience for 576 frames... +[2023-03-07 15:44:12,758][281946] Decorrelating experience for 992 frames... +[2023-03-07 15:44:12,759][281955] Decorrelating experience for 928 frames... +[2023-03-07 15:44:12,797][281962] Decorrelating experience for 736 frames... +[2023-03-07 15:44:12,848][283277] Decorrelating experience for 640 frames... +[2023-03-07 15:44:12,868][284047] Decorrelating experience for 640 frames... +[2023-03-07 15:44:12,885][287383] Decorrelating experience for 352 frames... +[2023-03-07 15:44:12,919][281580] Decorrelating experience for 960 frames... +[2023-03-07 15:44:12,924][283020] Decorrelating experience for 992 frames... +[2023-03-07 15:44:12,931][281974] Decorrelating experience for 864 frames... +[2023-03-07 15:44:12,959][283084] Decorrelating experience for 992 frames... +[2023-03-07 15:44:12,963][283281] Decorrelating experience for 864 frames... +[2023-03-07 15:44:12,973][283396] Decorrelating experience for 832 frames... +[2023-03-07 15:44:13,002][281667] Decorrelating experience for 416 frames... +[2023-03-07 15:44:13,043][284304] Decorrelating experience for 448 frames... +[2023-03-07 15:44:13,063][281806] Decorrelating experience for 768 frames... +[2023-03-07 15:44:13,104][281579] Decorrelating experience for 640 frames... +[2023-03-07 15:44:13,119][283641] Decorrelating experience for 832 frames... +[2023-03-07 15:44:13,124][283278] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,147][281813] Decorrelating experience for 800 frames... +[2023-03-07 15:44:13,158][281948] Decorrelating experience for 768 frames... +[2023-03-07 15:44:13,162][283469] Decorrelating experience for 768 frames... +[2023-03-07 15:44:13,185][282763] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,197][282574] Decorrelating experience for 800 frames... +[2023-03-07 15:44:13,240][284345] Decorrelating experience for 576 frames... +[2023-03-07 15:44:13,253][281946] Stopping RolloutWorker_w71... +[2023-03-07 15:44:13,253][281946] Loop rollout_proc71_evt_loop terminating... +[2023-03-07 15:44:13,253][236315] Component RolloutWorker_w71 stopped! +[2023-03-07 15:44:13,305][281993] Decorrelating experience for 928 frames... +[2023-03-07 15:44:13,324][281581] Decorrelating experience for 736 frames... +[2023-03-07 15:44:13,325][281582] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,349][281971] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,383][287383] Decorrelating experience for 384 frames... +[2023-03-07 15:44:13,403][281575] Decorrelating experience for 736 frames... +[2023-03-07 15:44:13,403][283020] Stopping RolloutWorker_w125... +[2023-03-07 15:44:13,404][283020] Loop rollout_proc125_evt_loop terminating... +[2023-03-07 15:44:13,404][236315] Component RolloutWorker_w125 stopped! +[2023-03-07 15:44:13,425][281667] Decorrelating experience for 448 frames... +[2023-03-07 15:44:13,431][281953] Decorrelating experience for 384 frames... +[2023-03-07 15:44:13,436][283084] Stopping RolloutWorker_w119... +[2023-03-07 15:44:13,436][281950] Decorrelating experience for 928 frames... +[2023-03-07 15:44:13,437][283084] Loop rollout_proc119_evt_loop terminating... +[2023-03-07 15:44:13,437][236315] Component RolloutWorker_w119 stopped! +[2023-03-07 15:44:13,442][282558] Decorrelating experience for 704 frames... +[2023-03-07 15:44:13,528][283150] Decorrelating experience for 800 frames... +[2023-03-07 15:44:13,530][281966] Decorrelating experience for 832 frames... +[2023-03-07 15:44:13,540][281579] Decorrelating experience for 672 frames... +[2023-03-07 15:44:13,558][283396] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,599][281992] Decorrelating experience for 704 frames... +[2023-03-07 15:44:13,640][282891] Decorrelating experience for 608 frames... +[2023-03-07 15:44:13,646][283641] Decorrelating experience for 864 frames... +[2023-03-07 15:44:13,651][283727] Decorrelating experience for 448 frames... +[2023-03-07 15:44:13,661][284345] Decorrelating experience for 608 frames... +[2023-03-07 15:44:13,678][281815] Decorrelating experience for 896 frames... +[2023-03-07 15:44:13,727][281800] Decorrelating experience for 576 frames... +[2023-03-07 15:44:13,734][281657] Decorrelating experience for 992 frames... +[2023-03-07 15:44:13,741][283281] Decorrelating experience for 896 frames... +[2023-03-07 15:44:13,755][283278] Decorrelating experience for 896 frames... +[2023-03-07 15:44:13,794][281968] Decorrelating experience for 832 frames... +[2023-03-07 15:44:13,840][281802] Decorrelating experience for 704 frames... +[2023-03-07 15:44:13,852][281662] Decorrelating experience for 640 frames... +[2023-03-07 15:44:13,860][281974] Decorrelating experience for 896 frames... +[2023-03-07 15:44:13,894][281667] Decorrelating experience for 480 frames... +[2023-03-07 15:44:13,922][281801] Decorrelating experience for 800 frames... +[2023-03-07 15:44:13,949][281813] Decorrelating experience for 832 frames... +[2023-03-07 15:44:13,970][287383] Decorrelating experience for 416 frames... +[2023-03-07 15:44:13,990][281656] Decorrelating experience for 736 frames... +[2023-03-07 15:44:13,991][281954] Decorrelating experience for 576 frames... +[2023-03-07 15:44:14,061][281967] Decorrelating experience for 704 frames... +[2023-03-07 15:44:14,062][284166] Decorrelating experience for 704 frames... +[2023-03-07 15:44:14,082][284345] Decorrelating experience for 640 frames... +[2023-03-07 15:44:14,120][282827] Decorrelating experience for 736 frames... +[2023-03-07 15:44:14,124][281993] Decorrelating experience for 960 frames... +[2023-03-07 15:44:14,148][281800] Decorrelating experience for 608 frames... +[2023-03-07 15:44:14,164][281665] Decorrelating experience for 672 frames... +[2023-03-07 15:44:14,185][283150] Decorrelating experience for 832 frames... +[2023-03-07 15:44:14,189][282695] Decorrelating experience for 736 frames... +[2023-03-07 15:44:14,202][281657] Stopping RolloutWorker_w44... +[2023-03-07 15:44:14,202][281657] Loop rollout_proc44_evt_loop terminating... +[2023-03-07 15:44:14,202][236315] Component RolloutWorker_w44 stopped! +[2023-03-07 15:44:14,257][282183] Decorrelating experience for 704 frames... +[2023-03-07 15:44:14,264][284047] Decorrelating experience for 672 frames... +[2023-03-07 15:44:14,314][281653] Decorrelating experience for 544 frames... +[2023-03-07 15:44:14,324][283149] Decorrelating experience for 768 frames... +[2023-03-07 15:44:14,343][281802] Decorrelating experience for 736 frames... +[2023-03-07 15:44:14,371][283641] Decorrelating experience for 896 frames... +[2023-03-07 15:44:14,393][281667] Decorrelating experience for 512 frames... +[2023-03-07 15:44:14,397][281974] Decorrelating experience for 928 frames... +[2023-03-07 15:44:14,475][281656] Decorrelating experience for 768 frames... +[2023-03-07 15:44:14,476][281799] Decorrelating experience for 928 frames... +[2023-03-07 15:44:14,476][281957] Decorrelating experience for 448 frames... +[2023-03-07 15:44:14,504][281651] Decorrelating experience for 480 frames... +[2023-03-07 15:44:14,519][284345] Decorrelating experience for 672 frames... +[2023-03-07 15:44:14,532][281967] Decorrelating experience for 736 frames... +[2023-03-07 15:44:14,574][283727] Decorrelating experience for 480 frames... +[2023-03-07 15:44:14,579][281655] Decorrelating experience for 768 frames... +[2023-03-07 15:44:14,619][281801] Decorrelating experience for 832 frames... +[2023-03-07 15:44:14,671][282891] Decorrelating experience for 640 frames... +[2023-03-07 15:44:14,671][281813] Decorrelating experience for 864 frames... +[2023-03-07 15:44:14,690][283150] Decorrelating experience for 864 frames... +[2023-03-07 15:44:14,707][281944] Decorrelating experience for 896 frames... +[2023-03-07 15:44:14,745][284304] Decorrelating experience for 480 frames... +[2023-03-07 15:44:14,776][283277] Decorrelating experience for 672 frames... +[2023-03-07 15:44:14,778][281582] Decorrelating experience for 896 frames... +[2023-03-07 15:44:14,814][281802] Decorrelating experience for 768 frames... +[2023-03-07 15:44:14,881][281815] Decorrelating experience for 928 frames... +[2023-03-07 15:44:14,883][281954] Decorrelating experience for 608 frames... +[2023-03-07 15:44:14,884][281575] Decorrelating experience for 768 frames... +[2023-03-07 15:44:14,898][281993] Decorrelating experience for 992 frames... +[2023-03-07 15:44:14,959][281665] Decorrelating experience for 704 frames... +[2023-03-07 15:44:14,966][281809] Decorrelating experience for 608 frames... +[2023-03-07 15:44:14,991][281667] Decorrelating experience for 544 frames... +[2023-03-07 15:44:15,014][281653] Decorrelating experience for 576 frames... +[2023-03-07 15:44:15,071][281974] Decorrelating experience for 960 frames... +[2023-03-07 15:44:15,082][281810] Decorrelating experience for 672 frames... +[2023-03-07 15:44:15,083][281661] Decorrelating experience for 704 frames... +[2023-03-07 15:44:15,103][281579] Decorrelating experience for 704 frames... +[2023-03-07 15:44:15,165][287383] Decorrelating experience for 448 frames... +[2023-03-07 15:44:15,170][281984] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:15,190][281970] Decorrelating experience for 512 frames... +[2023-03-07 15:44:15,209][283727] Decorrelating experience for 512 frames... +[2023-03-07 15:44:15,214][281664] Decorrelating experience for 832 frames... +[2023-03-07 15:44:15,239][283277] Decorrelating experience for 704 frames... +[2023-03-07 15:44:15,240][281944] Decorrelating experience for 928 frames... +[2023-03-07 15:44:15,276][284304] Decorrelating experience for 512 frames... +[2023-03-07 15:44:15,280][284200] Decorrelating experience for 736 frames... +[2023-03-07 15:44:15,308][281582] Decorrelating experience for 928 frames... +[2023-03-07 15:44:15,310][281952] Decorrelating experience for 640 frames... +[2023-03-07 15:44:15,357][281968] Decorrelating experience for 864 frames... +[2023-03-07 15:44:15,408][281654] Decorrelating experience for 608 frames... +[2023-03-07 15:44:15,420][283149] Decorrelating experience for 800 frames... +[2023-03-07 15:44:15,439][281667] Decorrelating experience for 576 frames... +[2023-03-07 15:44:15,455][281953] Decorrelating experience for 416 frames... +[2023-03-07 15:44:15,488][281962] Decorrelating experience for 768 frames... +[2023-03-07 15:44:15,501][281993] Stopping RolloutWorker_w64... +[2023-03-07 15:44:15,501][281993] Loop rollout_proc64_evt_loop terminating... +[2023-03-07 15:44:15,502][236315] Component RolloutWorker_w64 stopped! +[2023-03-07 15:44:15,514][281948] Decorrelating experience for 800 frames... +[2023-03-07 15:44:15,548][281967] Decorrelating experience for 768 frames... +[2023-03-07 15:44:15,552][281813] Decorrelating experience for 896 frames... +[2023-03-07 15:44:15,577][281970] Decorrelating experience for 544 frames... +[2023-03-07 15:44:15,636][281801] Decorrelating experience for 864 frames... +[2023-03-07 15:44:15,644][283960] Decorrelating experience for 416 frames... +[2023-03-07 15:44:15,655][282891] Decorrelating experience for 672 frames... +[2023-03-07 15:44:15,693][281974] Decorrelating experience for 992 frames... +[2023-03-07 15:44:15,715][282827] Decorrelating experience for 768 frames... +[2023-03-07 15:44:15,745][281799] Decorrelating experience for 960 frames... +[2023-03-07 15:44:15,746][283085] Decorrelating experience for 416 frames... +[2023-03-07 15:44:15,766][282283] Decorrelating experience for 896 frames... +[2023-03-07 15:44:15,776][281961] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:15,828][281654] Decorrelating experience for 640 frames... +[2023-03-07 15:44:15,852][281802] Decorrelating experience for 800 frames... +[2023-03-07 15:44:15,853][282697] Decorrelating experience for 544 frames... +[2023-03-07 15:44:15,871][281653] Decorrelating experience for 608 frames... +[2023-03-07 15:44:15,893][281660] Decorrelating experience for 832 frames... +[2023-03-07 15:44:15,911][283149] Decorrelating experience for 832 frames... +[2023-03-07 15:44:15,943][283281] Decorrelating experience for 928 frames... +[2023-03-07 15:44:15,944][281800] Decorrelating experience for 640 frames... +[2023-03-07 15:44:15,957][281953] Decorrelating experience for 448 frames... +[2023-03-07 15:44:16,046][281582] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,071][281970] Decorrelating experience for 576 frames... +[2023-03-07 15:44:16,083][281813] Decorrelating experience for 928 frames... +[2023-03-07 15:44:16,143][281971] Decorrelating experience for 896 frames... +[2023-03-07 15:44:16,152][281602] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,157][281974] Stopping RolloutWorker_w69... +[2023-03-07 15:44:16,157][281974] Loop rollout_proc69_evt_loop terminating... +[2023-03-07 15:44:16,157][281667] Decorrelating experience for 608 frames... +[2023-03-07 15:44:16,158][236315] Component RolloutWorker_w69 stopped! +[2023-03-07 15:44:16,174][283277] Decorrelating experience for 736 frames... +[2023-03-07 15:44:16,181][281651] Decorrelating experience for 512 frames... +[2023-03-07 15:44:16,254][281965] Decorrelating experience for 768 frames... +[2023-03-07 15:44:16,278][281968] Decorrelating experience for 896 frames... +[2023-03-07 15:44:16,289][282283] Decorrelating experience for 928 frames... +[2023-03-07 15:44:16,312][281948] Decorrelating experience for 832 frames... +[2023-03-07 15:44:16,333][281969] Decorrelating experience for 832 frames... +[2023-03-07 15:44:16,348][282697] Decorrelating experience for 576 frames... +[2023-03-07 15:44:16,378][281801] Decorrelating experience for 896 frames... +[2023-03-07 15:44:16,388][281800] Decorrelating experience for 672 frames... +[2023-03-07 15:44:16,455][282891] Decorrelating experience for 704 frames... +[2023-03-07 15:44:16,458][281954] Decorrelating experience for 640 frames... +[2023-03-07 15:44:16,464][283396] Decorrelating experience for 896 frames... +[2023-03-07 15:44:16,529][288232] Decorrelating experience for 608 frames... +[2023-03-07 15:44:16,545][281660] Decorrelating experience for 864 frames... +[2023-03-07 15:44:16,564][281953] Decorrelating experience for 480 frames... +[2023-03-07 15:44:16,632][283727] Decorrelating experience for 544 frames... +[2023-03-07 15:44:16,666][281813] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,668][281950] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,670][283149] Decorrelating experience for 864 frames... +[2023-03-07 15:44:16,732][281651] Decorrelating experience for 544 frames... +[2023-03-07 15:44:16,733][281667] Decorrelating experience for 640 frames... +[2023-03-07 15:44:16,751][281962] Decorrelating experience for 800 frames... +[2023-03-07 15:44:16,756][282697] Decorrelating experience for 608 frames... +[2023-03-07 15:44:16,828][281664] Decorrelating experience for 864 frames... +[2023-03-07 15:44:16,833][283277] Decorrelating experience for 768 frames... +[2023-03-07 15:44:16,875][281955] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,936][282283] Decorrelating experience for 960 frames... +[2023-03-07 15:44:16,952][281968] Decorrelating experience for 928 frames... +[2023-03-07 15:44:16,953][284053] Decorrelating experience for 736 frames... +[2023-03-07 15:44:16,957][288232] Decorrelating experience for 640 frames... +[2023-03-07 15:44:16,958][283281] Decorrelating experience for 960 frames... +[2023-03-07 15:44:17,034][281656] Decorrelating experience for 800 frames... +[2023-03-07 15:44:17,050][281954] Decorrelating experience for 672 frames... +[2023-03-07 15:44:17,054][281801] Decorrelating experience for 928 frames... +[2023-03-07 15:44:17,066][281956] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 2 +[2023-03-07 15:44:17,077][284345] Decorrelating experience for 704 frames... +[2023-03-07 15:44:17,147][281653] Decorrelating experience for 640 frames... +[2023-03-07 15:44:17,157][281582] Decorrelating experience for 992 frames... +[2023-03-07 15:44:17,162][282763] Decorrelating experience for 896 frames... +[2023-03-07 15:44:17,165][281948] Decorrelating experience for 864 frames... +[2023-03-07 15:44:17,189][283149] Decorrelating experience for 896 frames... +[2023-03-07 15:44:17,193][284047] Decorrelating experience for 704 frames... +[2023-03-07 15:44:17,249][281962] Decorrelating experience for 832 frames... +[2023-03-07 15:44:17,264][281660] Decorrelating experience for 896 frames... +[2023-03-07 15:44:17,272][282697] Decorrelating experience for 640 frames... +[2023-03-07 15:44:17,314][281970] Decorrelating experience for 608 frames... +[2023-03-07 15:44:17,342][281950] Decorrelating experience for 992 frames... +[2023-03-07 15:44:17,368][281945] Decorrelating experience for 992 frames... +[2023-03-07 15:44:17,369][281956] Decorrelating experience for 352 frames... +[2023-03-07 15:44:17,377][282891] Decorrelating experience for 736 frames... +[2023-03-07 15:44:17,388][284304] Decorrelating experience for 544 frames... +[2023-03-07 15:44:17,421][281958] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 2 +[2023-03-07 15:44:17,505][281955] Decorrelating experience for 992 frames... +[2023-03-07 15:44:17,546][281654] Decorrelating experience for 672 frames... +[2023-03-07 15:44:17,546][281651] Decorrelating experience for 576 frames... +[2023-03-07 15:44:17,551][281808] Decorrelating experience for 608 frames... +[2023-03-07 15:44:17,554][283277] Decorrelating experience for 800 frames... +[2023-03-07 15:44:17,565][282827] Decorrelating experience for 800 frames... +[2023-03-07 15:44:17,565][281669] Decorrelating experience for 384 frames... +[2023-03-07 15:44:17,584][281655] Decorrelating experience for 800 frames... +[2023-03-07 15:44:17,602][281664] Decorrelating experience for 896 frames... +[2023-03-07 15:44:17,621][281582] Stopping RolloutWorker_w23... +[2023-03-07 15:44:17,621][281582] Loop rollout_proc23_evt_loop terminating... +[2023-03-07 15:44:17,621][236315] Component RolloutWorker_w23 stopped! +[2023-03-07 15:44:17,746][281968] Decorrelating experience for 960 frames... +[2023-03-07 15:44:17,755][284166] Decorrelating experience for 736 frames... +[2023-03-07 15:44:17,760][281801] Decorrelating experience for 960 frames... +[2023-03-07 15:44:17,771][283408] Decorrelating experience for 768 frames... +[2023-03-07 15:44:17,775][281952] Decorrelating experience for 672 frames... +[2023-03-07 15:44:17,796][281965] Decorrelating experience for 800 frames... +[2023-03-07 15:44:17,807][281962] Decorrelating experience for 864 frames... +[2023-03-07 15:44:17,819][284345] Decorrelating experience for 736 frames... +[2023-03-07 15:44:17,822][281969] Decorrelating experience for 864 frames... +[2023-03-07 15:44:17,829][281950] Stopping RolloutWorker_w80... +[2023-03-07 15:44:17,830][281950] Loop rollout_proc80_evt_loop terminating... +[2023-03-07 15:44:17,830][236315] Component RolloutWorker_w80 stopped! +[2023-03-07 15:44:17,853][281945] Stopping RolloutWorker_w74... +[2023-03-07 15:44:17,854][281945] Loop rollout_proc74_evt_loop terminating... +[2023-03-07 15:44:17,854][236315] Component RolloutWorker_w74 stopped! +[2023-03-07 15:44:17,963][281800] Decorrelating experience for 704 frames... +[2023-03-07 15:44:17,978][281955] Stopping RolloutWorker_w53... +[2023-03-07 15:44:17,978][281955] Loop rollout_proc53_evt_loop terminating... +[2023-03-07 15:44:17,978][236315] Component RolloutWorker_w53 stopped! +[2023-03-07 15:44:17,987][283278] Decorrelating experience for 928 frames... +[2023-03-07 15:44:17,987][281667] Decorrelating experience for 672 frames... +[2023-03-07 15:44:17,995][281809] Decorrelating experience for 640 frames... +[2023-03-07 15:44:17,999][284304] Decorrelating experience for 576 frames... +[2023-03-07 15:44:18,000][281810] Decorrelating experience for 704 frames... +[2023-03-07 15:44:18,028][281954] Decorrelating experience for 704 frames... +[2023-03-07 15:44:18,045][281654] Decorrelating experience for 704 frames... +[2023-03-07 15:44:18,112][281967] Decorrelating experience for 800 frames... +[2023-03-07 15:44:18,162][281575] Decorrelating experience for 800 frames... +[2023-03-07 15:44:18,190][281655] Decorrelating experience for 832 frames... +[2023-03-07 15:44:18,191][281953] Decorrelating experience for 512 frames... +[2023-03-07 15:44:18,205][281944] Decorrelating experience for 960 frames... +[2023-03-07 15:44:18,205][283277] Decorrelating experience for 832 frames... +[2023-03-07 15:44:18,235][281813] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,240][284166] Decorrelating experience for 768 frames... +[2023-03-07 15:44:18,253][281799] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,306][282763] Decorrelating experience for 928 frames... +[2023-03-07 15:44:18,389][281971] Decorrelating experience for 928 frames... +[2023-03-07 15:44:18,416][282696] Decorrelating experience for 640 frames... +[2023-03-07 15:44:18,417][281664] Decorrelating experience for 928 frames... +[2023-03-07 15:44:18,429][282283] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,430][281669] Decorrelating experience for 416 frames... +[2023-03-07 15:44:18,436][283149] Decorrelating experience for 928 frames... +[2023-03-07 15:44:18,436][281809] Decorrelating experience for 672 frames... +[2023-03-07 15:44:18,454][281802] Decorrelating experience for 832 frames... +[2023-03-07 15:44:18,498][288232] Decorrelating experience for 672 frames... +[2023-03-07 15:44:18,546][283278] Decorrelating experience for 960 frames... +[2023-03-07 15:44:18,581][281806] Decorrelating experience for 800 frames... +[2023-03-07 15:44:18,622][281957] Decorrelating experience for 480 frames... +[2023-03-07 15:44:18,636][281801] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,637][284047] Decorrelating experience for 736 frames... +[2023-03-07 15:44:18,640][281969] Decorrelating experience for 896 frames... +[2023-03-07 15:44:18,659][283408] Decorrelating experience for 800 frames... +[2023-03-07 15:44:18,713][281813] Stopping RolloutWorker_w37... +[2023-03-07 15:44:18,713][281813] Loop rollout_proc37_evt_loop terminating... +[2023-03-07 15:44:18,713][236315] Component RolloutWorker_w37 stopped! +[2023-03-07 15:44:18,720][281808] Decorrelating experience for 640 frames... +[2023-03-07 15:44:18,740][281799] Stopping RolloutWorker_w39... +[2023-03-07 15:44:18,741][281799] Loop rollout_proc39_evt_loop terminating... +[2023-03-07 15:44:18,741][236315] Component RolloutWorker_w39 stopped! +[2023-03-07 15:44:18,788][281944] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,788][281580] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,790][281956] Decorrelating experience for 384 frames... +[2023-03-07 15:44:18,815][281965] Decorrelating experience for 832 frames... +[2023-03-07 15:44:18,846][281952] Decorrelating experience for 704 frames... +[2023-03-07 15:44:18,846][281967] Decorrelating experience for 832 frames... +[2023-03-07 15:44:18,853][284200] Decorrelating experience for 768 frames... +[2023-03-07 15:44:18,867][281651] Decorrelating experience for 608 frames... +[2023-03-07 15:44:18,895][282283] Stopping RolloutWorker_w55... +[2023-03-07 15:44:18,895][282283] Loop rollout_proc55_evt_loop terminating... +[2023-03-07 15:44:18,895][236315] Component RolloutWorker_w55 stopped! +[2023-03-07 15:44:18,921][281968] Decorrelating experience for 992 frames... +[2023-03-07 15:44:18,983][287383] Decorrelating experience for 480 frames... +[2023-03-07 15:44:18,999][288232] Decorrelating experience for 704 frames... +[2023-03-07 15:44:19,022][282697] Decorrelating experience for 672 frames... +[2023-03-07 15:44:19,071][281660] Decorrelating experience for 928 frames... +[2023-03-07 15:44:19,073][281602] Decorrelating experience for 992 frames... +[2023-03-07 15:44:19,079][281669] Decorrelating experience for 448 frames... +[2023-03-07 15:44:19,099][281802] Decorrelating experience for 864 frames... +[2023-03-07 15:44:19,110][281801] Stopping RolloutWorker_w13... +[2023-03-07 15:44:19,110][281801] Loop rollout_proc13_evt_loop terminating... +[2023-03-07 15:44:19,110][236315] Component RolloutWorker_w13 stopped! +[2023-03-07 15:44:19,148][284304] Decorrelating experience for 608 frames... +[2023-03-07 15:44:19,155][281655] Decorrelating experience for 864 frames... +[2023-03-07 15:44:19,178][281670] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,222][281662] Decorrelating experience for 672 frames... +[2023-03-07 15:44:19,276][281580] Stopping RolloutWorker_w20... +[2023-03-07 15:44:19,276][281580] Loop rollout_proc20_evt_loop terminating... +[2023-03-07 15:44:19,276][236315] Component RolloutWorker_w20 stopped! +[2023-03-07 15:44:19,280][281575] Decorrelating experience for 832 frames... +[2023-03-07 15:44:19,282][282763] Decorrelating experience for 960 frames... +[2023-03-07 15:44:19,297][281944] Stopping RolloutWorker_w40... +[2023-03-07 15:44:19,298][281944] Loop rollout_proc40_evt_loop terminating... +[2023-03-07 15:44:19,298][236315] Component RolloutWorker_w40 stopped! +[2023-03-07 15:44:19,299][284345] Decorrelating experience for 768 frames... +[2023-03-07 15:44:19,314][281952] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,326][283278] Decorrelating experience for 992 frames... +[2023-03-07 15:44:19,346][281965] Decorrelating experience for 864 frames... +[2023-03-07 15:44:19,350][281800] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,415][281968] Stopping RolloutWorker_w54... +[2023-03-07 15:44:19,416][281968] Loop rollout_proc54_evt_loop terminating... +[2023-03-07 15:44:19,416][236315] Component RolloutWorker_w54 stopped! +[2023-03-07 15:44:19,434][281966] Decorrelating experience for 864 frames... +[2023-03-07 15:44:19,474][281664] Decorrelating experience for 960 frames... +[2023-03-07 15:44:19,489][284053] Decorrelating experience for 768 frames... +[2023-03-07 15:44:19,507][281808] Decorrelating experience for 672 frames... +[2023-03-07 15:44:19,517][281651] Decorrelating experience for 640 frames... +[2023-03-07 15:44:19,530][281602] Stopping RolloutWorker_w32... +[2023-03-07 15:44:19,530][281602] Loop rollout_proc32_evt_loop terminating... +[2023-03-07 15:44:19,530][236315] Component RolloutWorker_w32 stopped! +[2023-03-07 15:44:19,545][281654] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,546][281962] Decorrelating experience for 896 frames... +[2023-03-07 15:44:19,623][281802] Decorrelating experience for 896 frames... +[2023-03-07 15:44:19,627][282694] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,672][281670] Decorrelating experience for 768 frames... +[2023-03-07 15:44:19,687][284309] Decorrelating experience for 960 frames... +[2023-03-07 15:44:19,721][282891] Decorrelating experience for 768 frames... +[2023-03-07 15:44:19,722][288232] Decorrelating experience for 736 frames... +[2023-03-07 15:44:19,750][281667] Decorrelating experience for 704 frames... +[2023-03-07 15:44:19,782][283278] Stopping RolloutWorker_w102... +[2023-03-07 15:44:19,782][283278] Loop rollout_proc102_evt_loop terminating... +[2023-03-07 15:44:19,783][236315] Component RolloutWorker_w102 stopped! +[2023-03-07 15:44:19,800][281575] Decorrelating experience for 864 frames... +[2023-03-07 15:44:19,825][282697] Decorrelating experience for 704 frames... +[2023-03-07 15:44:19,845][282763] Decorrelating experience for 992 frames... +[2023-03-07 15:44:19,870][287383] Decorrelating experience for 512 frames... +[2023-03-07 15:44:19,875][281965] Decorrelating experience for 896 frames... +[2023-03-07 15:44:19,884][281991] Decorrelating experience for 640 frames... +[2023-03-07 15:44:19,923][282696] Decorrelating experience for 672 frames... +[2023-03-07 15:44:19,928][281660] Decorrelating experience for 960 frames... +[2023-03-07 15:44:19,959][281651] Decorrelating experience for 672 frames... +[2023-03-07 15:44:19,963][281957] Decorrelating experience for 512 frames... +[2023-03-07 15:44:20,014][281954] Decorrelating experience for 736 frames... +[2023-03-07 15:44:20,078][284304] Decorrelating experience for 640 frames... +[2023-03-07 15:44:20,081][281664] Decorrelating experience for 992 frames... +[2023-03-07 15:44:20,117][281808] Decorrelating experience for 704 frames... +[2023-03-07 15:44:20,132][281971] Decorrelating experience for 960 frames... +[2023-03-07 15:44:20,153][282694] Decorrelating experience for 768 frames... +[2023-03-07 15:44:20,188][283890] Decorrelating experience for 960 frames... +[2023-03-07 15:44:20,199][281579] Decorrelating experience for 736 frames... +[2023-03-07 15:44:20,267][284200] Decorrelating experience for 800 frames... +[2023-03-07 15:44:20,281][282697] Decorrelating experience for 736 frames... +[2023-03-07 15:44:20,296][282763] Stopping RolloutWorker_w107... +[2023-03-07 15:44:20,297][282763] Loop rollout_proc107_evt_loop terminating... +[2023-03-07 15:44:20,297][236315] Component RolloutWorker_w107 stopped! +[2023-03-07 15:44:20,301][283352] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:20,302][281802] Decorrelating experience for 928 frames... +[2023-03-07 15:44:20,311][281667] Decorrelating experience for 736 frames... +[2023-03-07 15:44:20,332][283281] Decorrelating experience for 992 frames... +[2023-03-07 15:44:20,364][284053] Decorrelating experience for 800 frames... +[2023-03-07 15:44:20,384][281966] Decorrelating experience for 896 frames... +[2023-03-07 15:44:20,384][281973] Decorrelating experience for 736 frames... +[2023-03-07 15:44:20,454][283019] Decorrelating experience for 608 frames... +[2023-03-07 15:44:20,516][281991] Decorrelating experience for 672 frames... +[2023-03-07 15:44:20,528][281957] Decorrelating experience for 544 frames... +[2023-03-07 15:44:20,528][281664] Stopping RolloutWorker_w24... +[2023-03-07 15:44:20,528][281664] Loop rollout_proc24_evt_loop terminating... +[2023-03-07 15:44:20,528][236315] Component RolloutWorker_w24 stopped! +[2023-03-07 15:44:20,536][281660] Decorrelating experience for 992 frames... +[2023-03-07 15:44:20,560][282904] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:20,573][283408] Decorrelating experience for 832 frames... +[2023-03-07 15:44:20,585][281969] Decorrelating experience for 928 frames... +[2023-03-07 15:44:20,642][283149] Decorrelating experience for 960 frames... +[2023-03-07 15:44:20,649][282827] Decorrelating experience for 832 frames... +[2023-03-07 15:44:20,727][282694] Decorrelating experience for 800 frames... +[2023-03-07 15:44:20,736][281575] Decorrelating experience for 896 frames... +[2023-03-07 15:44:20,752][284200] Decorrelating experience for 832 frames... +[2023-03-07 15:44:20,777][283085] Decorrelating experience for 448 frames... +[2023-03-07 15:44:20,777][283281] Stopping RolloutWorker_w99... +[2023-03-07 15:44:20,778][283281] Loop rollout_proc99_evt_loop terminating... +[2023-03-07 15:44:20,778][236315] Component RolloutWorker_w99 stopped! +[2023-03-07 15:44:20,787][282184] Decorrelating experience for 928 frames... +[2023-03-07 15:44:20,800][281971] Decorrelating experience for 992 frames... +[2023-03-07 15:44:20,832][281804] Decorrelating experience for 704 frames... +[2023-03-07 15:44:20,853][281965] Decorrelating experience for 928 frames... +[2023-03-07 15:44:20,854][281802] Decorrelating experience for 960 frames... +[2023-03-07 15:44:20,928][282696] Decorrelating experience for 704 frames... +[2023-03-07 15:44:20,958][281667] Decorrelating experience for 768 frames... +[2023-03-07 15:44:20,973][283019] Decorrelating experience for 640 frames... +[2023-03-07 15:44:20,980][284047] Decorrelating experience for 768 frames... +[2023-03-07 15:44:20,993][283960] Decorrelating experience for 448 frames... +[2023-03-07 15:44:21,014][281660] Stopping RolloutWorker_w18... +[2023-03-07 15:44:21,015][281660] Loop rollout_proc18_evt_loop terminating... +[2023-03-07 15:44:21,014][236315] Component RolloutWorker_w18 stopped! +[2023-03-07 15:44:21,021][281966] Decorrelating experience for 928 frames... +[2023-03-07 15:44:21,028][281984] Decorrelating experience for 576 frames... +[2023-03-07 15:44:21,029][283890] Decorrelating experience for 992 frames... +[2023-03-07 15:44:21,057][281973] Decorrelating experience for 768 frames... +[2023-03-07 15:44:21,088][281962] Decorrelating experience for 928 frames... +[2023-03-07 15:44:21,167][281991] Decorrelating experience for 704 frames... +[2023-03-07 15:44:21,174][283396] Decorrelating experience for 928 frames... +[2023-03-07 15:44:21,178][281967] Decorrelating experience for 864 frames... +[2023-03-07 15:44:21,193][283085] Decorrelating experience for 480 frames... +[2023-03-07 15:44:21,200][283149] Decorrelating experience for 992 frames... +[2023-03-07 15:44:21,228][282694] Decorrelating experience for 832 frames... +[2023-03-07 15:44:21,236][281669] Decorrelating experience for 480 frames... +[2023-03-07 15:44:21,255][281971] Stopping RolloutWorker_w89... +[2023-03-07 15:44:21,255][281971] Loop rollout_proc89_evt_loop terminating... +[2023-03-07 15:44:21,255][236315] Component RolloutWorker_w89 stopped! +[2023-03-07 15:44:21,288][283408] Decorrelating experience for 864 frames... +[2023-03-07 15:44:21,295][281575] Decorrelating experience for 928 frames... +[2023-03-07 15:44:21,359][281954] Decorrelating experience for 768 frames... +[2023-03-07 15:44:21,364][281579] Decorrelating experience for 768 frames... +[2023-03-07 15:44:21,383][281969] Decorrelating experience for 960 frames... +[2023-03-07 15:44:21,389][281804] Decorrelating experience for 736 frames... +[2023-03-07 15:44:21,442][281984] Decorrelating experience for 608 frames... +[2023-03-07 15:44:21,486][281670] Decorrelating experience for 800 frames... +[2023-03-07 15:44:21,518][281965] Decorrelating experience for 960 frames... +[2023-03-07 15:44:21,551][281806] Decorrelating experience for 832 frames... +[2023-03-07 15:44:21,561][281802] Decorrelating experience for 992 frames... +[2023-03-07 15:44:21,580][283727] Decorrelating experience for 576 frames... +[2023-03-07 15:44:21,596][282574] Decorrelating experience for 832 frames... +[2023-03-07 15:44:21,629][283890] Stopping RolloutWorker_w100... +[2023-03-07 15:44:21,630][283890] Loop rollout_proc100_evt_loop terminating... +[2023-03-07 15:44:21,630][236315] Component RolloutWorker_w100 stopped! +[2023-03-07 15:44:21,654][283149] Stopping RolloutWorker_w93... +[2023-03-07 15:44:21,655][283149] Loop rollout_proc93_evt_loop terminating... +[2023-03-07 15:44:21,655][236315] Component RolloutWorker_w93 stopped! +[2023-03-07 15:44:21,682][281662] Decorrelating experience for 704 frames... +[2023-03-07 15:44:21,711][281966] Decorrelating experience for 960 frames... +[2023-03-07 15:44:21,732][281667] Decorrelating experience for 800 frames... +[2023-03-07 15:44:21,740][281973] Decorrelating experience for 800 frames... +[2023-03-07 15:44:21,770][284047] Decorrelating experience for 800 frames... +[2023-03-07 15:44:21,786][282694] Decorrelating experience for 864 frames... +[2023-03-07 15:44:21,802][282184] Decorrelating experience for 960 frames... +[2023-03-07 15:44:21,845][281575] Decorrelating experience for 960 frames... +[2023-03-07 15:44:21,887][281958] Decorrelating experience for 768 frames... +[2023-03-07 15:44:21,943][281579] Decorrelating experience for 800 frames... +[2023-03-07 15:44:21,986][283727] Decorrelating experience for 608 frames... +[2023-03-07 15:44:21,990][281969] Decorrelating experience for 992 frames... +[2023-03-07 15:44:21,998][281655] Decorrelating experience for 896 frames... +[2023-03-07 15:44:22,013][281577] Decorrelating experience for 800 frames... +[2023-03-07 15:44:22,016][281802] Stopping RolloutWorker_w22... +[2023-03-07 15:44:22,016][281802] Loop rollout_proc22_evt_loop terminating... +[2023-03-07 15:44:22,016][236315] Component RolloutWorker_w22 stopped! +[2023-03-07 15:44:22,060][284345] Decorrelating experience for 800 frames... +[2023-03-07 15:44:22,090][281962] Decorrelating experience for 960 frames... +[2023-03-07 15:44:22,104][282574] Decorrelating experience for 864 frames... +[2023-03-07 15:44:22,150][281967] Decorrelating experience for 896 frames... +[2023-03-07 15:44:22,203][281670] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,209][281954] Decorrelating experience for 800 frames... +[2023-03-07 15:44:22,209][281805] Decorrelating experience for 704 frames... +[2023-03-07 15:44:22,233][281973] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,292][282696] Decorrelating experience for 736 frames... +[2023-03-07 15:44:22,305][282694] Decorrelating experience for 896 frames... +[2023-03-07 15:44:22,329][281965] Decorrelating experience for 992 frames... +[2023-03-07 15:44:22,363][281966] Decorrelating experience for 992 frames... +[2023-03-07 15:44:22,399][281575] Decorrelating experience for 992 frames... +[2023-03-07 15:44:22,415][281952] Decorrelating experience for 768 frames... +[2023-03-07 15:44:22,415][281806] Decorrelating experience for 864 frames... +[2023-03-07 15:44:22,418][281956] Decorrelating experience for 416 frames... +[2023-03-07 15:44:22,434][281579] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,454][281667] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,499][281654] Decorrelating experience for 768 frames... +[2023-03-07 15:44:22,530][281655] Decorrelating experience for 928 frames... +[2023-03-07 15:44:22,621][283727] Decorrelating experience for 640 frames... +[2023-03-07 15:44:22,623][281969] Stopping RolloutWorker_w63... +[2023-03-07 15:44:22,623][281969] Loop rollout_proc63_evt_loop terminating... +[2023-03-07 15:44:22,623][236315] Component RolloutWorker_w63 stopped! +[2023-03-07 15:44:22,625][284047] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,652][282697] Decorrelating experience for 768 frames... +[2023-03-07 15:44:22,673][281970] Decorrelating experience for 640 frames... +[2023-03-07 15:44:22,721][282574] Decorrelating experience for 896 frames... +[2023-03-07 15:44:22,752][282696] Decorrelating experience for 768 frames... +[2023-03-07 15:44:22,764][284345] Decorrelating experience for 832 frames... +[2023-03-07 15:44:22,770][281807] Decorrelating experience for 800 frames... +[2023-03-07 15:44:22,811][281670] Decorrelating experience for 864 frames... +[2023-03-07 15:44:22,825][281965] Stopping RolloutWorker_w72... +[2023-03-07 15:44:22,825][281965] Loop rollout_proc72_evt_loop terminating... +[2023-03-07 15:44:22,826][236315] Component RolloutWorker_w72 stopped! +[2023-03-07 15:44:22,828][281966] Stopping RolloutWorker_w84... +[2023-03-07 15:44:22,828][281966] Loop rollout_proc84_evt_loop terminating... +[2023-03-07 15:44:22,828][236315] Component RolloutWorker_w84 stopped! +[2023-03-07 15:44:22,861][281962] Decorrelating experience for 992 frames... +[2023-03-07 15:44:22,873][283469] Decorrelating experience for 800 frames... +[2023-03-07 15:44:22,877][281575] Stopping RolloutWorker_w5... +[2023-03-07 15:44:22,877][281575] Loop rollout_proc5_evt_loop terminating... +[2023-03-07 15:44:22,877][236315] Component RolloutWorker_w5 stopped! +[2023-03-07 15:44:22,924][281963] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:22,936][281806] Decorrelating experience for 896 frames... +[2023-03-07 15:44:22,937][282694] Decorrelating experience for 928 frames... +[2023-03-07 15:44:22,957][281667] Decorrelating experience for 864 frames... +[2023-03-07 15:44:22,966][281972] Decorrelating experience for 736 frames... +[2023-03-07 15:44:22,985][281654] Decorrelating experience for 800 frames... +[2023-03-07 15:44:23,059][281954] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,071][283854] Decorrelating experience for 608 frames... +[2023-03-07 15:44:23,074][281577] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,082][281655] Decorrelating experience for 960 frames... +[2023-03-07 15:44:23,132][281988] Another process currently holds the lock /tmp/sf2_rolo/doom_008.lockfile, attempt: 1 +[2023-03-07 15:44:23,135][281662] Decorrelating experience for 736 frames... +[2023-03-07 15:44:23,170][284166] Decorrelating experience for 800 frames... +[2023-03-07 15:44:23,261][283277] Decorrelating experience for 864 frames... +[2023-03-07 15:44:23,269][284053] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,291][284345] Decorrelating experience for 864 frames... +[2023-03-07 15:44:23,301][284047] Decorrelating experience for 864 frames... +[2023-03-07 15:44:23,320][281962] Stopping RolloutWorker_w75... +[2023-03-07 15:44:23,321][281962] Loop rollout_proc75_evt_loop terminating... +[2023-03-07 15:44:23,321][236315] Component RolloutWorker_w75 stopped! +[2023-03-07 15:44:23,329][282697] Decorrelating experience for 800 frames... +[2023-03-07 15:44:23,340][281809] Decorrelating experience for 704 frames... +[2023-03-07 15:44:23,352][281800] Decorrelating experience for 768 frames... +[2023-03-07 15:44:23,447][281807] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,464][281975] Decorrelating experience for 672 frames... +[2023-03-07 15:44:23,465][283469] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,487][283854] Decorrelating experience for 640 frames... +[2023-03-07 15:44:23,508][282574] Decorrelating experience for 928 frames... +[2023-03-07 15:44:23,526][281810] Decorrelating experience for 736 frames... +[2023-03-07 15:44:23,552][283727] Decorrelating experience for 672 frames... +[2023-03-07 15:44:23,623][281655] Decorrelating experience for 992 frames... +[2023-03-07 15:44:23,658][281662] Decorrelating experience for 768 frames... +[2023-03-07 15:44:23,666][281956] Decorrelating experience for 448 frames... +[2023-03-07 15:44:23,723][281578] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:23,725][284166] Decorrelating experience for 832 frames... +[2023-03-07 15:44:23,749][281954] Decorrelating experience for 864 frames... +[2023-03-07 15:44:23,770][283277] Decorrelating experience for 896 frames... +[2023-03-07 15:44:23,797][281809] Decorrelating experience for 736 frames... +[2023-03-07 15:44:23,831][281952] Decorrelating experience for 800 frames... +[2023-03-07 15:44:23,848][284345] Decorrelating experience for 896 frames... +[2023-03-07 15:44:23,874][287383] Decorrelating experience for 544 frames... +[2023-03-07 15:44:23,929][283396] Decorrelating experience for 960 frames... +[2023-03-07 15:44:23,935][284200] Decorrelating experience for 864 frames... +[2023-03-07 15:44:23,990][281810] Decorrelating experience for 768 frames... +[2023-03-07 15:44:23,993][283727] Decorrelating experience for 704 frames... +[2023-03-07 15:44:24,004][281667] Decorrelating experience for 896 frames... +[2023-03-07 15:44:24,018][282904] Decorrelating experience for 448 frames... +[2023-03-07 15:44:24,040][282574] Decorrelating experience for 960 frames... +[2023-03-07 15:44:24,067][282697] Decorrelating experience for 832 frames... +[2023-03-07 15:44:24,073][281655] Stopping RolloutWorker_w12... +[2023-03-07 15:44:24,074][281655] Loop rollout_proc12_evt_loop terminating... +[2023-03-07 15:44:24,074][281654] Decorrelating experience for 832 frames... +[2023-03-07 15:44:24,074][236315] Component RolloutWorker_w12 stopped! +[2023-03-07 15:44:24,116][281992] Decorrelating experience for 736 frames... +[2023-03-07 15:44:24,126][282891] Decorrelating experience for 800 frames... +[2023-03-07 15:44:24,141][281662] Decorrelating experience for 800 frames... +[2023-03-07 15:44:24,212][281808] Decorrelating experience for 736 frames... +[2023-03-07 15:44:24,229][284166] Decorrelating experience for 864 frames... +[2023-03-07 15:44:24,273][287383] Decorrelating experience for 576 frames... +[2023-03-07 15:44:24,277][281807] Decorrelating experience for 864 frames... +[2023-03-07 15:44:24,294][283277] Decorrelating experience for 928 frames... +[2023-03-07 15:44:24,311][281975] Decorrelating experience for 704 frames... +[2023-03-07 15:44:24,327][281952] Decorrelating experience for 832 frames... +[2023-03-07 15:44:24,328][283408] Decorrelating experience for 896 frames... +[2023-03-07 15:44:24,377][282904] Decorrelating experience for 480 frames... +[2023-03-07 15:44:24,411][281578] Decorrelating experience for 800 frames... +[2023-03-07 15:44:24,485][281958] Decorrelating experience for 800 frames... +[2023-03-07 15:44:24,502][281581] Decorrelating experience for 768 frames... +[2023-03-07 15:44:24,506][281954] Decorrelating experience for 896 frames... +[2023-03-07 15:44:24,526][284053] Decorrelating experience for 864 frames... +[2023-03-07 15:44:24,527][284345] Decorrelating experience for 928 frames... +[2023-03-07 15:44:24,574][281992] Decorrelating experience for 768 frames... +[2023-03-07 15:44:24,603][284200] Decorrelating experience for 896 frames... +[2023-03-07 15:44:24,676][281654] Decorrelating experience for 864 frames... +[2023-03-07 15:44:24,680][283469] Decorrelating experience for 864 frames... +[2023-03-07 15:44:24,697][282695] Decorrelating experience for 768 frames... +[2023-03-07 15:44:24,709][281667] Decorrelating experience for 928 frames... +[2023-03-07 15:44:24,718][282891] Decorrelating experience for 832 frames... +[2023-03-07 15:44:24,732][283019] Decorrelating experience for 672 frames... +[2023-03-07 15:44:24,764][281975] Decorrelating experience for 736 frames... +[2023-03-07 15:44:24,782][287383] Decorrelating experience for 608 frames... +[2023-03-07 15:44:24,864][281807] Decorrelating experience for 896 frames... +[2023-03-07 15:44:24,888][281815] Decorrelating experience for 960 frames... +[2023-03-07 15:44:24,896][281972] Decorrelating experience for 768 frames... +[2023-03-07 15:44:24,910][282904] Decorrelating experience for 512 frames... +[2023-03-07 15:44:24,925][283396] Decorrelating experience for 992 frames... +[2023-03-07 15:44:24,947][283277] Decorrelating experience for 960 frames... +[2023-03-07 15:44:24,947][281810] Decorrelating experience for 800 frames... +[2023-03-07 15:44:24,955][281661] Decorrelating experience for 736 frames... +[2023-03-07 15:44:24,958][283408] Decorrelating experience for 928 frames... +[2023-03-07 15:44:24,998][281578] Decorrelating experience for 832 frames... +[2023-03-07 15:44:25,079][283854] Decorrelating experience for 672 frames... +[2023-03-07 15:44:25,092][281958] Decorrelating experience for 832 frames... +[2023-03-07 15:44:25,104][281579] Decorrelating experience for 864 frames... +[2023-03-07 15:44:25,137][281805] Decorrelating experience for 736 frames... +[2023-03-07 15:44:25,138][281581] Decorrelating experience for 800 frames... +[2023-03-07 15:44:25,144][281670] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,193][284053] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,216][283469] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,233][281954] Decorrelating experience for 928 frames... +[2023-03-07 15:44:25,366][282697] Decorrelating experience for 864 frames... +[2023-03-07 15:44:25,366][282558] Decorrelating experience for 736 frames... +[2023-03-07 15:44:25,370][281654] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,371][281662] Decorrelating experience for 832 frames... +[2023-03-07 15:44:25,372][284309] Decorrelating experience for 992 frames... +[2023-03-07 15:44:25,372][282695] Decorrelating experience for 800 frames... +[2023-03-07 15:44:25,374][281975] Decorrelating experience for 768 frames... +[2023-03-07 15:44:25,418][287383] Decorrelating experience for 640 frames... +[2023-03-07 15:44:25,449][282904] Decorrelating experience for 544 frames... +[2023-03-07 15:44:25,561][284166] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,574][283396] Stopping RolloutWorker_w114... +[2023-03-07 15:44:25,574][283396] Loop rollout_proc114_evt_loop terminating... +[2023-03-07 15:44:25,574][236315] Component RolloutWorker_w114 stopped! +[2023-03-07 15:44:25,581][282694] Decorrelating experience for 960 frames... +[2023-03-07 15:44:25,590][283352] Decorrelating experience for 768 frames... +[2023-03-07 15:44:25,591][282891] Decorrelating experience for 864 frames... +[2023-03-07 15:44:25,592][281956] Decorrelating experience for 480 frames... +[2023-03-07 15:44:25,601][283408] Decorrelating experience for 960 frames... +[2023-03-07 15:44:25,657][281807] Decorrelating experience for 928 frames... +[2023-03-07 15:44:25,659][281667] Decorrelating experience for 960 frames... +[2023-03-07 15:44:25,677][281952] Decorrelating experience for 864 frames... +[2023-03-07 15:44:25,756][283277] Decorrelating experience for 992 frames... +[2023-03-07 15:44:25,796][283150] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,809][281808] Decorrelating experience for 768 frames... +[2023-03-07 15:44:25,822][281579] Decorrelating experience for 896 frames... +[2023-03-07 15:44:25,826][281984] Decorrelating experience for 640 frames... +[2023-03-07 15:44:25,847][282558] Decorrelating experience for 768 frames... +[2023-03-07 15:44:25,870][284345] Decorrelating experience for 960 frames... +[2023-03-07 15:44:25,886][283854] Decorrelating experience for 704 frames... +[2023-03-07 15:44:25,930][283469] Decorrelating experience for 928 frames... +[2023-03-07 15:44:25,947][281972] Decorrelating experience for 800 frames... +[2023-03-07 15:44:25,984][281669] Decorrelating experience for 512 frames... +[2023-03-07 15:44:26,008][281992] Decorrelating experience for 800 frames... +[2023-03-07 15:44:26,011][281956] Decorrelating experience for 512 frames... +[2023-03-07 15:44:26,031][284309] Stopping RolloutWorker_w121... +[2023-03-07 15:44:26,031][284309] Loop rollout_proc121_evt_loop terminating... +[2023-03-07 15:44:26,031][236315] Component RolloutWorker_w121 stopped! +[2023-03-07 15:44:26,073][281975] Decorrelating experience for 800 frames... +[2023-03-07 15:44:26,109][282695] Decorrelating experience for 832 frames... +[2023-03-07 15:44:26,131][281581] Decorrelating experience for 832 frames... +[2023-03-07 15:44:26,143][282904] Decorrelating experience for 576 frames... +[2023-03-07 15:44:26,180][283641] Decorrelating experience for 928 frames... +[2023-03-07 15:44:26,216][281661] Decorrelating experience for 768 frames... +[2023-03-07 15:44:26,217][281958] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,217][281973] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,218][283277] Stopping RolloutWorker_w105... +[2023-03-07 15:44:26,218][236315] Component RolloutWorker_w105 stopped! +[2023-03-07 15:44:26,238][283277] Loop rollout_proc105_evt_loop terminating... +[2023-03-07 15:44:26,248][281807] Decorrelating experience for 960 frames... +[2023-03-07 15:44:26,290][282694] Decorrelating experience for 992 frames... +[2023-03-07 15:44:26,315][281808] Decorrelating experience for 800 frames... +[2023-03-07 15:44:26,319][281667] Decorrelating experience for 992 frames... +[2023-03-07 15:44:26,337][281662] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,409][282574] Decorrelating experience for 992 frames... +[2023-03-07 15:44:26,413][281954] Decorrelating experience for 960 frames... +[2023-03-07 15:44:26,433][284345] Decorrelating experience for 992 frames... +[2023-03-07 15:44:26,451][288232] Decorrelating experience for 768 frames... +[2023-03-07 15:44:26,492][281577] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,503][281992] Decorrelating experience for 832 frames... +[2023-03-07 15:44:26,510][282891] Decorrelating experience for 896 frames... +[2023-03-07 15:44:26,527][281670] Decorrelating experience for 928 frames... +[2023-03-07 15:44:26,543][282558] Decorrelating experience for 800 frames... +[2023-03-07 15:44:26,602][281956] Decorrelating experience for 544 frames... +[2023-03-07 15:44:26,611][281578] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,639][281961] Decorrelating experience for 736 frames... +[2023-03-07 15:44:26,695][281972] Decorrelating experience for 832 frames... +[2023-03-07 15:44:26,727][283150] Decorrelating experience for 928 frames... +[2023-03-07 15:44:26,740][282694] Stopping RolloutWorker_w92... +[2023-03-07 15:44:26,740][282694] Loop rollout_proc92_evt_loop terminating... +[2023-03-07 15:44:26,741][236315] Component RolloutWorker_w92 stopped! +[2023-03-07 15:44:26,770][282695] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,774][281653] Decorrelating experience for 672 frames... +[2023-03-07 15:44:26,783][281667] Stopping RolloutWorker_w36... +[2023-03-07 15:44:26,783][281667] Loop rollout_proc36_evt_loop terminating... +[2023-03-07 15:44:26,783][236315] Component RolloutWorker_w36 stopped! +[2023-03-07 15:44:26,809][284166] Decorrelating experience for 928 frames... +[2023-03-07 15:44:26,815][281973] Decorrelating experience for 896 frames... +[2023-03-07 15:44:26,819][281808] Decorrelating experience for 832 frames... +[2023-03-07 15:44:26,838][281804] Decorrelating experience for 768 frames... +[2023-03-07 15:44:26,856][281662] Decorrelating experience for 896 frames... +[2023-03-07 15:44:26,890][281581] Decorrelating experience for 864 frames... +[2023-03-07 15:44:26,897][284345] Stopping RolloutWorker_w127... +[2023-03-07 15:44:26,897][284345] Loop rollout_proc127_evt_loop terminating... +[2023-03-07 15:44:26,898][236315] Component RolloutWorker_w127 stopped! +[2023-03-07 15:44:26,937][283641] Decorrelating experience for 960 frames... +[2023-03-07 15:44:26,960][283469] Decorrelating experience for 960 frames... +[2023-03-07 15:44:26,979][281991] Decorrelating experience for 736 frames... +[2023-03-07 15:44:27,007][282574] Stopping RolloutWorker_w67... +[2023-03-07 15:44:27,008][282574] Loop rollout_proc67_evt_loop terminating... +[2023-03-07 15:44:27,008][236315] Component RolloutWorker_w67 stopped! +[2023-03-07 15:44:27,018][281807] Decorrelating experience for 992 frames... +[2023-03-07 15:44:27,021][283408] Decorrelating experience for 992 frames... +[2023-03-07 15:44:27,083][281975] Decorrelating experience for 832 frames... +[2023-03-07 15:44:27,096][288232] Decorrelating experience for 800 frames... +[2023-03-07 15:44:27,113][281958] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,151][282891] Decorrelating experience for 928 frames... +[2023-03-07 15:44:27,161][283854] Decorrelating experience for 736 frames... +[2023-03-07 15:44:27,171][281579] Decorrelating experience for 928 frames... +[2023-03-07 15:44:27,228][281670] Decorrelating experience for 960 frames... +[2023-03-07 15:44:27,228][281972] Decorrelating experience for 864 frames... +[2023-03-07 15:44:27,285][283085] Decorrelating experience for 512 frames... +[2023-03-07 15:44:27,304][281665] Decorrelating experience for 736 frames... +[2023-03-07 15:44:27,305][281956] Decorrelating experience for 576 frames... +[2023-03-07 15:44:27,314][281961] Decorrelating experience for 768 frames... +[2023-03-07 15:44:27,361][281653] Decorrelating experience for 704 frames... +[2023-03-07 15:44:27,364][281992] Decorrelating experience for 864 frames... +[2023-03-07 15:44:27,369][284166] Decorrelating experience for 960 frames... +[2023-03-07 15:44:27,395][283352] Decorrelating experience for 800 frames... +[2023-03-07 15:44:27,433][283150] Decorrelating experience for 960 frames... +[2023-03-07 15:44:27,437][281809] Decorrelating experience for 768 frames... +[2023-03-07 15:44:27,486][283408] Stopping RolloutWorker_w108... +[2023-03-07 15:44:27,486][281807] Stopping RolloutWorker_w31... +[2023-03-07 15:44:27,487][283408] Loop rollout_proc108_evt_loop terminating... +[2023-03-07 15:44:27,487][281807] Loop rollout_proc31_evt_loop terminating... +[2023-03-07 15:44:27,488][281669] Decorrelating experience for 544 frames... +[2023-03-07 15:44:27,487][236315] Component RolloutWorker_w108 stopped! +[2023-03-07 15:44:27,489][236315] Component RolloutWorker_w31 stopped! +[2023-03-07 15:44:27,501][281953] Decorrelating experience for 544 frames... +[2023-03-07 15:44:27,518][282904] Decorrelating experience for 608 frames... +[2023-03-07 15:44:27,529][281804] Decorrelating experience for 800 frames... +[2023-03-07 15:44:27,561][281973] Decorrelating experience for 928 frames... +[2023-03-07 15:44:27,565][281957] Decorrelating experience for 576 frames... +[2023-03-07 15:44:27,604][283641] Decorrelating experience for 992 frames... +[2023-03-07 15:44:27,641][283854] Decorrelating experience for 768 frames... +[2023-03-07 15:44:27,658][281661] Decorrelating experience for 800 frames... +[2023-03-07 15:44:27,695][281954] Decorrelating experience for 992 frames... +[2023-03-07 15:44:27,708][282891] Decorrelating experience for 960 frames... +[2023-03-07 15:44:27,751][281662] Decorrelating experience for 928 frames... +[2023-03-07 15:44:27,768][281984] Decorrelating experience for 672 frames... +[2023-03-07 15:44:27,768][281972] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,854][281670] Decorrelating experience for 992 frames... +[2023-03-07 15:44:27,854][281975] Decorrelating experience for 864 frames... +[2023-03-07 15:44:27,858][281952] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,858][282827] Decorrelating experience for 864 frames... +[2023-03-07 15:44:27,887][281578] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,897][281992] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,949][288232] Decorrelating experience for 832 frames... +[2023-03-07 15:44:27,963][281577] Decorrelating experience for 896 frames... +[2023-03-07 15:44:27,966][281669] Decorrelating experience for 576 frames... +[2023-03-07 15:44:28,052][283641] Stopping RolloutWorker_w94... +[2023-03-07 15:44:28,052][283641] Loop rollout_proc94_evt_loop terminating... +[2023-03-07 15:44:28,053][236315] Component RolloutWorker_w94 stopped! +[2023-03-07 15:44:28,055][281964] Decorrelating experience for 576 frames... +[2023-03-07 15:44:28,085][281800] Decorrelating experience for 800 frames... +[2023-03-07 15:44:28,095][282695] Decorrelating experience for 896 frames... +[2023-03-07 15:44:28,132][284200] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,134][281957] Decorrelating experience for 608 frames... +[2023-03-07 15:44:28,157][281953] Decorrelating experience for 576 frames... +[2023-03-07 15:44:28,160][281954] Stopping RolloutWorker_w50... +[2023-03-07 15:44:28,160][281954] Loop rollout_proc50_evt_loop terminating... +[2023-03-07 15:44:28,160][236315] Component RolloutWorker_w50 stopped! +[2023-03-07 15:44:28,163][283352] Decorrelating experience for 832 frames... +[2023-03-07 15:44:28,219][281665] Decorrelating experience for 768 frames... +[2023-03-07 15:44:28,253][281990] Decorrelating experience for 768 frames... +[2023-03-07 15:44:28,277][283854] Decorrelating experience for 800 frames... +[2023-03-07 15:44:28,293][282904] Decorrelating experience for 640 frames... +[2023-03-07 15:44:28,312][281670] Stopping RolloutWorker_w7... +[2023-03-07 15:44:28,312][281670] Loop rollout_proc7_evt_loop terminating... +[2023-03-07 15:44:28,312][236315] Component RolloutWorker_w7 stopped! +[2023-03-07 15:44:28,339][281579] Decorrelating experience for 960 frames... +[2023-03-07 15:44:28,349][282558] Decorrelating experience for 832 frames... +[2023-03-07 15:44:28,357][281972] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,428][281975] Decorrelating experience for 896 frames... +[2023-03-07 15:44:28,454][281958] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,466][281815] Decorrelating experience for 992 frames... +[2023-03-07 15:44:28,478][281992] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,506][281577] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,540][284166] Decorrelating experience for 992 frames... +[2023-03-07 15:44:28,554][281991] Decorrelating experience for 768 frames... +[2023-03-07 15:44:28,559][283150] Decorrelating experience for 992 frames... +[2023-03-07 15:44:28,641][281578] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,657][281951] Decorrelating experience for 832 frames... +[2023-03-07 15:44:28,659][281651] Decorrelating experience for 704 frames... +[2023-03-07 15:44:28,690][281809] Decorrelating experience for 800 frames... +[2023-03-07 15:44:28,745][281984] Decorrelating experience for 704 frames... +[2023-03-07 15:44:28,768][281973] Decorrelating experience for 960 frames... +[2023-03-07 15:44:28,768][281957] Decorrelating experience for 640 frames... +[2023-03-07 15:44:28,772][283854] Decorrelating experience for 832 frames... +[2023-03-07 15:44:28,828][282827] Decorrelating experience for 896 frames... +[2023-03-07 15:44:28,845][281952] Decorrelating experience for 928 frames... +[2023-03-07 15:44:28,861][283960] Decorrelating experience for 480 frames... +[2023-03-07 15:44:28,866][281970] Decorrelating experience for 672 frames... +[2023-03-07 15:44:28,895][281662] Decorrelating experience for 960 frames... +[2023-03-07 15:44:28,911][281815] Stopping RolloutWorker_w43... +[2023-03-07 15:44:28,911][281815] Loop rollout_proc43_evt_loop terminating... +[2023-03-07 15:44:28,911][236315] Component RolloutWorker_w43 stopped! +[2023-03-07 15:44:28,983][284200] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,009][283150] Stopping RolloutWorker_w96... +[2023-03-07 15:44:29,010][283150] Loop rollout_proc96_evt_loop terminating... +[2023-03-07 15:44:29,010][236315] Component RolloutWorker_w96 stopped! +[2023-03-07 15:44:29,021][284166] Stopping RolloutWorker_w112... +[2023-03-07 15:44:29,021][284166] Loop rollout_proc112_evt_loop terminating... +[2023-03-07 15:44:29,022][236315] Component RolloutWorker_w112 stopped! +[2023-03-07 15:44:29,024][288232] Decorrelating experience for 864 frames... +[2023-03-07 15:44:29,034][282558] Decorrelating experience for 864 frames... +[2023-03-07 15:44:29,036][283019] Decorrelating experience for 704 frames... +[2023-03-07 15:44:29,051][281972] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,064][281948] Decorrelating experience for 896 frames... +[2023-03-07 15:44:29,067][281665] Decorrelating experience for 800 frames... +[2023-03-07 15:44:29,093][281577] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,115][281808] Decorrelating experience for 864 frames... +[2023-03-07 15:44:29,125][281651] Decorrelating experience for 736 frames... +[2023-03-07 15:44:29,210][281578] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,233][282891] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,247][282695] Decorrelating experience for 928 frames... +[2023-03-07 15:44:29,272][282184] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,272][281975] Decorrelating experience for 928 frames... +[2023-03-07 15:44:29,281][281809] Decorrelating experience for 832 frames... +[2023-03-07 15:44:29,301][283469] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,314][281669] Decorrelating experience for 608 frames... +[2023-03-07 15:44:29,321][281581] Decorrelating experience for 896 frames... +[2023-03-07 15:44:29,339][281970] Decorrelating experience for 704 frames... +[2023-03-07 15:44:29,413][281984] Decorrelating experience for 736 frames... +[2023-03-07 15:44:29,443][283960] Decorrelating experience for 512 frames... +[2023-03-07 15:44:29,444][281991] Decorrelating experience for 800 frames... +[2023-03-07 15:44:29,466][281806] Decorrelating experience for 928 frames... +[2023-03-07 15:44:29,483][281959] Another process currently holds the lock /tmp/sf2_rolo/doom_000.lockfile, attempt: 1 +[2023-03-07 15:44:29,487][281973] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,504][281579] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,526][281953] Decorrelating experience for 608 frames... +[2023-03-07 15:44:29,549][288232] Decorrelating experience for 896 frames... +[2023-03-07 15:44:29,637][281956] Decorrelating experience for 608 frames... +[2023-03-07 15:44:29,638][281952] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,661][281662] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,666][284304] Decorrelating experience for 672 frames... +[2023-03-07 15:44:29,690][282891] Stopping RolloutWorker_w116... +[2023-03-07 15:44:29,690][282891] Loop rollout_proc116_evt_loop terminating... +[2023-03-07 15:44:29,691][236315] Component RolloutWorker_w116 stopped! +[2023-03-07 15:44:29,697][281577] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,709][283085] Decorrelating experience for 544 frames... +[2023-03-07 15:44:29,724][282184] Stopping RolloutWorker_w70... +[2023-03-07 15:44:29,724][282184] Loop rollout_proc70_evt_loop terminating... +[2023-03-07 15:44:29,724][236315] Component RolloutWorker_w70 stopped! +[2023-03-07 15:44:29,735][281957] Decorrelating experience for 672 frames... +[2023-03-07 15:44:29,754][281808] Decorrelating experience for 896 frames... +[2023-03-07 15:44:29,782][284200] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,792][281578] Decorrelating experience for 992 frames... +[2023-03-07 15:44:29,837][281964] Decorrelating experience for 608 frames... +[2023-03-07 15:44:29,876][281663] Decorrelating experience for 896 frames... +[2023-03-07 15:44:29,909][283469] Stopping RolloutWorker_w123... +[2023-03-07 15:44:29,909][283469] Loop rollout_proc123_evt_loop terminating... +[2023-03-07 15:44:29,910][281651] Decorrelating experience for 768 frames... +[2023-03-07 15:44:29,910][236315] Component RolloutWorker_w123 stopped! +[2023-03-07 15:44:29,911][282695] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,929][281975] Decorrelating experience for 960 frames... +[2023-03-07 15:44:29,940][282827] Decorrelating experience for 928 frames... +[2023-03-07 15:44:29,951][281973] Stopping RolloutWorker_w87... +[2023-03-07 15:44:29,951][281973] Loop rollout_proc87_evt_loop terminating... +[2023-03-07 15:44:29,951][281953] Decorrelating experience for 640 frames... +[2023-03-07 15:44:29,951][236315] Component RolloutWorker_w87 stopped! +[2023-03-07 15:44:29,983][281579] Stopping RolloutWorker_w14... +[2023-03-07 15:44:29,983][281579] Loop rollout_proc14_evt_loop terminating... +[2023-03-07 15:44:29,983][236315] Component RolloutWorker_w14 stopped! +[2023-03-07 15:44:29,992][281809] Decorrelating experience for 864 frames... +[2023-03-07 15:44:30,031][281656] Decorrelating experience for 832 frames... +[2023-03-07 15:44:30,081][281959] Decorrelating experience for 608 frames... +[2023-03-07 15:44:30,139][281967] Decorrelating experience for 928 frames... +[2023-03-07 15:44:30,146][288232] Decorrelating experience for 928 frames... +[2023-03-07 15:44:30,172][281970] Decorrelating experience for 736 frames... +[2023-03-07 15:44:30,181][281577] Stopping RolloutWorker_w8... +[2023-03-07 15:44:30,182][281577] Loop rollout_proc8_evt_loop terminating... +[2023-03-07 15:44:30,182][236315] Component RolloutWorker_w8 stopped! +[2023-03-07 15:44:30,189][281669] Decorrelating experience for 640 frames... +[2023-03-07 15:44:30,192][281661] Decorrelating experience for 832 frames... +[2023-03-07 15:44:30,237][281581] Decorrelating experience for 928 frames... +[2023-03-07 15:44:30,253][281662] Stopping RolloutWorker_w27... +[2023-03-07 15:44:30,253][281662] Loop rollout_proc27_evt_loop terminating... +[2023-03-07 15:44:30,253][236315] Component RolloutWorker_w27 stopped! +[2023-03-07 15:44:30,278][281951] Decorrelating experience for 864 frames... +[2023-03-07 15:44:30,285][281578] Stopping RolloutWorker_w11... +[2023-03-07 15:44:30,285][281578] Loop rollout_proc11_evt_loop terminating... +[2023-03-07 15:44:30,285][236315] Component RolloutWorker_w11 stopped! +[2023-03-07 15:44:30,297][281808] Decorrelating experience for 928 frames... +[2023-03-07 15:44:30,341][283854] Decorrelating experience for 864 frames... +[2023-03-07 15:44:30,351][284200] Stopping RolloutWorker_w106... +[2023-03-07 15:44:30,351][284200] Loop rollout_proc106_evt_loop terminating... +[2023-03-07 15:44:30,351][236315] Component RolloutWorker_w106 stopped! +[2023-03-07 15:44:30,359][281957] Decorrelating experience for 704 frames... +[2023-03-07 15:44:30,375][281972] Decorrelating experience for 992 frames... +[2023-03-07 15:44:30,382][283960] Decorrelating experience for 544 frames... +[2023-03-07 15:44:30,417][281663] Decorrelating experience for 928 frames... +[2023-03-07 15:44:30,418][281956] Decorrelating experience for 640 frames... +[2023-03-07 15:44:30,423][283085] Decorrelating experience for 576 frames... +[2023-03-07 15:44:30,476][283352] Decorrelating experience for 864 frames... +[2023-03-07 15:44:30,504][281964] Decorrelating experience for 640 frames... +[2023-03-07 15:44:30,543][281810] Decorrelating experience for 832 frames... +[2023-03-07 15:44:30,613][281651] Decorrelating experience for 800 frames... +[2023-03-07 15:44:30,615][281959] Decorrelating experience for 640 frames... +[2023-03-07 15:44:30,624][281809] Decorrelating experience for 896 frames... +[2023-03-07 15:44:30,629][281953] Decorrelating experience for 672 frames... +[2023-03-07 15:44:30,681][284047] Decorrelating experience for 896 frames... +[2023-03-07 15:44:30,691][282695] Decorrelating experience for 992 frames... +[2023-03-07 15:44:30,724][281669] Decorrelating experience for 672 frames... +[2023-03-07 15:44:30,744][283019] Decorrelating experience for 736 frames... +[2023-03-07 15:44:30,766][283727] Decorrelating experience for 736 frames... +[2023-03-07 15:44:30,826][281806] Decorrelating experience for 960 frames... +[2023-03-07 15:44:30,831][281665] Decorrelating experience for 832 frames... +[2023-03-07 15:44:30,832][281972] Stopping RolloutWorker_w66... +[2023-03-07 15:44:30,832][281972] Loop rollout_proc66_evt_loop terminating... +[2023-03-07 15:44:30,832][236315] Component RolloutWorker_w66 stopped! +[2023-03-07 15:44:30,844][281808] Decorrelating experience for 960 frames... +[2023-03-07 15:44:30,891][281967] Decorrelating experience for 960 frames... +[2023-03-07 15:44:30,911][281661] Decorrelating experience for 864 frames... +[2023-03-07 15:44:30,949][282696] Decorrelating experience for 800 frames... +[2023-03-07 15:44:30,958][281956] Decorrelating experience for 672 frames... +[2023-03-07 15:44:30,961][281663] Decorrelating experience for 960 frames... +[2023-03-07 15:44:31,025][281800] Decorrelating experience for 832 frames... +[2023-03-07 15:44:31,028][283960] Decorrelating experience for 576 frames... +[2023-03-07 15:44:31,108][281651] Decorrelating experience for 832 frames... +[2023-03-07 15:44:31,115][281957] Decorrelating experience for 736 frames... +[2023-03-07 15:44:31,138][282697] Decorrelating experience for 896 frames... +[2023-03-07 15:44:31,151][281970] Decorrelating experience for 768 frames... +[2023-03-07 15:44:31,153][281959] Decorrelating experience for 672 frames... +[2023-03-07 15:44:31,160][281809] Decorrelating experience for 928 frames... +[2023-03-07 15:44:31,165][281964] Decorrelating experience for 672 frames... +[2023-03-07 15:44:31,297][284304] Decorrelating experience for 704 frames... +[2023-03-07 15:44:31,301][281975] Decorrelating experience for 992 frames... +[2023-03-07 15:44:31,309][282827] Decorrelating experience for 960 frames... +[2023-03-07 15:44:31,311][282695] Stopping RolloutWorker_w79... +[2023-03-07 15:44:31,312][282695] Loop rollout_proc79_evt_loop terminating... +[2023-03-07 15:44:31,312][236315] Component RolloutWorker_w79 stopped! +[2023-03-07 15:44:31,324][281669] Decorrelating experience for 704 frames... +[2023-03-07 15:44:31,343][282183] Decorrelating experience for 736 frames... +[2023-03-07 15:44:31,348][283352] Decorrelating experience for 896 frames... +[2023-03-07 15:44:31,364][283019] Decorrelating experience for 768 frames... +[2023-03-07 15:44:31,391][281806] Decorrelating experience for 992 frames... +[2023-03-07 15:44:31,458][281581] Decorrelating experience for 960 frames... +[2023-03-07 15:44:31,507][281656] Decorrelating experience for 864 frames... +[2023-03-07 15:44:31,507][281992] Decorrelating experience for 960 frames... +[2023-03-07 15:44:31,515][288232] Decorrelating experience for 960 frames... +[2023-03-07 15:44:31,544][281952] Decorrelating experience for 992 frames... +[2023-03-07 15:44:31,586][284047] Decorrelating experience for 928 frames... +[2023-03-07 15:44:31,593][282696] Decorrelating experience for 832 frames... +[2023-03-07 15:44:31,610][281967] Decorrelating experience for 992 frames... +[2023-03-07 15:44:31,660][281808] Decorrelating experience for 992 frames... +[2023-03-07 15:44:31,707][281805] Decorrelating experience for 768 frames... +[2023-03-07 15:44:31,712][281956] Decorrelating experience for 704 frames... +[2023-03-07 15:44:31,718][283854] Decorrelating experience for 896 frames... +[2023-03-07 15:44:31,766][281975] Stopping RolloutWorker_w57... +[2023-03-07 15:44:31,766][281975] Loop rollout_proc57_evt_loop terminating... +[2023-03-07 15:44:31,766][236315] Component RolloutWorker_w57 stopped! +[2023-03-07 15:44:31,775][281957] Decorrelating experience for 768 frames... +[2023-03-07 15:44:31,785][281669] Decorrelating experience for 736 frames... +[2023-03-07 15:44:31,804][287383] Decorrelating experience for 672 frames... +[2023-03-07 15:44:31,814][282183] Decorrelating experience for 768 frames... +[2023-03-07 15:44:31,862][281806] Stopping RolloutWorker_w16... +[2023-03-07 15:44:31,862][281806] Loop rollout_proc16_evt_loop terminating... +[2023-03-07 15:44:31,862][236315] Component RolloutWorker_w16 stopped! +[2023-03-07 15:44:31,903][281804] Decorrelating experience for 832 frames... +[2023-03-07 15:44:31,945][281970] Decorrelating experience for 800 frames... +[2023-03-07 15:44:31,968][281964] Decorrelating experience for 704 frames... +[2023-03-07 15:44:32,000][281663] Decorrelating experience for 992 frames... +[2023-03-07 15:44:32,003][281952] Stopping RolloutWorker_w59... +[2023-03-07 15:44:32,003][281952] Loop rollout_proc59_evt_loop terminating... +[2023-03-07 15:44:32,003][236315] Component RolloutWorker_w59 stopped! +[2023-03-07 15:44:32,013][283019] Decorrelating experience for 800 frames... +[2023-03-07 15:44:32,014][281959] Decorrelating experience for 704 frames... +[2023-03-07 15:44:32,030][281656] Decorrelating experience for 896 frames... +[2023-03-07 15:44:32,064][281992] Decorrelating experience for 992 frames... +[2023-03-07 15:44:32,088][282827] Decorrelating experience for 992 frames... +[2023-03-07 15:44:32,093][281654] Decorrelating experience for 928 frames... +[2023-03-07 15:44:32,100][281967] Stopping RolloutWorker_w60... +[2023-03-07 15:44:32,100][281967] Loop rollout_proc60_evt_loop terminating... +[2023-03-07 15:44:32,100][236315] Component RolloutWorker_w60 stopped! +[2023-03-07 15:44:32,125][281808] Stopping RolloutWorker_w25... +[2023-03-07 15:44:32,125][281808] Loop rollout_proc25_evt_loop terminating... +[2023-03-07 15:44:32,125][236315] Component RolloutWorker_w25 stopped! +[2023-03-07 15:44:32,175][281956] Decorrelating experience for 736 frames... +[2023-03-07 15:44:32,183][281953] Decorrelating experience for 704 frames... +[2023-03-07 15:44:32,195][283960] Decorrelating experience for 608 frames... +[2023-03-07 15:44:32,197][281961] Decorrelating experience for 800 frames... +[2023-03-07 15:44:32,271][284304] Decorrelating experience for 736 frames... +[2023-03-07 15:44:32,289][281988] Decorrelating experience for 928 frames... +[2023-03-07 15:44:32,293][283727] Decorrelating experience for 768 frames... +[2023-03-07 15:44:32,306][282183] Decorrelating experience for 800 frames... +[2023-03-07 15:44:32,364][284047] Decorrelating experience for 960 frames... +[2023-03-07 15:44:32,376][281651] Decorrelating experience for 864 frames... +[2023-03-07 15:44:32,398][281665] Decorrelating experience for 864 frames... +[2023-03-07 15:44:32,403][281581] Decorrelating experience for 992 frames... +[2023-03-07 15:44:32,446][281663] Stopping RolloutWorker_w30... +[2023-03-07 15:44:32,446][281663] Loop rollout_proc30_evt_loop terminating... +[2023-03-07 15:44:32,446][236315] Component RolloutWorker_w30 stopped! +[2023-03-07 15:44:32,483][283352] Decorrelating experience for 928 frames... +[2023-03-07 15:44:32,507][281992] Stopping RolloutWorker_w85... +[2023-03-07 15:44:32,507][281992] Loop rollout_proc85_evt_loop terminating... +[2023-03-07 15:44:32,507][236315] Component RolloutWorker_w85 stopped! +[2023-03-07 15:44:32,510][287383] Decorrelating experience for 704 frames... +[2023-03-07 15:44:32,515][281804] Decorrelating experience for 864 frames... +[2023-03-07 15:44:32,543][281959] Decorrelating experience for 736 frames... +[2023-03-07 15:44:32,552][281800] Decorrelating experience for 864 frames... +[2023-03-07 15:44:32,602][281963] Decorrelating experience for 512 frames... +[2023-03-07 15:44:32,620][282827] Stopping RolloutWorker_w110... +[2023-03-07 15:44:32,621][282827] Loop rollout_proc110_evt_loop terminating... +[2023-03-07 15:44:32,621][236315] Component RolloutWorker_w110 stopped! +[2023-03-07 15:44:32,645][281957] Decorrelating experience for 800 frames... +[2023-03-07 15:44:32,666][281970] Decorrelating experience for 832 frames... +[2023-03-07 15:44:32,736][281961] Decorrelating experience for 832 frames... +[2023-03-07 15:44:32,748][281956] Decorrelating experience for 768 frames... +[2023-03-07 15:44:32,768][281964] Decorrelating experience for 736 frames... +[2023-03-07 15:44:32,805][281991] Decorrelating experience for 832 frames... +[2023-03-07 15:44:32,808][281953] Decorrelating experience for 736 frames... +[2023-03-07 15:44:32,828][282697] Decorrelating experience for 928 frames... +[2023-03-07 15:44:32,858][281669] Decorrelating experience for 768 frames... +[2023-03-07 15:44:32,940][282183] Decorrelating experience for 832 frames... +[2023-03-07 15:44:32,951][281581] Stopping RolloutWorker_w17... +[2023-03-07 15:44:32,952][281581] Loop rollout_proc17_evt_loop terminating... +[2023-03-07 15:44:32,951][236315] Component RolloutWorker_w17 stopped! +[2023-03-07 15:44:32,958][281665] Decorrelating experience for 896 frames... +[2023-03-07 15:44:32,986][281963] Decorrelating experience for 544 frames... +[2023-03-07 15:44:33,012][281984] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,019][284053] Decorrelating experience for 928 frames... +[2023-03-07 15:44:33,024][283727] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,029][281804] Decorrelating experience for 896 frames... +[2023-03-07 15:44:33,037][281654] Decorrelating experience for 960 frames... +[2023-03-07 15:44:33,062][281656] Decorrelating experience for 928 frames... +[2023-03-07 15:44:33,160][287383] Decorrelating experience for 736 frames... +[2023-03-07 15:44:33,194][281800] Decorrelating experience for 896 frames... +[2023-03-07 15:44:33,215][284047] Decorrelating experience for 992 frames... +[2023-03-07 15:44:33,222][282558] Decorrelating experience for 896 frames... +[2023-03-07 15:44:33,223][283960] Decorrelating experience for 640 frames... +[2023-03-07 15:44:33,230][281956] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,241][281651] Decorrelating experience for 896 frames... +[2023-03-07 15:44:33,292][282904] Decorrelating experience for 672 frames... +[2023-03-07 15:44:33,370][284304] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,378][281964] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,419][281653] Decorrelating experience for 736 frames... +[2023-03-07 15:44:33,437][281961] Decorrelating experience for 864 frames... +[2023-03-07 15:44:33,452][282696] Decorrelating experience for 864 frames... +[2023-03-07 15:44:33,481][288232] Decorrelating experience for 992 frames... +[2023-03-07 15:44:33,530][281970] Decorrelating experience for 864 frames... +[2023-03-07 15:44:33,546][281804] Decorrelating experience for 928 frames... +[2023-03-07 15:44:33,596][281669] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,619][281953] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,623][283085] Decorrelating experience for 608 frames... +[2023-03-07 15:44:33,661][284053] Decorrelating experience for 960 frames... +[2023-03-07 15:44:33,674][281984] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,676][281959] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,677][282697] Decorrelating experience for 960 frames... +[2023-03-07 15:44:33,750][283960] Decorrelating experience for 672 frames... +[2023-03-07 15:44:33,766][281651] Decorrelating experience for 928 frames... +[2023-03-07 15:44:33,794][283727] Decorrelating experience for 832 frames... +[2023-03-07 15:44:33,811][284047] Stopping RolloutWorker_w115... +[2023-03-07 15:44:33,812][284047] Loop rollout_proc115_evt_loop terminating... +[2023-03-07 15:44:33,812][236315] Component RolloutWorker_w115 stopped! +[2023-03-07 15:44:33,820][281956] Decorrelating experience for 832 frames... +[2023-03-07 15:44:33,826][287383] Decorrelating experience for 768 frames... +[2023-03-07 15:44:33,866][281805] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,868][281991] Decorrelating experience for 864 frames... +[2023-03-07 15:44:33,894][281957] Decorrelating experience for 832 frames... +[2023-03-07 15:44:33,904][281964] Decorrelating experience for 800 frames... +[2023-03-07 15:44:33,925][281800] Decorrelating experience for 928 frames... +[2023-03-07 15:44:33,927][288232] Stopping RolloutWorker_w1... +[2023-03-07 15:44:33,928][288232] Loop rollout_proc1_evt_loop terminating... +[2023-03-07 15:44:33,928][236315] Component RolloutWorker_w1 stopped! +[2023-03-07 15:44:33,992][281653] Decorrelating experience for 768 frames... +[2023-03-07 15:44:34,003][281961] Decorrelating experience for 896 frames... +[2023-03-07 15:44:34,028][281948] Decorrelating experience for 928 frames... +[2023-03-07 15:44:34,070][283854] Decorrelating experience for 928 frames... +[2023-03-07 15:44:34,076][282696] Decorrelating experience for 896 frames... +[2023-03-07 15:44:34,091][283085] Decorrelating experience for 640 frames... +[2023-03-07 15:44:34,096][281669] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,155][284304] Decorrelating experience for 800 frames... +[2023-03-07 15:44:34,188][281970] Decorrelating experience for 896 frames... +[2023-03-07 15:44:34,269][283019] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,271][281990] Decorrelating experience for 800 frames... +[2023-03-07 15:44:34,299][282183] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,301][281810] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,308][287383] Decorrelating experience for 800 frames... +[2023-03-07 15:44:34,355][281805] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,387][283727] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,399][281984] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,473][283352] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,479][281809] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,496][281964] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,524][281961] Decorrelating experience for 928 frames... +[2023-03-07 15:44:34,558][281948] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,562][282904] Decorrelating experience for 704 frames... +[2023-03-07 15:44:34,589][281669] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,596][281953] Decorrelating experience for 800 frames... +[2023-03-07 15:44:34,620][281653] Decorrelating experience for 800 frames... +[2023-03-07 15:44:34,678][283085] Decorrelating experience for 672 frames... +[2023-03-07 15:44:34,697][281957] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,745][281988] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,752][281951] Decorrelating experience for 896 frames... +[2023-03-07 15:44:34,767][284053] Decorrelating experience for 992 frames... +[2023-03-07 15:44:34,783][281956] Decorrelating experience for 864 frames... +[2023-03-07 15:44:34,789][287383] Decorrelating experience for 832 frames... +[2023-03-07 15:44:34,812][281970] Decorrelating experience for 928 frames... +[2023-03-07 15:44:34,889][281665] Decorrelating experience for 928 frames... +[2023-03-07 15:44:34,905][283854] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,946][281804] Decorrelating experience for 960 frames... +[2023-03-07 15:44:34,947][283960] Decorrelating experience for 704 frames... +[2023-03-07 15:44:34,972][281959] Decorrelating experience for 800 frames... +[2023-03-07 15:44:35,005][281964] Decorrelating experience for 864 frames... +[2023-03-07 15:44:35,010][281651] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,029][281809] Decorrelating experience for 992 frames... +[2023-03-07 15:44:35,090][281654] Decorrelating experience for 992 frames... +[2023-03-07 15:44:35,099][281805] Decorrelating experience for 864 frames... +[2023-03-07 15:44:35,102][281669] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,109][281653] Decorrelating experience for 832 frames... +[2023-03-07 15:44:35,134][281661] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,162][281961] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,188][281984] Decorrelating experience for 864 frames... +[2023-03-07 15:44:35,200][284053] Stopping RolloutWorker_w118... +[2023-03-07 15:44:35,200][284053] Loop rollout_proc118_evt_loop terminating... +[2023-03-07 15:44:35,200][236315] Component RolloutWorker_w118 stopped! +[2023-03-07 15:44:35,292][281991] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,293][282904] Decorrelating experience for 736 frames... +[2023-03-07 15:44:35,302][281956] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,342][281958] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,342][287383] Decorrelating experience for 864 frames... +[2023-03-07 15:44:35,351][281988] Decorrelating experience for 992 frames... +[2023-03-07 15:44:35,374][281800] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,409][282558] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,447][283854] Decorrelating experience for 992 frames... +[2023-03-07 15:44:35,464][281809] Stopping RolloutWorker_w28... +[2023-03-07 15:44:35,465][281809] Loop rollout_proc28_evt_loop terminating... +[2023-03-07 15:44:35,464][236315] Component RolloutWorker_w28 stopped! +[2023-03-07 15:44:35,489][283727] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,531][281654] Stopping RolloutWorker_w6... +[2023-03-07 15:44:35,531][281654] Loop rollout_proc6_evt_loop terminating... +[2023-03-07 15:44:35,531][236315] Component RolloutWorker_w6 stopped! +[2023-03-07 15:44:35,548][281951] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,598][283960] Decorrelating experience for 736 frames... +[2023-03-07 15:44:35,610][281653] Decorrelating experience for 864 frames... +[2023-03-07 15:44:35,615][281656] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,628][281669] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,629][281810] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,629][282696] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,642][281959] Decorrelating experience for 832 frames... +[2023-03-07 15:44:35,662][281661] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,714][281957] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,754][284304] Decorrelating experience for 832 frames... +[2023-03-07 15:44:35,782][281988] Stopping RolloutWorker_w76... +[2023-03-07 15:44:35,782][281988] Loop rollout_proc76_evt_loop terminating... +[2023-03-07 15:44:35,782][236315] Component RolloutWorker_w76 stopped! +[2023-03-07 15:44:35,819][281948] Decorrelating experience for 992 frames... +[2023-03-07 15:44:35,821][282183] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,830][281665] Decorrelating experience for 960 frames... +[2023-03-07 15:44:35,855][281984] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,872][281964] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,880][283854] Stopping RolloutWorker_w120... +[2023-03-07 15:44:35,880][283854] Loop rollout_proc120_evt_loop terminating... +[2023-03-07 15:44:35,880][236315] Component RolloutWorker_w120 stopped! +[2023-03-07 15:44:35,882][281805] Decorrelating experience for 896 frames... +[2023-03-07 15:44:35,886][281956] Decorrelating experience for 928 frames... +[2023-03-07 15:44:35,966][281651] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,015][287383] Decorrelating experience for 896 frames... +[2023-03-07 15:44:36,048][283352] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,058][281970] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,071][283960] Decorrelating experience for 768 frames... +[2023-03-07 15:44:36,103][282558] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,165][281656] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,169][282904] Decorrelating experience for 768 frames... +[2023-03-07 15:44:36,171][281961] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,219][281991] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,234][281959] Decorrelating experience for 864 frames... +[2023-03-07 15:44:36,257][281948] Stopping RolloutWorker_w47... +[2023-03-07 15:44:36,257][281948] Loop rollout_proc47_evt_loop terminating... +[2023-03-07 15:44:36,257][236315] Component RolloutWorker_w47 stopped! +[2023-03-07 15:44:36,260][281661] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,264][281958] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,271][282697] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,295][281957] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,342][282696] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,390][283727] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,415][281651] Stopping RolloutWorker_w41... +[2023-03-07 15:44:36,415][281651] Loop rollout_proc41_evt_loop terminating... +[2023-03-07 15:44:36,415][236315] Component RolloutWorker_w41 stopped! +[2023-03-07 15:44:36,440][284304] Decorrelating experience for 864 frames... +[2023-03-07 15:44:36,457][281805] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,458][281964] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,481][283352] Stopping RolloutWorker_w117... +[2023-03-07 15:44:36,482][283352] Loop rollout_proc117_evt_loop terminating... +[2023-03-07 15:44:36,482][236315] Component RolloutWorker_w117 stopped! +[2023-03-07 15:44:36,490][281653] Decorrelating experience for 896 frames... +[2023-03-07 15:44:36,546][287383] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,549][283960] Decorrelating experience for 800 frames... +[2023-03-07 15:44:36,563][281800] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,601][281656] Stopping RolloutWorker_w9... +[2023-03-07 15:44:36,601][281656] Loop rollout_proc9_evt_loop terminating... +[2023-03-07 15:44:36,601][236315] Component RolloutWorker_w9 stopped! +[2023-03-07 15:44:36,617][281961] Stopping RolloutWorker_w77... +[2023-03-07 15:44:36,618][281961] Loop rollout_proc77_evt_loop terminating... +[2023-03-07 15:44:36,617][236315] Component RolloutWorker_w77 stopped! +[2023-03-07 15:44:36,642][281990] Decorrelating experience for 832 frames... +[2023-03-07 15:44:36,646][281970] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,661][281810] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,700][281804] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,712][282697] Stopping RolloutWorker_w95... +[2023-03-07 15:44:36,712][282697] Loop rollout_proc95_evt_loop terminating... +[2023-03-07 15:44:36,712][236315] Component RolloutWorker_w95 stopped! +[2023-03-07 15:44:36,724][281958] Stopping RolloutWorker_w65... +[2023-03-07 15:44:36,725][281958] Loop rollout_proc65_evt_loop terminating... +[2023-03-07 15:44:36,725][236315] Component RolloutWorker_w65 stopped! +[2023-03-07 15:44:36,766][281951] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,796][281984] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,856][281957] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,899][281956] Decorrelating experience for 960 frames... +[2023-03-07 15:44:36,905][283019] Decorrelating experience for 864 frames... +[2023-03-07 15:44:36,926][282558] Decorrelating experience for 992 frames... +[2023-03-07 15:44:36,927][282904] Decorrelating experience for 800 frames... +[2023-03-07 15:44:36,987][282183] Decorrelating experience for 928 frames... +[2023-03-07 15:44:36,996][281963] Decorrelating experience for 576 frames... +[2023-03-07 15:44:37,023][281800] Stopping RolloutWorker_w4... +[2023-03-07 15:44:37,023][281800] Loop rollout_proc4_evt_loop terminating... +[2023-03-07 15:44:37,023][236315] Component RolloutWorker_w4 stopped! +[2023-03-07 15:44:37,057][284304] Decorrelating experience for 896 frames... +[2023-03-07 15:44:37,091][283085] Decorrelating experience for 704 frames... +[2023-03-07 15:44:37,093][282696] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,100][281964] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,118][281991] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,121][281661] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,140][281970] Stopping RolloutWorker_w51... +[2023-03-07 15:44:37,141][281970] Loop rollout_proc51_evt_loop terminating... +[2023-03-07 15:44:37,140][236315] Component RolloutWorker_w51 stopped! +[2023-03-07 15:44:37,146][281804] Stopping RolloutWorker_w45... +[2023-03-07 15:44:37,146][281804] Loop rollout_proc45_evt_loop terminating... +[2023-03-07 15:44:37,146][236315] Component RolloutWorker_w45 stopped! +[2023-03-07 15:44:37,193][281990] Decorrelating experience for 864 frames... +[2023-03-07 15:44:37,194][281805] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,295][281953] Decorrelating experience for 832 frames... +[2023-03-07 15:44:37,322][281653] Decorrelating experience for 928 frames... +[2023-03-07 15:44:37,336][287383] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,349][283960] Decorrelating experience for 832 frames... +[2023-03-07 15:44:37,413][281665] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,420][281957] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,426][281810] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,437][281963] Decorrelating experience for 608 frames... +[2023-03-07 15:44:37,506][282558] Stopping RolloutWorker_w58... +[2023-03-07 15:44:37,507][282558] Loop rollout_proc58_evt_loop terminating... +[2023-03-07 15:44:37,506][236315] Component RolloutWorker_w58 stopped! +[2023-03-07 15:44:37,529][282183] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,548][283085] Decorrelating experience for 736 frames... +[2023-03-07 15:44:37,549][281951] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,560][281661] Stopping RolloutWorker_w15... +[2023-03-07 15:44:37,561][281661] Loop rollout_proc15_evt_loop terminating... +[2023-03-07 15:44:37,560][236315] Component RolloutWorker_w15 stopped! +[2023-03-07 15:44:37,596][281984] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,617][283727] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,644][284304] Decorrelating experience for 928 frames... +[2023-03-07 15:44:37,668][281956] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,708][282696] Stopping RolloutWorker_w98... +[2023-03-07 15:44:37,709][282696] Loop rollout_proc98_evt_loop terminating... +[2023-03-07 15:44:37,709][236315] Component RolloutWorker_w98 stopped! +[2023-03-07 15:44:37,761][281990] Decorrelating experience for 896 frames... +[2023-03-07 15:44:37,762][281964] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,780][281991] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,844][281669] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,858][281957] Stopping RolloutWorker_w46... +[2023-03-07 15:44:37,859][281957] Loop rollout_proc46_evt_loop terminating... +[2023-03-07 15:44:37,859][281653] Decorrelating experience for 960 frames... +[2023-03-07 15:44:37,859][236315] Component RolloutWorker_w46 stopped! +[2023-03-07 15:44:37,866][281665] Stopping RolloutWorker_w33... +[2023-03-07 15:44:37,866][281665] Loop rollout_proc33_evt_loop terminating... +[2023-03-07 15:44:37,866][236315] Component RolloutWorker_w33 stopped! +[2023-03-07 15:44:37,875][283019] Decorrelating experience for 896 frames... +[2023-03-07 15:44:37,883][287383] Decorrelating experience for 992 frames... +[2023-03-07 15:44:37,926][282904] Decorrelating experience for 832 frames... +[2023-03-07 15:44:37,947][283960] Decorrelating experience for 864 frames... +[2023-03-07 15:44:37,997][281963] Decorrelating experience for 640 frames... +[2023-03-07 15:44:38,010][283085] Decorrelating experience for 768 frames... +[2023-03-07 15:44:38,073][281951] Stopping RolloutWorker_w83... +[2023-03-07 15:44:38,073][281951] Loop rollout_proc83_evt_loop terminating... +[2023-03-07 15:44:38,073][236315] Component RolloutWorker_w83 stopped! +[2023-03-07 15:44:38,124][281810] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,124][281956] Stopping RolloutWorker_w86... +[2023-03-07 15:44:38,124][281956] Loop rollout_proc86_evt_loop terminating... +[2023-03-07 15:44:38,124][236315] Component RolloutWorker_w86 stopped! +[2023-03-07 15:44:38,168][281953] Decorrelating experience for 864 frames... +[2023-03-07 15:44:38,169][281805] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,184][282183] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,191][284304] Decorrelating experience for 960 frames... +[2023-03-07 15:44:38,208][281959] Decorrelating experience for 896 frames... +[2023-03-07 15:44:38,215][281964] Stopping RolloutWorker_w81... +[2023-03-07 15:44:38,216][281964] Loop rollout_proc81_evt_loop terminating... +[2023-03-07 15:44:38,215][236315] Component RolloutWorker_w81 stopped! +[2023-03-07 15:44:38,225][281991] Stopping RolloutWorker_w61... +[2023-03-07 15:44:38,225][281991] Loop rollout_proc61_evt_loop terminating... +[2023-03-07 15:44:38,225][236315] Component RolloutWorker_w61 stopped! +[2023-03-07 15:44:38,313][287383] Stopping RolloutWorker_w0... +[2023-03-07 15:44:38,313][287383] Loop rollout_proc0_evt_loop terminating... +[2023-03-07 15:44:38,313][236315] Component RolloutWorker_w0 stopped! +[2023-03-07 15:44:38,335][281984] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,389][281669] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,391][283019] Decorrelating experience for 928 frames... +[2023-03-07 15:44:38,392][281990] Decorrelating experience for 928 frames... +[2023-03-07 15:44:38,419][281963] Decorrelating experience for 672 frames... +[2023-03-07 15:44:38,477][283085] Decorrelating experience for 800 frames... +[2023-03-07 15:44:38,488][283727] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,543][281810] Stopping RolloutWorker_w34... +[2023-03-07 15:44:38,543][281810] Loop rollout_proc34_evt_loop terminating... +[2023-03-07 15:44:38,543][236315] Component RolloutWorker_w34 stopped! +[2023-03-07 15:44:38,603][281653] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,609][282183] Stopping RolloutWorker_w90... +[2023-03-07 15:44:38,609][282183] Loop rollout_proc90_evt_loop terminating... +[2023-03-07 15:44:38,609][236315] Component RolloutWorker_w90 stopped! +[2023-03-07 15:44:38,636][281805] Stopping RolloutWorker_w19... +[2023-03-07 15:44:38,636][281805] Loop rollout_proc19_evt_loop terminating... +[2023-03-07 15:44:38,636][236315] Component RolloutWorker_w19 stopped! +[2023-03-07 15:44:38,671][282904] Decorrelating experience for 864 frames... +[2023-03-07 15:44:38,711][283960] Decorrelating experience for 896 frames... +[2023-03-07 15:44:38,730][284304] Decorrelating experience for 992 frames... +[2023-03-07 15:44:38,813][281669] Stopping RolloutWorker_w42... +[2023-03-07 15:44:38,813][281669] Loop rollout_proc42_evt_loop terminating... +[2023-03-07 15:44:38,813][236315] Component RolloutWorker_w42 stopped! +[2023-03-07 15:44:38,891][281953] Decorrelating experience for 896 frames... +[2023-03-07 15:44:38,897][281984] Stopping RolloutWorker_w49... +[2023-03-07 15:44:38,897][281984] Loop rollout_proc49_evt_loop terminating... +[2023-03-07 15:44:38,897][236315] Component RolloutWorker_w49 stopped! +[2023-03-07 15:44:38,910][283019] Decorrelating experience for 960 frames... +[2023-03-07 15:44:38,910][281963] Decorrelating experience for 704 frames... +[2023-03-07 15:44:38,920][281990] Decorrelating experience for 960 frames... +[2023-03-07 15:44:38,953][283085] Decorrelating experience for 832 frames... +[2023-03-07 15:44:39,039][281653] Stopping RolloutWorker_w3... +[2023-03-07 15:44:39,040][281653] Loop rollout_proc3_evt_loop terminating... +[2023-03-07 15:44:39,039][236315] Component RolloutWorker_w3 stopped! +[2023-03-07 15:44:39,073][283727] Stopping RolloutWorker_w126... +[2023-03-07 15:44:39,073][283727] Loop rollout_proc126_evt_loop terminating... +[2023-03-07 15:44:39,073][236315] Component RolloutWorker_w126 stopped! +[2023-03-07 15:44:39,115][281959] Decorrelating experience for 928 frames... +[2023-03-07 15:44:39,163][284304] Stopping RolloutWorker_w124... +[2023-03-07 15:44:39,164][284304] Loop rollout_proc124_evt_loop terminating... +[2023-03-07 15:44:39,163][236315] Component RolloutWorker_w124 stopped! +[2023-03-07 15:44:39,215][283960] Decorrelating experience for 928 frames... +[2023-03-07 15:44:39,337][282904] Decorrelating experience for 896 frames... +[2023-03-07 15:44:39,364][281963] Decorrelating experience for 736 frames... +[2023-03-07 15:44:39,435][283019] Decorrelating experience for 992 frames... +[2023-03-07 15:44:39,446][281990] Decorrelating experience for 992 frames... +[2023-03-07 15:44:39,512][281953] Decorrelating experience for 928 frames... +[2023-03-07 15:44:39,627][281959] Decorrelating experience for 960 frames... +[2023-03-07 15:44:39,636][283085] Decorrelating experience for 864 frames... +[2023-03-07 15:44:39,820][281963] Decorrelating experience for 768 frames... +[2023-03-07 15:44:39,834][283960] Decorrelating experience for 960 frames... +[2023-03-07 15:44:39,844][282904] Decorrelating experience for 928 frames... +[2023-03-07 15:44:39,853][283019] Stopping RolloutWorker_w122... +[2023-03-07 15:44:39,853][283019] Loop rollout_proc122_evt_loop terminating... +[2023-03-07 15:44:39,853][236315] Component RolloutWorker_w122 stopped! +[2023-03-07 15:44:39,865][281990] Stopping RolloutWorker_w82... +[2023-03-07 15:44:39,865][281990] Loop rollout_proc82_evt_loop terminating... +[2023-03-07 15:44:39,865][236315] Component RolloutWorker_w82 stopped! +[2023-03-07 15:44:40,023][281953] Decorrelating experience for 960 frames... +[2023-03-07 15:44:40,129][283085] Decorrelating experience for 896 frames... +[2023-03-07 15:44:40,148][281959] Decorrelating experience for 992 frames... +[2023-03-07 15:44:40,336][281963] Decorrelating experience for 800 frames... +[2023-03-07 15:44:40,356][283960] Decorrelating experience for 992 frames... +[2023-03-07 15:44:40,366][282904] Decorrelating experience for 960 frames... +[2023-03-07 15:44:40,545][281953] Decorrelating experience for 992 frames... +[2023-03-07 15:44:40,559][281959] Stopping RolloutWorker_w68... +[2023-03-07 15:44:40,559][281959] Loop rollout_proc68_evt_loop terminating... +[2023-03-07 15:44:40,559][236315] Component RolloutWorker_w68 stopped! +[2023-03-07 15:44:40,632][283085] Decorrelating experience for 928 frames... +[2023-03-07 15:44:40,772][283960] Stopping RolloutWorker_w103... +[2023-03-07 15:44:40,772][283960] Loop rollout_proc103_evt_loop terminating... +[2023-03-07 15:44:40,772][236315] Component RolloutWorker_w103 stopped! +[2023-03-07 15:44:40,820][281963] Decorrelating experience for 832 frames... +[2023-03-07 15:44:40,896][282904] Decorrelating experience for 992 frames... +[2023-03-07 15:44:40,956][281953] Stopping RolloutWorker_w62... +[2023-03-07 15:44:40,956][281953] Loop rollout_proc62_evt_loop terminating... +[2023-03-07 15:44:40,956][236315] Component RolloutWorker_w62 stopped! +[2023-03-07 15:44:41,140][283085] Decorrelating experience for 960 frames... +[2023-03-07 15:44:41,295][281963] Decorrelating experience for 864 frames... +[2023-03-07 15:44:41,311][282904] Stopping RolloutWorker_w113... +[2023-03-07 15:44:41,311][282904] Loop rollout_proc113_evt_loop terminating... +[2023-03-07 15:44:41,311][236315] Component RolloutWorker_w113 stopped! +[2023-03-07 15:44:41,663][283085] Decorrelating experience for 992 frames... +[2023-03-07 15:44:41,780][281963] Decorrelating experience for 896 frames... +[2023-03-07 15:44:42,079][283085] Stopping RolloutWorker_w91... +[2023-03-07 15:44:42,080][283085] Loop rollout_proc91_evt_loop terminating... +[2023-03-07 15:44:42,079][236315] Component RolloutWorker_w91 stopped! +[2023-03-07 15:44:42,280][281963] Decorrelating experience for 928 frames... +[2023-03-07 15:44:42,782][281963] Decorrelating experience for 960 frames... +[2023-03-07 15:44:43,296][281963] Decorrelating experience for 992 frames... +[2023-03-07 15:44:43,698][281963] Stopping RolloutWorker_w48... +[2023-03-07 15:44:43,698][281963] Loop rollout_proc48_evt_loop terminating... +[2023-03-07 15:44:43,698][236315] Component RolloutWorker_w48 stopped! +[2023-03-07 15:44:43,699][236315] Waiting for process learner_proc0 to stop... +[2023-03-07 15:44:43,701][236315] Waiting for process inference_proc0-0 to join... +[2023-03-07 15:44:43,703][236315] Waiting for process rollout_proc0 to join... +[2023-03-07 15:44:43,704][236315] Waiting for process rollout_proc1 to join... +[2023-03-07 15:44:43,705][236315] Waiting for process rollout_proc2 to join... +[2023-03-07 15:44:43,707][236315] Waiting for process rollout_proc3 to join... +[2023-03-07 15:44:43,708][236315] Waiting for process rollout_proc4 to join... +[2023-03-07 15:44:43,710][236315] Waiting for process rollout_proc5 to join... +[2023-03-07 15:44:43,711][236315] Waiting for process rollout_proc6 to join... +[2023-03-07 15:44:43,712][236315] Waiting for process rollout_proc7 to join... +[2023-03-07 15:44:43,714][236315] Waiting for process rollout_proc8 to join... +[2023-03-07 15:44:43,715][236315] Waiting for process rollout_proc9 to join... +[2023-03-07 15:44:43,716][236315] Waiting for process rollout_proc10 to join... +[2023-03-07 15:44:43,718][236315] Waiting for process rollout_proc11 to join... +[2023-03-07 15:44:43,719][236315] Waiting for process rollout_proc12 to join... +[2023-03-07 15:44:43,720][236315] Waiting for process rollout_proc13 to join... +[2023-03-07 15:44:43,722][236315] Waiting for process rollout_proc14 to join... +[2023-03-07 15:44:43,723][236315] Waiting for process rollout_proc15 to join... +[2023-03-07 15:44:43,724][236315] Waiting for process rollout_proc16 to join... +[2023-03-07 15:44:43,726][236315] Waiting for process rollout_proc17 to join... +[2023-03-07 15:44:43,727][236315] Waiting for process rollout_proc18 to join... +[2023-03-07 15:44:43,728][236315] Waiting for process rollout_proc19 to join... +[2023-03-07 15:44:43,730][236315] Waiting for process rollout_proc20 to join... +[2023-03-07 15:44:43,734][236315] Waiting for process rollout_proc21 to join... +[2023-03-07 15:44:43,735][236315] Waiting for process rollout_proc22 to join... +[2023-03-07 15:44:43,737][236315] Waiting for process rollout_proc23 to join... +[2023-03-07 15:44:43,738][236315] Waiting for process rollout_proc24 to join... +[2023-03-07 15:44:43,744][236315] Waiting for process rollout_proc25 to join... +[2023-03-07 15:44:43,746][236315] Waiting for process rollout_proc26 to join... +[2023-03-07 15:44:43,748][236315] Waiting for process rollout_proc27 to join... +[2023-03-07 15:44:43,750][236315] Waiting for process rollout_proc28 to join... +[2023-03-07 15:44:43,751][236315] Waiting for process rollout_proc29 to join... +[2023-03-07 15:44:43,753][236315] Waiting for process rollout_proc30 to join... +[2023-03-07 15:44:43,754][236315] Waiting for process rollout_proc31 to join... +[2023-03-07 15:44:43,756][236315] Waiting for process rollout_proc32 to join... +[2023-03-07 15:44:43,758][236315] Waiting for process rollout_proc33 to join... +[2023-03-07 15:44:43,760][236315] Waiting for process rollout_proc34 to join... +[2023-03-07 15:44:43,761][236315] Waiting for process rollout_proc35 to join... +[2023-03-07 15:44:43,764][236315] Waiting for process rollout_proc36 to join... +[2023-03-07 15:44:43,766][236315] Waiting for process rollout_proc37 to join... +[2023-03-07 15:44:43,767][236315] Waiting for process rollout_proc38 to join... +[2023-03-07 15:44:43,768][236315] Waiting for process rollout_proc39 to join... +[2023-03-07 15:44:43,770][236315] Waiting for process rollout_proc40 to join... +[2023-03-07 15:44:43,771][236315] Waiting for process rollout_proc41 to join... +[2023-03-07 15:44:43,773][236315] Waiting for process rollout_proc42 to join... +[2023-03-07 15:44:43,774][236315] Waiting for process rollout_proc43 to join... +[2023-03-07 15:44:43,775][236315] Waiting for process rollout_proc44 to join... +[2023-03-07 15:44:43,777][236315] Waiting for process rollout_proc45 to join... +[2023-03-07 15:44:43,781][236315] Waiting for process rollout_proc46 to join... +[2023-03-07 15:44:43,782][236315] Waiting for process rollout_proc47 to join... +[2023-03-07 15:44:43,784][236315] Waiting for process rollout_proc48 to join... +[2023-03-07 15:44:43,882][236315] Waiting for process rollout_proc49 to join... +[2023-03-07 15:44:43,883][236315] Waiting for process rollout_proc50 to join... +[2023-03-07 15:44:43,885][236315] Waiting for process rollout_proc51 to join... +[2023-03-07 15:44:43,886][236315] Waiting for process rollout_proc52 to join... +[2023-03-07 15:44:43,887][236315] Waiting for process rollout_proc53 to join... +[2023-03-07 15:44:43,889][236315] Waiting for process rollout_proc54 to join... +[2023-03-07 15:44:43,890][236315] Waiting for process rollout_proc55 to join... +[2023-03-07 15:44:43,892][236315] Waiting for process rollout_proc56 to join... +[2023-03-07 15:44:43,893][236315] Waiting for process rollout_proc57 to join... +[2023-03-07 15:44:43,894][236315] Waiting for process rollout_proc58 to join... +[2023-03-07 15:44:43,896][236315] Waiting for process rollout_proc59 to join... +[2023-03-07 15:44:43,897][236315] Waiting for process rollout_proc60 to join... +[2023-03-07 15:44:43,899][236315] Waiting for process rollout_proc61 to join... +[2023-03-07 15:44:43,900][236315] Waiting for process rollout_proc62 to join... +[2023-03-07 15:44:43,901][236315] Waiting for process rollout_proc63 to join... +[2023-03-07 15:44:43,903][236315] Waiting for process rollout_proc64 to join... +[2023-03-07 15:44:43,904][236315] Waiting for process rollout_proc65 to join... +[2023-03-07 15:44:43,906][236315] Waiting for process rollout_proc66 to join... +[2023-03-07 15:44:43,908][236315] Waiting for process rollout_proc67 to join... +[2023-03-07 15:44:43,909][236315] Waiting for process rollout_proc68 to join... +[2023-03-07 15:44:43,911][236315] Waiting for process rollout_proc69 to join... +[2023-03-07 15:44:43,912][236315] Waiting for process rollout_proc70 to join... +[2023-03-07 15:44:43,914][236315] Waiting for process rollout_proc71 to join... +[2023-03-07 15:44:43,917][236315] Waiting for process rollout_proc72 to join... +[2023-03-07 15:44:43,919][236315] Waiting for process rollout_proc73 to join... +[2023-03-07 15:44:43,920][236315] Waiting for process rollout_proc74 to join... +[2023-03-07 15:44:43,922][236315] Waiting for process rollout_proc75 to join... +[2023-03-07 15:44:43,924][236315] Waiting for process rollout_proc76 to join... +[2023-03-07 15:44:43,925][236315] Waiting for process rollout_proc77 to join... +[2023-03-07 15:44:43,927][236315] Waiting for process rollout_proc78 to join... +[2023-03-07 15:44:43,929][236315] Waiting for process rollout_proc79 to join... +[2023-03-07 15:44:43,931][236315] Waiting for process rollout_proc80 to join... +[2023-03-07 15:44:43,933][236315] Waiting for process rollout_proc81 to join... +[2023-03-07 15:44:43,934][236315] Waiting for process rollout_proc82 to join... +[2023-03-07 15:44:43,936][236315] Waiting for process rollout_proc83 to join... +[2023-03-07 15:44:43,937][236315] Waiting for process rollout_proc84 to join... +[2023-03-07 15:44:43,941][236315] Waiting for process rollout_proc85 to join... +[2023-03-07 15:44:43,942][236315] Waiting for process rollout_proc86 to join... +[2023-03-07 15:44:43,944][236315] Waiting for process rollout_proc87 to join... +[2023-03-07 15:44:43,945][236315] Waiting for process rollout_proc88 to join... +[2023-03-07 15:44:43,948][236315] Waiting for process rollout_proc89 to join... +[2023-03-07 15:44:43,950][236315] Waiting for process rollout_proc90 to join... +[2023-03-07 15:44:43,951][236315] Waiting for process rollout_proc91 to join... +[2023-03-07 15:44:43,953][236315] Waiting for process rollout_proc92 to join... +[2023-03-07 15:44:43,959][236315] Waiting for process rollout_proc93 to join... +[2023-03-07 15:44:43,961][236315] Waiting for process rollout_proc94 to join... +[2023-03-07 15:44:43,964][236315] Waiting for process rollout_proc95 to join... +[2023-03-07 15:44:43,966][236315] Waiting for process rollout_proc96 to join... +[2023-03-07 15:44:43,967][236315] Waiting for process rollout_proc97 to join... +[2023-03-07 15:44:43,969][236315] Waiting for process rollout_proc98 to join... +[2023-03-07 15:44:43,973][236315] Waiting for process rollout_proc99 to join... +[2023-03-07 15:44:43,975][236315] Waiting for process rollout_proc100 to join... +[2023-03-07 15:44:43,976][236315] Waiting for process rollout_proc101 to join... +[2023-03-07 15:44:43,978][236315] Waiting for process rollout_proc102 to join... +[2023-03-07 15:44:43,981][236315] Waiting for process rollout_proc103 to join... +[2023-03-07 15:44:43,982][236315] Waiting for process rollout_proc104 to join... +[2023-03-07 15:44:43,984][236315] Waiting for process rollout_proc105 to join... +[2023-03-07 15:44:43,985][236315] Waiting for process rollout_proc106 to join... +[2023-03-07 15:44:43,987][236315] Waiting for process rollout_proc107 to join... +[2023-03-07 15:44:43,989][236315] Waiting for process rollout_proc108 to join... +[2023-03-07 15:44:43,991][236315] Waiting for process rollout_proc109 to join... +[2023-03-07 15:44:43,993][236315] Waiting for process rollout_proc110 to join... +[2023-03-07 15:44:43,994][236315] Waiting for process rollout_proc111 to join... +[2023-03-07 15:44:43,996][236315] Waiting for process rollout_proc112 to join... +[2023-03-07 15:44:43,997][236315] Waiting for process rollout_proc113 to join... +[2023-03-07 15:44:44,001][236315] Waiting for process rollout_proc114 to join... +[2023-03-07 15:44:44,002][236315] Waiting for process rollout_proc115 to join... +[2023-03-07 15:44:44,004][236315] Waiting for process rollout_proc116 to join... +[2023-03-07 15:44:44,005][236315] Waiting for process rollout_proc117 to join... +[2023-03-07 15:44:44,009][236315] Waiting for process rollout_proc118 to join... +[2023-03-07 15:44:44,010][236315] Waiting for process rollout_proc119 to join... +[2023-03-07 15:44:44,011][236315] Waiting for process rollout_proc120 to join... +[2023-03-07 15:44:44,012][236315] Waiting for process rollout_proc121 to join... +[2023-03-07 15:44:44,013][236315] Waiting for process rollout_proc122 to join... +[2023-03-07 15:44:44,017][236315] Waiting for process rollout_proc123 to join... +[2023-03-07 15:44:44,018][236315] Waiting for process rollout_proc124 to join... +[2023-03-07 15:44:44,020][236315] Waiting for process rollout_proc125 to join... +[2023-03-07 15:44:44,021][236315] Waiting for process rollout_proc126 to join... +[2023-03-07 15:44:44,022][236315] Waiting for process rollout_proc127 to join... +[2023-03-07 15:44:44,023][236315] Batcher 0 profile tree view: +batching: 0.0148, releasing_batches: 0.0000 +[2023-03-07 15:44:44,024][236315] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0051 + wait_policy_total: 45.0994 +update_model: 0.0036 + weight_update: 0.0013 +one_step: 0.0022 + handle_policy_step: 0.4763 + deserialize: 0.0142, stack: 0.0010, obs_to_device_normalize: 0.0582, forward: 0.3643, send_messages: 0.0054 + prepare_outputs: 0.0254 + to_cpu: 0.0158 +[2023-03-07 15:44:44,024][236315] Learner 0 profile tree view: +misc: 0.0000, prepare_batch: 0.5622 +train: 0.2197 + epoch_init: 0.0000, minibatch_init: 0.0000, losses_postprocess: 0.0002, kl_divergence: 0.0007, after_optimizer: 0.0041 + calculate_losses: 0.0330 + losses_init: 0.0000, forward_head: 0.0289, bptt_initial: 0.0013, tail: 0.0004, advantages_returns: 0.0004, losses: 0.0012 + bptt: 0.0006 + bptt_forward_core: 0.0006 + update: 0.1814 + clip: 0.0012 +[2023-03-07 15:44:44,025][236315] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0007, enqueue_policy_requests: 0.0010 +[2023-03-07 15:44:44,026][236315] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0007, enqueue_policy_requests: 0.0010 +[2023-03-07 15:44:44,027][236315] Loop Runner_EvtLoop terminating... +[2023-03-07 15:44:44,029][236315] Runner profile tree view: +main_loop: 101.9645 +[2023-03-07 15:44:44,029][236315] Collected {0: 4014080}, FPS: 40.2 +[2023-03-07 15:47:48,456][236315] Environment doom_basic already registered, overwriting... +[2023-03-07 15:47:48,458][236315] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 15:47:48,459][236315] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 15:47:48,459][236315] Environment doom_dm already registered, overwriting... +[2023-03-07 15:47:48,460][236315] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 15:47:48,461][236315] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 15:47:48,461][236315] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 15:47:48,462][236315] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 15:47:48,464][236315] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 15:47:48,464][236315] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 15:47:48,465][236315] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 15:47:48,465][236315] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 15:47:48,467][236315] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 15:47:48,467][236315] Environment doom_battle already registered, overwriting... +[2023-03-07 15:47:48,468][236315] Environment doom_battle2 already registered, overwriting... +[2023-03-07 15:47:48,469][236315] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 15:47:48,469][236315] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 15:47:48,471][236315] Environment doom_duel already registered, overwriting... +[2023-03-07 15:47:48,471][236315] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 15:47:48,472][236315] Environment doom_benchmark already registered, overwriting... +[2023-03-07 15:47:48,473][236315] register_encoder_factory: +[2023-03-07 15:47:48,488][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:47:48,492][236315] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/default_experiment already exists! +[2023-03-07 15:47:48,493][236315] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/default_experiment... +[2023-03-07 15:47:48,494][236315] Weights and Biases integration disabled +[2023-03-07 15:47:48,496][236315] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 15:47:49,561][236315] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=default_experiment +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=120 --num_envs_per_worker=4 --train_for_env_steps=4000000 +cli_args={'env': 'doom_health_gathering_supreme', 'num_workers': 120, 'num_envs_per_worker': 4, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 15:47:49,562][236315] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json... +[2023-03-07 15:47:49,563][236315] Rollout worker 0 uses device cpu +[2023-03-07 15:47:49,563][236315] Rollout worker 1 uses device cpu +[2023-03-07 15:47:49,564][236315] Rollout worker 2 uses device cpu +[2023-03-07 15:47:49,564][236315] Rollout worker 3 uses device cpu +[2023-03-07 15:47:49,564][236315] Rollout worker 4 uses device cpu +[2023-03-07 15:47:49,565][236315] Rollout worker 5 uses device cpu +[2023-03-07 15:47:49,565][236315] Rollout worker 6 uses device cpu +[2023-03-07 15:47:49,566][236315] Rollout worker 7 uses device cpu +[2023-03-07 15:47:49,566][236315] Rollout worker 8 uses device cpu +[2023-03-07 15:47:49,566][236315] Rollout worker 9 uses device cpu +[2023-03-07 15:47:49,567][236315] Rollout worker 10 uses device cpu +[2023-03-07 15:47:49,567][236315] Rollout worker 11 uses device cpu +[2023-03-07 15:47:49,568][236315] Rollout worker 12 uses device cpu +[2023-03-07 15:47:49,568][236315] Rollout worker 13 uses device cpu +[2023-03-07 15:47:49,568][236315] Rollout worker 14 uses device cpu +[2023-03-07 15:47:49,569][236315] Rollout worker 15 uses device cpu +[2023-03-07 15:47:49,569][236315] Rollout worker 16 uses device cpu +[2023-03-07 15:47:49,569][236315] Rollout worker 17 uses device cpu +[2023-03-07 15:47:49,570][236315] Rollout worker 18 uses device cpu +[2023-03-07 15:47:49,570][236315] Rollout worker 19 uses device cpu +[2023-03-07 15:47:49,571][236315] Rollout worker 20 uses device cpu +[2023-03-07 15:47:49,571][236315] Rollout worker 21 uses device cpu +[2023-03-07 15:47:49,571][236315] Rollout worker 22 uses device cpu +[2023-03-07 15:47:49,572][236315] Rollout worker 23 uses device cpu +[2023-03-07 15:47:49,572][236315] Rollout worker 24 uses device cpu +[2023-03-07 15:47:49,573][236315] Rollout worker 25 uses device cpu +[2023-03-07 15:47:49,573][236315] Rollout worker 26 uses device cpu +[2023-03-07 15:47:49,573][236315] Rollout worker 27 uses device cpu +[2023-03-07 15:47:49,576][236315] Rollout worker 28 uses device cpu +[2023-03-07 15:47:49,576][236315] Rollout worker 29 uses device cpu +[2023-03-07 15:47:49,577][236315] Rollout worker 30 uses device cpu +[2023-03-07 15:47:49,577][236315] Rollout worker 31 uses device cpu +[2023-03-07 15:47:49,578][236315] Rollout worker 32 uses device cpu +[2023-03-07 15:47:49,578][236315] Rollout worker 33 uses device cpu +[2023-03-07 15:47:49,578][236315] Rollout worker 34 uses device cpu +[2023-03-07 15:47:49,579][236315] Rollout worker 35 uses device cpu +[2023-03-07 15:47:49,579][236315] Rollout worker 36 uses device cpu +[2023-03-07 15:47:49,579][236315] Rollout worker 37 uses device cpu +[2023-03-07 15:47:49,580][236315] Rollout worker 38 uses device cpu +[2023-03-07 15:47:49,580][236315] Rollout worker 39 uses device cpu +[2023-03-07 15:47:49,581][236315] Rollout worker 40 uses device cpu +[2023-03-07 15:47:49,581][236315] Rollout worker 41 uses device cpu +[2023-03-07 15:47:49,581][236315] Rollout worker 42 uses device cpu +[2023-03-07 15:47:49,582][236315] Rollout worker 43 uses device cpu +[2023-03-07 15:47:49,584][236315] Rollout worker 44 uses device cpu +[2023-03-07 15:47:49,585][236315] Rollout worker 45 uses device cpu +[2023-03-07 15:47:49,585][236315] Rollout worker 46 uses device cpu +[2023-03-07 15:47:49,586][236315] Rollout worker 47 uses device cpu +[2023-03-07 15:47:49,586][236315] Rollout worker 48 uses device cpu +[2023-03-07 15:47:49,587][236315] Rollout worker 49 uses device cpu +[2023-03-07 15:47:49,588][236315] Rollout worker 50 uses device cpu +[2023-03-07 15:47:49,588][236315] Rollout worker 51 uses device cpu +[2023-03-07 15:47:49,589][236315] Rollout worker 52 uses device cpu +[2023-03-07 15:47:49,589][236315] Rollout worker 53 uses device cpu +[2023-03-07 15:47:49,590][236315] Rollout worker 54 uses device cpu +[2023-03-07 15:47:49,590][236315] Rollout worker 55 uses device cpu +[2023-03-07 15:47:49,591][236315] Rollout worker 56 uses device cpu +[2023-03-07 15:47:49,592][236315] Rollout worker 57 uses device cpu +[2023-03-07 15:47:49,592][236315] Rollout worker 58 uses device cpu +[2023-03-07 15:47:49,593][236315] Rollout worker 59 uses device cpu +[2023-03-07 15:47:49,593][236315] Rollout worker 60 uses device cpu +[2023-03-07 15:47:49,594][236315] Rollout worker 61 uses device cpu +[2023-03-07 15:47:49,595][236315] Rollout worker 62 uses device cpu +[2023-03-07 15:47:49,595][236315] Rollout worker 63 uses device cpu +[2023-03-07 15:47:49,596][236315] Rollout worker 64 uses device cpu +[2023-03-07 15:47:49,596][236315] Rollout worker 65 uses device cpu +[2023-03-07 15:47:49,597][236315] Rollout worker 66 uses device cpu +[2023-03-07 15:47:49,597][236315] Rollout worker 67 uses device cpu +[2023-03-07 15:47:49,598][236315] Rollout worker 68 uses device cpu +[2023-03-07 15:47:49,599][236315] Rollout worker 69 uses device cpu +[2023-03-07 15:47:49,599][236315] Rollout worker 70 uses device cpu +[2023-03-07 15:47:49,600][236315] Rollout worker 71 uses device cpu +[2023-03-07 15:47:49,600][236315] Rollout worker 72 uses device cpu +[2023-03-07 15:47:49,601][236315] Rollout worker 73 uses device cpu +[2023-03-07 15:47:49,601][236315] Rollout worker 74 uses device cpu +[2023-03-07 15:47:49,602][236315] Rollout worker 75 uses device cpu +[2023-03-07 15:47:49,603][236315] Rollout worker 76 uses device cpu +[2023-03-07 15:47:49,603][236315] Rollout worker 77 uses device cpu +[2023-03-07 15:47:49,604][236315] Rollout worker 78 uses device cpu +[2023-03-07 15:47:49,604][236315] Rollout worker 79 uses device cpu +[2023-03-07 15:47:49,605][236315] Rollout worker 80 uses device cpu +[2023-03-07 15:47:49,605][236315] Rollout worker 81 uses device cpu +[2023-03-07 15:47:49,606][236315] Rollout worker 82 uses device cpu +[2023-03-07 15:47:49,607][236315] Rollout worker 83 uses device cpu +[2023-03-07 15:47:49,607][236315] Rollout worker 84 uses device cpu +[2023-03-07 15:47:49,608][236315] Rollout worker 85 uses device cpu +[2023-03-07 15:47:49,608][236315] Rollout worker 86 uses device cpu +[2023-03-07 15:47:49,609][236315] Rollout worker 87 uses device cpu +[2023-03-07 15:47:49,616][236315] Rollout worker 88 uses device cpu +[2023-03-07 15:47:49,617][236315] Rollout worker 89 uses device cpu +[2023-03-07 15:47:49,617][236315] Rollout worker 90 uses device cpu +[2023-03-07 15:47:49,618][236315] Rollout worker 91 uses device cpu +[2023-03-07 15:47:49,619][236315] Rollout worker 92 uses device cpu +[2023-03-07 15:47:49,619][236315] Rollout worker 93 uses device cpu +[2023-03-07 15:47:49,620][236315] Rollout worker 94 uses device cpu +[2023-03-07 15:47:49,620][236315] Rollout worker 95 uses device cpu +[2023-03-07 15:47:49,621][236315] Rollout worker 96 uses device cpu +[2023-03-07 15:47:49,621][236315] Rollout worker 97 uses device cpu +[2023-03-07 15:47:49,622][236315] Rollout worker 98 uses device cpu +[2023-03-07 15:47:49,622][236315] Rollout worker 99 uses device cpu +[2023-03-07 15:47:49,623][236315] Rollout worker 100 uses device cpu +[2023-03-07 15:47:49,623][236315] Rollout worker 101 uses device cpu +[2023-03-07 15:47:49,624][236315] Rollout worker 102 uses device cpu +[2023-03-07 15:47:49,624][236315] Rollout worker 103 uses device cpu +[2023-03-07 15:47:49,625][236315] Rollout worker 104 uses device cpu +[2023-03-07 15:47:49,625][236315] Rollout worker 105 uses device cpu +[2023-03-07 15:47:49,626][236315] Rollout worker 106 uses device cpu +[2023-03-07 15:47:49,626][236315] Rollout worker 107 uses device cpu +[2023-03-07 15:47:49,627][236315] Rollout worker 108 uses device cpu +[2023-03-07 15:47:49,628][236315] Rollout worker 109 uses device cpu +[2023-03-07 15:47:49,628][236315] Rollout worker 110 uses device cpu +[2023-03-07 15:47:49,629][236315] Rollout worker 111 uses device cpu +[2023-03-07 15:47:49,629][236315] Rollout worker 112 uses device cpu +[2023-03-07 15:47:49,630][236315] Rollout worker 113 uses device cpu +[2023-03-07 15:47:49,630][236315] Rollout worker 114 uses device cpu +[2023-03-07 15:47:49,631][236315] Rollout worker 115 uses device cpu +[2023-03-07 15:47:49,631][236315] Rollout worker 116 uses device cpu +[2023-03-07 15:47:49,632][236315] Rollout worker 117 uses device cpu +[2023-03-07 15:47:49,632][236315] Rollout worker 118 uses device cpu +[2023-03-07 15:47:49,633][236315] Rollout worker 119 uses device cpu +[2023-03-07 15:47:49,633][236315] Rollout worker 120 uses device cpu +[2023-03-07 15:47:49,634][236315] Rollout worker 121 uses device cpu +[2023-03-07 15:47:49,634][236315] Rollout worker 122 uses device cpu +[2023-03-07 15:47:49,635][236315] Rollout worker 123 uses device cpu +[2023-03-07 15:47:49,635][236315] Rollout worker 124 uses device cpu +[2023-03-07 15:47:49,636][236315] Rollout worker 125 uses device cpu +[2023-03-07 15:47:49,636][236315] Rollout worker 126 uses device cpu +[2023-03-07 15:47:49,637][236315] Rollout worker 127 uses device cpu +[2023-03-07 15:47:51,806][236315] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:47:51,806][236315] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:47:52,080][236315] Starting all processes... +[2023-03-07 15:47:52,081][236315] Starting process learner_proc0 +[2023-03-07 15:47:52,959][236315] Starting all processes... +[2023-03-07 15:47:52,960][382172] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:47:52,960][382172] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0 +[2023-03-07 15:47:52,964][236315] Starting process inference_proc0-0 +[2023-03-07 15:47:52,964][236315] Starting process rollout_proc2 +[2023-03-07 15:47:52,969][382172] Num visible devices: 1 +[2023-03-07 15:47:52,974][382172] Starting seed is not provided +[2023-03-07 15:47:52,974][382172] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:47:52,974][382172] Initializing actor-critic model on device cuda:0 +[2023-03-07 15:47:52,974][382172] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:47:52,975][382172] RunningMeanStd input shape: (1,) +[2023-03-07 15:47:52,965][236315] Starting process rollout_proc5 +[2023-03-07 15:47:52,966][236315] Starting process rollout_proc8 +[2023-03-07 15:47:52,966][236315] Starting process rollout_proc11 +[2023-03-07 15:47:52,985][382172] ConvEncoder: input_channels=3 +[2023-03-07 15:47:52,969][236315] Starting process rollout_proc14 +[2023-03-07 15:47:52,971][236315] Starting process rollout_proc17 +[2023-03-07 15:47:52,976][236315] Starting process rollout_proc20 +[2023-03-07 15:47:52,976][236315] Starting process rollout_proc23 +[2023-03-07 15:47:52,977][236315] Starting process rollout_proc26 +[2023-03-07 15:47:52,980][236315] Starting process rollout_proc29 +[2023-03-07 15:47:52,983][236315] Starting process rollout_proc32 +[2023-03-07 15:47:52,984][236315] Starting process rollout_proc35 +[2023-03-07 15:47:52,990][236315] Starting process rollout_proc38 +[2023-03-07 15:47:52,997][236315] Starting process rollout_proc41 +[2023-03-07 15:47:52,998][236315] Starting process rollout_proc44 +[2023-03-07 15:47:53,012][236315] Starting process rollout_proc6 +[2023-03-07 15:47:53,014][236315] Starting process rollout_proc3 +[2023-03-07 15:47:53,022][236315] Starting process rollout_proc12 +[2023-03-07 15:47:53,080][382172] Conv encoder output size: 512 +[2023-03-07 15:47:53,080][382172] Policy head output size: 512 +[2023-03-07 15:47:53,022][236315] Starting process rollout_proc9 +[2023-03-07 15:47:53,040][236315] Starting process rollout_proc24 +[2023-03-07 15:47:53,044][236315] Starting process rollout_proc18 +[2023-03-07 15:47:53,044][236315] Starting process rollout_proc21 +[2023-03-07 15:47:53,098][382172] Created Actor Critic model with architecture: +[2023-03-07 15:47:53,098][382172] ActorCriticSharedWeights( + (obs_normalizer): ObservationNormalizer( + (running_mean_std): RunningMeanStdDictInPlace( + (running_mean_std): ModuleDict( + (obs): RunningMeanStdInPlace() + ) + ) + ) + (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace) + (encoder): VizdoomEncoder( + (basic_encoder): ConvEncoder( + (enc): RecursiveScriptModule( + original_name=ConvEncoderImpl + (conv_head): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Conv2d) + (1): RecursiveScriptModule(original_name=ELU) + (2): RecursiveScriptModule(original_name=Conv2d) + (3): RecursiveScriptModule(original_name=ELU) + (4): RecursiveScriptModule(original_name=Conv2d) + (5): RecursiveScriptModule(original_name=ELU) + ) + (mlp_layers): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Linear) + (1): RecursiveScriptModule(original_name=ELU) + ) + ) + ) + ) + (core): ModelCoreRNN( + (core): GRU(512, 512) + ) + (decoder): MlpDecoder( + (mlp): Identity() + ) + (critic_linear): Linear(in_features=512, out_features=1, bias=True) + (action_parameterization): ActionParameterizationDefault( + (distribution_linear): Linear(in_features=512, out_features=5, bias=True) + ) +) +[2023-03-07 15:47:53,056][236315] Starting process rollout_proc15 +[2023-03-07 15:47:53,076][236315] Starting process rollout_proc27 +[2023-03-07 15:47:53,085][236315] Starting process rollout_proc30 +[2023-03-07 15:47:53,086][236315] Starting process rollout_proc36 +[2023-03-07 15:47:53,089][236315] Starting process rollout_proc33 +[2023-03-07 15:47:53,101][236315] Starting process rollout_proc39 +[2023-03-07 15:47:53,108][236315] Starting process rollout_proc42 +[2023-03-07 15:47:53,116][236315] Starting process rollout_proc7 +[2023-03-07 15:47:53,119][236315] Starting process rollout_proc45 +[2023-03-07 15:47:53,124][236315] Starting process rollout_proc13 +[2023-03-07 15:47:53,127][236315] Starting process rollout_proc4 +[2023-03-07 15:47:53,136][236315] Starting process rollout_proc10 +[2023-03-07 15:47:53,151][236315] Starting process rollout_proc19 +[2023-03-07 15:47:53,151][236315] Starting process rollout_proc22 +[2023-03-07 15:47:53,155][236315] Starting process rollout_proc25 +[2023-03-07 15:47:53,156][236315] Starting process rollout_proc16 +[2023-03-07 15:47:53,162][236315] Starting process rollout_proc28 +[2023-03-07 15:47:53,171][236315] Starting process rollout_proc31 +[2023-03-07 15:47:53,183][236315] Starting process rollout_proc34 +[2023-03-07 15:47:53,183][236315] Starting process rollout_proc37 +[2023-03-07 15:47:53,189][236315] Starting process rollout_proc40 +[2023-03-07 15:47:53,200][236315] Starting process rollout_proc43 +[2023-03-07 15:47:53,206][236315] Starting process rollout_proc46 +[2023-03-07 15:47:53,211][236315] Starting process rollout_proc47 +[2023-03-07 15:47:53,214][236315] Starting process rollout_proc50 +[2023-03-07 15:47:53,225][236315] Starting process rollout_proc53 +[2023-03-07 15:47:53,232][236315] Starting process rollout_proc56 +[2023-03-07 15:47:53,240][236315] Starting process rollout_proc59 +[2023-03-07 15:47:53,240][236315] Starting process rollout_proc62 +[2023-03-07 15:47:53,255][236315] Starting process rollout_proc65 +[2023-03-07 15:47:53,260][236315] Starting process rollout_proc68 +[2023-03-07 15:47:53,268][236315] Starting process rollout_proc71 +[2023-03-07 15:47:53,269][236315] Starting process rollout_proc74 +[2023-03-07 15:47:53,282][236315] Starting process rollout_proc77 +[2023-03-07 15:47:53,304][236315] Starting process rollout_proc80 +[2023-03-07 15:47:53,304][236315] Starting process rollout_proc83 +[2023-03-07 15:47:53,313][236315] Starting process rollout_proc86 +[2023-03-07 15:47:53,314][236315] Starting process rollout_proc89 +[2023-03-07 15:47:53,329][236315] Starting process rollout_proc51 +[2023-03-07 15:47:53,337][236315] Starting process rollout_proc54 +[2023-03-07 15:47:53,345][236315] Starting process rollout_proc57 +[2023-03-07 15:47:53,347][236315] Starting process rollout_proc48 +[2023-03-07 15:47:53,359][236315] Starting process rollout_proc60 +[2023-03-07 15:47:53,368][236315] Starting process rollout_proc63 +[2023-03-07 15:47:53,376][236315] Starting process rollout_proc66 +[2023-03-07 15:47:53,393][236315] Starting process rollout_proc72 +[2023-03-07 15:47:53,396][236315] Starting process rollout_proc75 +[2023-03-07 15:47:53,397][236315] Starting process rollout_proc69 +[2023-03-07 15:47:53,407][236315] Starting process rollout_proc78 +[2023-03-07 15:47:53,426][236315] Starting process rollout_proc84 +[2023-03-07 15:47:53,436][236315] Starting process rollout_proc81 +[2023-03-07 15:47:53,443][236315] Starting process rollout_proc90 +[2023-03-07 15:47:53,456][236315] Starting process rollout_proc87 +[2023-03-07 15:47:53,459][236315] Starting process rollout_proc55 +[2023-03-07 15:47:53,465][236315] Starting process rollout_proc58 +[2023-03-07 15:47:53,481][236315] Starting process rollout_proc49 +[2023-03-07 15:47:53,491][236315] Starting process rollout_proc52 +[2023-03-07 15:47:53,498][236315] Starting process rollout_proc61 +[2023-03-07 15:47:53,507][236315] Starting process rollout_proc67 +[2023-03-07 15:47:53,511][236315] Starting process rollout_proc64 +[2023-03-07 15:47:53,545][236315] Starting process rollout_proc73 +[2023-03-07 15:47:53,550][236315] Starting process rollout_proc76 +[2023-03-07 15:47:53,571][236315] Starting process rollout_proc70 +[2023-03-07 15:47:53,619][236315] Starting process rollout_proc82 +[2023-03-07 15:47:53,620][236315] Starting process rollout_proc92 +[2023-03-07 15:47:53,637][236315] Starting process rollout_proc95 +[2023-03-07 15:47:53,732][236315] Starting process rollout_proc85 +[2023-03-07 15:47:53,660][236315] Starting process rollout_proc101 +[2023-03-07 15:47:53,665][236315] Starting process rollout_proc79 +[2023-03-07 15:47:53,720][236315] Starting process rollout_proc91 +[2023-03-07 15:47:53,720][236315] Starting process rollout_proc88 +[2023-03-07 15:47:53,720][236315] Starting process rollout_proc104 +[2023-03-07 15:47:53,721][236315] Starting process rollout_proc107 +[2023-03-07 15:47:53,659][236315] Starting process rollout_proc98 +[2023-03-07 15:47:53,799][236315] Starting process rollout_proc110 +[2023-03-07 15:47:53,854][236315] Starting process rollout_proc113 +[2023-03-07 15:47:53,859][236315] Starting process rollout_proc116 +[2023-03-07 15:47:53,878][236315] Starting process rollout_proc119 +[2023-03-07 15:47:53,915][236315] Starting process rollout_proc122 +[2023-03-07 15:47:53,915][236315] Starting process rollout_proc93 +[2023-03-07 15:47:53,990][236315] Starting process rollout_proc96 +[2023-03-07 15:47:54,001][236315] Starting process rollout_proc108 +[2023-03-07 15:47:54,019][236315] Starting process rollout_proc125 +[2023-03-07 15:47:54,043][236315] Starting process rollout_proc105 +[2023-03-07 15:47:54,048][236315] Starting process rollout_proc102 +[2023-03-07 15:47:54,054][236315] Starting process rollout_proc111 +[2023-03-07 15:47:54,089][236315] Starting process rollout_proc114 +[2023-03-07 15:47:54,127][236315] Starting process rollout_proc99 +[2023-03-07 15:47:54,127][236315] Starting process rollout_proc123 +[2023-03-07 15:47:54,150][236315] Starting process rollout_proc120 +[2023-03-07 15:47:54,179][236315] Starting process rollout_proc94 +[2023-03-07 15:47:54,197][236315] Starting process rollout_proc117 +[2023-03-07 15:47:54,199][236315] Starting process rollout_proc97 +[2023-03-07 15:47:54,273][236315] Starting process rollout_proc126 +[2023-03-07 15:47:54,274][236315] Starting process rollout_proc109 +[2023-03-07 15:47:54,292][236315] Starting process rollout_proc103 +[2023-03-07 15:47:54,305][236315] Starting process rollout_proc106 +[2023-03-07 15:47:54,347][236315] Starting process rollout_proc112 +[2023-03-07 15:47:54,367][236315] Starting process rollout_proc100 +[2023-03-07 15:47:54,367][236315] Starting process rollout_proc115 +[2023-03-07 15:47:54,399][236315] Starting process rollout_proc124 +[2023-03-07 15:47:54,439][236315] Starting process rollout_proc121 +[2023-03-07 15:47:54,484][236315] Starting process rollout_proc127 +[2023-03-07 15:47:54,504][236315] Starting process rollout_proc118 +[2023-03-07 15:47:54,898][382327] Worker 20 uses CPU cores [20] +[2023-03-07 15:47:55,045][382324] Worker 8 uses CPU cores [8] +[2023-03-07 15:47:55,381][382396] Worker 35 uses CPU cores [35] +[2023-03-07 15:47:55,418][382393] Worker 26 uses CPU cores [26] +[2023-03-07 15:47:55,477][382401] Worker 3 uses CPU cores [3] +[2023-03-07 15:47:55,547][382399] Worker 6 uses CPU cores [6] +[2023-03-07 15:47:55,550][382326] Worker 23 uses CPU cores [23] +[2023-03-07 15:47:55,584][382421] Worker 22 uses CPU cores [22] +[2023-03-07 15:47:55,590][382321] Worker 5 uses CPU cores [5] +[2023-03-07 15:47:55,811][382325] Worker 17 uses CPU cores [17] +[2023-03-07 15:47:55,823][382323] Worker 11 uses CPU cores [11] +[2023-03-07 15:47:55,826][382403] Worker 9 uses CPU cores [9] +[2023-03-07 15:47:55,830][382395] Worker 32 uses CPU cores [32] +[2023-03-07 15:47:55,894][382394] Worker 29 uses CPU cores [29] +[2023-03-07 15:47:55,920][382322] Worker 2 uses CPU cores [2] +[2023-03-07 15:47:55,957][382406] Worker 24 uses CPU cores [24] +[2023-03-07 15:47:55,988][382409] Worker 27 uses CPU cores [27] +[2023-03-07 15:47:55,992][236315] Starting process rollout_proc0 +[2023-03-07 15:47:55,997][382259] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:47:55,998][382259] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0 +[2023-03-07 15:47:55,999][382402] Worker 12 uses CPU cores [12] +[2023-03-07 15:47:56,006][382328] Worker 14 uses CPU cores [14] +[2023-03-07 15:47:56,007][382172] Using optimizer +[2023-03-07 15:47:56,008][382172] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000980_4014080.pth... +[2023-03-07 15:47:56,010][382412] Worker 36 uses CPU cores [36] +[2023-03-07 15:47:56,017][382259] Num visible devices: 1 +[2023-03-07 15:47:56,090][382407] Worker 21 uses CPU cores [21] +[2023-03-07 15:47:56,115][382420] Worker 4 uses CPU cores [4] +[2023-03-07 15:47:56,120][382424] Worker 16 uses CPU cores [16] +[2023-03-07 15:47:56,122][236315] Starting process rollout_proc1 +[2023-03-07 15:47:56,129][382400] Worker 44 uses CPU cores [44] +[2023-03-07 15:47:56,127][382172] Loading model from checkpoint +[2023-03-07 15:47:56,137][382172] Loaded experiment state at self.train_step=980, self.env_steps=4014080 +[2023-03-07 15:47:56,141][382172] Initialized policy 0 weights for model version 980 +[2023-03-07 15:47:56,146][382172] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:47:56,159][382172] LearnerWorker_p0 finished initialization! +[2023-03-07 15:47:56,167][382398] Worker 41 uses CPU cores [41] +[2023-03-07 15:47:56,190][236315] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 4014080. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:47:56,199][382397] Worker 38 uses CPU cores [38] +[2023-03-07 15:47:56,241][382405] Worker 18 uses CPU cores [18] +[2023-03-07 15:47:56,246][382410] Worker 30 uses CPU cores [30] +[2023-03-07 15:47:56,291][382419] Worker 10 uses CPU cores [10] +[2023-03-07 15:47:56,310][382259] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:47:56,311][382259] RunningMeanStd input shape: (1,) +[2023-03-07 15:47:56,332][382259] ConvEncoder: input_channels=3 +[2023-03-07 15:47:56,363][382425] Worker 31 uses CPU cores [31] +[2023-03-07 15:47:56,382][382415] Worker 42 uses CPU cores [42] +[2023-03-07 15:47:56,490][382418] Worker 13 uses CPU cores [13] +[2023-03-07 15:47:56,542][382423] Worker 28 uses CPU cores [28] +[2023-03-07 15:47:56,553][382454] Worker 63 uses CPU cores [63] +[2023-03-07 15:47:56,585][382437] Worker 59 uses CPU cores [59] +[2023-03-07 15:47:56,614][382414] Worker 39 uses CPU cores [39] +[2023-03-07 15:47:56,620][382429] Worker 43 uses CPU cores [43] +[2023-03-07 15:47:56,637][382441] Worker 74 uses CPU cores [74] +[2023-03-07 15:47:56,662][382440] Worker 71 uses CPU cores [71] +[2023-03-07 15:47:56,674][382431] Worker 46 uses CPU cores [46] +[2023-03-07 15:47:56,676][382259] Conv encoder output size: 512 +[2023-03-07 15:47:56,677][382259] Policy head output size: 512 +[2023-03-07 15:47:56,689][382428] Worker 37 uses CPU cores [37] +[2023-03-07 15:47:56,690][382456] Worker 72 uses CPU cores [72] +[2023-03-07 15:47:56,694][382416] Worker 45 uses CPU cores [45] +[2023-03-07 15:47:56,703][382417] Worker 7 uses CPU cores [7] +[2023-03-07 15:47:56,706][382411] Worker 33 uses CPU cores [33] +[2023-03-07 15:47:56,750][382432] Worker 50 uses CPU cores [50] +[2023-03-07 15:47:56,766][382422] Worker 19 uses CPU cores [19] +[2023-03-07 15:47:56,772][382442] Worker 68 uses CPU cores [68] +[2023-03-07 15:47:56,789][382446] Worker 89 uses CPU cores [89] +[2023-03-07 15:47:56,798][382430] Worker 40 uses CPU cores [40] +[2023-03-07 15:47:56,820][382438] Worker 62 uses CPU cores [62] +[2023-03-07 15:47:56,834][382448] Worker 54 uses CPU cores [54] +[2023-03-07 15:47:56,846][383102] Worker 76 uses CPU cores [76] +[2023-03-07 15:47:56,858][382452] Worker 51 uses CPU cores [51] +[2023-03-07 15:47:56,870][383798] Worker 110 uses CPU cores [110] +[2023-03-07 15:47:56,870][382443] Worker 77 uses CPU cores [77] +[2023-03-07 15:47:56,870][382445] Worker 80 uses CPU cores [80] +[2023-03-07 15:47:56,870][382408] Worker 15 uses CPU cores [15] +[2023-03-07 15:47:56,873][382426] Worker 25 uses CPU cores [25] +[2023-03-07 15:47:56,885][382427] Worker 34 uses CPU cores [34] +[2023-03-07 15:47:56,890][382444] Worker 83 uses CPU cores [83] +[2023-03-07 15:47:56,898][382435] Worker 56 uses CPU cores [56] +[2023-03-07 15:47:56,906][382439] Worker 65 uses CPU cores [65] +[2023-03-07 15:47:56,926][382436] Worker 47 uses CPU cores [47] +[2023-03-07 15:47:56,928][382586] Worker 58 uses CPU cores [58] +[2023-03-07 15:47:56,947][382450] Worker 48 uses CPU cores [48] +[2023-03-07 15:47:56,954][383292] Worker 82 uses CPU cores [82] +[2023-03-07 15:47:56,962][384761] Worker 103 uses CPU cores [103] +[2023-03-07 15:47:56,962][382434] Worker 53 uses CPU cores [53] +[2023-03-07 15:47:56,972][382455] Worker 75 uses CPU cores [75] +[2023-03-07 15:47:56,974][382453] Worker 66 uses CPU cores [66] +[2023-03-07 15:47:56,983][382974] Worker 70 uses CPU cores [70] +[2023-03-07 15:47:56,990][382447] Worker 86 uses CPU cores [86] +[2023-03-07 15:47:56,994][383831] Worker 122 uses CPU cores [122] +[2023-03-07 15:47:56,998][382844] Worker 84 uses CPU cores [84] +[2023-03-07 15:47:57,002][382451] Worker 60 uses CPU cores [60] +[2023-03-07 15:47:57,002][383293] Worker 95 uses CPU cores [95] +[2023-03-07 15:47:57,006][382582] Worker 87 uses CPU cores [87] +[2023-03-07 15:47:57,008][382524] Worker 49 uses CPU cores [49] +[2023-03-07 15:47:57,010][382838] Worker 81 uses CPU cores [81] +[2023-03-07 15:47:57,018][382449] Worker 57 uses CPU cores [57] +[2023-03-07 15:47:57,039][383294] Worker 92 uses CPU cores [92] +[2023-03-07 15:47:57,058][382841] Worker 55 uses CPU cores [55] +[2023-03-07 15:47:57,060][382847] Worker 73 uses CPU cores [73] +[2023-03-07 15:47:57,076][382840] Worker 52 uses CPU cores [52] +[2023-03-07 15:47:57,093][383639] Worker 104 uses CPU cores [104] +[2023-03-07 15:47:57,094][382846] Worker 61 uses CPU cores [61] +[2023-03-07 15:47:57,102][382845] Worker 67 uses CPU cores [67] +[2023-03-07 15:47:57,108][383678] Worker 88 uses CPU cores [88] +[2023-03-07 15:47:57,116][382457] Worker 69 uses CPU cores [69] +[2023-03-07 15:47:57,120][383984] Worker 116 uses CPU cores [116] +[2023-03-07 15:47:57,126][382843] Worker 90 uses CPU cores [90] +[2023-03-07 15:47:57,130][383797] Worker 91 uses CPU cores [91] +[2023-03-07 15:47:57,130][383641] Worker 107 uses CPU cores [107] +[2023-03-07 15:47:57,146][383955] Worker 96 uses CPU cores [96] +[2023-03-07 15:47:57,161][385049] Worker 115 uses CPU cores [115] +[2023-03-07 15:47:57,177][384387] Worker 114 uses CPU cores [114] +[2023-03-07 15:47:57,181][383799] Worker 113 uses CPU cores [113] +[2023-03-07 15:47:57,185][384158] Worker 102 uses CPU cores [102] +[2023-03-07 15:47:57,202][383427] Worker 85 uses CPU cores [85] +[2023-03-07 15:47:57,222][384368] Worker 99 uses CPU cores [99] +[2023-03-07 15:47:57,246][382842] Worker 78 uses CPU cores [78] +[2023-03-07 15:47:57,249][383550] Worker 79 uses CPU cores [79] +[2023-03-07 15:47:57,250][383101] Worker 64 uses CPU cores [64] +[2023-03-07 15:47:57,255][385094] Worker 121 uses CPU cores [121] +[2023-03-07 15:47:57,263][385096] Worker 118 uses CPU cores [118] +[2023-03-07 15:47:57,266][385093] Worker 124 uses CPU cores [124] +[2023-03-07 15:47:57,270][384886] Worker 106 uses CPU cores [106] +[2023-03-07 15:47:57,270][384157] Worker 105 uses CPU cores [105] +[2023-03-07 15:47:57,273][383744] Worker 98 uses CPU cores [98] +[2023-03-07 15:47:57,276][384503] Worker 120 uses CPU cores [120] +[2023-03-07 15:47:57,279][383951] Worker 93 uses CPU cores [93] +[2023-03-07 15:47:57,308][384051] Worker 108 uses CPU cores [108] +[2023-03-07 15:47:57,323][385095] Worker 127 uses CPU cores [127] +[2023-03-07 15:47:57,336][383527] Worker 101 uses CPU cores [101] +[2023-03-07 15:47:57,340][384315] Worker 111 uses CPU cores [111] +[2023-03-07 15:47:57,368][383827] Worker 119 uses CPU cores [119] +[2023-03-07 15:47:57,377][384388] Worker 123 uses CPU cores [123] +[2023-03-07 15:47:57,381][384552] Worker 94 uses CPU cores [94] +[2023-03-07 15:47:57,384][384962] Worker 109 uses CPU cores [109] +[2023-03-07 15:47:57,388][384114] Worker 125 uses CPU cores [125] +[2023-03-07 15:47:57,395][385027] Worker 112 uses CPU cores [112] +[2023-03-07 15:47:57,396][384644] Worker 97 uses CPU cores [97] +[2023-03-07 15:47:57,399][385026] Worker 126 uses CPU cores [126] +[2023-03-07 15:47:57,480][385031] Worker 100 uses CPU cores [100] +[2023-03-07 15:47:57,532][384643] Worker 117 uses CPU cores [117] +[2023-03-07 15:47:57,715][389982] Worker 0 uses CPU cores [0] +[2023-03-07 15:47:57,812][390548] Worker 1 uses CPU cores [1] +[2023-03-07 15:47:57,850][236315] Inference worker 0-0 is ready! +[2023-03-07 15:47:57,851][236315] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:47:57,908][382449] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,910][382412] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,913][382441] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,916][382448] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,916][382840] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,916][383294] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,917][385049] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,917][382429] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,919][382434] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,920][382417] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,923][382437] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,924][382451] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,924][382328] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,925][382439] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,926][382524] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,927][382450] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,927][382454] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,928][385095] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,928][382403] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,929][382411] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,929][382408] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,929][382425] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,929][382455] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][382418] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][382457] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][382427] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][382846] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][383798] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,930][383831] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382416] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382426] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382430] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382435] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382586] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382438] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382841] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382322] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,931][382428] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,932][382398] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,932][382415] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][382436] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][384315] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][384368] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][382445] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][383951] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][382447] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,933][382324] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,934][382440] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,934][382422] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,934][382395] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,934][382419] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,934][382432] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382393] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][383799] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382423] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382838] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382974] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382582] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,935][382410] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382400] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382397] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][383102] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382414] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382453] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382406] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][383292] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382409] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382401] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,937][382443] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,937][382845] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,936][382327] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,938][383293] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,938][382323] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,938][384051] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,939][382446] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,939][382843] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,939][384886] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,939][384158] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,939][383527] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,940][382396] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,940][383955] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,940][382407] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][382402] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][383797] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][384761] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][382442] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][382431] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][382394] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,942][382424] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,941][382444] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,942][383427] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,943][382844] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,943][382421] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,943][382326] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,943][382325] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,944][382321] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,944][382405] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,945][382452] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,947][382399] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,947][382420] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,948][384387] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,950][382456] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,954][383639] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,962][383641] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,964][383678] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:57,986][385026] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,051][383101] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,076][383984] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,097][385096] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,105][382842] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,108][383550] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,109][385094] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,113][384157] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,130][383744] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,142][385093] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,149][382847] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,235][384388] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,240][384552] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,242][385027] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,302][384114] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,327][385031] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,363][384503] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,364][384962] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,440][383827] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,460][384643] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,499][384644] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,499][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:47:58,568][389982] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,809][390548] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:47:58,915][382439] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,918][382453] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,918][382586] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,919][382411] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,919][382840] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,919][382434] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,924][385049] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,929][382841] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,932][382412] Decorrelating experience for 0 frames... +[2023-03-07 15:47:58,943][382416] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,099][382327] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,101][382453] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,102][382586] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,104][382438] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,111][382847] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,116][383641] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,117][382412] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,120][383550] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,176][383427] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,176][382407] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,278][382440] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,286][382431] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,287][382441] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,294][382436] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,294][384387] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,296][382395] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,297][384051] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,299][382843] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,350][382407] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,351][382455] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,458][383293] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,467][389982] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,478][382326] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,479][382443] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,481][382449] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,481][382444] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,485][382428] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,486][383798] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,527][384388] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,534][384643] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,635][383294] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,645][382405] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,657][383951] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,661][385049] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,665][383798] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,671][382429] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,709][382401] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,711][382428] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,737][382454] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,737][385095] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,810][383294] Decorrelating experience for 32 frames... +[2023-03-07 15:47:59,839][382410] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,841][382399] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,845][383955] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,851][382393] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,862][382415] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,887][390548] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,913][383797] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,925][382322] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,931][384886] Decorrelating experience for 0 frames... +[2023-03-07 15:47:59,988][382847] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,021][382457] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,022][382838] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,037][383293] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,038][384644] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,041][382328] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,062][385096] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,089][383527] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,108][382845] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,113][385095] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,164][382431] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,198][389982] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,199][382400] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,215][390548] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,217][382399] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,225][382395] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,239][382407] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,266][383678] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,289][383797] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,304][382397] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,336][382415] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,375][382421] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,377][382418] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,395][382402] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,427][382434] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,450][382419] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,451][384158] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,467][382410] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,480][382843] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,484][382441] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,538][384643] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,555][382844] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,571][382415] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,577][382428] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,609][382452] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,628][382445] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,638][382419] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,644][384962] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,663][382841] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,719][382434] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,730][383527] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,732][382327] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,749][382321] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,752][383101] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,806][383798] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,815][382412] Decorrelating experience for 64 frames... +[2023-03-07 15:48:00,817][384962] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,820][382428] Decorrelating experience for 96 frames... +[2023-03-07 15:48:00,835][384552] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,900][382432] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,904][382418] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,911][382440] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,927][382451] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,928][382405] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,983][382422] Decorrelating experience for 0 frames... +[2023-03-07 15:48:00,996][383678] Decorrelating experience for 32 frames... +[2023-03-07 15:48:00,997][382426] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,001][382398] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,012][382395] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,076][382419] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,086][382452] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,106][385095] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,139][383293] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,162][382323] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,173][382415] Decorrelating experience for 96 frames... +[2023-03-07 15:48:01,173][382425] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,188][382414] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,193][389982] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,202][382838] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,256][382586] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,265][382394] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,279][384158] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,318][382327] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,335][382455] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,353][382453] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,355][384315] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,368][382846] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,385][382415] Decorrelating experience for 128 frames... +[2023-03-07 15:48:01,390][382838] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,429][382400] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,499][382440] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,521][382434] Decorrelating experience for 96 frames... +[2023-03-07 15:48:01,524][383827] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,525][382841] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,532][382398] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,533][382441] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,543][382407] Decorrelating experience for 96 frames... +[2023-03-07 15:48:01,582][384886] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,583][382449] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,606][382427] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,674][382409] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,710][385026] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,712][384368] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,713][383827] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,716][382395] Decorrelating experience for 96 frames... +[2023-03-07 15:48:01,719][382399] Decorrelating experience for 64 frames... +[2023-03-07 15:48:01,722][382327] Decorrelating experience for 96 frames... +[2023-03-07 15:48:01,761][382429] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,762][382445] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,782][382456] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,854][383102] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,888][382416] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,895][385031] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,896][382394] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,919][382437] Decorrelating experience for 0 frames... +[2023-03-07 15:48:01,940][384368] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,945][382846] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,958][382421] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,968][382845] Decorrelating experience for 32 frames... +[2023-03-07 15:48:01,968][382401] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,031][383799] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,073][384886] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,075][385031] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,076][385093] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,110][382457] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,116][384552] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,119][382423] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,135][384643] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,156][382407] Decorrelating experience for 128 frames... +[2023-03-07 15:48:02,158][382437] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,211][383641] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,251][382445] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,260][382451] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,263][384962] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,284][382327] Decorrelating experience for 128 frames... +[2023-03-07 15:48:02,295][382421] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,297][382457] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,312][382431] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,338][382427] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,338][382394] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,384][382420] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,441][382524] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,463][385095] Decorrelating experience for 96 frames... +[2023-03-07 15:48:02,475][382399] Decorrelating experience for 96 frames... +[2023-03-07 15:48:02,476][382444] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,490][382582] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,494][382455] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,513][382323] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,520][382586] Decorrelating experience for 96 frames... +[2023-03-07 15:48:02,522][383427] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,561][382439] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,615][383550] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,647][384387] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,654][382429] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,657][382403] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,667][382582] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,676][385095] Decorrelating experience for 128 frames... +[2023-03-07 15:48:02,689][383101] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,704][382440] Decorrelating experience for 96 frames... +[2023-03-07 15:48:02,722][382400] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,736][382442] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,790][385026] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,833][382843] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,834][382426] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,836][382395] Decorrelating experience for 128 frames... +[2023-03-07 15:48:02,844][382447] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,851][383641] Decorrelating experience for 64 frames... +[2023-03-07 15:48:02,865][383798] Decorrelating experience for 96 frames... +[2023-03-07 15:48:02,879][383984] Decorrelating experience for 0 frames... +[2023-03-07 15:48:02,910][382442] Decorrelating experience for 32 frames... +[2023-03-07 15:48:02,910][382440] Decorrelating experience for 128 frames... +[2023-03-07 15:48:03,010][382411] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,018][383292] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,021][384962] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,022][383799] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,023][383639] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,053][382418] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,057][383827] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,078][384644] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,083][382402] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,090][382436] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,187][382419] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,195][383292] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,195][382325] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,202][385095] Decorrelating experience for 160 frames... +[2023-03-07 15:48:03,235][383831] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,252][382324] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,265][389982] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,267][382448] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,288][382394] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,294][383101] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,366][384644] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,373][382400] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,374][382840] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,386][382449] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,409][384552] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,435][382454] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,442][382399] Decorrelating experience for 128 frames... +[2023-03-07 15:48:03,447][382396] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,478][382398] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:03,508][385027] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,543][382846] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,551][382421] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,554][382437] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,561][382409] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,586][382449] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,613][385093] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,623][382402] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,626][382422] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,656][382438] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,699][382397] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,728][382414] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,731][382436] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,745][382586] Decorrelating experience for 128 frames... +[2023-03-07 15:48:03,755][382457] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,765][383799] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,789][384368] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,807][383951] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,826][384114] Decorrelating experience for 0 frames... +[2023-03-07 15:48:03,834][382843] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,905][383639] Decorrelating experience for 32 frames... +[2023-03-07 15:48:03,907][382439] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,928][382398] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,930][382846] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,937][382397] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,948][383797] Decorrelating experience for 64 frames... +[2023-03-07 15:48:03,968][382431] Decorrelating experience for 96 frames... +[2023-03-07 15:48:03,983][382427] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,002][383678] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,008][382455] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,080][382436] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,089][385049] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,109][383641] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,111][382423] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,117][383831] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,123][382401] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,145][382438] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,160][384552] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,186][382397] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,186][382524] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,268][382421] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,270][382399] Decorrelating experience for 160 frames... +[2023-03-07 15:48:04,290][383292] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,292][382400] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,299][382843] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,299][383678] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,320][382456] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,339][384368] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,363][383827] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,369][383427] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,475][383799] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,477][382401] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,480][382410] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,503][384315] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,504][382452] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,511][382457] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,514][383293] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,539][383292] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,553][385049] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,604][382321] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,652][382448] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,657][384051] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,681][382454] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,697][382328] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,704][382450] Decorrelating experience for 0 frames... +[2023-03-07 15:48:04,713][384368] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,751][382396] Decorrelating experience for 32 frames... +[2023-03-07 15:48:04,782][382431] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,788][384552] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,796][382410] Decorrelating experience for 96 frames... +[2023-03-07 15:48:04,834][382430] Decorrelating experience for 0 frames... +[2023-03-07 15:48:04,838][382405] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,875][382408] Decorrelating experience for 0 frames... +[2023-03-07 15:48:04,881][384761] Decorrelating experience for 0 frames... +[2023-03-07 15:48:04,883][382840] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,887][382448] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,928][383294] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,960][382442] Decorrelating experience for 64 frames... +[2023-03-07 15:48:04,974][385049] Decorrelating experience for 128 frames... +[2023-03-07 15:48:04,993][382414] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,015][382411] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,020][382422] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,059][382394] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,060][385093] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,061][383831] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,074][383827] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,110][382423] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,175][382395] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,190][383798] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,197][385094] Decorrelating experience for 0 frames... +[2023-03-07 15:48:05,212][383527] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,223][382428] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,240][382326] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,244][382401] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,245][382324] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,259][382442] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,284][382402] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,355][384368] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,366][382451] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,374][382425] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,399][382406] Decorrelating experience for 0 frames... +[2023-03-07 15:48:05,401][382440] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,420][385096] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,429][383641] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,442][382405] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,443][382452] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,456][382844] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,529][382410] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,547][382407] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,550][382403] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,582][382586] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,584][382394] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,604][382841] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,624][383292] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,627][385094] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,633][382446] Decorrelating experience for 0 frames... +[2023-03-07 15:48:05,637][385031] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,707][382431] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,723][382323] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,726][382406] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,759][382847] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,762][382399] Decorrelating experience for 192 frames... +[2023-03-07 15:48:05,778][382326] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,810][382842] Decorrelating experience for 0 frames... +[2023-03-07 15:48:05,813][382423] Decorrelating experience for 96 frames... +[2023-03-07 15:48:05,815][382393] Decorrelating experience for 32 frames... +[2023-03-07 15:48:05,836][382324] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,899][382321] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,904][382415] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,909][385096] Decorrelating experience for 64 frames... +[2023-03-07 15:48:05,935][383293] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,940][382431] Decorrelating experience for 192 frames... +[2023-03-07 15:48:05,954][383641] Decorrelating experience for 160 frames... +[2023-03-07 15:48:05,998][383799] Decorrelating experience for 128 frames... +[2023-03-07 15:48:05,999][382414] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,002][382396] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,040][385031] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,076][382442] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,082][382843] Decorrelating experience for 160 frames... +[2023-03-07 15:48:06,085][382324] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,113][382974] Decorrelating experience for 0 frames... +[2023-03-07 15:48:06,124][382400] Decorrelating experience for 160 frames... +[2023-03-07 15:48:06,150][382416] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,175][382445] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,180][383427] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,184][384644] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,238][382454] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,254][382438] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,259][383951] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,263][382425] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,287][384552] Decorrelating experience for 160 frames... +[2023-03-07 15:48:06,300][382442] Decorrelating experience for 160 frames... +[2023-03-07 15:48:06,353][384962] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,356][382396] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,364][383955] Decorrelating experience for 32 frames... +[2023-03-07 15:48:06,378][382411] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,436][382407] Decorrelating experience for 192 frames... +[2023-03-07 15:48:06,445][382431] Decorrelating experience for 224 frames... +[2023-03-07 15:48:06,446][382974] Decorrelating experience for 32 frames... +[2023-03-07 15:48:06,450][382454] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,466][382409] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,476][382422] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,538][382445] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,540][382415] Decorrelating experience for 192 frames... +[2023-03-07 15:48:06,555][382393] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,620][383101] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,623][382394] Decorrelating experience for 192 frames... +[2023-03-07 15:48:06,625][382423] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,643][382322] Decorrelating experience for 32 frames... +[2023-03-07 15:48:06,651][382844] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,653][383955] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,678][389982] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,715][382425] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,718][385093] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,736][382420] Decorrelating experience for 32 frames... +[2023-03-07 15:48:06,805][384503] Decorrelating experience for 0 frames... +[2023-03-07 15:48:06,806][382321] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,819][382438] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,822][382437] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,830][383527] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,852][382844] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,854][383955] Decorrelating experience for 96 frames... +[2023-03-07 15:48:06,898][383101] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,918][382394] Decorrelating experience for 224 frames... +[2023-03-07 15:48:06,926][383427] Decorrelating experience for 128 frames... +[2023-03-07 15:48:06,983][382322] Decorrelating experience for 64 frames... +[2023-03-07 15:48:06,997][383294] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,014][382434] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,025][382456] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,032][383550] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,034][384503] Decorrelating experience for 32 frames... +[2023-03-07 15:48:07,036][382407] Decorrelating experience for 224 frames... +[2023-03-07 15:48:07,077][382401] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,091][382524] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,109][389982] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,163][385096] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,173][382419] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,190][382438] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,212][383294] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,214][384644] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,215][383955] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,280][382454] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,283][383798] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,288][382974] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,323][382431] Decorrelating experience for 256 frames... +[2023-03-07 15:48:07,338][382453] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,350][382328] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,366][384643] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,404][382403] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,414][382411] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,455][382402] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,478][382438] Decorrelating experience for 192 frames... +[2023-03-07 15:48:07,480][382417] Decorrelating experience for 0 frames... +[2023-03-07 15:48:07,512][385031] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,529][382838] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,530][382396] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,532][383102] Decorrelating experience for 32 frames... +[2023-03-07 15:48:07,549][382415] Decorrelating experience for 224 frames... +[2023-03-07 15:48:07,580][382425] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,600][382327] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,636][382322] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,658][382405] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,660][382416] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,717][384315] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,721][383827] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,721][385095] Decorrelating experience for 192 frames... +[2023-03-07 15:48:07,722][385026] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,726][384761] Decorrelating experience for 32 frames... +[2023-03-07 15:48:07,765][382846] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,832][382843] Decorrelating experience for 192 frames... +[2023-03-07 15:48:07,843][382455] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,846][382422] Decorrelating experience for 128 frames... +[2023-03-07 15:48:07,879][382393] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,896][382442] Decorrelating experience for 192 frames... +[2023-03-07 15:48:07,903][390548] Decorrelating experience for 64 frames... +[2023-03-07 15:48:07,905][382842] Decorrelating experience for 32 frames... +[2023-03-07 15:48:07,909][382425] Decorrelating experience for 160 frames... +[2023-03-07 15:48:07,919][382403] Decorrelating experience for 96 frames... +[2023-03-07 15:48:07,964][382402] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,008][382417] Decorrelating experience for 32 frames... +[2023-03-07 15:48:08,022][385026] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,030][382586] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,068][382398] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,082][382456] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,094][383294] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,107][382420] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,118][384315] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,118][383102] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,139][382582] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,183][383678] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,207][382396] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,214][384761] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,244][383798] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,257][382426] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,274][382449] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,285][382403] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,305][390548] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,309][382425] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,321][385027] Decorrelating experience for 32 frames... +[2023-03-07 15:48:08,359][383827] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,382][384157] Decorrelating experience for 0 frames... +[2023-03-07 15:48:08,407][383678] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,420][385095] Decorrelating experience for 224 frames... +[2023-03-07 15:48:08,439][382582] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,454][389982] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,471][382431] Decorrelating experience for 288 frames... +[2023-03-07 15:48:08,484][384643] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,490][382323] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:08,562][383641] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,562][384644] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,604][382322] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,607][382429] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,634][382395] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,638][385026] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,660][382524] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,662][382447] Decorrelating experience for 32 frames... +[2023-03-07 15:48:08,702][382438] Decorrelating experience for 224 frames... +[2023-03-07 15:48:08,703][382455] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,744][382430] Decorrelating experience for 32 frames... +[2023-03-07 15:48:08,747][382434] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,782][384503] Decorrelating experience for 64 frames... +[2023-03-07 15:48:08,787][382439] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,813][384368] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,842][382418] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,842][384552] Decorrelating experience for 192 frames... +[2023-03-07 15:48:08,855][382323] Decorrelating experience for 128 frames... +[2023-03-07 15:48:08,880][385095] Decorrelating experience for 256 frames... +[2023-03-07 15:48:08,907][385026] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,933][382457] Decorrelating experience for 160 frames... +[2023-03-07 15:48:08,936][382395] Decorrelating experience for 224 frames... +[2023-03-07 15:48:08,959][382328] Decorrelating experience for 96 frames... +[2023-03-07 15:48:08,963][382843] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,001][384644] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,020][382421] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,025][383639] Decorrelating experience for 64 frames... +[2023-03-07 15:48:09,042][384962] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,052][383798] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,089][382429] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,111][382448] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,116][382453] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,141][383101] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,144][382438] Decorrelating experience for 256 frames... +[2023-03-07 15:48:09,182][382432] Decorrelating experience for 32 frames... +[2023-03-07 15:48:09,199][382398] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,203][382439] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,218][382457] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,230][382445] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,264][382443] Decorrelating experience for 32 frames... +[2023-03-07 15:48:09,288][382451] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,330][384962] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,360][383799] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,360][385095] Decorrelating experience for 288 frames... +[2023-03-07 15:48:09,379][382442] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,383][383797] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,395][382454] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,397][383550] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,412][382426] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,447][382395] Decorrelating experience for 256 frames... +[2023-03-07 15:48:09,466][382457] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,507][382431] Decorrelating experience for 320 frames... +[2023-03-07 15:48:09,540][384315] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,587][383101] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,590][382844] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,597][383641] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,601][382841] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,618][382420] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,628][384962] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,633][383827] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,659][385031] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,713][382446] Decorrelating experience for 32 frames... +[2023-03-07 15:48:09,752][382442] Decorrelating experience for 256 frames... +[2023-03-07 15:48:09,769][384368] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,782][382401] Decorrelating experience for 192 frames... +[2023-03-07 15:48:09,789][382323] Decorrelating experience for 160 frames... +[2023-03-07 15:48:09,797][383797] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,814][382426] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,838][382424] Decorrelating experience for 0 frames... +[2023-03-07 15:48:09,877][382582] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,881][382326] Decorrelating experience for 96 frames... +[2023-03-07 15:48:09,890][384114] Decorrelating experience for 32 frames... +[2023-03-07 15:48:09,933][385095] Decorrelating experience for 320 frames... +[2023-03-07 15:48:09,969][382395] Decorrelating experience for 288 frames... +[2023-03-07 15:48:09,972][382393] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,980][382448] Decorrelating experience for 128 frames... +[2023-03-07 15:48:09,989][383101] Decorrelating experience for 224 frames... +[2023-03-07 15:48:09,995][383294] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,018][382445] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,056][382437] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,069][382455] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,071][382451] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,107][382418] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,147][382405] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,150][385026] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,174][382415] Decorrelating experience for 256 frames... +[2023-03-07 15:48:10,178][383797] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,182][384761] Decorrelating experience for 96 frames... +[2023-03-07 15:48:10,195][382394] Decorrelating experience for 256 frames... +[2023-03-07 15:48:10,240][384962] Decorrelating experience for 256 frames... +[2023-03-07 15:48:10,249][384552] Decorrelating experience for 224 frames... +[2023-03-07 15:48:10,283][384644] Decorrelating experience for 224 frames... +[2023-03-07 15:48:10,291][382426] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,329][382843] Decorrelating experience for 256 frames... +[2023-03-07 15:48:10,331][385095] Decorrelating experience for 352 frames... +[2023-03-07 15:48:10,363][382326] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,375][382324] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,384][382437] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,418][382440] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,425][382411] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,426][385027] Decorrelating experience for 64 frames... +[2023-03-07 15:48:10,466][383550] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,469][382424] Decorrelating experience for 32 frames... +[2023-03-07 15:48:10,510][382974] Decorrelating experience for 96 frames... +[2023-03-07 15:48:10,514][382448] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,542][382441] Decorrelating experience for 96 frames... +[2023-03-07 15:48:10,555][383678] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,599][384644] Decorrelating experience for 256 frames... +[2023-03-07 15:48:10,621][382437] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,645][382430] Decorrelating experience for 64 frames... +[2023-03-07 15:48:10,650][385026] Decorrelating experience for 224 frames... +[2023-03-07 15:48:10,667][382451] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,674][385093] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,691][382416] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,694][382321] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,722][384315] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,732][390548] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,776][385027] Decorrelating experience for 96 frames... +[2023-03-07 15:48:10,821][382327] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,834][383102] Decorrelating experience for 96 frames... +[2023-03-07 15:48:10,838][382420] Decorrelating experience for 128 frames... +[2023-03-07 15:48:10,843][382398] Decorrelating experience for 192 frames... +[2023-03-07 15:48:10,853][382453] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,893][383550] Decorrelating experience for 160 frames... +[2023-03-07 15:48:10,910][383984] Decorrelating experience for 32 frames... +[2023-03-07 15:48:10,911][382424] Decorrelating experience for 64 frames... +[2023-03-07 15:48:10,935][382442] Decorrelating experience for 288 frames... +[2023-03-07 15:48:11,015][382842] Decorrelating experience for 64 frames... +[2023-03-07 15:48:11,018][382456] Decorrelating experience for 128 frames... +[2023-03-07 15:48:11,019][382416] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,034][382411] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,065][382323] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,087][384644] Decorrelating experience for 288 frames... +[2023-03-07 15:48:11,094][384387] Decorrelating experience for 64 frames... +[2023-03-07 15:48:11,096][382439] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,099][382414] Decorrelating experience for 128 frames... +[2023-03-07 15:48:11,182][382455] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,197][383831] Decorrelating experience for 96 frames... +[2023-03-07 15:48:11,198][384503] Decorrelating experience for 96 frames... +[2023-03-07 15:48:11,206][383799] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,242][383427] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,253][382843] Decorrelating experience for 288 frames... +[2023-03-07 15:48:11,268][382446] Decorrelating experience for 64 frames... +[2023-03-07 15:48:11,275][385093] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,278][382582] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,350][382396] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,368][382442] Decorrelating experience for 320 frames... +[2023-03-07 15:48:11,376][382844] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,427][384114] Decorrelating experience for 64 frames... +[2023-03-07 15:48:11,435][382439] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,439][382445] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,449][382430] Decorrelating experience for 96 frames... +[2023-03-07 15:48:11,455][384368] Decorrelating experience for 256 frames... +[2023-03-07 15:48:11,463][382411] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,511][382846] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,546][382322] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,562][383831] Decorrelating experience for 128 frames... +[2023-03-07 15:48:11,616][382431] Decorrelating experience for 352 frames... +[2023-03-07 15:48:11,620][382842] Decorrelating experience for 96 frames... +[2023-03-07 15:48:11,633][382325] Decorrelating experience for 32 frames... +[2023-03-07 15:48:11,636][382424] Decorrelating experience for 96 frames... +[2023-03-07 15:48:11,637][382843] Decorrelating experience for 320 frames... +[2023-03-07 15:48:11,642][382323] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,666][382396] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,692][382422] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,761][382445] Decorrelating experience for 256 frames... +[2023-03-07 15:48:11,762][384503] Decorrelating experience for 128 frames... +[2023-03-07 15:48:11,798][385049] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,801][236315] Heartbeat connected on Batcher_0 +[2023-03-07 15:48:11,803][236315] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 15:48:11,815][382430] Decorrelating experience for 128 frames... +[2023-03-07 15:48:11,818][382442] Decorrelating experience for 352 frames... +[2023-03-07 15:48:11,820][384552] Decorrelating experience for 256 frames... +[2023-03-07 15:48:11,823][383827] Decorrelating experience for 256 frames... +[2023-03-07 15:48:11,834][382439] Decorrelating experience for 224 frames... +[2023-03-07 15:48:11,851][236315] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 15:48:11,865][382455] Decorrelating experience for 256 frames... +[2023-03-07 15:48:11,872][383797] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,938][383831] Decorrelating experience for 160 frames... +[2023-03-07 15:48:11,968][382443] Decorrelating experience for 64 frames... +[2023-03-07 15:48:11,979][382422] Decorrelating experience for 192 frames... +[2023-03-07 15:48:11,993][382405] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,003][382456] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,016][385093] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,019][382452] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,021][383955] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,051][383951] Decorrelating experience for 96 frames... +[2023-03-07 15:48:12,085][382416] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,120][382431] Decorrelating experience for 384 frames... +[2023-03-07 15:48:12,145][383827] Decorrelating experience for 288 frames... +[2023-03-07 15:48:12,160][383799] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,186][382974] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,196][383293] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,218][382441] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,222][382838] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,228][382422] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,228][384962] Decorrelating experience for 288 frames... +[2023-03-07 15:48:12,324][382447] Decorrelating experience for 64 frames... +[2023-03-07 15:48:12,331][383831] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,338][382420] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,366][383951] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,372][382586] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,401][382439] Decorrelating experience for 256 frames... +[2023-03-07 15:48:12,402][382410] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,407][383294] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,423][382841] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,426][382417] Decorrelating experience for 64 frames... +[2023-03-07 15:48:12,519][382974] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,521][382395] Decorrelating experience for 320 frames... +[2023-03-07 15:48:12,522][385093] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,543][382405] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,566][383102] Decorrelating experience for 128 frames... +[2023-03-07 15:48:12,579][382327] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,586][383293] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,591][383951] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,601][382398] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,626][382325] Decorrelating experience for 64 frames... +[2023-03-07 15:48:12,698][382453] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,713][382841] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,726][382420] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,731][382411] Decorrelating experience for 256 frames... +[2023-03-07 15:48:12,744][382430] Decorrelating experience for 160 frames... +[2023-03-07 15:48:12,759][383831] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,775][383797] Decorrelating experience for 224 frames... +[2023-03-07 15:48:12,779][382323] Decorrelating experience for 256 frames... +[2023-03-07 15:48:12,782][383744] Decorrelating experience for 0 frames... +[2023-03-07 15:48:12,800][382405] Decorrelating experience for 256 frames... +[2023-03-07 15:48:12,892][382410] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,911][382434] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,940][382394] Decorrelating experience for 288 frames... +[2023-03-07 15:48:12,942][382843] Decorrelating experience for 352 frames... +[2023-03-07 15:48:12,951][382442] Decorrelating experience for 384 frames... +[2023-03-07 15:48:12,956][382322] Decorrelating experience for 192 frames... +[2023-03-07 15:48:12,958][385095] Decorrelating experience for 384 frames... +[2023-03-07 15:48:12,985][382444] Decorrelating experience for 64 frames... +[2023-03-07 15:48:12,989][383955] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,036][382453] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,093][382425] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,122][384315] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,124][382395] Decorrelating experience for 352 frames... +[2023-03-07 15:48:13,141][383427] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,152][382328] Decorrelating experience for 128 frames... +[2023-03-07 15:48:13,152][382456] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,154][382402] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,163][382398] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,187][382429] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,218][383831] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,271][382405] Decorrelating experience for 288 frames... +[2023-03-07 15:48:13,323][383101] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,327][385031] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,339][382444] Decorrelating experience for 96 frames... +[2023-03-07 15:48:13,347][382420] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,355][382401] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,368][382415] Decorrelating experience for 288 frames... +[2023-03-07 15:48:13,372][382325] Decorrelating experience for 96 frames... +[2023-03-07 15:48:13,402][382456] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,421][382838] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,455][384503] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:13,500][382843] Decorrelating experience for 384 frames... +[2023-03-07 15:48:13,505][382425] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,554][382441] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,563][382405] Decorrelating experience for 320 frames... +[2023-03-07 15:48:13,566][382397] Decorrelating experience for 128 frames... +[2023-03-07 15:48:13,567][382845] Decorrelating experience for 64 frames... +[2023-03-07 15:48:13,596][383639] Decorrelating experience for 96 frames... +[2023-03-07 15:48:13,655][389982] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,668][383101] Decorrelating experience for 288 frames... +[2023-03-07 15:48:13,678][382394] Decorrelating experience for 320 frames... +[2023-03-07 15:48:13,684][382417] Decorrelating experience for 96 frames... +[2023-03-07 15:48:13,685][382430] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,744][383827] Decorrelating experience for 320 frames... +[2023-03-07 15:48:13,756][382453] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,778][382422] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,781][385031] Decorrelating experience for 224 frames... +[2023-03-07 15:48:13,817][382843] Decorrelating experience for 416 frames... +[2023-03-07 15:48:13,841][382414] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,862][382326] Decorrelating experience for 160 frames... +[2023-03-07 15:48:13,862][382838] Decorrelating experience for 192 frames... +[2023-03-07 15:48:13,874][384387] Decorrelating experience for 96 frames... +[2023-03-07 15:48:13,923][382444] Decorrelating experience for 128 frames... +[2023-03-07 15:48:13,928][385095] Decorrelating experience for 416 frames... +[2023-03-07 15:48:13,937][382396] Decorrelating experience for 256 frames... +[2023-03-07 15:48:13,962][382405] Decorrelating experience for 352 frames... +[2023-03-07 15:48:13,969][382406] Decorrelating experience for 64 frames... +[2023-03-07 15:48:14,031][382443] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,044][385093] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,048][385031] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,064][383827] Decorrelating experience for 352 frames... +[2023-03-07 15:48:14,074][382325] Decorrelating experience for 128 frames... +[2023-03-07 15:48:14,111][382400] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,113][382408] Decorrelating experience for 32 frames... +[2023-03-07 15:48:14,195][382425] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,207][383293] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,236][383639] Decorrelating experience for 128 frames... +[2023-03-07 15:48:14,238][382422] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,239][382841] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,250][382843] Decorrelating experience for 448 frames... +[2023-03-07 15:48:14,270][382846] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,311][382586] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,317][384114] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,321][382441] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,412][382430] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,423][384761] Decorrelating experience for 128 frames... +[2023-03-07 15:48:14,429][390548] Decorrelating experience for 160 frames... +[2023-03-07 15:48:14,437][383294] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,451][382435] Decorrelating experience for 0 frames... +[2023-03-07 15:48:14,452][382419] Decorrelating experience for 160 frames... +[2023-03-07 15:48:14,497][383797] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,502][382431] Decorrelating experience for 416 frames... +[2023-03-07 15:48:14,513][382411] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,545][382846] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,591][382446] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,609][384051] Decorrelating experience for 64 frames... +[2023-03-07 15:48:14,637][382449] Decorrelating experience for 160 frames... +[2023-03-07 15:48:14,643][382437] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,650][382840] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,654][382396] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,713][382420] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,724][382414] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,743][384368] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,768][383797] Decorrelating experience for 288 frames... +[2023-03-07 15:48:14,785][385026] Decorrelating experience for 256 frames... +[2023-03-07 15:48:14,815][384761] Decorrelating experience for 160 frames... +[2023-03-07 15:48:14,821][382422] Decorrelating experience for 320 frames... +[2023-03-07 15:48:14,843][382447] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,844][382419] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,847][382443] Decorrelating experience for 128 frames... +[2023-03-07 15:48:14,911][384051] Decorrelating experience for 96 frames... +[2023-03-07 15:48:14,918][390548] Decorrelating experience for 192 frames... +[2023-03-07 15:48:14,925][382416] Decorrelating experience for 224 frames... +[2023-03-07 15:48:14,945][382843] Decorrelating experience for 480 frames... +[2023-03-07 15:48:14,964][385095] Decorrelating experience for 448 frames... +[2023-03-07 15:48:14,998][383527] Decorrelating experience for 128 frames... +[2023-03-07 15:48:15,017][382840] Decorrelating experience for 128 frames... +[2023-03-07 15:48:15,025][382425] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,050][382421] Decorrelating experience for 192 frames... +[2023-03-07 15:48:15,076][382396] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,102][383797] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,108][382455] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,124][385093] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,159][382426] Decorrelating experience for 192 frames... +[2023-03-07 15:48:15,172][384368] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,178][385031] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,241][382447] Decorrelating experience for 128 frames... +[2023-03-07 15:48:15,247][382440] Decorrelating experience for 224 frames... +[2023-03-07 15:48:15,253][384157] Decorrelating experience for 32 frames... +[2023-03-07 15:48:15,293][382328] Decorrelating experience for 160 frames... +[2023-03-07 15:48:15,294][382586] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,305][382841] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,336][382429] Decorrelating experience for 192 frames... +[2023-03-07 15:48:15,364][382402] Decorrelating experience for 224 frames... +[2023-03-07 15:48:15,381][382409] Decorrelating experience for 96 frames... +[2023-03-07 15:48:15,429][384962] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,444][382455] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,477][382425] Decorrelating experience for 352 frames... +[2023-03-07 15:48:15,479][384051] Decorrelating experience for 128 frames... +[2023-03-07 15:48:15,486][382435] Decorrelating experience for 32 frames... +[2023-03-07 15:48:15,512][384157] Decorrelating experience for 64 frames... +[2023-03-07 15:48:15,518][382398] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,539][382846] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,551][383798] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,574][382841] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,627][382586] Decorrelating experience for 320 frames... +[2023-03-07 15:48:15,641][382410] Decorrelating experience for 224 frames... +[2023-03-07 15:48:15,679][382435] Decorrelating experience for 64 frames... +[2023-03-07 15:48:15,680][382402] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,691][382447] Decorrelating experience for 160 frames... +[2023-03-07 15:48:15,704][382397] Decorrelating experience for 160 frames... +[2023-03-07 15:48:15,748][383797] Decorrelating experience for 352 frames... +[2023-03-07 15:48:15,750][382401] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,791][382456] Decorrelating experience for 256 frames... +[2023-03-07 15:48:15,807][384761] Decorrelating experience for 192 frames... +[2023-03-07 15:48:15,825][384962] Decorrelating experience for 352 frames... +[2023-03-07 15:48:15,874][383798] Decorrelating experience for 288 frames... +[2023-03-07 15:48:15,885][382425] Decorrelating experience for 384 frames... +[2023-03-07 15:48:15,889][382429] Decorrelating experience for 224 frames... +[2023-03-07 15:48:15,921][382586] Decorrelating experience for 352 frames... +[2023-03-07 15:48:15,925][384368] Decorrelating experience for 352 frames... +[2023-03-07 15:48:15,946][383827] Decorrelating experience for 384 frames... +[2023-03-07 15:48:15,947][383292] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,004][382443] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,010][384051] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,055][383799] Decorrelating experience for 256 frames... +[2023-03-07 15:48:16,062][382396] Decorrelating experience for 352 frames... +[2023-03-07 15:48:16,079][382447] Decorrelating experience for 192 frames... +[2023-03-07 15:48:16,095][384644] Decorrelating experience for 320 frames... +[2023-03-07 15:48:16,105][383797] Decorrelating experience for 384 frames... +[2023-03-07 15:48:16,151][382431] Decorrelating experience for 448 frames... +[2023-03-07 15:48:16,177][382418] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,198][382455] Decorrelating experience for 352 frames... +[2023-03-07 15:48:16,205][382425] Decorrelating experience for 416 frames... +[2023-03-07 15:48:16,232][382435] Decorrelating experience for 96 frames... +[2023-03-07 15:48:16,233][382416] Decorrelating experience for 256 frames... +[2023-03-07 15:48:16,244][385095] Decorrelating experience for 480 frames... +[2023-03-07 15:48:16,259][382843] Decorrelating experience for 512 frames... +[2023-03-07 15:48:16,330][382405] Decorrelating experience for 384 frames... +[2023-03-07 15:48:16,354][384962] Decorrelating experience for 384 frames... +[2023-03-07 15:48:16,380][382409] Decorrelating experience for 128 frames... +[2023-03-07 15:48:16,382][383799] Decorrelating experience for 288 frames... +[2023-03-07 15:48:16,439][384368] Decorrelating experience for 384 frames... +[2023-03-07 15:48:16,445][384157] Decorrelating experience for 96 frames... +[2023-03-07 15:48:16,448][382846] Decorrelating experience for 288 frames... +[2023-03-07 15:48:16,453][382451] Decorrelating experience for 192 frames... +[2023-03-07 15:48:16,457][383292] Decorrelating experience for 192 frames... +[2023-03-07 15:48:16,504][382396] Decorrelating experience for 384 frames... +[2023-03-07 15:48:16,545][384644] Decorrelating experience for 352 frames... +[2023-03-07 15:48:16,571][382452] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,578][384114] Decorrelating experience for 128 frames... +[2023-03-07 15:48:16,582][382327] Decorrelating experience for 256 frames... +[2023-03-07 15:48:16,622][383827] Decorrelating experience for 416 frames... +[2023-03-07 15:48:16,649][382397] Decorrelating experience for 192 frames... +[2023-03-07 15:48:16,674][382431] Decorrelating experience for 480 frames... +[2023-03-07 15:48:16,696][382420] Decorrelating experience for 288 frames... +[2023-03-07 15:48:16,707][382425] Decorrelating experience for 448 frames... +[2023-03-07 15:48:16,715][383293] Decorrelating experience for 256 frames... +[2023-03-07 15:48:16,735][382447] Decorrelating experience for 224 frames... +[2023-03-07 15:48:16,752][383797] Decorrelating experience for 416 frames... +[2023-03-07 15:48:16,759][382402] Decorrelating experience for 288 frames... +[2023-03-07 15:48:16,809][382405] Decorrelating experience for 416 frames... +[2023-03-07 15:48:16,837][383798] Decorrelating experience for 320 frames... +[2023-03-07 15:48:16,852][382444] Decorrelating experience for 160 frames... +[2023-03-07 15:48:16,854][382582] Decorrelating experience for 192 frames... +[2023-03-07 15:48:16,907][382397] Decorrelating experience for 224 frames... +[2023-03-07 15:48:16,908][382323] Decorrelating experience for 288 frames... +[2023-03-07 15:48:16,910][382845] Decorrelating experience for 96 frames... +[2023-03-07 15:48:16,926][384157] Decorrelating experience for 128 frames... +[2023-03-07 15:48:16,949][385094] Decorrelating experience for 64 frames... +[2023-03-07 15:48:16,962][384643] Decorrelating experience for 160 frames... +[2023-03-07 15:48:17,024][382443] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,047][382447] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,052][382426] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,083][383292] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,094][382444] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,096][382425] Decorrelating experience for 480 frames... +[2023-03-07 15:48:17,101][382324] Decorrelating experience for 160 frames... +[2023-03-07 15:48:17,105][382423] Decorrelating experience for 160 frames... +[2023-03-07 15:48:17,128][385027] Decorrelating experience for 128 frames... +[2023-03-07 15:48:17,161][384503] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,231][382420] Decorrelating experience for 320 frames... +[2023-03-07 15:48:17,271][382405] Decorrelating experience for 448 frames... +[2023-03-07 15:48:17,280][382421] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,283][384368] Decorrelating experience for 416 frames... +[2023-03-07 15:48:17,283][383798] Decorrelating experience for 352 frames... +[2023-03-07 15:48:17,309][385031] Decorrelating experience for 320 frames... +[2023-03-07 15:48:17,311][382847] Decorrelating experience for 96 frames... +[2023-03-07 15:48:17,312][385093] Decorrelating experience for 320 frames... +[2023-03-07 15:48:17,323][382429] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,345][382423] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,410][382450] Decorrelating experience for 32 frames... +[2023-03-07 15:48:17,464][382452] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,469][382841] Decorrelating experience for 320 frames... +[2023-03-07 15:48:17,499][384643] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,507][382437] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,514][382454] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,515][383641] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,515][382406] Decorrelating experience for 96 frames... +[2023-03-07 15:48:17,528][382846] Decorrelating experience for 320 frames... +[2023-03-07 15:48:17,573][382434] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,590][382324] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,655][382393] Decorrelating experience for 160 frames... +[2023-03-07 15:48:17,692][383798] Decorrelating experience for 384 frames... +[2023-03-07 15:48:17,698][382421] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,700][382416] Decorrelating experience for 288 frames... +[2023-03-07 15:48:17,714][382446] Decorrelating experience for 128 frames... +[2023-03-07 15:48:17,715][383984] Decorrelating experience for 64 frames... +[2023-03-07 15:48:17,717][382452] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,751][384962] Decorrelating experience for 416 frames... +[2023-03-07 15:48:17,764][383955] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,771][382844] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,834][383293] Decorrelating experience for 288 frames... +[2023-03-07 15:48:17,875][384644] Decorrelating experience for 384 frames... +[2023-03-07 15:48:17,883][382406] Decorrelating experience for 128 frames... +[2023-03-07 15:48:17,914][382424] Decorrelating experience for 128 frames... +[2023-03-07 15:48:17,915][382432] Decorrelating experience for 64 frames... +[2023-03-07 15:48:17,922][382328] Decorrelating experience for 192 frames... +[2023-03-07 15:48:17,951][382423] Decorrelating experience for 224 frames... +[2023-03-07 15:48:17,954][382434] Decorrelating experience for 256 frames... +[2023-03-07 15:48:17,964][382442] Decorrelating experience for 416 frames... +[2023-03-07 15:48:18,000][384643] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,025][382395] Decorrelating experience for 384 frames... +[2023-03-07 15:48:18,060][382457] Decorrelating experience for 256 frames... +[2023-03-07 15:48:18,063][383827] Decorrelating experience for 448 frames... +[2023-03-07 15:48:18,109][385094] Decorrelating experience for 96 frames... +[2023-03-07 15:48:18,114][383744] Decorrelating experience for 32 frames... +[2023-03-07 15:48:18,117][385031] Decorrelating experience for 352 frames... +[2023-03-07 15:48:18,140][382401] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,141][382443] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,155][382428] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,210][382582] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,223][382420] Decorrelating experience for 352 frames... +[2023-03-07 15:48:18,240][383641] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,304][382400] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,315][384552] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,316][382394] Decorrelating experience for 352 frames... +[2023-03-07 15:48:18,320][384962] Decorrelating experience for 448 frames... +[2023-03-07 15:48:18,324][383984] Decorrelating experience for 96 frames... +[2023-03-07 15:48:18,338][382843] Decorrelating experience for 544 frames... +[2023-03-07 15:48:18,379][384051] Decorrelating experience for 192 frames... +[2023-03-07 15:48:18,392][382424] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,413][382450] Decorrelating experience for 64 frames... +[2023-03-07 15:48:18,418][382455] Decorrelating experience for 384 frames... +[2023-03-07 15:48:18,485][382846] Decorrelating experience for 352 frames... +[2023-03-07 15:48:18,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:18,504][382447] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,519][382417] Decorrelating experience for 128 frames... +[2023-03-07 15:48:18,519][384315] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,527][382407] Decorrelating experience for 256 frames... +[2023-03-07 15:48:18,528][382432] Decorrelating experience for 96 frames... +[2023-03-07 15:48:18,579][385027] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,585][382457] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,595][382324] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,603][384552] Decorrelating experience for 320 frames... +[2023-03-07 15:48:18,664][390548] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,686][382845] Decorrelating experience for 128 frames... +[2023-03-07 15:48:18,708][383744] Decorrelating experience for 64 frames... +[2023-03-07 15:48:18,716][383639] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,717][382406] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,754][383427] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,767][382327] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,771][383101] Decorrelating experience for 320 frames... +[2023-03-07 15:48:18,783][382328] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,784][383292] Decorrelating experience for 256 frames... +[2023-03-07 15:48:18,848][382395] Decorrelating experience for 416 frames... +[2023-03-07 15:48:18,866][382423] Decorrelating experience for 256 frames... +[2023-03-07 15:48:18,891][383984] Decorrelating experience for 128 frames... +[2023-03-07 15:48:18,924][382322] Decorrelating experience for 224 frames... +[2023-03-07 15:48:18,925][383102] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,932][384315] Decorrelating experience for 256 frames... +[2023-03-07 15:48:18,945][382417] Decorrelating experience for 160 frames... +[2023-03-07 15:48:18,957][382425] Decorrelating experience for 512 frames... +[2023-03-07 15:48:18,964][382445] Decorrelating experience for 288 frames... +[2023-03-07 15:48:18,972][382434] Decorrelating experience for 288 frames... +[2023-03-07 15:48:19,025][382450] Decorrelating experience for 96 frames... +[2023-03-07 15:48:19,054][382323] Decorrelating experience for 320 frames... +[2023-03-07 15:48:19,122][383101] Decorrelating experience for 352 frames... +[2023-03-07 15:48:19,123][382430] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,127][382524] Decorrelating experience for 128 frames... +[2023-03-07 15:48:19,129][382441] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,144][382442] Decorrelating experience for 448 frames... +[2023-03-07 15:48:19,148][383797] Decorrelating experience for 448 frames... +[2023-03-07 15:48:19,155][382400] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,175][383639] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,218][382845] Decorrelating experience for 160 frames... +[2023-03-07 15:48:19,233][382417] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,305][384644] Decorrelating experience for 416 frames... +[2023-03-07 15:48:19,315][382406] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,318][382324] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,321][383102] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,328][382843] Decorrelating experience for 576 frames... +[2023-03-07 15:48:19,337][383798] Decorrelating experience for 416 frames... +[2023-03-07 15:48:19,339][382416] Decorrelating experience for 320 frames... +[2023-03-07 15:48:19,373][382321] Decorrelating experience for 160 frames... +[2023-03-07 15:48:19,404][384552] Decorrelating experience for 352 frames... +[2023-03-07 15:48:19,419][384157] Decorrelating experience for 160 frames... +[2023-03-07 15:48:19,496][382418] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,502][382428] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,515][382328] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,521][383827] Decorrelating experience for 480 frames... +[2023-03-07 15:48:19,528][384388] Decorrelating experience for 32 frames... +[2023-03-07 15:48:19,583][383292] Decorrelating experience for 288 frames... +[2023-03-07 15:48:19,588][384761] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,596][382405] Decorrelating experience for 480 frames... +[2023-03-07 15:48:19,609][382445] Decorrelating experience for 320 frames... +[2023-03-07 15:48:19,617][382431] Decorrelating experience for 512 frames... +[2023-03-07 15:48:19,688][382395] Decorrelating experience for 448 frames... +[2023-03-07 15:48:19,702][382844] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,704][382447] Decorrelating experience for 320 frames... +[2023-03-07 15:48:19,708][383639] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,713][384157] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,775][382325] Decorrelating experience for 160 frames... +[2023-03-07 15:48:19,780][382426] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,793][382423] Decorrelating experience for 288 frames... +[2023-03-07 15:48:19,800][383678] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,830][389982] Decorrelating experience for 256 frames... +[2023-03-07 15:48:19,894][382407] Decorrelating experience for 288 frames... +[2023-03-07 15:48:19,898][383641] Decorrelating experience for 320 frames... +[2023-03-07 15:48:19,906][382450] Decorrelating experience for 128 frames... +[2023-03-07 15:48:19,908][382393] Decorrelating experience for 192 frames... +[2023-03-07 15:48:19,934][382406] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,953][382428] Decorrelating experience for 224 frames... +[2023-03-07 15:48:19,964][382445] Decorrelating experience for 352 frames... +[2023-03-07 15:48:19,976][385093] Decorrelating experience for 352 frames... +[2023-03-07 15:48:19,987][384886] Decorrelating experience for 96 frames... +[2023-03-07 15:48:20,028][382410] Decorrelating experience for 256 frames... +[2023-03-07 15:48:20,081][382400] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,116][382321] Decorrelating experience for 192 frames... +[2023-03-07 15:48:20,118][382846] Decorrelating experience for 384 frames... +[2023-03-07 15:48:20,133][383831] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,145][382402] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,170][382444] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,173][382449] Decorrelating experience for 192 frames... +[2023-03-07 15:48:20,174][382420] Decorrelating experience for 384 frames... +[2023-03-07 15:48:20,174][382442] Decorrelating experience for 480 frames... +[2023-03-07 15:48:20,234][384387] Decorrelating experience for 128 frames... +[2023-03-07 15:48:20,260][383798] Decorrelating experience for 448 frames... +[2023-03-07 15:48:20,303][384643] Decorrelating experience for 256 frames... +[2023-03-07 15:48:20,319][382325] Decorrelating experience for 192 frames... +[2023-03-07 15:48:20,361][382398] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,362][384886] Decorrelating experience for 128 frames... +[2023-03-07 15:48:20,366][382426] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,368][382321] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,377][382845] Decorrelating experience for 192 frames... +[2023-03-07 15:48:20,407][382401] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,432][382414] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,441][383101] Decorrelating experience for 384 frames... +[2023-03-07 15:48:20,483][383951] Decorrelating experience for 192 frames... +[2023-03-07 15:48:20,518][384387] Decorrelating experience for 160 frames... +[2023-03-07 15:48:20,547][382417] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,551][382402] Decorrelating experience for 352 frames... +[2023-03-07 15:48:20,557][382410] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,577][382436] Decorrelating experience for 128 frames... +[2023-03-07 15:48:20,587][385026] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,588][382846] Decorrelating experience for 416 frames... +[2023-03-07 15:48:20,621][383292] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,627][382420] Decorrelating experience for 416 frames... +[2023-03-07 15:48:20,665][382418] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,733][384503] Decorrelating experience for 224 frames... +[2023-03-07 15:48:20,764][382398] Decorrelating experience for 352 frames... +[2023-03-07 15:48:20,773][384552] Decorrelating experience for 384 frames... +[2023-03-07 15:48:20,774][382428] Decorrelating experience for 256 frames... +[2023-03-07 15:48:20,783][383427] Decorrelating experience for 256 frames... +[2023-03-07 15:48:20,783][382422] Decorrelating experience for 352 frames... +[2023-03-07 15:48:20,787][382438] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,806][382429] Decorrelating experience for 288 frames... +[2023-03-07 15:48:20,809][383798] Decorrelating experience for 480 frames... +[2023-03-07 15:48:20,845][382412] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:20,845][382414] Decorrelating experience for 256 frames... +[2023-03-07 15:48:20,919][383831] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,956][382447] Decorrelating experience for 352 frames... +[2023-03-07 15:48:20,976][383292] Decorrelating experience for 352 frames... +[2023-03-07 15:48:20,978][382400] Decorrelating experience for 320 frames... +[2023-03-07 15:48:20,983][385096] Decorrelating experience for 128 frames... +[2023-03-07 15:48:20,996][384368] Decorrelating experience for 448 frames... +[2023-03-07 15:48:20,997][382450] Decorrelating experience for 160 frames... +[2023-03-07 15:48:21,021][385026] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,028][382844] Decorrelating experience for 256 frames... +[2023-03-07 15:48:21,091][382327] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,101][382455] Decorrelating experience for 416 frames... +[2023-03-07 15:48:21,143][384552] Decorrelating experience for 416 frames... +[2023-03-07 15:48:21,190][382418] Decorrelating experience for 256 frames... +[2023-03-07 15:48:21,194][382586] Decorrelating experience for 384 frames... +[2023-03-07 15:48:21,195][384643] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,196][382448] Decorrelating experience for 192 frames... +[2023-03-07 15:48:21,206][382441] Decorrelating experience for 256 frames... +[2023-03-07 15:48:21,212][383427] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,220][382438] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,269][382414] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,286][382450] Decorrelating experience for 192 frames... +[2023-03-07 15:48:21,299][384158] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:21,325][382398] Decorrelating experience for 384 frames... +[2023-03-07 15:48:21,377][382402] Decorrelating experience for 384 frames... +[2023-03-07 15:48:21,383][382846] Decorrelating experience for 448 frames... +[2023-03-07 15:48:21,388][382325] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,389][382399] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,391][382326] Decorrelating experience for 192 frames... +[2023-03-07 15:48:21,431][384388] Decorrelating experience for 64 frames... +[2023-03-07 15:48:21,438][382455] Decorrelating experience for 448 frames... +[2023-03-07 15:48:21,449][382322] Decorrelating experience for 256 frames... +[2023-03-07 15:48:21,484][384643] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,505][382328] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,564][383797] Decorrelating experience for 480 frames... +[2023-03-07 15:48:21,568][382420] Decorrelating experience for 448 frames... +[2023-03-07 15:48:21,574][389982] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,580][382400] Decorrelating experience for 352 frames... +[2023-03-07 15:48:21,585][383527] Decorrelating experience for 160 frames... +[2023-03-07 15:48:21,616][382418] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,638][384157] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,640][382423] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,665][382327] Decorrelating experience for 352 frames... +[2023-03-07 15:48:21,690][382326] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,748][383293] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,750][384503] Decorrelating experience for 256 frames... +[2023-03-07 15:48:21,775][382393] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,783][384643] Decorrelating experience for 352 frames... +[2023-03-07 15:48:21,811][383798] Decorrelating experience for 512 frames... +[2023-03-07 15:48:21,821][382447] Decorrelating experience for 384 frames... +[2023-03-07 15:48:21,835][382441] Decorrelating experience for 288 frames... +[2023-03-07 15:48:21,842][382450] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,881][382449] Decorrelating experience for 224 frames... +[2023-03-07 15:48:21,917][382431] Decorrelating experience for 544 frames... +[2023-03-07 15:48:21,937][384388] Decorrelating experience for 96 frames... +[2023-03-07 15:48:21,943][384114] Decorrelating experience for 160 frames... +[2023-03-07 15:48:21,967][382426] Decorrelating experience for 320 frames... +[2023-03-07 15:48:21,976][382327] Decorrelating experience for 384 frames... +[2023-03-07 15:48:22,003][382396] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,013][382425] Decorrelating experience for 544 frames... +[2023-03-07 15:48:22,025][384503] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,035][383797] Decorrelating experience for 512 frames... +[2023-03-07 15:48:22,063][384051] Decorrelating experience for 224 frames... +[2023-03-07 15:48:22,100][390548] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,125][382323] Decorrelating experience for 352 frames... +[2023-03-07 15:48:22,150][382321] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,155][382326] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,186][382410] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,200][385093] Decorrelating experience for 384 frames... +[2023-03-07 15:48:22,206][382844] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,208][382399] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,244][382524] Decorrelating experience for 160 frames... +[2023-03-07 15:48:22,289][383293] Decorrelating experience for 352 frames... +[2023-03-07 15:48:22,306][382325] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,336][382435] Decorrelating experience for 128 frames... +[2023-03-07 15:48:22,345][382455] Decorrelating experience for 480 frames... +[2023-03-07 15:48:22,346][382453] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,364][384761] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,389][382445] Decorrelating experience for 384 frames... +[2023-03-07 15:48:22,398][384643] Decorrelating experience for 384 frames... +[2023-03-07 15:48:22,409][382398] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,426][390548] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,489][382396] Decorrelating experience for 448 frames... +[2023-03-07 15:48:22,497][382407] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,529][382321] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,530][382441] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,560][382838] Decorrelating experience for 224 frames... +[2023-03-07 15:48:22,564][383831] Decorrelating experience for 352 frames... +[2023-03-07 15:48:22,570][382426] Decorrelating experience for 352 frames... +[2023-03-07 15:48:22,602][382395] Decorrelating experience for 480 frames... +[2023-03-07 15:48:22,608][382448] Decorrelating experience for 224 frames... +[2023-03-07 15:48:22,615][383678] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,702][383293] Decorrelating experience for 384 frames... +[2023-03-07 15:48:22,710][382418] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,717][382420] Decorrelating experience for 480 frames... +[2023-03-07 15:48:22,741][382586] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,749][385027] Decorrelating experience for 192 frames... +[2023-03-07 15:48:22,754][382327] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,771][382457] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,800][382328] Decorrelating experience for 320 frames... +[2023-03-07 15:48:22,804][382455] Decorrelating experience for 512 frames... +[2023-03-07 15:48:22,810][382454] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,893][383294] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,905][382447] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,905][382325] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,926][384643] Decorrelating experience for 416 frames... +[2023-03-07 15:48:22,932][384315] Decorrelating experience for 288 frames... +[2023-03-07 15:48:22,939][382396] Decorrelating experience for 480 frames... +[2023-03-07 15:48:22,954][382444] Decorrelating experience for 256 frames... +[2023-03-07 15:48:22,984][384051] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,007][385027] Decorrelating experience for 224 frames... +[2023-03-07 15:48:23,025][382321] Decorrelating experience for 320 frames... +[2023-03-07 15:48:23,081][385096] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,125][382431] Decorrelating experience for 576 frames... +[2023-03-07 15:48:23,125][382419] Decorrelating experience for 224 frames... +[2023-03-07 15:48:23,128][383797] Decorrelating experience for 544 frames... +[2023-03-07 15:48:23,130][389982] Decorrelating experience for 320 frames... +[2023-03-07 15:48:23,132][382423] Decorrelating experience for 352 frames... +[2023-03-07 15:48:23,145][383678] Decorrelating experience for 288 frames... +[2023-03-07 15:48:23,189][384114] Decorrelating experience for 192 frames... +[2023-03-07 15:48:23,204][382446] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,215][384157] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,313][382422] Decorrelating experience for 384 frames... +[2023-03-07 15:48:23,325][384051] Decorrelating experience for 288 frames... +[2023-03-07 15:48:23,329][382445] Decorrelating experience for 416 frames... +[2023-03-07 15:48:23,330][382425] Decorrelating experience for 576 frames... +[2023-03-07 15:48:23,333][385093] Decorrelating experience for 416 frames... +[2023-03-07 15:48:23,334][383799] Decorrelating experience for 320 frames... +[2023-03-07 15:48:23,339][382426] Decorrelating experience for 384 frames... +[2023-03-07 15:48:23,368][382408] Decorrelating experience for 64 frames... +[2023-03-07 15:48:23,385][382419] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,412][382429] Decorrelating experience for 320 frames... +[2023-03-07 15:48:23,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:23,523][385049] Decorrelating experience for 192 frames... +[2023-03-07 15:48:23,528][384644] Decorrelating experience for 448 frames... +[2023-03-07 15:48:23,529][382840] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,529][382431] Decorrelating experience for 608 frames... +[2023-03-07 15:48:23,531][382406] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,536][382586] Decorrelating experience for 448 frames... +[2023-03-07 15:48:23,551][382524] Decorrelating experience for 192 frames... +[2023-03-07 15:48:23,563][382450] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,588][384387] Decorrelating experience for 192 frames... +[2023-03-07 15:48:23,593][382447] Decorrelating experience for 448 frames... +[2023-03-07 15:48:23,709][383951] Decorrelating experience for 224 frames... +[2023-03-07 15:48:23,717][382443] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,718][382582] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,723][382408] Decorrelating experience for 96 frames... +[2023-03-07 15:48:23,734][389982] Decorrelating experience for 352 frames... +[2023-03-07 15:48:23,741][382451] Decorrelating experience for 224 frames... +[2023-03-07 15:48:23,745][384503] Decorrelating experience for 320 frames... +[2023-03-07 15:48:23,746][382410] Decorrelating experience for 352 frames... +[2023-03-07 15:48:23,788][382840] Decorrelating experience for 192 frames... +[2023-03-07 15:48:23,865][383798] Decorrelating experience for 544 frames... +[2023-03-07 15:48:23,936][382409] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,937][382419] Decorrelating experience for 288 frames... +[2023-03-07 15:48:23,941][385049] Decorrelating experience for 224 frames... +[2023-03-07 15:48:23,942][382436] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,946][382418] Decorrelating experience for 352 frames... +[2023-03-07 15:48:23,948][383984] Decorrelating experience for 160 frames... +[2023-03-07 15:48:23,966][384157] Decorrelating experience for 288 frames... +[2023-03-07 15:48:23,971][383951] Decorrelating experience for 256 frames... +[2023-03-07 15:48:23,977][382398] Decorrelating experience for 448 frames... +[2023-03-07 15:48:24,020][382427] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:24,093][389982] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,130][385093] Decorrelating experience for 448 frames... +[2023-03-07 15:48:24,140][382321] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,140][385031] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,143][382838] Decorrelating experience for 256 frames... +[2023-03-07 15:48:24,144][382586] Decorrelating experience for 480 frames... +[2023-03-07 15:48:24,158][382401] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,160][384503] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,193][384368] Decorrelating experience for 480 frames... +[2023-03-07 15:48:24,221][382396] Decorrelating experience for 512 frames... +[2023-03-07 15:48:24,343][383797] Decorrelating experience for 576 frames... +[2023-03-07 15:48:24,344][382394] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,345][382428] Decorrelating experience for 288 frames... +[2023-03-07 15:48:24,346][382452] Decorrelating experience for 256 frames... +[2023-03-07 15:48:24,351][382582] Decorrelating experience for 288 frames... +[2023-03-07 15:48:24,362][384157] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,378][382325] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,382][382411] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,426][382420] Decorrelating experience for 512 frames... +[2023-03-07 15:48:24,468][385031] Decorrelating experience for 416 frames... +[2023-03-07 15:48:24,527][382419] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,538][382435] Decorrelating experience for 160 frames... +[2023-03-07 15:48:24,538][384643] Decorrelating experience for 448 frames... +[2023-03-07 15:48:24,539][382426] Decorrelating experience for 416 frames... +[2023-03-07 15:48:24,542][382422] Decorrelating experience for 416 frames... +[2023-03-07 15:48:24,558][382400] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,585][385094] Decorrelating experience for 128 frames... +[2023-03-07 15:48:24,617][382401] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,664][382394] Decorrelating experience for 416 frames... +[2023-03-07 15:48:24,671][382452] Decorrelating experience for 288 frames... +[2023-03-07 15:48:24,709][382843] Decorrelating experience for 608 frames... +[2023-03-07 15:48:24,731][382443] Decorrelating experience for 288 frames... +[2023-03-07 15:48:24,737][382421] Decorrelating experience for 288 frames... +[2023-03-07 15:48:24,739][382429] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,744][384368] Decorrelating experience for 512 frames... +[2023-03-07 15:48:24,776][384503] Decorrelating experience for 384 frames... +[2023-03-07 15:48:24,789][382449] Decorrelating experience for 256 frames... +[2023-03-07 15:48:24,828][383427] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,873][382426] Decorrelating experience for 448 frames... +[2023-03-07 15:48:24,888][382438] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,916][382403] Decorrelating experience for 160 frames... +[2023-03-07 15:48:24,916][382435] Decorrelating experience for 192 frames... +[2023-03-07 15:48:24,928][382457] Decorrelating experience for 352 frames... +[2023-03-07 15:48:24,932][382422] Decorrelating experience for 448 frames... +[2023-03-07 15:48:24,935][382431] Decorrelating experience for 640 frames... +[2023-03-07 15:48:24,958][382452] Decorrelating experience for 320 frames... +[2023-03-07 15:48:24,980][382325] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,045][383294] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,083][382419] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,090][382400] Decorrelating experience for 416 frames... +[2023-03-07 15:48:25,103][382412] Decorrelating experience for 96 frames... +[2023-03-07 15:48:25,121][385095] Decorrelating experience for 512 frames... +[2023-03-07 15:48:25,122][382328] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,126][384157] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,147][383101] Decorrelating experience for 416 frames... +[2023-03-07 15:48:25,148][383797] Decorrelating experience for 608 frames... +[2023-03-07 15:48:25,160][382844] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,224][383427] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,281][382415] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,302][382399] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,308][382403] Decorrelating experience for 192 frames... +[2023-03-07 15:48:25,323][382843] Decorrelating experience for 640 frames... +[2023-03-07 15:48:25,331][385094] Decorrelating experience for 160 frames... +[2023-03-07 15:48:25,336][382398] Decorrelating experience for 480 frames... +[2023-03-07 15:48:25,343][382436] Decorrelating experience for 192 frames... +[2023-03-07 15:48:25,348][383294] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,434][385026] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,460][383639] Decorrelating experience for 256 frames... +[2023-03-07 15:48:25,484][382456] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,491][382401] Decorrelating experience for 416 frames... +[2023-03-07 15:48:25,505][384503] Decorrelating experience for 416 frames... +[2023-03-07 15:48:25,536][385096] Decorrelating experience for 192 frames... +[2023-03-07 15:48:25,543][385095] Decorrelating experience for 544 frames... +[2023-03-07 15:48:25,590][389982] Decorrelating experience for 416 frames... +[2023-03-07 15:48:25,600][383101] Decorrelating experience for 448 frames... +[2023-03-07 15:48:25,641][385093] Decorrelating experience for 480 frames... +[2023-03-07 15:48:25,644][382411] Decorrelating experience for 352 frames... +[2023-03-07 15:48:25,668][385031] Decorrelating experience for 448 frames... +[2023-03-07 15:48:25,673][382443] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,718][382406] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,748][382435] Decorrelating experience for 224 frames... +[2023-03-07 15:48:25,774][382456] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,786][382328] Decorrelating experience for 384 frames... +[2023-03-07 15:48:25,826][382400] Decorrelating experience for 448 frames... +[2023-03-07 15:48:25,838][382430] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,856][382322] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,897][382434] Decorrelating experience for 320 frames... +[2023-03-07 15:48:25,905][383639] Decorrelating experience for 288 frames... +[2023-03-07 15:48:25,931][382422] Decorrelating experience for 480 frames... +[2023-03-07 15:48:25,938][382451] Decorrelating experience for 256 frames... +[2023-03-07 15:48:25,967][382418] Decorrelating experience for 384 frames... +[2023-03-07 15:48:25,981][382401] Decorrelating experience for 448 frames... +[2023-03-07 15:48:26,008][384158] Decorrelating experience for 64 frames... +[2023-03-07 15:48:26,019][383101] Decorrelating experience for 480 frames... +[2023-03-07 15:48:26,044][385027] Decorrelating experience for 256 frames... +[2023-03-07 15:48:26,045][385096] Decorrelating experience for 224 frames... +[2023-03-07 15:48:26,082][385095] Decorrelating experience for 576 frames... +[2023-03-07 15:48:26,092][383799] Decorrelating experience for 352 frames... +[2023-03-07 15:48:26,113][382403] Decorrelating experience for 224 frames... +[2023-03-07 15:48:26,121][382410] Decorrelating experience for 384 frames... +[2023-03-07 15:48:26,147][384503] Decorrelating experience for 448 frames... +[2023-03-07 15:48:26,190][382430] Decorrelating experience for 320 frames... +[2023-03-07 15:48:26,210][382328] Decorrelating experience for 416 frames... +[2023-03-07 15:48:26,244][384315] Decorrelating experience for 320 frames... +[2023-03-07 15:48:26,248][382446] Decorrelating experience for 192 frames... +[2023-03-07 15:48:26,272][382844] Decorrelating experience for 352 frames... +[2023-03-07 15:48:26,292][382426] Decorrelating experience for 480 frames... +[2023-03-07 15:48:26,297][382398] Decorrelating experience for 512 frames... +[2023-03-07 15:48:26,339][382422] Decorrelating experience for 512 frames... +[2023-03-07 15:48:26,366][382325] Decorrelating experience for 384 frames... +[2023-03-07 15:48:26,374][382838] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,390][382418] Decorrelating experience for 416 frames... +[2023-03-07 15:48:26,445][382448] Decorrelating experience for 256 frames... +[2023-03-07 15:48:26,460][382326] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,477][382449] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,486][382840] Decorrelating experience for 224 frames... +[2023-03-07 15:48:26,492][382327] Decorrelating experience for 448 frames... +[2023-03-07 15:48:26,494][384503] Decorrelating experience for 480 frames... +[2023-03-07 15:48:26,519][382321] Decorrelating experience for 384 frames... +[2023-03-07 15:48:26,577][383827] Decorrelating experience for 512 frames... +[2023-03-07 15:48:26,617][382444] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,654][382845] Decorrelating experience for 224 frames... +[2023-03-07 15:48:26,655][382405] Decorrelating experience for 512 frames... +[2023-03-07 15:48:26,670][382451] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,680][382452] Decorrelating experience for 352 frames... +[2023-03-07 15:48:26,683][382423] Decorrelating experience for 384 frames... +[2023-03-07 15:48:26,702][383527] Decorrelating experience for 192 frames... +[2023-03-07 15:48:26,727][385093] Decorrelating experience for 512 frames... +[2023-03-07 15:48:26,775][382400] Decorrelating experience for 480 frames... +[2023-03-07 15:48:26,809][382420] Decorrelating experience for 544 frames... +[2023-03-07 15:48:26,851][383639] Decorrelating experience for 320 frames... +[2023-03-07 15:48:26,855][382419] Decorrelating experience for 384 frames... +[2023-03-07 15:48:26,864][385096] Decorrelating experience for 256 frames... +[2023-03-07 15:48:26,873][382398] Decorrelating experience for 544 frames... +[2023-03-07 15:48:26,873][382449] Decorrelating experience for 320 frames... +[2023-03-07 15:48:26,895][382324] Decorrelating experience for 288 frames... +[2023-03-07 15:48:26,916][383797] Decorrelating experience for 640 frames... +[2023-03-07 15:48:26,954][383527] Decorrelating experience for 224 frames... +[2023-03-07 15:48:27,008][382443] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,038][382403] Decorrelating experience for 256 frames... +[2023-03-07 15:48:27,049][382430] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,056][382423] Decorrelating experience for 416 frames... +[2023-03-07 15:48:27,061][383799] Decorrelating experience for 384 frames... +[2023-03-07 15:48:27,067][382843] Decorrelating experience for 672 frames... +[2023-03-07 15:48:27,087][382426] Decorrelating experience for 512 frames... +[2023-03-07 15:48:27,099][382457] Decorrelating experience for 384 frames... +[2023-03-07 15:48:27,139][385096] Decorrelating experience for 288 frames... +[2023-03-07 15:48:27,148][382418] Decorrelating experience for 448 frames... +[2023-03-07 15:48:27,192][382452] Decorrelating experience for 384 frames... +[2023-03-07 15:48:27,220][382422] Decorrelating experience for 544 frames... +[2023-03-07 15:48:27,248][382327] Decorrelating experience for 480 frames... +[2023-03-07 15:48:27,255][389982] Decorrelating experience for 448 frames... +[2023-03-07 15:48:27,273][384387] Decorrelating experience for 224 frames... +[2023-03-07 15:48:27,299][382324] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,325][382405] Decorrelating experience for 544 frames... +[2023-03-07 15:48:27,329][383951] Decorrelating experience for 288 frames... +[2023-03-07 15:48:27,347][382456] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,372][382449] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,401][382322] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,425][385096] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,439][382412] Decorrelating experience for 128 frames... +[2023-03-07 15:48:27,452][382325] Decorrelating experience for 416 frames... +[2023-03-07 15:48:27,470][382435] Decorrelating experience for 256 frames... +[2023-03-07 15:48:27,516][382441] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,531][383101] Decorrelating experience for 512 frames... +[2023-03-07 15:48:27,537][382400] Decorrelating experience for 512 frames... +[2023-03-07 15:48:27,540][382524] Decorrelating experience for 224 frames... +[2023-03-07 15:48:27,557][382326] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,601][382401] Decorrelating experience for 480 frames... +[2023-03-07 15:48:27,620][382443] Decorrelating experience for 384 frames... +[2023-03-07 15:48:27,652][383827] Decorrelating experience for 544 frames... +[2023-03-07 15:48:27,676][382452] Decorrelating experience for 416 frames... +[2023-03-07 15:48:27,678][382412] Decorrelating experience for 160 frames... +[2023-03-07 15:48:27,711][382414] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,714][382444] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,745][382423] Decorrelating experience for 448 frames... +[2023-03-07 15:48:27,783][384368] Decorrelating experience for 544 frames... +[2023-03-07 15:48:27,794][382845] Decorrelating experience for 256 frames... +[2023-03-07 15:48:27,805][382524] Decorrelating experience for 256 frames... +[2023-03-07 15:48:27,807][389982] Decorrelating experience for 480 frames... +[2023-03-07 15:48:27,871][382403] Decorrelating experience for 288 frames... +[2023-03-07 15:48:27,874][383951] Decorrelating experience for 320 frames... +[2023-03-07 15:48:27,897][382420] Decorrelating experience for 576 frames... +[2023-03-07 15:48:27,903][382435] Decorrelating experience for 288 frames... +[2023-03-07 15:48:27,953][384387] Decorrelating experience for 256 frames... +[2023-03-07 15:48:27,955][382841] Decorrelating experience for 352 frames... +[2023-03-07 15:48:27,968][382454] Decorrelating experience for 288 frames... +[2023-03-07 15:48:27,987][385094] Decorrelating experience for 192 frames... +[2023-03-07 15:48:27,997][382421] Decorrelating experience for 320 frames... +[2023-03-07 15:48:28,012][383797] Decorrelating experience for 672 frames... +[2023-03-07 15:48:28,073][382411] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,083][383292] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,085][382843] Decorrelating experience for 704 frames... +[2023-03-07 15:48:28,141][382844] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,152][382321] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,183][382524] Decorrelating experience for 288 frames... +[2023-03-07 15:48:28,189][383827] Decorrelating experience for 576 frames... +[2023-03-07 15:48:28,207][382422] Decorrelating experience for 576 frames... +[2023-03-07 15:48:28,207][382445] Decorrelating experience for 448 frames... +[2023-03-07 15:48:28,219][389982] Decorrelating experience for 512 frames... +[2023-03-07 15:48:28,275][382438] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,283][382447] Decorrelating experience for 480 frames... +[2023-03-07 15:48:28,287][385093] Decorrelating experience for 544 frames... +[2023-03-07 15:48:28,330][384761] Decorrelating experience for 288 frames... +[2023-03-07 15:48:28,340][382407] Decorrelating experience for 352 frames... +[2023-03-07 15:48:28,375][382412] Decorrelating experience for 192 frames... +[2023-03-07 15:48:28,397][382420] Decorrelating experience for 608 frames... +[2023-03-07 15:48:28,399][382406] Decorrelating experience for 320 frames... +[2023-03-07 15:48:28,409][383831] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,447][382411] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,477][382846] Decorrelating experience for 480 frames... +[2023-03-07 15:48:28,479][382431] Decorrelating experience for 672 frames... +[2023-03-07 15:48:28,491][382419] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:28,514][382400] Decorrelating experience for 544 frames... +[2023-03-07 15:48:28,524][382408] Decorrelating experience for 128 frames... +[2023-03-07 15:48:28,558][382409] Decorrelating experience for 192 frames... +[2023-03-07 15:48:28,591][382321] Decorrelating experience for 448 frames... +[2023-03-07 15:48:28,612][383292] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,613][382410] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,670][382451] Decorrelating experience for 320 frames... +[2023-03-07 15:48:28,674][384761] Decorrelating experience for 320 frames... +[2023-03-07 15:48:28,695][382449] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,701][382406] Decorrelating experience for 352 frames... +[2023-03-07 15:48:28,713][383641] Decorrelating experience for 352 frames... +[2023-03-07 15:48:28,713][382438] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,752][382415] Decorrelating experience for 352 frames... +[2023-03-07 15:48:28,778][382840] Decorrelating experience for 256 frames... +[2023-03-07 15:48:28,814][382407] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,816][382397] Decorrelating experience for 256 frames... +[2023-03-07 15:48:28,857][382420] Decorrelating experience for 640 frames... +[2023-03-07 15:48:28,880][383294] Decorrelating experience for 384 frames... +[2023-03-07 15:48:28,887][383831] Decorrelating experience for 416 frames... +[2023-03-07 15:48:28,898][382325] Decorrelating experience for 448 frames... +[2023-03-07 15:48:28,904][382400] Decorrelating experience for 576 frames... +[2023-03-07 15:48:28,906][382422] Decorrelating experience for 608 frames... +[2023-03-07 15:48:28,933][385096] Decorrelating experience for 352 frames... +[2023-03-07 15:48:28,961][382421] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,003][382410] Decorrelating experience for 448 frames... +[2023-03-07 15:48:29,019][382455] Decorrelating experience for 544 frames... +[2023-03-07 15:48:29,038][382393] Decorrelating experience for 256 frames... +[2023-03-07 15:48:29,080][384761] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,087][382457] Decorrelating experience for 416 frames... +[2023-03-07 15:48:29,098][382441] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,098][383827] Decorrelating experience for 608 frames... +[2023-03-07 15:48:29,120][383101] Decorrelating experience for 544 frames... +[2023-03-07 15:48:29,191][382444] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,197][382447] Decorrelating experience for 512 frames... +[2023-03-07 15:48:29,209][383294] Decorrelating experience for 416 frames... +[2023-03-07 15:48:29,220][382403] Decorrelating experience for 320 frames... +[2023-03-07 15:48:29,225][382451] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,275][383831] Decorrelating experience for 448 frames... +[2023-03-07 15:48:29,285][382440] Decorrelating experience for 256 frames... +[2023-03-07 15:48:29,322][382415] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,333][382323] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,344][382321] Decorrelating experience for 480 frames... +[2023-03-07 15:48:29,375][384368] Decorrelating experience for 576 frames... +[2023-03-07 15:48:29,383][382436] Decorrelating experience for 224 frames... +[2023-03-07 15:48:29,410][382416] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,417][382445] Decorrelating experience for 480 frames... +[2023-03-07 15:48:29,421][383293] Decorrelating experience for 416 frames... +[2023-03-07 15:48:29,469][382398] Decorrelating experience for 576 frames... +[2023-03-07 15:48:29,487][382394] Decorrelating experience for 448 frames... +[2023-03-07 15:48:29,517][382428] Decorrelating experience for 320 frames... +[2023-03-07 15:48:29,539][382451] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,543][382421] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,567][382847] Decorrelating experience for 128 frames... +[2023-03-07 15:48:29,586][382409] Decorrelating experience for 224 frames... +[2023-03-07 15:48:29,590][382426] Decorrelating experience for 544 frames... +[2023-03-07 15:48:29,623][382437] Decorrelating experience for 288 frames... +[2023-03-07 15:48:29,641][382412] Decorrelating experience for 224 frames... +[2023-03-07 15:48:29,659][383292] Decorrelating experience for 448 frames... +[2023-03-07 15:48:29,701][389982] Decorrelating experience for 544 frames... +[2023-03-07 15:48:29,726][382416] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,747][382403] Decorrelating experience for 352 frames... +[2023-03-07 15:48:29,757][383797] Decorrelating experience for 704 frames... +[2023-03-07 15:48:29,758][382410] Decorrelating experience for 480 frames... +[2023-03-07 15:48:29,772][384503] Decorrelating experience for 512 frames... +[2023-03-07 15:48:29,782][382453] Decorrelating experience for 320 frames... +[2023-03-07 15:48:29,806][383293] Decorrelating experience for 448 frames... +[2023-03-07 15:48:29,860][383550] Decorrelating experience for 192 frames... +[2023-03-07 15:48:29,882][382429] Decorrelating experience for 384 frames... +[2023-03-07 15:48:29,954][382443] Decorrelating experience for 416 frames... +[2023-03-07 15:48:29,976][384387] Decorrelating experience for 288 frames... +[2023-03-07 15:48:29,977][382446] Decorrelating experience for 224 frames... +[2023-03-07 15:48:29,978][382440] Decorrelating experience for 288 frames... +[2023-03-07 15:48:29,987][382838] Decorrelating experience for 320 frames... +[2023-03-07 15:48:30,012][382408] Decorrelating experience for 160 frames... +[2023-03-07 15:48:30,042][382974] Decorrelating experience for 192 frames... +[2023-03-07 15:48:30,054][382403] Decorrelating experience for 384 frames... +[2023-03-07 15:48:30,133][382432] Decorrelating experience for 128 frames... +[2023-03-07 15:48:30,171][382415] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,176][383292] Decorrelating experience for 480 frames... +[2023-03-07 15:48:30,177][383639] Decorrelating experience for 352 frames... +[2023-03-07 15:48:30,177][382451] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,179][382844] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,204][383797] Decorrelating experience for 736 frames... +[2023-03-07 15:48:30,213][382444] Decorrelating experience for 384 frames... +[2023-03-07 15:48:30,241][382446] Decorrelating experience for 256 frames... +[2023-03-07 15:48:30,252][382418] Decorrelating experience for 480 frames... +[2023-03-07 15:48:30,320][382428] Decorrelating experience for 352 frames... +[2023-03-07 15:48:30,358][382401] Decorrelating experience for 512 frames... +[2023-03-07 15:48:30,374][382412] Decorrelating experience for 256 frames... +[2023-03-07 15:48:30,374][384051] Decorrelating experience for 320 frames... +[2023-03-07 15:48:30,378][382454] Decorrelating experience for 320 frames... +[2023-03-07 15:48:30,378][382847] Decorrelating experience for 160 frames... +[2023-03-07 15:48:30,418][382432] Decorrelating experience for 160 frames... +[2023-03-07 15:48:30,440][382838] Decorrelating experience for 352 frames... +[2023-03-07 15:48:30,470][382440] Decorrelating experience for 320 frames... +[2023-03-07 15:48:30,490][382443] Decorrelating experience for 448 frames... +[2023-03-07 15:48:30,507][382415] Decorrelating experience for 448 frames... +[2023-03-07 15:48:30,560][383293] Decorrelating experience for 480 frames... +[2023-03-07 15:48:30,568][382451] Decorrelating experience for 448 frames... +[2023-03-07 15:48:30,574][382446] Decorrelating experience for 288 frames... +[2023-03-07 15:48:30,575][384643] Decorrelating experience for 480 frames... +[2023-03-07 15:48:30,600][382322] Decorrelating experience for 352 frames... +[2023-03-07 15:48:30,621][382431] Decorrelating experience for 704 frames... +[2023-03-07 15:48:30,631][382403] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,757][382416] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,763][382453] Decorrelating experience for 352 frames... +[2023-03-07 15:48:30,768][382843] Decorrelating experience for 736 frames... +[2023-03-07 15:48:30,769][383799] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,770][382400] Decorrelating experience for 608 frames... +[2023-03-07 15:48:30,786][382428] Decorrelating experience for 384 frames... +[2023-03-07 15:48:30,789][382418] Decorrelating experience for 512 frames... +[2023-03-07 15:48:30,808][382436] Decorrelating experience for 256 frames... +[2023-03-07 15:48:30,864][382426] Decorrelating experience for 576 frames... +[2023-03-07 15:48:30,946][382444] Decorrelating experience for 416 frames... +[2023-03-07 15:48:30,959][385094] Decorrelating experience for 224 frames... +[2023-03-07 15:48:30,962][382435] Decorrelating experience for 320 frames... +[2023-03-07 15:48:30,979][383293] Decorrelating experience for 512 frames... +[2023-03-07 15:48:30,979][382844] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,001][385031] Decorrelating experience for 480 frames... +[2023-03-07 15:48:31,016][382450] Decorrelating experience for 288 frames... +[2023-03-07 15:48:31,027][385027] Decorrelating experience for 288 frames... +[2023-03-07 15:48:31,048][382446] Decorrelating experience for 320 frames... +[2023-03-07 15:48:31,062][383797] Decorrelating experience for 768 frames... +[2023-03-07 15:48:31,129][382457] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,140][382417] Decorrelating experience for 256 frames... +[2023-03-07 15:48:31,150][382443] Decorrelating experience for 480 frames... +[2023-03-07 15:48:31,186][382441] Decorrelating experience for 416 frames... +[2023-03-07 15:48:31,203][382524] Decorrelating experience for 320 frames... +[2023-03-07 15:48:31,209][382438] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,239][382327] Decorrelating experience for 512 frames... +[2023-03-07 15:48:31,261][382435] Decorrelating experience for 352 frames... +[2023-03-07 15:48:31,267][382451] Decorrelating experience for 480 frames... +[2023-03-07 15:48:31,278][382444] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,324][382410] Decorrelating experience for 512 frames... +[2023-03-07 15:48:31,329][382398] Decorrelating experience for 608 frames... +[2023-03-07 15:48:31,336][382400] Decorrelating experience for 640 frames... +[2023-03-07 15:48:31,370][383827] Decorrelating experience for 640 frames... +[2023-03-07 15:48:31,385][384315] Decorrelating experience for 352 frames... +[2023-03-07 15:48:31,396][384114] Decorrelating experience for 224 frames... +[2023-03-07 15:48:31,464][385027] Decorrelating experience for 320 frames... +[2023-03-07 15:48:31,470][382417] Decorrelating experience for 288 frames... +[2023-03-07 15:48:31,476][382431] Decorrelating experience for 736 frames... +[2023-03-07 15:48:31,504][382844] Decorrelating experience for 480 frames... +[2023-03-07 15:48:31,513][382397] Decorrelating experience for 288 frames... +[2023-03-07 15:48:31,518][382408] Decorrelating experience for 192 frames... +[2023-03-07 15:48:31,524][382323] Decorrelating experience for 416 frames... +[2023-03-07 15:48:31,557][382447] Decorrelating experience for 544 frames... +[2023-03-07 15:48:31,577][382840] Decorrelating experience for 288 frames... +[2023-03-07 15:48:31,580][385094] Decorrelating experience for 256 frames... +[2023-03-07 15:48:31,668][382842] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:31,674][382441] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,688][383831] Decorrelating experience for 480 frames... +[2023-03-07 15:48:31,697][382450] Decorrelating experience for 320 frames... +[2023-03-07 15:48:31,703][382406] Decorrelating experience for 384 frames... +[2023-03-07 15:48:31,705][382974] Decorrelating experience for 224 frames... +[2023-03-07 15:48:31,714][382418] Decorrelating experience for 544 frames... +[2023-03-07 15:48:31,753][382416] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,760][383799] Decorrelating experience for 448 frames... +[2023-03-07 15:48:31,760][382327] Decorrelating experience for 544 frames... +[2023-03-07 15:48:31,785][382322] Decorrelating experience for 384 frames... +[2023-03-07 15:48:31,894][382446] Decorrelating experience for 352 frames... +[2023-03-07 15:48:31,896][382326] Decorrelating experience for 352 frames... +[2023-03-07 15:48:31,913][382417] Decorrelating experience for 320 frames... +[2023-03-07 15:48:31,933][382449] Decorrelating experience for 416 frames... +[2023-03-07 15:48:31,936][382405] Decorrelating experience for 576 frames... +[2023-03-07 15:48:31,944][384643] Decorrelating experience for 512 frames... +[2023-03-07 15:48:31,960][382453] Decorrelating experience for 384 frames... +[2023-03-07 15:48:31,967][384114] Decorrelating experience for 256 frames... +[2023-03-07 15:48:32,024][385094] Decorrelating experience for 288 frames... +[2023-03-07 15:48:32,081][382843] Decorrelating experience for 768 frames... +[2023-03-07 15:48:32,109][383799] Decorrelating experience for 480 frames... +[2023-03-07 15:48:32,122][382443] Decorrelating experience for 512 frames... +[2023-03-07 15:48:32,128][382412] Decorrelating experience for 288 frames... +[2023-03-07 15:48:32,135][384387] Decorrelating experience for 320 frames... +[2023-03-07 15:48:32,137][382456] Decorrelating experience for 384 frames... +[2023-03-07 15:48:32,140][383797] Decorrelating experience for 800 frames... +[2023-03-07 15:48:32,150][385093] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,153][383678] Decorrelating experience for 320 frames... +[2023-03-07 15:48:32,241][383827] Decorrelating experience for 672 frames... +[2023-03-07 15:48:32,262][383294] Decorrelating experience for 448 frames... +[2023-03-07 15:48:32,317][382327] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,318][382326] Decorrelating experience for 384 frames... +[2023-03-07 15:48:32,326][382438] Decorrelating experience for 480 frames... +[2023-03-07 15:48:32,331][382428] Decorrelating experience for 416 frames... +[2023-03-07 15:48:32,331][382455] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,342][382453] Decorrelating experience for 416 frames... +[2023-03-07 15:48:32,345][382454] Decorrelating experience for 352 frames... +[2023-03-07 15:48:32,363][382424] Decorrelating experience for 192 frames... +[2023-03-07 15:48:32,466][382418] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,501][382439] Another process currently holds the lock /tmp/sf2_rolo/doom_003.lockfile, attempt: 1 +[2023-03-07 15:48:32,504][382445] Decorrelating experience for 512 frames... +[2023-03-07 15:48:32,513][385094] Decorrelating experience for 320 frames... +[2023-03-07 15:48:32,518][382846] Decorrelating experience for 512 frames... +[2023-03-07 15:48:32,520][382415] Decorrelating experience for 480 frames... +[2023-03-07 15:48:32,527][382411] Decorrelating experience for 448 frames... +[2023-03-07 15:48:32,533][382435] Decorrelating experience for 384 frames... +[2023-03-07 15:48:32,547][382447] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,575][382423] Decorrelating experience for 480 frames... +[2023-03-07 15:48:32,673][383799] Decorrelating experience for 512 frames... +[2023-03-07 15:48:32,689][384051] Decorrelating experience for 352 frames... +[2023-03-07 15:48:32,721][389982] Decorrelating experience for 576 frames... +[2023-03-07 15:48:32,722][382450] Decorrelating experience for 352 frames... +[2023-03-07 15:48:32,723][385049] Decorrelating experience for 256 frames... +[2023-03-07 15:48:32,728][382417] Decorrelating experience for 352 frames... +[2023-03-07 15:48:32,729][382838] Decorrelating experience for 384 frames... +[2023-03-07 15:48:32,771][382397] Decorrelating experience for 320 frames... +[2023-03-07 15:48:32,810][383294] Decorrelating experience for 480 frames... +[2023-03-07 15:48:32,815][385094] Decorrelating experience for 352 frames... +[2023-03-07 15:48:32,877][382840] Decorrelating experience for 320 frames... +[2023-03-07 15:48:32,901][384114] Decorrelating experience for 288 frames... +[2023-03-07 15:48:32,915][383639] Decorrelating experience for 384 frames... +[2023-03-07 15:48:32,917][382448] Decorrelating experience for 288 frames... +[2023-03-07 15:48:32,929][383827] Decorrelating experience for 704 frames... +[2023-03-07 15:48:32,964][382395] Decorrelating experience for 512 frames... +[2023-03-07 15:48:32,980][384158] Decorrelating experience for 96 frames... +[2023-03-07 15:48:32,999][384368] Decorrelating experience for 608 frames... +[2023-03-07 15:48:33,016][383955] Decorrelating experience for 256 frames... +[2023-03-07 15:48:33,026][382445] Decorrelating experience for 544 frames... +[2023-03-07 15:48:33,097][382400] Decorrelating experience for 672 frames... +[2023-03-07 15:48:33,101][383292] Decorrelating experience for 512 frames... +[2023-03-07 15:48:33,110][382843] Decorrelating experience for 800 frames... +[2023-03-07 15:48:33,115][389982] Decorrelating experience for 608 frames... +[2023-03-07 15:48:33,139][382397] Decorrelating experience for 352 frames... +[2023-03-07 15:48:33,165][382323] Decorrelating experience for 448 frames... +[2023-03-07 15:48:33,181][382417] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,219][384051] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,234][383639] Decorrelating experience for 416 frames... +[2023-03-07 15:48:33,285][382408] Decorrelating experience for 224 frames... +[2023-03-07 15:48:33,289][382396] Decorrelating experience for 544 frames... +[2023-03-07 15:48:33,300][382452] Decorrelating experience for 448 frames... +[2023-03-07 15:48:33,305][384387] Decorrelating experience for 352 frames... +[2023-03-07 15:48:33,329][385094] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,336][382840] Decorrelating experience for 352 frames... +[2023-03-07 15:48:33,358][382409] Decorrelating experience for 256 frames... +[2023-03-07 15:48:33,368][383678] Decorrelating experience for 352 frames... +[2023-03-07 15:48:33,412][382586] Decorrelating experience for 512 frames... +[2023-03-07 15:48:33,441][384114] Decorrelating experience for 320 frames... +[2023-03-07 15:48:33,474][382420] Decorrelating experience for 672 frames... +[2023-03-07 15:48:33,489][382847] Decorrelating experience for 192 frames... +[2023-03-07 15:48:33,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:33,516][382430] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,518][382844] Decorrelating experience for 512 frames... +[2023-03-07 15:48:33,536][382453] Decorrelating experience for 448 frames... +[2023-03-07 15:48:33,541][382841] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,576][384643] Decorrelating experience for 544 frames... +[2023-03-07 15:48:33,593][384886] Decorrelating experience for 160 frames... +[2023-03-07 15:48:33,651][385049] Decorrelating experience for 288 frames... +[2023-03-07 15:48:33,656][384552] Decorrelating experience for 448 frames... +[2023-03-07 15:48:33,672][382428] Decorrelating experience for 448 frames... +[2023-03-07 15:48:33,701][382582] Decorrelating experience for 320 frames... +[2023-03-07 15:48:33,706][382436] Decorrelating experience for 288 frames... +[2023-03-07 15:48:33,733][385031] Decorrelating experience for 512 frames... +[2023-03-07 15:48:33,733][383678] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,781][384387] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,811][383827] Decorrelating experience for 736 frames... +[2023-03-07 15:48:33,844][382431] Decorrelating experience for 768 frames... +[2023-03-07 15:48:33,845][384114] Decorrelating experience for 352 frames... +[2023-03-07 15:48:33,856][382448] Decorrelating experience for 320 frames... +[2023-03-07 15:48:33,895][382844] Decorrelating experience for 544 frames... +[2023-03-07 15:48:33,896][384157] Decorrelating experience for 384 frames... +[2023-03-07 15:48:33,915][382323] Decorrelating experience for 480 frames... +[2023-03-07 15:48:33,932][382453] Decorrelating experience for 480 frames... +[2023-03-07 15:48:33,962][382842] Decorrelating experience for 128 frames... +[2023-03-07 15:48:34,003][382838] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,027][382396] Decorrelating experience for 576 frames... +[2023-03-07 15:48:34,043][382430] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,048][382847] Decorrelating experience for 224 frames... +[2023-03-07 15:48:34,080][389982] Decorrelating experience for 640 frames... +[2023-03-07 15:48:34,151][385049] Decorrelating experience for 320 frames... +[2023-03-07 15:48:34,155][382846] Decorrelating experience for 544 frames... +[2023-03-07 15:48:34,158][382327] Decorrelating experience for 608 frames... +[2023-03-07 15:48:34,163][382328] Decorrelating experience for 448 frames... +[2023-03-07 15:48:34,190][382450] Decorrelating experience for 384 frames... +[2023-03-07 15:48:34,211][382421] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,255][382408] Decorrelating experience for 256 frames... +[2023-03-07 15:48:34,296][384114] Decorrelating experience for 384 frames... +[2023-03-07 15:48:34,297][384051] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,342][382429] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,345][382409] Decorrelating experience for 288 frames... +[2023-03-07 15:48:34,356][382436] Decorrelating experience for 320 frames... +[2023-03-07 15:48:34,357][382838] Decorrelating experience for 448 frames... +[2023-03-07 15:48:34,397][382457] Decorrelating experience for 480 frames... +[2023-03-07 15:48:34,458][382400] Decorrelating experience for 704 frames... +[2023-03-07 15:48:34,460][382431] Decorrelating experience for 800 frames... +[2023-03-07 15:48:34,485][383292] Decorrelating experience for 544 frames... +[2023-03-07 15:48:34,488][384368] Decorrelating experience for 640 frames... +[2023-03-07 15:48:34,523][382323] Decorrelating experience for 512 frames... +[2023-03-07 15:48:34,545][382437] Decorrelating experience for 320 frames... +[2023-03-07 15:48:34,548][383955] Decorrelating experience for 288 frames... +[2023-03-07 15:48:34,557][382448] Decorrelating experience for 352 frames... +[2023-03-07 15:48:34,596][382421] Decorrelating experience for 448 frames... +[2023-03-07 15:48:34,653][382844] Decorrelating experience for 576 frames... +[2023-03-07 15:48:34,662][382436] Decorrelating experience for 352 frames... +[2023-03-07 15:48:34,670][382426] Decorrelating experience for 608 frames... +[2023-03-07 15:48:34,676][382428] Decorrelating experience for 480 frames... +[2023-03-07 15:48:34,707][385049] Decorrelating experience for 352 frames... +[2023-03-07 15:48:34,730][382427] Decorrelating experience for 96 frames... +[2023-03-07 15:48:34,749][383294] Decorrelating experience for 512 frames... +[2023-03-07 15:48:34,761][383678] Decorrelating experience for 416 frames... +[2023-03-07 15:48:34,843][383955] Decorrelating experience for 320 frames... +[2023-03-07 15:48:34,863][384051] Decorrelating experience for 448 frames... +[2023-03-07 15:48:34,873][382448] Decorrelating experience for 384 frames... +[2023-03-07 15:48:34,883][382409] Decorrelating experience for 320 frames... +[2023-03-07 15:48:34,888][385095] Decorrelating experience for 608 frames... +[2023-03-07 15:48:34,889][382425] Decorrelating experience for 608 frames... +[2023-03-07 15:48:34,932][382403] Decorrelating experience for 448 frames... +[2023-03-07 15:48:34,936][382432] Decorrelating experience for 192 frames... +[2023-03-07 15:48:34,950][382427] Decorrelating experience for 128 frames... +[2023-03-07 15:48:34,974][382438] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,040][382431] Decorrelating experience for 832 frames... +[2023-03-07 15:48:35,103][382838] Decorrelating experience for 480 frames... +[2023-03-07 15:48:35,103][382405] Decorrelating experience for 608 frames... +[2023-03-07 15:48:35,104][382408] Decorrelating experience for 288 frames... +[2023-03-07 15:48:35,104][382321] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,104][382398] Decorrelating experience for 640 frames... +[2023-03-07 15:48:35,134][382415] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,134][382974] Decorrelating experience for 256 frames... +[2023-03-07 15:48:35,137][382394] Decorrelating experience for 480 frames... +[2023-03-07 15:48:35,174][389982] Decorrelating experience for 672 frames... +[2023-03-07 15:48:35,272][382428] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,313][382425] Decorrelating experience for 640 frames... +[2023-03-07 15:48:35,314][382421] Decorrelating experience for 480 frames... +[2023-03-07 15:48:35,321][383678] Decorrelating experience for 448 frames... +[2023-03-07 15:48:35,322][385095] Decorrelating experience for 640 frames... +[2023-03-07 15:48:35,339][383797] Decorrelating experience for 832 frames... +[2023-03-07 15:48:35,346][382586] Decorrelating experience for 544 frames... +[2023-03-07 15:48:35,347][384552] Decorrelating experience for 480 frames... +[2023-03-07 15:48:35,378][382841] Decorrelating experience for 416 frames... +[2023-03-07 15:48:35,402][383292] Decorrelating experience for 576 frames... +[2023-03-07 15:48:35,477][385049] Decorrelating experience for 384 frames... +[2023-03-07 15:48:35,500][382438] Decorrelating experience for 544 frames... +[2023-03-07 15:48:35,512][382415] Decorrelating experience for 544 frames... +[2023-03-07 15:48:35,530][382398] Decorrelating experience for 672 frames... +[2023-03-07 15:48:35,540][382847] Decorrelating experience for 256 frames... +[2023-03-07 15:48:35,545][382453] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,546][382846] Decorrelating experience for 576 frames... +[2023-03-07 15:48:35,567][382974] Decorrelating experience for 288 frames... +[2023-03-07 15:48:35,611][382400] Decorrelating experience for 736 frames... +[2023-03-07 15:48:35,617][382409] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,687][382430] Decorrelating experience for 448 frames... +[2023-03-07 15:48:35,707][382437] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,742][382434] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,751][382324] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,752][382447] Decorrelating experience for 608 frames... +[2023-03-07 15:48:35,752][382582] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,755][382394] Decorrelating experience for 512 frames... +[2023-03-07 15:48:35,803][385095] Decorrelating experience for 672 frames... +[2023-03-07 15:48:35,814][384368] Decorrelating experience for 672 frames... +[2023-03-07 15:48:35,878][383294] Decorrelating experience for 544 frames... +[2023-03-07 15:48:35,889][382424] Decorrelating experience for 224 frames... +[2023-03-07 15:48:35,935][382524] Decorrelating experience for 352 frames... +[2023-03-07 15:48:35,941][384387] Decorrelating experience for 416 frames... +[2023-03-07 15:48:35,948][382438] Decorrelating experience for 576 frames... +[2023-03-07 15:48:35,953][382846] Decorrelating experience for 608 frames... +[2023-03-07 15:48:35,954][382425] Decorrelating experience for 672 frames... +[2023-03-07 15:48:36,004][383292] Decorrelating experience for 608 frames... +[2023-03-07 15:48:36,011][382974] Decorrelating experience for 320 frames... +[2023-03-07 15:48:36,066][382324] Decorrelating experience for 384 frames... +[2023-03-07 15:48:36,080][382838] Decorrelating experience for 512 frames... +[2023-03-07 15:48:36,083][382417] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,136][382453] Decorrelating experience for 544 frames... +[2023-03-07 15:48:36,149][382406] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,150][382431] Decorrelating experience for 864 frames... +[2023-03-07 15:48:36,151][382429] Decorrelating experience for 448 frames... +[2023-03-07 15:48:36,153][382415] Decorrelating experience for 576 frames... +[2023-03-07 15:48:36,208][382424] Decorrelating experience for 256 frames... +[2023-03-07 15:48:36,222][382437] Decorrelating experience for 384 frames... +[2023-03-07 15:48:36,268][384051] Decorrelating experience for 480 frames... +[2023-03-07 15:48:36,284][384157] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,349][383427] Decorrelating experience for 384 frames... +[2023-03-07 15:48:36,349][382328] Decorrelating experience for 480 frames... +[2023-03-07 15:48:36,349][385094] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,416][382322] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,417][383798] Decorrelating experience for 576 frames... +[2023-03-07 15:48:36,418][383292] Decorrelating experience for 640 frames... +[2023-03-07 15:48:36,452][383797] Decorrelating experience for 864 frames... +[2023-03-07 15:48:36,480][382447] Decorrelating experience for 640 frames... +[2023-03-07 15:48:36,485][382426] Decorrelating experience for 640 frames... +[2023-03-07 15:48:36,494][382421] Decorrelating experience for 512 frames... +[2023-03-07 15:48:36,543][383639] Decorrelating experience for 448 frames... +[2023-03-07 15:48:36,544][382450] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,547][385095] Decorrelating experience for 704 frames... +[2023-03-07 15:48:36,610][382438] Decorrelating experience for 608 frames... +[2023-03-07 15:48:36,611][382415] Decorrelating experience for 608 frames... +[2023-03-07 15:48:36,617][389982] Decorrelating experience for 704 frames... +[2023-03-07 15:48:36,637][382452] Decorrelating experience for 480 frames... +[2023-03-07 15:48:36,683][383427] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,690][385094] Decorrelating experience for 448 frames... +[2023-03-07 15:48:36,692][383984] Decorrelating experience for 192 frames... +[2023-03-07 15:48:36,735][384051] Decorrelating experience for 512 frames... +[2023-03-07 15:48:36,748][385031] Decorrelating experience for 544 frames... +[2023-03-07 15:48:36,749][382420] Decorrelating experience for 704 frames... +[2023-03-07 15:48:36,808][382429] Decorrelating experience for 480 frames... +[2023-03-07 15:48:36,840][382394] Decorrelating experience for 544 frames... +[2023-03-07 15:48:36,853][382417] Decorrelating experience for 448 frames... +[2023-03-07 15:48:36,895][383678] Decorrelating experience for 480 frames... +[2023-03-07 15:48:36,896][382450] Decorrelating experience for 448 frames... +[2023-03-07 15:48:36,900][382326] Decorrelating experience for 416 frames... +[2023-03-07 15:48:36,922][382974] Decorrelating experience for 352 frames... +[2023-03-07 15:48:36,945][382405] Decorrelating experience for 640 frames... +[2023-03-07 15:48:36,989][382424] Decorrelating experience for 288 frames... +[2023-03-07 15:48:37,011][382440] Decorrelating experience for 352 frames... +[2023-03-07 15:48:37,024][383798] Decorrelating experience for 608 frames... +[2023-03-07 15:48:37,036][385026] Decorrelating experience for 384 frames... +[2023-03-07 15:48:37,094][385096] Decorrelating experience for 384 frames... +[2023-03-07 15:48:37,095][382327] Decorrelating experience for 640 frames... +[2023-03-07 15:48:37,112][383955] Decorrelating experience for 352 frames... +[2023-03-07 15:48:37,113][382428] Decorrelating experience for 544 frames... +[2023-03-07 15:48:37,133][385031] Decorrelating experience for 576 frames... +[2023-03-07 15:48:37,196][382407] Decorrelating experience for 416 frames... +[2023-03-07 15:48:37,216][384051] Decorrelating experience for 544 frames... +[2023-03-07 15:48:37,229][382398] Decorrelating experience for 704 frames... +[2023-03-07 15:48:37,240][382326] Decorrelating experience for 448 frames... +[2023-03-07 15:48:37,297][383550] Decorrelating experience for 224 frames... +[2023-03-07 15:48:37,300][382450] Decorrelating experience for 480 frames... +[2023-03-07 15:48:37,300][384157] Decorrelating experience for 448 frames... +[2023-03-07 15:48:37,318][382438] Decorrelating experience for 640 frames... +[2023-03-07 15:48:37,318][382323] Decorrelating experience for 544 frames... +[2023-03-07 15:48:37,320][389982] Decorrelating experience for 736 frames... +[2023-03-07 15:48:37,393][382454] Decorrelating experience for 384 frames... +[2023-03-07 15:48:37,400][390548] Decorrelating experience for 320 frames... +[2023-03-07 15:48:37,415][385049] Decorrelating experience for 416 frames... +[2023-03-07 15:48:37,423][383955] Decorrelating experience for 384 frames... +[2023-03-07 15:48:37,512][382437] Decorrelating experience for 416 frames... +[2023-03-07 15:48:37,514][383639] Decorrelating experience for 480 frames... +[2023-03-07 15:48:37,529][383294] Decorrelating experience for 576 frames... +[2023-03-07 15:48:37,530][382457] Decorrelating experience for 512 frames... +[2023-03-07 15:48:37,575][382431] Decorrelating experience for 896 frames... +[2023-03-07 15:48:37,575][383798] Decorrelating experience for 640 frames... +[2023-03-07 15:48:37,580][382582] Decorrelating experience for 384 frames... +[2023-03-07 15:48:37,642][382844] Decorrelating experience for 608 frames... +[2023-03-07 15:48:37,657][382321] Decorrelating experience for 544 frames... +[2023-03-07 15:48:37,704][382420] Decorrelating experience for 736 frames... +[2023-03-07 15:48:37,708][390548] Decorrelating experience for 352 frames... +[2023-03-07 15:48:37,722][382323] Decorrelating experience for 576 frames... +[2023-03-07 15:48:37,722][383550] Decorrelating experience for 256 frames... +[2023-03-07 15:48:37,760][382324] Decorrelating experience for 416 frames... +[2023-03-07 15:48:37,771][382405] Decorrelating experience for 672 frames... +[2023-03-07 15:48:37,788][382847] Decorrelating experience for 288 frames... +[2023-03-07 15:48:37,792][382327] Decorrelating experience for 672 frames... +[2023-03-07 15:48:37,830][382394] Decorrelating experience for 576 frames... +[2023-03-07 15:48:37,844][382412] Decorrelating experience for 320 frames... +[2023-03-07 15:48:37,891][382438] Decorrelating experience for 672 frames... +[2023-03-07 15:48:37,910][382432] Decorrelating experience for 224 frames... +[2023-03-07 15:48:37,932][384157] Decorrelating experience for 480 frames... +[2023-03-07 15:48:37,932][383639] Decorrelating experience for 512 frames... +[2023-03-07 15:48:37,945][382449] Decorrelating experience for 448 frames... +[2023-03-07 15:48:37,964][382582] Decorrelating experience for 416 frames... +[2023-03-07 15:48:37,984][389982] Decorrelating experience for 768 frames... +[2023-03-07 15:48:38,047][382446] Decorrelating experience for 384 frames... +[2023-03-07 15:48:38,054][382409] Decorrelating experience for 384 frames... +[2023-03-07 15:48:38,074][382417] Decorrelating experience for 480 frames... +[2023-03-07 15:48:38,091][383797] Decorrelating experience for 896 frames... +[2023-03-07 15:48:38,113][383550] Decorrelating experience for 288 frames... +[2023-03-07 15:48:38,137][383427] Decorrelating experience for 448 frames... +[2023-03-07 15:48:38,142][382424] Decorrelating experience for 320 frames... +[2023-03-07 15:48:38,185][382415] Decorrelating experience for 640 frames... +[2023-03-07 15:48:38,215][382456] Decorrelating experience for 416 frames... +[2023-03-07 15:48:38,228][382324] Decorrelating experience for 448 frames... +[2023-03-07 15:48:38,265][383984] Decorrelating experience for 224 frames... +[2023-03-07 15:48:38,279][385096] Decorrelating experience for 416 frames... +[2023-03-07 15:48:38,312][382426] Decorrelating experience for 672 frames... +[2023-03-07 15:48:38,322][383101] Decorrelating experience for 576 frames... +[2023-03-07 15:48:38,328][382323] Decorrelating experience for 608 frames... +[2023-03-07 15:48:38,334][382327] Decorrelating experience for 704 frames... +[2023-03-07 15:48:38,353][382408] Decorrelating experience for 320 frames... +[2023-03-07 15:48:38,401][383292] Decorrelating experience for 672 frames... +[2023-03-07 15:48:38,432][382394] Decorrelating experience for 608 frames... +[2023-03-07 15:48:38,434][382407] Decorrelating experience for 448 frames... +[2023-03-07 15:48:38,463][383798] Decorrelating experience for 672 frames... +[2023-03-07 15:48:38,477][382438] Decorrelating experience for 704 frames... +[2023-03-07 15:48:38,496][384114] Decorrelating experience for 416 frames... +[2023-03-07 15:48:38,497][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:38,517][384962] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:38,521][382447] Decorrelating experience for 672 frames... +[2023-03-07 15:48:38,523][382443] Decorrelating experience for 544 frames... +[2023-03-07 15:48:38,539][382457] Decorrelating experience for 544 frames... +[2023-03-07 15:48:38,595][382409] Decorrelating experience for 416 frames... +[2023-03-07 15:48:38,607][385031] Decorrelating experience for 608 frames... +[2023-03-07 15:48:38,651][383550] Decorrelating experience for 320 frames... +[2023-03-07 15:48:38,665][383639] Decorrelating experience for 544 frames... +[2023-03-07 15:48:38,719][389982] Decorrelating experience for 800 frames... +[2023-03-07 15:48:38,724][382428] Decorrelating experience for 576 frames... +[2023-03-07 15:48:38,724][382328] Decorrelating experience for 512 frames... +[2023-03-07 15:48:38,724][383678] Decorrelating experience for 512 frames... +[2023-03-07 15:48:38,767][383744] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:38,791][384552] Decorrelating experience for 512 frames... +[2023-03-07 15:48:38,792][382327] Decorrelating experience for 736 frames... +[2023-03-07 15:48:38,811][382431] Decorrelating experience for 928 frames... +[2023-03-07 15:48:38,847][382321] Decorrelating experience for 576 frames... +[2023-03-07 15:48:38,927][382419] Decorrelating experience for 448 frames... +[2023-03-07 15:48:38,931][383101] Decorrelating experience for 608 frames... +[2023-03-07 15:48:38,931][385049] Decorrelating experience for 448 frames... +[2023-03-07 15:48:38,993][382434] Decorrelating experience for 384 frames... +[2023-03-07 15:48:38,994][382395] Decorrelating experience for 544 frames... +[2023-03-07 15:48:38,995][382452] Decorrelating experience for 512 frames... +[2023-03-07 15:48:38,996][383955] Decorrelating experience for 416 frames... +[2023-03-07 15:48:39,033][385031] Decorrelating experience for 640 frames... +[2023-03-07 15:48:39,036][382421] Decorrelating experience for 544 frames... +[2023-03-07 15:48:39,075][383798] Decorrelating experience for 704 frames... +[2023-03-07 15:48:39,126][382446] Decorrelating experience for 416 frames... +[2023-03-07 15:48:39,127][382524] Decorrelating experience for 384 frames... +[2023-03-07 15:48:39,140][383984] Decorrelating experience for 256 frames... +[2023-03-07 15:48:39,198][382456] Decorrelating experience for 448 frames... +[2023-03-07 15:48:39,199][382429] Decorrelating experience for 512 frames... +[2023-03-07 15:48:39,199][382398] Decorrelating experience for 736 frames... +[2023-03-07 15:48:39,202][382846] Decorrelating experience for 640 frames... +[2023-03-07 15:48:39,230][382454] Decorrelating experience for 416 frames... +[2023-03-07 15:48:39,258][383639] Decorrelating experience for 576 frames... +[2023-03-07 15:48:39,315][382582] Decorrelating experience for 448 frames... +[2023-03-07 15:48:39,318][382434] Decorrelating experience for 416 frames... +[2023-03-07 15:48:39,326][382447] Decorrelating experience for 704 frames... +[2023-03-07 15:48:39,351][382327] Decorrelating experience for 768 frames... +[2023-03-07 15:48:39,388][390548] Decorrelating experience for 384 frames... +[2023-03-07 15:48:39,397][383292] Decorrelating experience for 704 frames... +[2023-03-07 15:48:39,400][382847] Decorrelating experience for 320 frames... +[2023-03-07 15:48:39,416][382417] Decorrelating experience for 512 frames... +[2023-03-07 15:48:39,422][383102] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:39,434][389982] Decorrelating experience for 832 frames... +[2023-03-07 15:48:39,509][382410] Decorrelating experience for 544 frames... +[2023-03-07 15:48:39,520][382431] Decorrelating experience for 960 frames... +[2023-03-07 15:48:39,545][383827] Decorrelating experience for 768 frames... +[2023-03-07 15:48:39,581][383984] Decorrelating experience for 288 frames... +[2023-03-07 15:48:39,587][382440] Decorrelating experience for 384 frames... +[2023-03-07 15:48:39,609][383678] Decorrelating experience for 544 frames... +[2023-03-07 15:48:39,610][382424] Decorrelating experience for 352 frames... +[2023-03-07 15:48:39,616][382446] Decorrelating experience for 448 frames... +[2023-03-07 15:48:39,639][383101] Decorrelating experience for 640 frames... +[2023-03-07 15:48:39,666][382841] Decorrelating experience for 448 frames... +[2023-03-07 15:48:39,696][382323] Decorrelating experience for 640 frames... +[2023-03-07 15:48:39,710][382398] Decorrelating experience for 768 frames... +[2023-03-07 15:48:39,750][382421] Decorrelating experience for 576 frames... +[2023-03-07 15:48:39,782][382419] Decorrelating experience for 480 frames... +[2023-03-07 15:48:39,802][382394] Decorrelating experience for 640 frames... +[2023-03-07 15:48:39,823][390548] Decorrelating experience for 416 frames... +[2023-03-07 15:48:39,855][382847] Decorrelating experience for 352 frames... +[2023-03-07 15:48:39,881][382426] Decorrelating experience for 704 frames... +[2023-03-07 15:48:39,885][382439] Decorrelating experience for 288 frames... +[2023-03-07 15:48:39,886][382450] Decorrelating experience for 512 frames... +[2023-03-07 15:48:39,932][382582] Decorrelating experience for 480 frames... +[2023-03-07 15:48:39,933][382424] Decorrelating experience for 384 frames... +[2023-03-07 15:48:39,943][383292] Decorrelating experience for 736 frames... +[2023-03-07 15:48:39,976][382328] Decorrelating experience for 544 frames... +[2023-03-07 15:48:39,997][384114] Decorrelating experience for 448 frames... +[2023-03-07 15:48:40,049][382410] Decorrelating experience for 576 frames... +[2023-03-07 15:48:40,082][382430] Decorrelating experience for 480 frames... +[2023-03-07 15:48:40,083][383797] Decorrelating experience for 928 frames... +[2023-03-07 15:48:40,088][382327] Decorrelating experience for 800 frames... +[2023-03-07 15:48:40,091][383951] Decorrelating experience for 352 frames... +[2023-03-07 15:48:40,130][382322] Decorrelating experience for 448 frames... +[2023-03-07 15:48:40,139][383101] Decorrelating experience for 672 frames... +[2023-03-07 15:48:40,161][382524] Decorrelating experience for 416 frames... +[2023-03-07 15:48:40,184][382448] Decorrelating experience for 416 frames... +[2023-03-07 15:48:40,259][384157] Decorrelating experience for 512 frames... +[2023-03-07 15:48:40,276][382414] Decorrelating experience for 352 frames... +[2023-03-07 15:48:40,282][383984] Decorrelating experience for 320 frames... +[2023-03-07 15:48:40,309][382846] Decorrelating experience for 672 frames... +[2023-03-07 15:48:40,329][382405] Decorrelating experience for 704 frames... +[2023-03-07 15:48:40,337][382426] Decorrelating experience for 736 frames... +[2023-03-07 15:48:40,338][382412] Decorrelating experience for 352 frames... +[2023-03-07 15:48:40,349][382407] Decorrelating experience for 480 frames... +[2023-03-07 15:48:40,349][382447] Decorrelating experience for 736 frames... +[2023-03-07 15:48:40,375][383827] Decorrelating experience for 800 frames... +[2023-03-07 15:48:40,452][382582] Decorrelating experience for 512 frames... +[2023-03-07 15:48:40,463][383831] Decorrelating experience for 512 frames... +[2023-03-07 15:48:40,475][382417] Decorrelating experience for 544 frames... +[2023-03-07 15:48:40,478][382442] Another process currently holds the lock /tmp/sf2_rolo/doom_009.lockfile, attempt: 1 +[2023-03-07 15:48:40,529][382325] Decorrelating experience for 480 frames... +[2023-03-07 15:48:40,532][382448] Decorrelating experience for 448 frames... +[2023-03-07 15:48:40,541][382397] Decorrelating experience for 384 frames... +[2023-03-07 15:48:40,557][382394] Decorrelating experience for 672 frames... +[2023-03-07 15:48:40,587][383292] Decorrelating experience for 768 frames... +[2023-03-07 15:48:40,591][382398] Decorrelating experience for 800 frames... +[2023-03-07 15:48:40,592][382327] Decorrelating experience for 832 frames... +[2023-03-07 15:48:40,652][382430] Decorrelating experience for 512 frames... +[2023-03-07 15:48:40,655][382524] Decorrelating experience for 448 frames... +[2023-03-07 15:48:40,666][383293] Decorrelating experience for 544 frames... +[2023-03-07 15:48:40,717][382418] Decorrelating experience for 608 frames... +[2023-03-07 15:48:40,744][383798] Decorrelating experience for 736 frames... +[2023-03-07 15:48:40,747][382846] Decorrelating experience for 704 frames... +[2023-03-07 15:48:40,770][382407] Decorrelating experience for 512 frames... +[2023-03-07 15:48:40,788][382408] Decorrelating experience for 352 frames... +[2023-03-07 15:48:40,788][384368] Decorrelating experience for 704 frames... +[2023-03-07 15:48:40,847][382426] Decorrelating experience for 768 frames... +[2023-03-07 15:48:40,856][384157] Decorrelating experience for 544 frames... +[2023-03-07 15:48:40,872][385093] Decorrelating experience for 608 frames... +[2023-03-07 15:48:40,872][385031] Decorrelating experience for 672 frames... +[2023-03-07 15:48:40,914][383678] Decorrelating experience for 576 frames... +[2023-03-07 15:48:40,937][382582] Decorrelating experience for 544 frames... +[2023-03-07 15:48:40,949][382432] Decorrelating experience for 256 frames... +[2023-03-07 15:48:40,988][382420] Decorrelating experience for 768 frames... +[2023-03-07 15:48:40,988][382457] Decorrelating experience for 576 frames... +[2023-03-07 15:48:41,043][382438] Decorrelating experience for 736 frames... +[2023-03-07 15:48:41,062][382428] Decorrelating experience for 608 frames... +[2023-03-07 15:48:41,074][382423] Decorrelating experience for 512 frames... +[2023-03-07 15:48:41,074][385094] Decorrelating experience for 480 frames... +[2023-03-07 15:48:41,103][383292] Decorrelating experience for 800 frames... +[2023-03-07 15:48:41,140][383101] Decorrelating experience for 704 frames... +[2023-03-07 15:48:41,153][382409] Decorrelating experience for 448 frames... +[2023-03-07 15:48:41,191][382838] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,194][382452] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,232][384114] Decorrelating experience for 480 frames... +[2023-03-07 15:48:41,252][382407] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,269][382844] Decorrelating experience for 640 frames... +[2023-03-07 15:48:41,341][382446] Decorrelating experience for 480 frames... +[2023-03-07 15:48:41,345][382442] Decorrelating experience for 512 frames... +[2023-03-07 15:48:41,385][382432] Decorrelating experience for 288 frames... +[2023-03-07 15:48:41,396][383294] Decorrelating experience for 608 frames... +[2023-03-07 15:48:41,397][382457] Decorrelating experience for 608 frames... +[2023-03-07 15:48:41,417][382399] Decorrelating experience for 320 frames... +[2023-03-07 15:48:41,449][382426] Decorrelating experience for 800 frames... +[2023-03-07 15:48:41,471][382431] Decorrelating experience for 992 frames... +[2023-03-07 15:48:41,528][382846] Decorrelating experience for 736 frames... +[2023-03-07 15:48:41,531][384552] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,538][382402] Another process currently holds the lock /tmp/sf2_rolo/doom_001.lockfile, attempt: 1 +[2023-03-07 15:48:41,554][382405] Decorrelating experience for 736 frames... +[2023-03-07 15:48:41,588][382838] Decorrelating experience for 576 frames... +[2023-03-07 15:48:41,593][382430] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,594][383101] Decorrelating experience for 736 frames... +[2023-03-07 15:48:41,603][382438] Decorrelating experience for 768 frames... +[2023-03-07 15:48:41,659][383427] Decorrelating experience for 480 frames... +[2023-03-07 15:48:41,673][382453] Decorrelating experience for 576 frames... +[2023-03-07 15:48:41,725][382447] Decorrelating experience for 768 frames... +[2023-03-07 15:48:41,726][383797] Decorrelating experience for 960 frames... +[2023-03-07 15:48:41,739][390548] Decorrelating experience for 448 frames... +[2023-03-07 15:48:41,776][382442] Decorrelating experience for 544 frames... +[2023-03-07 15:48:41,795][385093] Decorrelating experience for 640 frames... +[2023-03-07 15:48:41,839][236315] Heartbeat connected on RolloutWorker_w46 +[2023-03-07 15:48:41,851][384157] Decorrelating experience for 576 frames... +[2023-03-07 15:48:41,863][382407] Decorrelating experience for 576 frames... +[2023-03-07 15:48:41,881][384114] Decorrelating experience for 512 frames... +[2023-03-07 15:48:41,921][383639] Decorrelating experience for 608 frames... +[2023-03-07 15:48:41,937][383678] Decorrelating experience for 608 frames... +[2023-03-07 15:48:41,938][382436] Decorrelating experience for 384 frames... +[2023-03-07 15:48:41,940][384387] Decorrelating experience for 448 frames... +[2023-03-07 15:48:41,965][382457] Decorrelating experience for 640 frames... +[2023-03-07 15:48:41,985][384388] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:41,987][382582] Decorrelating experience for 576 frames... +[2023-03-07 15:48:42,064][382414] Decorrelating experience for 384 frames... +[2023-03-07 15:48:42,067][382400] Decorrelating experience for 768 frames... +[2023-03-07 15:48:42,075][383101] Decorrelating experience for 768 frames... +[2023-03-07 15:48:42,148][382974] Decorrelating experience for 384 frames... +[2023-03-07 15:48:42,148][382422] Decorrelating experience for 640 frames... +[2023-03-07 15:48:42,149][382395] Decorrelating experience for 576 frames... +[2023-03-07 15:48:42,151][384368] Decorrelating experience for 736 frames... +[2023-03-07 15:48:42,160][383984] Decorrelating experience for 352 frames... +[2023-03-07 15:48:42,178][383102] Decorrelating experience for 224 frames... +[2023-03-07 15:48:42,313][382409] Decorrelating experience for 480 frames... +[2023-03-07 15:48:42,313][384114] Decorrelating experience for 544 frames... +[2023-03-07 15:48:42,314][383427] Decorrelating experience for 512 frames... +[2023-03-07 15:48:42,340][383639] Decorrelating experience for 640 frames... +[2023-03-07 15:48:42,356][384157] Decorrelating experience for 608 frames... +[2023-03-07 15:48:42,361][382447] Decorrelating experience for 800 frames... +[2023-03-07 15:48:42,362][383292] Decorrelating experience for 832 frames... +[2023-03-07 15:48:42,404][382582] Decorrelating experience for 608 frames... +[2023-03-07 15:48:42,404][382457] Decorrelating experience for 672 frames... +[2023-03-07 15:48:42,405][382428] Decorrelating experience for 640 frames... +[2023-03-07 15:48:42,500][382427] Decorrelating experience for 160 frames... +[2023-03-07 15:48:42,542][382400] Decorrelating experience for 800 frames... +[2023-03-07 15:48:42,570][382442] Decorrelating experience for 576 frames... +[2023-03-07 15:48:42,571][382452] Decorrelating experience for 576 frames... +[2023-03-07 15:48:42,572][383294] Decorrelating experience for 640 frames... +[2023-03-07 15:48:42,572][382394] Decorrelating experience for 704 frames... +[2023-03-07 15:48:42,601][382451] Decorrelating experience for 512 frames... +[2023-03-07 15:48:42,609][389982] Decorrelating experience for 864 frames... +[2023-03-07 15:48:42,662][383101] Decorrelating experience for 800 frames... +[2023-03-07 15:48:42,694][382414] Decorrelating experience for 416 frames... +[2023-03-07 15:48:42,751][382453] Decorrelating experience for 608 frames... +[2023-03-07 15:48:42,758][383827] Decorrelating experience for 832 frames... +[2023-03-07 15:48:42,767][384368] Decorrelating experience for 768 frames... +[2023-03-07 15:48:42,775][384157] Decorrelating experience for 640 frames... +[2023-03-07 15:48:42,818][382321] Decorrelating experience for 608 frames... +[2023-03-07 15:48:42,826][382428] Decorrelating experience for 672 frames... +[2023-03-07 15:48:42,851][382395] Decorrelating experience for 608 frames... +[2023-03-07 15:48:42,891][383984] Decorrelating experience for 384 frames... +[2023-03-07 15:48:42,893][382456] Decorrelating experience for 480 frames... +[2023-03-07 15:48:42,894][382324] Decorrelating experience for 480 frames... +[2023-03-07 15:48:42,955][382447] Decorrelating experience for 832 frames... +[2023-03-07 15:48:42,960][382436] Decorrelating experience for 416 frames... +[2023-03-07 15:48:43,001][382846] Decorrelating experience for 768 frames... +[2023-03-07 15:48:43,030][382394] Decorrelating experience for 736 frames... +[2023-03-07 15:48:43,035][382322] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,060][382446] Decorrelating experience for 512 frames... +[2023-03-07 15:48:43,061][383427] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,087][382974] Decorrelating experience for 416 frames... +[2023-03-07 15:48:43,094][382414] Decorrelating experience for 448 frames... +[2023-03-07 15:48:43,105][385049] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,158][385096] Decorrelating experience for 448 frames... +[2023-03-07 15:48:43,195][382457] Decorrelating experience for 704 frames... +[2023-03-07 15:48:43,209][383797] Decorrelating experience for 992 frames... +[2023-03-07 15:48:43,258][382427] Decorrelating experience for 192 frames... +[2023-03-07 15:48:43,263][382844] Decorrelating experience for 672 frames... +[2023-03-07 15:48:43,275][382428] Decorrelating experience for 704 frames... +[2023-03-07 15:48:43,295][383101] Decorrelating experience for 832 frames... +[2023-03-07 15:48:43,308][383294] Decorrelating experience for 672 frames... +[2023-03-07 15:48:43,348][385093] Decorrelating experience for 672 frames... +[2023-03-07 15:48:43,391][382453] Decorrelating experience for 640 frames... +[2023-03-07 15:48:43,457][382451] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,460][382397] Decorrelating experience for 416 frames... +[2023-03-07 15:48:43,463][382838] Decorrelating experience for 608 frames... +[2023-03-07 15:48:43,496][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 17.8. Samples: 800. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:48:43,498][236315] Avg episode reward: [(0, '0.908')] +[2023-03-07 15:48:43,519][383827] Decorrelating experience for 864 frames... +[2023-03-07 15:48:43,522][382446] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,522][382419] Decorrelating experience for 512 frames... +[2023-03-07 15:48:43,526][382325] Decorrelating experience for 512 frames... +[2023-03-07 15:48:43,558][384503] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,584][236315] Heartbeat connected on RolloutWorker_w91 +[2023-03-07 15:48:43,589][384157] Decorrelating experience for 672 frames... +[2023-03-07 15:48:43,663][382394] Decorrelating experience for 768 frames... +[2023-03-07 15:48:43,669][382440] Decorrelating experience for 416 frames... +[2023-03-07 15:48:43,670][382443] Decorrelating experience for 576 frames... +[2023-03-07 15:48:43,722][384644] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:43,726][382401] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,727][382524] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,729][385096] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,752][382423] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,760][382420] Decorrelating experience for 800 frames... +[2023-03-07 15:48:43,864][382414] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,868][383831] Decorrelating experience for 544 frames... +[2023-03-07 15:48:43,926][384643] Decorrelating experience for 576 frames... +[2023-03-07 15:48:43,941][382586] Decorrelating experience for 576 frames... +[2023-03-07 15:48:43,948][382407] Decorrelating experience for 608 frames... +[2023-03-07 15:48:43,949][382974] Decorrelating experience for 448 frames... +[2023-03-07 15:48:43,949][383294] Decorrelating experience for 704 frames... +[2023-03-07 15:48:43,956][382441] Decorrelating experience for 480 frames... +[2023-03-07 15:48:43,987][382172] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000981_4018176.pth... +[2023-03-07 15:48:43,996][236315] Component Batcher_0 stopped! +[2023-03-07 15:48:43,993][382172] Stopping Batcher_0... +[2023-03-07 15:48:44,015][382440] Decorrelating experience for 448 frames... +[2023-03-07 15:48:44,013][382172] Loop batcher_evt_loop terminating... +[2023-03-07 15:48:44,031][382172] Removing /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000979_4009984.pth +[2023-03-07 15:48:44,036][382172] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000981_4018176.pth... +[2023-03-07 15:48:44,049][382259] Weights refcount: 2 0 +[2023-03-07 15:48:44,050][382259] Stopping InferenceWorker_p0-w0... +[2023-03-07 15:48:44,051][382259] Loop inference_proc0-0_evt_loop terminating... +[2023-03-07 15:48:44,050][236315] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 15:48:44,063][390548] Decorrelating experience for 480 frames... +[2023-03-07 15:48:44,088][383797] Stopping RolloutWorker_w91... +[2023-03-07 15:48:44,088][383797] Loop rollout_proc91_evt_loop terminating... +[2023-03-07 15:48:44,088][236315] Component RolloutWorker_w91 stopped! +[2023-03-07 15:48:44,092][382431] Stopping RolloutWorker_w46... +[2023-03-07 15:48:44,092][382431] Loop rollout_proc46_evt_loop terminating... +[2023-03-07 15:48:44,092][236315] Component RolloutWorker_w46 stopped! +[2023-03-07 15:48:44,121][384644] Decorrelating experience for 480 frames... +[2023-03-07 15:48:44,130][383427] Decorrelating experience for 576 frames... +[2023-03-07 15:48:44,135][382172] Stopping LearnerWorker_p0... +[2023-03-07 15:48:44,135][382172] Loop learner_proc0_evt_loop terminating... +[2023-03-07 15:48:44,135][236315] Component LearnerWorker_p0 stopped! +[2023-03-07 15:48:44,142][383984] Decorrelating experience for 416 frames... +[2023-03-07 15:48:44,149][382427] Decorrelating experience for 224 frames... +[2023-03-07 15:48:44,152][382325] Decorrelating experience for 544 frames... +[2023-03-07 15:48:44,165][382322] Decorrelating experience for 512 frames... +[2023-03-07 15:48:44,166][385096] Decorrelating experience for 512 frames... +[2023-03-07 15:48:44,288][382327] Decorrelating experience for 864 frames... +[2023-03-07 15:48:44,296][382423] Decorrelating experience for 576 frames... +[2023-03-07 15:48:44,334][382447] Decorrelating experience for 864 frames... +[2023-03-07 15:48:44,337][384643] Decorrelating experience for 608 frames... +[2023-03-07 15:48:44,344][382428] Decorrelating experience for 736 frames... +[2023-03-07 15:48:44,372][382444] Decorrelating experience for 480 frames... +[2023-03-07 15:48:44,372][382405] Decorrelating experience for 768 frames... +[2023-03-07 15:48:44,374][382419] Decorrelating experience for 544 frames... +[2023-03-07 15:48:44,386][382840] Decorrelating experience for 384 frames... +[2023-03-07 15:48:44,487][383827] Decorrelating experience for 896 frames... +[2023-03-07 15:48:44,499][382524] Decorrelating experience for 512 frames... +[2023-03-07 15:48:44,538][382453] Decorrelating experience for 672 frames... +[2023-03-07 15:48:44,538][389982] Decorrelating experience for 896 frames... +[2023-03-07 15:48:44,539][382456] Decorrelating experience for 512 frames... +[2023-03-07 15:48:44,563][384761] Decorrelating experience for 384 frames... +[2023-03-07 15:48:44,583][382440] Decorrelating experience for 480 frames... +[2023-03-07 15:48:44,586][384503] Decorrelating experience for 576 frames... +[2023-03-07 15:48:44,647][382421] Decorrelating experience for 608 frames... +[2023-03-07 15:48:44,676][382398] Decorrelating experience for 832 frames... +[2023-03-07 15:48:44,701][385096] Decorrelating experience for 544 frames... +[2023-03-07 15:48:44,735][382430] Decorrelating experience for 576 frames... +[2023-03-07 15:48:44,739][382443] Decorrelating experience for 608 frames... +[2023-03-07 15:48:44,741][382838] Decorrelating experience for 640 frames... +[2023-03-07 15:48:44,766][382399] Decorrelating experience for 352 frames... +[2023-03-07 15:48:44,772][384962] Decorrelating experience for 480 frames... +[2023-03-07 15:48:44,805][382428] Decorrelating experience for 768 frames... +[2023-03-07 15:48:44,823][382840] Decorrelating experience for 416 frames... +[2023-03-07 15:48:44,835][382441] Decorrelating experience for 512 frames... +[2023-03-07 15:48:44,886][382327] Decorrelating experience for 896 frames... +[2023-03-07 15:48:44,944][382452] Decorrelating experience for 608 frames... +[2023-03-07 15:48:44,949][382451] Decorrelating experience for 576 frames... +[2023-03-07 15:48:44,950][382322] Decorrelating experience for 544 frames... +[2023-03-07 15:48:44,974][382842] Decorrelating experience for 160 frames... +[2023-03-07 15:48:44,975][382847] Decorrelating experience for 384 frames... +[2023-03-07 15:48:45,003][382418] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,034][382456] Decorrelating experience for 544 frames... +[2023-03-07 15:48:45,035][382427] Decorrelating experience for 256 frames... +[2023-03-07 15:48:45,050][382447] Decorrelating experience for 896 frames... +[2023-03-07 15:48:45,103][382405] Decorrelating experience for 800 frames... +[2023-03-07 15:48:45,139][382409] Decorrelating experience for 512 frames... +[2023-03-07 15:48:45,147][382430] Decorrelating experience for 608 frames... +[2023-03-07 15:48:45,159][383984] Decorrelating experience for 448 frames... +[2023-03-07 15:48:45,163][383827] Decorrelating experience for 928 frames... +[2023-03-07 15:48:45,169][384552] Decorrelating experience for 576 frames... +[2023-03-07 15:48:45,193][383427] Decorrelating experience for 608 frames... +[2023-03-07 15:48:45,241][382582] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,244][382399] Decorrelating experience for 384 frames... +[2023-03-07 15:48:45,275][382428] Decorrelating experience for 800 frames... +[2023-03-07 15:48:45,323][385093] Decorrelating experience for 704 frames... +[2023-03-07 15:48:45,352][385049] Decorrelating experience for 512 frames... +[2023-03-07 15:48:45,361][382440] Decorrelating experience for 512 frames... +[2023-03-07 15:48:45,365][384643] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,387][383955] Decorrelating experience for 448 frames... +[2023-03-07 15:48:45,398][384157] Decorrelating experience for 704 frames... +[2023-03-07 15:48:45,421][382421] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,445][382443] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,446][382450] Decorrelating experience for 544 frames... +[2023-03-07 15:48:45,521][382838] Decorrelating experience for 672 frames... +[2023-03-07 15:48:45,545][382400] Decorrelating experience for 832 frames... +[2023-03-07 15:48:45,580][382974] Decorrelating experience for 480 frames... +[2023-03-07 15:48:45,581][382430] Decorrelating experience for 640 frames... +[2023-03-07 15:48:45,582][382436] Decorrelating experience for 448 frames... +[2023-03-07 15:48:45,597][384503] Decorrelating experience for 608 frames... +[2023-03-07 15:48:45,618][382327] Decorrelating experience for 928 frames... +[2023-03-07 15:48:45,647][382417] Decorrelating experience for 576 frames... +[2023-03-07 15:48:45,652][382846] Decorrelating experience for 800 frames... +[2023-03-07 15:48:45,653][385096] Decorrelating experience for 576 frames... +[2023-03-07 15:48:45,709][382453] Decorrelating experience for 704 frames... +[2023-03-07 15:48:45,751][382444] Decorrelating experience for 512 frames... +[2023-03-07 15:48:45,778][382426] Decorrelating experience for 832 frames... +[2023-03-07 15:48:45,781][385093] Decorrelating experience for 736 frames... +[2023-03-07 15:48:45,794][384643] Decorrelating experience for 672 frames... +[2023-03-07 15:48:45,828][383799] Decorrelating experience for 544 frames... +[2023-03-07 15:48:45,851][382450] Decorrelating experience for 576 frames... +[2023-03-07 15:48:45,856][383955] Decorrelating experience for 480 frames... +[2023-03-07 15:48:45,857][382405] Decorrelating experience for 832 frames... +[2023-03-07 15:48:45,905][382524] Decorrelating experience for 544 frames... +[2023-03-07 15:48:45,959][382442] Decorrelating experience for 608 frames... +[2023-03-07 15:48:45,964][382440] Decorrelating experience for 544 frames... +[2023-03-07 15:48:45,983][382406] Decorrelating experience for 448 frames... +[2023-03-07 15:48:46,009][384157] Decorrelating experience for 736 frames... +[2023-03-07 15:48:46,025][382455] Decorrelating experience for 608 frames... +[2023-03-07 15:48:46,042][384114] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,045][382436] Decorrelating experience for 480 frames... +[2023-03-07 15:48:46,070][382421] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,081][382422] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,120][382322] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,164][382327] Decorrelating experience for 960 frames... +[2023-03-07 15:48:46,164][382974] Decorrelating experience for 512 frames... +[2023-03-07 15:48:46,172][382397] Decorrelating experience for 448 frames... +[2023-03-07 15:48:46,216][384503] Decorrelating experience for 640 frames... +[2023-03-07 15:48:46,236][383527] Decorrelating experience for 256 frames... +[2023-03-07 15:48:46,241][382456] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,241][384643] Decorrelating experience for 704 frames... +[2023-03-07 15:48:46,276][383955] Decorrelating experience for 512 frames... +[2023-03-07 15:48:46,277][383984] Decorrelating experience for 480 frames... +[2023-03-07 15:48:46,317][382838] Decorrelating experience for 704 frames... +[2023-03-07 15:48:46,372][383827] Decorrelating experience for 960 frames... +[2023-03-07 15:48:46,372][385096] Decorrelating experience for 608 frames... +[2023-03-07 15:48:46,372][382430] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,423][383427] Decorrelating experience for 640 frames... +[2023-03-07 15:48:46,436][382416] Decorrelating experience for 480 frames... +[2023-03-07 15:48:46,474][384158] Decorrelating experience for 128 frames... +[2023-03-07 15:48:46,480][382409] Decorrelating experience for 544 frames... +[2023-03-07 15:48:46,527][382405] Decorrelating experience for 864 frames... +[2023-03-07 15:48:46,531][382418] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,550][382443] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,567][382417] Decorrelating experience for 608 frames... +[2023-03-07 15:48:46,573][383101] Decorrelating experience for 864 frames... +[2023-03-07 15:48:46,619][382436] Decorrelating experience for 512 frames... +[2023-03-07 15:48:46,629][382422] Decorrelating experience for 704 frames... +[2023-03-07 15:48:46,641][382325] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,686][385026] Decorrelating experience for 416 frames... +[2023-03-07 15:48:46,691][385031] Decorrelating experience for 704 frames... +[2023-03-07 15:48:46,734][382452] Decorrelating experience for 640 frames... +[2023-03-07 15:48:46,734][382425] Decorrelating experience for 704 frames... +[2023-03-07 15:48:46,768][384503] Decorrelating experience for 672 frames... +[2023-03-07 15:48:46,772][382847] Decorrelating experience for 416 frames... +[2023-03-07 15:48:46,848][382428] Decorrelating experience for 832 frames... +[2023-03-07 15:48:46,849][382427] Decorrelating experience for 288 frames... +[2023-03-07 15:48:46,854][382453] Decorrelating experience for 736 frames... +[2023-03-07 15:48:46,858][382440] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,884][385027] Decorrelating experience for 352 frames... +[2023-03-07 15:48:46,898][383550] Decorrelating experience for 352 frames... +[2023-03-07 15:48:46,933][382409] Decorrelating experience for 576 frames... +[2023-03-07 15:48:46,939][384552] Decorrelating experience for 608 frames... +[2023-03-07 15:48:46,975][383955] Decorrelating experience for 544 frames... +[2023-03-07 15:48:47,030][382457] Decorrelating experience for 736 frames... +[2023-03-07 15:48:47,068][382416] Decorrelating experience for 512 frames... +[2023-03-07 15:48:47,084][383798] Decorrelating experience for 768 frames... +[2023-03-07 15:48:47,096][382430] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,115][382407] Decorrelating experience for 640 frames... +[2023-03-07 15:48:47,116][382321] Decorrelating experience for 640 frames... +[2023-03-07 15:48:47,123][383427] Decorrelating experience for 672 frames... +[2023-03-07 15:48:47,170][382325] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,173][385031] Decorrelating experience for 736 frames... +[2023-03-07 15:48:47,210][382410] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,232][383294] Decorrelating experience for 736 frames... +[2023-03-07 15:48:47,268][384158] Decorrelating experience for 160 frames... +[2023-03-07 15:48:47,290][382449] Decorrelating experience for 480 frames... +[2023-03-07 15:48:47,297][382422] Decorrelating experience for 736 frames... +[2023-03-07 15:48:47,314][382846] Decorrelating experience for 832 frames... +[2023-03-07 15:48:47,315][383951] Decorrelating experience for 384 frames... +[2023-03-07 15:48:47,337][382452] Decorrelating experience for 672 frames... +[2023-03-07 15:48:47,369][382456] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,372][382443] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,402][384388] Decorrelating experience for 128 frames... +[2023-03-07 15:48:47,431][389982] Decorrelating experience for 928 frames... +[2023-03-07 15:48:47,476][383984] Decorrelating experience for 512 frames... +[2023-03-07 15:48:47,489][383293] Decorrelating experience for 576 frames... +[2023-03-07 15:48:47,509][384643] Decorrelating experience for 736 frames... +[2023-03-07 15:48:47,548][382847] Decorrelating experience for 448 frames... +[2023-03-07 15:48:47,549][384503] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,560][384552] Decorrelating experience for 640 frames... +[2023-03-07 15:48:47,580][382421] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,580][382450] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,595][382844] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,661][385094] Decorrelating experience for 512 frames... +[2023-03-07 15:48:47,676][382457] Decorrelating experience for 768 frames... +[2023-03-07 15:48:47,687][383102] Decorrelating experience for 256 frames... +[2023-03-07 15:48:47,713][384387] Decorrelating experience for 480 frames... +[2023-03-07 15:48:47,742][382451] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,767][385049] Decorrelating experience for 544 frames... +[2023-03-07 15:48:47,788][382444] Decorrelating experience for 544 frames... +[2023-03-07 15:48:47,798][383799] Decorrelating experience for 576 frames... +[2023-03-07 15:48:47,798][382418] Decorrelating experience for 704 frames... +[2023-03-07 15:48:47,799][382438] Decorrelating experience for 800 frames... +[2023-03-07 15:48:47,850][382586] Decorrelating experience for 608 frames... +[2023-03-07 15:48:47,889][383550] Decorrelating experience for 384 frames... +[2023-03-07 15:48:47,914][382395] Decorrelating experience for 640 frames... +[2023-03-07 15:48:47,932][383527] Decorrelating experience for 288 frames... +[2023-03-07 15:48:47,938][382435] Decorrelating experience for 416 frames... +[2023-03-07 15:48:47,960][385096] Decorrelating experience for 640 frames... +[2023-03-07 15:48:47,996][383827] Decorrelating experience for 992 frames... +[2023-03-07 15:48:48,001][382845] Decorrelating experience for 288 frames... +[2023-03-07 15:48:48,010][383427] Decorrelating experience for 704 frames... +[2023-03-07 15:48:48,031][382447] Decorrelating experience for 928 frames... +[2023-03-07 15:48:48,043][382445] Decorrelating experience for 576 frames... +[2023-03-07 15:48:48,087][382440] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,140][382446] Decorrelating experience for 576 frames... +[2023-03-07 15:48:48,169][383101] Decorrelating experience for 896 frames... +[2023-03-07 15:48:48,176][382421] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,203][384552] Decorrelating experience for 672 frames... +[2023-03-07 15:48:48,203][382582] Decorrelating experience for 672 frames... +[2023-03-07 15:48:48,203][383293] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,205][382407] Decorrelating experience for 672 frames... +[2023-03-07 15:48:48,226][382397] Decorrelating experience for 480 frames... +[2023-03-07 15:48:48,239][385095] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,279][383527] Decorrelating experience for 320 frames... +[2023-03-07 15:48:48,350][382840] Decorrelating experience for 448 frames... +[2023-03-07 15:48:48,361][382457] Decorrelating experience for 800 frames... +[2023-03-07 15:48:48,372][384761] Decorrelating experience for 416 frames... +[2023-03-07 15:48:48,410][382415] Decorrelating experience for 672 frames... +[2023-03-07 15:48:48,411][384388] Decorrelating experience for 160 frames... +[2023-03-07 15:48:48,411][382409] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,418][382430] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,422][382847] Decorrelating experience for 480 frames... +[2023-03-07 15:48:48,443][383641] Decorrelating experience for 384 frames... +[2023-03-07 15:48:48,463][383827] Stopping RolloutWorker_w119... +[2023-03-07 15:48:48,463][383827] Loop rollout_proc119_evt_loop terminating... +[2023-03-07 15:48:48,464][236315] Component RolloutWorker_w119 stopped! +[2023-03-07 15:48:48,522][382322] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,608][384503] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,624][385031] Decorrelating experience for 768 frames... +[2023-03-07 15:48:48,639][384114] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,641][382325] Decorrelating experience for 640 frames... +[2023-03-07 15:48:48,643][382838] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,645][382414] Decorrelating experience for 512 frames... +[2023-03-07 15:48:48,646][383527] Decorrelating experience for 352 frames... +[2023-03-07 15:48:48,653][383427] Decorrelating experience for 736 frames... +[2023-03-07 15:48:48,719][382453] Decorrelating experience for 768 frames... +[2023-03-07 15:48:48,742][382452] Decorrelating experience for 704 frames... +[2023-03-07 15:48:48,799][383744] Decorrelating experience for 96 frames... +[2023-03-07 15:48:48,811][382410] Decorrelating experience for 640 frames... +[2023-03-07 15:48:48,837][382406] Decorrelating experience for 480 frames... +[2023-03-07 15:48:48,838][384643] Decorrelating experience for 768 frames... +[2023-03-07 15:48:48,869][382423] Decorrelating experience for 608 frames... +[2023-03-07 15:48:48,875][382842] Decorrelating experience for 192 frames... +[2023-03-07 15:48:48,875][382411] Decorrelating experience for 480 frames... +[2023-03-07 15:48:48,876][382324] Decorrelating experience for 512 frames... +[2023-03-07 15:48:48,908][382415] Decorrelating experience for 704 frames... +[2023-03-07 15:48:48,950][382442] Decorrelating experience for 640 frames... +[2023-03-07 15:48:48,994][382454] Decorrelating experience for 448 frames... +[2023-03-07 15:48:49,032][383799] Decorrelating experience for 608 frames... +[2023-03-07 15:48:49,065][382456] Decorrelating experience for 640 frames... +[2023-03-07 15:48:49,075][385093] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,078][383984] Decorrelating experience for 544 frames... +[2023-03-07 15:48:49,084][382414] Decorrelating experience for 544 frames... +[2023-03-07 15:48:49,092][383744] Decorrelating experience for 128 frames... +[2023-03-07 15:48:49,126][382397] Decorrelating experience for 512 frames... +[2023-03-07 15:48:49,136][382427] Decorrelating experience for 320 frames... +[2023-03-07 15:48:49,157][382438] Decorrelating experience for 832 frames... +[2023-03-07 15:48:49,170][382393] Another process currently holds the lock /tmp/sf2_rolo/doom_001.lockfile, attempt: 1 +[2023-03-07 15:48:49,187][383102] Decorrelating experience for 288 frames... +[2023-03-07 15:48:49,258][384114] Decorrelating experience for 640 frames... +[2023-03-07 15:48:49,274][382403] Decorrelating experience for 480 frames... +[2023-03-07 15:48:49,274][382421] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,281][382417] Decorrelating experience for 640 frames... +[2023-03-07 15:48:49,298][382327] Decorrelating experience for 992 frames... +[2023-03-07 15:48:49,302][384158] Decorrelating experience for 192 frames... +[2023-03-07 15:48:49,313][383427] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,341][383550] Decorrelating experience for 416 frames... +[2023-03-07 15:48:49,353][382454] Decorrelating experience for 480 frames... +[2023-03-07 15:48:49,414][382418] Decorrelating experience for 736 frames... +[2023-03-07 15:48:49,478][383951] Decorrelating experience for 416 frames... +[2023-03-07 15:48:49,478][382321] Decorrelating experience for 672 frames... +[2023-03-07 15:48:49,482][382838] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,516][382427] Decorrelating experience for 352 frames... +[2023-03-07 15:48:49,516][384644] Decorrelating experience for 512 frames... +[2023-03-07 15:48:49,516][384552] Decorrelating experience for 704 frames... +[2023-03-07 15:48:49,538][382443] Decorrelating experience for 736 frames... +[2023-03-07 15:48:49,539][382430] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,555][382406] Decorrelating experience for 512 frames... +[2023-03-07 15:48:49,606][382420] Decorrelating experience for 832 frames... +[2023-03-07 15:48:49,684][382441] Decorrelating experience for 544 frames... +[2023-03-07 15:48:49,685][384962] Decorrelating experience for 512 frames... +[2023-03-07 15:48:49,689][382414] Decorrelating experience for 576 frames... +[2023-03-07 15:48:49,718][382417] Decorrelating experience for 672 frames... +[2023-03-07 15:48:49,737][383798] Decorrelating experience for 800 frames... +[2023-03-07 15:48:49,740][382324] Decorrelating experience for 544 frames... +[2023-03-07 15:48:49,759][382327] Stopping RolloutWorker_w20... +[2023-03-07 15:48:49,759][382327] Loop rollout_proc20_evt_loop terminating... +[2023-03-07 15:48:49,759][236315] Component RolloutWorker_w20 stopped! +[2023-03-07 15:48:49,779][384503] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,787][383744] Decorrelating experience for 160 frames... +[2023-03-07 15:48:49,795][382457] Decorrelating experience for 832 frames... +[2023-03-07 15:48:49,799][382842] Decorrelating experience for 224 frames... +[2023-03-07 15:48:49,893][383293] Decorrelating experience for 640 frames... +[2023-03-07 15:48:49,894][382436] Decorrelating experience for 544 frames... +[2023-03-07 15:48:49,899][383955] Decorrelating experience for 576 frames... +[2023-03-07 15:48:49,930][382405] Decorrelating experience for 896 frames... +[2023-03-07 15:48:49,933][382422] Decorrelating experience for 768 frames... +[2023-03-07 15:48:49,946][383550] Decorrelating experience for 448 frames... +[2023-03-07 15:48:49,993][382447] Decorrelating experience for 960 frames... +[2023-03-07 15:48:50,010][382446] Decorrelating experience for 608 frames... +[2023-03-07 15:48:50,013][382449] Decorrelating experience for 512 frames... +[2023-03-07 15:48:50,027][384761] Decorrelating experience for 448 frames... +[2023-03-07 15:48:50,087][385049] Decorrelating experience for 576 frames... +[2023-03-07 15:48:50,093][384368] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,096][384114] Decorrelating experience for 672 frames... +[2023-03-07 15:48:50,125][382420] Decorrelating experience for 864 frames... +[2023-03-07 15:48:50,129][384643] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,159][382453] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,187][383641] Decorrelating experience for 416 frames... +[2023-03-07 15:48:50,209][382398] Decorrelating experience for 864 frames... +[2023-03-07 15:48:50,228][384962] Decorrelating experience for 544 frames... +[2023-03-07 15:48:50,228][385095] Decorrelating experience for 768 frames... +[2023-03-07 15:48:50,289][382406] Decorrelating experience for 544 frames... +[2023-03-07 15:48:50,321][382455] Decorrelating experience for 640 frames... +[2023-03-07 15:48:50,326][382428] Decorrelating experience for 864 frames... +[2023-03-07 15:48:50,381][382407] Decorrelating experience for 704 frames... +[2023-03-07 15:48:50,382][382846] Decorrelating experience for 864 frames... +[2023-03-07 15:48:50,382][382322] Decorrelating experience for 640 frames... +[2023-03-07 15:48:50,383][382974] Decorrelating experience for 544 frames... +[2023-03-07 15:48:50,403][383550] Decorrelating experience for 480 frames... +[2023-03-07 15:48:50,430][383798] Decorrelating experience for 832 frames... +[2023-03-07 15:48:50,441][383639] Decorrelating experience for 672 frames... +[2023-03-07 15:48:50,510][382437] Decorrelating experience for 448 frames... +[2023-03-07 15:48:50,529][382456] Decorrelating experience for 672 frames... +[2023-03-07 15:48:50,567][383427] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,572][382394] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,589][383101] Decorrelating experience for 928 frames... +[2023-03-07 15:48:50,596][382416] Decorrelating experience for 544 frames... +[2023-03-07 15:48:50,614][382395] Decorrelating experience for 672 frames... +[2023-03-07 15:48:50,636][382446] Decorrelating experience for 640 frames... +[2023-03-07 15:48:50,641][382397] Decorrelating experience for 544 frames... +[2023-03-07 15:48:50,651][382430] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,737][384368] Decorrelating experience for 832 frames... +[2023-03-07 15:48:50,740][382452] Decorrelating experience for 736 frames... +[2023-03-07 15:48:50,759][382843] Decorrelating experience for 832 frames... +[2023-03-07 15:48:50,810][385095] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,810][385093] Decorrelating experience for 800 frames... +[2023-03-07 15:48:50,845][382409] Decorrelating experience for 640 frames... +[2023-03-07 15:48:50,864][382403] Decorrelating experience for 512 frames... +[2023-03-07 15:48:50,873][385049] Decorrelating experience for 608 frames... +[2023-03-07 15:48:50,873][384962] Decorrelating experience for 576 frames... +[2023-03-07 15:48:50,899][382451] Decorrelating experience for 640 frames... +[2023-03-07 15:48:50,941][382322] Decorrelating experience for 672 frames... +[2023-03-07 15:48:50,945][384761] Decorrelating experience for 480 frames... +[2023-03-07 15:48:50,951][382410] Decorrelating experience for 672 frames... +[2023-03-07 15:48:51,000][382435] Decorrelating experience for 448 frames... +[2023-03-07 15:48:51,016][382421] Decorrelating experience for 800 frames... +[2023-03-07 15:48:51,046][383744] Decorrelating experience for 192 frames... +[2023-03-07 15:48:51,086][382424] Decorrelating experience for 416 frames... +[2023-03-07 15:48:51,087][382422] Decorrelating experience for 800 frames... +[2023-03-07 15:48:51,088][382432] Decorrelating experience for 320 frames... +[2023-03-07 15:48:51,091][383641] Decorrelating experience for 448 frames... +[2023-03-07 15:48:51,154][382411] Decorrelating experience for 512 frames... +[2023-03-07 15:48:51,156][382428] Decorrelating experience for 896 frames... +[2023-03-07 15:48:51,161][384315] Decorrelating experience for 384 frames... +[2023-03-07 15:48:51,195][382586] Decorrelating experience for 640 frames... +[2023-03-07 15:48:51,215][382394] Decorrelating experience for 832 frames... +[2023-03-07 15:48:51,251][382442] Decorrelating experience for 672 frames... +[2023-03-07 15:48:51,296][382409] Decorrelating experience for 672 frames... +[2023-03-07 15:48:51,299][385094] Decorrelating experience for 544 frames... +[2023-03-07 15:48:51,300][383294] Decorrelating experience for 768 frames... +[2023-03-07 15:48:51,301][382417] Decorrelating experience for 704 frames... +[2023-03-07 15:48:51,359][382403] Decorrelating experience for 544 frames... +[2023-03-07 15:48:51,360][383984] Decorrelating experience for 576 frames... +[2023-03-07 15:48:51,364][385095] Decorrelating experience for 832 frames... +[2023-03-07 15:48:51,387][382974] Decorrelating experience for 576 frames... +[2023-03-07 15:48:51,410][382435] Decorrelating experience for 480 frames... +[2023-03-07 15:48:51,473][382447] Decorrelating experience for 992 frames... +[2023-03-07 15:48:51,493][385027] Decorrelating experience for 384 frames... +[2023-03-07 15:48:51,505][382451] Decorrelating experience for 672 frames... +[2023-03-07 15:48:51,506][390548] Decorrelating experience for 512 frames... +[2023-03-07 15:48:51,536][382397] Decorrelating experience for 576 frames... +[2023-03-07 15:48:51,566][384644] Decorrelating experience for 544 frames... +[2023-03-07 15:48:51,569][382434] Decorrelating experience for 448 frames... +[2023-03-07 15:48:51,580][382453] Decorrelating experience for 832 frames... +[2023-03-07 15:48:51,626][384158] Decorrelating experience for 224 frames... +[2023-03-07 15:48:51,628][383427] Decorrelating experience for 832 frames... +[2023-03-07 15:48:51,706][382442] Decorrelating experience for 704 frames... +[2023-03-07 15:48:51,708][383831] Decorrelating experience for 576 frames... +[2023-03-07 15:48:51,727][382840] Decorrelating experience for 480 frames... +[2023-03-07 15:48:51,731][382845] Decorrelating experience for 320 frames... +[2023-03-07 15:48:51,739][384503] Decorrelating experience for 800 frames... +[2023-03-07 15:48:51,765][382847] Decorrelating experience for 512 frames... +[2023-03-07 15:48:51,779][384761] Decorrelating experience for 512 frames... +[2023-03-07 15:48:51,783][382415] Decorrelating experience for 736 frames... +[2023-03-07 15:48:51,820][382445] Decorrelating experience for 608 frames... +[2023-03-07 15:48:51,831][382582] Decorrelating experience for 704 frames... +[2023-03-07 15:48:51,907][389982] Decorrelating experience for 960 frames... +[2023-03-07 15:48:51,945][382452] Decorrelating experience for 768 frames... +[2023-03-07 15:48:51,946][385027] Decorrelating experience for 416 frames... +[2023-03-07 15:48:51,947][382447] Stopping RolloutWorker_w86... +[2023-03-07 15:48:51,947][382447] Loop rollout_proc86_evt_loop terminating... +[2023-03-07 15:48:51,947][236315] Component RolloutWorker_w86 stopped! +[2023-03-07 15:48:51,952][382437] Decorrelating experience for 480 frames... +[2023-03-07 15:48:51,952][384387] Decorrelating experience for 512 frames... +[2023-03-07 15:48:51,976][382416] Decorrelating experience for 576 frames... +[2023-03-07 15:48:51,986][385095] Decorrelating experience for 864 frames... +[2023-03-07 15:48:51,987][382427] Decorrelating experience for 384 frames... +[2023-03-07 15:48:52,014][382417] Decorrelating experience for 736 frames... +[2023-03-07 15:48:52,039][383527] Decorrelating experience for 384 frames... +[2023-03-07 15:48:52,104][383639] Decorrelating experience for 704 frames... +[2023-03-07 15:48:52,151][383427] Decorrelating experience for 864 frames... +[2023-03-07 15:48:52,154][385093] Decorrelating experience for 832 frames... +[2023-03-07 15:48:52,174][382840] Decorrelating experience for 512 frames... +[2023-03-07 15:48:52,189][383831] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,192][382444] Decorrelating experience for 576 frames... +[2023-03-07 15:48:52,214][382454] Decorrelating experience for 512 frames... +[2023-03-07 15:48:52,227][383101] Decorrelating experience for 960 frames... +[2023-03-07 15:48:52,251][382842] Decorrelating experience for 256 frames... +[2023-03-07 15:48:52,266][382432] Decorrelating experience for 352 frames... +[2023-03-07 15:48:52,302][382974] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,353][384315] Decorrelating experience for 416 frames... +[2023-03-07 15:48:52,380][382841] Decorrelating experience for 480 frames... +[2023-03-07 15:48:52,385][382455] Decorrelating experience for 672 frames... +[2023-03-07 15:48:52,388][382408] Decorrelating experience for 384 frames... +[2023-03-07 15:48:52,401][384503] Decorrelating experience for 832 frames... +[2023-03-07 15:48:52,424][382429] Decorrelating experience for 544 frames... +[2023-03-07 15:48:52,437][382446] Decorrelating experience for 672 frames... +[2023-03-07 15:48:52,451][383798] Decorrelating experience for 864 frames... +[2023-03-07 15:48:52,482][383102] Decorrelating experience for 320 frames... +[2023-03-07 15:48:52,492][383951] Decorrelating experience for 448 frames... +[2023-03-07 15:48:52,546][382400] Decorrelating experience for 864 frames... +[2023-03-07 15:48:52,573][383955] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,583][382417] Decorrelating experience for 768 frames... +[2023-03-07 15:48:52,600][382444] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,623][382840] Decorrelating experience for 544 frames... +[2023-03-07 15:48:52,642][385049] Decorrelating experience for 640 frames... +[2023-03-07 15:48:52,656][382441] Decorrelating experience for 576 frames... +[2023-03-07 15:48:52,690][382456] Decorrelating experience for 704 frames... +[2023-03-07 15:48:52,693][384962] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,704][382452] Decorrelating experience for 800 frames... +[2023-03-07 15:48:52,738][382438] Decorrelating experience for 864 frames... +[2023-03-07 15:48:52,763][383639] Decorrelating experience for 736 frames... +[2023-03-07 15:48:52,781][382393] Decorrelating experience for 288 frames... +[2023-03-07 15:48:52,804][385027] Decorrelating experience for 448 frames... +[2023-03-07 15:48:52,814][382415] Decorrelating experience for 768 frames... +[2023-03-07 15:48:52,891][382434] Decorrelating experience for 480 frames... +[2023-03-07 15:48:52,901][389982] Decorrelating experience for 992 frames... +[2023-03-07 15:48:52,901][384114] Decorrelating experience for 704 frames... +[2023-03-07 15:48:52,934][383984] Decorrelating experience for 608 frames... +[2023-03-07 15:48:52,939][385031] Decorrelating experience for 800 frames... +[2023-03-07 15:48:52,962][382841] Decorrelating experience for 512 frames... +[2023-03-07 15:48:52,966][383101] Decorrelating experience for 992 frames... +[2023-03-07 15:48:52,986][382399] Decorrelating experience for 416 frames... +[2023-03-07 15:48:53,006][382842] Decorrelating experience for 288 frames... +[2023-03-07 15:48:53,012][382403] Decorrelating experience for 576 frames... +[2023-03-07 15:48:53,108][382432] Decorrelating experience for 384 frames... +[2023-03-07 15:48:53,148][382427] Decorrelating experience for 416 frames... +[2023-03-07 15:48:53,148][382429] Decorrelating experience for 576 frames... +[2023-03-07 15:48:53,149][383798] Decorrelating experience for 896 frames... +[2023-03-07 15:48:53,150][384158] Decorrelating experience for 256 frames... +[2023-03-07 15:48:53,156][385094] Decorrelating experience for 576 frames... +[2023-03-07 15:48:53,166][382407] Decorrelating experience for 736 frames... +[2023-03-07 15:48:53,201][382456] Decorrelating experience for 736 frames... +[2023-03-07 15:48:53,224][385049] Decorrelating experience for 672 frames... +[2023-03-07 15:48:53,276][382397] Decorrelating experience for 608 frames... +[2023-03-07 15:48:53,309][384315] Decorrelating experience for 448 frames... +[2023-03-07 15:48:53,349][382446] Decorrelating experience for 704 frames... +[2023-03-07 15:48:53,368][383550] Decorrelating experience for 512 frames... +[2023-03-07 15:48:53,369][382416] Decorrelating experience for 608 frames... +[2023-03-07 15:48:53,370][382582] Decorrelating experience for 736 frames... +[2023-03-07 15:48:53,370][382406] Decorrelating experience for 576 frames... +[2023-03-07 15:48:53,397][382412] Decorrelating experience for 384 frames... +[2023-03-07 15:48:53,404][382438] Decorrelating experience for 896 frames... +[2023-03-07 15:48:53,422][382444] Decorrelating experience for 640 frames... +[2023-03-07 15:48:53,509][382449] Decorrelating experience for 544 frames... +[2023-03-07 15:48:53,542][389982] Stopping RolloutWorker_w0... +[2023-03-07 15:48:53,543][389982] Loop rollout_proc0_evt_loop terminating... +[2023-03-07 15:48:53,542][236315] Component RolloutWorker_w0 stopped! +[2023-03-07 15:48:53,571][383984] Decorrelating experience for 640 frames... +[2023-03-07 15:48:53,572][382430] Decorrelating experience for 832 frames... +[2023-03-07 15:48:53,575][382395] Decorrelating experience for 704 frames... +[2023-03-07 15:48:53,620][383294] Decorrelating experience for 800 frames... +[2023-03-07 15:48:53,621][382432] Decorrelating experience for 416 frames... +[2023-03-07 15:48:53,621][385031] Decorrelating experience for 832 frames... +[2023-03-07 15:48:53,629][382451] Decorrelating experience for 704 frames... +[2023-03-07 15:48:53,636][382847] Decorrelating experience for 544 frames... +[2023-03-07 15:48:53,638][382403] Decorrelating experience for 608 frames... +[2023-03-07 15:48:53,641][383101] Stopping RolloutWorker_w64... +[2023-03-07 15:48:53,641][383101] Loop rollout_proc64_evt_loop terminating... +[2023-03-07 15:48:53,641][236315] Component RolloutWorker_w64 stopped! +[2023-03-07 15:48:53,686][384886] Another process currently holds the lock /tmp/sf2_rolo/doom_005.lockfile, attempt: 1 +[2023-03-07 15:48:53,709][382842] Decorrelating experience for 320 frames... +[2023-03-07 15:48:53,770][382434] Decorrelating experience for 512 frames... +[2023-03-07 15:48:53,771][383641] Decorrelating experience for 480 frames... +[2023-03-07 15:48:53,826][382325] Decorrelating experience for 672 frames... +[2023-03-07 15:48:53,827][384503] Decorrelating experience for 864 frames... +[2023-03-07 15:48:53,837][382441] Decorrelating experience for 608 frames... +[2023-03-07 15:48:53,837][384315] Decorrelating experience for 480 frames... +[2023-03-07 15:48:53,843][384761] Decorrelating experience for 544 frames... +[2023-03-07 15:48:53,850][385093] Decorrelating experience for 864 frames... +[2023-03-07 15:48:53,903][382840] Decorrelating experience for 576 frames... +[2023-03-07 15:48:53,926][382321] Decorrelating experience for 704 frames... +[2023-03-07 15:48:53,975][385027] Decorrelating experience for 480 frames... +[2023-03-07 15:48:53,976][384962] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,018][382420] Decorrelating experience for 896 frames... +[2023-03-07 15:48:54,028][384644] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,041][382393] Decorrelating experience for 320 frames... +[2023-03-07 15:48:54,057][382582] Decorrelating experience for 768 frames... +[2023-03-07 15:48:54,058][382427] Decorrelating experience for 448 frames... +[2023-03-07 15:48:54,062][383102] Decorrelating experience for 352 frames... +[2023-03-07 15:48:54,107][382444] Decorrelating experience for 672 frames... +[2023-03-07 15:48:54,128][383427] Decorrelating experience for 896 frames... +[2023-03-07 15:48:54,178][382400] Decorrelating experience for 896 frames... +[2023-03-07 15:48:54,225][382845] Decorrelating experience for 352 frames... +[2023-03-07 15:48:54,231][382414] Decorrelating experience for 608 frames... +[2023-03-07 15:48:54,234][382395] Decorrelating experience for 736 frames... +[2023-03-07 15:48:54,243][382396] Another process currently holds the lock /tmp/sf2_rolo/doom_006.lockfile, attempt: 1 +[2023-03-07 15:48:54,262][382449] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,262][382434] Decorrelating experience for 544 frames... +[2023-03-07 15:48:54,271][382328] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,308][382446] Decorrelating experience for 736 frames... +[2023-03-07 15:48:54,322][382325] Decorrelating experience for 704 frames... +[2023-03-07 15:48:54,322][383955] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,385][382406] Decorrelating experience for 608 frames... +[2023-03-07 15:48:54,415][382436] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,459][382403] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,462][382453] Decorrelating experience for 864 frames... +[2023-03-07 15:48:54,476][382324] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,520][384051] Decorrelating experience for 576 frames... +[2023-03-07 15:48:54,526][383294] Decorrelating experience for 832 frames... +[2023-03-07 15:48:54,527][390548] Decorrelating experience for 544 frames... +[2023-03-07 15:48:54,534][384158] Decorrelating experience for 288 frames... +[2023-03-07 15:48:54,542][382456] Decorrelating experience for 768 frames... +[2023-03-07 15:48:54,584][382408] Decorrelating experience for 416 frames... +[2023-03-07 15:48:54,607][382409] Decorrelating experience for 704 frames... +[2023-03-07 15:48:54,650][382444] Decorrelating experience for 704 frames... +[2023-03-07 15:48:54,757][382840] Decorrelating experience for 608 frames... +[2023-03-07 15:48:54,757][384387] Decorrelating experience for 544 frames... +[2023-03-07 15:48:54,764][382417] Decorrelating experience for 800 frames... +[2023-03-07 15:48:54,766][384315] Decorrelating experience for 512 frames... +[2023-03-07 15:48:54,766][382416] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,767][382396] Decorrelating experience for 608 frames... +[2023-03-07 15:48:54,769][382322] Decorrelating experience for 704 frames... +[2023-03-07 15:48:54,799][382455] Decorrelating experience for 704 frames... +[2023-03-07 15:48:54,800][382586] Decorrelating experience for 672 frames... +[2023-03-07 15:48:54,837][383102] Decorrelating experience for 384 frames... +[2023-03-07 15:48:54,949][382846] Decorrelating experience for 896 frames... +[2023-03-07 15:48:54,958][383427] Decorrelating experience for 928 frames... +[2023-03-07 15:48:54,960][382393] Decorrelating experience for 352 frames... +[2023-03-07 15:48:54,971][382441] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,976][382446] Decorrelating experience for 768 frames... +[2023-03-07 15:48:54,994][383799] Decorrelating experience for 640 frames... +[2023-03-07 15:48:54,999][382415] Decorrelating experience for 800 frames... +[2023-03-07 15:48:55,005][382845] Decorrelating experience for 384 frames... +[2023-03-07 15:48:55,045][382403] Decorrelating experience for 672 frames... +[2023-03-07 15:48:55,054][384962] Decorrelating experience for 672 frames... +[2023-03-07 15:48:55,140][383292] Decorrelating experience for 864 frames... +[2023-03-07 15:48:55,159][382428] Decorrelating experience for 928 frames... +[2023-03-07 15:48:55,178][382432] Decorrelating experience for 448 frames... +[2023-03-07 15:48:55,181][382452] Decorrelating experience for 832 frames... +[2023-03-07 15:48:55,203][382414] Decorrelating experience for 640 frames... +[2023-03-07 15:48:55,204][382420] Decorrelating experience for 928 frames... +[2023-03-07 15:48:55,251][382453] Decorrelating experience for 896 frames... +[2023-03-07 15:48:55,255][382455] Decorrelating experience for 736 frames... +[2023-03-07 15:48:55,270][382426] Decorrelating experience for 864 frames... +[2023-03-07 15:48:55,273][384761] Decorrelating experience for 576 frames... +[2023-03-07 15:48:55,338][382845] Decorrelating experience for 416 frames... +[2023-03-07 15:48:55,362][382841] Decorrelating experience for 544 frames... +[2023-03-07 15:48:55,398][382582] Decorrelating experience for 800 frames... +[2023-03-07 15:48:55,405][382844] Decorrelating experience for 736 frames... +[2023-03-07 15:48:55,427][383831] Decorrelating experience for 640 frames... +[2023-03-07 15:48:55,432][382416] Decorrelating experience for 672 frames... +[2023-03-07 15:48:55,471][382417] Decorrelating experience for 832 frames... +[2023-03-07 15:48:55,471][382429] Decorrelating experience for 608 frames... +[2023-03-07 15:48:55,495][382457] Decorrelating experience for 864 frames... +[2023-03-07 15:48:55,513][382586] Decorrelating experience for 704 frames... +[2023-03-07 15:48:55,531][382325] Decorrelating experience for 736 frames... +[2023-03-07 15:48:55,567][382445] Decorrelating experience for 640 frames... +[2023-03-07 15:48:55,612][384388] Decorrelating experience for 192 frames... +[2023-03-07 15:48:55,619][382405] Decorrelating experience for 928 frames... +[2023-03-07 15:48:55,640][382430] Decorrelating experience for 864 frames... +[2023-03-07 15:48:55,651][383427] Decorrelating experience for 960 frames... +[2023-03-07 15:48:55,673][385094] Decorrelating experience for 608 frames... +[2023-03-07 15:48:55,678][382410] Decorrelating experience for 704 frames... +[2023-03-07 15:48:55,710][382323] Decorrelating experience for 672 frames... +[2023-03-07 15:48:55,710][382456] Decorrelating experience for 800 frames... +[2023-03-07 15:48:55,730][383293] Decorrelating experience for 672 frames... +[2023-03-07 15:48:55,777][385093] Decorrelating experience for 896 frames... +[2023-03-07 15:48:55,824][382398] Decorrelating experience for 896 frames... +[2023-03-07 15:48:55,824][384503] Decorrelating experience for 896 frames... +[2023-03-07 15:48:55,850][382412] Decorrelating experience for 416 frames... +[2023-03-07 15:48:55,861][382841] Decorrelating experience for 576 frames... +[2023-03-07 15:48:55,876][384761] Decorrelating experience for 608 frames... +[2023-03-07 15:48:55,909][385027] Decorrelating experience for 512 frames... +[2023-03-07 15:48:55,910][382439] Decorrelating experience for 320 frames... +[2023-03-07 15:48:55,927][382416] Decorrelating experience for 704 frames... +[2023-03-07 15:48:55,957][382449] Decorrelating experience for 608 frames... +[2023-03-07 15:48:55,978][382844] Decorrelating experience for 768 frames... +[2023-03-07 15:48:56,023][382397] Decorrelating experience for 640 frames... +[2023-03-07 15:48:56,026][383951] Decorrelating experience for 480 frames... +[2023-03-07 15:48:56,060][382445] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,061][382974] Decorrelating experience for 640 frames... +[2023-03-07 15:48:56,091][382395] Decorrelating experience for 768 frames... +[2023-03-07 15:48:56,107][382418] Decorrelating experience for 768 frames... +[2023-03-07 15:48:56,110][382419] Decorrelating experience for 576 frames... +[2023-03-07 15:48:56,115][382409] Decorrelating experience for 736 frames... +[2023-03-07 15:48:56,146][382847] Decorrelating experience for 576 frames... +[2023-03-07 15:48:56,237][383292] Decorrelating experience for 896 frames... +[2023-03-07 15:48:56,254][382420] Decorrelating experience for 960 frames... +[2023-03-07 15:48:56,268][383550] Decorrelating experience for 544 frames... +[2023-03-07 15:48:56,282][382405] Decorrelating experience for 960 frames... +[2023-03-07 15:48:56,309][382432] Decorrelating experience for 480 frames... +[2023-03-07 15:48:56,309][382524] Decorrelating experience for 576 frames... +[2023-03-07 15:48:56,313][384643] Decorrelating experience for 832 frames... +[2023-03-07 15:48:56,341][382402] Decorrelating experience for 416 frames... +[2023-03-07 15:48:56,345][383955] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,359][382408] Decorrelating experience for 448 frames... +[2023-03-07 15:48:56,437][383984] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,464][383641] Decorrelating experience for 512 frames... +[2023-03-07 15:48:56,464][384503] Decorrelating experience for 928 frames... +[2023-03-07 15:48:56,482][382846] Decorrelating experience for 928 frames... +[2023-03-07 15:48:56,507][382397] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,507][382414] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,511][382838] Decorrelating experience for 800 frames... +[2023-03-07 15:48:56,537][382425] Decorrelating experience for 736 frames... +[2023-03-07 15:48:56,541][383951] Decorrelating experience for 512 frames... +[2023-03-07 15:48:56,550][382847] Decorrelating experience for 608 frames... +[2023-03-07 15:48:56,637][382322] Decorrelating experience for 736 frames... +[2023-03-07 15:48:56,667][385093] Decorrelating experience for 928 frames... +[2023-03-07 15:48:56,687][382416] Decorrelating experience for 736 frames... +[2023-03-07 15:48:56,712][383798] Decorrelating experience for 928 frames... +[2023-03-07 15:48:56,713][382398] Decorrelating experience for 928 frames... +[2023-03-07 15:48:56,716][383678] Decorrelating experience for 640 frames... +[2023-03-07 15:48:56,739][382409] Decorrelating experience for 768 frames... +[2023-03-07 15:48:56,740][382424] Decorrelating experience for 448 frames... +[2023-03-07 15:48:56,765][384114] Decorrelating experience for 736 frames... +[2023-03-07 15:48:56,769][382434] Decorrelating experience for 576 frames... +[2023-03-07 15:48:56,838][382324] Decorrelating experience for 608 frames... +[2023-03-07 15:48:56,876][383831] Decorrelating experience for 672 frames... +[2023-03-07 15:48:56,884][384388] Decorrelating experience for 224 frames... +[2023-03-07 15:48:56,922][382412] Decorrelating experience for 448 frames... +[2023-03-07 15:48:56,922][382420] Decorrelating experience for 992 frames... +[2023-03-07 15:48:56,923][382323] Decorrelating experience for 704 frames... +[2023-03-07 15:48:56,950][382845] Decorrelating experience for 448 frames... +[2023-03-07 15:48:56,985][384368] Decorrelating experience for 864 frames... +[2023-03-07 15:48:57,010][382453] Decorrelating experience for 928 frames... +[2023-03-07 15:48:57,045][382443] Decorrelating experience for 768 frames... +[2023-03-07 15:48:57,046][384962] Decorrelating experience for 704 frames... +[2023-03-07 15:48:57,090][383293] Decorrelating experience for 704 frames... +[2023-03-07 15:48:57,095][385049] Decorrelating experience for 704 frames... +[2023-03-07 15:48:57,118][382454] Decorrelating experience for 544 frames... +[2023-03-07 15:48:57,130][382450] Decorrelating experience for 640 frames... +[2023-03-07 15:48:57,153][382582] Decorrelating experience for 832 frames... +[2023-03-07 15:48:57,178][382434] Decorrelating experience for 608 frames... +[2023-03-07 15:48:57,192][382586] Decorrelating experience for 736 frames... +[2023-03-07 15:48:57,211][384388] Decorrelating experience for 256 frames... +[2023-03-07 15:48:57,241][382432] Decorrelating experience for 512 frames... +[2023-03-07 15:48:57,295][382393] Decorrelating experience for 384 frames... +[2023-03-07 15:48:57,327][382845] Decorrelating experience for 480 frames... +[2023-03-07 15:48:57,329][382428] Decorrelating experience for 960 frames... +[2023-03-07 15:48:57,331][382322] Decorrelating experience for 768 frames... +[2023-03-07 15:48:57,332][382411] Decorrelating experience for 544 frames... +[2023-03-07 15:48:57,360][384761] Decorrelating experience for 640 frames... +[2023-03-07 15:48:57,392][383102] Decorrelating experience for 416 frames... +[2023-03-07 15:48:57,397][382420] Stopping RolloutWorker_w4... +[2023-03-07 15:48:57,397][382420] Loop rollout_proc4_evt_loop terminating... +[2023-03-07 15:48:57,397][236315] Component RolloutWorker_w4 stopped! +[2023-03-07 15:48:57,427][382326] Another process currently holds the lock /tmp/sf2_rolo/doom_009.lockfile, attempt: 1 +[2023-03-07 15:48:57,429][382400] Decorrelating experience for 928 frames... +[2023-03-07 15:48:57,429][382416] Decorrelating experience for 768 frames... +[2023-03-07 15:48:57,493][382452] Decorrelating experience for 864 frames... +[2023-03-07 15:48:57,506][384157] Decorrelating experience for 768 frames... +[2023-03-07 15:48:57,538][382446] Decorrelating experience for 800 frames... +[2023-03-07 15:48:57,539][382419] Decorrelating experience for 608 frames... +[2023-03-07 15:48:57,539][382847] Decorrelating experience for 640 frames... +[2023-03-07 15:48:57,546][383293] Decorrelating experience for 736 frames... +[2023-03-07 15:48:57,561][383550] Decorrelating experience for 576 frames... +[2023-03-07 15:48:57,591][383799] Decorrelating experience for 672 frames... +[2023-03-07 15:48:57,632][382406] Decorrelating experience for 640 frames... +[2023-03-07 15:48:57,636][382323] Decorrelating experience for 736 frames... +[2023-03-07 15:48:57,705][384643] Decorrelating experience for 864 frames... +[2023-03-07 15:48:57,732][383951] Decorrelating experience for 544 frames... +[2023-03-07 15:48:57,750][383955] Decorrelating experience for 704 frames... +[2023-03-07 15:48:57,764][382582] Decorrelating experience for 864 frames... +[2023-03-07 15:48:57,782][382430] Decorrelating experience for 896 frames... +[2023-03-07 15:48:57,785][382321] Decorrelating experience for 736 frames... +[2023-03-07 15:48:57,790][382393] Decorrelating experience for 416 frames... +[2023-03-07 15:48:57,803][382457] Decorrelating experience for 896 frames... +[2023-03-07 15:48:57,835][382405] Decorrelating experience for 992 frames... +[2023-03-07 15:48:57,851][384761] Decorrelating experience for 672 frames... +[2023-03-07 15:48:57,913][385027] Decorrelating experience for 544 frames... +[2023-03-07 15:48:57,934][382432] Decorrelating experience for 544 frames... +[2023-03-07 15:48:57,960][382416] Decorrelating experience for 800 frames... +[2023-03-07 15:48:57,963][382415] Decorrelating experience for 832 frames... +[2023-03-07 15:48:57,982][382453] Decorrelating experience for 960 frames... +[2023-03-07 15:48:57,989][382841] Decorrelating experience for 608 frames... +[2023-03-07 15:48:57,994][383984] Decorrelating experience for 704 frames... +[2023-03-07 15:48:58,002][382524] Decorrelating experience for 608 frames... +[2023-03-07 15:48:58,034][382428] Decorrelating experience for 992 frames... +[2023-03-07 15:48:58,053][383744] Decorrelating experience for 224 frames... +[2023-03-07 15:48:58,115][382445] Decorrelating experience for 704 frames... +[2023-03-07 15:48:58,138][382411] Decorrelating experience for 576 frames... +[2023-03-07 15:48:58,159][382402] Decorrelating experience for 448 frames... +[2023-03-07 15:48:58,194][384051] Decorrelating experience for 608 frames... +[2023-03-07 15:48:58,195][382443] Decorrelating experience for 800 frames... +[2023-03-07 15:48:58,199][385094] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,241][382412] Decorrelating experience for 480 frames... +[2023-03-07 15:48:58,256][383639] Decorrelating experience for 768 frames... +[2023-03-07 15:48:58,259][382323] Decorrelating experience for 768 frames... +[2023-03-07 15:48:58,300][382405] Stopping RolloutWorker_w18... +[2023-03-07 15:48:58,301][382405] Loop rollout_proc18_evt_loop terminating... +[2023-03-07 15:48:58,301][236315] Component RolloutWorker_w18 stopped! +[2023-03-07 15:48:58,349][382432] Decorrelating experience for 576 frames... +[2023-03-07 15:48:58,350][382393] Decorrelating experience for 448 frames... +[2023-03-07 15:48:58,361][383955] Decorrelating experience for 736 frames... +[2023-03-07 15:48:58,392][382446] Decorrelating experience for 832 frames... +[2023-03-07 15:48:58,395][382322] Decorrelating experience for 800 frames... +[2023-03-07 15:48:58,419][382321] Decorrelating experience for 768 frames... +[2023-03-07 15:48:58,453][382408] Decorrelating experience for 480 frames... +[2023-03-07 15:48:58,458][383984] Decorrelating experience for 736 frames... +[2023-03-07 15:48:58,477][382449] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,520][382428] Stopping RolloutWorker_w37... +[2023-03-07 15:48:58,520][382428] Loop rollout_proc37_evt_loop terminating... +[2023-03-07 15:48:58,520][236315] Component RolloutWorker_w37 stopped! +[2023-03-07 15:48:58,533][382430] Decorrelating experience for 928 frames... +[2023-03-07 15:48:58,562][382416] Decorrelating experience for 832 frames... +[2023-03-07 15:48:58,562][382410] Decorrelating experience for 736 frames... +[2023-03-07 15:48:58,591][382439] Decorrelating experience for 352 frames... +[2023-03-07 15:48:58,625][384051] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,625][382974] Decorrelating experience for 672 frames... +[2023-03-07 15:48:58,655][383102] Decorrelating experience for 448 frames... +[2023-03-07 15:48:58,675][384886] Decorrelating experience for 192 frames... +[2023-03-07 15:48:58,678][382457] Decorrelating experience for 928 frames... +[2023-03-07 15:48:58,679][382452] Decorrelating experience for 896 frames... +[2023-03-07 15:48:58,725][385095] Decorrelating experience for 896 frames... +[2023-03-07 15:48:58,760][382434] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,764][382393] Decorrelating experience for 480 frames... +[2023-03-07 15:48:58,810][382395] Decorrelating experience for 800 frames... +[2023-03-07 15:48:58,824][382324] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,828][382586] Decorrelating experience for 768 frames... +[2023-03-07 15:48:58,856][382418] Decorrelating experience for 800 frames... +[2023-03-07 15:48:58,879][382397] Decorrelating experience for 704 frames... +[2023-03-07 15:48:58,880][382425] Decorrelating experience for 768 frames... +[2023-03-07 15:48:58,916][382432] Decorrelating experience for 608 frames... +[2023-03-07 15:48:58,924][382840] Decorrelating experience for 640 frames... +[2023-03-07 15:48:58,967][382841] Decorrelating experience for 640 frames... +[2023-03-07 15:48:59,006][382322] Decorrelating experience for 832 frames... +[2023-03-07 15:48:59,018][383639] Decorrelating experience for 800 frames... +[2023-03-07 15:48:59,034][382844] Decorrelating experience for 800 frames... +[2023-03-07 15:48:59,074][382439] Decorrelating experience for 384 frames... +[2023-03-07 15:48:59,075][385094] Decorrelating experience for 672 frames... +[2023-03-07 15:48:59,077][383798] Decorrelating experience for 960 frames... +[2023-03-07 15:48:59,122][385027] Decorrelating experience for 576 frames... +[2023-03-07 15:48:59,140][382449] Decorrelating experience for 672 frames... +[2023-03-07 15:48:59,173][382845] Decorrelating experience for 512 frames... +[2023-03-07 15:48:59,217][382396] Decorrelating experience for 640 frames... +[2023-03-07 15:48:59,220][383102] Decorrelating experience for 480 frames... +[2023-03-07 15:48:59,262][385095] Decorrelating experience for 928 frames... +[2023-03-07 15:48:59,287][382415] Decorrelating experience for 864 frames... +[2023-03-07 15:48:59,292][382321] Decorrelating experience for 800 frames... +[2023-03-07 15:48:59,296][382524] Decorrelating experience for 640 frames... +[2023-03-07 15:48:59,321][382412] Decorrelating experience for 512 frames... +[2023-03-07 15:48:59,331][382410] Decorrelating experience for 768 frames... +[2023-03-07 15:48:59,340][383799] Decorrelating experience for 704 frames... +[2023-03-07 15:48:59,413][384158] Decorrelating experience for 320 frames... +[2023-03-07 15:48:59,424][382416] Decorrelating experience for 864 frames... +[2023-03-07 15:48:59,428][382453] Decorrelating experience for 992 frames... +[2023-03-07 15:48:59,481][383292] Decorrelating experience for 928 frames... +[2023-03-07 15:48:59,491][382393] Decorrelating experience for 512 frames... +[2023-03-07 15:48:59,499][384387] Decorrelating experience for 576 frames... +[2023-03-07 15:48:59,537][384644] Decorrelating experience for 608 frames... +[2023-03-07 15:48:59,539][384643] Decorrelating experience for 896 frames... +[2023-03-07 15:48:59,540][382451] Decorrelating experience for 736 frames... +[2023-03-07 15:48:59,624][382844] Decorrelating experience for 832 frames... +[2023-03-07 15:48:59,643][382436] Decorrelating experience for 608 frames... +[2023-03-07 15:48:59,644][383102] Decorrelating experience for 512 frames... +[2023-03-07 15:48:59,686][382449] Decorrelating experience for 704 frames... +[2023-03-07 15:48:59,689][382408] Decorrelating experience for 512 frames... +[2023-03-07 15:48:59,710][383639] Decorrelating experience for 832 frames... +[2023-03-07 15:48:59,744][382450] Decorrelating experience for 672 frames... +[2023-03-07 15:48:59,745][382445] Decorrelating experience for 736 frames... +[2023-03-07 15:48:59,746][384886] Decorrelating experience for 224 frames... +[2023-03-07 15:48:59,806][384315] Decorrelating experience for 544 frames... +[2023-03-07 15:48:59,813][382586] Decorrelating experience for 800 frames... +[2023-03-07 15:48:59,836][383427] Decorrelating experience for 992 frames... +[2023-03-07 15:48:59,864][382845] Decorrelating experience for 544 frames... +[2023-03-07 15:48:59,900][382453] Stopping RolloutWorker_w66... +[2023-03-07 15:48:59,900][382453] Loop rollout_proc66_evt_loop terminating... +[2023-03-07 15:48:59,901][236315] Component RolloutWorker_w66 stopped! +[2023-03-07 15:48:59,930][382324] Decorrelating experience for 672 frames... +[2023-03-07 15:48:59,934][384158] Decorrelating experience for 352 frames... +[2023-03-07 15:48:59,940][382457] Decorrelating experience for 960 frames... +[2023-03-07 15:48:59,965][385093] Decorrelating experience for 960 frames... +[2023-03-07 15:49:00,016][382397] Decorrelating experience for 736 frames... +[2023-03-07 15:49:00,029][384157] Decorrelating experience for 800 frames... +[2023-03-07 15:49:00,030][383955] Decorrelating experience for 768 frames... +[2023-03-07 15:49:00,033][382321] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,034][382437] Decorrelating experience for 512 frames... +[2023-03-07 15:49:00,127][382322] Decorrelating experience for 864 frames... +[2023-03-07 15:49:00,139][382524] Decorrelating experience for 672 frames... +[2023-03-07 15:49:00,165][382396] Decorrelating experience for 672 frames... +[2023-03-07 15:49:00,166][385027] Decorrelating experience for 608 frames... +[2023-03-07 15:49:00,166][382411] Decorrelating experience for 608 frames... +[2023-03-07 15:49:00,227][382424] Decorrelating experience for 480 frames... +[2023-03-07 15:49:00,233][384114] Decorrelating experience for 768 frames... +[2023-03-07 15:49:00,269][382427] Decorrelating experience for 480 frames... +[2023-03-07 15:49:00,291][383427] Stopping RolloutWorker_w85... +[2023-03-07 15:49:00,291][383427] Loop rollout_proc85_evt_loop terminating... +[2023-03-07 15:49:00,292][236315] Component RolloutWorker_w85 stopped! +[2023-03-07 15:49:00,312][382398] Decorrelating experience for 960 frames... +[2023-03-07 15:49:00,318][383744] Decorrelating experience for 256 frames... +[2023-03-07 15:49:00,324][382408] Decorrelating experience for 544 frames... +[2023-03-07 15:49:00,348][382415] Decorrelating experience for 896 frames... +[2023-03-07 15:49:00,370][382323] Decorrelating experience for 800 frames... +[2023-03-07 15:49:00,440][384158] Decorrelating experience for 384 frames... +[2023-03-07 15:49:00,443][382452] Decorrelating experience for 928 frames... +[2023-03-07 15:49:00,444][382394] Decorrelating experience for 864 frames... +[2023-03-07 15:49:00,447][382451] Decorrelating experience for 768 frames... +[2023-03-07 15:49:00,468][382417] Decorrelating experience for 864 frames... +[2023-03-07 15:49:00,510][382418] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,517][382437] Decorrelating experience for 544 frames... +[2023-03-07 15:49:00,561][382586] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,568][385095] Decorrelating experience for 960 frames... +[2023-03-07 15:49:00,571][382450] Decorrelating experience for 704 frames... +[2023-03-07 15:49:00,636][383798] Decorrelating experience for 992 frames... +[2023-03-07 15:49:00,640][382325] Decorrelating experience for 768 frames... +[2023-03-07 15:49:00,648][382407] Decorrelating experience for 768 frames... +[2023-03-07 15:49:00,698][383955] Decorrelating experience for 800 frames... +[2023-03-07 15:49:00,713][382446] Decorrelating experience for 864 frames... +[2023-03-07 15:49:00,722][384051] Decorrelating experience for 672 frames... +[2023-03-07 15:49:00,746][382448] Another process currently holds the lock /tmp/sf2_rolo/doom_006.lockfile, attempt: 1 +[2023-03-07 15:49:00,770][383639] Decorrelating experience for 864 frames... +[2023-03-07 15:49:00,844][382395] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,851][382406] Decorrelating experience for 672 frames... +[2023-03-07 15:49:00,852][384387] Decorrelating experience for 608 frames... +[2023-03-07 15:49:00,853][384761] Decorrelating experience for 704 frames... +[2023-03-07 15:49:00,862][382323] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,891][385094] Decorrelating experience for 704 frames... +[2023-03-07 15:49:00,920][382412] Decorrelating experience for 544 frames... +[2023-03-07 15:49:00,965][382443] Decorrelating experience for 832 frames... +[2023-03-07 15:49:00,977][384886] Decorrelating experience for 256 frames... +[2023-03-07 15:49:01,058][382444] Decorrelating experience for 736 frames... +[2023-03-07 15:49:01,059][382441] Decorrelating experience for 672 frames... +[2023-03-07 15:49:01,067][383984] Decorrelating experience for 768 frames... +[2023-03-07 15:49:01,080][382844] Decorrelating experience for 864 frames... +[2023-03-07 15:49:01,104][383798] Stopping RolloutWorker_w110... +[2023-03-07 15:49:01,104][383798] Loop rollout_proc110_evt_loop terminating... +[2023-03-07 15:49:01,105][236315] Component RolloutWorker_w110 stopped! +[2023-03-07 15:49:01,118][382410] Decorrelating experience for 800 frames... +[2023-03-07 15:49:01,134][382432] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,166][382398] Decorrelating experience for 992 frames... +[2023-03-07 15:49:01,183][382450] Decorrelating experience for 736 frames... +[2023-03-07 15:49:01,261][382435] Decorrelating experience for 512 frames... +[2023-03-07 15:49:01,289][384886] Decorrelating experience for 288 frames... +[2023-03-07 15:49:01,289][382841] Decorrelating experience for 672 frames... +[2023-03-07 15:49:01,290][390548] Decorrelating experience for 576 frames... +[2023-03-07 15:49:01,295][382449] Decorrelating experience for 736 frames... +[2023-03-07 15:49:01,331][384644] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,335][382324] Decorrelating experience for 704 frames... +[2023-03-07 15:49:01,381][382446] Decorrelating experience for 896 frames... +[2023-03-07 15:49:01,453][384387] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,463][382394] Decorrelating experience for 896 frames... +[2023-03-07 15:49:01,485][383641] Decorrelating experience for 544 frames... +[2023-03-07 15:49:01,535][382321] Decorrelating experience for 864 frames... +[2023-03-07 15:49:01,553][383294] Decorrelating experience for 864 frames... +[2023-03-07 15:49:01,557][382423] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,588][382439] Decorrelating experience for 416 frames... +[2023-03-07 15:49:01,623][385027] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,625][382398] Stopping RolloutWorker_w41... +[2023-03-07 15:49:01,625][382398] Loop rollout_proc41_evt_loop terminating... +[2023-03-07 15:49:01,625][236315] Component RolloutWorker_w41 stopped! +[2023-03-07 15:49:01,668][382323] Decorrelating experience for 864 frames... +[2023-03-07 15:49:01,677][385049] Decorrelating experience for 736 frames... +[2023-03-07 15:49:01,723][382411] Decorrelating experience for 640 frames... +[2023-03-07 15:49:01,732][382840] Decorrelating experience for 672 frames... +[2023-03-07 15:49:01,732][382841] Decorrelating experience for 704 frames... +[2023-03-07 15:49:01,744][382400] Decorrelating experience for 960 frames... +[2023-03-07 15:49:01,783][384158] Decorrelating experience for 416 frames... +[2023-03-07 15:49:01,830][382412] Decorrelating experience for 576 frames... +[2023-03-07 15:49:01,834][383951] Decorrelating experience for 576 frames... +[2023-03-07 15:49:01,880][384644] Decorrelating experience for 672 frames... +[2023-03-07 15:49:01,883][383831] Decorrelating experience for 704 frames... +[2023-03-07 15:49:01,884][384761] Decorrelating experience for 736 frames... +[2023-03-07 15:49:01,931][385093] Decorrelating experience for 992 frames... +[2023-03-07 15:49:01,939][384643] Decorrelating experience for 928 frames... +[2023-03-07 15:49:01,943][383527] Decorrelating experience for 416 frames... +[2023-03-07 15:49:01,956][390548] Decorrelating experience for 608 frames... +[2023-03-07 15:49:01,979][382325] Decorrelating experience for 800 frames... +[2023-03-07 15:49:02,030][384114] Decorrelating experience for 800 frames... +[2023-03-07 15:49:02,056][382432] Decorrelating experience for 672 frames... +[2023-03-07 15:49:02,085][382419] Decorrelating experience for 640 frames... +[2023-03-07 15:49:02,088][384962] Decorrelating experience for 736 frames... +[2023-03-07 15:49:02,135][385096] Decorrelating experience for 672 frames... +[2023-03-07 15:49:02,171][382443] Decorrelating experience for 864 frames... +[2023-03-07 15:49:02,180][382394] Decorrelating experience for 928 frames... +[2023-03-07 15:49:02,196][383294] Decorrelating experience for 896 frames... +[2023-03-07 15:49:02,197][382323] Decorrelating experience for 896 frames... +[2023-03-07 15:49:02,219][382451] Decorrelating experience for 800 frames... +[2023-03-07 15:49:02,271][382449] Decorrelating experience for 768 frames... +[2023-03-07 15:49:02,274][382437] Decorrelating experience for 576 frames... +[2023-03-07 15:49:02,285][383641] Decorrelating experience for 576 frames... +[2023-03-07 15:49:02,336][382403] Decorrelating experience for 704 frames... +[2023-03-07 15:49:02,347][384388] Decorrelating experience for 288 frames... +[2023-03-07 15:49:02,380][382586] Decorrelating experience for 864 frames... +[2023-03-07 15:49:02,413][384761] Decorrelating experience for 768 frames... +[2023-03-07 15:49:02,421][382410] Decorrelating experience for 832 frames... +[2023-03-07 15:49:02,447][382416] Decorrelating experience for 896 frames... +[2023-03-07 15:49:02,466][383744] Decorrelating experience for 288 frames... +[2023-03-07 15:49:02,479][384644] Decorrelating experience for 704 frames... +[2023-03-07 15:49:02,507][382400] Decorrelating experience for 992 frames... +[2023-03-07 15:49:02,544][382425] Decorrelating experience for 800 frames... +[2023-03-07 15:49:02,561][385093] Stopping RolloutWorker_w124... +[2023-03-07 15:49:02,562][385093] Loop rollout_proc124_evt_loop terminating... +[2023-03-07 15:49:02,562][236315] Component RolloutWorker_w124 stopped! +[2023-03-07 15:49:02,584][385096] Decorrelating experience for 704 frames... +[2023-03-07 15:49:02,630][382450] Decorrelating experience for 768 frames... +[2023-03-07 15:49:02,632][385094] Decorrelating experience for 736 frames... +[2023-03-07 15:49:02,633][382325] Decorrelating experience for 832 frames... +[2023-03-07 15:49:02,639][382397] Decorrelating experience for 768 frames... +[2023-03-07 15:49:02,657][382426] Decorrelating experience for 896 frames... +[2023-03-07 15:49:02,674][390548] Decorrelating experience for 640 frames... +[2023-03-07 15:49:02,737][382429] Decorrelating experience for 640 frames... +[2023-03-07 15:49:02,774][382323] Decorrelating experience for 928 frames... +[2023-03-07 15:49:02,825][382424] Decorrelating experience for 512 frames... +[2023-03-07 15:49:02,833][384114] Decorrelating experience for 832 frames... +[2023-03-07 15:49:02,841][382444] Decorrelating experience for 768 frames... +[2023-03-07 15:49:02,851][382524] Decorrelating experience for 704 frames... +[2023-03-07 15:49:02,865][382427] Decorrelating experience for 512 frames... +[2023-03-07 15:49:02,909][382586] Decorrelating experience for 896 frames... +[2023-03-07 15:49:02,917][384051] Decorrelating experience for 704 frames... +[2023-03-07 15:49:02,945][382451] Decorrelating experience for 832 frames... +[2023-03-07 15:49:02,995][382840] Decorrelating experience for 704 frames... +[2023-03-07 15:49:03,030][384962] Decorrelating experience for 768 frames... +[2023-03-07 15:49:03,046][384761] Decorrelating experience for 800 frames... +[2023-03-07 15:49:03,052][384886] Decorrelating experience for 320 frames... +[2023-03-07 15:49:03,062][382432] Decorrelating experience for 704 frames... +[2023-03-07 15:49:03,088][382838] Decorrelating experience for 832 frames... +[2023-03-07 15:49:03,098][382400] Stopping RolloutWorker_w44... +[2023-03-07 15:49:03,098][382400] Loop rollout_proc44_evt_loop terminating... +[2023-03-07 15:49:03,098][236315] Component RolloutWorker_w44 stopped! +[2023-03-07 15:49:03,125][382450] Decorrelating experience for 800 frames... +[2023-03-07 15:49:03,142][382425] Decorrelating experience for 832 frames... +[2023-03-07 15:49:03,166][384158] Decorrelating experience for 448 frames... +[2023-03-07 15:49:03,187][382846] Decorrelating experience for 960 frames... +[2023-03-07 15:49:03,193][382439] Decorrelating experience for 448 frames... +[2023-03-07 15:49:03,259][382841] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,286][383294] Decorrelating experience for 928 frames... +[2023-03-07 15:49:03,313][382416] Decorrelating experience for 928 frames... +[2023-03-07 15:49:03,349][385027] Decorrelating experience for 672 frames... +[2023-03-07 15:49:03,354][382426] Decorrelating experience for 928 frames... +[2023-03-07 15:49:03,357][382397] Decorrelating experience for 800 frames... +[2023-03-07 15:49:03,373][382424] Decorrelating experience for 544 frames... +[2023-03-07 15:49:03,373][382322] Decorrelating experience for 896 frames... +[2023-03-07 15:49:03,389][382407] Decorrelating experience for 800 frames... +[2023-03-07 15:49:03,403][383641] Decorrelating experience for 608 frames... +[2023-03-07 15:49:03,454][382326] Decorrelating experience for 480 frames... +[2023-03-07 15:49:03,483][382427] Decorrelating experience for 544 frames... +[2023-03-07 15:49:03,505][382842] Decorrelating experience for 352 frames... +[2023-03-07 15:49:03,549][384644] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,562][385096] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,563][382451] Decorrelating experience for 864 frames... +[2023-03-07 15:49:03,571][383799] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,580][382432] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,614][382418] Decorrelating experience for 864 frames... +[2023-03-07 15:49:03,619][382840] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,657][383293] Decorrelating experience for 768 frames... +[2023-03-07 15:49:03,686][382436] Decorrelating experience for 640 frames... +[2023-03-07 15:49:03,704][382843] Decorrelating experience for 864 frames... +[2023-03-07 15:49:03,769][384051] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,810][385095] Decorrelating experience for 992 frames... +[2023-03-07 15:49:03,846][383744] Decorrelating experience for 320 frames... +[2023-03-07 15:49:03,847][382450] Decorrelating experience for 832 frames... +[2023-03-07 15:49:03,848][382325] Decorrelating experience for 864 frames... +[2023-03-07 15:49:03,850][382443] Decorrelating experience for 896 frames... +[2023-03-07 15:49:03,855][382524] Decorrelating experience for 736 frames... +[2023-03-07 15:49:03,863][382440] Decorrelating experience for 640 frames... +[2023-03-07 15:49:03,888][382445] Decorrelating experience for 768 frames... +[2023-03-07 15:49:03,905][382412] Decorrelating experience for 608 frames... +[2023-03-07 15:49:03,969][382401] Another process currently holds the lock /tmp/sf2_rolo/doom_006.lockfile, attempt: 1 +[2023-03-07 15:49:03,970][382323] Decorrelating experience for 960 frames... +[2023-03-07 15:49:04,047][384962] Decorrelating experience for 800 frames... +[2023-03-07 15:49:04,051][382394] Decorrelating experience for 960 frames... +[2023-03-07 15:49:04,063][382425] Decorrelating experience for 864 frames... +[2023-03-07 15:49:04,087][382846] Decorrelating experience for 992 frames... +[2023-03-07 15:49:04,091][382430] Decorrelating experience for 960 frames... +[2023-03-07 15:49:04,092][382397] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,099][383955] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,103][382442] Decorrelating experience for 736 frames... +[2023-03-07 15:49:04,125][385027] Decorrelating experience for 704 frames... +[2023-03-07 15:49:04,182][382841] Decorrelating experience for 768 frames... +[2023-03-07 15:49:04,255][382449] Decorrelating experience for 800 frames... +[2023-03-07 15:49:04,298][385095] Stopping RolloutWorker_w127... +[2023-03-07 15:49:04,298][385095] Loop rollout_proc127_evt_loop terminating... +[2023-03-07 15:49:04,299][236315] Component RolloutWorker_w127 stopped! +[2023-03-07 15:49:04,321][384643] Decorrelating experience for 960 frames... +[2023-03-07 15:49:04,323][382396] Decorrelating experience for 704 frames... +[2023-03-07 15:49:04,340][382847] Decorrelating experience for 672 frames... +[2023-03-07 15:49:04,341][383831] Decorrelating experience for 736 frames... +[2023-03-07 15:49:04,344][383293] Decorrelating experience for 800 frames... +[2023-03-07 15:49:04,344][382412] Decorrelating experience for 640 frames... +[2023-03-07 15:49:04,358][382586] Decorrelating experience for 928 frames... +[2023-03-07 15:49:04,399][382838] Decorrelating experience for 864 frames... +[2023-03-07 15:49:04,457][382414] Decorrelating experience for 704 frames... +[2023-03-07 15:49:04,521][382321] Decorrelating experience for 896 frames... +[2023-03-07 15:49:04,561][382328] Decorrelating experience for 608 frames... +[2023-03-07 15:49:04,575][382436] Decorrelating experience for 672 frames... +[2023-03-07 15:49:04,575][382450] Decorrelating experience for 864 frames... +[2023-03-07 15:49:04,576][384962] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,577][382440] Decorrelating experience for 672 frames... +[2023-03-07 15:49:04,579][382846] Stopping RolloutWorker_w61... +[2023-03-07 15:49:04,580][382846] Loop rollout_proc61_evt_loop terminating... +[2023-03-07 15:49:04,580][236315] Component RolloutWorker_w61 stopped! +[2023-03-07 15:49:04,582][382442] Decorrelating experience for 768 frames... +[2023-03-07 15:49:04,587][382454] Decorrelating experience for 576 frames... +[2023-03-07 15:49:04,606][382443] Decorrelating experience for 928 frames... +[2023-03-07 15:49:04,652][382430] Decorrelating experience for 992 frames... +[2023-03-07 15:49:04,727][382456] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,802][382437] Decorrelating experience for 608 frames... +[2023-03-07 15:49:04,802][382435] Decorrelating experience for 544 frames... +[2023-03-07 15:49:04,803][382422] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,804][382427] Decorrelating experience for 576 frames... +[2023-03-07 15:49:04,812][382429] Decorrelating experience for 672 frames... +[2023-03-07 15:49:04,819][382394] Decorrelating experience for 992 frames... +[2023-03-07 15:49:04,823][382840] Decorrelating experience for 768 frames... +[2023-03-07 15:49:04,850][382407] Decorrelating experience for 832 frames... +[2023-03-07 15:49:04,850][382425] Decorrelating experience for 896 frames... +[2023-03-07 15:49:04,921][385049] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,034][384051] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,040][384761] Decorrelating experience for 832 frames... +[2023-03-07 15:49:05,058][382403] Decorrelating experience for 736 frames... +[2023-03-07 15:49:05,059][382524] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,059][383831] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,060][382323] Decorrelating experience for 992 frames... +[2023-03-07 15:49:05,060][384552] Decorrelating experience for 736 frames... +[2023-03-07 15:49:05,060][383294] Decorrelating experience for 960 frames... +[2023-03-07 15:49:05,115][382415] Decorrelating experience for 928 frames... +[2023-03-07 15:49:05,130][382430] Stopping RolloutWorker_w40... +[2023-03-07 15:49:05,130][382430] Loop rollout_proc40_evt_loop terminating... +[2023-03-07 15:49:05,130][236315] Component RolloutWorker_w40 stopped! +[2023-03-07 15:49:05,206][382435] Decorrelating experience for 576 frames... +[2023-03-07 15:49:05,232][382325] Decorrelating experience for 896 frames... +[2023-03-07 15:49:05,294][382454] Decorrelating experience for 608 frames... +[2023-03-07 15:49:05,299][383292] Decorrelating experience for 960 frames... +[2023-03-07 15:49:05,305][382427] Decorrelating experience for 608 frames... +[2023-03-07 15:49:05,306][383102] Decorrelating experience for 544 frames... +[2023-03-07 15:49:05,306][383550] Decorrelating experience for 608 frames... +[2023-03-07 15:49:05,314][382394] Stopping RolloutWorker_w29... +[2023-03-07 15:49:05,314][382394] Loop rollout_proc29_evt_loop terminating... +[2023-03-07 15:49:05,314][236315] Component RolloutWorker_w29 stopped! +[2023-03-07 15:49:05,315][384962] Decorrelating experience for 864 frames... +[2023-03-07 15:49:05,318][383293] Decorrelating experience for 832 frames... +[2023-03-07 15:49:05,322][382399] Decorrelating experience for 448 frames... +[2023-03-07 15:49:05,405][385049] Decorrelating experience for 800 frames... +[2023-03-07 15:49:05,431][382437] Decorrelating experience for 640 frames... +[2023-03-07 15:49:05,496][384114] Decorrelating experience for 864 frames... +[2023-03-07 15:49:05,501][383641] Decorrelating experience for 640 frames... +[2023-03-07 15:49:05,526][382323] Stopping RolloutWorker_w11... +[2023-03-07 15:49:05,526][382323] Loop rollout_proc11_evt_loop terminating... +[2023-03-07 15:49:05,526][236315] Component RolloutWorker_w11 stopped! +[2023-03-07 15:49:05,544][384051] Decorrelating experience for 800 frames... +[2023-03-07 15:49:05,553][382395] Decorrelating experience for 864 frames... +[2023-03-07 15:49:05,553][385094] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,554][383799] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,563][382586] Decorrelating experience for 960 frames... +[2023-03-07 15:49:05,574][382423] Decorrelating experience for 672 frames... +[2023-03-07 15:49:05,660][382324] Decorrelating experience for 736 frames... +[2023-03-07 15:49:05,681][383831] Decorrelating experience for 800 frames... +[2023-03-07 15:49:05,733][382444] Decorrelating experience for 800 frames... +[2023-03-07 15:49:05,745][384552] Decorrelating experience for 768 frames... +[2023-03-07 15:49:05,748][385027] Decorrelating experience for 736 frames... +[2023-03-07 15:49:05,780][382435] Decorrelating experience for 608 frames... +[2023-03-07 15:49:05,783][382449] Decorrelating experience for 832 frames... +[2023-03-07 15:49:05,841][382456] Decorrelating experience for 864 frames... +[2023-03-07 15:49:05,850][382422] Decorrelating experience for 864 frames... +[2023-03-07 15:49:05,851][382524] Decorrelating experience for 800 frames... +[2023-03-07 15:49:05,896][384368] Decorrelating experience for 896 frames... +[2023-03-07 15:49:05,929][382437] Decorrelating experience for 672 frames... +[2023-03-07 15:49:05,933][382838] Decorrelating experience for 896 frames... +[2023-03-07 15:49:05,953][385049] Decorrelating experience for 832 frames... +[2023-03-07 15:49:05,979][382409] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,011][382396] Decorrelating experience for 736 frames... +[2023-03-07 15:49:06,049][382445] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,059][383955] Decorrelating experience for 864 frames... +[2023-03-07 15:49:06,059][382440] Decorrelating experience for 704 frames... +[2023-03-07 15:49:06,060][382328] Decorrelating experience for 640 frames... +[2023-03-07 15:49:06,100][382326] Decorrelating experience for 512 frames... +[2023-03-07 15:49:06,141][383641] Decorrelating experience for 672 frames... +[2023-03-07 15:49:06,145][382441] Decorrelating experience for 704 frames... +[2023-03-07 15:49:06,181][382450] Decorrelating experience for 896 frames... +[2023-03-07 15:49:06,187][383984] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,213][382455] Decorrelating experience for 768 frames... +[2023-03-07 15:49:06,271][382443] Decorrelating experience for 960 frames... +[2023-03-07 15:49:06,280][382435] Decorrelating experience for 640 frames... +[2023-03-07 15:49:06,286][382401] Decorrelating experience for 576 frames... +[2023-03-07 15:49:06,287][382841] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,303][382847] Decorrelating experience for 704 frames... +[2023-03-07 15:49:06,337][384051] Decorrelating experience for 832 frames... +[2023-03-07 15:49:06,387][382437] Decorrelating experience for 704 frames... +[2023-03-07 15:49:06,410][385031] Decorrelating experience for 864 frames... +[2023-03-07 15:49:06,416][382444] Decorrelating experience for 832 frames... +[2023-03-07 15:49:06,436][382456] Decorrelating experience for 896 frames... +[2023-03-07 15:49:06,474][382325] Decorrelating experience for 928 frames... +[2023-03-07 15:49:06,501][382442] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,501][384552] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,502][382454] Decorrelating experience for 640 frames... +[2023-03-07 15:49:06,522][383678] Decorrelating experience for 672 frames... +[2023-03-07 15:49:06,543][384644] Decorrelating experience for 768 frames... +[2023-03-07 15:49:06,588][383951] Decorrelating experience for 608 frames... +[2023-03-07 15:49:06,601][382842] Decorrelating experience for 384 frames... +[2023-03-07 15:49:06,648][384368] Decorrelating experience for 928 frames... +[2023-03-07 15:49:06,672][382322] Decorrelating experience for 928 frames... +[2023-03-07 15:49:06,692][383799] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,721][385026] Decorrelating experience for 448 frames... +[2023-03-07 15:49:06,765][385027] Decorrelating experience for 768 frames... +[2023-03-07 15:49:06,801][382840] Decorrelating experience for 800 frames... +[2023-03-07 15:49:06,810][382401] Decorrelating experience for 608 frames... +[2023-03-07 15:49:06,813][382450] Decorrelating experience for 928 frames... +[2023-03-07 15:49:06,814][383984] Decorrelating experience for 832 frames... +[2023-03-07 15:49:06,825][383955] Decorrelating experience for 896 frames... +[2023-03-07 15:49:06,876][382415] Decorrelating experience for 960 frames... +[2023-03-07 15:49:06,879][382440] Decorrelating experience for 736 frames... +[2023-03-07 15:49:06,955][382395] Decorrelating experience for 896 frames... +[2023-03-07 15:49:06,963][382449] Decorrelating experience for 864 frames... +[2023-03-07 15:49:06,981][382438] Decorrelating experience for 928 frames... +[2023-03-07 15:49:06,990][383641] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,008][385096] Decorrelating experience for 768 frames... +[2023-03-07 15:49:07,024][382325] Decorrelating experience for 960 frames... +[2023-03-07 15:49:07,024][383102] Decorrelating experience for 576 frames... +[2023-03-07 15:49:07,086][382845] Decorrelating experience for 576 frames... +[2023-03-07 15:49:07,118][382456] Decorrelating experience for 928 frames... +[2023-03-07 15:49:07,152][382442] Decorrelating experience for 832 frames... +[2023-03-07 15:49:07,155][382417] Decorrelating experience for 896 frames... +[2023-03-07 15:49:07,176][382418] Decorrelating experience for 896 frames... +[2023-03-07 15:49:07,191][382396] Decorrelating experience for 768 frames... +[2023-03-07 15:49:07,198][382843] Decorrelating experience for 896 frames... +[2023-03-07 15:49:07,219][382434] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,235][382435] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,239][382847] Decorrelating experience for 736 frames... +[2023-03-07 15:49:07,348][382406] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,377][382322] Decorrelating experience for 960 frames... +[2023-03-07 15:49:07,389][383678] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,393][382841] Decorrelating experience for 832 frames... +[2023-03-07 15:49:07,409][382429] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,423][382441] Decorrelating experience for 736 frames... +[2023-03-07 15:49:07,434][384368] Decorrelating experience for 960 frames... +[2023-03-07 15:49:07,443][382454] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,475][382403] Decorrelating experience for 768 frames... +[2023-03-07 15:49:07,499][385026] Decorrelating experience for 480 frames... +[2023-03-07 15:49:07,545][382427] Decorrelating experience for 640 frames... +[2023-03-07 15:49:07,583][383294] Decorrelating experience for 992 frames... +[2023-03-07 15:49:07,592][384388] Decorrelating experience for 320 frames... +[2023-03-07 15:49:07,618][382445] Decorrelating experience for 832 frames... +[2023-03-07 15:49:07,632][383527] Decorrelating experience for 448 frames... +[2023-03-07 15:49:07,652][382436] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,661][382443] Decorrelating experience for 992 frames... +[2023-03-07 15:49:07,678][385094] Decorrelating experience for 800 frames... +[2023-03-07 15:49:07,692][382438] Decorrelating experience for 960 frames... +[2023-03-07 15:49:07,720][382440] Decorrelating experience for 768 frames... +[2023-03-07 15:49:07,748][383799] Decorrelating experience for 832 frames... +[2023-03-07 15:49:07,792][382448] Decorrelating experience for 480 frames... +[2023-03-07 15:49:07,793][382411] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,831][383984] Decorrelating experience for 864 frames... +[2023-03-07 15:49:07,835][383639] Decorrelating experience for 896 frames... +[2023-03-07 15:49:07,858][382412] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,863][382328] Decorrelating experience for 672 frames... +[2023-03-07 15:49:07,895][382454] Decorrelating experience for 704 frames... +[2023-03-07 15:49:07,941][382406] Decorrelating experience for 736 frames... +[2023-03-07 15:49:07,952][384962] Decorrelating experience for 896 frames... +[2023-03-07 15:49:07,994][382444] Decorrelating experience for 864 frames... +[2023-03-07 15:49:07,995][382435] Decorrelating experience for 704 frames... +[2023-03-07 15:49:08,033][383831] Decorrelating experience for 832 frames... +[2023-03-07 15:49:08,037][382426] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,043][383294] Stopping RolloutWorker_w92... +[2023-03-07 15:49:08,044][383294] Loop rollout_proc92_evt_loop terminating... +[2023-03-07 15:49:08,044][236315] Component RolloutWorker_w92 stopped! +[2023-03-07 15:49:08,059][384368] Decorrelating experience for 992 frames... +[2023-03-07 15:49:08,067][382841] Decorrelating experience for 864 frames... +[2023-03-07 15:49:08,117][382436] Decorrelating experience for 736 frames... +[2023-03-07 15:49:08,119][384552] Decorrelating experience for 832 frames... +[2023-03-07 15:49:08,133][382443] Stopping RolloutWorker_w77... +[2023-03-07 15:49:08,133][382443] Loop rollout_proc77_evt_loop terminating... +[2023-03-07 15:49:08,133][236315] Component RolloutWorker_w77 stopped! +[2023-03-07 15:49:08,154][382457] Decorrelating experience for 992 frames... +[2023-03-07 15:49:08,206][384886] Decorrelating experience for 352 frames... +[2023-03-07 15:49:08,212][382417] Decorrelating experience for 928 frames... +[2023-03-07 15:49:08,240][382450] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,272][383955] Decorrelating experience for 928 frames... +[2023-03-07 15:49:08,276][382416] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,303][382440] Decorrelating experience for 800 frames... +[2023-03-07 15:49:08,326][382412] Decorrelating experience for 704 frames... +[2023-03-07 15:49:08,333][383951] Decorrelating experience for 640 frames... +[2023-03-07 15:49:08,337][382396] Decorrelating experience for 800 frames... +[2023-03-07 15:49:08,354][382399] Decorrelating experience for 480 frames... +[2023-03-07 15:49:08,403][382424] Decorrelating experience for 576 frames... +[2023-03-07 15:49:08,424][383678] Decorrelating experience for 736 frames... +[2023-03-07 15:49:08,492][382422] Decorrelating experience for 896 frames... +[2023-03-07 15:49:08,493][383639] Decorrelating experience for 928 frames... +[2023-03-07 15:49:08,493][382328] Decorrelating experience for 704 frames... +[2023-03-07 15:49:08,498][382393] Decorrelating experience for 544 frames... +[2023-03-07 15:49:08,533][382434] Decorrelating experience for 704 frames... +[2023-03-07 15:49:08,556][382437] Decorrelating experience for 736 frames... +[2023-03-07 15:49:08,573][382448] Decorrelating experience for 512 frames... +[2023-03-07 15:49:08,618][382421] Decorrelating experience for 832 frames... +[2023-03-07 15:49:08,632][384886] Decorrelating experience for 384 frames... +[2023-03-07 15:49:08,660][382457] Stopping RolloutWorker_w69... +[2023-03-07 15:49:08,660][382457] Loop rollout_proc69_evt_loop terminating... +[2023-03-07 15:49:08,660][236315] Component RolloutWorker_w69 stopped! +[2023-03-07 15:49:08,705][382439] Decorrelating experience for 480 frames... +[2023-03-07 15:49:08,712][382409] Decorrelating experience for 832 frames... +[2023-03-07 15:49:08,713][384644] Decorrelating experience for 800 frames... +[2023-03-07 15:49:08,722][384368] Stopping RolloutWorker_w99... +[2023-03-07 15:49:08,723][384368] Loop rollout_proc99_evt_loop terminating... +[2023-03-07 15:49:08,723][236315] Component RolloutWorker_w99 stopped! +[2023-03-07 15:49:08,731][382845] Decorrelating experience for 608 frames... +[2023-03-07 15:49:08,735][382456] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,784][385031] Decorrelating experience for 896 frames... +[2023-03-07 15:49:08,797][382399] Decorrelating experience for 512 frames... +[2023-03-07 15:49:08,816][382425] Decorrelating experience for 928 frames... +[2023-03-07 15:49:08,834][383955] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,846][382322] Decorrelating experience for 992 frames... +[2023-03-07 15:49:08,909][382326] Decorrelating experience for 544 frames... +[2023-03-07 15:49:08,946][382449] Decorrelating experience for 896 frames... +[2023-03-07 15:49:08,950][382417] Decorrelating experience for 960 frames... +[2023-03-07 15:49:08,961][382444] Decorrelating experience for 896 frames... +[2023-03-07 15:49:08,961][383292] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,006][382435] Decorrelating experience for 736 frames... +[2023-03-07 15:49:09,045][382434] Decorrelating experience for 736 frames... +[2023-03-07 15:49:09,049][385094] Decorrelating experience for 832 frames... +[2023-03-07 15:49:09,150][382841] Decorrelating experience for 896 frames... +[2023-03-07 15:49:09,158][382393] Decorrelating experience for 576 frames... +[2023-03-07 15:49:09,160][383102] Decorrelating experience for 608 frames... +[2023-03-07 15:49:09,203][383678] Decorrelating experience for 768 frames... +[2023-03-07 15:49:09,212][382402] Decorrelating experience for 480 frames... +[2023-03-07 15:49:09,213][382582] Decorrelating experience for 896 frames... +[2023-03-07 15:49:09,245][382325] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,267][382845] Decorrelating experience for 640 frames... +[2023-03-07 15:49:09,284][382456] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,321][382322] Stopping RolloutWorker_w2... +[2023-03-07 15:49:09,321][382322] Loop rollout_proc2_evt_loop terminating... +[2023-03-07 15:49:09,321][236315] Component RolloutWorker_w2 stopped! +[2023-03-07 15:49:09,343][382418] Decorrelating experience for 928 frames... +[2023-03-07 15:49:09,352][382450] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,417][383527] Decorrelating experience for 480 frames... +[2023-03-07 15:49:09,422][382419] Decorrelating experience for 672 frames... +[2023-03-07 15:49:09,435][383292] Stopping RolloutWorker_w82... +[2023-03-07 15:49:09,435][383292] Loop rollout_proc82_evt_loop terminating... +[2023-03-07 15:49:09,435][236315] Component RolloutWorker_w82 stopped! +[2023-03-07 15:49:09,443][384315] Decorrelating experience for 576 frames... +[2023-03-07 15:49:09,444][382843] Decorrelating experience for 928 frames... +[2023-03-07 15:49:09,445][385049] Decorrelating experience for 864 frames... +[2023-03-07 15:49:09,474][382409] Decorrelating experience for 864 frames... +[2023-03-07 15:49:09,503][382417] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,620][382436] Decorrelating experience for 768 frames... +[2023-03-07 15:49:09,627][382434] Decorrelating experience for 768 frames... +[2023-03-07 15:49:09,629][382424] Decorrelating experience for 608 frames... +[2023-03-07 15:49:09,662][382321] Decorrelating experience for 928 frames... +[2023-03-07 15:49:09,663][382449] Decorrelating experience for 928 frames... +[2023-03-07 15:49:09,663][383831] Decorrelating experience for 864 frames... +[2023-03-07 15:49:09,683][384643] Decorrelating experience for 992 frames... +[2023-03-07 15:49:09,684][385031] Decorrelating experience for 928 frames... +[2023-03-07 15:49:09,722][385027] Decorrelating experience for 800 frames... +[2023-03-07 15:49:09,728][382325] Stopping RolloutWorker_w17... +[2023-03-07 15:49:09,729][382325] Loop rollout_proc17_evt_loop terminating... +[2023-03-07 15:49:09,728][236315] Component RolloutWorker_w17 stopped! +[2023-03-07 15:49:09,752][382456] Stopping RolloutWorker_w72... +[2023-03-07 15:49:09,752][382456] Loop rollout_proc72_evt_loop terminating... +[2023-03-07 15:49:09,752][236315] Component RolloutWorker_w72 stopped! +[2023-03-07 15:49:09,783][383678] Decorrelating experience for 800 frames... +[2023-03-07 15:49:09,819][382328] Decorrelating experience for 736 frames... +[2023-03-07 15:49:09,853][383102] Decorrelating experience for 640 frames... +[2023-03-07 15:49:09,865][382326] Decorrelating experience for 576 frames... +[2023-03-07 15:49:09,875][382407] Decorrelating experience for 864 frames... +[2023-03-07 15:49:09,876][382845] Decorrelating experience for 672 frames... +[2023-03-07 15:49:09,892][382435] Decorrelating experience for 768 frames... +[2023-03-07 15:49:09,897][382406] Decorrelating experience for 768 frames... +[2023-03-07 15:49:09,902][384552] Decorrelating experience for 864 frames... +[2023-03-07 15:49:09,930][382840] Decorrelating experience for 832 frames... +[2023-03-07 15:49:09,967][382450] Stopping RolloutWorker_w48... +[2023-03-07 15:49:09,967][382450] Loop rollout_proc48_evt_loop terminating... +[2023-03-07 15:49:09,966][236315] Component RolloutWorker_w48 stopped! +[2023-03-07 15:49:09,985][382417] Stopping RolloutWorker_w7... +[2023-03-07 15:49:09,986][382417] Loop rollout_proc7_evt_loop terminating... +[2023-03-07 15:49:09,987][236315] Component RolloutWorker_w7 stopped! +[2023-03-07 15:49:10,000][382843] Decorrelating experience for 960 frames... +[2023-03-07 15:49:10,028][385049] Decorrelating experience for 896 frames... +[2023-03-07 15:49:10,083][382448] Decorrelating experience for 544 frames... +[2023-03-07 15:49:10,104][382841] Decorrelating experience for 928 frames... +[2023-03-07 15:49:10,112][382399] Decorrelating experience for 544 frames... +[2023-03-07 15:49:10,112][382409] Decorrelating experience for 896 frames... +[2023-03-07 15:49:10,130][385094] Decorrelating experience for 864 frames... +[2023-03-07 15:49:10,130][382586] Decorrelating experience for 992 frames... +[2023-03-07 15:49:10,134][384388] Decorrelating experience for 352 frames... +[2023-03-07 15:49:10,172][382402] Decorrelating experience for 512 frames... +[2023-03-07 15:49:10,196][383527] Decorrelating experience for 512 frames... +[2023-03-07 15:49:10,226][382419] Decorrelating experience for 704 frames... +[2023-03-07 15:49:10,280][384643] Stopping RolloutWorker_w117... +[2023-03-07 15:49:10,280][384643] Loop rollout_proc117_evt_loop terminating... +[2023-03-07 15:49:10,281][384761] Decorrelating experience for 864 frames... +[2023-03-07 15:49:10,281][236315] Component RolloutWorker_w117 stopped! +[2023-03-07 15:49:10,308][390548] Decorrelating experience for 672 frames... +[2023-03-07 15:49:10,339][382446] Decorrelating experience for 928 frames... +[2023-03-07 15:49:10,346][382845] Decorrelating experience for 704 frames... +[2023-03-07 15:49:10,350][384886] Decorrelating experience for 416 frames... +[2023-03-07 15:49:10,362][385027] Decorrelating experience for 832 frames... +[2023-03-07 15:49:10,370][382321] Decorrelating experience for 960 frames... +[2023-03-07 15:49:10,403][385096] Decorrelating experience for 800 frames... +[2023-03-07 15:49:10,406][382328] Decorrelating experience for 768 frames... +[2023-03-07 15:49:10,454][383678] Decorrelating experience for 832 frames... +[2023-03-07 15:49:10,472][385026] Decorrelating experience for 512 frames... +[2023-03-07 15:49:10,508][384387] Decorrelating experience for 672 frames... +[2023-03-07 15:49:10,543][382435] Decorrelating experience for 800 frames... +[2023-03-07 15:49:10,564][382427] Decorrelating experience for 672 frames... +[2023-03-07 15:49:10,594][382421] Decorrelating experience for 864 frames... +[2023-03-07 15:49:10,595][382423] Decorrelating experience for 704 frames... +[2023-03-07 15:49:10,599][382586] Stopping RolloutWorker_w58... +[2023-03-07 15:49:10,599][382586] Loop rollout_proc58_evt_loop terminating... +[2023-03-07 15:49:10,599][236315] Component RolloutWorker_w58 stopped! +[2023-03-07 15:49:10,602][385031] Decorrelating experience for 960 frames... +[2023-03-07 15:49:10,615][382402] Decorrelating experience for 544 frames... +[2023-03-07 15:49:10,622][382401] Decorrelating experience for 640 frames... +[2023-03-07 15:49:10,644][382436] Decorrelating experience for 800 frames... +[2023-03-07 15:49:10,675][385049] Decorrelating experience for 928 frames... +[2023-03-07 15:49:10,704][382974] Decorrelating experience for 704 frames... +[2023-03-07 15:49:10,753][384886] Decorrelating experience for 448 frames... +[2023-03-07 15:49:10,759][382409] Decorrelating experience for 928 frames... +[2023-03-07 15:49:10,787][382419] Decorrelating experience for 736 frames... +[2023-03-07 15:49:10,804][382407] Decorrelating experience for 896 frames... +[2023-03-07 15:49:10,827][382843] Decorrelating experience for 992 frames... +[2023-03-07 15:49:10,851][383951] Decorrelating experience for 672 frames... +[2023-03-07 15:49:10,857][382416] Decorrelating experience for 992 frames... +[2023-03-07 15:49:10,859][382448] Decorrelating experience for 576 frames... +[2023-03-07 15:49:10,876][382324] Decorrelating experience for 768 frames... +[2023-03-07 15:49:10,894][382847] Decorrelating experience for 768 frames... +[2023-03-07 15:49:10,946][382328] Decorrelating experience for 800 frames... +[2023-03-07 15:49:10,988][383744] Decorrelating experience for 352 frames... +[2023-03-07 15:49:11,029][384387] Decorrelating experience for 704 frames... +[2023-03-07 15:49:11,062][384315] Decorrelating experience for 608 frames... +[2023-03-07 15:49:11,062][383639] Decorrelating experience for 960 frames... +[2023-03-07 15:49:11,065][390548] Decorrelating experience for 704 frames... +[2023-03-07 15:49:11,079][385027] Decorrelating experience for 864 frames... +[2023-03-07 15:49:11,104][382838] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,114][385096] Decorrelating experience for 832 frames... +[2023-03-07 15:49:11,136][382393] Decorrelating experience for 608 frames... +[2023-03-07 15:49:11,193][383527] Decorrelating experience for 544 frames... +[2023-03-07 15:49:11,201][382427] Decorrelating experience for 704 frames... +[2023-03-07 15:49:11,248][384552] Decorrelating experience for 896 frames... +[2023-03-07 15:49:11,261][382444] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,261][382424] Decorrelating experience for 640 frames... +[2023-03-07 15:49:11,285][382395] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,313][382416] Stopping RolloutWorker_w45... +[2023-03-07 15:49:11,313][382416] Loop rollout_proc45_evt_loop terminating... +[2023-03-07 15:49:11,313][236315] Component RolloutWorker_w45 stopped! +[2023-03-07 15:49:11,322][384886] Decorrelating experience for 480 frames... +[2023-03-07 15:49:11,334][382407] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,381][382415] Decorrelating experience for 992 frames... +[2023-03-07 15:49:11,396][382429] Decorrelating experience for 736 frames... +[2023-03-07 15:49:11,405][382409] Decorrelating experience for 960 frames... +[2023-03-07 15:49:11,443][382401] Decorrelating experience for 672 frames... +[2023-03-07 15:49:11,444][382843] Stopping RolloutWorker_w90... +[2023-03-07 15:49:11,444][382843] Loop rollout_proc90_evt_loop terminating... +[2023-03-07 15:49:11,444][236315] Component RolloutWorker_w90 stopped! +[2023-03-07 15:49:11,475][384388] Decorrelating experience for 384 frames... +[2023-03-07 15:49:11,491][382406] Decorrelating experience for 800 frames... +[2023-03-07 15:49:11,504][382442] Decorrelating experience for 864 frames... +[2023-03-07 15:49:11,526][382841] Decorrelating experience for 960 frames... +[2023-03-07 15:49:11,531][390548] Decorrelating experience for 736 frames... +[2023-03-07 15:49:11,536][382403] Decorrelating experience for 800 frames... +[2023-03-07 15:49:11,584][382440] Decorrelating experience for 832 frames... +[2023-03-07 15:49:11,601][383527] Decorrelating experience for 576 frames... +[2023-03-07 15:49:11,610][385094] Decorrelating experience for 896 frames... +[2023-03-07 15:49:11,666][385026] Decorrelating experience for 544 frames... +[2023-03-07 15:49:11,689][382455] Decorrelating experience for 800 frames... +[2023-03-07 15:49:11,693][382328] Decorrelating experience for 832 frames... +[2023-03-07 15:49:11,707][382446] Decorrelating experience for 960 frames... +[2023-03-07 15:49:11,725][382399] Decorrelating experience for 576 frames... +[2023-03-07 15:49:11,734][383639] Decorrelating experience for 992 frames... +[2023-03-07 15:49:11,740][382451] Decorrelating experience for 896 frames... +[2023-03-07 15:49:11,777][384962] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,806][382424] Decorrelating experience for 672 frames... +[2023-03-07 15:49:11,813][382434] Decorrelating experience for 800 frames... +[2023-03-07 15:49:11,889][384552] Decorrelating experience for 928 frames... +[2023-03-07 15:49:11,894][385096] Decorrelating experience for 864 frames... +[2023-03-07 15:49:11,898][382326] Decorrelating experience for 608 frames... +[2023-03-07 15:49:11,935][385049] Decorrelating experience for 960 frames... +[2023-03-07 15:49:11,941][384315] Decorrelating experience for 640 frames... +[2023-03-07 15:49:11,959][382393] Decorrelating experience for 640 frames... +[2023-03-07 15:49:11,968][383955] Decorrelating experience for 992 frames... +[2023-03-07 15:49:11,975][382415] Stopping RolloutWorker_w42... +[2023-03-07 15:49:11,975][382415] Loop rollout_proc42_evt_loop terminating... +[2023-03-07 15:49:11,975][236315] Component RolloutWorker_w42 stopped! +[2023-03-07 15:49:12,000][382406] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,029][382429] Decorrelating experience for 768 frames... +[2023-03-07 15:49:12,066][384886] Decorrelating experience for 512 frames... +[2023-03-07 15:49:12,090][382847] Decorrelating experience for 800 frames... +[2023-03-07 15:49:12,098][382454] Decorrelating experience for 736 frames... +[2023-03-07 15:49:12,139][382418] Decorrelating experience for 960 frames... +[2023-03-07 15:49:12,144][382407] Decorrelating experience for 960 frames... +[2023-03-07 15:49:12,148][382422] Decorrelating experience for 928 frames... +[2023-03-07 15:49:12,163][382324] Decorrelating experience for 800 frames... +[2023-03-07 15:49:12,224][385026] Decorrelating experience for 576 frames... +[2023-03-07 15:49:12,227][383639] Stopping RolloutWorker_w104... +[2023-03-07 15:49:12,228][383639] Loop rollout_proc104_evt_loop terminating... +[2023-03-07 15:49:12,228][236315] Component RolloutWorker_w104 stopped! +[2023-03-07 15:49:12,239][382423] Decorrelating experience for 736 frames... +[2023-03-07 15:49:12,269][382409] Decorrelating experience for 992 frames... +[2023-03-07 15:49:12,316][384387] Decorrelating experience for 736 frames... +[2023-03-07 15:49:12,353][382841] Decorrelating experience for 992 frames... +[2023-03-07 15:49:12,354][384158] Decorrelating experience for 480 frames... +[2023-03-07 15:49:12,378][382424] Decorrelating experience for 704 frames... +[2023-03-07 15:49:12,379][382439] Decorrelating experience for 512 frames... +[2023-03-07 15:49:12,383][382403] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,432][383955] Stopping RolloutWorker_w96... +[2023-03-07 15:49:12,432][383955] Loop rollout_proc96_evt_loop terminating... +[2023-03-07 15:49:12,433][236315] Component RolloutWorker_w96 stopped! +[2023-03-07 15:49:12,441][382326] Decorrelating experience for 640 frames... +[2023-03-07 15:49:12,445][382455] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,510][383951] Decorrelating experience for 704 frames... +[2023-03-07 15:49:12,516][382974] Decorrelating experience for 736 frames... +[2023-03-07 15:49:12,577][382427] Decorrelating experience for 736 frames... +[2023-03-07 15:49:12,616][382399] Decorrelating experience for 608 frames... +[2023-03-07 15:49:12,624][382435] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,638][384503] Decorrelating experience for 960 frames... +[2023-03-07 15:49:12,651][385094] Decorrelating experience for 928 frames... +[2023-03-07 15:49:12,655][385049] Decorrelating experience for 992 frames... +[2023-03-07 15:49:12,661][382324] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,698][382418] Decorrelating experience for 992 frames... +[2023-03-07 15:49:12,704][382328] Decorrelating experience for 864 frames... +[2023-03-07 15:49:12,721][382409] Stopping RolloutWorker_w27... +[2023-03-07 15:49:12,721][382409] Loop rollout_proc27_evt_loop terminating... +[2023-03-07 15:49:12,721][236315] Component RolloutWorker_w27 stopped! +[2023-03-07 15:49:12,732][390548] Decorrelating experience for 768 frames... +[2023-03-07 15:49:12,771][382408] Decorrelating experience for 576 frames... +[2023-03-07 15:49:12,802][382841] Stopping RolloutWorker_w55... +[2023-03-07 15:49:12,803][382841] Loop rollout_proc55_evt_loop terminating... +[2023-03-07 15:49:12,803][236315] Component RolloutWorker_w55 stopped! +[2023-03-07 15:49:12,810][382436] Decorrelating experience for 832 frames... +[2023-03-07 15:49:12,820][383527] Decorrelating experience for 608 frames... +[2023-03-07 15:49:12,829][382407] Decorrelating experience for 992 frames... +[2023-03-07 15:49:12,859][384886] Decorrelating experience for 544 frames... +[2023-03-07 15:49:12,861][382393] Decorrelating experience for 672 frames... +[2023-03-07 15:49:12,927][382429] Decorrelating experience for 800 frames... +[2023-03-07 15:49:12,934][382446] Decorrelating experience for 992 frames... +[2023-03-07 15:49:13,005][383831] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,015][382439] Decorrelating experience for 544 frames... +[2023-03-07 15:49:13,027][383293] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,042][384114] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,046][382452] Decorrelating experience for 960 frames... +[2023-03-07 15:49:13,054][382455] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,070][382434] Decorrelating experience for 832 frames... +[2023-03-07 15:49:13,118][382974] Decorrelating experience for 768 frames... +[2023-03-07 15:49:13,139][382454] Decorrelating experience for 768 frames... +[2023-03-07 15:49:13,140][382421] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,159][382418] Stopping RolloutWorker_w13... +[2023-03-07 15:49:13,159][382418] Loop rollout_proc13_evt_loop terminating... +[2023-03-07 15:49:13,159][236315] Component RolloutWorker_w13 stopped! +[2023-03-07 15:49:13,204][382410] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,234][382842] Decorrelating experience for 416 frames... +[2023-03-07 15:49:13,244][382424] Decorrelating experience for 736 frames... +[2023-03-07 15:49:13,245][385049] Stopping RolloutWorker_w115... +[2023-03-07 15:49:13,245][385049] Loop rollout_proc115_evt_loop terminating... +[2023-03-07 15:49:13,245][236315] Component RolloutWorker_w115 stopped! +[2023-03-07 15:49:13,258][383527] Decorrelating experience for 640 frames... +[2023-03-07 15:49:13,259][385094] Decorrelating experience for 960 frames... +[2023-03-07 15:49:13,266][384886] Decorrelating experience for 576 frames... +[2023-03-07 15:49:13,279][382432] Decorrelating experience for 768 frames... +[2023-03-07 15:49:13,296][382407] Stopping RolloutWorker_w21... +[2023-03-07 15:49:13,296][382407] Loop rollout_proc21_evt_loop terminating... +[2023-03-07 15:49:13,296][236315] Component RolloutWorker_w21 stopped! +[2023-03-07 15:49:13,341][384315] Decorrelating experience for 672 frames... +[2023-03-07 15:49:13,344][382412] Decorrelating experience for 736 frames... +[2023-03-07 15:49:13,416][382397] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,458][383102] Decorrelating experience for 672 frames... +[2023-03-07 15:49:13,474][382393] Decorrelating experience for 704 frames... +[2023-03-07 15:49:13,481][382406] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,482][382845] Decorrelating experience for 736 frames... +[2023-03-07 15:49:13,489][382429] Decorrelating experience for 832 frames... +[2023-03-07 15:49:13,501][382446] Stopping RolloutWorker_w89... +[2023-03-07 15:49:13,501][382446] Loop rollout_proc89_evt_loop terminating... +[2023-03-07 15:49:13,501][236315] Component RolloutWorker_w89 stopped! +[2023-03-07 15:49:13,505][382426] Decorrelating experience for 992 frames... +[2023-03-07 15:49:13,544][383831] Decorrelating experience for 928 frames... +[2023-03-07 15:49:13,570][382399] Decorrelating experience for 640 frames... +[2023-03-07 15:49:13,609][382437] Decorrelating experience for 768 frames... +[2023-03-07 15:49:13,619][385031] Decorrelating experience for 992 frames... +[2023-03-07 15:49:13,647][382847] Decorrelating experience for 832 frames... +[2023-03-07 15:49:13,679][385096] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,679][384761] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,708][383293] Decorrelating experience for 896 frames... +[2023-03-07 15:49:13,709][390548] Decorrelating experience for 800 frames... +[2023-03-07 15:49:13,752][382449] Decorrelating experience for 960 frames... +[2023-03-07 15:49:13,771][382432] Decorrelating experience for 800 frames... +[2023-03-07 15:49:13,791][383951] Decorrelating experience for 736 frames... +[2023-03-07 15:49:13,818][382423] Decorrelating experience for 768 frames... +[2023-03-07 15:49:13,839][384315] Decorrelating experience for 704 frames... +[2023-03-07 15:49:13,887][382439] Decorrelating experience for 576 frames... +[2023-03-07 15:49:13,929][382421] Decorrelating experience for 928 frames... +[2023-03-07 15:49:13,941][382396] Decorrelating experience for 832 frames... +[2023-03-07 15:49:13,950][382414] Decorrelating experience for 736 frames... +[2023-03-07 15:49:13,962][382403] Decorrelating experience for 864 frames... +[2023-03-07 15:49:13,982][382454] Decorrelating experience for 800 frames... +[2023-03-07 15:49:13,991][385094] Decorrelating experience for 992 frames... +[2023-03-07 15:49:14,015][382426] Stopping RolloutWorker_w25... +[2023-03-07 15:49:14,015][382426] Loop rollout_proc25_evt_loop terminating... +[2023-03-07 15:49:14,015][236315] Component RolloutWorker_w25 stopped! +[2023-03-07 15:49:14,047][382842] Decorrelating experience for 448 frames... +[2023-03-07 15:49:14,047][382435] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,082][382411] Decorrelating experience for 704 frames... +[2023-03-07 15:49:14,128][385031] Stopping RolloutWorker_w100... +[2023-03-07 15:49:14,129][385031] Loop rollout_proc100_evt_loop terminating... +[2023-03-07 15:49:14,129][236315] Component RolloutWorker_w100 stopped! +[2023-03-07 15:49:14,138][382974] Decorrelating experience for 800 frames... +[2023-03-07 15:49:14,141][384886] Decorrelating experience for 608 frames... +[2023-03-07 15:49:14,152][382437] Decorrelating experience for 800 frames... +[2023-03-07 15:49:14,168][383102] Decorrelating experience for 704 frames... +[2023-03-07 15:49:14,181][382424] Decorrelating experience for 768 frames... +[2023-03-07 15:49:14,199][382429] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,258][383831] Decorrelating experience for 960 frames... +[2023-03-07 15:49:14,264][383951] Decorrelating experience for 768 frames... +[2023-03-07 15:49:14,285][382436] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,289][382408] Decorrelating experience for 608 frames... +[2023-03-07 15:49:14,342][384114] Decorrelating experience for 928 frames... +[2023-03-07 15:49:14,384][382452] Decorrelating experience for 992 frames... +[2023-03-07 15:49:14,387][390548] Decorrelating experience for 832 frames... +[2023-03-07 15:49:14,394][383293] Decorrelating experience for 928 frames... +[2023-03-07 15:49:14,407][383550] Decorrelating experience for 640 frames... +[2023-03-07 15:49:14,433][382399] Decorrelating experience for 672 frames... +[2023-03-07 15:49:14,476][382582] Decorrelating experience for 928 frames... +[2023-03-07 15:49:14,483][382454] Decorrelating experience for 832 frames... +[2023-03-07 15:49:14,496][384051] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,522][382847] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,561][385094] Stopping RolloutWorker_w121... +[2023-03-07 15:49:14,561][385094] Loop rollout_proc121_evt_loop terminating... +[2023-03-07 15:49:14,562][236315] Component RolloutWorker_w121 stopped! +[2023-03-07 15:49:14,575][382421] Decorrelating experience for 960 frames... +[2023-03-07 15:49:14,591][382412] Decorrelating experience for 768 frames... +[2023-03-07 15:49:14,600][382403] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,610][384552] Decorrelating experience for 960 frames... +[2023-03-07 15:49:14,633][384157] Decorrelating experience for 832 frames... +[2023-03-07 15:49:14,641][385096] Decorrelating experience for 928 frames... +[2023-03-07 15:49:14,668][382432] Decorrelating experience for 832 frames... +[2023-03-07 15:49:14,707][382844] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,709][382414] Decorrelating experience for 768 frames... +[2023-03-07 15:49:14,783][383102] Decorrelating experience for 736 frames... +[2023-03-07 15:49:14,789][382448] Decorrelating experience for 608 frames... +[2023-03-07 15:49:14,811][384644] Decorrelating experience for 832 frames... +[2023-03-07 15:49:14,811][382451] Decorrelating experience for 928 frames... +[2023-03-07 15:49:14,815][382396] Decorrelating experience for 864 frames... +[2023-03-07 15:49:14,846][382452] Stopping RolloutWorker_w51... +[2023-03-07 15:49:14,847][382452] Loop rollout_proc51_evt_loop terminating... +[2023-03-07 15:49:14,847][236315] Component RolloutWorker_w51 stopped! +[2023-03-07 15:49:14,858][382444] Decorrelating experience for 960 frames... +[2023-03-07 15:49:14,880][382408] Decorrelating experience for 640 frames... +[2023-03-07 15:49:14,904][382442] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,915][382410] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,923][382429] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,983][382435] Decorrelating experience for 896 frames... +[2023-03-07 15:49:14,996][390548] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,011][382395] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,021][382399] Decorrelating experience for 704 frames... +[2023-03-07 15:49:15,048][383293] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,088][382838] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,106][382397] Decorrelating experience for 896 frames... +[2023-03-07 15:49:15,117][384315] Decorrelating experience for 736 frames... +[2023-03-07 15:49:15,130][382411] Decorrelating experience for 736 frames... +[2023-03-07 15:49:15,188][382432] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,191][385027] Decorrelating experience for 896 frames... +[2023-03-07 15:49:15,208][383984] Decorrelating experience for 896 frames... +[2023-03-07 15:49:15,222][383831] Decorrelating experience for 992 frames... +[2023-03-07 15:49:15,282][382412] Decorrelating experience for 800 frames... +[2023-03-07 15:49:15,327][382847] Decorrelating experience for 896 frames... +[2023-03-07 15:49:15,327][382582] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,352][382454] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,364][382396] Decorrelating experience for 896 frames... +[2023-03-07 15:49:15,368][385096] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,399][382414] Decorrelating experience for 800 frames... +[2023-03-07 15:49:15,406][382441] Decorrelating experience for 768 frames... +[2023-03-07 15:49:15,437][382439] Decorrelating experience for 608 frames... +[2023-03-07 15:49:15,492][382403] Decorrelating experience for 928 frames... +[2023-03-07 15:49:15,514][382842] Decorrelating experience for 480 frames... +[2023-03-07 15:49:15,545][382448] Decorrelating experience for 640 frames... +[2023-03-07 15:49:15,551][384157] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,560][382451] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,575][382444] Decorrelating experience for 992 frames... +[2023-03-07 15:49:15,600][384962] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,605][382844] Decorrelating experience for 928 frames... +[2023-03-07 15:49:15,625][382840] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,627][382445] Decorrelating experience for 864 frames... +[2023-03-07 15:49:15,668][383831] Stopping RolloutWorker_w122... +[2023-03-07 15:49:15,669][383831] Loop rollout_proc122_evt_loop terminating... +[2023-03-07 15:49:15,669][236315] Component RolloutWorker_w122 stopped! +[2023-03-07 15:49:15,698][384114] Decorrelating experience for 960 frames... +[2023-03-07 15:49:15,755][384388] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,755][384388] EvtLoop [rollout_proc123_evt_loop, process=rollout_proc123] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,757][384388] Unhandled exception in evt loop rollout_proc123_evt_loop +[2023-03-07 15:49:15,761][382427] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,761][382844] EvtLoop [rollout_proc84_evt_loop, process=rollout_proc84] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,762][382844] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc84_evt_loop +[2023-03-07 15:49:15,763][382442] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,763][382845] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,762][382427] EvtLoop [rollout_proc34_evt_loop, process=rollout_proc34] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,763][382427] Unhandled exception in evt loop rollout_proc34_evt_loop +[2023-03-07 15:49:15,763][382845] EvtLoop [rollout_proc67_evt_loop, process=rollout_proc67] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,763][382442] EvtLoop [rollout_proc68_evt_loop, process=rollout_proc68] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,765][382845] Unhandled exception in evt loop rollout_proc67_evt_loop +[2023-03-07 15:49:15,765][382442] Unhandled exception in evt loop rollout_proc68_evt_loop +[2023-03-07 15:49:15,765][390548] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,766][382397] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,765][390548] EvtLoop [rollout_proc1_evt_loop, process=rollout_proc1] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,766][390548] Unhandled exception in evt loop rollout_proc1_evt_loop +[2023-03-07 15:49:15,766][382397] EvtLoop [rollout_proc38_evt_loop, process=rollout_proc38] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,767][382397] Unhandled exception in evt loop rollout_proc38_evt_loop +[2023-03-07 15:49:15,767][384552] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,768][384552] EvtLoop [rollout_proc94_evt_loop, process=rollout_proc94] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,769][384552] Unhandled exception in evt loop rollout_proc94_evt_loop +[2023-03-07 15:49:15,769][383293] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:15,768][382444] EvtLoop [rollout_proc83_evt_loop, process=rollout_proc83] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 117, in step + obs, info["reset_info"] = self.env.reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 346, in reset + self.game.new_episode() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,770][382444] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc83_evt_loop +[2023-03-07 15:49:15,769][383293] EvtLoop [rollout_proc95_evt_loop, process=rollout_proc95] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:15,771][383293] Unhandled exception in evt loop rollout_proc95_evt_loop +[2023-03-07 15:49:15,780][236315] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 236315], exiting... +[2023-03-07 15:49:15,785][236315] Runner profile tree view: +main_loop: 83.7042 +[2023-03-07 15:49:15,785][236315] Collected {0: 4018176}, FPS: 48.9 +[2023-03-07 15:49:15,786][384962] EvtLoop [rollout_proc109_evt_loop, process=rollout_proc109] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,787][384962] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc109_evt_loop +[2023-03-07 15:49:15,786][382451] EvtLoop [rollout_proc60_evt_loop, process=rollout_proc60] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,788][382451] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc60_evt_loop +[2023-03-07 15:49:15,787][384157] EvtLoop [rollout_proc105_evt_loop, process=rollout_proc105] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,788][384157] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc105_evt_loop +[2023-03-07 15:49:15,789][382445] EvtLoop [rollout_proc80_evt_loop, process=rollout_proc80] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,790][382445] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc80_evt_loop +[2023-03-07 15:49:15,789][384114] EvtLoop [rollout_proc125_evt_loop, process=rollout_proc125] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,790][384114] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc125_evt_loop +[2023-03-07 15:49:15,789][382448] EvtLoop [rollout_proc54_evt_loop, process=rollout_proc54] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,791][382448] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc54_evt_loop +[2023-03-07 15:49:15,790][382840] EvtLoop [rollout_proc52_evt_loop, process=rollout_proc52] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,791][382840] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc52_evt_loop +[2023-03-07 15:49:15,790][382403] EvtLoop [rollout_proc9_evt_loop, process=rollout_proc9] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,790][382847] EvtLoop [rollout_proc73_evt_loop, process=rollout_proc73] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,792][382403] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc9_evt_loop +[2023-03-07 15:49:15,792][382847] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc73_evt_loop +[2023-03-07 15:49:15,793][236315] Environment doom_basic already registered, overwriting... +[2023-03-07 15:49:15,794][236315] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 15:49:15,796][236315] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 15:49:15,796][236315] Environment doom_dm already registered, overwriting... +[2023-03-07 15:49:15,797][236315] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 15:49:15,798][236315] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 15:49:15,798][236315] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 15:49:15,799][236315] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 15:49:15,800][236315] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 15:49:15,800][236315] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 15:49:15,801][236315] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 15:49:15,801][236315] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 15:49:15,802][236315] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 15:49:15,802][236315] Environment doom_battle already registered, overwriting... +[2023-03-07 15:49:15,803][236315] Environment doom_battle2 already registered, overwriting... +[2023-03-07 15:49:15,803][236315] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 15:49:15,804][236315] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 15:49:15,804][236315] Environment doom_duel already registered, overwriting... +[2023-03-07 15:49:15,805][236315] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 15:49:15,806][236315] Environment doom_benchmark already registered, overwriting... +[2023-03-07 15:49:15,806][236315] register_encoder_factory: +[2023-03-07 15:49:15,822][236315] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:49:15,823][236315] Overriding arg 'train_for_env_steps' with value 8000000 passed from command line +[2023-03-07 15:49:15,828][236315] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/default_experiment already exists! +[2023-03-07 15:49:15,829][236315] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/default_experiment... +[2023-03-07 15:49:15,829][236315] Weights and Biases integration disabled +[2023-03-07 15:49:15,831][236315] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 15:49:15,849][385096] EvtLoop [rollout_proc118_evt_loop, process=rollout_proc118] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:15,850][385096] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc118_evt_loop +[2023-03-07 15:49:15,996][382434] Decorrelating experience for 864 frames... +[2023-03-07 15:49:16,023][384644] Decorrelating experience for 864 frames... +[2023-03-07 15:49:16,047][382582] Decorrelating experience for 992 frames... +[2023-03-07 15:49:16,050][382429] Decorrelating experience for 928 frames... +[2023-03-07 15:49:16,053][382435] Decorrelating experience for 928 frames... +[2023-03-07 15:49:16,057][383984] Decorrelating experience for 928 frames... +[2023-03-07 15:49:16,061][382401] Decorrelating experience for 704 frames... +[2023-03-07 15:49:16,062][382454] Decorrelating experience for 896 frames... +[2023-03-07 15:49:16,070][382321] Decorrelating experience for 992 frames... +[2023-03-07 15:49:16,266][382396] Decorrelating experience for 928 frames... +[2023-03-07 15:49:16,290][382399] Decorrelating experience for 736 frames... +[2023-03-07 15:49:16,296][382412] Decorrelating experience for 832 frames... +[2023-03-07 15:49:16,302][383799] Decorrelating experience for 864 frames... +[2023-03-07 15:49:16,312][383102] Decorrelating experience for 768 frames... +[2023-03-07 15:49:16,313][382421] Decorrelating experience for 992 frames... +[2023-03-07 15:49:16,373][382411] Decorrelating experience for 768 frames... +[2023-03-07 15:49:16,496][382328] Decorrelating experience for 896 frames... +[2023-03-07 15:49:16,536][382441] Decorrelating experience for 800 frames... +[2023-03-07 15:49:16,547][382414] Decorrelating experience for 832 frames... +[2023-03-07 15:49:16,563][382395] Decorrelating experience for 992 frames... +[2023-03-07 15:49:16,565][382455] Decorrelating experience for 896 frames... +[2023-03-07 15:49:16,586][384503] Decorrelating experience for 992 frames... +[2023-03-07 15:49:16,635][383744] Decorrelating experience for 384 frames... +[2023-03-07 15:49:16,659][382434] Decorrelating experience for 896 frames... +[2023-03-07 15:49:16,736][382429] Decorrelating experience for 960 frames... +[2023-03-07 15:49:16,784][382401] Decorrelating experience for 736 frames... +[2023-03-07 15:49:16,790][383550] Decorrelating experience for 672 frames... +[2023-03-07 15:49:16,826][382399] Decorrelating experience for 768 frames... +[2023-03-07 15:49:16,827][384315] Decorrelating experience for 768 frames... +[2023-03-07 15:49:16,827][384051] Decorrelating experience for 896 frames... +[2023-03-07 15:49:16,851][382524] Decorrelating experience for 832 frames... +[2023-03-07 15:49:16,867][384158] Decorrelating experience for 512 frames... +[2023-03-07 15:49:16,897][383102] Decorrelating experience for 800 frames... +[2023-03-07 15:49:17,022][383744] Decorrelating experience for 416 frames... +[2023-03-07 15:49:17,029][382842] Decorrelating experience for 512 frames... +[2023-03-07 15:49:17,051][382410] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,056][382838] Decorrelating experience for 992 frames... +[2023-03-07 15:49:17,079][382432] Decorrelating experience for 896 frames... +[2023-03-07 15:49:17,083][383984] Decorrelating experience for 960 frames... +[2023-03-07 15:49:17,098][382440] Decorrelating experience for 864 frames... +[2023-03-07 15:49:17,113][384886] Decorrelating experience for 640 frames... +[2023-03-07 15:49:17,143][383799] Decorrelating experience for 896 frames... +[2023-03-07 15:49:17,150][236315] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=default_experiment +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=8000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=120 --num_envs_per_worker=4 --train_for_env_steps=4000000 +cli_args={'env': 'doom_health_gathering_supreme', 'num_workers': 120, 'num_envs_per_worker': 4, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 15:49:17,151][236315] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json... +[2023-03-07 15:49:17,153][236315] Rollout worker 0 uses device cpu +[2023-03-07 15:49:17,153][236315] Rollout worker 1 uses device cpu +[2023-03-07 15:49:17,154][236315] Rollout worker 2 uses device cpu +[2023-03-07 15:49:17,154][236315] Rollout worker 3 uses device cpu +[2023-03-07 15:49:17,155][236315] Rollout worker 4 uses device cpu +[2023-03-07 15:49:17,156][236315] Rollout worker 5 uses device cpu +[2023-03-07 15:49:17,156][236315] Rollout worker 6 uses device cpu +[2023-03-07 15:49:17,156][236315] Rollout worker 7 uses device cpu +[2023-03-07 15:49:17,157][236315] Rollout worker 8 uses device cpu +[2023-03-07 15:49:17,157][236315] Rollout worker 9 uses device cpu +[2023-03-07 15:49:17,157][236315] Rollout worker 10 uses device cpu +[2023-03-07 15:49:17,158][236315] Rollout worker 11 uses device cpu +[2023-03-07 15:49:17,158][236315] Rollout worker 12 uses device cpu +[2023-03-07 15:49:17,159][236315] Rollout worker 13 uses device cpu +[2023-03-07 15:49:17,160][236315] Rollout worker 14 uses device cpu +[2023-03-07 15:49:17,160][236315] Rollout worker 15 uses device cpu +[2023-03-07 15:49:17,161][236315] Rollout worker 16 uses device cpu +[2023-03-07 15:49:17,161][236315] Rollout worker 17 uses device cpu +[2023-03-07 15:49:17,161][236315] Rollout worker 18 uses device cpu +[2023-03-07 15:49:17,162][236315] Rollout worker 19 uses device cpu +[2023-03-07 15:49:17,163][236315] Rollout worker 20 uses device cpu +[2023-03-07 15:49:17,163][236315] Rollout worker 21 uses device cpu +[2023-03-07 15:49:17,164][236315] Rollout worker 22 uses device cpu +[2023-03-07 15:49:17,164][236315] Rollout worker 23 uses device cpu +[2023-03-07 15:49:17,165][236315] Rollout worker 24 uses device cpu +[2023-03-07 15:49:17,165][236315] Rollout worker 25 uses device cpu +[2023-03-07 15:49:17,165][236315] Rollout worker 26 uses device cpu +[2023-03-07 15:49:17,167][236315] Rollout worker 27 uses device cpu +[2023-03-07 15:49:17,167][236315] Rollout worker 28 uses device cpu +[2023-03-07 15:49:17,168][236315] Rollout worker 29 uses device cpu +[2023-03-07 15:49:17,168][236315] Rollout worker 30 uses device cpu +[2023-03-07 15:49:17,169][236315] Rollout worker 31 uses device cpu +[2023-03-07 15:49:17,169][236315] Rollout worker 32 uses device cpu +[2023-03-07 15:49:17,170][236315] Rollout worker 33 uses device cpu +[2023-03-07 15:49:17,171][236315] Rollout worker 34 uses device cpu +[2023-03-07 15:49:17,171][236315] Rollout worker 35 uses device cpu +[2023-03-07 15:49:17,172][236315] Rollout worker 36 uses device cpu +[2023-03-07 15:49:17,172][236315] Rollout worker 37 uses device cpu +[2023-03-07 15:49:17,173][236315] Rollout worker 38 uses device cpu +[2023-03-07 15:49:17,174][236315] Rollout worker 39 uses device cpu +[2023-03-07 15:49:17,174][236315] Rollout worker 40 uses device cpu +[2023-03-07 15:49:17,175][236315] Rollout worker 41 uses device cpu +[2023-03-07 15:49:17,175][236315] Rollout worker 42 uses device cpu +[2023-03-07 15:49:17,176][236315] Rollout worker 43 uses device cpu +[2023-03-07 15:49:17,176][236315] Rollout worker 44 uses device cpu +[2023-03-07 15:49:17,177][236315] Rollout worker 45 uses device cpu +[2023-03-07 15:49:17,178][236315] Rollout worker 46 uses device cpu +[2023-03-07 15:49:17,178][236315] Rollout worker 47 uses device cpu +[2023-03-07 15:49:17,179][236315] Rollout worker 48 uses device cpu +[2023-03-07 15:49:17,180][236315] Rollout worker 49 uses device cpu +[2023-03-07 15:49:17,180][236315] Rollout worker 50 uses device cpu +[2023-03-07 15:49:17,181][236315] Rollout worker 51 uses device cpu +[2023-03-07 15:49:17,181][236315] Rollout worker 52 uses device cpu +[2023-03-07 15:49:17,182][236315] Rollout worker 53 uses device cpu +[2023-03-07 15:49:17,183][236315] Rollout worker 54 uses device cpu +[2023-03-07 15:49:17,183][236315] Rollout worker 55 uses device cpu +[2023-03-07 15:49:17,184][236315] Rollout worker 56 uses device cpu +[2023-03-07 15:49:17,184][236315] Rollout worker 57 uses device cpu +[2023-03-07 15:49:17,185][236315] Rollout worker 58 uses device cpu +[2023-03-07 15:49:17,185][236315] Rollout worker 59 uses device cpu +[2023-03-07 15:49:17,186][236315] Rollout worker 60 uses device cpu +[2023-03-07 15:49:17,187][236315] Rollout worker 61 uses device cpu +[2023-03-07 15:49:17,187][236315] Rollout worker 62 uses device cpu +[2023-03-07 15:49:17,188][236315] Rollout worker 63 uses device cpu +[2023-03-07 15:49:17,188][236315] Rollout worker 64 uses device cpu +[2023-03-07 15:49:17,189][236315] Rollout worker 65 uses device cpu +[2023-03-07 15:49:17,189][236315] Rollout worker 66 uses device cpu +[2023-03-07 15:49:17,190][236315] Rollout worker 67 uses device cpu +[2023-03-07 15:49:17,191][236315] Rollout worker 68 uses device cpu +[2023-03-07 15:49:17,191][236315] Rollout worker 69 uses device cpu +[2023-03-07 15:49:17,192][236315] Rollout worker 70 uses device cpu +[2023-03-07 15:49:17,192][236315] Rollout worker 71 uses device cpu +[2023-03-07 15:49:17,193][236315] Rollout worker 72 uses device cpu +[2023-03-07 15:49:17,193][236315] Rollout worker 73 uses device cpu +[2023-03-07 15:49:17,194][236315] Rollout worker 74 uses device cpu +[2023-03-07 15:49:17,194][236315] Rollout worker 75 uses device cpu +[2023-03-07 15:49:17,195][236315] Rollout worker 76 uses device cpu +[2023-03-07 15:49:17,196][236315] Rollout worker 77 uses device cpu +[2023-03-07 15:49:17,196][236315] Rollout worker 78 uses device cpu +[2023-03-07 15:49:17,197][236315] Rollout worker 79 uses device cpu +[2023-03-07 15:49:17,197][236315] Rollout worker 80 uses device cpu +[2023-03-07 15:49:17,198][236315] Rollout worker 81 uses device cpu +[2023-03-07 15:49:17,198][236315] Rollout worker 82 uses device cpu +[2023-03-07 15:49:17,199][236315] Rollout worker 83 uses device cpu +[2023-03-07 15:49:17,199][236315] Rollout worker 84 uses device cpu +[2023-03-07 15:49:17,200][236315] Rollout worker 85 uses device cpu +[2023-03-07 15:49:17,200][236315] Rollout worker 86 uses device cpu +[2023-03-07 15:49:17,201][236315] Rollout worker 87 uses device cpu +[2023-03-07 15:49:17,202][236315] Rollout worker 88 uses device cpu +[2023-03-07 15:49:17,202][236315] Rollout worker 89 uses device cpu +[2023-03-07 15:49:17,203][236315] Rollout worker 90 uses device cpu +[2023-03-07 15:49:17,203][236315] Rollout worker 91 uses device cpu +[2023-03-07 15:49:17,203][236315] Rollout worker 92 uses device cpu +[2023-03-07 15:49:17,204][236315] Rollout worker 93 uses device cpu +[2023-03-07 15:49:17,204][236315] Rollout worker 94 uses device cpu +[2023-03-07 15:49:17,205][236315] Rollout worker 95 uses device cpu +[2023-03-07 15:49:17,205][236315] Rollout worker 96 uses device cpu +[2023-03-07 15:49:17,206][236315] Rollout worker 97 uses device cpu +[2023-03-07 15:49:17,207][236315] Rollout worker 98 uses device cpu +[2023-03-07 15:49:17,208][236315] Rollout worker 99 uses device cpu +[2023-03-07 15:49:17,208][236315] Rollout worker 100 uses device cpu +[2023-03-07 15:49:17,209][236315] Rollout worker 101 uses device cpu +[2023-03-07 15:49:17,210][236315] Rollout worker 102 uses device cpu +[2023-03-07 15:49:17,210][236315] Rollout worker 103 uses device cpu +[2023-03-07 15:49:17,211][236315] Rollout worker 104 uses device cpu +[2023-03-07 15:49:17,212][236315] Rollout worker 105 uses device cpu +[2023-03-07 15:49:17,212][236315] Rollout worker 106 uses device cpu +[2023-03-07 15:49:17,213][236315] Rollout worker 107 uses device cpu +[2023-03-07 15:49:17,213][236315] Rollout worker 108 uses device cpu +[2023-03-07 15:49:17,214][236315] Rollout worker 109 uses device cpu +[2023-03-07 15:49:17,214][236315] Rollout worker 110 uses device cpu +[2023-03-07 15:49:17,215][236315] Rollout worker 111 uses device cpu +[2023-03-07 15:49:17,215][236315] Rollout worker 112 uses device cpu +[2023-03-07 15:49:17,216][236315] Rollout worker 113 uses device cpu +[2023-03-07 15:49:17,216][236315] Rollout worker 114 uses device cpu +[2023-03-07 15:49:17,217][236315] Rollout worker 115 uses device cpu +[2023-03-07 15:49:17,218][236315] Rollout worker 116 uses device cpu +[2023-03-07 15:49:17,218][236315] Rollout worker 117 uses device cpu +[2023-03-07 15:49:17,218][236315] Rollout worker 118 uses device cpu +[2023-03-07 15:49:17,219][236315] Rollout worker 119 uses device cpu +[2023-03-07 15:49:17,219][236315] Rollout worker 120 uses device cpu +[2023-03-07 15:49:17,220][236315] Rollout worker 121 uses device cpu +[2023-03-07 15:49:17,220][236315] Rollout worker 122 uses device cpu +[2023-03-07 15:49:17,221][236315] Rollout worker 123 uses device cpu +[2023-03-07 15:49:17,221][236315] Rollout worker 124 uses device cpu +[2023-03-07 15:49:17,222][236315] Rollout worker 125 uses device cpu +[2023-03-07 15:49:17,222][236315] Rollout worker 126 uses device cpu +[2023-03-07 15:49:17,223][236315] Rollout worker 127 uses device cpu +[2023-03-07 15:49:17,255][382414] Decorrelating experience for 864 frames... +[2023-03-07 15:49:17,317][382401] Decorrelating experience for 768 frames... +[2023-03-07 15:49:17,336][382441] Decorrelating experience for 832 frames... +[2023-03-07 15:49:17,359][382424] Decorrelating experience for 800 frames... +[2023-03-07 15:49:17,360][382422] Decorrelating experience for 960 frames... +[2023-03-07 15:49:17,383][382412] Decorrelating experience for 864 frames... +[2023-03-07 15:49:17,441][382328] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,514][384315] Decorrelating experience for 800 frames... +[2023-03-07 15:49:17,596][384051] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,612][385027] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,630][382455] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,650][382439] Decorrelating experience for 640 frames... +[2023-03-07 15:49:17,734][382440] Decorrelating experience for 896 frames... +[2023-03-07 15:49:17,742][383744] Decorrelating experience for 448 frames... +[2023-03-07 15:49:17,766][382410] Decorrelating experience for 960 frames... +[2023-03-07 15:49:17,830][382399] Decorrelating experience for 800 frames... +[2023-03-07 15:49:17,844][383102] Decorrelating experience for 832 frames... +[2023-03-07 15:49:17,918][382414] Decorrelating experience for 896 frames... +[2023-03-07 15:49:17,924][384761] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,977][383984] Decorrelating experience for 992 frames... +[2023-03-07 15:49:17,987][382432] Decorrelating experience for 928 frames... +[2023-03-07 15:49:17,997][382422] Decorrelating experience for 992 frames... +[2023-03-07 15:49:18,074][382328] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,075][382406] Decorrelating experience for 896 frames... +[2023-03-07 15:49:18,090][382424] Decorrelating experience for 832 frames... +[2023-03-07 15:49:18,169][382437] Decorrelating experience for 832 frames... +[2023-03-07 15:49:18,173][383744] Decorrelating experience for 480 frames... +[2023-03-07 15:49:18,187][382401] Decorrelating experience for 800 frames... +[2023-03-07 15:49:18,224][382455] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,231][384051] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,246][383799] Decorrelating experience for 928 frames... +[2023-03-07 15:49:18,297][382439] Decorrelating experience for 672 frames... +[2023-03-07 15:49:18,299][382324] Decorrelating experience for 864 frames... +[2023-03-07 15:49:18,382][382399] Decorrelating experience for 832 frames... +[2023-03-07 15:49:18,425][384886] Decorrelating experience for 672 frames... +[2023-03-07 15:49:18,459][383102] Decorrelating experience for 864 frames... +[2023-03-07 15:49:18,479][382412] Decorrelating experience for 896 frames... +[2023-03-07 15:49:18,498][382440] Decorrelating experience for 928 frames... +[2023-03-07 15:49:18,563][382438] Decorrelating experience for 992 frames... +[2023-03-07 15:49:18,651][383744] Decorrelating experience for 512 frames... +[2023-03-07 15:49:18,666][382393] Decorrelating experience for 736 frames... +[2023-03-07 15:49:18,675][382432] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,682][382414] Decorrelating experience for 928 frames... +[2023-03-07 15:49:18,692][382424] Decorrelating experience for 864 frames... +[2023-03-07 15:49:18,710][382401] Decorrelating experience for 832 frames... +[2023-03-07 15:49:18,729][385027] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,793][382396] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,827][384503] Stopping RolloutWorker_w120... +[2023-03-07 15:49:18,829][384503] Loop rollout_proc120_evt_loop terminating... +[2023-03-07 15:49:18,852][384051] Decorrelating experience for 992 frames... +[2023-03-07 15:49:18,917][384158] Decorrelating experience for 544 frames... +[2023-03-07 15:49:18,918][382439] Decorrelating experience for 704 frames... +[2023-03-07 15:49:18,929][382455] Decorrelating experience for 992 frames... +[2023-03-07 15:49:18,995][382421] Stopping RolloutWorker_w22... +[2023-03-07 15:49:18,996][384761] Decorrelating experience for 960 frames... +[2023-03-07 15:49:18,997][382421] Loop rollout_proc22_evt_loop terminating... +[2023-03-07 15:49:18,997][382324] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,004][382441] Decorrelating experience for 864 frames... +[2023-03-07 15:49:19,014][382838] Stopping RolloutWorker_w81... +[2023-03-07 15:49:19,015][382838] Loop rollout_proc81_evt_loop terminating... +[2023-03-07 15:49:19,016][382582] Stopping RolloutWorker_w87... +[2023-03-07 15:49:19,016][382582] Loop rollout_proc87_evt_loop terminating... +[2023-03-07 15:49:19,047][383984] Stopping RolloutWorker_w116... +[2023-03-07 15:49:19,048][383984] Loop rollout_proc116_evt_loop terminating... +[2023-03-07 15:49:19,052][382422] Stopping RolloutWorker_w19... +[2023-03-07 15:49:19,053][382422] Loop rollout_proc19_evt_loop terminating... +[2023-03-07 15:49:19,058][382438] Stopping RolloutWorker_w62... +[2023-03-07 15:49:19,060][382395] Stopping RolloutWorker_w32... +[2023-03-07 15:49:19,060][382321] Stopping RolloutWorker_w5... +[2023-03-07 15:49:19,061][382395] Loop rollout_proc32_evt_loop terminating... +[2023-03-07 15:49:19,061][382321] Loop rollout_proc5_evt_loop terminating... +[2023-03-07 15:49:19,061][382438] Loop rollout_proc62_evt_loop terminating... +[2023-03-07 15:49:19,103][382412] Decorrelating experience for 928 frames... +[2023-03-07 15:49:19,134][382423] Decorrelating experience for 800 frames... +[2023-03-07 15:49:19,171][383102] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,185][382437] Decorrelating experience for 864 frames... +[2023-03-07 15:49:19,206][382436] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,221][382393] Decorrelating experience for 768 frames... +[2023-03-07 15:49:19,237][382406] Decorrelating experience for 928 frames... +[2023-03-07 15:49:19,256][382401] Decorrelating experience for 864 frames... +[2023-03-07 15:49:19,298][384051] Stopping RolloutWorker_w108... +[2023-03-07 15:49:19,298][384051] Loop rollout_proc108_evt_loop terminating... +[2023-03-07 15:49:19,337][384387] Decorrelating experience for 768 frames... +[2023-03-07 15:49:19,342][385027] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,357][382455] Stopping RolloutWorker_w75... +[2023-03-07 15:49:19,358][382455] Loop rollout_proc75_evt_loop terminating... +[2023-03-07 15:49:19,380][382328] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,399][382435] Decorrelating experience for 960 frames... +[2023-03-07 15:49:19,404][382399] Decorrelating experience for 864 frames... +[2023-03-07 15:49:19,411][384158] Decorrelating experience for 576 frames... +[2023-03-07 15:49:19,457][382439] Decorrelating experience for 736 frames... +[2023-03-07 15:49:19,543][382429] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,572][382324] Decorrelating experience for 928 frames... +[2023-03-07 15:49:19,630][384886] Decorrelating experience for 704 frames... +[2023-03-07 15:49:19,636][382414] Decorrelating experience for 960 frames... +[2023-03-07 15:49:19,640][382402] Decorrelating experience for 576 frames... +[2023-03-07 15:49:19,651][382441] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,659][382423] Decorrelating experience for 832 frames... +[2023-03-07 15:49:19,678][236315] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:19,679][236315] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:49:19,684][382412] Decorrelating experience for 960 frames... +[2023-03-07 15:49:19,743][383550] Decorrelating experience for 704 frames... +[2023-03-07 15:49:19,764][385027] Stopping RolloutWorker_w112... +[2023-03-07 15:49:19,764][385027] Loop rollout_proc112_evt_loop terminating... +[2023-03-07 15:49:19,784][382401] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,805][382396] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,827][384158] Decorrelating experience for 608 frames... +[2023-03-07 15:49:19,830][382328] Stopping RolloutWorker_w14... +[2023-03-07 15:49:19,830][382328] Loop rollout_proc14_evt_loop terminating... +[2023-03-07 15:49:19,871][382432] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,898][382842] Decorrelating experience for 544 frames... +[2023-03-07 15:49:19,928][383102] Decorrelating experience for 928 frames... +[2023-03-07 15:49:19,936][382429] Stopping RolloutWorker_w43... +[2023-03-07 15:49:19,937][382429] Loop rollout_proc43_evt_loop terminating... +[2023-03-07 15:49:19,943][384761] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,951][384644] Decorrelating experience for 896 frames... +[2023-03-07 15:49:19,958][382435] Decorrelating experience for 992 frames... +[2023-03-07 15:49:19,983][236315] Starting all processes... +[2023-03-07 15:49:19,984][236315] Starting process learner_proc0 +[2023-03-07 15:49:20,051][382436] Decorrelating experience for 928 frames... +[2023-03-07 15:49:20,069][382410] Decorrelating experience for 992 frames... +[2023-03-07 15:49:20,088][382524] Decorrelating experience for 864 frames... +[2023-03-07 15:49:20,106][382324] Decorrelating experience for 960 frames... +[2023-03-07 15:49:20,152][383527] Decorrelating experience for 672 frames... +[2023-03-07 15:49:20,186][382396] Stopping RolloutWorker_w35... +[2023-03-07 15:49:20,186][382396] Loop rollout_proc35_evt_loop terminating... +[2023-03-07 15:49:20,187][384886] Decorrelating experience for 736 frames... +[2023-03-07 15:49:20,207][384387] Decorrelating experience for 800 frames... +[2023-03-07 15:49:20,238][382412] Decorrelating experience for 992 frames... +[2023-03-07 15:49:20,253][383550] Decorrelating experience for 736 frames... +[2023-03-07 15:49:20,272][382419] Decorrelating experience for 768 frames... +[2023-03-07 15:49:20,305][382401] Decorrelating experience for 928 frames... +[2023-03-07 15:49:20,338][382435] Stopping RolloutWorker_w56... +[2023-03-07 15:49:20,338][382435] Loop rollout_proc56_evt_loop terminating... +[2023-03-07 15:49:20,392][382432] Stopping RolloutWorker_w50... +[2023-03-07 15:49:20,392][382432] Loop rollout_proc50_evt_loop terminating... +[2023-03-07 15:49:20,392][382414] Decorrelating experience for 992 frames... +[2023-03-07 15:49:20,398][384158] Decorrelating experience for 640 frames... +[2023-03-07 15:49:20,428][384315] Decorrelating experience for 832 frames... +[2023-03-07 15:49:20,448][382842] Decorrelating experience for 576 frames... +[2023-03-07 15:49:20,456][382326] Decorrelating experience for 672 frames... +[2023-03-07 15:49:20,457][382410] Stopping RolloutWorker_w30... +[2023-03-07 15:49:20,458][382410] Loop rollout_proc30_evt_loop terminating... +[2023-03-07 15:49:20,472][383102] Decorrelating experience for 960 frames... +[2023-03-07 15:49:20,504][384761] Stopping RolloutWorker_w103... +[2023-03-07 15:49:20,505][384761] Loop rollout_proc103_evt_loop terminating... +[2023-03-07 15:49:20,636][382423] Decorrelating experience for 864 frames... +[2023-03-07 15:49:20,653][382524] Decorrelating experience for 896 frames... +[2023-03-07 15:49:20,655][382406] Decorrelating experience for 960 frames... +[2023-03-07 15:49:20,660][385026] Decorrelating experience for 608 frames... +[2023-03-07 15:49:20,666][382412] Stopping RolloutWorker_w36... +[2023-03-07 15:49:20,666][382412] Loop rollout_proc36_evt_loop terminating... +[2023-03-07 15:49:20,691][382441] Decorrelating experience for 928 frames... +[2023-03-07 15:49:20,714][384886] Decorrelating experience for 768 frames... +[2023-03-07 15:49:20,721][382424] Decorrelating experience for 896 frames... +[2023-03-07 15:49:20,829][382449] Decorrelating experience for 992 frames... +[2023-03-07 15:49:20,849][384387] Decorrelating experience for 832 frames... +[2023-03-07 15:49:20,864][382437] Decorrelating experience for 896 frames... +[2023-03-07 15:49:20,870][382842] Decorrelating experience for 608 frames... +[2023-03-07 15:49:20,897][382414] Stopping RolloutWorker_w39... +[2023-03-07 15:49:20,898][382414] Loop rollout_proc39_evt_loop terminating... +[2023-03-07 15:49:20,921][383678] Decorrelating experience for 864 frames... +[2023-03-07 15:49:20,946][382436] Decorrelating experience for 960 frames... +[2023-03-07 15:49:20,986][236315] Starting all processes... +[2023-03-07 15:49:20,988][469655] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:20,988][469655] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0 +[2023-03-07 15:49:20,998][469655] Num visible devices: 1 +[2023-03-07 15:49:21,004][469655] Starting seed is not provided +[2023-03-07 15:49:21,004][469655] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:21,004][469655] Initializing actor-critic model on device cuda:0 +[2023-03-07 15:49:21,005][469655] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:49:21,005][469655] RunningMeanStd input shape: (1,) +[2023-03-07 15:49:21,019][469655] ConvEncoder: input_channels=3 +[2023-03-07 15:49:21,032][382393] Decorrelating experience for 800 frames... +[2023-03-07 15:49:21,036][383550] Decorrelating experience for 768 frames... +[2023-03-07 15:49:21,058][383102] Decorrelating experience for 992 frames... +[2023-03-07 15:49:21,061][384315] Decorrelating experience for 864 frames... +[2023-03-07 15:49:21,072][385026] Decorrelating experience for 640 frames... +[2023-03-07 15:49:21,132][469655] Conv encoder output size: 512 +[2023-03-07 15:49:21,132][469655] Policy head output size: 512 +[2023-03-07 15:49:21,147][469655] Created Actor Critic model with architecture: +[2023-03-07 15:49:21,147][469655] ActorCriticSharedWeights( + (obs_normalizer): ObservationNormalizer( + (running_mean_std): RunningMeanStdDictInPlace( + (running_mean_std): ModuleDict( + (obs): RunningMeanStdInPlace() + ) + ) + ) + (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace) + (encoder): VizdoomEncoder( + (basic_encoder): ConvEncoder( + (enc): RecursiveScriptModule( + original_name=ConvEncoderImpl + (conv_head): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Conv2d) + (1): RecursiveScriptModule(original_name=ELU) + (2): RecursiveScriptModule(original_name=Conv2d) + (3): RecursiveScriptModule(original_name=ELU) + (4): RecursiveScriptModule(original_name=Conv2d) + (5): RecursiveScriptModule(original_name=ELU) + ) + (mlp_layers): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Linear) + (1): RecursiveScriptModule(original_name=ELU) + ) + ) + ) + ) + (core): ModelCoreRNN( + (core): GRU(512, 512) + ) + (decoder): MlpDecoder( + (mlp): Identity() + ) + (critic_linear): Linear(in_features=512, out_features=1, bias=True) + (action_parameterization): ActionParameterizationDefault( + (distribution_linear): Linear(in_features=512, out_features=5, bias=True) + ) +) +[2023-03-07 15:49:21,174][382454] Decorrelating experience for 928 frames... +[2023-03-07 15:49:21,186][384886] Decorrelating experience for 800 frames... +[2023-03-07 15:49:21,205][382449] Stopping RolloutWorker_w57... +[2023-03-07 15:49:21,205][382449] Loop rollout_proc57_evt_loop terminating... +[2023-03-07 15:49:21,226][382326] Decorrelating experience for 704 frames... +[2023-03-07 15:49:21,244][382423] Decorrelating experience for 896 frames... +[2023-03-07 15:49:21,252][384158] Decorrelating experience for 672 frames... +[2023-03-07 15:49:21,277][236315] Starting process inference_proc0-0 +[2023-03-07 15:49:21,277][236315] Starting process rollout_proc2 +[2023-03-07 15:49:21,278][236315] Starting process rollout_proc5 +[2023-03-07 15:49:21,278][236315] Starting process rollout_proc8 +[2023-03-07 15:49:21,279][236315] Starting process rollout_proc11 +[2023-03-07 15:49:21,280][236315] Starting process rollout_proc14 +[2023-03-07 15:49:21,281][236315] Starting process rollout_proc17 +[2023-03-07 15:49:21,284][382401] Decorrelating experience for 960 frames... +[2023-03-07 15:49:21,285][382399] Decorrelating experience for 896 frames... +[2023-03-07 15:49:21,282][236315] Starting process rollout_proc20 +[2023-03-07 15:49:21,282][236315] Starting process rollout_proc23 +[2023-03-07 15:49:21,282][236315] Starting process rollout_proc26 +[2023-03-07 15:49:21,282][236315] Starting process rollout_proc29 +[2023-03-07 15:49:21,283][236315] Starting process rollout_proc32 +[2023-03-07 15:49:21,284][236315] Starting process rollout_proc35 +[2023-03-07 15:49:21,285][236315] Starting process rollout_proc38 +[2023-03-07 15:49:21,286][236315] Starting process rollout_proc41 +[2023-03-07 15:49:21,286][236315] Starting process rollout_proc44 +[2023-03-07 15:49:21,362][383951] Decorrelating experience for 800 frames... +[2023-03-07 15:49:21,344][236315] Starting process rollout_proc9 +[2023-03-07 15:49:21,347][236315] Starting process rollout_proc3 +[2023-03-07 15:49:21,359][236315] Starting process rollout_proc6 +[2023-03-07 15:49:21,362][236315] Starting process rollout_proc12 +[2023-03-07 15:49:21,366][236315] Starting process rollout_proc27 +[2023-03-07 15:49:21,385][236315] Starting process rollout_proc15 +[2023-03-07 15:49:21,388][236315] Starting process rollout_proc24 +[2023-03-07 15:49:21,392][236315] Starting process rollout_proc21 +[2023-03-07 15:49:21,392][236315] Starting process rollout_proc18 +[2023-03-07 15:49:21,411][236315] Starting process rollout_proc33 +[2023-03-07 15:49:21,424][236315] Starting process rollout_proc30 +[2023-03-07 15:49:21,432][236315] Starting process rollout_proc36 +[2023-03-07 15:49:21,479][382437] Decorrelating experience for 928 frames... +[2023-03-07 15:49:21,441][236315] Starting process rollout_proc39 +[2023-03-07 15:49:21,441][236315] Starting process rollout_proc42 +[2023-03-07 15:49:21,493][383102] Stopping RolloutWorker_w76... +[2023-03-07 15:49:21,493][383102] Loop rollout_proc76_evt_loop terminating... +[2023-03-07 15:49:21,455][236315] Starting process rollout_proc45 +[2023-03-07 15:49:21,455][236315] Starting process rollout_proc10 +[2023-03-07 15:49:21,502][382324] Decorrelating experience for 992 frames... +[2023-03-07 15:49:21,481][236315] Starting process rollout_proc7 +[2023-03-07 15:49:21,516][382842] Decorrelating experience for 640 frames... +[2023-03-07 15:49:21,484][236315] Starting process rollout_proc13 +[2023-03-07 15:49:21,486][236315] Starting process rollout_proc4 +[2023-03-07 15:49:21,491][236315] Starting process rollout_proc16 +[2023-03-07 15:49:21,504][236315] Starting process rollout_proc25 +[2023-03-07 15:49:21,517][236315] Starting process rollout_proc22 +[2023-03-07 15:49:21,532][382439] Decorrelating experience for 768 frames... +[2023-03-07 15:49:21,517][236315] Starting process rollout_proc34 +[2023-03-07 15:49:21,534][385026] Decorrelating experience for 672 frames... +[2023-03-07 15:49:21,518][236315] Starting process rollout_proc28 +[2023-03-07 15:49:21,529][236315] Starting process rollout_proc37 +[2023-03-07 15:49:21,533][236315] Starting process rollout_proc31 +[2023-03-07 15:49:21,548][236315] Starting process rollout_proc46 +[2023-03-07 15:49:21,550][236315] Starting process rollout_proc19 +[2023-03-07 15:49:21,575][383550] Decorrelating experience for 800 frames... +[2023-03-07 15:49:21,573][236315] Starting process rollout_proc43 +[2023-03-07 15:49:21,573][236315] Starting process rollout_proc40 +[2023-03-07 15:49:21,584][236315] Starting process rollout_proc47 +[2023-03-07 15:49:21,588][236315] Starting process rollout_proc50 +[2023-03-07 15:49:21,590][236315] Starting process rollout_proc53 +[2023-03-07 15:49:21,630][383527] Decorrelating experience for 704 frames... +[2023-03-07 15:49:21,602][236315] Starting process rollout_proc56 +[2023-03-07 15:49:21,616][236315] Starting process rollout_proc59 +[2023-03-07 15:49:21,616][236315] Starting process rollout_proc62 +[2023-03-07 15:49:21,621][236315] Starting process rollout_proc65 +[2023-03-07 15:49:21,643][236315] Starting process rollout_proc68 +[2023-03-07 15:49:21,643][236315] Starting process rollout_proc71 +[2023-03-07 15:49:21,645][236315] Starting process rollout_proc74 +[2023-03-07 15:49:21,649][236315] Starting process rollout_proc77 +[2023-03-07 15:49:21,662][236315] Starting process rollout_proc80 +[2023-03-07 15:49:21,676][236315] Starting process rollout_proc83 +[2023-03-07 15:49:21,707][382524] Decorrelating experience for 928 frames... +[2023-03-07 15:49:21,683][236315] Starting process rollout_proc86 +[2023-03-07 15:49:21,694][236315] Starting process rollout_proc89 +[2023-03-07 15:49:21,698][236315] Starting process rollout_proc51 +[2023-03-07 15:49:21,719][236315] Starting process rollout_proc48 +[2023-03-07 15:49:21,731][236315] Starting process rollout_proc54 +[2023-03-07 15:49:21,740][236315] Starting process rollout_proc63 +[2023-03-07 15:49:21,749][236315] Starting process rollout_proc57 +[2023-03-07 15:49:21,762][236315] Starting process rollout_proc60 +[2023-03-07 15:49:21,779][236315] Starting process rollout_proc66 +[2023-03-07 15:49:21,784][236315] Starting process rollout_proc75 +[2023-03-07 15:49:21,812][382406] Decorrelating experience for 992 frames... +[2023-03-07 15:49:21,785][236315] Starting process rollout_proc69 +[2023-03-07 15:49:21,800][236315] Starting process rollout_proc72 +[2023-03-07 15:49:21,803][236315] Starting process rollout_proc81 +[2023-03-07 15:49:21,819][236315] Starting process rollout_proc78 +[2023-03-07 15:49:21,830][236315] Starting process rollout_proc84 +[2023-03-07 15:49:21,858][382436] Decorrelating experience for 992 frames... +[2023-03-07 15:49:21,833][236315] Starting process rollout_proc52 +[2023-03-07 15:49:21,862][382393] Decorrelating experience for 832 frames... +[2023-03-07 15:49:21,851][236315] Starting process rollout_proc90 +[2023-03-07 15:49:21,856][236315] Starting process rollout_proc87 +[2023-03-07 15:49:21,865][236315] Starting process rollout_proc49 +[2023-03-07 15:49:21,866][236315] Starting process rollout_proc64 +[2023-03-07 15:49:21,883][236315] Starting process rollout_proc58 +[2023-03-07 15:49:21,887][236315] Starting process rollout_proc55 +[2023-03-07 15:49:21,892][236315] Starting process rollout_proc61 +[2023-03-07 15:49:21,901][236315] Starting process rollout_proc70 +[2023-03-07 15:49:21,962][236315] Starting process rollout_proc76 +[2023-03-07 15:49:21,973][236315] Starting process rollout_proc67 +[2023-03-07 15:49:21,981][236315] Starting process rollout_proc79 +[2023-03-07 15:49:22,004][236315] Starting process rollout_proc82 +[2023-03-07 15:49:22,004][236315] Starting process rollout_proc73 +[2023-03-07 15:49:22,040][236315] Starting process rollout_proc92 +[2023-03-07 15:49:22,059][236315] Starting process rollout_proc95 +[2023-03-07 15:49:22,064][236315] Starting process rollout_proc88 +[2023-03-07 15:49:22,073][236315] Starting process rollout_proc91 +[2023-03-07 15:49:22,078][236315] Starting process rollout_proc98 +[2023-03-07 15:49:22,096][236315] Starting process rollout_proc85 +[2023-03-07 15:49:22,164][236315] Starting process rollout_proc101 +[2023-03-07 15:49:22,203][236315] Starting process rollout_proc104 +[2023-03-07 15:49:22,228][236315] Starting process rollout_proc107 +[2023-03-07 15:49:22,231][236315] Starting process rollout_proc110 +[2023-03-07 15:49:22,233][236315] Starting process rollout_proc113 +[2023-03-07 15:49:22,248][236315] Starting process rollout_proc116 +[2023-03-07 15:49:22,250][236315] Starting process rollout_proc119 +[2023-03-07 15:49:22,286][236315] Starting process rollout_proc122 +[2023-03-07 15:49:22,303][236315] Starting process rollout_proc96 +[2023-03-07 15:49:22,303][236315] Starting process rollout_proc99 +[2023-03-07 15:49:22,318][236315] Starting process rollout_proc125 +[2023-03-07 15:49:22,442][382324] Stopping RolloutWorker_w8... +[2023-03-07 15:49:22,443][382324] Loop rollout_proc8_evt_loop terminating... +[2023-03-07 15:49:22,464][382408] Decorrelating experience for 672 frames... +[2023-03-07 15:49:22,318][236315] Starting process rollout_proc93 +[2023-03-07 15:49:22,431][236315] Starting process rollout_proc102 +[2023-03-07 15:49:22,476][236315] Starting process rollout_proc105 +[2023-03-07 15:49:22,500][236315] Starting process rollout_proc108 +[2023-03-07 15:49:22,522][236315] Starting process rollout_proc117 +[2023-03-07 15:49:22,522][236315] Starting process rollout_proc111 +[2023-03-07 15:49:22,523][236315] Starting process rollout_proc123 +[2023-03-07 15:49:22,667][382842] Decorrelating experience for 672 frames... +[2023-03-07 15:49:22,525][236315] Starting process rollout_proc114 +[2023-03-07 15:49:22,575][236315] Starting process rollout_proc97 +[2023-03-07 15:49:22,669][236315] Starting process rollout_proc120 +[2023-03-07 15:49:22,671][236315] Starting process rollout_proc100 +[2023-03-07 15:49:22,792][236315] Starting process rollout_proc126 +[2023-03-07 15:49:22,793][236315] Starting process rollout_proc109 +[2023-03-07 15:49:22,808][236315] Starting process rollout_proc106 +[2023-03-07 15:49:22,820][236315] Starting process rollout_proc94 +[2023-03-07 15:49:22,821][236315] Starting process rollout_proc103 +[2023-03-07 15:49:22,821][236315] Starting process rollout_proc124 +[2023-03-07 15:49:22,834][236315] Starting process rollout_proc112 +[2023-03-07 15:49:22,881][236315] Starting process rollout_proc118 +[2023-03-07 15:49:22,898][236315] Starting process rollout_proc121 +[2023-03-07 15:49:22,899][236315] Starting process rollout_proc127 +[2023-03-07 15:49:22,900][236315] Starting process rollout_proc115 +[2023-03-07 15:49:22,936][383678] Decorrelating experience for 896 frames... +[2023-03-07 15:49:23,418][470096] Worker 5 uses CPU cores [5] +[2023-03-07 15:49:23,762][382419] Decorrelating experience for 800 frames... +[2023-03-07 15:49:23,770][470187] Worker 3 uses CPU cores [3] +[2023-03-07 15:49:23,795][469655] Using optimizer +[2023-03-07 15:49:23,796][469655] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000981_4018176.pth... +[2023-03-07 15:49:23,800][470094] Worker 2 uses CPU cores [2] +[2023-03-07 15:49:23,825][236315] Starting process rollout_proc0 +[2023-03-07 15:49:23,829][470099] Worker 11 uses CPU cores [11] +[2023-03-07 15:49:23,846][470095] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:23,846][470095] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0 +[2023-03-07 15:49:23,873][470183] Worker 41 uses CPU cores [41] +[2023-03-07 15:49:23,874][469655] Loading model from checkpoint +[2023-03-07 15:49:23,878][470095] Num visible devices: 1 +[2023-03-07 15:49:23,885][469655] Loaded experiment state at self.train_step=981, self.env_steps=4018176 +[2023-03-07 15:49:23,901][469655] Initialized policy 0 weights for model version 981 +[2023-03-07 15:49:23,903][469655] LearnerWorker_p0 finished initialization! +[2023-03-07 15:49:23,903][469655] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:23,919][470184] Worker 44 uses CPU cores [44] +[2023-03-07 15:49:23,934][470198] Worker 27 uses CPU cores [27] +[2023-03-07 15:49:23,941][383951] Decorrelating experience for 832 frames... +[2023-03-07 15:49:23,942][470104] Worker 20 uses CPU cores [20] +[2023-03-07 15:49:23,947][470115] Worker 29 uses CPU cores [29] +[2023-03-07 15:49:23,951][470101] Worker 26 uses CPU cores [26] +[2023-03-07 15:49:23,954][470179] Worker 35 uses CPU cores [35] +[2023-03-07 15:49:24,068][470103] Worker 14 uses CPU cores [14] +[2023-03-07 15:49:24,145][470107] Worker 23 uses CPU cores [23] +[2023-03-07 15:49:24,193][470095] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:49:24,193][470095] RunningMeanStd input shape: (1,) +[2023-03-07 15:49:24,236][470186] Worker 9 uses CPU cores [9] +[2023-03-07 15:49:24,244][470095] ConvEncoder: input_channels=3 +[2023-03-07 15:49:24,244][385026] Decorrelating experience for 704 frames... +[2023-03-07 15:49:24,250][470197] Worker 33 uses CPU cores [33] +[2023-03-07 15:49:24,255][470190] Worker 12 uses CPU cores [12] +[2023-03-07 15:49:24,269][470227] Worker 7 uses CPU cores [7] +[2023-03-07 15:49:24,275][470180] Worker 38 uses CPU cores [38] +[2023-03-07 15:49:24,277][470188] Worker 6 uses CPU cores [6] +[2023-03-07 15:49:24,306][470192] Worker 24 uses CPU cores [24] +[2023-03-07 15:49:24,329][470234] Worker 16 uses CPU cores [16] +[2023-03-07 15:49:24,354][470277] Worker 71 uses CPU cores [71] +[2023-03-07 15:49:24,374][470217] Worker 39 uses CPU cores [39] +[2023-03-07 15:49:24,410][470207] Worker 30 uses CPU cores [30] +[2023-03-07 15:49:24,415][470223] Worker 10 uses CPU cores [10] +[2023-03-07 15:49:24,416][470245] Worker 46 uses CPU cores [46] +[2023-03-07 15:49:24,436][470196] Worker 21 uses CPU cores [21] +[2023-03-07 15:49:24,515][470110] Worker 17 uses CPU cores [17] +[2023-03-07 15:49:24,523][470206] Worker 36 uses CPU cores [36] +[2023-03-07 15:49:24,575][470246] Worker 31 uses CPU cores [31] +[2023-03-07 15:49:24,645][470244] Worker 28 uses CPU cores [28] +[2023-03-07 15:49:24,661][470090] Worker 8 uses CPU cores [8] +[2023-03-07 15:49:24,670][470114] Worker 32 uses CPU cores [32] +[2023-03-07 15:49:24,679][470256] Worker 50 uses CPU cores [50] +[2023-03-07 15:49:24,680][470095] Conv encoder output size: 512 +[2023-03-07 15:49:24,681][470095] Policy head output size: 512 +[2023-03-07 15:49:24,686][470253] Worker 19 uses CPU cores [19] +[2023-03-07 15:49:24,708][470251] Worker 43 uses CPU cores [43] +[2023-03-07 15:49:24,725][470212] Worker 45 uses CPU cores [45] +[2023-03-07 15:49:24,726][470191] Worker 15 uses CPU cores [15] +[2023-03-07 15:49:24,750][470211] Worker 18 uses CPU cores [18] +[2023-03-07 15:49:24,762][470229] Worker 13 uses CPU cores [13] +[2023-03-07 15:49:24,771][470237] Worker 37 uses CPU cores [37] +[2023-03-07 15:49:24,895][470240] Worker 25 uses CPU cores [25] +[2023-03-07 15:49:24,935][382423] Decorrelating experience for 928 frames... +[2023-03-07 15:49:24,972][382399] Decorrelating experience for 928 frames... +[2023-03-07 15:49:25,062][470291] Worker 63 uses CPU cores [63] +[2023-03-07 15:49:25,128][470258] Worker 53 uses CPU cores [53] +[2023-03-07 15:49:25,150][470301] Worker 57 uses CPU cores [57] +[2023-03-07 15:49:25,162][470273] Worker 65 uses CPU cores [65] +[2023-03-07 15:49:25,198][470233] Worker 22 uses CPU cores [22] +[2023-03-07 15:49:25,210][470271] Worker 59 uses CPU cores [59] +[2023-03-07 15:49:25,214][470254] Worker 40 uses CPU cores [40] +[2023-03-07 15:49:25,222][470834] Worker 70 uses CPU cores [70] +[2023-03-07 15:49:25,222][470260] Worker 56 uses CPU cores [56] +[2023-03-07 15:49:25,236][470284] Worker 83 uses CPU cores [83] +[2023-03-07 15:49:25,236][470698] Worker 87 uses CPU cores [87] +[2023-03-07 15:49:25,254][470274] Worker 68 uses CPU cores [68] +[2023-03-07 15:49:25,262][470275] Worker 74 uses CPU cores [74] +[2023-03-07 15:49:25,269][382401] Decorrelating experience for 992 frames... +[2023-03-07 15:49:25,302][470703] Worker 49 uses CPU cores [49] +[2023-03-07 15:49:25,308][470700] Worker 64 uses CPU cores [64] +[2023-03-07 15:49:25,310][470438] Worker 72 uses CPU cores [72] +[2023-03-07 15:49:25,330][470248] Worker 34 uses CPU cores [34] +[2023-03-07 15:49:25,332][470289] Worker 86 uses CPU cores [86] +[2023-03-07 15:49:25,339][384158] Decorrelating experience for 704 frames... +[2023-03-07 15:49:25,343][470257] Worker 47 uses CPU cores [47] +[2023-03-07 15:49:25,349][470276] Worker 80 uses CPU cores [80] +[2023-03-07 15:49:25,350][470288] Worker 89 uses CPU cores [89] +[2023-03-07 15:49:25,354][470222] Worker 42 uses CPU cores [42] +[2023-03-07 15:49:25,378][470833] Worker 79 uses CPU cores [79] +[2023-03-07 15:49:25,406][470235] Worker 4 uses CPU cores [4] +[2023-03-07 15:49:25,422][470305] Worker 69 uses CPU cores [69] +[2023-03-07 15:49:25,426][470265] Worker 62 uses CPU cores [62] +[2023-03-07 15:49:25,454][472859] Worker 106 uses CPU cores [106] +[2023-03-07 15:49:25,464][470290] Worker 48 uses CPU cores [48] +[2023-03-07 15:49:25,488][470281] Worker 77 uses CPU cores [77] +[2023-03-07 15:49:25,497][382406] Stopping RolloutWorker_w24... +[2023-03-07 15:49:25,498][382406] Loop rollout_proc24_evt_loop terminating... +[2023-03-07 15:49:25,499][470668] Worker 84 uses CPU cores [84] +[2023-03-07 15:49:25,516][470303] Worker 54 uses CPU cores [54] +[2023-03-07 15:49:25,538][470442] Worker 78 uses CPU cores [78] +[2023-03-07 15:49:25,541][470283] Worker 51 uses CPU cores [51] +[2023-03-07 15:49:25,547][470705] Worker 55 uses CPU cores [55] +[2023-03-07 15:49:25,550][471304] Worker 91 uses CPU cores [91] +[2023-03-07 15:49:25,554][470308] Worker 75 uses CPU cores [75] +[2023-03-07 15:49:25,554][471283] Worker 98 uses CPU cores [98] +[2023-03-07 15:49:25,566][382326] Decorrelating experience for 736 frames... +[2023-03-07 15:49:25,571][382454] Decorrelating experience for 960 frames... +[2023-03-07 15:49:25,610][470295] Worker 60 uses CPU cores [60] +[2023-03-07 15:49:25,612][472901] Worker 103 uses CPU cores [103] +[2023-03-07 15:49:25,643][470479] Worker 90 uses CPU cores [90] +[2023-03-07 15:49:25,647][470704] Worker 58 uses CPU cores [58] +[2023-03-07 15:49:25,650][471189] Worker 73 uses CPU cores [73] +[2023-03-07 15:49:25,654][470873] Worker 67 uses CPU cores [67] +[2023-03-07 15:49:25,658][470702] Worker 61 uses CPU cores [61] +[2023-03-07 15:49:25,670][471256] Worker 95 uses CPU cores [95] +[2023-03-07 15:49:25,674][470309] Worker 81 uses CPU cores [81] +[2023-03-07 15:49:25,680][470699] Worker 52 uses CPU cores [52] +[2023-03-07 15:49:25,682][472386] Worker 105 uses CPU cores [105] +[2023-03-07 15:49:25,686][472899] Worker 126 uses CPU cores [126] +[2023-03-07 15:49:25,687][471068] Worker 82 uses CPU cores [82] +[2023-03-07 15:49:25,687][471306] Worker 88 uses CPU cores [88] +[2023-03-07 15:49:25,697][471395] Worker 101 uses CPU cores [101] +[2023-03-07 15:49:25,729][471734] Worker 113 uses CPU cores [113] +[2023-03-07 15:49:25,732][472473] Worker 111 uses CPU cores [111] +[2023-03-07 15:49:25,758][470307] Worker 66 uses CPU cores [66] +[2023-03-07 15:49:25,766][472381] Worker 108 uses CPU cores [108] +[2023-03-07 15:49:25,767][472833] Worker 120 uses CPU cores [120] +[2023-03-07 15:49:25,774][472746] Worker 100 uses CPU cores [100] +[2023-03-07 15:49:25,777][472900] Worker 94 uses CPU cores [94] +[2023-03-07 15:49:25,778][471867] Worker 96 uses CPU cores [96] +[2023-03-07 15:49:25,787][472495] Worker 123 uses CPU cores [123] +[2023-03-07 15:49:25,790][470832] Worker 76 uses CPU cores [76] +[2023-03-07 15:49:25,794][471541] Worker 104 uses CPU cores [104] +[2023-03-07 15:49:25,795][473018] Worker 112 uses CPU cores [112] +[2023-03-07 15:49:25,810][471190] Worker 92 uses CPU cores [92] +[2023-03-07 15:49:25,812][382401] Stopping RolloutWorker_w3... +[2023-03-07 15:49:25,813][382401] Loop rollout_proc3_evt_loop terminating... +[2023-03-07 15:49:25,818][471801] Worker 122 uses CPU cores [122] +[2023-03-07 15:49:25,825][471982] Worker 119 uses CPU cores [119] +[2023-03-07 15:49:25,830][471737] Worker 116 uses CPU cores [116] +[2023-03-07 15:49:25,834][472823] Worker 97 uses CPU cores [97] +[2023-03-07 15:49:25,846][473019] Worker 118 uses CPU cores [118] +[2023-03-07 15:49:25,851][472834] Worker 109 uses CPU cores [109] +[2023-03-07 15:49:25,862][382439] Decorrelating experience for 800 frames... +[2023-03-07 15:49:25,862][471871] Worker 99 uses CPU cores [99] +[2023-03-07 15:49:25,863][472902] Worker 124 uses CPU cores [124] +[2023-03-07 15:49:25,866][472380] Worker 102 uses CPU cores [102] +[2023-03-07 15:49:25,877][472534] Worker 117 uses CPU cores [117] +[2023-03-07 15:49:25,890][473020] Worker 127 uses CPU cores [127] +[2023-03-07 15:49:25,902][472342] Worker 93 uses CPU cores [93] +[2023-03-07 15:49:25,902][471693] Worker 110 uses CPU cores [110] +[2023-03-07 15:49:25,913][471605] Worker 107 uses CPU cores [107] +[2023-03-07 15:49:25,914][473023] Worker 115 uses CPU cores [115] +[2023-03-07 15:49:25,917][473024] Worker 121 uses CPU cores [121] +[2023-03-07 15:49:25,918][383550] Decorrelating experience for 832 frames... +[2023-03-07 15:49:25,949][472640] Worker 114 uses CPU cores [114] +[2023-03-07 15:49:25,968][472010] Worker 125 uses CPU cores [125] +[2023-03-07 15:49:25,982][382436] Stopping RolloutWorker_w47... +[2023-03-07 15:49:25,982][382436] Loop rollout_proc47_evt_loop terminating... +[2023-03-07 15:49:26,002][471477] Worker 85 uses CPU cores [85] +[2023-03-07 15:49:26,011][384886] Decorrelating experience for 832 frames... +[2023-03-07 15:49:26,057][382423] Decorrelating experience for 960 frames... +[2023-03-07 15:49:26,090][382393] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,101][236315] Starting process rollout_proc1 +[2023-03-07 15:49:26,103][475463] Worker 0 uses CPU cores [0] +[2023-03-07 15:49:26,105][382326] Decorrelating experience for 768 frames... +[2023-03-07 15:49:26,171][382974] Decorrelating experience for 832 frames... +[2023-03-07 15:49:26,208][382441] Decorrelating experience for 960 frames... +[2023-03-07 15:49:26,429][383678] Decorrelating experience for 928 frames... +[2023-03-07 15:49:26,455][383951] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,458][385026] Decorrelating experience for 736 frames... +[2023-03-07 15:49:26,508][384158] Decorrelating experience for 736 frames... +[2023-03-07 15:49:26,509][382408] Decorrelating experience for 704 frames... +[2023-03-07 15:49:26,509][383550] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,613][382423] Decorrelating experience for 992 frames... +[2023-03-07 15:49:26,649][382393] Decorrelating experience for 896 frames... +[2023-03-07 15:49:26,672][382454] Decorrelating experience for 992 frames... +[2023-03-07 15:49:26,724][384886] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,757][382974] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,786][382441] Decorrelating experience for 992 frames... +[2023-03-07 15:49:26,811][384644] Decorrelating experience for 928 frames... +[2023-03-07 15:49:26,851][384387] Decorrelating experience for 864 frames... +[2023-03-07 15:49:26,854][382326] Decorrelating experience for 800 frames... +[2023-03-07 15:49:26,878][382439] Decorrelating experience for 832 frames... +[2023-03-07 15:49:27,006][383678] Decorrelating experience for 960 frames... +[2023-03-07 15:49:27,023][384158] Decorrelating experience for 768 frames... +[2023-03-07 15:49:27,031][382423] Stopping RolloutWorker_w28... +[2023-03-07 15:49:27,031][382423] Loop rollout_proc28_evt_loop terminating... +[2023-03-07 15:49:27,043][383951] Decorrelating experience for 896 frames... +[2023-03-07 15:49:27,060][384315] Decorrelating experience for 896 frames... +[2023-03-07 15:49:27,071][382454] Stopping RolloutWorker_w63... +[2023-03-07 15:49:27,071][382454] Loop rollout_proc63_evt_loop terminating... +[2023-03-07 15:49:27,074][382411] Decorrelating experience for 800 frames... +[2023-03-07 15:49:27,154][383550] Decorrelating experience for 896 frames... +[2023-03-07 15:49:27,166][486568] Worker 1 uses CPU cores [1] +[2023-03-07 15:49:27,173][382441] Stopping RolloutWorker_w74... +[2023-03-07 15:49:27,173][382441] Loop rollout_proc74_evt_loop terminating... +[2023-03-07 15:49:27,172][236315] Inference worker 0-0 is ready! +[2023-03-07 15:49:27,174][236315] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:49:27,181][236315] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 4018176. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:49:27,245][470096] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,249][470479] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,252][471867] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,253][470115] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,253][470703] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,254][470668] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,254][470705] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,256][470251] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,257][470873] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,256][475463] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,257][470110] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,257][471189] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,257][470227] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,258][470192] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,258][472833] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,258][470256] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,258][470099] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,258][472010] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][471304] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][471477] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][472381] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][470190] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][471190] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][470188] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][470305] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,260][470276] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,259][472901] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,260][473020] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,260][471283] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,261][470260] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,261][470265] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,261][470211] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,261][470206] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,262][470289] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,263][470229] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,263][471871] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,263][473024] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,263][470308] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,263][470246] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,264][470700] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,264][470090] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,264][470223] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,265][470253] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,265][470094] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,265][470179] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,265][470184] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,265][471541] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,264][470698] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470207] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][471693] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470240] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][471256] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][471068] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470832] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470222] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470191] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][471801] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][470254] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,266][472834] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470288] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][472900] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470180] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][471395] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][472746] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470834] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][472534] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470307] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470257] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470704] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470103] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,267][470212] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470237] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470309] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470114] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][472495] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470104] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][472386] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][471737] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][473019] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470217] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470274] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,268][470295] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470196] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470281] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470198] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470442] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470235] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,269][470183] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470699] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470186] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470290] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][471605] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470303] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][472902] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470438] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,271][470301] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,270][470284] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,271][470283] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,271][473023] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,271][470187] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,271][471982] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,272][470271] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,272][470245] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,273][473018] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,273][470248] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,276][470107] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,276][470702] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,279][470233] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,285][470101] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,297][470258] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,310][383641] Another process currently holds the lock /tmp/sf2_rolo/doom_003.lockfile, attempt: 1 +[2023-03-07 15:49:27,312][470273] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,315][470291] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,321][470244] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,326][472640] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,336][472859] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,338][470275] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,330][472899] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,340][472380] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,343][472823] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,347][470197] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,347][470277] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,346][382393] Decorrelating experience for 928 frames... +[2023-03-07 15:49:27,352][471734] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,357][471306] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,380][470833] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,384][472342] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,366][472473] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,565][470234] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:27,635][384886] Decorrelating experience for 896 frames... +[2023-03-07 15:49:27,643][385026] Decorrelating experience for 768 frames... +[2023-03-07 15:49:27,829][382440] Decorrelating experience for 960 frames... +[2023-03-07 15:49:27,991][486568] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:28,007][382439] Decorrelating experience for 864 frames... +[2023-03-07 15:49:28,187][382326] Decorrelating experience for 832 frames... +[2023-03-07 15:49:28,213][384644] Decorrelating experience for 960 frames... +[2023-03-07 15:49:28,215][470479] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,219][471256] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,255][382974] Decorrelating experience for 896 frames... +[2023-03-07 15:49:28,270][470180] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,274][384387] Decorrelating experience for 896 frames... +[2023-03-07 15:49:28,290][382399] Decorrelating experience for 960 frames... +[2023-03-07 15:49:28,291][470223] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,319][470227] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,403][470196] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,411][384158] Decorrelating experience for 800 frames... +[2023-03-07 15:49:28,442][486568] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,445][472534] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,449][470704] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,473][383641] Decorrelating experience for 736 frames... +[2023-03-07 15:49:28,537][470698] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,559][470227] Decorrelating experience for 32 frames... +[2023-03-07 15:49:28,567][470101] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,567][470258] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,591][470099] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,602][470257] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,629][471283] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,637][383678] Decorrelating experience for 992 frames... +[2023-03-07 15:49:28,640][472859] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,695][382434] Decorrelating experience for 928 frames... +[2023-03-07 15:49:28,719][470244] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,751][470833] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,754][382440] Decorrelating experience for 992 frames... +[2023-03-07 15:49:28,755][470276] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,773][470260] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,787][470289] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,811][471737] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,823][470104] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,827][470229] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,875][382425] Decorrelating experience for 960 frames... +[2023-03-07 15:49:28,900][472823] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,940][384158] Decorrelating experience for 832 frames... +[2023-03-07 15:49:28,947][471734] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,956][470288] Decorrelating experience for 0 frames... +[2023-03-07 15:49:28,990][470222] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,004][473023] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,004][472902] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,011][470115] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,021][383678] Stopping RolloutWorker_w88... +[2023-03-07 15:49:29,022][383678] Loop rollout_proc88_evt_loop terminating... +[2023-03-07 15:49:29,025][472859] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,055][470308] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,079][471605] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,122][470281] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,129][382399] Decorrelating experience for 992 frames... +[2023-03-07 15:49:29,137][473019] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,140][382440] Stopping RolloutWorker_w71... +[2023-03-07 15:49:29,140][382440] Loop rollout_proc71_evt_loop terminating... +[2023-03-07 15:49:29,170][472901] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,200][471871] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,206][470257] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,207][382326] Decorrelating experience for 864 frames... +[2023-03-07 15:49:29,211][470090] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,237][470233] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,266][470094] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,308][472010] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,327][384315] Decorrelating experience for 928 frames... +[2023-03-07 15:49:29,388][470179] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,415][470438] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,432][472380] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,453][470479] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,487][382439] Decorrelating experience for 896 frames... +[2023-03-07 15:49:29,522][384387] Decorrelating experience for 928 frames... +[2023-03-07 15:49:29,523][470110] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,565][470273] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,565][470265] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,594][470253] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,609][470702] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,638][471189] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,665][471541] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,670][382399] Stopping RolloutWorker_w6... +[2023-03-07 15:49:29,670][382399] Loop rollout_proc6_evt_loop terminating... +[2023-03-07 15:49:29,704][470258] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,732][382393] Decorrelating experience for 960 frames... +[2023-03-07 15:49:29,754][473018] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,765][472640] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,776][382842] Decorrelating experience for 704 frames... +[2023-03-07 15:49:29,783][470188] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,792][382411] Decorrelating experience for 832 frames... +[2023-03-07 15:49:29,818][472859] Decorrelating experience for 64 frames... +[2023-03-07 15:49:29,853][470244] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,908][470180] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,918][471189] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,952][470438] Decorrelating experience for 32 frames... +[2023-03-07 15:49:29,953][470258] Decorrelating experience for 64 frames... +[2023-03-07 15:49:29,970][470237] Decorrelating experience for 0 frames... +[2023-03-07 15:49:29,974][383744] Decorrelating experience for 544 frames... +[2023-03-07 15:49:30,014][470273] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,033][470235] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,046][473019] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,067][470303] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,103][472901] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,115][475463] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,132][471734] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,156][470180] Decorrelating experience for 64 frames... +[2023-03-07 15:49:30,156][384886] Decorrelating experience for 928 frames... +[2023-03-07 15:49:30,158][382437] Decorrelating experience for 960 frames... +[2023-03-07 15:49:30,190][470276] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,214][472746] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,231][486568] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,249][470191] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,286][472823] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,293][470438] Decorrelating experience for 64 frames... +[2023-03-07 15:49:30,313][383641] Decorrelating experience for 768 frames... +[2023-03-07 15:49:30,336][471871] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,337][472386] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,356][470258] Decorrelating experience for 96 frames... +[2023-03-07 15:49:30,371][382393] Decorrelating experience for 992 frames... +[2023-03-07 15:49:30,394][470187] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,410][382326] Decorrelating experience for 896 frames... +[2023-03-07 15:49:30,428][470260] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,473][471734] Decorrelating experience for 64 frames... +[2023-03-07 15:49:30,480][470253] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,496][472902] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,520][472342] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,521][382842] Decorrelating experience for 736 frames... +[2023-03-07 15:49:30,552][471737] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,575][470233] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,588][384158] Decorrelating experience for 864 frames... +[2023-03-07 15:49:30,594][475463] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,608][470275] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,670][470206] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,704][470309] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,704][470265] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,729][470180] Decorrelating experience for 96 frames... +[2023-03-07 15:49:30,731][470099] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,759][470698] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,784][472473] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,801][470192] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,832][236315] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4018176. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:49:30,838][382393] Stopping RolloutWorker_w26... +[2023-03-07 15:49:30,838][382393] Loop rollout_proc26_evt_loop terminating... +[2023-03-07 15:49:30,850][470103] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,853][470206] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,885][471867] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,889][470254] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,920][470258] Decorrelating experience for 128 frames... +[2023-03-07 15:49:30,925][470257] Decorrelating experience for 64 frames... +[2023-03-07 15:49:30,944][470307] Decorrelating experience for 0 frames... +[2023-03-07 15:49:30,978][471256] Decorrelating experience for 32 frames... +[2023-03-07 15:49:30,991][470235] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,033][470191] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,036][472899] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,066][382524] Decorrelating experience for 960 frames... +[2023-03-07 15:49:31,079][470253] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,098][470438] Decorrelating experience for 96 frames... +[2023-03-07 15:49:31,103][470101] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,112][470094] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,124][470188] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,161][384315] Decorrelating experience for 960 frames... +[2023-03-07 15:49:31,179][471283] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,218][470190] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,224][470099] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,256][473020] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,291][471871] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,293][470307] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,294][470103] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,296][470104] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,304][382425] Decorrelating experience for 992 frames... +[2023-03-07 15:49:31,340][384158] Decorrelating experience for 896 frames... +[2023-03-07 15:49:31,360][470212] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,398][470305] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,436][470197] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,475][470251] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,488][470253] Decorrelating experience for 96 frames... +[2023-03-07 15:49:31,497][382842] Decorrelating experience for 768 frames... +[2023-03-07 15:49:31,499][470295] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,531][383641] Decorrelating experience for 800 frames... +[2023-03-07 15:49:31,537][472534] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,543][475463] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,575][470099] Decorrelating experience for 96 frames... +[2023-03-07 15:49:31,618][470277] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,686][382425] Stopping RolloutWorker_w31... +[2023-03-07 15:49:31,686][382425] Loop rollout_proc31_evt_loop terminating... +[2023-03-07 15:49:31,687][470305] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,689][472823] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,689][470256] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,692][472473] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,700][471737] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,716][470244] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,723][470254] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,744][470295] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,769][471801] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,805][472900] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,871][470873] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,876][470256] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,880][470705] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,888][470114] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,907][470308] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,909][471693] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,910][470198] Decorrelating experience for 0 frames... +[2023-03-07 15:49:31,951][470190] Decorrelating experience for 32 frames... +[2023-03-07 15:49:31,953][470235] Decorrelating experience for 64 frames... +[2023-03-07 15:49:31,983][383799] Decorrelating experience for 960 frames... +[2023-03-07 15:49:32,050][382326] Decorrelating experience for 928 frames... +[2023-03-07 15:49:32,058][473018] Decorrelating experience for 32 frames... +[2023-03-07 15:49:32,062][470479] Decorrelating experience for 64 frames... +[2023-03-07 15:49:32,084][470096] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,083][236315] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 236315], exiting... +[2023-03-07 15:49:32,085][470198] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,085][472746] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,084][470096] EvtLoop [rollout_proc5_evt_loop, process=rollout_proc5] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,085][470096] Unhandled exception in evt loop rollout_proc5_evt_loop +[2023-03-07 15:49:32,086][469655] Stopping Batcher_0... +[2023-03-07 15:49:32,086][382419] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,086][469655] Loop batcher_evt_loop terminating... +[2023-03-07 15:49:32,086][472534] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,086][470295] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,087][469655] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000981_4018176.pth... +[2023-03-07 15:49:32,086][486568] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,085][470198] EvtLoop [rollout_proc27_evt_loop, process=rollout_proc27] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,087][470198] Unhandled exception in evt loop rollout_proc27_evt_loop +[2023-03-07 15:49:32,087][471605] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,086][236315] Runner profile tree view: +main_loop: 12.1030 +[2023-03-07 15:49:32,086][470479] EvtLoop [rollout_proc90_evt_loop, process=rollout_proc90] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 439, in _reset + observations, rew, terminated, truncated, info = e.step(actions) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 129, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 115, in step + obs, rew, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 33, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 384, in step + observation, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 88, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 319, in step + return self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 54, in step + obs, reward, terminated, truncated, info = self.env.step(action) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 452, in step + reward = self.game.make_action(actions_flattened, self.skip_frames) +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. +[2023-03-07 15:49:32,087][470479] Unhandled exception Signal SIGINT received. ViZDoom instance has been closed. in evt loop rollout_proc90_evt_loop +[2023-03-07 15:49:32,086][472746] EvtLoop [rollout_proc100_evt_loop, process=rollout_proc100] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,087][470103] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,086][382419] EvtLoop [rollout_proc10_evt_loop, process=rollout_proc10] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,088][472746] Unhandled exception in evt loop rollout_proc100_evt_loop +[2023-03-07 15:49:32,088][382419] Unhandled exception in evt loop rollout_proc10_evt_loop +[2023-03-07 15:49:32,087][470295] EvtLoop [rollout_proc60_evt_loop, process=rollout_proc60] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,087][472534] EvtLoop [rollout_proc117_evt_loop, process=rollout_proc117] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,088][470295] Unhandled exception in evt loop rollout_proc60_evt_loop +[2023-03-07 15:49:32,087][236315] Collected {0: 4018176}, FPS: 0.0 +[2023-03-07 15:49:32,088][472534] Unhandled exception in evt loop rollout_proc117_evt_loop +[2023-03-07 15:49:32,087][471605] EvtLoop [rollout_proc107_evt_loop, process=rollout_proc107] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,088][471605] Unhandled exception in evt loop rollout_proc107_evt_loop +[2023-03-07 15:49:32,087][486568] EvtLoop [rollout_proc1_evt_loop, process=rollout_proc1] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,089][486568] Unhandled exception in evt loop rollout_proc1_evt_loop +[2023-03-07 15:49:32,088][470103] EvtLoop [rollout_proc14_evt_loop, process=rollout_proc14] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,091][470103] Unhandled exception in evt loop rollout_proc14_evt_loop +[2023-03-07 15:49:32,090][470099] VizDoom game.init() threw an exception SignalException('Signal SIGINT received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,092][470099] EvtLoop [rollout_proc11_evt_loop, process=rollout_proc11] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGINT received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,107][470099] Unhandled exception in evt loop rollout_proc11_evt_loop +[2023-03-07 15:49:32,130][470095] Weights refcount: 2 0 +[2023-03-07 15:49:32,131][470095] Stopping InferenceWorker_p0-w0... +[2023-03-07 15:49:32,131][470095] Loop inference_proc0-0_evt_loop terminating... +[2023-03-07 15:49:32,136][469655] Stopping LearnerWorker_p0... +[2023-03-07 15:49:32,136][469655] Loop learner_proc0_evt_loop terminating... +[2023-03-07 15:49:32,255][382439] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,256][382439] EvtLoop [rollout_proc65_evt_loop, process=rollout_proc65] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,259][382439] Unhandled exception in evt loop rollout_proc65_evt_loop +[2023-03-07 15:49:32,273][470110] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,277][470254] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,277][470211] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,274][470110] EvtLoop [rollout_proc17_evt_loop, process=rollout_proc17] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,278][470110] Unhandled exception in evt loop rollout_proc17_evt_loop +[2023-03-07 15:49:32,277][470254] EvtLoop [rollout_proc40_evt_loop, process=rollout_proc40] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,281][470254] Unhandled exception in evt loop rollout_proc40_evt_loop +[2023-03-07 15:49:32,277][470211] EvtLoop [rollout_proc18_evt_loop, process=rollout_proc18] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,281][470211] Unhandled exception in evt loop rollout_proc18_evt_loop +[2023-03-07 15:49:32,288][470290] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,288][470705] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,288][470290] EvtLoop [rollout_proc48_evt_loop, process=rollout_proc48] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,289][470705] EvtLoop [rollout_proc55_evt_loop, process=rollout_proc55] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,292][470290] Unhandled exception in evt loop rollout_proc48_evt_loop +[2023-03-07 15:49:32,292][470705] Unhandled exception in evt loop rollout_proc55_evt_loop +[2023-03-07 15:49:32,293][471190] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,293][471190] EvtLoop [rollout_proc92_evt_loop, process=rollout_proc92] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,297][471190] Unhandled exception in evt loop rollout_proc92_evt_loop +[2023-03-07 15:49:32,301][472901] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,301][472901] EvtLoop [rollout_proc103_evt_loop, process=rollout_proc103] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,305][472901] Unhandled exception in evt loop rollout_proc103_evt_loop +[2023-03-07 15:49:32,310][470438] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,311][470438] EvtLoop [rollout_proc72_evt_loop, process=rollout_proc72] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,323][470438] Unhandled exception in evt loop rollout_proc72_evt_loop +[2023-03-07 15:49:32,333][472899] VizDoom game.init() threw an exception SignalException('Signal SIGTERM received. ViZDoom instance has been closed.'). Terminate process... +[2023-03-07 15:49:32,333][472899] EvtLoop [rollout_proc126_evt_loop, process=rollout_proc126] unhandled exception in slot='init' connected to emitter=Emitter(object_id='Sampler', signal_name='_inference_workers_initialized'), args=() +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 228, in _game_init + self.game.init() +vizdoom.vizdoom.SignalException: Signal SIGTERM received. ViZDoom instance has been closed. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/signal_slot/signal_slot.py", line 355, in _process_signal + slot_callable(*args) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/rollout_worker.py", line 150, in init + env_runner.init(self.timing) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 418, in init + self._reset() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/sampling/non_batched_sampling.py", line 430, in _reset + observations, info = e.reset(seed=seed) # new way of doing seeding since Gym 0.26.0 + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 125, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/algo/utils/make_env.py", line 110, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/scenario_wrappers/gathering_reward_shaping.py", line 30, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 379, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sample_factory/envs/env_wrappers.py", line 84, in reset + obs, info = self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/gym/core.py", line 323, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/wrappers/multiplayer_stats.py", line 51, in reset + return self.env.reset(**kwargs) + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 323, in reset + self._ensure_initialized() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 274, in _ensure_initialized + self.initialize() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 269, in initialize + self._game_init() + File "/home/rolo/.pyenv/versions/3.10.9/envs/samplefactory/lib/python3.10/site-packages/sf_examples/vizdoom/doom/doom_gym.py", line 244, in _game_init + raise EnvCriticalError() +sample_factory.envs.env_utils.EnvCriticalError +[2023-03-07 15:49:32,343][472899] Unhandled exception in evt loop rollout_proc126_evt_loop +[2023-03-07 15:49:42,764][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json... +[2023-03-07 15:49:42,765][488352] Rollout worker 0 uses device cpu +[2023-03-07 15:49:42,766][488352] Rollout worker 1 uses device cpu +[2023-03-07 15:49:42,766][488352] Rollout worker 2 uses device cpu +[2023-03-07 15:49:42,767][488352] Rollout worker 3 uses device cpu +[2023-03-07 15:49:42,767][488352] Rollout worker 4 uses device cpu +[2023-03-07 15:49:42,767][488352] Rollout worker 5 uses device cpu +[2023-03-07 15:49:42,768][488352] Rollout worker 6 uses device cpu +[2023-03-07 15:49:42,768][488352] Rollout worker 7 uses device cpu +[2023-03-07 15:49:42,769][488352] Rollout worker 8 uses device cpu +[2023-03-07 15:49:42,769][488352] Rollout worker 9 uses device cpu +[2023-03-07 15:49:42,769][488352] Rollout worker 10 uses device cpu +[2023-03-07 15:49:42,770][488352] Rollout worker 11 uses device cpu +[2023-03-07 15:49:42,770][488352] Rollout worker 12 uses device cpu +[2023-03-07 15:49:42,771][488352] Rollout worker 13 uses device cpu +[2023-03-07 15:49:42,772][488352] Rollout worker 14 uses device cpu +[2023-03-07 15:49:42,772][488352] Rollout worker 15 uses device cpu +[2023-03-07 15:49:42,773][488352] Rollout worker 16 uses device cpu +[2023-03-07 15:49:42,773][488352] Rollout worker 17 uses device cpu +[2023-03-07 15:49:42,773][488352] Rollout worker 18 uses device cpu +[2023-03-07 15:49:42,774][488352] Rollout worker 19 uses device cpu +[2023-03-07 15:49:42,774][488352] Rollout worker 20 uses device cpu +[2023-03-07 15:49:42,774][488352] Rollout worker 21 uses device cpu +[2023-03-07 15:49:42,775][488352] Rollout worker 22 uses device cpu +[2023-03-07 15:49:42,775][488352] Rollout worker 23 uses device cpu +[2023-03-07 15:49:42,776][488352] Rollout worker 24 uses device cpu +[2023-03-07 15:49:42,776][488352] Rollout worker 25 uses device cpu +[2023-03-07 15:49:42,776][488352] Rollout worker 26 uses device cpu +[2023-03-07 15:49:42,777][488352] Rollout worker 27 uses device cpu +[2023-03-07 15:49:42,777][488352] Rollout worker 28 uses device cpu +[2023-03-07 15:49:42,778][488352] Rollout worker 29 uses device cpu +[2023-03-07 15:49:42,778][488352] Rollout worker 30 uses device cpu +[2023-03-07 15:49:42,780][488352] Rollout worker 31 uses device cpu +[2023-03-07 15:49:42,780][488352] Rollout worker 32 uses device cpu +[2023-03-07 15:49:42,781][488352] Rollout worker 33 uses device cpu +[2023-03-07 15:49:42,781][488352] Rollout worker 34 uses device cpu +[2023-03-07 15:49:42,782][488352] Rollout worker 35 uses device cpu +[2023-03-07 15:49:42,782][488352] Rollout worker 36 uses device cpu +[2023-03-07 15:49:42,783][488352] Rollout worker 37 uses device cpu +[2023-03-07 15:49:42,783][488352] Rollout worker 38 uses device cpu +[2023-03-07 15:49:42,784][488352] Rollout worker 39 uses device cpu +[2023-03-07 15:49:42,784][488352] Rollout worker 40 uses device cpu +[2023-03-07 15:49:42,784][488352] Rollout worker 41 uses device cpu +[2023-03-07 15:49:42,785][488352] Rollout worker 42 uses device cpu +[2023-03-07 15:49:42,785][488352] Rollout worker 43 uses device cpu +[2023-03-07 15:49:42,785][488352] Rollout worker 44 uses device cpu +[2023-03-07 15:49:42,787][488352] Rollout worker 45 uses device cpu +[2023-03-07 15:49:42,787][488352] Rollout worker 46 uses device cpu +[2023-03-07 15:49:42,787][488352] Rollout worker 47 uses device cpu +[2023-03-07 15:49:42,788][488352] Rollout worker 48 uses device cpu +[2023-03-07 15:49:42,788][488352] Rollout worker 49 uses device cpu +[2023-03-07 15:49:42,788][488352] Rollout worker 50 uses device cpu +[2023-03-07 15:49:42,789][488352] Rollout worker 51 uses device cpu +[2023-03-07 15:49:42,789][488352] Rollout worker 52 uses device cpu +[2023-03-07 15:49:42,790][488352] Rollout worker 53 uses device cpu +[2023-03-07 15:49:42,790][488352] Rollout worker 54 uses device cpu +[2023-03-07 15:49:42,790][488352] Rollout worker 55 uses device cpu +[2023-03-07 15:49:42,791][488352] Rollout worker 56 uses device cpu +[2023-03-07 15:49:42,791][488352] Rollout worker 57 uses device cpu +[2023-03-07 15:49:42,792][488352] Rollout worker 58 uses device cpu +[2023-03-07 15:49:42,792][488352] Rollout worker 59 uses device cpu +[2023-03-07 15:49:42,792][488352] Rollout worker 60 uses device cpu +[2023-03-07 15:49:42,793][488352] Rollout worker 61 uses device cpu +[2023-03-07 15:49:42,793][488352] Rollout worker 62 uses device cpu +[2023-03-07 15:49:42,794][488352] Rollout worker 63 uses device cpu +[2023-03-07 15:49:42,794][488352] Rollout worker 64 uses device cpu +[2023-03-07 15:49:42,796][488352] Rollout worker 65 uses device cpu +[2023-03-07 15:49:42,796][488352] Rollout worker 66 uses device cpu +[2023-03-07 15:49:42,797][488352] Rollout worker 67 uses device cpu +[2023-03-07 15:49:42,797][488352] Rollout worker 68 uses device cpu +[2023-03-07 15:49:42,797][488352] Rollout worker 69 uses device cpu +[2023-03-07 15:49:42,798][488352] Rollout worker 70 uses device cpu +[2023-03-07 15:49:42,798][488352] Rollout worker 71 uses device cpu +[2023-03-07 15:49:42,799][488352] Rollout worker 72 uses device cpu +[2023-03-07 15:49:42,800][488352] Rollout worker 73 uses device cpu +[2023-03-07 15:49:42,800][488352] Rollout worker 74 uses device cpu +[2023-03-07 15:49:42,800][488352] Rollout worker 75 uses device cpu +[2023-03-07 15:49:42,801][488352] Rollout worker 76 uses device cpu +[2023-03-07 15:49:42,801][488352] Rollout worker 77 uses device cpu +[2023-03-07 15:49:42,802][488352] Rollout worker 78 uses device cpu +[2023-03-07 15:49:42,802][488352] Rollout worker 79 uses device cpu +[2023-03-07 15:49:42,803][488352] Rollout worker 80 uses device cpu +[2023-03-07 15:49:42,803][488352] Rollout worker 81 uses device cpu +[2023-03-07 15:49:42,804][488352] Rollout worker 82 uses device cpu +[2023-03-07 15:49:42,804][488352] Rollout worker 83 uses device cpu +[2023-03-07 15:49:42,804][488352] Rollout worker 84 uses device cpu +[2023-03-07 15:49:42,805][488352] Rollout worker 85 uses device cpu +[2023-03-07 15:49:42,805][488352] Rollout worker 86 uses device cpu +[2023-03-07 15:49:42,806][488352] Rollout worker 87 uses device cpu +[2023-03-07 15:49:42,806][488352] Rollout worker 88 uses device cpu +[2023-03-07 15:49:42,806][488352] Rollout worker 89 uses device cpu +[2023-03-07 15:49:42,807][488352] Rollout worker 90 uses device cpu +[2023-03-07 15:49:42,807][488352] Rollout worker 91 uses device cpu +[2023-03-07 15:49:42,807][488352] Rollout worker 92 uses device cpu +[2023-03-07 15:49:42,808][488352] Rollout worker 93 uses device cpu +[2023-03-07 15:49:42,808][488352] Rollout worker 94 uses device cpu +[2023-03-07 15:49:42,809][488352] Rollout worker 95 uses device cpu +[2023-03-07 15:49:42,809][488352] Rollout worker 96 uses device cpu +[2023-03-07 15:49:42,809][488352] Rollout worker 97 uses device cpu +[2023-03-07 15:49:42,810][488352] Rollout worker 98 uses device cpu +[2023-03-07 15:49:42,812][488352] Rollout worker 99 uses device cpu +[2023-03-07 15:49:42,812][488352] Rollout worker 100 uses device cpu +[2023-03-07 15:49:42,813][488352] Rollout worker 101 uses device cpu +[2023-03-07 15:49:42,813][488352] Rollout worker 102 uses device cpu +[2023-03-07 15:49:42,813][488352] Rollout worker 103 uses device cpu +[2023-03-07 15:49:42,814][488352] Rollout worker 104 uses device cpu +[2023-03-07 15:49:42,815][488352] Rollout worker 105 uses device cpu +[2023-03-07 15:49:42,815][488352] Rollout worker 106 uses device cpu +[2023-03-07 15:49:42,815][488352] Rollout worker 107 uses device cpu +[2023-03-07 15:49:42,816][488352] Rollout worker 108 uses device cpu +[2023-03-07 15:49:42,816][488352] Rollout worker 109 uses device cpu +[2023-03-07 15:49:42,817][488352] Rollout worker 110 uses device cpu +[2023-03-07 15:49:42,817][488352] Rollout worker 111 uses device cpu +[2023-03-07 15:49:42,817][488352] Rollout worker 112 uses device cpu +[2023-03-07 15:49:42,818][488352] Rollout worker 113 uses device cpu +[2023-03-07 15:49:42,819][488352] Rollout worker 114 uses device cpu +[2023-03-07 15:49:42,819][488352] Rollout worker 115 uses device cpu +[2023-03-07 15:49:42,820][488352] Rollout worker 116 uses device cpu +[2023-03-07 15:49:42,820][488352] Rollout worker 117 uses device cpu +[2023-03-07 15:49:42,820][488352] Rollout worker 118 uses device cpu +[2023-03-07 15:49:42,821][488352] Rollout worker 119 uses device cpu +[2023-03-07 15:49:42,821][488352] Rollout worker 120 uses device cpu +[2023-03-07 15:49:42,822][488352] Rollout worker 121 uses device cpu +[2023-03-07 15:49:42,822][488352] Rollout worker 122 uses device cpu +[2023-03-07 15:49:42,822][488352] Rollout worker 123 uses device cpu +[2023-03-07 15:49:42,823][488352] Rollout worker 124 uses device cpu +[2023-03-07 15:49:42,823][488352] Rollout worker 125 uses device cpu +[2023-03-07 15:49:42,823][488352] Rollout worker 126 uses device cpu +[2023-03-07 15:49:42,824][488352] Rollout worker 127 uses device cpu +[2023-03-07 15:49:42,994][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:42,995][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:49:43,346][488352] Starting all processes... +[2023-03-07 15:49:43,347][488352] Starting process learner_proc0 +[2023-03-07 15:49:44,241][488352] Starting all processes... +[2023-03-07 15:49:44,243][488625] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:44,244][488625] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0 +[2023-03-07 15:49:44,248][488352] Starting process inference_proc0-0 +[2023-03-07 15:49:44,249][488352] Starting process rollout_proc2 +[2023-03-07 15:49:44,253][488625] Num visible devices: 1 +[2023-03-07 15:49:44,249][488352] Starting process rollout_proc5 +[2023-03-07 15:49:44,257][488625] Starting seed is not provided +[2023-03-07 15:49:44,257][488625] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:44,258][488625] Initializing actor-critic model on device cuda:0 +[2023-03-07 15:49:44,258][488625] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:49:44,258][488625] RunningMeanStd input shape: (1,) +[2023-03-07 15:49:44,249][488352] Starting process rollout_proc8 +[2023-03-07 15:49:44,250][488352] Starting process rollout_proc11 +[2023-03-07 15:49:44,250][488352] Starting process rollout_proc14 +[2023-03-07 15:49:44,268][488625] ConvEncoder: input_channels=3 +[2023-03-07 15:49:44,251][488352] Starting process rollout_proc17 +[2023-03-07 15:49:44,252][488352] Starting process rollout_proc20 +[2023-03-07 15:49:44,255][488352] Starting process rollout_proc23 +[2023-03-07 15:49:44,256][488352] Starting process rollout_proc26 +[2023-03-07 15:49:44,256][488352] Starting process rollout_proc29 +[2023-03-07 15:49:44,258][488352] Starting process rollout_proc32 +[2023-03-07 15:49:44,258][488352] Starting process rollout_proc35 +[2023-03-07 15:49:44,259][488352] Starting process rollout_proc38 +[2023-03-07 15:49:44,262][488352] Starting process rollout_proc41 +[2023-03-07 15:49:44,264][488352] Starting process rollout_proc44 +[2023-03-07 15:49:44,292][488352] Starting process rollout_proc3 +[2023-03-07 15:49:44,299][488352] Starting process rollout_proc9 +[2023-03-07 15:49:44,300][488352] Starting process rollout_proc6 +[2023-03-07 15:49:44,301][488352] Starting process rollout_proc12 +[2023-03-07 15:49:44,308][488352] Starting process rollout_proc15 +[2023-03-07 15:49:44,311][488352] Starting process rollout_proc18 +[2023-03-07 15:49:44,325][488352] Starting process rollout_proc21 +[2023-03-07 15:49:44,331][488352] Starting process rollout_proc27 +[2023-03-07 15:49:44,333][488352] Starting process rollout_proc24 +[2023-03-07 15:49:44,335][488352] Starting process rollout_proc30 +[2023-03-07 15:49:44,356][488352] Starting process rollout_proc39 +[2023-03-07 15:49:44,356][488352] Starting process rollout_proc33 +[2023-03-07 15:49:44,373][488625] Conv encoder output size: 512 +[2023-03-07 15:49:44,367][488352] Starting process rollout_proc42 +[2023-03-07 15:49:44,375][488625] Policy head output size: 512 +[2023-03-07 15:49:44,371][488352] Starting process rollout_proc45 +[2023-03-07 15:49:44,372][488352] Starting process rollout_proc36 +[2023-03-07 15:49:44,387][488625] Created Actor Critic model with architecture: +[2023-03-07 15:49:44,379][488352] Starting process rollout_proc10 +[2023-03-07 15:49:44,387][488625] ActorCriticSharedWeights( + (obs_normalizer): ObservationNormalizer( + (running_mean_std): RunningMeanStdDictInPlace( + (running_mean_std): ModuleDict( + (obs): RunningMeanStdInPlace() + ) + ) + ) + (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace) + (encoder): VizdoomEncoder( + (basic_encoder): ConvEncoder( + (enc): RecursiveScriptModule( + original_name=ConvEncoderImpl + (conv_head): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Conv2d) + (1): RecursiveScriptModule(original_name=ELU) + (2): RecursiveScriptModule(original_name=Conv2d) + (3): RecursiveScriptModule(original_name=ELU) + (4): RecursiveScriptModule(original_name=Conv2d) + (5): RecursiveScriptModule(original_name=ELU) + ) + (mlp_layers): RecursiveScriptModule( + original_name=Sequential + (0): RecursiveScriptModule(original_name=Linear) + (1): RecursiveScriptModule(original_name=ELU) + ) + ) + ) + ) + (core): ModelCoreRNN( + (core): GRU(512, 512) + ) + (decoder): MlpDecoder( + (mlp): Identity() + ) + (critic_linear): Linear(in_features=512, out_features=1, bias=True) + (action_parameterization): ActionParameterizationDefault( + (distribution_linear): Linear(in_features=512, out_features=5, bias=True) + ) +) +[2023-03-07 15:49:44,383][488352] Starting process rollout_proc4 +[2023-03-07 15:49:44,387][488352] Starting process rollout_proc13 +[2023-03-07 15:49:44,390][488352] Starting process rollout_proc7 +[2023-03-07 15:49:44,394][488352] Starting process rollout_proc16 +[2023-03-07 15:49:44,407][488352] Starting process rollout_proc19 +[2023-03-07 15:49:44,408][488352] Starting process rollout_proc22 +[2023-03-07 15:49:44,411][488352] Starting process rollout_proc28 +[2023-03-07 15:49:44,413][488352] Starting process rollout_proc25 +[2023-03-07 15:49:44,418][488352] Starting process rollout_proc31 +[2023-03-07 15:49:44,420][488352] Starting process rollout_proc40 +[2023-03-07 15:49:44,425][488352] Starting process rollout_proc34 +[2023-03-07 15:49:44,432][488352] Starting process rollout_proc43 +[2023-03-07 15:49:44,437][488352] Starting process rollout_proc37 +[2023-03-07 15:49:44,437][488352] Starting process rollout_proc47 +[2023-03-07 15:49:44,447][488352] Starting process rollout_proc46 +[2023-03-07 15:49:44,447][488352] Starting process rollout_proc50 +[2023-03-07 15:49:44,449][488352] Starting process rollout_proc53 +[2023-03-07 15:49:44,453][488352] Starting process rollout_proc56 +[2023-03-07 15:49:44,454][488352] Starting process rollout_proc59 +[2023-03-07 15:49:44,461][488352] Starting process rollout_proc62 +[2023-03-07 15:49:44,467][488352] Starting process rollout_proc65 +[2023-03-07 15:49:44,467][488352] Starting process rollout_proc68 +[2023-03-07 15:49:44,477][488352] Starting process rollout_proc71 +[2023-03-07 15:49:44,478][488352] Starting process rollout_proc74 +[2023-03-07 15:49:44,490][488352] Starting process rollout_proc77 +[2023-03-07 15:49:44,494][488352] Starting process rollout_proc80 +[2023-03-07 15:49:44,504][488352] Starting process rollout_proc83 +[2023-03-07 15:49:44,506][488352] Starting process rollout_proc86 +[2023-03-07 15:49:44,510][488352] Starting process rollout_proc48 +[2023-03-07 15:49:44,522][488352] Starting process rollout_proc51 +[2023-03-07 15:49:44,528][488352] Starting process rollout_proc89 +[2023-03-07 15:49:44,534][488352] Starting process rollout_proc54 +[2023-03-07 15:49:44,538][488352] Starting process rollout_proc57 +[2023-03-07 15:49:44,539][488352] Starting process rollout_proc60 +[2023-03-07 15:49:44,546][488352] Starting process rollout_proc63 +[2023-03-07 15:49:44,557][488352] Starting process rollout_proc72 +[2023-03-07 15:49:44,564][488352] Starting process rollout_proc66 +[2023-03-07 15:49:44,565][488352] Starting process rollout_proc69 +[2023-03-07 15:49:44,573][488352] Starting process rollout_proc75 +[2023-03-07 15:49:44,578][488352] Starting process rollout_proc78 +[2023-03-07 15:49:44,586][488352] Starting process rollout_proc81 +[2023-03-07 15:49:44,591][488352] Starting process rollout_proc84 +[2023-03-07 15:49:44,597][488352] Starting process rollout_proc87 +[2023-03-07 15:49:44,609][488352] Starting process rollout_proc49 +[2023-03-07 15:49:44,613][488352] Starting process rollout_proc58 +[2023-03-07 15:49:44,682][488352] Starting process rollout_proc79 +[2023-03-07 15:49:44,683][488352] Starting process rollout_proc85 +[2023-03-07 15:49:44,684][488352] Starting process rollout_proc52 +[2023-03-07 15:49:44,686][488352] Starting process rollout_proc61 +[2023-03-07 15:49:44,692][488352] Starting process rollout_proc90 +[2023-03-07 15:49:44,692][488352] Starting process rollout_proc76 +[2023-03-07 15:49:44,692][488352] Starting process rollout_proc82 +[2023-03-07 15:49:44,695][488352] Starting process rollout_proc55 +[2023-03-07 15:49:44,696][488352] Starting process rollout_proc67 +[2023-03-07 15:49:44,701][488352] Starting process rollout_proc73 +[2023-03-07 15:49:44,705][488352] Starting process rollout_proc88 +[2023-03-07 15:49:44,710][488352] Starting process rollout_proc70 +[2023-03-07 15:49:44,712][488352] Starting process rollout_proc64 +[2023-03-07 15:49:44,713][488352] Starting process rollout_proc92 +[2023-03-07 15:49:44,714][488352] Starting process rollout_proc95 +[2023-03-07 15:49:44,750][488352] Starting process rollout_proc98 +[2023-03-07 15:49:44,762][488352] Starting process rollout_proc101 +[2023-03-07 15:49:44,764][488352] Starting process rollout_proc104 +[2023-03-07 15:49:44,767][488352] Starting process rollout_proc107 +[2023-03-07 15:49:44,771][488352] Starting process rollout_proc91 +[2023-03-07 15:49:44,783][488352] Starting process rollout_proc110 +[2023-03-07 15:49:44,799][488352] Starting process rollout_proc113 +[2023-03-07 15:49:44,805][488352] Starting process rollout_proc116 +[2023-03-07 15:49:44,808][488352] Starting process rollout_proc119 +[2023-03-07 15:49:44,810][488352] Starting process rollout_proc122 +[2023-03-07 15:49:44,830][488352] Starting process rollout_proc93 +[2023-03-07 15:49:44,832][488352] Starting process rollout_proc125 +[2023-03-07 15:49:44,902][488352] Starting process rollout_proc96 +[2023-03-07 15:49:44,923][488352] Starting process rollout_proc99 +[2023-03-07 15:49:44,926][488352] Starting process rollout_proc105 +[2023-03-07 15:49:44,954][488352] Starting process rollout_proc111 +[2023-03-07 15:49:44,961][488352] Starting process rollout_proc108 +[2023-03-07 15:49:44,985][488352] Starting process rollout_proc120 +[2023-03-07 15:49:44,986][488352] Starting process rollout_proc94 +[2023-03-07 15:49:44,986][488352] Starting process rollout_proc123 +[2023-03-07 15:49:44,986][488352] Starting process rollout_proc102 +[2023-03-07 15:49:44,989][488352] Starting process rollout_proc114 +[2023-03-07 15:49:45,001][488352] Starting process rollout_proc97 +[2023-03-07 15:49:45,001][488352] Starting process rollout_proc126 +[2023-03-07 15:49:45,004][488352] Starting process rollout_proc100 +[2023-03-07 15:49:45,023][488352] Starting process rollout_proc117 +[2023-03-07 15:49:45,045][488352] Starting process rollout_proc106 +[2023-03-07 15:49:45,125][488352] Starting process rollout_proc109 +[2023-03-07 15:49:45,138][488352] Starting process rollout_proc112 +[2023-03-07 15:49:45,162][488352] Starting process rollout_proc118 +[2023-03-07 15:49:45,186][488352] Starting process rollout_proc121 +[2023-03-07 15:49:45,187][488352] Starting process rollout_proc115 +[2023-03-07 15:49:45,188][488352] Starting process rollout_proc127 +[2023-03-07 15:49:45,188][488352] Starting process rollout_proc103 +[2023-03-07 15:49:45,302][488352] Starting process rollout_proc124 +[2023-03-07 15:49:46,614][488789] Worker 14 uses CPU cores [14] +[2023-03-07 15:49:46,675][488787] Worker 8 uses CPU cores [8] +[2023-03-07 15:49:46,751][488786] Worker 5 uses CPU cores [5] +[2023-03-07 15:49:46,869][488858] Worker 32 uses CPU cores [32] +[2023-03-07 15:49:46,899][488785] Worker 2 uses CPU cores [2] +[2023-03-07 15:49:46,961][488859] Worker 38 uses CPU cores [38] +[2023-03-07 15:49:47,047][488910] Worker 80 uses CPU cores [80] +[2023-03-07 15:49:47,050][488793] Worker 26 uses CPU cores [26] +[2023-03-07 15:49:47,087][488875] Worker 24 uses CPU cores [24] +[2023-03-07 15:49:47,104][488868] Worker 3 uses CPU cores [3] +[2023-03-07 15:49:47,127][488872] Worker 18 uses CPU cores [18] +[2023-03-07 15:49:47,136][488874] Worker 27 uses CPU cores [27] +[2023-03-07 15:49:47,185][488877] Worker 39 uses CPU cores [39] +[2023-03-07 15:49:47,195][488791] Worker 20 uses CPU cores [20] +[2023-03-07 15:49:47,228][488352] Starting process rollout_proc0 +[2023-03-07 15:49:47,264][488783] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:47,265][488783] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0 +[2023-03-07 15:49:47,279][488882] Worker 36 uses CPU cores [36] +[2023-03-07 15:49:47,303][488783] Num visible devices: 1 +[2023-03-07 15:49:47,331][488352] Starting process rollout_proc1 +[2023-03-07 15:49:47,337][488870] Worker 6 uses CPU cores [6] +[2023-03-07 15:49:47,363][488792] Worker 23 uses CPU cores [23] +[2023-03-07 15:49:47,387][488794] Worker 29 uses CPU cores [29] +[2023-03-07 15:49:47,391][488790] Worker 17 uses CPU cores [17] +[2023-03-07 15:49:47,458][488889] Worker 19 uses CPU cores [19] +[2023-03-07 15:49:47,485][488863] Worker 35 uses CPU cores [35] +[2023-03-07 15:49:47,495][488904] Worker 62 uses CPU cores [62] +[2023-03-07 15:49:47,529][488894] Worker 40 uses CPU cores [40] +[2023-03-07 15:49:47,530][488866] Worker 44 uses CPU cores [44] +[2023-03-07 15:49:47,532][488896] Worker 43 uses CPU cores [43] +[2023-03-07 15:49:47,549][488878] Worker 33 uses CPU cores [33] +[2023-03-07 15:49:47,586][488883] Worker 10 uses CPU cores [10] +[2023-03-07 15:49:47,594][488876] Worker 30 uses CPU cores [30] +[2023-03-07 15:49:47,612][488906] Worker 65 uses CPU cores [65] +[2023-03-07 15:49:47,617][488901] Worker 53 uses CPU cores [53] +[2023-03-07 15:49:47,623][488788] Worker 11 uses CPU cores [11] +[2023-03-07 15:49:47,697][488871] Worker 15 uses CPU cores [15] +[2023-03-07 15:49:47,702][488869] Worker 12 uses CPU cores [12] +[2023-03-07 15:49:47,708][488888] Worker 22 uses CPU cores [22] +[2023-03-07 15:49:47,713][488907] Worker 68 uses CPU cores [68] +[2023-03-07 15:49:47,737][488880] Worker 42 uses CPU cores [42] +[2023-03-07 15:49:47,738][488905] Worker 71 uses CPU cores [71] +[2023-03-07 15:49:47,750][488881] Worker 45 uses CPU cores [45] +[2023-03-07 15:49:47,752][488935] Worker 82 uses CPU cores [82] +[2023-03-07 15:49:47,767][488864] Worker 41 uses CPU cores [41] +[2023-03-07 15:49:47,768][488884] Worker 7 uses CPU cores [7] +[2023-03-07 15:49:47,768][488912] Worker 86 uses CPU cores [86] +[2023-03-07 15:49:47,778][488909] Worker 77 uses CPU cores [77] +[2023-03-07 15:49:47,774][488893] Worker 31 uses CPU cores [31] +[2023-03-07 15:49:47,782][488890] Worker 28 uses CPU cores [28] +[2023-03-07 15:49:47,786][488900] Worker 46 uses CPU cores [46] +[2023-03-07 15:49:47,798][488902] Worker 56 uses CPU cores [56] +[2023-03-07 15:49:47,806][488873] Worker 21 uses CPU cores [21] +[2023-03-07 15:49:47,806][488886] Worker 13 uses CPU cores [13] +[2023-03-07 15:49:47,838][488887] Worker 16 uses CPU cores [16] +[2023-03-07 15:49:47,862][488895] Worker 37 uses CPU cores [37] +[2023-03-07 15:49:47,862][488867] Worker 9 uses CPU cores [9] +[2023-03-07 15:49:47,866][488897] Worker 34 uses CPU cores [34] +[2023-03-07 15:49:47,874][488891] Worker 25 uses CPU cores [25] +[2023-03-07 15:49:47,874][488913] Worker 48 uses CPU cores [48] +[2023-03-07 15:49:47,878][488899] Worker 50 uses CPU cores [50] +[2023-03-07 15:49:47,886][488898] Worker 47 uses CPU cores [47] +[2023-03-07 15:49:47,890][488903] Worker 59 uses CPU cores [59] +[2023-03-07 15:49:47,902][488928] Worker 49 uses CPU cores [49] +[2023-03-07 15:49:47,918][488908] Worker 74 uses CPU cores [74] +[2023-03-07 15:49:47,922][488911] Worker 83 uses CPU cores [83] +[2023-03-07 15:49:47,934][488885] Worker 4 uses CPU cores [4] +[2023-03-07 15:49:47,974][489212] Worker 105 uses CPU cores [105] +[2023-03-07 15:49:47,976][488919] Worker 89 uses CPU cores [89] +[2023-03-07 15:49:47,986][488918] Worker 60 uses CPU cores [60] +[2023-03-07 15:49:47,990][488916] Worker 78 uses CPU cores [78] +[2023-03-07 15:49:47,998][488917] Worker 51 uses CPU cores [51] +[2023-03-07 15:49:48,002][489467] Worker 108 uses CPU cores [108] +[2023-03-07 15:49:48,010][488914] Worker 57 uses CPU cores [57] +[2023-03-07 15:49:48,014][489136] Worker 104 uses CPU cores [104] +[2023-03-07 15:49:48,034][488933] Worker 61 uses CPU cores [61] +[2023-03-07 15:49:48,047][489209] Worker 125 uses CPU cores [125] +[2023-03-07 15:49:48,065][488924] Worker 81 uses CPU cores [81] +[2023-03-07 15:49:48,066][488926] Worker 63 uses CPU cores [63] +[2023-03-07 15:49:48,069][488934] Worker 90 uses CPU cores [90] +[2023-03-07 15:49:48,074][488943] Worker 88 uses CPU cores [88] +[2023-03-07 15:49:48,092][489139] Worker 107 uses CPU cores [107] +[2023-03-07 15:49:48,102][488922] Worker 66 uses CPU cores [66] +[2023-03-07 15:49:48,106][488941] Worker 64 uses CPU cores [64] +[2023-03-07 15:49:48,112][488930] Worker 79 uses CPU cores [79] +[2023-03-07 15:49:48,125][488936] Worker 76 uses CPU cores [76] +[2023-03-07 15:49:48,130][488942] Worker 92 uses CPU cores [92] +[2023-03-07 15:49:48,130][488969] Worker 98 uses CPU cores [98] +[2023-03-07 15:49:48,143][489072] Worker 101 uses CPU cores [101] +[2023-03-07 15:49:48,154][488920] Worker 75 uses CPU cores [75] +[2023-03-07 15:49:48,158][488915] Worker 84 uses CPU cores [84] +[2023-03-07 15:49:48,158][488921] Worker 54 uses CPU cores [54] +[2023-03-07 15:49:48,165][489140] Worker 113 uses CPU cores [113] +[2023-03-07 15:49:48,166][489204] Worker 91 uses CPU cores [91] +[2023-03-07 15:49:48,174][489206] Worker 119 uses CPU cores [119] +[2023-03-07 15:49:48,183][488944] Worker 95 uses CPU cores [95] +[2023-03-07 15:49:48,187][488929] Worker 58 uses CPU cores [58] +[2023-03-07 15:49:48,197][488927] Worker 69 uses CPU cores [69] +[2023-03-07 15:49:48,213][488940] Worker 70 uses CPU cores [70] +[2023-03-07 15:49:48,222][488938] Worker 73 uses CPU cores [73] +[2023-03-07 15:49:48,226][488932] Worker 85 uses CPU cores [85] +[2023-03-07 15:49:48,228][488925] Worker 87 uses CPU cores [87] +[2023-03-07 15:49:48,264][489717] Worker 94 uses CPU cores [94] +[2023-03-07 15:49:48,274][488931] Worker 52 uses CPU cores [52] +[2023-03-07 15:49:48,308][489729] Worker 109 uses CPU cores [109] +[2023-03-07 15:49:48,314][489920] Worker 118 uses CPU cores [118] +[2023-03-07 15:49:48,326][488939] Worker 67 uses CPU cores [67] +[2023-03-07 15:49:48,331][488937] Worker 55 uses CPU cores [55] +[2023-03-07 15:49:48,336][489727] Worker 106 uses CPU cores [106] +[2023-03-07 15:49:48,346][489724] Worker 97 uses CPU cores [97] +[2023-03-07 15:49:48,352][489211] Worker 99 uses CPU cores [99] +[2023-03-07 15:49:48,355][490263] Worker 121 uses CPU cores [121] +[2023-03-07 15:49:48,363][489213] Worker 111 uses CPU cores [111] +[2023-03-07 15:49:48,366][489208] Worker 96 uses CPU cores [96] +[2023-03-07 15:49:48,370][489730] Worker 112 uses CPU cores [112] +[2023-03-07 15:49:48,377][489207] Worker 93 uses CPU cores [93] +[2023-03-07 15:49:48,383][490807] Worker 124 uses CPU cores [124] +[2023-03-07 15:49:48,386][489670] Worker 120 uses CPU cores [120] +[2023-03-07 15:49:48,389][489725] Worker 100 uses CPU cores [100] +[2023-03-07 15:49:48,396][488923] Worker 72 uses CPU cores [72] +[2023-03-07 15:49:48,406][489728] Worker 117 uses CPU cores [117] +[2023-03-07 15:49:48,413][489720] Worker 114 uses CPU cores [114] +[2023-03-07 15:49:48,428][489137] Worker 110 uses CPU cores [110] +[2023-03-07 15:49:48,450][490343] Worker 127 uses CPU cores [127] +[2023-03-07 15:49:48,461][490617] Worker 103 uses CPU cores [103] +[2023-03-07 15:49:48,485][490342] Worker 115 uses CPU cores [115] +[2023-03-07 15:49:48,488][489726] Worker 126 uses CPU cores [126] +[2023-03-07 15:49:48,498][489210] Worker 116 uses CPU cores [116] +[2023-03-07 15:49:48,516][489718] Worker 123 uses CPU cores [123] +[2023-03-07 15:49:48,522][489719] Worker 102 uses CPU cores [102] +[2023-03-07 15:49:48,562][489205] Worker 122 uses CPU cores [122] +[2023-03-07 15:49:48,761][497235] Worker 1 uses CPU cores [1] +[2023-03-07 15:49:48,912][488625] Using optimizer +[2023-03-07 15:49:48,913][488625] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000981_4018176.pth... +[2023-03-07 15:49:48,935][488625] Loading model from checkpoint +[2023-03-07 15:49:48,939][488625] Loaded experiment state at self.train_step=981, self.env_steps=4018176 +[2023-03-07 15:49:48,939][488625] Initialized policy 0 weights for model version 981 +[2023-03-07 15:49:48,940][488625] LearnerWorker_p0 finished initialization! +[2023-03-07 15:49:48,941][496838] Worker 0 uses CPU cores [0] +[2023-03-07 15:49:48,941][488625] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:49:48,998][488783] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:49:48,998][488783] RunningMeanStd input shape: (1,) +[2023-03-07 15:49:49,008][488783] ConvEncoder: input_channels=3 +[2023-03-07 15:49:49,083][488783] Conv encoder output size: 512 +[2023-03-07 15:49:49,083][488783] Policy head output size: 512 +[2023-03-07 15:49:49,959][488352] Inference worker 0-0 is ready! +[2023-03-07 15:49:49,960][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:49:50,040][489467] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,039][489207] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,041][488918] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,042][489205] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,042][488920] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,042][488899] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,042][490617] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,043][488969] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,043][488903] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,045][489140] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,045][489211] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,045][488894] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,045][489717] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][488874] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][489718] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][488921] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][488900] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][488919] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,046][488932] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,047][489209] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,047][489920] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,047][488915] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,048][488944] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,048][488936] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488872] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488904] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,048][488912] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488934] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488867] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488871] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488910] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488906] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][489670] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,049][488924] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][488916] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][488863] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][490342] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][489212] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][496838] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,050][489724] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488913] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488922] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488870] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488929] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488940] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488935] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488792] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][497235] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][489213] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][488884] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,051][488880] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][488931] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][488868] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][489210] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][488866] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][489720] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,052][489139] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488895] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488786] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488891] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][489725] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][489727] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488882] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488942] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488885] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][489072] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,053][488896] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,054][488876] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,054][489719] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,054][488859] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,054][488791] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,055][488875] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,056][488883] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,056][490343] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488873] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488887] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488930] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488788] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][489136] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488943] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,058][488928] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488793] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488893] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488878] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][489728] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488858] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488787] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488790] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,059][488881] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488937] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][489729] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488941] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488927] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488933] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488923] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488902] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488886] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][489206] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][490807] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,060][488869] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488789] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][489208] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488864] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488917] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488926] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488898] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488908] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488909] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][489730] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][490263] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][489137] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488905] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][489726] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][489204] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488785] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488939] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488877] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488901] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488911] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488907] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488888] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,061][488914] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488938] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488794] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488897] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,062][488889] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,063][488890] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,074][488925] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:49:50,258][488920] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,258][488918] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,403][489467] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,411][488900] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,412][488899] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,413][489205] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,413][489207] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,414][488867] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,418][488969] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,418][489211] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,435][488859] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,439][488943] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,585][488871] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,595][488873] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,601][488793] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,602][488939] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,622][488925] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,622][488859] Decorrelating experience for 32 frames... +[2023-03-07 15:49:50,631][488894] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,635][488915] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,673][488787] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,673][488923] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,733][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 4018176. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:49:50,789][489920] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,790][488873] Decorrelating experience for 32 frames... +[2023-03-07 15:49:50,791][489724] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,791][488935] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,819][488871] Decorrelating experience for 32 frames... +[2023-03-07 15:49:50,838][488903] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,839][489140] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,839][489209] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,874][488943] Decorrelating experience for 32 frames... +[2023-03-07 15:49:50,884][488912] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,970][488915] Decorrelating experience for 32 frames... +[2023-03-07 15:49:50,991][488904] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,994][488792] Decorrelating experience for 0 frames... +[2023-03-07 15:49:50,995][489210] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,011][488923] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,024][489140] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,068][488864] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,071][489209] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,092][488872] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,130][488935] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,158][489211] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,176][488903] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,181][488792] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,196][490343] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,220][488858] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,233][489670] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,248][489205] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,270][488912] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,288][488902] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,341][488922] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,343][488916] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,355][488867] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,368][488894] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,390][488932] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,404][488886] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,413][488876] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,431][490617] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,448][488905] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,477][489724] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,525][489920] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,544][489213] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,546][488895] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,551][488933] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,579][488916] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,592][488906] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,594][488866] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,632][488932] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,634][490343] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,665][490617] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,735][488904] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,738][489210] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,738][488858] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,743][488896] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,779][488942] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,781][488866] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,789][488944] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,829][488906] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,846][488880] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,853][489208] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,932][488902] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,934][489213] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,946][488787] Decorrelating experience for 32 frames... +[2023-03-07 15:49:51,963][489212] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,976][489136] Decorrelating experience for 0 frames... +[2023-03-07 15:49:51,984][488625] Signal inference workers to stop experience collection... +[2023-03-07 15:49:51,990][488783] InferenceWorker_p0-w0: stopping experience collection +[2023-03-07 15:49:51,993][488925] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,018][488863] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,024][488944] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,043][488919] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,048][489139] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,115][488969] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,134][488880] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,145][488921] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,148][497235] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,159][488938] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,205][488930] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,206][489206] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,250][488886] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,250][488793] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,296][488868] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,299][488934] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,326][488890] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,331][488922] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,344][489208] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,346][488874] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,392][489718] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,433][488913] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,443][488938] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,497][496838] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,506][488891] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,508][489137] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,527][488928] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,541][489730] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,548][488919] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,553][489720] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,583][488934] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,627][488942] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,629][489718] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,695][489137] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,702][489719] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,710][488891] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,711][488926] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,722][488913] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,732][488874] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,749][489727] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,777][488870] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,808][488888] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,814][490263] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,851][488625] Signal inference workers to resume experience collection... +[2023-03-07 15:49:52,852][488783] InferenceWorker_p0-w0: resuming experience collection +[2023-03-07 15:49:52,855][488625] Stopping Batcher_0... +[2023-03-07 15:49:52,855][488625] Loop batcher_evt_loop terminating... +[2023-03-07 15:49:52,863][488352] Component Batcher_0 stopped! +[2023-03-07 15:49:52,873][489205] Stopping RolloutWorker_w122... +[2023-03-07 15:49:52,873][488943] Stopping RolloutWorker_w88... +[2023-03-07 15:49:52,873][489205] Loop rollout_proc122_evt_loop terminating... +[2023-03-07 15:49:52,873][488873] Stopping RolloutWorker_w21... +[2023-03-07 15:49:52,873][488943] Loop rollout_proc88_evt_loop terminating... +[2023-03-07 15:49:52,873][488969] Stopping RolloutWorker_w98... +[2023-03-07 15:49:52,873][488903] Stopping RolloutWorker_w59... +[2023-03-07 15:49:52,873][488873] Loop rollout_proc21_evt_loop terminating... +[2023-03-07 15:49:52,873][488793] Stopping RolloutWorker_w26... +[2023-03-07 15:49:52,873][488969] Loop rollout_proc98_evt_loop terminating... +[2023-03-07 15:49:52,873][490343] Stopping RolloutWorker_w127... +[2023-03-07 15:49:52,873][488903] Loop rollout_proc59_evt_loop terminating... +[2023-03-07 15:49:52,873][488793] Loop rollout_proc26_evt_loop terminating... +[2023-03-07 15:49:52,874][490343] Loop rollout_proc127_evt_loop terminating... +[2023-03-07 15:49:52,874][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 15:49:52,875][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 15:49:52,875][488894] Stopping RolloutWorker_w40... +[2023-03-07 15:49:52,875][488894] Loop rollout_proc40_evt_loop terminating... +[2023-03-07 15:49:52,876][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 15:49:52,877][488874] Stopping RolloutWorker_w27... +[2023-03-07 15:49:52,877][488915] Stopping RolloutWorker_w84... +[2023-03-07 15:49:52,877][488874] Loop rollout_proc27_evt_loop terminating... +[2023-03-07 15:49:52,877][488915] Loop rollout_proc84_evt_loop terminating... +[2023-03-07 15:49:52,877][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 15:49:52,878][490617] Stopping RolloutWorker_w103... +[2023-03-07 15:49:52,878][488919] Stopping RolloutWorker_w89... +[2023-03-07 15:49:52,878][490617] Loop rollout_proc103_evt_loop terminating... +[2023-03-07 15:49:52,878][488919] Loop rollout_proc89_evt_loop terminating... +[2023-03-07 15:49:52,878][489140] Stopping RolloutWorker_w113... +[2023-03-07 15:49:52,879][489140] Loop rollout_proc113_evt_loop terminating... +[2023-03-07 15:49:52,879][488906] Stopping RolloutWorker_w65... +[2023-03-07 15:49:52,879][488906] Loop rollout_proc65_evt_loop terminating... +[2023-03-07 15:49:52,879][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 15:49:52,880][488880] Stopping RolloutWorker_w42... +[2023-03-07 15:49:52,880][488922] Stopping RolloutWorker_w66... +[2023-03-07 15:49:52,880][488880] Loop rollout_proc42_evt_loop terminating... +[2023-03-07 15:49:52,880][488902] Stopping RolloutWorker_w56... +[2023-03-07 15:49:52,880][488913] Stopping RolloutWorker_w48... +[2023-03-07 15:49:52,880][488922] Loop rollout_proc66_evt_loop terminating... +[2023-03-07 15:49:52,880][488942] Stopping RolloutWorker_w92... +[2023-03-07 15:49:52,880][489211] Stopping RolloutWorker_w99... +[2023-03-07 15:49:52,880][489213] Stopping RolloutWorker_w111... +[2023-03-07 15:49:52,880][489137] Stopping RolloutWorker_w110... +[2023-03-07 15:49:52,880][488912] Stopping RolloutWorker_w86... +[2023-03-07 15:49:52,880][488792] Stopping RolloutWorker_w23... +[2023-03-07 15:49:52,880][488902] Loop rollout_proc56_evt_loop terminating... +[2023-03-07 15:49:52,880][489724] Stopping RolloutWorker_w97... +[2023-03-07 15:49:52,881][488913] Loop rollout_proc48_evt_loop terminating... +[2023-03-07 15:49:52,880][489920] Stopping RolloutWorker_w118... +[2023-03-07 15:49:52,880][488858] Stopping RolloutWorker_w32... +[2023-03-07 15:49:52,880][488904] Stopping RolloutWorker_w62... +[2023-03-07 15:49:52,880][488944] Stopping RolloutWorker_w95... +[2023-03-07 15:49:52,880][489718] Stopping RolloutWorker_w123... +[2023-03-07 15:49:52,880][488886] Stopping RolloutWorker_w13... +[2023-03-07 15:49:52,880][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 15:49:52,881][488942] Loop rollout_proc92_evt_loop terminating... +[2023-03-07 15:49:52,881][489211] Loop rollout_proc99_evt_loop terminating... +[2023-03-07 15:49:52,881][489213] Loop rollout_proc111_evt_loop terminating... +[2023-03-07 15:49:52,881][489137] Loop rollout_proc110_evt_loop terminating... +[2023-03-07 15:49:52,881][488912] Loop rollout_proc86_evt_loop terminating... +[2023-03-07 15:49:52,880][488934] Stopping RolloutWorker_w90... +[2023-03-07 15:49:52,881][489724] Loop rollout_proc97_evt_loop terminating... +[2023-03-07 15:49:52,881][488792] Loop rollout_proc23_evt_loop terminating... +[2023-03-07 15:49:52,881][488858] Loop rollout_proc32_evt_loop terminating... +[2023-03-07 15:49:52,881][488904] Loop rollout_proc62_evt_loop terminating... +[2023-03-07 15:49:52,881][489718] Loop rollout_proc123_evt_loop terminating... +[2023-03-07 15:49:52,881][488944] Loop rollout_proc95_evt_loop terminating... +[2023-03-07 15:49:52,881][489920] Loop rollout_proc118_evt_loop terminating... +[2023-03-07 15:49:52,881][488886] Loop rollout_proc13_evt_loop terminating... +[2023-03-07 15:49:52,881][488934] Loop rollout_proc90_evt_loop terminating... +[2023-03-07 15:49:52,881][488935] Stopping RolloutWorker_w82... +[2023-03-07 15:49:52,882][488935] Loop rollout_proc82_evt_loop terminating... +[2023-03-07 15:49:52,881][488932] Stopping RolloutWorker_w85... +[2023-03-07 15:49:52,881][488925] Stopping RolloutWorker_w87... +[2023-03-07 15:49:52,882][488859] Stopping RolloutWorker_w38... +[2023-03-07 15:49:52,881][488866] Stopping RolloutWorker_w44... +[2023-03-07 15:49:52,882][488916] Stopping RolloutWorker_w78... +[2023-03-07 15:49:52,882][488871] Stopping RolloutWorker_w15... +[2023-03-07 15:49:52,882][489209] Stopping RolloutWorker_w125... +[2023-03-07 15:49:52,882][488938] Stopping RolloutWorker_w73... +[2023-03-07 15:49:52,882][488867] Stopping RolloutWorker_w9... +[2023-03-07 15:49:52,882][488923] Stopping RolloutWorker_w72... +[2023-03-07 15:49:52,882][488787] Stopping RolloutWorker_w8... +[2023-03-07 15:49:52,882][488932] Loop rollout_proc85_evt_loop terminating... +[2023-03-07 15:49:52,882][489208] Stopping RolloutWorker_w96... +[2023-03-07 15:49:52,882][488859] Loop rollout_proc38_evt_loop terminating... +[2023-03-07 15:49:52,882][488891] Stopping RolloutWorker_w25... +[2023-03-07 15:49:52,882][488925] Loop rollout_proc87_evt_loop terminating... +[2023-03-07 15:49:52,882][488866] Loop rollout_proc44_evt_loop terminating... +[2023-03-07 15:49:52,882][488871] Loop rollout_proc15_evt_loop terminating... +[2023-03-07 15:49:52,882][489209] Loop rollout_proc125_evt_loop terminating... +[2023-03-07 15:49:52,882][489208] Loop rollout_proc96_evt_loop terminating... +[2023-03-07 15:49:52,882][488916] Loop rollout_proc78_evt_loop terminating... +[2023-03-07 15:49:52,882][488923] Loop rollout_proc72_evt_loop terminating... +[2023-03-07 15:49:52,882][488787] Loop rollout_proc8_evt_loop terminating... +[2023-03-07 15:49:52,882][488867] Loop rollout_proc9_evt_loop terminating... +[2023-03-07 15:49:52,882][488938] Loop rollout_proc73_evt_loop terminating... +[2023-03-07 15:49:52,882][488891] Loop rollout_proc25_evt_loop terminating... +[2023-03-07 15:49:52,882][489210] Stopping RolloutWorker_w116... +[2023-03-07 15:49:52,883][489210] Loop rollout_proc116_evt_loop terminating... +[2023-03-07 15:49:52,883][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 15:49:52,884][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 15:49:52,885][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 15:49:52,886][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 15:49:52,888][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 15:49:52,889][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 15:49:52,890][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 15:49:52,891][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 15:49:52,892][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 15:49:52,893][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 15:49:52,894][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 15:49:52,895][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 15:49:52,896][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 15:49:52,897][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 15:49:52,899][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 15:49:52,900][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 15:49:52,901][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 15:49:52,902][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 15:49:52,903][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 15:49:52,904][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 15:49:52,905][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 15:49:52,906][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 15:49:52,907][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 15:49:52,909][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 15:49:52,909][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 15:49:52,911][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 15:49:52,912][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 15:49:52,913][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 15:49:52,914][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 15:49:52,915][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 15:49:52,916][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 15:49:52,916][488933] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,917][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 15:49:52,918][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 15:49:52,919][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 15:49:52,920][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 15:49:52,921][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 15:49:52,922][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 15:49:52,923][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 15:49:52,925][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 15:49:52,925][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 15:49:52,927][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 15:49:52,933][488941] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,936][489467] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,939][488933] Stopping RolloutWorker_w61... +[2023-03-07 15:49:52,939][488933] Loop rollout_proc61_evt_loop terminating... +[2023-03-07 15:49:52,939][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 15:49:52,940][488789] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,961][489467] Stopping RolloutWorker_w108... +[2023-03-07 15:49:52,962][489467] Loop rollout_proc108_evt_loop terminating... +[2023-03-07 15:49:52,962][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 15:49:52,973][488882] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,977][488920] Decorrelating experience for 32 frames... +[2023-03-07 15:49:52,986][488936] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,995][488869] Decorrelating experience for 0 frames... +[2023-03-07 15:49:52,996][488920] Stopping RolloutWorker_w75... +[2023-03-07 15:49:52,997][488920] Loop rollout_proc75_evt_loop terminating... +[2023-03-07 15:49:52,997][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 15:49:53,015][489720] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,037][489720] Stopping RolloutWorker_w114... +[2023-03-07 15:49:53,037][489720] Loop rollout_proc114_evt_loop terminating... +[2023-03-07 15:49:53,037][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 15:49:53,050][488926] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,075][488926] Stopping RolloutWorker_w63... +[2023-03-07 15:49:53,075][488926] Loop rollout_proc63_evt_loop terminating... +[2023-03-07 15:49:53,075][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 15:49:53,120][489206] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,139][489206] Stopping RolloutWorker_w119... +[2023-03-07 15:49:53,139][489206] Loop rollout_proc119_evt_loop terminating... +[2023-03-07 15:49:53,139][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 15:49:53,158][488783] Weights refcount: 2 0 +[2023-03-07 15:49:53,159][488783] Stopping InferenceWorker_p0-w0... +[2023-03-07 15:49:53,160][488783] Loop inference_proc0-0_evt_loop terminating... +[2023-03-07 15:49:53,160][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 15:49:53,186][489212] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,192][489136] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,193][488794] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,206][489212] Stopping RolloutWorker_w105... +[2023-03-07 15:49:53,207][489212] Loop rollout_proc105_evt_loop terminating... +[2023-03-07 15:49:53,207][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 15:49:53,217][489136] Stopping RolloutWorker_w104... +[2023-03-07 15:49:53,217][489136] Loop rollout_proc104_evt_loop terminating... +[2023-03-07 15:49:53,217][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 15:49:53,218][488881] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,220][488885] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,236][488907] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,249][488928] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,275][488928] Stopping RolloutWorker_w49... +[2023-03-07 15:49:53,275][488928] Loop rollout_proc49_evt_loop terminating... +[2023-03-07 15:49:53,275][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 15:49:53,279][488888] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,303][488888] Stopping RolloutWorker_w22... +[2023-03-07 15:49:53,303][488888] Loop rollout_proc22_evt_loop terminating... +[2023-03-07 15:49:53,303][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 15:49:53,357][490263] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,389][490263] Stopping RolloutWorker_w121... +[2023-03-07 15:49:53,389][490263] Loop rollout_proc121_evt_loop terminating... +[2023-03-07 15:49:53,390][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 15:49:53,394][488941] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,404][489717] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,405][488917] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,414][488941] Stopping RolloutWorker_w64... +[2023-03-07 15:49:53,414][488941] Loop rollout_proc64_evt_loop terminating... +[2023-03-07 15:49:53,414][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 15:49:53,431][488931] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,448][488625] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 15:49:53,456][488936] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,470][489728] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,478][488936] Stopping RolloutWorker_w76... +[2023-03-07 15:49:53,478][488936] Loop rollout_proc76_evt_loop terminating... +[2023-03-07 15:49:53,479][488899] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,479][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 15:49:53,489][488625] Removing /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000980_4014080.pth +[2023-03-07 15:49:53,493][488625] Saving /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 15:49:53,496][488899] Stopping RolloutWorker_w50... +[2023-03-07 15:49:53,496][488899] Loop rollout_proc50_evt_loop terminating... +[2023-03-07 15:49:53,496][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 15:49:53,500][488914] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,545][488887] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,587][488789] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,589][488625] Stopping LearnerWorker_p0... +[2023-03-07 15:49:53,589][488625] Loop learner_proc0_evt_loop terminating... +[2023-03-07 15:49:53,589][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 15:49:53,604][488789] Stopping RolloutWorker_w14... +[2023-03-07 15:49:53,604][488789] Loop rollout_proc14_evt_loop terminating... +[2023-03-07 15:49:53,604][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 15:49:53,628][488918] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,644][488918] Stopping RolloutWorker_w60... +[2023-03-07 15:49:53,645][488918] Loop rollout_proc60_evt_loop terminating... +[2023-03-07 15:49:53,645][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 15:49:53,646][488869] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,649][496838] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,657][489728] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,661][488939] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,668][489207] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,668][496838] Stopping RolloutWorker_w0... +[2023-03-07 15:49:53,668][488869] Stopping RolloutWorker_w12... +[2023-03-07 15:49:53,669][496838] Loop rollout_proc0_evt_loop terminating... +[2023-03-07 15:49:53,669][488869] Loop rollout_proc12_evt_loop terminating... +[2023-03-07 15:49:53,669][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 15:49:53,671][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 15:49:53,674][489728] Stopping RolloutWorker_w117... +[2023-03-07 15:49:53,675][489728] Loop rollout_proc117_evt_loop terminating... +[2023-03-07 15:49:53,675][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 15:49:53,679][488939] Stopping RolloutWorker_w67... +[2023-03-07 15:49:53,680][488939] Loop rollout_proc67_evt_loop terminating... +[2023-03-07 15:49:53,680][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 15:49:53,681][488889] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,685][489207] Stopping RolloutWorker_w93... +[2023-03-07 15:49:53,685][489207] Loop rollout_proc93_evt_loop terminating... +[2023-03-07 15:49:53,686][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 15:49:53,761][488882] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,784][488882] Stopping RolloutWorker_w36... +[2023-03-07 15:49:53,784][488882] Loop rollout_proc36_evt_loop terminating... +[2023-03-07 15:49:53,784][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 15:49:53,793][489717] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,809][489717] Stopping RolloutWorker_w94... +[2023-03-07 15:49:53,810][489717] Loop rollout_proc94_evt_loop terminating... +[2023-03-07 15:49:53,810][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 15:49:53,827][488908] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,838][489730] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,847][488895] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,854][489730] Stopping RolloutWorker_w112... +[2023-03-07 15:49:53,855][489730] Loop rollout_proc112_evt_loop terminating... +[2023-03-07 15:49:53,855][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 15:49:53,864][488910] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,865][488895] Stopping RolloutWorker_w37... +[2023-03-07 15:49:53,865][488895] Loop rollout_proc37_evt_loop terminating... +[2023-03-07 15:49:53,865][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 15:49:53,866][488790] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,876][489725] Decorrelating experience for 0 frames... +[2023-03-07 15:49:53,887][488914] Decorrelating experience for 32 frames... +[2023-03-07 15:49:53,904][488914] Stopping RolloutWorker_w57... +[2023-03-07 15:49:53,904][488914] Loop rollout_proc57_evt_loop terminating... +[2023-03-07 15:49:53,904][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 15:49:53,940][488878] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,011][488872] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,018][488881] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,018][488900] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,028][488872] Stopping RolloutWorker_w18... +[2023-03-07 15:49:54,028][488872] Loop rollout_proc18_evt_loop terminating... +[2023-03-07 15:49:54,028][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 15:49:54,035][488881] Stopping RolloutWorker_w45... +[2023-03-07 15:49:54,035][488900] Stopping RolloutWorker_w46... +[2023-03-07 15:49:54,035][488881] Loop rollout_proc45_evt_loop terminating... +[2023-03-07 15:49:54,035][488900] Loop rollout_proc46_evt_loop terminating... +[2023-03-07 15:49:54,035][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 15:49:54,036][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 15:49:54,049][488893] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,057][488790] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,059][488910] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,064][488870] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,075][488790] Stopping RolloutWorker_w17... +[2023-03-07 15:49:54,076][488790] Loop rollout_proc17_evt_loop terminating... +[2023-03-07 15:49:54,075][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 15:49:54,081][488910] Stopping RolloutWorker_w80... +[2023-03-07 15:49:54,081][488870] Stopping RolloutWorker_w6... +[2023-03-07 15:49:54,082][488910] Loop rollout_proc80_evt_loop terminating... +[2023-03-07 15:49:54,082][488870] Loop rollout_proc6_evt_loop terminating... +[2023-03-07 15:49:54,082][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 15:49:54,083][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 15:49:54,121][488890] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,137][488890] Stopping RolloutWorker_w28... +[2023-03-07 15:49:54,137][488890] Loop rollout_proc28_evt_loop terminating... +[2023-03-07 15:49:54,137][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 15:49:54,192][489727] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,208][489727] Stopping RolloutWorker_w106... +[2023-03-07 15:49:54,208][489727] Loop rollout_proc106_evt_loop terminating... +[2023-03-07 15:49:54,208][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 15:49:54,218][488937] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,229][489072] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,247][488921] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,262][488929] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,264][488921] Stopping RolloutWorker_w54... +[2023-03-07 15:49:54,264][488921] Loop rollout_proc54_evt_loop terminating... +[2023-03-07 15:49:54,264][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 15:49:54,279][488875] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,305][488887] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,317][488884] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,328][488887] Stopping RolloutWorker_w16... +[2023-03-07 15:49:54,328][488887] Loop rollout_proc16_evt_loop terminating... +[2023-03-07 15:49:54,328][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 15:49:54,371][490807] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,405][488937] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,414][489072] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,419][488791] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,422][488937] Stopping RolloutWorker_w55... +[2023-03-07 15:49:54,422][488937] Loop rollout_proc55_evt_loop terminating... +[2023-03-07 15:49:54,422][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 15:49:54,430][489072] Stopping RolloutWorker_w101... +[2023-03-07 15:49:54,430][489072] Loop rollout_proc101_evt_loop terminating... +[2023-03-07 15:49:54,430][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 15:49:54,433][488897] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,469][489204] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,490][488875] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,499][489719] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,507][488875] Stopping RolloutWorker_w24... +[2023-03-07 15:49:54,507][488875] Loop rollout_proc24_evt_loop terminating... +[2023-03-07 15:49:54,507][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 15:49:54,516][489719] Stopping RolloutWorker_w102... +[2023-03-07 15:49:54,517][489719] Loop rollout_proc102_evt_loop terminating... +[2023-03-07 15:49:54,516][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 15:49:54,520][488785] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,552][497235] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,568][497235] Stopping RolloutWorker_w1... +[2023-03-07 15:49:54,568][497235] Loop rollout_proc1_evt_loop terminating... +[2023-03-07 15:49:54,568][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 15:49:54,600][488929] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,608][490807] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,617][488929] Stopping RolloutWorker_w58... +[2023-03-07 15:49:54,617][488929] Loop rollout_proc58_evt_loop terminating... +[2023-03-07 15:49:54,617][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 15:49:54,623][488897] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,624][490807] Stopping RolloutWorker_w124... +[2023-03-07 15:49:54,624][490807] Loop rollout_proc124_evt_loop terminating... +[2023-03-07 15:49:54,624][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 15:49:54,626][488927] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,640][488897] Stopping RolloutWorker_w34... +[2023-03-07 15:49:54,640][488897] Loop rollout_proc34_evt_loop terminating... +[2023-03-07 15:49:54,640][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 15:49:54,654][488884] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,672][488884] Stopping RolloutWorker_w7... +[2023-03-07 15:49:54,672][488884] Loop rollout_proc7_evt_loop terminating... +[2023-03-07 15:49:54,672][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 15:49:54,674][489726] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,678][488883] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,702][488791] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,719][488791] Stopping RolloutWorker_w20... +[2023-03-07 15:49:54,719][488791] Loop rollout_proc20_evt_loop terminating... +[2023-03-07 15:49:54,719][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 15:49:54,721][488863] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,737][488907] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,745][488863] Stopping RolloutWorker_w35... +[2023-03-07 15:49:54,745][488863] Loop rollout_proc35_evt_loop terminating... +[2023-03-07 15:49:54,745][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 15:49:54,758][488907] Stopping RolloutWorker_w68... +[2023-03-07 15:49:54,758][488907] Loop rollout_proc68_evt_loop terminating... +[2023-03-07 15:49:54,758][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 15:49:54,814][489204] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,814][488785] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,823][488901] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,830][489204] Stopping RolloutWorker_w91... +[2023-03-07 15:49:54,830][489204] Loop rollout_proc91_evt_loop terminating... +[2023-03-07 15:49:54,830][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 15:49:54,831][488785] Stopping RolloutWorker_w2... +[2023-03-07 15:49:54,831][488785] Loop rollout_proc2_evt_loop terminating... +[2023-03-07 15:49:54,832][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 15:49:54,833][488924] Decorrelating experience for 0 frames... +[2023-03-07 15:49:54,863][489726] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,865][488883] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,880][489726] Stopping RolloutWorker_w126... +[2023-03-07 15:49:54,880][489726] Loop rollout_proc126_evt_loop terminating... +[2023-03-07 15:49:54,880][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 15:49:54,881][488883] Stopping RolloutWorker_w10... +[2023-03-07 15:49:54,882][488883] Loop rollout_proc10_evt_loop terminating... +[2023-03-07 15:49:54,882][488889] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,882][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 15:49:54,899][488889] Stopping RolloutWorker_w19... +[2023-03-07 15:49:54,899][488889] Loop rollout_proc19_evt_loop terminating... +[2023-03-07 15:49:54,899][489139] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,899][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 15:49:54,916][489139] Stopping RolloutWorker_w107... +[2023-03-07 15:49:54,916][489139] Loop rollout_proc107_evt_loop terminating... +[2023-03-07 15:49:54,917][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 15:49:54,977][488885] Decorrelating experience for 32 frames... +[2023-03-07 15:49:54,999][488885] Stopping RolloutWorker_w4... +[2023-03-07 15:49:54,999][488885] Loop rollout_proc4_evt_loop terminating... +[2023-03-07 15:49:54,999][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 15:49:55,023][488877] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,044][489670] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,061][489670] Stopping RolloutWorker_w120... +[2023-03-07 15:49:55,061][489670] Loop rollout_proc120_evt_loop terminating... +[2023-03-07 15:49:55,061][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 15:49:55,076][488909] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,086][488927] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,087][490342] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,096][488876] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,102][488786] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,108][488927] Stopping RolloutWorker_w69... +[2023-03-07 15:49:55,109][488927] Loop rollout_proc69_evt_loop terminating... +[2023-03-07 15:49:55,109][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 15:49:55,112][488876] Stopping RolloutWorker_w30... +[2023-03-07 15:49:55,112][488876] Loop rollout_proc30_evt_loop terminating... +[2023-03-07 15:49:55,112][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 15:49:55,220][488896] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,226][489729] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,232][488864] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,237][488896] Stopping RolloutWorker_w43... +[2023-03-07 15:49:55,237][488896] Loop rollout_proc43_evt_loop terminating... +[2023-03-07 15:49:55,237][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 15:49:55,249][488864] Stopping RolloutWorker_w41... +[2023-03-07 15:49:55,250][488864] Loop rollout_proc41_evt_loop terminating... +[2023-03-07 15:49:55,250][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 15:49:55,257][488877] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,274][488877] Stopping RolloutWorker_w39... +[2023-03-07 15:49:55,274][488877] Loop rollout_proc39_evt_loop terminating... +[2023-03-07 15:49:55,274][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 15:49:55,275][488898] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,285][488878] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,297][488940] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,302][488878] Stopping RolloutWorker_w33... +[2023-03-07 15:49:55,302][488878] Loop rollout_proc33_evt_loop terminating... +[2023-03-07 15:49:55,302][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 15:49:55,350][488786] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,368][488786] Stopping RolloutWorker_w5... +[2023-03-07 15:49:55,369][488786] Loop rollout_proc5_evt_loop terminating... +[2023-03-07 15:49:55,368][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 15:49:55,404][488917] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,413][489729] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,421][488917] Stopping RolloutWorker_w51... +[2023-03-07 15:49:55,422][488917] Loop rollout_proc51_evt_loop terminating... +[2023-03-07 15:49:55,422][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 15:49:55,424][488924] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,429][489729] Stopping RolloutWorker_w109... +[2023-03-07 15:49:55,430][489729] Loop rollout_proc109_evt_loop terminating... +[2023-03-07 15:49:55,430][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 15:49:55,433][488931] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,440][488924] Stopping RolloutWorker_w81... +[2023-03-07 15:49:55,440][488924] Loop rollout_proc81_evt_loop terminating... +[2023-03-07 15:49:55,440][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 15:49:55,451][488931] Stopping RolloutWorker_w52... +[2023-03-07 15:49:55,451][488931] Loop rollout_proc52_evt_loop terminating... +[2023-03-07 15:49:55,451][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 15:49:55,463][488898] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,465][488901] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,481][488898] Stopping RolloutWorker_w47... +[2023-03-07 15:49:55,481][488898] Loop rollout_proc47_evt_loop terminating... +[2023-03-07 15:49:55,481][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 15:49:55,483][488901] Stopping RolloutWorker_w53... +[2023-03-07 15:49:55,483][488901] Loop rollout_proc53_evt_loop terminating... +[2023-03-07 15:49:55,483][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 15:49:55,487][488940] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,491][488893] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,503][488940] Stopping RolloutWorker_w70... +[2023-03-07 15:49:55,504][488940] Loop rollout_proc70_evt_loop terminating... +[2023-03-07 15:49:55,503][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 15:49:55,508][488893] Stopping RolloutWorker_w31... +[2023-03-07 15:49:55,508][488893] Loop rollout_proc31_evt_loop terminating... +[2023-03-07 15:49:55,508][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 15:49:55,567][488905] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,584][488905] Stopping RolloutWorker_w71... +[2023-03-07 15:49:55,584][488905] Loop rollout_proc71_evt_loop terminating... +[2023-03-07 15:49:55,584][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 15:49:55,602][488868] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,618][488868] Stopping RolloutWorker_w3... +[2023-03-07 15:49:55,618][488868] Loop rollout_proc3_evt_loop terminating... +[2023-03-07 15:49:55,618][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 15:49:55,662][488909] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,679][488909] Stopping RolloutWorker_w77... +[2023-03-07 15:49:55,679][488909] Loop rollout_proc77_evt_loop terminating... +[2023-03-07 15:49:55,679][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 15:49:55,714][490342] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,731][490342] Stopping RolloutWorker_w115... +[2023-03-07 15:49:55,731][490342] Loop rollout_proc115_evt_loop terminating... +[2023-03-07 15:49:55,731][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 15:49:55,763][488911] Decorrelating experience for 0 frames... +[2023-03-07 15:49:55,796][488794] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,812][488794] Stopping RolloutWorker_w29... +[2023-03-07 15:49:55,813][488794] Loop rollout_proc29_evt_loop terminating... +[2023-03-07 15:49:55,813][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 15:49:55,876][489725] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,892][489725] Stopping RolloutWorker_w100... +[2023-03-07 15:49:55,892][489725] Loop rollout_proc100_evt_loop terminating... +[2023-03-07 15:49:55,892][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 15:49:55,951][488911] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,965][488930] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,967][488911] Stopping RolloutWorker_w83... +[2023-03-07 15:49:55,967][488911] Loop rollout_proc83_evt_loop terminating... +[2023-03-07 15:49:55,967][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 15:49:55,974][488908] Decorrelating experience for 32 frames... +[2023-03-07 15:49:55,981][488930] Stopping RolloutWorker_w79... +[2023-03-07 15:49:55,981][488930] Loop rollout_proc79_evt_loop terminating... +[2023-03-07 15:49:55,981][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 15:49:55,990][488908] Stopping RolloutWorker_w74... +[2023-03-07 15:49:55,991][488908] Loop rollout_proc74_evt_loop terminating... +[2023-03-07 15:49:55,990][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 15:49:56,082][488788] Decorrelating experience for 0 frames... +[2023-03-07 15:49:56,267][488788] Decorrelating experience for 32 frames... +[2023-03-07 15:49:56,284][488788] Stopping RolloutWorker_w11... +[2023-03-07 15:49:56,284][488788] Loop rollout_proc11_evt_loop terminating... +[2023-03-07 15:49:56,284][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 15:49:56,285][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 15:49:56,286][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 15:49:56,287][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 15:49:56,288][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 15:49:56,289][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 15:49:56,291][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 15:49:56,292][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 15:49:56,293][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 15:49:56,294][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 15:49:56,295][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 15:49:56,296][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 15:49:56,297][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 15:49:56,298][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 15:49:56,299][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 15:49:56,464][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 15:49:56,465][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 15:49:56,466][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 15:49:56,467][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 15:49:56,468][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 15:49:56,469][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 15:49:56,471][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 15:49:56,472][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 15:49:56,473][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 15:49:56,474][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 15:49:56,475][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 15:49:56,476][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 15:49:56,477][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 15:49:56,478][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 15:49:56,479][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 15:49:56,480][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 15:49:56,481][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 15:49:56,482][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 15:49:56,483][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 15:49:56,484][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 15:49:56,485][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 15:49:56,486][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 15:49:56,487][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 15:49:56,488][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 15:49:56,489][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 15:49:56,491][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 15:49:56,492][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 15:49:56,493][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 15:49:56,494][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 15:49:56,495][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 15:49:56,496][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 15:49:56,497][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 15:49:56,498][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 15:49:56,499][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 15:49:56,500][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 15:49:56,501][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 15:49:56,503][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 15:49:56,504][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 15:49:56,505][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 15:49:56,506][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 15:49:56,509][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 15:49:56,510][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 15:49:56,511][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 15:49:56,512][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 15:49:56,513][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 15:49:56,517][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 15:49:56,518][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 15:49:56,519][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 15:49:56,520][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 15:49:56,521][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 15:49:56,522][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 15:49:56,524][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 15:49:56,525][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 15:49:56,526][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 15:49:56,527][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 15:49:56,528][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 15:49:56,529][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 15:49:56,531][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 15:49:56,532][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 15:49:56,533][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 15:49:56,535][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 15:49:56,536][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 15:49:56,537][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 15:49:56,540][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 15:49:56,541][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 15:49:56,543][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 15:49:56,544][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 15:49:56,545][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 15:49:56,546][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 15:49:56,547][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 15:49:56,548][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 15:49:56,549][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 15:49:56,551][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 15:49:56,552][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 15:49:56,553][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 15:49:56,554][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 15:49:56,556][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 15:49:56,557][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 15:49:56,558][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 15:49:56,559][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 15:49:56,560][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 15:49:56,561][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 15:49:56,562][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 15:49:56,563][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 15:49:56,564][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 15:49:56,565][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 15:49:56,567][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 15:49:56,568][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 15:49:56,569][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 15:49:56,571][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 15:49:56,573][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 15:49:56,574][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 15:49:56,575][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 15:49:56,576][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 15:49:56,577][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 15:49:56,579][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 15:49:56,580][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 15:49:56,581][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 15:49:56,586][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 15:49:56,587][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 15:49:56,588][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 15:49:56,590][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 15:49:56,593][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 15:49:56,594][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 15:49:56,595][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 15:49:56,597][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 15:49:56,598][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 15:49:56,599][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 15:49:56,600][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 15:49:56,602][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 15:49:56,603][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 15:49:56,604][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 15:49:56,605][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 15:49:56,606][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 15:49:56,608][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 15:49:56,609][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 15:49:56,611][488352] Batcher 0 profile tree view: +batching: 0.0889, releasing_batches: 0.0005 +[2023-03-07 15:49:56,612][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0045 + wait_policy_total: 1.3187 +update_model: 0.2426 + weight_update: 0.2403 +one_step: 0.0125 + handle_policy_step: 0.7122 + deserialize: 0.0242, stack: 0.0012, obs_to_device_normalize: 0.0588, forward: 0.5365, send_messages: 0.0513 + prepare_outputs: 0.0309 + to_cpu: 0.0171 +[2023-03-07 15:49:56,613][488352] Learner 0 profile tree view: +misc: 0.0000, prepare_batch: 1.3773 +train: 0.2697 + epoch_init: 0.0000, minibatch_init: 0.0000, losses_postprocess: 0.0008, kl_divergence: 0.0004, after_optimizer: 0.0030 + calculate_losses: 0.0285 + losses_init: 0.0000, forward_head: 0.0165, bptt_initial: 0.0078, tail: 0.0006, advantages_returns: 0.0005, losses: 0.0016 + bptt: 0.0012 + bptt_forward_core: 0.0012 + update: 0.2363 + clip: 0.0020 +[2023-03-07 15:49:56,614][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0001, enqueue_policy_requests: 0.0003 +[2023-03-07 15:49:56,615][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0003, enqueue_policy_requests: 0.0123, env_step: 0.0884, overhead: 0.0091, complete_rollouts: 0.0004 +save_policy_outputs: 0.0107 + split_output_tensors: 0.0052 +[2023-03-07 15:49:56,616][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 15:49:56,620][488352] Runner profile tree view: +main_loop: 13.2739 +[2023-03-07 15:49:56,621][488352] Collected {0: 4026368}, FPS: 617.2 +[2023-03-07 15:51:10,061][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 15:51:10,063][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 15:51:10,064][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 15:51:10,065][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 15:51:10,066][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 15:51:10,067][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 15:51:10,068][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 15:51:10,069][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 15:51:10,070][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 15:51:10,071][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 15:51:10,071][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 15:51:10,072][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 15:51:10,073][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 15:51:10,074][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 15:51:10,075][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 15:51:10,075][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 15:51:10,076][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 15:51:10,077][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 15:51:10,077][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 15:51:10,079][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 15:51:10,079][488352] register_encoder_factory: +[2023-03-07 15:51:10,094][488352] Saved parameter configuration for experiment doom_health_threads not found! +[2023-03-07 15:51:10,095][488352] Starting experiment from scratch! +[2023-03-07 15:51:10,100][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads already exists! +[2023-03-07 15:51:10,100][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads... +[2023-03-07 15:51:10,101][488352] Weights and Biases integration disabled +[2023-03-07 15:51:10,104][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 15:51:11,147][488352] Automatically setting recurrence to 32 +[2023-03-07 15:51:11,148][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=16 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=16 --train_for_env_steps=4000000 --experiment=doom_health_threads +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads', 'num_workers': 128, 'num_envs_per_worker': 16, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 15:51:11,149][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json... +[2023-03-07 15:51:11,151][488352] Rollout worker 0 uses device cpu +[2023-03-07 15:51:11,151][488352] Rollout worker 1 uses device cpu +[2023-03-07 15:51:11,152][488352] Rollout worker 2 uses device cpu +[2023-03-07 15:51:11,153][488352] Rollout worker 3 uses device cpu +[2023-03-07 15:51:11,153][488352] Rollout worker 4 uses device cpu +[2023-03-07 15:51:11,154][488352] Rollout worker 5 uses device cpu +[2023-03-07 15:51:11,154][488352] Rollout worker 6 uses device cpu +[2023-03-07 15:51:11,155][488352] Rollout worker 7 uses device cpu +[2023-03-07 15:51:11,155][488352] Rollout worker 8 uses device cpu +[2023-03-07 15:51:11,156][488352] Rollout worker 9 uses device cpu +[2023-03-07 15:51:11,157][488352] Rollout worker 10 uses device cpu +[2023-03-07 15:51:11,157][488352] Rollout worker 11 uses device cpu +[2023-03-07 15:51:11,158][488352] Rollout worker 12 uses device cpu +[2023-03-07 15:51:11,158][488352] Rollout worker 13 uses device cpu +[2023-03-07 15:51:11,159][488352] Rollout worker 14 uses device cpu +[2023-03-07 15:51:11,160][488352] Rollout worker 15 uses device cpu +[2023-03-07 15:51:11,160][488352] Rollout worker 16 uses device cpu +[2023-03-07 15:51:11,161][488352] Rollout worker 17 uses device cpu +[2023-03-07 15:51:11,161][488352] Rollout worker 18 uses device cpu +[2023-03-07 15:51:11,162][488352] Rollout worker 19 uses device cpu +[2023-03-07 15:51:11,163][488352] Rollout worker 20 uses device cpu +[2023-03-07 15:51:11,163][488352] Rollout worker 21 uses device cpu +[2023-03-07 15:51:11,164][488352] Rollout worker 22 uses device cpu +[2023-03-07 15:51:11,164][488352] Rollout worker 23 uses device cpu +[2023-03-07 15:51:11,165][488352] Rollout worker 24 uses device cpu +[2023-03-07 15:51:11,166][488352] Rollout worker 25 uses device cpu +[2023-03-07 15:51:11,170][488352] Rollout worker 26 uses device cpu +[2023-03-07 15:51:11,170][488352] Rollout worker 27 uses device cpu +[2023-03-07 15:51:11,171][488352] Rollout worker 28 uses device cpu +[2023-03-07 15:51:11,172][488352] Rollout worker 29 uses device cpu +[2023-03-07 15:51:11,172][488352] Rollout worker 30 uses device cpu +[2023-03-07 15:51:11,173][488352] Rollout worker 31 uses device cpu +[2023-03-07 15:51:11,173][488352] Rollout worker 32 uses device cpu +[2023-03-07 15:51:11,174][488352] Rollout worker 33 uses device cpu +[2023-03-07 15:51:11,174][488352] Rollout worker 34 uses device cpu +[2023-03-07 15:51:11,175][488352] Rollout worker 35 uses device cpu +[2023-03-07 15:51:11,176][488352] Rollout worker 36 uses device cpu +[2023-03-07 15:51:11,176][488352] Rollout worker 37 uses device cpu +[2023-03-07 15:51:11,177][488352] Rollout worker 38 uses device cpu +[2023-03-07 15:51:11,177][488352] Rollout worker 39 uses device cpu +[2023-03-07 15:51:11,178][488352] Rollout worker 40 uses device cpu +[2023-03-07 15:51:11,179][488352] Rollout worker 41 uses device cpu +[2023-03-07 15:51:11,179][488352] Rollout worker 42 uses device cpu +[2023-03-07 15:51:11,180][488352] Rollout worker 43 uses device cpu +[2023-03-07 15:51:11,180][488352] Rollout worker 44 uses device cpu +[2023-03-07 15:51:11,181][488352] Rollout worker 45 uses device cpu +[2023-03-07 15:51:11,182][488352] Rollout worker 46 uses device cpu +[2023-03-07 15:51:11,182][488352] Rollout worker 47 uses device cpu +[2023-03-07 15:51:11,183][488352] Rollout worker 48 uses device cpu +[2023-03-07 15:51:11,183][488352] Rollout worker 49 uses device cpu +[2023-03-07 15:51:11,184][488352] Rollout worker 50 uses device cpu +[2023-03-07 15:51:11,185][488352] Rollout worker 51 uses device cpu +[2023-03-07 15:51:11,185][488352] Rollout worker 52 uses device cpu +[2023-03-07 15:51:11,186][488352] Rollout worker 53 uses device cpu +[2023-03-07 15:51:11,186][488352] Rollout worker 54 uses device cpu +[2023-03-07 15:51:11,191][488352] Rollout worker 55 uses device cpu +[2023-03-07 15:51:11,192][488352] Rollout worker 56 uses device cpu +[2023-03-07 15:51:11,193][488352] Rollout worker 57 uses device cpu +[2023-03-07 15:51:11,194][488352] Rollout worker 58 uses device cpu +[2023-03-07 15:51:11,195][488352] Rollout worker 59 uses device cpu +[2023-03-07 15:51:11,195][488352] Rollout worker 60 uses device cpu +[2023-03-07 15:51:11,196][488352] Rollout worker 61 uses device cpu +[2023-03-07 15:51:11,197][488352] Rollout worker 62 uses device cpu +[2023-03-07 15:51:11,198][488352] Rollout worker 63 uses device cpu +[2023-03-07 15:51:11,199][488352] Rollout worker 64 uses device cpu +[2023-03-07 15:51:11,200][488352] Rollout worker 65 uses device cpu +[2023-03-07 15:51:11,201][488352] Rollout worker 66 uses device cpu +[2023-03-07 15:51:11,201][488352] Rollout worker 67 uses device cpu +[2023-03-07 15:51:11,203][488352] Rollout worker 68 uses device cpu +[2023-03-07 15:51:11,203][488352] Rollout worker 69 uses device cpu +[2023-03-07 15:51:11,204][488352] Rollout worker 70 uses device cpu +[2023-03-07 15:51:11,205][488352] Rollout worker 71 uses device cpu +[2023-03-07 15:51:11,205][488352] Rollout worker 72 uses device cpu +[2023-03-07 15:51:11,206][488352] Rollout worker 73 uses device cpu +[2023-03-07 15:51:11,208][488352] Rollout worker 74 uses device cpu +[2023-03-07 15:51:11,208][488352] Rollout worker 75 uses device cpu +[2023-03-07 15:51:11,209][488352] Rollout worker 76 uses device cpu +[2023-03-07 15:51:11,210][488352] Rollout worker 77 uses device cpu +[2023-03-07 15:51:11,211][488352] Rollout worker 78 uses device cpu +[2023-03-07 15:51:11,212][488352] Rollout worker 79 uses device cpu +[2023-03-07 15:51:11,212][488352] Rollout worker 80 uses device cpu +[2023-03-07 15:51:11,213][488352] Rollout worker 81 uses device cpu +[2023-03-07 15:51:11,214][488352] Rollout worker 82 uses device cpu +[2023-03-07 15:51:11,215][488352] Rollout worker 83 uses device cpu +[2023-03-07 15:51:11,216][488352] Rollout worker 84 uses device cpu +[2023-03-07 15:51:11,217][488352] Rollout worker 85 uses device cpu +[2023-03-07 15:51:11,218][488352] Rollout worker 86 uses device cpu +[2023-03-07 15:51:11,219][488352] Rollout worker 87 uses device cpu +[2023-03-07 15:51:11,220][488352] Rollout worker 88 uses device cpu +[2023-03-07 15:51:11,220][488352] Rollout worker 89 uses device cpu +[2023-03-07 15:51:11,221][488352] Rollout worker 90 uses device cpu +[2023-03-07 15:51:11,222][488352] Rollout worker 91 uses device cpu +[2023-03-07 15:51:11,223][488352] Rollout worker 92 uses device cpu +[2023-03-07 15:51:11,224][488352] Rollout worker 93 uses device cpu +[2023-03-07 15:51:11,225][488352] Rollout worker 94 uses device cpu +[2023-03-07 15:51:11,225][488352] Rollout worker 95 uses device cpu +[2023-03-07 15:51:11,226][488352] Rollout worker 96 uses device cpu +[2023-03-07 15:51:11,227][488352] Rollout worker 97 uses device cpu +[2023-03-07 15:51:11,227][488352] Rollout worker 98 uses device cpu +[2023-03-07 15:51:11,228][488352] Rollout worker 99 uses device cpu +[2023-03-07 15:51:11,229][488352] Rollout worker 100 uses device cpu +[2023-03-07 15:51:11,229][488352] Rollout worker 101 uses device cpu +[2023-03-07 15:51:11,232][488352] Rollout worker 102 uses device cpu +[2023-03-07 15:51:11,232][488352] Rollout worker 103 uses device cpu +[2023-03-07 15:51:11,233][488352] Rollout worker 104 uses device cpu +[2023-03-07 15:51:11,234][488352] Rollout worker 105 uses device cpu +[2023-03-07 15:51:11,235][488352] Rollout worker 106 uses device cpu +[2023-03-07 15:51:11,236][488352] Rollout worker 107 uses device cpu +[2023-03-07 15:51:11,237][488352] Rollout worker 108 uses device cpu +[2023-03-07 15:51:11,238][488352] Rollout worker 109 uses device cpu +[2023-03-07 15:51:11,239][488352] Rollout worker 110 uses device cpu +[2023-03-07 15:51:11,240][488352] Rollout worker 111 uses device cpu +[2023-03-07 15:51:11,240][488352] Rollout worker 112 uses device cpu +[2023-03-07 15:51:11,241][488352] Rollout worker 113 uses device cpu +[2023-03-07 15:51:11,242][488352] Rollout worker 114 uses device cpu +[2023-03-07 15:51:11,243][488352] Rollout worker 115 uses device cpu +[2023-03-07 15:51:11,244][488352] Rollout worker 116 uses device cpu +[2023-03-07 15:51:11,244][488352] Rollout worker 117 uses device cpu +[2023-03-07 15:51:11,245][488352] Rollout worker 118 uses device cpu +[2023-03-07 15:51:11,246][488352] Rollout worker 119 uses device cpu +[2023-03-07 15:51:11,247][488352] Rollout worker 120 uses device cpu +[2023-03-07 15:51:11,248][488352] Rollout worker 121 uses device cpu +[2023-03-07 15:51:11,249][488352] Rollout worker 122 uses device cpu +[2023-03-07 15:51:11,249][488352] Rollout worker 123 uses device cpu +[2023-03-07 15:51:11,251][488352] Rollout worker 124 uses device cpu +[2023-03-07 15:51:11,252][488352] Rollout worker 125 uses device cpu +[2023-03-07 15:51:11,252][488352] Rollout worker 126 uses device cpu +[2023-03-07 15:51:11,253][488352] Rollout worker 127 uses device cpu +[2023-03-07 15:51:12,370][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:51:12,371][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:51:12,700][488352] Starting all processes... +[2023-03-07 15:51:12,701][488352] Starting process learner_proc0 +[2023-03-07 15:51:13,588][488352] Starting all processes... +[2023-03-07 15:51:13,593][488352] Starting process inference_proc0-0 +[2023-03-07 15:51:13,593][488352] Starting process rollout_proc2 +[2023-03-07 15:51:13,593][488352] Starting process rollout_proc5 +[2023-03-07 15:51:13,594][488352] Starting process rollout_proc8 +[2023-03-07 15:51:13,594][488352] Starting process rollout_proc11 +[2023-03-07 15:51:13,594][488352] Starting process rollout_proc14 +[2023-03-07 15:51:13,595][488352] Starting process rollout_proc17 +[2023-03-07 15:51:13,595][488352] Starting process rollout_proc20 +[2023-03-07 15:51:13,595][488352] Starting process rollout_proc23 +[2023-03-07 15:51:13,600][488352] Starting process rollout_proc26 +[2023-03-07 15:51:13,604][488352] Starting process rollout_proc29 +[2023-03-07 15:51:13,604][488352] Starting process rollout_proc32 +[2023-03-07 15:51:13,605][488352] Starting process rollout_proc35 +[2023-03-07 15:51:13,606][488352] Starting process rollout_proc38 +[2023-03-07 15:51:13,611][488352] Starting process rollout_proc41 +[2023-03-07 15:51:13,614][488352] Starting process rollout_proc44 +[2023-03-07 15:51:13,639][488352] Starting process rollout_proc6 +[2023-03-07 15:51:13,640][488352] Starting process rollout_proc3 +[2023-03-07 15:51:13,648][488352] Starting process rollout_proc12 +[2023-03-07 15:51:13,650][488352] Starting process rollout_proc9 +[2023-03-07 15:51:13,654][488352] Starting process rollout_proc21 +[2023-03-07 15:51:13,662][488352] Starting process rollout_proc18 +[2023-03-07 15:51:13,668][488352] Starting process rollout_proc24 +[2023-03-07 15:51:13,675][488352] Starting process rollout_proc15 +[2023-03-07 15:51:13,678][488352] Starting process rollout_proc30 +[2023-03-07 15:51:13,695][488352] Starting process rollout_proc33 +[2023-03-07 15:51:13,711][488352] Starting process rollout_proc27 +[2023-03-07 15:51:13,716][488352] Starting process rollout_proc39 +[2023-03-07 15:51:13,718][488352] Starting process rollout_proc36 +[2023-03-07 15:51:13,723][488352] Starting process rollout_proc42 +[2023-03-07 15:51:13,726][488352] Starting process rollout_proc45 +[2023-03-07 15:51:13,737][488352] Starting process rollout_proc7 +[2023-03-07 15:51:13,750][488352] Starting process rollout_proc10 +[2023-03-07 15:51:13,754][488352] Starting process rollout_proc13 +[2023-03-07 15:51:13,754][488352] Starting process rollout_proc4 +[2023-03-07 15:51:13,760][488352] Starting process rollout_proc22 +[2023-03-07 15:51:13,767][488352] Starting process rollout_proc19 +[2023-03-07 15:51:13,775][488352] Starting process rollout_proc25 +[2023-03-07 15:51:13,779][488352] Starting process rollout_proc16 +[2023-03-07 15:51:13,783][488352] Starting process rollout_proc34 +[2023-03-07 15:51:13,785][488352] Starting process rollout_proc28 +[2023-03-07 15:51:13,788][488352] Starting process rollout_proc31 +[2023-03-07 15:51:13,792][488352] Starting process rollout_proc40 +[2023-03-07 15:51:13,795][488352] Starting process rollout_proc37 +[2023-03-07 15:51:13,805][488352] Starting process rollout_proc43 +[2023-03-07 15:51:13,807][488352] Starting process rollout_proc46 +[2023-03-07 15:51:13,811][488352] Starting process rollout_proc47 +[2023-03-07 15:51:13,819][488352] Starting process rollout_proc50 +[2023-03-07 15:51:13,826][488352] Starting process rollout_proc53 +[2023-03-07 15:51:13,827][488352] Starting process rollout_proc56 +[2023-03-07 15:51:13,836][488352] Starting process rollout_proc59 +[2023-03-07 15:51:13,845][488352] Starting process rollout_proc62 +[2023-03-07 15:51:13,859][488352] Starting process rollout_proc65 +[2023-03-07 15:51:13,863][488352] Starting process rollout_proc68 +[2023-03-07 15:51:13,870][488352] Starting process rollout_proc71 +[2023-03-07 15:51:13,873][488352] Starting process rollout_proc74 +[2023-03-07 15:51:13,875][488352] Starting process rollout_proc77 +[2023-03-07 15:51:13,883][488352] Starting process rollout_proc80 +[2023-03-07 15:51:13,889][488352] Starting process rollout_proc83 +[2023-03-07 15:51:13,900][488352] Starting process rollout_proc86 +[2023-03-07 15:51:13,903][488352] Starting process rollout_proc89 +[2023-03-07 15:51:13,908][488352] Starting process rollout_proc48 +[2023-03-07 15:51:13,965][488352] Starting process rollout_proc69 +[2023-03-07 15:51:13,966][488352] Starting process rollout_proc54 +[2023-03-07 15:51:13,968][488352] Starting process rollout_proc72 +[2023-03-07 15:51:13,972][488352] Starting process rollout_proc66 +[2023-03-07 15:51:13,976][488352] Starting process rollout_proc51 +[2023-03-07 15:51:13,976][488352] Starting process rollout_proc57 +[2023-03-07 15:51:13,979][488352] Starting process rollout_proc78 +[2023-03-07 15:51:13,986][488352] Starting process rollout_proc75 +[2023-03-07 15:51:13,987][488352] Starting process rollout_proc60 +[2023-03-07 15:51:13,995][488352] Starting process rollout_proc63 +[2023-03-07 15:51:13,996][488352] Starting process rollout_proc81 +[2023-03-07 15:51:13,999][488352] Starting process rollout_proc84 +[2023-03-07 15:51:14,006][488352] Starting process rollout_proc87 +[2023-03-07 15:51:14,024][488352] Starting process rollout_proc90 +[2023-03-07 15:51:14,028][488352] Starting process rollout_proc49 +[2023-03-07 15:51:14,049][488352] Starting process rollout_proc70 +[2023-03-07 15:51:14,057][488352] Starting process rollout_proc55 +[2023-03-07 15:51:14,072][488352] Starting process rollout_proc73 +[2023-03-07 15:51:14,072][488352] Starting process rollout_proc67 +[2023-03-07 15:51:14,076][488352] Starting process rollout_proc52 +[2023-03-07 15:51:14,088][488352] Starting process rollout_proc58 +[2023-03-07 15:51:14,095][488352] Starting process rollout_proc79 +[2023-03-07 15:51:14,098][488352] Starting process rollout_proc64 +[2023-03-07 15:51:14,107][488352] Starting process rollout_proc76 +[2023-03-07 15:51:14,117][488352] Starting process rollout_proc61 +[2023-03-07 15:51:14,123][488352] Starting process rollout_proc82 +[2023-03-07 15:51:14,126][488352] Starting process rollout_proc88 +[2023-03-07 15:51:14,157][488352] Starting process rollout_proc92 +[2023-03-07 15:51:14,165][488352] Starting process rollout_proc85 +[2023-03-07 15:51:14,177][488352] Starting process rollout_proc91 +[2023-03-07 15:51:14,207][488352] Starting process rollout_proc95 +[2023-03-07 15:51:14,211][488352] Starting process rollout_proc98 +[2023-03-07 15:51:14,274][488352] Starting process rollout_proc101 +[2023-03-07 15:51:14,279][488352] Starting process rollout_proc104 +[2023-03-07 15:51:14,284][488352] Starting process rollout_proc107 +[2023-03-07 15:51:14,310][488352] Starting process rollout_proc110 +[2023-03-07 15:51:14,311][488352] Starting process rollout_proc113 +[2023-03-07 15:51:14,316][488352] Starting process rollout_proc116 +[2023-03-07 15:51:14,320][488352] Starting process rollout_proc119 +[2023-03-07 15:51:14,321][488352] Starting process rollout_proc122 +[2023-03-07 15:51:14,346][488352] Starting process rollout_proc125 +[2023-03-07 15:51:14,358][488352] Starting process rollout_proc99 +[2023-03-07 15:51:14,409][488352] Starting process rollout_proc93 +[2023-03-07 15:51:14,410][488352] Starting process rollout_proc96 +[2023-03-07 15:51:14,454][488352] Starting process rollout_proc102 +[2023-03-07 15:51:14,476][488352] Starting process rollout_proc123 +[2023-03-07 15:51:14,476][488352] Starting process rollout_proc117 +[2023-03-07 15:51:14,476][488352] Starting process rollout_proc108 +[2023-03-07 15:51:14,491][488352] Starting process rollout_proc111 +[2023-03-07 15:51:14,506][488352] Starting process rollout_proc105 +[2023-03-07 15:51:14,507][488352] Starting process rollout_proc114 +[2023-03-07 15:51:14,520][488352] Starting process rollout_proc126 +[2023-03-07 15:51:14,562][488352] Starting process rollout_proc120 +[2023-03-07 15:51:14,602][488352] Starting process rollout_proc94 +[2023-03-07 15:51:14,609][488352] Starting process rollout_proc97 +[2023-03-07 15:51:14,644][488352] Starting process rollout_proc115 +[2023-03-07 15:51:14,690][488352] Starting process rollout_proc124 +[2023-03-07 15:51:14,690][488352] Starting process rollout_proc127 +[2023-03-07 15:51:14,691][488352] Starting process rollout_proc103 +[2023-03-07 15:51:14,691][488352] Starting process rollout_proc100 +[2023-03-07 15:51:14,732][488352] Starting process rollout_proc106 +[2023-03-07 15:51:14,733][488352] Starting process rollout_proc109 +[2023-03-07 15:51:14,733][488352] Starting process rollout_proc118 +[2023-03-07 15:51:14,733][488352] Starting process rollout_proc112 +[2023-03-07 15:51:14,792][488352] Starting process rollout_proc121 +[2023-03-07 15:51:16,514][488352] Starting process rollout_proc0 +[2023-03-07 15:51:16,654][488352] Starting process rollout_proc1 +[2023-03-07 15:51:19,277][488352] Inference worker 0-0 is ready! +[2023-03-07 15:51:19,278][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:51:20,104][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:51:25,104][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:51:30,104][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:51:32,366][488352] Heartbeat connected on Batcher_0 +[2023-03-07 15:51:32,368][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 15:51:32,400][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 15:51:35,104][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:51:38,169][488352] Heartbeat connected on RolloutWorker_w35 +[2023-03-07 15:51:38,447][488352] Heartbeat connected on RolloutWorker_w45 +[2023-03-07 15:51:39,387][488352] Heartbeat connected on RolloutWorker_w13 +[2023-03-07 15:51:40,104][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:51:40,105][488352] Avg episode reward: [(0, '1.072')] +[2023-03-07 15:51:40,973][488352] Heartbeat connected on RolloutWorker_w30 +[2023-03-07 15:51:42,224][488352] Heartbeat connected on RolloutWorker_w54 +[2023-03-07 15:51:43,559][488352] Heartbeat connected on RolloutWorker_w89 +[2023-03-07 15:51:44,086][488352] Heartbeat connected on RolloutWorker_w103 +[2023-03-07 15:51:44,200][488352] Heartbeat connected on RolloutWorker_w17 +[2023-03-07 15:51:45,104][488352] Fps is (10 sec: 4915.2, 60 sec: 1966.1, 300 sec: 1966.1). Total num frames: 49152. Throughput: 0: 344.3. Samples: 8608. Policy #0 lag: (min: 0.0, avg: 1.8, max: 4.0) +[2023-03-07 15:51:45,105][488352] Avg episode reward: [(0, '3.353')] +[2023-03-07 15:51:45,633][488352] Heartbeat connected on RolloutWorker_w127 +[2023-03-07 15:51:46,464][488352] Heartbeat connected on RolloutWorker_w58 +[2023-03-07 15:51:46,965][488352] Heartbeat connected on RolloutWorker_w42 +[2023-03-07 15:51:47,108][488352] Heartbeat connected on RolloutWorker_w14 +[2023-03-07 15:51:47,783][488352] Heartbeat connected on RolloutWorker_w85 +[2023-03-07 15:51:47,988][488352] Heartbeat connected on RolloutWorker_w119 +[2023-03-07 15:51:48,327][488352] Heartbeat connected on RolloutWorker_w0 +[2023-03-07 15:51:48,594][488352] Heartbeat connected on RolloutWorker_w90 +[2023-03-07 15:51:48,930][488352] Heartbeat connected on RolloutWorker_w114 +[2023-03-07 15:51:49,059][488352] Heartbeat connected on RolloutWorker_w56 +[2023-03-07 15:51:49,070][488352] Heartbeat connected on RolloutWorker_w43 +[2023-03-07 15:51:50,104][488352] Fps is (10 sec: 14745.7, 60 sec: 4915.2, 300 sec: 4915.2). Total num frames: 147456. Throughput: 0: 1241.1. Samples: 37232. Policy #0 lag: (min: 0.0, avg: 4.0, max: 7.0) +[2023-03-07 15:51:50,105][488352] Avg episode reward: [(0, '3.077')] +[2023-03-07 15:51:50,193][488352] Heartbeat connected on RolloutWorker_w87 +[2023-03-07 15:51:51,110][488352] Heartbeat connected on RolloutWorker_w26 +[2023-03-07 15:51:51,207][488352] Heartbeat connected on RolloutWorker_w111 +[2023-03-07 15:51:51,321][488352] Heartbeat connected on RolloutWorker_w102 +[2023-03-07 15:51:51,686][488352] Heartbeat connected on RolloutWorker_w29 +[2023-03-07 15:51:51,851][488352] Heartbeat connected on RolloutWorker_w71 +[2023-03-07 15:51:52,061][488352] Heartbeat connected on RolloutWorker_w86 +[2023-03-07 15:51:52,115][488352] Heartbeat connected on RolloutWorker_w9 +[2023-03-07 15:51:52,383][488352] Heartbeat connected on RolloutWorker_w65 +[2023-03-07 15:51:52,561][488352] Heartbeat connected on RolloutWorker_w49 +[2023-03-07 15:51:52,856][488352] Heartbeat connected on RolloutWorker_w39 +[2023-03-07 15:51:52,918][488352] Heartbeat connected on RolloutWorker_w25 +[2023-03-07 15:51:53,121][488352] Heartbeat connected on RolloutWorker_w91 +[2023-03-07 15:51:53,243][488352] Heartbeat connected on RolloutWorker_w24 +[2023-03-07 15:51:53,495][488352] Heartbeat connected on RolloutWorker_w79 +[2023-03-07 15:51:53,601][488352] Heartbeat connected on RolloutWorker_w69 +[2023-03-07 15:51:53,673][488352] Heartbeat connected on RolloutWorker_w96 +[2023-03-07 15:51:53,945][488352] Heartbeat connected on RolloutWorker_w117 +[2023-03-07 15:51:54,053][488352] Heartbeat connected on RolloutWorker_w124 +[2023-03-07 15:51:54,180][488352] Heartbeat connected on RolloutWorker_w77 +[2023-03-07 15:51:54,306][488352] Heartbeat connected on RolloutWorker_w100 +[2023-03-07 15:51:54,668][488352] Heartbeat connected on RolloutWorker_w63 +[2023-03-07 15:51:54,843][488352] Heartbeat connected on RolloutWorker_w121 +[2023-03-07 15:51:54,940][488352] Heartbeat connected on RolloutWorker_w59 +[2023-03-07 15:51:54,997][488352] Heartbeat connected on RolloutWorker_w115 +[2023-03-07 15:51:54,999][488352] Heartbeat connected on RolloutWorker_w73 +[2023-03-07 15:51:55,104][488352] Fps is (10 sec: 49971.2, 60 sec: 15681.8, 300 sec: 15681.8). Total num frames: 548864. Throughput: 0: 2115.0. Samples: 74024. Policy #0 lag: (min: 0.0, avg: 8.8, max: 19.0) +[2023-03-07 15:51:55,105][488352] Avg episode reward: [(0, '3.601')] +[2023-03-07 15:51:55,241][488352] Heartbeat connected on RolloutWorker_w84 +[2023-03-07 15:51:55,310][488352] Heartbeat connected on RolloutWorker_w60 +[2023-03-07 15:51:55,591][488352] Heartbeat connected on RolloutWorker_w5 +[2023-03-07 15:51:55,876][488352] Heartbeat connected on RolloutWorker_w98 +[2023-03-07 15:51:56,041][488352] Heartbeat connected on RolloutWorker_w76 +[2023-03-07 15:51:56,160][488352] Heartbeat connected on RolloutWorker_w48 +[2023-03-07 15:51:56,195][488352] Heartbeat connected on RolloutWorker_w105 +[2023-03-07 15:51:56,369][488352] Heartbeat connected on RolloutWorker_w107 +[2023-03-07 15:51:56,392][488352] Heartbeat connected on RolloutWorker_w61 +[2023-03-07 15:51:56,444][488352] Heartbeat connected on RolloutWorker_w38 +[2023-03-07 15:51:56,960][488352] Heartbeat connected on RolloutWorker_w120 +[2023-03-07 15:51:57,574][488352] Heartbeat connected on RolloutWorker_w109 +[2023-03-07 15:51:57,814][488352] Heartbeat connected on RolloutWorker_w3 +[2023-03-07 15:51:57,947][488352] Heartbeat connected on RolloutWorker_w27 +[2023-03-07 15:51:58,184][488352] Heartbeat connected on RolloutWorker_w50 +[2023-03-07 15:51:58,389][488352] Heartbeat connected on RolloutWorker_w75 +[2023-03-07 15:51:58,454][488352] Heartbeat connected on RolloutWorker_w113 +[2023-03-07 15:51:58,663][488352] Heartbeat connected on RolloutWorker_w74 +[2023-03-07 15:51:58,912][488352] Heartbeat connected on RolloutWorker_w64 +[2023-03-07 15:51:58,974][488352] Heartbeat connected on RolloutWorker_w106 +[2023-03-07 15:51:59,241][488352] Heartbeat connected on RolloutWorker_w51 +[2023-03-07 15:51:59,286][488352] Heartbeat connected on RolloutWorker_w80 +[2023-03-07 15:51:59,632][488352] Heartbeat connected on RolloutWorker_w47 +[2023-03-07 15:52:00,078][488352] Heartbeat connected on RolloutWorker_w55 +[2023-03-07 15:52:00,081][488352] Heartbeat connected on RolloutWorker_w36 +[2023-03-07 15:52:00,104][488352] Fps is (10 sec: 109360.5, 60 sec: 31027.0, 300 sec: 31027.0). Total num frames: 1241088. Throughput: 0: 6471.0. Samples: 258840. Policy #0 lag: (min: 4.0, avg: 17.4, max: 32.0) +[2023-03-07 15:52:00,106][488352] Avg episode reward: [(0, '4.299')] +[2023-03-07 15:52:00,413][488352] Heartbeat connected on RolloutWorker_w16 +[2023-03-07 15:52:00,481][488352] Heartbeat connected on RolloutWorker_w92 +[2023-03-07 15:52:00,543][488352] Heartbeat connected on RolloutWorker_w94 +[2023-03-07 15:52:00,651][488352] Heartbeat connected on RolloutWorker_w101 +[2023-03-07 15:52:00,697][488352] Heartbeat connected on RolloutWorker_w97 +[2023-03-07 15:52:00,948][488352] Heartbeat connected on RolloutWorker_w118 +[2023-03-07 15:52:00,959][488352] Heartbeat connected on RolloutWorker_w8 +[2023-03-07 15:52:01,065][488352] Heartbeat connected on RolloutWorker_w52 +[2023-03-07 15:52:01,341][488352] Heartbeat connected on RolloutWorker_w31 +[2023-03-07 15:52:01,469][488352] Heartbeat connected on RolloutWorker_w12 +[2023-03-07 15:52:01,516][488352] Heartbeat connected on RolloutWorker_w72 +[2023-03-07 15:52:01,632][488352] Heartbeat connected on RolloutWorker_w37 +[2023-03-07 15:52:01,759][488352] Heartbeat connected on RolloutWorker_w83 +[2023-03-07 15:52:01,810][488352] Heartbeat connected on RolloutWorker_w99 +[2023-03-07 15:52:01,848][488352] Heartbeat connected on RolloutWorker_w93 +[2023-03-07 15:52:01,958][488352] Heartbeat connected on RolloutWorker_w104 +[2023-03-07 15:52:02,076][488352] Heartbeat connected on RolloutWorker_w34 +[2023-03-07 15:52:02,152][488352] Heartbeat connected on RolloutWorker_w116 +[2023-03-07 15:52:02,190][488352] Heartbeat connected on RolloutWorker_w62 +[2023-03-07 15:52:02,338][488352] Heartbeat connected on RolloutWorker_w7 +[2023-03-07 15:52:02,394][488352] Heartbeat connected on RolloutWorker_w57 +[2023-03-07 15:52:02,791][488352] Heartbeat connected on RolloutWorker_w122 +[2023-03-07 15:52:03,019][488352] Heartbeat connected on RolloutWorker_w18 +[2023-03-07 15:52:03,033][488352] Heartbeat connected on RolloutWorker_w88 +[2023-03-07 15:52:03,036][488352] Heartbeat connected on RolloutWorker_w21 +[2023-03-07 15:52:03,300][488352] Heartbeat connected on RolloutWorker_w20 +[2023-03-07 15:52:03,456][488352] Heartbeat connected on RolloutWorker_w46 +[2023-03-07 15:52:03,540][488352] Heartbeat connected on RolloutWorker_w95 +[2023-03-07 15:52:03,558][488352] Heartbeat connected on RolloutWorker_w53 +[2023-03-07 15:52:03,588][488352] Heartbeat connected on RolloutWorker_w40 +[2023-03-07 15:52:03,598][488352] Heartbeat connected on RolloutWorker_w112 +[2023-03-07 15:52:03,872][488352] Heartbeat connected on RolloutWorker_w66 +[2023-03-07 15:52:04,219][488352] Heartbeat connected on RolloutWorker_w41 +[2023-03-07 15:52:04,279][488352] Heartbeat connected on RolloutWorker_w33 +[2023-03-07 15:52:04,422][488352] Heartbeat connected on RolloutWorker_w1 +[2023-03-07 15:52:04,720][488352] Heartbeat connected on RolloutWorker_w78 +[2023-03-07 15:52:04,795][488352] Heartbeat connected on RolloutWorker_w32 +[2023-03-07 15:52:04,822][488352] Heartbeat connected on RolloutWorker_w2 +[2023-03-07 15:52:04,940][488352] Heartbeat connected on RolloutWorker_w126 +[2023-03-07 15:52:05,076][488352] Heartbeat connected on RolloutWorker_w28 +[2023-03-07 15:52:05,098][488352] Heartbeat connected on RolloutWorker_w19 +[2023-03-07 15:52:05,104][488352] Fps is (10 sec: 132708.2, 60 sec: 41688.0, 300 sec: 41688.0). Total num frames: 1875968. Throughput: 0: 10532.9. Samples: 473984. Policy #0 lag: (min: 3.0, avg: 26.2, max: 52.0) +[2023-03-07 15:52:05,106][488352] Avg episode reward: [(0, '4.415')] +[2023-03-07 15:52:05,462][488352] Heartbeat connected on RolloutWorker_w110 +[2023-03-07 15:52:05,647][488352] Heartbeat connected on RolloutWorker_w22 +[2023-03-07 15:52:05,716][488352] Heartbeat connected on RolloutWorker_w44 +[2023-03-07 15:52:05,756][488352] Heartbeat connected on RolloutWorker_w108 +[2023-03-07 15:52:05,925][488352] Heartbeat connected on RolloutWorker_w11 +[2023-03-07 15:52:06,057][488352] Heartbeat connected on RolloutWorker_w81 +[2023-03-07 15:52:06,165][488352] Heartbeat connected on RolloutWorker_w15 +[2023-03-07 15:52:06,421][488352] Heartbeat connected on RolloutWorker_w125 +[2023-03-07 15:52:06,485][488352] Heartbeat connected on RolloutWorker_w4 +[2023-03-07 15:52:06,526][488352] Heartbeat connected on RolloutWorker_w67 +[2023-03-07 15:52:06,680][488352] Heartbeat connected on RolloutWorker_w23 +[2023-03-07 15:52:06,974][488352] Heartbeat connected on RolloutWorker_w68 +[2023-03-07 15:52:06,999][488352] Heartbeat connected on RolloutWorker_w123 +[2023-03-07 15:52:07,131][488352] Heartbeat connected on RolloutWorker_w70 +[2023-03-07 15:52:07,180][488352] Heartbeat connected on RolloutWorker_w10 +[2023-03-07 15:52:07,236][488352] Heartbeat connected on RolloutWorker_w82 +[2023-03-07 15:52:08,006][488352] Heartbeat connected on RolloutWorker_w6 +[2023-03-07 15:52:10,104][488352] Fps is (10 sec: 128617.4, 60 sec: 50544.6, 300 sec: 50544.6). Total num frames: 2527232. Throughput: 0: 12673.4. Samples: 570304. Policy #0 lag: (min: 8.0, avg: 36.9, max: 74.0) +[2023-03-07 15:52:10,105][488352] Avg episode reward: [(0, '4.420')] +[2023-03-07 15:52:15,104][488352] Fps is (10 sec: 134350.2, 60 sec: 58535.5, 300 sec: 58535.5). Total num frames: 3219456. Throughput: 0: 18666.5. Samples: 839992. Policy #0 lag: (min: 11.0, avg: 42.4, max: 73.0) +[2023-03-07 15:52:15,105][488352] Avg episode reward: [(0, '5.214')] +[2023-03-07 15:52:19,772][488352] Component Batcher_0 stopped! +[2023-03-07 15:52:19,816][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 15:52:20,391][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 15:52:20,399][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 15:52:20,404][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 15:52:20,408][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 15:52:20,412][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 15:52:20,413][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 15:52:20,415][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 15:52:20,417][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 15:52:20,418][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 15:52:20,420][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 15:52:20,427][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 15:52:20,431][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 15:52:20,433][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 15:52:20,435][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 15:52:20,436][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 15:52:20,437][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 15:52:20,438][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 15:52:20,439][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 15:52:20,440][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 15:52:20,441][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 15:52:20,443][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 15:52:20,444][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 15:52:20,445][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 15:52:20,446][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 15:52:20,447][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 15:52:20,448][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 15:52:20,449][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 15:52:20,461][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 15:52:20,462][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 15:52:20,463][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 15:52:20,465][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 15:52:20,466][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 15:52:20,468][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 15:52:20,470][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 15:52:20,471][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 15:52:20,471][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 15:52:20,473][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 15:52:20,474][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 15:52:20,475][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 15:52:20,476][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 15:52:20,477][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 15:52:20,478][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 15:52:20,479][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 15:52:20,480][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 15:52:20,482][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 15:52:20,483][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 15:52:20,484][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 15:52:20,488][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 15:52:20,490][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 15:52:20,491][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 15:52:20,492][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 15:52:20,493][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 15:52:20,494][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 15:52:20,495][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 15:52:20,496][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 15:52:20,497][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 15:52:20,498][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 15:52:20,500][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 15:52:20,501][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 15:52:20,502][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 15:52:20,503][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 15:52:20,504][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 15:52:20,504][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 15:52:20,506][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 15:52:20,507][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 15:52:20,508][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 15:52:20,509][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 15:52:20,510][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 15:52:20,511][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 15:52:20,512][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 15:52:20,513][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 15:52:20,514][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 15:52:20,515][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 15:52:20,516][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 15:52:20,516][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 15:52:20,517][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 15:52:20,518][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 15:52:20,520][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 15:52:20,521][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 15:52:20,522][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 15:52:20,522][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 15:52:20,523][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 15:52:20,531][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 15:52:20,532][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 15:52:20,533][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 15:52:20,533][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 15:52:20,534][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 15:52:20,535][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 15:52:20,536][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 15:52:20,536][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 15:52:20,537][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 15:52:20,537][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 15:52:20,538][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 15:52:20,544][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 15:52:20,545][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 15:52:20,545][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 15:52:20,546][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 15:52:20,547][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 15:52:20,547][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 15:52:20,548][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 15:52:20,548][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 15:52:20,550][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 15:52:20,551][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 15:52:20,552][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 15:52:20,552][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 15:52:20,553][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 15:52:20,554][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 15:52:20,554][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 15:52:20,555][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 15:52:20,555][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 15:52:20,556][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 15:52:20,557][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 15:52:20,562][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 15:52:20,564][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 15:52:20,565][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 15:52:20,565][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 15:52:20,566][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 15:52:20,566][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 15:52:20,567][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 15:52:20,567][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 15:52:20,568][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 15:52:20,568][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 15:52:20,569][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 15:52:20,570][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 15:52:20,570][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 15:52:20,571][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 15:52:20,571][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 15:52:20,603][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 15:52:20,620][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 15:52:20,622][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 15:52:22,182][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 15:52:22,183][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 15:52:22,184][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 15:52:22,186][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 15:52:22,187][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 15:52:22,188][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 15:52:22,189][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 15:52:22,190][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 15:52:22,191][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 15:52:22,192][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 15:52:22,193][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 15:52:22,194][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 15:52:22,195][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 15:52:22,196][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 15:52:22,197][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 15:52:22,198][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 15:52:22,199][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 15:52:22,200][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 15:52:22,201][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 15:52:22,202][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 15:52:22,203][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 15:52:22,204][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 15:52:22,205][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 15:52:22,207][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 15:52:22,208][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 15:52:22,209][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 15:52:22,210][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 15:52:22,213][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 15:52:22,214][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 15:52:22,215][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 15:52:22,216][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 15:52:22,217][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 15:52:22,218][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 15:52:22,220][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 15:52:22,222][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 15:52:22,224][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 15:52:22,225][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 15:52:22,227][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 15:52:22,229][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 15:52:22,230][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 15:52:22,232][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 15:52:22,234][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 15:52:22,235][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 15:52:22,237][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 15:52:22,239][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 15:52:22,240][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 15:52:22,242][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 15:52:22,243][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 15:52:22,245][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 15:52:22,249][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 15:52:22,250][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 15:52:22,252][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 15:52:22,253][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 15:52:22,255][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 15:52:22,257][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 15:52:22,259][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 15:52:22,261][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 15:52:22,262][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 15:52:22,264][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 15:52:22,266][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 15:52:22,268][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 15:52:22,269][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 15:52:22,272][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 15:52:22,274][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 15:52:22,276][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 15:52:22,278][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 15:52:22,281][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 15:52:22,282][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 15:52:22,284][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 15:52:22,286][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 15:52:22,287][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 15:52:22,289][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 15:52:22,291][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 15:52:22,293][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 15:52:22,300][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 15:52:22,301][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 15:52:22,302][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 15:52:22,303][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 15:52:22,305][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 15:52:22,306][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 15:52:22,307][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 15:52:22,308][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 15:52:22,309][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 15:52:22,311][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 15:52:22,312][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 15:52:22,313][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 15:52:22,314][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 15:52:22,315][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 15:52:22,316][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 15:52:22,317][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 15:52:22,319][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 15:52:22,320][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 15:52:22,321][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 15:52:22,325][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 15:52:22,326][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 15:52:22,328][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 15:52:22,329][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 15:52:22,330][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 15:52:22,332][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 15:52:22,334][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 15:52:22,335][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 15:52:22,336][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 15:52:22,337][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 15:52:22,343][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 15:52:22,345][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 15:52:22,346][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 15:52:22,347][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 15:52:22,348][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 15:52:22,349][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 15:52:22,351][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 15:52:22,352][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 15:52:22,353][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 15:52:22,354][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 15:52:22,356][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 15:52:22,357][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 15:52:22,358][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 15:52:22,360][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 15:52:22,361][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 15:52:22,362][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 15:52:22,363][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 15:52:22,364][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 15:52:22,365][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 15:52:22,368][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 15:52:22,369][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 15:52:22,370][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 15:52:22,371][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 15:52:22,372][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 15:52:22,374][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 15:52:22,376][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 15:52:22,377][488352] Batcher 0 profile tree view: +batching: 2.2164, releasing_batches: 10.5474 +[2023-03-07 15:52:22,378][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0003 + wait_policy_total: 31.7612 +update_model: 1.4893 + weight_update: 0.0008 +one_step: 0.0140 + handle_policy_step: 25.7161 + deserialize: 6.0890, stack: 0.0763, obs_to_device_normalize: 5.6382, forward: 7.5461, send_messages: 2.7665 + prepare_outputs: 3.1118 + to_cpu: 1.4099 +[2023-03-07 15:52:22,379][488352] Learner 0 profile tree view: +misc: 0.0045, prepare_batch: 10.1715 +train: 17.8911 + epoch_init: 0.0045, minibatch_init: 0.0047, losses_postprocess: 0.5540, kl_divergence: 0.4576, after_optimizer: 0.9465 + calculate_losses: 7.6486 + losses_init: 0.0029, forward_head: 0.6441, bptt_initial: 4.1491, tail: 0.5483, advantages_returns: 0.1307, losses: 0.9404 + bptt: 1.0857 + bptt_forward_core: 1.0375 + update: 7.9721 + clip: 1.0980 +[2023-03-07 15:52:22,379][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0150, enqueue_policy_requests: 0.6305, env_step: 10.1836, overhead: 0.9249, complete_rollouts: 0.0071 +save_policy_outputs: 1.1199 + split_output_tensors: 0.5268 +[2023-03-07 15:52:22,380][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0153, enqueue_policy_requests: 0.8282, env_step: 13.4191, overhead: 1.2370, complete_rollouts: 0.0092 +save_policy_outputs: 1.4471 + split_output_tensors: 0.6815 +[2023-03-07 15:52:22,381][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 15:52:22,383][488352] Runner profile tree view: +main_loop: 69.6832 +[2023-03-07 15:52:22,384][488352] Collected {0: 4014080}, FPS: 57604.7 +[2023-03-07 15:53:48,396][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 15:53:48,396][488352] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 15:53:48,397][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:53:48,397][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:53:48,398][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:53:48,398][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:53:48,399][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:53:48,399][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:53:48,400][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 15:53:48,401][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 15:53:48,401][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:53:48,402][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:53:48,402][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:53:48,403][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:53:48,403][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:53:48,411][488352] Doom resolution: 160x120, resize resolution: (128, 72) +[2023-03-07 15:53:48,412][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:53:48,413][488352] RunningMeanStd input shape: (1,) +[2023-03-07 15:53:48,423][488352] ConvEncoder: input_channels=3 +[2023-03-07 15:53:48,513][488352] Conv encoder output size: 512 +[2023-03-07 15:53:48,514][488352] Policy head output size: 512 +[2023-03-07 15:53:50,121][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 15:53:51,054][488352] Num frames 100... +[2023-03-07 15:53:51,138][488352] Num frames 200... +[2023-03-07 15:53:51,224][488352] Num frames 300... +[2023-03-07 15:53:51,311][488352] Num frames 400... +[2023-03-07 15:53:51,406][488352] Avg episode rewards: #0: 5.480, true rewards: #0: 4.480 +[2023-03-07 15:53:51,407][488352] Avg episode reward: 5.480, avg true_objective: 4.480 +[2023-03-07 15:53:51,471][488352] Num frames 500... +[2023-03-07 15:53:51,556][488352] Num frames 600... +[2023-03-07 15:53:51,641][488352] Num frames 700... +[2023-03-07 15:53:51,727][488352] Num frames 800... +[2023-03-07 15:53:51,811][488352] Num frames 900... +[2023-03-07 15:53:51,915][488352] Avg episode rewards: #0: 6.800, true rewards: #0: 4.800 +[2023-03-07 15:53:51,916][488352] Avg episode reward: 6.800, avg true_objective: 4.800 +[2023-03-07 15:53:51,952][488352] Num frames 1000... +[2023-03-07 15:53:52,037][488352] Num frames 1100... +[2023-03-07 15:53:52,124][488352] Num frames 1200... +[2023-03-07 15:53:52,210][488352] Num frames 1300... +[2023-03-07 15:53:52,301][488352] Avg episode rewards: #0: 5.813, true rewards: #0: 4.480 +[2023-03-07 15:53:52,302][488352] Avg episode reward: 5.813, avg true_objective: 4.480 +[2023-03-07 15:53:52,368][488352] Num frames 1400... +[2023-03-07 15:53:52,451][488352] Num frames 1500... +[2023-03-07 15:53:52,537][488352] Num frames 1600... +[2023-03-07 15:53:52,621][488352] Num frames 1700... +[2023-03-07 15:53:52,751][488352] Avg episode rewards: #0: 5.730, true rewards: #0: 4.480 +[2023-03-07 15:53:52,752][488352] Avg episode reward: 5.730, avg true_objective: 4.480 +[2023-03-07 15:53:52,762][488352] Num frames 1800... +[2023-03-07 15:53:52,861][488352] Num frames 1900... +[2023-03-07 15:53:52,945][488352] Num frames 2000... +[2023-03-07 15:53:53,031][488352] Num frames 2100... +[2023-03-07 15:53:53,116][488352] Num frames 2200... +[2023-03-07 15:53:53,205][488352] Avg episode rewards: #0: 5.680, true rewards: #0: 4.480 +[2023-03-07 15:53:53,206][488352] Avg episode reward: 5.680, avg true_objective: 4.480 +[2023-03-07 15:53:53,281][488352] Num frames 2300... +[2023-03-07 15:53:53,363][488352] Num frames 2400... +[2023-03-07 15:53:53,448][488352] Num frames 2500... +[2023-03-07 15:53:53,534][488352] Num frames 2600... +[2023-03-07 15:53:53,664][488352] Avg episode rewards: #0: 5.647, true rewards: #0: 4.480 +[2023-03-07 15:53:53,664][488352] Avg episode reward: 5.647, avg true_objective: 4.480 +[2023-03-07 15:53:53,678][488352] Num frames 2700... +[2023-03-07 15:53:53,778][488352] Num frames 2800... +[2023-03-07 15:53:53,864][488352] Num frames 2900... +[2023-03-07 15:53:53,951][488352] Num frames 3000... +[2023-03-07 15:53:54,067][488352] Avg episode rewards: #0: 5.389, true rewards: #0: 4.389 +[2023-03-07 15:53:54,068][488352] Avg episode reward: 5.389, avg true_objective: 4.389 +[2023-03-07 15:53:54,104][488352] Num frames 3100... +[2023-03-07 15:53:54,195][488352] Num frames 3200... +[2023-03-07 15:53:54,279][488352] Num frames 3300... +[2023-03-07 15:53:54,362][488352] Num frames 3400... +[2023-03-07 15:53:54,448][488352] Num frames 3500... +[2023-03-07 15:53:54,518][488352] Avg episode rewards: #0: 5.400, true rewards: #0: 4.400 +[2023-03-07 15:53:54,519][488352] Avg episode reward: 5.400, avg true_objective: 4.400 +[2023-03-07 15:53:54,588][488352] Num frames 3600... +[2023-03-07 15:53:54,674][488352] Num frames 3700... +[2023-03-07 15:53:54,759][488352] Num frames 3800... +[2023-03-07 15:53:54,845][488352] Num frames 3900... +[2023-03-07 15:53:54,931][488352] Num frames 4000... +[2023-03-07 15:53:55,017][488352] Num frames 4100... +[2023-03-07 15:53:55,095][488352] Avg episode rewards: #0: 5.809, true rewards: #0: 4.587 +[2023-03-07 15:53:55,096][488352] Avg episode reward: 5.809, avg true_objective: 4.587 +[2023-03-07 15:53:55,174][488352] Num frames 4200... +[2023-03-07 15:53:55,257][488352] Num frames 4300... +[2023-03-07 15:53:55,340][488352] Num frames 4400... +[2023-03-07 15:53:55,423][488352] Num frames 4500... +[2023-03-07 15:53:55,539][488352] Avg episode rewards: #0: 5.776, true rewards: #0: 4.576 +[2023-03-07 15:53:55,540][488352] Avg episode reward: 5.776, avg true_objective: 4.576 +[2023-03-07 15:54:00,892][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 15:57:00,737][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 15:57:00,739][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 15:57:00,740][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 15:57:00,740][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:57:00,741][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 15:57:00,742][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 15:57:00,743][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 15:57:00,744][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 15:57:00,745][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 15:57:00,745][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 15:57:00,746][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 15:57:00,747][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 15:57:00,747][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 15:57:00,748][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 15:57:00,753][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 15:57:00,755][488352] RunningMeanStd input shape: (1,) +[2023-03-07 15:57:00,770][488352] ConvEncoder: input_channels=3 +[2023-03-07 15:57:00,815][488352] Conv encoder output size: 512 +[2023-03-07 15:57:00,816][488352] Policy head output size: 512 +[2023-03-07 15:57:00,842][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/checkpoint_p0/checkpoint_000000980_4014080.pth... +[2023-03-07 15:57:01,404][488352] Num frames 100... +[2023-03-07 15:57:01,489][488352] Num frames 200... +[2023-03-07 15:57:01,575][488352] Num frames 300... +[2023-03-07 15:57:01,700][488352] Avg episode rewards: #0: 3.840, true rewards: #0: 3.840 +[2023-03-07 15:57:01,701][488352] Avg episode reward: 3.840, avg true_objective: 3.840 +[2023-03-07 15:57:01,716][488352] Num frames 400... +[2023-03-07 15:57:01,801][488352] Num frames 500... +[2023-03-07 15:57:01,886][488352] Num frames 600... +[2023-03-07 15:57:01,970][488352] Num frames 700... +[2023-03-07 15:57:02,057][488352] Num frames 800... +[2023-03-07 15:57:02,108][488352] Avg episode rewards: #0: 4.500, true rewards: #0: 4.000 +[2023-03-07 15:57:02,109][488352] Avg episode reward: 4.500, avg true_objective: 4.000 +[2023-03-07 15:57:02,213][488352] Num frames 900... +[2023-03-07 15:57:02,297][488352] Num frames 1000... +[2023-03-07 15:57:02,381][488352] Num frames 1100... +[2023-03-07 15:57:02,465][488352] Num frames 1200... +[2023-03-07 15:57:02,586][488352] Avg episode rewards: #0: 5.267, true rewards: #0: 4.267 +[2023-03-07 15:57:02,587][488352] Avg episode reward: 5.267, avg true_objective: 4.267 +[2023-03-07 15:57:02,611][488352] Num frames 1300... +[2023-03-07 15:57:02,706][488352] Num frames 1400... +[2023-03-07 15:57:02,789][488352] Num frames 1500... +[2023-03-07 15:57:02,873][488352] Num frames 1600... +[2023-03-07 15:57:02,956][488352] Num frames 1700... +[2023-03-07 15:57:03,059][488352] Avg episode rewards: #0: 5.650, true rewards: #0: 4.400 +[2023-03-07 15:57:03,060][488352] Avg episode reward: 5.650, avg true_objective: 4.400 +[2023-03-07 15:57:03,109][488352] Num frames 1800... +[2023-03-07 15:57:03,194][488352] Num frames 1900... +[2023-03-07 15:57:03,276][488352] Num frames 2000... +[2023-03-07 15:57:03,360][488352] Num frames 2100... +[2023-03-07 15:57:03,444][488352] Num frames 2200... +[2023-03-07 15:57:03,505][488352] Avg episode rewards: #0: 5.616, true rewards: #0: 4.416 +[2023-03-07 15:57:03,506][488352] Avg episode reward: 5.616, avg true_objective: 4.416 +[2023-03-07 15:57:03,604][488352] Num frames 2300... +[2023-03-07 15:57:03,688][488352] Num frames 2400... +[2023-03-07 15:57:03,774][488352] Num frames 2500... +[2023-03-07 15:57:03,902][488352] Avg episode rewards: #0: 5.310, true rewards: #0: 4.310 +[2023-03-07 15:57:03,903][488352] Avg episode reward: 5.310, avg true_objective: 4.310 +[2023-03-07 15:57:03,920][488352] Num frames 2600... +[2023-03-07 15:57:04,017][488352] Num frames 2700... +[2023-03-07 15:57:04,099][488352] Num frames 2800... +[2023-03-07 15:57:04,181][488352] Num frames 2900... +[2023-03-07 15:57:04,292][488352] Avg episode rewards: #0: 5.100, true rewards: #0: 4.243 +[2023-03-07 15:57:04,293][488352] Avg episode reward: 5.100, avg true_objective: 4.243 +[2023-03-07 15:57:04,331][488352] Num frames 3000... +[2023-03-07 15:57:04,423][488352] Num frames 3100... +[2023-03-07 15:57:04,507][488352] Num frames 3200... +[2023-03-07 15:57:04,590][488352] Num frames 3300... +[2023-03-07 15:57:04,689][488352] Avg episode rewards: #0: 4.943, true rewards: #0: 4.192 +[2023-03-07 15:57:04,690][488352] Avg episode reward: 4.943, avg true_objective: 4.192 +[2023-03-07 15:57:04,728][488352] Num frames 3400... +[2023-03-07 15:57:04,813][488352] Num frames 3500... +[2023-03-07 15:57:04,897][488352] Num frames 3600... +[2023-03-07 15:57:04,981][488352] Num frames 3700... +[2023-03-07 15:57:05,065][488352] Num frames 3800... +[2023-03-07 15:57:05,121][488352] Avg episode rewards: #0: 5.002, true rewards: #0: 4.224 +[2023-03-07 15:57:05,122][488352] Avg episode reward: 5.002, avg true_objective: 4.224 +[2023-03-07 15:57:05,221][488352] Num frames 3900... +[2023-03-07 15:57:05,305][488352] Num frames 4000... +[2023-03-07 15:57:05,389][488352] Num frames 4100... +[2023-03-07 15:57:05,473][488352] Num frames 4200... +[2023-03-07 15:57:05,569][488352] Avg episode rewards: #0: 5.050, true rewards: #0: 4.250 +[2023-03-07 15:57:05,569][488352] Avg episode reward: 5.050, avg true_objective: 4.250 +[2023-03-07 15:57:10,623][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/replay.mp4! +[2023-03-07 15:59:15,783][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 15:59:15,785][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 15:59:15,785][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 15:59:15,786][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 15:59:15,787][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 15:59:15,788][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 15:59:15,789][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 15:59:15,790][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 15:59:15,790][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 15:59:15,791][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 15:59:15,792][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 15:59:15,792][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 15:59:15,793][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 15:59:15,794][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 15:59:15,795][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 15:59:15,796][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 15:59:15,796][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 15:59:15,798][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 15:59:15,798][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 15:59:15,799][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 15:59:15,800][488352] register_encoder_factory: +[2023-03-07 15:59:15,829][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 15:59:15,829][488352] Overriding arg 'num_envs_per_worker' with value 32 passed from command line +[2023-03-07 15:59:15,830][488352] Overriding arg 'train_for_env_steps' with value 20000000 passed from command line +[2023-03-07 15:59:15,835][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads already exists! +[2023-03-07 15:59:15,836][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads... +[2023-03-07 15:59:15,837][488352] Weights and Biases integration disabled +[2023-03-07 15:59:15,839][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 15:59:16,892][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=20000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=16 --train_for_env_steps=4000000 --experiment=doom_health_threads +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads', 'num_workers': 128, 'num_envs_per_worker': 16, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 15:59:16,892][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json... +[2023-03-07 15:59:16,894][488352] Rollout worker 0 uses device cpu +[2023-03-07 15:59:16,895][488352] Rollout worker 1 uses device cpu +[2023-03-07 15:59:16,895][488352] Rollout worker 2 uses device cpu +[2023-03-07 15:59:16,896][488352] Rollout worker 3 uses device cpu +[2023-03-07 15:59:16,896][488352] Rollout worker 4 uses device cpu +[2023-03-07 15:59:16,897][488352] Rollout worker 5 uses device cpu +[2023-03-07 15:59:16,897][488352] Rollout worker 6 uses device cpu +[2023-03-07 15:59:16,898][488352] Rollout worker 7 uses device cpu +[2023-03-07 15:59:16,898][488352] Rollout worker 8 uses device cpu +[2023-03-07 15:59:16,899][488352] Rollout worker 9 uses device cpu +[2023-03-07 15:59:16,900][488352] Rollout worker 10 uses device cpu +[2023-03-07 15:59:16,900][488352] Rollout worker 11 uses device cpu +[2023-03-07 15:59:16,901][488352] Rollout worker 12 uses device cpu +[2023-03-07 15:59:16,901][488352] Rollout worker 13 uses device cpu +[2023-03-07 15:59:16,902][488352] Rollout worker 14 uses device cpu +[2023-03-07 15:59:16,903][488352] Rollout worker 15 uses device cpu +[2023-03-07 15:59:16,903][488352] Rollout worker 16 uses device cpu +[2023-03-07 15:59:16,904][488352] Rollout worker 17 uses device cpu +[2023-03-07 15:59:16,904][488352] Rollout worker 18 uses device cpu +[2023-03-07 15:59:16,905][488352] Rollout worker 19 uses device cpu +[2023-03-07 15:59:16,905][488352] Rollout worker 20 uses device cpu +[2023-03-07 15:59:16,906][488352] Rollout worker 21 uses device cpu +[2023-03-07 15:59:16,907][488352] Rollout worker 22 uses device cpu +[2023-03-07 15:59:16,907][488352] Rollout worker 23 uses device cpu +[2023-03-07 15:59:16,908][488352] Rollout worker 24 uses device cpu +[2023-03-07 15:59:16,908][488352] Rollout worker 25 uses device cpu +[2023-03-07 15:59:16,909][488352] Rollout worker 26 uses device cpu +[2023-03-07 15:59:16,909][488352] Rollout worker 27 uses device cpu +[2023-03-07 15:59:16,910][488352] Rollout worker 28 uses device cpu +[2023-03-07 15:59:16,911][488352] Rollout worker 29 uses device cpu +[2023-03-07 15:59:16,912][488352] Rollout worker 30 uses device cpu +[2023-03-07 15:59:16,913][488352] Rollout worker 31 uses device cpu +[2023-03-07 15:59:16,913][488352] Rollout worker 32 uses device cpu +[2023-03-07 15:59:16,914][488352] Rollout worker 33 uses device cpu +[2023-03-07 15:59:16,915][488352] Rollout worker 34 uses device cpu +[2023-03-07 15:59:16,916][488352] Rollout worker 35 uses device cpu +[2023-03-07 15:59:16,917][488352] Rollout worker 36 uses device cpu +[2023-03-07 15:59:16,918][488352] Rollout worker 37 uses device cpu +[2023-03-07 15:59:16,919][488352] Rollout worker 38 uses device cpu +[2023-03-07 15:59:16,919][488352] Rollout worker 39 uses device cpu +[2023-03-07 15:59:16,920][488352] Rollout worker 40 uses device cpu +[2023-03-07 15:59:16,921][488352] Rollout worker 41 uses device cpu +[2023-03-07 15:59:16,922][488352] Rollout worker 42 uses device cpu +[2023-03-07 15:59:16,923][488352] Rollout worker 43 uses device cpu +[2023-03-07 15:59:16,924][488352] Rollout worker 44 uses device cpu +[2023-03-07 15:59:16,924][488352] Rollout worker 45 uses device cpu +[2023-03-07 15:59:16,925][488352] Rollout worker 46 uses device cpu +[2023-03-07 15:59:16,926][488352] Rollout worker 47 uses device cpu +[2023-03-07 15:59:16,927][488352] Rollout worker 48 uses device cpu +[2023-03-07 15:59:16,928][488352] Rollout worker 49 uses device cpu +[2023-03-07 15:59:16,928][488352] Rollout worker 50 uses device cpu +[2023-03-07 15:59:16,929][488352] Rollout worker 51 uses device cpu +[2023-03-07 15:59:16,930][488352] Rollout worker 52 uses device cpu +[2023-03-07 15:59:16,931][488352] Rollout worker 53 uses device cpu +[2023-03-07 15:59:16,932][488352] Rollout worker 54 uses device cpu +[2023-03-07 15:59:16,933][488352] Rollout worker 55 uses device cpu +[2023-03-07 15:59:16,934][488352] Rollout worker 56 uses device cpu +[2023-03-07 15:59:16,935][488352] Rollout worker 57 uses device cpu +[2023-03-07 15:59:16,935][488352] Rollout worker 58 uses device cpu +[2023-03-07 15:59:16,936][488352] Rollout worker 59 uses device cpu +[2023-03-07 15:59:16,937][488352] Rollout worker 60 uses device cpu +[2023-03-07 15:59:16,937][488352] Rollout worker 61 uses device cpu +[2023-03-07 15:59:16,939][488352] Rollout worker 62 uses device cpu +[2023-03-07 15:59:16,940][488352] Rollout worker 63 uses device cpu +[2023-03-07 15:59:16,940][488352] Rollout worker 64 uses device cpu +[2023-03-07 15:59:16,941][488352] Rollout worker 65 uses device cpu +[2023-03-07 15:59:16,942][488352] Rollout worker 66 uses device cpu +[2023-03-07 15:59:16,943][488352] Rollout worker 67 uses device cpu +[2023-03-07 15:59:16,944][488352] Rollout worker 68 uses device cpu +[2023-03-07 15:59:16,945][488352] Rollout worker 69 uses device cpu +[2023-03-07 15:59:16,946][488352] Rollout worker 70 uses device cpu +[2023-03-07 15:59:16,947][488352] Rollout worker 71 uses device cpu +[2023-03-07 15:59:16,947][488352] Rollout worker 72 uses device cpu +[2023-03-07 15:59:16,948][488352] Rollout worker 73 uses device cpu +[2023-03-07 15:59:16,949][488352] Rollout worker 74 uses device cpu +[2023-03-07 15:59:16,950][488352] Rollout worker 75 uses device cpu +[2023-03-07 15:59:16,951][488352] Rollout worker 76 uses device cpu +[2023-03-07 15:59:16,951][488352] Rollout worker 77 uses device cpu +[2023-03-07 15:59:16,952][488352] Rollout worker 78 uses device cpu +[2023-03-07 15:59:16,953][488352] Rollout worker 79 uses device cpu +[2023-03-07 15:59:16,953][488352] Rollout worker 80 uses device cpu +[2023-03-07 15:59:16,955][488352] Rollout worker 81 uses device cpu +[2023-03-07 15:59:16,956][488352] Rollout worker 82 uses device cpu +[2023-03-07 15:59:16,956][488352] Rollout worker 83 uses device cpu +[2023-03-07 15:59:16,957][488352] Rollout worker 84 uses device cpu +[2023-03-07 15:59:16,958][488352] Rollout worker 85 uses device cpu +[2023-03-07 15:59:16,959][488352] Rollout worker 86 uses device cpu +[2023-03-07 15:59:16,960][488352] Rollout worker 87 uses device cpu +[2023-03-07 15:59:16,961][488352] Rollout worker 88 uses device cpu +[2023-03-07 15:59:16,961][488352] Rollout worker 89 uses device cpu +[2023-03-07 15:59:16,963][488352] Rollout worker 90 uses device cpu +[2023-03-07 15:59:16,963][488352] Rollout worker 91 uses device cpu +[2023-03-07 15:59:16,964][488352] Rollout worker 92 uses device cpu +[2023-03-07 15:59:16,965][488352] Rollout worker 93 uses device cpu +[2023-03-07 15:59:16,966][488352] Rollout worker 94 uses device cpu +[2023-03-07 15:59:16,967][488352] Rollout worker 95 uses device cpu +[2023-03-07 15:59:16,968][488352] Rollout worker 96 uses device cpu +[2023-03-07 15:59:16,968][488352] Rollout worker 97 uses device cpu +[2023-03-07 15:59:16,969][488352] Rollout worker 98 uses device cpu +[2023-03-07 15:59:16,970][488352] Rollout worker 99 uses device cpu +[2023-03-07 15:59:16,971][488352] Rollout worker 100 uses device cpu +[2023-03-07 15:59:16,972][488352] Rollout worker 101 uses device cpu +[2023-03-07 15:59:16,972][488352] Rollout worker 102 uses device cpu +[2023-03-07 15:59:16,973][488352] Rollout worker 103 uses device cpu +[2023-03-07 15:59:16,974][488352] Rollout worker 104 uses device cpu +[2023-03-07 15:59:16,975][488352] Rollout worker 105 uses device cpu +[2023-03-07 15:59:16,976][488352] Rollout worker 106 uses device cpu +[2023-03-07 15:59:16,977][488352] Rollout worker 107 uses device cpu +[2023-03-07 15:59:16,977][488352] Rollout worker 108 uses device cpu +[2023-03-07 15:59:16,979][488352] Rollout worker 109 uses device cpu +[2023-03-07 15:59:16,979][488352] Rollout worker 110 uses device cpu +[2023-03-07 15:59:16,980][488352] Rollout worker 111 uses device cpu +[2023-03-07 15:59:16,981][488352] Rollout worker 112 uses device cpu +[2023-03-07 15:59:16,982][488352] Rollout worker 113 uses device cpu +[2023-03-07 15:59:16,983][488352] Rollout worker 114 uses device cpu +[2023-03-07 15:59:16,984][488352] Rollout worker 115 uses device cpu +[2023-03-07 15:59:16,984][488352] Rollout worker 116 uses device cpu +[2023-03-07 15:59:16,985][488352] Rollout worker 117 uses device cpu +[2023-03-07 15:59:16,986][488352] Rollout worker 118 uses device cpu +[2023-03-07 15:59:16,987][488352] Rollout worker 119 uses device cpu +[2023-03-07 15:59:16,988][488352] Rollout worker 120 uses device cpu +[2023-03-07 15:59:16,989][488352] Rollout worker 121 uses device cpu +[2023-03-07 15:59:16,990][488352] Rollout worker 122 uses device cpu +[2023-03-07 15:59:16,990][488352] Rollout worker 123 uses device cpu +[2023-03-07 15:59:16,991][488352] Rollout worker 124 uses device cpu +[2023-03-07 15:59:16,992][488352] Rollout worker 125 uses device cpu +[2023-03-07 15:59:16,993][488352] Rollout worker 126 uses device cpu +[2023-03-07 15:59:16,994][488352] Rollout worker 127 uses device cpu +[2023-03-07 15:59:19,154][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 15:59:19,155][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 15:59:19,419][488352] Starting all processes... +[2023-03-07 15:59:19,419][488352] Starting process learner_proc0 +[2023-03-07 15:59:20,306][488352] Starting all processes... +[2023-03-07 15:59:20,311][488352] Starting process inference_proc0-0 +[2023-03-07 15:59:20,312][488352] Starting process rollout_proc2 +[2023-03-07 15:59:20,312][488352] Starting process rollout_proc5 +[2023-03-07 15:59:20,313][488352] Starting process rollout_proc8 +[2023-03-07 15:59:20,313][488352] Starting process rollout_proc11 +[2023-03-07 15:59:20,323][488352] Starting process rollout_proc14 +[2023-03-07 15:59:20,326][488352] Starting process rollout_proc17 +[2023-03-07 15:59:20,327][488352] Starting process rollout_proc20 +[2023-03-07 15:59:20,330][488352] Starting process rollout_proc23 +[2023-03-07 15:59:20,332][488352] Starting process rollout_proc26 +[2023-03-07 15:59:20,333][488352] Starting process rollout_proc29 +[2023-03-07 15:59:20,338][488352] Starting process rollout_proc32 +[2023-03-07 15:59:20,343][488352] Starting process rollout_proc35 +[2023-03-07 15:59:20,343][488352] Starting process rollout_proc38 +[2023-03-07 15:59:20,344][488352] Starting process rollout_proc41 +[2023-03-07 15:59:20,346][488352] Starting process rollout_proc44 +[2023-03-07 15:59:20,360][488352] Starting process rollout_proc3 +[2023-03-07 15:59:20,366][488352] Starting process rollout_proc12 +[2023-03-07 15:59:20,369][488352] Starting process rollout_proc9 +[2023-03-07 15:59:20,369][488352] Starting process rollout_proc6 +[2023-03-07 15:59:20,378][488352] Starting process rollout_proc15 +[2023-03-07 15:59:20,388][488352] Starting process rollout_proc18 +[2023-03-07 15:59:20,396][488352] Starting process rollout_proc21 +[2023-03-07 15:59:20,410][488352] Starting process rollout_proc24 +[2023-03-07 15:59:20,411][488352] Starting process rollout_proc27 +[2023-03-07 15:59:20,423][488352] Starting process rollout_proc30 +[2023-03-07 15:59:20,433][488352] Starting process rollout_proc33 +[2023-03-07 15:59:20,439][488352] Starting process rollout_proc39 +[2023-03-07 15:59:20,451][488352] Starting process rollout_proc36 +[2023-03-07 15:59:20,466][488352] Starting process rollout_proc4 +[2023-03-07 15:59:20,471][488352] Starting process rollout_proc42 +[2023-03-07 15:59:20,475][488352] Starting process rollout_proc45 +[2023-03-07 15:59:20,477][488352] Starting process rollout_proc13 +[2023-03-07 15:59:20,486][488352] Starting process rollout_proc10 +[2023-03-07 15:59:20,491][488352] Starting process rollout_proc7 +[2023-03-07 15:59:20,491][488352] Starting process rollout_proc16 +[2023-03-07 15:59:20,499][488352] Starting process rollout_proc22 +[2023-03-07 15:59:20,502][488352] Starting process rollout_proc19 +[2023-03-07 15:59:20,513][488352] Starting process rollout_proc25 +[2023-03-07 15:59:20,513][488352] Starting process rollout_proc28 +[2023-03-07 15:59:20,518][488352] Starting process rollout_proc31 +[2023-03-07 15:59:20,526][488352] Starting process rollout_proc34 +[2023-03-07 15:59:20,540][488352] Starting process rollout_proc47 +[2023-03-07 15:59:20,540][488352] Starting process rollout_proc40 +[2023-03-07 15:59:20,546][488352] Starting process rollout_proc37 +[2023-03-07 15:59:20,552][488352] Starting process rollout_proc43 +[2023-03-07 15:59:20,564][488352] Starting process rollout_proc46 +[2023-03-07 15:59:20,568][488352] Starting process rollout_proc50 +[2023-03-07 15:59:20,574][488352] Starting process rollout_proc53 +[2023-03-07 15:59:20,582][488352] Starting process rollout_proc56 +[2023-03-07 15:59:20,587][488352] Starting process rollout_proc59 +[2023-03-07 15:59:20,599][488352] Starting process rollout_proc62 +[2023-03-07 15:59:20,600][488352] Starting process rollout_proc65 +[2023-03-07 15:59:20,606][488352] Starting process rollout_proc68 +[2023-03-07 15:59:20,615][488352] Starting process rollout_proc71 +[2023-03-07 15:59:20,628][488352] Starting process rollout_proc74 +[2023-03-07 15:59:20,628][488352] Starting process rollout_proc48 +[2023-03-07 15:59:20,644][488352] Starting process rollout_proc77 +[2023-03-07 15:59:20,653][488352] Starting process rollout_proc80 +[2023-03-07 15:59:20,659][488352] Starting process rollout_proc83 +[2023-03-07 15:59:20,665][488352] Starting process rollout_proc86 +[2023-03-07 15:59:20,681][488352] Starting process rollout_proc89 +[2023-03-07 15:59:20,684][488352] Starting process rollout_proc54 +[2023-03-07 15:59:20,690][488352] Starting process rollout_proc51 +[2023-03-07 15:59:20,706][488352] Starting process rollout_proc63 +[2023-03-07 15:59:20,712][488352] Starting process rollout_proc57 +[2023-03-07 15:59:20,715][488352] Starting process rollout_proc60 +[2023-03-07 15:59:20,720][488352] Starting process rollout_proc69 +[2023-03-07 15:59:20,731][488352] Starting process rollout_proc66 +[2023-03-07 15:59:20,748][488352] Starting process rollout_proc72 +[2023-03-07 15:59:20,749][488352] Starting process rollout_proc49 +[2023-03-07 15:59:20,761][488352] Starting process rollout_proc81 +[2023-03-07 15:59:20,772][488352] Starting process rollout_proc75 +[2023-03-07 15:59:20,784][488352] Starting process rollout_proc78 +[2023-03-07 15:59:20,784][488352] Starting process rollout_proc84 +[2023-03-07 15:59:20,792][488352] Starting process rollout_proc87 +[2023-03-07 15:59:20,826][488352] Starting process rollout_proc55 +[2023-03-07 15:59:20,834][488352] Starting process rollout_proc90 +[2023-03-07 15:59:20,837][488352] Starting process rollout_proc58 +[2023-03-07 15:59:20,838][488352] Starting process rollout_proc64 +[2023-03-07 15:59:20,844][488352] Starting process rollout_proc52 +[2023-03-07 15:59:20,849][488352] Starting process rollout_proc61 +[2023-03-07 15:59:20,857][488352] Starting process rollout_proc70 +[2023-03-07 15:59:20,874][488352] Starting process rollout_proc67 +[2023-03-07 15:59:20,879][488352] Starting process rollout_proc73 +[2023-03-07 15:59:20,903][488352] Starting process rollout_proc92 +[2023-03-07 15:59:20,992][488352] Starting process rollout_proc76 +[2023-03-07 15:59:21,022][488352] Starting process rollout_proc82 +[2023-03-07 15:59:21,034][488352] Starting process rollout_proc88 +[2023-03-07 15:59:21,050][488352] Starting process rollout_proc79 +[2023-03-07 15:59:21,051][488352] Starting process rollout_proc85 +[2023-03-07 15:59:21,053][488352] Starting process rollout_proc95 +[2023-03-07 15:59:21,082][488352] Starting process rollout_proc98 +[2023-03-07 15:59:21,089][488352] Starting process rollout_proc91 +[2023-03-07 15:59:21,089][488352] Starting process rollout_proc101 +[2023-03-07 15:59:21,089][488352] Starting process rollout_proc104 +[2023-03-07 15:59:21,092][488352] Starting process rollout_proc107 +[2023-03-07 15:59:21,098][488352] Starting process rollout_proc110 +[2023-03-07 15:59:21,116][488352] Starting process rollout_proc113 +[2023-03-07 15:59:21,135][488352] Starting process rollout_proc93 +[2023-03-07 15:59:21,162][488352] Starting process rollout_proc116 +[2023-03-07 15:59:21,162][488352] Starting process rollout_proc119 +[2023-03-07 15:59:21,162][488352] Starting process rollout_proc122 +[2023-03-07 15:59:21,195][488352] Starting process rollout_proc96 +[2023-03-07 15:59:21,243][488352] Starting process rollout_proc125 +[2023-03-07 15:59:21,344][488352] Starting process rollout_proc105 +[2023-03-07 15:59:21,346][488352] Starting process rollout_proc99 +[2023-03-07 15:59:21,394][488352] Starting process rollout_proc102 +[2023-03-07 15:59:21,430][488352] Starting process rollout_proc111 +[2023-03-07 15:59:21,444][488352] Starting process rollout_proc120 +[2023-03-07 15:59:21,482][488352] Starting process rollout_proc114 +[2023-03-07 15:59:21,483][488352] Starting process rollout_proc108 +[2023-03-07 15:59:21,519][488352] Starting process rollout_proc117 +[2023-03-07 15:59:21,519][488352] Starting process rollout_proc97 +[2023-03-07 15:59:21,519][488352] Starting process rollout_proc94 +[2023-03-07 15:59:21,554][488352] Starting process rollout_proc112 +[2023-03-07 15:59:21,554][488352] Starting process rollout_proc106 +[2023-03-07 15:59:21,554][488352] Starting process rollout_proc123 +[2023-03-07 15:59:21,581][488352] Starting process rollout_proc126 +[2023-03-07 15:59:21,581][488352] Starting process rollout_proc121 +[2023-03-07 15:59:21,614][488352] Starting process rollout_proc103 +[2023-03-07 15:59:21,616][488352] Starting process rollout_proc100 +[2023-03-07 15:59:21,618][488352] Starting process rollout_proc115 +[2023-03-07 15:59:21,694][488352] Starting process rollout_proc109 +[2023-03-07 15:59:21,727][488352] Starting process rollout_proc118 +[2023-03-07 15:59:21,857][488352] Starting process rollout_proc124 +[2023-03-07 15:59:21,896][488352] Starting process rollout_proc127 +[2023-03-07 15:59:22,978][488352] Starting process rollout_proc0 +[2023-03-07 15:59:24,891][488352] Starting process rollout_proc1 +[2023-03-07 15:59:25,867][488352] Inference worker 0-0 is ready! +[2023-03-07 15:59:25,907][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 15:59:25,912][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 4014080. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:25,927][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:30,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:35,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:39,150][488352] Heartbeat connected on Batcher_0 +[2023-03-07 15:59:39,152][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 15:59:39,203][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 15:59:40,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:45,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:50,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 15:59:55,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:00:00,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:00:05,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:00:10,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:00:15,839][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 4014080. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:00:17,302][488352] Heartbeat connected on RolloutWorker_w54 +[2023-03-07 16:00:20,092][488352] Heartbeat connected on RolloutWorker_w100 +[2023-03-07 16:00:20,839][488352] Fps is (10 sec: 1228.8, 60 sec: 223.7, 300 sec: 223.7). Total num frames: 4026368. Throughput: 0: 38.4. Samples: 1728. Policy #0 lag: (min: 0.0, avg: 0.0, max: 0.0) +[2023-03-07 16:00:20,840][488352] Avg episode reward: [(0, '2.118')] +[2023-03-07 16:00:23,626][488352] Heartbeat connected on RolloutWorker_w121 +[2023-03-07 16:00:24,512][488352] Heartbeat connected on RolloutWorker_w111 +[2023-03-07 16:00:24,546][488352] Heartbeat connected on RolloutWorker_w63 +[2023-03-07 16:00:25,839][488352] Fps is (10 sec: 6963.2, 60 sec: 1162.2, 300 sec: 1161.9). Total num frames: 4083712. Throughput: 0: 354.5. Samples: 15952. Policy #0 lag: (min: 0.0, avg: 1.6, max: 3.0) +[2023-03-07 16:00:25,841][488352] Avg episode reward: [(0, '3.692')] +[2023-03-07 16:00:28,152][488352] Heartbeat connected on RolloutWorker_w70 +[2023-03-07 16:00:28,193][488352] Heartbeat connected on RolloutWorker_w21 +[2023-03-07 16:00:29,330][488352] Heartbeat connected on RolloutWorker_w42 +[2023-03-07 16:00:30,203][488352] Heartbeat connected on RolloutWorker_w87 +[2023-03-07 16:00:30,602][488352] Heartbeat connected on RolloutWorker_w106 +[2023-03-07 16:00:30,839][488352] Fps is (10 sec: 16383.9, 60 sec: 2935.5, 300 sec: 2712.7). Total num frames: 4190208. Throughput: 0: 679.5. Samples: 30576. Policy #0 lag: (min: 0.0, avg: 3.2, max: 7.0) +[2023-03-07 16:00:30,840][488352] Avg episode reward: [(0, '4.345')] +[2023-03-07 16:00:32,445][488352] Heartbeat connected on RolloutWorker_w40 +[2023-03-07 16:00:33,739][488352] Heartbeat connected on RolloutWorker_w31 +[2023-03-07 16:00:34,354][488352] Heartbeat connected on RolloutWorker_w84 +[2023-03-07 16:00:34,914][488352] Heartbeat connected on RolloutWorker_w65 +[2023-03-07 16:00:35,201][488352] Heartbeat connected on RolloutWorker_w53 +[2023-03-07 16:00:35,594][488352] Heartbeat connected on RolloutWorker_w36 +[2023-03-07 16:00:35,840][488352] Fps is (10 sec: 29081.3, 60 sec: 6007.4, 300 sec: 5154.6). Total num frames: 4374528. Throughput: 0: 1794.1. Samples: 80736. Policy #0 lag: (min: 1.0, avg: 6.7, max: 15.0) +[2023-03-07 16:00:35,841][488352] Avg episode reward: [(0, '4.257')] +[2023-03-07 16:00:36,122][488352] Heartbeat connected on RolloutWorker_w5 +[2023-03-07 16:00:36,443][488352] Heartbeat connected on RolloutWorker_w8 +[2023-03-07 16:00:36,594][488352] Heartbeat connected on RolloutWorker_w61 +[2023-03-07 16:00:37,005][488352] Heartbeat connected on RolloutWorker_w67 +[2023-03-07 16:00:37,162][488352] Heartbeat connected on RolloutWorker_w60 +[2023-03-07 16:00:38,349][488352] Heartbeat connected on RolloutWorker_w120 +[2023-03-07 16:00:39,112][488352] Heartbeat connected on RolloutWorker_w4 +[2023-03-07 16:00:39,621][488352] Heartbeat connected on RolloutWorker_w1 +[2023-03-07 16:00:39,669][488352] Heartbeat connected on RolloutWorker_w77 +[2023-03-07 16:00:40,043][488352] Heartbeat connected on RolloutWorker_w118 +[2023-03-07 16:00:40,622][488352] Heartbeat connected on RolloutWorker_w117 +[2023-03-07 16:00:40,797][488352] Heartbeat connected on RolloutWorker_w47 +[2023-03-07 16:00:40,839][488352] Fps is (10 sec: 47923.6, 60 sec: 10922.7, 300 sec: 8746.6). Total num frames: 4669440. Throughput: 0: 3728.7. Samples: 167792. Policy #0 lag: (min: 1.0, avg: 9.6, max: 19.0) +[2023-03-07 16:00:40,840][488352] Avg episode reward: [(0, '5.410')] +[2023-03-07 16:00:41,319][488352] Heartbeat connected on RolloutWorker_w93 +[2023-03-07 16:00:41,576][488352] Heartbeat connected on RolloutWorker_w96 +[2023-03-07 16:00:41,809][488352] Heartbeat connected on RolloutWorker_w89 +[2023-03-07 16:00:41,824][488352] Heartbeat connected on RolloutWorker_w45 +[2023-03-07 16:00:41,987][488352] Heartbeat connected on RolloutWorker_w12 +[2023-03-07 16:00:42,106][488352] Heartbeat connected on RolloutWorker_w49 +[2023-03-07 16:00:42,354][488352] Heartbeat connected on RolloutWorker_w41 +[2023-03-07 16:00:42,824][488352] Heartbeat connected on RolloutWorker_w9 +[2023-03-07 16:00:42,914][488352] Heartbeat connected on RolloutWorker_w3 +[2023-03-07 16:00:43,198][488352] Heartbeat connected on RolloutWorker_w109 +[2023-03-07 16:00:43,469][488352] Heartbeat connected on RolloutWorker_w10 +[2023-03-07 16:00:43,522][488352] Heartbeat connected on RolloutWorker_w30 +[2023-03-07 16:00:43,909][488352] Heartbeat connected on RolloutWorker_w37 +[2023-03-07 16:00:45,532][488352] Heartbeat connected on RolloutWorker_w83 +[2023-03-07 16:00:45,839][488352] Fps is (10 sec: 79054.0, 60 sec: 19182.9, 300 sec: 14400.2). Total num frames: 5165056. Throughput: 0: 5307.0. Samples: 238816. Policy #0 lag: (min: 5.0, avg: 20.5, max: 34.0) +[2023-03-07 16:00:45,841][488352] Avg episode reward: [(0, '5.693')] +[2023-03-07 16:00:46,083][488352] Heartbeat connected on RolloutWorker_w115 +[2023-03-07 16:00:46,443][488352] Heartbeat connected on RolloutWorker_w44 +[2023-03-07 16:00:46,925][488352] Heartbeat connected on RolloutWorker_w125 +[2023-03-07 16:00:46,992][488352] Heartbeat connected on RolloutWorker_w126 +[2023-03-07 16:00:48,202][488352] Heartbeat connected on RolloutWorker_w58 +[2023-03-07 16:00:48,357][488352] Heartbeat connected on RolloutWorker_w72 +[2023-03-07 16:00:48,471][488352] Heartbeat connected on RolloutWorker_w56 +[2023-03-07 16:00:48,631][488352] Heartbeat connected on RolloutWorker_w17 +[2023-03-07 16:00:48,721][488352] Heartbeat connected on RolloutWorker_w6 +[2023-03-07 16:00:48,986][488352] Heartbeat connected on RolloutWorker_w71 +[2023-03-07 16:00:49,034][488352] Heartbeat connected on RolloutWorker_w101 +[2023-03-07 16:00:49,587][488352] Heartbeat connected on RolloutWorker_w32 +[2023-03-07 16:00:49,724][488352] Heartbeat connected on RolloutWorker_w7 +[2023-03-07 16:00:49,755][488352] Heartbeat connected on RolloutWorker_w29 +[2023-03-07 16:00:50,150][488352] Heartbeat connected on RolloutWorker_w35 +[2023-03-07 16:00:50,388][488352] Heartbeat connected on RolloutWorker_w95 +[2023-03-07 16:00:50,520][488352] Heartbeat connected on RolloutWorker_w91 +[2023-03-07 16:00:50,839][488352] Fps is (10 sec: 101580.5, 60 sec: 27852.8, 300 sec: 19677.6). Total num frames: 5685248. Throughput: 0: 9572.6. Samples: 430768. Policy #0 lag: (min: 6.0, avg: 23.3, max: 41.0) +[2023-03-07 16:00:50,841][488352] Avg episode reward: [(0, '6.595')] +[2023-03-07 16:00:50,966][488352] Heartbeat connected on RolloutWorker_w80 +[2023-03-07 16:00:50,987][488352] Heartbeat connected on RolloutWorker_w94 +[2023-03-07 16:00:51,170][488352] Heartbeat connected on RolloutWorker_w114 +[2023-03-07 16:00:51,234][488352] Heartbeat connected on RolloutWorker_w81 +[2023-03-07 16:00:51,352][488352] Heartbeat connected on RolloutWorker_w18 +[2023-03-07 16:00:51,777][488352] Heartbeat connected on RolloutWorker_w79 +[2023-03-07 16:00:52,127][488352] Heartbeat connected on RolloutWorker_w51 +[2023-03-07 16:00:52,293][488352] Heartbeat connected on RolloutWorker_w15 +[2023-03-07 16:00:52,377][488352] Heartbeat connected on RolloutWorker_w85 +[2023-03-07 16:00:52,594][488352] Heartbeat connected on RolloutWorker_w50 +[2023-03-07 16:00:53,214][488352] Heartbeat connected on RolloutWorker_w39 +[2023-03-07 16:00:53,756][488352] Heartbeat connected on RolloutWorker_w11 +[2023-03-07 16:00:55,079][488352] Heartbeat connected on RolloutWorker_w88 +[2023-03-07 16:00:55,646][488352] Heartbeat connected on RolloutWorker_w119 +[2023-03-07 16:00:55,742][488352] Heartbeat connected on RolloutWorker_w69 +[2023-03-07 16:00:55,831][488352] Heartbeat connected on RolloutWorker_w48 +[2023-03-07 16:00:55,840][488352] Fps is (10 sec: 97074.3, 60 sec: 35362.1, 300 sec: 23593.7). Total num frames: 6135808. Throughput: 0: 14163.9. Samples: 637376. Policy #0 lag: (min: 75.0, avg: 94.6, max: 113.0) +[2023-03-07 16:00:55,841][488352] Avg episode reward: [(0, '6.720')] +[2023-03-07 16:00:56,263][488352] Heartbeat connected on RolloutWorker_w97 +[2023-03-07 16:00:56,744][488352] Heartbeat connected on RolloutWorker_w98 +[2023-03-07 16:00:57,070][488352] Heartbeat connected on RolloutWorker_w38 +[2023-03-07 16:00:57,280][488352] Heartbeat connected on RolloutWorker_w25 +[2023-03-07 16:00:57,347][488352] Heartbeat connected on RolloutWorker_w108 +[2023-03-07 16:00:57,742][488352] Heartbeat connected on RolloutWorker_w13 +[2023-03-07 16:00:58,256][488352] Heartbeat connected on RolloutWorker_w107 +[2023-03-07 16:00:58,751][488352] Heartbeat connected on RolloutWorker_w78 +[2023-03-07 16:00:58,766][488352] Heartbeat connected on RolloutWorker_w102 +[2023-03-07 16:00:59,102][488352] Heartbeat connected on RolloutWorker_w57 +[2023-03-07 16:01:00,057][488352] Heartbeat connected on RolloutWorker_w27 +[2023-03-07 16:01:00,536][488352] Heartbeat connected on RolloutWorker_w33 +[2023-03-07 16:01:00,627][488352] Heartbeat connected on RolloutWorker_w46 +[2023-03-07 16:01:00,831][488352] Heartbeat connected on RolloutWorker_w19 +[2023-03-07 16:01:00,839][488352] Fps is (10 sec: 94617.4, 60 sec: 43622.4, 300 sec: 27572.0). Total num frames: 6631424. Throughput: 0: 16898.5. Samples: 760432. Policy #0 lag: (min: 41.0, avg: 63.9, max: 86.0) +[2023-03-07 16:01:00,841][488352] Avg episode reward: [(0, '6.908')] +[2023-03-07 16:01:01,208][488352] Heartbeat connected on RolloutWorker_w116 +[2023-03-07 16:01:02,508][488352] Heartbeat connected on RolloutWorker_w124 +[2023-03-07 16:01:02,778][488352] Heartbeat connected on RolloutWorker_w62 +[2023-03-07 16:01:03,220][488352] Heartbeat connected on RolloutWorker_w43 +[2023-03-07 16:01:03,221][488352] Heartbeat connected on RolloutWorker_w66 +[2023-03-07 16:01:03,455][488352] Heartbeat connected on RolloutWorker_w26 +[2023-03-07 16:01:03,627][488352] Heartbeat connected on RolloutWorker_w112 +[2023-03-07 16:01:03,749][488352] Heartbeat connected on RolloutWorker_w24 +[2023-03-07 16:01:03,799][488352] Heartbeat connected on RolloutWorker_w20 +[2023-03-07 16:01:03,800][488352] Heartbeat connected on RolloutWorker_w2 +[2023-03-07 16:01:03,935][488352] Heartbeat connected on RolloutWorker_w127 +[2023-03-07 16:01:04,694][488352] Heartbeat connected on RolloutWorker_w110 +[2023-03-07 16:01:04,932][488352] Heartbeat connected on RolloutWorker_w113 +[2023-03-07 16:01:05,254][488352] Heartbeat connected on RolloutWorker_w14 +[2023-03-07 16:01:05,815][488352] Heartbeat connected on RolloutWorker_w105 +[2023-03-07 16:01:05,839][488352] Fps is (10 sec: 108546.7, 60 sec: 53452.8, 300 sec: 32094.9). Total num frames: 7221248. Throughput: 0: 21971.2. Samples: 990432. Policy #0 lag: (min: 258.0, avg: 283.7, max: 308.0) +[2023-03-07 16:01:05,840][488352] Avg episode reward: [(0, '9.092')] +[2023-03-07 16:01:06,907][488352] Heartbeat connected on RolloutWorker_w68 +[2023-03-07 16:01:07,268][488352] Heartbeat connected on RolloutWorker_w16 +[2023-03-07 16:01:07,726][488352] Heartbeat connected on RolloutWorker_w22 +[2023-03-07 16:01:07,798][488352] Heartbeat connected on RolloutWorker_w28 +[2023-03-07 16:01:07,926][488352] Heartbeat connected on RolloutWorker_w86 +[2023-03-07 16:01:08,414][488352] Heartbeat connected on RolloutWorker_w75 +[2023-03-07 16:01:08,999][488352] Heartbeat connected on RolloutWorker_w73 +[2023-03-07 16:01:09,060][488352] Heartbeat connected on RolloutWorker_w99 +[2023-03-07 16:01:09,962][488352] Heartbeat connected on RolloutWorker_w55 +[2023-03-07 16:01:10,619][488352] Heartbeat connected on RolloutWorker_w122 +[2023-03-07 16:01:10,839][488352] Fps is (10 sec: 127796.7, 60 sec: 64921.8, 300 sec: 37123.7). Total num frames: 7909376. Throughput: 0: 26214.2. Samples: 1195584. Policy #0 lag: (min: 12.0, avg: 33.0, max: 55.0) +[2023-03-07 16:01:10,840][488352] Avg episode reward: [(0, '9.182')] +[2023-03-07 16:01:10,857][488352] Heartbeat connected on RolloutWorker_w92 +[2023-03-07 16:01:11,046][488352] Heartbeat connected on RolloutWorker_w34 +[2023-03-07 16:01:11,601][488352] Heartbeat connected on RolloutWorker_w90 +[2023-03-07 16:01:11,924][488352] Heartbeat connected on RolloutWorker_w104 +[2023-03-07 16:01:12,458][488352] Heartbeat connected on RolloutWorker_w103 +[2023-03-07 16:01:12,504][488352] Heartbeat connected on RolloutWorker_w64 +[2023-03-07 16:01:12,981][488352] Heartbeat connected on RolloutWorker_w59 +[2023-03-07 16:01:13,239][488352] Heartbeat connected on RolloutWorker_w82 +[2023-03-07 16:01:13,292][488352] Heartbeat connected on RolloutWorker_w0 +[2023-03-07 16:01:14,593][488352] Heartbeat connected on RolloutWorker_w123 +[2023-03-07 16:01:15,261][488352] Heartbeat connected on RolloutWorker_w76 +[2023-03-07 16:01:15,294][488352] Heartbeat connected on RolloutWorker_w52 +[2023-03-07 16:01:15,839][488352] Fps is (10 sec: 140083.1, 60 sec: 76800.2, 300 sec: 41918.5). Total num frames: 8622080. Throughput: 0: 28176.1. Samples: 1298496. Policy #0 lag: (min: 8.0, avg: 27.8, max: 49.0) +[2023-03-07 16:01:15,840][488352] Avg episode reward: [(0, '9.048')] +[2023-03-07 16:01:17,550][488352] Heartbeat connected on RolloutWorker_w23 +[2023-03-07 16:01:17,757][488352] Heartbeat connected on RolloutWorker_w74 +[2023-03-07 16:01:20,839][488352] Fps is (10 sec: 145815.3, 60 sec: 89019.6, 300 sec: 46581.2). Total num frames: 9367552. Throughput: 0: 31969.9. Samples: 1519376. Policy #0 lag: (min: 354.0, avg: 374.3, max: 397.0) +[2023-03-07 16:01:20,841][488352] Avg episode reward: [(0, '10.322')] +[2023-03-07 16:01:25,839][488352] Fps is (10 sec: 154828.1, 60 sec: 101444.5, 300 sec: 51333.4). Total num frames: 10170368. Throughput: 0: 35304.5. Samples: 1756496. Policy #0 lag: (min: 49.0, avg: 72.1, max: 97.0) +[2023-03-07 16:01:25,840][488352] Avg episode reward: [(0, '10.182')] +[2023-03-07 16:01:30,839][488352] Fps is (10 sec: 156468.1, 60 sec: 112367.1, 300 sec: 55377.2). Total num frames: 10932224. Throughput: 0: 36337.8. Samples: 1874016. Policy #0 lag: (min: 26.0, avg: 47.0, max: 68.0) +[2023-03-07 16:01:30,840][488352] Avg episode reward: [(0, '11.717')] +[2023-03-07 16:01:35,839][488352] Fps is (10 sec: 154829.2, 60 sec: 122402.7, 300 sec: 59299.0). Total num frames: 11718656. Throughput: 0: 37183.7. Samples: 2104032. Policy #0 lag: (min: 36.0, avg: 59.1, max: 79.0) +[2023-03-07 16:01:35,840][488352] Avg episode reward: [(0, '11.496')] +[2023-03-07 16:01:40,839][488352] Fps is (10 sec: 161791.6, 60 sec: 131345.0, 300 sec: 63264.0). Total num frames: 12550144. Throughput: 0: 38031.8. Samples: 2348800. Policy #0 lag: (min: 11.0, avg: 31.6, max: 53.0) +[2023-03-07 16:01:40,841][488352] Avg episode reward: [(0, '12.478')] +[2023-03-07 16:01:45,839][488352] Fps is (10 sec: 160151.7, 60 sec: 135918.9, 300 sec: 66506.6). Total num frames: 13320192. Throughput: 0: 38007.4. Samples: 2470768. Policy #0 lag: (min: 47.0, avg: 67.2, max: 89.0) +[2023-03-07 16:01:45,840][488352] Avg episode reward: [(0, '11.880')] +[2023-03-07 16:01:50,839][488352] Fps is (10 sec: 149912.7, 60 sec: 139400.3, 300 sec: 69242.8). Total num frames: 14049280. Throughput: 0: 38196.5. Samples: 2709280. Policy #0 lag: (min: 11.0, avg: 29.8, max: 48.0) +[2023-03-07 16:01:50,840][488352] Avg episode reward: [(0, '13.752')] +[2023-03-07 16:01:55,839][488352] Fps is (10 sec: 145000.1, 60 sec: 143906.7, 300 sec: 71741.9). Total num frames: 14770176. Throughput: 0: 38874.6. Samples: 2944944. Policy #0 lag: (min: 30.0, avg: 49.6, max: 67.0) +[2023-03-07 16:01:55,841][488352] Avg episode reward: [(0, '12.105')] +[2023-03-07 16:02:00,839][488352] Fps is (10 sec: 142541.4, 60 sec: 147387.7, 300 sec: 73973.9). Total num frames: 15474688. Throughput: 0: 39164.7. Samples: 3060912. Policy #0 lag: (min: 463.0, avg: 764.3, max: 1068.0) +[2023-03-07 16:02:00,841][488352] Avg episode reward: [(0, '12.862')] +[2023-03-07 16:02:05,839][488352] Fps is (10 sec: 143357.7, 60 sec: 149708.3, 300 sec: 76220.0). Total num frames: 16203776. Throughput: 0: 39575.4. Samples: 3300272. Policy #0 lag: (min: 10.0, avg: 27.1, max: 47.0) +[2023-03-07 16:02:05,841][488352] Avg episode reward: [(0, '14.067')] +[2023-03-07 16:02:10,839][488352] Fps is (10 sec: 144177.9, 60 sec: 150117.8, 300 sec: 78230.6). Total num frames: 16916480. Throughput: 0: 39604.8. Samples: 3538720. Policy #0 lag: (min: 54.0, avg: 78.7, max: 102.0) +[2023-03-07 16:02:10,841][488352] Avg episode reward: [(0, '14.002')] +[2023-03-07 16:02:15,839][488352] Fps is (10 sec: 145407.5, 60 sec: 150595.7, 300 sec: 80291.6). Total num frames: 17657856. Throughput: 0: 39706.5. Samples: 3660816. Policy #0 lag: (min: 41.0, avg: 61.4, max: 80.0) +[2023-03-07 16:02:15,841][488352] Avg episode reward: [(0, '14.992')] +[2023-03-07 16:02:20,839][488352] Fps is (10 sec: 151554.3, 60 sec: 151074.3, 300 sec: 82422.2). Total num frames: 18432000. Throughput: 0: 39731.9. Samples: 3891968. Policy #0 lag: (min: 13.0, avg: 33.8, max: 54.0) +[2023-03-07 16:02:20,840][488352] Avg episode reward: [(0, '12.618')] +[2023-03-07 16:02:25,839][488352] Fps is (10 sec: 151143.1, 60 sec: 149981.5, 300 sec: 84229.4). Total num frames: 19169280. Throughput: 0: 39684.5. Samples: 4134608. Policy #0 lag: (min: 12.0, avg: 63.6, max: 116.0) +[2023-03-07 16:02:25,841][488352] Avg episode reward: [(0, '16.797')] +[2023-03-07 16:02:30,839][488352] Fps is (10 sec: 146637.7, 60 sec: 149435.9, 300 sec: 85894.7). Total num frames: 19898368. Throughput: 0: 39547.2. Samples: 4250384. Policy #0 lag: (min: 13.0, avg: 31.9, max: 51.0) +[2023-03-07 16:02:30,840][488352] Avg episode reward: [(0, '16.563')] +[2023-03-07 16:02:31,599][488352] Component Batcher_0 stopped! +[2023-03-07 16:02:31,677][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 16:02:31,949][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 16:02:33,379][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 16:02:33,399][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 16:02:33,401][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 16:02:33,403][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 16:02:33,403][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 16:02:33,405][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 16:02:33,408][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 16:02:33,426][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 16:02:33,432][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 16:02:33,435][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 16:02:33,437][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 16:02:33,441][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 16:02:33,445][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 16:02:33,448][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 16:02:33,451][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 16:02:33,456][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 16:02:33,459][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 16:02:33,467][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 16:02:33,473][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 16:02:33,480][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 16:02:33,482][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 16:02:33,492][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 16:02:33,494][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 16:02:33,496][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 16:02:33,500][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 16:02:33,502][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 16:02:33,509][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 16:02:33,511][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 16:02:33,520][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 16:02:33,522][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 16:02:33,524][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 16:02:33,525][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 16:02:33,526][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 16:02:33,527][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 16:02:33,528][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 16:02:33,529][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 16:02:33,530][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 16:02:33,531][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 16:02:33,532][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 16:02:33,533][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 16:02:33,534][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 16:02:33,552][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 16:02:33,554][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 16:02:33,555][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 16:02:33,556][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 16:02:33,557][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 16:02:33,559][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 16:02:33,560][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 16:02:33,560][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 16:02:33,563][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 16:02:33,564][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 16:02:33,565][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 16:02:33,566][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 16:02:33,567][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 16:02:33,568][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 16:02:33,570][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 16:02:33,570][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 16:02:33,572][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 16:02:33,573][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 16:02:33,574][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 16:02:33,575][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 16:02:33,576][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 16:02:33,577][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 16:02:33,578][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 16:02:33,579][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 16:02:33,580][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 16:02:33,581][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 16:02:33,582][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 16:02:33,583][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 16:02:33,584][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 16:02:33,584][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 16:02:33,585][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 16:02:33,586][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 16:02:33,587][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 16:02:33,588][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 16:02:33,589][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 16:02:33,590][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 16:02:33,591][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 16:02:33,591][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 16:02:33,592][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 16:02:33,593][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 16:02:33,594][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 16:02:33,594][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 16:02:33,595][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 16:02:33,596][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 16:02:33,597][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 16:02:33,598][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 16:02:33,599][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 16:02:33,599][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 16:02:33,600][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 16:02:33,601][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 16:02:33,602][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 16:02:33,603][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 16:02:33,603][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 16:02:33,604][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 16:02:33,605][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 16:02:33,606][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 16:02:33,606][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 16:02:33,607][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 16:02:33,608][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 16:02:33,609][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 16:02:33,610][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 16:02:33,611][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 16:02:33,611][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 16:02:33,612][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 16:02:33,613][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 16:02:33,614][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 16:02:33,614][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 16:02:33,615][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 16:02:33,616][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 16:02:33,617][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 16:02:33,617][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 16:02:33,618][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 16:02:33,619][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 16:02:33,619][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 16:02:33,620][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 16:02:33,621][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 16:02:33,622][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 16:02:33,623][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 16:02:33,624][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 16:02:33,624][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 16:02:33,625][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 16:02:33,626][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 16:02:33,627][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 16:02:33,627][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 16:02:33,628][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 16:02:33,628][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 16:02:33,640][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 16:02:33,642][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 16:02:34,727][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 16:02:34,729][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 16:02:34,730][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 16:02:34,732][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 16:02:34,733][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 16:02:34,734][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 16:02:34,736][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 16:02:34,737][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 16:02:34,738][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 16:02:34,740][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 16:02:34,741][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 16:02:34,742][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 16:02:34,744][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 16:02:34,745][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 16:02:34,746][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 16:02:34,748][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 16:02:34,750][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 16:02:34,751][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 16:02:34,753][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 16:02:34,754][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 16:02:34,756][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 16:02:34,757][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 16:02:34,758][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 16:02:34,760][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 16:02:34,762][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 16:02:34,763][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 16:02:34,764][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 16:02:34,766][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 16:02:34,772][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 16:02:34,774][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 16:02:34,775][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 16:02:34,777][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 16:02:34,782][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 16:02:34,784][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 16:02:34,785][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 16:02:34,786][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 16:02:34,788][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 16:02:34,789][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 16:02:34,792][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 16:02:34,793][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 16:02:34,795][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 16:02:34,796][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 16:02:34,798][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 16:02:34,801][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 16:02:34,802][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 16:02:34,804][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 16:02:34,805][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 16:02:34,810][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 16:02:34,812][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 16:02:34,813][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 16:02:34,816][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 16:02:34,818][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 16:02:34,819][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 16:02:34,820][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 16:02:34,822][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 16:02:34,823][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 16:02:34,825][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 16:02:34,831][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 16:02:34,835][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 16:02:34,836][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 16:02:34,838][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 16:02:34,839][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 16:02:34,841][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 16:02:34,842][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 16:02:34,844][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 16:02:34,845][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 16:02:34,847][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 16:02:34,848][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 16:02:34,849][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 16:02:34,853][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 16:02:34,855][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 16:02:34,856][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 16:02:34,857][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 16:02:34,859][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 16:02:34,861][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 16:02:34,862][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 16:02:34,863][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 16:02:34,865][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 16:02:34,868][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 16:02:34,869][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 16:02:34,871][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 16:02:34,873][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 16:02:34,876][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 16:02:34,878][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 16:02:34,880][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 16:02:34,881][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 16:02:34,883][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 16:02:34,885][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 16:02:34,886][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 16:02:34,888][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 16:02:34,889][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 16:02:34,892][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 16:02:34,893][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 16:02:34,895][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 16:02:34,896][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 16:02:34,898][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 16:02:34,901][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 16:02:34,902][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 16:02:34,904][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 16:02:34,905][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 16:02:34,907][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 16:02:34,908][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 16:02:34,910][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 16:02:34,911][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 16:02:34,913][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 16:02:34,914][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 16:02:34,916][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 16:02:34,917][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 16:02:34,918][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 16:02:34,920][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 16:02:34,922][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 16:02:34,925][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 16:02:34,926][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 16:02:34,928][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 16:02:34,929][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 16:02:34,933][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 16:02:34,935][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 16:02:34,936][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 16:02:34,938][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 16:02:34,940][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 16:02:34,942][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 16:02:34,945][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 16:02:34,946][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 16:02:34,948][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 16:02:34,950][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 16:02:34,951][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 16:02:34,953][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 16:02:34,955][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 16:02:34,956][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 16:02:34,958][488352] Batcher 0 profile tree view: +batching: 2.8709, releasing_batches: 41.3343 +[2023-03-07 16:02:34,959][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0198 + wait_policy_total: 102.2752 +update_model: 4.5276 + weight_update: 0.0009 +one_step: 0.0179 + handle_policy_step: 68.4109 + deserialize: 19.4202, stack: 0.2065, obs_to_device_normalize: 18.6262, forward: 12.6791, send_messages: 4.8130 + prepare_outputs: 11.5026 + to_cpu: 5.1017 +[2023-03-07 16:02:34,960][488352] Learner 0 profile tree view: +misc: 0.0134, prepare_batch: 38.0995 +train: 74.7122 + epoch_init: 0.0167, minibatch_init: 0.0164, losses_postprocess: 1.9463, kl_divergence: 1.6633, after_optimizer: 0.7645 + calculate_losses: 25.8419 + losses_init: 0.0104, forward_head: 3.2238, bptt_initial: 14.0620, tail: 1.5249, advantages_returns: 0.4133, losses: 3.0905 + bptt: 3.0329 + bptt_forward_core: 2.9000 + update: 43.3769 + clip: 3.7180 +[2023-03-07 16:02:34,960][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0759, enqueue_policy_requests: 0.0601, env_step: 0.9430, overhead: 0.0868, complete_rollouts: 0.0004 +save_policy_outputs: 0.0932 + split_output_tensors: 0.0437 +[2023-03-07 16:02:34,961][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0586, enqueue_policy_requests: 2.3243, env_step: 42.4431, overhead: 3.5947, complete_rollouts: 0.0180 +save_policy_outputs: 3.5870 + split_output_tensors: 1.7032 +[2023-03-07 16:02:34,963][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 16:02:34,966][488352] Runner profile tree view: +main_loop: 195.5471 +[2023-03-07 16:02:34,966][488352] Collected {0: 20008960}, FPS: 81795.5 +[2023-03-07 16:02:34,977][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:02:34,978][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:02:34,979][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:02:34,980][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:02:34,980][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:02:34,981][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:02:34,982][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:02:34,982][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 16:02:34,984][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 16:02:34,985][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:02:34,985][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:02:34,986][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:02:34,987][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:02:34,987][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:02:34,991][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:02:34,993][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:02:35,002][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:02:35,033][488352] Conv encoder output size: 512 +[2023-03-07 16:02:35,034][488352] Policy head output size: 512 +[2023-03-07 16:02:35,051][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/checkpoint_p0/checkpoint_000004885_20008960.pth... +[2023-03-07 16:02:35,589][488352] Num frames 100... +[2023-03-07 16:02:35,680][488352] Num frames 200... +[2023-03-07 16:02:35,771][488352] Num frames 300... +[2023-03-07 16:02:35,869][488352] Num frames 400... +[2023-03-07 16:02:35,961][488352] Num frames 500... +[2023-03-07 16:02:36,050][488352] Num frames 600... +[2023-03-07 16:02:36,138][488352] Num frames 700... +[2023-03-07 16:02:36,257][488352] Avg episode rewards: #0: 12.680, true rewards: #0: 7.680 +[2023-03-07 16:02:36,259][488352] Avg episode reward: 12.680, avg true_objective: 7.680 +[2023-03-07 16:02:36,302][488352] Num frames 800... +[2023-03-07 16:02:36,400][488352] Num frames 900... +[2023-03-07 16:02:36,494][488352] Num frames 1000... +[2023-03-07 16:02:36,585][488352] Num frames 1100... +[2023-03-07 16:02:36,675][488352] Num frames 1200... +[2023-03-07 16:02:36,767][488352] Avg episode rewards: #0: 9.685, true rewards: #0: 6.185 +[2023-03-07 16:02:36,768][488352] Avg episode reward: 9.685, avg true_objective: 6.185 +[2023-03-07 16:02:36,832][488352] Num frames 1300... +[2023-03-07 16:02:36,921][488352] Num frames 1400... +[2023-03-07 16:02:37,011][488352] Num frames 1500... +[2023-03-07 16:02:37,102][488352] Num frames 1600... +[2023-03-07 16:02:37,234][488352] Avg episode rewards: #0: 8.283, true rewards: #0: 5.617 +[2023-03-07 16:02:37,235][488352] Avg episode reward: 8.283, avg true_objective: 5.617 +[2023-03-07 16:02:37,256][488352] Num frames 1700... +[2023-03-07 16:02:37,364][488352] Num frames 1800... +[2023-03-07 16:02:37,455][488352] Num frames 1900... +[2023-03-07 16:02:37,544][488352] Num frames 2000... +[2023-03-07 16:02:37,634][488352] Num frames 2100... +[2023-03-07 16:02:37,724][488352] Num frames 2200... +[2023-03-07 16:02:37,816][488352] Num frames 2300... +[2023-03-07 16:02:37,901][488352] Num frames 2400... +[2023-03-07 16:02:37,994][488352] Num frames 2500... +[2023-03-07 16:02:38,082][488352] Num frames 2600... +[2023-03-07 16:02:38,170][488352] Num frames 2700... +[2023-03-07 16:02:38,265][488352] Num frames 2800... +[2023-03-07 16:02:38,346][488352] Num frames 2900... +[2023-03-07 16:02:38,429][488352] Num frames 3000... +[2023-03-07 16:02:38,521][488352] Num frames 3100... +[2023-03-07 16:02:38,655][488352] Avg episode rewards: #0: 14.723, true rewards: #0: 7.972 +[2023-03-07 16:02:38,655][488352] Avg episode reward: 14.723, avg true_objective: 7.972 +[2023-03-07 16:02:38,668][488352] Num frames 3200... +[2023-03-07 16:02:38,773][488352] Num frames 3300... +[2023-03-07 16:02:38,863][488352] Num frames 3400... +[2023-03-07 16:02:38,957][488352] Num frames 3500... +[2023-03-07 16:02:39,040][488352] Num frames 3600... +[2023-03-07 16:02:39,128][488352] Avg episode rewards: #0: 12.874, true rewards: #0: 7.274 +[2023-03-07 16:02:39,129][488352] Avg episode reward: 12.874, avg true_objective: 7.274 +[2023-03-07 16:02:39,200][488352] Num frames 3700... +[2023-03-07 16:02:39,286][488352] Num frames 3800... +[2023-03-07 16:02:39,374][488352] Num frames 3900... +[2023-03-07 16:02:39,460][488352] Num frames 4000... +[2023-03-07 16:02:39,533][488352] Avg episode rewards: #0: 11.368, true rewards: #0: 6.702 +[2023-03-07 16:02:39,534][488352] Avg episode reward: 11.368, avg true_objective: 6.702 +[2023-03-07 16:02:39,607][488352] Num frames 4100... +[2023-03-07 16:02:39,697][488352] Num frames 4200... +[2023-03-07 16:02:39,787][488352] Num frames 4300... +[2023-03-07 16:02:39,851][488352] Avg episode rewards: #0: 10.441, true rewards: #0: 6.156 +[2023-03-07 16:02:39,851][488352] Avg episode reward: 10.441, avg true_objective: 6.156 +[2023-03-07 16:02:39,969][488352] Num frames 4400... +[2023-03-07 16:02:40,056][488352] Num frames 4500... +[2023-03-07 16:02:40,144][488352] Num frames 4600... +[2023-03-07 16:02:40,253][488352] Avg episode rewards: #0: 9.826, true rewards: #0: 5.826 +[2023-03-07 16:02:40,254][488352] Avg episode reward: 9.826, avg true_objective: 5.826 +[2023-03-07 16:02:40,292][488352] Num frames 4700... +[2023-03-07 16:02:40,383][488352] Num frames 4800... +[2023-03-07 16:02:40,473][488352] Num frames 4900... +[2023-03-07 16:02:40,562][488352] Num frames 5000... +[2023-03-07 16:02:40,653][488352] Num frames 5100... +[2023-03-07 16:02:40,742][488352] Num frames 5200... +[2023-03-07 16:02:40,830][488352] Avg episode rewards: #0: 9.819, true rewards: #0: 5.819 +[2023-03-07 16:02:40,831][488352] Avg episode reward: 9.819, avg true_objective: 5.819 +[2023-03-07 16:02:40,905][488352] Num frames 5300... +[2023-03-07 16:02:40,994][488352] Num frames 5400... +[2023-03-07 16:02:41,084][488352] Num frames 5500... +[2023-03-07 16:02:41,174][488352] Num frames 5600... +[2023-03-07 16:02:41,264][488352] Num frames 5700... +[2023-03-07 16:02:41,352][488352] Num frames 5800... +[2023-03-07 16:02:41,441][488352] Num frames 5900... +[2023-03-07 16:02:41,529][488352] Num frames 6000... +[2023-03-07 16:02:41,589][488352] Avg episode rewards: #0: 10.205, true rewards: #0: 6.005 +[2023-03-07 16:02:41,590][488352] Avg episode reward: 10.205, avg true_objective: 6.005 +[2023-03-07 16:02:49,067][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/replay.mp4! +[2023-03-07 16:16:02,530][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:16:02,531][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:16:02,532][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:16:02,532][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:16:02,533][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:16:02,533][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:16:02,534][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:16:02,535][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 16:16:02,535][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 16:16:02,536][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:16:02,536][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:16:02,537][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:16:02,537][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:16:02,540][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:16:02,545][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:16:02,546][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:16:02,555][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:16:02,585][488352] Conv encoder output size: 512 +[2023-03-07 16:16:02,586][488352] Policy head output size: 512 +[2023-03-07 16:16:02,609][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/checkpoint_p0/checkpoint_000004885_20008960.pth... +[2023-03-07 16:16:03,139][488352] Num frames 100... +[2023-03-07 16:16:03,224][488352] Num frames 200... +[2023-03-07 16:16:03,308][488352] Num frames 300... +[2023-03-07 16:16:03,391][488352] Num frames 400... +[2023-03-07 16:16:03,476][488352] Num frames 500... +[2023-03-07 16:16:03,562][488352] Num frames 600... +[2023-03-07 16:16:03,646][488352] Num frames 700... +[2023-03-07 16:16:03,729][488352] Num frames 800... +[2023-03-07 16:16:03,816][488352] Num frames 900... +[2023-03-07 16:16:03,901][488352] Num frames 1000... +[2023-03-07 16:16:03,985][488352] Num frames 1100... +[2023-03-07 16:16:04,069][488352] Num frames 1200... +[2023-03-07 16:16:04,141][488352] Avg episode rewards: #0: 26.170, true rewards: #0: 12.170 +[2023-03-07 16:16:04,142][488352] Avg episode reward: 26.170, avg true_objective: 12.170 +[2023-03-07 16:16:04,233][488352] Num frames 1300... +[2023-03-07 16:16:04,317][488352] Num frames 1400... +[2023-03-07 16:16:04,401][488352] Num frames 1500... +[2023-03-07 16:16:04,487][488352] Num frames 1600... +[2023-03-07 16:16:04,572][488352] Num frames 1700... +[2023-03-07 16:16:04,656][488352] Num frames 1800... +[2023-03-07 16:16:04,739][488352] Num frames 1900... +[2023-03-07 16:16:04,823][488352] Num frames 2000... +[2023-03-07 16:16:04,906][488352] Num frames 2100... +[2023-03-07 16:16:04,989][488352] Num frames 2200... +[2023-03-07 16:16:05,073][488352] Num frames 2300... +[2023-03-07 16:16:05,132][488352] Avg episode rewards: #0: 25.525, true rewards: #0: 11.525 +[2023-03-07 16:16:05,133][488352] Avg episode reward: 25.525, avg true_objective: 11.525 +[2023-03-07 16:16:05,231][488352] Num frames 2400... +[2023-03-07 16:16:05,316][488352] Num frames 2500... +[2023-03-07 16:16:05,399][488352] Num frames 2600... +[2023-03-07 16:16:05,485][488352] Num frames 2700... +[2023-03-07 16:16:05,573][488352] Num frames 2800... +[2023-03-07 16:16:05,663][488352] Num frames 2900... +[2023-03-07 16:16:05,799][488352] Avg episode rewards: #0: 22.660, true rewards: #0: 9.993 +[2023-03-07 16:16:05,800][488352] Avg episode reward: 22.660, avg true_objective: 9.993 +[2023-03-07 16:16:05,802][488352] Num frames 3000... +[2023-03-07 16:16:05,913][488352] Num frames 3100... +[2023-03-07 16:16:05,996][488352] Num frames 3200... +[2023-03-07 16:16:06,080][488352] Num frames 3300... +[2023-03-07 16:16:06,167][488352] Num frames 3400... +[2023-03-07 16:16:06,251][488352] Num frames 3500... +[2023-03-07 16:16:06,336][488352] Num frames 3600... +[2023-03-07 16:16:06,420][488352] Num frames 3700... +[2023-03-07 16:16:06,505][488352] Num frames 3800... +[2023-03-07 16:16:06,586][488352] Avg episode rewards: #0: 20.575, true rewards: #0: 9.575 +[2023-03-07 16:16:06,587][488352] Avg episode reward: 20.575, avg true_objective: 9.575 +[2023-03-07 16:16:06,657][488352] Num frames 3900... +[2023-03-07 16:16:06,743][488352] Num frames 4000... +[2023-03-07 16:16:06,828][488352] Num frames 4100... +[2023-03-07 16:16:06,913][488352] Num frames 4200... +[2023-03-07 16:16:06,979][488352] Avg episode rewards: #0: 17.228, true rewards: #0: 8.428 +[2023-03-07 16:16:06,980][488352] Avg episode reward: 17.228, avg true_objective: 8.428 +[2023-03-07 16:16:07,071][488352] Num frames 4300... +[2023-03-07 16:16:07,158][488352] Num frames 4400... +[2023-03-07 16:16:07,247][488352] Num frames 4500... +[2023-03-07 16:16:07,331][488352] Num frames 4600... +[2023-03-07 16:16:07,415][488352] Num frames 4700... +[2023-03-07 16:16:07,499][488352] Num frames 4800... +[2023-03-07 16:16:07,583][488352] Num frames 4900... +[2023-03-07 16:16:07,706][488352] Avg episode rewards: #0: 16.470, true rewards: #0: 8.303 +[2023-03-07 16:16:07,707][488352] Avg episode reward: 16.470, avg true_objective: 8.303 +[2023-03-07 16:16:07,727][488352] Num frames 5000... +[2023-03-07 16:16:07,823][488352] Num frames 5100... +[2023-03-07 16:16:07,907][488352] Num frames 5200... +[2023-03-07 16:16:07,993][488352] Num frames 5300... +[2023-03-07 16:16:08,077][488352] Num frames 5400... +[2023-03-07 16:16:08,162][488352] Num frames 5500... +[2023-03-07 16:16:08,265][488352] Avg episode rewards: #0: 15.512, true rewards: #0: 7.940 +[2023-03-07 16:16:08,267][488352] Avg episode reward: 15.512, avg true_objective: 7.940 +[2023-03-07 16:16:08,320][488352] Num frames 5600... +[2023-03-07 16:16:08,409][488352] Num frames 5700... +[2023-03-07 16:16:08,495][488352] Num frames 5800... +[2023-03-07 16:16:08,581][488352] Num frames 5900... +[2023-03-07 16:16:08,671][488352] Avg episode rewards: #0: 14.053, true rewards: #0: 7.427 +[2023-03-07 16:16:08,672][488352] Avg episode reward: 14.053, avg true_objective: 7.427 +[2023-03-07 16:16:08,722][488352] Num frames 6000... +[2023-03-07 16:16:08,807][488352] Num frames 6100... +[2023-03-07 16:16:08,895][488352] Num frames 6200... +[2023-03-07 16:16:08,980][488352] Num frames 6300... +[2023-03-07 16:16:09,057][488352] Avg episode rewards: #0: 12.918, true rewards: #0: 7.029 +[2023-03-07 16:16:09,058][488352] Avg episode reward: 12.918, avg true_objective: 7.029 +[2023-03-07 16:16:09,133][488352] Num frames 6400... +[2023-03-07 16:16:09,217][488352] Num frames 6500... +[2023-03-07 16:16:09,302][488352] Num frames 6600... +[2023-03-07 16:16:09,402][488352] Avg episode rewards: #0: 11.954, true rewards: #0: 6.654 +[2023-03-07 16:16:09,403][488352] Avg episode reward: 11.954, avg true_objective: 6.654 +[2023-03-07 16:16:17,468][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/replay.mp4! +[2023-03-07 16:17:33,672][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 16:17:33,673][488352] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 16:17:33,674][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:17:33,675][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:17:33,675][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:17:33,676][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:17:33,677][488352] Adding new argument 'max_num_frames'=100000 that is not in the saved config file! +[2023-03-07 16:17:33,677][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:17:33,678][488352] Adding new argument 'push_to_hub'=True that is not in the saved config file! +[2023-03-07 16:17:33,679][488352] Adding new argument 'hf_repository'='Rolo/rl_course_vizdoom_health_gathering_supreme' that is not in the saved config file! +[2023-03-07 16:17:33,679][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:17:33,680][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:17:33,681][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:17:33,681][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:17:33,682][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:17:33,691][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:17:33,693][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:17:33,711][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:17:33,750][488352] Conv encoder output size: 512 +[2023-03-07 16:17:33,751][488352] Policy head output size: 512 +[2023-03-07 16:17:33,776][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 16:17:34,314][488352] Num frames 100... +[2023-03-07 16:17:34,401][488352] Num frames 200... +[2023-03-07 16:17:34,490][488352] Num frames 300... +[2023-03-07 16:17:34,577][488352] Num frames 400... +[2023-03-07 16:17:34,680][488352] Avg episode rewards: #0: 6.500, true rewards: #0: 4.500 +[2023-03-07 16:17:34,681][488352] Avg episode reward: 6.500, avg true_objective: 4.500 +[2023-03-07 16:17:34,726][488352] Num frames 500... +[2023-03-07 16:17:34,811][488352] Num frames 600... +[2023-03-07 16:17:34,899][488352] Num frames 700... +[2023-03-07 16:17:34,988][488352] Num frames 800... +[2023-03-07 16:17:35,099][488352] Avg episode rewards: #0: 5.830, true rewards: #0: 4.330 +[2023-03-07 16:17:35,100][488352] Avg episode reward: 5.830, avg true_objective: 4.330 +[2023-03-07 16:17:35,143][488352] Num frames 900... +[2023-03-07 16:17:35,233][488352] Num frames 1000... +[2023-03-07 16:17:35,318][488352] Num frames 1100... +[2023-03-07 16:17:35,405][488352] Num frames 1200... +[2023-03-07 16:17:35,502][488352] Avg episode rewards: #0: 5.167, true rewards: #0: 4.167 +[2023-03-07 16:17:35,503][488352] Avg episode reward: 5.167, avg true_objective: 4.167 +[2023-03-07 16:17:35,563][488352] Num frames 1300... +[2023-03-07 16:17:35,647][488352] Num frames 1400... +[2023-03-07 16:17:35,730][488352] Num frames 1500... +[2023-03-07 16:17:35,816][488352] Num frames 1600... +[2023-03-07 16:17:35,927][488352] Avg episode rewards: #0: 5.415, true rewards: #0: 4.165 +[2023-03-07 16:17:35,928][488352] Avg episode reward: 5.415, avg true_objective: 4.165 +[2023-03-07 16:17:35,956][488352] Num frames 1700... +[2023-03-07 16:17:36,040][488352] Num frames 1800... +[2023-03-07 16:17:36,126][488352] Num frames 1900... +[2023-03-07 16:17:36,212][488352] Num frames 2000... +[2023-03-07 16:17:36,310][488352] Avg episode rewards: #0: 5.500, true rewards: #0: 4.100 +[2023-03-07 16:17:36,311][488352] Avg episode reward: 5.500, avg true_objective: 4.100 +[2023-03-07 16:17:36,373][488352] Num frames 2100... +[2023-03-07 16:17:36,457][488352] Num frames 2200... +[2023-03-07 16:17:36,543][488352] Num frames 2300... +[2023-03-07 16:17:36,628][488352] Num frames 2400... +[2023-03-07 16:17:36,717][488352] Num frames 2500... +[2023-03-07 16:17:36,850][488352] Avg episode rewards: #0: 5.823, true rewards: #0: 4.323 +[2023-03-07 16:17:36,851][488352] Avg episode reward: 5.823, avg true_objective: 4.323 +[2023-03-07 16:17:36,856][488352] Num frames 2600... +[2023-03-07 16:17:36,957][488352] Num frames 2700... +[2023-03-07 16:17:37,040][488352] Num frames 2800... +[2023-03-07 16:17:37,127][488352] Num frames 2900... +[2023-03-07 16:17:37,213][488352] Num frames 3000... +[2023-03-07 16:17:37,302][488352] Avg episode rewards: #0: 5.774, true rewards: #0: 4.346 +[2023-03-07 16:17:37,303][488352] Avg episode reward: 5.774, avg true_objective: 4.346 +[2023-03-07 16:17:37,368][488352] Num frames 3100... +[2023-03-07 16:17:37,453][488352] Num frames 3200... +[2023-03-07 16:17:37,537][488352] Num frames 3300... +[2023-03-07 16:17:37,623][488352] Num frames 3400... +[2023-03-07 16:17:37,709][488352] Num frames 3500... +[2023-03-07 16:17:37,796][488352] Num frames 3600... +[2023-03-07 16:17:37,883][488352] Num frames 3700... +[2023-03-07 16:17:37,969][488352] Num frames 3800... +[2023-03-07 16:17:38,033][488352] Avg episode rewards: #0: 6.763, true rewards: #0: 4.762 +[2023-03-07 16:17:38,034][488352] Avg episode reward: 6.763, avg true_objective: 4.762 +[2023-03-07 16:17:38,110][488352] Num frames 3900... +[2023-03-07 16:17:38,194][488352] Num frames 4000... +[2023-03-07 16:17:38,281][488352] Num frames 4100... +[2023-03-07 16:17:38,368][488352] Num frames 4200... +[2023-03-07 16:17:38,453][488352] Num frames 4300... +[2023-03-07 16:17:38,526][488352] Avg episode rewards: #0: 6.802, true rewards: #0: 4.802 +[2023-03-07 16:17:38,527][488352] Avg episode reward: 6.802, avg true_objective: 4.802 +[2023-03-07 16:17:38,594][488352] Num frames 4400... +[2023-03-07 16:17:38,680][488352] Num frames 4500... +[2023-03-07 16:17:38,765][488352] Num frames 4600... +[2023-03-07 16:17:38,851][488352] Num frames 4700... +[2023-03-07 16:17:38,967][488352] Avg episode rewards: #0: 6.670, true rewards: #0: 4.770 +[2023-03-07 16:17:38,968][488352] Avg episode reward: 6.670, avg true_objective: 4.770 +[2023-03-07 16:17:44,679][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 16:18:07,837][488352] The model has been pushed to https://huggingface.co/Rolo/rl_course_vizdoom_health_gathering_supreme +[2023-03-07 16:22:37,722][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 16:22:37,724][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 16:22:37,725][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 16:22:37,726][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 16:22:37,727][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 16:22:37,728][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 16:22:37,728][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 16:22:37,729][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 16:22:37,730][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 16:22:37,731][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 16:22:37,732][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 16:22:37,733][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 16:22:37,733][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 16:22:37,734][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 16:22:37,735][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 16:22:37,736][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 16:22:37,737][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 16:22:37,738][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 16:22:37,739][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 16:22:37,740][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 16:22:37,740][488352] register_encoder_factory: +[2023-03-07 16:22:37,770][488352] Saved parameter configuration for experiment doom_health_threads --batch_size=4096 not found! +[2023-03-07 16:22:37,771][488352] Starting experiment from scratch! +[2023-03-07 16:22:37,773][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads --batch_size=4096 already exists! +[2023-03-07 16:22:37,774][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads --batch_size=4096... +[2023-03-07 16:22:37,775][488352] Weights and Biases integration disabled +[2023-03-07 16:22:37,777][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 16:22:38,845][488352] Automatically setting recurrence to 32 +[2023-03-07 16:22:38,846][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads --batch_size=4096 +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=1024 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=32 --train_for_env_steps=4000000 --experiment=doom_health_threads --batch_size=4096 +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads --batch_size=4096', 'num_workers': 128, 'num_envs_per_worker': 32, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 16:22:38,847][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads --batch_size=4096/config.json... +[2023-03-07 16:22:38,848][488352] Rollout worker 0 uses device cpu +[2023-03-07 16:22:38,849][488352] Rollout worker 1 uses device cpu +[2023-03-07 16:22:38,849][488352] Rollout worker 2 uses device cpu +[2023-03-07 16:22:38,850][488352] Rollout worker 3 uses device cpu +[2023-03-07 16:22:38,851][488352] Rollout worker 4 uses device cpu +[2023-03-07 16:22:38,851][488352] Rollout worker 5 uses device cpu +[2023-03-07 16:22:38,852][488352] Rollout worker 6 uses device cpu +[2023-03-07 16:22:38,852][488352] Rollout worker 7 uses device cpu +[2023-03-07 16:22:38,853][488352] Rollout worker 8 uses device cpu +[2023-03-07 16:22:38,853][488352] Rollout worker 9 uses device cpu +[2023-03-07 16:22:38,854][488352] Rollout worker 10 uses device cpu +[2023-03-07 16:22:38,855][488352] Rollout worker 11 uses device cpu +[2023-03-07 16:22:38,855][488352] Rollout worker 12 uses device cpu +[2023-03-07 16:22:38,856][488352] Rollout worker 13 uses device cpu +[2023-03-07 16:22:38,856][488352] Rollout worker 14 uses device cpu +[2023-03-07 16:22:38,857][488352] Rollout worker 15 uses device cpu +[2023-03-07 16:22:38,857][488352] Rollout worker 16 uses device cpu +[2023-03-07 16:22:38,858][488352] Rollout worker 17 uses device cpu +[2023-03-07 16:22:38,859][488352] Rollout worker 18 uses device cpu +[2023-03-07 16:22:38,859][488352] Rollout worker 19 uses device cpu +[2023-03-07 16:22:38,860][488352] Rollout worker 20 uses device cpu +[2023-03-07 16:22:38,860][488352] Rollout worker 21 uses device cpu +[2023-03-07 16:22:38,861][488352] Rollout worker 22 uses device cpu +[2023-03-07 16:22:38,862][488352] Rollout worker 23 uses device cpu +[2023-03-07 16:22:38,862][488352] Rollout worker 24 uses device cpu +[2023-03-07 16:22:38,863][488352] Rollout worker 25 uses device cpu +[2023-03-07 16:22:38,863][488352] Rollout worker 26 uses device cpu +[2023-03-07 16:22:38,864][488352] Rollout worker 27 uses device cpu +[2023-03-07 16:22:38,868][488352] Rollout worker 28 uses device cpu +[2023-03-07 16:22:38,869][488352] Rollout worker 29 uses device cpu +[2023-03-07 16:22:38,870][488352] Rollout worker 30 uses device cpu +[2023-03-07 16:22:38,870][488352] Rollout worker 31 uses device cpu +[2023-03-07 16:22:38,871][488352] Rollout worker 32 uses device cpu +[2023-03-07 16:22:38,871][488352] Rollout worker 33 uses device cpu +[2023-03-07 16:22:38,872][488352] Rollout worker 34 uses device cpu +[2023-03-07 16:22:38,873][488352] Rollout worker 35 uses device cpu +[2023-03-07 16:22:38,873][488352] Rollout worker 36 uses device cpu +[2023-03-07 16:22:38,874][488352] Rollout worker 37 uses device cpu +[2023-03-07 16:22:38,874][488352] Rollout worker 38 uses device cpu +[2023-03-07 16:22:38,875][488352] Rollout worker 39 uses device cpu +[2023-03-07 16:22:38,876][488352] Rollout worker 40 uses device cpu +[2023-03-07 16:22:38,876][488352] Rollout worker 41 uses device cpu +[2023-03-07 16:22:38,877][488352] Rollout worker 42 uses device cpu +[2023-03-07 16:22:38,877][488352] Rollout worker 43 uses device cpu +[2023-03-07 16:22:38,878][488352] Rollout worker 44 uses device cpu +[2023-03-07 16:22:38,878][488352] Rollout worker 45 uses device cpu +[2023-03-07 16:22:38,879][488352] Rollout worker 46 uses device cpu +[2023-03-07 16:22:38,880][488352] Rollout worker 47 uses device cpu +[2023-03-07 16:22:38,880][488352] Rollout worker 48 uses device cpu +[2023-03-07 16:22:38,881][488352] Rollout worker 49 uses device cpu +[2023-03-07 16:22:38,882][488352] Rollout worker 50 uses device cpu +[2023-03-07 16:22:38,886][488352] Rollout worker 51 uses device cpu +[2023-03-07 16:22:38,886][488352] Rollout worker 52 uses device cpu +[2023-03-07 16:22:38,887][488352] Rollout worker 53 uses device cpu +[2023-03-07 16:22:38,888][488352] Rollout worker 54 uses device cpu +[2023-03-07 16:22:38,889][488352] Rollout worker 55 uses device cpu +[2023-03-07 16:22:38,889][488352] Rollout worker 56 uses device cpu +[2023-03-07 16:22:38,891][488352] Rollout worker 57 uses device cpu +[2023-03-07 16:22:38,892][488352] Rollout worker 58 uses device cpu +[2023-03-07 16:22:38,893][488352] Rollout worker 59 uses device cpu +[2023-03-07 16:22:38,893][488352] Rollout worker 60 uses device cpu +[2023-03-07 16:22:38,894][488352] Rollout worker 61 uses device cpu +[2023-03-07 16:22:38,895][488352] Rollout worker 62 uses device cpu +[2023-03-07 16:22:38,896][488352] Rollout worker 63 uses device cpu +[2023-03-07 16:22:38,897][488352] Rollout worker 64 uses device cpu +[2023-03-07 16:22:38,898][488352] Rollout worker 65 uses device cpu +[2023-03-07 16:22:38,899][488352] Rollout worker 66 uses device cpu +[2023-03-07 16:22:38,899][488352] Rollout worker 67 uses device cpu +[2023-03-07 16:22:38,900][488352] Rollout worker 68 uses device cpu +[2023-03-07 16:22:38,901][488352] Rollout worker 69 uses device cpu +[2023-03-07 16:22:38,902][488352] Rollout worker 70 uses device cpu +[2023-03-07 16:22:38,903][488352] Rollout worker 71 uses device cpu +[2023-03-07 16:22:38,904][488352] Rollout worker 72 uses device cpu +[2023-03-07 16:22:38,904][488352] Rollout worker 73 uses device cpu +[2023-03-07 16:22:38,905][488352] Rollout worker 74 uses device cpu +[2023-03-07 16:22:38,906][488352] Rollout worker 75 uses device cpu +[2023-03-07 16:22:38,907][488352] Rollout worker 76 uses device cpu +[2023-03-07 16:22:38,908][488352] Rollout worker 77 uses device cpu +[2023-03-07 16:22:38,909][488352] Rollout worker 78 uses device cpu +[2023-03-07 16:22:38,910][488352] Rollout worker 79 uses device cpu +[2023-03-07 16:22:38,911][488352] Rollout worker 80 uses device cpu +[2023-03-07 16:22:38,912][488352] Rollout worker 81 uses device cpu +[2023-03-07 16:22:38,912][488352] Rollout worker 82 uses device cpu +[2023-03-07 16:22:38,913][488352] Rollout worker 83 uses device cpu +[2023-03-07 16:22:38,914][488352] Rollout worker 84 uses device cpu +[2023-03-07 16:22:38,915][488352] Rollout worker 85 uses device cpu +[2023-03-07 16:22:38,916][488352] Rollout worker 86 uses device cpu +[2023-03-07 16:22:38,916][488352] Rollout worker 87 uses device cpu +[2023-03-07 16:22:38,918][488352] Rollout worker 88 uses device cpu +[2023-03-07 16:22:38,918][488352] Rollout worker 89 uses device cpu +[2023-03-07 16:22:38,919][488352] Rollout worker 90 uses device cpu +[2023-03-07 16:22:38,920][488352] Rollout worker 91 uses device cpu +[2023-03-07 16:22:38,920][488352] Rollout worker 92 uses device cpu +[2023-03-07 16:22:38,921][488352] Rollout worker 93 uses device cpu +[2023-03-07 16:22:38,922][488352] Rollout worker 94 uses device cpu +[2023-03-07 16:22:38,923][488352] Rollout worker 95 uses device cpu +[2023-03-07 16:22:38,924][488352] Rollout worker 96 uses device cpu +[2023-03-07 16:22:38,925][488352] Rollout worker 97 uses device cpu +[2023-03-07 16:22:38,925][488352] Rollout worker 98 uses device cpu +[2023-03-07 16:22:38,926][488352] Rollout worker 99 uses device cpu +[2023-03-07 16:22:38,927][488352] Rollout worker 100 uses device cpu +[2023-03-07 16:22:38,927][488352] Rollout worker 101 uses device cpu +[2023-03-07 16:22:38,928][488352] Rollout worker 102 uses device cpu +[2023-03-07 16:22:38,929][488352] Rollout worker 103 uses device cpu +[2023-03-07 16:22:38,929][488352] Rollout worker 104 uses device cpu +[2023-03-07 16:22:38,932][488352] Rollout worker 105 uses device cpu +[2023-03-07 16:22:38,933][488352] Rollout worker 106 uses device cpu +[2023-03-07 16:22:38,933][488352] Rollout worker 107 uses device cpu +[2023-03-07 16:22:38,934][488352] Rollout worker 108 uses device cpu +[2023-03-07 16:22:38,935][488352] Rollout worker 109 uses device cpu +[2023-03-07 16:22:38,936][488352] Rollout worker 110 uses device cpu +[2023-03-07 16:22:38,937][488352] Rollout worker 111 uses device cpu +[2023-03-07 16:22:38,937][488352] Rollout worker 112 uses device cpu +[2023-03-07 16:22:38,938][488352] Rollout worker 113 uses device cpu +[2023-03-07 16:22:38,939][488352] Rollout worker 114 uses device cpu +[2023-03-07 16:22:38,940][488352] Rollout worker 115 uses device cpu +[2023-03-07 16:22:38,941][488352] Rollout worker 116 uses device cpu +[2023-03-07 16:22:38,941][488352] Rollout worker 117 uses device cpu +[2023-03-07 16:22:38,943][488352] Rollout worker 118 uses device cpu +[2023-03-07 16:22:38,943][488352] Rollout worker 119 uses device cpu +[2023-03-07 16:22:38,944][488352] Rollout worker 120 uses device cpu +[2023-03-07 16:22:38,945][488352] Rollout worker 121 uses device cpu +[2023-03-07 16:22:38,946][488352] Rollout worker 122 uses device cpu +[2023-03-07 16:22:38,947][488352] Rollout worker 123 uses device cpu +[2023-03-07 16:22:38,948][488352] Rollout worker 124 uses device cpu +[2023-03-07 16:22:38,949][488352] Rollout worker 125 uses device cpu +[2023-03-07 16:22:38,949][488352] Rollout worker 126 uses device cpu +[2023-03-07 16:22:38,950][488352] Rollout worker 127 uses device cpu +[2023-03-07 16:22:40,983][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 16:22:40,984][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 16:22:41,251][488352] Starting all processes... +[2023-03-07 16:22:41,252][488352] Starting process learner_proc0 +[2023-03-07 16:22:42,137][488352] Starting all processes... +[2023-03-07 16:22:42,141][488352] Starting process inference_proc0-0 +[2023-03-07 16:22:42,142][488352] Starting process rollout_proc2 +[2023-03-07 16:22:42,142][488352] Starting process rollout_proc5 +[2023-03-07 16:22:42,143][488352] Starting process rollout_proc8 +[2023-03-07 16:22:42,143][488352] Starting process rollout_proc11 +[2023-03-07 16:22:42,144][488352] Starting process rollout_proc14 +[2023-03-07 16:22:42,147][488352] Starting process rollout_proc17 +[2023-03-07 16:22:42,149][488352] Starting process rollout_proc20 +[2023-03-07 16:22:42,151][488352] Starting process rollout_proc23 +[2023-03-07 16:22:42,152][488352] Starting process rollout_proc26 +[2023-03-07 16:22:42,152][488352] Starting process rollout_proc29 +[2023-03-07 16:22:42,153][488352] Starting process rollout_proc32 +[2023-03-07 16:22:42,154][488352] Starting process rollout_proc35 +[2023-03-07 16:22:42,155][488352] Starting process rollout_proc38 +[2023-03-07 16:22:42,160][488352] Starting process rollout_proc41 +[2023-03-07 16:22:42,164][488352] Starting process rollout_proc44 +[2023-03-07 16:22:42,187][488352] Starting process rollout_proc6 +[2023-03-07 16:22:42,189][488352] Starting process rollout_proc9 +[2023-03-07 16:22:42,200][488352] Starting process rollout_proc3 +[2023-03-07 16:22:42,202][488352] Starting process rollout_proc12 +[2023-03-07 16:22:42,202][488352] Starting process rollout_proc18 +[2023-03-07 16:22:42,207][488352] Starting process rollout_proc15 +[2023-03-07 16:22:42,216][488352] Starting process rollout_proc21 +[2023-03-07 16:22:42,222][488352] Starting process rollout_proc27 +[2023-03-07 16:22:42,241][488352] Starting process rollout_proc30 +[2023-03-07 16:22:42,250][488352] Starting process rollout_proc24 +[2023-03-07 16:22:42,252][488352] Starting process rollout_proc33 +[2023-03-07 16:22:42,271][488352] Starting process rollout_proc36 +[2023-03-07 16:22:42,274][488352] Starting process rollout_proc39 +[2023-03-07 16:22:42,278][488352] Starting process rollout_proc42 +[2023-03-07 16:22:42,279][488352] Starting process rollout_proc45 +[2023-03-07 16:22:42,291][488352] Starting process rollout_proc7 +[2023-03-07 16:22:42,293][488352] Starting process rollout_proc10 +[2023-03-07 16:22:42,299][488352] Starting process rollout_proc4 +[2023-03-07 16:22:42,308][488352] Starting process rollout_proc13 +[2023-03-07 16:22:42,312][488352] Starting process rollout_proc16 +[2023-03-07 16:22:42,321][488352] Starting process rollout_proc22 +[2023-03-07 16:22:42,321][488352] Starting process rollout_proc28 +[2023-03-07 16:22:42,321][488352] Starting process rollout_proc19 +[2023-03-07 16:22:42,328][488352] Starting process rollout_proc31 +[2023-03-07 16:22:42,336][488352] Starting process rollout_proc34 +[2023-03-07 16:22:42,338][488352] Starting process rollout_proc25 +[2023-03-07 16:22:42,345][488352] Starting process rollout_proc37 +[2023-03-07 16:22:42,350][488352] Starting process rollout_proc40 +[2023-03-07 16:22:42,358][488352] Starting process rollout_proc43 +[2023-03-07 16:22:42,362][488352] Starting process rollout_proc46 +[2023-03-07 16:22:42,366][488352] Starting process rollout_proc47 +[2023-03-07 16:22:42,373][488352] Starting process rollout_proc50 +[2023-03-07 16:22:42,380][488352] Starting process rollout_proc53 +[2023-03-07 16:22:42,389][488352] Starting process rollout_proc56 +[2023-03-07 16:22:42,391][488352] Starting process rollout_proc59 +[2023-03-07 16:22:42,397][488352] Starting process rollout_proc62 +[2023-03-07 16:22:42,407][488352] Starting process rollout_proc65 +[2023-03-07 16:22:42,416][488352] Starting process rollout_proc68 +[2023-03-07 16:22:42,416][488352] Starting process rollout_proc71 +[2023-03-07 16:22:42,424][488352] Starting process rollout_proc74 +[2023-03-07 16:22:42,428][488352] Starting process rollout_proc77 +[2023-03-07 16:22:42,438][488352] Starting process rollout_proc80 +[2023-03-07 16:22:42,446][488352] Starting process rollout_proc83 +[2023-03-07 16:22:42,453][488352] Starting process rollout_proc86 +[2023-03-07 16:22:42,463][488352] Starting process rollout_proc89 +[2023-03-07 16:22:42,476][488352] Starting process rollout_proc48 +[2023-03-07 16:22:42,476][488352] Starting process rollout_proc54 +[2023-03-07 16:22:42,488][488352] Starting process rollout_proc57 +[2023-03-07 16:22:42,496][488352] Starting process rollout_proc51 +[2023-03-07 16:22:42,503][488352] Starting process rollout_proc60 +[2023-03-07 16:22:42,512][488352] Starting process rollout_proc66 +[2023-03-07 16:22:42,521][488352] Starting process rollout_proc63 +[2023-03-07 16:22:42,531][488352] Starting process rollout_proc72 +[2023-03-07 16:22:42,531][488352] Starting process rollout_proc78 +[2023-03-07 16:22:42,537][488352] Starting process rollout_proc75 +[2023-03-07 16:22:42,540][488352] Starting process rollout_proc69 +[2023-03-07 16:22:42,550][488352] Starting process rollout_proc81 +[2023-03-07 16:22:42,557][488352] Starting process rollout_proc84 +[2023-03-07 16:22:42,565][488352] Starting process rollout_proc87 +[2023-03-07 16:22:42,581][488352] Starting process rollout_proc55 +[2023-03-07 16:22:42,587][488352] Starting process rollout_proc90 +[2023-03-07 16:22:42,612][488352] Starting process rollout_proc49 +[2023-03-07 16:22:42,612][488352] Starting process rollout_proc58 +[2023-03-07 16:22:42,623][488352] Starting process rollout_proc52 +[2023-03-07 16:22:42,634][488352] Starting process rollout_proc64 +[2023-03-07 16:22:42,638][488352] Starting process rollout_proc67 +[2023-03-07 16:22:42,647][488352] Starting process rollout_proc73 +[2023-03-07 16:22:42,651][488352] Starting process rollout_proc61 +[2023-03-07 16:22:42,653][488352] Starting process rollout_proc79 +[2023-03-07 16:22:42,656][488352] Starting process rollout_proc76 +[2023-03-07 16:22:42,666][488352] Starting process rollout_proc70 +[2023-03-07 16:22:42,677][488352] Starting process rollout_proc85 +[2023-03-07 16:22:42,685][488352] Starting process rollout_proc88 +[2023-03-07 16:22:42,696][488352] Starting process rollout_proc82 +[2023-03-07 16:22:42,748][488352] Starting process rollout_proc92 +[2023-03-07 16:22:42,750][488352] Starting process rollout_proc91 +[2023-03-07 16:22:42,750][488352] Starting process rollout_proc95 +[2023-03-07 16:22:42,784][488352] Starting process rollout_proc98 +[2023-03-07 16:22:42,843][488352] Starting process rollout_proc101 +[2023-03-07 16:22:42,856][488352] Starting process rollout_proc104 +[2023-03-07 16:22:42,875][488352] Starting process rollout_proc113 +[2023-03-07 16:22:42,857][488352] Starting process rollout_proc110 +[2023-03-07 16:22:42,876][488352] Starting process rollout_proc116 +[2023-03-07 16:22:42,856][488352] Starting process rollout_proc107 +[2023-03-07 16:22:42,893][488352] Starting process rollout_proc119 +[2023-03-07 16:22:42,918][488352] Starting process rollout_proc122 +[2023-03-07 16:22:42,952][488352] Starting process rollout_proc125 +[2023-03-07 16:22:43,014][488352] Starting process rollout_proc102 +[2023-03-07 16:22:43,074][488352] Starting process rollout_proc93 +[2023-03-07 16:22:43,074][488352] Starting process rollout_proc96 +[2023-03-07 16:22:43,075][488352] Starting process rollout_proc99 +[2023-03-07 16:22:43,106][488352] Starting process rollout_proc105 +[2023-03-07 16:22:43,106][488352] Starting process rollout_proc120 +[2023-03-07 16:22:43,110][488352] Starting process rollout_proc114 +[2023-03-07 16:22:43,111][488352] Starting process rollout_proc111 +[2023-03-07 16:22:43,155][488352] Starting process rollout_proc117 +[2023-03-07 16:22:43,198][488352] Starting process rollout_proc126 +[2023-03-07 16:22:43,221][488352] Starting process rollout_proc123 +[2023-03-07 16:22:43,257][488352] Starting process rollout_proc103 +[2023-03-07 16:22:43,257][488352] Starting process rollout_proc108 +[2023-03-07 16:22:43,269][488352] Starting process rollout_proc97 +[2023-03-07 16:22:43,287][488352] Starting process rollout_proc100 +[2023-03-07 16:22:43,287][488352] Starting process rollout_proc121 +[2023-03-07 16:22:43,287][488352] Starting process rollout_proc94 +[2023-03-07 16:22:43,331][488352] Starting process rollout_proc115 +[2023-03-07 16:22:43,382][488352] Starting process rollout_proc106 +[2023-03-07 16:22:43,404][488352] Starting process rollout_proc118 +[2023-03-07 16:22:43,406][488352] Starting process rollout_proc112 +[2023-03-07 16:22:43,437][488352] Starting process rollout_proc127 +[2023-03-07 16:22:43,514][488352] Starting process rollout_proc109 +[2023-03-07 16:22:43,519][488352] Starting process rollout_proc124 +[2023-03-07 16:22:45,711][488352] Starting process rollout_proc0 +[2023-03-07 16:22:45,863][488352] Starting process rollout_proc1 +[2023-03-07 16:22:47,313][488352] Inference worker 0-0 is ready! +[2023-03-07 16:22:47,315][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 16:22:47,779][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:22:52,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:22:57,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:00,979][488352] Heartbeat connected on Batcher_0 +[2023-03-07 16:23:00,981][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 16:23:01,009][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 16:23:02,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:07,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:12,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:17,778][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 0. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:21,701][488352] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 488352], exiting... +[2023-03-07 16:23:21,704][488352] Runner profile tree view: +main_loop: 40.4531 +[2023-03-07 16:23:21,705][488352] Collected {0: 0}, FPS: 0.0 +[2023-03-07 16:23:31,633][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 16:23:31,634][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 16:23:31,635][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 16:23:31,636][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 16:23:31,637][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 16:23:31,637][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 16:23:31,639][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 16:23:31,640][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 16:23:31,640][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 16:23:31,641][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 16:23:31,642][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 16:23:31,643][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 16:23:31,644][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 16:23:31,645][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 16:23:31,645][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 16:23:31,647][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 16:23:31,648][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 16:23:31,649][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 16:23:31,650][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 16:23:31,651][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 16:23:31,651][488352] register_encoder_factory: +[2023-03-07 16:23:31,681][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:23:31,682][488352] Overriding arg 'batch_size' with value 4096 passed from command line +[2023-03-07 16:23:31,682][488352] Overriding arg 'train_for_env_steps' with value 4000000 passed from command line +[2023-03-07 16:23:31,687][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads already exists! +[2023-03-07 16:23:31,688][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads... +[2023-03-07 16:23:31,689][488352] Weights and Biases integration disabled +[2023-03-07 16:23:31,692][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 16:23:32,849][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=32 +batch_size=4096 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=16 --train_for_env_steps=4000000 --experiment=doom_health_threads +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads', 'num_workers': 128, 'num_envs_per_worker': 16, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 16:23:32,850][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json... +[2023-03-07 16:23:32,851][488352] Rollout worker 0 uses device cpu +[2023-03-07 16:23:32,852][488352] Rollout worker 1 uses device cpu +[2023-03-07 16:23:32,852][488352] Rollout worker 2 uses device cpu +[2023-03-07 16:23:32,853][488352] Rollout worker 3 uses device cpu +[2023-03-07 16:23:32,853][488352] Rollout worker 4 uses device cpu +[2023-03-07 16:23:32,853][488352] Rollout worker 5 uses device cpu +[2023-03-07 16:23:32,854][488352] Rollout worker 6 uses device cpu +[2023-03-07 16:23:32,855][488352] Rollout worker 7 uses device cpu +[2023-03-07 16:23:32,856][488352] Rollout worker 8 uses device cpu +[2023-03-07 16:23:32,856][488352] Rollout worker 9 uses device cpu +[2023-03-07 16:23:32,857][488352] Rollout worker 10 uses device cpu +[2023-03-07 16:23:32,857][488352] Rollout worker 11 uses device cpu +[2023-03-07 16:23:32,857][488352] Rollout worker 12 uses device cpu +[2023-03-07 16:23:32,858][488352] Rollout worker 13 uses device cpu +[2023-03-07 16:23:32,859][488352] Rollout worker 14 uses device cpu +[2023-03-07 16:23:32,859][488352] Rollout worker 15 uses device cpu +[2023-03-07 16:23:32,859][488352] Rollout worker 16 uses device cpu +[2023-03-07 16:23:32,860][488352] Rollout worker 17 uses device cpu +[2023-03-07 16:23:32,860][488352] Rollout worker 18 uses device cpu +[2023-03-07 16:23:32,861][488352] Rollout worker 19 uses device cpu +[2023-03-07 16:23:32,861][488352] Rollout worker 20 uses device cpu +[2023-03-07 16:23:32,861][488352] Rollout worker 21 uses device cpu +[2023-03-07 16:23:32,862][488352] Rollout worker 22 uses device cpu +[2023-03-07 16:23:32,863][488352] Rollout worker 23 uses device cpu +[2023-03-07 16:23:32,863][488352] Rollout worker 24 uses device cpu +[2023-03-07 16:23:32,864][488352] Rollout worker 25 uses device cpu +[2023-03-07 16:23:32,864][488352] Rollout worker 26 uses device cpu +[2023-03-07 16:23:32,865][488352] Rollout worker 27 uses device cpu +[2023-03-07 16:23:32,866][488352] Rollout worker 28 uses device cpu +[2023-03-07 16:23:32,866][488352] Rollout worker 29 uses device cpu +[2023-03-07 16:23:32,866][488352] Rollout worker 30 uses device cpu +[2023-03-07 16:23:32,867][488352] Rollout worker 31 uses device cpu +[2023-03-07 16:23:32,867][488352] Rollout worker 32 uses device cpu +[2023-03-07 16:23:32,868][488352] Rollout worker 33 uses device cpu +[2023-03-07 16:23:32,868][488352] Rollout worker 34 uses device cpu +[2023-03-07 16:23:32,868][488352] Rollout worker 35 uses device cpu +[2023-03-07 16:23:32,870][488352] Rollout worker 36 uses device cpu +[2023-03-07 16:23:32,870][488352] Rollout worker 37 uses device cpu +[2023-03-07 16:23:32,871][488352] Rollout worker 38 uses device cpu +[2023-03-07 16:23:32,871][488352] Rollout worker 39 uses device cpu +[2023-03-07 16:23:32,871][488352] Rollout worker 40 uses device cpu +[2023-03-07 16:23:32,872][488352] Rollout worker 41 uses device cpu +[2023-03-07 16:23:32,872][488352] Rollout worker 42 uses device cpu +[2023-03-07 16:23:32,873][488352] Rollout worker 43 uses device cpu +[2023-03-07 16:23:32,873][488352] Rollout worker 44 uses device cpu +[2023-03-07 16:23:32,873][488352] Rollout worker 45 uses device cpu +[2023-03-07 16:23:32,874][488352] Rollout worker 46 uses device cpu +[2023-03-07 16:23:32,875][488352] Rollout worker 47 uses device cpu +[2023-03-07 16:23:32,876][488352] Rollout worker 48 uses device cpu +[2023-03-07 16:23:32,876][488352] Rollout worker 49 uses device cpu +[2023-03-07 16:23:32,876][488352] Rollout worker 50 uses device cpu +[2023-03-07 16:23:32,877][488352] Rollout worker 51 uses device cpu +[2023-03-07 16:23:32,877][488352] Rollout worker 52 uses device cpu +[2023-03-07 16:23:32,878][488352] Rollout worker 53 uses device cpu +[2023-03-07 16:23:32,878][488352] Rollout worker 54 uses device cpu +[2023-03-07 16:23:32,879][488352] Rollout worker 55 uses device cpu +[2023-03-07 16:23:32,879][488352] Rollout worker 56 uses device cpu +[2023-03-07 16:23:32,879][488352] Rollout worker 57 uses device cpu +[2023-03-07 16:23:32,880][488352] Rollout worker 58 uses device cpu +[2023-03-07 16:23:32,880][488352] Rollout worker 59 uses device cpu +[2023-03-07 16:23:32,881][488352] Rollout worker 60 uses device cpu +[2023-03-07 16:23:32,881][488352] Rollout worker 61 uses device cpu +[2023-03-07 16:23:32,881][488352] Rollout worker 62 uses device cpu +[2023-03-07 16:23:32,882][488352] Rollout worker 63 uses device cpu +[2023-03-07 16:23:32,884][488352] Rollout worker 64 uses device cpu +[2023-03-07 16:23:32,884][488352] Rollout worker 65 uses device cpu +[2023-03-07 16:23:32,884][488352] Rollout worker 66 uses device cpu +[2023-03-07 16:23:32,885][488352] Rollout worker 67 uses device cpu +[2023-03-07 16:23:32,885][488352] Rollout worker 68 uses device cpu +[2023-03-07 16:23:32,886][488352] Rollout worker 69 uses device cpu +[2023-03-07 16:23:32,887][488352] Rollout worker 70 uses device cpu +[2023-03-07 16:23:32,887][488352] Rollout worker 71 uses device cpu +[2023-03-07 16:23:32,887][488352] Rollout worker 72 uses device cpu +[2023-03-07 16:23:32,888][488352] Rollout worker 73 uses device cpu +[2023-03-07 16:23:32,888][488352] Rollout worker 74 uses device cpu +[2023-03-07 16:23:32,889][488352] Rollout worker 75 uses device cpu +[2023-03-07 16:23:32,889][488352] Rollout worker 76 uses device cpu +[2023-03-07 16:23:32,889][488352] Rollout worker 77 uses device cpu +[2023-03-07 16:23:32,890][488352] Rollout worker 78 uses device cpu +[2023-03-07 16:23:32,890][488352] Rollout worker 79 uses device cpu +[2023-03-07 16:23:32,891][488352] Rollout worker 80 uses device cpu +[2023-03-07 16:23:32,891][488352] Rollout worker 81 uses device cpu +[2023-03-07 16:23:32,892][488352] Rollout worker 82 uses device cpu +[2023-03-07 16:23:32,892][488352] Rollout worker 83 uses device cpu +[2023-03-07 16:23:32,892][488352] Rollout worker 84 uses device cpu +[2023-03-07 16:23:32,893][488352] Rollout worker 85 uses device cpu +[2023-03-07 16:23:32,893][488352] Rollout worker 86 uses device cpu +[2023-03-07 16:23:32,894][488352] Rollout worker 87 uses device cpu +[2023-03-07 16:23:32,894][488352] Rollout worker 88 uses device cpu +[2023-03-07 16:23:32,894][488352] Rollout worker 89 uses device cpu +[2023-03-07 16:23:32,895][488352] Rollout worker 90 uses device cpu +[2023-03-07 16:23:32,895][488352] Rollout worker 91 uses device cpu +[2023-03-07 16:23:32,896][488352] Rollout worker 92 uses device cpu +[2023-03-07 16:23:32,896][488352] Rollout worker 93 uses device cpu +[2023-03-07 16:23:32,896][488352] Rollout worker 94 uses device cpu +[2023-03-07 16:23:32,897][488352] Rollout worker 95 uses device cpu +[2023-03-07 16:23:32,897][488352] Rollout worker 96 uses device cpu +[2023-03-07 16:23:32,898][488352] Rollout worker 97 uses device cpu +[2023-03-07 16:23:32,898][488352] Rollout worker 98 uses device cpu +[2023-03-07 16:23:32,899][488352] Rollout worker 99 uses device cpu +[2023-03-07 16:23:32,901][488352] Rollout worker 100 uses device cpu +[2023-03-07 16:23:32,902][488352] Rollout worker 101 uses device cpu +[2023-03-07 16:23:32,902][488352] Rollout worker 102 uses device cpu +[2023-03-07 16:23:32,903][488352] Rollout worker 103 uses device cpu +[2023-03-07 16:23:32,903][488352] Rollout worker 104 uses device cpu +[2023-03-07 16:23:32,904][488352] Rollout worker 105 uses device cpu +[2023-03-07 16:23:32,904][488352] Rollout worker 106 uses device cpu +[2023-03-07 16:23:32,904][488352] Rollout worker 107 uses device cpu +[2023-03-07 16:23:32,905][488352] Rollout worker 108 uses device cpu +[2023-03-07 16:23:32,905][488352] Rollout worker 109 uses device cpu +[2023-03-07 16:23:32,907][488352] Rollout worker 110 uses device cpu +[2023-03-07 16:23:32,908][488352] Rollout worker 111 uses device cpu +[2023-03-07 16:23:32,908][488352] Rollout worker 112 uses device cpu +[2023-03-07 16:23:32,909][488352] Rollout worker 113 uses device cpu +[2023-03-07 16:23:32,909][488352] Rollout worker 114 uses device cpu +[2023-03-07 16:23:32,909][488352] Rollout worker 115 uses device cpu +[2023-03-07 16:23:32,910][488352] Rollout worker 116 uses device cpu +[2023-03-07 16:23:32,910][488352] Rollout worker 117 uses device cpu +[2023-03-07 16:23:32,911][488352] Rollout worker 118 uses device cpu +[2023-03-07 16:23:32,911][488352] Rollout worker 119 uses device cpu +[2023-03-07 16:23:32,912][488352] Rollout worker 120 uses device cpu +[2023-03-07 16:23:32,912][488352] Rollout worker 121 uses device cpu +[2023-03-07 16:23:32,912][488352] Rollout worker 122 uses device cpu +[2023-03-07 16:23:32,913][488352] Rollout worker 123 uses device cpu +[2023-03-07 16:23:32,913][488352] Rollout worker 124 uses device cpu +[2023-03-07 16:23:32,914][488352] Rollout worker 125 uses device cpu +[2023-03-07 16:23:32,914][488352] Rollout worker 126 uses device cpu +[2023-03-07 16:23:32,914][488352] Rollout worker 127 uses device cpu +[2023-03-07 16:23:35,230][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 16:23:35,230][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 16:23:35,552][488352] Starting all processes... +[2023-03-07 16:23:35,553][488352] Starting process learner_proc0 +[2023-03-07 16:23:36,516][488352] Starting all processes... +[2023-03-07 16:23:36,520][488352] Starting process inference_proc0-0 +[2023-03-07 16:23:36,521][488352] Starting process rollout_proc2 +[2023-03-07 16:23:36,521][488352] Starting process rollout_proc5 +[2023-03-07 16:23:36,521][488352] Starting process rollout_proc8 +[2023-03-07 16:23:36,521][488352] Starting process rollout_proc11 +[2023-03-07 16:23:36,522][488352] Starting process rollout_proc14 +[2023-03-07 16:23:36,522][488352] Starting process rollout_proc17 +[2023-03-07 16:23:36,522][488352] Starting process rollout_proc20 +[2023-03-07 16:23:36,523][488352] Starting process rollout_proc23 +[2023-03-07 16:23:36,523][488352] Starting process rollout_proc26 +[2023-03-07 16:23:36,523][488352] Starting process rollout_proc29 +[2023-03-07 16:23:36,524][488352] Starting process rollout_proc32 +[2023-03-07 16:23:36,524][488352] Starting process rollout_proc35 +[2023-03-07 16:23:36,524][488352] Starting process rollout_proc38 +[2023-03-07 16:23:36,524][488352] Starting process rollout_proc41 +[2023-03-07 16:23:36,525][488352] Starting process rollout_proc44 +[2023-03-07 16:23:36,584][488352] Starting process rollout_proc3 +[2023-03-07 16:23:36,592][488352] Starting process rollout_proc6 +[2023-03-07 16:23:36,597][488352] Starting process rollout_proc12 +[2023-03-07 16:23:36,598][488352] Starting process rollout_proc15 +[2023-03-07 16:23:36,608][488352] Starting process rollout_proc9 +[2023-03-07 16:23:36,619][488352] Starting process rollout_proc21 +[2023-03-07 16:23:36,637][488352] Starting process rollout_proc24 +[2023-03-07 16:23:36,638][488352] Starting process rollout_proc18 +[2023-03-07 16:23:36,653][488352] Starting process rollout_proc27 +[2023-03-07 16:23:36,659][488352] Starting process rollout_proc45 +[2023-03-07 16:23:36,664][488352] Starting process rollout_proc33 +[2023-03-07 16:23:36,664][488352] Starting process rollout_proc30 +[2023-03-07 16:23:36,674][488352] Starting process rollout_proc39 +[2023-03-07 16:23:36,675][488352] Starting process rollout_proc36 +[2023-03-07 16:23:36,689][488352] Starting process rollout_proc42 +[2023-03-07 16:23:36,708][488352] Starting process rollout_proc7 +[2023-03-07 16:23:36,718][488352] Starting process rollout_proc4 +[2023-03-07 16:23:36,721][488352] Starting process rollout_proc19 +[2023-03-07 16:23:36,726][488352] Starting process rollout_proc13 +[2023-03-07 16:23:36,728][488352] Starting process rollout_proc25 +[2023-03-07 16:23:36,740][488352] Starting process rollout_proc10 +[2023-03-07 16:23:36,745][488352] Starting process rollout_proc16 +[2023-03-07 16:23:36,749][488352] Starting process rollout_proc28 +[2023-03-07 16:23:36,760][488352] Starting process rollout_proc22 +[2023-03-07 16:23:36,771][488352] Starting process rollout_proc46 +[2023-03-07 16:23:36,775][488352] Starting process rollout_proc37 +[2023-03-07 16:23:36,787][488352] Starting process rollout_proc31 +[2023-03-07 16:23:36,790][488352] Starting process rollout_proc34 +[2023-03-07 16:23:36,807][488352] Starting process rollout_proc43 +[2023-03-07 16:23:36,816][488352] Starting process rollout_proc40 +[2023-03-07 16:23:36,822][488352] Starting process rollout_proc47 +[2023-03-07 16:23:36,833][488352] Starting process rollout_proc50 +[2023-03-07 16:23:36,839][488352] Starting process rollout_proc53 +[2023-03-07 16:23:36,850][488352] Starting process rollout_proc56 +[2023-03-07 16:23:36,858][488352] Starting process rollout_proc59 +[2023-03-07 16:23:36,863][488352] Starting process rollout_proc62 +[2023-03-07 16:23:36,878][488352] Starting process rollout_proc65 +[2023-03-07 16:23:36,880][488352] Starting process rollout_proc68 +[2023-03-07 16:23:36,901][488352] Starting process rollout_proc71 +[2023-03-07 16:23:36,901][488352] Starting process rollout_proc74 +[2023-03-07 16:23:36,902][488352] Starting process rollout_proc77 +[2023-03-07 16:23:36,911][488352] Starting process rollout_proc80 +[2023-03-07 16:23:36,925][488352] Starting process rollout_proc83 +[2023-03-07 16:23:36,931][488352] Starting process rollout_proc86 +[2023-03-07 16:23:36,937][488352] Starting process rollout_proc89 +[2023-03-07 16:23:36,948][488352] Starting process rollout_proc48 +[2023-03-07 16:23:36,961][488352] Starting process rollout_proc51 +[2023-03-07 16:23:36,975][488352] Starting process rollout_proc60 +[2023-03-07 16:23:36,976][488352] Starting process rollout_proc54 +[2023-03-07 16:23:37,001][488352] Starting process rollout_proc63 +[2023-03-07 16:23:39,023][488352] Starting process rollout_proc0 +[2023-03-07 16:23:39,076][488352] Starting process rollout_proc1 +[2023-03-07 16:23:39,213][488352] Starting process rollout_proc69 +[2023-03-07 16:23:39,291][488352] Starting process rollout_proc72 +[2023-03-07 16:23:39,302][488352] Starting process rollout_proc57 +[2023-03-07 16:23:39,350][488352] Starting process rollout_proc64 +[2023-03-07 16:23:39,357][488352] Starting process rollout_proc49 +[2023-03-07 16:23:39,358][488352] Starting process rollout_proc90 +[2023-03-07 16:23:39,363][488352] Starting process rollout_proc66 +[2023-03-07 16:23:39,368][488352] Starting process rollout_proc55 +[2023-03-07 16:23:39,402][488352] Starting process rollout_proc84 +[2023-03-07 16:23:39,408][488352] Starting process rollout_proc52 +[2023-03-07 16:23:39,426][488352] Starting process rollout_proc61 +[2023-03-07 16:23:39,434][488352] Starting process rollout_proc78 +[2023-03-07 16:23:39,458][488352] Starting process rollout_proc75 +[2023-03-07 16:23:39,458][488352] Starting process rollout_proc81 +[2023-03-07 16:23:39,485][488352] Starting process rollout_proc87 +[2023-03-07 16:23:40,661][488352] Starting process rollout_proc92 +[2023-03-07 16:23:40,682][488352] Starting process rollout_proc73 +[2023-03-07 16:23:40,735][488352] Starting process rollout_proc70 +[2023-03-07 16:23:40,807][488352] Starting process rollout_proc58 +[2023-03-07 16:23:40,848][488352] Starting process rollout_proc85 +[2023-03-07 16:23:40,857][488352] Starting process rollout_proc79 +[2023-03-07 16:23:40,868][488352] Starting process rollout_proc76 +[2023-03-07 16:23:40,878][488352] Starting process rollout_proc95 +[2023-03-07 16:23:40,912][488352] Starting process rollout_proc98 +[2023-03-07 16:23:40,930][488352] Starting process rollout_proc91 +[2023-03-07 16:23:40,940][488352] Starting process rollout_proc101 +[2023-03-07 16:23:40,945][488352] Starting process rollout_proc104 +[2023-03-07 16:23:40,954][488352] Starting process rollout_proc67 +[2023-03-07 16:23:40,962][488352] Starting process rollout_proc88 +[2023-03-07 16:23:40,966][488352] Starting process rollout_proc82 +[2023-03-07 16:23:40,971][488352] Starting process rollout_proc107 +[2023-03-07 16:23:42,022][488352] Starting process rollout_proc93 +[2023-03-07 16:23:42,035][488352] Starting process rollout_proc110 +[2023-03-07 16:23:42,058][488352] Starting process rollout_proc113 +[2023-03-07 16:23:42,068][488352] Starting process rollout_proc116 +[2023-03-07 16:23:42,081][488352] Starting process rollout_proc119 +[2023-03-07 16:23:42,184][488352] Starting process rollout_proc122 +[2023-03-07 16:23:42,253][488352] Starting process rollout_proc96 +[2023-03-07 16:23:42,310][488352] Starting process rollout_proc125 +[2023-03-07 16:23:42,311][488352] Starting process rollout_proc99 +[2023-03-07 16:23:42,399][488352] Starting process rollout_proc105 +[2023-03-07 16:23:42,408][488352] Starting process rollout_proc102 +[2023-03-07 16:23:42,452][488352] Starting process rollout_proc108 +[2023-03-07 16:23:43,251][488352] Starting process rollout_proc94 +[2023-03-07 16:23:43,299][488352] Starting process rollout_proc114 +[2023-03-07 16:23:43,321][488352] Starting process rollout_proc111 +[2023-03-07 16:23:43,338][488352] Starting process rollout_proc120 +[2023-03-07 16:23:43,397][488352] Starting process rollout_proc117 +[2023-03-07 16:23:43,488][488352] Starting process rollout_proc123 +[2023-03-07 16:23:43,565][488352] Starting process rollout_proc100 +[2023-03-07 16:23:43,566][488352] Starting process rollout_proc97 +[2023-03-07 16:23:43,613][488352] Starting process rollout_proc126 +[2023-03-07 16:23:43,736][488352] Starting process rollout_proc103 +[2023-03-07 16:23:43,753][488352] Starting process rollout_proc106 +[2023-03-07 16:23:43,847][488352] Starting process rollout_proc109 +[2023-03-07 16:23:44,469][488352] Starting process rollout_proc115 +[2023-03-07 16:23:44,567][488352] Starting process rollout_proc121 +[2023-03-07 16:23:44,575][488352] Starting process rollout_proc112 +[2023-03-07 16:23:44,689][488352] Starting process rollout_proc118 +[2023-03-07 16:23:44,828][488352] Starting process rollout_proc124 +[2023-03-07 16:23:44,937][488352] Starting process rollout_proc127 +[2023-03-07 16:23:46,091][488352] Inference worker 0-0 is ready! +[2023-03-07 16:23:46,093][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 16:23:46,099][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 20008960. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:46,118][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:46,695][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:51,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:23:55,225][488352] Heartbeat connected on Batcher_0 +[2023-03-07 16:23:55,227][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 16:23:55,266][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 16:23:56,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:01,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:06,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:11,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:16,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:21,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:26,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:31,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:36,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:41,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:46,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:51,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:24:56,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:01,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:06,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:11,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:16,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:21,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:22,646][488352] Heartbeat connected on RolloutWorker_w68 +[2023-03-07 16:25:25,060][488352] Heartbeat connected on RolloutWorker_w16 +[2023-03-07 16:25:26,692][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20008960. Throughput: 0: 88.5. Samples: 3984. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:25:26,693][488352] Avg episode reward: [(0, '2.858')] +[2023-03-07 16:25:27,073][488352] Heartbeat connected on RolloutWorker_w24 +[2023-03-07 16:25:27,110][488352] Heartbeat connected on RolloutWorker_w60 +[2023-03-07 16:25:27,235][488352] Heartbeat connected on RolloutWorker_w32 +[2023-03-07 16:25:27,307][488352] Component Batcher_0 stopped! +[2023-03-07 16:25:27,356][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 16:25:27,458][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 16:25:27,464][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 16:25:27,466][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 16:25:27,468][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 16:25:27,481][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 16:25:27,488][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 16:25:27,951][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 16:25:29,552][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 16:25:29,834][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 16:25:33,181][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 16:25:34,147][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 16:25:34,715][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 16:25:35,282][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 16:25:35,696][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 16:25:36,228][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 16:25:36,273][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 16:25:37,663][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 16:25:38,521][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 16:25:38,919][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 16:25:39,697][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 16:25:39,724][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 16:25:39,801][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 16:25:40,457][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 16:25:40,848][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 16:25:40,900][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 16:25:40,915][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 16:25:41,250][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 16:25:42,469][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 16:25:42,660][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 16:25:42,732][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 16:25:43,047][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 16:25:43,091][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 16:25:43,158][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 16:25:43,254][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 16:25:43,273][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 16:25:43,623][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 16:25:44,458][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 16:25:44,749][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 16:25:45,067][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 16:25:45,389][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 16:25:45,513][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 16:25:45,545][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 16:25:45,750][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 16:25:46,093][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 16:25:46,480][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 16:25:46,609][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 16:25:47,146][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 16:25:47,805][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 16:25:47,918][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 16:25:48,092][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 16:25:48,316][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 16:25:48,488][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 16:25:48,670][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 16:25:49,260][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 16:25:49,289][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 16:25:49,420][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 16:25:49,895][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 16:25:51,334][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 16:25:51,561][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 16:25:51,874][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 16:25:52,043][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 16:25:52,125][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 16:25:52,152][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 16:25:52,164][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 16:25:52,319][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 16:25:52,367][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 16:25:53,270][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 16:25:53,515][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 16:25:53,662][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 16:25:54,013][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 16:25:54,341][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 16:25:54,819][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 16:25:55,123][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 16:25:55,163][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 16:25:55,562][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 16:25:55,655][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 16:25:55,949][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 16:25:56,032][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 16:25:56,390][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 16:25:56,637][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 16:25:56,924][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 16:25:57,318][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 16:25:57,358][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 16:25:57,438][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 16:25:57,478][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 16:25:57,943][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 16:25:58,175][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 16:25:58,259][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 16:25:58,384][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 16:25:58,474][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 16:25:58,475][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 16:25:59,213][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 16:25:59,273][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 16:25:59,296][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 16:25:59,496][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 16:25:59,715][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 16:25:59,847][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 16:26:00,323][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 16:26:00,572][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 16:26:00,694][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 16:26:01,114][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 16:26:01,274][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 16:26:01,929][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 16:26:02,006][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 16:26:02,385][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 16:26:02,600][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 16:26:03,154][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 16:26:03,224][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 16:26:03,249][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 16:26:03,328][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 16:26:03,407][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 16:26:03,645][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 16:26:03,949][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 16:26:04,067][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 16:26:04,175][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 16:26:04,475][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 16:26:04,917][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 16:26:05,108][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 16:26:05,150][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 16:26:05,193][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 16:26:05,660][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 16:26:05,979][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 16:26:06,520][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 16:26:06,914][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 16:26:07,111][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 16:26:07,410][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 16:26:08,533][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 16:26:10,759][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 16:26:11,506][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 16:26:11,507][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 16:26:11,509][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 16:26:11,511][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 16:26:11,513][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 16:26:11,515][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 16:26:11,517][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 16:26:11,519][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 16:26:11,520][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 16:26:11,522][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 16:26:11,524][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 16:26:11,526][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 16:26:11,527][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 16:26:11,529][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 16:26:11,534][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 16:26:11,536][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 16:26:11,538][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 16:26:11,540][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 16:26:11,542][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 16:26:11,544][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 16:26:11,546][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 16:26:11,548][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 16:26:11,549][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 16:26:11,551][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 16:26:11,553][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 16:26:11,557][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 16:26:11,559][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 16:26:11,561][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 16:26:11,566][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 16:26:11,568][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 16:26:11,570][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 16:26:11,572][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 16:26:11,574][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 16:26:11,576][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 16:26:11,578][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 16:26:11,580][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 16:26:11,582][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 16:26:11,584][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 16:26:11,586][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 16:26:11,588][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 16:26:11,589][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 16:26:11,591][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 16:26:11,593][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 16:26:11,595][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 16:26:11,597][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 16:26:11,599][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 16:26:11,601][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 16:26:11,605][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 16:26:11,607][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 16:26:11,609][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 16:26:11,610][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 16:26:11,612][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 16:26:11,614][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 16:26:11,616][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 16:26:11,618][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 16:26:11,620][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 16:26:11,622][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 16:26:11,626][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 16:26:11,628][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 16:26:11,629][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 16:26:11,631][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 16:26:11,632][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 16:26:11,633][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 16:26:11,635][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 16:26:11,637][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 16:26:11,638][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 16:26:11,640][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 16:26:11,641][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 16:26:11,643][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 16:26:11,644][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 16:26:11,646][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 16:26:11,650][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 16:26:11,651][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 16:26:11,653][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 16:26:11,658][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 16:26:11,660][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 16:26:11,661][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 16:26:11,662][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 16:26:11,664][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 16:26:11,665][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 16:26:11,667][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 16:26:11,668][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 16:26:11,669][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 16:26:11,671][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 16:26:11,673][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 16:26:11,674][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 16:26:11,676][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 16:26:11,677][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 16:26:11,681][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 16:26:11,683][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 16:26:11,684][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 16:26:11,685][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 16:26:11,687][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 16:26:11,688][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 16:26:11,690][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 16:26:11,691][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 16:26:11,693][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 16:26:11,695][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 16:26:11,697][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 16:26:11,699][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 16:26:11,700][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 16:26:11,702][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 16:26:11,704][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 16:26:11,705][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 16:26:11,707][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 16:26:11,709][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 16:26:11,711][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 16:26:11,714][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 16:26:11,715][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 16:26:11,717][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 16:26:11,719][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 16:26:11,721][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 16:26:11,722][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 16:26:11,724][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 16:26:11,725][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 16:26:11,727][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 16:26:11,729][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 16:26:11,731][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 16:26:11,733][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 16:26:11,735][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 16:26:11,736][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 16:26:11,738][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 16:26:11,739][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 16:26:11,741][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 16:26:11,742][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 16:26:11,744][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 16:26:11,745][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 16:26:11,747][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 16:26:11,748][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 16:26:11,750][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 16:26:11,753][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 16:26:11,755][488352] Batcher 0 profile tree view: +batching: 0.0246, releasing_batches: 0.0000 +[2023-03-07 16:26:11,755][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0051 + wait_policy_total: 104.3917 +update_model: 0.0076 + weight_update: 0.0014 +one_step: 0.0023 + handle_policy_step: 0.9133 + deserialize: 0.0438, stack: 0.0024, obs_to_device_normalize: 0.1436, forward: 0.6238, send_messages: 0.0139 + prepare_outputs: 0.0688 + to_cpu: 0.0462 +[2023-03-07 16:26:11,756][488352] Learner 0 profile tree view: +misc: 0.0000, prepare_batch: 0.6562 +train: 0.5419 + epoch_init: 0.0000, minibatch_init: 0.0000, losses_postprocess: 0.0003, kl_divergence: 0.0005, after_optimizer: 0.0122 + calculate_losses: 0.1453 + losses_init: 0.0000, forward_head: 0.1337, bptt_initial: 0.0064, tail: 0.0017, advantages_returns: 0.0006, losses: 0.0016 + bptt: 0.0010 + bptt_forward_core: 0.0010 + update: 0.3833 + clip: 0.0015 +[2023-03-07 16:26:11,757][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0007, enqueue_policy_requests: 0.0011 +[2023-03-07 16:26:11,757][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0007, enqueue_policy_requests: 0.0011 +[2023-03-07 16:26:11,758][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 16:26:11,760][488352] Runner profile tree view: +main_loop: 156.2079 +[2023-03-07 16:26:11,761][488352] Collected {0: 20025344}, FPS: 104.9 +[2023-03-07 16:26:11,768][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:26:11,769][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:26:11,770][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:26:11,770][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:26:11,771][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:26:11,772][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:26:11,772][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:26:11,773][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 16:26:11,773][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 16:26:11,774][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:26:11,774][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:26:11,775][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:26:11,776][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:26:11,776][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:26:11,782][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:26:11,783][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:26:11,792][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:26:11,823][488352] Conv encoder output size: 512 +[2023-03-07 16:26:11,824][488352] Policy head output size: 512 +[2023-03-07 16:26:11,856][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/checkpoint_p0/checkpoint_000004886_20025344.pth... +[2023-03-07 16:26:12,245][488352] Num frames 100... +[2023-03-07 16:26:12,333][488352] Num frames 200... +[2023-03-07 16:26:12,420][488352] Num frames 300... +[2023-03-07 16:26:12,506][488352] Num frames 400... +[2023-03-07 16:26:12,597][488352] Num frames 500... +[2023-03-07 16:26:12,683][488352] Num frames 600... +[2023-03-07 16:26:12,800][488352] Avg episode rewards: #0: 15.720, true rewards: #0: 6.720 +[2023-03-07 16:26:12,801][488352] Avg episode reward: 15.720, avg true_objective: 6.720 +[2023-03-07 16:26:12,826][488352] Num frames 700... +[2023-03-07 16:26:12,911][488352] Num frames 800... +[2023-03-07 16:26:12,996][488352] Num frames 900... +[2023-03-07 16:26:13,082][488352] Num frames 1000... +[2023-03-07 16:26:13,167][488352] Num frames 1100... +[2023-03-07 16:26:13,256][488352] Num frames 1200... +[2023-03-07 16:26:13,323][488352] Avg episode rewards: #0: 12.580, true rewards: #0: 6.080 +[2023-03-07 16:26:13,324][488352] Avg episode reward: 12.580, avg true_objective: 6.080 +[2023-03-07 16:26:13,401][488352] Num frames 1300... +[2023-03-07 16:26:13,487][488352] Num frames 1400... +[2023-03-07 16:26:13,573][488352] Num frames 1500... +[2023-03-07 16:26:13,660][488352] Num frames 1600... +[2023-03-07 16:26:13,745][488352] Num frames 1700... +[2023-03-07 16:26:13,833][488352] Num frames 1800... +[2023-03-07 16:26:13,919][488352] Num frames 1900... +[2023-03-07 16:26:14,004][488352] Num frames 2000... +[2023-03-07 16:26:14,090][488352] Num frames 2100... +[2023-03-07 16:26:14,176][488352] Num frames 2200... +[2023-03-07 16:26:14,263][488352] Num frames 2300... +[2023-03-07 16:26:14,351][488352] Num frames 2400... +[2023-03-07 16:26:14,438][488352] Num frames 2500... +[2023-03-07 16:26:14,523][488352] Num frames 2600... +[2023-03-07 16:26:14,611][488352] Num frames 2700... +[2023-03-07 16:26:14,697][488352] Num frames 2800... +[2023-03-07 16:26:14,784][488352] Num frames 2900... +[2023-03-07 16:26:14,872][488352] Num frames 3000... +[2023-03-07 16:26:14,963][488352] Num frames 3100... +[2023-03-07 16:26:15,051][488352] Num frames 3200... +[2023-03-07 16:26:15,137][488352] Num frames 3300... +[2023-03-07 16:26:15,205][488352] Avg episode rewards: #0: 25.386, true rewards: #0: 11.053 +[2023-03-07 16:26:15,206][488352] Avg episode reward: 25.386, avg true_objective: 11.053 +[2023-03-07 16:26:15,299][488352] Num frames 3400... +[2023-03-07 16:26:15,386][488352] Num frames 3500... +[2023-03-07 16:26:15,472][488352] Num frames 3600... +[2023-03-07 16:26:15,560][488352] Num frames 3700... +[2023-03-07 16:26:15,646][488352] Num frames 3800... +[2023-03-07 16:26:15,731][488352] Num frames 3900... +[2023-03-07 16:26:15,816][488352] Num frames 4000... +[2023-03-07 16:26:15,916][488352] Avg episode rewards: #0: 22.380, true rewards: #0: 10.130 +[2023-03-07 16:26:15,917][488352] Avg episode reward: 22.380, avg true_objective: 10.130 +[2023-03-07 16:26:15,958][488352] Num frames 4100... +[2023-03-07 16:26:16,043][488352] Num frames 4200... +[2023-03-07 16:26:16,128][488352] Num frames 4300... +[2023-03-07 16:26:16,214][488352] Num frames 4400... +[2023-03-07 16:26:16,301][488352] Num frames 4500... +[2023-03-07 16:26:16,387][488352] Num frames 4600... +[2023-03-07 16:26:16,474][488352] Num frames 4700... +[2023-03-07 16:26:16,563][488352] Num frames 4800... +[2023-03-07 16:26:16,653][488352] Num frames 4900... +[2023-03-07 16:26:16,743][488352] Num frames 5000... +[2023-03-07 16:26:16,831][488352] Num frames 5100... +[2023-03-07 16:26:16,920][488352] Num frames 5200... +[2023-03-07 16:26:17,009][488352] Num frames 5300... +[2023-03-07 16:26:17,098][488352] Num frames 5400... +[2023-03-07 16:26:17,190][488352] Num frames 5500... +[2023-03-07 16:26:17,280][488352] Num frames 5600... +[2023-03-07 16:26:17,410][488352] Avg episode rewards: #0: 24.968, true rewards: #0: 11.368 +[2023-03-07 16:26:17,411][488352] Avg episode reward: 24.968, avg true_objective: 11.368 +[2023-03-07 16:26:17,432][488352] Num frames 5700... +[2023-03-07 16:26:17,536][488352] Num frames 5800... +[2023-03-07 16:26:17,627][488352] Num frames 5900... +[2023-03-07 16:26:17,717][488352] Num frames 6000... +[2023-03-07 16:26:17,805][488352] Num frames 6100... +[2023-03-07 16:26:17,894][488352] Num frames 6200... +[2023-03-07 16:26:17,983][488352] Num frames 6300... +[2023-03-07 16:26:18,074][488352] Num frames 6400... +[2023-03-07 16:26:18,174][488352] Avg episode rewards: #0: 22.920, true rewards: #0: 10.753 +[2023-03-07 16:26:18,175][488352] Avg episode reward: 22.920, avg true_objective: 10.753 +[2023-03-07 16:26:18,240][488352] Num frames 6500... +[2023-03-07 16:26:18,328][488352] Num frames 6600... +[2023-03-07 16:26:18,417][488352] Num frames 6700... +[2023-03-07 16:26:18,506][488352] Num frames 6800... +[2023-03-07 16:26:18,595][488352] Num frames 6900... +[2023-03-07 16:26:18,686][488352] Num frames 7000... +[2023-03-07 16:26:18,776][488352] Num frames 7100... +[2023-03-07 16:26:18,868][488352] Num frames 7200... +[2023-03-07 16:26:18,957][488352] Num frames 7300... +[2023-03-07 16:26:19,047][488352] Num frames 7400... +[2023-03-07 16:26:19,137][488352] Num frames 7500... +[2023-03-07 16:26:19,227][488352] Num frames 7600... +[2023-03-07 16:26:19,317][488352] Num frames 7700... +[2023-03-07 16:26:19,406][488352] Num frames 7800... +[2023-03-07 16:26:19,493][488352] Num frames 7900... +[2023-03-07 16:26:19,584][488352] Num frames 8000... +[2023-03-07 16:26:19,672][488352] Num frames 8100... +[2023-03-07 16:26:19,760][488352] Num frames 8200... +[2023-03-07 16:26:19,825][488352] Avg episode rewards: #0: 25.303, true rewards: #0: 11.731 +[2023-03-07 16:26:19,826][488352] Avg episode reward: 25.303, avg true_objective: 11.731 +[2023-03-07 16:26:19,921][488352] Num frames 8300... +[2023-03-07 16:26:20,011][488352] Num frames 8400... +[2023-03-07 16:26:20,102][488352] Num frames 8500... +[2023-03-07 16:26:20,192][488352] Num frames 8600... +[2023-03-07 16:26:20,282][488352] Num frames 8700... +[2023-03-07 16:26:20,357][488352] Avg episode rewards: #0: 23.155, true rewards: #0: 10.905 +[2023-03-07 16:26:20,358][488352] Avg episode reward: 23.155, avg true_objective: 10.905 +[2023-03-07 16:26:20,446][488352] Num frames 8800... +[2023-03-07 16:26:20,535][488352] Num frames 8900... +[2023-03-07 16:26:20,623][488352] Num frames 9000... +[2023-03-07 16:26:20,713][488352] Num frames 9100... +[2023-03-07 16:26:20,803][488352] Num frames 9200... +[2023-03-07 16:26:20,891][488352] Num frames 9300... +[2023-03-07 16:26:20,979][488352] Num frames 9400... +[2023-03-07 16:26:21,069][488352] Num frames 9500... +[2023-03-07 16:26:21,159][488352] Num frames 9600... +[2023-03-07 16:26:21,250][488352] Num frames 9700... +[2023-03-07 16:26:21,319][488352] Avg episode rewards: #0: 22.795, true rewards: #0: 10.796 +[2023-03-07 16:26:21,320][488352] Avg episode reward: 22.795, avg true_objective: 10.796 +[2023-03-07 16:26:21,414][488352] Num frames 9800... +[2023-03-07 16:26:21,505][488352] Num frames 9900... +[2023-03-07 16:26:21,597][488352] Num frames 10000... +[2023-03-07 16:26:21,677][488352] Num frames 10100... +[2023-03-07 16:26:21,758][488352] Num frames 10200... +[2023-03-07 16:26:21,842][488352] Num frames 10300... +[2023-03-07 16:26:21,930][488352] Num frames 10400... +[2023-03-07 16:26:22,020][488352] Num frames 10500... +[2023-03-07 16:26:22,110][488352] Num frames 10600... +[2023-03-07 16:26:22,235][488352] Avg episode rewards: #0: 22.576, true rewards: #0: 10.676 +[2023-03-07 16:26:22,236][488352] Avg episode reward: 22.576, avg true_objective: 10.676 +[2023-03-07 16:26:35,551][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/replay.mp4! +[2023-03-07 16:26:35,947][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 16:26:35,948][488352] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 16:26:35,949][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:26:35,950][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:26:35,950][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:26:35,951][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:26:35,952][488352] Adding new argument 'max_num_frames'=100000 that is not in the saved config file! +[2023-03-07 16:26:35,952][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:26:35,953][488352] Adding new argument 'push_to_hub'=True that is not in the saved config file! +[2023-03-07 16:26:35,953][488352] Adding new argument 'hf_repository'='Rolo/rl_course_vizdoom_health_gathering_supreme' that is not in the saved config file! +[2023-03-07 16:26:35,954][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:26:35,954][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:26:35,955][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:26:35,956][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:26:35,956][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:26:35,963][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:26:35,964][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:26:35,974][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:26:36,004][488352] Conv encoder output size: 512 +[2023-03-07 16:26:36,006][488352] Policy head output size: 512 +[2023-03-07 16:26:36,018][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 16:26:36,558][488352] Num frames 100... +[2023-03-07 16:26:36,645][488352] Num frames 200... +[2023-03-07 16:26:36,732][488352] Num frames 300... +[2023-03-07 16:26:36,817][488352] Num frames 400... +[2023-03-07 16:26:36,903][488352] Num frames 500... +[2023-03-07 16:26:36,992][488352] Num frames 600... +[2023-03-07 16:26:37,081][488352] Num frames 700... +[2023-03-07 16:26:37,169][488352] Num frames 800... +[2023-03-07 16:26:37,257][488352] Num frames 900... +[2023-03-07 16:26:37,364][488352] Avg episode rewards: #0: 17.600, true rewards: #0: 9.600 +[2023-03-07 16:26:37,365][488352] Avg episode reward: 17.600, avg true_objective: 9.600 +[2023-03-07 16:26:37,399][488352] Num frames 1000... +[2023-03-07 16:26:37,483][488352] Num frames 1100... +[2023-03-07 16:26:37,569][488352] Num frames 1200... +[2023-03-07 16:26:37,657][488352] Num frames 1300... +[2023-03-07 16:26:37,722][488352] Avg episode rewards: #0: 11.060, true rewards: #0: 6.560 +[2023-03-07 16:26:37,723][488352] Avg episode reward: 11.060, avg true_objective: 6.560 +[2023-03-07 16:26:37,820][488352] Num frames 1400... +[2023-03-07 16:26:37,905][488352] Num frames 1500... +[2023-03-07 16:26:37,990][488352] Num frames 1600... +[2023-03-07 16:26:38,077][488352] Num frames 1700... +[2023-03-07 16:26:38,184][488352] Avg episode rewards: #0: 9.200, true rewards: #0: 5.867 +[2023-03-07 16:26:38,185][488352] Avg episode reward: 9.200, avg true_objective: 5.867 +[2023-03-07 16:26:38,234][488352] Num frames 1800... +[2023-03-07 16:26:38,320][488352] Num frames 1900... +[2023-03-07 16:26:38,407][488352] Num frames 2000... +[2023-03-07 16:26:38,494][488352] Num frames 2100... +[2023-03-07 16:26:38,580][488352] Num frames 2200... +[2023-03-07 16:26:38,664][488352] Num frames 2300... +[2023-03-07 16:26:38,778][488352] Avg episode rewards: #0: 9.170, true rewards: #0: 5.920 +[2023-03-07 16:26:38,779][488352] Avg episode reward: 9.170, avg true_objective: 5.920 +[2023-03-07 16:26:38,821][488352] Num frames 2400... +[2023-03-07 16:26:38,914][488352] Num frames 2500... +[2023-03-07 16:26:39,001][488352] Num frames 2600... +[2023-03-07 16:26:39,089][488352] Num frames 2700... +[2023-03-07 16:26:39,177][488352] Num frames 2800... +[2023-03-07 16:26:39,245][488352] Avg episode rewards: #0: 8.432, true rewards: #0: 5.632 +[2023-03-07 16:26:39,246][488352] Avg episode reward: 8.432, avg true_objective: 5.632 +[2023-03-07 16:26:39,336][488352] Num frames 2900... +[2023-03-07 16:26:39,422][488352] Num frames 3000... +[2023-03-07 16:26:39,507][488352] Num frames 3100... +[2023-03-07 16:26:39,593][488352] Num frames 3200... +[2023-03-07 16:26:39,677][488352] Num frames 3300... +[2023-03-07 16:26:39,764][488352] Num frames 3400... +[2023-03-07 16:26:39,850][488352] Num frames 3500... +[2023-03-07 16:26:39,922][488352] Avg episode rewards: #0: 8.867, true rewards: #0: 5.867 +[2023-03-07 16:26:39,924][488352] Avg episode reward: 8.867, avg true_objective: 5.867 +[2023-03-07 16:26:40,008][488352] Num frames 3600... +[2023-03-07 16:26:40,094][488352] Num frames 3700... +[2023-03-07 16:26:40,181][488352] Num frames 3800... +[2023-03-07 16:26:40,268][488352] Num frames 3900... +[2023-03-07 16:26:40,326][488352] Avg episode rewards: #0: 8.149, true rewards: #0: 5.577 +[2023-03-07 16:26:40,327][488352] Avg episode reward: 8.149, avg true_objective: 5.577 +[2023-03-07 16:26:40,427][488352] Num frames 4000... +[2023-03-07 16:26:40,512][488352] Num frames 4100... +[2023-03-07 16:26:40,597][488352] Num frames 4200... +[2023-03-07 16:26:40,727][488352] Avg episode rewards: #0: 7.610, true rewards: #0: 5.360 +[2023-03-07 16:26:40,729][488352] Avg episode reward: 7.610, avg true_objective: 5.360 +[2023-03-07 16:26:40,744][488352] Num frames 4300... +[2023-03-07 16:26:40,842][488352] Num frames 4400... +[2023-03-07 16:26:40,930][488352] Num frames 4500... +[2023-03-07 16:26:41,017][488352] Num frames 4600... +[2023-03-07 16:26:41,105][488352] Avg episode rewards: #0: 7.267, true rewards: #0: 5.156 +[2023-03-07 16:26:41,106][488352] Avg episode reward: 7.267, avg true_objective: 5.156 +[2023-03-07 16:26:41,160][488352] Num frames 4700... +[2023-03-07 16:26:41,247][488352] Num frames 4800... +[2023-03-07 16:26:41,337][488352] Num frames 4900... +[2023-03-07 16:26:41,425][488352] Num frames 5000... +[2023-03-07 16:26:41,514][488352] Num frames 5100... +[2023-03-07 16:26:41,586][488352] Avg episode rewards: #0: 7.420, true rewards: #0: 5.120 +[2023-03-07 16:26:41,587][488352] Avg episode reward: 7.420, avg true_objective: 5.120 +[2023-03-07 16:26:47,762][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 16:26:56,400][488352] The model has been pushed to https://huggingface.co/Rolo/rl_course_vizdoom_health_gathering_supreme +[2023-03-07 16:34:21,132][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 16:34:21,134][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 16:34:21,135][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 16:34:21,136][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 16:34:21,137][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 16:34:21,138][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 16:34:21,139][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 16:34:21,140][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 16:34:21,140][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 16:34:21,141][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 16:34:21,142][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 16:34:21,143][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 16:34:21,144][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 16:34:21,144][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 16:34:21,146][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 16:34:21,146][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 16:34:21,147][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 16:34:21,148][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 16:34:21,149][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 16:34:21,149][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 16:34:21,151][488352] register_encoder_factory: +[2023-03-07 16:34:22,878][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:34:22,879][488352] Overriding arg 'num_envs_per_worker' with value 64 passed from command line +[2023-03-07 16:34:22,880][488352] Overriding arg 'batch_size' with value 8192 passed from command line +[2023-03-07 16:34:22,881][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads already exists! +[2023-03-07 16:34:22,881][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads... +[2023-03-07 16:34:22,882][488352] Weights and Biases integration disabled +[2023-03-07 16:34:22,884][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 16:34:24,033][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=64 +batch_size=8192 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=4000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=16 --train_for_env_steps=4000000 --experiment=doom_health_threads +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads', 'num_workers': 128, 'num_envs_per_worker': 16, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 16:34:24,034][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json... +[2023-03-07 16:34:24,035][488352] Rollout worker 0 uses device cpu +[2023-03-07 16:34:24,036][488352] Rollout worker 1 uses device cpu +[2023-03-07 16:34:24,036][488352] Rollout worker 2 uses device cpu +[2023-03-07 16:34:24,036][488352] Rollout worker 3 uses device cpu +[2023-03-07 16:34:24,037][488352] Rollout worker 4 uses device cpu +[2023-03-07 16:34:24,037][488352] Rollout worker 5 uses device cpu +[2023-03-07 16:34:24,038][488352] Rollout worker 6 uses device cpu +[2023-03-07 16:34:24,038][488352] Rollout worker 7 uses device cpu +[2023-03-07 16:34:24,038][488352] Rollout worker 8 uses device cpu +[2023-03-07 16:34:24,039][488352] Rollout worker 9 uses device cpu +[2023-03-07 16:34:24,040][488352] Rollout worker 10 uses device cpu +[2023-03-07 16:34:24,040][488352] Rollout worker 11 uses device cpu +[2023-03-07 16:34:24,041][488352] Rollout worker 12 uses device cpu +[2023-03-07 16:34:24,041][488352] Rollout worker 13 uses device cpu +[2023-03-07 16:34:24,042][488352] Rollout worker 14 uses device cpu +[2023-03-07 16:34:24,042][488352] Rollout worker 15 uses device cpu +[2023-03-07 16:34:24,043][488352] Rollout worker 16 uses device cpu +[2023-03-07 16:34:24,043][488352] Rollout worker 17 uses device cpu +[2023-03-07 16:34:24,044][488352] Rollout worker 18 uses device cpu +[2023-03-07 16:34:24,044][488352] Rollout worker 19 uses device cpu +[2023-03-07 16:34:24,044][488352] Rollout worker 20 uses device cpu +[2023-03-07 16:34:24,045][488352] Rollout worker 21 uses device cpu +[2023-03-07 16:34:24,045][488352] Rollout worker 22 uses device cpu +[2023-03-07 16:34:24,046][488352] Rollout worker 23 uses device cpu +[2023-03-07 16:34:24,047][488352] Rollout worker 24 uses device cpu +[2023-03-07 16:34:24,047][488352] Rollout worker 25 uses device cpu +[2023-03-07 16:34:24,047][488352] Rollout worker 26 uses device cpu +[2023-03-07 16:34:24,048][488352] Rollout worker 27 uses device cpu +[2023-03-07 16:34:24,048][488352] Rollout worker 28 uses device cpu +[2023-03-07 16:34:24,049][488352] Rollout worker 29 uses device cpu +[2023-03-07 16:34:24,049][488352] Rollout worker 30 uses device cpu +[2023-03-07 16:34:24,049][488352] Rollout worker 31 uses device cpu +[2023-03-07 16:34:24,050][488352] Rollout worker 32 uses device cpu +[2023-03-07 16:34:24,051][488352] Rollout worker 33 uses device cpu +[2023-03-07 16:34:24,051][488352] Rollout worker 34 uses device cpu +[2023-03-07 16:34:24,052][488352] Rollout worker 35 uses device cpu +[2023-03-07 16:34:24,052][488352] Rollout worker 36 uses device cpu +[2023-03-07 16:34:24,052][488352] Rollout worker 37 uses device cpu +[2023-03-07 16:34:24,053][488352] Rollout worker 38 uses device cpu +[2023-03-07 16:34:24,053][488352] Rollout worker 39 uses device cpu +[2023-03-07 16:34:24,054][488352] Rollout worker 40 uses device cpu +[2023-03-07 16:34:24,054][488352] Rollout worker 41 uses device cpu +[2023-03-07 16:34:24,055][488352] Rollout worker 42 uses device cpu +[2023-03-07 16:34:24,055][488352] Rollout worker 43 uses device cpu +[2023-03-07 16:34:24,056][488352] Rollout worker 44 uses device cpu +[2023-03-07 16:34:24,056][488352] Rollout worker 45 uses device cpu +[2023-03-07 16:34:24,057][488352] Rollout worker 46 uses device cpu +[2023-03-07 16:34:24,057][488352] Rollout worker 47 uses device cpu +[2023-03-07 16:34:24,057][488352] Rollout worker 48 uses device cpu +[2023-03-07 16:34:24,058][488352] Rollout worker 49 uses device cpu +[2023-03-07 16:34:24,059][488352] Rollout worker 50 uses device cpu +[2023-03-07 16:34:24,060][488352] Rollout worker 51 uses device cpu +[2023-03-07 16:34:24,060][488352] Rollout worker 52 uses device cpu +[2023-03-07 16:34:24,060][488352] Rollout worker 53 uses device cpu +[2023-03-07 16:34:24,061][488352] Rollout worker 54 uses device cpu +[2023-03-07 16:34:24,061][488352] Rollout worker 55 uses device cpu +[2023-03-07 16:34:24,062][488352] Rollout worker 56 uses device cpu +[2023-03-07 16:34:24,062][488352] Rollout worker 57 uses device cpu +[2023-03-07 16:34:24,063][488352] Rollout worker 58 uses device cpu +[2023-03-07 16:34:24,063][488352] Rollout worker 59 uses device cpu +[2023-03-07 16:34:24,064][488352] Rollout worker 60 uses device cpu +[2023-03-07 16:34:24,064][488352] Rollout worker 61 uses device cpu +[2023-03-07 16:34:24,065][488352] Rollout worker 62 uses device cpu +[2023-03-07 16:34:24,065][488352] Rollout worker 63 uses device cpu +[2023-03-07 16:34:24,066][488352] Rollout worker 64 uses device cpu +[2023-03-07 16:34:24,066][488352] Rollout worker 65 uses device cpu +[2023-03-07 16:34:24,067][488352] Rollout worker 66 uses device cpu +[2023-03-07 16:34:24,067][488352] Rollout worker 67 uses device cpu +[2023-03-07 16:34:24,068][488352] Rollout worker 68 uses device cpu +[2023-03-07 16:34:24,068][488352] Rollout worker 69 uses device cpu +[2023-03-07 16:34:24,068][488352] Rollout worker 70 uses device cpu +[2023-03-07 16:34:24,069][488352] Rollout worker 71 uses device cpu +[2023-03-07 16:34:24,069][488352] Rollout worker 72 uses device cpu +[2023-03-07 16:34:24,069][488352] Rollout worker 73 uses device cpu +[2023-03-07 16:34:24,070][488352] Rollout worker 74 uses device cpu +[2023-03-07 16:34:24,071][488352] Rollout worker 75 uses device cpu +[2023-03-07 16:34:24,071][488352] Rollout worker 76 uses device cpu +[2023-03-07 16:34:24,072][488352] Rollout worker 77 uses device cpu +[2023-03-07 16:34:24,072][488352] Rollout worker 78 uses device cpu +[2023-03-07 16:34:24,073][488352] Rollout worker 79 uses device cpu +[2023-03-07 16:34:24,073][488352] Rollout worker 80 uses device cpu +[2023-03-07 16:34:24,073][488352] Rollout worker 81 uses device cpu +[2023-03-07 16:34:24,074][488352] Rollout worker 82 uses device cpu +[2023-03-07 16:34:24,074][488352] Rollout worker 83 uses device cpu +[2023-03-07 16:34:24,074][488352] Rollout worker 84 uses device cpu +[2023-03-07 16:34:24,075][488352] Rollout worker 85 uses device cpu +[2023-03-07 16:34:24,075][488352] Rollout worker 86 uses device cpu +[2023-03-07 16:34:24,076][488352] Rollout worker 87 uses device cpu +[2023-03-07 16:34:24,076][488352] Rollout worker 88 uses device cpu +[2023-03-07 16:34:24,076][488352] Rollout worker 89 uses device cpu +[2023-03-07 16:34:24,077][488352] Rollout worker 90 uses device cpu +[2023-03-07 16:34:24,077][488352] Rollout worker 91 uses device cpu +[2023-03-07 16:34:24,077][488352] Rollout worker 92 uses device cpu +[2023-03-07 16:34:24,078][488352] Rollout worker 93 uses device cpu +[2023-03-07 16:34:24,078][488352] Rollout worker 94 uses device cpu +[2023-03-07 16:34:24,079][488352] Rollout worker 95 uses device cpu +[2023-03-07 16:34:24,079][488352] Rollout worker 96 uses device cpu +[2023-03-07 16:34:24,079][488352] Rollout worker 97 uses device cpu +[2023-03-07 16:34:24,080][488352] Rollout worker 98 uses device cpu +[2023-03-07 16:34:24,080][488352] Rollout worker 99 uses device cpu +[2023-03-07 16:34:24,081][488352] Rollout worker 100 uses device cpu +[2023-03-07 16:34:24,081][488352] Rollout worker 101 uses device cpu +[2023-03-07 16:34:24,081][488352] Rollout worker 102 uses device cpu +[2023-03-07 16:34:24,082][488352] Rollout worker 103 uses device cpu +[2023-03-07 16:34:24,082][488352] Rollout worker 104 uses device cpu +[2023-03-07 16:34:24,083][488352] Rollout worker 105 uses device cpu +[2023-03-07 16:34:24,083][488352] Rollout worker 106 uses device cpu +[2023-03-07 16:34:24,083][488352] Rollout worker 107 uses device cpu +[2023-03-07 16:34:24,084][488352] Rollout worker 108 uses device cpu +[2023-03-07 16:34:24,084][488352] Rollout worker 109 uses device cpu +[2023-03-07 16:34:24,087][488352] Rollout worker 110 uses device cpu +[2023-03-07 16:34:24,088][488352] Rollout worker 111 uses device cpu +[2023-03-07 16:34:24,088][488352] Rollout worker 112 uses device cpu +[2023-03-07 16:34:24,089][488352] Rollout worker 113 uses device cpu +[2023-03-07 16:34:24,089][488352] Rollout worker 114 uses device cpu +[2023-03-07 16:34:24,089][488352] Rollout worker 115 uses device cpu +[2023-03-07 16:34:24,090][488352] Rollout worker 116 uses device cpu +[2023-03-07 16:34:24,091][488352] Rollout worker 117 uses device cpu +[2023-03-07 16:34:24,091][488352] Rollout worker 118 uses device cpu +[2023-03-07 16:34:24,092][488352] Rollout worker 119 uses device cpu +[2023-03-07 16:34:24,092][488352] Rollout worker 120 uses device cpu +[2023-03-07 16:34:24,092][488352] Rollout worker 121 uses device cpu +[2023-03-07 16:34:24,093][488352] Rollout worker 122 uses device cpu +[2023-03-07 16:34:24,093][488352] Rollout worker 123 uses device cpu +[2023-03-07 16:34:24,094][488352] Rollout worker 124 uses device cpu +[2023-03-07 16:34:24,094][488352] Rollout worker 125 uses device cpu +[2023-03-07 16:34:24,094][488352] Rollout worker 126 uses device cpu +[2023-03-07 16:34:24,095][488352] Rollout worker 127 uses device cpu +[2023-03-07 16:34:28,445][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 16:34:28,446][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 16:34:28,720][488352] Starting all processes... +[2023-03-07 16:34:28,721][488352] Starting process learner_proc0 +[2023-03-07 16:34:29,622][488352] Starting all processes... +[2023-03-07 16:34:29,628][488352] Starting process inference_proc0-0 +[2023-03-07 16:34:29,628][488352] Starting process rollout_proc2 +[2023-03-07 16:34:29,628][488352] Starting process rollout_proc5 +[2023-03-07 16:34:29,629][488352] Starting process rollout_proc8 +[2023-03-07 16:34:29,629][488352] Starting process rollout_proc11 +[2023-03-07 16:34:29,630][488352] Starting process rollout_proc14 +[2023-03-07 16:34:29,630][488352] Starting process rollout_proc17 +[2023-03-07 16:34:29,631][488352] Starting process rollout_proc20 +[2023-03-07 16:34:29,638][488352] Starting process rollout_proc23 +[2023-03-07 16:34:29,639][488352] Starting process rollout_proc26 +[2023-03-07 16:34:29,639][488352] Starting process rollout_proc29 +[2023-03-07 16:34:29,641][488352] Starting process rollout_proc32 +[2023-03-07 16:34:29,641][488352] Starting process rollout_proc35 +[2023-03-07 16:34:29,644][488352] Starting process rollout_proc38 +[2023-03-07 16:34:29,644][488352] Starting process rollout_proc41 +[2023-03-07 16:34:29,645][488352] Starting process rollout_proc44 +[2023-03-07 16:34:29,669][488352] Starting process rollout_proc3 +[2023-03-07 16:34:29,677][488352] Starting process rollout_proc6 +[2023-03-07 16:34:29,680][488352] Starting process rollout_proc12 +[2023-03-07 16:34:29,686][488352] Starting process rollout_proc9 +[2023-03-07 16:34:29,693][488352] Starting process rollout_proc15 +[2023-03-07 16:34:29,694][488352] Starting process rollout_proc18 +[2023-03-07 16:34:29,707][488352] Starting process rollout_proc27 +[2023-03-07 16:34:29,708][488352] Starting process rollout_proc24 +[2023-03-07 16:34:29,722][488352] Starting process rollout_proc21 +[2023-03-07 16:34:29,728][488352] Starting process rollout_proc30 +[2023-03-07 16:34:29,737][488352] Starting process rollout_proc33 +[2023-03-07 16:34:29,740][488352] Starting process rollout_proc39 +[2023-03-07 16:34:29,754][488352] Starting process rollout_proc42 +[2023-03-07 16:34:29,767][488352] Starting process rollout_proc4 +[2023-03-07 16:34:29,772][488352] Starting process rollout_proc36 +[2023-03-07 16:34:29,773][488352] Starting process rollout_proc45 +[2023-03-07 16:34:29,778][488352] Starting process rollout_proc7 +[2023-03-07 16:34:29,787][488352] Starting process rollout_proc13 +[2023-03-07 16:34:29,791][488352] Starting process rollout_proc10 +[2023-03-07 16:34:29,792][488352] Starting process rollout_proc16 +[2023-03-07 16:34:29,797][488352] Starting process rollout_proc28 +[2023-03-07 16:34:29,806][488352] Starting process rollout_proc19 +[2023-03-07 16:34:29,810][488352] Starting process rollout_proc22 +[2023-03-07 16:34:29,813][488352] Starting process rollout_proc25 +[2023-03-07 16:34:29,820][488352] Starting process rollout_proc40 +[2023-03-07 16:34:29,822][488352] Starting process rollout_proc31 +[2023-03-07 16:34:29,829][488352] Starting process rollout_proc34 +[2023-03-07 16:34:29,833][488352] Starting process rollout_proc43 +[2023-03-07 16:34:29,839][488352] Starting process rollout_proc47 +[2023-03-07 16:34:29,843][488352] Starting process rollout_proc37 +[2023-03-07 16:34:29,850][488352] Starting process rollout_proc46 +[2023-03-07 16:34:29,860][488352] Starting process rollout_proc50 +[2023-03-07 16:34:29,862][488352] Starting process rollout_proc53 +[2023-03-07 16:34:29,864][488352] Starting process rollout_proc56 +[2023-03-07 16:34:29,868][488352] Starting process rollout_proc59 +[2023-03-07 16:34:29,872][488352] Starting process rollout_proc62 +[2023-03-07 16:34:29,890][488352] Starting process rollout_proc65 +[2023-03-07 16:34:29,891][488352] Starting process rollout_proc68 +[2023-03-07 16:34:29,895][488352] Starting process rollout_proc71 +[2023-03-07 16:34:29,913][488352] Starting process rollout_proc74 +[2023-03-07 16:34:29,923][488352] Starting process rollout_proc77 +[2023-03-07 16:34:29,923][488352] Starting process rollout_proc80 +[2023-03-07 16:34:29,927][488352] Starting process rollout_proc83 +[2023-03-07 16:34:29,937][488352] Starting process rollout_proc86 +[2023-03-07 16:34:29,946][488352] Starting process rollout_proc51 +[2023-03-07 16:34:29,948][488352] Starting process rollout_proc48 +[2023-03-07 16:34:29,959][488352] Starting process rollout_proc89 +[2023-03-07 16:34:29,964][488352] Starting process rollout_proc54 +[2023-03-07 16:34:29,979][488352] Starting process rollout_proc60 +[2023-03-07 16:34:29,979][488352] Starting process rollout_proc57 +[2023-03-07 16:34:29,995][488352] Starting process rollout_proc63 +[2023-03-07 16:34:29,996][488352] Starting process rollout_proc69 +[2023-03-07 16:34:30,007][488352] Starting process rollout_proc66 +[2023-03-07 16:34:30,016][488352] Starting process rollout_proc72 +[2023-03-07 16:34:30,017][488352] Starting process rollout_proc75 +[2023-03-07 16:34:30,024][488352] Starting process rollout_proc78 +[2023-03-07 16:34:30,038][488352] Starting process rollout_proc84 +[2023-03-07 16:34:30,044][488352] Starting process rollout_proc81 +[2023-03-07 16:34:30,050][488352] Starting process rollout_proc52 +[2023-03-07 16:34:30,062][488352] Starting process rollout_proc87 +[2023-03-07 16:34:30,073][488352] Starting process rollout_proc49 +[2023-03-07 16:34:30,089][488352] Starting process rollout_proc90 +[2023-03-07 16:34:30,112][488352] Starting process rollout_proc58 +[2023-03-07 16:34:30,115][488352] Starting process rollout_proc55 +[2023-03-07 16:34:30,121][488352] Starting process rollout_proc64 +[2023-03-07 16:34:30,127][488352] Starting process rollout_proc70 +[2023-03-07 16:34:30,130][488352] Starting process rollout_proc61 +[2023-03-07 16:34:30,135][488352] Starting process rollout_proc67 +[2023-03-07 16:34:30,141][488352] Starting process rollout_proc73 +[2023-03-07 16:34:30,152][488352] Starting process rollout_proc76 +[2023-03-07 16:34:30,154][488352] Starting process rollout_proc82 +[2023-03-07 16:34:30,162][488352] Starting process rollout_proc79 +[2023-03-07 16:34:30,162][488352] Starting process rollout_proc85 +[2023-03-07 16:34:30,173][488352] Starting process rollout_proc92 +[2023-03-07 16:34:30,181][488352] Starting process rollout_proc88 +[2023-03-07 16:34:30,199][488352] Starting process rollout_proc91 +[2023-03-07 16:34:30,251][488352] Starting process rollout_proc95 +[2023-03-07 16:34:30,253][488352] Starting process rollout_proc98 +[2023-03-07 16:34:30,307][488352] Starting process rollout_proc101 +[2023-03-07 16:34:30,339][488352] Starting process rollout_proc104 +[2023-03-07 16:34:30,340][488352] Starting process rollout_proc107 +[2023-03-07 16:34:30,351][488352] Starting process rollout_proc110 +[2023-03-07 16:34:30,374][488352] Starting process rollout_proc113 +[2023-03-07 16:34:30,377][488352] Starting process rollout_proc116 +[2023-03-07 16:34:30,378][488352] Starting process rollout_proc119 +[2023-03-07 16:34:30,399][488352] Starting process rollout_proc122 +[2023-03-07 16:34:30,413][488352] Starting process rollout_proc125 +[2023-03-07 16:34:30,413][488352] Starting process rollout_proc93 +[2023-03-07 16:34:30,439][488352] Starting process rollout_proc105 +[2023-03-07 16:34:30,468][488352] Starting process rollout_proc99 +[2023-03-07 16:34:30,521][488352] Starting process rollout_proc102 +[2023-03-07 16:34:30,575][488352] Starting process rollout_proc108 +[2023-03-07 16:34:30,578][488352] Starting process rollout_proc96 +[2023-03-07 16:34:30,644][488352] Starting process rollout_proc111 +[2023-03-07 16:34:30,676][488352] Starting process rollout_proc117 +[2023-03-07 16:34:30,677][488352] Starting process rollout_proc114 +[2023-03-07 16:34:30,700][488352] Starting process rollout_proc120 +[2023-03-07 16:34:30,758][488352] Starting process rollout_proc123 +[2023-03-07 16:34:30,816][488352] Starting process rollout_proc126 +[2023-03-07 16:34:30,818][488352] Starting process rollout_proc100 +[2023-03-07 16:34:30,826][488352] Starting process rollout_proc109 +[2023-03-07 16:34:30,836][488352] Starting process rollout_proc94 +[2023-03-07 16:34:30,866][488352] Starting process rollout_proc97 +[2023-03-07 16:34:30,866][488352] Starting process rollout_proc112 +[2023-03-07 16:34:30,867][488352] Starting process rollout_proc103 +[2023-03-07 16:34:30,882][488352] Starting process rollout_proc106 +[2023-03-07 16:34:30,930][488352] Starting process rollout_proc118 +[2023-03-07 16:34:30,935][488352] Starting process rollout_proc124 +[2023-03-07 16:34:30,988][488352] Starting process rollout_proc121 +[2023-03-07 16:34:30,995][488352] Starting process rollout_proc115 +[2023-03-07 16:34:31,029][488352] Starting process rollout_proc127 +[2023-03-07 16:34:32,836][488352] Starting process rollout_proc0 +[2023-03-07 16:34:33,014][488352] Starting process rollout_proc1 +[2023-03-07 16:34:34,837][488352] Inference worker 0-0 is ready! +[2023-03-07 16:34:34,838][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 16:34:37,884][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 20025344. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:34:42,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:34:47,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:34:48,441][488352] Heartbeat connected on Batcher_0 +[2023-03-07 16:34:48,443][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 16:34:48,478][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 16:34:52,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:34:57,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:02,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:07,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:12,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:17,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:22,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:27,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:32,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:37,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:42,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:47,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:52,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:35:57,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:02,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:07,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:12,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:17,885][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:22,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:27,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:32,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:37,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:42,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:47,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:52,208][488352] Heartbeat connected on RolloutWorker_w4 +[2023-03-07 16:36:52,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:52,885][488352] Avg episode reward: [(0, '0.200')] +[2023-03-07 16:36:57,884][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20025344. Throughput: 0: 112.4. Samples: 5056. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:36:57,885][488352] Avg episode reward: [(0, '3.257')] +[2023-03-07 16:36:59,358][488352] Component Batcher_0 stopped! +[2023-03-07 16:36:59,410][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 16:36:59,470][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 16:36:59,739][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 16:37:18,778][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 16:37:19,338][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 16:37:21,977][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 16:37:21,981][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 16:37:26,006][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 16:37:26,278][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 16:37:28,490][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 16:37:29,892][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 16:37:29,986][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 16:37:30,277][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 16:37:35,242][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 16:37:37,295][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 16:37:38,216][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 16:37:39,026][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 16:37:43,165][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 16:37:43,271][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 16:37:44,784][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 16:37:45,482][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 16:37:47,578][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 16:37:48,314][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 16:37:49,058][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 16:37:49,650][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 16:37:50,226][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 16:37:50,354][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 16:37:53,598][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 16:37:54,520][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 16:37:54,808][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 16:37:55,517][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 16:37:55,967][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 16:37:56,737][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 16:37:57,889][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 16:37:58,705][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 16:37:59,827][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 16:37:59,844][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 16:38:00,508][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 16:38:01,724][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 16:38:02,380][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 16:38:02,494][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 16:38:03,105][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 16:38:03,889][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 16:38:05,131][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 16:38:05,142][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 16:38:05,224][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 16:38:05,416][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 16:38:05,740][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 16:38:06,159][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 16:38:06,206][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 16:38:06,915][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 16:38:07,056][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 16:38:07,474][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 16:38:07,502][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 16:38:07,724][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 16:38:08,085][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 16:38:08,324][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 16:38:08,398][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 16:38:08,522][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 16:38:08,661][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 16:38:09,776][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 16:38:10,292][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 16:38:10,321][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 16:38:10,377][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 16:38:10,424][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 16:38:10,557][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 16:38:10,734][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 16:38:11,165][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 16:38:11,492][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 16:38:11,611][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 16:38:12,330][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 16:38:12,417][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 16:38:12,419][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 16:38:12,696][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 16:38:12,882][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 16:38:12,983][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 16:38:13,065][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 16:38:13,257][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 16:38:13,577][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 16:38:14,382][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 16:38:14,689][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 16:38:14,936][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 16:38:15,059][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 16:38:15,613][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 16:38:15,653][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 16:38:16,184][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 16:38:16,202][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 16:38:17,061][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 16:38:17,203][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 16:38:17,770][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 16:38:18,012][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 16:38:18,302][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 16:38:18,545][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 16:38:18,609][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 16:38:18,672][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 16:38:19,180][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 16:38:19,302][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 16:38:19,333][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 16:38:19,466][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 16:38:19,598][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 16:38:20,630][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 16:38:21,062][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 16:38:21,159][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 16:38:21,675][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 16:38:21,809][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 16:38:21,982][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 16:38:22,388][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 16:38:22,859][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 16:38:23,271][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 16:38:23,641][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 16:38:24,619][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 16:38:24,965][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 16:38:25,155][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 16:38:25,513][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 16:38:25,731][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 16:38:25,768][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 16:38:25,956][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 16:38:26,259][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 16:38:26,518][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 16:38:26,775][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 16:38:26,892][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 16:38:27,187][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 16:38:27,911][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 16:38:27,990][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 16:38:29,206][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 16:38:29,289][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 16:38:29,533][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 16:38:30,519][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 16:38:31,364][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 16:38:34,535][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 16:38:34,536][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 16:38:34,538][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 16:38:34,539][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 16:38:34,541][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 16:38:34,542][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 16:38:34,544][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 16:38:34,545][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 16:38:34,547][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 16:38:34,548][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 16:38:34,549][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 16:38:34,551][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 16:38:34,552][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 16:38:34,554][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 16:38:34,555][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 16:38:34,557][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 16:38:34,558][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 16:38:34,560][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 16:38:34,561][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 16:38:34,562][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 16:38:34,564][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 16:38:34,729][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 16:38:34,731][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 16:38:34,732][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 16:38:34,734][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 16:38:34,735][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 16:38:34,737][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 16:38:34,738][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 16:38:34,740][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 16:38:34,741][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 16:38:34,743][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 16:38:34,744][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 16:38:34,747][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 16:38:34,748][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 16:38:34,750][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 16:38:34,751][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 16:38:34,753][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 16:38:34,756][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 16:38:34,758][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 16:38:34,759][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 16:38:34,761][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 16:38:34,765][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 16:38:34,767][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 16:38:34,769][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 16:38:34,770][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 16:38:34,772][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 16:38:34,773][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 16:38:34,775][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 16:38:34,776][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 16:38:34,778][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 16:38:34,779][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 16:38:34,781][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 16:38:34,783][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 16:38:34,784][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 16:38:34,786][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 16:38:34,787][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 16:38:34,789][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 16:38:34,792][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 16:38:34,793][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 16:38:34,795][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 16:38:34,796][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 16:38:34,798][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 16:38:34,800][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 16:38:34,801][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 16:38:34,805][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 16:38:34,806][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 16:38:34,807][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 16:38:34,809][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 16:38:34,810][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 16:38:34,811][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 16:38:34,812][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 16:38:34,813][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 16:38:34,815][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 16:38:34,817][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 16:38:34,818][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 16:38:34,820][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 16:38:34,821][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 16:38:34,822][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 16:38:34,823][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 16:38:34,825][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 16:38:34,826][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 16:38:34,827][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 16:38:34,828][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 16:38:34,829][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 16:38:34,831][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 16:38:34,832][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 16:38:34,833][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 16:38:34,835][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 16:38:34,836][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 16:38:34,837][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 16:38:34,841][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 16:38:34,843][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 16:38:34,844][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 16:38:34,845][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 16:38:34,847][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 16:38:34,848][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 16:38:34,849][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 16:38:34,850][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 16:38:34,851][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 16:38:34,852][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 16:38:34,854][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 16:38:34,856][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 16:38:34,857][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 16:38:34,858][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 16:38:34,859][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 16:38:34,860][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 16:38:34,862][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 16:38:34,864][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 16:38:34,865][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 16:38:34,866][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 16:38:34,867][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 16:38:34,868][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 16:38:34,870][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 16:38:34,872][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 16:38:34,874][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 16:38:34,875][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 16:38:34,876][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 16:38:34,877][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 16:38:34,878][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 16:38:34,879][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 16:38:34,881][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 16:38:34,882][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 16:38:34,883][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 16:38:34,884][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 16:38:34,885][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 16:38:34,887][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 16:38:34,888][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 16:38:34,889][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 16:38:34,890][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 16:38:34,891][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 16:38:34,892][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 16:38:34,893][488352] Batcher 0 profile tree view: +batching: 0.0382, releasing_batches: 0.0000 +[2023-03-07 16:38:34,894][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0051 + wait_policy_total: 141.7888 +update_model: 0.0134 + weight_update: 0.0015 +one_step: 0.0045 + handle_policy_step: 1.0126 + deserialize: 0.0648, stack: 0.0043, obs_to_device_normalize: 0.2028, forward: 0.5950, send_messages: 0.0179 + prepare_outputs: 0.0996 + to_cpu: 0.0629 +[2023-03-07 16:38:34,895][488352] Learner 0 profile tree view: +misc: 0.0000, prepare_batch: 0.6207 +train: 0.6673 + epoch_init: 0.0000, minibatch_init: 0.0000, losses_postprocess: 0.0002, kl_divergence: 0.0004, after_optimizer: 0.0071 + calculate_losses: 0.2450 + losses_init: 0.0000, forward_head: 0.2340, bptt_initial: 0.0055, tail: 0.0014, advantages_returns: 0.0007, losses: 0.0020 + bptt: 0.0011 + bptt_forward_core: 0.0011 + update: 0.4142 + clip: 0.0018 +[2023-03-07 16:38:34,895][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0015, enqueue_policy_requests: 0.0019 +[2023-03-07 16:38:34,896][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0013, enqueue_policy_requests: 0.0020 +[2023-03-07 16:38:34,897][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 16:38:34,898][488352] Runner profile tree view: +main_loop: 246.1790 +[2023-03-07 16:38:34,899][488352] Collected {0: 20058112}, FPS: 133.1 +[2023-03-07 16:48:50,284][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 16:48:50,286][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 16:48:50,287][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 16:48:50,287][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 16:48:50,288][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 16:48:50,289][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 16:48:50,289][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 16:48:50,290][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 16:48:50,291][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 16:48:50,291][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 16:48:50,292][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 16:48:50,293][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 16:48:50,293][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 16:48:50,294][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 16:48:50,294][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 16:48:50,297][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 16:48:50,297][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 16:48:50,298][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 16:48:50,299][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 16:48:50,300][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 16:48:50,300][488352] register_encoder_factory: +[2023-03-07 16:48:50,329][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:48:50,330][488352] Overriding arg 'train_for_env_steps' with value 10000000 passed from command line +[2023-03-07 16:48:50,335][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads already exists! +[2023-03-07 16:48:50,335][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads... +[2023-03-07 16:48:50,336][488352] Weights and Biases integration disabled +[2023-03-07 16:48:50,338][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 16:48:51,432][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=64 +batch_size=8192 +num_batches_per_epoch=1 +num_epochs=1 +rollout=32 +recurrence=32 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=10000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=16 --train_for_env_steps=4000000 --experiment=doom_health_threads +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads', 'num_workers': 128, 'num_envs_per_worker': 16, 'train_for_env_steps': 4000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 16:48:51,432][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json... +[2023-03-07 16:48:51,434][488352] Rollout worker 0 uses device cpu +[2023-03-07 16:48:51,435][488352] Rollout worker 1 uses device cpu +[2023-03-07 16:48:51,435][488352] Rollout worker 2 uses device cpu +[2023-03-07 16:48:51,436][488352] Rollout worker 3 uses device cpu +[2023-03-07 16:48:51,436][488352] Rollout worker 4 uses device cpu +[2023-03-07 16:48:51,437][488352] Rollout worker 5 uses device cpu +[2023-03-07 16:48:51,437][488352] Rollout worker 6 uses device cpu +[2023-03-07 16:48:51,438][488352] Rollout worker 7 uses device cpu +[2023-03-07 16:48:51,438][488352] Rollout worker 8 uses device cpu +[2023-03-07 16:48:51,439][488352] Rollout worker 9 uses device cpu +[2023-03-07 16:48:51,440][488352] Rollout worker 10 uses device cpu +[2023-03-07 16:48:51,441][488352] Rollout worker 11 uses device cpu +[2023-03-07 16:48:51,441][488352] Rollout worker 12 uses device cpu +[2023-03-07 16:48:51,442][488352] Rollout worker 13 uses device cpu +[2023-03-07 16:48:51,442][488352] Rollout worker 14 uses device cpu +[2023-03-07 16:48:51,443][488352] Rollout worker 15 uses device cpu +[2023-03-07 16:48:51,443][488352] Rollout worker 16 uses device cpu +[2023-03-07 16:48:51,444][488352] Rollout worker 17 uses device cpu +[2023-03-07 16:48:51,445][488352] Rollout worker 18 uses device cpu +[2023-03-07 16:48:51,445][488352] Rollout worker 19 uses device cpu +[2023-03-07 16:48:51,446][488352] Rollout worker 20 uses device cpu +[2023-03-07 16:48:51,446][488352] Rollout worker 21 uses device cpu +[2023-03-07 16:48:51,447][488352] Rollout worker 22 uses device cpu +[2023-03-07 16:48:51,448][488352] Rollout worker 23 uses device cpu +[2023-03-07 16:48:51,448][488352] Rollout worker 24 uses device cpu +[2023-03-07 16:48:51,449][488352] Rollout worker 25 uses device cpu +[2023-03-07 16:48:51,449][488352] Rollout worker 26 uses device cpu +[2023-03-07 16:48:51,450][488352] Rollout worker 27 uses device cpu +[2023-03-07 16:48:51,450][488352] Rollout worker 28 uses device cpu +[2023-03-07 16:48:51,451][488352] Rollout worker 29 uses device cpu +[2023-03-07 16:48:51,451][488352] Rollout worker 30 uses device cpu +[2023-03-07 16:48:51,452][488352] Rollout worker 31 uses device cpu +[2023-03-07 16:48:51,452][488352] Rollout worker 32 uses device cpu +[2023-03-07 16:48:51,453][488352] Rollout worker 33 uses device cpu +[2023-03-07 16:48:51,453][488352] Rollout worker 34 uses device cpu +[2023-03-07 16:48:51,455][488352] Rollout worker 35 uses device cpu +[2023-03-07 16:48:51,455][488352] Rollout worker 36 uses device cpu +[2023-03-07 16:48:51,456][488352] Rollout worker 37 uses device cpu +[2023-03-07 16:48:51,456][488352] Rollout worker 38 uses device cpu +[2023-03-07 16:48:51,457][488352] Rollout worker 39 uses device cpu +[2023-03-07 16:48:51,457][488352] Rollout worker 40 uses device cpu +[2023-03-07 16:48:51,458][488352] Rollout worker 41 uses device cpu +[2023-03-07 16:48:51,458][488352] Rollout worker 42 uses device cpu +[2023-03-07 16:48:51,459][488352] Rollout worker 43 uses device cpu +[2023-03-07 16:48:51,459][488352] Rollout worker 44 uses device cpu +[2023-03-07 16:48:51,460][488352] Rollout worker 45 uses device cpu +[2023-03-07 16:48:51,460][488352] Rollout worker 46 uses device cpu +[2023-03-07 16:48:51,461][488352] Rollout worker 47 uses device cpu +[2023-03-07 16:48:51,461][488352] Rollout worker 48 uses device cpu +[2023-03-07 16:48:51,462][488352] Rollout worker 49 uses device cpu +[2023-03-07 16:48:51,463][488352] Rollout worker 50 uses device cpu +[2023-03-07 16:48:51,464][488352] Rollout worker 51 uses device cpu +[2023-03-07 16:48:51,464][488352] Rollout worker 52 uses device cpu +[2023-03-07 16:48:51,465][488352] Rollout worker 53 uses device cpu +[2023-03-07 16:48:51,465][488352] Rollout worker 54 uses device cpu +[2023-03-07 16:48:51,466][488352] Rollout worker 55 uses device cpu +[2023-03-07 16:48:51,466][488352] Rollout worker 56 uses device cpu +[2023-03-07 16:48:51,467][488352] Rollout worker 57 uses device cpu +[2023-03-07 16:48:51,467][488352] Rollout worker 58 uses device cpu +[2023-03-07 16:48:51,468][488352] Rollout worker 59 uses device cpu +[2023-03-07 16:48:51,468][488352] Rollout worker 60 uses device cpu +[2023-03-07 16:48:51,469][488352] Rollout worker 61 uses device cpu +[2023-03-07 16:48:51,469][488352] Rollout worker 62 uses device cpu +[2023-03-07 16:48:51,470][488352] Rollout worker 63 uses device cpu +[2023-03-07 16:48:51,471][488352] Rollout worker 64 uses device cpu +[2023-03-07 16:48:51,472][488352] Rollout worker 65 uses device cpu +[2023-03-07 16:48:51,472][488352] Rollout worker 66 uses device cpu +[2023-03-07 16:48:51,473][488352] Rollout worker 67 uses device cpu +[2023-03-07 16:48:51,473][488352] Rollout worker 68 uses device cpu +[2023-03-07 16:48:51,474][488352] Rollout worker 69 uses device cpu +[2023-03-07 16:48:51,475][488352] Rollout worker 70 uses device cpu +[2023-03-07 16:48:51,475][488352] Rollout worker 71 uses device cpu +[2023-03-07 16:48:51,476][488352] Rollout worker 72 uses device cpu +[2023-03-07 16:48:51,476][488352] Rollout worker 73 uses device cpu +[2023-03-07 16:48:51,477][488352] Rollout worker 74 uses device cpu +[2023-03-07 16:48:51,477][488352] Rollout worker 75 uses device cpu +[2023-03-07 16:48:51,478][488352] Rollout worker 76 uses device cpu +[2023-03-07 16:48:51,478][488352] Rollout worker 77 uses device cpu +[2023-03-07 16:48:51,479][488352] Rollout worker 78 uses device cpu +[2023-03-07 16:48:51,479][488352] Rollout worker 79 uses device cpu +[2023-03-07 16:48:51,479][488352] Rollout worker 80 uses device cpu +[2023-03-07 16:48:51,481][488352] Rollout worker 81 uses device cpu +[2023-03-07 16:48:51,481][488352] Rollout worker 82 uses device cpu +[2023-03-07 16:48:51,482][488352] Rollout worker 83 uses device cpu +[2023-03-07 16:48:51,482][488352] Rollout worker 84 uses device cpu +[2023-03-07 16:48:51,484][488352] Rollout worker 85 uses device cpu +[2023-03-07 16:48:51,484][488352] Rollout worker 86 uses device cpu +[2023-03-07 16:48:51,485][488352] Rollout worker 87 uses device cpu +[2023-03-07 16:48:51,486][488352] Rollout worker 88 uses device cpu +[2023-03-07 16:48:51,487][488352] Rollout worker 89 uses device cpu +[2023-03-07 16:48:51,488][488352] Rollout worker 90 uses device cpu +[2023-03-07 16:48:51,489][488352] Rollout worker 91 uses device cpu +[2023-03-07 16:48:51,489][488352] Rollout worker 92 uses device cpu +[2023-03-07 16:48:51,490][488352] Rollout worker 93 uses device cpu +[2023-03-07 16:48:51,491][488352] Rollout worker 94 uses device cpu +[2023-03-07 16:48:51,492][488352] Rollout worker 95 uses device cpu +[2023-03-07 16:48:51,493][488352] Rollout worker 96 uses device cpu +[2023-03-07 16:48:51,493][488352] Rollout worker 97 uses device cpu +[2023-03-07 16:48:51,494][488352] Rollout worker 98 uses device cpu +[2023-03-07 16:48:51,495][488352] Rollout worker 99 uses device cpu +[2023-03-07 16:48:51,496][488352] Rollout worker 100 uses device cpu +[2023-03-07 16:48:51,497][488352] Rollout worker 101 uses device cpu +[2023-03-07 16:48:51,498][488352] Rollout worker 102 uses device cpu +[2023-03-07 16:48:51,498][488352] Rollout worker 103 uses device cpu +[2023-03-07 16:48:51,499][488352] Rollout worker 104 uses device cpu +[2023-03-07 16:48:51,500][488352] Rollout worker 105 uses device cpu +[2023-03-07 16:48:51,500][488352] Rollout worker 106 uses device cpu +[2023-03-07 16:48:51,502][488352] Rollout worker 107 uses device cpu +[2023-03-07 16:48:51,502][488352] Rollout worker 108 uses device cpu +[2023-03-07 16:48:51,503][488352] Rollout worker 109 uses device cpu +[2023-03-07 16:48:51,504][488352] Rollout worker 110 uses device cpu +[2023-03-07 16:48:51,505][488352] Rollout worker 111 uses device cpu +[2023-03-07 16:48:51,506][488352] Rollout worker 112 uses device cpu +[2023-03-07 16:48:51,507][488352] Rollout worker 113 uses device cpu +[2023-03-07 16:48:51,507][488352] Rollout worker 114 uses device cpu +[2023-03-07 16:48:51,508][488352] Rollout worker 115 uses device cpu +[2023-03-07 16:48:51,509][488352] Rollout worker 116 uses device cpu +[2023-03-07 16:48:51,510][488352] Rollout worker 117 uses device cpu +[2023-03-07 16:48:51,511][488352] Rollout worker 118 uses device cpu +[2023-03-07 16:48:51,511][488352] Rollout worker 119 uses device cpu +[2023-03-07 16:48:51,512][488352] Rollout worker 120 uses device cpu +[2023-03-07 16:48:51,513][488352] Rollout worker 121 uses device cpu +[2023-03-07 16:48:51,513][488352] Rollout worker 122 uses device cpu +[2023-03-07 16:48:51,514][488352] Rollout worker 123 uses device cpu +[2023-03-07 16:48:51,514][488352] Rollout worker 124 uses device cpu +[2023-03-07 16:48:51,515][488352] Rollout worker 125 uses device cpu +[2023-03-07 16:48:51,515][488352] Rollout worker 126 uses device cpu +[2023-03-07 16:48:51,516][488352] Rollout worker 127 uses device cpu +[2023-03-07 16:48:55,864][488352] Using GPUs [0] for process 0 (actually maps to GPUs [0]) +[2023-03-07 16:48:55,865][488352] InferenceWorker_p0-w0: min num requests: 42 +[2023-03-07 16:48:56,137][488352] Starting all processes... +[2023-03-07 16:48:56,138][488352] Starting process learner_proc0 +[2023-03-07 16:48:57,052][488352] Starting all processes... +[2023-03-07 16:48:57,057][488352] Starting process inference_proc0-0 +[2023-03-07 16:48:57,058][488352] Starting process rollout_proc2 +[2023-03-07 16:48:57,059][488352] Starting process rollout_proc5 +[2023-03-07 16:48:57,059][488352] Starting process rollout_proc8 +[2023-03-07 16:48:57,060][488352] Starting process rollout_proc11 +[2023-03-07 16:48:57,064][488352] Starting process rollout_proc14 +[2023-03-07 16:48:57,072][488352] Starting process rollout_proc17 +[2023-03-07 16:48:57,080][488352] Starting process rollout_proc20 +[2023-03-07 16:48:57,081][488352] Starting process rollout_proc23 +[2023-03-07 16:48:57,084][488352] Starting process rollout_proc26 +[2023-03-07 16:48:57,085][488352] Starting process rollout_proc29 +[2023-03-07 16:48:57,085][488352] Starting process rollout_proc32 +[2023-03-07 16:48:57,086][488352] Starting process rollout_proc35 +[2023-03-07 16:48:57,087][488352] Starting process rollout_proc38 +[2023-03-07 16:48:57,088][488352] Starting process rollout_proc41 +[2023-03-07 16:48:57,101][488352] Starting process rollout_proc44 +[2023-03-07 16:48:57,102][488352] Starting process rollout_proc9 +[2023-03-07 16:48:57,108][488352] Starting process rollout_proc6 +[2023-03-07 16:48:57,111][488352] Starting process rollout_proc3 +[2023-03-07 16:48:57,119][488352] Starting process rollout_proc12 +[2023-03-07 16:48:57,123][488352] Starting process rollout_proc18 +[2023-03-07 16:48:57,133][488352] Starting process rollout_proc15 +[2023-03-07 16:48:57,144][488352] Starting process rollout_proc21 +[2023-03-07 16:48:57,163][488352] Starting process rollout_proc27 +[2023-03-07 16:48:57,165][488352] Starting process rollout_proc30 +[2023-03-07 16:48:57,179][488352] Starting process rollout_proc24 +[2023-03-07 16:48:57,184][488352] Starting process rollout_proc33 +[2023-03-07 16:48:57,197][488352] Starting process rollout_proc39 +[2023-03-07 16:48:57,200][488352] Starting process rollout_proc45 +[2023-03-07 16:48:57,212][488352] Starting process rollout_proc36 +[2023-03-07 16:48:57,213][488352] Starting process rollout_proc10 +[2023-03-07 16:48:57,218][488352] Starting process rollout_proc42 +[2023-03-07 16:48:57,226][488352] Starting process rollout_proc7 +[2023-03-07 16:48:57,230][488352] Starting process rollout_proc4 +[2023-03-07 16:48:57,230][488352] Starting process rollout_proc13 +[2023-03-07 16:48:57,241][488352] Starting process rollout_proc16 +[2023-03-07 16:48:57,241][488352] Starting process rollout_proc19 +[2023-03-07 16:48:57,248][488352] Starting process rollout_proc22 +[2023-03-07 16:48:57,253][488352] Starting process rollout_proc31 +[2023-03-07 16:48:57,258][488352] Starting process rollout_proc28 +[2023-03-07 16:48:57,266][488352] Starting process rollout_proc34 +[2023-03-07 16:48:57,272][488352] Starting process rollout_proc25 +[2023-03-07 16:48:57,276][488352] Starting process rollout_proc40 +[2023-03-07 16:48:57,284][488352] Starting process rollout_proc46 +[2023-03-07 16:48:57,295][488352] Starting process rollout_proc47 +[2023-03-07 16:48:57,302][488352] Starting process rollout_proc43 +[2023-03-07 16:48:57,308][488352] Starting process rollout_proc37 +[2023-03-07 16:48:57,312][488352] Starting process rollout_proc50 +[2023-03-07 16:48:57,324][488352] Starting process rollout_proc53 +[2023-03-07 16:48:57,324][488352] Starting process rollout_proc56 +[2023-03-07 16:48:57,339][488352] Starting process rollout_proc59 +[2023-03-07 16:48:57,339][488352] Starting process rollout_proc62 +[2023-03-07 16:48:57,355][488352] Starting process rollout_proc65 +[2023-03-07 16:48:57,361][488352] Starting process rollout_proc68 +[2023-03-07 16:48:57,367][488352] Starting process rollout_proc71 +[2023-03-07 16:48:57,381][488352] Starting process rollout_proc74 +[2023-03-07 16:48:57,395][488352] Starting process rollout_proc77 +[2023-03-07 16:48:57,395][488352] Starting process rollout_proc80 +[2023-03-07 16:48:57,400][488352] Starting process rollout_proc83 +[2023-03-07 16:48:57,413][488352] Starting process rollout_proc48 +[2023-03-07 16:48:57,425][488352] Starting process rollout_proc86 +[2023-03-07 16:48:57,431][488352] Starting process rollout_proc89 +[2023-03-07 16:48:57,441][488352] Starting process rollout_proc51 +[2023-03-07 16:48:57,445][488352] Starting process rollout_proc54 +[2023-03-07 16:48:57,458][488352] Starting process rollout_proc57 +[2023-03-07 16:48:57,465][488352] Starting process rollout_proc60 +[2023-03-07 16:48:57,469][488352] Starting process rollout_proc63 +[2023-03-07 16:48:57,479][488352] Starting process rollout_proc66 +[2023-03-07 16:48:57,486][488352] Starting process rollout_proc69 +[2023-03-07 16:48:57,497][488352] Starting process rollout_proc72 +[2023-03-07 16:48:57,504][488352] Starting process rollout_proc75 +[2023-03-07 16:48:57,513][488352] Starting process rollout_proc78 +[2023-03-07 16:48:57,529][488352] Starting process rollout_proc81 +[2023-03-07 16:48:57,539][488352] Starting process rollout_proc84 +[2023-03-07 16:48:57,539][488352] Starting process rollout_proc49 +[2023-03-07 16:48:57,552][488352] Starting process rollout_proc87 +[2023-03-07 16:48:57,560][488352] Starting process rollout_proc52 +[2023-03-07 16:48:57,582][488352] Starting process rollout_proc90 +[2023-03-07 16:48:57,590][488352] Starting process rollout_proc55 +[2023-03-07 16:48:57,590][488352] Starting process rollout_proc58 +[2023-03-07 16:48:57,591][488352] Starting process rollout_proc61 +[2023-03-07 16:48:57,602][488352] Starting process rollout_proc64 +[2023-03-07 16:48:57,630][488352] Starting process rollout_proc70 +[2023-03-07 16:48:57,639][488352] Starting process rollout_proc67 +[2023-03-07 16:48:57,704][488352] Starting process rollout_proc73 +[2023-03-07 16:48:57,746][488352] Starting process rollout_proc79 +[2023-03-07 16:48:57,759][488352] Starting process rollout_proc76 +[2023-03-07 16:48:57,778][488352] Starting process rollout_proc92 +[2023-03-07 16:48:57,779][488352] Starting process rollout_proc82 +[2023-03-07 16:48:57,785][488352] Starting process rollout_proc85 +[2023-03-07 16:48:57,788][488352] Starting process rollout_proc95 +[2023-03-07 16:48:57,788][488352] Starting process rollout_proc91 +[2023-03-07 16:48:57,817][488352] Starting process rollout_proc88 +[2023-03-07 16:48:57,818][488352] Starting process rollout_proc98 +[2023-03-07 16:48:57,818][488352] Starting process rollout_proc101 +[2023-03-07 16:48:57,909][488352] Starting process rollout_proc104 +[2023-03-07 16:48:57,910][488352] Starting process rollout_proc107 +[2023-03-07 16:48:57,934][488352] Starting process rollout_proc110 +[2023-03-07 16:48:57,940][488352] Starting process rollout_proc113 +[2023-03-07 16:48:57,971][488352] Starting process rollout_proc116 +[2023-03-07 16:48:57,979][488352] Starting process rollout_proc119 +[2023-03-07 16:48:57,995][488352] Starting process rollout_proc122 +[2023-03-07 16:48:57,995][488352] Starting process rollout_proc93 +[2023-03-07 16:48:58,006][488352] Starting process rollout_proc125 +[2023-03-07 16:48:58,134][488352] Starting process rollout_proc111 +[2023-03-07 16:48:58,172][488352] Starting process rollout_proc114 +[2023-03-07 16:48:58,172][488352] Starting process rollout_proc99 +[2023-03-07 16:48:58,173][488352] Starting process rollout_proc102 +[2023-03-07 16:48:58,173][488352] Starting process rollout_proc96 +[2023-03-07 16:48:58,198][488352] Starting process rollout_proc105 +[2023-03-07 16:48:58,198][488352] Starting process rollout_proc120 +[2023-03-07 16:48:58,199][488352] Starting process rollout_proc108 +[2023-03-07 16:48:58,226][488352] Starting process rollout_proc126 +[2023-03-07 16:48:58,226][488352] Starting process rollout_proc117 +[2023-03-07 16:48:58,244][488352] Starting process rollout_proc123 +[2023-03-07 16:48:58,359][488352] Starting process rollout_proc94 +[2023-03-07 16:48:58,360][488352] Starting process rollout_proc112 +[2023-03-07 16:48:58,386][488352] Starting process rollout_proc115 +[2023-03-07 16:48:58,387][488352] Starting process rollout_proc109 +[2023-03-07 16:48:58,414][488352] Starting process rollout_proc106 +[2023-03-07 16:48:58,422][488352] Starting process rollout_proc127 +[2023-03-07 16:48:58,423][488352] Starting process rollout_proc103 +[2023-03-07 16:48:58,443][488352] Starting process rollout_proc121 +[2023-03-07 16:48:58,443][488352] Starting process rollout_proc100 +[2023-03-07 16:48:58,507][488352] Starting process rollout_proc124 +[2023-03-07 16:48:58,515][488352] Starting process rollout_proc118 +[2023-03-07 16:48:58,542][488352] Starting process rollout_proc97 +[2023-03-07 16:48:59,603][488352] Starting process rollout_proc0 +[2023-03-07 16:48:59,820][488352] Starting process rollout_proc1 +[2023-03-07 16:49:02,161][488352] Inference worker 0-0 is ready! +[2023-03-07 16:49:02,162][488352] All inference workers are ready! Signal rollout workers to start! +[2023-03-07 16:49:05,338][488352] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 20058112. Throughput: 0: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:10,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:15,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:15,860][488352] Heartbeat connected on Batcher_0 +[2023-03-07 16:49:15,862][488352] Heartbeat connected on LearnerWorker_p0 +[2023-03-07 16:49:15,903][488352] Heartbeat connected on InferenceWorker_p0-w0 +[2023-03-07 16:49:20,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:25,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:30,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:35,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:40,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:45,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:50,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:49:55,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:00,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:05,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:10,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:15,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:20,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:25,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:30,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:35,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:40,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:45,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:50,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:50:55,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:00,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:05,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:10,339][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:15,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:20,180][488352] Heartbeat connected on RolloutWorker_w31 +[2023-03-07 16:51:20,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 0.0. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:25,338][488352] Fps is (10 sec: 0.0, 60 sec: 0.0, 300 sec: 0.0). Total num frames: 20058112. Throughput: 0: 92.4. Samples: 4160. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0) +[2023-03-07 16:51:25,339][488352] Avg episode reward: [(0, '3.321')] +[2023-03-07 16:51:27,320][488352] Component Batcher_0 stopped! +[2023-03-07 16:51:27,370][488352] Component InferenceWorker_p0-w0 stopped! +[2023-03-07 16:51:27,458][488352] Component LearnerWorker_p0 stopped! +[2023-03-07 16:51:27,779][488352] Component RolloutWorker_w31 stopped! +[2023-03-07 16:51:29,686][488352] Component RolloutWorker_w55 stopped! +[2023-03-07 16:51:43,488][488352] Component RolloutWorker_w36 stopped! +[2023-03-07 16:51:46,707][488352] Component RolloutWorker_w113 stopped! +[2023-03-07 16:51:52,714][488352] Component RolloutWorker_w106 stopped! +[2023-03-07 16:51:58,007][488352] Component RolloutWorker_w32 stopped! +[2023-03-07 16:51:59,487][488352] Component RolloutWorker_w110 stopped! +[2023-03-07 16:52:00,524][488352] Component RolloutWorker_w63 stopped! +[2023-03-07 16:52:05,132][488352] Component RolloutWorker_w64 stopped! +[2023-03-07 16:52:05,236][488352] Component RolloutWorker_w84 stopped! +[2023-03-07 16:52:10,530][488352] Component RolloutWorker_w65 stopped! +[2023-03-07 16:52:10,958][488352] Component RolloutWorker_w13 stopped! +[2023-03-07 16:52:11,923][488352] Component RolloutWorker_w39 stopped! +[2023-03-07 16:52:14,728][488352] Component RolloutWorker_w58 stopped! +[2023-03-07 16:52:15,189][488352] Component RolloutWorker_w20 stopped! +[2023-03-07 16:52:17,382][488352] Component RolloutWorker_w52 stopped! +[2023-03-07 16:52:19,697][488352] Component RolloutWorker_w7 stopped! +[2023-03-07 16:52:19,917][488352] Component RolloutWorker_w127 stopped! +[2023-03-07 16:52:20,213][488352] Component RolloutWorker_w109 stopped! +[2023-03-07 16:52:22,647][488352] Component RolloutWorker_w88 stopped! +[2023-03-07 16:52:23,199][488352] Component RolloutWorker_w121 stopped! +[2023-03-07 16:52:23,559][488352] Component RolloutWorker_w122 stopped! +[2023-03-07 16:52:23,956][488352] Component RolloutWorker_w37 stopped! +[2023-03-07 16:52:24,420][488352] Component RolloutWorker_w86 stopped! +[2023-03-07 16:52:24,478][488352] Component RolloutWorker_w77 stopped! +[2023-03-07 16:52:24,845][488352] Component RolloutWorker_w97 stopped! +[2023-03-07 16:52:26,409][488352] Component RolloutWorker_w104 stopped! +[2023-03-07 16:52:26,452][488352] Component RolloutWorker_w92 stopped! +[2023-03-07 16:52:27,159][488352] Component RolloutWorker_w25 stopped! +[2023-03-07 16:52:27,777][488352] Component RolloutWorker_w27 stopped! +[2023-03-07 16:52:29,054][488352] Component RolloutWorker_w93 stopped! +[2023-03-07 16:52:30,232][488352] Component RolloutWorker_w89 stopped! +[2023-03-07 16:52:30,376][488352] Component RolloutWorker_w100 stopped! +[2023-03-07 16:52:30,389][488352] Component RolloutWorker_w15 stopped! +[2023-03-07 16:52:30,480][488352] Component RolloutWorker_w21 stopped! +[2023-03-07 16:52:31,103][488352] Component RolloutWorker_w61 stopped! +[2023-03-07 16:52:31,511][488352] Component RolloutWorker_w83 stopped! +[2023-03-07 16:52:32,030][488352] Component RolloutWorker_w60 stopped! +[2023-03-07 16:52:32,550][488352] Component RolloutWorker_w6 stopped! +[2023-03-07 16:52:32,847][488352] Component RolloutWorker_w103 stopped! +[2023-03-07 16:52:33,073][488352] Component RolloutWorker_w28 stopped! +[2023-03-07 16:52:33,541][488352] Component RolloutWorker_w56 stopped! +[2023-03-07 16:52:34,305][488352] Component RolloutWorker_w18 stopped! +[2023-03-07 16:52:34,606][488352] Component RolloutWorker_w45 stopped! +[2023-03-07 16:52:34,635][488352] Component RolloutWorker_w67 stopped! +[2023-03-07 16:52:36,131][488352] Component RolloutWorker_w116 stopped! +[2023-03-07 16:52:37,343][488352] Component RolloutWorker_w94 stopped! +[2023-03-07 16:52:37,423][488352] Component RolloutWorker_w96 stopped! +[2023-03-07 16:52:37,728][488352] Component RolloutWorker_w33 stopped! +[2023-03-07 16:52:38,010][488352] Component RolloutWorker_w40 stopped! +[2023-03-07 16:52:38,239][488352] Component RolloutWorker_w98 stopped! +[2023-03-07 16:52:38,393][488352] Component RolloutWorker_w117 stopped! +[2023-03-07 16:52:38,755][488352] Component RolloutWorker_w62 stopped! +[2023-03-07 16:52:38,810][488352] Component RolloutWorker_w9 stopped! +[2023-03-07 16:52:38,930][488352] Component RolloutWorker_w119 stopped! +[2023-03-07 16:52:39,587][488352] Component RolloutWorker_w42 stopped! +[2023-03-07 16:52:39,615][488352] Component RolloutWorker_w108 stopped! +[2023-03-07 16:52:39,630][488352] Component RolloutWorker_w90 stopped! +[2023-03-07 16:52:39,995][488352] Component RolloutWorker_w34 stopped! +[2023-03-07 16:52:40,105][488352] Component RolloutWorker_w114 stopped! +[2023-03-07 16:52:40,256][488352] Component RolloutWorker_w0 stopped! +[2023-03-07 16:52:40,305][488352] Component RolloutWorker_w5 stopped! +[2023-03-07 16:52:41,341][488352] Component RolloutWorker_w124 stopped! +[2023-03-07 16:52:41,976][488352] Component RolloutWorker_w8 stopped! +[2023-03-07 16:52:42,098][488352] Component RolloutWorker_w70 stopped! +[2023-03-07 16:52:42,328][488352] Component RolloutWorker_w1 stopped! +[2023-03-07 16:52:43,152][488352] Component RolloutWorker_w54 stopped! +[2023-03-07 16:52:43,390][488352] Component RolloutWorker_w107 stopped! +[2023-03-07 16:52:43,625][488352] Component RolloutWorker_w115 stopped! +[2023-03-07 16:52:43,884][488352] Component RolloutWorker_w3 stopped! +[2023-03-07 16:52:44,223][488352] Component RolloutWorker_w69 stopped! +[2023-03-07 16:52:44,556][488352] Component RolloutWorker_w112 stopped! +[2023-03-07 16:52:44,573][488352] Component RolloutWorker_w126 stopped! +[2023-03-07 16:52:44,747][488352] Component RolloutWorker_w44 stopped! +[2023-03-07 16:52:45,773][488352] Component RolloutWorker_w26 stopped! +[2023-03-07 16:52:45,881][488352] Component RolloutWorker_w23 stopped! +[2023-03-07 16:52:46,210][488352] Component RolloutWorker_w2 stopped! +[2023-03-07 16:52:46,420][488352] Component RolloutWorker_w35 stopped! +[2023-03-07 16:52:46,650][488352] Component RolloutWorker_w85 stopped! +[2023-03-07 16:52:46,760][488352] Component RolloutWorker_w91 stopped! +[2023-03-07 16:52:46,882][488352] Component RolloutWorker_w46 stopped! +[2023-03-07 16:52:47,085][488352] Component RolloutWorker_w30 stopped! +[2023-03-07 16:52:47,272][488352] Component RolloutWorker_w16 stopped! +[2023-03-07 16:52:47,526][488352] Component RolloutWorker_w123 stopped! +[2023-03-07 16:52:47,780][488352] Component RolloutWorker_w38 stopped! +[2023-03-07 16:52:47,820][488352] Component RolloutWorker_w75 stopped! +[2023-03-07 16:52:48,098][488352] Component RolloutWorker_w12 stopped! +[2023-03-07 16:52:48,222][488352] Component RolloutWorker_w48 stopped! +[2023-03-07 16:52:48,516][488352] Component RolloutWorker_w47 stopped! +[2023-03-07 16:52:48,637][488352] Component RolloutWorker_w120 stopped! +[2023-03-07 16:52:48,656][488352] Component RolloutWorker_w14 stopped! +[2023-03-07 16:52:49,009][488352] Component RolloutWorker_w41 stopped! +[2023-03-07 16:52:49,196][488352] Component RolloutWorker_w79 stopped! +[2023-03-07 16:52:49,297][488352] Component RolloutWorker_w118 stopped! +[2023-03-07 16:52:49,441][488352] Component RolloutWorker_w82 stopped! +[2023-03-07 16:52:49,569][488352] Component RolloutWorker_w22 stopped! +[2023-03-07 16:52:49,940][488352] Component RolloutWorker_w125 stopped! +[2023-03-07 16:52:51,263][488352] Component RolloutWorker_w73 stopped! +[2023-03-07 16:52:51,696][488352] Component RolloutWorker_w11 stopped! +[2023-03-07 16:52:51,960][488352] Component RolloutWorker_w66 stopped! +[2023-03-07 16:52:52,493][488352] Component RolloutWorker_w105 stopped! +[2023-03-07 16:52:52,578][488352] Component RolloutWorker_w68 stopped! +[2023-03-07 16:52:52,701][488352] Component RolloutWorker_w76 stopped! +[2023-03-07 16:52:52,885][488352] Component RolloutWorker_w50 stopped! +[2023-03-07 16:52:52,951][488352] Component RolloutWorker_w99 stopped! +[2023-03-07 16:52:53,238][488352] Component RolloutWorker_w10 stopped! +[2023-03-07 16:52:53,365][488352] Component RolloutWorker_w49 stopped! +[2023-03-07 16:52:53,392][488352] Component RolloutWorker_w53 stopped! +[2023-03-07 16:52:53,972][488352] Component RolloutWorker_w57 stopped! +[2023-03-07 16:52:54,442][488352] Component RolloutWorker_w59 stopped! +[2023-03-07 16:52:55,162][488352] Component RolloutWorker_w19 stopped! +[2023-03-07 16:52:55,186][488352] Component RolloutWorker_w72 stopped! +[2023-03-07 16:52:55,291][488352] Component RolloutWorker_w102 stopped! +[2023-03-07 16:52:55,375][488352] Component RolloutWorker_w24 stopped! +[2023-03-07 16:52:56,524][488352] Component RolloutWorker_w71 stopped! +[2023-03-07 16:52:57,612][488352] Component RolloutWorker_w78 stopped! +[2023-03-07 16:52:57,664][488352] Component RolloutWorker_w29 stopped! +[2023-03-07 16:52:57,852][488352] Component RolloutWorker_w74 stopped! +[2023-03-07 16:52:58,903][488352] Component RolloutWorker_w4 stopped! +[2023-03-07 16:52:59,590][488352] Component RolloutWorker_w80 stopped! +[2023-03-07 16:52:59,719][488352] Component RolloutWorker_w43 stopped! +[2023-03-07 16:53:00,302][488352] Component RolloutWorker_w111 stopped! +[2023-03-07 16:53:00,336][488352] Component RolloutWorker_w81 stopped! +[2023-03-07 16:53:00,653][488352] Component RolloutWorker_w51 stopped! +[2023-03-07 16:53:01,117][488352] Component RolloutWorker_w101 stopped! +[2023-03-07 16:53:02,085][488352] Component RolloutWorker_w87 stopped! +[2023-03-07 16:53:02,694][488352] Component RolloutWorker_w95 stopped! +[2023-03-07 16:53:05,337][488352] Component RolloutWorker_w17 stopped! +[2023-03-07 16:53:05,339][488352] Waiting for process learner_proc0 to stop... +[2023-03-07 16:53:05,340][488352] Waiting for process inference_proc0-0 to join... +[2023-03-07 16:53:05,342][488352] Waiting for process rollout_proc0 to join... +[2023-03-07 16:53:05,344][488352] Waiting for process rollout_proc1 to join... +[2023-03-07 16:53:05,346][488352] Waiting for process rollout_proc2 to join... +[2023-03-07 16:53:05,348][488352] Waiting for process rollout_proc3 to join... +[2023-03-07 16:53:05,351][488352] Waiting for process rollout_proc4 to join... +[2023-03-07 16:53:05,353][488352] Waiting for process rollout_proc5 to join... +[2023-03-07 16:53:05,355][488352] Waiting for process rollout_proc6 to join... +[2023-03-07 16:53:05,357][488352] Waiting for process rollout_proc7 to join... +[2023-03-07 16:53:05,359][488352] Waiting for process rollout_proc8 to join... +[2023-03-07 16:53:05,361][488352] Waiting for process rollout_proc9 to join... +[2023-03-07 16:53:05,363][488352] Waiting for process rollout_proc10 to join... +[2023-03-07 16:53:05,365][488352] Waiting for process rollout_proc11 to join... +[2023-03-07 16:53:05,368][488352] Waiting for process rollout_proc12 to join... +[2023-03-07 16:53:05,370][488352] Waiting for process rollout_proc13 to join... +[2023-03-07 16:53:05,372][488352] Waiting for process rollout_proc14 to join... +[2023-03-07 16:53:05,378][488352] Waiting for process rollout_proc15 to join... +[2023-03-07 16:53:05,380][488352] Waiting for process rollout_proc16 to join... +[2023-03-07 16:53:05,381][488352] Waiting for process rollout_proc17 to join... +[2023-03-07 16:53:05,531][488352] Waiting for process rollout_proc18 to join... +[2023-03-07 16:53:05,533][488352] Waiting for process rollout_proc19 to join... +[2023-03-07 16:53:05,534][488352] Waiting for process rollout_proc20 to join... +[2023-03-07 16:53:05,536][488352] Waiting for process rollout_proc21 to join... +[2023-03-07 16:53:05,538][488352] Waiting for process rollout_proc22 to join... +[2023-03-07 16:53:05,539][488352] Waiting for process rollout_proc23 to join... +[2023-03-07 16:53:05,541][488352] Waiting for process rollout_proc24 to join... +[2023-03-07 16:53:05,542][488352] Waiting for process rollout_proc25 to join... +[2023-03-07 16:53:05,543][488352] Waiting for process rollout_proc26 to join... +[2023-03-07 16:53:05,545][488352] Waiting for process rollout_proc27 to join... +[2023-03-07 16:53:05,547][488352] Waiting for process rollout_proc28 to join... +[2023-03-07 16:53:05,549][488352] Waiting for process rollout_proc29 to join... +[2023-03-07 16:53:05,551][488352] Waiting for process rollout_proc30 to join... +[2023-03-07 16:53:05,553][488352] Waiting for process rollout_proc31 to join... +[2023-03-07 16:53:05,555][488352] Waiting for process rollout_proc32 to join... +[2023-03-07 16:53:05,557][488352] Waiting for process rollout_proc33 to join... +[2023-03-07 16:53:05,560][488352] Waiting for process rollout_proc34 to join... +[2023-03-07 16:53:05,562][488352] Waiting for process rollout_proc35 to join... +[2023-03-07 16:53:05,564][488352] Waiting for process rollout_proc36 to join... +[2023-03-07 16:53:05,566][488352] Waiting for process rollout_proc37 to join... +[2023-03-07 16:53:05,568][488352] Waiting for process rollout_proc38 to join... +[2023-03-07 16:53:05,570][488352] Waiting for process rollout_proc39 to join... +[2023-03-07 16:53:05,573][488352] Waiting for process rollout_proc40 to join... +[2023-03-07 16:53:05,575][488352] Waiting for process rollout_proc41 to join... +[2023-03-07 16:53:05,577][488352] Waiting for process rollout_proc42 to join... +[2023-03-07 16:53:05,581][488352] Waiting for process rollout_proc43 to join... +[2023-03-07 16:53:05,584][488352] Waiting for process rollout_proc44 to join... +[2023-03-07 16:53:05,586][488352] Waiting for process rollout_proc45 to join... +[2023-03-07 16:53:05,588][488352] Waiting for process rollout_proc46 to join... +[2023-03-07 16:53:05,590][488352] Waiting for process rollout_proc47 to join... +[2023-03-07 16:53:05,592][488352] Waiting for process rollout_proc48 to join... +[2023-03-07 16:53:05,594][488352] Waiting for process rollout_proc49 to join... +[2023-03-07 16:53:05,595][488352] Waiting for process rollout_proc50 to join... +[2023-03-07 16:53:05,598][488352] Waiting for process rollout_proc51 to join... +[2023-03-07 16:53:05,600][488352] Waiting for process rollout_proc52 to join... +[2023-03-07 16:53:05,602][488352] Waiting for process rollout_proc53 to join... +[2023-03-07 16:53:05,604][488352] Waiting for process rollout_proc54 to join... +[2023-03-07 16:53:05,609][488352] Waiting for process rollout_proc55 to join... +[2023-03-07 16:53:05,611][488352] Waiting for process rollout_proc56 to join... +[2023-03-07 16:53:05,613][488352] Waiting for process rollout_proc57 to join... +[2023-03-07 16:53:05,618][488352] Waiting for process rollout_proc58 to join... +[2023-03-07 16:53:05,620][488352] Waiting for process rollout_proc59 to join... +[2023-03-07 16:53:05,621][488352] Waiting for process rollout_proc60 to join... +[2023-03-07 16:53:05,626][488352] Waiting for process rollout_proc61 to join... +[2023-03-07 16:53:05,628][488352] Waiting for process rollout_proc62 to join... +[2023-03-07 16:53:05,630][488352] Waiting for process rollout_proc63 to join... +[2023-03-07 16:53:05,632][488352] Waiting for process rollout_proc64 to join... +[2023-03-07 16:53:05,633][488352] Waiting for process rollout_proc65 to join... +[2023-03-07 16:53:05,634][488352] Waiting for process rollout_proc66 to join... +[2023-03-07 16:53:05,636][488352] Waiting for process rollout_proc67 to join... +[2023-03-07 16:53:05,638][488352] Waiting for process rollout_proc68 to join... +[2023-03-07 16:53:05,639][488352] Waiting for process rollout_proc69 to join... +[2023-03-07 16:53:05,641][488352] Waiting for process rollout_proc70 to join... +[2023-03-07 16:53:05,642][488352] Waiting for process rollout_proc71 to join... +[2023-03-07 16:53:05,644][488352] Waiting for process rollout_proc72 to join... +[2023-03-07 16:53:05,646][488352] Waiting for process rollout_proc73 to join... +[2023-03-07 16:53:05,647][488352] Waiting for process rollout_proc74 to join... +[2023-03-07 16:53:05,649][488352] Waiting for process rollout_proc75 to join... +[2023-03-07 16:53:05,650][488352] Waiting for process rollout_proc76 to join... +[2023-03-07 16:53:05,652][488352] Waiting for process rollout_proc77 to join... +[2023-03-07 16:53:05,653][488352] Waiting for process rollout_proc78 to join... +[2023-03-07 16:53:05,655][488352] Waiting for process rollout_proc79 to join... +[2023-03-07 16:53:05,656][488352] Waiting for process rollout_proc80 to join... +[2023-03-07 16:53:05,657][488352] Waiting for process rollout_proc81 to join... +[2023-03-07 16:53:05,659][488352] Waiting for process rollout_proc82 to join... +[2023-03-07 16:53:05,661][488352] Waiting for process rollout_proc83 to join... +[2023-03-07 16:53:05,664][488352] Waiting for process rollout_proc84 to join... +[2023-03-07 16:53:05,665][488352] Waiting for process rollout_proc85 to join... +[2023-03-07 16:53:05,667][488352] Waiting for process rollout_proc86 to join... +[2023-03-07 16:53:05,668][488352] Waiting for process rollout_proc87 to join... +[2023-03-07 16:53:05,670][488352] Waiting for process rollout_proc88 to join... +[2023-03-07 16:53:05,673][488352] Waiting for process rollout_proc89 to join... +[2023-03-07 16:53:05,674][488352] Waiting for process rollout_proc90 to join... +[2023-03-07 16:53:05,676][488352] Waiting for process rollout_proc91 to join... +[2023-03-07 16:53:05,677][488352] Waiting for process rollout_proc92 to join... +[2023-03-07 16:53:05,679][488352] Waiting for process rollout_proc93 to join... +[2023-03-07 16:53:05,680][488352] Waiting for process rollout_proc94 to join... +[2023-03-07 16:53:05,682][488352] Waiting for process rollout_proc95 to join... +[2023-03-07 16:53:05,683][488352] Waiting for process rollout_proc96 to join... +[2023-03-07 16:53:05,685][488352] Waiting for process rollout_proc97 to join... +[2023-03-07 16:53:05,687][488352] Waiting for process rollout_proc98 to join... +[2023-03-07 16:53:05,688][488352] Waiting for process rollout_proc99 to join... +[2023-03-07 16:53:05,690][488352] Waiting for process rollout_proc100 to join... +[2023-03-07 16:53:05,691][488352] Waiting for process rollout_proc101 to join... +[2023-03-07 16:53:05,693][488352] Waiting for process rollout_proc102 to join... +[2023-03-07 16:53:05,696][488352] Waiting for process rollout_proc103 to join... +[2023-03-07 16:53:05,697][488352] Waiting for process rollout_proc104 to join... +[2023-03-07 16:53:05,699][488352] Waiting for process rollout_proc105 to join... +[2023-03-07 16:53:05,700][488352] Waiting for process rollout_proc106 to join... +[2023-03-07 16:53:05,702][488352] Waiting for process rollout_proc107 to join... +[2023-03-07 16:53:05,709][488352] Waiting for process rollout_proc108 to join... +[2023-03-07 16:53:05,710][488352] Waiting for process rollout_proc109 to join... +[2023-03-07 16:53:05,712][488352] Waiting for process rollout_proc110 to join... +[2023-03-07 16:53:05,713][488352] Waiting for process rollout_proc111 to join... +[2023-03-07 16:53:05,715][488352] Waiting for process rollout_proc112 to join... +[2023-03-07 16:53:05,716][488352] Waiting for process rollout_proc113 to join... +[2023-03-07 16:53:05,718][488352] Waiting for process rollout_proc114 to join... +[2023-03-07 16:53:05,719][488352] Waiting for process rollout_proc115 to join... +[2023-03-07 16:53:05,721][488352] Waiting for process rollout_proc116 to join... +[2023-03-07 16:53:05,722][488352] Waiting for process rollout_proc117 to join... +[2023-03-07 16:53:05,724][488352] Waiting for process rollout_proc118 to join... +[2023-03-07 16:53:05,725][488352] Waiting for process rollout_proc119 to join... +[2023-03-07 16:53:05,728][488352] Waiting for process rollout_proc120 to join... +[2023-03-07 16:53:05,730][488352] Waiting for process rollout_proc121 to join... +[2023-03-07 16:53:05,731][488352] Waiting for process rollout_proc122 to join... +[2023-03-07 16:53:05,733][488352] Waiting for process rollout_proc123 to join... +[2023-03-07 16:53:05,738][488352] Waiting for process rollout_proc124 to join... +[2023-03-07 16:53:05,740][488352] Waiting for process rollout_proc125 to join... +[2023-03-07 16:53:05,741][488352] Waiting for process rollout_proc126 to join... +[2023-03-07 16:53:05,746][488352] Waiting for process rollout_proc127 to join... +[2023-03-07 16:53:05,748][488352] Batcher 0 profile tree view: +batching: 0.0343, releasing_batches: 0.0000 +[2023-03-07 16:53:05,748][488352] InferenceWorker_p0-w0 profile tree view: +wait_policy: 0.0051 + wait_policy_total: 142.5174 +update_model: 0.0097 + weight_update: 0.0014 +one_step: 0.0036 + handle_policy_step: 0.9825 + deserialize: 0.0640, stack: 0.0036, obs_to_device_normalize: 0.1941, forward: 0.5748, send_messages: 0.0156 + prepare_outputs: 0.1044 + to_cpu: 0.0691 +[2023-03-07 16:53:05,749][488352] Learner 0 profile tree view: +misc: 0.0000, prepare_batch: 0.6404 +train: 0.6519 + epoch_init: 0.0000, minibatch_init: 0.0000, losses_postprocess: 0.0001, kl_divergence: 0.0005, after_optimizer: 0.0059 + calculate_losses: 0.2365 + losses_init: 0.0000, forward_head: 0.2286, bptt_initial: 0.0048, tail: 0.0006, advantages_returns: 0.0004, losses: 0.0012 + bptt: 0.0007 + bptt_forward_core: 0.0007 + update: 0.4085 + clip: 0.0012 +[2023-03-07 16:53:05,750][488352] RolloutWorker_w0 profile tree view: +wait_for_trajectories: 0.0014, enqueue_policy_requests: 0.0020 +[2023-03-07 16:53:05,752][488352] RolloutWorker_w127 profile tree view: +wait_for_trajectories: 0.0013, enqueue_policy_requests: 0.0019 +[2023-03-07 16:53:05,753][488352] Loop Runner_EvtLoop terminating... +[2023-03-07 16:53:05,755][488352] Runner profile tree view: +main_loop: 249.6183 +[2023-03-07 16:53:05,756][488352] Collected {0: 20090880}, FPS: 131.3 +[2023-03-07 16:57:25,088][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/config.json +[2023-03-07 16:57:25,089][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:57:25,089][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:57:25,090][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:57:25,091][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:57:25,092][488352] Adding new argument 'max_num_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:57:25,093][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:57:25,093][488352] Adding new argument 'push_to_hub'=False that is not in the saved config file! +[2023-03-07 16:57:25,095][488352] Adding new argument 'hf_repository'=None that is not in the saved config file! +[2023-03-07 16:57:25,095][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:57:25,096][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:57:25,097][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:57:25,097][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:57:25,099][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:57:25,104][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:57:25,105][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:57:25,121][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:57:25,156][488352] Conv encoder output size: 512 +[2023-03-07 16:57:25,157][488352] Policy head output size: 512 +[2023-03-07 16:57:25,189][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/checkpoint_p0/checkpoint_000004888_20090880.pth... +[2023-03-07 16:57:25,759][488352] Num frames 100... +[2023-03-07 16:57:25,847][488352] Num frames 200... +[2023-03-07 16:57:25,928][488352] Num frames 300... +[2023-03-07 16:57:26,010][488352] Num frames 400... +[2023-03-07 16:57:26,089][488352] Num frames 500... +[2023-03-07 16:57:26,170][488352] Num frames 600... +[2023-03-07 16:57:26,255][488352] Num frames 700... +[2023-03-07 16:57:26,340][488352] Avg episode rewards: #0: 12.360, true rewards: #0: 7.360 +[2023-03-07 16:57:26,341][488352] Avg episode reward: 12.360, avg true_objective: 7.360 +[2023-03-07 16:57:26,410][488352] Num frames 800... +[2023-03-07 16:57:26,493][488352] Num frames 900... +[2023-03-07 16:57:26,581][488352] Num frames 1000... +[2023-03-07 16:57:26,667][488352] Num frames 1100... +[2023-03-07 16:57:26,754][488352] Num frames 1200... +[2023-03-07 16:57:26,840][488352] Num frames 1300... +[2023-03-07 16:57:26,925][488352] Num frames 1400... +[2023-03-07 16:57:27,004][488352] Num frames 1500... +[2023-03-07 16:57:27,081][488352] Num frames 1600... +[2023-03-07 16:57:27,158][488352] Num frames 1700... +[2023-03-07 16:57:27,235][488352] Num frames 1800... +[2023-03-07 16:57:27,311][488352] Num frames 1900... +[2023-03-07 16:57:27,386][488352] Num frames 2000... +[2023-03-07 16:57:27,475][488352] Avg episode rewards: #0: 19.740, true rewards: #0: 10.240 +[2023-03-07 16:57:27,476][488352] Avg episode reward: 19.740, avg true_objective: 10.240 +[2023-03-07 16:57:27,518][488352] Num frames 2100... +[2023-03-07 16:57:27,599][488352] Num frames 2200... +[2023-03-07 16:57:27,684][488352] Num frames 2300... +[2023-03-07 16:57:27,770][488352] Num frames 2400... +[2023-03-07 16:57:27,857][488352] Num frames 2500... +[2023-03-07 16:57:27,990][488352] Avg episode rewards: #0: 16.640, true rewards: #0: 8.640 +[2023-03-07 16:57:27,991][488352] Avg episode reward: 16.640, avg true_objective: 8.640 +[2023-03-07 16:57:27,999][488352] Num frames 2600... +[2023-03-07 16:57:28,098][488352] Num frames 2700... +[2023-03-07 16:57:28,178][488352] Num frames 2800... +[2023-03-07 16:57:28,255][488352] Num frames 2900... +[2023-03-07 16:57:28,332][488352] Num frames 3000... +[2023-03-07 16:57:28,409][488352] Num frames 3100... +[2023-03-07 16:57:28,487][488352] Num frames 3200... +[2023-03-07 16:57:28,538][488352] Avg episode rewards: #0: 14.750, true rewards: #0: 8.000 +[2023-03-07 16:57:28,539][488352] Avg episode reward: 14.750, avg true_objective: 8.000 +[2023-03-07 16:57:28,626][488352] Num frames 3300... +[2023-03-07 16:57:28,702][488352] Num frames 3400... +[2023-03-07 16:57:28,778][488352] Num frames 3500... +[2023-03-07 16:57:28,855][488352] Num frames 3600... +[2023-03-07 16:57:28,941][488352] Num frames 3700... +[2023-03-07 16:57:29,028][488352] Num frames 3800... +[2023-03-07 16:57:29,114][488352] Num frames 3900... +[2023-03-07 16:57:29,201][488352] Num frames 4000... +[2023-03-07 16:57:29,287][488352] Num frames 4100... +[2023-03-07 16:57:29,392][488352] Avg episode rewards: #0: 15.720, true rewards: #0: 8.320 +[2023-03-07 16:57:29,393][488352] Avg episode reward: 15.720, avg true_objective: 8.320 +[2023-03-07 16:57:29,429][488352] Num frames 4200... +[2023-03-07 16:57:29,515][488352] Num frames 4300... +[2023-03-07 16:57:29,601][488352] Num frames 4400... +[2023-03-07 16:57:29,691][488352] Num frames 4500... +[2023-03-07 16:57:29,783][488352] Avg episode rewards: #0: 13.740, true rewards: #0: 7.573 +[2023-03-07 16:57:29,783][488352] Avg episode reward: 13.740, avg true_objective: 7.573 +[2023-03-07 16:57:29,860][488352] Num frames 4600... +[2023-03-07 16:57:29,945][488352] Num frames 4700... +[2023-03-07 16:57:30,031][488352] Num frames 4800... +[2023-03-07 16:57:30,117][488352] Num frames 4900... +[2023-03-07 16:57:30,203][488352] Num frames 5000... +[2023-03-07 16:57:30,290][488352] Num frames 5100... +[2023-03-07 16:57:30,362][488352] Avg episode rewards: #0: 13.172, true rewards: #0: 7.314 +[2023-03-07 16:57:30,363][488352] Avg episode reward: 13.172, avg true_objective: 7.314 +[2023-03-07 16:57:30,437][488352] Num frames 5200... +[2023-03-07 16:57:30,522][488352] Num frames 5300... +[2023-03-07 16:57:30,608][488352] Num frames 5400... +[2023-03-07 16:57:30,698][488352] Num frames 5500... +[2023-03-07 16:57:30,787][488352] Num frames 5600... +[2023-03-07 16:57:30,877][488352] Num frames 5700... +[2023-03-07 16:57:30,966][488352] Num frames 5800... +[2023-03-07 16:57:31,054][488352] Num frames 5900... +[2023-03-07 16:57:31,140][488352] Num frames 6000... +[2023-03-07 16:57:31,231][488352] Num frames 6100... +[2023-03-07 16:57:31,318][488352] Num frames 6200... +[2023-03-07 16:57:31,403][488352] Num frames 6300... +[2023-03-07 16:57:31,490][488352] Num frames 6400... +[2023-03-07 16:57:31,579][488352] Num frames 6500... +[2023-03-07 16:57:31,668][488352] Num frames 6600... +[2023-03-07 16:57:31,744][488352] Avg episode rewards: #0: 15.905, true rewards: #0: 8.280 +[2023-03-07 16:57:31,745][488352] Avg episode reward: 15.905, avg true_objective: 8.280 +[2023-03-07 16:57:31,832][488352] Num frames 6700... +[2023-03-07 16:57:31,919][488352] Num frames 6800... +[2023-03-07 16:57:32,006][488352] Num frames 6900... +[2023-03-07 16:57:32,092][488352] Num frames 7000... +[2023-03-07 16:57:32,179][488352] Num frames 7100... +[2023-03-07 16:57:32,263][488352] Num frames 7200... +[2023-03-07 16:57:32,348][488352] Num frames 7300... +[2023-03-07 16:57:32,433][488352] Num frames 7400... +[2023-03-07 16:57:32,519][488352] Num frames 7500... +[2023-03-07 16:57:32,617][488352] Avg episode rewards: #0: 15.947, true rewards: #0: 8.391 +[2023-03-07 16:57:32,619][488352] Avg episode reward: 15.947, avg true_objective: 8.391 +[2023-03-07 16:57:32,674][488352] Num frames 7600... +[2023-03-07 16:57:32,761][488352] Num frames 7700... +[2023-03-07 16:57:32,847][488352] Num frames 7800... +[2023-03-07 16:57:32,933][488352] Num frames 7900... +[2023-03-07 16:57:33,021][488352] Num frames 8000... +[2023-03-07 16:57:33,107][488352] Num frames 8100... +[2023-03-07 16:57:33,194][488352] Num frames 8200... +[2023-03-07 16:57:33,283][488352] Num frames 8300... +[2023-03-07 16:57:33,371][488352] Num frames 8400... +[2023-03-07 16:57:33,462][488352] Num frames 8500... +[2023-03-07 16:57:33,551][488352] Num frames 8600... +[2023-03-07 16:57:33,638][488352] Num frames 8700... +[2023-03-07 16:57:33,723][488352] Num frames 8800... +[2023-03-07 16:57:33,810][488352] Num frames 8900... +[2023-03-07 16:57:33,898][488352] Num frames 9000... +[2023-03-07 16:57:33,985][488352] Num frames 9100... +[2023-03-07 16:57:34,057][488352] Avg episode rewards: #0: 18.020, true rewards: #0: 9.120 +[2023-03-07 16:57:34,058][488352] Avg episode reward: 18.020, avg true_objective: 9.120 +[2023-03-07 16:57:45,255][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads/replay.mp4! +[2023-03-07 16:58:38,583][488352] Loading existing experiment configuration from /mnt/Lata/projects/samplefactory/train_dir/default_experiment/config.json +[2023-03-07 16:58:38,584][488352] Overriding arg 'num_workers' with value 1 passed from command line +[2023-03-07 16:58:38,585][488352] Adding new argument 'no_render'=True that is not in the saved config file! +[2023-03-07 16:58:38,586][488352] Adding new argument 'save_video'=True that is not in the saved config file! +[2023-03-07 16:58:38,587][488352] Adding new argument 'video_frames'=1000000000.0 that is not in the saved config file! +[2023-03-07 16:58:38,587][488352] Adding new argument 'video_name'=None that is not in the saved config file! +[2023-03-07 16:58:38,588][488352] Adding new argument 'max_num_frames'=100000 that is not in the saved config file! +[2023-03-07 16:58:38,589][488352] Adding new argument 'max_num_episodes'=10 that is not in the saved config file! +[2023-03-07 16:58:38,590][488352] Adding new argument 'push_to_hub'=True that is not in the saved config file! +[2023-03-07 16:58:38,591][488352] Adding new argument 'hf_repository'='Rolo/doom_health_threads' that is not in the saved config file! +[2023-03-07 16:58:38,592][488352] Adding new argument 'policy_index'=0 that is not in the saved config file! +[2023-03-07 16:58:38,592][488352] Adding new argument 'eval_deterministic'=False that is not in the saved config file! +[2023-03-07 16:58:38,593][488352] Adding new argument 'train_script'=None that is not in the saved config file! +[2023-03-07 16:58:38,594][488352] Adding new argument 'enjoy_script'=None that is not in the saved config file! +[2023-03-07 16:58:38,595][488352] Using frameskip 1 and render_action_repeat=4 for evaluation +[2023-03-07 16:58:38,600][488352] RunningMeanStd input shape: (3, 72, 128) +[2023-03-07 16:58:38,602][488352] RunningMeanStd input shape: (1,) +[2023-03-07 16:58:38,622][488352] ConvEncoder: input_channels=3 +[2023-03-07 16:58:38,659][488352] Conv encoder output size: 512 +[2023-03-07 16:58:38,670][488352] Policy head output size: 512 +[2023-03-07 16:58:38,688][488352] Loading state from checkpoint /mnt/Lata/projects/samplefactory/train_dir/default_experiment/checkpoint_p0/checkpoint_000000983_4026368.pth... +[2023-03-07 16:58:39,199][488352] Num frames 100... +[2023-03-07 16:58:39,287][488352] Num frames 200... +[2023-03-07 16:58:39,376][488352] Num frames 300... +[2023-03-07 16:58:39,465][488352] Num frames 400... +[2023-03-07 16:58:39,560][488352] Avg episode rewards: #0: 6.480, true rewards: #0: 4.480 +[2023-03-07 16:58:39,561][488352] Avg episode reward: 6.480, avg true_objective: 4.480 +[2023-03-07 16:58:39,622][488352] Num frames 500... +[2023-03-07 16:58:39,712][488352] Num frames 600... +[2023-03-07 16:58:39,796][488352] Num frames 700... +[2023-03-07 16:58:39,880][488352] Num frames 800... +[2023-03-07 16:58:39,962][488352] Avg episode rewards: #0: 5.160, true rewards: #0: 4.160 +[2023-03-07 16:58:39,963][488352] Avg episode reward: 5.160, avg true_objective: 4.160 +[2023-03-07 16:58:40,034][488352] Num frames 900... +[2023-03-07 16:58:40,119][488352] Num frames 1000... +[2023-03-07 16:58:40,205][488352] Num frames 1100... +[2023-03-07 16:58:40,290][488352] Num frames 1200... +[2023-03-07 16:58:40,386][488352] Avg episode rewards: #0: 5.160, true rewards: #0: 4.160 +[2023-03-07 16:58:40,387][488352] Avg episode reward: 5.160, avg true_objective: 4.160 +[2023-03-07 16:58:40,431][488352] Num frames 1300... +[2023-03-07 16:58:40,515][488352] Num frames 1400... +[2023-03-07 16:58:40,598][488352] Num frames 1500... +[2023-03-07 16:58:40,682][488352] Num frames 1600... +[2023-03-07 16:58:40,770][488352] Num frames 1700... +[2023-03-07 16:58:40,857][488352] Num frames 1800... +[2023-03-07 16:58:40,946][488352] Num frames 1900... +[2023-03-07 16:58:41,033][488352] Num frames 2000... +[2023-03-07 16:58:41,156][488352] Avg episode rewards: #0: 7.450, true rewards: #0: 5.200 +[2023-03-07 16:58:41,156][488352] Avg episode reward: 7.450, avg true_objective: 5.200 +[2023-03-07 16:58:41,181][488352] Num frames 2100... +[2023-03-07 16:58:41,281][488352] Num frames 2200... +[2023-03-07 16:58:41,365][488352] Num frames 2300... +[2023-03-07 16:58:41,450][488352] Num frames 2400... +[2023-03-07 16:58:41,532][488352] Avg episode rewards: #0: 6.864, true rewards: #0: 4.864 +[2023-03-07 16:58:41,533][488352] Avg episode reward: 6.864, avg true_objective: 4.864 +[2023-03-07 16:58:41,603][488352] Num frames 2500... +[2023-03-07 16:58:41,688][488352] Num frames 2600... +[2023-03-07 16:58:41,774][488352] Num frames 2700... +[2023-03-07 16:58:41,860][488352] Num frames 2800... +[2023-03-07 16:58:41,945][488352] Num frames 2900... +[2023-03-07 16:58:42,030][488352] Num frames 3000... +[2023-03-07 16:58:42,117][488352] Num frames 3100... +[2023-03-07 16:58:42,202][488352] Avg episode rewards: #0: 7.560, true rewards: #0: 5.227 +[2023-03-07 16:58:42,203][488352] Avg episode reward: 7.560, avg true_objective: 5.227 +[2023-03-07 16:58:42,260][488352] Num frames 3200... +[2023-03-07 16:58:42,345][488352] Num frames 3300... +[2023-03-07 16:58:42,430][488352] Num frames 3400... +[2023-03-07 16:58:42,517][488352] Num frames 3500... +[2023-03-07 16:58:42,588][488352] Avg episode rewards: #0: 7.029, true rewards: #0: 5.029 +[2023-03-07 16:58:42,588][488352] Avg episode reward: 7.029, avg true_objective: 5.029 +[2023-03-07 16:58:42,672][488352] Num frames 3600... +[2023-03-07 16:58:42,757][488352] Num frames 3700... +[2023-03-07 16:58:42,844][488352] Num frames 3800... +[2023-03-07 16:58:42,929][488352] Num frames 3900... +[2023-03-07 16:58:43,043][488352] Avg episode rewards: #0: 6.835, true rewards: #0: 4.960 +[2023-03-07 16:58:43,044][488352] Avg episode reward: 6.835, avg true_objective: 4.960 +[2023-03-07 16:58:43,086][488352] Num frames 4000... +[2023-03-07 16:58:43,178][488352] Num frames 4100... +[2023-03-07 16:58:43,265][488352] Num frames 4200... +[2023-03-07 16:58:43,351][488352] Num frames 4300... +[2023-03-07 16:58:43,477][488352] Avg episode rewards: #0: 6.649, true rewards: #0: 4.871 +[2023-03-07 16:58:43,478][488352] Avg episode reward: 6.649, avg true_objective: 4.871 +[2023-03-07 16:58:43,500][488352] Num frames 4400... +[2023-03-07 16:58:43,601][488352] Num frames 4500... +[2023-03-07 16:58:43,690][488352] Num frames 4600... +[2023-03-07 16:58:43,778][488352] Num frames 4700... +[2023-03-07 16:58:43,866][488352] Num frames 4800... +[2023-03-07 16:58:43,952][488352] Num frames 4900... +[2023-03-07 16:58:44,041][488352] Num frames 5000... +[2023-03-07 16:58:44,128][488352] Num frames 5100... +[2023-03-07 16:58:44,258][488352] Avg episode rewards: #0: 7.387, true rewards: #0: 5.187 +[2023-03-07 16:58:44,259][488352] Avg episode reward: 7.387, avg true_objective: 5.187 +[2023-03-07 16:58:50,297][488352] Replay video saved to /mnt/Lata/projects/samplefactory/train_dir/default_experiment/replay.mp4! +[2023-03-07 16:59:19,347][488352] The model has been pushed to https://huggingface.co/Rolo/doom_health_threads +[2023-03-07 17:03:31,561][488352] Environment doom_basic already registered, overwriting... +[2023-03-07 17:03:31,564][488352] Environment doom_two_colors_easy already registered, overwriting... +[2023-03-07 17:03:31,609][488352] Environment doom_two_colors_hard already registered, overwriting... +[2023-03-07 17:03:31,610][488352] Environment doom_dm already registered, overwriting... +[2023-03-07 17:03:31,612][488352] Environment doom_dwango5 already registered, overwriting... +[2023-03-07 17:03:31,612][488352] Environment doom_my_way_home_flat_actions already registered, overwriting... +[2023-03-07 17:03:31,613][488352] Environment doom_defend_the_center_flat_actions already registered, overwriting... +[2023-03-07 17:03:31,614][488352] Environment doom_my_way_home already registered, overwriting... +[2023-03-07 17:03:31,615][488352] Environment doom_deadly_corridor already registered, overwriting... +[2023-03-07 17:03:31,616][488352] Environment doom_defend_the_center already registered, overwriting... +[2023-03-07 17:03:31,616][488352] Environment doom_defend_the_line already registered, overwriting... +[2023-03-07 17:03:31,617][488352] Environment doom_health_gathering already registered, overwriting... +[2023-03-07 17:03:31,618][488352] Environment doom_health_gathering_supreme already registered, overwriting... +[2023-03-07 17:03:31,619][488352] Environment doom_battle already registered, overwriting... +[2023-03-07 17:03:31,620][488352] Environment doom_battle2 already registered, overwriting... +[2023-03-07 17:03:31,620][488352] Environment doom_duel_bots already registered, overwriting... +[2023-03-07 17:03:31,621][488352] Environment doom_deathmatch_bots already registered, overwriting... +[2023-03-07 17:03:31,622][488352] Environment doom_duel already registered, overwriting... +[2023-03-07 17:03:31,623][488352] Environment doom_deathmatch_full already registered, overwriting... +[2023-03-07 17:03:31,624][488352] Environment doom_benchmark already registered, overwriting... +[2023-03-07 17:03:31,625][488352] register_encoder_factory: +[2023-03-07 17:03:31,657][488352] Saved parameter configuration for experiment doom_health_threads_bigger not found! +[2023-03-07 17:03:31,658][488352] Starting experiment from scratch! +[2023-03-07 17:03:31,663][488352] Experiment dir /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads_bigger already exists! +[2023-03-07 17:03:31,664][488352] Resuming existing experiment from /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads_bigger... +[2023-03-07 17:03:31,665][488352] Weights and Biases integration disabled +[2023-03-07 17:03:31,667][488352] Environment var CUDA_VISIBLE_DEVICES is 0 + +[2023-03-07 17:03:32,749][488352] Automatically setting recurrence to 128 +[2023-03-07 17:03:32,750][488352] Starting experiment with the following configuration: +help=False +algo=APPO +env=doom_health_gathering_supreme +experiment=doom_health_threads_bigger +train_dir=/mnt/Lata/projects/samplefactory/train_dir +restart_behavior=resume +device=gpu +seed=None +num_policies=1 +async_rl=True +serial_mode=False +batched_sampling=False +num_batches_to_accumulate=2 +worker_num_splits=2 +policy_workers_per_policy=1 +max_policy_lag=1000 +num_workers=128 +num_envs_per_worker=128 +batch_size=8192 +num_batches_per_epoch=1 +num_epochs=1 +rollout=128 +recurrence=128 +shuffle_minibatches=False +gamma=0.99 +reward_scale=1.0 +reward_clip=1000.0 +value_bootstrap=False +normalize_returns=True +exploration_loss_coeff=0.001 +value_loss_coeff=0.5 +kl_loss_coeff=0.0 +exploration_loss=symmetric_kl +gae_lambda=0.95 +ppo_clip_ratio=0.1 +ppo_clip_value=0.2 +with_vtrace=False +vtrace_rho=1.0 +vtrace_c=1.0 +optimizer=adam +adam_eps=1e-06 +adam_beta1=0.9 +adam_beta2=0.999 +max_grad_norm=4.0 +learning_rate=0.0001 +lr_schedule=constant +lr_schedule_kl_threshold=0.008 +lr_adaptive_min=1e-06 +lr_adaptive_max=0.01 +obs_subtract_mean=0.0 +obs_scale=255.0 +normalize_input=True +normalize_input_keys=None +decorrelate_experience_max_seconds=0 +decorrelate_envs_on_one_worker=True +actor_worker_gpus=[] +set_workers_cpu_affinity=True +force_envs_single_thread=False +default_niceness=0 +log_to_file=True +experiment_summaries_interval=10 +flush_summaries_interval=30 +stats_avg=100 +summaries_use_frameskip=True +heartbeat_interval=20 +heartbeat_reporting_interval=600 +train_for_env_steps=50000000 +train_for_seconds=10000000000 +save_every_sec=120 +keep_checkpoints=2 +load_checkpoint_kind=latest +save_milestones_sec=-1 +save_best_every_sec=5 +save_best_metric=reward +save_best_after=100000 +benchmark=False +encoder_mlp_layers=[512, 512] +encoder_conv_architecture=convnet_simple +encoder_conv_mlp_layers=[512] +use_rnn=True +rnn_size=512 +rnn_type=gru +rnn_num_layers=1 +decoder_mlp_layers=[] +nonlinearity=elu +policy_initialization=orthogonal +policy_init_gain=1.0 +actor_critic_share_weights=True +adaptive_stddev=True +continuous_tanh_scale=0.0 +initial_stddev=1.0 +use_env_info_cache=False +env_gpu_actions=False +env_gpu_observations=True +env_frameskip=4 +env_framestack=1 +pixel_format=CHW +use_record_episode_statistics=False +with_wandb=False +wandb_user=None +wandb_project=sample_factory +wandb_group=None +wandb_job_type=SF +wandb_tags=[] +with_pbt=False +pbt_mix_policies_in_one_env=True +pbt_period_env_steps=5000000 +pbt_start_mutation=20000000 +pbt_replace_fraction=0.3 +pbt_mutation_rate=0.15 +pbt_replace_reward_gap=0.1 +pbt_replace_reward_gap_absolute=1e-06 +pbt_optimize_gamma=False +pbt_target_objective=true_objective +pbt_perturb_min=1.1 +pbt_perturb_max=1.5 +num_agents=-1 +num_humans=0 +num_bots=-1 +start_bot_difficulty=None +timelimit=None +res_w=128 +res_h=72 +wide_aspect_ratio=False +eval_env_frameskip=1 +fps=35 +command_line=--env=doom_health_gathering_supreme --num_workers=128 --num_envs_per_worker=128 --train_for_env_steps=50000000 --experiment=doom_health_threads_bigger --rollout=128 --batch_size=8192 +cli_args={'env': 'doom_health_gathering_supreme', 'experiment': 'doom_health_threads_bigger', 'num_workers': 128, 'num_envs_per_worker': 128, 'batch_size': 8192, 'rollout': 128, 'train_for_env_steps': 50000000} +git_hash=unknown +git_repo_name=not a git repository +[2023-03-07 17:03:32,751][488352] Saving configuration to /mnt/Lata/projects/samplefactory/train_dir/doom_health_threads_bigger/config.json... +[2023-03-07 17:03:32,753][488352] Rollout worker 0 uses device cpu +[2023-03-07 17:03:32,753][488352] Rollout worker 1 uses device cpu +[2023-03-07 17:03:32,754][488352] Rollout worker 2 uses device cpu +[2023-03-07 17:03:32,754][488352] Rollout worker 3 uses device cpu +[2023-03-07 17:03:32,755][488352] Rollout worker 4 uses device cpu +[2023-03-07 17:03:32,755][488352] Rollout worker 5 uses device cpu +[2023-03-07 17:03:32,756][488352] Rollout worker 6 uses device cpu +[2023-03-07 17:03:32,756][488352] Rollout worker 7 uses device cpu +[2023-03-07 17:03:32,757][488352] Rollout worker 8 uses device cpu +[2023-03-07 17:03:32,757][488352] Rollout worker 9 uses device cpu +[2023-03-07 17:03:32,758][488352] Rollout worker 10 uses device cpu +[2023-03-07 17:03:32,758][488352] Rollout worker 11 uses device cpu +[2023-03-07 17:03:32,759][488352] Rollout worker 12 uses device cpu +[2023-03-07 17:03:32,759][488352] Rollout worker 13 uses device cpu +[2023-03-07 17:03:32,760][488352] Rollout worker 14 uses device cpu +[2023-03-07 17:03:32,760][488352] Rollout worker 15 uses device cpu +[2023-03-07 17:03:32,761][488352] Rollout worker 16 uses device cpu +[2023-03-07 17:03:32,761][488352] Rollout worker 17 uses device cpu +[2023-03-07 17:03:32,763][488352] Rollout worker 18 uses device cpu +[2023-03-07 17:03:32,764][488352] Rollout worker 19 uses device cpu +[2023-03-07 17:03:32,764][488352] Rollout worker 20 uses device cpu +[2023-03-07 17:03:32,765][488352] Rollout worker 21 uses device cpu +[2023-03-07 17:03:32,765][488352] Rollout worker 22 uses device cpu +[2023-03-07 17:03:32,766][488352] Rollout worker 23 uses device cpu +[2023-03-07 17:03:32,766][488352] Rollout worker 24 uses device cpu +[2023-03-07 17:03:32,766][488352] Rollout worker 25 uses device cpu +[2023-03-07 17:03:32,767][488352] Rollout worker 26 uses device cpu +[2023-03-07 17:03:32,767][488352] Rollout worker 27 uses device cpu +[2023-03-07 17:03:32,768][488352] Rollout worker 28 uses device cpu +[2023-03-07 17:03:32,768][488352] Rollout worker 29 uses device cpu +[2023-03-07 17:03:32,768][488352] Rollout worker 30 uses device cpu +[2023-03-07 17:03:32,769][488352] Rollout worker 31 uses device cpu +[2023-03-07 17:03:32,769][488352] Rollout worker 32 uses device cpu +[2023-03-07 17:03:32,770][488352] Rollout worker 33 uses device cpu +[2023-03-07 17:03:32,770][488352] Rollout worker 34 uses device cpu +[2023-03-07 17:03:32,770][488352] Rollout worker 35 uses device cpu +[2023-03-07 17:03:32,771][488352] Rollout worker 36 uses device cpu +[2023-03-07 17:03:32,771][488352] Rollout worker 37 uses device cpu +[2023-03-07 17:03:32,772][488352] Rollout worker 38 uses device cpu +[2023-03-07 17:03:32,772][488352] Rollout worker 39 uses device cpu +[2023-03-07 17:03:32,772][488352] Rollout worker 40 uses device cpu +[2023-03-07 17:03:32,773][488352] Rollout worker 41 uses device cpu +[2023-03-07 17:03:32,773][488352] Rollout worker 42 uses device cpu +[2023-03-07 17:03:32,774][488352] Rollout worker 43 uses device cpu +[2023-03-07 17:03:32,774][488352] Rollout worker 44 uses device cpu +[2023-03-07 17:03:32,774][488352] Rollout worker 45 uses device cpu +[2023-03-07 17:03:32,775][488352] Rollout worker 46 uses device cpu +[2023-03-07 17:03:32,775][488352] Rollout worker 47 uses device cpu +[2023-03-07 17:03:32,776][488352] Rollout worker 48 uses device cpu +[2023-03-07 17:03:32,776][488352] Rollout worker 49 uses device cpu +[2023-03-07 17:03:32,776][488352] Rollout worker 50 uses device cpu +[2023-03-07 17:03:32,777][488352] Rollout worker 51 uses device cpu +[2023-03-07 17:03:32,777][488352] Rollout worker 52 uses device cpu +[2023-03-07 17:03:32,778][488352] Rollout worker 53 uses device cpu +[2023-03-07 17:03:32,778][488352] Rollout worker 54 uses device cpu +[2023-03-07 17:03:32,779][488352] Rollout worker 55 uses device cpu +[2023-03-07 17:03:32,779][488352] Rollout worker 56 uses device cpu +[2023-03-07 17:03:32,780][488352] Rollout worker 57 uses device cpu +[2023-03-07 17:03:32,780][488352] Rollout worker 58 uses device cpu +[2023-03-07 17:03:32,781][488352] Rollout worker 59 uses device cpu +[2023-03-07 17:03:32,782][488352] Rollout worker 60 uses device cpu +[2023-03-07 17:03:32,782][488352] Rollout worker 61 uses device cpu +[2023-03-07 17:03:32,783][488352] Rollout worker 62 uses device cpu +[2023-03-07 17:03:32,783][488352] Rollout worker 63 uses device cpu +[2023-03-07 17:03:32,784][488352] Rollout worker 64 uses device cpu +[2023-03-07 17:03:32,785][488352] Rollout worker 65 uses device cpu +[2023-03-07 17:03:32,785][488352] Rollout worker 66 uses device cpu +[2023-03-07 17:03:32,786][488352] Rollout worker 67 uses device cpu +[2023-03-07 17:03:32,786][488352] Rollout worker 68 uses device cpu +[2023-03-07 17:03:32,787][488352] Rollout worker 69 uses device cpu +[2023-03-07 17:03:32,787][488352] Rollout worker 70 uses device cpu +[2023-03-07 17:03:32,788][488352] Rollout worker 71 uses device cpu +[2023-03-07 17:03:32,789][488352] Rollout worker 72 uses device cpu +[2023-03-07 17:03:32,789][488352] Rollout worker 73 uses device cpu +[2023-03-07 17:03:32,790][488352] Rollout worker 74 uses device cpu +[2023-03-07 17:03:32,790][488352] Rollout worker 75 uses device cpu +[2023-03-07 17:03:32,791][488352] Rollout worker 76 uses device cpu +[2023-03-07 17:03:32,791][488352] Rollout worker 77 uses device cpu +[2023-03-07 17:03:32,792][488352] Rollout worker 78 uses device cpu +[2023-03-07 17:03:32,792][488352] Rollout worker 79 uses device cpu +[2023-03-07 17:03:32,792][488352] Rollout worker 80 uses device cpu +[2023-03-07 17:03:32,793][488352] Rollout worker 81 uses device cpu +[2023-03-07 17:03:32,793][488352] Rollout worker 82 uses device cpu +[2023-03-07 17:03:32,794][488352] Rollout worker 83 uses device cpu +[2023-03-07 17:03:32,794][488352] Rollout worker 84 uses device cpu +[2023-03-07 17:03:32,794][488352] Rollout worker 85 uses device cpu +[2023-03-07 17:03:32,795][488352] Rollout worker 86 uses device cpu +[2023-03-07 17:03:32,795][488352] Rollout worker 87 uses device cpu +[2023-03-07 17:03:32,796][488352] Rollout worker 88 uses device cpu +[2023-03-07 17:03:32,796][488352] Rollout worker 89 uses device cpu +[2023-03-07 17:03:32,796][488352] Rollout worker 90 uses device cpu +[2023-03-07 17:03:32,797][488352] Rollout worker 91 uses device cpu +[2023-03-07 17:03:32,797][488352] Rollout worker 92 uses device cpu +[2023-03-07 17:03:32,798][488352] Rollout worker 93 uses device cpu +[2023-03-07 17:03:32,798][488352] Rollout worker 94 uses device cpu +[2023-03-07 17:03:32,804][488352] Rollout worker 95 uses device cpu +[2023-03-07 17:03:32,804][488352] Rollout worker 96 uses device cpu +[2023-03-07 17:03:32,805][488352] Rollout worker 97 uses device cpu +[2023-03-07 17:03:32,805][488352] Rollout worker 98 uses device cpu +[2023-03-07 17:03:32,806][488352] Rollout worker 99 uses device cpu +[2023-03-07 17:03:32,806][488352] Rollout worker 100 uses device cpu +[2023-03-07 17:03:32,806][488352] Rollout worker 101 uses device cpu +[2023-03-07 17:03:32,807][488352] Rollout worker 102 uses device cpu +[2023-03-07 17:03:32,807][488352] Rollout worker 103 uses device cpu +[2023-03-07 17:03:32,808][488352] Rollout worker 104 uses device cpu +[2023-03-07 17:03:32,808][488352] Rollout worker 105 uses device cpu +[2023-03-07 17:03:32,808][488352] Rollout worker 106 uses device cpu +[2023-03-07 17:03:32,809][488352] Rollout worker 107 uses device cpu +[2023-03-07 17:03:32,809][488352] Rollout worker 108 uses device cpu +[2023-03-07 17:03:32,810][488352] Rollout worker 109 uses device cpu +[2023-03-07 17:03:32,810][488352] Rollout worker 110 uses device cpu +[2023-03-07 17:03:32,810][488352] Rollout worker 111 uses device cpu +[2023-03-07 17:03:32,811][488352] Rollout worker 112 uses device cpu +[2023-03-07 17:03:32,811][488352] Rollout worker 113 uses device cpu +[2023-03-07 17:03:32,812][488352] Rollout worker 114 uses device cpu +[2023-03-07 17:03:32,812][488352] Rollout worker 115 uses device cpu +[2023-03-07 17:03:32,813][488352] Rollout worker 116 uses device cpu +[2023-03-07 17:03:32,813][488352] Rollout worker 117 uses device cpu +[2023-03-07 17:03:32,813][488352] Rollout worker 118 uses device cpu +[2023-03-07 17:03:32,814][488352] Rollout worker 119 uses device cpu +[2023-03-07 17:03:32,816][488352] Rollout worker 120 uses device cpu +[2023-03-07 17:03:32,816][488352] Rollout worker 121 uses device cpu +[2023-03-07 17:03:32,817][488352] Rollout worker 122 uses device cpu +[2023-03-07 17:03:32,817][488352] Rollout worker 123 uses device cpu +[2023-03-07 17:03:32,817][488352] Rollout worker 124 uses device cpu +[2023-03-07 17:03:32,818][488352] Rollout worker 125 uses device cpu +[2023-03-07 17:03:32,819][488352] Rollout worker 126 uses device cpu +[2023-03-07 17:03:32,819][488352] Rollout worker 127 uses device cpu