v5.28

SoccerTwos.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:202d27e9b185d21c323ee28ace81dfb33f1c3ecaca500eb0fad56d4c8da1b1db
+oid sha256:693e78241b1c621eb93840614cc30d6774aae00e0a21ee2ba50c41320a9f0cba
 size 2815689

SoccerTwos/SoccerTwos-27499864.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4e64a090b692673693a3117b17b0489487a39ec4f7715d09cd5c0739418ea3ad
-size 34728689

SoccerTwos/SoccerTwos-27999982.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1344942dab22c6e7ff69dc58293fa8e966d7c4c37384c73582c64079f205b436
-size 34728689

SoccerTwos/SoccerTwos-28477740.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:202d27e9b185d21c323ee28ace81dfb33f1c3ecaca500eb0fad56d4c8da1b1db
-size 2815689

SoccerTwos/SoccerTwos-28477740.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b45e82a0ecf22630c5a3a855bd4421054dc4baee92da623a59a52ce074e6ce4d
-size 34728689

SoccerTwos/{SoccerTwos-26499995.onnx → SoccerTwos-31999878.onnx}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2ce18e5f4c53b1dd5c379fa8822348c8065ccdae2769a9a4db7503ff68d522e
+oid sha256:2523ce43e63cbd68fc969bc3fcf28dad06cba673f5e3d6a3254c27173e051aef
 size 2815689

SoccerTwos/{SoccerTwos-26499995.pt → SoccerTwos-31999878.pt}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be60d96bf0a8fcc51db2e308530bdc753106dd18bfb300db95dbdf75e1564bf0
-size 34728689
+oid sha256:e826095b182fc6be7f89d0abf6955e86f652c2867540092e02203ae9e2a5860f
+size 35525901

SoccerTwos/{SoccerTwos-27499864.onnx → SoccerTwos-32499928.onnx}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d439691acfe5223f9548ef594388216f78634665d706eb087be7108a06048a5
+oid sha256:7d1f31fb6ea15cd9042054c000696cbbcfb927df8fc33463acaac6c4c818ef2f
 size 2815689

SoccerTwos/{SoccerTwos-26999977.pt → SoccerTwos-32499928.pt}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecb2c7ab1fe045868b5dbf4de308c9dcb328b9e6dc27ac09506e4d922599dd7a
-size 34728689
+oid sha256:eb3b5e6795c6808a6b1c5f3ff401483df0d2efca9b27d5bb191f4764fd1dfaad
+size 35525901

SoccerTwos/{SoccerTwos-26999977.onnx → SoccerTwos-32999999.onnx}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78f368728b0a1e786778e6d82352a6901eb152e5ebd56cebb82f13c117cb210c
+oid sha256:80e9bf911d475e6a0eb9bd23f9d44ebed1e1658acf51945e75c7782ffd5268af
 size 2815689

SoccerTwos/SoccerTwos-32999999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605980ab99386002a6e8058bb25cf1dbc9afe2d9309e50d6c62f1197fb5aa7c0
+size 35525901

SoccerTwos/{SoccerTwos-27999982.onnx → SoccerTwos-33499982.onnx}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c188b966c85ecccfe488de2995a282941c551e9a93648a5fd0b3bf40ccd41108
+oid sha256:7ba39e2a4ecad130abeb21d661f822cb70b6af2369e0d51f2781c7396b73cd62
 size 2815689

SoccerTwos/SoccerTwos-33499982.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b02564f86db0026ca1f1d41845e9b5cb0a491ee6d507baae7e9bc161076534
+size 35525901

SoccerTwos/SoccerTwos-33646596.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693e78241b1c621eb93840614cc30d6774aae00e0a21ee2ba50c41320a9f0cba
+size 2815689

SoccerTwos/SoccerTwos-33646596.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d8dd78178fede114090ae8138e9fcf2c23e1de7efed7d9567e9eee82e78b11b
+size 35525901

SoccerTwos/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b45e82a0ecf22630c5a3a855bd4421054dc4baee92da623a59a52ce074e6ce4d
-size 34728689
+oid sha256:2d8dd78178fede114090ae8138e9fcf2c23e1de7efed7d9567e9eee82e78b11b
+size 35525901

SoccerTwos/events.out.tfevents.1680172555.linux.fritz.box.18610.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e325fc91e09ae0a47bf1293cce940c6547a2690a7524ea340faa1fd9c6b08c3
+size 5349667

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 4096, "buffer_size": 204800, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 3, "vis_encode_type": "fully_connected", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.995, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 30000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 10000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos/SoccerTwos.exe", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos5", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 4096, "buffer_size": 204800, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 3, "vis_encode_type": "fully_connected", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.995, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "curiosity": {"gamma": 0.9, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0003, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 90000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 10000, "window": 30, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos/SoccerTwos.exe", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos5", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -34,10 +34,23 @@ behaviors:
           memory: null
           goal_conditioning_type: hyper
           deterministic: false
+      curiosity:
+        gamma: 0.9
+        strength: 0.01
+        network_settings:
+          normalize: false
+          hidden_units: 128
+          num_layers: 2
+          vis_encode_type: simple
+          memory: null
+          goal_conditioning_type: hyper
+          deterministic: false
+        learning_rate: 0.0003
+        encoding_size: null
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 30000000
+    max_steps: 90000000
     time_horizon: 1000
     summary_freq: 10000
     threaded: false
@@ -45,7 +58,7 @@ behaviors:
       save_steps: 50000
       team_change: 250000
       swap_steps: 10000
-      window: 10
+      window: 30
       play_against_latest_model_ratio: 0.5
       initial_elo: 1200.0
     behavioral_cloning: null

run_logs/Player-0.log

@@ -11,7 +11,7 @@ NullGfxDevice:
     Renderer: Null Device
     Vendor:   Unity Technologies
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.155 seconds
+- Completed reload, in  0.081 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -36,7 +36,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.744856 ms
+UnloadTime: 0.668976 ms
 Registered Communicator in Agent.
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
@@ -45,7 +45,7 @@ Memory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [8.0 KB-16.0 KB]: 183132 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [8.0 KB-16.0 KB]: 97182 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.1 MB
@@ -73,7 +73,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 0]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0 B
+      Peak Allocated Bytes 0.7 KB
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 10]
       Initial Block Size 32.0 KB
@@ -143,7 +143,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 2]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.7 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 3]
       Initial Block Size 32.0 KB
@@ -177,16 +177,16 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [4.0 MB-8.0 MB]: 183133 frames
+      Peak usage frame count: [4.0 MB-8.0 MB]: 97183 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 4.6 MB
+      Peak Allocated memory 4.8 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 183133 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 97183 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 17.6 MB
+      Peak Allocated memory 17.4 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -215,13 +215,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 183132 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 97182 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 65.6 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [128.0 KB-256.0 KB]: 183133 frames
+      Peak usage frame count: [128.0 KB-256.0 KB]: 97183 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 173.5 KB
@@ -233,16 +233,16 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 183133 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 97183 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 0.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 183132 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 97182 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
-      Peak Allocated memory 4.4 MB
+      Peak Allocated memory 4.5 MB
       Peak Large allocation bytes 0 B
 [ALLOC_TYPETREE] Dual Thread Allocator
   Peak main deferred allocation count 0
@@ -251,13 +251,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 183133 frames
+      Peak usage frame count: [0-1.0 KB]: 97183 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 183132 frames, [2.0 KB-4.0 KB]: 1 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 97182 frames, [2.0 KB-4.0 KB]: 1 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 2.2 KB

run_logs/timers.json

@@ -2,201 +2,261 @@
     "name": "root",
     "gauges": {
         "SoccerTwos.Policy.Entropy.mean": {
-            "value": 1.7348119020462036,
-            "min": 1.7348119020462036,
-            "max": 2.1338491439819336,
-            "count": 974
+            "value": 1.6544617414474487,
+            "min": 1.6383187770843506,
+            "max": 1.771085500717163,
+            "count": 517
         },
         "SoccerTwos.Policy.Entropy.sum": {
-            "value": 29144.83984375,
-            "min": 1606.896484375,
-            "max": 48490.890625,
-            "count": 974
+            "value": 32295.09375,
+            "min": 10663.513671875,
+            "max": 39872.62890625,
+            "count": 517
         },
         "SoccerTwos.Environment.EpisodeLength.mean": {
-            "value": 55.86206896551724,
-            "min": 22.0,
-            "max": 85.24137931034483,
-            "count": 974
+            "value": 52.989010989010985,
+            "min": 36.21212121212121,
+            "max": 75.50769230769231,
+            "count": 517
         },
         "SoccerTwos.Environment.EpisodeLength.sum": {
-            "value": 19440.0,
-            "min": 88.0,
-            "max": 21752.0,
-            "count": 974
+            "value": 19288.0,
+            "min": 4780.0,
+            "max": 20732.0,
+            "count": 517
         },
         "SoccerTwos.Self-play.ELO.mean": {
-            "value": 1761.567949531139,
-            "min": 1718.6108505610346,
-            "max": 1803.8036335559661,
-            "count": 974
+            "value": 1783.9345993395016,
+            "min": 1739.4789078559804,
+            "max": 1819.578822229663,
+            "count": 517
         },
         "SoccerTwos.Self-play.ELO.sum": {
-            "value": 306512.82321841817,
-            "min": 3474.259222005546,
-            "max": 447654.8648728988,
-            "count": 974
-        },
-        "SoccerTwos.IsTraining.mean": {
-            "value": 1.0,
-            "min": 1.0,
-            "max": 1.0,
-            "count": 974
-        },
-        "SoccerTwos.IsTraining.sum": {
-            "value": 1.0,
-            "min": 1.0,
-            "max": 1.0,
-            "count": 974
+            "value": 324676.09707978927,
+            "min": 115798.43448098522,
+            "max": 439690.6603627888,
+            "count": 517
         },
         "SoccerTwos.Step.mean": {
-            "value": 28469980.0,
-            "min": 18749904.0,
-            "max": 28469980.0,
-            "count": 973
+            "value": 33639986.0,
+            "min": 28479978.0,
+            "max": 33639986.0,
+            "count": 517
         },
         "SoccerTwos.Step.sum": {
-            "value": 28469980.0,
-            "min": 18749904.0,
-            "max": 28469980.0,
-            "count": 973
+            "value": 33639986.0,
+            "min": 28479978.0,
+            "max": 33639986.0,
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.mean": {
-            "value": -0.029856232926249504,
-            "min": -0.13772714138031006,
-            "max": 0.06465017795562744,
-            "count": 973
+            "value": -0.018630528822541237,
+            "min": -0.11188879609107971,
+            "max": 0.07426375895738602,
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.sum": {
-            "value": -5.2248406410217285,
-            "min": -23.138160705566406,
-            "max": 12.154233932495117,
-            "count": 973
+            "value": -3.390756130218506,
+            "min": -21.461830139160156,
+            "max": 13.765193939208984,
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.mean": {
-            "value": -0.02988698147237301,
-            "min": -0.14075849950313568,
-            "max": 0.06273878365755081,
-            "count": 973
+            "value": -0.018748609349131584,
+            "min": -0.10551068186759949,
+            "max": 0.07412952184677124,
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.sum": {
-            "value": -5.230221748352051,
-            "min": -23.647428512573242,
-            "max": 11.794891357421875,
-            "count": 973
+            "value": -3.4122469425201416,
+            "min": -20.84265899658203,
+            "max": 14.455087661743164,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityBaselineEstimate.mean": {
+            "value": 0.006533732172101736,
+            "min": 0.006031353026628494,
+            "max": 0.09678366780281067,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityBaselineEstimate.sum": {
+            "value": 1.1891392469406128,
+            "min": 0.9304066896438599,
+            "max": 20.50408935546875,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityValueEstimate.mean": {
+            "value": 0.006480127107352018,
+            "min": 0.005918653681874275,
+            "max": 0.09726906567811966,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityValueEstimate.sum": {
+            "value": 1.1793831586837769,
+            "min": 0.913135290145874,
+            "max": 20.615158081054688,
+            "count": 517
         },
         "SoccerTwos.Environment.CumulativeReward.mean": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 973
+            "count": 517
         },
         "SoccerTwos.Environment.CumulativeReward.sum": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 973
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicReward.mean": {
-            "value": -0.05570971488952637,
-            "min": -0.37514444523387486,
-            "max": 0.24607526935556884,
-            "count": 973
+            "value": -0.07899780391336797,
+            "min": -0.32739889358288676,
+            "max": 0.33040844974383504,
+            "count": 517
         },
         "SoccerTwos.Policy.ExtrinsicReward.sum": {
-            "value": -9.749200105667114,
-            "min": -67.52600014209747,
-            "max": 45.7700001001358,
-            "count": 973
+            "value": -14.377600312232971,
+            "min": -59.2591997385025,
+            "max": 47.84479957818985,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityReward.mean": {
+            "value": 0.03525229465794637,
+            "min": 0.0,
+            "max": 0.24388014633286667,
+            "count": 517
+        },
+        "SoccerTwos.Policy.CuriosityReward.sum": {
+            "value": 6.415917627746239,
+            "min": 0.0,
+            "max": 36.89434967190027,
+            "count": 517
         },
         "SoccerTwos.Environment.GroupCumulativeReward.mean": {
-            "value": -0.05570971488952637,
-            "min": -0.37514444523387486,
-            "max": 0.24607526935556884,
-            "count": 973
+            "value": -0.07899780391336797,
+            "min": -0.32739889358288676,
+            "max": 0.33040844974383504,
+            "count": 517
         },
         "SoccerTwos.Environment.GroupCumulativeReward.sum": {
-            "value": -9.749200105667114,
-            "min": -67.52600014209747,
-            "max": 45.7700001001358,
-            "count": 973
+            "value": -14.377600312232971,
+            "min": -59.2591997385025,
+            "max": 47.84479957818985,
+            "count": 517
+        },
+        "SoccerTwos.IsTraining.mean": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 517
+        },
+        "SoccerTwos.IsTraining.sum": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 517
         },
         "SoccerTwos.Losses.PolicyLoss.mean": {
-            "value": 0.012733476008288563,
-            "min": 0.011228136497666128,
-            "max": 0.013726944572093391,
-            "count": 47
+            "value": 0.01082317113565902,
+            "min": 0.010641817656190445,
+            "max": 0.013228910765095254,
+            "count": 25
         },
         "SoccerTwos.Losses.PolicyLoss.sum": {
-            "value": 0.012733476008288563,
-            "min": 0.011228136497666128,
-            "max": 0.013726944572093391,
-            "count": 47
+            "value": 0.01082317113565902,
+            "min": 0.010641817656190445,
+            "max": 0.013228910765095254,
+            "count": 25
         },
         "SoccerTwos.Losses.ValueLoss.mean": {
-            "value": 0.12843565613031388,
-            "min": 0.11572077537576357,
-            "max": 0.12922100678086282,
-            "count": 47
+            "value": 0.0611632056782643,
+            "min": 0.058945538699626925,
+            "max": 0.06389266346891721,
+            "count": 25
         },
         "SoccerTwos.Losses.ValueLoss.sum": {
-            "value": 0.12843565613031388,
-            "min": 0.11572077537576357,
-            "max": 0.12922100678086282,
-            "count": 47
+            "value": 0.0611632056782643,
+            "min": 0.058945538699626925,
+            "max": 0.06389266346891721,
+            "count": 25
         },
         "SoccerTwos.Losses.BaselineLoss.mean": {
-            "value": 0.12908015037576356,
-            "min": 0.11725246061881384,
-            "max": 0.12977993567784626,
-            "count": 47
+            "value": 0.06231740226348241,
+            "min": 0.06012462628384431,
+            "max": 0.06539666548371315,
+            "count": 25
         },
         "SoccerTwos.Losses.BaselineLoss.sum": {
-            "value": 0.12908015037576356,
-            "min": 0.11725246061881384,
-            "max": 0.12977993567784626,
-            "count": 47
+            "value": 0.06231740226348241,
+            "min": 0.06012462628384431,
+            "max": 0.06539666548371315,
+            "count": 25
         },
         "SoccerTwos.Policy.LearningRate.mean": {
-            "value": 1.6292574569173326e-05,
-            "min": 1.6292574569173326e-05,
-            "max": 0.00011054928315026,
-            "count": 47
+            "value": 0.0001879993440002311,
+            "min": 0.0001879993440002311,
+            "max": 0.00020439037186988668,
+            "count": 25
         },
         "SoccerTwos.Policy.LearningRate.sum": {
-            "value": 1.6292574569173326e-05,
-            "min": 1.6292574569173326e-05,
-            "max": 0.00011054928315026,
-            "count": 47
+            "value": 0.0001879993440002311,
+            "min": 0.0001879993440002311,
+            "max": 0.00020439037186988668,
+            "count": 25
         },
         "SoccerTwos.Policy.Epsilon.mean": {
-            "value": 0.10543082666666669,
-            "min": 0.10543082666666669,
-            "max": 0.13684974,
-            "count": 47
+            "value": 0.16266643555555557,
+            "min": 0.16266643555555557,
+            "max": 0.1681301133333333,
+            "count": 25
         },
         "SoccerTwos.Policy.Epsilon.sum": {
-            "value": 0.10543082666666669,
-            "min": 0.10543082666666669,
-            "max": 0.13684974,
-            "count": 47
+            "value": 0.16266643555555557,
+            "min": 0.16266643555555557,
+            "max": 0.1681301133333333,
+            "count": 25
         },
         "SoccerTwos.Policy.Beta.mean": {
-            "value": 0.00028099825066666665,
-            "min": 0.00028099825066666665,
-            "max": 0.001848802026,
-            "count": 47
+            "value": 0.003137055134222223,
+            "min": 0.003137055134222223,
+            "max": 0.003409692655333334,
+            "count": 25
         },
         "SoccerTwos.Policy.Beta.sum": {
-            "value": 0.00028099825066666665,
-            "min": 0.00028099825066666665,
-            "max": 0.001848802026,
-            "count": 47
+            "value": 0.003137055134222223,
+            "min": 0.003137055134222223,
+            "max": 0.003409692655333334,
+            "count": 25
+        },
+        "SoccerTwos.Losses.CuriosityForwardLoss.mean": {
+            "value": 0.06363795998195808,
+            "min": 0.06363795998195808,
+            "max": 0.4287375779946645,
+            "count": 25
+        },
+        "SoccerTwos.Losses.CuriosityForwardLoss.sum": {
+            "value": 0.06363795998195808,
+            "min": 0.06363795998195808,
+            "max": 0.4287375779946645,
+            "count": 25
+        },
+        "SoccerTwos.Losses.CuriosityInverseLoss.mean": {
+            "value": 1.1895974318186442,
+            "min": 1.1853395628929138,
+            "max": 2.4184607744216917,
+            "count": 25
+        },
+        "SoccerTwos.Losses.CuriosityInverseLoss.sum": {
+            "value": 1.1895974318186442,
+            "min": 1.1853395628929138,
+            "max": 2.4184607744216917,
+            "count": 25
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1680129716",
+        "start_time_seconds": "1680172554",
         "python_version": "3.9.16 | packaged by conda-forge | (main, Feb  1 2023, 21:39:03) \n[GCC 11.3.0]",
         "command_line_arguments": "/home/ringo/mambaforge/envs/hfrl/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos/SoccerTwos.exe --run-id=SoccerTwos5 --no-graphics --resume",
         "mlagents_version": "0.31.0.dev0",
@@ -204,59 +264,59 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "1.11.0+cu102",
         "numpy_version": "1.21.2",
-        "end_time_seconds": "1680172438"
+        "end_time_seconds": "1680191712"
     },
-    "total": 42722.574667928,
+    "total": 19158.371048654983,
     "count": 1,
-    "self": 0.0759030100016389,
+    "self": 0.06926428701262921,
     "children": {
         "run_training.setup": {
-            "total": 0.025251507002394646,
+            "total": 0.012148113979492337,
             "count": 1,
-            "self": 0.025251507002394646
+            "self": 0.012148113979492337
         },
         "TrainerController.start_learning": {
-            "total": 42722.473513410994,
+            "total": 19158.28963625399,
             "count": 1,
-            "self": 19.858428147112136,
+            "self": 8.16942810654291,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 2.5805474780063378,
-                    "count": 40,
-                    "self": 2.5805474780063378
+                    "total": 1.2355314569722395,
+                    "count": 22,
+                    "self": 1.2355314569722395
                 },
                 "TrainerController.advance": {
-                    "total": 42699.72945140685,
-                    "count": 679082,
-                    "self": 23.77234519814374,
+                    "total": 19147.71053276246,
+                    "count": 361257,
+                    "self": 8.56901838304475,
                     "children": {
                         "env_step": {
-                            "total": 15710.199912431635,
-                            "count": 679082,
-                            "self": 12746.711335640968,
+                            "total": 6734.64510125207,
+                            "count": 361257,
+                            "self": 5508.992406987061,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 2952.1668921852106,
-                                    "count": 679082,
-                                    "self": 136.18300451725372,
+                                    "total": 1220.7081321968872,
+                                    "count": 361257,
+                                    "self": 53.134995022497606,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 2815.983887667957,
-                                            "count": 1220900,
-                                            "self": 2815.983887667957
+                                            "total": 1167.5731371743896,
+                                            "count": 647900,
+                                            "self": 1167.5731371743896
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 11.32168460545654,
-                                    "count": 679081,
+                                    "total": 4.944562068121741,
+                                    "count": 361257,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 42684.878815808435,
-                                            "count": 679081,
+                                            "total": 19140.3656414171,
+                                            "count": 361257,
                                             "is_parallel": true,
-                                            "self": 32150.423927959564,
+                                            "self": 14556.282447089936,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -265,48 +325,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.0065970009891316295,
+                                                            "total": 0.002159021998522803,
                                                             "count": 2,
                                                             "is_parallel": true,
-                                                            "self": 0.001290915985009633,
+                                                            "self": 0.0005102219292894006,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0053060850041219965,
+                                                                    "total": 0.0016488000692334026,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0053060850041219965
+                                                                    "self": 0.0016488000692334026
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.10075123800197616,
+                                                            "total": 0.03421102600987069,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0012871539947809651,
+                                                            "self": 0.0008790829742792994,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0007557589997304603,
+                                                                    "total": 0.0006237560010049492,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0007557589997304603
+                                                                    "self": 0.0006237560010049492
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.0864543719944777,
+                                                                    "total": 0.029803024022839963,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0864543719944777
+                                                                    "self": 0.029803024022839963
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.01225395301298704,
+                                                                    "total": 0.002905163011746481,
                                                                     "count": 2,
                                                                     "is_parallel": true,
-                                                                    "self": 0.007683153031393886,
+                                                                    "self": 0.0005354399618227035,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.004570799981593154,
+                                                                            "total": 0.0023697230499237776,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.004570799981593154
+                                                                            "self": 0.0023697230499237776
                                                                         }
                                                                     }
                                                                 }
@@ -315,48 +375,48 @@
                                                     }
                                                 },
                                                 "steps_from_proto": {
-                                                    "total": 0.10548005497548729,
-                                                    "count": 78,
+                                                    "total": 0.04302037303568795,
+                                                    "count": 42,
                                                     "is_parallel": true,
-                                                    "self": 0.019453318280284293,
+                                                    "self": 0.007955353416036814,
                                                     "children": {
                                                         "_process_rank_one_or_two_observation": {
-                                                            "total": 0.086026736695203,
-                                                            "count": 312,
+                                                            "total": 0.03506501961965114,
+                                                            "count": 168,
                                                             "is_parallel": true,
-                                                            "self": 0.086026736695203
+                                                            "self": 0.03506501961965114
                                                         }
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 10534.349407793896,
-                                                    "count": 679080,
+                                                    "total": 4584.040173954127,
+                                                    "count": 361256,
                                                     "is_parallel": true,
-                                                    "self": 671.891442082604,
+                                                    "self": 294.0414173450845,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 436.96546818055504,
-                                                            "count": 679080,
+                                                            "total": 205.20476716093253,
+                                                            "count": 361256,
                                                             "is_parallel": true,
-                                                            "self": 436.96546818055504
+                                                            "self": 205.20476716093253
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 7533.343012046884,
-                                                            "count": 679080,
+                                                            "total": 3238.665957780555,
+                                                            "count": 361256,
                                                             "is_parallel": true,
-                                                            "self": 7533.343012046884
+                                                            "self": 3238.665957780555
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 1892.149485483853,
-                                                            "count": 1358160,
+                                                            "total": 846.1280316675548,
+                                                            "count": 722512,
                                                             "is_parallel": true,
-                                                            "self": 372.890244903494,
+                                                            "self": 159.79457659239415,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 1519.259240580359,
-                                                                    "count": 5432640,
+                                                                    "total": 686.3334550751606,
+                                                                    "count": 2890048,
                                                                     "is_parallel": true,
-                                                                    "self": 1519.259240580359
+                                                                    "self": 686.3334550751606
                                                                 }
                                                             }
                                                         }
@@ -369,31 +429,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 26965.75719377707,
-                            "count": 679081,
-                            "self": 128.03335191983206,
+                            "total": 12404.496413127345,
+                            "count": 361257,
+                            "self": 50.02537562244106,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 3997.3622430362157,
-                                    "count": 679081,
-                                    "self": 3991.13507309719,
+                                    "total": 1759.4455397337733,
+                                    "count": 361257,
+                                    "self": 1757.0021228148253,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 6.227169939025771,
-                                            "count": 19,
-                                            "self": 6.227169939025771
+                                            "total": 2.44341691894806,
+                                            "count": 11,
+                                            "self": 2.44341691894806
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 22840.361598821022,
-                                    "count": 47,
-                                    "self": 1953.111261621816,
+                                    "total": 10595.02549777113,
+                                    "count": 25,
+                                    "self": 1625.9843534287647,
                                     "children": {
                                         "TorchPOCAOptimizer.update": {
-                                            "total": 20887.250337199206,
-                                            "count": 7050,
-                                            "self": 20887.250337199206
+                                            "total": 8969.041144342365,
+                                            "count": 3750,
+                                            "self": 8969.041144342365
                                         }
                                     }
                                 }
@@ -402,19 +462,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 8.970091585069895e-07,
+                    "total": 2.4730106815695763e-06,
                     "count": 1,
-                    "self": 8.970091585069895e-07
+                    "self": 2.4730106815695763e-06
                 },
                 "TrainerController._save_models": {
-                    "total": 0.30508548201760277,
+                    "total": 1.1741414550051559,
                     "count": 1,
-                    "self": 0.0037372090155258775,
+                    "self": 0.006163765996461734,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.3013482730020769,
+                            "total": 1.1679776890086941,
                             "count": 1,
-                            "self": 0.3013482730020769
+                            "self": 1.1679776890086941
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,59 +2,59 @@
     "SoccerTwos": {
         "checkpoints": [
             {
-                "steps": 26499995,
-                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-26499995.onnx",
+                "steps": 31999878,
+                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-31999878.onnx",
                 "reward": 0.0,
-                "creation_time": 1680163679.061089,
+                "creation_time": 1680185500.9602473,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-26499995.pt"
+                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-31999878.pt"
                 ]
             },
             {
-                "steps": 26999977,
-                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-26999977.onnx",
+                "steps": 32499928,
+                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-32499928.onnx",
                 "reward": 0.0,
-                "creation_time": 1680166516.4965715,
+                "creation_time": 1680187156.8123388,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-26999977.pt"
+                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-32499928.pt"
                 ]
             },
             {
-                "steps": 27499864,
-                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-27499864.onnx",
+                "steps": 32999999,
+                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-32999999.onnx",
                 "reward": 0.0,
-                "creation_time": 1680168793.6688595,
+                "creation_time": 1680189217.7823765,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-27499864.pt"
+                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-32999999.pt"
                 ]
             },
             {
-                "steps": 27999982,
-                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-27999982.onnx",
+                "steps": 33499982,
+                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-33499982.onnx",
                 "reward": 0.0,
-                "creation_time": 1680170873.503431,
+                "creation_time": 1680190866.9587436,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-27999982.pt"
+                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-33499982.pt"
                 ]
             },
             {
-                "steps": 28477740,
-                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-28477740.onnx",
+                "steps": 33646596,
+                "file_path": "results/SoccerTwos5/SoccerTwos/SoccerTwos-33646596.onnx",
                 "reward": 0.0,
-                "creation_time": 1680172438.7580104,
+                "creation_time": 1680191712.6597888,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-28477740.pt"
+                    "results/SoccerTwos5/SoccerTwos/SoccerTwos-33646596.pt"
                 ]
             }
         ],
-        "elo": 1755.2629314981316,
+        "elo": 1786.6899060684966,
         "final_checkpoint": {
-            "steps": 28477740,
+            "steps": 33646596,
             "file_path": "results/SoccerTwos5/SoccerTwos.onnx",
             "reward": 0.0,
-            "creation_time": 1680172438.7580104,
+            "creation_time": 1680191712.6597888,
             "auxillary_file_paths": [
-                "results/SoccerTwos5/SoccerTwos/SoccerTwos-28477740.pt"
+                "results/SoccerTwos5/SoccerTwos/SoccerTwos-33646596.pt"
             ]
         }
     },