diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-06-02 18:04:26,681 INFO StreamThr :121 [internal.py:wandb_internal():86] W&B internal server running at pid: 121, started at: 2024-06-02 18:04:26.680861 -2024-06-02 18:04:26,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status -2024-06-02 18:04:27,411 INFO WriterThread:121 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240602_180426-ocheitu1/run-ocheitu1.wandb -2024-06-02 18:04:27,411 DEBUG SenderThread:121 [sender.py:send():379] send: header -2024-06-02 18:04:27,415 DEBUG SenderThread:121 [sender.py:send():379] send: run -2024-06-02 18:04:27,759 INFO SenderThread:121 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files -2024-06-02 18:04:27,760 INFO SenderThread:121 [sender.py:_start_run_threads():1124] run started: ocheitu1 with start time 1717351466.682356 -2024-06-02 18:04:27,769 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: check_version -2024-06-02 18:04:27,770 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: check_version -2024-06-02 18:04:27,828 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: run_start -2024-06-02 18:04:27,839 DEBUG HandlerThread:121 [system_info.py:__init__():26] System info init -2024-06-02 18:04:27,839 DEBUG HandlerThread:121 [system_info.py:__init__():41] System info init done -2024-06-02 18:04:27,839 INFO HandlerThread:121 [system_monitor.py:start():194] Starting system monitor -2024-06-02 18:04:27,839 INFO SystemMonitor:121 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 18:04:27,839 INFO HandlerThread:121 [system_monitor.py:probe():214] Collecting system info -2024-06-02 18:04:27,840 INFO SystemMonitor:121 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 18:04:27,841 INFO SystemMonitor:121 [interfaces.py:start():190] Started disk monitoring -2024-06-02 18:04:27,842 INFO SystemMonitor:121 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 18:04:27,843 INFO SystemMonitor:121 [interfaces.py:start():190] Started memory monitoring -2024-06-02 18:04:27,843 INFO SystemMonitor:121 [interfaces.py:start():190] Started network monitoring -2024-06-02 18:04:27,858 DEBUG HandlerThread:121 [system_info.py:probe():150] Probing system -2024-06-02 18:04:27,860 DEBUG HandlerThread:121 [gitlib.py:_init_repo():56] git repository is invalid -2024-06-02 18:04:27,860 DEBUG HandlerThread:121 [system_info.py:probe():198] Probing system done -2024-06-02 18:04:27,860 DEBUG HandlerThread:121 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-06-02T18:04:27.858572', 'startedAt': '2024-06-02T18:04:26.674360', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'a8697b0bf589', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.156, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.156, 'min': 0.0, 'max': 0.0}, {'current': 2000.156, 'min': 0.0, 'max': 0.0}, {'current': 2000.156, 'min': 0.0, 'max': 0.0}, {'current': 2000.156, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5643.919727325439}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357563018798828}} -2024-06-02 18:04:27,860 INFO HandlerThread:121 [system_monitor.py:probe():224] Finished collecting system info -2024-06-02 18:04:27,860 INFO HandlerThread:121 [system_monitor.py:probe():227] Publishing system info -2024-06-02 18:04:27,861 DEBUG HandlerThread:121 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-06-02 18:04:28,761 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/conda-environment.yaml -2024-06-02 18:04:42,876 ERROR HandlerThread:121 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-06-02 20:13:53,460 INFO StreamThr :81 [internal.py:wandb_internal():86] W&B internal server running at pid: 81, started at: 2024-06-02 20:13:53.459265 +2024-06-02 20:13:53,461 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status +2024-06-02 20:13:53,843 INFO WriterThread:81 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/run-3mh4qp2s.wandb +2024-06-02 20:13:53,844 DEBUG SenderThread:81 [sender.py:send():379] send: header +2024-06-02 20:13:53,847 DEBUG SenderThread:81 [sender.py:send():379] send: run +2024-06-02 20:13:53,953 INFO SenderThread:81 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files +2024-06-02 20:13:53,954 INFO SenderThread:81 [sender.py:_start_run_threads():1124] run started: 3mh4qp2s with start time 1717359233.460694 +2024-06-02 20:13:53,962 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: check_version +2024-06-02 20:13:53,962 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: check_version +2024-06-02 20:13:54,052 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: run_start +2024-06-02 20:13:54,063 DEBUG HandlerThread:81 [system_info.py:__init__():26] System info init +2024-06-02 20:13:54,063 DEBUG HandlerThread:81 [system_info.py:__init__():41] System info init done +2024-06-02 20:13:54,063 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:13:54,064 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:13:54,064 INFO HandlerThread:81 [system_monitor.py:probe():214] Collecting system info +2024-06-02 20:13:54,064 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:13:54,065 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:13:54,067 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:13:54,068 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:13:54,069 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:13:54,078 DEBUG HandlerThread:81 [system_info.py:probe():150] Probing system +2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [gitlib.py:_init_repo():56] git repository is invalid +2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_info.py:probe():198] Probing system done +2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-06-02T20:13:54.078341', 'startedAt': '2024-06-02T20:13:53.453169', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '8f29a1fd4865', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.168, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5643.874462127686}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357563018798828}} +2024-06-02 20:13:54,080 INFO HandlerThread:81 [system_monitor.py:probe():224] Finished collecting system info +2024-06-02 20:13:54,080 INFO HandlerThread:81 [system_monitor.py:probe():227] Publishing system info +2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-06-02 20:13:54,956 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/conda-environment.yaml +2024-06-02 20:14:09,095 ERROR HandlerThread:81 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,2829 +37,560 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-06-02 18:04:42,879 DEBUG HandlerThread:121 [system_info.py:_save_conda():222] Saving conda packages done -2024-06-02 18:04:42,879 INFO HandlerThread:121 [system_monitor.py:probe():229] Finished publishing system info -2024-06-02 18:04:42,887 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:42,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 18:04:42,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:42,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 18:04:42,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:42,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 18:04:42,889 DEBUG SenderThread:121 [sender.py:send():379] send: files -2024-06-02 18:04:42,889 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-06-02 18:04:43,462 INFO wandb-upload_0:121 [upload_job.py:push():131] Uploaded file /tmp/tmphzfx4knlwandb/24rg94qw-wandb-metadata.json -2024-06-02 18:04:43,764 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-metadata.json -2024-06-02 18:04:44,209 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: python_packages -2024-06-02 18:04:44,210 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: python_packages -2024-06-02 18:04:44,213 DEBUG SenderThread:121 [sender.py:send():379] send: telemetry -2024-06-02 18:04:44,299 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:04:44,304 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:04:44,305 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:04:44,517 DEBUG SenderThread:121 [sender.py:send():379] send: config -2024-06-02 18:04:44,524 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:04:44,524 DEBUG SenderThread:121 [sender.py:send():379] send: telemetry -2024-06-02 18:04:44,524 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:04:44,524 WARNING SenderThread:121 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-06-02 18:04:44,525 DEBUG SenderThread:121 [sender.py:send():379] send: telemetry -2024-06-02 18:04:44,764 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/requirements.txt -2024-06-02 18:04:44,765 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:04:46,765 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:04:47,525 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:50,767 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:04:53,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:58,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:04:59,216 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:04:59,221 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:04:59,901 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:05:00,826 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/config.yaml -2024-06-02 18:05:05,101 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:10,102 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:14,210 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:05:14,211 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:05:14,251 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:05:15,409 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:20,410 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:25,411 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:27,843 DEBUG SystemMonitor:121 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 18:05:27,845 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:05:29,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:05:29,211 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:05:29,251 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:05:31,393 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:32,608 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:05:32,610 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:05:32,610 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:05:32,610 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:05:32,610 DEBUG SenderThread:121 [sender.py:send():379] send: metric -2024-06-02 18:05:32,610 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:05:32,611 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:05:32,611 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:05:32,838 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:05:36,612 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:41,613 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:44,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:05:44,212 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:05:44,212 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:05:47,453 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:52,453 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:57,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:05:57,846 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:05:59,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:05:59,211 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:05:59,251 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:06:03,400 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:05,903 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/config.yaml -2024-06-02 18:06:09,924 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:14,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:06:14,211 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:06:14,252 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:06:15,410 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:16,271 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:06:16,272 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:06:16,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:06:16,275 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:06:16,907 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:06:21,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:26,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:27,847 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:06:29,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:06:29,212 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:06:29,215 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:06:31,395 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:36,396 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:41,397 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:44,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:06:44,212 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:06:44,252 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:06:46,448 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:51,449 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:56,449 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:06:57,849 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:06:59,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:06:59,212 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:06:59,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:06:59,917 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:06:59,918 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:06:59,918 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:06:59,919 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:06:59,922 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:07:01,920 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:06,920 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:11,921 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:14,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:07:14,212 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:07:14,213 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:07:17,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:22,459 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:27,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:27,849 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:07:29,212 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:07:29,212 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:07:29,252 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:07:33,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:38,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:42,964 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:07:42,965 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:07:42,965 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:07:42,967 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:07:43,938 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:07:43,968 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:44,398 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:07:44,398 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:07:44,704 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:07:48,940 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:07:49,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:54,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:07:57,850 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:07:59,234 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:07:59,234 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:07:59,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:07:59,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:04,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:09,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:14,236 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:08:14,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:08:14,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:08:15,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:20,455 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:25,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:27,851 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:08:29,234 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:08:29,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:08:29,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:08:31,450 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:32,431 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:08:32,432 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:08:32,432 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:08:32,433 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:08:32,956 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:08:37,434 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:42,435 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:44,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:08:44,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:08:44,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:08:47,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:52,459 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:57,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:08:57,852 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:08:59,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:08:59,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:08:59,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:09:02,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:07,476 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:12,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:14,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:09:14,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:09:14,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:09:18,288 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:09:18,289 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:09:18,289 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:09:18,290 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:18,291 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:09:18,973 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:09:23,292 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:27,853 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:09:28,855 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:29,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:09:29,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:09:29,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:09:34,472 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:39,473 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:44,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:09:44,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:09:44,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:09:45,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:50,471 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:55,472 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:09:57,855 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:09:59,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:09:59,235 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:09:59,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:10:01,414 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:02,815 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:10:02,817 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:10:02,817 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:10:02,817 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:10:02,990 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:10:06,818 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:11,819 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:14,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:10:14,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:10:14,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:10:17,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:22,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:27,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:27,855 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:10:29,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:10:29,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:10:29,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:10:32,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:37,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:42,408 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:44,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:10:44,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:10:44,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:10:46,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:10:46,347 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:10:46,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:10:46,347 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:10:47,006 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:10:48,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:53,009 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:10:53,789 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:57,856 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:10:58,857 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:10:59,236 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:10:59,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:10:59,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:11:04,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:09,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:14,236 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:11:14,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:11:14,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:11:15,415 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:20,416 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:25,416 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:27,857 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:11:29,239 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:11:29,239 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:11:29,280 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:11:30,499 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:33,848 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:11:33,849 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:11:33,850 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:11:33,850 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:11:34,029 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:11:35,851 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:40,852 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:44,239 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:11:44,239 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:11:44,279 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:11:46,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:51,463 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:56,464 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:11:57,858 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:11:59,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:11:59,240 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:11:59,280 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:12:02,405 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:07,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:12,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:14,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:12:14,240 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:12:14,280 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:12:17,413 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:17,753 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:12:17,754 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:12:17,755 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:12:17,755 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:12:18,045 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:12:22,756 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:27,757 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:27,859 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:12:29,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:12:29,240 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:12:29,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:12:33,398 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:38,398 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:43,399 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:44,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:12:44,240 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:12:44,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:12:48,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:53,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:57,860 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:12:58,860 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:12:59,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:12:59,241 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:12:59,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:13:00,478 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:13:00,479 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:13:00,479 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:13:00,479 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:13:01,060 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:13:04,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:09,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:14,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:13:14,241 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:13:14,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:13:15,452 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:20,453 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:25,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:27,861 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:13:29,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:13:29,241 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:13:29,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:13:30,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:35,476 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:40,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:43,234 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:13:43,236 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:13:43,236 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:13:43,236 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:13:44,076 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:13:44,357 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:13:44,358 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:13:44,588 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:13:45,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:49,078 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:13:50,937 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:55,938 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:13:59,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:13:59,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:14:00,938 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:02,616 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:14:02,617 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:14:06,760 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:11,761 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:14,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:14:14,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:14:14,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:14:17,436 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:22,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:27,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:27,862 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:14:29,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:14:29,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:14:29,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:14:29,921 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:14:30,196 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:14:30,196 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:14:30,197 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:14:31,094 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:14:33,198 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:38,198 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:43,199 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:44,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:14:44,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:14:44,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:14:48,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:53,440 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:57,863 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:14:58,864 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:14:59,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:14:59,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:14:59,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:15:04,516 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:09,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:11,927 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:15:11,928 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:15:11,929 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:15:11,929 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:15:12,109 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:15:14,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:15:14,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:15:14,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:15:15,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:20,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:25,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:27,864 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:15:29,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:15:29,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:15:29,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:15:31,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:36,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:41,482 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:44,254 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:15:44,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:15:44,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:15:46,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:51,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:53,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:15:53,266 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:15:53,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:15:53,267 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:15:54,123 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:15:57,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:15:57,865 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:15:59,254 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:15:59,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:15:59,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:16:02,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:07,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:12,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:14,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:16:14,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:16:14,265 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:16:18,413 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:23,414 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:27,867 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:16:28,868 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:29,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:16:29,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:16:29,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:16:34,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:35,287 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:16:35,289 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:16:35,290 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:16:35,291 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:16:36,139 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:16:40,233 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:41,141 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:16:44,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:16:44,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:16:44,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:16:45,464 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:50,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:55,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:16:57,868 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:16:59,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:16:59,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:16:59,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:17:00,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:05,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:10,524 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:14,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:17:14,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:17:14,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:17:16,430 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:21,054 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:17:21,055 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:17:21,056 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:17:21,056 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:17:21,156 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:17:22,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:27,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:27,869 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:17:29,256 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:17:29,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:17:29,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:17:32,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:37,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:42,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:44,256 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:17:44,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:17:44,266 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:17:47,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:52,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:57,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:17:57,870 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:17:59,258 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:17:59,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:17:59,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:18:03,272 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:18:03,273 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:18:03,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:18:03,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:03,274 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:18:04,172 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:18:08,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:13,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:14,256 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:18:14,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:18:14,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:18:18,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:23,467 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:27,871 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:18:28,872 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:29,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:18:29,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:18:29,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:18:34,428 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:39,429 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:44,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:18:44,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:18:44,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:18:44,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:44,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:18:44,493 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:18:44,494 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:18:44,494 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:18:45,187 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:18:49,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:54,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:18:57,872 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:18:59,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:18:59,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:18:59,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:19:00,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:05,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:10,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:14,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:19:14,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:19:14,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:19:15,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:20,467 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:25,451 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:19:25,452 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:19:25,452 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:19:25,453 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:19:26,203 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:19:26,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:27,873 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:19:29,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:19:29,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:19:29,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:19:32,206 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:19:32,405 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:37,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:42,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:44,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:19:44,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:19:44,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:19:47,529 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:52,530 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:57,531 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:19:57,873 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:19:59,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:19:59,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:19:59,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:20:03,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:08,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:10,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:20:10,347 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:20:10,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:20:10,347 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:20:11,220 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:20:14,232 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:14,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:20:14,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:20:14,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:20:19,408 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:24,409 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:27,874 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:20:29,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:20:29,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:20:29,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:20:29,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:34,490 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:39,491 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:44,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:20:44,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:20:44,267 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:20:44,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:49,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:52,530 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:20:52,531 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:20:52,531 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:20:52,532 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:20:53,236 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:20:54,532 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:20:57,875 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:20:59,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:20:59,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:20:59,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:21:00,440 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:05,441 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:10,441 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:14,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:21:14,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:21:14,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:21:15,518 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:20,519 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:25,520 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:27,876 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:21:29,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:21:29,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:21:29,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:21:31,482 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:33,623 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:21:33,624 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:21:33,624 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:21:33,625 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:21:34,251 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:21:36,625 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:41,626 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:44,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:21:44,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:21:44,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:21:47,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:52,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:57,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:21:57,877 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:21:59,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:21:59,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:21:59,268 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:22:02,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:07,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:12,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:13,880 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:22:13,881 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:22:13,882 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:22:13,882 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:22:14,266 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:22:14,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:22:14,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:22:14,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:22:17,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:20,268 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:22:22,793 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:27,795 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:27,878 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:22:29,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:22:29,269 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:22:29,309 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:22:33,468 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:38,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:43,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:44,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:22:44,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:22:44,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:22:48,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:53,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:57,879 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:22:58,801 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:22:58,802 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:22:58,803 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:22:58,803 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:22:58,804 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:22:59,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:22:59,269 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:22:59,283 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:22:59,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:23:04,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:09,490 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:14,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:23:14,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:23:14,274 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:23:15,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:20,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:25,439 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:27,880 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:23:29,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:23:29,269 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:23:29,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:23:30,441 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:35,442 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:39,421 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:23:39,422 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:23:39,423 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:23:39,423 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:23:40,299 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:23:41,424 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:44,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:23:44,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:23:44,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:23:46,424 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:51,425 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:56,426 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:23:57,881 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:23:59,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:23:59,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:23:59,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:24:01,450 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:06,451 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:11,452 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:14,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:24:14,271 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:24:14,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:24:16,491 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:21,492 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:22,560 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:24:22,562 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:24:22,562 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:24:22,562 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:24:23,315 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:24:26,563 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:27,882 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:24:29,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:24:29,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:24:29,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:24:32,431 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:37,432 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:42,433 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:44,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:24:44,270 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:24:44,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:24:47,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:52,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:57,504 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:24:57,883 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:24:59,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:24:59,271 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:24:59,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:25:02,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:04,989 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:25:04,990 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:25:04,991 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:25:04,991 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:25:05,330 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:25:07,992 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:10,332 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:25:13,779 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:14,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:25:14,271 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:25:14,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:25:19,474 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:24,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:27,884 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:25:29,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:25:29,271 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:25:29,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:25:29,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:34,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:39,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:44,272 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:25:44,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:25:44,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:25:45,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:50,226 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:25:50,227 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:25:50,228 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:25:50,228 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:25:50,347 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:25:51,229 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:56,230 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:25:57,885 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:25:59,272 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:25:59,272 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:25:59,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:26:01,435 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:06,436 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:11,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:14,272 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:26:14,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:26:14,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:26:16,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:21,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:26,459 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:27,885 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:26:29,272 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:26:29,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:26:29,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:26:31,126 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:26:31,127 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:26:31,127 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:26:31,128 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:26:31,362 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:26:32,128 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:37,129 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:42,130 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:44,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:26:44,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:26:44,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:26:47,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:52,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:57,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:26:57,886 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:26:59,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:26:59,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:26:59,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:27:03,426 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:08,427 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:13,429 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:13,634 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:27:13,636 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:27:13,636 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:27:13,636 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:27:14,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:27:14,273 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:27:14,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:27:14,378 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:27:18,443 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:23,444 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:27,887 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:27:28,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:29,274 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:27:29,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:27:29,315 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:27:34,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:39,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:44,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:27:44,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:27:44,315 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:27:44,531 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:49,532 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:54,533 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:27:55,368 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:27:55,370 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:27:55,370 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:27:55,371 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:27:55,393 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:27:57,888 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:27:59,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:27:59,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:27:59,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:28:00,395 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:28:00,498 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:05,499 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:10,500 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:14,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:28:14,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:28:14,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:28:16,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:21,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:26,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:27,889 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:28:29,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:28:29,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:28:29,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:28:31,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:36,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:41,200 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:28:41,201 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:28:41,201 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:28:41,202 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:28:41,409 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:28:42,202 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:44,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:28:44,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:28:44,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:28:47,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:52,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:57,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:28:57,894 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:28:59,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:28:59,275 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:28:59,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:29:03,468 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:08,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:13,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:14,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:29:14,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:29:14,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:29:18,537 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:23,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:29:23,390 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:29:23,391 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:29:23,391 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:29:23,425 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:29:24,392 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:27,891 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:29:29,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:29:29,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:29:29,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:29:29,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:34,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:39,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:44,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:29:44,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:29:44,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:29:45,446 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:50,447 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:55,448 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:29:57,893 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:29:59,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:29:59,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:29:59,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:30:00,464 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:05,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:05,808 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:30:05,809 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:30:05,809 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:30:05,809 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:30:06,442 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:30:10,810 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:14,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:30:14,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:30:14,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:30:16,468 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:21,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:26,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:27,892 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:30:29,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:30:29,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:30:29,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:30:31,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:36,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:41,482 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:44,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:30:44,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:30:44,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:30:46,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:47,815 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:30:47,817 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:30:47,817 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:30:47,817 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:30:48,457 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:30:51,589 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:54,460 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:30:56,590 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:30:57,893 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:30:59,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:30:59,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:30:59,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:31:02,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:07,515 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:12,516 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:14,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:31:14,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:31:14,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:31:17,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:22,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:27,524 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:27,894 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:31:29,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:31:29,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:31:29,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:31:32,642 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:34,091 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:31:34,092 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:31:34,092 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:31:34,093 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:31:34,475 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:31:38,093 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:43,094 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:44,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:31:44,277 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:31:44,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:31:48,420 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:53,421 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:57,895 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:31:58,896 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:31:59,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:31:59,279 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:31:59,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:32:04,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:09,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:14,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:32:14,278 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:32:14,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:32:14,533 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:17,868 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:32:17,869 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:32:17,869 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:32:17,869 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:32:18,491 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:32:19,870 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:24,871 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:27,896 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:32:29,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:32:29,278 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:32:29,320 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:32:30,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:35,511 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:40,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:44,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:32:44,279 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:32:44,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:32:45,516 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:50,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:55,518 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:32:57,897 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:32:59,279 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:32:59,279 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:32:59,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:33:00,285 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:33:00,287 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:33:00,287 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:33:00,288 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:33:00,507 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:33:01,288 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:06,289 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:11,290 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:14,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:33:14,279 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:33:14,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:33:16,414 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:21,415 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:26,416 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:27,898 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:33:29,279 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:33:29,279 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:33:29,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:33:31,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:36,497 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:41,498 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:43,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:33:43,318 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:33:43,319 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:33:43,319 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:33:43,523 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:33:44,552 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:33:44,552 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:33:44,592 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:33:46,764 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:48,525 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:33:52,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:57,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:33:57,899 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:33:59,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:33:59,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:33:59,318 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:34:02,468 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:07,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:12,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:14,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:34:14,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:34:14,318 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:34:17,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:22,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:27,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:27,899 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:34:29,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:34:29,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:34:29,319 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:34:29,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:34:29,559 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:34:29,559 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:34:29,560 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:34:30,539 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:34:32,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:37,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:42,562 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:44,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:34:44,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:34:44,319 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:34:48,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:53,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:57,901 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:34:58,901 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:34:59,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:34:59,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:34:59,318 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:35:04,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:09,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:12,152 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:35:12,154 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:35:12,154 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:35:12,154 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:35:12,555 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:35:14,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:35:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:35:14,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:35:14,542 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:19,543 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:24,544 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:27,901 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:35:29,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:35:29,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:35:29,323 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:35:29,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:34,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:39,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:44,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:35:44,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:35:44,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:35:45,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:50,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:54,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:35:54,068 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:35:54,069 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:35:54,069 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:35:54,570 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:35:56,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:35:57,903 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:35:59,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:35:59,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:35:59,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:36:01,492 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:06,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:11,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:14,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:36:14,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:36:14,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:36:16,557 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:21,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:26,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:27,903 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:36:29,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:36:29,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:36:29,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:36:32,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:37,177 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:36:37,178 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:36:37,178 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:36:37,178 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:36:37,586 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:36:38,179 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:42,588 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:36:44,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:44,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:36:44,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:36:44,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:36:49,545 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:54,546 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:36:57,904 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:36:59,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:36:59,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:36:59,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:36:59,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:04,562 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:09,563 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:14,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:37:14,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:37:14,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:37:15,511 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:20,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:23,239 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:37:23,241 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:37:23,241 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:37:23,241 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:37:23,603 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:37:26,242 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:27,905 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:37:29,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:37:29,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:37:29,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:37:31,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:36,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:41,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:44,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:37:44,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:37:44,324 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:37:46,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:51,549 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:56,550 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:37:57,906 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:37:59,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:37:59,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:37:59,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:38:02,483 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:05,812 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:38:05,814 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:38:05,814 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:38:05,814 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:38:06,619 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:38:07,815 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:12,816 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:14,308 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:38:14,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:38:14,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:38:18,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:23,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:27,907 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:38:28,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:29,308 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:38:29,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:38:29,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:38:34,474 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:39,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:44,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:38:44,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:38:44,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:38:44,541 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:47,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:38:47,832 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:38:47,832 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:38:47,833 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:38:48,634 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:38:49,833 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:54,834 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:38:57,907 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:38:59,312 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:38:59,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:38:59,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:39:00,553 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:05,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:10,555 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:14,312 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:39:14,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:39:14,325 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:39:16,539 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:21,540 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:26,541 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:27,909 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:39:29,312 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:39:29,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:39:29,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:39:29,772 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:39:29,773 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:39:29,773 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:39:29,774 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:39:30,650 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:39:31,774 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:34,652 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:39:36,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:41,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:44,312 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:39:44,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:39:44,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:39:47,537 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:52,538 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:57,539 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:39:57,909 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:39:59,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:39:59,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:39:59,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:40:03,472 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:08,473 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:13,474 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:14,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:40:14,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:40:14,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:40:14,420 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:40:14,540 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:40:14,540 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:40:14,541 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:40:14,667 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:40:18,542 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:23,543 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:27,910 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:40:28,911 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:29,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:40:29,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:40:29,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:40:34,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:39,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:44,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:40:44,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:40:44,326 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:40:44,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:49,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:54,560 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:40:56,726 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:40:56,727 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:40:56,727 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:40:56,727 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:40:57,683 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:40:57,911 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:40:59,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:40:59,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:40:59,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:41:00,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:05,504 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:10,505 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:14,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:41:14,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:41:14,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:41:16,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:21,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:26,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:27,912 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:41:29,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:41:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:41:29,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:41:31,544 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:36,545 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:39,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:41:39,019 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:41:39,019 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:41:39,020 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:41:39,699 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:41:42,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:44,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:41:44,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:41:44,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:41:47,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:52,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:57,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:41:57,913 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:41:59,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:41:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:41:59,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:42:02,564 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:07,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:12,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:14,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:42:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:42:14,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:42:17,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:42:17,307 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:42:17,307 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:42:17,307 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:42:17,712 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:42:18,308 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:22,715 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:42:23,971 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:27,914 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:42:29,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:29,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:42:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:42:29,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:42:34,506 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:39,507 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:44,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:42:44,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:42:44,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:42:44,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:49,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:54,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:42:57,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:42:57,458 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:42:57,458 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:42:57,459 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:42:57,729 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:42:57,915 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:42:59,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:42:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:42:59,327 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:43:00,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:05,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:10,524 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:14,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:43:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:43:14,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:43:15,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:20,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:25,549 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:27,916 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:43:29,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:43:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:43:29,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:43:30,569 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:34,295 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:43:34,301 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:43:34,301 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:43:34,301 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:43:34,742 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:43:36,302 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:41,303 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:44,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:43:44,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:43:44,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:43:46,574 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:51,575 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:56,576 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:43:57,917 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:43:59,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:43:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:43:59,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:44:02,555 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:07,556 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:11,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:44:11,566 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:44:11,566 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:44:11,566 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:44:11,756 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:44:12,567 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:44:14,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:44:14,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:44:18,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:23,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:27,919 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:44:28,920 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:29,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:44:29,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:44:29,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:44:34,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:39,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:44,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:44:44,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:44:44,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:44:44,521 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:47,362 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:44:47,363 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:44:47,363 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:44:47,364 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:44:47,769 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:44:50,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:53,771 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:44:56,233 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:44:57,920 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:44:59,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:44:59,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:44:59,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:45:01,473 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:06,474 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:11,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:14,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:45:14,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:45:14,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:45:16,516 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:21,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:26,518 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:27,208 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:45:27,209 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:45:27,209 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:45:27,210 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:45:27,784 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:45:27,921 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:45:29,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:45:29,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:45:29,328 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:45:31,553 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:36,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:41,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:44,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:45:44,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:45:44,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:45:47,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:52,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:57,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:45:57,921 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:45:59,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:45:59,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:45:59,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:46:02,533 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:04,226 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:46:04,228 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:46:04,228 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:46:04,228 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:46:04,798 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:46:08,229 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:13,230 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:14,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:46:14,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:46:14,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:46:18,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:23,563 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:27,922 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:46:28,923 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:29,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:46:29,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:46:29,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:46:34,501 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:39,502 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:39,824 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:46:39,825 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:46:39,825 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:46:39,826 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:46:40,813 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:46:44,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:46:44,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:46:44,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:46:44,546 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:49,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:54,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:46:57,923 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:46:59,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:46:59,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:46:59,329 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:47:00,501 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:05,502 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:10,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:14,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:47:14,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:47:14,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:47:16,499 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:16,812 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:47:16,813 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:47:16,814 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:47:16,814 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:47:16,826 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:47:21,828 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:47:22,286 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:27,287 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:27,924 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:47:29,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:47:29,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:47:29,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:47:32,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:37,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:42,482 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:44,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:47:44,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:47:44,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:47:47,557 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:52,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:56,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:47:56,504 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:47:56,504 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:47:56,504 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:47:56,842 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:47:57,925 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:47:57,926 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:47:59,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:47:59,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:47:59,330 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:48:03,499 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:08,500 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:13,501 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:14,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:48:14,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:48:14,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:48:18,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:23,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:27,926 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:48:28,927 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:48:29,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:48:29,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:48:33,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:48:33,064 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:48:33,064 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:48:33,065 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:48:33,856 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:48:34,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:39,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:44,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:44,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:48:44,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:48:44,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:48:49,538 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:54,538 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:48:57,927 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:48:59,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:48:59,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:48:59,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:48:59,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:04,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:09,182 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:49:09,183 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:49:09,183 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:49:09,184 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:49:09,869 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:49:10,184 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:14,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:49:14,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:49:14,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:49:15,589 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:20,590 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:25,590 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:27,928 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:49:29,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:49:29,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:49:29,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:49:31,534 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:36,535 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:41,535 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:44,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:49:44,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:49:44,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:49:45,386 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:49:45,387 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:49:45,388 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:49:45,388 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:49:45,883 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:49:47,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:51,886 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:49:53,173 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:57,929 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:49:58,930 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:49:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:49:59,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:49:59,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:50:04,556 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:09,557 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:50:14,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:50:14,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:50:15,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:20,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:24,769 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:50:24,770 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:50:24,771 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:50:24,771 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:50:24,898 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:50:25,772 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:27,930 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:50:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:50:29,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:50:29,331 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:50:31,519 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:36,520 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:41,521 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:44,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:50:44,332 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:50:44,332 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:50:47,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:52,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:57,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:50:57,930 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:50:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:50:59,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:50:59,332 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:51:01,361 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:51:01,363 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:51:01,363 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:51:01,363 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:51:01,912 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:51:03,364 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:08,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:13,366 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:51:14,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:51:14,332 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:51:18,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:23,522 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:27,931 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:51:28,932 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:29,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:51:29,332 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:51:29,332 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:51:34,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:37,895 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:51:37,896 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:51:37,897 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:51:37,897 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:51:37,925 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:51:39,898 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:44,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:51:44,333 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:51:44,333 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:51:45,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:50,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:55,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:51:57,932 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:51:59,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:51:59,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:51:59,334 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:52:00,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:05,583 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:10,584 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:14,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:52:14,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:52:14,334 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:52:14,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:52:14,521 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:52:14,521 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:52:14,522 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:52:14,941 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:52:16,523 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:19,943 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:52:22,309 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:27,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:27,933 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:52:29,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:52:29,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:52:29,334 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:52:32,592 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:37,593 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:42,594 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:44,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:52:44,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:52:44,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:52:48,565 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:53,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:54,980 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:52:54,981 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:52:54,982 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:52:54,982 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:52:55,956 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:52:57,934 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:52:58,935 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:52:59,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:52:59,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:52:59,334 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:53:04,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:09,478 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:14,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:53:14,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:53:14,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:53:14,569 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:19,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:24,571 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:27,935 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:53:29,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:53:29,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:53:29,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:53:30,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:31,225 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:53:31,226 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:53:31,226 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:53:31,227 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:53:31,970 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:53:36,228 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:41,229 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:44,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:53:44,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:53:44,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:53:46,581 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:51,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:56,583 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:53:57,936 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:53:59,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:53:59,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:53:59,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:54:02,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:07,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:09,917 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:54:09,918 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:54:09,918 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:54:09,919 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:54:09,985 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:54:12,920 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:14,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:54:14,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:54:14,335 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:54:18,569 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:23,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:27,937 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:54:28,938 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:29,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:54:29,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:54:29,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:54:34,483 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:39,484 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:44,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:54:44,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:54:44,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:54:45,467 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:48,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:54:48,032 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:54:48,032 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:54:48,032 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:54:49,000 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:54:51,033 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:56,649 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:54:57,937 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:54:58,004 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:54:59,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:54:59,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:54:59,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:55:02,553 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:07,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:12,555 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:14,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:55:14,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:55:14,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:55:17,581 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:22,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:27,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:27,938 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:55:29,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:55:29,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:55:29,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:55:32,371 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:55:32,372 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:55:32,373 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:55:32,373 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:55:33,017 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:55:33,374 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:38,375 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:43,376 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:44,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:55:44,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:55:44,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:55:48,544 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:53,545 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:57,939 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:55:58,940 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:55:59,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:55:59,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:55:59,336 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:56:04,483 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:09,231 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:56:09,232 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:56:09,232 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:56:09,233 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:56:10,031 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:56:10,234 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:14,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:56:14,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:56:14,338 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:56:15,491 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:20,492 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:25,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:27,940 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:56:29,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:56:29,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:56:29,337 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:56:30,499 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:35,500 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:40,500 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:56:44,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:56:44,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:56:45,315 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:56:45,316 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:56:45,317 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:56:45,317 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:56:46,046 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:56:46,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:51,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:56,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:56:57,941 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:56:59,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:56:59,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:56:59,343 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:57:01,501 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:06,502 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:11,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:14,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:57:14,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:57:14,343 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:57:16,584 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:21,585 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:21,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:57:21,685 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:57:21,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:57:21,686 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:57:22,061 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:57:26,835 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:27,942 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:57:28,063 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 18:57:29,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:57:29,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:57:29,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:57:32,577 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:37,578 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:42,578 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:44,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:57:44,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:57:44,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:57:48,550 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:53,550 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:57,948 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:57:58,949 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:57:59,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:57:59,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:57:59,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:58:03,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:58:03,830 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:58:03,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:58:03,831 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:58:04,077 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:58:04,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:09,832 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:14,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:58:14,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:58:14,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:58:15,569 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:20,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:25,571 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:27,944 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:58:29,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:58:29,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:58:29,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:58:31,518 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:36,519 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:41,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:58:41,038 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:58:41,038 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:58:41,040 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:58:41,091 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:58:42,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:44,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:58:44,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:58:44,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:58:47,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:52,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:57,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:58:57,945 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:58:59,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:58:59,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:58:59,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:59:02,569 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:07,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:12,571 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:14,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:59:14,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:59:14,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:59:17,577 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:19,103 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:59:19,104 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:59:19,104 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:59:19,106 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:59:19,107 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:59:23,107 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:27,945 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:59:28,946 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:29,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:59:29,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:59:29,344 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:59:34,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:39,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:44,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:59:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:59:44,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 18:59:45,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:50,559 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:55,560 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 18:59:56,850 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 18:59:56,851 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 18:59:56,851 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 18:59:56,853 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 18:59:57,121 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 18:59:57,946 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 18:59:59,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 18:59:59,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 18:59:59,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:00:01,550 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:02,123 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:00:06,551 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:11,558 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:14,338 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:00:14,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:00:14,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:00:17,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:22,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:27,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:27,947 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:00:29,338 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:00:29,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:00:29,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:00:32,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:37,435 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:00:37,436 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:00:37,436 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:00:37,436 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:00:38,137 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:00:38,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:43,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:44,338 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:00:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:00:44,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:00:48,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:53,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:57,948 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:00:58,949 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:00:59,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:00:59,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:00:59,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:01:04,538 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:09,539 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:14,339 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:01:14,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:01:14,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:01:14,591 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:15,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:01:15,276 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:01:15,276 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:01:15,277 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:01:16,151 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:01:20,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:25,279 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:27,949 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:01:29,339 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:01:29,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:01:29,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:01:30,581 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:35,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:40,583 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:44,339 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:01:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:01:44,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:01:46,572 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:51,572 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:53,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:01:53,069 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:01:53,070 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:01:53,070 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:01:53,165 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:01:57,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:01:57,950 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:01:59,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:01:59,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:01:59,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:02:02,527 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:07,528 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:12,528 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:14,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:02:14,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:02:14,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:02:17,598 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:22,599 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:27,600 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:27,951 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:02:29,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:02:29,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:02:29,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:02:30,691 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:02:30,692 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:02:30,692 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:02:30,693 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:02:31,180 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:02:32,693 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:37,694 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:42,184 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:02:43,207 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:44,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:02:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:02:44,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:02:48,571 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:53,572 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:57,952 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:02:58,953 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:02:59,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:02:59,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:02:59,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:03:04,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:09,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:14,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:03:14,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:03:14,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:03:14,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:17,620 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:03:17,621 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:03:17,622 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:03:17,622 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:03:18,198 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:03:19,623 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:24,624 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:27,953 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:03:29,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:03:29,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:03:29,346 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:03:30,528 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:35,529 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:40,529 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:44,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:03:44,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:03:44,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:03:46,520 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:51,521 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:53,735 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:03:53,736 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:03:53,736 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:03:53,737 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:03:54,211 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:03:56,738 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:03:57,955 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:03:59,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:03:59,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:03:59,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:04:02,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:07,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:12,515 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:14,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:04:14,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:04:14,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:04:17,534 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:22,535 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:27,536 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:27,955 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:04:29,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:04:29,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:04:29,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:04:31,635 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:04:31,636 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:04:31,637 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:04:31,637 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:04:32,227 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:04:32,638 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:37,638 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:42,639 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:44,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:04:44,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:04:44,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:04:48,625 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:53,626 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:57,957 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:04:58,958 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:04:59,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:04:59,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:04:59,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:05:04,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:09,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:10,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:05:10,320 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:05:10,320 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:05:10,320 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:05:11,242 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:05:14,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:05:14,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:05:14,347 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:05:14,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:16,244 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:05:19,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:24,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:27,958 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:05:29,357 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:05:29,358 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:05:29,398 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:05:29,516 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:34,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:39,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:44,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:05:44,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:05:44,352 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:05:44,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:49,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:51,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:05:51,494 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:05:51,494 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:05:51,495 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:05:52,258 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:05:55,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:05:57,959 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:05:59,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:05:59,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:05:59,352 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:06:00,507 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:05,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:10,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:14,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:06:14,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:06:14,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:06:15,547 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:20,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:25,549 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:27,960 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:06:28,775 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:06:28,776 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:06:28,776 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:06:28,776 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:06:29,273 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:06:29,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:06:29,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:06:29,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:06:30,593 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:35,594 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:40,594 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:44,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:06:44,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:06:44,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:06:46,544 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:51,545 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:56,546 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:06:57,961 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:06:59,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:06:59,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:06:59,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:07:01,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:06,571 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:07,400 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:07:07,401 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:07:07,401 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:07:07,402 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:07:08,287 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:07:12,403 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:14,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:07:14,353 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:07:14,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:07:17,616 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:22,617 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:27,618 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:27,962 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:07:29,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:07:29,353 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:07:29,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:07:33,581 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:38,582 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:43,583 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:44,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:07:44,353 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:07:44,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:07:45,420 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:07:45,421 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:07:45,421 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:07:45,422 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:07:46,301 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:07:49,423 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:51,303 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:07:54,502 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:07:57,963 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:07:59,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:07:59,353 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:07:59,353 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:07:59,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:04,567 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:09,568 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:14,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:08:14,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:08:14,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:08:14,617 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:19,618 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:24,619 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:27,260 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:08:27,261 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:08:27,261 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:08:27,262 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:08:27,316 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:08:27,964 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:08:29,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:08:29,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:08:29,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:08:30,551 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:35,552 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:40,553 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:44,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:08:44,353 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:08:44,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:08:45,584 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:50,585 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:55,586 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:08:57,965 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:08:59,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:08:59,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:08:59,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:09:01,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:05,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:09:05,389 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:09:05,390 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:09:05,390 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:09:06,331 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:09:07,391 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:12,392 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:14,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:09:14,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:09:14,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:09:17,618 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:22,619 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:27,620 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:27,966 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:09:29,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:09:29,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:09:29,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:09:33,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:38,497 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:43,498 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:43,784 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:09:43,785 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:09:43,785 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:09:43,786 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:09:44,346 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:09:44,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:09:44,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:09:44,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:09:48,504 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:53,505 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:57,967 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:09:58,968 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:09:59,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:09:59,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:09:59,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:10:04,584 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:09,585 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:14,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:10:14,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:10:14,354 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:10:14,603 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:19,603 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:21,597 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:10:21,598 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:10:21,599 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:10:21,599 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:10:22,361 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:10:25,600 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:27,363 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:10:27,968 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:10:29,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:10:29,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:10:29,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:10:31,598 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:36,599 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:41,600 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:44,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:10:44,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:10:44,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:10:47,535 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:52,536 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:57,537 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:10:57,969 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:10:59,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:10:59,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:10:59,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:11:02,596 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:03,688 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:11:03,689 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:11:03,689 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:11:03,689 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:11:04,377 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:11:07,690 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:12,691 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:14,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:11:14,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:11:14,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:11:18,483 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:23,484 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:27,970 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:11:28,970 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:29,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:11:29,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:11:29,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:11:34,520 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:39,521 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:42,295 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:11:42,296 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:11:42,297 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:11:42,297 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:11:42,391 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:11:44,351 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:11:44,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:11:44,356 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:11:44,620 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:49,620 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:54,621 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:11:57,970 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:11:59,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:11:59,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:11:59,355 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:12:00,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:05,567 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:10,568 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:14,351 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:12:14,356 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:12:14,357 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:12:15,599 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:20,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:12:20,339 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:12:20,339 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:12:20,339 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:12:20,405 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:12:21,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:26,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:27,975 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:12:29,359 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:12:29,360 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:12:29,360 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:12:31,594 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:36,595 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:41,596 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:44,357 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:12:44,358 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:12:44,399 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:12:46,604 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:51,605 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:56,606 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:12:57,972 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:12:58,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:12:58,366 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:12:58,366 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:12:58,368 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:12:58,419 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:12:59,473 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:12:59,538 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:12:59,539 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:13:01,689 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:05,422 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:13:07,245 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:12,246 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:14,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:13:14,366 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:13:14,408 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:13:17,608 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:22,609 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:27,610 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:27,973 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:13:29,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:13:29,366 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:13:29,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:13:33,536 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:38,537 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:41,385 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:13:41,386 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:13:41,386 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:13:41,388 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:13:41,435 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:13:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:44,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:13:44,366 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:13:44,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:13:49,604 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:54,604 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:13:57,973 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:13:59,366 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:13:59,366 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:13:59,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:14:00,590 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:05,592 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:10,592 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:14,367 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:14:14,368 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:14:14,368 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:14:15,609 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:20,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:14:20,351 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:14:20,351 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:14:20,353 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:14:20,449 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:14:21,354 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:26,355 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:27,975 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:14:29,366 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:14:29,367 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:14:29,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:14:31,632 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:36,633 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:41,634 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:44,366 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:14:44,367 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:14:44,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:14:47,604 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:52,605 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:57,606 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:14:57,975 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:14:58,914 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:14:58,915 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:14:58,915 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:14:58,916 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:14:59,465 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:14:59,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:14:59,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:14:59,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:15:02,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:07,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:12,909 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:14,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:15:14,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:15:14,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:15:18,875 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:23,876 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:27,976 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:15:28,977 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:29,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:15:29,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:15:29,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:15:34,851 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:35,479 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:15:35,480 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:15:35,480 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:15:35,481 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:15:36,478 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:15:40,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:41,480 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:15:44,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:15:44,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:15:44,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:15:45,859 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:50,860 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:55,861 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:15:57,977 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:15:59,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:15:59,682 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:15:59,722 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:16:00,907 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:05,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:10,909 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:14,681 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:16:14,682 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:16:14,722 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:16:15,927 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:17,652 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:16:17,653 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:16:17,653 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:16:17,653 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:16:18,494 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:16:21,654 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:26,655 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:27,978 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:16:29,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:16:29,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:16:29,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:16:31,891 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:36,892 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:41,893 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:44,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:16:44,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:16:44,723 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:16:47,886 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:52,887 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:56,794 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:16:56,795 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:16:56,795 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:16:56,796 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:16:57,509 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:16:57,979 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:16:57,982 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:16:59,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:16:59,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:16:59,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:17:03,848 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:08,849 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:13,849 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:14,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:17:14,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:17:14,723 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:17:18,922 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:23,923 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:27,980 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:17:28,981 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:29,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:17:29,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:17:29,723 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:17:34,397 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:17:34,398 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:17:34,399 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:17:34,399 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:34,400 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:17:34,523 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:17:39,401 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:44,402 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:44,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:17:44,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:17:44,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:17:49,835 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:54,836 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:17:57,981 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:17:59,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:17:59,683 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:17:59,723 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:17:59,887 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:04,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:09,889 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:12,208 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:18:12,210 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:18:12,210 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:18:12,210 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:18:12,538 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:18:14,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:18:14,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:18:14,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:18:15,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:17,540 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:18:21,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:26,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:27,982 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:18:29,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:18:29,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:18:29,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:18:31,901 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:36,902 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:41,903 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:44,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:18:44,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:18:44,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:18:47,862 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:52,863 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:53,826 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:18:53,827 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:18:53,827 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:18:53,827 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:18:54,554 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:18:57,983 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:18:57,986 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:18:59,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:18:59,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:18:59,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:19:03,907 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:08,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:13,909 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:14,683 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:19:14,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:19:14,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:19:19,849 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:24,850 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:27,984 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:19:29,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:19:29,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:19:29,684 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:19:29,933 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:31,402 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:19:31,404 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:19:31,404 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:19:31,404 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:19:31,568 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:19:35,405 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:40,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:44,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:19:44,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:19:44,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:19:45,886 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:50,887 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:55,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:19:57,985 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:19:59,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:19:59,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:19:59,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:20:01,868 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:06,869 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:09,438 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:20:09,439 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:20:09,439 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:20:09,440 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:20:09,582 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:20:12,441 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:14,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:20:14,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:20:14,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:20:17,883 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:22,884 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:27,885 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:27,985 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:20:29,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:20:29,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:20:29,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:20:33,834 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:38,835 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:43,836 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:44,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:20:44,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:20:44,725 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:20:47,464 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:20:47,465 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:20:47,465 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:20:47,466 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:20:47,597 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:20:49,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:53,599 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:20:54,588 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:20:57,986 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:20:59,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:20:59,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:20:59,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:20:59,924 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:04,925 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:09,926 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:14,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:21:14,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:21:14,725 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:21:15,862 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:20,863 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:25,864 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:27,987 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:21:29,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:21:29,344 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:21:29,345 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:21:29,345 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:21:29,615 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:21:29,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:21:29,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:21:29,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:21:30,873 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:35,873 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:40,874 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:44,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:21:44,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:21:44,725 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:21:45,888 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:50,889 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:55,889 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:21:57,988 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:21:59,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:21:59,685 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:21:59,726 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:22:00,906 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:05,907 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:08,171 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:22:08,172 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:22:08,173 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:22:08,173 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:22:08,630 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:22:11,174 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:14,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:22:14,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:22:14,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:22:16,896 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:21,897 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:26,897 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:27,990 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:22:29,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:22:29,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:22:29,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:22:32,853 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:37,854 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:42,855 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:44,685 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:22:44,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:22:44,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:22:45,802 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:22:45,803 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:22:45,804 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:22:45,804 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:22:46,644 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:22:48,805 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:53,806 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:57,990 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:22:58,991 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:22:59,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:22:59,688 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:22:59,688 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:23:04,827 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:09,828 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:14,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:23:14,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:23:14,726 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:23:14,872 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:19,873 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:23,367 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:23:23,369 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:23:23,369 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:23:23,369 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:23:23,657 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:23:25,370 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:27,992 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:23:29,660 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:23:29,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:23:29,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:23:29,688 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:23:30,835 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:35,836 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:40,837 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:44,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:23:44,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:23:44,688 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:23:45,844 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:50,845 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:55,846 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:23:57,992 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:23:59,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:23:59,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:23:59,690 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:24:00,874 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:05,875 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:06,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:24:06,274 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:24:06,274 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:24:06,275 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:24:06,672 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:24:11,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:14,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:24:14,686 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:24:14,689 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:24:16,921 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:21,922 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:26,923 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:27,993 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:24:29,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:24:29,687 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:24:29,727 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:24:32,914 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:37,915 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:42,916 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:43,145 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:24:43,146 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:24:43,146 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:24:43,146 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:24:43,687 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:24:44,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:24:44,687 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:24:44,689 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:24:48,915 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:53,916 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:57,994 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:24:58,995 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:24:59,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:24:59,688 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:24:59,728 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:25:04,912 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:09,913 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:14,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:25:14,687 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:25:14,727 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:25:15,850 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:20,225 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:25:20,226 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:25:20,227 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:25:20,227 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:25:20,704 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:25:21,228 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:26,229 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:27,995 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:25:29,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:25:29,687 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:25:29,690 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:25:31,908 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:36,909 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:41,910 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:44,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:25:44,687 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:25:44,690 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:25:47,858 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:52,859 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:57,860 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:25:57,997 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:25:58,889 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:25:58,890 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:25:58,890 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:25:58,891 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:25:59,719 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:25:59,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:25:59,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:25:59,832 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:26:03,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:05,721 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:26:08,109 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:13,110 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:14,828 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:26:14,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:26:14,829 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:26:19,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:24,047 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:27,998 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:26:29,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:26:29,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:26:29,832 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:26:29,832 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:35,028 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:40,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:40,596 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:26:40,597 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:26:40,597 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:26:40,598 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:26:40,734 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:26:44,828 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:26:44,829 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:26:44,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:26:45,985 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:50,986 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:55,987 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:26:57,998 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:26:59,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:26:59,829 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:26:59,869 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:27:00,996 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:05,997 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:10,998 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:14,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:27:14,829 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:27:14,869 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:27:16,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:20,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:27:20,513 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:27:20,513 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:27:20,513 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:27:20,750 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:27:21,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:26,515 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:28,000 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:27:29,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:27:29,829 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:27:29,831 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:27:32,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:37,020 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:42,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:44,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:27:44,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:27:44,870 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:27:48,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:53,015 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:58,000 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:27:59,001 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:27:59,726 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:27:59,727 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:27:59,727 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:27:59,727 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:27:59,764 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:27:59,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:27:59,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:27:59,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:28:04,054 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:09,055 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:14,056 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:14,829 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:28:14,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:28:14,870 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:28:20,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:25,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:28,002 DEBUG SenderThread:121 [sender.py:send():379] send: stats -2024-06-02 19:28:29,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:28:29,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:28:29,870 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:28:30,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:35,036 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:37,846 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 19:28:37,847 DEBUG SenderThread:121 [sender.py:send():379] send: history -2024-06-02 19:28:37,847 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: summary_record -2024-06-02 19:28:37,847 INFO SenderThread:121 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 19:28:38,779 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/wandb-summary.json -2024-06-02 19:28:40,848 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:43,781 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_180426-ocheitu1/files/output.log -2024-06-02 19:28:44,830 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 19:28:44,830 DEBUG SenderThread:121 [sender.py:send_request():406] send_request: stop_status -2024-06-02 19:28:44,833 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 19:28:46,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 19:28:51,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:09,098 DEBUG HandlerThread:81 [system_info.py:_save_conda():222] Saving conda packages done +2024-06-02 20:14:09,099 INFO HandlerThread:81 [system_monitor.py:probe():229] Finished publishing system info +2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:14:09,108 DEBUG SenderThread:81 [sender.py:send():379] send: files +2024-06-02 20:14:09,108 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-06-02 20:14:09,360 INFO wandb-upload_0:81 [upload_job.py:push():131] Uploaded file /tmp/tmp1abbn3iqwandb/snb0frox-wandb-metadata.json +2024-06-02 20:14:09,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-metadata.json +2024-06-02 20:14:10,745 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: python_packages +2024-06-02 20:14:10,746 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: python_packages +2024-06-02 20:14:10,748 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry +2024-06-02 20:14:10,825 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:14:10,825 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:14:10,842 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:14:10,864 DEBUG SenderThread:81 [sender.py:send():379] send: config +2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry +2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:14:10,870 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry +2024-06-02 20:14:10,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/requirements.txt +2024-06-02 20:14:10,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:14:11,862 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:14:11,862 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:14:11,862 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:14:11,863 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:14:11,863 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:14:11,863 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:14:11,864 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:14:11,864 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:14:11,873 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:14:12,960 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:14:14,874 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:19,875 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:24,884 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:25,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:14:25,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:14:25,803 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:14:25,965 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml +2024-06-02 20:14:30,888 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:35,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:40,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:14:40,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:14:40,786 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:14:41,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:46,852 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:51,853 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:14:55,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:14:55,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:14:55,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:14:57,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:02,808 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:07,809 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:10,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:15:10,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:15:10,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:15:12,823 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:17,824 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:22,825 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:25,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:15:25,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:15:25,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:15:28,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:33,794 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:38,795 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:40,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:15:40,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:15:40,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:15:43,876 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:48,877 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:53,878 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:15:55,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:15:55,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:15:55,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:15:59,826 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:04,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:09,828 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:10,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:16:10,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:16:10,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:16:14,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:19,890 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:24,891 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:25,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:16:25,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:16:25,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:16:30,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:35,794 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:40,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:16:40,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:16:40,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:16:40,813 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:45,814 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:50,815 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:16:55,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:16:55,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:16:55,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:16:55,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:00,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:05,868 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:10,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:17:10,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:17:10,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:17:11,831 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:16,832 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:21,833 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:25,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:17:25,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:17:25,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:17:26,840 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:31,841 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:36,843 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:40,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:17:40,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:17:40,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:17:42,808 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:47,809 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:52,810 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:17:55,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:17:55,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:17:55,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:17:58,804 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:03,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:08,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:10,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:18:10,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:18:10,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:18:13,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:18,851 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:23,852 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:25,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:18:25,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:18:25,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:18:28,865 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:33,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:38,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:40,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:18:40,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:18:40,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:18:44,340 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:18:44,340 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:18:44,340 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:18:44,341 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:18:44,341 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:18:44,342 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:18:44,343 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:18:44,343 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:18:44,345 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:18:44,381 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:18:44,381 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:18:44,381 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:18:44,381 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:18:44,381 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:18:44,382 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:18:44,382 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:18:44,383 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:18:44,390 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:18:44,391 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:49,392 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:54,393 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:18:55,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:18:55,750 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:18:55,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:18:59,871 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:04,070 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:19:04,070 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:19:04,070 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:19:04,070 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:19:04,071 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:19:04,072 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:19:04,073 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:19:04,073 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:19:04,074 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:19:04,110 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:19:04,110 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:19:04,110 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:19:04,111 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:19:04,111 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:19:04,122 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:19:04,122 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:19:04,122 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:19:04,123 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:19:04,123 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:19:04,123 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:19:04,124 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:19:04,125 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:19:04,125 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:19:04,126 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:19:04,234 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:19:04,234 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:19:04,234 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:19:04,234 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:19:04,235 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:19:04,235 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:19:04,235 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:19:04,235 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:19:04,244 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:19:04,245 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:19:04,245 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:19:04,245 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:19:04,245 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:19:04,246 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:19:04,246 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:19:04,247 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:19:04,247 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:19:04,248 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:19:04,883 DEBUG SenderThread:81 [sender.py:send():379] send: config +2024-06-02 20:19:04,901 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:05,950 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:19:05,950 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:19:05,951 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-06-02 20:19:06,156 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml +2024-06-02 20:19:06,157 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:19:08,157 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:19:10,368 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:19:10,369 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:19:10,369 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:19:10,374 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:19:10,374 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:19:10,385 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:19:10,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:19:10,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:19:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:19:11,872 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:16,874 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:21,875 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:25,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:19:25,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:19:25,790 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:19:27,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:32,828 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:37,837 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:39,200 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml +2024-06-02 20:19:40,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:19:40,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:19:40,790 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:19:43,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:48,829 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:53,830 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:19:54,035 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:19:54,035 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:19:54,035 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:19:54,035 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:19:54,036 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:19:54,036 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:19:54,037 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:19:54,037 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:19:54,038 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:19:54,075 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:19:54,075 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:19:54,075 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:19:54,075 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:19:54,075 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:19:54,076 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:19:54,077 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:19:54,077 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:19:54,086 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:19:55,750 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:19:55,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:19:55,751 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:19:58,862 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:00,656 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:00,656 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:00,656 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:00,656 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:00,657 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:00,657 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:00,658 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:00,659 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:00,660 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:00,696 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:00,696 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:00,696 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:00,697 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:00,697 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:20:00,708 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:00,708 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:00,708 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:00,708 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:00,709 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:00,709 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:00,709 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:00,710 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:00,711 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:00,712 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:02,240 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,240 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,240 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,240 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,241 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,253 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:02,254 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:02,254 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:20:02,254 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,255 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,255 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,255 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,256 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,256 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:02,258 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:02,380 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,380 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,380 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,381 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,381 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,392 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:02,393 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:02,393 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:20:02,393 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,393 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,393 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,394 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,394 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,394 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,394 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,395 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,395 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,396 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,398 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,398 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,398 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,398 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,398 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,398 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,399 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,399 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,400 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,400 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,400 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,402 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:02,402 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,403 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,403 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:02,403 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,403 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,404 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,406 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,425 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:02,426 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:02,426 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,426 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,426 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,426 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,427 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,427 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,427 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,427 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,427 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,429 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,429 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,429 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,429 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,429 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,430 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,430 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,431 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,432 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:02,433 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:02,433 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:02,456 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,456 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,456 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,456 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,457 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,457 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,457 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,457 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:20:02,470 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,470 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,470 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,471 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,471 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,472 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,472 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,473 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:02,473 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:02,474 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:02,538 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause +2024-06-02 20:20:02,538 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-06-02 20:20:02,538 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor +2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-06-02 20:20:02,539 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor +2024-06-02 20:20:02,539 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor +2024-06-02 20:20:02,551 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor +2024-06-02 20:20:02,552 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor +2024-06-02 20:20:02,552 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor +2024-06-02 20:20:02,552 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume +2024-06-02 20:20:02,552 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread +2024-06-02 20:20:02,552 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:20:02,552 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:20:02,553 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:20:02,553 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:20:02,553 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:20:02,554 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:20:02,555 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:20:02,555 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:20:03,232 DEBUG SenderThread:81 [sender.py:send():379] send: config +2024-06-02 20:20:03,237 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:03,237 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:03,237 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-06-02 20:20:04,209 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:20:04,238 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:06,209 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:20:09,361 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:20:10,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:20:10,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:20:11,270 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml +2024-06-02 20:20:15,865 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:20,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:20:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:20:25,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:20:25,888 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:30,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:35,890 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:40,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:20:40,752 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:20:40,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:20:41,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:44,446 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history +2024-06-02 20:20:44,448 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric +2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: history +2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record +2024-06-02 20:20:44,449 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-06-02 20:20:45,283 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json +2024-06-02 20:20:47,450 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:52,451 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:20:55,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:20:55,754 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:20:55,755 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:20:57,873 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:02,555 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:21:02,557 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:21:03,558 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:08,559 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:21:10,752 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:21:10,792 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:21:13,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:15,355 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml +2024-06-02 20:21:19,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:24,868 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:25,035 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history +2024-06-02 20:21:25,036 DEBUG SenderThread:81 [sender.py:send():379] send: history +2024-06-02 20:21:25,037 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record +2024-06-02 20:21:25,039 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-06-02 20:21:25,359 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json +2024-06-02 20:21:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:21:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:21:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:21:29,869 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:32,558 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:21:35,559 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:40,560 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:40,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:21:40,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:21:40,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:21:45,835 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:50,836 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:21:55,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:21:55,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:21:55,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:21:56,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:01,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:02,561 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:22:05,261 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history +2024-06-02 20:22:05,262 DEBUG SenderThread:81 [sender.py:send():379] send: history +2024-06-02 20:22:05,262 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record +2024-06-02 20:22:05,263 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-06-02 20:22:05,375 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json +2024-06-02 20:22:07,263 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:22:10,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:22:10,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:22:12,884 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:17,885 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:22,886 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:25,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:22:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:22:25,785 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:22:28,786 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:32,560 DEBUG SenderThread:81 [sender.py:send():379] send: stats +2024-06-02 20:22:34,561 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:39,562 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:40,754 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:22:40,754 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:22:40,791 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:22:44,415 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history +2024-06-02 20:22:44,416 DEBUG SenderThread:81 [sender.py:send():379] send: history +2024-06-02 20:22:44,416 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record +2024-06-02 20:22:44,417 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-06-02 20:22:45,390 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json +2024-06-02 20:22:45,418 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:50,392 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log +2024-06-02 20:22:50,594 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:55,594 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:22:55,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:22:55,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:22:55,756 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages