diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-06-02 20:13:53,460 INFO StreamThr :81 [internal.py:wandb_internal():86] W&B internal server running at pid: 81, started at: 2024-06-02 20:13:53.459265 -2024-06-02 20:13:53,461 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status -2024-06-02 20:13:53,843 INFO WriterThread:81 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/run-3mh4qp2s.wandb -2024-06-02 20:13:53,844 DEBUG SenderThread:81 [sender.py:send():379] send: header -2024-06-02 20:13:53,847 DEBUG SenderThread:81 [sender.py:send():379] send: run -2024-06-02 20:13:53,953 INFO SenderThread:81 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files -2024-06-02 20:13:53,954 INFO SenderThread:81 [sender.py:_start_run_threads():1124] run started: 3mh4qp2s with start time 1717359233.460694 -2024-06-02 20:13:53,962 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: check_version -2024-06-02 20:13:53,962 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: check_version -2024-06-02 20:13:54,052 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: run_start -2024-06-02 20:13:54,063 DEBUG HandlerThread:81 [system_info.py:__init__():26] System info init -2024-06-02 20:13:54,063 DEBUG HandlerThread:81 [system_info.py:__init__():41] System info init done -2024-06-02 20:13:54,063 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:13:54,064 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:13:54,064 INFO HandlerThread:81 [system_monitor.py:probe():214] Collecting system info -2024-06-02 20:13:54,064 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:13:54,065 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:13:54,067 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:13:54,068 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:13:54,069 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:13:54,078 DEBUG HandlerThread:81 [system_info.py:probe():150] Probing system -2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [gitlib.py:_init_repo():56] git repository is invalid -2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_info.py:probe():198] Probing system done -2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-06-02T20:13:54.078341', 'startedAt': '2024-06-02T20:13:53.453169', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '8f29a1fd4865', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.168, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}, {'current': 2000.168, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5643.874462127686}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357563018798828}} -2024-06-02 20:13:54,080 INFO HandlerThread:81 [system_monitor.py:probe():224] Finished collecting system info -2024-06-02 20:13:54,080 INFO HandlerThread:81 [system_monitor.py:probe():227] Publishing system info -2024-06-02 20:13:54,080 DEBUG HandlerThread:81 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-06-02 20:13:54,956 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/conda-environment.yaml -2024-06-02 20:14:09,095 ERROR HandlerThread:81 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-06-02 20:47:46,140 INFO StreamThr :82 [internal.py:wandb_internal():86] W&B internal server running at pid: 82, started at: 2024-06-02 20:47:46.139860 +2024-06-02 20:47:46,142 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status +2024-06-02 20:47:46,717 INFO WriterThread:82 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240602_204746-wowpw30n/run-wowpw30n.wandb +2024-06-02 20:47:46,718 DEBUG SenderThread:82 [sender.py:send():379] send: header +2024-06-02 20:47:46,721 DEBUG SenderThread:82 [sender.py:send():379] send: run +2024-06-02 20:47:46,819 INFO SenderThread:82 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files +2024-06-02 20:47:46,819 INFO SenderThread:82 [sender.py:_start_run_threads():1124] run started: wowpw30n with start time 1717361266.141329 +2024-06-02 20:47:46,829 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: check_version +2024-06-02 20:47:46,829 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: check_version +2024-06-02 20:47:46,921 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: run_start +2024-06-02 20:47:46,933 DEBUG HandlerThread:82 [system_info.py:__init__():26] System info init +2024-06-02 20:47:46,933 DEBUG HandlerThread:82 [system_info.py:__init__():41] System info init done +2024-06-02 20:47:46,933 INFO HandlerThread:82 [system_monitor.py:start():194] Starting system monitor +2024-06-02 20:47:46,933 INFO SystemMonitor:82 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-06-02 20:47:46,934 INFO HandlerThread:82 [system_monitor.py:probe():214] Collecting system info +2024-06-02 20:47:46,934 INFO SystemMonitor:82 [interfaces.py:start():190] Started cpu monitoring +2024-06-02 20:47:46,935 INFO SystemMonitor:82 [interfaces.py:start():190] Started disk monitoring +2024-06-02 20:47:46,936 INFO SystemMonitor:82 [interfaces.py:start():190] Started gpu monitoring +2024-06-02 20:47:46,936 INFO SystemMonitor:82 [interfaces.py:start():190] Started memory monitoring +2024-06-02 20:47:46,937 INFO SystemMonitor:82 [interfaces.py:start():190] Started network monitoring +2024-06-02 20:47:46,948 DEBUG HandlerThread:82 [system_info.py:probe():150] Probing system +2024-06-02 20:47:46,950 DEBUG HandlerThread:82 [gitlib.py:_init_repo():56] git repository is invalid +2024-06-02 20:47:46,950 DEBUG HandlerThread:82 [system_info.py:probe():198] Probing system done +2024-06-02 20:47:46,950 DEBUG HandlerThread:82 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-06-02T20:47:46.948167', 'startedAt': '2024-06-02T20:47:46.133534', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '96cd76fa9870', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.204, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.204, 'min': 0.0, 'max': 0.0}, {'current': 2000.204, 'min': 0.0, 'max': 0.0}, {'current': 2000.204, 'min': 0.0, 'max': 0.0}, {'current': 2000.204, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5643.874465942383}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357563018798828}} +2024-06-02 20:47:46,950 INFO HandlerThread:82 [system_monitor.py:probe():224] Finished collecting system info +2024-06-02 20:47:46,950 INFO HandlerThread:82 [system_monitor.py:probe():227] Publishing system info +2024-06-02 20:47:46,950 DEBUG HandlerThread:82 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-06-02 20:47:47,821 INFO Thread-12 :82 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/conda-environment.yaml +2024-06-02 20:48:01,965 ERROR HandlerThread:82 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,918 +37,220 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-06-02 20:14:09,098 DEBUG HandlerThread:81 [system_info.py:_save_conda():222] Saving conda packages done -2024-06-02 20:14:09,099 INFO HandlerThread:81 [system_monitor.py:probe():229] Finished publishing system info -2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 20:14:09,107 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:09,108 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: keepalive -2024-06-02 20:14:09,108 DEBUG SenderThread:81 [sender.py:send():379] send: files -2024-06-02 20:14:09,108 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-06-02 20:14:09,360 INFO wandb-upload_0:81 [upload_job.py:push():131] Uploaded file /tmp/tmp1abbn3iqwandb/snb0frox-wandb-metadata.json -2024-06-02 20:14:09,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-metadata.json -2024-06-02 20:14:10,745 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: python_packages -2024-06-02 20:14:10,746 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: python_packages -2024-06-02 20:14:10,748 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry -2024-06-02 20:14:10,825 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:14:10,825 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:14:10,842 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:14:10,864 DEBUG SenderThread:81 [sender.py:send():379] send: config -2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry -2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:14:10,870 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-06-02 20:14:10,870 DEBUG SenderThread:81 [sender.py:send():379] send: telemetry -2024-06-02 20:14:10,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/requirements.txt -2024-06-02 20:14:10,959 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:14:11,862 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:14:11,862 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:14:11,862 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:14:11,863 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:14:11,863 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:14:11,863 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:14:11,864 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:14:11,864 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:14:11,872 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:14:11,873 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:14:12,960 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:14:14,874 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:19,875 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:24,884 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:25,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:14:25,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:14:25,803 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:14:25,965 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml -2024-06-02 20:14:30,888 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:35,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:40,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:14:40,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:14:40,786 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:14:41,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:46,852 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:51,853 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:14:55,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:14:55,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:14:55,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:14:57,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:02,808 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:07,809 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:10,746 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:15:10,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:15:10,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:15:12,823 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:17,824 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:22,825 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:25,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:15:25,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:15:25,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:15:28,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:33,794 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:38,795 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:40,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:15:40,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:15:40,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:15:43,876 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:48,877 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:53,878 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:15:55,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:15:55,747 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:15:55,787 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:15:59,826 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:04,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:09,828 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:10,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:16:10,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:16:10,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:16:14,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:19,890 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:24,891 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:25,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:16:25,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:16:25,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:16:30,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:35,794 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:40,747 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:16:40,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:16:40,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:16:40,813 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:45,814 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:50,815 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:16:55,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:16:55,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:16:55,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:16:55,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:00,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:05,868 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:10,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:17:10,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:17:10,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:17:11,831 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:16,832 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:21,833 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:25,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:17:25,748 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:17:25,788 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:17:26,840 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:31,841 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:36,843 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:40,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:17:40,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:17:40,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:17:42,808 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:47,809 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:52,810 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:17:55,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:17:55,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:17:55,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:17:58,804 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:03,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:08,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:10,748 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:18:10,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:18:10,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:18:13,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:18,851 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:23,852 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:25,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:18:25,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:18:25,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:18:28,865 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:33,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:38,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:40,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:18:40,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:18:40,789 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:18:44,340 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:18:44,340 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:18:44,340 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:18:44,341 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:18:44,341 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:18:44,342 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:18:44,343 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:18:44,343 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:18:44,345 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:18:44,381 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:18:44,381 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:18:44,381 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:18:44,381 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:18:44,381 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:18:44,382 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:18:44,382 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:18:44,383 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:18:44,390 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:18:44,390 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:18:44,391 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:49,392 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:54,393 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:18:55,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:18:55,750 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:18:55,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:18:59,871 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:04,070 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:19:04,070 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:19:04,070 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:19:04,070 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:19:04,071 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:19:04,072 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:19:04,073 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:19:04,073 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:19:04,074 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:19:04,110 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:19:04,110 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:19:04,110 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:19:04,111 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:19:04,111 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:19:04,116 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:19:04,122 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:19:04,122 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:19:04,122 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:19:04,122 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:19:04,123 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:19:04,123 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:19:04,123 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:19:04,124 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:19:04,125 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:19:04,125 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:19:04,126 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:19:04,234 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:19:04,234 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:19:04,234 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:19:04,234 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:19:04,235 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:19:04,235 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:19:04,235 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:19:04,235 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:19:04,244 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:19:04,244 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:19:04,245 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:19:04,245 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:19:04,245 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:19:04,245 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:19:04,246 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:19:04,246 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:19:04,247 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:19:04,247 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:19:04,248 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:19:04,883 DEBUG SenderThread:81 [sender.py:send():379] send: config -2024-06-02 20:19:04,901 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:05,950 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:19:05,950 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:19:05,951 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-06-02 20:19:06,156 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml -2024-06-02 20:19:06,157 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:19:08,157 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:19:10,368 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:19:10,369 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:19:10,369 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:19:10,369 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:19:10,374 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:19:10,374 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:19:10,380 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:19:10,385 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:19:10,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:19:10,749 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:19:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:19:11,872 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:16,874 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:21,875 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:25,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:19:25,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:19:25,790 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:19:27,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:32,828 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:37,837 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:39,200 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml -2024-06-02 20:19:40,749 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:19:40,750 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:19:40,790 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:19:43,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:48,829 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:53,830 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:19:54,035 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:19:54,035 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:19:54,035 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:19:54,035 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:19:54,036 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:19:54,036 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:19:54,037 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:19:54,037 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:19:54,038 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:19:54,075 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:19:54,075 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:19:54,075 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:19:54,075 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:19:54,075 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:19:54,076 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:19:54,077 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:19:54,077 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:19:54,086 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:19:54,086 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:19:55,750 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:19:55,751 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:19:55,751 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:19:58,862 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:00,656 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:00,656 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:00,656 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:00,656 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:00,657 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:00,657 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:00,658 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:00,659 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:00,660 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:00,696 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:00,696 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:00,696 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:00,697 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:00,697 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:00,697 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:00,708 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:20:00,708 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:00,708 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:00,708 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:00,708 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:00,709 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:00,709 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:00,709 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:00,710 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:00,711 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:00,712 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:02,240 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,240 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,240 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,240 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,241 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,241 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,253 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:02,254 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:02,254 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:20:02,254 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,255 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,255 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,255 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,256 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,256 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:02,257 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:02,258 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:02,380 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,380 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,380 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,380 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,381 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,381 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,392 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:02,393 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:02,393 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:20:02,393 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,393 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,393 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,394 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,394 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,394 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,394 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,395 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,395 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,396 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,396 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,398 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,398 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,398 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,398 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,398 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,398 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,399 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,399 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,400 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,400 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,400 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,402 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:02,402 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,403 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,403 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:02,403 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,403 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,404 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,406 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,425 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:02,426 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:02,426 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,426 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,426 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,426 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,427 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,427 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,427 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,427 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,427 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,428 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,429 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,429 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,429 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,429 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,429 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,430 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,430 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,431 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,432 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:02,433 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:02,433 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:02,456 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,456 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,456 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,456 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,457 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,457 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,457 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,457 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:02,470 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:20:02,470 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,470 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,470 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,471 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,471 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,472 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,472 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,473 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:02,473 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:02,474 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:02,538 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: pause -2024-06-02 20:20:02,538 INFO HandlerThread:81 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-06-02 20:20:02,538 INFO HandlerThread:81 [system_monitor.py:finish():203] Stopping system monitor -2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-02 20:20:02,538 DEBUG SystemMonitor:81 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-02 20:20:02,539 INFO HandlerThread:81 [interfaces.py:finish():202] Joined cpu monitor -2024-06-02 20:20:02,539 INFO HandlerThread:81 [interfaces.py:finish():202] Joined disk monitor -2024-06-02 20:20:02,551 INFO HandlerThread:81 [interfaces.py:finish():202] Joined gpu monitor -2024-06-02 20:20:02,552 INFO HandlerThread:81 [interfaces.py:finish():202] Joined memory monitor -2024-06-02 20:20:02,552 INFO HandlerThread:81 [interfaces.py:finish():202] Joined network monitor -2024-06-02 20:20:02,552 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: resume -2024-06-02 20:20:02,552 INFO HandlerThread:81 [handler.py:handle_request_resume():699] starting system metrics thread -2024-06-02 20:20:02,552 INFO HandlerThread:81 [system_monitor.py:start():194] Starting system monitor -2024-06-02 20:20:02,552 INFO SystemMonitor:81 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-02 20:20:02,553 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:20:02,553 INFO SystemMonitor:81 [interfaces.py:start():190] Started cpu monitoring -2024-06-02 20:20:02,553 INFO SystemMonitor:81 [interfaces.py:start():190] Started disk monitoring -2024-06-02 20:20:02,554 INFO SystemMonitor:81 [interfaces.py:start():190] Started gpu monitoring -2024-06-02 20:20:02,555 INFO SystemMonitor:81 [interfaces.py:start():190] Started memory monitoring -2024-06-02 20:20:02,555 INFO SystemMonitor:81 [interfaces.py:start():190] Started network monitoring -2024-06-02 20:20:03,232 DEBUG SenderThread:81 [sender.py:send():379] send: config -2024-06-02 20:20:03,237 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:03,237 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:03,237 WARNING SenderThread:81 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-06-02 20:20:04,209 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:20:04,238 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:06,209 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:20:09,361 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:20:10,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:20:10,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:20:11,270 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml -2024-06-02 20:20:15,865 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:20,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:20:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:20:25,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:20:25,888 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:30,889 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:35,890 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:40,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:20:40,752 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:20:40,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:20:41,866 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:44,446 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:20:44,448 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: metric -2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:20:44,449 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:20:44,449 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:20:45,283 INFO Thread-12 :81 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:20:47,450 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:52,451 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:20:55,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:20:55,754 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:20:55,755 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:20:57,873 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:02,555 DEBUG SystemMonitor:81 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-02 20:21:02,557 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:21:03,558 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:08,559 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:21:10,752 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:21:10,792 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:21:13,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:15,355 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/config.yaml -2024-06-02 20:21:19,867 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:24,868 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:25,035 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:21:25,036 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:21:25,037 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:21:25,039 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:21:25,359 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:21:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:21:25,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:21:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:21:29,869 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:32,558 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:21:35,559 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:40,560 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:40,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:21:40,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:21:40,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:21:45,835 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:50,836 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:21:55,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:21:55,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:21:55,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:21:56,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:01,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:02,561 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:22:05,261 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:22:05,262 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:22:05,262 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:22:05,263 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:22:05,375 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:22:07,263 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:10,752 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:22:10,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:22:10,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:22:12,884 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:17,885 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:22,886 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:25,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:22:25,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:22:25,785 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:22:28,786 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:32,560 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:22:34,561 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:39,562 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:40,754 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:22:40,754 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:22:40,791 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:22:44,415 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:22:44,416 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:22:44,416 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:22:44,417 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:22:45,390 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:22:45,418 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:50,392 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:22:50,594 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:55,594 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:22:55,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:22:55,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:22:55,756 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:23:00,812 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:02,561 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:23:06,562 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:10,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:23:10,753 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:23:10,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:23:11,841 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:16,842 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:21,843 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:25,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:23:25,754 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:23:25,793 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:23:27,049 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:23:27,050 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:23:27,051 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:23:27,051 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:27,052 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:23:27,407 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:23:32,053 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:32,562 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:23:37,563 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:40,753 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:23:40,754 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:23:40,756 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:23:42,816 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:47,817 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:52,818 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:23:55,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:23:55,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:23:55,799 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:23:58,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:02,563 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:24:04,564 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:07,133 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:24:07,134 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:24:07,134 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:24:07,135 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:24:07,423 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:24:10,135 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:10,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:24:10,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:24:10,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:24:15,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:20,806 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:25,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:24:25,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:24:25,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:24:25,886 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:30,887 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:32,564 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:24:36,565 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:40,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:24:40,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:24:40,757 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:24:41,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:46,857 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:47,212 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:24:47,213 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:24:47,214 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:24:47,214 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:24:47,440 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:24:52,215 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:24:55,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:24:55,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:24:55,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:24:57,891 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:02,565 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:25:03,566 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:08,567 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:10,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:25:10,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:25:10,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:25:13,883 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:18,884 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:23,885 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:25,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:25:25,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:25:25,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:25:26,997 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:25:26,998 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:25:26,998 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:25:26,999 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:25:27,455 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:25:29,000 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:32,457 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:25:32,566 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:25:34,567 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:39,567 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:40,757 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:25:40,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:25:40,758 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:25:44,875 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:49,876 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:54,877 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:25:55,759 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:25:55,759 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:25:55,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:26:00,870 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:02,567 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:26:06,568 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:10,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:26:10,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:26:10,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:26:10,928 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:26:10,929 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:26:10,929 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:26:10,930 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:26:11,472 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:26:11,930 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:16,931 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:21,932 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:25,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:26:25,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:26:25,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:26:27,797 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:32,572 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:26:33,573 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:38,574 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:40,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:26:40,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:26:40,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:26:43,823 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:48,824 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:51,423 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:26:51,424 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:26:51,424 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:26:51,425 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:26:51,487 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:26:54,426 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:26:55,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:26:55,759 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:26:55,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:26:59,857 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:02,568 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:27:05,569 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:10,570 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:10,759 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:27:10,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:27:10,760 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:27:15,846 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:20,847 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:25,758 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:27:25,759 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:27:25,799 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:27:26,837 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:31,508 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:27:31,510 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:27:31,510 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:27:31,512 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:27:32,504 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:27:32,512 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:32,569 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:27:37,570 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:40,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:27:40,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:27:40,761 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:27:42,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:47,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:52,857 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:27:55,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:27:55,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:27:55,761 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:27:58,840 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:02,571 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:28:04,573 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:09,574 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:10,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:28:10,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:28:10,761 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:28:11,611 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:28:11,612 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:28:11,612 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:28:11,612 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:28:12,520 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:28:14,613 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:16,521 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:28:20,197 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:25,198 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:25,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:28:25,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:28:25,761 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:28:30,843 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:32,571 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:28:36,572 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:40,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:28:40,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:28:40,762 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:28:41,797 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:46,798 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:51,799 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:28:55,378 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:28:55,379 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:28:55,379 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:28:55,379 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:28:55,537 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:28:55,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:28:55,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:28:55,761 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:28:56,815 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:01,816 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:02,572 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:29:07,574 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:10,760 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:29:10,762 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:29:10,762 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:29:12,853 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:17,854 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:22,855 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:25,761 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:29:25,762 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:29:25,764 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:29:27,885 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:32,573 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:29:33,574 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:34,824 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:29:34,825 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:29:34,825 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:29:34,826 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:29:35,552 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:29:38,827 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:40,772 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:29:40,773 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:29:40,776 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:29:44,814 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:49,815 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:54,816 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:29:55,766 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:29:55,767 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:29:55,807 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:29:59,892 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:02,574 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:30:05,575 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:10,577 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:10,766 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:30:10,766 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:30:10,807 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:30:14,013 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:30:14,014 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:30:14,015 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:30:14,015 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:30:14,567 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:30:16,016 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:21,017 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:25,766 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:30:25,767 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:30:25,771 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:30:26,804 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:31,805 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:32,576 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:30:37,578 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:40,766 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:30:40,767 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:30:40,807 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:30:42,898 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:47,899 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:52,900 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:53,781 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:30:53,783 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:30:53,783 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:30:53,783 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:30:54,583 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:30:56,031 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:30:56,031 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:30:56,066 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:30:58,515 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:30:58,585 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:31:02,576 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:31:03,577 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:08,580 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:10,849 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:31:10,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:31:10,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:31:13,984 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:18,984 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:23,985 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:25,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:31:25,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:31:25,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:31:29,971 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:32,579 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:31:35,580 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:36,928 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:31:36,929 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:31:36,930 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:31:36,930 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:31:37,599 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:31:40,773 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:40,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:31:40,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:31:40,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:31:45,948 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:50,948 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:31:55,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:31:55,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:31:55,890 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:31:55,979 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:00,980 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:02,578 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:32:06,580 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:10,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:32:10,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:32:10,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:32:11,939 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:16,441 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:32:16,442 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:32:16,442 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:32:16,443 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:32:16,613 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:32:17,443 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:22,444 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:25,850 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:32:25,851 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:32:25,851 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:32:27,978 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:32,579 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:32:33,580 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:38,581 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:40,858 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:32:40,859 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:32:40,860 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:32:43,994 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:48,994 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:53,995 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:32:55,089 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:32:55,091 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:32:55,091 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:32:55,091 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:32:55,629 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:32:55,855 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:32:55,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:32:55,856 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:32:59,978 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:02,580 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:33:05,581 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:10,582 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:10,855 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:33:10,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:33:10,856 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:33:15,914 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:20,915 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:25,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:33:25,857 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:33:25,857 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:33:25,943 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:30,944 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:32,581 DEBUG SenderThread:81 [sender.py:send():379] send: stats -2024-06-02 20:33:33,904 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: partial_history -2024-06-02 20:33:33,905 DEBUG SenderThread:81 [sender.py:send():379] send: history -2024-06-02 20:33:33,906 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: summary_record -2024-06-02 20:33:33,906 INFO SenderThread:81 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-06-02 20:33:34,643 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/wandb-summary.json -2024-06-02 20:33:36,907 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:38,645 INFO Thread-12 :81 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_201353-3mh4qp2s/files/output.log -2024-06-02 20:33:40,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: internal_messages -2024-06-02 20:33:40,856 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: stop_status -2024-06-02 20:33:40,857 DEBUG SenderThread:81 [sender.py:send_request():406] send_request: stop_status -2024-06-02 20:33:42,877 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report -2024-06-02 20:33:47,878 DEBUG HandlerThread:81 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:01,969 DEBUG HandlerThread:82 [system_info.py:_save_conda():222] Saving conda packages done +2024-06-02 20:48:01,970 INFO HandlerThread:82 [system_monitor.py:probe():229] Finished publishing system info +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:01,978 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: keepalive +2024-06-02 20:48:01,979 DEBUG SenderThread:82 [sender.py:send():379] send: files +2024-06-02 20:48:01,979 INFO SenderThread:82 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-06-02 20:48:02,281 INFO wandb-upload_0:82 [upload_job.py:push():131] Uploaded file /tmp/tmp2t2ujinlwandb/a7gdpmj8-wandb-metadata.json +2024-06-02 20:48:02,824 INFO Thread-12 :82 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/wandb-metadata.json +2024-06-02 20:48:03,595 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: python_packages +2024-06-02 20:48:03,595 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: python_packages +2024-06-02 20:48:03,598 DEBUG SenderThread:82 [sender.py:send():379] send: telemetry +2024-06-02 20:48:03,675 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:48:03,693 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:48:03,700 DEBUG SenderThread:82 [sender.py:send():379] send: config +2024-06-02 20:48:03,705 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:48:03,802 DEBUG SenderThread:82 [sender.py:send():379] send: metric +2024-06-02 20:48:03,803 DEBUG SenderThread:82 [sender.py:send():379] send: telemetry +2024-06-02 20:48:03,803 DEBUG SenderThread:82 [sender.py:send():379] send: metric +2024-06-02 20:48:03,803 WARNING SenderThread:82 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-06-02 20:48:03,803 DEBUG SenderThread:82 [sender.py:send():379] send: telemetry +2024-06-02 20:48:03,825 INFO Thread-12 :82 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/output.log +2024-06-02 20:48:03,825 INFO Thread-12 :82 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/requirements.txt +2024-06-02 20:48:05,825 INFO Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/output.log +2024-06-02 20:48:07,804 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:11,828 INFO Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/output.log +2024-06-02 20:48:13,131 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:18,137 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:18,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:48:18,606 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:48:19,167 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:48:19,892 INFO Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/config.yaml +2024-06-02 20:48:24,249 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:29,249 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:33,596 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:48:33,597 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:48:33,598 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:48:34,631 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:39,632 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:44,634 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:46,938 DEBUG SystemMonitor:82 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-06-02 20:48:46,939 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:48:48,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:48:48,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:48:48,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:48:49,726 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:54,727 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:48:59,728 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:03,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:49:03,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:49:03,598 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:49:05,659 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:10,660 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:15,661 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:16,940 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:49:18,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:49:18,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:49:18,599 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:49:20,716 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:25,717 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:30,718 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:33,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:49:33,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:49:33,599 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:49:36,661 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:41,662 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:46,663 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:46,941 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:49:48,598 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:49:48,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:49:48,599 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:49:51,719 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:49:56,720 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:01,722 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:03,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:50:03,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:50:03,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:50:07,698 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:12,699 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:16,942 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:50:17,943 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:18,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:50:18,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:50:18,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:50:23,653 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:28,654 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:33,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:50:33,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:50:33,642 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:50:33,658 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:38,659 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:43,660 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:46,943 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:50:48,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:50:48,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:50:48,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:50:48,717 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:53,718 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:50:58,719 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:03,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:51:03,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:51:03,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:51:04,638 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:09,639 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:14,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:16,944 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:51:18,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:51:18,599 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:51:18,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:51:19,692 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:24,693 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:29,694 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:33,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:51:33,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:51:33,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:51:35,684 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:40,685 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:45,686 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:46,945 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:51:48,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:51:48,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:51:48,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:51:50,739 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:51:55,740 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:00,741 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:03,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:52:03,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:52:03,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:52:06,717 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:11,718 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:16,720 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:16,946 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:52:18,599 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:52:18,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:52:18,636 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:52:22,637 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:27,638 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:32,639 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:33,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:52:33,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:52:33,640 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:52:37,670 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:42,671 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:46,947 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:52:47,948 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:48,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:52:48,600 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:52:48,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:52:53,721 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:52:58,722 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:03,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:53:03,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:53:03,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:53:04,653 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:09,654 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:14,655 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:16,948 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:53:18,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:53:18,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:53:18,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:53:19,731 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:24,732 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:29,733 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:33,600 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:53:33,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:53:33,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:53:35,654 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:40,655 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:45,656 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:46,949 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:53:48,602 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:53:48,602 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:53:48,642 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:53:50,726 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:53:55,727 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:00,729 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:03,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:54:03,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:54:03,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:54:06,728 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:11,729 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:16,730 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:16,950 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:54:18,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:54:18,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:54:18,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:54:22,649 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:27,650 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:32,651 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:33,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:54:33,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:54:33,641 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:54:37,731 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:42,732 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:46,951 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:54:47,952 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:48,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:54:48,601 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:54:48,642 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:54:53,727 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:54:58,728 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:55:03,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:55:03,602 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:55:03,642 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:55:04,712 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:55:09,713 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:55:14,714 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:55:16,952 DEBUG SenderThread:82 [sender.py:send():379] send: stats +2024-06-02 20:55:18,601 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status +2024-06-02 20:55:18,602 DEBUG SenderThread:82 [sender.py:send_request():406] send_request: stop_status +2024-06-02 20:55:18,602 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages +2024-06-02 20:55:20,701 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report +2024-06-02 20:55:21,054 INFO Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_204746-wowpw30n/files/output.log +2024-06-02 20:55:25,702 DEBUG HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report